text:
<?php

namespace PlentymarketsAdapter\ServiceBus\QueryHandler\Manufacturer;

use Exception;
use PlentyConnector\Connector\ServiceBus\Query\FetchTransferObjectQuery;
use PlentyConnector\Connector\ServiceBus\Query\QueryInterface;
use PlentyConnector\Connector\ServiceBus\QueryHandler\QueryHandlerInterface;
use PlentyConnector\Connector\ServiceBus\QueryType;
use PlentyConnector\Connector\TransferObject\Manufacturer\Manufacturer;
use PlentyConnector\Console\OutputHandler\OutputHandlerInterface;
use PlentymarketsAdapter\Client\ClientInterface;
use PlentymarketsAdapter\PlentymarketsAdapter;
use PlentymarketsAdapter\ResponseParser\Manufacturer\ManufacturerResponseParserInterface;
use Psr\Log\LoggerInterface;

class FetchAllManufacturersQueryHandler implements QueryHandlerInterface
{
    /**
     * @var ClientInterface
     */
    private $client;

    /**
     * @var ManufacturerResponseParserInterface
     */
    private $responseParser;

    /**
     * @var LoggerInterface
     */
    private $logger;

    /**
     * @var OutputHandlerInterface
     */
    private $outputHandler;

    public function __construct(
        ClientInterface $client,
        ManufacturerResponseParserInterface $responseParser,
        LoggerInterface $logger,
        OutputHandlerInterface $outputHandler
    ) {
        $this->client = $client;
        $this->responseParser = $responseParser;
        $this->logger = $logger;
        $this->outputHandler = $outputHandler;
    }

    /**
     * {@inheritdoc}
     */
    public function supports(QueryInterface $query)
    {
        return $query instanceof FetchTransferObjectQuery &&
            $query->getAdapterName() === PlentymarketsAdapter::NAME &&
            $query->getObjectType() === Manufacturer::TYPE &&
            $query->getQueryType() === QueryType::ALL;
    }

    /**
     * {@inheritdoc}
     */
    public function handle(QueryInterface $query)
    {
        $elements = $this->client->getIterator('items/manufacturers');

        $this->outputHandler->startProgressBar(count($elements));

        foreach ($elements as $element) {
            try {
                $result = $this->responseParser->parse($element);
            } catch (Exception $exception) {
                $this->logger->error($exception->getMessage());

                $result = null;
            }

            if (empty($result)) {
                $result = [];
            }

            $result = array_filter($result);

            foreach ($result as $parsedElement) {
                yield $parsedElement;
            }

            $this->outputHandler->advanceProgressBar();
        }

        $this->outputHandler->finishProgressBar();
    }
}

redpajama_set_name: RedPajamaGithub


text:
function kern = rbfperiodicKernParamInit(kern)

% RBFPERIODICKERNPARAMINIT RBFPERIODIC kernel parameter initialisation.
% This kernel is a periodic kernel constructed by mapping a one
% dimensional input into a two dimensional space,
%
% u_1(x) = cos(x), u_2(x) = sin(x)
%
% A radial basis function kernel is then applied in the resulting
% two dimensional space. The resulting form of the covariance is
% then
%
% k(x_i, x_j) = sigma2 * exp(-2*gamma *sin^2((x_i - x_j)/2)').
%
% The kernel, thereby, generates periodic functions in the space of
% (u_1, u_2).
%
% See Rasmussen & Williams (2006) page 92 and and MacKay's 1998
% introduction to Gaussian processes for further details.
%
% SEEALSO : rbfKernParamInit
%
% FORMAT
% DESC initialises the RBF derived periodic
%  kernel structure with some default parameters.
% ARG kern : the kernel structure which requires initialisation.
% RETURN kern : the kernel structure with the default parameters placed in.
%
% SEEALSO : kernCreate, kernParamInit
%
% COPYRIGHT : Neil D. Lawrence, 2007

% KERN

if kern.inputDimension > 1
  error('PERIODIC kernel only valid for one-D input.')
end

kern.inverseWidth = 1;
kern.variance = 1;
kern.nParams = 2;

kern.period = 2*pi;

% Constrains parameters positive for optimisation.
kern.transforms.index = [1 2];
kern.transforms.type = optimiDefaultConstraint('positive');
kern.isStationary = true;


text:
require "spec_helper"
require "open3"
require 'active_support/core_ext/object/blank'

describe "Pliny integration test" do
  describe "bin/setup" do
    around do |example|
      original_dir = Dir.pwd
      test_dir = Dir.mktmpdir("plinytest-")
      Dir.chdir(test_dir)

      bash "pliny-new myapp"

      Dir.chdir("myapp")
      example.run
      Dir.chdir(original_dir)
    end

    it "generates .env" do
      bash "bin/setup"

      assert File.exist?("./.env")
    end
  end

  describe "pliny-generate scaffold" do
    around do |example|
      DB.tables.each { |t| DB.drop_table(t) }

      original_dir = Dir.pwd
      test_dir = Dir.mktmpdir("plinytest-")
      Dir.chdir(test_dir)

      bash "pliny-new myapp"

      Dir.chdir("myapp")
      bash "bin/setup"

      example.run
      Dir.chdir(original_dir)
    end

    before do
      bash "pliny-generate scaffold artist"
    end

    it "creates the model file" do
      assert File.exist?("./lib/models/artist.rb")
    end

    it "creates the endpoint file" do
      assert File.exist?("./lib/endpoints/artists.rb")
    end

    it "creates the serializer file" do
      assert File.exist?("./lib/serializers/artist.rb")
    end

    it "creates the schema file" do
      assert File.exist?("./schema/schemata/artist.yaml")
    end
  end

  describe "rake db:migrate:status" do
    around do |example|
      DB.tables.each { |t| DB.drop_table(t) }

      original_dir = Dir.pwd
      test_dir = Dir.mktmpdir("plinytest-")
      Dir.chdir(test_dir)

      bash "pliny-new myapp"

      Dir.chdir("myapp")
      bash "bin/setup"

      example.run
      Dir.chdir(original_dir)
    end

    it "returns a migration in the DOWN state when not migrated" do
      bash "pliny-generate model artist"
      migration_file = Dir.glob('db/migrate/*').first

      stdout, stderr = bash_with_output("rake db:migrate:status")

      statuses = Hash[stdout.to_s.split(/\+[-]+\+[-]+\+/)[2..-1].map { |s| s.tr("\n", "") }.select(&:present?).map { |s| s.split("|").map { |s| s.tr(" ", "") }.select(&:present?).reverse }]
      assert statuses[migration_file.split("/").last] == "DOWN"
    end

    it "returns a migration in the UP state when not migrated" do
      bash "pliny-generate model artist"
      migration_file = Dir.glob('db/migrate/*').first
      bash "rake db:migrate"

      stdout, stderr = bash_with_output("rake db:migrate:status")

      statuses = Hash[stdout.to_s.split(/\+[-]+\+[-]+\+/)[2..-1].map { |s| s.tr("\n", "") }.select(&:present?).map { |s| s.split("|").map { |s| s.tr(" ", "") }.select(&:present?).reverse }]
      assert statuses[migration_file.split("/").last] == "UP"
    end

    it "returns a migration in the FILE MISSING state when the file is missing" do
      bash "pliny-generate model artist"
      migration_file = Dir.glob('db/migrate/*').first
      bash "rake db:migrate"

      FileUtils.rm_f(migration_file)

      stdout, stderr = bash_with_output("rake db:migrate:status")

      statuses = Hash[stdout.to_s.split(/\+[-]+\+[-]+\+/)[2..-1].map { |s| s.tr("\n", "") }.select(&:present?).map { |s| s.split("|").map { |s| s.gsub(/(^[ ]+|[ ]+$)/, "") }.select(&:present?).reverse }]
      assert statuses[migration_file.split("/").last] == "FILE MISSING"
    end
  end

  def bash_with_output(cmd)
    bin  = File.expand_path('../bin', File.dirname(__FILE__))
    path = "#{bin}:#{ENV["PATH"]}"
    env = { "PATH" => path }
    stdout, stderr, status = Open3.capture3(env, cmd)

    unless status.success?
      raise "Failed to run #{cmd}, error was #{stderr}"
    end

    return stdout, stderr
  end

  def bash(cmd)
    bin  = File.expand_path('../bin', File.dirname(__FILE__))
    path = "#{bin}:#{ENV["PATH"]}"
    env = { "PATH" => path }
    unless system(env, "#{cmd} > /dev/null")
      raise "Failed to run #{cmd}"
    end
  end
end


text:
namespace web_restrictions {

class WebRestrictionsClient;

class WebRestrictionsResourceThrottle : public content::ResourceThrottle {
 public:
  WebRestrictionsResourceThrottle(WebRestrictionsClient* provider,
                                  const GURL& request_url,
                                  bool is_main_frame);
  ~WebRestrictionsResourceThrottle() override;

  // content::ResourceThrottle implementation:
  void WillStartRequest(bool* defer) override;
  void WillRedirectRequest(const net::RedirectInfo& redirect_info,
                           bool* defer) override;
  const char* GetNameForLogging() const override;

 private:
  bool ShouldDefer(const GURL& url);

  void OnCheckResult(bool should_proceed);

  WebRestrictionsClient* provider_;  // Not owned.

  const GURL request_url_;
  bool is_main_frame_;
  base::WeakPtrFactory<WebRestrictionsResourceThrottle> weak_ptr_factory_;

  DISALLOW_COPY_AND_ASSIGN(WebRestrictionsResourceThrottle);
};

}  // namespace web_restrictions

#endif  // COMPONENTS_WEB_RESTRICTION_WEB_RESTRICTION_RESOURCE_THROTTLE_H_


text:
"""
Pipeline Preprocessing algorithms for Quicklook
"""

import numpy as np
import os,sys
import astropy
import astropy.io.fits as fits
from desispec import io
from desispec.io import read_raw,read_image
from desispec.io.meta import findfile
from desispec.io.fluxcalibration import read_average_flux_calibration
from desispec.calibfinder import findcalibfile
from desispec.quicklook import pas
from desispec.quicklook import qlexceptions,qllogger
from desispec.image import Image as im
from desispec.frame import Frame as fr
from desispec.io.xytraceset import read_xytraceset
from desispec.maskbits import ccdmask

qlog=qllogger.QLLogger("QuickLook",20)
log=qlog.getlog()

class Initialize(pas.PipelineAlg):
    """
    This PA takes information from the fibermap and raw header
    and adds it to the general info section of the merged dictionary
    """
    def __init__(self,name,config,logger=None):
        if name is None or name.strip() == "":
            name="Ready"
        rawtype=astropy.io.fits.hdu.hdulist.HDUList
        pas.PipelineAlg.__init__(self,name,rawtype,rawtype,config,logger)

    def run(self,*args,**kwargs):
        if len(args) == 0 :
            log.critical("Missing input parameter!")
            sys.exit()
        if not self.is_compatible(type(args[0])):
            log.critical("Incompatible input!")
            sys.exit("Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))

        raw=args[0]
        flavor=kwargs['Flavor']
        peaks=None
        fibermap=None
        if flavor != 'bias' and flavor != 'dark':
            fibermap=kwargs['FiberMap']
            peaks=kwargs['Peaks']
        camera=kwargs['Camera']

        return self.run_pa(raw,fibermap,camera,peaks,flavor)

    def run_pa(self,raw,fibermap,camera,peaks,flavor):
        import pytz
        import datetime
        from desitarget.targetmask import desi_mask
        from desispec.fluxcalibration import isStdStar

        #- Create general info dictionary to be sent to merged json
        general_info={}

        #- Get information from raw header
        general_info['PROGRAM']=program=raw[0].header['PROGRAM'].upper()
        calibs=['arcs','flat','bias','dark']

        if not flavor in calibs:
            general_info['AIRMASS']=raw[0].header['AIRMASS']
            general_info['SEEING']=raw[0].header['SEEING']

            #- Get information from fibermap

            #- Limit flux info to fibers in camera
            minfiber=int(camera[1])*500
            maxfiber=minfiber+499
            fibermags=[]
            for flux in ['FLUX_G','FLUX_R','FLUX_Z']:
                fibermags.append(22.5-2.5*np.log10(fibermap[flux][minfiber:maxfiber+1]))

            #- Set sky/no flux fibers to 30 mag
            for i in range(3):
                skyfibs=np.where(fibermags[i]==0.)[0]
                noflux=np.where(fibermags[i]==np.inf)[0]
                badmags=np.array(list(set(skyfibs) | set(noflux)))
                fibermags[i][badmags]=30.

            general_info['FIBER_MAGS']=fibermags

            #- Limit RA and DEC to 5 decimal places
            targetra=fibermap['TARGET_RA'][minfiber:maxfiber+1]
            general_info['RA']=[float("%.5f"%ra) for ra in targetra]
            targetdec=fibermap['TARGET_DEC'][minfiber:maxfiber+1]
            general_info['DEC']=[float("%.5f"%dec) for dec in targetdec]

            #- Find fibers in camera per target type
            elgfibers=np.where((fibermap['DESI_TARGET']&desi_mask.ELG)!=0)[0]
            general_info['ELG_FIBERID']=[elgfib for elgfib in elgfibers if minfiber <= elgfib <= maxfiber]
            lrgfibers=np.where((fibermap['DESI_TARGET']&desi_mask.LRG)!=0)[0]
            general_info['LRG_FIBERID']=[lrgfib for lrgfib in lrgfibers if minfiber <= lrgfib <= maxfiber]
            qsofibers=np.where((fibermap['DESI_TARGET']&desi_mask.QSO)!=0)[0]
            general_info['QSO_FIBERID']=[qsofib for qsofib in qsofibers if minfiber <= qsofib <= maxfiber]
            skyfibers=np.where((fibermap['DESI_TARGET']&desi_mask.SKY)!=0)[0]
            general_info['SKY_FIBERID']=[skyfib for skyfib in skyfibers if minfiber <= skyfib <= maxfiber]
            general_info['NSKY_FIB']=len(general_info['SKY_FIBERID'])
            stdfibers=np.where(isStdStar(fibermap))[0]
            general_info['STAR_FIBERID']=[stdfib for stdfib in stdfibers if minfiber <= stdfib <= maxfiber]

        general_info['EXPTIME']=raw[0].header['EXPTIME']
#        general_info['FITS_DESISPEC_VERION']=raw[0].header['FITS_DESISPEC_VERSION']
#        general_info['PROC_DESISPEC_VERION']=raw[0].header['PROC_DESISPEC_VERSION']
#        general_info['PROC_QuickLook_VERION']=raw[0].header['PROC_QuickLook_VERSION']

        #- Get peaks from configuration file
        if not flavor != 'arcs' and flavor in calibs:
            general_info['B_PEAKS']=peaks['B_PEAKS']
            general_info['R_PEAKS']=peaks['R_PEAKS']
            general_info['Z_PEAKS']=peaks['Z_PEAKS']

        #- Get current time information
        general_info['QLrun_datime_UTC']=datetime.datetime.now(tz=pytz.utc).isoformat()

        return (raw,general_info)

class Preproc(pas.PipelineAlg):
    #- TODO: currently io itself seems to have the preproc inside it. And preproc does bias, pi
     # xelflat, etc in one step.
    from desispec.maskbits import ccdmask

    def __init__(self,name,config,logger=None):
        if name is None or name.strip() == "":
            name="Preproc"

        rawtype=astropy.io.fits.hdu.hdulist.HDUList
        pas.PipelineAlg.__init__(self,name,rawtype,im,config,logger)

    def run(self,*args,**kwargs):
        if len(args) == 0 :
            #raise qlexceptions.ParameterException("Missing input parameter")
            log.critical("Missing input parameter!")
            sys.exit()

        if not self.is_compatible(type(args[0])):
            #raise qlexceptions.ParameterException("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))
            log.critical("Incompatible input!")
            sys.exit("Was expecting {} got {}".format(type(self.__inpType__),type(args[0])))

        input_raw=args[0][0]

        dumpfile=None
        if "dumpfile" in kwargs:
            dumpfile=kwargs["dumpfile"]

        if 'camera' not in kwargs:
            #raise qlexceptions.ParameterException("Need Camera to run preprocess on raw files")
            log.critical("Need Camera to run preprocess on raw files")
            sys.exit()

        else:
            camera=kwargs["camera"]
        if camera.upper() not in input_raw:
            raise IOError('Camera {} not in raw input'.format(camera))
        if "Bias" in kwargs:
            bias=kwargs["Bias"]
        else: bias=False

        if "Pixflat" in kwargs:
            pixflat=kwargs["Pixflat"]
        else: pixflat=False

        if "Mask" in kwargs:
            mask=kwargs["Mask"]
        else: mask=False

        return self.run_pa(input_raw,camera,bias=bias,pixflat=pixflat,mask=mask,dumpfile=dumpfile)

    def run_pa(self,input_raw,camera,bias=False,pixflat=False,mask=True,dumpfile='ttt1.fits'):
        import desispec.preproc

        rawimage=input_raw[camera.upper()].data
        header=input_raw[camera.upper()].header
        primary_header=input_raw[0].header
        if 'INHERIT' in header and header['INHERIT']:
            h0 = input_raw[0].header
            for key in h0:
                if key not in header:
                    header[key] = h0[key]
        #- WARNING!!!This is a hack for QL to run on old raw images for QLF to be working on old set of data
        #if "PROGRAM" not in header:
        #    log.warning("Temporary hack for QL to add header key PROGRAM. Only to facilitate QLF to work on their dataset. Remove this after some time and run with new data set")
        #    header["PROGRAM"]= 'dark'
        #if header["FLAVOR"] not in [None,'bias','arc','flat','science']:
        #    header["FLAVOR"] = 'science'

        img = desispec.preproc.preproc(rawimage,header,primary_header,bias=bias,pixflat=pixflat,mask=mask)

        if img.mask is not None :
            img.pix *= (img.mask==0)

        if dumpfile is not None:
            night = img.meta['NIGHT']
            expid = img.meta['EXPID']
            io.write_image(dumpfile, img)
            log.debug("Wrote intermediate file %s after %s"%(dumpfile,self.name))

        return img

class Flexure(pas.PipelineAlg):
    """ Use desi_compute_trace_shifts to output modified psf file
    """
    def __init__(self,name,config,logger=None):
        if name is None or name.strip() == "":
            name="Flexure"
        pas.PipelineAlg.__init__(self,name,im,fr,config,logger)

    def run(self,*args,**kwargs):
        if 'preprocFile' not in kwargs:
            #raise qlexceptions.ParameterException("Must provide preproc file for desi_compute_trace_shifts")
            log.critical("Must provide preproc file for desi_compute_trace_shifts")
            sys.exit()

        if 'inputPSFFile' not in kwargs:
            #raise qlexceptions.ParameterException("Must provide input psf file desi_compute_trace_shifts")
            log.critical("Must provide input psf file desi_compute_trace_shifts")
            sys.exit()

        if 'outputPSFFile' not in kwargs:
            #raise qlexceptions.ParameterException("Must provide output psf file")
            log.critical("Must provide output psf file")
            sys.exit()

        preproc_file=kwargs["preprocFile"]
        input_file=kwargs["inputPSFFile"]
        output_file=kwargs["outputPSFFile"]

        return self.run_pa(preproc_file,input_file,output_file,args)

    def run_pa(self,preproc_file,input_file,output_file,args):
        from desispec.util import runcmd
        #- Generate modified psf file
        cmd="desi_compute_trace_shifts --image {} --psf {} --outpsf {}".format(preproc_file,input_file,output_file)
        if not runcmd(cmd)[1]:
            raise RuntimeError('desi_compute_trace_shifts failed, psftrace not written')

        #- return image object to pass to boxcar for extraction
        img=args[0]
        return img

class BoxcarExtract(pas.PipelineAlg):
    from desispec.quicklook.qlboxcar import do_boxcar
    from desispec.maskbits import ccdmask

    def __init__(self,name,config,logger=None):
        if name is None or name.strip() == "":
            name="BoxcarExtract"
        pas.PipelineAlg.__init__(self,name,im,fr,config,logger)

    def run(self,*args,**kwargs):

        if len(args) == 0 :
            #raise qlexceptions.ParameterException("Missing input parameter")
            log.critical("Missing input parameter")
            sys.exit()

        if not self.is_compatible(type(args[0])):
            #raise qlexceptions.ParameterException("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))
            log.critical("Incompatible input!")
            sys.exit("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))

        if "PSFFile" not in kwargs:
            #raise qlexceptions.ParameterException("Need PSF File")
            log.critical("Need PSF File")
            sys.exit()

        input_image=args[0]

        dumpfile=None
        if "dumpfile" in kwargs:
            dumpfile=kwargs["dumpfile"]

        flavor=kwargs["Flavor"]

        psf_filename=kwargs["PSFFile"]
        #psf = PSF(psf_filename)
        tset = read_xytraceset(psf_filename)
        boxwidth=kwargs["BoxWidth"]
        nspec=kwargs["Nspec"]
        quickRes=kwargs["QuickResolution"] if "QuickResolution" in kwargs else False
        if "usesigma" in kwargs:
             usesigma=kwargs["usesigma"]
        else: usesigma = False

        if "Wavelength" not in kwargs:
            wstart = np.ceil(tset.wavemin)
            wstop = np.floor(tset.wavemax)
            dw = 0.5
        else:
            wavelength=kwargs["Wavelength"]
            if kwargs["Wavelength"] is not None: #- should be in wstart,wstop,dw format
                wstart, wstop, dw = [float(w) for w in wavelength]
            else:
                wstart = np.ceil(tset.wavemin)
                wstop = np.floor(tset.wavemax)
                dw = 0.5
        wave = np.arange(wstart, wstop+dw/2.0, dw)
        if "Specmin" not in kwargs:
            specmin=0
        else:
            specmin=kwargs["Specmin"]
            if kwargs["Specmin"] is None:
               specmin=0

        if "Nspec" not in kwargs:
            nspec = tset.nspec
        else:
            nspec=kwargs["Nspec"]
            if nspec is None:
                nspec=tset.nspec

        specmax = specmin + nspec

        camera = input_image.meta['CAMERA'].lower()     #- b0, r1, .. z9
        spectrograph = int(camera[1])
        fibermin = spectrograph*500 + specmin
        if "FiberMap" not in kwargs:
            fibermap = None
            fibers = np.arange(fibermin, fibermin+nspec, dtype='i4')
        else:
            fibermap=kwargs["FiberMap"]
            fibermap = fibermap[fibermin:fibermin+nspec]
            fibers = fibermap['FIBER']
        if "Outfile" in kwargs:
            outfile=kwargs["Outfile"]
        else:
            outfile=None
        maskFile=None
        if "MaskFile" in kwargs:
            maskFile=kwargs['MaskFile']

        #- Add some header keys relevant for this extraction
        input_image.meta['NSPEC']   = (nspec, 'Number of spectra')
        input_image.meta['WAVEMIN'] = (wstart, 'First wavelength [Angstroms]')
        input_image.meta['WAVEMAX'] = (wstop, 'Last wavelength [Angstroms]')
        input_image.meta['WAVESTEP']= (dw, 'Wavelength step size [Angstroms]')

        return self.run_pa(input_image,flavor,tset,wave,boxwidth,nspec,
                           fibers=fibers,fibermap=fibermap,dumpfile=dumpfile,
                           maskFile=maskFile,usesigma=usesigma,quick_resolution=quickRes)

    def run_pa(self,input_image,flavor,tset,outwave,boxwidth,nspec,
               fibers=None,fibermap=None,dumpfile=None,
               maskFile=None,usesigma=False,quick_resolution=False):
        from desispec.quicklook.qlboxcar import do_boxcar
        #import desispec.tset
        flux,ivar,Rdata=do_boxcar(input_image, tset, outwave, boxwidth=boxwidth,
                                  nspec=nspec,maskFile=maskFile,usesigma=usesigma,
                                  quick_resolution=quick_resolution)

        #- write to a frame object
        qndiag=21
        wsigma=None
        if quick_resolution:
            log.warning("deprecated, please use QFrame format to store sigma values")
            wsigma=np.zeros(flux.shape)
            if tset.ysig_vs_wave_traceset is not None :
                dw = np.gradient(outwave)
                for i in range(nspec):
                    ysig = tset.ysig_vs_wave(i,outwave)
                    y    = tset.y_vs_wave(i,outwave)
                    dydw = np.gradient(y)/dw
                    wsigma[i] = ysig/dydw # in A
        frame = fr(outwave, flux, ivar, resolution_data=Rdata,fibers=fibers,
                   meta=input_image.meta, fibermap=fibermap,
                   wsigma=wsigma,ndiag=qndiag)

        if dumpfile is not None:
            night = frame.meta['NIGHT']
            expid = frame.meta['EXPID']
            io.write_frame(dumpfile, frame)
            log.debug("Wrote intermediate file %s after %s"%(dumpfile,self.name))

        return frame

    def get_default_config(self):
        return {("BoxWidth",2.5,"Boxcar halfwidth"),
                ("PSFFile","%%PSFFile","PSFFile to use"),
                ("DeltaW",0.5,"Binwidth of extrapolated wavelength array"),
                ("Nspec",500,"number of spectra to extract")
                }

# TODO 2d extraction runs fine as well. Will need more testing of the setup.

class Extraction_2d(pas.PipelineAlg):
    """
       Offline 2D extraction for offline QuickLook
    """
    def __init__(self,name,config,logger=None):
        if name is None or name.strip() == "":
            name="2D Extraction" # using specter.extract.ex2d
        pas.PipelineAlg.__init__(self,name,im,fr,config,logger)

    def run(self,*args,**kwargs):

        if len(args) == 0 :
            #raise qlexceptions.ParameterException("Missing input parameter")
            log.critical("Missing input parameter")
            sys.exit()

        if not self.is_compatible(type(args[0])):
            #raise qlexceptions.ParameterException("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))
            log.critical("Incompatible input!")
            sys.exit("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))

        if "PSFFile_sp" not in kwargs:
            #raise qlexceptions.ParameterException("Need PSF File")
            log.critical("Need PSF File")
            sys.exit()

        from specter.psf import load_psf

        input_image=args[0]
        psffile=kwargs["PSFFile_sp"]
        psf=load_psf(psffile)

        if "Wavelength" not in kwargs:
            wstart = np.ceil(psf.wmin_all)
            wstop = np.floor(psf.wmax_all)
            dw = 0.5
        else:
            wavelength=kwargs["Wavelength"]
            if kwargs["Wavelength"] is not None: #- should be in wstart,wstop,dw format
                wstart, wstop, dw = [float(w) for w in wavelength]
            else:
                wstart = np.ceil(psf.wmin_all)
                wstop = np.floor(psf.wmax_all)
                dw = 0.5
        wave = np.arange(wstart, wstop+dw/2.0, dw)

        if "Specmin" not in kwargs:
            specmin=0
        else:
            specmin=kwargs["Specmin"]
            if kwargs["Specmin"] is None:
               specmin=0

        if "Nspec" not in kwargs:
            nspec = psf.nspec
        else:
            nspec=kwargs["Nspec"]
            if nspec is None:
                nspec=psf.nspec

        specmax = specmin + nspec

        camera = input_image.meta['CAMERA'].lower()     #- b0, r1, .. z9
        spectrograph = int(camera[1])
        fibermin = spectrograph*500 + specmin

        if "FiberMap" not in kwargs:
            fibermap = None
            fibers = np.arange(fibermin, fibermin+nspec, dtype='i4')
        else:
            fibermap=kwargs["FiberMap"]
            fibermap = fibermap[fibermin:fibermin+nspec]
            fibers = fibermap['FIBER']
        if "Regularize" in kwargs:
            regularize=kwargs["Regularize"]
        else:
            regularize=False
        if "ndecorr" in kwargs:
            ndecorr=ndecorr
        else:
            ndecorr=True

        bundlesize=25 #- hard coded

        if "Outfile" in kwargs:
            outfile=kwargs["Outfile"]
        else:
            outfile=None

        if "Nwavestep" in kwargs:
            wavesize=kwargs["Nwavestep"]
        else:
            wavesize=50

        return self.run_pa(input_image,psf,specmin,nspec,wave,regularize=regularize,ndecorr=ndecorr, bundlesize=bundlesize, wavesize=wavesize,outfile=outfile,fibers=fibers,fibermap=fibermap)

    def run_pa(self,input_image,psf,specmin,nspec,wave,regularize=None,ndecorr=True,bundlesize=25,wavesize=50, outfile=None,fibers=None,fibermap=None):
        import specter
        from specter.extract import ex2d

        flux,ivar,Rdata=ex2d(input_image.pix,input_image.ivar*(input_image.mask==0),psf,specmin,nspec,wave,regularize=regularize,ndecorr=ndecorr,bundlesize=bundlesize,wavesize=wavesize)

        #- Augment input image header for output
        input_image.meta['NSPEC']   = (nspec, 'Number of spectra')
        input_image.meta['WAVEMIN'] = (wave[0], 'First wavelength [Angstroms]')
        input_image.meta['WAVEMAX'] = (wave[-1], 'Last wavelength [Angstroms]')
        input_image.meta['WAVESTEP']= (wave[1]-wave[0], 'Wavelength step size [Angstroms]')
        input_image.meta['SPECTER'] = (specter.__version__, 'https://github.com/desihub/specter')
        #input_image.meta['IN_PSF']  = (_trim(psf_file), 'Input spectral PSF')
        #input_image.meta['IN_IMG']  = (_trim(input_file), 'Input image')

        frame = fr(wave, flux, ivar, resolution_data=Rdata,fibers=fibers, meta=input_image.meta, fibermap=fibermap)

        if outfile is not None:  #- writing to a frame file if needed.
            io.write_frame(outfile,frame)
            log.debug("wrote frame output file  %s"%outfile)

        return frame

class ComputeFiberflat(pas.PipelineAlg):
    """ PA to compute fiberflat field correction from a DESI continuum lamp frame
    """
    def __init__(self,name,config,logger=None):
        if name is None or name.strip() == "":
            name="ComputeFiberflat"
        pas.PipelineAlg.__init__(self,name,fr,fr,config,logger)

    def run(self,*args,**kwargs):
        if len(args) == 0 :
            #raise qlexceptions.ParameterException("Missing input parameter")
            log.critical("Missing input parameter")
            sys.exit()

        if not self.is_compatible(type(args[0])):
            #raise qlexceptions.ParameterException("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))
            log.critical("Incompatible input!")
            sys.exit("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))

        input_frame=args[0] #- frame object to calculate fiberflat from
        if "outputFile" not in kwargs:
            #raise qlexceptions.ParameterException("Need output file name to write fiberflat File")
            log.critical("Need output file name to write fiberflat File")
            sys.exit()

        outputfile=kwargs["outputFile"]

        return self.run_pa(input_frame,outputfile)

    def run_pa(self,input_frame,outputfile):
        from desispec.fiberflat import compute_fiberflat
        import desispec.io.fiberflat as ffIO
        fiberflat=compute_fiberflat(input_frame)
        ffIO.write_fiberflat(outputfile,fiberflat,header=input_frame.meta)
        log.debug("Fiberflat file wrtten. Exiting Quicklook for this configuration") #- File written no need to go further
        # !!!!! SAMI to whoever wrote this
        # PA's or any other components *CANNOT* call sys.exit()!! this needs to be fixed!!!!!
        sys.exit(0)

class ComputeFiberflat_QL(pas.PipelineAlg):
    """ PA to compute fiberflat field correction from a DESI continuum lamp frame
    """
    def __init__(self,name,config,logger=None):
        if name is None or name.strip() == "":
            name="ComputeFiberflat"
        pas.PipelineAlg.__init__(self,name,fr,fr,config,logger)

    def run(self,*args,**kwargs):
        if len(args) == 0 :
            #raise qlexceptions.ParameterException("Missing input parameter")
            log.critical("Missing input parameter")
            sys.exit()

        if not self.is_compatible(type(args[0])):
            #raise qlexceptions.ParameterException("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))
            log.critical("Incompatible input!")
            sys.exit("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))

        input_frame=args[0] #- frame object to calculate fiberflat from
        if "outputFile" not in kwargs:
            #raise qlexceptions.ParameterException("Need output file name to write fiberflat File")
            log.critical("Need output file name to write fiberflat File")
            sys.exit()

        outputfile=kwargs["outputFile"]

        return self.run_pa(input_frame,outputfile)

    def run_pa(self,frame,outputfile):
        from desispec.fiberflat import FiberFlat
        import desispec.io.fiberflat as ffIO
        from desispec.linalg import cholesky_solve
        nwave=frame.nwave
        nfibers=frame.nspec
        wave = frame.wave  #- this will become part of output too
        flux = frame.flux
        sumFlux=np.zeros((nwave))
        realFlux=np.zeros(flux.shape)
        ivar = frame.ivar*(frame.mask==0)
        #deconv
        for fib in range(nfibers):
            Rf=frame.R[fib].todense()
            B=flux[fib]
            try:
                realFlux[fib]=cholesky_solve(Rf,B)
            except:
                log.warning("cholesky_solve failed for fiber {}, using numpy.linalg.solve instead.".format(fib))
                realFlux[fib]=np.linalg.solve(Rf,B)
            sumFlux+=realFlux[fib]
        #iflux=nfibers/sumFlux
        flat = np.zeros(flux.shape)
        flat_ivar=np.zeros(ivar.shape)
        avg=sumFlux/nfibers
        for fib in range(nfibers):
            Rf=frame.R[fib]
            # apply and reconvolute
            M=Rf.dot(avg)
            M0=(M==0)
            flat[fib]=(~M0)*flux[fib]/(M+M0) +M0
            flat_ivar[fib]=ivar[fib]*M**2
        fibflat=FiberFlat(frame.wave.copy(),flat,flat_ivar,frame.mask.copy(),avg)

        #fiberflat=compute_fiberflat(input_frame)
        ffIO.write_fiberflat(outputfile,fibflat,header=frame.meta)
        log.info("Wrote fiberflat file {}".format(outputfile))

        fflatfile = ffIO.read_fiberflat(outputfile)

        return fflatfile

class ApplyFiberFlat(pas.PipelineAlg):
    """
       PA to Apply the fiberflat field to the given frame
    """
    def __init__(self,name,config,logger=None):
        if name is None or name.strip() == "":
            name="ApplyFiberFlat"
        pas.PipelineAlg.__init__(self,name,fr,fr,config,logger)

    def run(self,*args,**kwargs):
        if len(args) == 0 :
            #raise qlexceptions.ParameterException("Missing input parameter")
            log.critical("Missing input parameter")
            sys.exit()

        if not self.is_compatible(type(args[0])):
            #raise qlexceptions.ParameterException("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))
            log.critical("Incompatible input!")
            sys.exit("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))

        if "FiberFlatFile" not in kwargs:
            #raise qlexceptions.ParameterException("Need Fiberflat file")
            log.critical("Need Fiberflat file")
            sys.exit()

        input_frame=args[0]

        fiberflat=kwargs["FiberFlatFile"]

        return self.run_pa(input_frame,fiberflat)

    def run_pa(self,input_frame,fiberflat):

        from desispec.fiberflat import apply_fiberflat
        apply_fiberflat(input_frame,fiberflat)
        return input_frame

class ApplyFiberFlat_QL(pas.PipelineAlg):
    """
       PA to Apply the fiberflat field (QL) to the given frame
    """
    def __init__(self,name,config,logger=None):
        if name is None or name.strip() == "":
            name="Apply FiberFlat"
        pas.PipelineAlg.__init__(self,name,fr,fr,config,logger)

    def run(self,*args,**kwargs):
        if len(args) == 0 :
            #raise qlexceptions.ParameterException("Missing input parameter")
            log.critical("Missing input parameter!")
            sys.exit()

        if not self.is_compatible(type(args[0])):
            #raise qlexceptions.ParameterException("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))
            log.critical("Incompatible input!")
            sys.exit("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))

        if "FiberFlatFile" not in kwargs:
            #raise qlexceptions.ParameterException("Need Fiberflat file")
            log.critical("Need Fiberflat file")
            sys.exit()

        input_frame=args[0]

        dumpfile=None
        if "dumpfile" in kwargs:
            dumpfile=kwargs["dumpfile"]

        fiberflat=kwargs["FiberFlatFile"]

        return self.run_pa(input_frame,fiberflat,dumpfile=dumpfile)

    def run_pa(self,input_frame,fiberflat,dumpfile=None):

        from desispec.quicklook.quickfiberflat import apply_fiberflat
        fframe=apply_fiberflat(input_frame,fiberflat)

        if dumpfile is not None:
            night = fframe.meta['NIGHT']
            expid = fframe.meta['EXPID']
            io.write_frame(dumpfile, fframe)
            log.debug("Wrote intermediate file %s after %s"%(dumpfile,self.name))

        return fframe

class ComputeSky(pas.PipelineAlg):
    """ PA to compute sky model from a DESI frame
    """
    def __init__(self,name,config,logger=None):
        if name is None or name.strip() == "":
            name="ComputeSky"
        pas.PipelineAlg.__init__(self,name,fr,fr,config,logger)

    def run(self,*args,**kwargs):
        if len(args) == 0 :
            #raise qlexceptions.ParameterException("Missing input parameter")
            log.critical("Missing input parameter!")
            sys.exit()

        if not self.is_compatible(type(args[0])):
            #raise qlexceptions.ParameterException("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))
            log.critical("Incompatible input!")
            sys.exit("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))

        if "FiberFlatFile" not in kwargs: #- need this as fiberflat has to apply to frame first
            #raise qlexceptions.ParameterException("Need Fiberflat frame file")
            log.critical("Need Fiberflat frame file!")
            sys.exit()

        input_frame=args[0] #- frame object to calculate sky from
        if "FiberMap" in kwargs:
            fibermap=kwargs["FiberMap"]
        if "Outfile" not in kwargs:
            #raise qlexceptions.ParameterException("Need output file name to write skymodel")
            log.critical("Incompatible input!")
            sys.exit("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))

        fiberflat=kwargs["FiberFlatFile"]
        outputfile=kwargs["Outfile"]
        return self.run_pa(input_frame,fiberflat,outputfile)

    def run_pa(self,input_frame,fiberflat,outputfile):
        from desispec.fiberflat import apply_fiberflat
        from desispec.sky import compute_sky
        from desispec.io.sky import write_sky

        #- First apply fiberflat to sky fibers
        apply_fiberflat(input_frame,fiberflat)

        #- calculate the model
        skymodel=compute_sky(input_frame)
        write_sky(outputfile,skymodel,input_frame.meta)
        log.debug("Sky Model file wrtten. Exiting pipeline for this configuration")
        sys.exit(0)

class ComputeSky_QL(pas.PipelineAlg):
    """ PA to compute sky model from a DESI frame
    """
    def __init__(self,name,config,logger=None):
        if name is None or name.strip() == "":
            name="ComputeSky_QL"
        pas.PipelineAlg.__init__(self,name,fr,fr,config,logger)

    def run(self,*args,**kwargs):
        if len(args) == 0 :
            #raise qlexceptions.ParameterException("Missing input parameter")
            log.critical("Missing input parameter!")
            sys.exit()

        if not self.is_compatible(type(args[0])):
            #raise qlexceptions.ParameterException("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))
            log.critical("Incompatible input!")
            sys.exit("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))

        input_frame=args[0] #- frame object to calculate sky from. Should be fiber flat corrected
        if "FiberMap" in kwargs:
            fibermap=kwargs["FiberMap"]
        else: fibermap=None

        if "Apply_resolution" in kwargs:
            apply_resolution=kwargs["Apply_resolution"]

        if "Outfile" not in kwargs:
            #raise qlexceptions.ParameterException("Need output file name to write skymodel")
            log.critical("Need output file name to write skymodel!")
            sys.exit()

        outputfile=kwargs["Outfile"]
        return self.run_pa(input_frame,outputfile,fibermap=fibermap,apply_resolution=apply_resolution)

    def run_pa(self,input_frame,outputfile,fibermap=None,apply_resolution=False): #- input frame should be already fiberflat fielded
        from desispec.io.sky import write_sky
        from desispec.quicklook.quicksky import compute_sky

        skymodel=compute_sky(input_frame,fibermap,apply_resolution=apply_resolution)

        write_sky(outputfile,skymodel,input_frame.meta)
        # SEE ABOVE COMMENT!!!!
        log.debug("Sky Model file wrtten. Exiting the pipeline for this configuration")
        sys.exit(0)

class SkySub(pas.PipelineAlg):

    def __init__(self,name,config,logger=None):
        if name is None or name.strip() == "":
            name="SkySub"
        pas.PipelineAlg.__init__(self,name,fr,fr,config,logger)

    def run(self,*args,**kwargs):
        if len(args) == 0 :
            #raise qlexceptions.ParameterException("Missing input parameter")
            log.critical("Missing input parameter!")
            sys.exit()

        if not self.is_compatible(type(args[0])):
            #raise qlexceptions.ParameterException("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))
            log.critical("Incompatible input!")
            sys.exit("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))

        if "SkyFile" not in kwargs:
            #raise qlexceptions.ParameterException("Need Skymodel file")
            log.critical("Need Skymodel file!")
            sys.exit()

        input_frame=args[0] #- this must be flat field applied before sky subtraction in the pipeline
        skyfile=kwargs["SkyFile"]    #- Read sky model file itself from an argument
        from desispec.io.sky import read_sky
        skymodel=read_sky(skyfile)

        return self.run_pa(input_frame,skymodel)

    def run_pa(self,input_frame,skymodel):
        from desispec.sky import subtract_sky
        subtract_sky(input_frame,skymodel)
        return (input_frame, skymodel)

class SkySub_QL(pas.PipelineAlg):
    """
       This is for QL Sky subtraction. The input frame object should be fiber flat corrected.
       Unlike offline, if no skymodel file is given as input, a sky compute method is called
       to create a skymodel object and then subtraction is performed. Outputing that skymodel
       to a file is optional and can be configured.
    """
    def __init__(self,name,config,logger=None):
        if name is None or name.strip() == "":
            name="SkySub_QL"
        pas.PipelineAlg.__init__(self,name,fr,type(tuple),config,logger)

    def run(self,*args,**kwargs):
        if len(args) == 0 :
            #raise qlexceptions.ParameterException("Missing input parameter")
            log.critical("Missing input parameter!")
            sys.exit()

        if not self.is_compatible(type(args[0])):
            #raise qlexceptions.ParameterException("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))
            log.critical("Incompatible input!")
            sys.exit("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))

        input_frame=args[0] #- this must be flat field applied before sky subtraction in the pipeline

        dumpfile=None
        if "dumpfile" in kwargs:
            dumpfile=kwargs["dumpfile"]

        if "SkyFile" in kwargs:
            from desispec.io.sky import read_sky
            skyfile=kwargs["SkyFile"]    #- Read sky model file itself from an argument
            log.debug("Using given sky file %s for subtraction"%skyfile)

            skymodel=read_sky(skyfile)

        else:
            if "Outskyfile" in kwargs:
                outskyfile=kwargs["Outskyfile"]
            else: outskyfile=None

            log.debug("No sky file given. Computing sky first")
            from desispec.quicklook.quicksky import compute_sky
            if "Apply_resolution" in kwargs:
                apply_resolution=kwargs["Apply_resolution"]
                log.debug("Apply fiber to fiber resolution variation in computing sky")
            else: apply_resolution = False
            fibermap=input_frame.fibermap
            skymodel=compute_sky(input_frame,fibermap,apply_resolution=apply_resolution)
            if outskyfile is not None:
                from desispec.io.sky import write_sky
                log.debug("writing an output sky model file %s "%outskyfile)
                write_sky(outskyfile,skymodel,input_frame.meta)

        #- now do the subtraction
        return self.run_pa(input_frame,skymodel,dumpfile=dumpfile)

    def run_pa(self,input_frame,skymodel,dumpfile=None):
        from desispec.quicklook.quicksky import subtract_sky
        sframe=subtract_sky(input_frame,skymodel)

        if dumpfile is not None:
            night = sframe.meta['NIGHT']
            expid = sframe.meta['EXPID']
            io.write_frame(dumpfile, sframe)
            log.debug("Wrote intermediate file %s after %s"%(dumpfile,self.name))

        return (sframe,skymodel)

class ApplyFluxCalibration(pas.PipelineAlg):
    """PA to apply flux calibration to the given sframe
    """
    def __init__(self,name,config,logger=None):
        if name is None or name.strip() == "":
            name="Apply Flux Calibration"
        pas.PipelineAlg.__init__(self,name,fr,fr,config,logger)

    def run(self,*args,**kwargs):
        if len(args) == 0 :
            log.critical("Missing input parameter!")
            sys.exit()

        if not self.is_compatible(type(args[0][0])):
            log.critical("Incompatible input!")
            sys.exit("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0][0])))

        input_frame=args[0][0]

        if "outputfile" in kwargs:
            outputfile=kwargs["outputfile"]
        else:
            log.critical("Must provide output file to write cframe")
            sys.exit()

        return self.run_pa(input_frame,outputfile=outputfile)

    def run_pa(self,frame,outputfile=None):
        night=frame.meta['NIGHT']
        camera=frame.meta['CAMERA']
        expid=frame.meta['EXPID']

        rawfile=findfile('raw',night,expid,rawdata_dir=os.environ['QL_SPEC_DATA'])
        rawfits=fits.open(rawfile)
        primary_header=rawfits[0].header
        image=read_raw(rawfile,camera)

        fluxcalib_filename=findcalibfile([image.meta,primary_header],"FLUXCALIB")

        fluxcalib = read_average_flux_calibration(fluxcalib_filename)
        log.info("read average calib in {}".format(fluxcalib_filename))
        seeing  = frame.meta["SEEING"]
        airmass = frame.meta["AIRMASS"]
        exptime = frame.meta["EXPTIME"]
        exposure_calib = fluxcalib.value(seeing=seeing,airmass=airmass)
        for q in range(frame.nspec) :
            fiber_calib=np.interp(frame.wave[q],fluxcalib.wave,exposure_calib)*exptime
            inv_calib = (fiber_calib>0)/(fiber_calib + (fiber_calib==0))
            frame.flux[q] *= inv_calib
            frame.ivar[q] *= fiber_calib**2*(fiber_calib>0)

        write_qframe(outputfile,frame)
        log.info("Wrote flux calibrated frame file %s after %s"%(outputfile,self.name))

        return frame

class ResolutionFit(pas.PipelineAlg):

    """
    Fitting of Arc lines on extracted arc spectra, polynomial expansion of the fitted sigmas, and updating
    the coefficients to the new traceset file
    """
    def __init__(self,name,config,logger=None):
        if name is None or name.strip() == "":
            name="ResolutionFit"
        pas.PipelineAlg.__init__(self,name,fr,fr,config,logger)

    def run(self,*args,**kwargs):
        if len(args) == 0 :
            #raise qlexceptions.ParameterException("Missing input parameter")
            log.critical("Missing input parameter!")
            sys.exit()

        if not self.is_compatible(type(args[0])):
            #raise qlexceptions.ParameterException("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))
            log.critical("Incompatible input!")
            sys.exit("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))

        if "PSFoutfile" not in kwargs:
            #raise qlexceptions.ParameterException("Missing psfoutfile in the arguments")
            log.critical("Missing psfoutfile in the arguments!")
            sys.exit()

        psfoutfile=kwargs["PSFoutfile"]
        psfinfile=kwargs["PSFinputfile"]

        if "usesigma" in kwargs:
             usesigma=kwargs["usesigma"]
        else: usesigma = False

        tset = read_xytraceset(psfinfile)
        domain=(tset.wavemin,tset.wavemax)

        input_frame=args[0]

        linelist=None
        if "Linelist" in kwargs:
            linelist=kwargs["Linelist"]

        npoly=2
        if "NPOLY" in kwargs:
            npoly=kwargs["NPOLY"]
        nbins=2
        if "NBINS" in kwargs:
            nbins=kwargs["NBINS"]

        return self.run_pa(input_frame,psfinfile,psfoutfile,usesigma,linelist=linelist,npoly=npoly,nbins=nbins,domain=domain)

    def run_pa(self,input_frame,psfinfile,outfile,usesigma,linelist=None,npoly=2,nbins=2,domain=None):
        from desispec.quicklook.arcprocess import process_arc,write_psffile
        from desispec.quicklook.palib import get_resolution

        wcoeffs,wavemin,wavemax =process_arc(input_frame,linelist=linelist,npoly=npoly,nbins=nbins,domain=domain)
        write_psffile(psfinfile,wcoeffs,wavemin,wavemax,outfile)
        log.debug("Wrote xytraceset file {}".format(outfile))

        #- update the arc frame resolution from new coeffs
        tset = read_xytraceset(outfile)
        input_frame.resolution_data=get_resolution(input_frame.wave,input_frame.nspec,tset,usesigma=usesigma)

        return (tset,input_frame)

# =======================
# qproc algorithms
# =======================

from desispec.sky import SkyModel
from desispec.qproc.io import write_qframe
from desispec.qproc.qextract import qproc_boxcar_extraction
from desispec.qproc.qfiberflat import qproc_apply_fiberflat
from desispec.qproc.qsky import qproc_sky_subtraction

class Extract_QP(pas.PipelineAlg):

    def __init__(self,name,config,logger=None):
        if name is None or name.strip() == "":
            name="Extract_QP"
        pas.PipelineAlg.__init__(self,name,im,fr,config,logger)

    def run(self,*args,**kwargs):

        if len(args) == 0 :
            #raise qlexceptions.ParameterException("Missing input parameter")
            log.critical("Missing input parameter!")
            sys.exit()

        if not self.is_compatible(type(args[0])):
            #raise qlexceptions.ParameterException("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))
            log.critical("Incompatible input!")
            sys.exit("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))

        if "PSFFile" not in kwargs:
            #raise qlexceptions.ParameterException("Need PSF File")
            log.critical("Need PSF file!")
            sys.exit()

        input_image=args[0]

        dumpfile=None
        if "dumpfile" in kwargs:
            dumpfile=kwargs["dumpfile"]

        psf_filename=kwargs["PSFFile"]
        print("psf_filename=",psf_filename)

        traceset = read_xytraceset(psf_filename)

        width=kwargs["FullWidth"]
        nspec=kwargs["Nspec"]
        if "Wavelength" not in kwargs:
            wstart = np.ceil(traceset.wavemin)
            wstop = np.floor(traceset.wavemax)
            dw = 0.5
        else:
            wavelength=kwargs["Wavelength"]
            print('kwargs["Wavelength"]=',kwargs["Wavelength"])
            if kwargs["Wavelength"] is not None: #- should be in wstart,wstop,dw format
                wstart, wstop, dw = [float(w) for w in wavelength]
            else:
                wstart = np.ceil(traceset.wmin)
                wstop = np.floor(traceset.wmax)
                dw = 0.5
        wave = np.arange(wstart, wstop+dw/2.0, dw)
        if "Specmin" not in kwargs:
            specmin=0
        else:
            specmin=kwargs["Specmin"]
            if kwargs["Specmin"] is None:
               specmin=0

        if "Nspec" not in kwargs:
            nspec = traceset.nspec
        else:
            nspec=kwargs["Nspec"]
            if nspec is None:
                nspec=traceset.nspec

        specmax = specmin + nspec

        camera = input_image.meta['CAMERA'].lower()     #- b0, r1, .. z9
        spectrograph = int(camera[1])
        fibermin = spectrograph*500 + specmin
        if "FiberMap" not in kwargs:
            fibermap = None
            fibers = np.arange(fibermin, fibermin+nspec, dtype='i4')
        else:
            fibermap=kwargs["FiberMap"]
            fibermap = fibermap[fibermin:fibermin+nspec]
            fibers = fibermap['FIBER']
        if "Outfile" in kwargs:
            outfile=kwargs["Outfile"]
        else:
            outfile=None
        maskFile=None
        if "MaskFile" in kwargs:
            maskFile=kwargs['MaskFile']

        #- Add some header keys relevant for this extraction
        input_image.meta['NSPEC']   = (nspec, 'Number of spectra')
        input_image.meta['WAVEMIN'] = (wstart, 'First wavelength [Angstroms]')
        input_image.meta['WAVEMAX'] = (wstop, 'Last wavelength [Angstroms]')
        input_image.meta['WAVESTEP']= (dw, 'Wavelength step size [Angstroms]')

        return self.run_pa(input_image,traceset,wave,width,nspec,
                           fibers=fibers,fibermap=fibermap,dumpfile=dumpfile,
                           maskFile=maskFile)

    def run_pa(self,input_image,traceset,outwave,width,nspec,
               fibers=None,fibermap=None,dumpfile=None,
               maskFile=None):

        qframe = qproc_boxcar_extraction(traceset,input_image,fibers=fibers, width=width, fibermap=fibermap)

        if dumpfile is not None:
            write_qframe(dumpfile, qframe, fibermap=fibermap)
            log.debug("Wrote intermediate file %s after %s"%(dumpfile,self.name))

        return qframe

    def get_default_config(self):
        return {("FullWidth",7,"Boxcar full width"),
                ("PSFFile","%%PSFFile","PSFFile to use"),
                ("DeltaW",0.5,"Binwidth of extrapolated wavelength array"),
                ("Nspec",500,"number of spectra to extract")
                }

class ComputeFiberflat_QP(pas.PipelineAlg):
    def __init__(self,name,config,logger=None):
        if name is None or name.strip() == "":
            name="ComputeFiberflat"
        pas.PipelineAlg.__init__(self,name,fr,fr,config,logger)

    def run(self,*args,**kwargs):
        if len(args) == 0 :
            raise qlexceptions.ParameterException("Missing input parameter")
        if not self.is_compatible(type(args[0])):
            raise qlexceptions.ParameterException("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))
        input_frame=args[0] #- frame object to calculate fiberflat from
        if "outputFile" not in kwargs:
            raise qlexceptions.ParameterException("Need output file name to write fiberflat File")
        outputfile=kwargs["outputFile"]

        return self.run_pa(input_frame,outputfile)

    def run_pa(self,qframe,outputfile):
        from desispec.qproc.qfiberflat import qproc_compute_fiberflat
        import desispec.io.fiberflat as ffIO

        fibflat=qproc_compute_fiberflat(qframe)

        ffIO.write_fiberflat(outputfile,fibflat,header=qframe.meta)
        log.info("Wrote fiberflat file {}".format(outputfile))

        fflatfile = ffIO.read_fiberflat(outputfile)

        return fflatfile

class ApplyFiberFlat_QP(pas.PipelineAlg):
    """
       PA to Apply the fiberflat field (QP) to the given qframe
    """
    def __init__(self,name,config,logger=None):
        if name is None or name.strip() == "":
            name="Apply FiberFlat"
        pas.PipelineAlg.__init__(self,name,fr,fr,config,logger)

    def run(self,*args,**kwargs):
        if len(args) == 0 :
            #raise qlexceptions.ParameterException("Missing input parameter")
            log.critical("Missing input parameter!")
            sys.exit()

        if not self.is_compatible(type(args[0])):
            #raise qlexceptions.ParameterException("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))
            log.critical("Incompatible input!")
            sys.exit("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))

        if "FiberFlatFile" not in kwargs:
            #raise qlexceptions.ParameterException("Need Fiberflat file")
            log.critical("Need Fiberflat file!")
            sys.exit()

        input_qframe=args[0]

        dumpfile=None
        if "dumpfile" in kwargs:
            dumpfile=kwargs["dumpfile"]

        fiberflat=kwargs["FiberFlatFile"]

        return self.run_pa(input_qframe,fiberflat,dumpfile=dumpfile)

    def run_pa(self,qframe,fiberflat,dumpfile=None):

        qproc_apply_fiberflat(qframe,fiberflat)

        if dumpfile is not None:
            night = qframe.meta['NIGHT']
            expid = qframe.meta['EXPID']
            write_qframe(dumpfile, qframe)
            log.debug("Wrote intermediate file %s after %s"%(dumpfile,self.name))

        return qframe

class SkySub_QP(pas.PipelineAlg):
    """
       Sky subtraction. The input frame object should be fiber flat corrected.
       No sky model is saved for now
    """
    def __init__(self,name,config,logger=None):
        if name is None or name.strip() == "":
            name="SkySub_QP"
        pas.PipelineAlg.__init__(self,name,fr,type(tuple),config,logger)

    def run(self,*args,**kwargs):
        if len(args) == 0 :
            #raise qlexceptions.ParameterException("Missing input parameter")
            log.critical("Missing input parameter!")
            sys.exit()
        if not self.is_compatible(type(args[0])):
            #raise qlexceptions.ParameterException("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))
            log.critical("Incompatible input!")
            sys.exit("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0])))

        input_qframe=args[0] #- this must be flat field applied before sky subtraction in the pipeline

        dumpfile=None
        if "dumpfile" in kwargs:
            dumpfile=kwargs["dumpfile"]

        #- now do the subtraction
        return self.run_pa(input_qframe,dumpfile=dumpfile)

    def run_pa(self,qframe,dumpfile=None):

        skymodel = qproc_sky_subtraction(qframe,return_skymodel=True)
        #qproc_sky_subtraction(qframe)

        if dumpfile is not None:
            night = qframe.meta['NIGHT']
            expid = qframe.meta['EXPID']
            write_qframe(dumpfile, qframe)
            log.debug("Wrote intermediate file %s after %s"%(dumpfile,self.name))

        # convert for QA
#        sframe=qframe.asframe()
#        tmpsky=np.interp(sframe.wave,qframe.wave[0],skymodel[0])
#        skymodel = SkyModel(sframe.wave,np.tile(tmpsky,(sframe.nspec,1)),np.ones(sframe.flux.shape),np.zeros(sframe.flux.shape,dtype="int32"))

        return (qframe,skymodel)


text:
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">

<title>SWI-Prolog 8.3.28 Reference Manual: Section 1.1</title><link rel="home" href="index.html">
<link rel="contents" href="Contents.html">
<link rel="index" href="DocIndex.html">
<link rel="summary" href="summary.html">
<link rel="previous" href="intro.html">
<link rel="next" href="status.html">

1.1 Positioning
SWI-Prolog

Most implementations of the Prolog language are designed to serve a
limited set of use cases. SWI-Prolog is no exception to this rule.
SWI-Prolog positions itself primarily as a Prolog environment for‘programming
in the large' and use cases where it plays a central role in an
application, i.e., where it acts as‘glue' between components. At
the same time, SWI-Prolog aims at providing a productive rapid
prototyping environment. Its orientation towards programming in the
large is backed up by scalability, compiler speed, program structuring
(modules), support for multithreading to accommodate servers, Unicode
and interfaces to a large number of document formats, protocols and
programming languages. Prototyping is facilitated by good development
tools, both for command line usage and for usage with graphical
development tools. Demand loading of predicates from the library and a‘make'
facility avoids the requirement for using declarations and
reduces typing.

SWI-Prolog is traditionally strong in education because it is free
and portable, but also because of its compatibility with textbooks and
its easy-to-use environment.

Note that these positions do not imply that the system cannot be used
with other scenarios. SWI-Prolog is used as an embedded language where
it serves as a small rule subsystem in a large application. It is also
used as a deductive database. In some cases, this is the right choice
because SWI-Prolog has features that are required in the application,
such as threading or Unicode support. In general though, for example:
GNU-Prolog is more suited for embedding because it is small and can
compile to native code; XSB is better for deductive databases because it
provides a mature implementation of tabling including support
for incremental updates and Well Founded Semantics1Sponsored
by Kyndi and with help from the XSB developers Theresa Swift and David
S. Warren, SWI-Prolog now supports many of the XSB features.;
and ECLiPSe is better at constraint handling.

The syntax and set of built-in predicates is based on the ISO
standard
Hodgson,
1998. Most extensions follow the‘Edinburgh tradition'
(DEC10 Prolog and C-Prolog) and Quintus Prolog Qui,
1997. The infrastructure for constraint programming is based
on hProlog
Demoen, 2002.
Some libraries are copied from the YAP2http://www.dcc.fc.up.pt/\~{}vsc/Yap/
system. Together with YAP, we developed a portability framework (see
section C). This framework
has been filled for SICStus Prolog, YAP, IF/Prolog and Ciao. SWI-Prolog
version 7 introduces various extensions to the Prolog language (see section
5). The
string data type and its supporting set of built-in predicates
is compatible with ECLiPSe.


text:
package org.apache.tamaya.model.spi;

import org.apache.tamaya.Configuration;
import org.apache.tamaya.model.Validation;
import org.apache.tamaya.model.ValidationResult;
import org.apache.tamaya.model.ValidationType;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Objects;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 * Default configuration Model for a configuration parameter.
 */
public class ParameterValidation extends AbstractValidation {
    /** Optional regular expression for validating the value. */
    private String regEx;
    /** The target type into which the value must be convertible. */
    private Class type;

    /**
     * Internal constructor.
     * @param builder the builder, not null.
     */
    protected ParameterValidation(Builder builder) {
        super(builder.name, builder.required, builder.description, builder.provider);
        this.regEx = builder.regEx;
        this.type = builder.type;
    }

    @Override
    public ValidationType getType() {
        return ValidationType.Parameter;
    }

    /**
     * Get the required parameter type.
     *
     * @return the type.
     */
    public Class getParameterType() {
        return type;
    }

    @Override
    public Collection<ValidationResult> validate(Configuration config) {
        List<ValidationResult> result = new ArrayList<>(1);
        String configValue = config.get(getName());
        if (configValue == null && isRequired()) {
            result.add(ValidationResult.ofMissing(this));
        }
        if (configValue != null && regEx != null) {
            if (!configValue.matches(regEx)) {
                result.add(ValidationResult.ofError(this, "Config value not matching expression: " + regEx + ", was " +
                        configValue));
            }
        }
        return result;
    }

    @Override
    public String toString() {
        StringBuilder b = new StringBuilder();
        b.append(getType()).append(": ").append(getName());
        if (isRequired()) {
            b.append(", required: " + isRequired());
        }
        if (regEx != null) {
            b.append(", expression: " + regEx);
        }
        return b.toString();
    }

    /**
     * Creates a new Builder instance.
     * @param name the fully qualified parameter name.
     * @return
     */
    public static Builder builder(String name) {
        return new Builder(name);
    }

    /**
     * Creates a new Validation
     * @param name the fully qualified parameter name.
     * @param required the required flag.
     * @param expression an optional regular expression to validate a value.
     * @return the new Validation instance.
     */
    public static Validation of(String name, boolean required, String expression) {
        return new Builder(name).setRequired(required).setExpression(expression).build();
    }

    /**
     * Creates a new Validation
     * @param name the fully qualified parameter name.
     * @param required the required flag.
     * @return the new Validation instance.
     */
    public static Validation of(String name, boolean required) {
        return new Builder(name).setRequired(required).build();
    }

    /**
     * Creates a new Validation. The parameter will be defined as optional.
     * @param name the fully qualified parameter name.
     * @return the new Validation instance.
     */
    public static Validation of(String name) {
        return new Builder(name).setRequired(false).build();
    }

    /**
     * A new Builder for creating ParameterValidation instances.
     */
    public static class Builder {
        /** The parameter's target type. */
        private Class type;
        /** The fully qualified parameter name. */
        private String name;
        /** The optional provider. */
        private String provider;
        /** The optional validation expression. */
        private String regEx;
        /** The optional description. */
        private String description;
        /** The required flag. */
        private boolean required;

        /**
         * Creates a new Builder.
         * @param name the fully qualified parameter name, not null.
         */
        public Builder(String name) {
            this.name = Objects.requireNonNull(name);
        }

        /**
         * Sets the target type.
         * @param type the type, not null.
         * @return the Builder for chaining
         */
        public Builder setType(String type) {
            try {
                this.type = Class.forName(type);
            } catch (ClassNotFoundException e) {
                try {
                    this.type = Class.forName("java.lang."+type);
                } catch (ClassNotFoundException e2) {
                    Logger.getLogger(getClass().getName()).log(Level.WARNING, "Failed to load parameter type: " + type, e2);
                }
            }
            return this;
        }

        /**
         * Sets the required flag.
         * @param required the required flag.
         * @return the Builder for chaining
         */
        public Builder setRequired(boolean required) {
            this.required = required;
            return this;
        }

        /**
         * Sets the optional description
         * @param description the description
         * @return the Builder for chaining
         */
        public Builder setDescription(String description) {
            this.description = description;
            return this;
        }

        /**
         * Sets the optional validation expression
         * @param expression the validation expression
         * @return the Builder for chaining
         */
        public Builder setExpression(String expression) {
            this.regEx = expression;
            return this;
        }

        /**
         * Sets the fully qualified parameter name.
         * @param name the fully qualified parameter name, not null.
         * @return the Builder for chaining
         */
        public Builder setName(String name) {
            this.name = Objects.requireNonNull(name);
            return this;
        }

        /**
         * Set the provider.
         * @param provider the provider.
         * @return the Builder for chaining
         */
        public Builder setProvider(String provider) {
            this.provider = provider;
            return this;
        }

        /**
         * Creates a new Validation with the given parameters.
         * @return a new Validation , never null.
         */
        public Validation build() {
            return new ParameterValidation(this);
        }
    }
}


text:
from direct.directnotify import DirectNotifyGlobal
from otp.avatar import DistributedAvatarAI
from toontown.battle import BattleExperienceAI
from toontown.toonbase import ToontownGlobals
from toontown.toonbase import ToontownBattleGlobals
from toontown.toon import InventoryBase
from toontown.battle import DistributedBattleFinalAI
from toontown.building import SuitPlannerInteriorAI
from toontown.battle import BattleBase
from pandac.PandaModules import *
import SuitDNA
import random
AllBossCogs = []

class DistributedBossCogAI(DistributedAvatarAI.DistributedAvatarAI):
    notify = DirectNotifyGlobal.directNotify.newCategory('DistributedBossCogAI')

    def __init__(self, air, dept):
        DistributedAvatarAI.DistributedAvatarAI.__init__(self, air)
        self.dept = dept
        self.dna = SuitDNA.SuitDNA()
        self.dna.newBossCog(self.dept)
        self.deptIndex = SuitDNA.suitDepts.index(self.dept)
        self.resetBattleCounters()
        self.looseToons = []
        self.involvedToons = []
        self.toonsA = []
        self.toonsB = []
        self.nearToons = []
        self.suitsA = []
        self.activeSuitsA = []
        self.suitsB = []
        self.activeSuitsB = []
        self.reserveSuits = []
        self.barrier = None
        self.keyStates = ['BattleOne',
         'BattleTwo',
         'BattleThree',
         'Victory']
        self.bossDamage = 0
        self.battleThreeStart = 0
        self.battleThreeDuration = 1800
        self.attackCode = None
        self.attackAvId = 0
        self.hitCount = 0
        AllBossCogs.append(self)
        return

    def delete(self):
        self.ignoreAll()
        if self in AllBossCogs:
            i = AllBossCogs.index(self)
            del AllBossCogs[i]
        return DistributedAvatarAI.DistributedAvatarAI.delete(self)

    def getDNAString(self):
        return self.dna.makeNetString()

    def avatarEnter(self):
        avId = self.air.getAvatarIdFromSender()
        self.addToon(avId)

    def avatarExit(self):
        avId = self.air.getAvatarIdFromSender()
        self.removeToon(avId)

    def avatarNearEnter(self):
        avId = self.air.getAvatarIdFromSender()
        if avId not in self.nearToons:
            self.nearToons.append(avId)

    def avatarNearExit(self):
        avId = self.air.getAvatarIdFromSender()
        try:
            self.nearToons.remove(avId)
        except:
            pass

    def __handleUnexpectedExit(self, avId):
        self.removeToon(avId)

    def addToon(self, avId):
        if avId not in self.looseToons and avId not in self.involvedToons:
            self.looseToons.append(avId)
            event = self.air.getAvatarExitEvent(avId)
            self.acceptOnce(event, self.__handleUnexpectedExit, extraArgs=[avId])

    def removeToon(self, avId):
        resendIds = 0
        try:
            self.looseToons.remove(avId)
        except:
            pass

        try:
            self.involvedToons.remove(avId)
            resendIds = 1
        except:
            pass

        try:
            self.toonsA.remove(avId)
        except:
            pass

        try:
            self.toonsB.remove(avId)
        except:
            pass

        try:
            self.nearToons.remove(avId)
        except:
            pass

        event = self.air.getAvatarExitEvent(avId)
        self.ignore(event)
        if not self.hasToons():
            taskMgr.doMethodLater(10, self.__bossDone, self.uniqueName('BossDone'))

    def __bossDone(self, task):
        self.b_setState('Off')
        messenger.send(self.uniqueName('BossDone'))
        self.ignoreAll()

    def hasToons(self):
        return self.looseToons or self.involvedToons

    def hasToonsAlive(self):
        alive = 0
        for toonId in self.involvedToons:
            toon = self.air.doId2do.get(toonId)
            if toon:
                hp = toon.getHp()
                if hp > 0:
                    alive = 1

        return alive

    def sendBattleIds(self):
        self.sendUpdate('setBattleIds', [self.battleNumber, self.battleAId, self.battleBId])

    def sendToonIds(self):
        self.sendUpdate('setToonIds', [self.involvedToons, self.toonsA, self.toonsB])

    def damageToon(self, toon, deduction):
        toon.takeDamage(deduction)
        if toon.getHp() <= 0:
            self.sendUpdate('toonDied', [toon.doId])
            empty = InventoryBase.InventoryBase(toon)
            toon.b_setInventory(empty.makeNetString())
            self.removeToon(toon.doId)

    def healToon(self, toon, increment):
        toon.toonUp(increment)

    def d_setBattleExperience(self):
        self.sendUpdate('setBattleExperience', self.getBattleExperience())

    def getBattleExperience(self):
        result = BattleExperienceAI.getBattleExperience(8, self.involvedToons, self.toonExp, self.toonSkillPtsGained, self.toonOrigQuests, self.toonItems, self.toonOrigMerits, self.toonMerits, self.toonParts, self.suitsKilled, self.helpfulToons)
        return result

    def b_setArenaSide(self, arenaSide):
        self.setArenaSide(arenaSide)
        self.d_setArenaSide(arenaSide)

    def setArenaSide(self, arenaSide):
        self.arenaSide = arenaSide

    def d_setArenaSide(self, arenaSide):
        self.sendUpdate('setArenaSide', [arenaSide])

    def b_setState(self, state):
        self.setState(state)
        self.d_setState(state)

    def d_setState(self, state):
        self.sendUpdate('setState', [state])

    def setState(self, state):
        self.demand(state)
        if self.air:
            if state in self.keyStates:
                self.air.writeServerEvent('bossBattle', self.doId, '%s|%s|%s|%s' % (self.dept,
                 state,
                 self.involvedToons,
                 self.formatReward()))

    def getState(self):
        return self.state

    def formatReward(self):
        return 'unspecified'

    def enterOff(self):
        self.resetBattles()
        self.resetToons()
        self.resetBattleCounters()

    def exitOff(self):
        pass

    def enterWaitForToons(self):
        self.acceptNewToons()
        self.barrier = self.beginBarrier('WaitForToons', self.involvedToons, 5, self.__doneWaitForToons)

    def __doneWaitForToons(self, toons):
        self.b_setState('Elevator')

    def exitWaitForToons(self):
        self.ignoreBarrier(self.barrier)

    def enterElevator(self):
        if self.notify.getDebug():
            for toonId in self.involvedToons:
                toon = simbase.air.doId2do.get(toonId)
                if toon:
                    self.notify.debug('%s. involved toon %s, %s/%s' % (self.doId,
                     toonId,
                     toon.getHp(),
                     toon.getMaxHp()))

        self.resetBattles()
        self.barrier = self.beginBarrier('Elevator', self.involvedToons, 30, self.__doneElevator)

    def __doneElevator(self, avIds):
        self.b_setState('Introduction')

    def exitElevator(self):
        self.ignoreBarrier(self.barrier)

    def enterIntroduction(self):
        self.resetBattles()
        self.arenaSide = None
        self.makeBattleOneBattles()
        self.barrier = self.beginBarrier('Introduction', self.involvedToons, 45, self.doneIntroduction)
        return

    def doneIntroduction(self, avIds):
        self.b_setState('BattleOne')

    def exitIntroduction(self):
        self.ignoreBarrier(self.barrier)
        for toonId in self.involvedToons:
            toon = simbase.air.doId2do.get(toonId)
            if toon:
                toon.b_setCogIndex(-1)

    def enterBattleOne(self):
        if self.battleA:
            self.battleA.startBattle(self.toonsA, self.suitsA)
        if self.battleB:
            self.battleB.startBattle(self.toonsB, self.suitsB)

    def exitBattleOne(self):
        self.resetBattles()

    def enterReward(self):
        self.resetBattles()
        self.barrier = self.beginBarrier('Reward', self.involvedToons, BattleBase.BUILDING_REWARD_TIMEOUT, self.__doneReward)

    def __doneReward(self, avIds):
        self.b_setState('Epilogue')

    def exitReward(self):
        pass

    def enterEpilogue(self):
        pass

    def exitEpilogue(self):
        pass

    def enterFrolic(self):
        self.resetBattles()

    def exitFrolic(self):
        pass

    def resetBattleCounters(self):
        self.battleNumber = 0
        self.battleA = None
        self.battleAId = 0
        self.battleB = None
        self.battleBId = 0
        self.arenaSide = None
        self.toonSkillPtsGained = {}
        self.toonExp = {}
        self.toonOrigQuests = {}
        self.toonItems = {}
        self.toonOrigMerits = {}
        self.toonMerits = {}
        self.toonParts = {}
        self.suitsKilled = []
        self.helpfulToons = []
        return

    def resetBattles(self):
        sendReset = 0
        if self.battleA:
            self.battleA.requestDelete()
            self.battleA = None
            self.battleAId = 0
            sendReset = 1
        if self.battleB:
            self.battleB.requestDelete()
            self.battleB = None
            self.battleBId = 0
            sendReset = 1
        for suit in self.suitsA + self.suitsB:
            suit.requestDelete()

        for suit, joinChance in self.reserveSuits:
            suit.requestDelete()

        self.suitsA = []
        self.activeSuitsA = []
        self.suitsB = []
        self.activeSuitsB = []
        self.reserveSuits = []
        self.battleNumber = 0
        if sendReset:
            self.sendBattleIds()
        return

    def resetToons(self):
        if self.toonsA or self.toonsB:
            self.looseToons = self.looseToons + self.involvedToons
            self.involvedToons = []
            self.toonsA = []
            self.toonsB = []
            self.sendToonIds()

    def divideToons(self):
        toons = self.involvedToons[:]
        random.shuffle(toons)
        numToons = min(len(toons), 8)
        if numToons < 4:
            numToonsB = numToons / 2
        else:
            numToonsB = (numToons + random.choice([0, 1])) / 2
        self.toonsA = toons[numToonsB:numToons]
        self.toonsB = toons[:numToonsB]
        self.looseToons += toons[numToons:]
        self.sendToonIds()

    def acceptNewToons(self):
        sourceToons = self.looseToons
        self.looseToons = []
        for toonId in sourceToons:
            toon = self.air.doId2do.get(toonId)
            if toon and not toon.ghostMode:
                self.involvedToons.append(toonId)
            else:
                self.looseToons.append(toonId)

        for avId in self.involvedToons:
            toon = self.air.doId2do.get(avId)
            if toon:
                p = []
                for t in ToontownBattleGlobals.Tracks:
                    p.append(toon.experience.getExp(t))

                self.toonExp[avId] = p
                self.toonOrigMerits[avId] = toon.cogMerits[:]

        self.divideToons()

    def initializeBattles(self, battleNumber, bossCogPosHpr):
        self.resetBattles()
        if not self.involvedToons:
            self.notify.warning('initializeBattles: no toons!')
            return
        self.battleNumber = battleNumber
        suitHandles = self.generateSuits(battleNumber)
        self.suitsA = suitHandles['activeSuits']
        self.activeSuitsA = self.suitsA[:]
        self.reserveSuits = suitHandles['reserveSuits']
        suitHandles = self.generateSuits(battleNumber)
        self.suitsB = suitHandles['activeSuits']
        self.activeSuitsB = self.suitsB[:]
        self.reserveSuits += suitHandles['reserveSuits']
        if self.toonsA:
            self.battleA = self.makeBattle(bossCogPosHpr, ToontownGlobals.BossCogBattleAPosHpr, self.handleRoundADone, self.handleBattleADone, battleNumber, 0)
            self.battleAId = self.battleA.doId
        else:
            self.moveSuits(self.activeSuitsA)
            self.suitsA = []
            self.activeSuitsA = []
            if self.arenaSide == None:
                self.b_setArenaSide(0)
        if self.toonsB:
            self.battleB = self.makeBattle(bossCogPosHpr, ToontownGlobals.BossCogBattleBPosHpr, self.handleRoundBDone, self.handleBattleBDone, battleNumber, 1)
            self.battleBId = self.battleB.doId
        else:
            self.moveSuits(self.activeSuitsB)
            self.suitsB = []
            self.activeSuitsB = []
            if self.arenaSide == None:
                self.b_setArenaSide(1)
        self.sendBattleIds()
        return

    def makeBattle(self, bossCogPosHpr, battlePosHpr, roundCallback, finishCallback, battleNumber, battleSide):
        battle = DistributedBattleFinalAI.DistributedBattleFinalAI(self.air, self, roundCallback, finishCallback, battleSide)
        self.setBattlePos(battle, bossCogPosHpr, battlePosHpr)
        battle.suitsKilled = self.suitsKilled
        battle.battleCalc.toonSkillPtsGained = self.toonSkillPtsGained
        battle.toonExp = self.toonExp
        battle.toonOrigQuests = self.toonOrigQuests
        battle.toonItems = self.toonItems
        battle.toonOrigMerits = self.toonOrigMerits
        battle.toonMerits = self.toonMerits
        battle.toonParts = self.toonParts
        battle.helpfulToons = self.helpfulToons
        mult = ToontownBattleGlobals.getBossBattleCreditMultiplier(battleNumber)
        battle.battleCalc.setSkillCreditMultiplier(mult)
        battle.generateWithRequired(self.zoneId)
        return battle

    def setBattlePos(self, battle, cogPosHpr, battlePosHpr):
        bossNode = NodePath('bossNode')
        bossNode.setPosHpr(*cogPosHpr)
        battleNode = bossNode.attachNewNode('battleNode')
        battleNode.setPosHpr(*battlePosHpr)
        suitNode = battleNode.attachNewNode('suitNode')
        suitNode.setPos(0, 1, 0)
        battle.pos = battleNode.getPos(NodePath())
        battle.initialSuitPos = suitNode.getPos(NodePath())

    def moveSuits(self, active):
        for suit in active:
            self.reserveSuits.append((suit, 0))

    def handleRoundADone(self, toonIds, totalHp, deadSuits):
        if self.battleA:
            self.handleRoundDone(self.battleA, self.suitsA, self.activeSuitsA, toonIds, totalHp, deadSuits)

    def handleRoundBDone(self, toonIds, totalHp, deadSuits):
        if self.battleB:
            self.handleRoundDone(self.battleB, self.suitsB, self.activeSuitsB, toonIds, totalHp, deadSuits)

    def handleBattleADone(self, zoneId, toonIds):
        if self.battleA:
            self.battleA.requestDelete()
            self.battleA = None
            self.battleAId = 0
            self.sendBattleIds()
        if self.arenaSide == None:
            self.b_setArenaSide(0)
        if not self.battleB and self.hasToons() and self.hasToonsAlive():
            self.b_setState(self.postBattleState)
        return

    def handleBattleBDone(self, zoneId, toonIds):
        if self.battleB:
            self.battleB.requestDelete()
            self.battleB = None
            self.battleBId = 0
            self.sendBattleIds()
        if self.arenaSide == None:
            self.b_setArenaSide(1)
        if not self.battleA and self.hasToons() and self.hasToonsAlive():
            self.b_setState(self.postBattleState)
        return

    def invokeSuitPlanner(self, buildingCode, skelecog):
        planner = SuitPlannerInteriorAI.SuitPlannerInteriorAI(1, buildingCode, self.dna.dept, self.zoneId)
        planner.respectInvasions = 0
        suits = planner.genFloorSuits(0)
        if skelecog:
            for suit in suits['activeSuits']:
                suit.b_setSkelecog(1)

            for reserve in suits['reserveSuits']:
                suit = reserve[0]
                suit.b_setSkelecog(1)

        return suits

    def generateSuits(self, battleNumber):
        raise StandardError, 'generateSuits unimplemented'

    def handleRoundDone(self, battle, suits, activeSuits, toonIds, totalHp, deadSuits):
        totalMaxHp = 0
        for suit in suits:
            totalMaxHp += suit.maxHP

        for suit in deadSuits:
            activeSuits.remove(suit)

        joinedReserves = []
        if len(self.reserveSuits) > 0 and len(activeSuits) < 4:
            hpPercent = 100 - totalHp / totalMaxHp * 100.0
            for info in self.reserveSuits:
                if info[1] <= hpPercent and len(activeSuits) < 4:
                    suits.append(info[0])
                    activeSuits.append(info[0])
                    joinedReserves.append(info)

            for info in joinedReserves:
                self.reserveSuits.remove(info)

        battle.resume(joinedReserves)

    def getBattleThreeTime(self):
        elapsed = globalClock.getFrameTime() - self.battleThreeStart
        t1 = elapsed / float(self.battleThreeDuration)
        return t1

    def progressValue(self, fromValue, toValue):
        t0 = float(self.bossDamage) / float(self.bossMaxDamage)
        elapsed = globalClock.getFrameTime() - self.battleThreeStart
        t1 = elapsed / float(self.battleThreeDuration)
        t = max(t0, t1)
        return fromValue + (toValue - fromValue) * min(t, 1)

    def progressRandomValue(self, fromValue, toValue, radius = 0.2):
        t = self.progressValue(0, 1)
        radius = radius * (1.0 - abs(t - 0.5) * 2.0)
        t += radius * random.uniform(-1, 1)
        t = max(min(t, 1.0), 0.0)
        return fromValue + (toValue - fromValue) * t

    def reportToonHealth(self):
        if self.notify.getDebug():
            str = ''
            for toonId in self.involvedToons:
                toon = self.air.doId2do.get(toonId)
                if toon:
                    str += ', %s (%s/%s)' % (toonId, toon.getHp(), toon.getMaxHp())

            self.notify.debug('%s.toons = %s' % (self.doId, str[2:]))

    def getDamageMultiplier(self):
        return 1.0

    def zapToon(self, x, y, z, h, p, r, bpx, bpy, attackCode, timestamp):
        avId = self.air.getAvatarIdFromSender()
        if not self.validate(avId, avId in self.involvedToons, 'zapToon from unknown avatar'):
            return
        if attackCode == ToontownGlobals.BossCogLawyerAttack and self.dna.dept != 'l':
            self.notify.warning('got lawyer attack but not in CJ boss battle')
            return
        toon = simbase.air.doId2do.get(avId)
        if toon:
            self.d_showZapToon(avId, x, y, z, h, p, r, attackCode, timestamp)
            damage = ToontownGlobals.BossCogDamageLevels.get(attackCode)
            if damage == None:
                self.notify.warning('No damage listed for attack code %s' % attackCode)
                damage = 5
            damage *= self.getDamageMultiplier()
            self.damageToon(toon, damage)
            currState = self.getCurrentOrNextState()
            if attackCode == ToontownGlobals.BossCogElectricFence and (currState == 'RollToBattleTwo' or currState == 'BattleThree'):
                if bpy < 0 and abs(bpx / bpy) > 0.5:
                    if bpx < 0:
                        self.b_setAttackCode(ToontownGlobals.BossCogSwatRight)
                    else:
                        self.b_setAttackCode(ToontownGlobals.BossCogSwatLeft)
        return

    def d_showZapToon(self, avId, x, y, z, h, p, r, attackCode, timestamp):
        self.sendUpdate('showZapToon', [avId,
         x,
         y,
         z,
         h,
         p,
         r,
         attackCode,
         timestamp])

    def b_setAttackCode(self, attackCode, avId = 0):
        self.d_setAttackCode(attackCode, avId)
        self.setAttackCode(attackCode, avId)

    def setAttackCode(self, attackCode, avId = 0):
        self.attackCode = attackCode
        self.attackAvId = avId
        if attackCode == ToontownGlobals.BossCogDizzy or attackCode == ToontownGlobals.BossCogDizzyNow:
            delayTime = self.progressValue(20, 5)
            self.hitCount = 0
        elif attackCode == ToontownGlobals.BossCogSlowDirectedAttack:
            delayTime = ToontownGlobals.BossCogAttackTimes.get(attackCode)
            delayTime += self.progressValue(10, 0)
        else:
            delayTime = ToontownGlobals.BossCogAttackTimes.get(attackCode)
            if delayTime == None:
                return
        self.waitForNextAttack(delayTime)
        return

    def d_setAttackCode(self, attackCode, avId = 0):
        self.sendUpdate('setAttackCode', [attackCode, avId])

    def waitForNextAttack(self, delayTime):
        currState = self.getCurrentOrNextState()
        if currState == 'BattleThree':
            taskName = self.uniqueName('NextAttack')
            taskMgr.remove(taskName)
            taskMgr.doMethodLater(delayTime, self.doNextAttack, taskName)

    def stopAttacks(self):
        taskName = self.uniqueName('NextAttack')
        taskMgr.remove(taskName)

    def doNextAttack(self, task):
        self.b_setAttackCode(ToontownGlobals.BossCogNoAttack)


text:
package ch.wsl.box.rest

import ch.wsl.box.jdbc.FullDatabase
import ch.wsl.box.model.shared.JSONID
import ch.wsl.box.rest.logic.FormActions
import ch.wsl.box.rest.metadata.FormMetadataFactory
import ch.wsl.box.rest.utils.UserProfile
import ch.wsl.box.services.Services
import ch.wsl.box.testmodel.EntityActionsRegistry
import ch.wsl.box.jdbc.PostgresProfile.api._
import _root_.io.circe._
import _root_.io.circe.syntax._
import _root_.io.circe.generic.auto._
import ch.wsl.box.BaseSpec
import ch.wsl.box.rest.fixtures.{DbManagedIdFixtures, FormFixtures}
import ch.wsl.box.shared.utils.JSONUtils.EnhancedJson
import ch.wsl.box.testmodel.Entities.Simple_row
import org.scalatest.Assertion

import scala.concurrent.Future

class UpdateWithDeleteFormSpec extends BaseSpec {

  def insert(formName:String,json:Json)(implicit services:Services):Future[(JSONID,Json)] = {

    implicit val up = UserProfile(services.connection.adminUser)
    implicit val fdb = FullDatabase(services.connection.adminDB,services.connection.adminDB)

    for{
      form <- up.db.run(FormMetadataFactory().of(formName,"it"))
      actions = FormActions(form,EntityActionsRegistry.apply,FormMetadataFactory())
      i <- up.db.run(actions.insert(json).transactionally)
      result <- up.db.run(actions.getById(JSONID.fromData(i,form).get))
    } yield {
      assert(i == result.get.dropBoxObjectId)
      (JSONID.fromData(i,form).get,result.get)
    }
  }

  def update(formName:String,id:JSONID,json:Json)(implicit services:Services) = {

    implicit val up = UserProfile(services.connection.adminUser)
    implicit val fdb = FullDatabase(services.connection.adminDB,services.connection.adminDB)

    for{
      form <- up.db.run(FormMetadataFactory().of(formName,"it"))
      actions = FormActions(form,EntityActionsRegistry.apply,FormMetadataFactory())
      i <- up.db.run(actions.update(id,json).transactionally)
      result <- up.db.run(actions.getById(id))
    } yield (i,result)
  }

  def upsert(formName:String,id:JSONID,json:Json)(implicit services:Services) = {

    implicit val up = UserProfile(services.connection.adminUser)
    implicit val fdb = FullDatabase(services.connection.adminDB,services.connection.adminDB)

    for{
      form <- up.db.run(FormMetadataFactory().of(formName,"it"))
      actions = FormActions(form,EntityActionsRegistry.apply,FormMetadataFactory())
      i <- up.db.run(actions.upsertIfNeeded(Some(id),json).transactionally)
      result <- up.db.run(actions.getById(id))
    } yield (i,result)
  }

  "Form"  should "update a row deleting a field"  in withServices[Assertion] { implicit services =>
      implicit val up = UserProfile(services.connection.adminUser)
      implicit val fdb = FullDatabase(services.connection.adminDB,services.connection.adminDB)

      val base = Simple_row(name = Some("test"))

       for{
         (formName,_) <- FormFixtures.insertSimple(up.db,services.executionContext)
         (idEntry,result) <- insert(formName,base.asJson)
         resultSimple = result.as[Simple_row].toOption.get
         resultWithDeletion = resultSimple.copy(name = None).asJson.dropBoxObjectId
         (r1,r2) <- update(formName,idEntry,resultWithDeletion)
       } yield {
         resultSimple.name shouldBe base.name
         r1 shouldBe r2.get.dropBoxObjectId
         r1 shouldBe resultWithDeletion
         r2.get.dropBoxObjectId shouldBe resultWithDeletion
       }
  }

  "Form with empty child"  should "update a row deleting a field"  in withServices[Assertion] { implicit services =>
    implicit val up = UserProfile(services.connection.adminUser)
    implicit val fdb = FullDatabase(services.connection.adminDB,services.connection.adminDB)

    val base = stringToJson(DbManagedIdFixtures.layers(1))

    for{
      (formName,_,_,_) <- new FormFixtures("db_").insertForm(up.db)
      (idEntry,result) <- insert(formName,base)
      resultWithDeletion = Json.fromFields(result.as[JsonObject].toOption.get.toList.filterNot(_._1 == "name")).dropBoxObjectId
      (r1,r2) <- update(formName,idEntry,resultWithDeletion)
    } yield {
      r1 shouldBe r2.get.dropBoxObjectId
      r1.dropNullValues shouldBe resultWithDeletion.dropNullValues
      r2.get.dropNullValues.dropBoxObjectId shouldBe resultWithDeletion.dropNullValues
    }
  }

  "Form with empty child"  should "upsert a row deleting a field"  in withServices[Assertion] { implicit services =>
    implicit val up = UserProfile(services.connection.adminUser)
    implicit val fdb = FullDatabase(services.connection.adminDB,services.connection.adminDB)

    val base = stringToJson(DbManagedIdFixtures.layers(1))

    for{
      (formName,_,_,_) <- new FormFixtures("db_").insertForm(up.db)
      (idEntry,result) <- insert(formName,base)
      resultWithDeletion = Json.fromFields(result.as[JsonObject].toOption.get.toList.filterNot(_._1 == "name")).dropBoxObjectId
      (r1,r2) <- upsert(formName,idEntry,resultWithDeletion)
    } yield {
      r1 shouldBe r2.get.dropBoxObjectId
      r1.dropNullValues shouldBe resultWithDeletion.dropNullValues
      r2.get.dropNullValues.dropBoxObjectId shouldBe resultWithDeletion.dropNullValues
    }
  }

}