text:
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System;
using System.Collections.Generic;
using Debug = System.Diagnostics.Debug;

using Internal.TypeSystem;
using Internal.TypeSystem.Ecma;
using ILCompiler.DependencyAnalysis;

namespace ILCompiler
{
    public abstract class PreInitFixupInfo : IComparable<PreInitFixupInfo>
    {
        /// <summary>
        /// Offset into the blob
        /// </summary>
        public int Offset { get; }

        public PreInitFixupInfo(int offset)
        {
            Offset = offset;
        }

        int IComparable<PreInitFixupInfo>.CompareTo(PreInitFixupInfo other)
        {
            return this.Offset - other.Offset;
        }

        /// <summary>
        /// Writes fixup data into current ObjectDataBuilder. Caller needs to make sure ObjectDataBuilder is
        /// at correct offset before writing.
        /// </summary>
        public abstract void WriteData(ref ObjectDataBuilder builder, NodeFactory factory);
    }

    public class PreInitTypeFixupInfo : PreInitFixupInfo
    {
        public TypeDesc TypeFixup { get; }

        public PreInitTypeFixupInfo(int offset, TypeDesc type)
            :base(offset)
        {
            TypeFixup = type;
        }

        public override void WriteData(ref ObjectDataBuilder builder, NodeFactory factory)
        {
            builder.EmitPointerRelocOrIndirectionReference(factory.NecessaryTypeSymbol(TypeFixup));
        }
    }

    public class PreInitMethodFixupInfo : PreInitFixupInfo
    {
        public MethodDesc MethodFixup { get; }

        public PreInitMethodFixupInfo(int offset, MethodDesc method)
            : base(offset)
        {
            MethodFixup = method;
        }

        public override void WriteData(ref ObjectDataBuilder builder, NodeFactory factory)
        {
            builder.EmitPointerReloc(factory.ExactCallableAddress(MethodFixup));
        }
    }

    public class PreInitFieldFixupInfo : PreInitFixupInfo
    {
        public FieldDesc FieldFixup { get; }

        public PreInitFieldFixupInfo(int offset, FieldDesc field)
            : base(offset)
        {
            FieldFixup = field;
        }

        public override void WriteData(ref ObjectDataBuilder builder, NodeFactory factory)
        {
            MetadataType type = (MetadataType)FieldFixup.OwningType;

            // Do not support fixing up fields from external modules
            if (!factory.CompilationModuleGroup.ContainsType(type))
                throw new BadImageFormatException();

            ISymbolNode staticBase = FieldFixup.HasGCStaticBase ? factory.TypeGCStaticsSymbol(type) : factory.TypeNonGCStaticsSymbol(type);
            builder.EmitPointerReloc(staticBase, FieldFixup.Offset.AsInt);
        }
    }

    public class PreInitFieldInfo
    {
        public FieldDesc Field { get; }

        /// <summary>
        /// The type of the real field data. This could be a subtype of the field type.
        /// </summary>
        public TypeDesc Type { get; }

        /// <summary>
        /// Points to the underlying contents of the data.
        /// </summary>
        public byte[] Data { get; }

        /// <summary>
        /// Start offset of the real contents in the data.
        /// </summary>
        public int Offset { get; }

        /// <summary>
        /// Number of elements, if this is a frozen array.
        /// </summary>
        public int Length { get; }

        /// <summary>
        /// List of fixup to be apply to the data blob
        /// This is needed for information that can't be encoded into blob ahead of time before codegen
        /// </summary>
        private List<PreInitFixupInfo> FixupInfos;

        public PreInitFieldInfo(FieldDesc field, TypeDesc type, byte[] data, int offset, int length, List<PreInitFixupInfo> fixups)
        {
            Field = field;
            Type = type;
            Data = data;
            Offset = offset;
            Length = length;
            FixupInfos = fixups;

            if (FixupInfos != null)
                FixupInfos.Sort();
        }

        public void WriteData(ref ObjectDataBuilder builder, NodeFactory factory, bool relocsOnly = false)
        {
            int offset = Offset;

            if (FixupInfos != null)
            {
                int startOffset = builder.CountBytes;

                for (int i = 0; i < FixupInfos.Count; ++i)
                {
                    var fixupInfo = FixupInfos[i];

                    // do we have overlapping fixups?
                    if (fixupInfo.Offset < offset)
                        throw new BadImageFormatException();

                    if (!relocsOnly)
                    {
                        // emit bytes before fixup
                        builder.EmitBytes(Data, offset, fixupInfo.Offset - offset);
                    }

                    // write the fixup
                    FixupInfos[i].WriteData(ref builder, factory);

                    // move pointer past the fixup
                    offset = Offset + builder.CountBytes - startOffset;
                }
            }

            if (offset > Data.Length)
                throw new BadImageFormatException();

            if (!relocsOnly)
            {
                // Emit remaining bytes
                builder.EmitBytes(Data, offset, Data.Length - offset);
            }
        }

        public static List<PreInitFieldInfo> GetPreInitFieldInfos(TypeDesc type, bool hasGCStaticBase)
        {
            List<PreInitFieldInfo> list = null;

            foreach (var field in type.GetFields())
            {
                if (!field.IsStatic || field.IsThreadStatic)
                    continue;

                if (field.HasGCStaticBase != hasGCStaticBase)
                    continue;

                var dataField = GetPreInitDataField(field);
                if (dataField != null)
                {
                    if (list == null)
                        list = new List<PreInitFieldInfo>();
                    list.Add(ConstructPreInitFieldInfo(field, dataField));
                }
            }

            return list;
        }

        /// <summary>
        /// Retrieves the corresponding static preinitialized data field by looking at various attributes
        /// </summary>
        private static FieldDesc GetPreInitDataField(FieldDesc thisField)
        {
            Debug.Assert(thisField.IsStatic);

            var field = thisField as EcmaField;
            if (field == null)
                return null;

            var decoded = field.GetDecodedCustomAttribute("System.Runtime.CompilerServices", "InitDataBlobAttribute");
            if (decoded == null)
                return null;

            var decodedValue = decoded.Value;
            if (decodedValue.FixedArguments.Length != 2)
                throw new BadImageFormatException();

            var typeDesc = decodedValue.FixedArguments[0].Value as TypeDesc;
            if (typeDesc == null)
                throw new BadImageFormatException();

            if (decodedValue.FixedArguments[1].Type != field.Context.GetWellKnownType(WellKnownType.String))
                throw new BadImageFormatException();

            var fieldName = (string)decodedValue.FixedArguments[1].Value;
            var dataField = typeDesc.GetField(fieldName);
            if (dataField== null)
                throw new BadImageFormatException();

            return dataField;
        }

        /// <summary>
        /// Extract preinitialize data as byte[] from a RVA field, and perform necessary validations.
        /// </summary>
        private static PreInitFieldInfo ConstructPreInitFieldInfo(FieldDesc field, FieldDesc dataField)
        {
            if (!dataField.HasRva)
                throw new BadImageFormatException();

            var ecmaDataField = dataField as EcmaField;
            if (ecmaDataField == null)
                throw new NotSupportedException();

            var rvaData = ecmaDataField.GetFieldRvaData();
            var fieldType = field.FieldType;
            int elementCount;
            int realDataOffset;
            TypeDesc realDataType = null;

            //
            // Construct fixups
            //
            List<PreInitFixupInfo> fixups = null;

            var typeFixupAttrs = ecmaDataField.GetDecodedCustomAttributes("System.Runtime.CompilerServices", "TypeHandleFixupAttribute");
            foreach (var typeFixupAttr in typeFixupAttrs)
            {
                if (typeFixupAttr.FixedArguments[0].Type != field.Context.GetWellKnownType(WellKnownType.Int32))
                    throw new BadImageFormatException();

                int offset = (int)typeFixupAttr.FixedArguments[0].Value;
                var typeArg = typeFixupAttr.FixedArguments[1].Value;
                var fixupType = typeArg as TypeDesc;
                if (fixupType == null)
                {
                    if (typeArg is string fixupTypeName)
                    {
                        fixupType = CustomAttributeTypeNameParser.GetTypeByCustomAttributeTypeName(ecmaDataField.Module, fixupTypeName, throwIfNotFound: true);
                    }
                    else
                    {
                        throw new BadImageFormatException();
                    }
                }

                fixups = fixups ?? new List<PreInitFixupInfo>();

                if (offset == 0 && fieldType.IsSzArray)
                {
                    // For array field, offset 0 is the element type handle followed by the element count
                    realDataType = fixupType;
                }
                else
                {
                    fixups.Add(new PreInitTypeFixupInfo(offset, fixupType));
                }
            }

            var methodFixupAttrs = ecmaDataField.GetDecodedCustomAttributes("System.Runtime.CompilerServices", "MethodAddrFixupAttribute");
            foreach (var methodFixupAttr in methodFixupAttrs)
            {
                if (methodFixupAttr.FixedArguments[0].Type != field.Context.GetWellKnownType(WellKnownType.Int32))
                    throw new BadImageFormatException();

                int offset = (int)methodFixupAttr.FixedArguments[0].Value;
                TypeDesc fixupType = methodFixupAttr.FixedArguments[1].Value as TypeDesc;
                if (fixupType == null)
                    throw new BadImageFormatException();

                string methodName = methodFixupAttr.FixedArguments[2].Value as string;
                if (methodName == null)
                    throw new BadImageFormatException();

                var method = fixupType.GetMethod(methodName, signature : null);
                if (method == null)
                    throw new BadImageFormatException();

                fixups = fixups ?? new List<PreInitFixupInfo>();

                fixups.Add(new PreInitMethodFixupInfo(offset, method));
            }

            var fieldFixupAttrs = ecmaDataField.GetDecodedCustomAttributes("System.Runtime.CompilerServices", "FieldAddrFixupAttribute");
            foreach (var fieldFixupAttr in fieldFixupAttrs)
            {
                if (fieldFixupAttr.FixedArguments[0].Type != field.Context.GetWellKnownType(WellKnownType.Int32))
                    throw new BadImageFormatException();

                int offset = (int)fieldFixupAttr.FixedArguments[0].Value;
                TypeDesc fixupType = fieldFixupAttr.FixedArguments[1].Value as TypeDesc;
                if (fixupType == null)
                    throw new BadImageFormatException();

                string fieldName = fieldFixupAttr.FixedArguments[2].Value as string;
                if (fieldName == null)
                    throw new BadImageFormatException();

                var fixupField = fixupType.GetField(fieldName);
                if (fixupField == null)
                    throw new BadImageFormatException();

                if (!fixupField.IsStatic)
                    throw new BadImageFormatException();

                fixups = fixups ?? new List<PreInitFixupInfo>();

                fixups.Add(new PreInitFieldFixupInfo(offset, fixupField));
            }

            if (fieldType.IsValueType || fieldType.IsPointer)
            {
                elementCount = -1;
                realDataOffset = 0;
                realDataType = fieldType;
            }
            else if (fieldType.IsSzArray)
            {
                // Offset 0 is the element type handle fixup followed by the element count
                if (realDataType == null)
                    throw new BadImageFormatException();

                int ptrSize = fieldType.Context.Target.PointerSize;
                elementCount = rvaData[ptrSize] | rvaData[ptrSize + 1] << 8 | rvaData[ptrSize + 2] << 16 | rvaData[ptrSize + 3] << 24;
                realDataOffset = ptrSize * 2;
                realDataType = realDataType.MakeArrayType();
            }
            else
            {
                throw new NotSupportedException();
            }

            return new PreInitFieldInfo(field, realDataType, rvaData, realDataOffset, elementCount, fixups);
        }

        public static int FieldDescCompare(PreInitFieldInfo fieldInfo1, PreInitFieldInfo fieldInfo2)
        {
            return fieldInfo1.Field.Offset.AsInt - fieldInfo2.Field.Offset.AsInt;
        }
    }
}


text:
//////////////////////////////////////////////////////////////////
// (c) Copyright 2003- by Jeongnim Kim
//////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////
//   Jeongnim Kim
//   jnkim@ornl.gov
//////////////////////////////////////////////////////////////////
// -*- C++ -*-
namespace qmcplusplus
{

/** branch engine for VMC using |Phi|^2
 */
struct VMCBranch
{
  inline VMCBranch(BranchBase& bb)
  {}

  /** return true if r*r is zero
   */
  inline bool phaseChanged(double r, double phi) const
  {
    return r*r<numeric_limits<double>::epsilon();
  }

  /** return weight=1
   */
  inline double weight(double enew, double eold) const
  {
    return 1.0;
  }
};

/** branch engine for fixed-node DMC or RMC */
struct FixedNodeCBranch
{
  const BranchBase& brancher;
  inline FixedNodeBranch(BranchBase& bb)
    : brancher(bb)
  {}

  /** return true if r*r is zero
   */
  inline bool phaseChanged(double r, double phi) const
  {
#if defined(QMC_COMPLEX)
    return false;
#else
    return std::cos(psi0) < numeric_limits<RealType>::epsilon();
#endif
  }

  /** return the DMC weight
   */
  inline double weight(double enew, double eold) const
  {
    return brancher.Branch(enew,eold);
  }
};

template<typename BB>
inline void QMCUpdate::advanceWalkerPbyP(Walker_t& thisWalker, int nbranch, int nsub)
{
  BB brancher(*branchEngine);
  Walker_t::Buffer_t& w_buffer(thisWalker.DataSet);
  W.loadWalker(thisWalker,true);
  Psi.copyFromBuffer(W,w_buffer);
  for(int ibranch=0; ibranch<nbranch; ++ibranch)
  {
    int nAcceptTemp(0);
    int nRejectTemp(0);
    RealType eold(thisWalker.Properties(LOCALENERGY));
    RealType enew(eold);
    RealType rr_proposed=0.0;
    RealType rr_accepted=0.0;
    RealType gf_acc=1.0;
    for(int iter=0; iter<nsub; ++iter)
    {
      makeGaussRandomWithEngine(deltaR,RandomGen);
      for(int ig=0; ig<W.groups(); ++ig) //loop over species
      {
        RealType tauovermass = Tau*MassInvS[ig];
        RealType oneover2tau = 0.5/(tauovermass);
        RealType sqrttau = std::sqrt(tauovermass);
        for (int iat=W.first(ig); iat<W.last(ig); ++iat)
        {
          //get the displacement
          GradType grad_iat=Psi.evalGrad(W,iat);
          PosType dr;
          getScaledDrift(tauovermass, grad_iat, dr);
          dr += sqrttau * deltaR[iat];
          RealType rr=tauovermass*dot(deltaR[iat],deltaR[iat]);
          rr_proposed+=rr;

          //singular gradient, reject
          if(rr>m_r2max)
          {
            ++nRejectTemp;
            continue;
          }

          //move is out of bound, reject
          if(!W.makeMoveAndCheck(iat,dr))
          {
            ++nRejectTemp;
            continue;
          }

          PosType newpos(W.R[iat]);
          RealType ratio = Psi.ratioGrad(W,iat,grad_iat);
          bool valid_move=false;
          //first check, if phase changed
          if (brancher.phaseChanged(ratio,Psi.getPhaseDiff()))
          {
            ++nRejectTemp;
            ++nNodeCrossing;
            W.rejectMove(iat);
            Psi.rejectMove(iat);
          }
          else
          {
            RealType logGf = -0.5*dot(deltaR[iat],deltaR[iat]);
            getScaledDrift(tauovermass, grad_iat, dr);
            dr = thisWalker.R[iat] - newpos - dr;
            RealType logGb = -oneover2tau*dot(dr,dr);
            RealType prob = ratio*ratio*std::exp(logGb-logGf);
            if(RandomGen() < prob)
            {
              valid_move=true;
              ++nAcceptTemp;
              W.acceptMove(iat);
              Psi.acceptMove(W,iat);
              rr_accepted+=rr;
              gf_acc *=prob;//accumulate the ratio
            }
            else
            {
              ++nRejectTemp;
              W.rejectMove(iat);
              Psi.rejectMove(iat);
            }
          }
        }//iat
      }//ig
    } //iter=substeps without energy evaluations

    if(UseTMove) nonLocalOps.reset();
    bool advanced=true;
    if(nAcceptTemp>0)
    {
      //need to overwrite the walker properties
      thisWalker.Age=0;
      thisWalker.R = W.R;
      RealType logpsi = Psi.updateBuffer(W,w_buffer,false);
      W.saveWalker(thisWalker);
      if(UseTMove)
        enew= H.evaluate(W,nonLocalOps.Txy);
      else
        enew= H.evaluate(W);
      thisWalker.resetProperty(logpsi,Psi.getPhase(),enew,rr_accepted,rr_proposed,1.0 );
      thisWalker.Weight *= brancher.weight(enew,eold);
      //this is always called at the end
      //H.auxHevaluate(W,thisWalker);
      H.saveProperty(thisWalker.getPropertyBase());
    }
    else
    {
      //all moves are rejected: does not happen normally with reasonable wavefunctions
      advanced=false;
      thisWalker.Age++;
      thisWalker.Properties(R2ACCEPTED)=0.0;
      H.rejectedMove(W,thisWalker);
      ++nAllRejected;
      enew=eold;//copy back old energy
      gf_acc=1.0;
      thisWalker.Weight *= brancher.weight(enew,eold);
    }

    if(UseTMove)
    {
      int ibar = nonLocalOps.selectMove(RandomGen());
      //make a non-local move
      if(ibar)
      {
        int iat=nonLocalOps.id(ibar);
        if(!W.makeMoveAndCheck(iat,nonLocalOps.delta(ibar)))
          continue;
        RealType ratio=Psi.ratio(W,iat,dG,dL);
        W.acceptMove(iat);
        Psi.acceptMove(W,iat);
        W.G += dG;
        W.L += dL;
        RealType logpsi = Psi.evaluateLog(W,w_buffer);
        W.saveWalker(thisWalker);
        //PAOps<RealType,OHMMS_DIM>::copy(W.G,thisWalker.Drift);
        ++NonLocalMoveAccepted;
      }
    }
    nAccept += nAcceptTemp;
    nReject += nRejectTemp;
  }

  //now compute all the other observables per walker
  H.auxHevaluate(W,thisWalker);
}

}

/***************************************************************************
 * $RCSfile: DMCUpdatePbyP.cpp,v $   $Author: mcminis2 $
 * $Revision: 3697 $   $Date: 2009-03-24 19:30:46 -0400 (Tue, 24 Mar 2009) $
 * $Id: DMCUpdatePbyP.cpp 3697 2009-03-24 23:30:46Z mcminis2 $
 ***************************************************************************/

redpajama_set_name: RedPajamaGithub


text:
<?php

/**
 * @class HookLoader
 * @file HookLoader.php
 * @author Sascha Weidner support[at]sioweb.de
 * @package sioweb.mvc.core
 * @singleton 1
 *
 * @brief Stellt Hooks zur Verfügung mit denen der Standardablauf des Programmcodes erweitert werden kann
 */

namespace sioweb\core;

class HookLoader {

  static private $instance = null;
  private $Hooks = [];

  static public function getInstance(){
    if (null === self::$instance)
      self::$instance = new self;
    return self::$instance;
  }

  private function __construct(){}
  private function __clone(){}

  public function __call($func,$values) {
    $Active = preg_split('/(?=[A-Z])/',$func);
    if(method_exists($this,strtolower($Active[0])))
      return $this->$Active[0](implode('',array_slice($Active,1)),$values);
  }

  private function registrate($hook, $values) {
    $this->Hooks[$hook][] = $values;
  }

  private function load($hook, $values) {
    if(!empty($this->Hooks[$hook])) {
      foreach($this->Hooks[$hook] as $Hook) {
        if(method_exists($Hook[0],'getInstance'))
          $class = $Hook[0]::getInstance();
        else $class = new $Hook[0]($hook,$values);
        return $class->$Hook[1]($values);
      }
    }
  }
}


text:
<?php

namespace UserRegister\Service\Factories;

use UserRegister\Service\AdminRegisterService;
use Zend\ServiceManager\FactoryInterface;
use Zend\ServiceManager\ServiceLocatorInterface;

class AdminRegisterServiceFactory implements FactoryInterface
{
    public function createService(ServiceLocatorInterface $serviceLocator)
    {
        return new AdminRegisterService($serviceLocator);
    }
}


text:
/* $Id: PDFBoxAdapter.java 1740069 2016-04-20 08:51:49Z ssteiner $ */

package org.apache.fop.render.pdf.pdfbox;

import java.awt.Rectangle;
import java.awt.geom.AffineTransform;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;

import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import org.apache.pdfbox.cos.COSArray;
import org.apache.pdfbox.cos.COSBase;
import org.apache.pdfbox.cos.COSBoolean;
import org.apache.pdfbox.cos.COSDictionary;
import org.apache.pdfbox.cos.COSFloat;
import org.apache.pdfbox.cos.COSInteger;
import org.apache.pdfbox.cos.COSName;
import org.apache.pdfbox.cos.COSNull;
import org.apache.pdfbox.cos.COSObject;
import org.apache.pdfbox.cos.COSStream;
import org.apache.pdfbox.cos.COSString;

import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDResources;
import org.apache.pdfbox.pdmodel.common.COSObjectable;
import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.apache.pdfbox.pdmodel.common.PDStream;

import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
import org.apache.pdfbox.pdmodel.interactive.form.PDAcroForm;

import org.apache.fop.fonts.FontInfo;
import org.apache.fop.fonts.Typeface;
import org.apache.fop.pdf.PDFArray;
import org.apache.fop.pdf.PDFDictionary;
import org.apache.fop.pdf.PDFDocument;
import org.apache.fop.pdf.PDFName;
import org.apache.fop.pdf.PDFNumber;
import org.apache.fop.pdf.PDFObject;
import org.apache.fop.pdf.PDFPage;
import org.apache.fop.pdf.PDFRoot;
import org.apache.fop.pdf.PDFStream;

/**
 * This class provides an adapter for transferring content from a PDFBox PDDocument to
 * FOP's PDFDocument. It is used to parse PDF using PDFBox and write content using
 * FOP's PDF library.
 */
public class PDFBoxAdapter {

    /** logging instance */
    protected static final Log log = LogFactory.getLog(PDFBoxAdapter.class);

    private static final Set FILTER_FILTER = new HashSet(
            Arrays.asList(new String[] {"Filter", "DecodeParms"}));

    private final PDFPage targetPage;
    private final PDFDocument pdfDoc;

    private final Map clonedVersion;
    private Map<COSName, String> newXObj = new HashMap<COSName, String>();
    private Map<Integer, PDFArray> pageNumbers;
    private Collection<String> parentFonts = new ArrayList<String>();

    private int currentMCID;

    /**
     * Creates a new PDFBoxAdapter.
     * @param targetPage The target FOP PDF page object
     * @param objectCache the object cache for reusing objects shared by multiple pages.
     * @param pageNumbers references to page object numbers
     */
    public PDFBoxAdapter(PDFPage targetPage, Map objectCache, Map<Integer, PDFArray> pageNumbers) {
        this.targetPage = targetPage;
        this.pdfDoc = this.targetPage.getDocument();
        this.clonedVersion = objectCache;
        this.pageNumbers = pageNumbers;
    }

    public PDFPage getTargetPage() {
        return targetPage;
    }

    public int getCurrentMCID() {
        return currentMCID;
    }

    public void setCurrentMCID(int currentMCID) {
        this.currentMCID = currentMCID;
    }

    protected Object cloneForNewDocument(Object base) throws IOException {
        return cloneForNewDocument(base, base);
    }

    protected Object cloneForNewDocument(Object base, Object keyBase) throws IOException {
        return cloneForNewDocument(base, keyBase, Collections.EMPTY_LIST);
    }

    protected Object cloneForNewDocument(Object base, Object keyBase, Collection exclude) throws IOException {
        if (base == null) {
            return null;
        }
        Object cached = getCachedClone(keyBase);
        if (cached != null) {
            // we are done, it has already been converted.
            return cached;
        } else if (base instanceof List) {
            PDFArray array = new PDFArray();
            cacheClonedObject(keyBase, array);
            List list = (List)base;
            for (Object o : list) {
                array.add(cloneForNewDocument(o, o, exclude));
            }
            return array;
        } else if (base instanceof COSObjectable && !(base instanceof COSBase)) {
            Object o = ((COSObjectable)base).getCOSObject();
            Object retval = cloneForNewDocument(o, o, exclude);
            return cacheClonedObject(keyBase, retval);
        } else if (base instanceof COSObject) {
            return readCOSObject((COSObject) base, exclude);
        } else if (base instanceof COSArray) {
            PDFArray newArray = new PDFArray();
            cacheClonedObject(keyBase, newArray);
            COSArray array = (COSArray)base;
            for (int i = 0; i < array.size(); i++) {
                newArray.add(cloneForNewDocument(array.get(i), array.get(i), exclude));
            }
            return newArray;
//        } else if (base instanceof COSStreamArray) {
//            COSStreamArray array = (COSStreamArray)base;
//            PDFArray newArray = new PDFArray();
//            cacheClonedObject(keyBase, newArray);
//            for (int i = 0, c = array.getStreamCount(); i < c; i++) {
//                newArray.add(cloneForNewDocument(array.get(i)));
//            }
//            return newArray;
        } else if (base instanceof COSStream) {
            return readCOSStream((COSStream) base, keyBase);
        } else if (base instanceof COSDictionary) {
            return readCOSDictionary((COSDictionary) base, keyBase, exclude);
        } else if (base instanceof COSName) {
            PDFName newName = new PDFName(((COSName)base).getName());
            return cacheClonedObject(keyBase, newName);
        } else if (base instanceof COSInteger) {
            PDFNumber number = new PDFNumber();
            number.setNumber(((COSInteger)base).longValue());
            return cacheClonedObject(keyBase, number);
        } else if (base instanceof COSFloat) {
            PDFNumber number = new PDFNumber();
            number.setNumber(((COSFloat)base).floatValue());
            return cacheClonedObject(keyBase, number);
        } else if (base instanceof COSBoolean) {
            //TODO Do we need a PDFBoolean here?
            Boolean retval = ((COSBoolean)base).getValueAsObject();
            if (keyBase instanceof COSObject) {
                return cacheClonedObject(keyBase, new PDFBoolean(retval));
            } else {
                return cacheClonedObject(keyBase, retval);
            }
        } else if (base instanceof COSString) {
            return readCOSString((COSString) base, keyBase);
        } else if (base instanceof COSNull) {
            return cacheClonedObject(keyBase, null);
        } else {
            throw new UnsupportedOperationException("NYI: " + base.getClass().getName());
        }
    }

    private PDFDictionary readCOSDictionary(COSDictionary dic, Object keyBase, Collection exclude) throws IOException {
        PDFDictionary newDict = new PDFDictionary();
        cacheClonedObject(keyBase, newDict);
        for (Map.Entry<COSName, COSBase> e : dic.entrySet()) {
            if (!exclude.contains(e.getKey())) {
                newDict.put(e.getKey().getName(), cloneForNewDocument(e.getValue(), e.getValue(), exclude));
            }
        }
        return newDict;
    }

    private Object readCOSObject(COSObject object, Collection exclude) throws IOException {
        if (log.isTraceEnabled()) {
            log.trace("Cloning indirect object: "
                    + object.getObjectNumber()
                    + " " + object.getGenerationNumber());
        }
        Object obj = cloneForNewDocument(object.getObject(), object, exclude);
        if (obj instanceof PDFObject) {
            PDFObject pdfobj = (PDFObject)obj;
            //pdfDoc.registerObject(pdfobj);
            if (!pdfobj.hasObjectNumber()) {
                throw new IllegalStateException("PDF object was not registered!");
            }
            if (log.isTraceEnabled()) {
                log.trace("Object registered: "
                        + pdfobj.getObjectNumber()
                        + " " + pdfobj.getGeneration()
                        + " for COSObject: "
                        + object.getObjectNumber()
                        + " " + object.getGenerationNumber());
            }
        }
        return obj;
    }

    private Object readCOSString(COSString string, Object keyBase) {
        //retval = ((COSString)base).getString(); //this is unsafe for binary content
        byte[] bytes = string.getBytes();
        //Be on the safe side and use the byte array to avoid encoding problems
        //as PDFBox doesn't indicate whether the string is just
        //a string (PDF 1.4, 3.2.3) or a text string (PDF 1.4, 3.8.1).
        if (keyBase instanceof COSObject) {
            return cacheClonedObject(keyBase, new PDFString(bytes));
        } else {
            if (PDFString.isUSASCII(bytes)) {
                return cacheClonedObject(keyBase, string.getString());
            } else {
                return cacheClonedObject(keyBase, bytes);
            }
        }
    }

    private Object readCOSStream(COSStream originalStream, Object keyBase) throws IOException {
        InputStream in;
        Set filter;
        if (pdfDoc.isEncryptionActive()
                || (originalStream.containsKey(COSName.DECODE_PARMS) && !originalStream.containsKey(COSName.FILTER))) {
            in = originalStream.getUnfilteredStream();
            filter = FILTER_FILTER;
        } else {
            //transfer encoded data (don't reencode)
            in = originalStream.getFilteredStream();
            filter = Collections.EMPTY_SET;
        }
        PDFStream stream = new PDFStream();
        OutputStream out = stream.getBufferOutputStream();
        IOUtils.copyLarge(in, out);
        transferDict(originalStream, stream, filter);
        return cacheClonedObject(keyBase, stream);
    }

    protected Object getCachedClone(Object base) {
        return clonedVersion.get(getBaseKey(base));
    }

    protected Object cacheClonedObject(Object base, Object cloned) {
        Object key = getBaseKey(base);
        if (key == null) {
            return cloned;
        }
        PDFObject pdfobj = (PDFObject) cloned;
        if (pdfobj != null && !pdfobj.hasObjectNumber() && !(base instanceof COSDictionary)) {
            pdfDoc.registerObject(pdfobj);
            if (log.isTraceEnabled()) {
                log.trace(key + ": " + pdfobj.getClass().getName() + " registered as "
                        + pdfobj.getObjectNumber() + " " + pdfobj.getGeneration());
            }
        }
        clonedVersion.put(key, cloned);
        return cloned;
    }

    private Object getBaseKey(Object base) {
        if (base instanceof COSObject) {
            COSObject obj = (COSObject)base;
            return obj.getObjectNumber() + " " + obj.getGenerationNumber();
        } else if (base instanceof COSDictionary) {
            return base;
        } else {
            return null;
        }
    }

    private void transferDict(COSDictionary orgDict, PDFStream targetDict, Set filter) throws IOException {
        transferDict(orgDict, targetDict, filter, false);
    }

    private void transferDict(COSDictionary orgDict, PDFStream targetDict, Set filter, boolean inclusive)
        throws IOException {
        Set<COSName> keys = orgDict.keySet();
        for (COSName key : keys) {
            if (inclusive && !filter.contains(key.getName())) {
                continue;
            } else if (!inclusive && filter.contains(key.getName())) {
                continue;
            }
            targetDict.put(key.getName(),
                    cloneForNewDocument(orgDict.getItem(key)));
        }
    }

    /**
     * Creates a stream (from FOP's PDF library) from a PDF page parsed with PDFBox.
     * @param sourceDoc the source PDF the given page to be copied belongs to
     * @param page the page to transform into a stream
     * @param key value to use as key for the stream
     * @param atdoc adjustment for stream
     * @param fontinfo fonts
     * @param pos rectangle
     * @return the stream
     * @throws IOException if an I/O error occurs
     */
    public String createStreamFromPDFBoxPage(PDDocument sourceDoc, PDPage page, String key,
                                             AffineTransform atdoc, FontInfo fontinfo, Rectangle pos)
        throws IOException {
        handleAnnotations(sourceDoc, page, atdoc);
        if (pageNumbers.containsKey(targetPage.getPageIndex())) {
            pageNumbers.get(targetPage.getPageIndex()).set(0, targetPage.makeReference());
        }
        PDResources sourcePageResources = page.getResources();
        PDStream pdStream = getContents(page);

        COSDictionary fonts = (COSDictionary)sourcePageResources.getCOSObject().getDictionaryObject(COSName.FONT);
        COSDictionary fontsBackup = null;
        UniqueName uniqueName = new UniqueName(key, sourcePageResources);
        String newStream = null;
        if (fonts != null && pdfDoc.isMergeFontsEnabled()) {
            fontsBackup = new COSDictionary(fonts);
            MergeFontsPDFWriter m = new MergeFontsPDFWriter(fonts, fontinfo, uniqueName, parentFonts, currentMCID);
            newStream = m.writeText(pdStream);
//            if (newStream != null) {
//                for (Object f : fonts.keySet().toArray()) {
//                    COSDictionary fontdata = (COSDictionary)fonts.getDictionaryObject((COSName)f);
//                    if (getUniqueFontName(fontdata) != null) {
//                        fonts.removeItem((COSName)f);
//                    }
//                }
//            }
        }
        if (newStream == null) {
            PDFWriter writer = new PDFWriter(uniqueName, currentMCID);
            newStream = writer.writeText(pdStream);
            currentMCID = writer.getCurrentMCID();

        }
        pdStream = new PDStream(sourceDoc, new ByteArrayInputStream(newStream.getBytes("ISO-8859-1")));
        mergeXObj(sourcePageResources.getCOSObject(), fontinfo, uniqueName);
        PDFDictionary pageResources = (PDFDictionary)cloneForNewDocument(sourcePageResources.getCOSObject());

        PDFDictionary fontDict = (PDFDictionary)pageResources.get("Font");
        if (fontDict != null && pdfDoc.isMergeFontsEnabled()) {
            for (Map.Entry<String, Typeface> fontEntry : fontinfo.getUsedFonts().entrySet()) {
                Typeface font = fontEntry.getValue();
                if (font instanceof FOPPDFFont) {
                    FOPPDFFont pdfFont = (FOPPDFFont)font;
                    if (pdfFont.getRef() == null) {
                        pdfFont.setRef(new PDFDictionary());
                        pdfDoc.assignObjectNumber(pdfFont.getRef());
                    }
                    fontDict.put(fontEntry.getKey(), pdfFont.getRef());
                }
            }
        }
        updateXObj(sourcePageResources.getCOSObject(), pageResources);
        if (fontsBackup != null) {
            sourcePageResources.getCOSObject().setItem(COSName.FONT, fontsBackup);
        }

        COSStream originalPageContents = pdStream.getCOSObject();

        bindOptionalContent(sourceDoc);

        PDFStream pageStream;
        Set filter;
//        if (originalPageContents instanceof COSStreamArray) {
//            COSStreamArray array = (COSStreamArray)originalPageContents;
//            pageStream = new PDFStream();
//            InputStream in = array.getUnfilteredStream();
//            OutputStream out = pageStream.getBufferOutputStream();
//            IOUtils.copyLarge(in, out);
//            filter = FILTER_FILTER;
//        } else {
            pageStream = (PDFStream)cloneForNewDocument(originalPageContents);
            filter = Collections.EMPTY_SET;
//        }
        if (pageStream == null) {
            pageStream = new PDFStream();
        }
        if (originalPageContents != null) {
            transferDict(originalPageContents, pageStream, filter);
        }

        transferPageDict(fonts, uniqueName, sourcePageResources);

        PDRectangle mediaBox = page.getMediaBox();
        PDRectangle cropBox = page.getCropBox();
        PDRectangle viewBox = cropBox != null ? cropBox : mediaBox;

        //Handle the /Rotation entry on the page dict
        int rotation = PDFUtil.getNormalizedRotation(page);

        //Transform to FOP's user space
        float w = (float)pos.getWidth() / 1000f;
        float h = (float)pos.getHeight() / 1000f;
        if (rotation == 90 || rotation == 270) {
            float tmp = w;
            w = h;
            h = tmp;
        }
        atdoc.setTransform(AffineTransform.getScaleInstance(w / viewBox.getWidth(), h / viewBox.getHeight()));
        atdoc.translate(0, viewBox.getHeight());
        atdoc.rotate(-Math.PI);
        atdoc.scale(-1, 1);
        atdoc.translate(-viewBox.getLowerLeftX(), -viewBox.getLowerLeftY());

        rotate(rotation, viewBox, atdoc);

        StringBuilder boxStr = new StringBuilder();
        boxStr.append(PDFNumber.doubleOut(mediaBox.getLowerLeftX())).append(' ')
                .append(PDFNumber.doubleOut(mediaBox.getLowerLeftY())).append(' ')
                .append(PDFNumber.doubleOut(mediaBox.getWidth())).append(' ')
                .append(PDFNumber.doubleOut(mediaBox.getHeight())).append(" re W n\n");
        return boxStr.toString() + IOUtils.toString(pdStream.createInputStream(null), "ISO-8859-1");
    }

    private void rotate(int rotation, PDRectangle viewBox, AffineTransform atdoc) {
        float x = viewBox.getWidth() + viewBox.getLowerLeftX();
        float y = viewBox.getHeight() + viewBox.getLowerLeftY();
        switch (rotation) {
            case 90:
                atdoc.scale(viewBox.getWidth() / viewBox.getHeight(), viewBox.getHeight() / viewBox.getWidth());
                atdoc.translate(0, viewBox.getWidth());
                atdoc.rotate(-Math.PI / 2.0);
                atdoc.scale(viewBox.getWidth() / viewBox.getHeight(), viewBox.getHeight() / viewBox.getWidth());
                break;
            case 180:
                atdoc.translate(x, y);
                atdoc.rotate(-Math.PI);
                atdoc.translate(-viewBox.getLowerLeftX(), -viewBox.getLowerLeftY());
                break;
            case 270:
                atdoc.translate(viewBox.getLowerLeftX(), y);
                atdoc.rotate(Math.toRadians(270 + 180));
                atdoc.translate(-x, -y);
                break;
            default:
                //no additional transformations necessary
                break;
        }
    }

    private PDStream getContents(PDPage page) throws IOException {
        PDStream pdStream = new PDStream(new COSStream());
        OutputStream os = pdStream.createOutputStream();
        IOUtils.copy(page.getContents(), os);
        os.close();
        return pdStream;
    }

    private void mergeXObj(COSDictionary sourcePageResources, FontInfo fontinfo, UniqueName uniqueName)
        throws IOException {
        COSDictionary xobj = (COSDictionary) sourcePageResources.getDictionaryObject(COSName.XOBJECT);
        if (xobj != null && pdfDoc.isMergeFontsEnabled()) {
            for (Map.Entry<COSName, COSBase> i : xobj.entrySet()) {
                COSObject v = (COSObject) i.getValue();
                COSStream stream = (COSStream) v.getObject();
                COSDictionary res = (COSDictionary) stream.getDictionaryObject(COSName.RESOURCES);
                if (res != null) {
                    COSDictionary src = (COSDictionary) res.getDictionaryObject(COSName.FONT);
                    if (src != null) {
                        COSDictionary target = (COSDictionary) sourcePageResources.getDictionaryObject(COSName.FONT);
                        if (target == null) {
                            sourcePageResources.setItem(COSName.FONT, src);
                        } else {
                            for (Map.Entry<COSName, COSBase> entry : src.entrySet()) {
                                if (!target.keySet().contains(entry.getKey())) {
                                    target.setItem(uniqueName.getName(entry.getKey()), entry.getValue());
                                }
                            }
                        }
                        PDFWriter writer = new MergeFontsPDFWriter(src, fontinfo, uniqueName, parentFonts, 0);
                        String c = writer.writeText(new PDStream(stream));
                        if (c != null) {
                            stream.removeItem(COSName.FILTER);
                            newXObj.put(i.getKey(), c);
                            for (Object e : src.keySet().toArray()) {
                                COSName name = (COSName) e;
                                src.setItem(uniqueName.getName(name), src.getItem(name));
                                src.removeItem(name);
                            }
                        }
                    }
                }
            }
        }
    }

    private void updateXObj(COSDictionary sourcePageResources, PDFDictionary pageResources) throws IOException {
        COSDictionary xobj = (COSDictionary) sourcePageResources.getDictionaryObject(COSName.XOBJECT);
        if (xobj != null && pdfDoc.isMergeFontsEnabled()) {
            PDFDictionary target = (PDFDictionary) pageResources.get("XObject");
            for (COSName entry : xobj.keySet()) {
                if (newXObj.containsKey(entry)) {
                    PDFStream s = (PDFStream) target.get(entry.getName());
                    s.setData(newXObj.get(entry).getBytes("ISO-8859-1"));
                    PDFDictionary xobjr = (PDFDictionary) s.get("Resources");
                    xobjr.put("Font", pageResources.get("Font"));
                }
            }
        }
    }

    private void transferPageDict(COSDictionary fonts, UniqueName uniqueName, PDResources sourcePageResources)
        throws IOException {
        if (fonts != null) {
            for (Map.Entry<COSName, COSBase> f : fonts.entrySet()) {
                String name = uniqueName.getName(f.getKey());
                targetPage.getPDFResources().addFont(name, (PDFDictionary)cloneForNewDocument(f.getValue()));
            }
        }
        for (Map.Entry<COSName, COSBase> e : sourcePageResources.getCOSObject().entrySet()) {
            transferDict(e, uniqueName);
        }
    }

    private void transferDict(Map.Entry<COSName, COSBase> dict, UniqueName uniqueName) throws IOException {
        COSBase src;
        if (dict.getValue() instanceof COSObject) {
            src = ((COSObject) dict.getValue()).getObject();
        } else {
            src = dict.getValue();
        }
        if (dict.getKey() != COSName.FONT && src instanceof COSDictionary) {
            String name = dict.getKey().getName();
            PDFDictionary newDict = (PDFDictionary) targetPage.getPDFResources().get(name);
            if (newDict == null) {
                newDict = new PDFDictionary(targetPage.getPDFResources());
            }
            COSDictionary srcDict = (COSDictionary) src;
            for (Map.Entry<COSName, COSBase> v : srcDict.entrySet()) {
                newDict.put(uniqueName.getName(v.getKey()), cloneForNewDocument(v.getValue()));
            }
            targetPage.getPDFResources().put(name, newDict);
        }
    }

    private void bindOptionalContent(PDDocument sourceDoc) throws IOException {
        /*
         * PDOptionalContentProperties ocProperties =
         * sourceDoc.getDocumentCatalog().getOCProperties(); PDFDictionary ocDictionary =
         * (PDFDictionary) cloneForNewDocument(ocProperties); if (ocDictionary != null) {
         * this.pdfDoc.getRoot().put(COSName.OCPROPERTIES.getName(), ocDictionary); }
         */
    }

    private void handleAnnotations(PDDocument sourceDoc, PDPage page, AffineTransform at) throws IOException {
        PDDocumentCatalog srcCatalog = sourceDoc.getDocumentCatalog();
        PDAcroForm srcAcroForm = srcCatalog.getAcroForm();
        List pageAnnotations = page.getAnnotations();
        if (srcAcroForm == null && pageAnnotations.isEmpty()) {
            return;
        }

        moveAnnotations(page, pageAnnotations, at);

        //Pseudo-cache the target page in place of the original source page.
        //This essentially replaces the original page reference with the target page.
        COSObject cosPage = null;
        COSDictionary parentDic = (COSDictionary) page.getCOSObject().getDictionaryObject(COSName.PARENT, COSName.P);
        COSArray kids = (COSArray) parentDic.getDictionaryObject(COSName.KIDS);
        for (int i = 0; i < kids.size(); i++) {
            //Hopefully safe to cast, as kids need to be indirect objects
            COSObject kid = (COSObject) kids.get(i);
            if (!pageNumbers.containsKey(i)) {
                PDFArray a = new PDFArray();
                a.add(null);
                pdfDoc.assignObjectNumber(a);
                pdfDoc.addTrailerObject(a);
                pageNumbers.put(i, a);
            }
            cacheClonedObject(kid, pageNumbers.get(i));
            if (kid.getObject() == page.getCOSObject()) {
                cosPage = kid;
            }
        }
        if (cosPage == null) {
            throw new IOException("Illegal PDF. Page not part of parent page node.");
        }

        Set<COSObject> fields = copyAnnotations(page);

        boolean formAlreadyCopied = getCachedClone(srcAcroForm) != null;
        PDFRoot catalog = this.pdfDoc.getRoot();
        PDFDictionary destAcroForm = (PDFDictionary)catalog.get(COSName.ACRO_FORM.getName());
        if (formAlreadyCopied) {
            //skip, already copied
        } else if (destAcroForm == null) {
            if (srcAcroForm != null) {
                //With this, only the first PDF's AcroForm is copied over. If later AcroForms have
                //different properties besides the actual fields, these get lost. Only fields
                //get merged.
                Collection exclude = Collections.singletonList(COSName.FIELDS);
                destAcroForm = (PDFDictionary)cloneForNewDocument(srcAcroForm, srcAcroForm, exclude);
            } else {
                //Work-around for incorrectly split PDFs which lack an AcroForm but have widgets
                //on pages. This doesn't handle the case where field dicts have "C" entries
                //(for the "CO" entry), so this may produce problems, but we have almost no chance
                //to guess the calculation order.
                destAcroForm = new PDFDictionary(pdfDoc.getRoot());
            }
            pdfDoc.registerObject(destAcroForm);
            catalog.put(COSName.ACRO_FORM.getName(), destAcroForm);
        }
        PDFArray clonedFields = (PDFArray) destAcroForm.get(COSName.FIELDS.getName());
        if (clonedFields == null) {
            clonedFields = new PDFArray();
            destAcroForm.put(COSName.FIELDS.getName(), clonedFields);
        }
        for (COSObject field : fields) {
            PDFDictionary clone = (PDFDictionary) cloneForNewDocument(field, field, Arrays.asList(COSName.KIDS));
            clonedFields.add(clone);
        }
    }

    private void updateAnnotationLink(PDFDictionary clonedAnnot) {
        Object a = clonedAnnot.get("A");
        if (a instanceof PDFDictionary) {
            PDFDictionary annot = (PDFDictionary) a;
            Object oldarrayObj = annot.get("D");
            if (oldarrayObj instanceof PDFArray) {
                PDFArray oldarray = (PDFArray) oldarrayObj;
                Object newarrayObj = oldarray.get(0);
                if (newarrayObj instanceof PDFArray) {
                    PDFArray newarray = (PDFArray) newarrayObj;
                    for (int i = 1; i < oldarray.length(); i++) {
                        newarray.add(oldarray.get(i));
                    }
                    annot.put("D", oldarray.get(0));
                }
            }
        }
    }

    private Set<COSObject> copyAnnotations(PDPage page) throws IOException {
        COSArray annots = (COSArray) page.getCOSObject().getDictionaryObject(COSName.ANNOTS);
        Set<COSObject> fields = Collections.emptySet();
        if (annots != null) {
            fields = new TreeSet<COSObject>(new CompareFields());
            for (Object annot1 : annots) {
                Collection<COSName> exclude = new ArrayList<COSName>();
                exclude.add(COSName.P);
                if (annot1 instanceof COSObject) {
                    COSObject annot = (COSObject) annot1;
                    COSObject fieldObject = annot;
                    COSDictionary field = (COSDictionary) fieldObject.getObject();
                    COSObject parent;
                    while ((parent = (COSObject) field.getItem(COSName.PARENT)) != null) {
                        fieldObject = parent;
                        field = (COSDictionary) fieldObject.getObject();
                    }
                    fields.add(fieldObject);

                    if (((COSDictionary) annot.getObject()).getItem(COSName.getPDFName("StructParent")) != null) {
                        exclude.add(COSName.PARENT);
                    }
                }

                PDFObject clonedAnnot = (PDFObject) cloneForNewDocument(annot1, annot1, exclude);
                if (clonedAnnot instanceof PDFDictionary) {
                    clonedAnnot.setParent(targetPage);
                    updateAnnotationLink((PDFDictionary) clonedAnnot);
                }
                targetPage.addAnnotation(clonedAnnot);
            }
        }
        return fields;
    }

    private void moveAnnotations(PDPage page, List pageAnnotations, AffineTransform at) {
        PDRectangle mediaBox = page.getMediaBox();
        PDRectangle cropBox = page.getCropBox();
        PDRectangle viewBox = cropBox != null ? cropBox : mediaBox;
        for (Object obj : pageAnnotations) {
            PDAnnotation annot = (PDAnnotation)obj;
            PDRectangle rect = annot.getRectangle();
            float translateX = (float) (at.getTranslateX() - viewBox.getLowerLeftX());
            float translateY = (float) (at.getTranslateY() - viewBox.getLowerLeftY());
            if (rect != null) {
                rect.setUpperRightX(rect.getUpperRightX() + translateX);
                rect.setLowerLeftX(rect.getLowerLeftX() + translateX);
                rect.setUpperRightY(rect.getUpperRightY() + translateY);
                rect.setLowerLeftY(rect.getLowerLeftY() + translateY);
                annot.setRectangle(rect);
            }
//            COSArray vertices = (COSArray) annot.getCOSObject().getDictionaryObject("Vertices");
//            if (vertices != null) {
//                Iterator iter = vertices.iterator();
//                while (iter.hasNext()) {
//                    COSFloat x = (COSFloat) iter.next();
//                    COSFloat y = (COSFloat) iter.next();
//                    x.setValue(x.floatValue() + translateX);
//                    y.setValue(y.floatValue() + translateY);
//                }
//            }
        }
    }

    static class CompareFields implements Comparator<COSObject>, Serializable {
        private static final long serialVersionUID = -6081505461660440801L;

        public int compare(COSObject o1, COSObject o2) {
            return (int) (o1.getObjectNumber() - o2.getObjectNumber());
        }
    }
}


text:
import _ from 'underscore';
import utils from '../utils/index';

export default StepFunction;

function StepFunction(stepName, fn, options = {}) {

  _.extend(this, options);

  this.stepName = stepName;
  this.fn = fn || function() {};
  this.regex = this.createRegex();

  this.extractAssertions();

  this.isAsync = !! fn.toString().match(/function[^\)]+done\s*\)/);
}

var fn = StepFunction.prototype;

fn.createRegex = function() {
  var step = this.stepName;
  var isRegex = false;
  var variableRegex = /\$\{[^\}]+\}/g;
  var regex;

  if (step instanceof RegExp) {
    isRegex = true;
    regex = step;
  } else if (step.match(variableRegex)) {
    regex = new RegExp('^' + step.replace(variableRegex, '(.+)') + '$', 'i');
    isRegex = true;
  } else{
    regex = new RegExp('^' + step + '$', 'i');
  }

  return regex;
};

fn.getArgumentsObject = function(stepRecord) {
  var variableValues = this.getVariableValues(stepRecord);

  if (! this.variableNames) {
    this.variableNames = this.getVariableNames();
  }

  var variableNames = this.variableNames;

  var argsObject = {};

  for (var i = 0; i < variableNames.length; i++) {
    argsObject[variableNames[i]] = variableValues[i];
  }

  return argsObject;
};

fn.getVariableNames = function() {
  var variableNames = [];
  var regex = /\$\{([^\}]+)\}/g;
  var match;
  while (match = regex.exec(this.stepName)) {
    variableNames.push(match[1]);
  }

  return variableNames;
};

fn.getVariableValues = function(stepRecord) {
  var match = stepRecord.name.match(this.regex);
  if (! match) {
    throw new Error('stepFunction was given a mismatched stepRecord');
  }

  var args = [];
  var i, value;

  for (i = 1; i < match.length; i++) {
    value = match[i];
    value = utils.smartTypeCast(value);
    args.push(value);
  }

  if (stepRecord.table) {
    args.push(stepRecord.table);
  }

  return args;
};

fn.extractAssertions = function extractAssertions() {
  var str = this.fn.toString();
  var regex = /\s*((expect|assert)[\(\.][^;]+;)/g;
  var match;

  str = utils.stripComments(str);

  this.assertions = [];
  while (match = regex.exec(str)) {
    let assertion = utils.htmlEscape( match[1].trim() );
    this.assertions.push(assertion);
  }
  return this.assertions;
};


text:
  Bragayrac est
       un village
       situé dans le département de Haute-Garonne en Midi-Pyrénées. Elle totalisait 266 habitants en 2008.

Le nombre d'habitations, à Bragayrac, était réparti en 2011 en six appartements et 110 maisons soit
 un marché plutôt équilibré.

Si vous pensez venir habiter à Bragayrac, vous pourrez aisément trouver une maison à vendre.

À proximité de Bragayrac sont localisées les communes de
Beaufort à 4 km, 348 habitants,
Pébées située à 4 km, 88 habitants,
Lahage localisée à 4 km, 240 habitants,
Nizas localisée à 5 km, 175 habitants,
Empeaux localisée à 5 km, 195 habitants,
Seysses-Savès située à 3 km, 257 habitants,
 entre autres. De plus, Bragayrac est située à seulement 29 km de Fort-de-France.


text:
package com.evanzeimet.queryinfo.jpa.predicate.directive;

import javax.persistence.criteria.Predicate;

import com.evanzeimet.queryinfo.QueryInfoException;
import com.evanzeimet.queryinfo.condition.ConditionGroup;
import com.evanzeimet.queryinfo.jpa.entity.QueryInfoEntityContextRegistry;
import com.evanzeimet.queryinfo.jpa.jpacontext.QueryInfoJPAContext;
import com.evanzeimet.queryinfo.jpa.jpacontext.QueryInfoJPAContexts;

public interface DirectiveConditionGroupPredicateFactory<RootEntity> {

	Predicate createPredicate(QueryInfoEntityContextRegistry entityContextRegistry,
			QueryInfoJPAContexts<?, ?> jpaContexts,
			QueryInfoJPAContext<RootEntity, ?> currentJpaContext,
			ConditionGroup conditionGroup) throws QueryInfoException;

}


text:
"""
Created on Sat Oct  7 18:35:28 2017

@author: caleb
"""

import json
from collections import Counter

f = open('Parsed scifinder metadata.json').read()
data = json.loads(f)
all_names = []
for CAS, d in data.items():
    if 'Other Names' in d:
        all_names.extend(d['Other Names'])
    if 'Name' in d:
        all_names.append(d['Name'])
dup_names =  [item for item, count in Counter(all_names).items() if count > 1]
#print(len(all_names), len(set(all_names)))
#print(dup_names)


text:
/* global $:false */
/*jshint devel:true */

'use strict';

angular.module('wasabi.controllers').
    controller('PageManagementCtrl', ['$scope', '$filter', '$http', '$stateParams', '$timeout', '$cookies', 'PagesFactory', 'ApplicationsFactory', 'DialogsFactory', 'ExperimentsFactory', '$uibModal', 'UtilitiesFactory', '$rootScope', 'AUTH_EVENTS', '$state', 'PERMISSIONS', 'ConfigFactory',
        function ($scope, $filter, $http, $stateParams, $timeout, $cookies, PagesFactory, ApplicationsFactory, DialogsFactory, ExperimentsFactory, $uibModal, UtilitiesFactory, $rootScope, AUTH_EVENTS, $state, PERMISSIONS, ConfigFactory) {

            $scope.data = {
                query: '',
                applicationName: '', // This is bound to the selection in the application name drop down menu.
                selectedPage: '',
                selectedExperiments: []
            };
            $scope.appNames = [];
            $scope.pages = [];
            $scope.experiments = [];
            $scope.allExperiments = [];
            // This is passed in as a parameter on the URL. The selection in the drop down will cause an URL
            // with this parameter to be hit. This is necessary so that going "back" from the details page will
            // come back to the correct form of the PageManagement table.
            $scope.applicationName = $stateParams.appname;
            $scope.currentPage = 1;
            $scope.totalItems = 0;
            $scope.itemsPerPage = 10;

            $scope.orderByField = 'time';
            $scope.reverseSort = true;

            $scope.help = ConfigFactory.help;

            UtilitiesFactory.hideHeading(false);
            UtilitiesFactory.selectTopLevelTab('Tools');

            $scope.changePage = function() {
                $cookies.wasabiDefaultApplication = $scope.data.applicationName;
                $state.go('pages', {'appname': $scope.data.applicationName});
            };

            $scope.onSelectAppName = function(selectedApp) {
                if (selectedApp) {
                    $scope.applicationName = $cookies.wasabiDefaultApplication = selectedApp;
                    var options =
                        {
                            applicationName: selectedApp
                        };
                    PagesFactory.query(options).$promise.then(function (data) {
                        $scope.pages = $filter('orderBy')(data, 'name', false);

                        UtilitiesFactory.doTrackingInit();

                        UtilitiesFactory.trackEvent('loadedDialog',
                            {key: 'dialog_name', value: 'pagesList'},
                            {key: 'application_name', value: selectedApp});
                    }, function(response) {
                        UtilitiesFactory.handleGlobalError(response, 'The list of pages could not be retrieved.');
                    });
                    ApplicationsFactory.getExperiments({appName: selectedApp}).$promise.then(function (data) {
                        $scope.allExperiments = data;

                        UtilitiesFactory.doTrackingInit();

                        UtilitiesFactory.trackEvent('loadedDialog',
                            {key: 'dialog_name', value: 'allExperimentsList'},
                            {key: 'application_name', value: selectedApp});
                    }, function(response) {
                        UtilitiesFactory.handleGlobalError(response, 'The list of all experiments could not be retrieved.');
                    });
                }
            };

            $scope.onSelectPage = function() {
                $scope.getExperiments($scope.data.applicationName, $scope.data.selectedPage.name);
            };

            // init controller
            $scope.init = function() {
                var appNames = UtilitiesFactory.getAdministeredApplications();
                for (var i = 0; i < appNames.length; i++) {
                    $scope.appNames.push(appNames[i].label);
                }

                if ($scope.appNames.length === 1) {
                    $scope.onSelectAppName($scope.appNames[0]);
                }
            };
            $scope.init();

            // If we are on a version of this page for a specific application, this will cause the $watch below
            // to populate the table with the correct data for the correct application.
            if (!$scope.applicationName && $cookies.wasabiDefaultApplication) {
                $scope.applicationName = $cookies.wasabiDefaultApplication;
            }

            // If we are on a version of this page for a specific application, this will cause the $watch below
            // to populate the table with the correct data for the correct application.
            // TODO: Reconcile this with code above.
            $scope.data.applicationName = $scope.applicationName;
            $scope.$watch(function() {
                    return $scope.appNames.length;
                },
                function() {
                    if ($scope.appNames.length === 0) {
                        return;
                    }
                    $scope.$evalAsync(function() {
                        // As a workaround of the fact that modifying the model *DOES NOT* seem to cause the menu to be set
                        // to reflect it, we are using jQuery to move the menu to the correct value.
                        var choiceIndex = 0;
                        for (var i = 0; i < $scope.appNames.length; i++) {
                            if ($scope.appNames[i] === $scope.applicationName) {
                                choiceIndex = i + 1;
                            }
                        }
                        $('#applicationNameChoice').prop('selectedIndex', choiceIndex);
                        $scope.onSelectAppName($scope.applicationName);
                    });
                }
            );

            $scope.capitalizeFirstLetter = function (string) {
                return UtilitiesFactory.capitalizeFirstLetter(string);
            };

            $scope.getExperiments = function(applicationName, pageName) {
                PagesFactory.getExperimentsWithPage({
                    applicationName: applicationName,
                    id: pageName
                }).$promise.then(function (data) {
                    $scope.experiments = $filter('orderBy')(data, 'label', false);

                    UtilitiesFactory.doTrackingInit();

                    UtilitiesFactory.trackEvent('loadedDialog',
                        {key: 'dialog_name', value: 'experimentsList'},
                        {key: 'application_name', value: applicationName});
                }, function(response) {
                    UtilitiesFactory.handleGlobalError(response, 'The list of experiments could not be retrieved.');
                });
            };

            $scope.addExperimentToPage = function() {
                $scope.openAddToPageModal();
            };

            $scope.createNameList = function(objectsList, nameAttrName) {
                return UtilitiesFactory.createNameList(objectsList, nameAttrName);
            };

            $scope.removeExperimentsFromPage = function() {
                var numExperimentsToAdd;

                function removeExperiment(experiment, page) {
                    ExperimentsFactory.removePage({
                        id: experiment.id,
                        pageName: page
                    }).$promise.then(function () {
                        UtilitiesFactory.trackEvent('deleteItemSuccess',
                            {key: 'dialog_name', value: 'deleteExperimentFromPage'},
                            {key: 'experiment_id', value: experiment.id},
                            {key: 'item_id', value: page});

                        numExperimentsToAdd--;
                        if (!numExperimentsToAdd) {
                            var expmtStr = 'experiment',
                                hasBeenStr = 'has been';
                            if ($scope.data.selectedExperiments.length > 1) {
                                expmtStr = 'experiments';
                                hasBeenStr = 'have been';
                            }
                            var removedExperimentNames = $scope.createNameList($scope.data.selectedExperiments, 'label');
                            UtilitiesFactory.displaySuccessWithCacheWarning('Experiments Removed', 'The ' + expmtStr + ', ' + removedExperimentNames + ', ' + hasBeenStr + ' removed from the page, ' + page + '.');
                            $scope.onSelectPage();
                        }
                    }, function(response) {
                        UtilitiesFactory.handleGlobalError(response, 'Your experiment could not be removed from the page.');
                        numExperimentsToAdd--;
                        if (!numExperimentsToAdd) {
                            $scope.onSelectPage();
                        }
                    });
                }

                if (!$scope.data.selectedExperiments.length) {
                    return;
                }
                var expmtStr = 'experiment';
                if ($scope.data.selectedExperiments.length > 1) {
                    expmtStr = 'experiments';
                }
                var namesToRemove = $scope.createNameList($scope.data.selectedExperiments, 'label');
                DialogsFactory.confirmDialog(
                    'Are you sure you want to remove your ' + expmtStr + ', ' + namesToRemove + ', from the page, ' + $scope.data.selectedPage.name + '?',
                    'Remove Experiments',
                    function() {
                        numExperimentsToAdd = $scope.data.selectedExperiments.length;
                        for (var i = 0; i < $scope.data.selectedExperiments.length; i++) {
                            removeExperiment($scope.data.selectedExperiments[i], $scope.data.selectedPage.name);
                        }
                    },
                    function() { /* Do nothing */ },
                    'Remove Experiments',
                    'Cancel');
            };

            $scope.openAddToPageModal = function () {
                if (!$scope.data.selectedPage) {
                    return;
                }
                var modalInstance = $uibModal.open({
                    templateUrl: 'views/AddExperimentsToPageModal.html',
                    controller: 'AddExperimentsToPageModalCtrl',
                    windowClass: 'larger-dialog',
                    backdrop: 'static',
                    size: 'lg',
                    resolve: {
                        experiments: function () {
                            return $scope.experiments;
                        },
                        allExperiments: function () {
                            return $scope.allExperiments;
                        },
                        page: function() {
                            return $scope.data.selectedPage;
                        }
                    }
                });

                // This closes the dialog if we encounter an expired token and redirects to the Sign In page.
                // Note that this will also broadcast an event that will be caught by the parent modal dialog
                // so that it, too, can close.
                UtilitiesFactory.failIfTokenExpired(modalInstance);

                modalInstance.result.then(function () {
                    $scope.onSelectPage();
                });
            };

            $scope.pageChanged = function() {
                // The widget has updated the currentPage member.  By simply triggering the code to get the
                // pageManagement list, we should update the page.
                $scope.onSelectAppName($scope.applicationName);
            };

            $scope.sortBy = function (orderByField) {
                if ($scope.orderByField === orderByField) {
                    $scope.reverseSort = !$scope.reverseSort;
                }
                else {
                    $scope.reverseSort = true;
                }

                $scope.orderByField = orderByField;

                if ($scope.orderByField !== '') {
                    $scope.onSelectAppName($scope.applicationName);
                }
            };

            $scope.clearSearch = function() {
                $scope.data.query = '';
                $scope.onSelectAppName($scope.applicationName);
            };

            $scope.doSearch = function() {
                $scope.onSelectAppName($scope.applicationName);
            };

            $scope.search = function () {
                if ($.trim($scope.data.query).length > 0 && $scope.data.applicationName.length > 0) {
                    if ($scope.searchTimer) {
                        $timeout.cancel($scope.searchTimer);
                    }
                    $scope.searchTimer = $timeout($scope.doSearch, 400);
                }
            };
        }]);


text:
<?xml version="1.0" encoding="utf-8"?>
<shape xmlns:android="http://schemas.android.com/apk/res/android"
    android:shape="rectangle" >

    <solid android:color="#9E9E9E" /> <!-- Grey 500 -->
    <corners
        android:bottomLeftRadius="2dp"
        android:bottomRightRadius="2dp"
        android:topLeftRadius="2dp"
        android:topRightRadius="2dp" />

</shape>


text:
package example.atfreaks.co.jp.imgwatermark;

import android.app.Application;
import android.test.ApplicationTestCase;

/**
 * Testing Fundamentals
 */
public class ApplicationTest extends ApplicationTestCase<Application> {
    public ApplicationTest() {
        super(Application.class);
    }
}


text:
<?php

$installer = $this;
$this->startSetup();
$this->run("
ALTER TABLE `{$installer->getTable('sales_quote')}`
    change `is_active` `is_active` tinyint (1)UNSIGNED  DEFAULT '1' NULL ,
    change `is_virtual` `is_virtual` tinyint (1)UNSIGNED  DEFAULT '0' NULL ,
    change `is_multi_shipping` `is_multi_shipping` tinyint (1)UNSIGNED  DEFAULT '0' NULL ,
    change `is_multi_payment` `is_multi_payment` tinyint (1)UNSIGNED  DEFAULT '0' NULL ,
    change `customer_note_notify` `customer_note_notify` tinyint (1)UNSIGNED  DEFAULT '1' NULL ,
    change `customer_is_guest` `customer_is_guest` tinyint (1)UNSIGNED  DEFAULT '0' NULL ,
    change `quote_status_id` `quote_status_id` int (10)UNSIGNED  DEFAULT '0' NULL ,
    change `billing_address_id` `billing_address_id` int (10)UNSIGNED  DEFAULT '0' NULL ,
    change `orig_order_id` `orig_order_id` int (10)UNSIGNED  DEFAULT '0' NULL ,
    change `customer_id` `customer_id` int (10)UNSIGNED  DEFAULT '0' NULL ,
    change `customer_tax_class_id` `customer_tax_class_id` int (10)UNSIGNED  DEFAULT '0' NULL ,
    change `customer_group_id` `customer_group_id` int (10)UNSIGNED  DEFAULT '0' NULL ,
    change `items_count` `items_count` int (10)UNSIGNED  DEFAULT '0' NULL ,
    change `items_qty` `items_qty` decimal (12,4) DEFAULT '0.0000' NULL ,
    change `store_to_base_rate` `store_to_base_rate` decimal (12,4) DEFAULT '0.0000' NULL ,
    change `store_to_quote_rate` `store_to_quote_rate` decimal (12,4) DEFAULT '0.0000' NULL ,
    change `grand_total` `grand_total` decimal (12,4) DEFAULT '0.0000' NULL ,
    change `base_grand_total` `base_grand_total` decimal (12,4) DEFAULT '0.0000' NULL ,
    change `custbalance_amount` `custbalance_amount` decimal (12,4) DEFAULT '0.0000' NULL ,
    change `checkout_method` `checkout_method` varchar (255)  NULL  COLLATE utf8_general_ci ,
    change `password_hash` `password_hash` varchar (255)  NULL  COLLATE utf8_general_ci ,
    change `coupon_code` `coupon_code` varchar (255)  NULL  COLLATE utf8_general_ci ,
    change `base_currency_code` `base_currency_code` varchar (255)  NULL  COLLATE utf8_general_ci ,
    change `store_currency_code` `store_currency_code` varchar (255)  NULL  COLLATE utf8_general_ci ,
    change `quote_currency_code` `quote_currency_code` varchar (255)  NULL  COLLATE utf8_general_ci ,
    change `customer_email` `customer_email` varchar (255)  NULL  COLLATE utf8_general_ci ,
    change `customer_firstname` `customer_firstname` varchar (255)  NULL  COLLATE utf8_general_ci ,
    change `customer_lastname` `customer_lastname` varchar (255)  NULL  COLLATE utf8_general_ci ,
    change `customer_note` `customer_note` varchar (255)  NULL  COLLATE utf8_general_ci ,
    change `remote_ip` `remote_ip` varchar (255)  NULL  COLLATE utf8_general_ci ,
    change `applied_rule_ids` `applied_rule_ids` varchar (255)  NULL  COLLATE utf8_general_ci
");
$this->endSetup();
$this->installEntities();


text:
"""
This module provides the foundation to define jobs.

 .. moduleauthor:: Michael Markert <markert.michael@googlemail.com>
 .. moduleauthor:: Fabian Hirschmann <fabian@hirschmann.email>

 :copyright: PenchY Developers 2011-2012, see AUTHORS
 :license: MIT License, see LICENSE
"""
import logging
import os
import subprocess
from collections import defaultdict
from functools import partial
from hashlib import sha1
from itertools import groupby, chain
from operator import attrgetter
from tempfile import NamedTemporaryFile

from penchy.compat import update_hasher, write
from penchy.jobs.dependency import build_keys, edgesort
from penchy.jobs.elements import PipelineElement, SystemFilter
from penchy.jobs.filters import Receive, Send, WrongInputError
from penchy.jobs.plots import Plot
from penchy.jobs.hooks import Hook
from penchy.jobs.typecheck import TypeCheckError
from penchy.maven import get_classpath, setup_dependencies
from penchy.util import tempdir, default

log = logging.getLogger(__name__)

class NodeSetting(object):
    """
    Represents a configuration of a node.
    """

    def __init__(self, host, ssh_port, username, path,
                 basepath, description="", password=None,
                 keyfile=None, timeout_factor=1):
        """
        :param host: hostname (or IP) of node
        :type host: string
        :param ssh_port: port number of ssh server on node
        :type ssh_port: int
        :param username: login name for penchy on node
        :type username: string
        :param path: working directory on the node (this is where
                     the job will be uploaded to and where the
                     temporary files and directories will be created)
        :type path: string
        :param basepath: basepath for JVMs on this node
        :type basepath: string
        :param description: Textual description of node
        :type description: string
        :param password: this is either the password for a given username
                         or the passphrase to unlock the keyfile
        :type password: string
        :param keyfile: path to the ssh keyfile to use (defaults to system keyfile)
        :type keyfile: string
        :param timeout_factor: this is either an integer or a function
                               which gets executed client-side and returns
                               an integer. In either case, the resulting
                               integer will get multiplied with the timeout
                               for this node.
        :type timeout_factor: int or function
        """
        self.host = host
        self.ssh_port = ssh_port
        self.username = username
        self.path = path
        self.basepath = basepath
        self.description = description
        self.password = password
        self.keyfile = keyfile
        self._timeout_factor = timeout_factor

    @property
    def identifier(self):
        """
        A unique identifier for this node.
        """
        return self.host

    @property
    def timeout_factor(self):
        """
        The factor by which the timeout should get multiplied with.
        """
        if callable(self._timeout_factor):
            return self._timeout_factor()

        return self._timeout_factor

    def __eq__(self, other):
        return isinstance(other, NodeSetting) and \
                self.identifier == other.identifier

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return hash(self.identifier)

    def __str__(self):
        return self.identifier

    def hash(self):
        """
        Return the sha1 hexdigest.

        Used for identifying :class:`SystemComposition` across server and
        client.

        :returns: sha1 hexdigest of instance
        :rtype: string
        """
        hasher = sha1()
        update_hasher(hasher, self.identifier)
        return hasher.hexdigest()

class SystemComposition(object):
    """
    This class represents a combination of a :class:`~penchy.jobs.jvms.JVM`
    and a :class:`NodeSetting`.

    A :class:`SystemComposition` is a unique identifier that groups the
    results of its execution for server consumption.

    The ``flow`` of the :class:`SystemComposition` has to be set before
    executing the job::

        composition = SystemComposition(jvm, node)
        w = Dacapo('fop')
        jvm.workload = w
        composition.flow = [w >> Print()]

    """

    def __init__(self, jvm, node_setting, name=None):
        """
        :param jvm: the associated jvm
        :type jvm: :class:`~penchy.jobs.jvms.JVM`
        :param node_setting: the node setting
        :type node_setting: :class:`NodeSetting`
        """
        self.name = default(name, "{0} @ {1}".format(jvm, node_setting))
        self.jvm = jvm
        self.node_setting = node_setting
        self._flow = []

    def __eq__(self, other):
        try:
            return self.hash() == other.hash()
        except Exception:
            return False

    def __ne__(self, other):
        return self.jvm != other.jvm or self.node_setting != other.node_setting

    def __hash__(self):
        return hash(hash(self.jvm) + hash(self.node_setting))

    def __str__(self):  # pragma: no cover
        return self.name

    def hash(self):
        """
        Return the sha1 hexdigest.

        Used for identifying :class:`SystemComposition` across server and
        client.

        :returns: sha1 hexdigest of instance
        :rtype: str
        """
        hasher = sha1()
        update_hasher(hasher, self.jvm.hash())
        update_hasher(hasher, self.node_setting.hash())
        return hasher.hexdigest()

    def set_timeout_function(self, fun):
        """
        Set the timeout function of this composition.

        :param fun: timeout function
        :type fun: callable
        """
        if not self.timeout:
            return

        timeout_hook = Hook(setup=lambda: fun(self.hash(), self.timeout),
                            teardown=lambda: fun(self.hash(), 0))
        self.jvm.hooks.append(timeout_hook)

    @property
    def flow(self):
        """
        The flow of the pipeline.
        """
        return self._flow

    @flow.setter
    def flow(self, flow):
        """
        The flow of the pipeline.

        :param flow: flow of execution after JVM
        :type flow: List of :class:`Edge` or :class:`Pipeline`
        """
        self._flow = list(chain.from_iterable(e.edges for e in flow))

    @property
    def starts(self):
        """
        The starts of the pipeline.

        All :class:`~penchy.jobs.elements.PipelineElement` that
        are executed by the jvm.
        """
        return [e for e in (self.jvm.workload, self.jvm.tool, self.jvm)
                if isinstance(e, PipelineElement)]

    @property
    def elements(self):
        """
        All :class:`~penchy.jobs.elements.PipelineElement` of this composition.
        """
        if not self._flow:
            log.warn('No flow set')

        elements = set(e.source for e in self._flow)
        elements.update(e.sink for e in self._flow)
        if self.jvm.workload:
            elements.add(self.jvm.workload)
        if self.jvm.tool:
            elements.add(self.jvm.tool)
        if isinstance(self.jvm, PipelineElement):
            elements.add(self.jvm)

        return elements

    @property
    def timeout(self):
        """
        Timeout of this Composition.
        """
        return self.jvm.timeout * self.node_setting.timeout_factor

    def _reset(self):
        """
        Reset the elements..
        """
        for e in self.elements:
            e.reset()

class Job(object):
    """
    Represents a job.

    Those attributes have to be set from the outside before a :class:`Job`
    instance is ``run``:

    - ``job.send`` to be set on client to a function with a signature
      ``(hash, data)`` where ``hash`` identifies the :class:`SystemComposition`
      and ``data`` the data to send

    - ``job.receive`` to be set on server to a function that takes no arguments
      and returns the results of all :class:`SystemComposition` with the
      :class:`SystemComposition` as key and the results as value.

    - ``job.filename`` has to be set to the filename of the job
    """

    def __init__(self, compositions, server_flow, invocations=1):
        """
        :param compositions: :class:`SystemComposition` to execute jobs on
        :type compositions: List of :class:`SystemComposition`
                              or :class:`SystemComposition`
        :param server_flow: describes the execution of the job on the server
        :type server_flow: List of :class:`~penchy.jobs.dependency.Edge` or
                           :class:`~penchy.jobs.dependency.Pipeline`
        :param invocations: number of times to run job on each configuration
        :type invocations: int
        """
        self.compositions = compositions if isinstance(compositions, list) \
                            else [compositions]
        self.server_flow = list(chain.from_iterable(dep.edges for dep in server_flow))
        self.invocations = invocations
        self.send = None
        self.timeout = None
        self.receive = None
        self._composition = None
        self.filename = None

    def run(self, composition):
        """
        Run clientside Job.

        :param composition: composition to run.
        :type composition: :class:`SystemComposition`
        """
        # setup
        pomfile = os.path.join(composition.node_setting.path, 'pom.xml')
        setup_dependencies(pomfile, self._get_client_dependencies(composition))
        composition.jvm.add_to_cp(get_classpath(pomfile))
        self._composition = composition

        # save send for restoring
        send = self.send
        # replace with one that knows how to identify the composition if it is set
        if self.send is not None:
            self.send = partial(self.send, composition.hash())

        composition.jvm.basepath = composition.node_setting.basepath

        _, edge_order = edgesort(composition.starts, composition.flow)

        for i in range(1, self.invocations + 1):
            log.info('Run invocation {0}'.format(i))
            with tempdir(prefix='penchy-invocation{0}-'.format(i)):
                composition.jvm.run()

        log.info('Run pipeline')
        for sink, group in groupby(edge_order, attrgetter('sink')):
            kwargs = build_keys(group)
            if isinstance(sink, SystemFilter):
                kwargs[':environment:'] = self._build_environment()
            log.debug('Passing this input to {0}:\n{1}'
                      .format(sink.__class__.__name__,
                              kwargs))
            try:
                sink.run(**kwargs)
            except TypeCheckError:
                log.error('Type check failed on component {0} and arguments {1}'
                          .format(sink.__class__.__name__, kwargs))
                raise
            except WrongInputError:
                log.error('Run failed on component {0} and arguments {1}'
                          .format(sink.__class__.__name__, kwargs))
                raise
            log.debug('{0} transformed input to:\n{1}'
                      .format(sink.__class__.__name__, sink.out))

        # reset state of filters for running multiple configurations
        composition._reset()
        # restore send
        self.send = send
        self._composition = None

    def _get_client_dependencies(self, composition):
        """
        Return all clientside :class:`MavenDependency` of this job for a given
        :class:`SystemComposition`.

        :raises: :exc:`ValueError` if ``composition`` is not part of this job

        :param composition: composition to analyze.
        :type composition::class:`SystemComposition`
        :returns: Set of :class:`MavenDependency`.
        :rtype: set
        """
        if composition not in self.compositions:
            raise ValueError('composition not part of this job')

        return set(chain.from_iterable(e.DEPENDENCIES for e in composition.elements))

    def run_server_pipeline(self):
        """
        Run the serverside pipeline.

        :raise: :exc:`ValueError` if there is no
                :class:`~penchy.jobs.filters.Receive` in the serverside pipeline
        """
        # Do nothing if server_flow is empty
        if not self.server_flow:
            return

        starts = [edge.source for edge in self.server_flow
                  if isinstance(edge.source, Receive)]

        if not starts:
            log.error('There is no Receiver in the serverside flow. Aborting.')
            raise ValueError('There is no Receiver in the serverside flow')

        _, edge_order = edgesort(starts, self.server_flow)

        # all starts are receivers, run them with the environment
        for start in starts:
            start.run(**{':environment:': self._build_environment()})

        # run other filters
        for sink, group in groupby(edge_order, attrgetter('sink')):
            kwargs = build_keys(group)
            if isinstance(sink, SystemFilter):
                kwargs[':environment:'] = self._build_environment()
            log.debug('Passing this input to {0}:\n{1}'
                      .format(sink.__class__.__name__,
                              kwargs))
            try:
                sink.run(**kwargs)
            except TypeCheckError:
                log.error('Type check failed on component {0} and arguments {1}'
                          .format(sink.__class__.__name__, kwargs))
                raise
            except WrongInputError:
                log.error('Run failed on component {0} and arguments {1}'
                          .format(sink.__class__.__name__, kwargs))
                raise
            log.debug('{0} transformed input to:\n{1}'
                      .format(sink.__class__.__name__, sink.out))

    def _get_server_dependencies(self):
        """
        Return the serverside dependencies of the job.

        :returns: Set of :class:`MavenDependency`.
        :rtype: set
        """
        return set(chain.from_iterable(element.DEPENDENCIES for element
                                       in self.server_flow))

    def _build_environment(self):
        """
        Return the environment for a :class:`SystemFilter`.

        Contains:
        - ``receive``: to get all data that has been received, takes no arguments
                       returns a dict with :class:`SystemComposition` as keys
        - ``send``: to send data to the server, takes one datum as argument
        - ``job``: the filename this job
        - ``current_composition``: the current :class:`SystemComposition` or ``None``

        :returns: environment for a SystemFilter
        :rtype: dict
        """
        # replace receive and send with dummy functions if not set to avoid
        # corner cases in pipeline
        receive = default(self.receive, lambda: {})
        send = default(self.send, lambda data: None)

        return dict(receive=receive,
                    send=send,
                    job=self.filename,
                    current_composition=self._composition)

    def compositions_for_node(self, identifier):
        """
        Return the compositions of this job that are to be run on the node
        that corresponds to ``identifier``.

        :param identifier: identifier for node.
        :type host: str
        :returns: :class:`SystemComposition` of job that run on host
        :rtype: list
        """
        return [c for c in self.compositions if c.node_setting.identifier == identifier]

    def check(self):
        """
        Check job for plausibility.

        :returns: if job is plausible, i.e. no failure is expected.
        :rtype: bool
        """
        valid = True
        for composition in self.compositions:
            # check if workload is set
            if composition.jvm.workload is None:
                log.error('Check: composition "{0}" has no workload'
                          .format(composition))
                valid = False

            # check if there are no starts in client pipelines
            if not composition.starts:
                log.error('Check: composition {0} has no starts'
                          .format(composition.name))
                valid = False
            # check if there are cycles in client pipelines
            try:
                edgesort(composition.starts, composition.flow)
            except ValueError:
                log.exception('Check: cycle on composition "{0}"'
                              .format(composition))
                valid = False

            if not any(isinstance(e, Send) for e in composition.elements):
                log.error('Check: there is no Send in composition "{0}"'
                          .format(composition))
                valid = False

            if any(isinstance(e, Plot) for e in composition.elements):
                log.error('Check: there is a Plot in composition "{0}"'
                          .format(composition))
                valid = False

        # check if there are cycles in server pipeline
        starts = [edge.source for edge in self.server_flow
                  if isinstance(edge.source, Receive)]
        if not starts:
            # if there are no receivers the server pipeline is not valid
            log.error('Check: There is no Receiver in server pipeline')
            valid = False
        else:
            try:
                edgesort(starts, self.server_flow)
            except ValueError:
                log.exception('Check: cycle in server pipeline')
                valid = False

        for flow in chain(chain(c.flow for c in self.compositions),
                         [self.server_flow]):
            valid = valid and self._check_flow(flow)

        return valid

    def _check_flow(self, flow):
        """
        Check the flow for validity.

        That is:
        - all Edges are valid (inputs/outputs exist)
        - all Sinks are saturated

        :param flow: flow to check
        :type flow: sequence of :class:`~penchy.jobs.dependency.Edge`
        :returns: if flow is valid
        :rtype: bool
        """
        valid = True
        sources_of_sinks = defaultdict(list)
        for edge in flow:
            if not edge.check():
                log.error('Error at connection from "{0}" to "{1}"'
                          .format(edge.source.__class__.__name__,
                                  edge.sink.__class__.__name__))
                valid = False
            source_name_mapping = (edge.source.__class__.__name__,
                                   default(edge.map_,
                                           [(name, name) for name
                                            in edge.source.outputs.names]))
            sources_of_sinks[edge.sink].append(source_name_mapping)
        for sink, source_mappings in sources_of_sinks.items():
            if not sink.inputs.check_sink(source_mappings):
                log.error('Error at sink "{0}"'.format(sink.__class__.__name__))
                valid = False

        return valid

    def visualize(self, format='png', dot='dot'):
        """
        Visualize job via graphviz.

        .. note::

            The ``dot`` executable has to be in the path or passed as ``dot``
            parameter.

        :param format: output format (has to be supported by dot)
        :type format: str
        :param dot: path to dot executable
        :type dot: str

        :returns: the path to the generated file
        :rtype: str
        """
        def edges_of_flow(flow, flow_id=0):
            """
            :param flow: flow to visualize
            :type flow: list of :class:`~penchy.jobs.dependency.Edge`
            :param flow_id: id to differentiate between multiple flows with same elements
            :type flow_id: int
            :returns: edges in graphviz format
            :rtype: list of str
            """

            return [""" node{source_id}{id} [label = "{source}"];
                        node{sink_id}{id} [label = "{sink}"];
                        node{source_id}{id} -> node{sink_id}{id} [label = "{decoration}"];
                    """
                    .format(source=e.source,
                            source_id=id(e.source),
                            sink=e.sink,
                            sink_id=id(e.sink),
                            id=flow_id,
                            decoration=', '.join('{0} -> {1}'
                                                 .format(m[0], m[1])
                                                 if m[0] != m[1]
                                                 else m[0]
                                                 for m in e.map_
                                             )
                                    if e.map_ else '')
                    for e in flow]

        clients = ["""
                   subgraph cluster_client%d {
                       label = "%s";
                       %s
                   }
                   """ % (i, c.name, '\n'.join(edges_of_flow(c.flow, i)))
                   for i, c in enumerate(self.compositions)]

        server_edges = edges_of_flow(self.server_flow)
        s = """
        digraph G {
            rankdir = LR
            subgraph cluster_server {
                color = blue;
                %s
                label = "Server";
            }
            subgraph cluster_client {
                color = black;
                label = "Clients";
                %s
            }
        }
        """ % ('\n'.join(server_edges), '\n'.join(clients))
        with NamedTemporaryFile(delete=False) as f:
            fname = f.name
            write(f, s)
        subprocess.call([dot, '-T', format, '-O', fname])
        os.remove(fname)
        return '{0}.{1}'.format(fname, format)


text:
package com.apptentive.android.sdk.storage;

/**
 * @author Sky Kelsey
 */
public interface EventStore extends PayloadStore {

}