text:
#include "vrtdataset.h"

#include "cpl_minixml.h"
#include "cpl_string.h"
#include "ogr_spatialref.h"

#include <algorithm>

CPL_CVSID("$Id: vrtdataset.cpp 33808 2016-03-29 21:15:28Z goatbar $");

/************************************************************************/
/*                            VRTDataset()                             */
/************************************************************************/

VRTDataset::VRTDataset( int nXSize, int nYSize ) :
    m_pszProjection(NULL),
    m_bGeoTransformSet(FALSE),
    m_nGCPCount(0),
    m_pasGCPList(NULL),
    m_pszGCPProjection(NULL),
    m_bNeedsFlush(FALSE),
    m_bWritable(TRUE),
    m_pszVRTPath(NULL),
    m_poMaskBand(NULL),
    m_bCompatibleForDatasetIO(-1)
{
    nRasterXSize = nXSize;
    nRasterYSize = nYSize;

    m_adfGeoTransform[0] = 0.0;
    m_adfGeoTransform[1] = 1.0;
    m_adfGeoTransform[2] = 0.0;
    m_adfGeoTransform[3] = 0.0;
    m_adfGeoTransform[4] = 0.0;
    m_adfGeoTransform[5] = 1.0;

    GDALRegister_VRT();

    poDriver = reinterpret_cast<GDALDriver *>( GDALGetDriverByName( "VRT" ) );
}

/************************************************************************/
/*                              VRTCreate()                             */
/************************************************************************/

/**
 * @see VRTDataset::VRTDataset()
 */

VRTDatasetH CPL_STDCALL VRTCreate(int nXSize, int nYSize)

{
    return ( new VRTDataset(nXSize, nYSize) );
}

/************************************************************************/
/*                            ~VRTDataset()                            */
/************************************************************************/

VRTDataset::~VRTDataset()

{
    FlushCache();
    CPLFree( m_pszProjection );

    CPLFree( m_pszGCPProjection );
    if( m_nGCPCount > 0 )
    {
        GDALDeinitGCPs( m_nGCPCount, m_pasGCPList );
        CPLFree( m_pasGCPList );
    }
    CPLFree( m_pszVRTPath );

    delete m_poMaskBand;

    for(size_t i=0;i<m_apoOverviews.size();i++)
        delete m_apoOverviews[i];
    for(size_t i=0;i<m_apoOverviewsBak.size();i++)
        delete m_apoOverviewsBak[i];
}

/************************************************************************/
/*                             FlushCache()                             */
/************************************************************************/

void VRTDataset::FlushCache()

{
    GDALDataset::FlushCache();

    if( !m_bNeedsFlush || m_bWritable == FALSE)
        return;

    m_bNeedsFlush = FALSE;

    // We don't write to disk if there is no filename.  This is a
    // memory only dataset.
    if( strlen(GetDescription()) == 0
        || STARTS_WITH_CI(GetDescription(), "<VRTDataset") )
        return;

    /* -------------------------------------------------------------------- */
    /*      Create the output file.                                         */
    /* -------------------------------------------------------------------- */
    VSILFILE *fpVRT = VSIFOpenL( GetDescription(), "w" );
    if( fpVRT == NULL )
    {
        CPLError( CE_Failure, CPLE_AppDefined,
                  "Failed to write .vrt file in FlushCache()." );
        return;
    }

    /* -------------------------------------------------------------------- */
    /*      Convert tree to a single block of XML text.                     */
    /* -------------------------------------------------------------------- */
    char** papszContent = GetMetadata("xml:VRT");
    bool bOK = true;
    if( papszContent && papszContent[0] )
    {
        /* ------------------------------------------------------------------ */
        /*      Write to disk.                                                */
        /* ------------------------------------------------------------------ */
        bOK &= VSIFWriteL( papszContent[0], 1, strlen(papszContent[0]), fpVRT ) == strlen(papszContent[0]);
    }
    if( VSIFCloseL( fpVRT ) != 0 )
        bOK = false;
    if( !bOK )
    {
        CPLError( CE_Failure, CPLE_AppDefined,
                  "Failed to write .vrt file in FlushCache()." );
        return;
    }
}

/************************************************************************/
/*                            GetMetadata()                             */
/************************************************************************/

char** VRTDataset::GetMetadata( const char *pszDomain )
{
    if( pszDomain != NULL && EQUAL(pszDomain, "xml:VRT") )
    {
        /* ------------------------------------------------------------------ */
        /*      Convert tree to a single block of XML text.                   */
        /* ------------------------------------------------------------------ */
        char *l_pszVRTPath = CPLStrdup(CPLGetPath(GetDescription()));
        CPLXMLNode *psDSTree = SerializeToXML( l_pszVRTPath );
        char *pszXML = CPLSerializeXMLTree( psDSTree );

        CPLDestroyXMLNode( psDSTree );

        CPLFree( l_pszVRTPath );

        char* apszContent[2] = { pszXML, NULL };
        GDALDataset::SetMetadata(apszContent, "xml:VRT");
        CPLFree(pszXML);
    }

    return GDALDataset::GetMetadata(pszDomain);
}

/************************************************************************/
/*                            VRTFlushCache()                           */
/************************************************************************/

/**
 * @see VRTDataset::FlushCache()
 */

void CPL_STDCALL VRTFlushCache( VRTDatasetH hDataset )
{
    VALIDATE_POINTER0( hDataset, "VRTFlushCache" );

    reinterpret_cast<VRTDataset *>( hDataset )->FlushCache();
}

/************************************************************************/
/*                           SerializeToXML()                           */
/************************************************************************/

CPLXMLNode *VRTDataset::SerializeToXML( const char *pszVRTPathIn )

{
    /* -------------------------------------------------------------------- */
    /*      Setup root node and attributes.                                 */
    /* -------------------------------------------------------------------- */
    CPLXMLNode *psDSTree = CPLCreateXMLNode( NULL, CXT_Element, "VRTDataset" );

    char szNumber[128];
    snprintf( szNumber, sizeof(szNumber), "%d", GetRasterXSize() );
    CPLSetXMLValue( psDSTree, "#rasterXSize", szNumber );

    snprintf( szNumber, sizeof(szNumber), "%d", GetRasterYSize() );
    CPLSetXMLValue( psDSTree, "#rasterYSize", szNumber );

 /* -------------------------------------------------------------------- */
 /*      SRS                                                             */
 /* -------------------------------------------------------------------- */
    if( m_pszProjection != NULL && strlen(m_pszProjection) > 0 )
        CPLSetXMLValue( psDSTree, "SRS", m_pszProjection );

 /* -------------------------------------------------------------------- */
 /*      Geotransform.                                                   */
 /* -------------------------------------------------------------------- */
    if( m_bGeoTransformSet )
    {
        CPLSetXMLValue(
            psDSTree, "GeoTransform",
            CPLSPrintf( "%24.16e,%24.16e,%24.16e,%24.16e,%24.16e,%24.16e",
                        m_adfGeoTransform[0],
                        m_adfGeoTransform[1],
                        m_adfGeoTransform[2],
                        m_adfGeoTransform[3],
                        m_adfGeoTransform[4],
                        m_adfGeoTransform[5] ) );
    }

/* -------------------------------------------------------------------- */
/*      Metadata                                                        */
/* -------------------------------------------------------------------- */
    CPLXMLNode *psMD = oMDMD.Serialize();
    if( psMD != NULL )
    {
        CPLAddXMLChild( psDSTree, psMD );
    }

 /* -------------------------------------------------------------------- */
 /*      GCPs                                                            */
 /* -------------------------------------------------------------------- */
    if( m_nGCPCount > 0 )
    {
        GDALSerializeGCPListToXML( psDSTree,
                                   m_pasGCPList,
                                   m_nGCPCount,
                                   m_pszGCPProjection );
    }

    /* -------------------------------------------------------------------- */
    /*      Serialize bands.                                                */
    /* -------------------------------------------------------------------- */
    for( int iBand = 0; iBand < nBands; iBand++ )
    {
        CPLXMLNode *psBandTree =
            reinterpret_cast<VRTRasterBand *>(
                papoBands[iBand])->SerializeToXML( pszVRTPathIn );

        if( psBandTree != NULL )
            CPLAddXMLChild( psDSTree, psBandTree );
    }

    /* -------------------------------------------------------------------- */
    /*      Serialize dataset mask band.                                    */
    /* -------------------------------------------------------------------- */
    if (m_poMaskBand)
    {
        CPLXMLNode *psBandTree =
            m_poMaskBand->SerializeToXML(pszVRTPathIn);

        if( psBandTree != NULL )
        {
            CPLXMLNode *psMaskBandElement
                = CPLCreateXMLNode( psDSTree, CXT_Element, "MaskBand" );
            CPLAddXMLChild( psMaskBandElement, psBandTree );
        }
    }

    return psDSTree;
}

/************************************************************************/
/*                          VRTSerializeToXML()                         */
/************************************************************************/

/**
 * @see VRTDataset::SerializeToXML()
 */

CPLXMLNode * CPL_STDCALL VRTSerializeToXML( VRTDatasetH hDataset,
                                            const char *pszVRTPath )
{
    VALIDATE_POINTER1( hDataset, "VRTSerializeToXML", NULL );

    return reinterpret_cast<VRTDataset *>(
        hDataset )->SerializeToXML(pszVRTPath);
}

/************************************************************************/
/*                              XMLInit()                               */
/************************************************************************/

CPLErr VRTDataset::XMLInit( CPLXMLNode *psTree, const char *pszVRTPathIn )

{
    if( pszVRTPathIn != NULL )
        m_pszVRTPath = CPLStrdup(pszVRTPathIn);

/* -------------------------------------------------------------------- */
/*      Check for an SRS node.                                          */
/* -------------------------------------------------------------------- */
    if( strlen(CPLGetXMLValue(psTree, "SRS", "")) > 0 )
    {
        CPLFree( m_pszProjection );
        m_pszProjection = NULL;

        OGRSpatialReference oSRS;
        if( oSRS.SetFromUserInput( CPLGetXMLValue(psTree, "SRS", "") )
            == OGRERR_NONE )
            oSRS.exportToWkt( &m_pszProjection );
    }

/* -------------------------------------------------------------------- */
/*      Check for a GeoTransform node.                                  */
/* -------------------------------------------------------------------- */
    if( strlen(CPLGetXMLValue(psTree, "GeoTransform", "")) > 0 )
    {
        const char *pszGT = CPLGetXMLValue(psTree, "GeoTransform", "");
        char **papszTokens
            = CSLTokenizeStringComplex( pszGT, ",", FALSE, FALSE );
        if( CSLCount(papszTokens) != 6 )
        {
            CPLError( CE_Warning, CPLE_AppDefined,
                      "GeoTransform node does not have expected six values.");
        }
        else
        {
            for( int iTA = 0; iTA < 6; iTA++ )
                m_adfGeoTransform[iTA] = CPLAtof(papszTokens[iTA]);
            m_bGeoTransformSet = TRUE;
        }

        CSLDestroy( papszTokens );
    }

/* -------------------------------------------------------------------- */
/*      Check for GCPs.                                                 */
/* -------------------------------------------------------------------- */
    CPLXMLNode *psGCPList = CPLGetXMLNode( psTree, "GCPList" );

    if( psGCPList != NULL )
    {
        GDALDeserializeGCPListFromXML( psGCPList,
                                       &m_pasGCPList,
                                       &m_nGCPCount,
                                       &m_pszGCPProjection );
    }

/* -------------------------------------------------------------------- */
/*      Apply any dataset level metadata.                               */
/* -------------------------------------------------------------------- */
    oMDMD.XMLInit( psTree, TRUE );

/* -------------------------------------------------------------------- */
/*      Create dataset mask band.                                       */
/* -------------------------------------------------------------------- */

    /* Parse dataset mask band first */
    CPLXMLNode* psMaskBandNode = CPLGetXMLNode(psTree, "MaskBand");

    CPLXMLNode *psChild;
    if (psMaskBandNode)
        psChild = psMaskBandNode->psChild;
    else
        psChild = NULL;

    for( ; psChild != NULL; psChild=psChild->psNext )
    {
        if( psChild->eType == CXT_Element
            && EQUAL(psChild->pszValue,"VRTRasterBand") )
        {
            const char *pszSubclass = CPLGetXMLValue( psChild, "subclass",
                                                      "VRTSourcedRasterBand" );

            VRTRasterBand  *poBand = NULL;
            if( EQUAL(pszSubclass,"VRTSourcedRasterBand") )
                poBand = new VRTSourcedRasterBand( this, 0 );
            else if( EQUAL(pszSubclass, "VRTDerivedRasterBand") )
                poBand = new VRTDerivedRasterBand( this, 0 );
            else if( EQUAL(pszSubclass, "VRTRawRasterBand") )
                poBand = new VRTRawRasterBand( this, 0 );
            else if( EQUAL(pszSubclass, "VRTWarpedRasterBand") &&
                     dynamic_cast<VRTWarpedDataset*>(this) != NULL )
                poBand = new VRTWarpedRasterBand( this, 0 );
            //else if( EQUAL(pszSubclass, "VRTPansharpenedRasterBand") )
            //    poBand = new VRTPansharpenedRasterBand( this, 0 );
            else
                CPLError( CE_Failure, CPLE_AppDefined,
                          "VRTRasterBand of unrecognized subclass '%s'.",
                          pszSubclass );

            if( poBand != NULL
                && poBand->XMLInit( psChild, pszVRTPathIn ) == CE_None )
            {
                SetMaskBand(poBand);
                break;
            }
            else
            {
                if( poBand )
                    delete poBand;
                return CE_Failure;
            }
        }
    }

/* -------------------------------------------------------------------- */
/*      Create band information objects.                                */
/* -------------------------------------------------------------------- */
    int		l_nBands = 0;
    for( psChild=psTree->psChild; psChild != NULL; psChild=psChild->psNext )
    {
        if( psChild->eType == CXT_Element
            && EQUAL(psChild->pszValue,"VRTRasterBand") )
        {
            const char *pszSubclass = CPLGetXMLValue( psChild, "subclass",
                                                      "VRTSourcedRasterBand" );

            VRTRasterBand  *poBand = NULL;
            if( EQUAL(pszSubclass,"VRTSourcedRasterBand") )
                poBand = new VRTSourcedRasterBand( this, l_nBands+1 );
            else if( EQUAL(pszSubclass, "VRTDerivedRasterBand") )
                poBand = new VRTDerivedRasterBand( this, l_nBands+1 );
            else if( EQUAL(pszSubclass, "VRTRawRasterBand") )
                poBand = new VRTRawRasterBand( this, l_nBands+1 );
            else if( EQUAL(pszSubclass, "VRTWarpedRasterBand") &&
                     dynamic_cast<VRTWarpedDataset*>(this) != NULL )
                poBand = new VRTWarpedRasterBand( this, l_nBands+1 );
            else if( EQUAL(pszSubclass, "VRTPansharpenedRasterBand") &&
                     dynamic_cast<VRTPansharpenedDataset*>(this) != NULL )
                poBand = new VRTPansharpenedRasterBand( this, l_nBands+1 );
            else
                CPLError( CE_Failure, CPLE_AppDefined,
                          "VRTRasterBand of unrecognized subclass '%s'.",
                          pszSubclass );

            if( poBand != NULL
                && poBand->XMLInit( psChild, pszVRTPathIn ) == CE_None )
            {
                SetBand( ++l_nBands, poBand );
            }
            else
            {
                if( poBand )
                    delete poBand;
                return CE_Failure;
            }
        }
    }

    return CE_None;
}

/************************************************************************/
/*                            GetGCPCount()                             */
/************************************************************************/

int VRTDataset::GetGCPCount()

{
    return m_nGCPCount;
}

/************************************************************************/
/*                          GetGCPProjection()                          */
/************************************************************************/

const char *VRTDataset::GetGCPProjection()

{
    if( m_pszGCPProjection == NULL )
        return "";

    return m_pszGCPProjection;
}

/************************************************************************/
/*                               GetGCPs()                              */
/************************************************************************/

const GDAL_GCP *VRTDataset::GetGCPs()

{
    return m_pasGCPList;
}

/************************************************************************/
/*                              SetGCPs()                               */
/************************************************************************/

CPLErr VRTDataset::SetGCPs( int nGCPCountIn, const GDAL_GCP *pasGCPListIn,
                            const char *pszGCPProjectionIn )

{
    CPLFree( m_pszGCPProjection );
    if( m_nGCPCount > 0 )
    {
        GDALDeinitGCPs( m_nGCPCount, m_pasGCPList );
        CPLFree( m_pasGCPList );
    }

    m_pszGCPProjection = CPLStrdup(pszGCPProjectionIn);

    m_nGCPCount = nGCPCountIn;

    m_pasGCPList = GDALDuplicateGCPs( nGCPCountIn, pasGCPListIn );

    m_bNeedsFlush = TRUE;

    return CE_None;
}

/************************************************************************/
/*                           SetProjection()                            */
/************************************************************************/

CPLErr VRTDataset::SetProjection( const char *pszWKT )

{
    CPLFree( m_pszProjection );
    m_pszProjection = NULL;

    if( pszWKT != NULL )
        m_pszProjection = CPLStrdup(pszWKT);

    m_bNeedsFlush = TRUE;

    return CE_None;
}

/************************************************************************/
/*                          GetProjectionRef()                          */
/************************************************************************/

const char *VRTDataset::GetProjectionRef()

{
    if( m_pszProjection == NULL )
        return "";

    return m_pszProjection;
}

/************************************************************************/
/*                          SetGeoTransform()                           */
/************************************************************************/

CPLErr VRTDataset::SetGeoTransform( double *padfGeoTransformIn )

{
    memcpy( m_adfGeoTransform, padfGeoTransformIn, sizeof(double) * 6 );
    m_bGeoTransformSet = TRUE;

    m_bNeedsFlush = TRUE;

    return CE_None;
}

/************************************************************************/
/*                          GetGeoTransform()                           */
/************************************************************************/

CPLErr VRTDataset::GetGeoTransform( double * padfGeoTransform )

{
    memcpy( padfGeoTransform, m_adfGeoTransform, sizeof(double) * 6 );

    return m_bGeoTransformSet ? CE_None : CE_Failure;
}

/************************************************************************/
/*                            SetMetadata()                             */
/************************************************************************/

CPLErr VRTDataset::SetMetadata( char **papszMetadata,
                                const char *pszDomain )

{
    SetNeedsFlush();

    return GDALDataset::SetMetadata( papszMetadata, pszDomain );
}

/************************************************************************/
/*                          SetMetadataItem()                           */
/************************************************************************/

CPLErr VRTDataset::SetMetadataItem( const char *pszName,
                                    const char *pszValue,
                                    const char *pszDomain )

{
    SetNeedsFlush();

    return GDALDataset::SetMetadataItem( pszName, pszValue, pszDomain );
}

/************************************************************************/
/*                              Identify()                              */
/************************************************************************/

int VRTDataset::Identify( GDALOpenInfo * poOpenInfo )

{
    if( poOpenInfo->nHeaderBytes > 20
         && strstr((const char *)poOpenInfo->pabyHeader,"<VRTDataset") != NULL )
        return TRUE;

    if( strstr(poOpenInfo->pszFilename,"<VRTDataset") != NULL )
        return TRUE;

    return FALSE;
}

/************************************************************************/
/*                                Open()                                */
/************************************************************************/

GDALDataset *VRTDataset::Open( GDALOpenInfo * poOpenInfo )

{
/* -------------------------------------------------------------------- */
/*      Does this appear to be a virtual dataset definition XML         */
/*      file?                                                           */
/* -------------------------------------------------------------------- */
    if( !Identify( poOpenInfo ) )
        return NULL;

/* -------------------------------------------------------------------- */
/*	Try to read the whole file into memory.				*/
/* -------------------------------------------------------------------- */
    char *pszXML = NULL;
    VSILFILE *fp = poOpenInfo->fpL;

    char *pszVRTPath = NULL;
    if( fp != NULL )
    {
        unsigned int nLength;

        poOpenInfo->fpL = NULL;

        if( strcmp(poOpenInfo->pszFilename, "/vsistdin/") == 0 )
        {
            nLength = 0;
            pszXML = reinterpret_cast<char *>( VSIMalloc(1024+1) );
            while( true )
            {
                const int nRead = static_cast<int>(
                    VSIFReadL( pszXML + nLength, 1, 1024, fp) );
                nLength += nRead;
                if( nRead < 1024 )
                    break;
                char* pszXMLNew = reinterpret_cast<char *>(
                    VSIRealloc( pszXML, nLength + 1024 + 1) );
                if( pszXMLNew == NULL )
                {
                    VSIFree(pszXML);
                    return NULL;
                }
                pszXML = pszXMLNew;
            }
        }
        else
        {
            CPL_IGNORE_RET_VAL(VSIFSeekL( fp, 0, SEEK_END ));
            nLength = static_cast<int>( VSIFTellL( fp ) );
            CPL_IGNORE_RET_VAL(VSIFSeekL( fp, 0, SEEK_SET ));

            pszXML = reinterpret_cast<char *>( VSI_MALLOC_VERBOSE(nLength+1) );

            if( pszXML == NULL )
            {
                CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
                return NULL;
            }

            if( VSIFReadL( pszXML, 1, nLength, fp ) != nLength )
            {
                CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
                CPLFree( pszXML );
                CPLError( CE_Failure, CPLE_FileIO,
                          "Failed to read %d bytes from VRT xml file.",
                          nLength );
                return NULL;
            }
        }

        pszXML[nLength] = '\0';

        char* pszCurDir = CPLGetCurrentDir();
        const char *currentVrtFilename
            = CPLProjectRelativeFilename(pszCurDir, poOpenInfo->pszFilename);
        CPLFree(pszCurDir);

#if defined(HAVE_READLINK) && defined(HAVE_LSTAT)
        char filenameBuffer[2048];

        while( true ) {
            VSIStatBuf statBuffer;
            int lstatCode = lstat( currentVrtFilename, &statBuffer );
            if ( lstatCode == -1 ) {
                if (errno == ENOENT) {
                    // The file could be a virtual file, let later checks handle it.
                    break;
                } else {
                    CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
                    CPLFree( pszXML );
                    CPLError( CE_Failure, CPLE_FileIO,
                              "Failed to lstat %s: %s",
                              currentVrtFilename,
                              VSIStrerror(errno) );
                    return NULL;
                }
            }

            if ( !VSI_ISLNK(statBuffer.st_mode) ) {
                break;
            }

            const int bufferSize = static_cast<int>(readlink( currentVrtFilename,
                                             filenameBuffer,
                                             sizeof(filenameBuffer) ));
            if (bufferSize != -1) {
                filenameBuffer[std::min(bufferSize, static_cast<int>(
                    sizeof(filenameBuffer) ) - 1)] = 0;
                // The filename in filenameBuffer might be a relative path
                // from the linkfile resolve it before looping
                currentVrtFilename = CPLProjectRelativeFilename(
                    CPLGetDirname( currentVrtFilename ), filenameBuffer);
            } else {
                CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
                CPLFree( pszXML );
                CPLError( CE_Failure, CPLE_FileIO,
                          "Failed to read filename from symlink %s: %s",
                          currentVrtFilename,
                          VSIStrerror(errno) );
                return NULL;
            }
        }
#endif  // HAVE_READLINK && HAVE_LSTAT

        pszVRTPath = CPLStrdup(CPLGetPath(currentVrtFilename));

        CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
    }
/* -------------------------------------------------------------------- */
/*      Or use the filename as the XML input.                           */
/* -------------------------------------------------------------------- */
    else
    {
        pszXML = CPLStrdup( poOpenInfo->pszFilename );
    }

    if( CSLFetchNameValue(poOpenInfo->papszOpenOptions, "ROOT_PATH") != NULL )
    {
        CPLFree(pszVRTPath);
        pszVRTPath = CPLStrdup(
            CSLFetchNameValue( poOpenInfo->papszOpenOptions, "ROOT_PATH" ) );
    }

/* -------------------------------------------------------------------- */
/*      Turn the XML representation into a VRTDataset.                  */
/* -------------------------------------------------------------------- */
    VRTDataset *poDS = reinterpret_cast<VRTDataset *>(
        OpenXML( pszXML, pszVRTPath, poOpenInfo->eAccess ) );

    if( poDS != NULL )
        poDS->m_bNeedsFlush = FALSE;

    CPLFree( pszXML );
    CPLFree( pszVRTPath );

/* -------------------------------------------------------------------- */
/*      Initialize info for later overview discovery.                   */
/* -------------------------------------------------------------------- */
    if( fp != NULL && poDS != NULL )
    {
        poDS->oOvManager.Initialize( poDS, poOpenInfo->pszFilename );
        if( poOpenInfo->AreSiblingFilesLoaded() )
            poDS->oOvManager.TransferSiblingFiles( poOpenInfo->StealSiblingFiles() );
    }

    return poDS;
}

/************************************************************************/
/*                              OpenXML()                               */
/*                                                                      */
/*      Create an open VRTDataset from a supplied XML representation    */
/*      of the dataset.                                                 */
/************************************************************************/

GDALDataset *VRTDataset::OpenXML( const char *pszXML, const char *pszVRTPath,
                                  GDALAccess eAccess)

{
 /* -------------------------------------------------------------------- */
 /*      Parse the XML.                                                  */
 /* -------------------------------------------------------------------- */
    CPLXMLNode	*psTree = CPLParseXMLString( pszXML );
    if( psTree == NULL )
        return NULL;

    CPLXMLNode *psRoot = CPLGetXMLNode( psTree, "=VRTDataset" );
    if (psRoot == NULL)
    {
        CPLError( CE_Failure, CPLE_AppDefined,
                  "Missing VRTDataset element." );
        CPLDestroyXMLNode( psTree );
        return NULL;
    }

    const bool bIsPansharpened
        = strstr( pszXML, "VRTPansharpenedDataset" ) != NULL;

    if( !bIsPansharpened &&
        (CPLGetXMLNode( psRoot, "rasterXSize" ) == NULL
        || CPLGetXMLNode( psRoot, "rasterYSize" ) == NULL
        || CPLGetXMLNode( psRoot, "VRTRasterBand" ) == NULL) )
    {
        CPLError( CE_Failure, CPLE_AppDefined,
                  "Missing one of rasterXSize, rasterYSize or bands on"
                  " VRTDataset." );
        CPLDestroyXMLNode( psTree );
        return NULL;
    }

/* -------------------------------------------------------------------- */
/*      Create the new virtual dataset object.                          */
/* -------------------------------------------------------------------- */
    const int nXSize = atoi(CPLGetXMLValue(psRoot,"rasterXSize","0"));
    const int nYSize = atoi(CPLGetXMLValue(psRoot,"rasterYSize","0"));

    if ( !bIsPansharpened &&
         !GDALCheckDatasetDimensions( nXSize, nYSize ) )
    {
        CPLDestroyXMLNode( psTree );
        return NULL;
    }

    VRTDataset *poDS;
    if( strstr(pszXML,"VRTWarpedDataset") != NULL )
        poDS = new VRTWarpedDataset( nXSize, nYSize );
    else if( bIsPansharpened )
        poDS = new VRTPansharpenedDataset( nXSize, nYSize );
    else
    {
        poDS = new VRTDataset( nXSize, nYSize );
        poDS->eAccess = eAccess;
    }

    if( poDS->XMLInit( psRoot, pszVRTPath ) != CE_None )
    {
        delete poDS;
        poDS = NULL;
    }

/* -------------------------------------------------------------------- */
/*      Try to return a regular handle on the file.                     */
/* -------------------------------------------------------------------- */
    CPLDestroyXMLNode( psTree );

    return poDS;
}

/************************************************************************/
/*                              AddBand()                               */
/************************************************************************/

CPLErr VRTDataset::AddBand( GDALDataType eType, char **papszOptions )

{
    m_bNeedsFlush = TRUE;

/* ==================================================================== */
/*      Handle a new raw band.                                          */
/* ==================================================================== */
    const char *pszSubClass = CSLFetchNameValue(papszOptions, "subclass");

    if( pszSubClass != NULL && EQUAL(pszSubClass,"VRTRawRasterBand") )
    {
        const int nWordDataSize = GDALGetDataTypeSizeBytes( eType );

/* -------------------------------------------------------------------- */
/*      Collect required information.                                   */
/* -------------------------------------------------------------------- */
        const char* pszImageOffset = CSLFetchNameValueDef(papszOptions, "ImageOffset", "0");
        vsi_l_offset nImageOffset = CPLScanUIntBig(
                        pszImageOffset, static_cast<int>(strlen(pszImageOffset)));

        int nPixelOffset = nWordDataSize;
        if( CSLFetchNameValue(papszOptions, "PixelOffset") != NULL )
            nPixelOffset = atoi(CSLFetchNameValue(papszOptions,"PixelOffset"));

        int nLineOffset = nWordDataSize * GetRasterXSize();
        if( CSLFetchNameValue(papszOptions, "LineOffset") != NULL )
            nLineOffset = atoi(CSLFetchNameValue(papszOptions, "LineOffset"));

        const char *pszByteOrder = NULL;
        if( CSLFetchNameValue(papszOptions, "ByteOrder") != NULL )
            pszByteOrder = CSLFetchNameValue(papszOptions, "ByteOrder");

        const char *pszFilename;
        if( CSLFetchNameValue(papszOptions, "SourceFilename") != NULL )
            pszFilename = CSLFetchNameValue(papszOptions, "SourceFilename");
        else
        {
            CPLError( CE_Failure, CPLE_AppDefined,
                      "AddBand() requires a SourceFilename option for "
                      "VRTRawRasterBands." );
            return CE_Failure;
        }

        const bool bRelativeToVRT =
            CPL_TO_BOOL(CSLFetchBoolean( papszOptions, "relativeToVRT", FALSE ));

/* -------------------------------------------------------------------- */
/*      Create and initialize the band.                                 */
/* -------------------------------------------------------------------- */

        VRTRawRasterBand *poBand =
            new VRTRawRasterBand( this, GetRasterCount() + 1, eType );

        char* l_pszVRTPath = CPLStrdup(CPLGetPath(GetDescription()));
        if EQUAL(l_pszVRTPath, "")
        {
            CPLFree(l_pszVRTPath);
            l_pszVRTPath = NULL;
        }

        CPLErr eErr =
            poBand->SetRawLink( pszFilename, l_pszVRTPath, bRelativeToVRT,
                                nImageOffset, nPixelOffset, nLineOffset,
                                pszByteOrder );
        CPLFree(l_pszVRTPath);
        if( eErr != CE_None )
        {
            delete poBand;
            return eErr;
        }

        SetBand( GetRasterCount() + 1, poBand );

        return CE_None;
    }

/* ==================================================================== */
/*      Handle a new "sourced" band.                                    */
/* ==================================================================== */
    else
    {
        VRTSourcedRasterBand *poBand;

	/* ---- Check for our sourced band 'derived' subclass ---- */
        if(pszSubClass != NULL && EQUAL(pszSubClass,"VRTDerivedRasterBand")) {

            /* We'll need a pointer to the subclass in case we need */
            /* to set the new band's pixel function below. */
            VRTDerivedRasterBand* poDerivedBand = new VRTDerivedRasterBand(
                this, GetRasterCount() + 1, eType,
                GetRasterXSize(), GetRasterYSize());

            /* Set the pixel function options it provided. */
            const char* pszFuncName =
                CSLFetchNameValue(papszOptions, "PixelFunctionType");
            if (pszFuncName != NULL)
                poDerivedBand->SetPixelFunctionName(pszFuncName);

            const char* pszTransferTypeName =
                CSLFetchNameValue(papszOptions, "SourceTransferType");
            if (pszTransferTypeName != NULL) {
                const GDALDataType eTransferType =
                    GDALGetDataTypeByName(pszTransferTypeName);
                if (eTransferType == GDT_Unknown) {
                    CPLError( CE_Failure, CPLE_AppDefined,
                              "invalid SourceTransferType: \"%s\".",
                              pszTransferTypeName);
                    delete poDerivedBand;
                    return CE_Failure;
                }
                poDerivedBand->SetSourceTransferType(eTransferType);
            }

            /* We're done with the derived band specific stuff, so */
            /* we can assigned the base class pointer now. */
            poBand = poDerivedBand;
        }
        else {

            /* ---- Standard sourced band ---- */
            poBand = new VRTSourcedRasterBand(
                this, GetRasterCount() + 1, eType,
                GetRasterXSize(), GetRasterYSize());
        }

        SetBand( GetRasterCount() + 1, poBand );

        for( int i=0; papszOptions != NULL && papszOptions[i] != NULL; i++ )
        {
            if( STARTS_WITH_CI(papszOptions[i], "AddFuncSource=") )
            {
                char **papszTokens
                    = CSLTokenizeStringComplex( papszOptions[i]+14,
                                                ",", TRUE, FALSE );
                if( CSLCount(papszTokens) < 1 )
                {
                    CPLError( CE_Failure, CPLE_AppDefined,
                              "AddFuncSource(): required argument missing." );
                    // TODO: How should this error be handled?  Return
                    // CE_Failure?
                }

                VRTImageReadFunc pfnReadFunc = NULL;
                sscanf( papszTokens[0], "%p", &pfnReadFunc );

                void *pCBData = NULL;
                if( CSLCount(papszTokens) > 1 )
                    sscanf( papszTokens[1], "%p", &pCBData );

                const double dfNoDataValue =
                    ( CSLCount(papszTokens) > 2 ) ?
                    CPLAtof( papszTokens[2] ) : VRT_NODATA_UNSET;

                poBand->AddFuncSource( pfnReadFunc, pCBData, dfNoDataValue );

                CSLDestroy( papszTokens );
            }
        }

        return CE_None;
    }
}

/************************************************************************/
/*                              VRTAddBand()                            */
/************************************************************************/

/**
 * @see VRTDataset::VRTAddBand().
 */

int CPL_STDCALL VRTAddBand( VRTDatasetH hDataset, GDALDataType eType,
                            char **papszOptions )

{
    VALIDATE_POINTER1( hDataset, "VRTAddBand", 0 );

    return reinterpret_cast<VRTDataset *>(
        hDataset )->AddBand( eType, papszOptions );
}

/************************************************************************/
/*                               Create()                               */
/************************************************************************/

GDALDataset *
VRTDataset::Create( const char * pszName,
                    int nXSize, int nYSize, int nBands,
                    GDALDataType eType, char ** papszOptions )

{

    if( STARTS_WITH_CI(pszName, "<VRTDataset") )
    {
        GDALDataset *poDS = OpenXML( pszName, NULL, GA_Update );
        if ( poDS != NULL )
            poDS->SetDescription( "<FromXML>" );
        return poDS;
    }

    const char *pszSubclass = CSLFetchNameValue( papszOptions,
                                                 "SUBCLASS" );

    VRTDataset *poDS = NULL;

    if( pszSubclass == NULL || EQUAL(pszSubclass,"VRTDataset") )
        poDS = new VRTDataset( nXSize, nYSize );
    else if( EQUAL(pszSubclass,"VRTWarpedDataset") )
    {
        poDS = new VRTWarpedDataset( nXSize, nYSize );
    }
    else
    {
        CPLError( CE_Failure, CPLE_AppDefined,
                  "SUBCLASS=%s not recognised.",
                  pszSubclass );
        return NULL;
    }
    poDS->eAccess = GA_Update;

    poDS->SetDescription( pszName );

    for( int iBand = 0; iBand < nBands; iBand++ )
        poDS->AddBand( eType, NULL );

    poDS->m_bNeedsFlush = TRUE;

    poDS->oOvManager.Initialize( poDS, pszName );

    return poDS;
}

/************************************************************************/
/*                            GetFileList()                             */
/************************************************************************/

char** VRTDataset::GetFileList()
{
    char** papszFileList = GDALDataset::GetFileList();

    int nSize = CSLCount(papszFileList);
    int nMaxSize = nSize;

    // Do not need an element deallocator as each string points to an
    // element of the papszFileList.
    CPLHashSet* hSetFiles = CPLHashSetNew(CPLHashSetHashStr,
                                          CPLHashSetEqualStr,
                                          NULL);

    for( int iBand = 0; iBand < nBands; iBand++ )
    {
      reinterpret_cast<VRTRasterBand *>(
          papoBands[iBand])->GetFileList(
              &papszFileList, &nSize, &nMaxSize, hSetFiles );
    }

    CPLHashSetDestroy(hSetFiles);

    return papszFileList;
}

/************************************************************************/
/*                              Delete()                                */
/************************************************************************/

/* We implement Delete() to avoid that the default implementation */
/* in GDALDriver::Delete() destroys the source files listed by GetFileList(),*/
/* which would be an undesired effect... */
CPLErr VRTDataset::Delete( const char * pszFilename )
{
    GDALDriverH hDriver = GDALIdentifyDriver(pszFilename, NULL);

    if (!hDriver || !EQUAL( GDALGetDriverShortName(hDriver), "VRT" ) )
        return CE_Failure;

    if( strstr(pszFilename, "<VRTDataset") == NULL &&
        VSIUnlink( pszFilename ) != 0 )
    {
        CPLError( CE_Failure, CPLE_AppDefined,
                  "Deleting %s failed:\n%s",
                  pszFilename,
                  VSIStrerror(errno) );
        return CE_Failure;
    }

    return CE_None;
}

/************************************************************************/
/*                          CreateMaskBand()                            */
/************************************************************************/

CPLErr VRTDataset::CreateMaskBand( int )
{
    if (m_poMaskBand != NULL)
    {
        CPLError(CE_Failure, CPLE_AppDefined,
                 "This VRT dataset has already a mask band");
        return CE_Failure;
    }

    SetMaskBand(new VRTSourcedRasterBand( this, 0 ));

    return CE_None;
}

/************************************************************************/
/*                           SetMaskBand()                              */
/************************************************************************/

void VRTDataset::SetMaskBand(VRTRasterBand* poMaskBandIn)
{
    delete m_poMaskBand;
    m_poMaskBand = poMaskBandIn;
    m_poMaskBand->SetIsMaskBand();
}

/************************************************************************/
/*                        CloseDependentDatasets()                      */
/************************************************************************/

int VRTDataset::CloseDependentDatasets()
{
    /* We need to call it before removing the sources, otherwise */
    /* we would remove them from the serizalized VRT */
    FlushCache();

    int bHasDroppedRef = GDALDataset::CloseDependentDatasets();

    for( int iBand = 0; iBand < nBands; iBand++ )
    {
        bHasDroppedRef |= reinterpret_cast<VRTRasterBand *>(
            papoBands[iBand] )->CloseDependentDatasets();
    }

    return bHasDroppedRef;
}

/************************************************************************/
/*                      CheckCompatibleForDatasetIO()                   */
/************************************************************************/

/* We will return TRUE only if all the bands are VRTSourcedRasterBands */
/* made of identical sources, that are strictly VRTSimpleSource, and that */
/* the band number of each source is the band number of the VRTSouredRasterBand */

int VRTDataset::CheckCompatibleForDatasetIO()
{
    int nSources = 0;
    VRTSource    **papoSources = NULL;
    CPLString osResampling;

    for(int iBand = 0; iBand < nBands; iBand++)
    {
        if (!reinterpret_cast<VRTRasterBand *>(
                papoBands[iBand] )->IsSourcedRasterBand() )
            return FALSE;

        VRTSourcedRasterBand* poBand
            = reinterpret_cast<VRTSourcedRasterBand*>( papoBands[iBand] );

        if (iBand == 0)
        {
            nSources = poBand->nSources;
            papoSources = poBand->papoSources;
            for(int iSource = 0; iSource < nSources; iSource++)
            {
                if (!papoSources[iSource]->IsSimpleSource())
                    return FALSE;

                VRTSimpleSource* poSource = reinterpret_cast<VRTSimpleSource *>(
                    papoSources[iSource] );
                if (!EQUAL(poSource->GetType(), "SimpleSource"))
                    return FALSE;

                GDALRasterBand *srcband = poSource->GetBand();
                if (srcband == NULL)
                    return FALSE;
                if (srcband->GetDataset() == NULL)
                    return FALSE;
                if (srcband->GetDataset()->GetRasterCount() <= iBand)
                    return FALSE;
                if (srcband->GetDataset()->GetRasterBand(iBand + 1) != srcband)
                    return FALSE;
                osResampling = poSource->GetResampling();
            }
        }
        else if (nSources != poBand->nSources)
        {
            return FALSE;
        }
        else
        {
            for(int iSource = 0; iSource < nSources; iSource++)
            {
                if (!poBand->papoSources[iSource]->IsSimpleSource())
                    return FALSE;
                VRTSimpleSource* poRefSource
                    = reinterpret_cast<VRTSimpleSource *>(
                        papoSources[iSource] );

                VRTSimpleSource* poSource = reinterpret_cast<VRTSimpleSource *>(
                    poBand->papoSources[iSource] );
                if (!EQUAL(poSource->GetType(), "SimpleSource"))
                    return FALSE;
                if (!poSource->IsSameExceptBandNumber(poRefSource))
                    return FALSE;

                GDALRasterBand *srcband = poSource->GetBand();
                if (srcband == NULL)
                    return FALSE;
                if (srcband->GetDataset() == NULL)
                    return FALSE;
                if (srcband->GetDataset()->GetRasterCount() <= iBand)
                    return FALSE;
                if (srcband->GetDataset()->GetRasterBand(iBand + 1) != srcband)
                    return FALSE;
                if (osResampling.compare(poSource->GetResampling()) != 0)
                    return FALSE;
            }
        }
    }

    return nSources != 0;
}

/************************************************************************/
/*                         GetSingleSimpleSource()                      */
/*                                                                      */
/* Returns a non-NULL dataset if the VRT is made of a single source     */
/* that is a simple source, in its full extent, and with all of its     */
/* bands. Basically something produced by :                             */
/*   gdal_translate src dst.vrt -of VRT (-a_srs / -a_ullr)              */
/************************************************************************/

GDALDataset* VRTDataset::GetSingleSimpleSource()
{
    if (!CheckCompatibleForDatasetIO())
        return NULL;

    VRTSourcedRasterBand* poVRTBand
        = reinterpret_cast<VRTSourcedRasterBand *>( papoBands[0] );
    if( poVRTBand->nSources != 1 )
        return NULL;

    VRTSimpleSource* poSource = reinterpret_cast<VRTSimpleSource *>(
        poVRTBand->papoSources[0] );

    GDALRasterBand* poBand = poSource->GetBand();
    if (poBand == NULL)
        return NULL;

    GDALDataset* poSrcDS = poBand->GetDataset();
    if (poSrcDS == NULL)
        return NULL;

    /* Check that it uses the full source dataset */
    double dfReqXOff, dfReqYOff, dfReqXSize, dfReqYSize;
    int nReqXOff, nReqYOff, nReqXSize, nReqYSize;
    int nOutXOff, nOutYOff, nOutXSize, nOutYSize;
    if (! poSource->GetSrcDstWindow( 0, 0,
                                     poSrcDS->GetRasterXSize(),
                                     poSrcDS->GetRasterYSize(),
                                     poSrcDS->GetRasterXSize(),
                                     poSrcDS->GetRasterYSize(),
                                     &dfReqXOff, &dfReqYOff,
                                     &dfReqXSize, &dfReqYSize,
                                     &nReqXOff, &nReqYOff,
                                     &nReqXSize, &nReqYSize,
                                     &nOutXOff, &nOutYOff,
                                     &nOutXSize, &nOutYSize ) )
        return NULL;

    if (nReqXOff != 0 || nReqYOff != 0 ||
        nReqXSize != poSrcDS->GetRasterXSize() ||
        nReqYSize != poSrcDS->GetRasterYSize())
        return NULL;

    if (nOutXOff != 0 || nOutYOff != 0 ||
        nOutXSize != poSrcDS->GetRasterXSize() ||
        nOutYSize != poSrcDS->GetRasterYSize())
        return NULL;

    return poSrcDS;
}

/************************************************************************/
/*                              IRasterIO()                             */
/************************************************************************/

CPLErr VRTDataset::IRasterIO( GDALRWFlag eRWFlag,
                               int nXOff, int nYOff, int nXSize, int nYSize,
                               void * pData, int nBufXSize, int nBufYSize,
                               GDALDataType eBufType,
                               int nBandCount, int *panBandMap,
                               GSpacing nPixelSpace, GSpacing nLineSpace,
                               GSpacing nBandSpace,
                               GDALRasterIOExtraArg* psExtraArg)
{
    if (m_bCompatibleForDatasetIO < 0)
    {
        m_bCompatibleForDatasetIO = CheckCompatibleForDatasetIO();
    }

    bool bLocalCompatibleForDatasetIO = CPL_TO_BOOL(m_bCompatibleForDatasetIO);
    if (bLocalCompatibleForDatasetIO && eRWFlag == GF_Read &&
        (nBufXSize < nXSize || nBufYSize < nYSize))
    {
        int bTried;
        CPLErr eErr = TryOverviewRasterIO( eRWFlag,
                                    nXOff, nYOff, nXSize, nYSize,
                                    pData, nBufXSize, nBufYSize,
                                    eBufType,
                                    nBandCount, panBandMap,
                                    nPixelSpace, nLineSpace,
                                    nBandSpace,
                                    psExtraArg,
                                    &bTried );
        if( bTried )
            return eErr;

        for(int iBand = 0; iBand < nBands; iBand++)
        {
            VRTSourcedRasterBand* poBand
                = reinterpret_cast<VRTSourcedRasterBand *>( papoBands[iBand] );

            // If there are overviews, let VRTSourcedRasterBand::IRasterIO()
            // do the job.
            if( poBand->GetOverviewCount() != 0 )
            {
                bLocalCompatibleForDatasetIO = false;
                break;
            }
        }
    }

    // If resampling with non-nearest neighbour, we need to be careful
    // if the VRT band exposes a nodata value, but the sources do not have it
    if (bLocalCompatibleForDatasetIO && eRWFlag == GF_Read &&
        (nXSize != nBufXSize || nYSize != nBufYSize) &&
        psExtraArg->eResampleAlg != GRIORA_NearestNeighbour )
    {
        for(int iBandIndex=0; iBandIndex<nBandCount; iBandIndex++)
        {
            VRTSourcedRasterBand* poBand =
                reinterpret_cast<VRTSourcedRasterBand*>(
                    GetRasterBand(panBandMap[iBandIndex]) );
            int bHasNoData = FALSE;
            const double dfNoDataValue = poBand->GetNoDataValue(&bHasNoData);
            if( bHasNoData )
            {
                for( int i = 0; i < poBand->nSources; i++ )
                {
                    VRTSimpleSource* poSource
                        = reinterpret_cast<VRTSimpleSource *>(
                            poBand->papoSources[i] );
                    int bSrcHasNoData = FALSE;
                    const double dfSrcNoData
                        = poSource->GetBand()->GetNoDataValue(&bSrcHasNoData);
                    if( !bSrcHasNoData || dfSrcNoData != dfNoDataValue )
                    {
                        bLocalCompatibleForDatasetIO = false;
                        break;
                    }
                }
                if( !bLocalCompatibleForDatasetIO )
                    break;
            }
        }
    }

    if (bLocalCompatibleForDatasetIO && eRWFlag == GF_Read)
    {
        for(int iBandIndex=0; iBandIndex<nBandCount; iBandIndex++)
        {
            VRTSourcedRasterBand* poBand
                = reinterpret_cast<VRTSourcedRasterBand *>(
                    GetRasterBand( panBandMap[iBandIndex] ) );

            /* Dirty little trick to initialize the buffer without doing */
            /* any real I/O */
            const int nSavedSources = poBand->nSources;
            poBand->nSources = 0;

            GByte *pabyBandData
                = reinterpret_cast<GByte *>( pData ) + iBandIndex * nBandSpace;

            poBand->IRasterIO(GF_Read, nXOff, nYOff, nXSize, nYSize,
                                pabyBandData, nBufXSize, nBufYSize,
                                eBufType,
                                nPixelSpace, nLineSpace, psExtraArg);

            poBand->nSources = nSavedSources;
        }

        CPLErr eErr = CE_None;

        GDALProgressFunc  pfnProgressGlobal = psExtraArg->pfnProgress;
        void             *pProgressDataGlobal = psExtraArg->pProgressData;

        // Use the last band, because when sources reference a GDALProxyDataset,
        // they don't necessary instantiate all underlying rasterbands.
        VRTSourcedRasterBand* poBand = reinterpret_cast<VRTSourcedRasterBand *>(
            papoBands[nBands - 1] );
        for( int iSource = 0;
             eErr == CE_None && iSource < poBand->nSources;
             iSource++ )
        {
            psExtraArg->pfnProgress = GDALScaledProgress;
            psExtraArg->pProgressData =
                GDALCreateScaledProgress( 1.0 * iSource / poBand->nSources,
                                        1.0 * (iSource + 1) / poBand->nSources,
                                        pfnProgressGlobal,
                                        pProgressDataGlobal );

            VRTSimpleSource* poSource = reinterpret_cast<VRTSimpleSource *>(
                poBand->papoSources[iSource] );

            eErr = poSource->DatasetRasterIO( nXOff, nYOff, nXSize, nYSize,
                                              pData, nBufXSize, nBufYSize,
                                              eBufType,
                                              nBandCount, panBandMap,
                                              nPixelSpace, nLineSpace, nBandSpace,
                                              psExtraArg);

            GDALDestroyScaledProgress( psExtraArg->pProgressData );
        }

        psExtraArg->pfnProgress = pfnProgressGlobal;
        psExtraArg->pProgressData = pProgressDataGlobal;

        return eErr;
    }

    return GDALDataset::IRasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize,
                                  pData, nBufXSize, nBufYSize,
                                  eBufType,
                                  nBandCount, panBandMap,
                                  nPixelSpace, nLineSpace, nBandSpace,
                                  psExtraArg);
}

/************************************************************************/
/*                  UnsetPreservedRelativeFilenames()                   */
/************************************************************************/

void VRTDataset::UnsetPreservedRelativeFilenames()
{
    for(int iBand = 0; iBand < nBands; iBand++)
    {
        if( !reinterpret_cast<VRTRasterBand *>(
               papoBands[iBand] )->IsSourcedRasterBand() )
            continue;

        VRTSourcedRasterBand* poBand
            = reinterpret_cast<VRTSourcedRasterBand *>( papoBands[iBand] );
        const int nSources = poBand->nSources;
        VRTSource** papoSources = poBand->papoSources;
        for(int iSource = 0; iSource < nSources; iSource++)
        {
            if (!papoSources[iSource]->IsSimpleSource())
                continue;

            VRTSimpleSource* poSource = reinterpret_cast<VRTSimpleSource *>(
                papoSources[iSource] );
            poSource->UnsetPreservedRelativeFilenames();
        }
    }
}

/************************************************************************/
/*                        BuildVirtualOverviews()                       */
/************************************************************************/

void VRTDataset::BuildVirtualOverviews()
{
    // Currently we expose virtual overviews only if the dataset is made of
    // a single SimpleSource/ComplexSource, in each band.
    // And if the underlying sources have overviews of course
    if( m_apoOverviews.size() || m_apoOverviewsBak.size() )
        return;

    int nOverviews = 0;
    GDALRasterBand* poFirstBand = NULL;
    for(int iBand = 0; iBand < nBands; iBand++)
    {
        if( !reinterpret_cast<VRTRasterBand *>(
               papoBands[iBand] )->IsSourcedRasterBand())
            return;

        VRTSourcedRasterBand* poVRTBand
            = reinterpret_cast<VRTSourcedRasterBand *>( papoBands[iBand] );
        if( poVRTBand->nSources != 1 )
            return;
        if (!poVRTBand->papoSources[0]->IsSimpleSource())
            return;

        VRTSimpleSource* poSource
            = reinterpret_cast<VRTSimpleSource *>( poVRTBand->papoSources[0] );
        if (!EQUAL(poSource->GetType(), "SimpleSource") &&
            !EQUAL(poSource->GetType(), "ComplexSource"))
            return;
        GDALRasterBand* poSrcBand = poSource->GetBand();
        if (poSrcBand == NULL)
            return;

        // To prevent recursion
        m_apoOverviewsBak.push_back(NULL);
        const int nOvrCount = poSrcBand->GetOverviewCount();
        m_apoOverviewsBak.resize(0);

        if( nOvrCount == 0 )
            return;
        if( iBand == 0 )
        {
            poFirstBand = poSrcBand;
            nOverviews = nOvrCount;
        }
        else if( nOvrCount < nOverviews )
            nOverviews = nOvrCount;
    }

    for(int j=0;j<nOverviews;j++)
    {
        const double dfXRatio = static_cast<double>(
            poFirstBand->GetOverview(j)->GetXSize() ) / poFirstBand->GetXSize();
        const double dfYRatio = static_cast<double>(
            poFirstBand->GetOverview(j)->GetYSize() ) / poFirstBand->GetYSize();
        const int nOvrXSize = static_cast<int>(0.5 + nRasterXSize * dfXRatio);
        const int nOvrYSize = static_cast<int>(0.5 + nRasterYSize * dfYRatio);
        if( nOvrXSize < 128 || nOvrYSize < 128 )
            break;
        VRTDataset* poOvrVDS = new VRTDataset(nOvrXSize, nOvrYSize);
        m_apoOverviews.push_back(poOvrVDS);

        for(int i=0;i<nBands;i++)
        {
            VRTSourcedRasterBand* poVRTBand
                = reinterpret_cast<VRTSourcedRasterBand *>(
                    GetRasterBand(i+1) );
            VRTSourcedRasterBand* poOvrVRTBand = new VRTSourcedRasterBand(
                poOvrVDS,
                poOvrVDS->GetRasterCount() + 1,
                poVRTBand->GetRasterDataType(),
                nOvrXSize, nOvrYSize);
            poOvrVDS->SetBand( poOvrVDS->GetRasterCount() + 1, poOvrVRTBand );

            VRTSimpleSource* poSrcSource = reinterpret_cast<VRTSimpleSource *>(
                poVRTBand->papoSources[0] );
            VRTSimpleSource* poNewSource = NULL;
            if( EQUAL(poSrcSource->GetType(), "SimpleSource") )
                poNewSource = new VRTSimpleSource(poSrcSource, dfXRatio, dfYRatio);
            else if( EQUAL(poSrcSource->GetType(), "ComplexSource") )
              poNewSource = new VRTComplexSource(
                  reinterpret_cast<VRTComplexSource *>( poSrcSource ),
                  dfXRatio, dfYRatio );
            else
            {
                CPLAssert(FALSE);
            }
            if( poNewSource->GetBand()->GetDataset() )
                poNewSource->GetBand()->GetDataset()->Reference();
            poOvrVRTBand->AddSource(poNewSource);
        }
    }
}

/************************************************************************/
/*                          IBuildOverviews()                           */
/************************************************************************/

CPLErr
VRTDataset::IBuildOverviews( const char *pszResampling,
                             int nOverviews,
                             int *panOverviewList,
                             int nListBands,
                             int *panBandList,
                             GDALProgressFunc pfnProgress,
                             void * pProgressData )
{
    /* Make implicit overviews invisible, but do not destroy */
    /* them in case they are already used (not sure that the client */
    /* has the right to do that. behaviour undefined in GDAL API I think) */
    if( m_apoOverviews.size() )
    {
        m_apoOverviewsBak = m_apoOverviews;
        m_apoOverviews.resize(0);
    }
    else
    {
        // Add a dummy overview so that GDALDataset::IBuildOverviews()
        // doesn't manage to get a virtual implicit overview
        m_apoOverviews.push_back(NULL);
    }

    return GDALDataset::IBuildOverviews(pszResampling,
                                               nOverviews,
                                               panOverviewList,
                                               nListBands,
                                               panBandList,
                                               pfnProgress,
                                               pProgressData);
}


text:
import collections
import logging
import sys
from functools import lru_cache
from os import chmod
from os.path import basename, exists, join
from shutil import move
from typing import Dict, List, Union, BinaryIO
from urllib.parse import parse_qs

import click
import yaml

from dsl_parser import exceptions as dsl_parser_exceptions
from dsl_parser.constants import (
    WORKFLOW_PLUGINS_TO_INSTALL,
    DEPLOYMENT_PLUGINS_TO_INSTALL,
    HOST_AGENT_PLUGINS_TO_INSTALL,
    PLUGIN_PACKAGE_NAME,
    PLUGIN_PACKAGE_VERSION,
)
from dsl_parser.models import Plan

from manager_rest.flask_utils import get_tenant_by_name, set_tenant_in_app
from manager_rest.resolver_with_catalog_support import (
    ResolverWithCatalogSupport,
)
from manager_rest.shell import common
from manager_rest.storage import models
from manager_rest.storage.storage_manager import SQLStorageManager

try:
    from packaging.version import parse as parse_version
except ImportError:
    from distutils.version import LooseVersion as parse_version

REST_AUTHORIZATION_CONFIG_PATH = join(
    common.REST_HOME_DIR, "authorization.conf"
)

AT_LEAST = "at_least"
BLUEPRINT_LINE = "blueprint_line"
CURRENT_IMPORT_FROM = "current_import_from"
CURRENT_IS_PINNED = "current_is_pinned"
CURRENT_VERSION = "current_version"
END_POS = "end_pos"
EXECUTORS = [
    DEPLOYMENT_PLUGINS_TO_INSTALL,
    WORKFLOW_PLUGINS_TO_INSTALL,
    HOST_AGENT_PLUGINS_TO_INSTALL,
]
FINE = "fine"
IMPORT_FROM_MANAGED = "managed"
IMPORT_FROM_SOURCE = "source"
IMPORT_FROM_URL = "url"
IMPORTS = "imports"
IS_PINNED = True
IS_NOT_PINNED = False
IS_UNKNOWN = True
IS_NOT_UNKNOWN = False
PLUGIN_NAME = "plugin_name"
REPLACEMENT = "replacement"
EXACT_VERSION = "exact_version"
START_POS = "start_pos"
SUGGESTED_IMPORT_FROM = "suggested_import_from"
SUGGESTED_IS_PINNED = "suggested_is_pinned"
SUGGESTED_VERSION = "suggested_version"
UNKNOWN = "unknown"
UPDATES = "updates"
VERSIONS = "versions"
MAX_IMPORT_TOKEN_LENGTH = 200

logging.basicConfig(
    format="%(asctime)s.%(msecs)03d %(levelname)s "
    "[%(module)s.%(funcName)s] %(message)s",
    datefmt="%H:%M:%S",
    level=logging.INFO,
)
logger = logging.getLogger(basename(sys.argv[0]))

class PluginSpec:
    def __init__(self, versions: List[str], exact=None, at_least=None):
        self.versions = sorted(
            versions,
            key=parse_version,
            reverse=True,
        )
        self.exact = exact
        self.at_least = at_least

CLOUDIFY_PLUGINS: Dict[str, PluginSpec] = {
    "cloudify-aws-plugin": PluginSpec(
        [
            "3.0.5",
            "2.14.0",
            "2.13.0",
            "2.12.14",
            "2.11.1",
            "2.10.0",
            "2.9.2",
            "2.8.0",
            "2.6.0",
            "2.5.14",
            "2.4.4",
            "2.3.5",
            "2.2.1",
            "2.1.0",
            "2.0.2",
            "1.5.1.2",
        ]
    ),
    "cloudify-azure-plugin": PluginSpec(
        [
            "3.7.1",
            "3.6.1",
            "3.5.6",
            "3.4.1",
            "3.3.1",
            "3.2.1",
            "3.1.0",
            "3.0.11",
            "2.1.10",
            "2.0.0",
            "1.8.0",
            "1.7.3",
            "1.6.2",
            "1.5.1.1",
            "1.4.3",
        ],
    ),
    "cloudify-gcp-plugin": PluginSpec(
        [
            "1.8.1",
            "1.7.0",
            "1.6.9",
            "1.5.1",
            "1.4.5",
            "1.3.0.1",
            "1.2.0",
            "1.1.0",
            "1.0.1",
        ]
    ),
    "cloudify-openstack-plugin": PluginSpec(
        [
            "3.3.3",
            "2.14.24",
            "3.2.21",
            "3.1.1",
            "3.0.0",
            "2.13.1",
            "2.12.0",
            "2.11.1",
            "2.10.0",
            "2.9.8",
            "2.8.2",
            "2.7.6",
            "2.6.0",
            "2.5.3",
            "2.4.1.1",
            "2.3.0",
            "2.2.0",
        ]
    ),
    "cloudify-vsphere-plugin": PluginSpec(
        [
            "2.19.10",
            "2.18.13",
            "2.17.0",
            "2.16.2",
            "2.15.1",
            "2.14.0",
            "2.13.1",
            "2.12.0",
            "2.11.0",
            "2.10.0",
            "2.9.3",
            "2.8.0",
            "2.7.0",
            "2.6.1",
            "2.5.0",
            "2.4.1",
            "2.3.0",
            "2.2.2",
        ],
    ),
    "cloudify-terraform-plugin": PluginSpec(
        [
            "0.18.17",
            "0.17.7",
            "0.16.5",
            "0.15.5",
            "0.14.4",
            "0.13.4",
            "0.12.0",
            "0.11.0",
            "0.10",
            "0.9",
            "0.7",
        ],
    ),
    "cloudify-ansible-plugin": PluginSpec(
        [
            "2.13.7",
            "2.12.1",
            "2.11.1",
            "2.10.1",
            "2.9.4",
            "2.8.2",
            "2.7.1",
            "2.6.0",
            "2.5.0",
            "2.4.0",
            "2.3.0",
            "2.2.0",
            "2.1.1",
            "2.0.4",
        ],
    ),
    "cloudify-kubernetes-plugin": PluginSpec(
        [
            "2.13.9",
            "2.12.1",
            "2.11.2",
            "2.10.0",
            "2.9.4",
            "2.8.3",
            "2.7.2",
            "2.6.5",
            "2.5.0",
            "2.4.1",
            "2.3.2",
            "2.2.2",
            "2.1.0",
            "2.0.0.1",
            "1.4.0",
            "1.3.1.1",
            "1.2.2",
            "1.1.0",
            "1.0.0",
        ],
    ),
    "cloudify-docker-plugin": PluginSpec(
        ["2.0.5", "1.3.2", "1.2", "1.1"],
    ),
    "cloudify-netconf-plugin": PluginSpec(
        ["0.4.4", "0.3.1", "0.2.1", "0.1.0"],
    ),
    "cloudify-fabric-plugin": PluginSpec(
        [
            "2.0.13",
            "1.7.0",
            "1.6.0",
            "1.5.3",
            "1.4.3",
            "1.3.1",
            "1.2.1",
            "1.1",
        ],
        at_least="2.0.6",
    ),
    "cloudify-libvirt-plugin": PluginSpec(
        [
            "0.9.0",
            "0.8.1",
            "0.7.0",
            "0.6.0",
            "0.5.0",
            "0.4.1",
            "0.3",
            "0.2",
            "0.1",
        ],
    ),
    "cloudify-utilities-plugin": PluginSpec(
        [
            "1.25.7",
            "1.24.4",
            "1.23.12",
            "1.22.1",
            "1.21.0",
            "1.20.0",
            "1.19.0",
            "1.18.0",
            "1.17.0",
            "1.16.1",
            "1.15.3",
            "1.14.0",
            "1.13.0",
            "1.12.5",
            "1.11.2",
            "1.10.2",
            "1.9.8",
            "1.8.3",
            "1.7.3",
            "1.6.1",
            "1.5.4",
            "1.4.5",
            "1.3.1",
            "1.2.5",
            "1.1.1",
            "1.0.0",
        ],
    ),
    "cloudify-host-pool-plugin": PluginSpec(
        ["1.5.3", "1.4", "1.2"],
    ),
    "cloudify-diamond-plugin": PluginSpec(
        ["1.3.19", "1.2.1", "1.2", "1.1"],
    ),
    "tosca-vcloud-plugin": PluginSpec(
        ["1.6.1", "1.5.1", "1.4.1"],
    ),
    "cloudify-vcloud-plugin": PluginSpec(
        ["2.0.1"],
    ),
    "cloudify-helm-plugin": PluginSpec(
        ["0.2.7", "0.1.1", "0.0.8"],
    ),
}

def load_imports(blueprint: models.Blueprint) -> List[str]:
    file_name = common.blueprint_file_name(blueprint)
    try:
        with open(file_name, "rb") as blueprint_file:
            try:
                return yaml.safe_load(blueprint_file)[IMPORTS]
            except yaml.YAMLError as ex:
                raise common.UpdateException(
                    f"Cannot load imports from {file_name}: {ex}"
                )
            except KeyError as ex:
                raise common.UpdateException(
                    f"Cannot find imports definition in {file_name}: {ex}"
                )
    except OSError as ex:
        raise common.UpdateException(
            f"Blueprint file {file_name} cannot be read: {ex}"
        )
    return []

def load_mappings(file_name: str) -> Dict:
    try:
        with open(file_name, "r") as mapping_file:
            try:
                mappings = yaml.safe_load(mapping_file)
            except yaml.YAMLError as ex:
                raise common.UpdateException(
                    f"Cannot load mappings from {file_name}: {ex}"
                )
    except OSError as ex:
        raise common.UpdateException(
            f"Mappings file {file_name} cannot be read: {ex}"
        )
    return mappings

@lru_cache(maxsize=2048)
def spec_from_url(resolver: ResolverWithCatalogSupport, url: str) -> tuple:
    try:
        response_text = resolver.fetch_import(url)
    except dsl_parser_exceptions.DSLParsingLogicException as ex:
        logger.warning("Cannot retrieve %s: %s", url, ex)
        return None, None
    try:
        plugin_yaml = yaml.safe_load(response_text)
    except yaml.YAMLError as ex:
        logger.warning("Cannot load imports from %s: %s", url, ex)
        return None, None
    for _, spec in plugin_yaml.get("plugins", {}).items():
        if spec.get(PLUGIN_PACKAGE_NAME) and spec.get(PLUGIN_PACKAGE_VERSION):
            return spec[PLUGIN_PACKAGE_NAME], spec[PLUGIN_PACKAGE_VERSION]
    return None, None

def spec_from_import(plugin_line: str) -> tuple:
    # More or less copy of ResolverWithCatalogSupport._resolve_plugin_yaml_url
    spec = plugin_line.replace("plugin:", "", 1).strip()
    name, _, params = spec.partition("?")
    for filter_name, filter_value in parse_qs(params).items():
        if filter_name == "version":
            if len(filter_value) == 1:
                return name, filter_value[0].strip()
            return name, None
    return name, None

def spec_from_source(import_line):
    if import_line.startswith("file://"):
        import_line = import_line.replace("file://", "", 1)
    if not exists(import_line):
        return None, None
    try:
        with open(import_line, "tr") as fh:
            plugin = yaml.safe_load(fh)
        plugins_dict = plugin.get("plugins")
        if len(plugins_dict) != 1:
            raise Exception("There should be exactly one `plugins` entry")
        plugin_id, plugin_spec = plugins_dict.popitem()
        return plugin_spec.get("package_name"), plugin_spec.get(
            "package_version"
        )
    except Exception:
        return None, None

def plugin_spec(
    resolver: ResolverWithCatalogSupport, import_line: str
) -> tuple:
    if import_line.startswith("http://") or import_line.startswith("https://"):
        name, version = spec_from_url(resolver, import_line)
        if name and version:
            return IS_PINNED, IS_NOT_UNKNOWN, IMPORT_FROM_URL, name, version
        return IS_PINNED, IS_UNKNOWN, IMPORT_FROM_URL, None, None
    if import_line.startswith("plugin:"):
        name, version = spec_from_import(import_line)
        if version and version.startswith(">"):
            return (
                IS_NOT_PINNED,
                IS_NOT_UNKNOWN,
                IMPORT_FROM_MANAGED,
                name,
                version,
            )
        if not version:
            return (
                IS_NOT_PINNED,
                IS_NOT_UNKNOWN,
                IMPORT_FROM_MANAGED,
                name,
                version,
            )
        return IS_PINNED, IS_NOT_UNKNOWN, IMPORT_FROM_MANAGED, name, version
    # Try if the `import_line` points to a local file
    name, version = spec_from_source(import_line)
    if not name:
        return IS_NOT_PINNED, IS_UNKNOWN, IMPORT_FROM_SOURCE, None, None
    if not version:
        return IS_NOT_PINNED, IS_NOT_UNKNOWN, IMPORT_FROM_SOURCE, name, None
    return IS_PINNED, IS_NOT_UNKNOWN, IMPORT_FROM_SOURCE, name, version

def plugins_in_a_plan(plan: Plan) -> collections.Iterable:
    for executor in [
        DEPLOYMENT_PLUGINS_TO_INSTALL,
        WORKFLOW_PLUGINS_TO_INSTALL,
        HOST_AGENT_PLUGINS_TO_INSTALL,
    ]:
        if executor not in plan:
            continue
        for plugin in plan[executor]:
            if plugin[PLUGIN_PACKAGE_NAME] and plugin[PLUGIN_PACKAGE_VERSION]:
                yield plugin

def find_plugin_in_a_plan(plan: Plan, plugin_name: str) -> dict:
    for plugin in plugins_in_a_plan(plan):
        if plugin[PLUGIN_PACKAGE_NAME] == plugin_name:
            return plugin
    return {}

def suggest_version(plugin_name: str, plugin_version: str) -> str:
    if plugin_name not in CLOUDIFY_PLUGINS:
        return plugin_version
    if CLOUDIFY_PLUGINS[plugin_name].exact is not None:
        return CLOUDIFY_PLUGINS[plugin_name].exact
    if CLOUDIFY_PLUGINS[plugin_name].at_least and (
        parse_version(CLOUDIFY_PLUGINS[plugin_name].at_least)
        > parse_version(plugin_version)
    ):
        base_version = CLOUDIFY_PLUGINS[plugin_name].at_least
    else:
        base_version = plugin_version
    plugin_major_version = base_version.split(".")[0]
    for available_version in CLOUDIFY_PLUGINS[plugin_name].versions:
        if available_version.split(".")[0] == plugin_major_version:
            return available_version
    return base_version

def scan_blueprint(
    resolver: ResolverWithCatalogSupport,
    blueprint: models.Blueprint,
    plugin_names: tuple,
) -> tuple:
    def add_mapping(genre: str, content: object):
        if genre not in mappings:
            mappings[genre] = []
        mappings[genre].append(content)

    def update_stats(
        current_import_from: str = None,
        suggested_import_from: str = None,
        current_is_pinned: bool = None,
        suggested_is_pinned: bool = None,
    ):
        if current_import_from is not None:
            current_import_from_stats[current_import_from] += 1
        if suggested_import_from is not None:
            suggested_import_from_stats[suggested_import_from] += 1
        if current_is_pinned:
            pinned_stats[CURRENT_IS_PINNED] += 1
        if suggested_is_pinned:
            pinned_stats[SUGGESTED_IS_PINNED] += 1

    imports = load_imports(blueprint)
    mappings: Dict[str, List] = {}
    plugins_install_suggestions = {}
    current_import_from_stats: Dict[str, int] = collections.defaultdict(
        lambda: 0
    )
    suggested_import_from_stats: Dict[str, int] = collections.defaultdict(
        lambda: 0
    )
    pinned_stats = {
        CURRENT_IS_PINNED: 0,
        SUGGESTED_IS_PINNED: 0,
    }

    for import_line in imports:
        if import_line.endswith("/types.yaml"):
            continue
        (
            is_pinned_version,
            is_unknown,
            import_from,
            plugin_name,
            _,
        ) = plugin_spec(resolver, import_line)
        if plugin_names and plugin_name not in plugin_names:
            continue
        update_stats(
            current_import_from=import_from,
            current_is_pinned=is_pinned_version,
        )
        if is_unknown:
            add_mapping(UNKNOWN, import_line)
            update_stats(
                suggested_import_from=import_from,
                suggested_is_pinned=is_pinned_version,
            )
            continue
        plugin_in_plan = find_plugin_in_a_plan(blueprint.plan, plugin_name)
        if not plugin_in_plan:
            update_stats(
                suggested_import_from=import_from,
                suggested_is_pinned=is_pinned_version,
            )
            continue
        suggested_version = suggest_version(
            plugin_name, plugin_in_plan[PLUGIN_PACKAGE_VERSION]
        )
        if not suggested_version:
            add_mapping(UNKNOWN, import_line)
            update_stats(
                suggested_import_from=import_from,
                suggested_is_pinned=is_pinned_version,
            )
            continue
        if plugin_name not in plugins_install_suggestions:
            plugins_install_suggestions[plugin_name] = suggested_version
        if not is_pinned_version:
            add_mapping(FINE, import_line)
            update_stats(
                suggested_import_from=import_from,
                suggested_is_pinned=is_pinned_version,
            )
            continue
        add_mapping(
            UPDATES,
            {
                import_line: {
                    PLUGIN_NAME: plugin_name,
                    CURRENT_VERSION: plugin_in_plan[PLUGIN_PACKAGE_VERSION],
                    SUGGESTED_VERSION: suggested_version,
                }
            },
        )
        update_stats(
            suggested_import_from=IMPORT_FROM_MANAGED,
            suggested_is_pinned=False,
        )
    stats: Dict[str, Union[Dict[str, int], int]] = {
        CURRENT_IMPORT_FROM: current_import_from_stats,
        CURRENT_IS_PINNED: pinned_stats[CURRENT_IS_PINNED],
        SUGGESTED_IMPORT_FROM: suggested_import_from_stats,
        SUGGESTED_IS_PINNED: pinned_stats[SUGGESTED_IS_PINNED],
    }
    return mappings, stats, plugins_install_suggestions

def get_imports(blueprint_file: BinaryIO) -> dict:
    level = 0
    imports_token = None
    import_lines = {}
    blueprint_file.seek(0, 0)
    for t in yaml.scan(blueprint_file):
        if isinstance(
            t,
            (
                yaml.tokens.BlockMappingStartToken,
                yaml.tokens.BlockSequenceStartToken,
                yaml.tokens.FlowMappingStartToken,
                yaml.tokens.FlowSequenceStartToken,
            ),
        ):
            level += 1
        if isinstance(
            t,
            (
                yaml.tokens.BlockEndToken,
                yaml.tokens.FlowMappingEndToken,
                yaml.tokens.FlowSequenceEndToken,
            ),
        ):
            level -= 1

        if isinstance(t, yaml.tokens.ScalarToken):
            if level == 1 and t.value == "imports":
                imports_token = t
                continue

            token_length = t.end_mark.index - t.start_mark.index

            if (
                level >= 1
                and imports_token
                and token_length < MAX_IMPORT_TOKEN_LENGTH
            ):
                import_lines[t.value] = {
                    START_POS: t.start_mark.index,
                    END_POS: t.end_mark.index,
                }

        if isinstance(t, yaml.tokens.KeyToken) and imports_token:
            break

    return import_lines

def write_updated_blueprint(
    input_file_name: str, output_file_name: str, import_updates: list
):
    try:
        with open(input_file_name, "rb") as input_file:
            with open(output_file_name, "wb") as output_file:
                for update in import_updates:
                    content = input_file.read(
                        update[START_POS] - input_file.tell()
                    )
                    output_file.write(content)
                    output_file.write(
                        update[REPLACEMENT].encode("utf-8", "ignore")
                    )
                    input_file.read(update[END_POS] - update[START_POS])
                content = input_file.read()
                output_file.write(content)
    except OSError as ex:
        raise common.UpdateException(
            f"Cannot update blueprint file source {input_file_name}, "
            f"destination {output_file_name}: {ex}"
        )

def correct_blueprint(
    blueprint: models.Blueprint, plugin_names: tuple, mappings: Dict[str, List]
) -> str:
    def line_replacement(mapping_spec: Dict[str, str]) -> str:
        suggested_version = mapping_spec.get(SUGGESTED_VERSION, "")
        next_major_version = int(suggested_version.split(".")[0]) + 1
        return "plugin:{0}?version=>={1},<{2}".format(
            mapping_spec.get(PLUGIN_NAME),
            suggested_version,
            next_major_version,
        )

    if not mappings:
        return UNKNOWN
    if not mappings.get(UPDATES):
        if mappings.get(UNKNOWN):
            return UNKNOWN
        return FINE
    file_name = common.blueprint_file_name(blueprint)
    new_file_name = common.blueprint_updated_file_name(blueprint)
    try:
        with open(file_name, "rb") as blueprint_file:
            import_lines = get_imports(blueprint_file)
    except OSError as ex:
        raise common.UpdateException(
            f"Cannot load blueprint from {file_name}: {ex}"
        )
    import_updates = []
    for mapping_updates in mappings.get(UPDATES, []):
        for blueprint_line, spec in mapping_updates.items():
            if blueprint_line not in import_lines:
                continue
            if plugin_names and spec.get(PLUGIN_NAME) not in plugin_names:
                continue
            import_updates.append(
                {
                    REPLACEMENT: line_replacement(spec),
                    START_POS: import_lines[blueprint_line][START_POS],
                    END_POS: import_lines[blueprint_line][END_POS],
                }
            )

    write_updated_blueprint(
        file_name,
        new_file_name,
        sorted(import_updates, key=lambda u: u[START_POS]),
    )
    common.write_blueprint_diff(
        file_name, new_file_name, common.blueprint_diff_file_name(blueprint)
    )
    common.update_archive(blueprint, new_file_name)
    move(new_file_name, file_name)
    return UPDATES

def printout_scanning_stats(
    total_blueprints: int, mappings: dict, stats: dict
):
    number_of_unknown_or_updates = len(
        [b for b in mappings.values() if UPDATES in b or UNKNOWN in b]
    )
    number_of_unknown = len([b for b in mappings.values() if UNKNOWN in b])
    pinned = (
        sum(s.get(CURRENT_IS_PINNED) for s in stats.values()),
        sum(s.get(SUGGESTED_IS_PINNED) for s in stats.values()),
    )
    url_import = (
        sum(
            s.get(CURRENT_IMPORT_FROM, {}).get(IMPORT_FROM_URL, 0)
            for s in stats.values()
        ),
        sum(
            s.get(SUGGESTED_IMPORT_FROM, {}).get(IMPORT_FROM_URL, 0)
            for s in stats.values()
        ),
    )
    source_import = (
        sum(
            s.get(CURRENT_IMPORT_FROM, {}).get(IMPORT_FROM_SOURCE, 0)
            for s in stats.values()
        ),
        sum(
            s.get(SUGGESTED_IMPORT_FROM, {}).get(IMPORT_FROM_SOURCE, 0)
            for s in stats.values()
        ),
    )
    print("\n\n                             SCANNING STATS")
    print("----------------------------------------------+---------------")
    print(
        " Number blueprints scanned                    | {0:14d}".format(
            total_blueprints
        )
    )
    print("                                              +---------------")
    print("                                              | BEFORE | AFTER")
    print("----------------------------------------------+--------+------")
    print(
        " Number of blueprints with one or more issues | {0:6d} | {1:5d}"
        .format(number_of_unknown_or_updates, number_of_unknown)
    )
    print(
        " Number of blueprints with version lock       | {0:6d} | {1:5d}"
        .format(pinned[0], pinned[1])
    )
    print(
        " Number of blueprints with URL import         | {0:6d} | {1:5d}"
        .format(url_import[0], url_import[1])
    )
    print(
        " Number of blueprints with source import      | {0:6d} | {1:5d}"
        .format(source_import[0], source_import[1])
    )
    if source_import[1] > 0:
        # There are some plugins that are imported from source
        print(
            "\nWARNING! There will be still some plugins imported from "
            "source left.\n         Those are listed in mappings "
            "file under the `unknown` nodes.\n         Please verify that "
            "you can replace those with managed plugins\n         and edit "
            "generated mappings file accordingly."
        )

def printout_install_suggestions(install_suggestions: dict):
    print("\n\nMake sure those plugins are installed (in suggested versions):")
    for plugin_name, suggested_versions in install_suggestions.items():
        print("  {0}: {1}".format(plugin_name, ", ".join(suggested_versions)))

def printout_correction_stats(stats):
    total_blueprints_scanned = sum([len(s) for s in stats.values()])
    number_of_unknown = len(stats.get(UNKNOWN, []))

    # Collect: the number of blueprints scanned,
    #          the number of blueprints corrected,
    #          the number of blueprints that are now valid (total)
    #          the number of blueprints that still require manual attention !!!

    print("\n\n                   CORRECTION STATS")
    print("----------------------------------------------+-------")
    print(
        " Number blueprints scanned                    | {0:5d}".format(
            total_blueprints_scanned
        )
    )
    print("----------------------------------------------+-------")
    print(
        " Number of blueprints corrected               | {0:5d}".format(
            len(stats.get(UPDATES, []))
        )
    )
    print(
        " Number of blueprints that are valid          | {0:5d}".format(
            len(stats.get(UPDATES, [])) + len(stats.get(FINE, []))
        )
    )
    print(
        " Number of blueprints that require attention  | {0:5d}".format(
            number_of_unknown
        )
    )
    if number_of_unknown:
        print(
            "\n\nMake sure to manually attend to {0} more "
            "blueprints".format(number_of_unknown)
        )
        if number_of_unknown < 20:
            print(
                "These blueprints require manual attention: {0}".format(
                    ", ".join(stats.get(UNKNOWN, []))
                )
            )

@click.command()
@click.option(
    "-t",
    "--tenant-name",
    "tenant_names",
    multiple=True,
    help="Tenant name; mutually exclusivele with --all-tenants.",
)
@click.option(
    "-a",
    "--all-tenants",
    is_flag=True,
    help="Include resources from all tenants.",
)
@click.option(
    "--plugin-name",
    "plugin_names",
    multiple=True,
    help="Plugin(s) to update (you can provide " "multiple --plugin-name(s).",
)
@click.option(
    "-b",
    "--blueprint",
    "blueprint_ids",
    multiple=True,
    help="Blueprint(s) to update (you can provide " "multiple --blueprint(s).",
)
@click.option(
    "--mapping",
    "mapping_file",
    multiple=False,
    required=True,
    help="Provide a mapping file generated with ",
)
@click.option(
    "--correct",
    is_flag=True,
    default=False,
    help="Update the blueprints using provided mapping file.",
)
def main(
    tenant_names,
    all_tenants,
    plugin_names,
    blueprint_ids,
    mapping_file,
    correct,
):
    def update_suggestions(new_suggestion: dict):
        for plugin_name, plugin_version in new_suggestion.items():
            if plugin_name not in install_suggestions:
                install_suggestions[plugin_name] = []
            if plugin_version not in install_suggestions[plugin_name]:
                install_suggestions[plugin_name].append(plugin_version)

    if all_tenants and tenant_names:
        logger.critical(
            "--all-tenants and --tenant-name options are " "mutually exclusive"
        )
        sys.exit(1)
    if not tenant_names:
        tenant_names = (common.DEFAULT_TENANT,)

    common.setup_environment()
    admin = models.User.query.get(0)
    sm = SQLStorageManager(user=admin)
    resolver = common.get_resolver(sm)
    tenants = (
        sm.list(models.Tenant, get_all_results=True)
        if all_tenants
        else [get_tenant_by_name(name) for name in tenant_names]
    )
    blueprint_filter = {"state": "uploaded"}
    if blueprint_ids:
        blueprint_filter["id"] = blueprint_ids
    mappings = load_mappings(mapping_file) if correct else {}
    stats = {}
    install_suggestions: Dict[str, List[str]] = {}
    blueprints_processed = 0

    for tenant in tenants:
        set_tenant_in_app(get_tenant_by_name(tenant.name))
        sm = SQLStorageManager(user=admin)
        blueprint_filter["tenant"] = tenant
        blueprints = sm.list(
            models.Blueprint, filters=blueprint_filter, get_all_results=True
        )

        for blueprint in blueprints:
            blueprint_identifier = "{0}::{1}".format(tenant.name, blueprint.id)
            if correct:
                logger.info(
                    "Correcting tenant's `%s` blueprint `%s`",
                    blueprint.tenant.name,
                    blueprint.id,
                )
                try:
                    result = correct_blueprint(
                        blueprint,
                        plugin_names,
                        mappings.get(tenant.name, {}).get(blueprint.id),
                    )
                except common.UpdateException as ex:
                    logger.error(
                        "Error updating tenant's `%s` blueprint `%s`: %s",
                        blueprint.tenant.name,
                        blueprint.id,
                        ex,
                    )
                else:
                    blueprints_processed += 1
                    if result not in stats:
                        stats[result] = [blueprint_identifier]
                    else:
                        stats[result].append(blueprint_identifier)
            else:
                logger.info(
                    "Scanning tenant's `%s` blueprint `%s`",
                    blueprint.tenant.name,
                    blueprint.id,
                )
                try:
                    a_mapping, a_statistic, a_suggestion = scan_blueprint(
                        resolver, blueprint, plugin_names
                    )
                except common.UpdateException as ex:
                    logger.error(
                        "Error scanning tenant's `%s` blueprint `%s`: %s",
                        blueprint.tenant.name,
                        blueprint.id,
                        ex,
                    )
                else:
                    blueprints_processed += 1
                    if a_mapping:
                        if tenant.name not in mappings:
                            mappings[tenant.name] = {}
                        mappings[tenant.name][blueprint.id] = a_mapping
                    if a_statistic:
                        stats[blueprint_identifier] = a_statistic
                    if a_suggestion:
                        update_suggestions(a_suggestion)

    # Wrap it up
    if correct:
        printout_correction_stats(stats)
    else:
        with open(mapping_file, "w") as output_file:
            yaml.dump(mappings, output_file, default_flow_style=False)
        chmod(mapping_file, 0o644)
        logger.info("Mapping file saved: %s", mapping_file)
        printout_scanning_stats(blueprints_processed, mappings, stats)
        if install_suggestions:
            printout_install_suggestions(install_suggestions)

redpajama_set_name: RedPajamaGithub


text:
$.register_module({
	name: 'og.views.convention_forms.swapindexconvention',
	dependencies: [
		'og.api.rest',
		'og.common.util.ui'
	],
	obj: function () {
		var ui = og.common.util.ui,
			forms = og.views.forms,
			api = og.api.rest,
			Form = ui.Form,
			ATTR = 'attributes',
			EIDS = 'externalIdBundle',
			INDX = '<INDEX>',
			EMPT = '<EMPTY>',
			type_map = [
				[['0', INDX].join('.'),								Form.type.STR],
				['name', 											Form.type.STR],
				['fixingTime', 										Form.type.STR],
				['swapConvention',		 							Form.type.STR],
				['underlyingConventionName', 						Form.type.STR],
				['uniqueId',										Form.type.STR],
				[[EIDS, 'ID', INDX, 'Scheme'].join('.'),	 		Form.type.STR],
				[[EIDS, 'ID', INDX, 'Value'].join('.'),				Form.type.STR],
				[['id', EMPT, 'scheme'].join('.'),					Form.type.STR],
				[['id', EMPT, 'value'].join('.'),					Form.type.STR],
				[[ATTR, EMPT].join('.'),							Form.type.STR],
				[[ATTR, INDX, 'Key'].join('.'),						Form.type.STR],
				[[ATTR, INDX, 'Value'].join('.'),					Form.type.STR],
				].reduce(function (acc, val) { return acc[val[0]] = val[1], acc; }, {});
        var arr = function (obj) { return arr && $.isArray(obj) ? obj : typeof obj !== 'undefined' ? [obj] : [] };
        var constructor = function (config) {
        	var load_handler = config.handler || $.noop,
	        	selector = config.selector,
	        	loading = config.loading || $.noop,
	        	deleted = config.data.template_data.deleted,
	        	is_new = config.is_new,
	        	orig_name = config.data.template_data.name,
	        	resource_id = config.data.template_data.object_id,
	        	save_new_handler = config.save_new_handler,
	        	save_handler = config.save_handler,
	        	master = config.data.template_data.configJSON.data,
	        	convention_type = config.type,
	        	fixingTime = master.fixingTime,
	        	underlyingConventionName = master.underlyingConventionName,
	        	form = new Form({
	        		module: 'og.views.forms.swap-index-convention_tash',
	        		data: master,
	        		type_map: type_map,
	        		selector: selector,
	        		extras: {
	        			name: master.name,
	        		},
	        		processor: function (data) {
	        			data.id = data.id.filter(function (v) { return v !== void 0; });
            			data[EIDS] = arr(data[EIDS]).filter(function (v) { return v !== void 0; });
            			data[ATTR] = arr(data[ATTR]).filter(function (v) { return v !== void 0; });
	        		}
	        	}),
	        	form_id = '#' + form.id,
        		save_resource = function (result) {
        			var data = result.data,
        				meta = result.meta,
        				as_new = result.extras.as_new;
        			if (as_new && (orig_name == data.name)) { return window.alert('Please select a new name.') };
        			if (!data.externalIdBundle.ID.length) { return window.alert('Please add at least one external identifier') };
        			api.conventions.put({
        				id: as_new ? void 0 : resource_id,
        				name: data.name,
        				json: JSON.stringify({ data: data, meta: meta }),
        				type: convention_type,
        				loading: loading,
        				handler: as_new ? save_new_handler : save_handler
        			});
        		},
        		load_resource = function () {
            		var header = '\
            			\

\

\
            			' + master.name + '\

\
            			  &nbsp(Swap Index Convention)\
            			\
            			';
            		$('.OG-layout-admin-details-center .ui-layout-header').html(header);
            		$(form_id + ' input[name=fixingTime]').val(fixingTime);
            		$(form_id + ' p[id=warning]')[0].hidden = ($('#' + form.children[0].id)[0].options.length > 1);
            		setTimeout(load_handler.partial(form));
                };
            form.on('form:submit', save_resource)
            	.on('form:load', load_resource);
            form.children = [
            	new ui.Dropdown({
            		form: form,
            		placeholder: 'Please select...',
            		value: !underlyingConventionName ? "" : underlyingConventionName,
            		resource: 'conventions.conventionIds',
            		data_generator: function (handler) {
            			api.conventions.convention_ids.get({ conventionType: 'Swap'}).pipe(function (result) {
            				handler(result.data.map(function (convention) {
            					var split = convention.split('|');
            					return { value: split[0], text: split[0], selected: split[0] === underlyingConventionName };
            				}))
            			})
            		},
            		index: 'swapConvention'
            	}),
            	// item_1
            	new og.views.convention_forms.ExternalIdBundle({
            		form: form,
            		data: master.externalIdBundle,
            		index: 'externalIdBundle',
            	}),
            	// item_2
            	new og.views.convention_forms.Attributes({
            		form: form,
            		attributes: master.attributes,
            		index: 'attributes'
            	})
            ];
            form.dom();
        };
        constructor.type_map = type_map;
        return constructor;
	}
})


text:
"""
main.py
"""
import septa
import dispatch

def printData(data):
	print "Data Recieved:"
	print data
	pass

def main():
	"""
	Take input
	Instantiate Trips.
	While...run dispatch for each trip
	"""

	trips = []
	print "Septa-Dispatch"
	#add more stuff here
	routeName = raw_input("Enter Bus Name to start tracking: ")
	print "route accepted, getting data"

	pass

main()


text:
#ifndef __WIDGET_TWO_H__
#define __WIDGET_TWO_H__

#include <mycommon.h>
#include <mytrace.h>

#include "participant.h"

struct mediator;
/** our channel of communication */
struct widget_two {
	struct participant participant;
	struct mediator * _mediator;
};

/** constructor(). */
void widget_two_init(struct widget_two *widget_two, struct mediator *mediator);

/** forward to mediator */
char * widget_two_talk(struct widget_two *widget_two, char *);

#endif /* __WIDGET_TWO_H__ */