private T retrieveAndProcessFeed (String originalId) {
        // see if we have it cached locally
        String id = cleanId(originalId);
        String key = bucketFolder != null ? String.join("/", bucketFolder, id) : id;
        File dbFile = new File(cacheDir, id + ".db");
        GTFSFeed feed;
        if (dbFile.exists()) {
            LOG.info("Processed GTFS was found cached locally");
            try {
                feed = new GTFSFeed(dbFile.getAbsolutePath());
                if (feed != null) {
                    return processFeed(feed);
                }
            } catch (Exception e) {
                LOG.warn("Error loading local MapDB.", e);
                deleteLocalDBFiles(id);
            }
        }

        if (bucket != null) {
            try {
                LOG.info("Attempting to download cached GTFS MapDB from S3: {}/{}.db", bucket, key);
                S3Object db = s3.getObject(bucket, key + ".db");
                InputStream is = db.getObjectContent();
                FileOutputStream fos = new FileOutputStream(dbFile);
                ByteStreams.copy(is, fos);
                is.close();
                fos.close();

                S3Object dbp = s3.getObject(bucket, key + ".db.p");
                InputStream isp = dbp.getObjectContent();
                FileOutputStream fosp = new FileOutputStream(new File(cacheDir, id + ".db.p"));
                ByteStreams.copy(isp, fosp);
                isp.close();
                fosp.close();

                LOG.info("Returning processed GTFS from S3");
                feed = new GTFSFeed(dbFile.getAbsolutePath());
                if (feed != null) {
                    return processFeed(feed);
                }
            } catch (AmazonS3Exception e) {
                LOG.warn("MapDB file for key '{}' does not exist on S3.", key);
            } catch (ExecutionException | IOException e) {
                LOG.warn("Error retrieving MapDB file from S3, will re-create one from the original GTFS.", e);
            }
        }

        // if we fell through to here, getting the mapdb was unsuccessful
        // grab GTFS from S3 if it is not found locally
        File feedFile = new File(cacheDir, id + ".zip");
        if (feedFile.exists()) {
            LOG.info("Loading feed from local cache directory...");
        }

        if (!feedFile.exists() && bucket != null) {
            LOG.info("Feed not found locally, downloading from S3.");
            try {
                S3Object gtfs = s3.getObject(bucket, key + ".zip");
                InputStream is = gtfs.getObjectContent();
                FileOutputStream fos = new FileOutputStream(feedFile);
                ByteStreams.copy(is, fos);
                is.close();
                fos.close();
            } catch (Exception e) {
                LOG.error("Could not download feed at s3://{}/{}.", bucket, key);
                throw new RuntimeException(e);
            }
        }

        if (feedFile.exists()) {
            // TODO this will also re-upload the original feed ZIP to S3.
            try {
                return put(originalId, feedFile);
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        } else {
            LOG.warn("Feed {} not found locally", originalId);
            throw new NoSuchElementException(originalId);
        }
    }
private T retrieveAndProcessFeed (String originalId) {
        
        String id = cleanId(originalId);
        String key = bucketFolder != null ? String.join("/", bucketFolder, id) : id;
        File dbFile = new File(cacheDir, id + ".db");
        GTFSFeed feed;
        if (dbFile.exists()) {
            LOG.info("Processed GTFS was found cached locally");
            try {
                feed = new GTFSFeed(dbFile.getAbsolutePath());
                if (feed != null) {
                    return processFeed(feed);
                }
            } catch (Exception e) {
                LOG.warn("Error loading local MapDB.", e);
                deleteLocalDBFiles(id);
            }
        }

        if (bucket != null) {
            try {
                LOG.info("Attempting to download cached GTFS MapDB from S3: {}/{}.db", bucket, key);
                S3Object db = s3.getObject(bucket, key + ".db");
                InputStream is = db.getObjectContent();
                FileOutputStream fos = new FileOutputStream(dbFile);
                ByteStreams.copy(is, fos);
                is.close();
                fos.close();

                S3Object dbp = s3.getObject(bucket, key + ".db.p");
                InputStream isp = dbp.getObjectContent();
                FileOutputStream fosp = new FileOutputStream(new File(cacheDir, id + ".db.p"));
                ByteStreams.copy(isp, fosp);
                isp.close();
                fosp.close();

                LOG.info("Returning processed GTFS from S3");
                feed = new GTFSFeed(dbFile.getAbsolutePath());
                if (feed != null) {
                    return processFeed(feed);
                }
            } catch (AmazonS3Exception e) {
                LOG.warn("MapDB file for key '{}' does not exist on S3.", key);
            } catch (ExecutionException | IOException e) {
                LOG.warn("Error retrieving MapDB file from S3, will re-create one from the original GTFS.", e);
            }
        }

        
        
        File feedFile = new File(cacheDir, id + ".zip");
        if (feedFile.exists()) {
            LOG.info("Loading feed from local cache directory...");
        }

        if (!feedFile.exists() && bucket != null) {
            LOG.info("Feed not found locally, downloading from S3.");
            try {
                S3Object gtfs = s3.getObject(bucket, key + ".zip");
                InputStream is = gtfs.getObjectContent();
                FileOutputStream fos = new FileOutputStream(feedFile);
                ByteStreams.copy(is, fos);
                is.close();
                fos.close();
            } catch (Exception e) {
                LOG.error("Could not download feed at s3:
                throw new RuntimeException(e);
            }
        }

        if (feedFile.exists()) {
            
            try {
                return put(originalId, feedFile);
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        } else {
            LOG.warn("Feed {} not found locally", originalId);
            throw new NoSuchElementException(originalId);
        }
    }
private T retrieveAndProcessFeed ( String originalId ) { // see if we have it cached locally String id = cleanId ( originalId ) ; String key = bucketFolder != null ? String . join ( "/" , bucketFolder , id ) : id ; File dbFile = new File ( cacheDir , id + ".db" ) ; GTFSFeed feed ; if ( dbFile . exists ( ) ) { LOG . info ( "Processed GTFS was found cached locally" ) ; try { feed = new GTFSFeed ( dbFile . getAbsolutePath ( ) ) ; if ( feed != null ) { return processFeed ( feed ) ; } } catch ( Exception e ) { LOG . warn ( "Error loading local MapDB." , e ) ; deleteLocalDBFiles ( id ) ; } } if ( bucket != null ) { try { LOG . info ( "Attempting to download cached GTFS MapDB from S3: {}/{}.db" , bucket , key ) ; S3Object db = s3 . getObject ( bucket , key + ".db" ) ; InputStream is = db . getObjectContent ( ) ; FileOutputStream fos = new FileOutputStream ( dbFile ) ; ByteStreams . copy ( is , fos ) ; is . close ( ) ; fos . close ( ) ; S3Object dbp = s3 . getObject ( bucket , key + ".db.p" ) ; InputStream isp = dbp . getObjectContent ( ) ; FileOutputStream fosp = new FileOutputStream ( new File ( cacheDir , id + ".db.p" ) ) ; ByteStreams . copy ( isp , fosp ) ; isp . close ( ) ; fosp . close ( ) ; LOG . info ( "Returning processed GTFS from S3" ) ; feed = new GTFSFeed ( dbFile . getAbsolutePath ( ) ) ; if ( feed != null ) { return processFeed ( feed ) ; } } catch ( AmazonS3Exception e ) { LOG . warn ( "MapDB file for key '{}' does not exist on S3." , key ) ; } catch ( ExecutionException | IOException e ) { LOG . warn ( "Error retrieving MapDB file from S3, will re-create one from the original GTFS." , e ) ; } } // if we fell through to here, getting the mapdb was unsuccessful // grab GTFS from S3 if it is not found locally File feedFile = new File ( cacheDir , id + ".zip" ) ; if ( feedFile . exists ( ) ) { LOG . info ( "Loading feed from local cache directory..." ) ; } if ( ! feedFile . exists ( ) && bucket != null ) { LOG . info ( "Feed not found locally, downloading from S3." ) ; try { S3Object gtfs = s3 . getObject ( bucket , key + ".zip" ) ; InputStream is = gtfs . getObjectContent ( ) ; FileOutputStream fos = new FileOutputStream ( feedFile ) ; ByteStreams . copy ( is , fos ) ; is . close ( ) ; fos . close ( ) ; } catch ( Exception e ) { LOG . error ( "Could not download feed at s3://{}/{}." , bucket , key ) ; throw new RuntimeException ( e ) ; } } if ( feedFile . exists ( ) ) { // TODO this will also re-upload the original feed ZIP to S3. try { return put ( originalId , feedFile ) ; } catch ( Exception e ) { throw new RuntimeException ( e ) ; } } else { LOG . warn ( "Feed {} not found locally" , originalId ) ; throw new NoSuchElementException ( originalId ) ; } }
retrieve a feed from local cache or S3
retrieve a feed from local cache or S3