public FeedLoadResult loadTables () {

        // This result object will be returned to the caller to summarize the feed and report any critical errors.
        FeedLoadResult result = new FeedLoadResult();

        try {
            // Begin tracking time. FIXME: should this follow the connect/register and begin with the table loads?
            long startTime = System.currentTimeMillis();
            // We get a single connection object and share it across several different methods.
            // This ensures that actions taken in one method are visible to all subsequent SQL statements.
            // If we create a schema or table on one connection, then access it in a separate connection, we have no
            // guarantee that it exists when the accessing statement is executed.
            connection = dataSource.getConnection();
            // Generate a unique prefix that will identify this feed.
            // Prefixes ("schema" names) based on feed_id and feed_version get very messy, so we use random unique IDs.
            // We don't want to use an auto-increment numeric primary key because these need to be alphabetical.
            // Although ID collisions are theoretically possible, they are improbable in the extreme because our IDs
            // are long enough to have as much entropy as a UUID. So we don't really need to check for uniqueness and
            // retry in a loop.
            // TODO handle the case where we don't want any prefix.
            this.tablePrefix = this.namespace = randomIdString();
            result.uniqueIdentifier = tablePrefix;
            registerFeed();
            // Include the dot separator in the table prefix.
            // This allows everything to work even when there's no prefix.
            this.tablePrefix += ".";
            this.errorStorage = new SQLErrorStorage(connection, tablePrefix, true);

            // Get entity lists that are nested in feed.
            List<Calendar> calendars = gtfsFeed.services.values()
                    .stream()
                    .map(service -> service.calendar)
                    .collect(Collectors.toList());
            List<CalendarDate> calendarDates = gtfsFeed.services.values()
                    .stream()
                    .map(service -> service.calendar_dates.values())
                    .flatMap(Collection::stream)
                    .collect(Collectors.toList());
            List<FareAttribute> fareAttributes = gtfsFeed.fares.values()
                    .stream()
                    .map(fare -> fare.fare_attribute)
                    .collect(Collectors.toList());
            List<FareRule> fareRules = gtfsFeed.fares.values()
                    .stream()
                    .map(fare -> fare.fare_rules)
                    .flatMap(List::stream)
                    .collect(Collectors.toList());
            List<Frequency> frequencies = gtfsFeed.frequencies.stream()
                    .map(stringFrequencyTuple2 -> stringFrequencyTuple2.b)
                    .collect(Collectors.toList());

            // Copy all tables (except for PATTERN_STOPS, which does not exist in GTFSFeed).
            copyEntityToSql(gtfsFeed.agency.values(), Table.AGENCY);
            copyEntityToSql(calendars, Table.CALENDAR);
            copyEntityToSql(calendarDates, Table.CALENDAR_DATES);
            copyEntityToSql(gtfsFeed.routes.values(), Table.ROUTES);
            copyEntityToSql(gtfsFeed.patterns.values(), Table.PATTERNS);
            // FIXME: How to handle pattern stops?
//            copyEntityToSql(gtfsFeed.patterns.values(), Table.PATTERN_STOP);
            copyEntityToSql(fareAttributes, Table.FARE_ATTRIBUTES);
            copyEntityToSql(fareRules, Table.FARE_RULES);
            copyEntityToSql(gtfsFeed.feedInfo.values(), Table.FEED_INFO);
            copyEntityToSql(gtfsFeed.shape_points.values(), Table.SHAPES);
            copyEntityToSql(gtfsFeed.stops.values(), Table.STOPS);
            copyEntityToSql(gtfsFeed.transfers.values(), Table.TRANSFERS);
            copyEntityToSql(gtfsFeed.trips.values(), Table.TRIPS); // refs routes
            copyEntityToSql(frequencies, Table.FREQUENCIES); // refs trips
            copyEntityToSql(gtfsFeed.stop_times.values(), Table.STOP_TIMES);
//            result.errorCount = errorStorage.getErrorCount();
            // This will commit and close the single connection that has been shared between all preceding load steps.
            errorStorage.commitAndClose();
            result.completionTime = System.currentTimeMillis();
            result.loadTimeMillis = result.completionTime - startTime;
            LOG.info("Loading tables took {} sec", result.loadTimeMillis / 1000);
        } catch (Exception ex) {
            try {
                connection.rollback();
            } catch (SQLException e) {
                e.printStackTrace();
            }
            // TODO catch exceptions separately while loading each table so load can continue, store in TableLoadResult
            LOG.error("Exception while loading GTFS file: {}", ex.toString());
            ex.printStackTrace();
            result.fatalException = ex.getMessage();
        } finally {
            DbUtils.closeQuietly(connection);
        }
        return result;
    }
public FeedLoadResult loadTables () {

        
        FeedLoadResult result = new FeedLoadResult();

        try {
            
            long startTime = System.currentTimeMillis();
            
            
            
            
            connection = dataSource.getConnection();
            
            
            
            
            
            
            
            this.tablePrefix = this.namespace = randomIdString();
            result.uniqueIdentifier = tablePrefix;
            registerFeed();
            
            
            this.tablePrefix += ".";
            this.errorStorage = new SQLErrorStorage(connection, tablePrefix, true);

            
            List<Calendar> calendars = gtfsFeed.services.values()
                    .stream()
                    .map(service -> service.calendar)
                    .collect(Collectors.toList());
            List<CalendarDate> calendarDates = gtfsFeed.services.values()
                    .stream()
                    .map(service -> service.calendar_dates.values())
                    .flatMap(Collection::stream)
                    .collect(Collectors.toList());
            List<FareAttribute> fareAttributes = gtfsFeed.fares.values()
                    .stream()
                    .map(fare -> fare.fare_attribute)
                    .collect(Collectors.toList());
            List<FareRule> fareRules = gtfsFeed.fares.values()
                    .stream()
                    .map(fare -> fare.fare_rules)
                    .flatMap(List::stream)
                    .collect(Collectors.toList());
            List<Frequency> frequencies = gtfsFeed.frequencies.stream()
                    .map(stringFrequencyTuple2 -> stringFrequencyTuple2.b)
                    .collect(Collectors.toList());

            
            copyEntityToSql(gtfsFeed.agency.values(), Table.AGENCY);
            copyEntityToSql(calendars, Table.CALENDAR);
            copyEntityToSql(calendarDates, Table.CALENDAR_DATES);
            copyEntityToSql(gtfsFeed.routes.values(), Table.ROUTES);
            copyEntityToSql(gtfsFeed.patterns.values(), Table.PATTERNS);
            

            copyEntityToSql(fareAttributes, Table.FARE_ATTRIBUTES);
            copyEntityToSql(fareRules, Table.FARE_RULES);
            copyEntityToSql(gtfsFeed.feedInfo.values(), Table.FEED_INFO);
            copyEntityToSql(gtfsFeed.shape_points.values(), Table.SHAPES);
            copyEntityToSql(gtfsFeed.stops.values(), Table.STOPS);
            copyEntityToSql(gtfsFeed.transfers.values(), Table.TRANSFERS);
            copyEntityToSql(gtfsFeed.trips.values(), Table.TRIPS); 
            copyEntityToSql(frequencies, Table.FREQUENCIES); 
            copyEntityToSql(gtfsFeed.stop_times.values(), Table.STOP_TIMES);

            
            errorStorage.commitAndClose();
            result.completionTime = System.currentTimeMillis();
            result.loadTimeMillis = result.completionTime - startTime;
            LOG.info("Loading tables took {} sec", result.loadTimeMillis / 1000);
        } catch (Exception ex) {
            try {
                connection.rollback();
            } catch (SQLException e) {
                e.printStackTrace();
            }
            
            LOG.error("Exception while loading GTFS file: {}", ex.toString());
            ex.printStackTrace();
            result.fatalException = ex.getMessage();
        } finally {
            DbUtils.closeQuietly(connection);
        }
        return result;
    }
public FeedLoadResult loadTables ( ) { // This result object will be returned to the caller to summarize the feed and report any critical errors. FeedLoadResult result = new FeedLoadResult ( ) ; try { // Begin tracking time. FIXME: should this follow the connect/register and begin with the table loads? long startTime = System . currentTimeMillis ( ) ; // We get a single connection object and share it across several different methods. // This ensures that actions taken in one method are visible to all subsequent SQL statements. // If we create a schema or table on one connection, then access it in a separate connection, we have no // guarantee that it exists when the accessing statement is executed. connection = dataSource . getConnection ( ) ; // Generate a unique prefix that will identify this feed. // Prefixes ("schema" names) based on feed_id and feed_version get very messy, so we use random unique IDs. // We don't want to use an auto-increment numeric primary key because these need to be alphabetical. // Although ID collisions are theoretically possible, they are improbable in the extreme because our IDs // are long enough to have as much entropy as a UUID. So we don't really need to check for uniqueness and // retry in a loop. // TODO handle the case where we don't want any prefix. this . tablePrefix = this . namespace = randomIdString ( ) ; result . uniqueIdentifier = tablePrefix ; registerFeed ( ) ; // Include the dot separator in the table prefix. // This allows everything to work even when there's no prefix. this . tablePrefix += "." ; this . errorStorage = new SQLErrorStorage ( connection , tablePrefix , true ) ; // Get entity lists that are nested in feed. List < Calendar > calendars = gtfsFeed . services . values ( ) . stream ( ) . map ( service -> service . calendar ) . collect ( Collectors . toList ( ) ) ; List < CalendarDate > calendarDates = gtfsFeed . services . values ( ) . stream ( ) . map ( service -> service . calendar_dates . values ( ) ) . flatMap ( Collection :: stream ) . collect ( Collectors . toList ( ) ) ; List < FareAttribute > fareAttributes = gtfsFeed . fares . values ( ) . stream ( ) . map ( fare -> fare . fare_attribute ) . collect ( Collectors . toList ( ) ) ; List < FareRule > fareRules = gtfsFeed . fares . values ( ) . stream ( ) . map ( fare -> fare . fare_rules ) . flatMap ( List :: stream ) . collect ( Collectors . toList ( ) ) ; List < Frequency > frequencies = gtfsFeed . frequencies . stream ( ) . map ( stringFrequencyTuple2 -> stringFrequencyTuple2 . b ) . collect ( Collectors . toList ( ) ) ; // Copy all tables (except for PATTERN_STOPS, which does not exist in GTFSFeed). copyEntityToSql ( gtfsFeed . agency . values ( ) , Table . AGENCY ) ; copyEntityToSql ( calendars , Table . CALENDAR ) ; copyEntityToSql ( calendarDates , Table . CALENDAR_DATES ) ; copyEntityToSql ( gtfsFeed . routes . values ( ) , Table . ROUTES ) ; copyEntityToSql ( gtfsFeed . patterns . values ( ) , Table . PATTERNS ) ; // FIXME: How to handle pattern stops? //            copyEntityToSql(gtfsFeed.patterns.values(), Table.PATTERN_STOP); copyEntityToSql ( fareAttributes , Table . FARE_ATTRIBUTES ) ; copyEntityToSql ( fareRules , Table . FARE_RULES ) ; copyEntityToSql ( gtfsFeed . feedInfo . values ( ) , Table . FEED_INFO ) ; copyEntityToSql ( gtfsFeed . shape_points . values ( ) , Table . SHAPES ) ; copyEntityToSql ( gtfsFeed . stops . values ( ) , Table . STOPS ) ; copyEntityToSql ( gtfsFeed . transfers . values ( ) , Table . TRANSFERS ) ; copyEntityToSql ( gtfsFeed . trips . values ( ) , Table . TRIPS ) ; // refs routes copyEntityToSql ( frequencies , Table . FREQUENCIES ) ; // refs trips copyEntityToSql ( gtfsFeed . stop_times . values ( ) , Table . STOP_TIMES ) ; //            result.errorCount = errorStorage.getErrorCount(); // This will commit and close the single connection that has been shared between all preceding load steps. errorStorage . commitAndClose ( ) ; result . completionTime = System . currentTimeMillis ( ) ; result . loadTimeMillis = result . completionTime - startTime ; LOG . info ( "Loading tables took {} sec" , result . loadTimeMillis / 1000 ) ; } catch ( Exception ex ) { try { connection . rollback ( ) ; } catch ( SQLException e ) { e . printStackTrace ( ) ; } // TODO catch exceptions separately while loading each table so load can continue, store in TableLoadResult LOG . error ( "Exception while loading GTFS file: {}" , ex . toString ( ) ) ; ex . printStackTrace ( ) ; result . fatalException = ex . getMessage ( ) ; } finally { DbUtils . closeQuietly ( connection ) ; } return result ; }
There appears to be no advantage to loading tables in parallel, as the whole loading process is I/O bound.
There appears to be no advantage to loading tables in parallel as the whole loading process is I / O bound .