protected List<URL> getDriversFromMirror(URL driverUrl) throws IOException {
        if (mirrorLog) {
            log.info("Crawling driver list from mirror {}", driverUrl);
            mirrorLog = true;
        } else {
            log.trace("[Recursive call] Crawling driver list from mirror {}",
                    driverUrl);
        }

        String driverStr = driverUrl.toString();
        String driverUrlContent = driverUrl.getPath();

        HttpResponse response = httpClient
                .execute(httpClient.createHttpGet(driverUrl));
        try (InputStream in = response.getEntity().getContent()) {
            org.jsoup.nodes.Document doc = Jsoup.parse(in, null, "");
            Iterator<org.jsoup.nodes.Element> iterator = doc.select("a")
                    .iterator();
            List<URL> urlList = new ArrayList<>();

            while (iterator.hasNext()) {
                String link = iterator.next().attr("href");
                if (link.contains("mirror") && link.endsWith(SLASH)) {
                    urlList.addAll(getDriversFromMirror(new URL(
                            driverStr + link.replace(driverUrlContent, ""))));
                } else if (link.startsWith(driverUrlContent)
                        && !link.contains("icons")) {
                    urlList.add(new URL(
                            driverStr + link.replace(driverUrlContent, "")));
                }
            }
            return urlList;
        }
    }
protected List<URL> getDriversFromMirror(URL driverUrl) throws IOException {
        if (mirrorLog) {
            log.info("Crawling driver list from mirror {}", driverUrl);
            mirrorLog = true;
        } else {
            log.trace("[Recursive call] Crawling driver list from mirror {}",
                    driverUrl);
        }

        String driverStr = driverUrl.toString();
        String driverUrlContent = driverUrl.getPath();

        HttpResponse response = httpClient
                .execute(httpClient.createHttpGet(driverUrl));
        try (InputStream in = response.getEntity().getContent()) {
            org.jsoup.nodes.Document doc = Jsoup.parse(in, null, "");
            Iterator<org.jsoup.nodes.Element> iterator = doc.select("a")
                    .iterator();
            List<URL> urlList = new ArrayList<>();

            while (iterator.hasNext()) {
                String link = iterator.next().attr("href");
                if (link.contains("mirror") && link.endsWith(SLASH)) {
                    urlList.addAll(getDriversFromMirror(new URL(
                            driverStr + link.replace(driverUrlContent, ""))));
                } else if (link.startsWith(driverUrlContent)
                        && !link.contains("icons")) {
                    urlList.add(new URL(
                            driverStr + link.replace(driverUrlContent, "")));
                }
            }
            return urlList;
        }
    }
protected List < URL > getDriversFromMirror ( URL driverUrl ) throws IOException { if ( mirrorLog ) { log . info ( "Crawling driver list from mirror {}" , driverUrl ) ; mirrorLog = true ; } else { log . trace ( "[Recursive call] Crawling driver list from mirror {}" , driverUrl ) ; } String driverStr = driverUrl . toString ( ) ; String driverUrlContent = driverUrl . getPath ( ) ; HttpResponse response = httpClient . execute ( httpClient . createHttpGet ( driverUrl ) ) ; try ( InputStream in = response . getEntity ( ) . getContent ( ) ) { org . jsoup . nodes . Document doc = Jsoup . parse ( in , null , "" ) ; Iterator < org . jsoup . nodes . Element > iterator = doc . select ( "a" ) . iterator ( ) ; List < URL > urlList = new ArrayList <> ( ) ; while ( iterator . hasNext ( ) ) { String link = iterator . next ( ) . attr ( "href" ) ; if ( link . contains ( "mirror" ) && link . endsWith ( SLASH ) ) { urlList . addAll ( getDriversFromMirror ( new URL ( driverStr + link . replace ( driverUrlContent , "" ) ) ) ) ; } else if ( link . startsWith ( driverUrlContent ) && ! link . contains ( "icons" ) ) { urlList . add ( new URL ( driverStr + link . replace ( driverUrlContent , "" ) ) ) ; } } return urlList ; } }
This method works also for http://npm.taobao.org/ and
https://bitbucket.org/ mirrors.
This method works also for http : // npm . taobao . org / and https : // bitbucket . org / mirrors .