title: Web crawler stops at first page
body: <p>I'm working on a webcrawler which should be working like this:</p>
  <ol>
  <li>go to a website, crawl all links from the site</li>
  <li>download all images (starting from the startpage)</li>
  <li>if there are no images left on the current page, go to the next link found in step 1 and do step 2 and 3 until there are no links/images left.</li>
  </ol>
  <p>It seems like the code below is somehow working, like when I try to crawl some sites, I get some images to download.</p>
  <p><em>(even I dont understand the images I get, cause I cant find them on the website, it seems like the crawler does not start with the startpage of the website)</em>.</p>
  <p>After a few images (~25-500), the crawler is done and stops, no errors, it just stops. I tried this with multiple websites and after a few images it just stops. I think the crawler somehow ignores step 3.</p>
  <pre><code>package main
  
  import (
      &quot;fmt&quot;
      &quot;io&quot;
      &quot;log&quot;
      &quot;net/http&quot;
      &quot;os&quot;
      &quot;strconv&quot;
      &quot;strings&quot;
  
      &quot;github.com/PuerkitoBio/goquery&quot;
  )
  
  var (
      currWebsite  string = &quot;https://www.youtube.com&quot;
      imageCount   int    = 0
      crawlWebsite string
  )
  
  func processElement(index int, element *goquery.Selection) {
      href, exists := element.Attr(&quot;href&quot;)
      if exists &amp;&amp; strings.HasPrefix(href, &quot;http&quot;) {
          crawlWebsite = href
          response, err := http.Get(crawlWebsite)
          if err != nil {
              log.Fatalf(&quot;error on current website&quot;)
          }
  
          defer response.Body.Close()
  
          document, err := goquery.NewDocumentFromReader(response.Body)
          if err != nil {
              log.Fatal(&quot;Error loading HTTP response body.&quot;, err)
          }
  
          document.Find(&quot;img&quot;).Each(func(index int, element *goquery.Selection) {
              imgSrc, exists := element.Attr(&quot;src&quot;)
              if strings.HasPrefix(imgSrc, &quot;http&quot;) &amp;&amp; exists {
                  fileName := fmt.Sprintf(&quot;./images/img&quot; + strconv.Itoa(imageCount) + &quot;.jpg&quot;)
                  currWebsite := fmt.Sprint(imgSrc)
                  fmt.Println(&quot;[+]&quot;, currWebsite)
                  DownloadFile(fileName, currWebsite)
                  imageCount++
              }
          })
      }
  }
  
  func main() {
      err := os.MkdirAll(&quot;./images/&quot;, 0777)
      if err != nil {
          log.Fatalln(&quot;error on creating directory&quot;)
      }
  
      response, err := http.Get(currWebsite)
      if err != nil {
          log.Fatalln(&quot;error on searching website&quot;)
      }
  
      defer response.Body.Close()
  
      document, err := goquery.NewDocumentFromReader(response.Body)
      if err != nil {
          log.Fatalln(&quot;Error loading HTTP response body. &quot;, err)
      }
  
      document.Find(&quot;a&quot;).Each(processElement)
  }
  
  func DownloadFile(filepath string, url string) {
      response, err := http.Get(url)
      if err != nil {
          log.Fatalln(&quot;error getting the website infos&quot;)
      }
      defer response.Body.Close()
  
      if response.StatusCode != 200 {
          log.Fatalln(&quot;received non 200 response code&quot;)
      }
  
      file, err := os.Create(filepath)
      if err != nil {
          log.Fatalf(&quot;error creating file at %v\n&quot;, filepath)
      }
  
      defer file.Close()
  
      _, err = io.Copy(file, response.Body)
      if err != nil {
          log.Fatalln(&quot;error copy file from src to dst&quot;)
      }
  }
  </code></pre>