title: Multiple goroutines access/modify a list/map
body: <p>I am trying to implement a multithreaded crawler using a go lang as a sample task to learn the language.</p>
  
  <p>It supposed to scan pages, follow links and save them do DB.</p>
  
  <p>To avoid duplicates I'm trying to use map where I save all the URLs I've already saved.</p>
  
  <p>The synchronous version works fine, but I have troubles when I'm trying to use goroutines.</p>
  
  <p>I'm trying to use mutex as a sync object for map, and channel as a way to coordinate goroutines. But obviously I don't have clear understanding of them. </p>
  
  <p>The problem is that I have many duplicate entries, so my map store/check does not work properly.</p>
  
  <p>Here is my code:</p>
  
  <pre><code>package main
  
  import (
      "fmt"
      "net/http"
      "golang.org/x/net/html"
      "strings"
      "database/sql"
      _ "github.com/ziutek/mymysql/godrv"
      "io/ioutil"
      "runtime/debug"
      "sync"
  )
  
  const maxDepth = 2;
  
  var workers = make(chan bool)
  
  type Pages struct {
      mu sync.Mutex
      pagesMap map[string]bool
  }
  
  func main() {
      var pagesMutex Pages
      fmt.Println("Start")
      const database = "gotest"
      const user = "root"
      const password = "123"
  
      //open connection to DB
      con, err := sql.Open("mymysql", database + "/" + user + "/" + password)
      if err != nil { /* error handling */
          fmt.Printf("%s", err)
          debug.PrintStack()
      }
  
      fmt.Println("call 1st save site")
      pagesMutex.pagesMap = make(map[string]bool)
      go pagesMutex.saveSite(con, "http://golang.org/", 0)
  
      fmt.Println("saving true to channel")
      workers &lt;- true
  
      fmt.Println("finishing in main")
      defer con.Close()
  }
  
  
  func (p *Pages) saveSite(con *sql.DB, url string, depth int) {
      fmt.Println("Save ", url, depth)
      fmt.Println("trying to lock")
      p.mu.Lock()
      fmt.Println("locked on mutex")
      pageDownloaded := p.pagesMap[url] == true
      if pageDownloaded {
          p.mu.Unlock()
          return
      } else {
          p.pagesMap[url] = true
      }
      p.mu.Unlock()
  
      response, err := http.Get(url)
      if err != nil {
          fmt.Printf("%s", err)
          debug.PrintStack()
      } else {
          defer response.Body.Close()
  
          contents, err := ioutil.ReadAll(response.Body)
          if err != nil {
              if err != nil {
                  fmt.Printf("%s", err)
                  debug.PrintStack()
              }
          }
  
          _, err = con.Exec("insert into pages (url) values (?)", string(url))
          if err != nil {
              fmt.Printf("%s", err)
              debug.PrintStack()
          }
          z := html.NewTokenizer(strings.NewReader((string(contents))))
  
          for {
              tokenType := z.Next()
              if tokenType == html.ErrorToken {
                  return
              }
  
              token := z.Token()
              switch tokenType {
              case html.StartTagToken: // &lt;tag&gt;
  
                  tagName := token.Data
                  if strings.Compare(string(tagName), "a") == 0 {
                      for _, attr := range token.Attr {
                          if strings.Compare(attr.Key, "href") == 0 {
                              if depth &lt; maxDepth  {
                                  urlNew := attr.Val
                                  if !strings.HasPrefix(urlNew, "http")  {
                                      if strings.HasPrefix(urlNew, "/")  {
                                          urlNew = urlNew[1:]
                                      }
                                      urlNew = url + urlNew
                                  }
                                  //urlNew = path.Clean(urlNew)
                                  go  p.saveSite(con, urlNew, depth + 1)
  
                              }
                          }
                      }
  
                  }
              case html.TextToken: // text between start and end tag
              case html.EndTagToken: // &lt;/tag&gt;
              case html.SelfClosingTagToken: // &lt;tag/&gt;
  
              }
  
          }
  
      }
      val := &lt;-workers
      fmt.Println("finished Save Site", val)
  }
  </code></pre>
  
  <p>Could someone explain to me how to do this properly, please?</p>