title: How to make Web Crawler to crawl pages in background?
body: <p>i am currently developing a web crawler in PHP and it still is a simple one but what i want to know is how can i make my crawler to crawl pages in background and not use my bandwidth, do i have to use some cron jobs and i want it to automatically store the data in database.</p>
  
  <p>Here what i have done:</p>
  
  <pre><code>&lt;?php 
  $conn = mysqli_connect("localhost","root","","crawler") or die(mysqli_error());
  ini_set('max_execution_time', 4000);
  $to_crawl = "http://hootpile.com";
  $c = array();
  function get_links($url){
      global $c;
      $input = file_get_contents($url);
      $regexp = "&lt;a\s[^&gt;]*href=(\"??)([^\" &gt;]*?)\\1[^&gt;]*&gt;(.*)&lt;\/a&gt;";
      preg_match_all("/$regexp/siU", $input, $matches);
      $base_url = parse_url($url, PHP_URL_HOST);
      $l = $matches[2];
      foreach($l as $link) {
          if(strpos($link, "#")) {
              $link = substr($link,0, strpos($link, "#"));
          }
          if(substr($link,0,1) == ".") {
              $link = substr($link, 1);
          }
          if(substr($link,0,7)=="http://") {
              $link = $link;
          }
          else if(substr($link,0,8) =="https://") {
              $link = $link;
          }
          else if(substr($link,0,2) =="//") {
              $link = substr($link, 2);
          }
          else if(substr($link,0,2) =="#") {
              $link = $url;
          }
          else if(substr($link,0,2) =="mailto:") {
              $link = "[".$link."]";
          }
          else {
              if(substr($link,0,1) != "/") {
              $link = $base_url."/".$link;
          }
          else {
              $link = $base_url.$link;
          }
          }
          if(substr($link, 0, 7)=="http://" &amp;&amp; substr($link, 0, 8)!="https://" &amp;&amp; substr($link, 0, 1)=="[") {
              if(substr($url, 0, 8) == "https://") {
                  $link = "https://".$link;
              }
              else {
                  $link = "http://".$link;
              }
          }
          //echo $link."&lt;br /&gt;";
          if(!in_array($link,$c)) {
              array_push($c,$link);
          }
      }
  }
  get_links($to_crawl);
  foreach ($c as $page) {
      get_links($page);
  }
  foreach ($c as $page) {
      $query = mysqli_query($conn,"INSERT INTO LINKS VALUES('','$page')");
      echo $page."&lt;br /&gt;";
  }
  ?&gt;
  </code></pre>