protected function buildRegExpressions(&$applying_lines, $base_url)
  { 
    // First, get all "Disallow:"-pathes
    $disallow_pathes = array();
    for ($x=0; $x<count($applying_lines); $x++)
    {
      if (preg_match("#^Disallow:# i", $applying_lines[$x]))
      {
        preg_match("#^Disallow:[ ]*(.*)#", $applying_lines[$x], $match);
        $disallow_pathes[] = trim($match[1]);
      }
    }
    
    // Works like this:
    // The base-url is http://www.foo.com.
    // The driective says: "Disallow: /bla/"
    // This means: The nonFollowMatch is "#^http://www\.foo\.com/bla/#"
    
    $normalized_base_url = PHPCrawlerUtils::normalizeURL($base_url);
    
    $non_follow_expressions = array();
    
    for ($x=0; $x<count($disallow_pathes); $x++)
    {
      // If the disallow-path is empty -> simply ignore it
      if ($disallow_pathes[$x] == "") continue;
      
      $non_follow_path_complpete = $normalized_base_url.substr($disallow_pathes[$x], 1); // "http://www.foo.com/bla/"
      $non_follow_exp = preg_quote($non_follow_path_complpete, "#"); // "http://www\.foo\.com/bla/"
      $non_follow_exp = "#^".$non_follow_exp."#"; // "#^http://www\.foo\.com/bla/#"
        
      $non_follow_expressions[] = $non_follow_exp;
    }
    
    return $non_follow_expressions;
  }
protected function buildRegExpressions(&$applying_lines, $base_url)
  { 
    // First, get all "Disallow:"-pathes
    $disallow_pathes = array();
    for ($x=0; $x<count($applying_lines); $x++)
    {
      if (preg_match("#^Disallow:# i", $applying_lines[$x]))
      {
        preg_match("#^Disallow:[ ]*(.*)#", $applying_lines[$x], $match);
        $disallow_pathes[] = trim($match[1]);
      }
    }
    
    // Works like this:
    // The base-url is http://www.foo.com.
    // The driective says: "Disallow: /bla/"
    // This means: The nonFollowMatch is "#^http://www\.foo\.com/bla/#"
    
    $normalized_base_url = PHPCrawlerUtils::normalizeURL($base_url);
    
    $non_follow_expressions = array();
    
    for ($x=0; $x<count($disallow_pathes); $x++)
    {
      // If the disallow-path is empty -> simply ignore it
      if ($disallow_pathes[$x] == "") continue;
      
      $non_follow_path_complpete = $normalized_base_url.substr($disallow_pathes[$x], 1); // "http://www.foo.com/bla/"
      $non_follow_exp = preg_quote($non_follow_path_complpete, "#"); // "http://www\.foo\.com/bla/"
      $non_follow_exp = "#^".$non_follow_exp."#"; // "#^http://www\.foo\.com/bla/#"
        
      $non_follow_expressions[] = $non_follow_exp;
    }
    
    return $non_follow_expressions;
  }
protected function buildRegExpressions ( & $ applying_lines , $ base_url ) { // First, get all "Disallow:"-pathes
 $ disallow_pathes = array ( ) ; for ( $ x = 0 ; $ x < count ( $ applying_lines ) ; $ x ++ ) { if ( preg_match ( "#^Disallow:# i" , $ applying_lines [ $ x ] ) ) { preg_match ( "#^Disallow:[ ]*(.*)#" , $ applying_lines [ $ x ] , $ match ) ; $ disallow_pathes [ ] = trim ( $ match [ 1 ] ) ; } } // Works like this:
 // The base-url is http://www.foo.com.
 // The driective says: "Disallow: /bla/"
 // This means: The nonFollowMatch is "#^http://www\.foo\.com/bla/#"
 $ normalized_base_url = PHPCrawlerUtils :: normalizeURL ( $ base_url ) ; $ non_follow_expressions = array ( ) ; for ( $ x = 0 ; $ x < count ( $ disallow_pathes ) ; $ x ++ ) { // If the disallow-path is empty -> simply ignore it
 if ( $ disallow_pathes [ $ x ] == "" ) continue ; $ non_follow_path_complpete = $ normalized_base_url . substr ( $ disallow_pathes [ $ x ] , 1 ) ; // "http://www.foo.com/bla/"
 $ non_follow_exp = preg_quote ( $ non_follow_path_complpete , "#" ) ; // "http://www\.foo\.com/bla/"
 $ non_follow_exp = "#^" . $ non_follow_exp . "#" ; // "#^http://www\.foo\.com/bla/#"
 $ non_follow_expressions [ ] = $ non_follow_exp ; } return $ non_follow_expressions ; }
Returns an array containig regular-expressions corresponding
to the given robots.txt-style "Disallow"-lines

@param array &$applying_lines Numeric array containing "disallow"-lines.
@param string $base_url       Base-URL the robots.txt-file was found in.

@return array  Numeric array containing regular-expresseions created for each "disallow"-line.
Returns an array containig regular - expressions corresponding to the given robots . txt - style Disallow - lines