public function parseRobotsTxt(PHPCrawlerURLDescriptor $Url, $user_agent_string)
  {
    PHPCrawlerBenchmark::start("processing_robotstxt");
    
    // URL of robots-txt
    $RobotsTxtUrl = self::getRobotsTxtURL($Url);
    
    // Get robots.txt-content related to the given URL
    $robots_txt_content = $this->getRobotsTxtContent($RobotsTxtUrl);
    
    $non_follow_reg_exps = array();
    
    // If content was found
    if ($robots_txt_content != null)
    {
      // Get all lines in the robots.txt-content that are adressed to our user-agent.
      $applying_lines = $this->getApplyingLines($robots_txt_content, $user_agent_string);
      
      // Get valid reg-expressions for the given disallow-pathes.
      $non_follow_reg_exps = $this->buildRegExpressions($applying_lines, PHPCrawlerUtils::getRootUrl($Url->url_rebuild));
    }
    
    PHPCrawlerBenchmark::stop("processing_robots.txt");
    
    return $non_follow_reg_exps;
  }
public function parseRobotsTxt(PHPCrawlerURLDescriptor $Url, $user_agent_string)
  {
    PHPCrawlerBenchmark::start("processing_robotstxt");
    
    // URL of robots-txt
    $RobotsTxtUrl = self::getRobotsTxtURL($Url);
    
    // Get robots.txt-content related to the given URL
    $robots_txt_content = $this->getRobotsTxtContent($RobotsTxtUrl);
    
    $non_follow_reg_exps = array();
    
    // If content was found
    if ($robots_txt_content != null)
    {
      // Get all lines in the robots.txt-content that are adressed to our user-agent.
      $applying_lines = $this->getApplyingLines($robots_txt_content, $user_agent_string);
      
      // Get valid reg-expressions for the given disallow-pathes.
      $non_follow_reg_exps = $this->buildRegExpressions($applying_lines, PHPCrawlerUtils::getRootUrl($Url->url_rebuild));
    }
    
    PHPCrawlerBenchmark::stop("processing_robots.txt");
    
    return $non_follow_reg_exps;
  }
public function parseRobotsTxt ( PHPCrawlerURLDescriptor $ Url , $ user_agent_string ) { PHPCrawlerBenchmark :: start ( "processing_robotstxt" ) ; // URL of robots-txt
 $ RobotsTxtUrl = self :: getRobotsTxtURL ( $ Url ) ; // Get robots.txt-content related to the given URL
 $ robots_txt_content = $ this -> getRobotsTxtContent ( $ RobotsTxtUrl ) ; $ non_follow_reg_exps = array ( ) ; // If content was found
 if ( $ robots_txt_content != null ) { // Get all lines in the robots.txt-content that are adressed to our user-agent.
 $ applying_lines = $ this -> getApplyingLines ( $ robots_txt_content , $ user_agent_string ) ; // Get valid reg-expressions for the given disallow-pathes.
 $ non_follow_reg_exps = $ this -> buildRegExpressions ( $ applying_lines , PHPCrawlerUtils :: getRootUrl ( $ Url -> url_rebuild ) ) ; } PHPCrawlerBenchmark :: stop ( "processing_robots.txt" ) ; return $ non_follow_reg_exps ; }
Parses the robots.txt-file related to the given URL and returns regular-expression-rules
corresponding to the containing "disallow"-rules that are adressed to the given user-agent.

@param PHPCrawlerURLDescriptor $Url The URL
@param string $user_agent_string User-agent.

@return array Numeric array containing regular-expressions for each "disallow"-rule defined in the robots.txt-file
that's adressed to the given user-agent.
Parses the robots . txt - file related to the given URL and returns regular - expression - rules corresponding to the containing disallow - rules that are adressed to the given user - agent .