public function parseRobotsTxt(PHPCrawlerURLDescriptor $Url, $user_agent_string)
  {
    PHPCrawlerBenchmark::start("processing_robotstxt");
    
    // URL of robots-txt
    $RobotsTxtUrl = self::getRobotsTxtURL($Url);
    
    // Get robots.txt-content related to the given URL
    $robots_txt_content = $this->getRobotsTxtContent($RobotsTxtUrl);
    
    $non_follow_reg_exps = array();
    
    // If content was found
    if ($robots_txt_content != null)
    {
      // Get all lines in the robots.txt-content that are adressed to our user-agent.
      $applying_lines = $this->getApplyingLines($robots_txt_content, $user_agent_string);
      
      // Get valid reg-expressions for the given disallow-pathes.
      $non_follow_reg_exps = $this->buildRegExpressions($applying_lines, PHPCrawlerUtils::getRootUrl($Url->url_rebuild));
    }
    
    PHPCrawlerBenchmark::stop("processing_robots.txt");
    
    return $non_follow_reg_exps;
  }