protected function getRobotsTxtContent(PHPCrawlerURLDescriptor $Url)
  {
    // Request robots-txt
    $this->PageRequest->setUrl($Url);
    $PageInfo = $this->PageRequest->sendRequest();

    // Return content of the robots.txt-file if it was found, otherwie
    // reutrn NULL
    if ($PageInfo->http_status_code == 200)
    {
      return $PageInfo->content;
    }
    else
    {
      return null;
    }
  }
protected function getRobotsTxtContent(PHPCrawlerURLDescriptor $Url)
  {
    // Request robots-txt
    $this->PageRequest->setUrl($Url);
    $PageInfo = $this->PageRequest->sendRequest();

    // Return content of the robots.txt-file if it was found, otherwie
    // reutrn NULL
    if ($PageInfo->http_status_code == 200)
    {
      return $PageInfo->content;
    }
    else
    {
      return null;
    }
  }
protected function getRobotsTxtContent ( PHPCrawlerURLDescriptor $ Url ) { // Request robots-txt
 $ this -> PageRequest -> setUrl ( $ Url ) ; $ PageInfo = $ this -> PageRequest -> sendRequest ( ) ; // Return content of the robots.txt-file if it was found, otherwie
 // reutrn NULL
 if ( $ PageInfo -> http_status_code == 200 ) { return $ PageInfo -> content ; } else { return null ; } }
Retreives the content of a robots.txt-file

@param PHPCrawlerURLDescriptor $Url The URL of the robots.txt-file
@return string The content of the robots.txt or NULL if no robots.txt was found.
Retreives the content of a robots . txt - file