protected function getRobotsTxtContent(PHPCrawlerURLDescriptor $Url)
  {
    // Request robots-txt
    $this->PageRequest->setUrl($Url);
    $PageInfo = $this->PageRequest->sendRequest();

    // Return content of the robots.txt-file if it was found, otherwie
    // reutrn NULL
    if ($PageInfo->http_status_code == 200)
    {
      return $PageInfo->content;
    }
    else
    {
      return null;
    }
  }