public static function getRobotsTxtURL($url)
  {
    $url_parts = PHPCrawlerUtils::splitURL($url); 
    $robots_txt_url = $url_parts["protocol"].$url_parts["host"].":".$url_parts["port"] . "/robots.txt";
    
    return $robots_txt_url;
  }