src/SitemapParser.php
protected function parseRobotstxt($robotstxt)
    {
        // Split lines into array
        $lines = array_filter(array_map('trim', mb_split('\r\n|\n|\r', $robotstxt)));
        // Parse each line individually
        foreach ($lines as $line) {
            // Remove comments
            $line = mb_split('#', $line, 2)[0];
            // Split by directive and rule
            $pair = array_map('trim', mb_split(':', $line, 2));
            // Check if the line contains a sitemap
            if (
                mb_strtolower($pair[0]) !== self::XML_TAG_SITEMAP ||
                empty($pair[1])
            ) {
                // Line does not contain any supported directive
                continue;
            }
            $url = $this->urlEncode($pair[1]);
            if ($this->urlValidate($url)) {
                $this->addArray(self::XML_TAG_SITEMAP, ['loc' => $url]);
            }
        }
        return true;
    }
protected function parseRobotstxt($robotstxt)
    {
        // Split lines into array
        $lines = array_filter(array_map('trim', mb_split('\r\n|\n|\r', $robotstxt)));
        // Parse each line individually
        foreach ($lines as $line) {
            // Remove comments
            $line = mb_split('#', $line, 2)[0];
            // Split by directive and rule
            $pair = array_map('trim', mb_split(':', $line, 2));
            // Check if the line contains a sitemap
            if (
                mb_strtolower($pair[0]) !== self::XML_TAG_SITEMAP ||
                empty($pair[1])
            ) {
                // Line does not contain any supported directive
                continue;
            }
            $url = $this->urlEncode($pair[1]);
            if ($this->urlValidate($url)) {
                $this->addArray(self::XML_TAG_SITEMAP, ['loc' => $url]);
            }
        }
        return true;
    }
protected function parseRobotstxt ( $ robotstxt ) { // Split lines into array $ lines = array_filter ( array_map ( 'trim' , mb_split ( '\r\n|\n|\r' , $ robotstxt ) ) ) ; // Parse each line individually foreach ( $ lines as $ line ) { // Remove comments $ line = mb_split ( '#' , $ line , 2 ) [ 0 ] ; // Split by directive and rule $ pair = array_map ( 'trim' , mb_split ( ':' , $ line , 2 ) ) ; // Check if the line contains a sitemap if ( mb_strtolower ( $ pair [ 0 ] ) !== self :: XML_TAG_SITEMAP || empty ( $ pair [ 1 ] ) ) { // Line does not contain any supported directive continue ; } $ url = $ this -> urlEncode ( $ pair [ 1 ] ) ; if ( $ this -> urlValidate ( $ url ) ) { $ this -> addArray ( self :: XML_TAG_SITEMAP , [ 'loc' => $ url ] ) ; } } return true ; }
Search for sitemaps in the robots.txt content

@param string $robotstxt
@return bool
Search for sitemaps in the robots . txt content