<repo_name>PiedWeb/UrlHarvester<file_sep>/tests/HarvestTest.php
<?php

declare(strict_types=1);

namespace PiedWeb\UrlHarvester\Test;

use PiedWeb\Curl\ResponseFromCache;
use PiedWeb\UrlHarvester\Harvest;
use PiedWeb\UrlHarvester\Indexable;
use PiedWeb\UrlHarvester\Link;

class HarvestTest extends \PHPUnit\Framework\TestCase
{
    private static $harvest;

    private function getUrl()
    {
        return 'https://piedweb.com/seo/crawler';
    }

    private function getHarvest()
    {
        if (null === self::$harvest) {
            self::$harvest = Harvest::fromUrl(
                $this->getUrl(),
                'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0'
            );
        }

        return self::$harvest;
    }

    private function getHarvestFromCache()
    {
        $response = new ResponseFromCache(
            'HTTP/1.1 200 OK'.\PHP_EOL.\PHP_EOL.file_get_contents(__DIR__.'/page.html'),
            $this->getUrl(),
            ['content_type' => 'text/html; charset=UTF-8']
        );

        $harvest = new Harvest($response);

        return $harvest;
    }

    public function testHarvestWordCount()
    {
        $harvest = $this->getHarvestFromCache();

        $this->assertTrue($harvest->getWordCount() > 50);
    }

    public function testHarvestLinksForReal()
    {
        $harvest = $this->getHarvestFromCache();
        $url = $this->getUrl();

        $this->assertSame(2, \count($harvest->getLinks(Link::LINK_SELF)));
        $this->assertTrue(4 == \count($harvest->getLinks(Link::LINK_INTERNAL)));
        $this->assertTrue(3 == \count($harvest->getLinks(Link::LINK_EXTERNAL)));
        $this->assertTrue(2 == \count($harvest->getLinks(Link::LINK_SUB)));
        $this->assertSame(11, \count($harvest->getLinks()));
    }

    public function testHarvest()
    {
        $harvest = $this->getHarvest();
        $url = $this->getUrl();

        // Just check Curl  is doing is job
        $this->assertTrue($harvest->getResponse()->getInfo('total_time') > 0.00000001);
        $this->assertTrue(\strlen($harvest->getTag('h1')) > 2);
        $this->assertTrue(\strlen($harvest->getMeta('description')) > 2);
        $this->assertTrue('https://piedweb.com/seo/crawler' == $harvest->getCanonical());
        $this->assertTrue($harvest->isCanonicalCorrect());
        $this->assertTrue($harvest->getRatioTxtCode() > 2);
        $this->assertTrue(\is_array($harvest->getKws()));

        $this->assertTrue(\strlen($harvest->getUniqueTag('head title')) > 10);

        $this->assertTrue(null === $harvest->getUniqueTag('h12'));

        $this->assertTrue(\is_array($harvest->getBreadCrumb()));

        $this->assertSame('piedweb.com', $harvest->url()->getRegistrableDomain());
        $this->assertSame('https://piedweb.com/seo/crawler', $harvest->getBaseUrl());
    }

    public function testHarvestLinks()
    {
        $harvest = $this->getHarvest();
        $url = $this->getUrl();

        $this->assertTrue(\is_array($harvest->getLinkedRessources()));
        $this->assertTrue(\is_array($harvest->getLinks()));
        $this->assertTrue(\is_array($harvest->getLinks(Link::LINK_SELF)));
        $this->assertTrue(\is_array($harvest->getLinks(Link::LINK_INTERNAL)));
        $this->assertTrue(\is_array($harvest->getLinks(Link::LINK_SUB)));
        $this->assertTrue(\is_array($harvest->getLinks(Link::LINK_EXTERNAL)));
        $this->assertTrue(\is_int($harvest->getNbrDuplicateLinks()));
    }

    public function testFollow()
    {
        $harvest = $this->getHarvest();

        $this->assertTrue($harvest->getLinks()[0]->mayFollow());
        $this->assertTrue($harvest->mayFollow());
    }

    public function testRedirection()
    {
        $url = 'https://www.piedweb.com/';
        $harvest = Harvest::fromUrl(
            $url,
            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0'
        );

        $this->assertSame('https://piedweb.com/', $harvest->getRedirection());
        $this->assertSame(Indexable::NOT_INDEXABLE_3XX, $harvest->indexable());
    }

    public function testIndexable()
    {
        $indexable = new Indexable($this->getHarvest());
        $this->assertTrue($indexable->metaAllows());
        $this->assertTrue($indexable->headersAllow());
        $this->assertSame(Indexable::INDEXABLE, $this->getHarvest()->indexable());

        $url = 'https://dev.piedweb.com/disallow';
        $harvest = Harvest::fromUrl(
            $url,
            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0'
        );

        $harvest->setRobotsTxt('User-agent: *'."\n".'Disallow: /disallow');
        $this->assertSame(Indexable::NOT_INDEXABLE_ROBOTS, $harvest->indexable());

        $harvest2 = new Harvest(new ResponseFromCache(
            'HTTP/1.1 200 OK'.\PHP_EOL.'X-robots-tag: noindex'.\PHP_EOL.\PHP_EOL.'<!DOCTYPE html><html>'
            .'<head></head><body><p>Tests</p></body>',
            'https://piedweb.com/noindex-headers',
            ['content_type' => 'text/html; charset=UTF-8']
        ));
        $harvest2->setRobotsTxt($harvest->getRobotsTxt());
        $this->assertSame(Indexable::NOT_INDEXABLE_HEADER, $harvest2->indexable());

        $harvest2 = new Harvest(new ResponseFromCache(
            'HTTP/1.1 200 OK'.\PHP_EOL.\PHP_EOL.'<!DOCTYPE html><html>'
            .'<head><meta name="robots" content="noindex"></head><body><p>Tests</p></body>',
            'https://piedweb.com/',
            ['content_type' => 'text/html; charset=UTF-8']
        ));
        $harvest2->setRobotsTxt($harvest->getRobotsTxt());
        $this->assertSame(Indexable::NOT_INDEXABLE_META, $harvest2->indexable());

        $harvest2 = new Harvest(new ResponseFromCache(
            'HTTP/1.1 200 OK'.\PHP_EOL.\PHP_EOL.'<!DOCTYPE html><html>'
            .'<head><link rel="canonical" href="https://piedweb.com/seo" /></head><body><p>Tests</p></body>',
            'https://piedweb.com/',
            ['content_type' => 'text/html; charset=UTF-8']
        ));
        $harvest2->setRobotsTxt($harvest->getRobotsTxt());
        $this->assertSame(Indexable::NOT_INDEXABLE_CANONICAL, $harvest2->indexable());

        $harvest2 = new Harvest(new ResponseFromCache(
            'HTTP/1.1 404 OK'.\PHP_EOL.\PHP_EOL.'<!DOCTYPE html><html>'
            .'<head></head><body><p>Tests</p></body>',
            'https://piedweb.com/',
            ['content_type' => 'text/html; charset=UTF-8']
        ));
        $harvest2->setRobotsTxt($harvest->getRobotsTxt());
        $this->assertSame(Indexable::NOT_INDEXABLE_4XX, $harvest2->indexable());

        $harvest2 = new Harvest(new ResponseFromCache(
            'HTTP/1.1 510 OK'.\PHP_EOL.\PHP_EOL.'<!DOCTYPE html><html>'
            .'<head></head><body><p>Tests</p></body>',
            'https://piedweb.com/',
            ['content_type' => 'text/html; charset=UTF-8']
        ));
        $harvest2->setRobotsTxt($harvest->getRobotsTxt());
        $this->assertSame(Indexable::NOT_INDEXABLE_5XX, $harvest2->indexable());

        $harvest2 = new Harvest(new ResponseFromCache(
            'HTTP/1.1 301 Moved Permanently'.\PHP_EOL.\PHP_EOL.'<!DOCTYPE html><html>'
            .'<head></head><body><p>Tests</p></body>',
            'https://piedweb.com/',
            ['content_type' => 'text/html; charset=UTF-8']
        ));
        $harvest2->setRobotsTxt($harvest->getRobotsTxt());
        $this->assertSame(Indexable::NOT_INDEXABLE_3XX, $harvest2->indexable());
    }

    public function testHarvestWithPreviousRequest()
    {
        $harvest = $this->getHarvest();

        $newHarvest = Harvest::fromUrl(
            $this->getUrl(),
            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0',
            'fr',
            $harvest->getResponse()->getRequest()
        );

        $this->assertTrue(\strlen($harvest->getUniqueTag('head title')) > 10);
    }

    public function testHarvestFromCache()
    {
        $harvest = new Harvest(new ResponseFromCache(
            'HTTP/1.1 200 OK'.\PHP_EOL.\PHP_EOL.'<!DOCTYPE html><html><body><p>Tests</p></body>',
            'https://piedweb.com/',
            ['content_type' => 'text/html; charset=UTF-8']
        ));

        $this->assertTrue($harvest->isIndexable());
    }

    public function testTextAnalysis()
    {
        $this->assertTrue(\count($this->getHarvest()->getTextAnalysis()->getExpressionsByDensity()) > 1);
    }

    public function testCanonical()
    {
        $harvest = Harvest::fromUrl(
            'https://piedweb.com',
            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0'
        );

        $this->assertTrue($harvest->isCanonicalCorrect('https://piedweb.com'));
        $this->assertTrue($harvest->isCanonicalCorrect('https://piedweb.com/'));
        $this->assertFalse($harvest->isCanonicalCorrect('https://piedweb.com//'));
    }

    public function testHarvestTripAdvisor()
    {
        $url = 'https://www.tripadvisor.fr/Attraction_Review-g196719-d21232770-Reviews-Accompagnateur_Pied_Vert_Rando_Vercors-Villard_de_Lans_Isere_Auvergne_Rhone_Alpe.html';
        $harvest = Harvest::fromUrl(
            $url,
            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0'
        );

        $this->assertTrue(!is_int($harvest->getResponse()));
    }
}
<file_sep>/src/ExtractLinks.php
<?php

namespace PiedWeb\UrlHarvester;

class ExtractLinks
{
    public const SELECT_A = 'a[href]';

    public const SELECT_ALL = '[href],[src]';

    /** @var Harvest */
    private $harvest;

    /** @var string */
    private $selector;

    public static function get(Harvest $harvest, $selector = self::SELECT_A): array
    {
        $self = new self();

        $self->selector = $selector;
        $self->harvest = $harvest;

        return $self->extractLinks();
    }

    private function __construct()
    {
    }

    /**
     * @return array
     */
    private function extractLinks()
    {
        $links = [];
        $elements = $this->harvest->getDom()->filter($this->selector); // what happen if find nothing

        foreach ($elements as $element) {
            //var_dump(get_class_methods($element->getNode()));
            //if (!$element instanceof \DomElement) { continue; } // wtf ?
            $url = $this->extractUrl($element);
            //$type = $element->getAttribute('href') ? Link::LINK_A : Link::LINK_SRC;
            if (null !== $url) {
                //$links[] = (new Link($url, $element, $type))->setParent($this->parentUrl);
                $links[] = (new Link($url, $this->harvest, $element));
            }
        }

        return $links;
    }

    /**
     * @return string|null absolute url
     */
    private function extractUrl(\DomElement $element): ?string
    {
        $attributes = explode(',', str_replace(['a[', '*[', '[', ']'], '', $this->selector));
        foreach ($attributes as $attribute) {
            $url = $element->getAttribute($attribute);
            if ($url) {
                break;
            }
        }

        if (! $url || ! $this->isWebLink($url)) {
            return null;
        }

        return $this->harvest->url()->resolve($url);
    }

    public static function isWebLink(string $url)
    {
        return preg_match('@^((?:(http:|https:)//([\w\d-]+\.)+[\w\d-]+){0,1}(/?[\w~,;\-\./?%&+#=]*))$@', $url);
    }
}
<file_sep>/src/Request.php
<?php

namespace PiedWeb\UrlHarvester;

use PiedWeb\Curl\Request as CurlRequest;
use PiedWeb\Curl\Response;

/**
 * Request a page and get it only if it's an html page.
 */
class Request
{
    private string $url;

    private string $userAgent;

    private string $language;

    private ?string $proxy;

    public int $maxSize = 1000000;

    /**
     * @param bool $tryHttps
     *
     * @return Response|int corresponding to the curl error
     */
    public static function make(
        string $url,
        string $userAgent,
        string $language = 'en,en-US;q=0.5',
        ?string $proxy = null,
        int $documentMaxSize = 1000000
    ) {
        return self::makeFromRequest(null, $url, $userAgent, $language, $proxy, $documentMaxSize);
    }

    public static function makeFromRequest(
        ?CurlRequest $curlRequest = null,
        string $url,
        string $userAgent,
        string $language = 'en,en-US;q=0.5',
        ?string $proxy = null,
        int $documentMaxSize = 1000000
    ) {
        $request = new self($url);

        $request->userAgent = $userAgent;
        $request->language = $language;
        $request->proxy = $proxy;
        $request->maxSize = $documentMaxSize;

        return $request->request($curlRequest);
    }

    private function __construct($url)
    {
        /*
        if (!filter_var($string, FILTER_VALIDATE_URL)) {
            throw new \Exception('URL invalid: '.$string);
        }**/
        $this->url = $url;
    }

    /**
     * Prepare headers as a normal browser (same order, same content).
     */
    private function prepareHeadersForRequest(): array
    {
        //$host = parse_url($this->url, PHP_URL_HOST);

        $headers = [];
        $headers[] = 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8';
        $headers[] = 'Accept-Encoding: gzip, deflate';
        $headers[] = 'Accept-Language: '.$this->language;
        $headers[] = 'Connection: keep-alive';

        //if ($host) {
        //$headers[] =  'Host: '.$host;
        //}
        // Referer

        $headers[] = 'Upgrade-Insecure-Requests: 1';
        $headers[] = 'User-Agent: '.$this->userAgent;

        return $headers;
    }

    /**
     * @return Response|int corresponding to the curl error
     */
    private function request(?CurlRequest $request = null)
    {
        $request = null !== $request ? $request : new CurlRequest();
        $request
            ->setUrl($this->url)
            ->setReturnHeader()
            ->setEncodingGzip()
            ->setUserAgent($this->userAgent)
            ->setDefaultSpeedOptions()
            ->setOpt(\CURLOPT_SSL_VERIFYHOST, 0)
            ->setOpt(\CURLOPT_SSL_VERIFYPEER, 0)
            ->setOpt(\CURLOPT_MAXREDIRS, 0)
            ->setOpt(\CURLOPT_FOLLOWLOCATION, false)
            ->setOpt(\CURLOPT_COOKIE, false)
            ->setOpt(\CURLOPT_CONNECTTIMEOUT, 20)
            ->setOpt(\CURLOPT_TIMEOUT, 80)
            ->setAbortIfTooBig($this->maxSize); // 2Mo

        if ($this->proxy) {
            $request->setProxy($this->proxy);
        }

        $request->setOpt(\CURLOPT_HTTPHEADER, $this->prepareHeadersForRequest());

        $response = $request->exec();
        //dd($this->request->exec());

        return $response;
    }
}
<file_sep>/tests/LinkTest.php
<?php

declare(strict_types=1);

namespace PiedWeb\UrlHarvester\Test;

use PiedWeb\UrlHarvester\Harvest;
use PiedWeb\UrlHarvester\Link;
use PiedWeb\UrlHarvester\Url;
use Symfony\Component\DomCrawler\Crawler as DomCrawler;

class LinkTest extends \PHPUnit\Framework\TestCase
{
    private static $harvest;

    private function getUrl()
    {
        return 'https://piedweb.com/';
    }

    private function getHarvest()
    {
        if (null === self::$harvest) {
            self::$harvest = Harvest::fromUrl(
                $this->getUrl(),
                'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0'
            );
        }

        return self::$harvest;
    }

    public function testLinkNormalization()
    {
        $link = new Link('https://www.piedweb.com', $this->getHarvest());

        $this->assertSame($link->getPageUrl(), 'https://www.piedweb.com/');
        $this->assertSame($link->getUrl()->getRelativizedDocumentUrl(), '/');
    }

    private function getDomElement()
    {
        $html = '<a href="/test" rel=nofollow>test</a>';
        $dom = new DomCrawler($html);

        return $dom->filter('a')->getNode(0);
    }

    public function testNofollow()
    {
        $link = new Link('https://piedweb.com/test', $this->getHarvest(), $this->getDomElement());

        $this->assertTrue(! $link->mayFollow());
    }

    public function testAnchor()
    {
        $link = new Link('https://piedweb.com', $this->getHarvest(), $this->getDomElement());

        $this->assertSame($link->getAnchor(), 'test');
    }

    public function testShortcutsForRedir()
    {
        $link = Link::createRedirection('https://piedweb.com', $this->getHarvest());
        $this->assertTrue($link->mayFollow());
        $this->assertSame($link->getAnchor(), null);
    }

    public function testLinkType()
    {
        $link = new Link('https://external.com/test', $this->getHarvest(), $this->getDomElement());

        $this->assertTrue(! $link->isInternalLink());
        $this->assertTrue(! $link->isSubLink());
        $this->assertTrue(! $link->isSelfLink());
    }

    public function testUrl()
    {
        $url = new Url($this->getUrl());

        $this->assertTrue('https://piedweb.com' == $url->resolve('//piedweb.com'));
    }

    public function testRelativize()
    {
        $url = new Url($this->getUrl());

        $this->assertSame($url->relativize(), '/');
    }
}
<file_sep>/tests/RequestTest.php
<?php

declare(strict_types=1);

namespace PiedWeb\UrlHarvester\Test;

use PiedWeb\UrlHarvester\Request;

class RequestTest extends \PHPUnit\Framework\TestCase
{
    public function testFailedRequest()
    {
        $request = Request::make('https://dzejnd'.rand(10000, 9999999999).'.biz', 'Hello :)');

        $this->assertSame(6, $request);
    }

    public function testRequest()
    {
        $request = Request::make(
            'https://piedweb.com/',
            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0'
        );

        $this->assertTrue(\strlen($request->getContent()) > 10);
        $this->assertTrue(! empty($request->getHeaders()));
    }

    public function testRequestRedir()
    {
        $request = Request::make(
            'https://www.piedweb.com/',
            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0'
        );

        $this->assertSame(
            $request->getHeaders()['location'] ?? $request->getHeaders()['Location'],
            'https://piedweb.com/'
        );
    }
}
<file_sep>/src/Url.php
<?php

/**
 * Wrapper for League\Uri.
 *
 * Permits to cache registrableDomain and Origin
 */

namespace PiedWeb\UrlHarvester;

use League\Uri\Http;
use League\Uri\UriInfo;
use League\Uri\UriResolver;

class Url
{
    protected $http;

    protected $origin;

    protected $registrableDomain;

    public function __construct(string $url)
    {
        $this->http = Http::createFromString($url);

        if (! UriInfo::isAbsolute($this->http)) {
            throw new \Exception('$url must be absolute (`'.$url.'`)');
        }
    }

    public function resolve($url): string
    {
        $resolved = UriResolver::resolve(Http::createFromString($url), $this->http);

        return $resolved->__toString();
    }

    public function getHttp()
    {
        return $this->http;
    }

    public function getScheme()
    {
        return $this->http->getScheme();
    }

    public function getHost()
    {
        return $this->http->getHost();
    }

    public function getOrigin()
    {
        $this->origin = $this->origin ?? $this->origin = UriInfo::getOrigin($this->http);

        return $this->origin;
    }

    public function getRegistrableDomain()
    {
        return $this->registrableDomain
            ?? ($this->registrableDomain = Domain::getRegistrableDomain($this->http->getHost()));
    }

    public function getDocumentUrl(): string
    {
        return $this->http->withFragment('');
    }

    public function getRelativizedDocumentUrl(): string
    {
        return substr($this->http->withFragment(''), \strlen($this->getOrigin()));
    }

    public function get(): string
    {
        return $this->__toString();
    }

    public function __toString(): string
    {
        return (string) $this->http;
    }

    public function relativize()
    {
        return substr($this->get(), \strlen($this->getOrigin()));
    }
}
<file_sep>/tests/ExtractorTest.php
<?php

declare(strict_types=1);

namespace PiedWeb\UrlHarvester\Test;

use PiedWeb\UrlHarvester\ExtractBreadcrumb;
use PiedWeb\UrlHarvester\ExtractLinks;
use PiedWeb\UrlHarvester\Harvest;
use PiedWeb\UrlHarvester\Helper;

class ExtractorTest extends \PHPUnit\Framework\TestCase
{
    private static $dom;

    private static $harvest;

    private function getUrl()
    {
        return 'https://piedweb.com/seo/crawler';
    }

    private function getHarvest()
    {
        if (null === self::$harvest) {
            self::$harvest = Harvest::fromUrl(
                $this->getUrl(),
                'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0'
            );
        }

        return self::$harvest;
    }

    public function testExtractLinks()
    {
        $links = ExtractLinks::get($this->getHarvest());

        foreach ($links as $link) {
            $this->assertTrue(\strlen($link->getUrl()->get()) > 10);
            $this->assertTrue(\strlen($link->getAnchor()) > 1);
            $this->assertTrue('' !== $link->getElement()->getAttribute('href'));
        }

        $links = ExtractLinks::get($this->getHarvest(), ExtractLinks::SELECT_ALL);
        foreach ($links as $link) {
            $this->assertTrue(\strlen($link->getUrl()->get()) > 10);
        }
    }

    public function testExtractBreadcrumb()
    {
        $bcItems = ExtractBreadcrumb::get($this->getHarvest());

        foreach ($bcItems as $item) {
            $this->assertTrue(\strlen($item->getUrl()) > 10);
            $this->assertTrue(\strlen($item->getName()) > 1);
            $this->assertTrue(\strlen($item->getCleanName()) > 1);
        }
    }

    public function testHelperClean()
    {
        $this->assertSame('Hello Toi', Helper::clean('Hello  Toi '));
    }
}
<file_sep>/src/HarvestLinksTrait.php
<?php

namespace PiedWeb\UrlHarvester;

trait HarvestLinksTrait
{
    /**
     * @var array
     */
    protected $links;

    protected $linksPerType;

    abstract public function getDom();

    public function getLinkedRessources()
    {
        return ExtractLinks::get($this, ExtractLinks::SELECT_ALL);
    }

    public function getLinks($type = null): array
    {
        if (null === $this->links) {
            $this->links = ExtractLinks::get($this, ExtractLinks::SELECT_A);
            $this->classifyLinks();
        }

        switch ($type) {
            case Link::LINK_SELF:
                return $this->linksPerType[Link::LINK_SELF] ?? [];
            case Link::LINK_INTERNAL:
                return $this->linksPerType[Link::LINK_INTERNAL] ?? [];
            case Link::LINK_SUB:
                return $this->linksPerType[Link::LINK_SUB] ?? [];
            case Link::LINK_EXTERNAL:
                return $this->linksPerType[Link::LINK_EXTERNAL] ?? [];
            default:
                return $this->links;
        }
    }

    /**
     * Return duplicate links
     * /test and /test#2 are not duplicates.
     */
    public function getNbrDuplicateLinks(): int
    {
        $links = $this->getLinks();
        $u = [];
        foreach ($links as $link) {
            $u[(string) $link->getUrl()] = 1;
        }

        return \count($links) - \count($u);
    }

    public function classifyLinks()
    {
        $links = $this->getLinks();

        foreach ($links as $link) {
            $this->linksPerType[$link->getType()][] = $link;
        }
    }
}
<file_sep>/src/Helper.php
<?php

namespace PiedWeb\UrlHarvester;

use ForceUTF8\Encoding;

class Helper
{
    public static function clean(string $source)
    {
        return trim(preg_replace('/\s{2,}/', ' ', Encoding::toUTF8($source)));
    }

    public static function htmlToPlainText($str, $keepN = false)
    {
        $str = preg_replace('#<(style|script).*</(style|script)>#siU', ' ', $str);
        $str = preg_replace('#</?(br|p|div)>#siU', "\n", $str);
        $str = preg_replace('/<\/[a-z]+>/siU', ' ', $str);
        $str = str_replace(["\r", "\t"], ' ', $str);
        $str = strip_tags(preg_replace('/<[^<]+?>/', ' ', $str));
        if ($keepN) {
            $str = preg_replace('/ {2,}/', ' ', $str);
        } else {
            $str = preg_replace('/\s+/', ' ', $str);
        }

        return $str;
    }
}
<file_sep>/README.md
<p align="center"><a href="https://dev.piedweb.com">
<img src="https://raw.githubusercontent.com/PiedWeb/piedweb-devoluix-theme/master/src/img/logo_title.png" width="200" height="200" alt="Open Source Package" />
</a></p>

# Url Meta Data Harvester

[![Latest Version](https://img.shields.io/github/tag/PiedWeb/UrlHarvester.svg?style=flat&label=release)](https://github.com/PiedWeb/UrlHarvester/tags)
[![Software License](https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat)](LICENSE)
[![GitHub Tests Action Status](https://img.shields.io/github/workflow/status/PiedWeb/UrlHarvester/Tests?label=tests)](https://github.com/PiedWeb/UrlHarvester/actions)
[![Quality Score](https://img.shields.io/scrutinizer/g/PiedWeb/UrlHarvester.svg?style=flat)](https://scrutinizer-ci.com/g/PiedWeb/UrlHarvester)
[![Code Coverage](https://codecov.io/gh/PiedWeb/UrlHarvester/branch/main/graph/badge.svg)](https://codecov.io/gh/PiedWeb/UrlHarvester/branch/main)
[![Type Coverage](https://shepherd.dev/github/PiedWeb/UrlHarvester/coverage.svg)](https://shepherd.dev/github/PiedWeb/UrlHarvester)
[![Total Downloads](https://img.shields.io/packagist/dt/piedweb/url-harvester.svg?style=flat)](https://packagist.org/packages/piedweb/url-harvester)

Harvest statistics and meta data from an URL or his source code (seo oriented).

Implemented in [Seo Pocket Crawler](https://piedweb.com/seo/crawler) ([source on github](https://github.com/PiedWeb/SeoPocketCrawler)).

## Install

Via [Packagist](https://img.shields.io/packagist/dt/piedweb/url-harvester.svg?style=flat)

```bash
$ composer require piedweb/url-harvester
```

## Usage

Harvest Methods :

```php
use \PiedWeb\UrlHarvester\Harvest;
use \PiedWeb\UrlHarvester\Link;

$url = 'https://piedweb.com';

Harvest::fromUrl($url)
    ->getResponse()->getInfo('total_time') // load time
    ->getResponse()->getInfo('size_download')
    ->getResponse()->getStatusCode()
    ->getResponse()->getContentType()
    ->getRes...

    ->getTag('h1') // @return first tag content (could be html)
    ->getUniqueTag('h1') // @return first tag content in utf8 (could contain html)
    ->getMeta('description') // @return string from content attribute or NULL
    ->getCanonical() // @return string|NULL
    ->isCanonicalCorrect() // @return bool
    ->getRatioTxtCode() // @return int
    ->getTextAnalysis() // @return \PiedWeb\TextAnalyzer\Analysis
    ->getKws() // @return 10 more used words
    ->getBreadCrumb()
    ->indexable($userAgent = 'googlebot') // @return int corresponding to a const from Indexable

    ->getLinks()
    ->getLinks(Link::LINK_SELF)
    ->getLinks(Link::LINK_INTERNAL)
    ->getLinks(Link::LINK_SUB)
    ->getLinks(Link::LINK_EXTERNAL)
    ->getLinkedRessources() // Return an array with all attributes containing a href or a src property
    ->mayFollow() // check headers and meta and return bool

    ->getDomain()
    ->getBaseUrl()

    ->getRobotsTxt() // @return \Spatie\Robots\RobotsTxt or empty string
    ->setRobotsTxt($content) // @param string or RobotsTxt
```

## Testing

```bash
$ composer test
```

## Contributing

Please see [contributing](https://dev.piedweb.com/contributing)

## Credits

- [Pied Web](https://piedweb.com)
- [All Contributors](https://github.com/PiedWeb/:package_skake/graphs/contributors)

## License

The MIT License (MIT). Please see [License File](LICENSE) for more information.
<file_sep>/src/RobotsTxtTrait.php
<?php

namespace PiedWeb\UrlHarvester;

use PiedWeb\Curl\Request as CurlRequest;
use Spatie\Robots\RobotsTxt;

trait RobotsTxtTrait
{
    /** @var RobotsTxt|string (empty string) */
    protected $robotsTxt;

    //abstract public function getDomainAndScheme();

    abstract public function getResponse();

    abstract public function url();

    /**
     * @return RobotsTxt|string containing the current Robots.txt or NULL if an error occured
     *                          or empty string if robots is empty file
     */
    public function getRobotsTxt()
    {
        if (null === $this->robotsTxt) {
            $url = $this->url()->getOrigin().'/robots.txt';

            $request = $this->getResponse()->getRequest();
            $userAgent = $request ? $request->getUserAgent() : Harvest::DEFAULT_USER_AGENT;

            $request = new CurlRequest($url);
            $request
                ->setDefaultSpeedOptions()
                ->setDownloadOnly('0-500000')
                ->setUserAgent($userAgent)
            ;
            $result = $request->exec();

            if (! $result instanceof \PiedWeb\Curl\Response
                || false === stripos($result->getContentType(), 'text/plain')
                || empty(trim($result->getContent()))
            ) {
                $this->robotsTxt = '';
            } else {
                $this->robotsTxt = new RobotsTxt($result->getContent());
            }
        }

        return $this->robotsTxt;
    }

    /**
     * @param RobotsTxt|string $robotsTxt
     *
     * @return self
     */
    public function setRobotsTxt($robotsTxt)
    {
        $this->robotsTxt = \is_string($robotsTxt) ? (empty($robotsTxt) ? '' : new RobotsTxt($robotsTxt)) : $robotsTxt;

        return $this;
    }
}
<file_sep>/test.php
<?php

header('Location: mailto:tagada');
<file_sep>/src/Domain.php
<?php

/**
 * Entity.
 */

namespace PiedWeb\UrlHarvester;

use Pdp\Cache;
use Pdp\CurlHttpClient;
use Pdp\Manager;
use Pdp\Rules;

class Domain
{
    protected static $rules;

    public static function resolve(string $host): \Pdp\ResolvedDomainName
    {
        return self::getRules()->resolve($host);
    }

    public static function getRegistrableDomain(string $host): ?string
    {
        return self::resolve($host)->registrableDomain()->toString();
    }

    protected static function getRules()
    {
        if (null !== self::$rules) {
            return self::$rules;
        }

        $reflector = new \ReflectionClass("Pdp\Rules");
        $base = \dirname($reflector->getFileName(), 2);

        return self::$rules = Rules::fromPath($base.'/test_data/public_suffix_list.dat');

        //return self::$rules = (new Manager(new Cache(), new CurlHttpClient()))->getRules();
    }
}
<file_sep>/src/Link.php
<?php

/**
 * Entity.
 */

namespace PiedWeb\UrlHarvester;

use Symfony\Component\DomCrawler\Crawler as DomCrawler;

class Link
{
    /** @var Url */
    protected $url;

    /** @var string cache */
    protected $anchor;

    /** @var \DomElement */
    protected $element;

    /** @var Harvest */
    protected $parentDoc;

    /** @var int */
    protected $wrapper;

    /** @var int */
    protected $type;

    // Ce serait dans une liste, dans une phrase...
    protected $context;

    // wrapper related
    public const LINK_A = 1;

    public const LINK_SRC = 4;

    public const LINK_3XX = 2;

    public const LINK_301 = 3;

    // type related
    public const LINK_SELF = 1;

    public const LINK_INTERNAL = 2;

    public const LINK_SUB = 3;

    public const LINK_EXTERNAL = 4;

    /**
     * Add trailing slash for domain. Eg: https://piedweb.com => https://piedweb.com/ and '/test ' = '/test'.
     */
    public static function normalizeUrl(string $url): string
    {
        $url = trim($url);

        if ('' == preg_replace('@(.*\://?([^\/]+))@', '', $url)) {
            $url .= '/';
        }

        return $url;
    }

    protected static function getWrapperFromElement(\DomElement $element): ?int
    {
        if ('a' == $element->tagName && $element->getAttribute('href')) {
            return self::LINK_A;
        }

        if ($element->getAttribute('src')) {
            return self::LINK_SRC;
        }

        return null;
    }

    /**
     * Always submit absoute Url !
     */
    public function __construct(string $url, Harvest $parent, \DOMElement $element = null, int $wrapper = null)
    {
        $this->url = new Url(self::normalizeUrl($url));

        $this->parentDoc = $parent;

        if (null !== $element) {
            $this->setAnchor($element);
        }

        $this->element = $element;

        $this->wrapper = $wrapper ?? (null !== $element ? self::getWrapperFromElement($element) : null);
    }

    public static function createRedirection(string $url, Harvest $parent, int $redirType = null): self
    {
        return new self($url, $parent, null, $redirType ?? self::LINK_3XX);
    }

    public function getWrapper(): ?int
    {
        return $this->wrapper;
    }

    protected function setAnchor(\DomElement $element)
    {
        // Get classic text anchor
        $this->anchor = $element->textContent;

        // If get nothing, then maybe we can get an alternative text (eg: img)
        if (empty($this->anchor)) {
            $alt = (new DomCrawler($element))->filter('*[alt]');
            if ($alt->count() > 0) {
                $this->anchor = $alt->eq(0)->attr('alt') ?? '';
            }
        }

        // Limit to 100 characters
        // Totally subjective
        $this->anchor = substr(Helper::clean($this->anchor), 0, 99);

        return $this;
    }

    public function getUrl(): Url
    {
        return $this->url;
    }

    public function getPageUrl(): string
    {
        return $this->url->getDocumentUrl(); //return preg_replace('/(\#.*)/si', '', $this->url->get());
    }

    public function getParentUrl(): Url
    {
        return $this->parentDoc->getUrl();
    }

    public function getAnchor()
    {
        return $this->anchor;
    }

    public function getElement()
    {
        return $this->element;
    }

    /**
     * @return bool
     */
    public function mayFollow()
    {
        // check meta robots and headers
        if (null !== $this->parentDoc && ! $this->parentDoc->mayFollow()) {
            return false;
        }

        // check "wrapper" rel
        if (null !== $this->element && $this->element->getAttribute('rel')) {
            if (preg_match('(nofollow|sponsored|ugc)', $this->element->getAttribute('rel'))) {
                return false;
            }
        }

        return true;
    }

    /**
     * @return string
     */
    public function getRelAttribute(): ?string
    {
        return null !== $this->element ? $this->element->getAttribute('rel') : null;
    }

    public function isInternalLink(): bool
    {
        return $this->url->getOrigin() == $this->getParentUrl()->getOrigin();
    }

    public function isSubLink(): bool
    {
        return ! $this->isInternalLink()
            && $this->url->getRegistrableDomain() == $this->getParentUrl()->getRegistrableDomain();
        //&& strtolower(substr($this->getHost(), -strlen($this->parentDomain))) === $this->parentDomain;
    }

    public function isSelfLink(): bool
    {
        return $this->isInternalLink()
            && $this->url->getDocumentUrl() == $this->getParentUrl()->getDocumentUrl();
    }

    public function getType()
    {
        if ($this->isSelfLink()) {
            return self::LINK_SELF;
        }

        if ($this->isInternalLink()) {
            return self::LINK_INTERNAL;
        }

        if ($this->isSubLink()) {
            return self::LINK_SUB;
        }

        return self::LINK_EXTERNAL;
    }
}
<file_sep>/src/BreadcrumbItem.php
<?php

namespace PiedWeb\UrlHarvester;

class BreadcrumbItem
{
    private $url;

    private $name;

    public function __construct($url, $name)
    {
        $this->url = $url;
        $this->name = $name;
    }

    public function getUrl()
    {
        return $this->url;
    }

    public function getName()
    {
        return $this->name;
    }

    public function getCleanName()
    {
        return substr(strip_tags($this->name), 0, 100);
    }
}
<file_sep>/src/Harvest.php
<?php

namespace PiedWeb\UrlHarvester;

use PiedWeb\Curl\Request as CurlRequest;
use PiedWeb\Curl\Response;
use PiedWeb\TextAnalyzer\Analysis;
use PiedWeb\TextAnalyzer\Analyzer as TextAnalyzer;
use Spatie\Robots\RobotsHeaders;
use Symfony\Component\DomCrawler\Crawler as DomCrawler;

class Harvest
{
    use HarvestLinksTrait;
    use RobotsTxtTrait;

    public const DEFAULT_USER_AGENT = 'SeoPocketCrawler - Open Source Bot for SEO Metrics';

    protected Response $response;

    protected DomCrawler $dom;

    protected string $baseUrl;

    protected bool $follow;

    private Analysis $textAnalysis;

    protected Url $urlRequested;

    protected Url $url;

    /**
     * @return self|int
     */
    public static function fromUrl(
        string $url,
        string $userAgent = self::DEFAULT_USER_AGENT,
        string $language = 'en,en-US;q=0.5',
        ?CurlRequest $previousRequest = null
    ) {
        $url = Link::normalizeUrl($url); // add trailing slash for domain

        $response = Request::makeFromRequest($previousRequest, $url, $userAgent, $language);

        if ($response instanceof Response) {
            return new self($response);
        }

        return $response;
    }

    public function __construct(Response $response)
    {
        $this->response = $response;

        $this->url = new Url($this->response->getEffectiveUrl());
        $this->urlRequested = new Url($this->response->getUrl());
    }

    public function urlRequested(): Url
    {
        return $this->urlRequested;
    }

    /**
     * Return url response (curl effective url)
     * // todo : check if urlRequested can be diffenrent than url (depends on curl wrench).
     */
    public function url(): Url
    {
        return $this->url;
    }

    public function getUrl(): Url
    {
        return $this->url;
    }

    public function getResponse(): Response
    {
        return $this->response;
    }

    /** @psalm-suppress RedundantPropertyInitializationCheck */
    public function getDom()
    {
        $this->dom = isset($this->dom) ? $this->dom : new DomCrawler($this->response->getContent());

        return $this->dom;
    }

    private function find($selector, $i = null): DomCrawler
    {
        return null !== $i ? $this->getDom()->filter($selector)->eq($i) : $this->getDom()->filter($selector);
    }

    /**
     * Alias for find($selector, 0).
     */
    private function findOne($selector): DomCrawler
    {
        return $this->find($selector, 0);
    }

    /**
     * Return content inside a selector.
     * Eg.: getTag('title').
     *
     * @return ?string
     */
    public function getTag($selector)
    {
        $found = $this->findOne($selector);

        return $found->count() > 0 ? Helper::clean($found->text()) : null;
    }

    public function getUniqueTag($selector = 'title')
    {
        $found = $this->find($selector);

        if (0 === $found->count()) {
            return null;
        }

        if ($found->count() > 1) {
            return $found->count().' `'.$selector.'` /!\ ';
        }

        return Helper::clean($found->eq(0)->text());
    }

    /**
     * Return content inside a meta.
     *
     * @return string|null from content attribute
     */
    public function getMeta(string $name): ?string
    {
        $meta = $this->findOne('meta[name='.$name.']');

        return $meta->count() > 0 ? (null !== $meta->attr('content') ? Helper::clean($meta->attr('content')) : '')
            : null;
    }

    /**
     * Renvoie le contenu de l'attribut href de la balise link rel=canonical.
     */
    public function getCanonical(): ?string
    {
        $canonical = $this->findOne('link[rel=canonical]');

        return $canonical->count() > 0 ? (null !== $canonical->attr('href') ? $canonical->attr('href') : '') : null;
    }

    /*
     * @return bool true si canonical = url requested or no canonical balise
     */
    public function isCanonicalCorrect(?string $urlRequested = null): bool
    {
        $canonical = $this->getCanonical();

        if (null === $canonical) {
            return true;
        }

        $urlRequested = $urlRequested ?? $this->urlRequested()->get();

        if ($urlRequested == $canonical) {
            return true;
        }

        return $this->checkCanonicalException($urlRequested, $canonical);
    }

    private function checkCanonicalException(string $urlRequested, string $canonical): bool
    {
        if (false !== preg_match('/^.+?[^\/:](?=[?\/]|$)/', $urlRequested, $match)
            && $match[0] === ltrim($urlRequested, '/')
            && ($match[0] == $canonical || $match[0].'/' == $canonical)) {
            return true;
        }

        return false;
    }

    /** @psalm-suppress RedundantPropertyInitializationCheck */
    public function getTextAnalysis()
    {
        if (isset($this->textAnalysis)) {
            return $this->textAnalysis;
        }

        return $this->textAnalysis = $this->getDom()->count() > 0 ? TextAnalyzer::get(
            $this->getDom()->text(),
            true,   // only sentences
            1,      // no expression, just words
            0      // keep trail
        ) : null;
    }

    public function getWordCount(): int
    {
        return (int) str_word_count($this->getDom()->text('') ?? '');
    }

    public function getKws()
    {
        return $this->getTextAnalysis()->getExpressions(10);
    }

    public function getRatioTxtCode(): int
    {
        $textLenght = \strlen($this->getDom()->text(''));
        $htmlLenght = \strlen(Helper::clean($this->response->getContent()));

        return (int) ($htmlLenght > 0 ? round($textLenght / $htmlLenght * 100) : 0);
    }

    /**
     * Return an array of object with two elements Link and anchor.
     */
    public function getBreadCrumb(?string $separator = null)
    {
        $breadcrumb = ExtractBreadcrumb::get($this);

        if (null !== $separator && \is_array($breadcrumb)) {
            $breadcrumb = array_map(function ($item) {
                return $item->getCleanName();
            }, $breadcrumb);
            $breadcrumb = implode($separator, $breadcrumb);
        }

        return $breadcrumb;
    }

    /**
     * @return ?string absolute url
     */
    public function getRedirection(): ?string
    {
        $headers = $this->response->getHeaders();
        $headers = array_change_key_case($headers ?: []);
        if (isset($headers['location']) && ExtractLinks::isWebLink($headers['location'])) {
            return $this->url()->resolve($headers['location']);
        }

        return null;
    }

    public function getRedirectionLink(): ?Link
    {
        $redirection = $this->getRedirection();

        if (null !== $redirection) {
            return Link::createRedirection($redirection, $this);
        }

        return null;
    }

    public function isRedirectToHttps(): bool
    {
        $redirUrl = $this->getRedirection();

        return null !== $redirUrl && preg_replace('#^http:#', 'https:', $this->urlRequested()->get(), 1) == $redirUrl;
    }

    /**
     * Return the value in base tag if exist, else, current Url.
     *
     * @psalm-suppress RedundantPropertyInitializationCheck
     */
    public function getBaseUrl(): string
    {
        if (! isset($this->baseUrl)) {
            $base = $this->findOne('base');
            if ($base->getBaseHref() && filter_var($base->getBaseHref(), \FILTER_VALIDATE_URL)) {
                $this->baseUrl = $base->getBaseHref();
            } else {
                $this->baseUrl = $this->url()->get();
            }
        }

        return (string) $this->baseUrl;
    }

    /**
     * @return int correspond to a const from Indexable
     */
    public function indexable(string $userAgent = 'googlebot'): int
    {
        return Indexable::indexable($this, $userAgent);
    }

    public function isIndexable(string $userAgent = 'googlebot'): bool
    {
        return Indexable::INDEXABLE === $this->indexable($userAgent);
    }

    protected function metaAuthorizeToFollow()
    {
        return ! (strpos($this->getMeta('googlebot'), 'nofollow') || strpos($this->getMeta('robots'), 'nofollow'));
    }

    /** @psalm-suppress RedundantPropertyInitializationCheck */
    public function mayFollow()
    {
        if (! isset($this->follow)) {
            $robotsHeaders = new RobotsHeaders((array) $this->response->getHeaders());
            $this->follow = $robotsHeaders->mayFollow() && $this->metaAuthorizeToFollow() ? true : false;
        }

        return $this->follow;
    }
}
<file_sep>/src/ExtractBreadcrumb.php
<?php

/**
 * Entity.
 */

namespace PiedWeb\UrlHarvester;

/**
 * Quelques notes :
 * - Un bc ne contient pas l'élément courant.
 */
class ExtractBreadcrumb
{
    protected $breadcrumb = [];

    protected $parentDoc;

    public const BC_RGX = '#<(div|p|nav|ul)[^>]*(id|class)="?(breadcrumbs?|fil_?d?arian?ne)"?[^>]*>(.*)<\/(\1)>#siU';

    public const BC_DIVIDER = [
        'class="?navigation-pipe"?',
        '&gt;',
        'class="?divider"?',
        '›',
        '</li>',
    ];

    /**
     * @param string $source  HTML code from the page
     * @param string $baseUrl To get absolute urls
     * @param string $current The current url. If not set we thing it's the same than $baseUrl
     *
     * @return array|null
     */
    public static function get(Harvest $parent)
    {
        $self = new self();

        $self->parentDoc = $parent;

        return $self->extractBreadcrumb();
    }

    protected function __construct()
    {
    }

    /**
     * @return array|null
     */
    public function extractBreadcrumb()
    {
        $breadcrumb = $this->findBreadcrumb();
        if (null !== $breadcrumb) {
            foreach (self::BC_DIVIDER as $divider) {
                $exploded = $this->divideBreadcrumb($breadcrumb, $divider);
                if (false !== $exploded) {
                    $this->extractBreadcrumbData($exploded);

                    return $this->breadcrumb;
                }
            }
        }
    }

    protected function findBreadcrumb()
    {
        if (preg_match(self::BC_RGX, $this->parentDoc->getResponse()->getContent(), $match)) {
            return $match[4];
        }
    }

    protected function divideBreadcrumb($breadcrumb, $divider)
    {
        $exploded = preg_split('/'.str_replace('/', '\/', $divider).'/si', $breadcrumb);

        return false !== $exploded && \count($exploded) > 1 ? $exploded : false;
    }

    /**
     * On essaye d'extraire l'url et l'ancre.
     */
    protected function extractBreadcrumbData($array)
    {
        foreach ($array as $a) {
            $link = $this->extractHref($a);
            if (null === $link || $link == $this->parentDoc->getUrl()->get()) {
                break;
            }
            $this->breadcrumb[] = new BreadcrumbItem(
                $link,
                $this->extractAnchor($a)
            );
        }
    }

    protected function extractAnchor($str)
    {
        return trim(strtolower(Helper::htmlToPlainText($str)), '> ');
    }

    protected function extractHref($str)
    {
        $regex = [
            'href="([^"]*)"',
            'href=\'([^\']*)\'',
            'href=(\S+) ',
        ];
        foreach ($regex as $r) {
            if (preg_match('/'.$r.'/siU', $str, $match)) {
                if (ExtractLinks::isWebLink($match[1])) {
                    return $this->parentDoc->getUrl()->resolve($match[1]);
                }
            }
        }
    }
}
<file_sep>/src/Indexable.php
<?php

namespace PiedWeb\UrlHarvester;

use Spatie\Robots\RobotsHeaders;

class Indexable
{
    // https://stackoverflow.com/questions/1880148/how-to-get-name-of-the-constant
    public const INDEXABLE = 0;

    public const NOT_INDEXABLE_ROBOTS = 1;

    public const NOT_INDEXABLE_HEADER = 2;

    public const NOT_INDEXABLE_META = 3;

    public const NOT_INDEXABLE_CANONICAL = 4;

    public const NOT_INDEXABLE_4XX = 5;

    public const NOT_INDEXABLE_5XX = 6;

    public const NOT_INDEXABLE_NETWORK_ERROR = 7;

    public const NOT_INDEXABLE_TOO_BIG = 10;

    public const NOT_INDEXABLE_3XX = 8;

    public const NOT_INDEXABLE_NOT_HTML = 9;

    /** @var Harvest */
    protected $harvest;

    /** @var string */
    protected $isIndexableFor;

    public function __construct(Harvest $harvest, string $isIndexableFor = 'googlebot')
    {
        $this->harvest = $harvest;
        $this->isIndexableFor = $isIndexableFor;
    }

    public function robotsTxtAllows()
    {
        $url = $this->harvest->getResponse()->getUrl();
        $robotsTxt = $this->harvest->getRobotsTxt();

        return '' === $robotsTxt ? true : $robotsTxt->allows($url, $this->isIndexableFor);
    }

    public function metaAllows()
    {
        $meta = $this->harvest->getMeta($this->isIndexableFor);
        $generic = $this->harvest->getMeta('robots');

        return ! (false !== stripos($meta, 'noindex') || false !== stripos($generic, 'noindex'));
    }

    public function headersAllow()
    {
        $headers = explode(\PHP_EOL, $this->harvest->getResponse()->getHeaders(false));

        return RobotsHeaders::create($headers)->mayIndex($this->isIndexableFor);
    }

    public static function indexable(Harvest $harvest, string $isIndexableFor = 'googlebot'): int
    {
        $self = new self($harvest, $isIndexableFor);

        // robots
        if (! $self->robotsTxtAllows()) {
            return self::NOT_INDEXABLE_ROBOTS;
        }

        if (! $self->headersAllow()) {
            return self::NOT_INDEXABLE_HEADER;
        }

        if (! $self->metaAllows()) {
            return self::NOT_INDEXABLE_META;
        }

        // canonical
        if (! $harvest->isCanonicalCorrect()) {
            return self::NOT_INDEXABLE_CANONICAL;
        }

        $statusCode = $harvest->getResponse()->getStatusCode();

        // status 4XX
        if ($statusCode < 500 && $statusCode > 399) {
            return self::NOT_INDEXABLE_4XX;
        }

        // status 5XX
        if ($statusCode < 600 && $statusCode > 499) {
            return self::NOT_INDEXABLE_5XX;
        }

        // status 3XX
        if ($statusCode < 400 && $statusCode > 299) {
            return self::NOT_INDEXABLE_3XX;
        }

        return self::INDEXABLE;
    }
}
