protected function configureRobotsFile(RobotsFileInterface $robots)
    {
        $robots->setCrawlDelay($this->robots['crawl_delay']);
        foreach ($this->robots['allow'] as $path => $userAgent) {
            if (strpos($path, '/') === false) {
                $path = $this->router->generate($path);
            }
            $robots->addAllowEntry($path, $userAgent);
        }
        foreach ($this->robots['disallow'] as $path => $userAgent) {
            if (strpos($path, '/') === false) {
                $path = $this->router->generate($path);
            }
            $robots->addDisallowEntry($path, $userAgent);
        }
        foreach ($this->robots['clean_param'] as $path => $value) {
            $robots->addCleanParamEntry($value['parameters'], $path);
        }
    }