<file_sep>xdebug.default_enable = on
xdebug.remote_enable = on
xdebug.remote_autostart = on
xdebug.remote_connect_back = on
xdebug.remote_port = 9000
xdebug.idekey = PHPSTORM
xdebug.max_nesting_level = 500
xdebug.profiler_enable = off
xdebug.remote_log = "php://stdout"
xdebug.profiler_output_dir = "/var/www/dev/var/tmp"
<file_sep><?php declare(strict_types=1);

namespace App\Crawler\Page;

use DateTime;
use DateTimeZone;
use Symfony\Component\DomCrawler\Crawler;

class NewsArticlePage
{
    /**
     * @var Crawler
     */
    private $crawler;

    /**
     * @param Crawler $crawler
     */
    public function __construct(Crawler $crawler)
    {
        $this->crawler = $crawler;
    }

    public function getTitle() : string
    {
        return $this->crawler->filter('div div#content article header h1')->text();
    }

    public function getDescription() : string
    {
        return $this->crawler->filter('div div#content article header h2')->text();
    }

    public function getContent() : string
    {
        return $this->crawler->filter('div div#content article div.contentBody')->html();
    }

    public function getAuthor() : string
    {
        return $this->crawler->filter('div div#content article header div.noticias_media div.autor strong')->text();
    }

    public function getPublishDate() : DateTime
    {
        $time = $this->crawler->filter('div div#content article header div.noticias_media time')->text();

        return $this->getDateTime($time);
    }

    private function getDateTime(string $time) : DateTime
    {
        $timezone = new DateTimeZone('America/Sao_Paulo');

        return DateTime::createFromFormat('Y-m-d H:i', $time, $timezone);
    }
}
<file_sep>ARG BASE_IMAGE

FROM ${BASE_IMAGE} AS base

MAINTAINER <NAME> <<EMAIL>>

# Install Xdebug
RUN yes | pecl install xdebug && \
    docker-php-ext-enable xdebug
<file_sep><?php declare(strict_types=1);

namespace App\Command;

use App\Crawler\Crawler\SymfonyCrawler;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Contracts\HttpClient\Exception\ClientExceptionInterface;
use Symfony\Contracts\HttpClient\Exception\DecodingExceptionInterface;
use Symfony\Contracts\HttpClient\Exception\RedirectionExceptionInterface;
use Symfony\Contracts\HttpClient\Exception\ServerExceptionInterface;
use Symfony\Contracts\HttpClient\Exception\TransportExceptionInterface;

class GetSymfonyReleasesCommand extends Command
{
    protected static $defaultName = 'app:symfony:releases';

    /**
     * @var SymfonyCrawler
     */
    private $crawler;

    public function __construct(SymfonyCrawler $crawler)
    {
        $this->crawler = $crawler;

        parent::__construct();
    }

    protected function configure() : void
    {
        $this
            ->setDescription('Get information about the Symfony releases.')
            ->setHelp('This command allows you to get information about the Symfony releases.');
    }

    /**
     * @param InputInterface $input
     * @param OutputInterface $output
     * @throws ClientExceptionInterface
     * @throws DecodingExceptionInterface
     * @throws RedirectionExceptionInterface
     * @throws ServerExceptionInterface
     * @throws TransportExceptionInterface
     */
    protected function execute(InputInterface $input, OutputInterface $output) : void
    {
        $releases = $this->crawler->getReleases();

        if (!empty($releases)) {
            $output->writeln('========== Symfony Releases ==========');
            $output->writeln(sprintf('LTS: %s', $releases['symfony_versions']['lts']));
            $output->writeln(sprintf('Stable: %s', $releases['symfony_versions']['stable']));
            $output->writeln(sprintf('Next: %s', $releases['symfony_versions']['next']));
            $output->writeln(sprintf('Latest Stable Version: %s', $releases['latest_stable_version']));
            $output->writeln('Supported Versions:');

            foreach ($releases['supported_versions'] as $version) {
                $output->writeln($version);
            }
        }
    }
}
<file_sep><?php declare(strict_types=1);

namespace App\Command;

use App\Crawler\Crawler\CityHallNewsCrawler;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Contracts\HttpClient\Exception\ClientExceptionInterface;
use Symfony\Contracts\HttpClient\Exception\RedirectionExceptionInterface;
use Symfony\Contracts\HttpClient\Exception\ServerExceptionInterface;
use Symfony\Contracts\HttpClient\Exception\TransportExceptionInterface;

class GetLatestCityHallNewsCommand extends Command
{
    protected static $defaultName = 'app:news:latest';

    /**
     * @var CityHallNewsCrawler
     */
    private $crawler;

    public function __construct(CityHallNewsCrawler $crawler)
    {
        $this->crawler = $crawler;

        parent::__construct();
    }

    protected function configure() : void
    {
        $this
            ->setDescription('Get the latest news from the São Paulo City Hall.')
            ->setHelp('This command allows you to get the latest news from the São Paulo City Hall.');
    }

    /**
     * @param InputInterface $input
     * @param OutputInterface $output
     * @throws ClientExceptionInterface
     * @throws RedirectionExceptionInterface
     * @throws ServerExceptionInterface
     * @throws TransportExceptionInterface
     */
    protected function execute(InputInterface $input, OutputInterface $output) : void
    {
        $searchPageUrl = 'http://www.capital.sp.gov.br/@@busca_atualizada?pt_toggle=%23&b_start:int=0&portal_type:list=News%20Item';
        $searchPage = $this->crawler->getNewsArticleSearchPage($searchPageUrl);

        if ($searchPage->getNewsArticlesTotal() > 0) {
            $output->writeln('========== Latest News - São Paulo City Hall ==========');

            foreach ($searchPage->getNewsArticleLinks() as $link) {
                $output->writeln($link->getUri());
            }
        }
    }
}
<file_sep><?php declare(strict_types=1);

namespace App\Crawler\Crawler;

use RuntimeException;
use Symfony\Component\HttpClient\HttpClient;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Contracts\HttpClient\Exception\ClientExceptionInterface;
use Symfony\Contracts\HttpClient\Exception\DecodingExceptionInterface;
use Symfony\Contracts\HttpClient\Exception\RedirectionExceptionInterface;
use Symfony\Contracts\HttpClient\Exception\ServerExceptionInterface;
use Symfony\Contracts\HttpClient\Exception\TransportExceptionInterface;
use Symfony\Contracts\HttpClient\HttpClientInterface;

class SymfonyCrawler
{
    /**
     * @var HttpClientInterface
     */
    private $httpClient;

    public function __construct(HttpClientInterface $httpClient)
    {
        $this->httpClient = $httpClient;
    }

    /**
     * @return array
     * @throws TransportExceptionInterface
     * @throws ClientExceptionInterface
     * @throws DecodingExceptionInterface
     * @throws RedirectionExceptionInterface
     * @throws ServerExceptionInterface
     */
    public function getVersions() : array
    {
        $response = $this->httpClient->request('GET', 'https://symfony.com/versions.json');

        if (Response::HTTP_OK === $response->getStatusCode()) {
            return $response->toArray();
        }

        throw new RuntimeException('Could not retrieve the Symfony versions.');
    }

    /**
     * @return array
     * @throws TransportExceptionInterface
     * @throws ClientExceptionInterface
     * @throws DecodingExceptionInterface
     * @throws RedirectionExceptionInterface
     * @throws ServerExceptionInterface
     */
    public function getReleases() : array
    {
        $httpClient = HttpClient::create();
        $response = $httpClient->request('GET', 'https://symfony.com/releases.json');

        if (200 === $response->getStatusCode()) {
            return $response->toArray();
        }

        throw new RuntimeException('Could not retrieve the Symfony releases.');
    }
}
<file_sep><?php declare(strict_types=1);

namespace App\Command;

use App\Crawler\Crawler\CityHallNewsCrawler;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Contracts\HttpClient\Exception\ClientExceptionInterface;
use Symfony\Contracts\HttpClient\Exception\RedirectionExceptionInterface;
use Symfony\Contracts\HttpClient\Exception\ServerExceptionInterface;
use Symfony\Contracts\HttpClient\Exception\TransportExceptionInterface;

class GetCityHallNewsArticleCommand extends Command
{
    protected static $defaultName = 'app:news:article';

    /**
     * @var CityHallNewsCrawler
     */
    private $crawler;

    public function __construct(CityHallNewsCrawler $crawler)
    {
        $this->crawler = $crawler;

        parent::__construct();
    }

    protected function configure() : void
    {
        $this
            ->setDescription('Get the news article from URL.')
            ->setHelp('This command allows you to get the news article for the given URL.')
            ->addArgument('url', InputArgument::REQUIRED, 'News article URL');
    }

    /**
     * @param InputInterface $input
     * @param OutputInterface $output
     * @throws ClientExceptionInterface
     * @throws RedirectionExceptionInterface
     * @throws ServerExceptionInterface
     * @throws TransportExceptionInterface
     */
    protected function execute(InputInterface $input, OutputInterface $output) : void
    {
        $url = $input->getArgument('url');
        $newsArticle = $this->crawler->getNewsArticle($url);

        $output->writeln('========== News Article - São Paulo City Hall ==========');
        $output->writeln($newsArticle->getTitle());
        $output->writeln($newsArticle->getContent());
    }
}
<file_sep><?php declare(strict_types=1);

namespace App\Crawler\Page;

use Symfony\Component\DomCrawler\Crawler;
use Symfony\Component\DomCrawler\Link;

class NewsArticleSearchPage
{
    /**
     * @var Crawler
     */
    private $crawler;

    /**
     * @param Crawler $crawler
     */
    public function __construct(Crawler $crawler)
    {
        $this->crawler = $crawler;
    }

    /**
     * @return int
     */
    public function getNewsArticlesTotal() : int
    {
        return (int) $this->crawler->filter('#updated-search-results-number')->text();
    }

    /**
     * @return Link[]
     */
    public function getNewsArticleLinks() : array
    {
        return $this->crawler->filter(
            'dl.searchResults div.contenttype-news-item a.state-published'
        )->links();
    }

    /**
     * @return Link
     */
    public function getNextSearchPageLink() : Link
    {
        return $this->crawler->filter('#search-results ul.paginacao.listingBar li a.proximo')->link();
    }
}
<file_sep><?php declare(strict_types=1);

namespace App\Command;

use App\Crawler\Crawler\SymfonyCrawler;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Contracts\HttpClient\Exception\ClientExceptionInterface;
use Symfony\Contracts\HttpClient\Exception\DecodingExceptionInterface;
use Symfony\Contracts\HttpClient\Exception\RedirectionExceptionInterface;
use Symfony\Contracts\HttpClient\Exception\ServerExceptionInterface;
use Symfony\Contracts\HttpClient\Exception\TransportExceptionInterface;

class GetSymfonyVersionsCommand extends Command
{
    protected static $defaultName = 'app:symfony:versions';

    /**
     * @var SymfonyCrawler
     */
    private $crawler;

    public function __construct(SymfonyCrawler $crawler)
    {
        $this->crawler = $crawler;

        parent::__construct();
    }

    protected function configure() : void
    {
        $this
            ->setDescription('Get information about the Symfony versions.')
            ->setHelp('This command allows you to get information about the Symfony versions.');
    }

    /**
     * @param InputInterface $input
     * @param OutputInterface $output
     * @throws ClientExceptionInterface
     * @throws DecodingExceptionInterface
     * @throws RedirectionExceptionInterface
     * @throws ServerExceptionInterface
     * @throws TransportExceptionInterface
     */
    protected function execute(InputInterface $input, OutputInterface $output) : void
    {
        $versions = $this->crawler->getVersions();

        if (!empty($versions)) {
            $output->writeln('========== Symfony Versions ==========');

            foreach ($versions as $key => $version) {
                // Skipping some keys because there are a lot of versions
                if ($key === 'non_installable' || $key === 'installable') {
                    continue;
                }

                $output->writeln(sprintf('%s: %s', $key, $version));
            }
        }
    }
}
<file_sep># Development

## Requirements

1. Install [Docker][docker-install] and [Docker Compose][compose-install],
   and add the binaries to the `PATH` environment variable.
   
   _Note: The following configurations are based on a Linux setup. Adjustments
   may be required to use Docker Compose in the Windows setup._

## Installation

### Environment Variables

1. Access the `.deploy/local` directory, create an `.env` file from `.env.dist`,
   and set up the variables used on the containers:

    | Variable              | Description                                                           |
    | --------------------- | --------------------------------------------------------------------- |
    | COMPOSE_FILE          | The project docker-compose yaml file(s).                              |
    | COMPOSE_PROJECT_DIR   | The project docker-compose directory for local config.                |
    | COMPOSE_PROJECT_NAME  | Project name used as prefix when creating containers.                 |
    | COMPOSER_MEMORY_LIMIT | Prevent memory limit errors as explained [here][memory-limit-errors]. |
    | DEV_COMPOSER_DIR      | Your local composer directory, on your host machine, used for cache.  |
    | DEV_DOCKER_IMAGE_PHP  | Docker image used by the php service.                                 |
    | DEV_HOST_PORT_HTTP    | HTTP port used on the host machine. Default: 80.                      |
    | DEV_HOST_PORT_MYSQL   | MySQL port used on host machine. Default: 3306.                       |
    | DEV_GID               | The user group id, used to prevent permissions errors.                |
    | DEV_UID               | The user id, used to prevent permissions errors.                      |
    | MYSQL_DATABASE        | Default database.                                                     |
    | MYSQL_PASSWORD        | <PASSWORD> user password.                                                  |
    | MYSQL_ROOT_PASSWORD   | MySQL root password.                                                  |
    | MYSQL_USER            | MySQL user.                                                           |

    To get the `$GID` and `$UID` run the following commands:

    ```
    id -u <user>

    id -g <group>
    ```

2. Then you can execute the helper scripts from the `.deploy/local/bin` folder:

    ```
    .deploy/local
    ├── bin
    │   ├── check-env.sh
    │   ├── install-env.sh
    │   └── set-aliases.sh
    ├── services
    │   ├── mysql
    │   ├── nginx
    │   └── php
    ├── docker-compose.yaml
    └── .env.dist
    .env
    ```

### Docker

1. From the **project root folder**, run the following commands to create
   aliases for the development tools:

    ```
    COMPOSE_PROJECT_DIR=${PWD}/.deploy/local

    . .deploy/local/bin/install-env.sh
    ```

2. The following aliases will be created:

    * composer
    * console
    * dc
    * php

1. Build the docker images:

    ```
    docker build -t ${DEV_DOCKER_IMAGE_PHP}:base .deploy/docker/base

    dc build --parallel
    ```

1. Run the containers:

    ```
    dc up -d
    ```

    _Note: Make sure the host ports set up to the services on the
    `docker-compose.yaml` file are free. The `ports` directive maps ports on the
    host machine to ports on the containers and follows the format
    `<host-port>:<container-port>`. More info on the
    [Compose file reference][compose-ports]._

1. To stop the containers, run:

    ```
    dc down
    ```

### Application

1. Access the root directory, create an `.env.local` file from `.env`, and set
   up the variables used on the application.

2. Then, run the following command:

    ```
    composer install
    ```

[compose-install]: https://docs.docker.com/compose/install/
[compose-ports]: https://docs.docker.com/compose/compose-file/#ports
[docker-install]: https://docs.docker.com/install/
[memory-limit-errors]: https://getcomposer.org/doc/articles/troubleshooting.md#memory-limit-errors
<file_sep>#!/usr/bin/env bash

alias composer="dc \
    exec php composer $@"

alias console="dc \
    exec php bin/console $@"

alias dc="docker-compose \
    --project-directory ${COMPOSE_PROJECT_DIR} \
    --file ${COMPOSE_PROJECT_DIR}/docker-compose.yaml \
    $@"

alias php="dc \
    exec php php $@"
<file_sep>FROM composer:latest AS composer
FROM php:fpm-alpine AS php

MAINTAINER <NAME> <<EMAIL>>

# Install dependencies and tools
RUN apk add --no-cache --update --virtual .build-deps \
    autoconf \
    g++ \
    git \
    libxml2-dev \
    libzip-dev \
    make \
    openssl-dev \
    zip

# Build and install PHP extensions
RUN docker-php-ext-configure \
        opcache && \
    docker-php-ext-install \
        mysqli \
        opcache \
        pdo_mysql \
        zip && \
    pecl install \
        mongodb && \
    docker-php-ext-enable \
        mongodb \
        opcache

# Remove dependencies
RUN apk del --purge \
        autoconf \
        g++ \
        libxml2-dev \
        make \
        openssl-dev && \
    docker-php-source delete && \
    rm -rf /tmp/* /var/cache/apk/*

# Install Composer
COPY --from=composer /usr/bin/composer /usr/bin/composer
ENV COMPOSER_HOME /composer
ENV PATH ${PATH}:/composer/vendor/bin
ENV COMPOSER_ALLOW_SUPERUSER 0
<file_sep># SymfonyCon Amsterdam 2019 - Crawling the Web with the New Symfony Components

## Documentation

[Symfony Documentation][doc-symfony]

## Requirements

- MySQL 8.0
- PHP 7.3
- Symfony 4.3

## Development

* [Local][doc-local]

[doc-symfony]: https://symfony.com/doc/current/index.html
[doc-local]: .deploy/local/README.md
<file_sep><?php declare(strict_types=1);

namespace App\Crawler\Crawler;

use App\Crawler\Page\NewsArticlePage;
use App\Crawler\Page\NewsArticleSearchPage;
use RuntimeException;
use Symfony\Component\DomCrawler\Crawler;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Contracts\HttpClient\Exception\ClientExceptionInterface;
use Symfony\Contracts\HttpClient\Exception\RedirectionExceptionInterface;
use Symfony\Contracts\HttpClient\Exception\ServerExceptionInterface;
use Symfony\Contracts\HttpClient\Exception\TransportExceptionInterface;
use Symfony\Contracts\HttpClient\HttpClientInterface;

class CityHallNewsCrawler
{
    /**
     * @var HttpClientInterface
     */
    private $httpClient;

    public function __construct(HttpClientInterface $httpClient)
    {
        $this->httpClient = $httpClient;
    }

    /**
     * @param string $url
     * @return NewsArticleSearchPage
     * @throws ClientExceptionInterface
     * @throws RedirectionExceptionInterface
     * @throws ServerExceptionInterface
     * @throws TransportExceptionInterface
     */
    public function getNewsArticleSearchPage(string $url) : NewsArticleSearchPage
    {
        $response = $this->httpClient->request('GET', $url);

        if (Response::HTTP_OK === $response->getStatusCode()) {
            $content = $response->getContent();
            $crawler = new Crawler($content);

            // Create an object to handle the target DOM elements
            return new NewsArticleSearchPage($crawler);
        }

        throw new RuntimeException('Could not retrieve the news articles.');
    }

    /**
     * @param string $url
     * @return NewsArticlePage
     * @throws ClientExceptionInterface
     * @throws RedirectionExceptionInterface
     * @throws ServerExceptionInterface
     * @throws TransportExceptionInterface
     */
    public function getNewsArticle(string $url) : NewsArticlePage
    {
        $response = $this->httpClient->request('GET', $url);

        if (Response::HTTP_OK === $response->getStatusCode()) {
            $content = $response->getContent();
            $crawler = new Crawler($content);

            // Create an object to handle the target DOM elements
            return new NewsArticlePage($crawler);
        }

        throw new RuntimeException('Could not retrieve the news article.');
    }
}
