<repo_name>StanislavAshykhmin/parser<file_sep>/public/js/select.js
document.addEventListener("DOMContentLoaded", function(event) {
    $('select[name="system_id[]"]').select2({
        placeholder: 'Systems',
    }).change(function () {
        $(this).valid();
    });
});
<file_sep>/tests/Browser/tag/CreateTagTest.php
<?php

namespace Tests\Browser\Tag;

use App\User;
use Tests\DuskTestCase;
use Laravel\Dusk\Browser;
use Illuminate\Foundation\Testing\DatabaseMigrations;

class CreateTagTest extends DuskTestCase
{
    /**
     * A Dusk test example.
     *
     * @return void
     * @group regresssion
     * @group tag
     */
    public function testExample()
    {
        $user = User::find(1);
        $this->browse(function ($browser) use ($user) {
            $browser->visit('/login')
                ->type('email', $user->email)
                ->type('password', '<PASSWORD>')
                ->press('Login')
                ->waitForText('Tags')
                ->click('.item2')
                ->waitForText('Create tag')
                ->press('Create tag')
                ->waitForText('Save tag')
                ->type('name', 'ruby')
                ->press('Save tag')
                ->waitForText('Tag created')
                ->assertSee('Ruby');
        });
    }
}
<file_sep>/readme.md
<p align="center"><img src="https://laravel.com/assets/img/components/logo-laravel.svg"></p>

<p align="center">
<a href="https://travis-ci.org/laravel/framework"><img src="https://travis-ci.org/laravel/framework.svg" alt="Build Status"></a>
<a href="https://packagist.org/packages/laravel/framework"><img src="https://poser.pugx.org/laravel/framework/d/total.svg" alt="Total Downloads"></a>
<a href="https://packagist.org/packages/laravel/framework"><img src="https://poser.pugx.org/laravel/framework/v/stable.svg" alt="Latest Stable Version"></a>
<a href="https://packagist.org/packages/laravel/framework"><img src="https://poser.pugx.org/laravel/framework/license.svg" alt="License"></a>
</p>

## Get Started Tool

1. "git clone https://github.com/StanislavAshykhmin/parser.git" - clone with https or "git clone <EMAIL>:StanislavAshykhmin/parser.git" - clone with SSH;
2. "cp .env.example to .env" - copy basic settings;
3. "docker-compose up -d --build" - create docker container;
4. "docker exec -it parser-web-container bash" - open container;
5. "composer install" - install packages;
6. "chown -R www-data:www-data ." - opening access to reading;
7.  "php artisan key:generate" - generate key;
8. "php artisan migrate" - create table on data base;
9. "php artisan db:seed" - seeding table.
<file_sep>/routes/web.php
<?php

/*
|--------------------------------------------------------------------------
| Web Routes
|--------------------------------------------------------------------------
|
| Here is where you can register web routes for your application. These
| routes are loaded by the RouteServiceProvider within a group which
| contains the "web" middleware group. Now create something great!
|
*/


Route::get('/home', function () {
    return redirect('/dashboard');
});
Auth::routes();

Route::group(['middleware' => ['web', 'auth']], function () {
    Route::get('/dashboard', 'Dashboard\DashboardController@index')->name('dashboard');
    Route::get('/dashboard/graphic/{id}/{from?}/{before?}', 'Dashboard\DashboardController@getGraphics')->name('select');
    Route::get('/tags', 'Dashboard\TagsController@index')->name('tags');
    Route::post('/tag/create', 'Dashboard\TagsController@store')->name('create');
    Route::delete('/tag/delete/{id}', 'Dashboard\TagsController@destroy')->name('delete');
    Route::get('/parser/{tag}', 'Parser\ParserController@getContent')->name('content');
    Route::get('/tag/{id}', 'Dashboard\TagsController@show')->name('tag');
    Route::get('/systems', 'Dashboard\CmsController@index')->name('systems');
    Route::get('/messages', 'Dashboard\MessageController@index')->name('message');
    Route::get('/messages/send/{id}', 'Mail\MailController@send')->name('send');
    Route::post('/messages/create', 'Dashboard\MessageController@store')->name('message_create');
    Route::get('/messages/update/{id}', 'Dashboard\MessageController@edit')->name('edit');
    Route::post('/messages/update', 'Dashboard\MessageController@update')->name('update');
    Route::delete('/messages/delete/{id}', 'Dashboard\MessageController@destroy')->name('message_delete');
    Route::get('/systems/{id}', 'Dashboard\CmsController@show')->name('sites');
    Route::get('/site/{id}', 'Dashboard\EmailController@show')->name('email');
});



Route::group(['middleware' => ['web']], function () {
    Route::get('/messages/check/{id}', 'Mail\MailController@checkMail')->name('check');
    Route::post('/landing/contact', 'Landing\LandingController@contact')->name('contact');
    Route::get('/', 'Landing\LandingController@index')->name('landing');
});
<file_sep>/tests/Feature/Parser/ParserTest.php
<?php

namespace Tests\Feature\Parser;

use App\Helpers\ParserStackOverflow;
use Tests\TestCase;
use Illuminate\Foundation\Testing\WithoutMiddleware;

class ParserTest extends TestCase
{
    use WithoutMiddleware;
    /**
     * A basic test example.
     *
     * @return void
     */
    public function testExample()
    {
        $tag = 'Laravel';
        $content = new ParserStackOverflow();
        $records = $content->index($tag);
        $result = count($records);
        $this->assertEquals($result, 75);

    }
}
<file_sep>/app/Enums/StatusType.php
<?php

namespace App\Enums;

use BenSampo\Enum\Enum;

final class StatusType extends Enum
{
    const NotAvailable = 0;
    const EmailNotFound = 1;
    const LangNotFound = 2;
    const NotProcessed = 3;
    const Processed = 4;
    const NotSend = 5;
    const SentTo = 6;
    const Open = 7;
}
<file_sep>/app/Http/Controllers/Dashboard/CmsController.php
<?php

namespace App\Http\Controllers\Dashboard;

use App\Model\Parser\Site;
use App\Model\Dashboard\System;
use App\User;
use App\Http\Controllers\Controller;

class CmsController extends Controller
{
    public function index(){
        return view('dashboard.systems.index', ['allCms' => System::withCount('sites')->get(), 'user' => User::find(1)]);
    }

    public function show($id){
        return view('dashboard.systems.show', ['system' => System::find($id), 'sites' => Site::where('system_id', $id)->orderBy('status', 'desc')->orderByRaw("FIELD(lang , 'en-US', 'en') desc")->paginate(6), 'user' => User::find(1)]);
    }
}
<file_sep>/data/docker/web/dev/Dockerfile
FROM php:7.1-apache

MAINTAINER <NAME> <<EMAIL>>

ARG HOST_UID=1000

VOLUME ["/var/www/html"]

ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update && apt-get install -my wget gnupg
RUN apt-get update && apt-get install -y apt-utils
RUN apt-get install -y \
        zlib1g-dev \
        libicu-dev \
        libpq-dev \
        libpcre3-dev \
        git \
        nano \
        zip \
        libfreetype6-dev \
        libjpeg62-turbo-dev \
        libmcrypt-dev \
        libpng-dev \
        supervisor \
        mysql-client \
        cron \
    && docker-php-ext-install -j$(nproc) iconv mcrypt \
    && docker-php-ext-configure gd --with-freetype-dir=/usr/include/ --with-jpeg-dir=/usr/include/ \
    && docker-php-ext-install -j$(nproc) gd \
    && docker-php-ext-install intl zip pdo_mysql \
    && docker-php-ext-install exif \
    && apt-get clean \
    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

# Install Node.js
RUN curl -sL https://deb.nodesource.com/setup_8.x | bash - \
    && apt-get update && apt-get install -y nodejs

# Install google-chrome-stable
RUN apt-get install -y wget && \
    wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub |  apt-key add - && \
    sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list' && \
    apt-get update && apt-get install -y google-chrome-stable && \
    apt-get install -y xvfb

# Install Xdebug
RUN yes | pecl install xdebug \
    && echo "zend_extension=$(find /usr/local/lib/php/extensions/ -name xdebug.so)" \
    > /usr/local/etc/php/conf.d/xdebug.ini \
    && echo "xdebug.remote_enable=on" >> /usr/local/etc/php/conf.d/xdebug.ini \
    && echo "xdebug.remote_autostart=off" >> /usr/local/etc/php/conf.d/xdebug.ini

COPY ./.bashrc /root/.bashrc
COPY ./apache.conf /etc/apache2/sites-available/000-default.conf
COPY ./php.ini /usr/local/etc/php/

RUN echo "LogFormat \"%a %l %u %t \\\"%r\\\" %>s %O \\\"%{User-Agent}i\\\"\" mainlog" >> /etc/apache2/apache2.conf
RUN a2enmod rewrite remoteip

# Install composer
RUN set -x && curl -sS https://getcomposer.org/installer | php && mv composer.phar /usr/local/bin/composer

# Setup permissions for user `www-data`
RUN usermod -u ${HOST_UID} www-data && groupmod -g ${HOST_UID} www-data && chsh -s /bin/bash www-data

# Setup cron
ADD ./crontab /etc/cron.d/cron-jobs
RUN chmod 0644 /etc/cron.d/cron-jobs && touch /var/log/cron.log

# Setup supervisor
RUN mkdir -p /var/log/supervisor
COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf

EXPOSE 80

CMD ["/usr/bin/supervisord"]
<file_sep>/tests/Browser/GraphicTest.php
<?php

namespace Tests\Browser;

use App\User;
use Tests\DuskTestCase;
use Laravel\Dusk\Browser;
use Illuminate\Foundation\Testing\DatabaseMigrations;

class GraphicTest extends DuskTestCase
{
    /**
     * A Dusk test example.
     *
     * @return void
     * @group regresssion
     * @group Graphic
     */
    public function testExample()
    {
        $user = User::find(1);
        $this->browse(function ($browser) use ($user) {
            $browser->visit('/login')
                ->type('email', $user->email)
                ->type('password', '<PASSWORD>')
                ->press('Login')
                ->waitForText('Dashboard')
                ->press('Print Graphic')
                ->pause(1000)
                ->assertVisible('.chart');
        });
    }
}
<file_sep>/app/Helpers/ParserStackOverflow.php
<?php
/**
 * Created by PhpStorm.
 * User: stanislav
 * Date: 21.12.18
 * Time: 19:41
 */

namespace App\Helpers;


use Symfony\Component\DomCrawler\Crawler;

class ParserStackOverflow
{
    public function index($tag)
    {
        $result = [];
        for ($page = 1; $page < 6; $page++) {
            $link = 'https://stackoverflow.com/questions/tagged/' . $tag . '?sort=newest&page=' . $page . '&pagesize=15';
            $html = file_get_contents($link);
            $crawler = new Crawler(null, $link);
            $crawler->addHtmlContent($html, 'UTF-8');
            $title = $crawler->filter('h3 .question-hyperlink')->each(function ($node) {
                return $node->text();
            });
            $links = $crawler->filter('h3 .question-hyperlink ')->each(function ($node) {
                return $node->attr('href');
            });
            $vote = $crawler->filter('.vote')->each(function ($node) {
                return $node->text();
            });
            $answers = $crawler->filter('.status')->each(function ($node) {
                return $node->text();
            });
            $views = $crawler->filter('.views')->each(function ($node) {
                return $node->text();
            });
            $date = $crawler->filter('.user-action-time span')->each(function ($node) {
                return $node->attr('title');
            });
            for ($id = 0; $id < 15; $id++) {
                if (!empty($links[$id])) {
                    $result['link'] = $links[$id];
                    $result['title'] = $title[$id];
                    $result['vote'] = trim(str_replace(["\r\n", "votes", "vote"], '', $vote[$id]));
                    $result['answer'] = trim(str_replace(["\r\n", "answers", "answer"], '', $answers[$id]));
                    $result['view'] = trim(str_replace(["\r\n", "views", "view"], '', $views[$id]));
                    $result['parser_date'] = trim(str_replace("Z", '', $date[$id]));
                    $fullResult[] = $result;
                } else {
                    $fullResult[] = 0;
                    break;
                }
            }
        }
        return array_reverse($fullResult);
    }
}
<file_sep>/app/Listeners/ParserListener.php
<?php

namespace App\Listeners;

use App\Events\ParserEvent;
use App\Jobs\ParserJob;

class ParserListener
{
    /**
     * Create the event listener.
     *
     * @return void
     */
    public function __construct()
    {
        //
    }

    /**
     * Handle the event.
     *
     * @param  object $event
     * @return void
     */
    public function handle(ParserEvent $event)
    {
        ParserJob::dispatch($event->tag)->onQueue('high');
    }
}
<file_sep>/database/factories/RecordFactory.php
<?php

use Faker\Generator as Faker;

$factory->define(App\Model\Parser\Record::class, function (Faker $faker) {
    return [
        'title' => $faker->title,
        'vote' => '0',
        'answer' => '1',
        'view' => '1000',
        'url' => $faker->url,
        'tag_id' => '10', // Laravel
    ];
});
<file_sep>/app/Http/Controllers/Dashboard/TagsController.php
<?php

namespace App\Http\Controllers\Dashboard;

use App\Helpers\ParserStackOverflow;
use App\Http\Controllers\Controller;
use App\Http\Requests\TagRequest;
use App\Model\Dashboard\Tag;
use App\Model\Parser\Record;
use Helmesvs\Notify\Facades\Notify;
use Illuminate\Support\Facades\Auth;

class TagsController extends Controller
{
    public function index()
    {
        return view('dashboard.tags.index', ['user' => Auth::user(), 'tags' => Tag::withCount('records')->get()]);
    }

    public function store(TagRequest $request)
    {
        $data = $request->all();
        $content = new ParserStackOverflow();
        $records = $content->index($data['name']);
        if ($records[4] != 0) {
            Tag::create([
                'name' => $data['name'],
            ]);
            Notify::success('Tag created', 'Success', $options = []);
            return redirect()->back();
        } else
            return redirect()->back()->with('messageCreateError', 'Tag does not exist. Please enter new tag.');
    }

    public function destroy($id)
    {
        $tag = Tag::find($id);
        $tag->delete();
        Notify::success('Tag deleted', 'Success', $options = []);
        return redirect()->back();
    }

    public function show($id)
    {
        $tag = Tag::find($id);
        $records = Record::where('tag_id', $id)->orderBy('id', 'desc')->paginate(10);
        return view('dashboard.tags.show', ['user' => Auth::user(), 'records' => $records, 'tag' => $tag]);
    }

}
<file_sep>/app/Model/Parser/Contact.php
<?php

namespace App\Model\Parser;

use Illuminate\Database\Eloquent\Model;

class Contact extends Model
{
    protected $fillable = ['url_id', 'email'];
    public function site(){
        return $this->belongsTo('App\Model\Parser\Site');
    }

    public function messages(){
        return $this->belongsToMany('App\Model\Dashboard\Message')->withPivot('status')->withTimestamps();
    }

}
<file_sep>/app/Console/Commands/RecordsSeed.php
<?php

namespace App\Console\Commands;

use App\Model\Parser\Record;
use Illuminate\Console\Command;

class RecordsSeed extends Command
{
    /**
     * The name and signature of the console command.
     *
     * @var string
     */
    protected $signature = 'seed:records';

    /**
     * The console command description.
     *
     * @var string
     */
    protected $description = 'Command description';

    /**
     * Create a new command instance.
     *
     * @return void
     */
    public function __construct()
    {
        parent::__construct();
    }

    /**
     * Execute the console command.
     *
     * @return mixed
     */
    public function handle()
    {
        factory(Record::class, 100000)->create();
    }
}
<file_sep>/app/Model/Parser/Site.php
<?php

namespace App\Model\Parser;

use Illuminate\Database\Eloquent\Model;

class Site extends Model
{
    protected $fillable = ['cms_id', 'url', 'status', 'lang', 'email'];

    public function contacts(){
        return $this->hasMany('App\Model\Parser\Contact');
    }
    public function system(){
        return $this->belongsTo('App\Model\Dashboard\System');
    }
}
<file_sep>/app/Http/Controllers/Dashboard/EmailController.php
<?php

namespace App\Http\Controllers\Dashboard;

use App\Model\Parser\Contact;
use App\Model\Parser\Site;
use App\User;
use App\Http\Controllers\Controller;

class EmailController extends Controller
{
    public function show($id){
        return view('dashboard.site.show', [
            'site' => Site::find($id),
            'contact' => Contact::where('site_id', $id)->pluck('email')->first(),
            'messages' => Contact::where('site_id', $id)->first()->messages,
            'user' => User::find(1)]);
    }
}
<file_sep>/app/Model/Response/Response.php
<?php

namespace App\Model\Response;

use Illuminate\Database\Eloquent\Model;

class Response extends Model
{
    protected $fillable = ['name', 'last_name', 'email', 'url', 'text'];
}
<file_sep>/tests/Browser/LoginTest.php
<?php

namespace Tests\Browser;

use App\User;
use Tests\DuskTestCase;

class ExampleTest extends DuskTestCase
{
    /**
     * A basic browser test example.
     *
     * @return void
     * @group regresssion
     * @group Login
     */
    public function testBasicExample()
    {
        $user = User::find(1);

        $this->browse(function ($browser) use ($user) {
            $browser->visit('/login')
                ->type('email', $user->email)
                ->type('password', '<PASSWORD>')
                ->press('Login')
                ->assertPathIs('/dashboard');
        });
    }
}
<file_sep>/app/Model/Parser/Record.php
<?php

namespace App\Model\Parser;

use Illuminate\Database\Eloquent\Model;

class Record extends Model
{
    protected $fillable = ['title', 'vote', 'answer', 'view', 'url', 'tag_id', 'parser_date'];


    public function tag()
    {
        return $this->belongsTo('App\Model\Dashboard\Tag');
    }
}
<file_sep>/app/Mail/SendMail.php
<?php

namespace App\Mail;

use Illuminate\Bus\Queueable;
use Illuminate\Mail\Mailable;
use Illuminate\Queue\SerializesModels;

class SendMail extends Mailable
{
    use Queueable, SerializesModels;

    protected $id;
    protected $title;
    protected $text;
    /**
     * Create a new message instance.
     *
     * @return void
     */
    public function __construct($email, $message)
    {
        $this->id = $email['id'];
        $this->title = $message['title'];
        $this->text = $message['text'];
    }

    /**
     * Build the message.
     *
     * @return $this
     */
    public function build()
    {
        return $this->view('mail.mail')->with([
            'id' => $this->id,
            'title' => $this->title,
            'text' => $this->text,
        ]);
    }
}
<file_sep>/app/Jobs/ParserJob.php
<?php

namespace App\Jobs;

use App\Helpers\ParserStackOverflow;
use App\Model\Dashboard\Tag;
use Helmesvs\Notify\Facades\Notify;
use Illuminate\Bus\Queueable;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;

class ParserJob implements ShouldQueue
{
    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;

    protected $tag;

    /**
     * Create a new job instance.
     *
     * @return void
     */
    public function __construct($tag)
    {
        $this->tag = $tag;
    }

    /**
     * Execute the job.
     *
     * @return void
     */
    public function handle()
    {
        $content = new ParserStackOverflow();
        $records = $content->index($this->tag);
        $save = Tag::where('name', $this->tag)->first();
        foreach ($records as $record) {
            $save->records()->updateOrCreate([
                'title' => $record['title'],
            ],
                [
                    'vote' => $record['vote'],
                    'answer' => $record['answer'],
                    'view' => $record['view'],
                    'url' => $record['link'],
                    'parser_date' => $record['parser_date']
                ]);
        }
    }
}
<file_sep>/tests/Browser/message/CreateMessageTest.php
<?php

namespace Tests\Browser\Message;

use App\User;
use Tests\DuskTestCase;
use Laravel\Dusk\Browser;
use Illuminate\Foundation\Testing\DatabaseMigrations;

class CreateMessageTest extends DuskTestCase
{
    /**
     * A Dusk test example.
     *
     * @return void
     * @group regresssion
     * @group message
     */
    public function testExample()
    {
        $user = User::find(1);
        $this->browse(function ($browser) use ($user) {
            $browser->visit('/login')
                ->type('email', $user->email)
                ->type('password', '<PASSWORD>')
                ->press('Login')
                ->waitForText('Tags')
                ->click('.item4')
                ->waitForText('Systems')
                ->press('Create message')
                ->waitForText('Text')
                ->type('#label_message', 'first-test')
                ->type('#title_message', 'first-test')
                ->type('text', 'first-test')
                ->click('.select2')
                ->click('option:nth-child(1)')
                ->click('#label_message')
                ->press('Save message')
                ->waitForText('Create message')
                ->assertSee('Message created');
        });
    }
}
<file_sep>/app/Http/Controllers/Mail/MailController.php
<?php

namespace App\Http\Controllers\Mail;

use App\Enums\StatusType;
use App\Mail\SendMail;
use App\Model\Parser\Contact;
use App\Model\Dashboard\Message;
use App\Http\Controllers\Controller;
use Helmesvs\Notify\Facades\Notify;
use Illuminate\Support\Facades\Mail;

class MailController extends Controller
{
    public function send($id){
        $message = Message::find($id);
        $emails = $message->contacts;
        foreach ($emails as $email){
            Mail::to($email['email'])->send(new SendMail($email, $message));
            $site = Contact::where('id', $email->id)->first()->site->update(['status' => StatusType::Processed]);
            $message->contacts()->sync([$email->id =>['status' => StatusType::SentTo]], false);
            }
        Notify::success('Message sent', 'Success', $options = []);
        return redirect()->back();
    }

    public function checkMail($id){
        $contact = Contact::find($id);
        $messages = $contact->messages;
        foreach ($messages as $message){
            $contact->messages()->sync([$message->id =>['status' => StatusType::Open]], false);
        }
        return response(base64_decode('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQMAAAAl21bKAAAAA1BMVEUAAACnej3aAAAAAXRSTlMAQObYZgAAAApJREFUCNdjYAAAAAIAAeIhvDMAAAAASUVORK5CYII='), 200, ['Content-Type' => 'image/png']);
    }
}
<file_sep>/app/Mail/ResponseMail.php
<?php

namespace App\Mail;

use Illuminate\Bus\Queueable;
use Illuminate\Mail\Mailable;
use Illuminate\Queue\SerializesModels;

class ResponseMail extends Mailable
{
    use Queueable, SerializesModels;

    protected $name;
    protected $lastName;
    protected $email;
    protected $url;
    protected $text;
    /**
     * Create a new message instance.
     *
     * @return void
     */
    public function __construct($data)
    {
        $this->name = $data['first_name'];
        $this->lastName = $data['last_name'];
        $this->email = $data['email'];
        $this->url = $data['url'];
        $this->text = $data['text'];
    }

    /**
     * Build the message.
     *
     * @return $this
     */
    public function build()
    {
        return $this->view('mail.response')->with([
            'name' => $this->name,
            'lastName' => $this->lastName,
            'email' => $this->email,
            'url' => $this->url,
            'text' => $this->text,
        ]);
    }
}
<file_sep>/app/Helpers/ParserCmsSite.php
<?php
/**
 * Created by PhpStorm.
 * User: stanislav
 * Date: 15.01.19
 * Time: 14:20
 */

namespace App\Helpers;



use App\Enums\StatusType;
use App\Model\Dashboard\System;
use App\Model\Parser\Site;
use Illuminate\Support\Facades\Storage;
use Symfony\Component\DomCrawler\Crawler;

class ParserCmsSite
{
    public function getContent($cms){
        System::create([
            'name' => $cms,
        ]);
        $link = '/public/systems/'.$cms.'.csv';
        $file = Storage::get($link);
        $urls = explode("\n", $file);
        $urls = array_diff($urls, array(''));
        foreach ($urls as $url){
                try {
                    $html = $this->get_web_page($url);
                    $crawler = new Crawler(null, $url);
                    $crawler->addHtmlContent($html['content'], 'UTF-8');
                    $lang = $crawler->filter('html')->each(function ($node) {
                        return $node->attr('lang');
                    });
                    $text = preg_match('/[A-Za-z0-9._%+-]+\@[A-Za-z0-9.-]+\.[A-Za-z]{2,6}/', $crawler->text(), $email);
                    if (empty($email)) {
                        $status = StatusType::EmailNotFound;
                    } elseif (empty($lang)) {
                        $status = StatusType::LangNotFound;
                    } else {
                        $status = StatusType::NotProcessed;
                    }
                    $saveSite = System::where('name', $cms)->first();
                    $saveSite->sites()->create([
                        'url' => $url,
                        'lang' => implode('', $lang),
                        'status' => $status,
                    ]);
                    if ($email) {
                        $saveEmail = Site::where('url', $url)->first();
                        $saveEmail->contacts()->create([
                            'email' => implode('', $email),
                        ]);
                    }
                } catch (\Exception $e) {
                    $saveSite = System::where('name', $cms)->first();
                    $saveSite->sites()->create([
                        'url' => $url,
                        'lang' => implode('', $lang),
                        'status' => StatusType::NotAvailable,
                    ]);
                    echo 'Выброшено исключение: ',  $e->getMessage(), "\n";
                }
            }
    }

    public function get_web_page( $url )
    {
       // $user_agent='Mozilla/5.0 (Windows NT 6.1; rv:8.0) Gecko/20100101 Firefox/8.0';

        $options = array(

            CURLOPT_CUSTOMREQUEST  =>"GET",        //set request type post or get
            CURLOPT_POST           =>false,        //set to GET
           // CURLOPT_USERAGENT      => $user_agent, //set user agent
          //  CURLOPT_COOKIEFILE     =>"cookie.txt", //set cookie file
         //   CURLOPT_COOKIEJAR      =>"cookie.txt", //set cookie jar
            CURLOPT_RETURNTRANSFER => true,     // return web page
            CURLOPT_HEADER         => false,    // don't return headers
            CURLOPT_FOLLOWLOCATION => true,     // follow redirects
            CURLOPT_ENCODING       => "",       // handle all encodings
            CURLOPT_AUTOREFERER    => true,     // set referer on redirect
            CURLOPT_CONNECTTIMEOUT => 60,      // timeout on connect
            CURLOPT_TIMEOUT        => 60,      // timeout on response
            CURLOPT_MAXREDIRS      => 10,       // stop after 10 redirects
        );

        $ch      = curl_init( $url );
        curl_setopt_array( $ch, $options );
        $content = curl_exec( $ch );
        $err     = curl_errno( $ch );
        $errmsg  = curl_error( $ch );
        $header  = curl_getinfo( $ch );
        curl_close( $ch );

        $header['errno']   = $err;
        $header['errmsg']  = $errmsg;
        $header['content'] = $content;
        return $header;
    }
}
<file_sep>/app/Model/Dashboard/Tag.php
<?php

namespace App\Model\Dashboard;

use Illuminate\Database\Eloquent\Model;

class Tag extends Model
{
    protected $fillable = ['name',];

    public function records()
    {
        return $this->hasMany('App\Model\Parser\Record');
    }
}
<file_sep>/database/seeds/TagsSeeder.php
<?php

use Illuminate\Database\Seeder;
use App\Model\Dashboard\Tag;

class TagsSeeder extends Seeder
{
    /**
     * Run the database seeds.
     *
     * @return void
     */
    public function run()
    {
        Tag::create([
            'name'=>"Laravel",
        ]);
    }
}
<file_sep>/app/Http/Controllers/Dashboard/MessageController.php
<?php

namespace App\Http\Controllers\Dashboard;

use App\Enums\StatusType;
use App\Http\Requests\MessageRequest;
use App\Model\Dashboard\Message;
use App\Model\Dashboard\System;
use App\User;
use App\Http\Controllers\Controller;
use Helmesvs\Notify\Facades\Notify;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Validator;

class MessageController extends Controller
{
    public function index()
    {
        return view('dashboard.message.index', ['messages' => Message::with('systems')->get(), 'user' => User::find(1), 'systems' => System::all()]);
    }

    public function store(MessageRequest $request)
    {
        $data = $request->all();
        Message::create([
            'label' => $data['label'],
            'title' => $data['title'],
            'text' => $data['text'],
        ]);
        foreach ($data['system_id'] as $system) {
            $emails = System::where('id', $system)->first()->contacts;
            $save = Message::where('label', $data['label'])->first();
            $save->systems()->attach(
                ['system_id' => $system]
            );
            foreach ($emails as $email) {
                $save->contacts()->attach(
                    ['contact_id' => $email->id],
                    ['status' => StatusType::NotSend]);
            }
        }
        Notify::success('Message created', 'Success', $options = []);
        return redirect()->back();
    }

    public function edit($id)
    {
        $message = Message::whereId($id)->first();
        if ($message) {
            return response(['message' => $message], 200);
        }
        return response(['message' => 'Error !!'], 422);
    }

    public function update(Request $request)
    {
        $data = $request->except('_token');
        $validator = Validator::make($data, [
            'label'=>'required',
            'title'=>'required',
            'text'=>'required',
            'system_id'=>'required',
        ]);
        if ($validator->fails()) {
            return response()->json(['errors' => $validator->errors()->all()]);
        }
        $message = Message::find($data['id']);
        $message->systems()->detach();
        $message->contacts()->detach();
        $message->label = $data['label'];
        $message->title = $data['title'];
        $message->text = $data['text'];
        $res = $message->saveMessage()->save($message);

        foreach ($data['system_id'] as $system) {
            $emails = System::where('id', $system)->first()->contacts;
            $message->systems()->attach(
                ['system_id' => $system]
            );
            foreach ($emails as $email) {
                $message->contacts()->attach(
                    ['contact_id' => $email->id],
                    ['status' => StatusType::NotSend]);
            }
        }
        return response()->json(['success' => 'Message updated']);
    }

    public function destroy($id)
    {
        $message = Message::find($id);
        $message->delete();
        Notify::success('Message deleted', 'Success', $options = []);
        return redirect()->back();
    }

}
<file_sep>/data/docker/web/prod/Dockerfile
FROM php:7.1-apache

MAINTAINER <NAME> <<EMAIL>>

ARG HOST_UID=1000

VOLUME ["/var/www/html"]

ENV DEBIAN_FRONTEND noninteractive

RUN apt-get update && apt-get install -y apt-utils
RUN apt-get install -y \
        cron \
        zlib1g-dev \
        libicu-dev \
        libpq-dev \
        libpcre3-dev \
        git \
        nano \
        zip \
        libfreetype6-dev \
        libjpeg62-turbo-dev \
        libmcrypt-dev \
        libpng12-dev \
        sudo \
        openssh-server \
        supervisor \
        mysql-client \
        cron \
    && docker-php-ext-install -j$(nproc) iconv mcrypt \
    && docker-php-ext-configure gd --with-freetype-dir=/usr/include/ --with-jpeg-dir=/usr/include/ \
    && docker-php-ext-install -j$(nproc) gd \
    && docker-php-ext-install intl zip pdo_mysql \
    && docker-php-ext-install exif \
    && apt-get clean \
    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

# Install Node.js
RUN curl -sL https://deb.nodesource.com/setup_8.x | bash - \
    && apt-get update && apt-get install -y nodejs

COPY ./.bashrc /root/.bashrc
COPY ./apache.conf /etc/apache2/sites-available/000-default.conf
COPY ./php.ini /usr/local/etc/php/

RUN echo "LogFormat \"%a %l %u %t \\\"%r\\\" %>s %O \\\"%{User-Agent}i\\\"\" mainlog" >> /etc/apache2/apache2.conf
RUN a2enmod rewrite remoteip

# Install composer
RUN set -x && curl -sS https://getcomposer.org/installer | php && mv composer.phar /usr/local/bin/composer

# Setup permissions for user `www-data`
RUN usermod -u ${HOST_UID} www-data && groupmod -g ${HOST_UID} www-data && chsh -s /bin/bash www-data

# Setup cron
ADD ./crontab /etc/cron.d/cron-jobs
RUN chmod 0644 /etc/cron.d/cron-jobs && touch /var/log/cron.log

# Setup supervisor
RUN mkdir -p /var/log/supervisor
COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf

EXPOSE 80

CMD ["/usr/bin/supervisord"]
<file_sep>/app/Http/Controllers/Dashboard/DashboardController.php
<?php

namespace App\Http\Controllers\Dashboard;

use App\Http\Controllers\Controller;
use App\Model\Dashboard\Tag;
use App\Model\Parser\Record;
use Carbon\Carbon;
use Illuminate\Support\Facades\Auth;

class DashboardController extends Controller
{
    public function index()
    {
        $tags = Tag::all();
        $records = Record::where([['tag_id', '>', 0], ['parser_date', '>', Carbon::now()->startOfDay()]])->orderBy('view', 'desc')->limit(5)->get();
        return view('dashboard.dashboard', ['user' => Auth::user(), 'records' => $records, 'tags' => $tags]);
    }

    public function getGraphics($id, $from = null, $before = null)
    {
        $count = [];
        $day = [];
        if ($before != null && $from != null) {
            $before = Carbon::parse($before);
            $from = Carbon::parse($from);
        } elseif ($from != null) {
            $before = Carbon::now();
            $from = Carbon::parse($from);
        } else{
            $before = Carbon::now();
            $from = Carbon::now()->startOfMonth();
        }
        while ($from <= $before) {
             $count[] = Record::whereDate('parser_date', '=', $from->toDateString())->whereTagId($id)->count('id');
             $day[] = date_format($from, 'd');
             $from->addDay();
        }
        if ($count && $day) {
            return response(['count' => $count, 'day' => $day, 'countDay' => count($day)], 200);
        }
        return response(['message' => 'Error !!'], 422);
    }
}
<file_sep>/app/Console/Commands/Parser.php
<?php

namespace App\Console\Commands;

use App\Jobs\ParserJob;
use App\Model\Dashboard\Tag;
use Illuminate\Console\Command;

class Parser extends Command
{
    /**
     * The name and signature of the console command.
     *
     * @var string
     */
    protected $signature = 'parser:tag';

    /**
     * The console command description.
     *
     * @var string
     */
    protected $description = 'Command description';

    /**
     * Create a new command instance.
     *
     * @return void
     */
    public function __construct()
    {
        parent::__construct();
    }

    /**
     * Execute the console command.
     *
     * @return mixed
     */
    public function handle()
    {
        $tags = Tag::all();
        foreach ($tags as $tag) {
            ParserJob::dispatch($tag->name);
        }
    }
}
<file_sep>/app/Model/Dashboard/Message.php
<?php

namespace App\Model\Dashboard;

use Illuminate\Database\Eloquent\Model;

class Message extends Model
{
    protected $fillable = ['label', 'title', 'text', 'system_id', 'message_id', 'status'];

    protected $with = [
      'systems'
    ];

    public function contacts(){
        return $this->belongsToMany('App\Model\Parser\Contact');
    }
    public function sites(){
        return $this->hasManyThrough('App\Model\Parser\Site', 'App\Model\Parser\Contact');
    }

    public function systems(){
        return $this->belongsToMany('App\Model\Dashboard\System');
    }

    public function saveMessage(){
        return $this->hasMany('App\Model\Dashboard\Message', 'id');
    }
}
<file_sep>/app/Http/Controllers/Landing/LandingController.php
<?php

namespace App\Http\Controllers\Landing;

use App\Mail\ResponseMail;
use App\Model\Response\Response;
use App\User;
use App\Http\Controllers\Controller;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Mail;
use Illuminate\Support\Facades\Validator;


class LandingController extends Controller
{
    public function index(){
        return view('landing.landing', ['user' => User::find(1)]);
    }

    public function contact(Request $request){
        $data = $request->all();
        $validator = Validator::make($data, [
            'first_name'=>'required',
            'last_name'=>'required',
            'email'=>'required|email',
            'url'=>'required|url',
            'text'=>'required',
        ]);
        if ($validator->fails())
        {
            return response()->json(['errors'=>$validator->errors()->all()]);
        }else {
            $user = User::where('id', '>', 0)->first();
            Response::create([
                'name' => $data['first_name'],
                'last_name' => $data['last_name'],
                'email' => $data['email'],
                'url' => $data['url'],
                'text' => $data['text'],
            ]);
            Mail::to($user->email)->send(new ResponseMail($data));
            return response()->json(['success' => 'Thanks you. We will contact you within 24 hours.']);
        }
    }
}
<file_sep>/app/Http/Controllers/Parser/ParserController.php
<?php

namespace App\Http\Controllers\Parser;

use App\Events\ParserEvent;
use App\Http\Controllers\Controller;
use Helmesvs\Notify\Facades\Notify;

class ParserController extends Controller
{
    public function getContent($tag){
        event(new ParserEvent($tag));
        Notify::success('Parser started. Please reload page after 2 minutes!', 'Success', $options = []);
        return redirect()->back();
    }

}
<file_sep>/app/Model/Dashboard/System.php
<?php

namespace App\Model\Dashboard;

use Illuminate\Database\Eloquent\Model;

class System extends Model
{
    protected $fillable = ['name', 'status'];
    public function sites(){
        return $this->hasMany('App\Model\Parser\Site');
    }
    public function contacts(){
        return $this->hasManyThrough('App\Model\Parser\Contact', 'App\Model\Parser\Site');
    }

}
<file_sep>/.env.example
HOST_UID=1000

APP_NAME=parser
APP_ENV=local
APP_KEY=<KEY>
APP_DEBUG=true
APP_LOG_LEVEL=debug
APP_URL=http://localhost:45010

DB_CONNECTION=mysql
DB_HOST=parser_db
DB_PORT=3306
DB_DATABASE=parser
DB_USERNAME=docker
DB_PASSWORD=<PASSWORD>



MIX_PUSHER_APP_KEY="${PUSHER_APP_KEY}"
MIX_PUSHER_APP_CLUSTER="${PUSHER_APP_CLUSTER}"

CACHE_DRIVER=file
SESSION_DRIVER=file
SESSION_LIFETIME=120
QUEUE_DRIVER=redis

REDIS_HOST=127.0.0.1
REDIS_PASSWORD=<PASSWORD>
REDIS_PORT=6379

MAIL_DRIVER=log
MAIL_HOST=smtp.gmail.com
MAIL_PORT=587
MAIL_USERNAME=null
MAIL_PASSWORD=<PASSWORD>
MAIL_ENCRYPTION=tls

PUSHER_APP_ID=
PUSHER_APP_KEY=
PUSHER_APP_SECRET=
PUSHER_APP_CLUSTER=mt1

