<|start_filename|>app/ScrapShell/ChromeLauncher.cs<|end_filename|>
﻿using Microsoft.Win32;
using System;
using System.Diagnostics;

namespace ScrapShell
{
    internal static class ChromeLauncher
    {
        private const string ChromeAppKey = @"\Software\Microsoft\Windows\CurrentVersion\App Paths\chrome.exe";

        private static string ChromeAppFileName
        {
            get
            {
                return (string)(Registry.GetValue("HKEY_LOCAL_MACHINE" + ChromeAppKey, "", null) ??
                                    Registry.GetValue("HKEY_CURRENT_USER" + ChromeAppKey, "", null));
            }
        }

        public static void OpenLink(string url)
        {
            string chromeAppFileName = ChromeAppFileName;
            if (string.IsNullOrEmpty(chromeAppFileName))
                throw new TypeAccessException("Could not find chrome.exe!");
            
            Process.Start(chromeAppFileName, "--disable-javascript --disabled --no-network-profile-warning --disable-renderer-backgrounding --disable-background-timer-throttling --start-maximized --disable-background-networking --disable-javascript-harmony-shipping --disable-translate --incognito " + url);
        }

        public static void OpenFile(string fileName)
        {
            var file = $"file:///{fileName.Replace("\\", "/")}";
            OpenLink(file);
        }
    }
}

<|start_filename|>src/SpiderSharp/Helpers/Extractors.cs<|end_filename|>
﻿using Newtonsoft.Json.Linq;

using Slugify;

using System.Text.RegularExpressions;

namespace SpiderSharp.Helpers
{
    public static class Extractors
    {
        public static int? IntOrNull(string val)
        {
            int num;
            if (int.TryParse(val, out num))
            {
                return num;
            }

            return null;
        }

        public static int IntOrOne(string val)
        {
            int num;
            if (int.TryParse(val, out num))
            {
                return num;
            }

            return 1;
        }

        public static int? NumbersOrNull(string value)
        {
            if (value == null)
                return null;

            var num = Regex.Match(value, @"\d+").Value;
            return IntOrNull(num);
        }

        public static string NumbericOrNull(string value)
        {
            if (value == null)
                return null;

            return Regex.Match(value, @"\d+").Value;
        }

        public static JToken JsonList(object obj)
        {
            return JToken.FromObject(obj);
        }

        public static string RegexUrl(string text)
        {
            if (string.IsNullOrWhiteSpace(text))
                return null;

            var match = RegexLibrary.Url.Match(text);
            if (match.Success)
                return match.Value;

            return null;
        }

        public static string RegexFirstGroup(string text, string pattern)
        {
            if (string.IsNullOrEmpty(text))
                return null;

            Regex regex = new Regex(pattern);
            if (regex.IsMatch(text))
                return regex.Match(text).Groups[0].Value;
            else
                return null;
        }

        public static string RegexLastGroup(string text, string pattern)
        {
            if (string.IsNullOrEmpty(text))
                return null;

            Regex regex = new Regex(pattern);
            if (regex.IsMatch(text))
            {
                var match = regex.Match(text);
                return match.Groups[match.Groups.Count - 1].Value;
            }
            else
                return null;
        }

        public static string RegexGroup(string text, string pattern, string group)
        {
            if (string.IsNullOrEmpty(text))
                return null;

            Regex regex = new Regex(pattern);
            if (regex.IsMatch(text))
            {
                var match = regex.Match(text);
                return match.Groups[group].Value;
            }
            else
                return null;
        }

        /// <summary>
        /// https://github.com/ctolkien/Slugify
        /// </summary>
        /// <param name="text"></param>
        /// <returns></returns>
        public static string Slug(string text)
        {
            SlugHelper helper = new SlugHelper();
            return helper.GenerateSlug(text);
        }
    }
}

<|start_filename|>src/SpiderSharp/RunPipelines/RunSaveToElasticSearchPipelineAsync.cs<|end_filename|>
﻿using DataFoundation.ElasticSearch;
using Humanizer;
using Newtonsoft.Json.Linq;
using Serilog;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Text;
using System.Threading;
using System.Threading.Tasks;

namespace SpiderSharp
{
    public partial class SpiderContext
    {
        public async Task RunSaveToElasticSearchPipelineAsync(string index, string type = null, string primaryKeyField = null)
        {
            var it = this.Data;

            var elastic = new ElasticConnection(index, GlobalSettings.ElasticSearchConnectionString, type: type);
            JObject obj = new JObject(it);
            var json = obj.ToString();

            if (primaryKeyField == null)
            {
                var ret = await elastic.IndexAsync(json);
                if (!ret.Success)
                    Log.Error(ret.Body);
            }
            else
            {
                var id = (string)it[primaryKeyField];
                var ret = await elastic.IndexAsync(id, json);
                if (!ret.Success)
                {
                    Log.Error($"{id} -> {ret.Body}");
                }
            }
        }
    }
}

<|start_filename|>src/SpiderSharp/RunPipelines/RunSafeUrlsPipeline.cs<|end_filename|>
﻿using Newtonsoft.Json.Linq;
using Serilog;
using System;
using System.Collections.Generic;
using System.Dynamic;
using System.Text;

namespace SpiderSharp
{
    public partial class SpiderContext
    {
        public void RunSafeUrlsPipeline(string baseUrl, params string[] tokens)
        {
            var baseUrlSlashed = baseUrl.EndsWith("/") ? baseUrl : baseUrl + "/";

            JObject json = JObject.FromObject(this.Data);

            foreach (var token in tokens)
            {
                JToken jtoken = json.SelectToken(token);

                if (jtoken == null)
                {
                    Log.Warning($"Token {token} don't is on context");
                    continue;
                }

                if (jtoken.Type == JTokenType.Array)
                {
                    JArray jarr = (JArray) jtoken;

                    for (int i=0; i<jarr.Count; i++)
                    {
                        if (jarr[i].Type == JTokenType.String)
                        {
                            if (!jarr[i].ToString().ToLower().StartsWith(baseUrlSlashed.ToLower()))
                            {
                                jarr[i] = $"{baseUrlSlashed}{jarr[i].ToString().TrimStart('/')}";
                            }
                        }
                    }
                }
                else if (jtoken.Type == JTokenType.String)
                {
                    string link = jtoken.ToString().ToLower();

                    if (!string.IsNullOrEmpty(link))
                    {
                        // add schema/domain/port if needed
                        if (!link.StartsWith(baseUrlSlashed.ToLower().Trim()))
                        {
                            var concat = jtoken.ToString().TrimStart('/');
                            if (Uri.TryCreate(baseUrlSlashed + concat, UriKind.RelativeOrAbsolute, out Uri uri))
                            {
                                JValue property = (JValue)jtoken;
                                property.Value = uri.ToString();
                            }
                            else
                            {
                                throw new Exception($"Cannot parse uri {uri}");
                            }
                        }
                    }
                }
            }

            this.Data = json;
        }
    }
}

<|start_filename|>src/SpiderSharp/AddPipelines/AddSaveToMongoDBAsyncPipeline.cs<|end_filename|>
﻿using System;

namespace SpiderSharp
{
    public abstract partial class SpiderEngine
    {
        [Obsolete("Use RunSaveToMongoDBAsyncPipeline in SpiderContext")]
        public void AddSaveToMongoDBAsyncPipeline(string collection, string primaryKeyField)
        {
            this.AddPipeline(it =>
            {
                //try
                //{
                //    var mongo = new MongoConnection(GlobalSettings.MongoDatabase, GlobalSettings.MongoConnectionString);
                //    var pkValue = (string)it[primaryKeyField];
                //    mongo.UpdateDefinitionAsync(collection, primaryKeyField, pkValue, it.ToString());
                //}
                //catch (Exception ex)
                //{
                //    Console.WriteLine(ex.Message);
                //}

                return it;
            });
        }
    }
}

<|start_filename|>src/SpiderSharp/Helpers/Json.cs<|end_filename|>
﻿using Newtonsoft.Json.Linq;

using System;
using System.Collections.Generic;
using System.Linq;

namespace SpiderSharp.Helpers
{
    public static class Json
    {
        public static bool IsJson(string content)
        {
            return TryParse(content) != null;
        }

        public static JToken TryParse(string content)
        {
            try
            {
                // check if json
                return JToken.Parse(content);
            }
            catch
            {
            }

            return null;
        }

        public static JToken CloneRenaming(JToken json, Func<string, string> map)
        {
            var prop = json as JProperty;
            if (prop != null)
            {
                return new JProperty(map(prop.Name), CloneRenaming(prop.Value, map));
            }

            var arr = json as JArray;
            if (arr != null)
            {
                var cont = arr.Select(el => CloneRenaming(el, map));
                return new JArray(cont);
            }

            var o = json as JObject;
            if (o != null)
            {
                var cont = o.Properties().Select(el => CloneRenaming(el, map));
                return new JObject(cont);
            }

            return json;
        }

        public static JToken CloneRenaming(JToken json, Dictionary<string, string> map)
        {
            return CloneRenaming(json, name => map.ContainsKey(name) ? map[name] : name);
        }

        public static void Rename(JToken item, Func<string, string> map)
        {
            JObject json = (JObject)item;
            foreach (JProperty it in json.DescendantsAndSelf().Where(w => w.Type == JTokenType.Property).ToList())
            {
                if (it.Type == JTokenType.Property)
                {
                    var oldFieldName = it.Name;
                    var newFiledName = map(it.Name);
                    if (oldFieldName != newFiledName)
                    {
                        //it.Parent.Add(new JProperty(map(it.Name), it.Value));
                        //it.Remove();
                        it.Replace(new JProperty(map(it.Name), it.Value));
                    }
                }
            }
        }

        public static void RenameProperty(JToken item, string from, string to)
        {
            JObject json = (JObject)item;
            var prop = json.Property(from);
            prop.Replace(new JProperty(to, prop.Value));
        }
    }
}

<|start_filename|>src/SpiderSharp/Helpers/RegexHelper.cs<|end_filename|>
﻿using System;
using System.Collections.Generic;
using System.Text;
using System.Text.RegularExpressions;

namespace SpiderSharp.Helpers
{
    public static class RegexHelper
    {
        public static string RemoveAllNthChild(string url)
        {
            return Regex.Replace(url, @":nth-child\([0-9]+\)", string.Empty);
        }
    }
}


<|start_filename|>src/SpiderSharp/RunPipelines/RunRenameFieldsPipeline.cs<|end_filename|>
﻿using Humanizer;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Dynamic;
using System.Text;

namespace SpiderSharp
{
    public partial class SpiderContext
    {
        public void RunRenameFieldsPipeline(string from, string to)
        {
            var it = this.Data;
            var jobject = JObject.FromObject(it);
            Helpers.Json.RenameProperty(jobject, from, to);
            it = jobject;
        }
    }
}

<|start_filename|>src/SpiderSharp/DownloaderSpider.cs<|end_filename|>
﻿using HttpRequester;
using Newtonsoft.Json.Linq;
using Serilog;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;

namespace SpiderSharp
{
    public class DownloaderMiddleware
    {
        public void ForceCookies(string cookies)
        {
            //this.client.ForceCookies = cookies;
            this.client.SetHeader("Cookie", cookies);
        }

        //public string PrefixCache { get; set; }

        public Dictionary<string, string> DefaultHeaders = new Dictionary<string, string>();

        public string HttpMethod { get; set; }
        public string HttpBody { get; set; }

        public List<KeyValuePair<string, string>> PostData = new List<KeyValuePair<string, string>>();

        public string RedisConnectrionstring { get; set; }

        public string RedisPassword { get; set; }
        public bool UseRedisCache { get; set; }
        public TimeSpan? Duration { get; set; }
        public string Cookies { get; set; }
        //public bool ExecuteTidy { get; set; }
        public EnumHttpProvider HttpProvider { get; set; }
        public HttpRequester.RequesterCached client;
        public HttpRequester.RequesterCached cachedRequester;

        public int TotalRequestOnline { get; private set; }
        public int TotalRequestCached { get; private set; }

        public bool IsFirstMultipleByNumberOnlineRequest(int number)
        {
            return this.TotalRequestOnline % number == 0;
        }

        public DownloaderMiddleware(HttpRequester.EnumHttpProvider provider)
        {
            this.HttpProvider = provider;
            this.client = client ?? new HttpRequester.RequesterCached(this.HttpProvider);

            this.UseRedisCache = GlobalSettings.UseRedisCache ?? false;
            if (this.UseRedisCache)
            {
                this.RedisConnectrionstring = GlobalSettings.RedisConnectionString ?? null;
                this.RedisPassword = GlobalSettings.RedisPassword ?? null;

                var redis = new DataFoundation.Redis.RedisConnection(this.RedisConnectrionstring, this.RedisPassword);
                this.cachedRequester = this.cachedRequester ?? new HttpRequester.RequesterCached(this.HttpProvider, new CacheProvider(redis, this.Duration));
            }
            else
            {
                this.cachedRequester = this.client;
            }
        }

        //public async Task SimplePostAsync(string url)
        //{
        //    Log.Information($"SIMPLE POST: {url}");
        //    //client.DefaultHeaders = DefaultHeaders;
        //    DefaultHeaders.ToList().ForEach(a => client.SetHeader(a.Key, a.Value));
        //    //client.Cookies = this.Cookies;
        //    client.SetHeader("Cookie", this.Cookies);
        //    await client.PostAsync(url, (string) null);
        //    Cookies = client.LastCookie;
        //}

        public async Task<string> RunAsync(string url)
        {
            string content = string.Empty;
        
            try
            {
                string rc = null;
                if (string.IsNullOrEmpty(this.HttpMethod) || this.HttpMethod == "GET")
                {
                    var ct = await cachedRequester.GetAsync(url);
                    if (ct.HasErrors)
                    {
                        throw ct.Exception;
                    }

                        rc = ct.StringContent;

                    if (ct.HasUsedCache)
                        TotalRequestCached++;
                    else
                        TotalRequestOnline++;
                }
                else if(this.HttpMethod == "POST")
                {
                    ResponseContext ct = null;
                    if (!string.IsNullOrEmpty(this.HttpBody))
                    {
                        ct = await cachedRequester.PostAsync(url, this.HttpBody);
                        if (ct.HasErrors)
                        {
                            throw ct.Exception;
                        }
                        rc = ct.StringContent;
                    }
                    else if (PostData.Any())
                    {
                        ct = await cachedRequester.PostAsync(url, this.PostData);
                        if (ct.HasErrors)
                        {
                            throw ct.Exception;
                        }
                        rc = ct.StringContent;
                    }
                    else
                    {
                        ct = await cachedRequester.PostAsync(url, string.Empty);
                        if (ct.HasErrors)
                        {
                            throw ct.Exception;
                        }
                    }

                    if (ct.HasUsedCache)
                        TotalRequestCached++;
                    else
                        TotalRequestOnline++;
                }

                content = rc;
            }
            catch (HttpRequestException ex) when (ex.Message.Contains("404"))
            {
                return string.Empty;
            }
         
            return content;
        }
    }
}

<|start_filename|>src/SpiderSharp/RunPipelines/RunCleanUrlBookmarkPipeline.cs<|end_filename|>
﻿using Newtonsoft.Json.Linq;

namespace SpiderSharp
{
    public partial class SpiderContext
    {
        public void RunCleanUrlBookmarkPipeline(params string[] tokens)
        {
            JObject json = JObject.FromObject(this.Data);

            foreach (var token in tokens)
            {
                JToken jtoken = json.SelectToken(token);

                if (jtoken.Type == JTokenType.Array)
                {
                    JArray jarr = (JArray)jtoken;

                    for (int i = 0; i < jarr.Count; i++)
                    {
                        if (jarr[i].Type == JTokenType.String)
                        {
                            jarr[i] = SpiderSharp.Helpers.Url.ReplaceBookmark(jarr[i].ToString(), string.Empty);
                        }
                    }
                }
                else if (jtoken.Type == JTokenType.String)
                {
                    string link = jtoken.Value<string>().ToString();

                    JValue property = (JValue)jtoken;
                    property.Value = SpiderSharp.Helpers.Url.ReplaceBookmark(link, string.Empty);
                }
            }

            this.Data = json;
        }
    }
}

<|start_filename|>src/SpiderSharp/AngleDocument.cs<|end_filename|>
﻿using System;
using AngleSharp.Html.Dom;
using AngleSharp.Html.Parser;

namespace SpiderSharp
{
    public static class AngleDocument
    {
        public static IHtmlDocument TryParse(string content)
        {
            try
            {
                var parser = new HtmlParser();
                return parser.ParseDocument(content);
            }
            catch (Exception ex)
            {

            }

            return null;
        }
    }
}



<|start_filename|>src/SpiderSharp/RunPipelines/RunTrimStringPipeline.cs<|end_filename|>
﻿using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Dynamic;
using System.Text;

namespace SpiderSharp
{
    public partial class SpiderContext
    {
        public void RunTrimStringPipeline(params string[] tokens)
        {
            JObject json = JObject.FromObject(this.Data);

            foreach (var token in tokens)
            {
                JToken jtoken = json.SelectToken(token);

                if (jtoken.Type == JTokenType.Array)
                {
                    JArray jarr = (JArray) jtoken;

                    for (int i=0; i<jarr.Count; i++)
                    {
                        if (jarr[i].Type == JTokenType.String)
                        {
                            jarr[i] = jarr[i].ToString().Trim();
                        }
                    }
                }
                else if (jtoken.Type == JTokenType.String)
                {
                    string link = jtoken.Value<string>().ToString();

                    JValue property = (JValue)jtoken;
                    property.Value = link.Trim();
                }
            }

            this.Data = json;
        }
    }
}

<|start_filename|>src/SpiderSharp/Extensions/AngleSharpExtension.cs<|end_filename|>
﻿using Newtonsoft.Json.Linq;
using System;
using System.Linq;

namespace AngleSharp.Dom
{
    public static class AngleSharpExtension
    {
        public static JObject GetJson(this IElement doc, bool addPath = false)
        {
            JObject obj = new JObject();
            obj.Add(new JProperty(doc?.NodeName, IElementToJObject(doc)));

            if (addPath)
            {
                var idx = 0;
                JObject path = new JObject();
                foreach (JToken item in obj.DescendantsAndSelf())
                {
                    if (item.Type == JTokenType.Property)
                    {
                        var prop = (JProperty)item;
                        if (!prop.Name.StartsWith("@") && !prop.Name.StartsWith("#"))
                        {
                            idx++;
                            path.Add(prop.Name + "_" + idx, item.Path.ToLower().Replace(".", " > "));
                        }
                    }
                }

                obj.Add("PATH", path);
            }

            return obj;
        }

        static JObject IElementToJObject(this IElement node)
        {
            JObject obj = new JObject();

            // child
            node?.ChildNodes.ToList().ForEach(a =>
            {
                // add childs
                if (a.NodeType == NodeType.Element)
                {
                    var no = IElementToJObject((IElement)a);
                    if (no.Children().Count() > 0)
                    {
                        // verifiy if is unique or array
                        if (node.ChildNodes.Any(w => w.NodeName == a.NodeName))
                        {
                            // is array

                            // verify if is first
                            if (obj.ContainsKey(a.NodeName))
                            {
                                // second, third
                                JArray arr = obj[a.NodeName] as JArray;
                                arr.Add(no);
                            }
                            else
                            {
                                // first
                                obj.Add(new JProperty(a.NodeName, new JArray(no)));
                            }
                        }
                        else
                        {
                            // is unique
                            obj.Add(new JProperty(a.NodeName, no));
                        }
                    }
                }
                else if (a.NodeType == NodeType.Text)
                {
                    if (!string.IsNullOrWhiteSpace(a.TextContent))
                    {
                        var ct = System.Web.HttpUtility.HtmlDecode(a.TextContent.Trim());
                        if (!obj.ContainsKey(a.NodeName))
                        {
                            obj.Add(new JProperty(a.NodeName, ct));
                        }
                        else
                        {
                            obj.Add(new JProperty(a.NodeName + "_" + obj.Count, ct));
                        }
                    }
                }
                else
                {
                    throw new NotImplementedException();
                }
            });

            // add attributes
            node?.Attributes.ToList().ForEach(a => {

                if (!string.IsNullOrWhiteSpace(a.Value))
                {
                    string ct = System.Web.HttpUtility.HtmlDecode(a.Value.ToString());

                    // verifiy if is unique or array
                    if (node.Attributes.Any(w => w.Name == a.Name))
                    {
                        // is array

                        // verify if is first
                        if (obj.ContainsKey($"@{a.Name}"))
                        {
                            // second, third
                            JArray arr = obj[$"@{a.Name}"] as JArray;
                            arr.Add(ct);
                        }
                        else
                        {
                            // first
                            obj.Add(new JProperty($"@{a.Name}", new JArray(ct)));
                        }
                    }
                    else
                    {
                        // is single
                        obj.Add(new JProperty($"@{a.Name}", ct));
                    }
                }
            });

            return obj;
        }
    }
}


<|start_filename|>src/SpiderSharp/Helpers/Url.cs<|end_filename|>
﻿using System;
using System.Web;

namespace SpiderSharp.Helpers
{
    public static class Url
    {
        public static string SetBaseUrl(string baseUrl, string url)
        {
            baseUrl = baseUrl.Trim().TrimEnd('/');
            url = url.Trim().TrimStart('/');

            if (!url.Trim().ToLower().StartsWith(baseUrl.Trim().ToLower()))
            {
                if (Uri.TryCreate($"{baseUrl}/{url}", UriKind.RelativeOrAbsolute, out Uri uri))
                    return uri.ToString();

                throw new Exception("The baseurl cannot be set");
            }

            return url;
        }

        public static string ReplaceQueryString(string url, string newQueryString)
        {
            if (string.IsNullOrEmpty(url))
                return url;

            UriBuilder uri = new UriBuilder(url);
            uri.Query = newQueryString;
            
            return System.Web.HttpUtility.UrlPathEncode(uri.Uri.ToString());
        }

        public static string ReplaceBookmark(string url, string newQueryString)
        {
            if (string.IsNullOrEmpty(url))
                return url;

            UriBuilder uri = new UriBuilder(url);
            uri.Fragment = newQueryString;

            return uri.Uri.ToString();
        }

        public static string RemoveQueryString(string url)
        {
            return ReplaceQueryString(url, string.Empty);
        }

        public static string SetQueryString(string url, string queryStringKey, string queryStringValue)
        {
            var uri = new UriBuilder(url);

            var qs = HttpUtility.ParseQueryString(uri.Query);
            qs.Set(queryStringKey, queryStringValue);

            uri.Query = qs.ToString();

            return System.Web.HttpUtility.UrlPathEncode(uri.Uri.ToString());
        }
    }
}

<|start_filename|>src/SpiderSharp/AddPipelines/AddPrintToConsolePipeline.cs<|end_filename|>
﻿using Newtonsoft.Json.Linq;

using System;
using System.Linq;

namespace SpiderSharp
{
    public abstract partial class SpiderEngine
    {
        [Obsolete("Use RunPrintToConsolePipeline in SpiderContext")]
        public void AddPrintToConsolePipeline(params string[] fields)
        {
            this.AddPipeline(it =>
            {
                if (fields == null || fields.Count() == 0)
                {
                    JToken obj = JToken.FromObject(it);
                    Console.WriteLine(obj);
                }
                else
                {
                    foreach (var item in fields)
                    {
                        if (it[item] != null)
                            Console.WriteLine($"{item} => {it[item].Value}");
                    }
                }

                return it;
            });
        }
    }
}

<|start_filename|>src/SpiderSharp/AddPipelines/AddRenameFields.cs<|end_filename|>
﻿using Newtonsoft.Json.Linq;

using System;

namespace SpiderSharp
{
    public abstract partial class SpiderEngine
    {
        [Obsolete("Use RunRenameFields in SpiderContext")]
        public void AddRenameFields(string from, string to)
        {
            this.AddPipeline(it =>
            {
                var jobject = JObject.FromObject(it);
                Helpers.Json.RenameProperty(jobject, from, to);
                it = jobject;

                return it;
            });
        }
    }
}

<|start_filename|>src/SpiderSharp/Nodes.cs<|end_filename|>
﻿using HtmlAgilityPack;
using HtmlAgilityPack.CssSelectors.NetCore;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using SpiderSharp.Helpers;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Xml;

namespace SpiderSharp
{
    public class Nodes
    {
        private HtmlNode doc;

        public Nodes(string sourceCode)
        {
            var document = new HtmlDocument();
            document.LoadHtml(sourceCode);

            this.doc = document.DocumentNode;
        }

        public Nodes(HtmlNode node)
        {
            this.doc = node;
        }

        public bool Exist(string cssSelector)
        {
            var node = doc.QuerySelector(cssSelector);
            return node != null;
        }


        public string GetAttribute(string attribute)
        {
            return doc?.Attributes[attribute]?.Value;
        }

        public string GetAttribute(string cssSelector, string attribute)
        {
            var node = doc.QuerySelector(cssSelector);
            return node?.Attributes[attribute]?.Value;
        }

        public string GetAttributeValue(string attribute)
        {
            return this.doc.Attributes[attribute]?.Value;
        }

        public string GetAttributeValue(string cssSelector, string attribute)
        {
            var node = doc.QuerySelector(cssSelector);
            return node?.Attributes[attribute]?.Value;
        }

        public string GetClass(string cssSelector)
        {
            return this.GetAttribute(cssSelector, "class");
        }

        public string GetHref()
        {
            return this.doc.Attributes["href"].Value;
        }

        public bool HasHref()
        {
            return this.doc.Attributes.Any(w => w.Name == "href");
        }

        public bool HasClass()
        {
            return this.doc.Attributes.Any(w => w.Name == "class");
        }

        public string GetHref(string cssSelector)
        {
            return GetAttribute(cssSelector, "href");
        }

        public string GetTitle(string cssSelector)
        {
            return GetAttribute(cssSelector, "title");
        }

        public Nodes GetParentNode()
        {
            return new Nodes(this.doc.ParentNode);
        }

        public string GetSrc(string cssSelector)
        {
            return GetAttribute(cssSelector, "src");
        }

        public string GetInnerText(string cssSelector)
        {
            var node = doc.QuerySelector(cssSelector);
            return System.Web.HttpUtility.HtmlDecode(node?.InnerText);
        }

        public string GetFirstChildInnerText(string cssSelector)
        {
            var node = doc.QuerySelector(cssSelector);
            return System.Web.HttpUtility.HtmlDecode(node?.FirstChild.InnerText);
        }

        public string GetInnerText()
        {
            return System.Web.HttpUtility.HtmlDecode(doc?.InnerText);
        }

        public string GetInnerHtml()
        {
            return System.Web.HttpUtility.HtmlDecode(doc?.InnerHtml);
        }

        public JObject GetJson()
        {
            JObject obj = new JObject();
            obj.Add(new JProperty(doc?.Name, HtmlNodeToJObject(doc)));

            return obj;
        }


        static JObject HtmlNodeToJObject(HtmlNode node)
        {
            JObject obj = new JObject();

            node?.ChildNodes.ToList().ForEach(a =>
            {
                // add childs
                if (a.NodeType == HtmlNodeType.Element)
                {
                    var no = HtmlNodeToJObject(a);
                    if (no.Children().Count() > 0)
                    {
                        // verifiy if is unique or array
                        if (node?.ChildNodes.Where(w => w.Name == a.Name).Count() > 1)
                        {
                            // is array

                            // verify if is first
                            if (obj.ContainsKey(a.Name))
                            {
                                // second, third
                                JArray arr = obj[a.Name] as JArray;
                                arr.Add(no);
                            }
                            else
                            {
                                // first
                                obj.Add(new JProperty(a.Name, new JArray(no)));
                            }
                        }
                        else
                        {
                            // is unique
                            obj.Add(new JProperty(a.Name, no));
                        }
                    }
                }
                else if (a.NodeType == HtmlNodeType.Text)
                {
                    if (!string.IsNullOrWhiteSpace(a.InnerText))
                    {
                        var ct = System.Web.HttpUtility.HtmlDecode(a.InnerText.Trim());
                        if (!obj.ContainsKey(a.Name))
                        {
                            obj.Add(new JProperty(a.Name, ct));
                        }
                        else
                        {
                            obj.Add(new JProperty(a.Name + "_" + obj.Count, ct));
                        }
                    }
                }
            });

            // add attributes
            node?.Attributes.ToList().ForEach(a => {

                if (!string.IsNullOrWhiteSpace(a.Value))
                {
                    string ct = System.Web.HttpUtility.HtmlDecode(a.Value.ToString());
                    // verifiy if is unique or array
                    if (node?.Attributes.Where(w => w.Name == a.Name).Count() > 1)
                    {
                        // is array

                        // verify if is first
                        if (obj.ContainsKey($"@{a.Name}"))
                        {
                            // second, third
                            JArray arr = obj[$"@{a.Name}"] as JArray;
                            arr.Add(ct);
                        }
                        else
                        {
                            // first
                            obj.Add(new JProperty($"@{a.Name}", new JArray(ct)));
                        }
                    }
                    else
                    {
                        // is single
                        obj.Add(new JProperty($"@{a.Name}", ct));
                    }
                }
            });

            return obj;
        }


        public string GetOuterHtml()
        {
            return System.Web.HttpUtility.HtmlDecode(doc?.OuterHtml);
        }

        public string GetHtml()
        {
            return doc.OwnerDocument.DocumentNode.OuterHtml;
        }

        public IEnumerable<string> GetLinksAll(string cssSelector)
        {
            IList<HtmlNode> nodes = doc.QuerySelectorAll(cssSelector);
            return nodes
                .Select(s => s.GetAttributeValue("href", null))
                .Where(w => w != null);
        }

        public IEnumerable<KeyValuePair<string, string>> GetLinksEnumerable(string cssSelector)
        {
            IList<HtmlNode> nodes = doc.QuerySelectorAll(cssSelector);
            return nodes
                .Where(w => w.GetAttributeValue("href", null) != null)
                .Select(s => new KeyValuePair<string, string>(s.GetAttributeValue("href", null), s.InnerText));
        }

        public IEnumerable<string> SelectAttribute(string cssSelector, string attribute)
        {
            var nodes = doc.QuerySelectorAll(cssSelector);
            return nodes?.Select(s => s.Attributes[attribute].Value);
        }

        public IEnumerable<string> SelectHref(string cssSelector)
        {
            var nodes = doc.QuerySelectorAll(cssSelector);
            return nodes?
                .Where(w => w.Attributes.Contains("href"))
                .Select(s => s.Attributes["href"].Value);
        }

        public IEnumerable<string> SelectInnerText(string cssSelector)
        {
            var nodes = doc.QuerySelectorAll(cssSelector);
            return nodes?.Select(s => System.Web.HttpUtility.HtmlDecode(s.InnerText));
        }

        public IEnumerable<Nodes> SelectNodes(string cssSelector)
        {
            return doc.QuerySelectorAll(cssSelector)
                .Select(s => new Nodes(s));
        }

        public IEnumerable<Nodes> XSelectNodes(string xpath)
        {
            return doc.SelectNodes(xpath)
                .Select(s => new Nodes(s));
        }

        public Nodes GetNode(string cssSelector)
        {
            var node = doc.QuerySelector(cssSelector);
            return new Nodes(node);
        }
    }
}

<|start_filename|>src/SpiderSharp.Test/LoadWebsiteTest.cs<|end_filename|>
﻿using System;
using Microsoft.VisualStudio.TestTools.UnitTesting;

namespace SpiderSharp.Text
{
    [TestClass]
    public class LoadWebsiteTest
    {
        [TestMethod]
        public void LoadTest()
        {
            // https://www.aliexpress.com/country/us-united-states.html

        }
    }
}


<|start_filename|>src/SpiderSharp/Helpers/Collection.cs<|end_filename|>
﻿using System.Collections.Generic;
using System.Linq;

namespace SpiderSharp.Helpers
{
    public static class Collection
    {
        public static T GetLastN<T>(IEnumerable<T> ienumerable, int n)
        {
            return ienumerable.ElementAt(ienumerable.Count() - n);
        }

        public static IEnumerable<T> RemoveFirst<T>(IEnumerable<T> ienumerable)
        {
            var l = ienumerable.ToList();
            l.RemoveAt(0);
            return l;
        }
    }
}

<|start_filename|>src/SpiderSharp/SpiderContext.cs<|end_filename|>
﻿using System;
using System.Collections.Generic;
using System.Dynamic;
using System.Text;
using Newtonsoft.Json.Linq;

namespace SpiderSharp
{
    public partial class SpiderContext
    {
        public dynamic Bag { get; set; }
        public dynamic Data { get; set;  }
        public Exception Error { get; set;  }
        public bool HasError { get { return this.Error != null; } }

        public string Url { get; set; }

        public string Spider { get; set; }

        public SpiderContext()
        {
            //dynamic json = new ExpandoObject();
            //this.Data = JObject.FromObject(new ExpandoObject());
            this.Data = new ExpandoObject();
        }

        public override string ToString()
        {
            if (this.Data != null)
                return this.Data.ToString();
            else
                return null;
        }
    }
}


<|start_filename|>src/SpiderSharp/RunPipelines/RunPrintToConsolePipeline.cs<|end_filename|>
﻿using Humanizer;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

namespace SpiderSharp
{
    public partial class SpiderContext
    {
        public void RunPrintToConsolePipeline(params string[] fields)
        {
            var it = this.Data;

            if (fields == null || fields.Count() == 0)
            {
                JToken obj = JToken.FromObject(it);
                Console.WriteLine(obj);
            }
            else
            {
                foreach (var item in fields)
                {
                    if (it[item] != null)
                        Console.WriteLine($"{item} => {it[item].Value}");
                }
            }
        }
    }
}

<|start_filename|>src/SpiderSharp/ISpiderEngine.cs<|end_filename|>
﻿using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;

namespace SpiderSharp
{
    public interface ISpiderEngine
    {
        Task<bool> RunAsync();
        void AddBag(string key, object value);
        void SetUrl(string url);
        void SetNofollow();
    }
}

<|start_filename|>src/SpiderSharp/CommonSetings.cs<|end_filename|>
﻿using System;
using System.Collections.Generic;
using System.Text;

namespace SpiderSharp
{
    public class CommonSetings
    {
    }
}


<|start_filename|>src/SpiderSharp/AddPipelines/AddSaveToElasticSearchAsyncPipeline.cs<|end_filename|>
﻿using DataFoundation.ElasticSearch;

using Newtonsoft.Json.Linq;

using System;

namespace SpiderSharp
{
    public abstract partial class SpiderEngine
    {
        [Obsolete("Use RunSaveToElasticSearchAsyncPipeline in SpiderContext")]
        public void AddSaveToElasticSearchAsyncPipeline(string index, string primaryKeyField)
        {
            this.AddPipeline(it =>
            {
                var elastic = new ElasticConnection(index, GlobalSettings.ElasticSearchConnectionString);
                JObject obj = JObject.Parse(it.ToString());
                var id = (string)it[primaryKeyField];
                elastic.Index(id, obj.ToString());

                return it;
            });
        }
    }
}

<|start_filename|>src/SpiderSharp/Helpers/Html.cs<|end_filename|>
﻿using System;
using System.Text.RegularExpressions;

namespace SpiderSharp.Helpers
{
    public static class Html
    {
        public static Nodes TryParse(string content)
        {
            try
            {
                return new Nodes(content);
            }
            catch (Exception ex)
            {
            }

            return null;
        }

        /// <summary>
        /// 
        /// </summary>
        /// <see cref="https://stackoverflow.com/questions/18153998/how-do-i-remove-all-html-tags-from-a-string-without-knowing-which-tags-are-in-it"/>
        /// <param name="input"></param>
        /// <returns></returns>
        public static string StripHtml(string input)
        {
            return Regex.Replace(input, "<.*?>", String.Empty);
        }
    }
}

<|start_filename|>src/SpiderSharp/AddPipelines/AddDasherizePipeline.cs<|end_filename|>
﻿using Humanizer;

using Newtonsoft.Json.Linq;

using System;

namespace SpiderSharp
{
    public abstract partial class SpiderEngine
    {
        [Obsolete("Use RunDasherizePipeline in SpiderContext")]
        public void AddDasherizePipeline()
        {
            this.AddPipeline(it =>
            {
                JObject obj = JObject.FromObject(it);
                Helpers.Json.Rename(obj, name => name.ToString().Underscore().Replace("_", "-"));
                return obj;
            });
        }
    }
}

<|start_filename|>src/SpiderSharp/RunPipelines/RunEmbedMetadataPipeline.cs<|end_filename|>
﻿using Newtonsoft.Json.Linq;

using System;

namespace SpiderSharp
{
    public partial class SpiderContext
    {
        public void RunEmbedMetadataPipeline(string exceptionProperty = "exception", string urlProperty = "url", string nowProperty = "now", string spiderName = "spider")
        {
            JObject obj = JObject.FromObject(this.Data);

            if (this.Error != null)
                obj.Add(exceptionProperty, JObject.FromObject(this.Error));

            if (!string.IsNullOrEmpty(this.Url))
                obj.Add(urlProperty, this.Url);

            obj.Add(spiderName, this.Spider);
            obj.Add(nowProperty, DateTime.Now);

            this.Data = obj;
        }
    }
}

<|start_filename|>src/SpiderSharp/RunPipelines/RunDasherizePipeline.cs<|end_filename|>
﻿using Humanizer;

using Newtonsoft.Json.Linq;

namespace SpiderSharp
{
    public partial class SpiderContext
    {
        public void RunDasherizePipeline()
        {
            var it = this.Data;
            JObject obj = JObject.FromObject(it);
            Helpers.Json.Rename(obj, name => name.ToString().Underscore().Replace("_", "-"));
            this.Data = obj;
        }
    }
}

<|start_filename|>src/SpiderSharp/RunPipelines/RunSaveToMongoDBPipeline.cs<|end_filename|>
﻿using DataFoundation.MongoDB;
using Humanizer;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;

namespace SpiderSharp
{
    public partial class SpiderContext
    {
        public async Task RunSaveToMongoDBAsyncPipelineAsync(string collection, string primaryKeyField)
        {
            var it = this.Data;
            
            try
            {
                var mongo = new MongoConnection(GlobalSettings.MongoDatabase, GlobalSettings.MongoConnectionString);
                var pkValue = (string)it[primaryKeyField];
                await mongo.UpdateDefinitionAsync(collection, primaryKeyField, pkValue, it.ToString());
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
            }
        }
    }
}

<|start_filename|>src/SpiderSharp.Test/RegexTest.cs<|end_filename|>
﻿using System;
using Microsoft.VisualStudio.TestTools.UnitTesting;

namespace SpiderSharp.Text
{
    [TestClass]
    public class RegexTest
    {
        [TestMethod]
        public void RemoveNthChildTest()
        {
            var clear = SpiderSharp.Helpers.RegexHelper.RemoveAllNthChild(@"$$("".list-unstyled > li:nth-child(3) > div:nth-child(2) > ul:nth-child(3) > li > a"")");
            Assert.AreEqual(clear, @"$$("".list-unstyled > li > div > ul > li > a"")");

        }
    }
}


<|start_filename|>custom-deploy.cmd<|end_filename|>
REM https://docs.microsoft.com/pt-br/dotnet/core/tools/custom-templates
rd /s /q .\src\SpiderSharp\bin\Release
REM dotnet build -c Release .\src\SpiderSharp\SpiderSharp.csproj
dotnet build -c Release .\SpiderSharp.sln
nuget pack SpiderSharp.nuspec
nuget push .\*.nupkg -Source https://api.nuget.org/v3/index.json
del /f /s /q .\*.nupkg

<|start_filename|>src/SpiderSharp/AddPipelines/AddYoutubeDetailPipeline.cs<|end_filename|>
﻿using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using Newtonsoft.Json.Linq;

using System;

using YoutubeExplode;

namespace SpiderSharp
{
    public abstract partial class SpiderEngine
    {
        [Obsolete("Use RunYoutubeDetailPipeline in SpiderContext")]
        public void AddYoutubeDetailPipeline(string urlField, string detailField, string videoField, string streamField, string captionField)
        {
            this.AddPipeline(it =>
            {
                JObject obj = JObject.FromObject(it);
                var urlVideo = obj[urlField].ToString();

                var client = new YoutubeClient();
                var id = YoutubeClient.ParseVideoId(urlVideo); // "bnsUkE8i0tU"

                obj[detailField] = new JObject();
                if (!string.IsNullOrEmpty(videoField))
                {
                    var video = client.GetVideoAsync(id).Result;
                    var json1 = JsonConvert.SerializeObject(video, new StringEnumConverter());
                    obj[detailField][videoField] = JObject.Parse(json1);
                }

                if (!string.IsNullOrEmpty(streamField))
                {
                    var streamInfoSet = client.GetVideoMediaStreamInfosAsync(id).Result;
                    var json2 = JsonConvert.SerializeObject(streamInfoSet, new StringEnumConverter());
                    obj[detailField][streamField] = JObject.Parse(json2);
                }

                if (!string.IsNullOrEmpty(captionField))
                {
                    var caption = client.GetVideoClosedCaptionTrackInfosAsync(id).Result;
                    var json3 = JsonConvert.SerializeObject(caption, new StringEnumConverter());
                    obj[detailField][captionField] = JArray.Parse(json3);
                }

                //var streamInfo = (YoutubeExplode.Models.MediaStreams.MuxedStreamInfo)streamInfoSet.Muxed.OrderByDescending(o => o.Size).FirstOrDefault();

                //var tmp = $"{Path.GetTempFileName()}.{streamInfo.Container.ToString().ToLower().Trim()}";
                //var tas = client.DownloadMediaStreamAsync(streamInfo, tmp);
                //tas.Wait();

                return obj;
            });
        }

        public void AddYoutubeDetailPipeline(string urlField, string detailField)
        {
            this.AddYoutubeDetailPipeline(urlField, detailField, "video", "stream", "caption");
        }

        public void AddYoutubeDetailPipeline(string urlField)
        {
            this.AddYoutubeDetailPipeline(urlField, urlField + "-detail");
        }
    }
}

<|start_filename|>src/SpiderSharp/AddPipelines/AddPipeline.cs<|end_filename|>
﻿using System;

namespace SpiderSharp
{
    public abstract partial class SpiderEngine
    {
        [Obsolete]
        public void AddPipeline(Func<dynamic, dynamic> act)
        {
            this.pipelines.Add(act);
        }
    }
}

<|start_filename|>src/SpiderSharp/RunPipelines/RunHtmlDecodePipeline.cs<|end_filename|>
﻿using Newtonsoft.Json.Linq;

using System.Linq;

namespace SpiderSharp
{
    public partial class SpiderContext
    {
        public void RunHtmlDecodePipeline()
        {
            var it = this.Data;
            JObject json = JObject.FromObject(it);

            foreach (var token in json.Properties().Select(p => p.Name))
            {
                JProperty prop = json.Property(token);

                if (prop != null && prop.HasValues && prop.Type == JTokenType.Property && prop.Value.Type == JTokenType.String)
                {
                    prop.Value = System.Web.HttpUtility.HtmlDecode(prop.Value.ToString());
                }
            }

            this.Data = json;
        }
    }
}

<|start_filename|>src/SpiderSharp/RunPipelines/RunYoutubeDetailPipeline.cs<|end_filename|>
﻿using System;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using Newtonsoft.Json.Linq;
using YoutubeExplode;
using YoutubeExplode.Models;
using YoutubeExplode.Models.ClosedCaptions;
using YoutubeExplode.Models.MediaStreams;

namespace SpiderSharp
{
    public partial class SpiderContext
    {
        public void RunYoutubeDetailPipeline(string urlField, string detailField, string videoField, string streamField, string captionField)
        {
            var it = this.Data;
            JObject obj = JObject.FromObject(it);
            var urlVideo = obj[urlField].ToString();

            var client = new YoutubeClient();
            var id = YoutubeClient.ParseVideoId(urlVideo); // "bnsUkE8i0tU"

            obj[detailField] = new JObject();
            if (!string.IsNullOrEmpty(videoField))
            {
                var video = client.GetVideoAsync(id).Result;
                var json1 = JsonConvert.SerializeObject(video, new StringEnumConverter());
                obj[detailField][videoField] = JObject.Parse(json1);
            }

            if (!string.IsNullOrEmpty(streamField))
            {
                var streamInfoSet = client.GetVideoMediaStreamInfosAsync(id).Result;
                var json2 = JsonConvert.SerializeObject(streamInfoSet, new StringEnumConverter());
                obj[detailField][streamField] = JObject.Parse(json2);
            }

            if (!string.IsNullOrEmpty(captionField))
            {
                var caption = client.GetVideoClosedCaptionTrackInfosAsync(id).Result;
                var json3 = JsonConvert.SerializeObject(caption, new StringEnumConverter());
                obj[detailField][captionField] = JArray.Parse(json3);
            }
        }

        public void RunYoutubeDetailPipeline(string urlField, string detailField)
        {
            this.RunYoutubeDetailPipeline(urlField, detailField, "video", "stream", "caption");
        }

        public void RunYoutubeDetailPipeline(string urlField)
        {
            this.RunYoutubeDetailPipeline(urlField, urlField + "-detail");
        }
    }
}


<|start_filename|>sample/ScrapQuotes/Program.cs<|end_filename|>
﻿using System;
using SpiderSharp;

namespace ScrapQuotes
{
    class Program
    {
        static void Main(string[] args)
        {
            ScrapQuotesSpider spider = new ScrapQuotesSpider();
            spider.RunAsync().Wait();
        }
    }
}


<|start_filename|>src/SpiderSharp/GlobalSettings.cs<|end_filename|>
﻿using System;
using System.Collections.Generic;
using System.Text;
using HttpRequester;

namespace SpiderSharp
{
    public static class GlobalSettings
    {
        public static string RedisPassword { get; set; }
        public static string RedisConnectionString { get; set; }
        public static string MongoConnectionString { get; set; }
        public static string ElasticSearchConnectionString { get; set; }
        public static string MongoDatabase { get; set; }
        public static bool? UseRedisCache { get; set; }
        public static TimeSpan? RedisCacheDuration { get; set; }
        public static EnumHttpProvider? HttpProvider { get; set; }
        //public static Dictionary<string, string> DefaultHeaders = new Dictionary<string, string>();
    }
}


<|start_filename|>app/ScrapShell/Program.cs<|end_filename|>
﻿using AngleSharp.Dom;
using Newtonsoft.Json.Linq;
using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Text.RegularExpressions;

namespace ScrapShell
{
    public class Program
    {
        static HttpRequester.RequesterCached requester;
        static string content;
        static IElement doc;
        static CommandMenu currentMenu;
        static string filename;

        static void LoadCommand(Match match)
        {
            var url = match.Groups["url"].Value;

            if (File.Exists(url))
            {
                content = System.IO.File.ReadAllText(url);
            }
            else
            {
                if (requester != null)
                {
                    content = requester.GetAsync(url).Result.StringContent;
                }
                else
                {
                    HttpRequester.Engine.ChromePersistentClient httpRequester = new HttpRequester.Engine.ChromePersistentClient(true);
                    content = httpRequester.GetContentAsync(url).Result;
                }
            }

            filename = System.IO.Path.GetTempFileName() + ".txt";
            System.IO.File.WriteAllText(filename, content);

            doc = SpiderSharp.AngleDocument.TryParse(content)?.DocumentElement;

            var msg = $"{content.Count()} bytes loaded";
            var m  = nameof(msg);
            Console.WriteLine(msg);
        }

        static void SetCommand(Match match)
        {
            switch (match.Groups["driver"].Value)
            {
                case "anglesharp":
                    requester = new HttpRequester.RequesterCached(HttpRequester.EnumHttpProvider.AngleSharp);
                    Console.WriteLine("AngleSharp has been set");
                    break;

                case "httpclient":
                    requester = new HttpRequester.RequesterCached(HttpRequester.EnumHttpProvider.HttpClient);
                    Console.WriteLine("HttpClient has been set");
                    break;
            }
        }

        static void SaveCommand(Match match)
        {
            var filename = match.Groups["filename"].Value;

            if (string.IsNullOrEmpty(content))
            {
                Console.WriteLine("You must load some url before execute save command");
            }
            else
            {
                if (string.IsNullOrEmpty(filename))
                {
                    filename = Path.GetTempFileName() + ".html";
                }

                System.IO.File.WriteAllText(filename, content);
                Console.WriteLine($"Saved {filename}");
            }
        }

        static void HelpCommand(Match match)
        {
            currentMenu.PrintHelp();
        }

        static void BrowseCommand(Match match)
        {
            if (string.IsNullOrEmpty(content))
            {
                Console.WriteLine("You must load some content before execute open command");
            }
            else
            {
                var tmp = System.IO.Path.GetTempFileName() + ".html";
                System.IO.File.WriteAllText(tmp, content);
                OpenBrowser(tmp);

                Console.WriteLine("Browser opened");
            }
        }

        static void NotepadCommand(Match match)
        {
            if (string.IsNullOrEmpty(filename))
            {
                Console.WriteLine("You must load some content before execute open command");
            }
            else
            {
                OpenNotepad(filename);
                Console.WriteLine("notepad opened");
            }
        }

        static void ClearCommand(Match match)
        {
            Console.Clear();
        }

        static void InnerTextCommand(Match match)
        {
            var selector = match.Groups["selector"].Value;
            if (string.IsNullOrEmpty(selector))
            {
                Console.WriteLine("You must type the selector");
            }
            else
            {
                StringBuilder sb = new StringBuilder();

                var res = doc.QuerySelectorAll(selector);
                res.ToList().ForEach(a => sb.AppendLine(a.TextContent));
                sb.AppendLine($"{res.Count()} itens found");
                Console.WriteLine(sb.ToString());

                filename = Path.GetTempFileName() + ".txt";
                System.IO.File.WriteAllText(filename, sb.ToString());
            }
        }

        static void AttributeCommand(Match match)
        {
            var selector = match.Groups["selector"].Value;
            if (string.IsNullOrEmpty(selector))
            {
                Console.WriteLine("You must type the selector");
            }
            else
            {
                var att = match.Groups["attribute"].Value;

                StringBuilder sb = new StringBuilder();
                var res = doc.QuerySelectorAll(selector);
                res.Where(w => w.HasAttribute(att)).ToList().ForEach(a => sb.AppendLine(a.Attributes[att].Value));
                sb.AppendLine($"{res.Where(w => w.HasAttribute(att)).Count()} itens found");
                Console.WriteLine(sb.ToString());

                filename = Path.GetTempFileName() + ".txt";
                System.IO.File.WriteAllText(filename, sb.ToString());
            }
        }

        static void InnerHtmlCommand(Match match)
        {
            var selector = match.Groups["selector"].Value;
            if (string.IsNullOrEmpty(selector))
            {
                Console.WriteLine("You must type the selector");
            }
            else
            {
                StringBuilder sb = new StringBuilder();

                var res = doc.QuerySelectorAll(selector);
                res.ToList().ForEach(a => sb.AppendLine(a.InnerHtml));
                sb.AppendLine($"{res.Count()} itens found");
                Console.WriteLine(sb.ToString());

                filename = Path.GetTempFileName() + ".txt";
                System.IO.File.WriteAllText(filename, sb.ToString());
            }
        }

        static void OuterHtmlCommand(Match match)
        {
            var selector = match.Groups["selector"].Value;
            if (string.IsNullOrEmpty(selector))
            {
                Console.WriteLine("You must type the selector");
            }
            else
            {
                StringBuilder sb = new StringBuilder();

                var res = doc.QuerySelectorAll(selector);
                res.ToList().ForEach(a => sb.AppendLine(a.OuterHtml));
                sb.AppendLine($"{res.Count()} itens found");
                Console.WriteLine(sb.ToString());

                filename = Path.GetTempFileName() + ".txt";
                System.IO.File.WriteAllText(filename, sb.ToString());
            }
        }

        static void JsonCommand(Match match)
        {
            var selector = match.Groups["selector"].Value;
            if (string.IsNullOrEmpty(selector))
            {
                Console.WriteLine("You must type the selector");
            }
            else
            {
                var res = doc.QuerySelectorAll(selector);

                JArray arr = new JArray();
                var sb = new StringBuilder();
                res.ToList().ForEach(a => {
                    var json = a.GetJson(false);
                    arr.Add(json);
                    sb.AppendLine(json.ToString());
                });

                sb.AppendLine($"{res.ToList().Count} itens found");
                Console.WriteLine(sb.ToString());

                filename = Path.GetTempFileName() + ".json";
                System.IO.File.WriteAllText(filename, sb.ToString());
            }
        }

        static void PathCommand(Match match)
        {
            var selector = match.Groups["selector"].Value;
            if (string.IsNullOrEmpty(selector))
            {
                Console.WriteLine("You must type the selector");
            }
            else
            {
                var res = doc.QuerySelectorAll(selector);

                JArray arr = new JArray();
                var sb = new StringBuilder();
                res.ToList().ForEach(a => {
                    var json = a.GetJson(true);
                    arr.Add(json);
                    sb.AppendLine(json.ToString());
                });

                sb.AppendLine($"{res.ToList().Count} itens found");
                Console.WriteLine(sb.ToString());

                filename = Path.GetTempFileName() + ".json";
                System.IO.File.WriteAllText(filename, sb.ToString());
            }
        }

        static void LinksCommand(Match match)
        {
            var selector = match.Groups["selector"].Value;
            if (string.IsNullOrEmpty(selector))
            {
                Console.WriteLine("You must type the selector");
            }
            else
            {
                StringBuilder sb = new StringBuilder();

                var res = doc.QuerySelectorAll(selector);
                res.ToList().ForEach(a => sb.AppendLine(a.ToString()));
                sb.AppendLine($"{res.Count()} itens found");
                Console.WriteLine(sb.ToString());

                filename = Path.GetTempFileName() + ".txt";
                System.IO.File.WriteAllText(filename, sb.ToString());
            }
        }
        
        static void Main(string[] args)
        {
            ReadLine.AutoCompletionHandler = new AutoCompletionHandler();

            var mainMenu = new CommandMenu("Main");
            mainMenu.AddCommand("set (?<driver>anglesharp|httpclient)", SetCommand, "set => set httpclient or anglesharp driver to request");
            mainMenu.AddCommand("load (?<url>.*)", LoadCommand, "load => load url or local file");
            mainMenu.AddCommand("save (?<filename>.*)", SaveCommand, "save => save content to local file");
            mainMenu.AddCommand("innertext (?<selector>.*)", InnerTextCommand, "innertext => select innertext using css selector");
            mainMenu.AddCommand("attribute (?<attribute>.*?) (?<selector>.*)", AttributeCommand, "attribute => select attribute value using css selector");
            mainMenu.AddCommand("innerhtml (?<selector>.*)", InnerHtmlCommand, "innerhtml => select innerhtml using css selector");
            mainMenu.AddCommand("outerhtml (?<selector>.*)", OuterHtmlCommand, "outerhtml => select outerhtml using css selector");
            mainMenu.AddCommand("links (?<selector>.*)", LinksCommand, "links => select links using css selector");
            mainMenu.AddCommand("json (?<selector>.*)", JsonCommand, "json => select links using css selector");
            mainMenu.AddCommand("path (?<selector>.*)", PathCommand, "path => select links using css selector and print json path");
            mainMenu.AddCommand("cls|clear", ClearCommand, "cls => clear screen");
            mainMenu.AddCommand("browse", BrowseCommand, "browse => open browser with content");
            mainMenu.AddCommand("notepad", NotepadCommand, "notepad => open notepad with content");
            mainMenu.AddCommand("help", HelpCommand, "help => show all commands");
            mainMenu.AddExitCommand("exit|quit|back", "quit => exit program");
            mainMenu.PrintHelp();
            Console.WriteLine(string.Empty);
            currentMenu = mainMenu;
            mainMenu.Execute();
        }

        static void OpenBrowser(string filename)
        {
            bool hasRegistry = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
            if (hasRegistry)
            {
                ChromeLauncher.OpenFile(filename);
            }
            else
            {
                System.Console.WriteLine("Platform not supported");
            }
        }

        static void OpenNotepad(string filename)
        {
            bool hasRegistry = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
            if (hasRegistry)
            {
                Process.Start(@"c:\windows\explorer.exe", filename);
            }
            else
            {
                System.Console.WriteLine("Platform not supported");
            }
        }
    }

    class AutoCompletionHandler : IAutoCompleteHandler
    {
        // characters to start completion from
        public char[] Separators { get; set; } = new char[] { ' ', '.', '/' };

        // text - The current text entered in the console
        // index - The index of the terminal cursor within {text}
        public string[] GetSuggestions(string text, int index)
        {
            if (text.StartsWith("set "))
                return new string[] { "anglesharp", "httpclient" };
            else
                return null;
        }
    }
}


<|start_filename|>src/SpiderSharp/Helpers/RegexLibrary.cs<|end_filename|>
﻿using System.Text.RegularExpressions;

namespace SpiderSharp.Helpers
{
    public static class RegexLibrary
    {
        /// <summary>
        /// 
        /// </summary>
        /// <see cref="https://stackoverflow.com/questions/19233413/how-to-parse-multiple-cookies-values-from-set-cookie-contains-commas-and-semico"/>
        public static readonly Regex Cookies = new Regex(@"(.*?)=(.*?)($|;|,(?! ))", RegexOptions.Singleline);

        /// <summary>
        /// 
        /// </summary>
        /// <see cref="https://stackoverflow.com/questions/6038061/regular-expression-to-find-urls-within-a-string"/>
        public static readonly Regex Url = new Regex(@"(?:(?:https?|ftp):\/\/|\b(?:[a-z\d]+\.))(?:(?:[^\s()<>]+|\((?:[^\s()<>]+|(?:\([^\s()<>]+\)))?\))+(?:\((?:[^\s()<>]+|(?:\(?:[^\s()<>]+\)))?\)|[^\s`!()\[\]{};:'"".,<>?«»“”‘’]))?", RegexOptions.Singleline);
    }
}

<|start_filename|>app/ScrapShell/CommandMenu.cs<|end_filename|>
﻿using System;
using System.Collections.Generic;
using System.Text.RegularExpressions;

namespace ScrapShell
{
    class CommandItem
    {
        public Regex Regex { get; set; }
        public Action<Match> Action { get; set; }
        public string Description { get; set; }
    }

    public class CommandMenu
    {
        Dictionary<string, CommandItem> dict;
        string prefix;
        public delegate void CommandNotFound();
        public event CommandNotFound OnCommandNotFound;

        public CommandMenu(string menuName)
        {
            prefix = menuName;
            dict = new Dictionary<string, CommandItem>();
        }

        private void CommandMenu_OnCommandNotFound()
        {
            Console.WriteLine("Command not found");
        }

        public void AddCommand(string regex, Action<Match> action, string description = null)
        {
            if (action == null)
                throw new Exception("The action cannot be null");

            if (CheckCollide(regex))
                throw new Exception($"The '{regex}' will collide has other command");

            var reg = new Regex(regex, RegexOptions.IgnoreCase);
            dict.Add(regex, new CommandItem { Regex = reg, Action = action, Description = description });
        }

        public static string StripNameGroups(string regex)
        {
            return Regex.Replace(regex, @"\?<.*?>", String.Empty);
        }

        bool Collide(string regex1, string regex2)
        {
            var regexStriped1 = StripNameGroups(regex1);
            var regexStriped2 = StripNameGroups(regex2);

            var automaton1 = new Fare.RegExp(regexStriped1).ToAutomaton();
            var automaton2 = new Fare.RegExp(regexStriped2).ToAutomaton();

            var intersection = automaton1.Intersection(automaton2);

            var random1 = new Random(Environment.TickCount);
            var sut1 = new Fare.Xeger(regexStriped1, random1);

            var random2 = new Random(Environment.TickCount);
            var sut2 = new Fare.Xeger(regexStriped2, random2);

            var gen1 = sut1.Generate();
            var gen2 = sut2.Generate();

            var result1 = intersection.Run(gen1);
            var result2 = intersection.Run(gen2);

            return result1 || result2;
        }

        bool CheckCollide(string regex)
        {
            foreach (var item in dict)
            {
                if (Collide(item.Key, regex))
                    return true;
            }

            return false;
        }

        public void AddExitCommand(string regex, string description = null)
        {
            dict.Add(regex, new CommandItem { Regex = new Regex(regex, RegexOptions.IgnoreCase), Action = null, Description = description });
        }

        public void PrintHelp(char separator = '=')
        {
            if (!string.IsNullOrEmpty(prefix))
            {
                Console.WriteLine(prefix);
                Console.WriteLine(new String(separator, prefix.Length));
            }

            foreach (var item in dict)
            {
                Console.WriteLine(item.Value.Description ?? item.Key);
            }
        }

        public void Execute()
        {
            ReadLine.HistoryEnabled = true;

            while (true)
            {
                var isValid = false;

                var restoreColor = Console.ForegroundColor;

                Console.ForegroundColor = ConsoleColor.Green;
                Console.Write($"{prefix} # ");
                Console.ForegroundColor = ConsoleColor.White;
                string input = ReadLine.Read();

                Console.ForegroundColor = restoreColor;

                foreach (var item in dict)
                {
                    if (item.Value.Regex.IsMatch(input))
                    {
                        isValid = true;
                        if (item.Value.Action == null)
                            return;

                        var match = item.Value.Regex.Match(input);
                        try
                        {
                            item.Value.Action(match);
                        }
                        catch (Exception ex)
                        {
                            Console.WriteLine(ex);
                        }

                        break;
                    }
                }

                if (!string.IsNullOrWhiteSpace(input))
                {
                    if (!isValid)
                    {
                        if (OnCommandNotFound == null)
                            OnCommandNotFound += CommandMenu_OnCommandNotFound;

                        OnCommandNotFound();
                    }

                    //Console.WriteLine(string.Empty);   
                }
            }
        }
    }
}


<|start_filename|>src/SpiderSharp/AddPipelines/AddSafeUrlsPipeline.cs<|end_filename|>
﻿using Newtonsoft.Json.Linq;

using System;

namespace SpiderSharp
{
    public abstract partial class SpiderEngine
    {
        [Obsolete("Use RunSafeUrlsPipeline in SpiderContext")]
        public void AddSafeUrlsPipeline(string domain, params string[] tokens)
        {
            this.AddPipeline(it =>
            {
                JObject json = JObject.FromObject(it);

                foreach (var token in tokens)
                {
                    JProperty prop = json.Property(token);

                    if (prop != null && prop.HasValues)
                    {
                        string link = prop.Value.ToString().ToLower();

                        if (!string.IsNullOrEmpty(link))
                        {
                            domain = domain.EndsWith("/") ? domain : domain + "/";

                            // add schema/domain/port if needed
                            if (!link.StartsWith(domain.ToLower().Trim()))
                            {
                                var concat = prop.Value.ToString();
                                if (concat.StartsWith("/"))
                                {
                                    concat = concat.Substring(1);
                                }

                                link = domain + concat;

                                Uri uri = null;
                                if (Uri.TryCreate(link, UriKind.RelativeOrAbsolute, out uri))
                                {
                                    prop.Value = uri.ToString();
                                }
                            }
                        }
                    }
                }

                dynamic dyn = json;
                it = dyn;

                return dyn;
            });
        }
    }
}

<|start_filename|>sample/ScrapQuotes/ScrapQuotesSpider.cs<|end_filename|>
﻿using System;
using System.Collections.Generic;
using System.Text;
using SpiderSharp;
using System.Dynamic;
using System.Threading.Tasks;

namespace ScrapQuotes
{
    public class ScrapQuotesSpider : SpiderEngine, ISpiderEngine
    {
        public ScrapQuotesSpider()
        {
            this.SetUrl("http://quotes.toscrape.com");
        }

        protected override string FollowPage()
        {
            return $"http://quotes.toscrape.com{this.node.GetHref("ul > li > a")}";
        }

        protected override IEnumerable<SpiderContext> OnRun()
        {
            var quotes = this.node.SelectNodes("div.quote");

            foreach (var item in quotes)
            {
                yield return this.Fetch(() => {
                    ct.Data.text = item.GetInnerText("span.text");
                    ct.Data.author = item.GetInnerText("small.author");
                    ct.Data.tags = item.SelectInnerText("div.tags > a.tag");
                });
            }
        }

        protected async override Task SuccessPipelineAsync(SpiderContext context)
        {
            context.RunPrintToConsolePipeline();
        }

        protected async override Task ErrorPipelineAsync(SpiderContext context)
        {
            context.RunEmbedMetadataPipeline();
            context.RunPrintToConsolePipeline();
        }
    }
}


<|start_filename|>src/SpiderSharp/SpiderEngine.cs<|end_filename|>
﻿using EnsureThat;
using Humanizer;
using Newtonsoft.Json.Linq;
using DataFoundation.MongoDB;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Dynamic;
using Newtonsoft.Json;
using System.Threading.Tasks;
using Serilog;
using System.Diagnostics;
using AngleSharp.Html.Dom;

namespace SpiderSharp
{
    public abstract partial class SpiderEngine
    {
        public DownloaderMiddleware Downloader = new DownloaderMiddleware(GlobalSettings.HttpProvider ?? HttpRequester.EnumHttpProvider.HttpClient);
        readonly List<Func<dynamic, dynamic>> pipelines = new List<Func<dynamic, dynamic>>();
        string sourceCode;
        string url;
        bool nofollow;
        public string Cookies { get; private set; }

        protected Nodes node;

        protected IHtmlDocument AngleDocument { get; set; }

        protected JToken Json { get; set; }

        public string SpiderName { get; private set; }

        private Dictionary<string, object> ViewBag = new Dictionary<string, object>();
        
        public void AddBag(string key, object value)
        {
            if (ViewBag.ContainsKey(key))
                ViewBag[key] = value;
            else
                ViewBag.Add(key, value);

            if (ct != null)
                ct.Bag = JObject.FromObject(ViewBag);
        }

        protected SpiderContext ct;

        public SpiderEngine()
        {
            this.SpiderName = this.GetType().Name.Underscore().Replace("_", "-");
        }

        public SpiderEngine(string url)
        {
            this.SpiderName = this.GetType().Name.Underscore().Replace("_", "-");
            this.SetUrl(url);
        }

        protected virtual void After(dynamic jObject)
        {
            // Executed after Run()
            foreach (Func<dynamic, dynamic> item in pipelines)
            {
                Console.WriteLine("Executing Pipeline > " + item.Method.Name.ToString());
                jObject = item(jObject);
            }
        }

        protected async virtual Task BeforeAsync()
        {
            // Executed before Run()
            System.Console.WriteLine("Executing Middlewares");

            Ensure.That(url).IsNotNullOrWhiteSpace();
            // Ensure.That(downloader).IsNotNull();

            sourceCode = await this.Downloader.RunAsync(url);
            Cookies = this.Downloader.Cookies;
        }

        protected virtual string FollowPage()
        {
            return null;
        }

        // Implement to execute code
        protected abstract IEnumerable<SpiderContext> OnRun();
        
        public string GetUrl()
        {
            return this.url;
        }


        public dynamic Fetch(Action action)
        {
            try
            {
                action();
            }
            catch (Exception ex)
            {
#if DEBUG
                Debugger.Break();
#endif
                ct.Error = ex;
            }

            return ct;
        }


        string DebugFile;

        public async Task<bool> RunAsync()
        {
            Log.Information($"Starting RunAsync ... {this.SpiderName}");
            try
            {
                this.ct = new SpiderContext();
                ct.Url = this.url;
                ct.Spider = this.SpiderName;
                ct.Bag = JObject.FromObject(ViewBag);

                await SetupBeforeRunAsync();

                var hasNextPage = false;
                do
                {
                    await RunDownloaderAsync();

                    Log.Information("Running... " + this.SpiderName);
                    ct.Url = this.url;
                    ct.Spider = this.SpiderName;
                    ct.Bag = JObject.FromObject(ViewBag);
                    var obj = OnRun();

                    foreach (var item in obj)
                    {
                        ct = new SpiderContext();
                        ct.Url = this.url;
                        ct.Spider = this.SpiderName;
                        ct.Bag = JObject.FromObject(ViewBag);

                        try
                        {
                            // After(item.Data);
                            if (item.Error == null)
                                await SuccessPipelineAsync(item);
                            else
                                await ErrorPipelineAsync(item);
                        }
                        catch (Exception ex)
                        {
                            Console.WriteLine(JsonConvert.SerializeObject(this));
                            Console.WriteLine(JsonConvert.SerializeObject(item));
                            Console.WriteLine(JsonConvert.SerializeObject(ex));

                            Log.Error(ex, "Error looping spider engine");
                        }
                    }

                    System.Console.WriteLine("Finish... " + this.SpiderName);

                    if (!nofollow)
                    {
                        var nextPage = this.FollowPage();
                        if (this.Downloader.HttpMethod == "POST")
                        {
                            // post the nextPage can be same of url
                            hasNextPage = !string.IsNullOrEmpty(nextPage);
                        }
                        else
                        {
                            // get required that nextPage different of url
                            hasNextPage = !string.IsNullOrEmpty(nextPage) && nextPage != url;
                        }

                        if (hasNextPage)
                        {
                            this.SetUrl(nextPage);
                        }
                    }

                } while (hasNextPage);

                return true;
            }
            catch (Exception ex)
            {
                ct.Data.SpiderEngine = JsonConvert.SerializeObject(this);
                ct.Data.Exception = JsonConvert.SerializeObject(ex);
                //ct.RunEmbedMetadataPipeline();
                Log.Error(ex, ct.Data.ToString());
                throw ex;
            }

            return false;
        }

        protected async virtual Task SuccessPipelineAsync(SpiderContext context)
        {
            Log.Debug("Sucess Pipeline");
        }

        protected async virtual Task ErrorPipelineAsync(SpiderContext context)
        {
            Log.Debug("Error Pipeline");
        }

        protected async virtual Task SetupBeforeRunAsync()
        {
            Log.Debug("Setup for run");
        }

        protected async virtual Task RunDownloaderAsync()
        {
            Log.Information($"URL: {url}");
            await BeforeAsync();

            this.Json = SpiderSharp.Helpers.Json.TryParse(sourceCode);
            this.node = SpiderSharp.Helpers.Html.TryParse(sourceCode);
            this.AngleDocument = SpiderSharp.AngleDocument.TryParse(sourceCode);

#if DEBUG
            if (this.node != null)
            {
                Log.Debug($"URL: {url}");
                DebugFile = System.IO.Path.GetTempFileName() + ".html";
                System.IO.File.WriteAllText(DebugFile, sourceCode);

                Log.Debug($"Debug: {DebugFile}");
            }
#endif
        }

        public void SetUrl(string url)
        {
            this.url = url;
        }

        public void SetNofollow()
        {
            this.nofollow = true;
        }
    }
}

<|start_filename|>src/SpiderSharp/Helpers/Numeric.cs<|end_filename|>
﻿using System;

namespace SpiderSharp.Helpers
{
    public static class Numeric
    {
        public static float? CalculateDiscount(double? originalPrice, double currentPrice)
        {
            if (!originalPrice.HasValue || originalPrice <= 0 || currentPrice <= 0)
            {
                return null;
            }

            var discountCalc = (originalPrice - currentPrice) / originalPrice;
            return Math.Max(Math.Min(Convert.ToInt32(discountCalc * 100), 100), 0);
        }

        public static double? CalculateOriginalPrice(double? discount, double currentPrice)
        {
            if (!discount.HasValue || discount <= 0 || currentPrice <= 0)
            {
                return null;
            }

            var originalPrice = currentPrice / (1 - (discount.Value / 100));
            return Math.Round(originalPrice, 2);
        }
    }
}