File: testUpdateHostsFile.py

#!/usr/bin/env python

# Script by gfyoung
# https://github.com/gfyoung
#
# Python script for testing updateHostFiles.py

import json
import locale
import os
import platform
import re
import shutil
import sys
import tempfile
import unittest
import unittest.mock as mock
from io import BytesIO, StringIO

import requests

import updateHostsFile
from updateHostsFile import (
    Colors,
    colorize,
    display_exclusion_options,
    domain_to_idna,
    exclude_domain,
    flush_dns_cache,
    gather_custom_exclusions,
    get_defaults,
    get_file_by_url,
    is_valid_user_provided_domain_format,
    matches_exclusions,
    move_hosts_file_into_place,
    normalize_rule,
    path_join_robust,
    print_failure,
    print_success,
    prompt_for_exclusions,
    prompt_for_flush_dns_cache,
    prompt_for_move,
    prompt_for_update,
    query_yes_no,
    recursive_glob,
    remove_old_hosts_file,
    sort_sources,
    strip_rule,
    supports_color,
    update_all_sources,
    update_readme_data,
    update_sources_data,
    write_data,
    write_opening_header,
)

unicode = str


# Test Helper Objects
class Base(unittest.TestCase):
    @staticmethod
    def mock_property(name):
        return mock.patch(name, new_callable=mock.PropertyMock)

    @property
    def sep(self):
        if platform.system().lower() == "windows":
            return "\\"
        return os.sep

    def assert_called_once(self, mock_method):
        self.assertEqual(mock_method.call_count, 1)


class BaseStdout(Base):
    def setUp(self):
        sys.stdout = StringIO()

    def tearDown(self):
        sys.stdout.close()
        sys.stdout = sys.__stdout__


class BaseMockDir(Base):
    @property
    def dir_count(self):
        return len(os.listdir(self.test_dir))

    def setUp(self):
        self.test_dir = tempfile.mkdtemp()

    def tearDown(self):
        shutil.rmtree(self.test_dir)


# End Test Helper Objects


# Project Settings
class TestGetDefaults(Base):
    def test_get_defaults(self):
        with self.mock_property("updateHostsFile.BASEDIR_PATH"):
            updateHostsFile.BASEDIR_PATH = "foo"
            actual = get_defaults()
            expected = {
                "numberofrules": 0,
                "datapath": "foo" + self.sep + "data",
                "freshen": True,
                "replace": False,
                "backup": False,
                "skipstatichosts": False,
                "keepdomaincomments": True,
                "extensionspath": "foo" + self.sep + "extensions",
                "extensions": [],
                "nounifiedhosts": False,
                "compress": False,
                "minimise": False,
                "outputsubfolder": "",
                "hostfilename": "hosts",
                "targetip": "0.0.0.0",
                "sourcedatafilename": "update.json",
                "sourcesdata": [],
                "readmefilename": "readme.md",
                "readmetemplate": ("foo" + self.sep + "readme_template.md"),
                "readmedata": {},
                "readmedatafilename": ("foo" + self.sep + "readmeData.json"),
                "exclusionpattern": r"([a-zA-Z\d-]+\.){0,}",
                "exclusionregexes": [],
                "exclusions": [],
                "commonexclusions": ["hulu.com"],
                "blacklistfile": "foo" + self.sep + "blacklist",
                "whitelistfile": "foo" + self.sep + "whitelist",
            }
            self.assertDictEqual(actual, expected)


# End Project Settings


class TestSortSources(Base):
    def test_sort_sources_simple(self):
        given = [
            "sbc.io",
            "example.com",
            "github.com",
        ]

        expected = ["example.com", "github.com", "sbc.io"]

        actual = sort_sources(given)

        self.assertEqual(actual, expected)

    def test_live_data(self):
        given = [
            "data/KADhosts/update.json",
            "data/someonewhocares.org/update.json",
            "data/StevenBlack/update.json",
            "data/adaway.org/update.json",
            "data/URLHaus/update.json",
            "data/UncheckyAds/update.json",
            "data/add.2o7Net/update.json",
            "data/mvps.org/update.json",
            "data/add.Spam/update.json",
            "data/add.Dead/update.json",
            "data/malwaredomainlist.com/update.json",
            "data/Badd-Boyz-Hosts/update.json",
            "data/hostsVN/update.json",
            "data/yoyo.org/update.json",
            "data/add.Risk/update.json",
            "data/tiuxo/update.json",
            "extensions/gambling/update.json",
            "extensions/porn/clefspeare13/update.json",
            "extensions/porn/sinfonietta-snuff/update.json",
            "extensions/porn/tiuxo/update.json",
            "extensions/porn/sinfonietta/update.json",
            "extensions/fakenews/update.json",
            "extensions/social/tiuxo/update.json",
            "extensions/social/sinfonietta/update.json",
        ]

        expected = [
            "data/StevenBlack/update.json",
            "data/adaway.org/update.json",
            "data/add.2o7Net/update.json",
            "data/add.Dead/update.json",
            "data/add.Risk/update.json",
            "data/add.Spam/update.json",
            "data/Badd-Boyz-Hosts/update.json",
            "data/hostsVN/update.json",
            "data/KADhosts/update.json",
            "data/malwaredomainlist.com/update.json",
            "data/mvps.org/update.json",
            "data/someonewhocares.org/update.json",
            "data/tiuxo/update.json",
            "data/UncheckyAds/update.json",
            "data/URLHaus/update.json",
            "data/yoyo.org/update.json",
            "extensions/fakenews/update.json",
            "extensions/gambling/update.json",
            "extensions/porn/clefspeare13/update.json",
            "extensions/porn/sinfonietta/update.json",
            "extensions/porn/sinfonietta-snuff/update.json",
            "extensions/porn/tiuxo/update.json",
            "extensions/social/sinfonietta/update.json",
            "extensions/social/tiuxo/update.json",
        ]

        actual = sort_sources(given)

        self.assertEqual(actual, expected)


# Prompt the User
class TestPromptForUpdate(BaseStdout, BaseMockDir):
    def setUp(self):
        BaseStdout.setUp(self)
        BaseMockDir.setUp(self)

    def test_no_freshen_no_new_file(self):
        hosts_file = os.path.join(self.test_dir, "hosts")
        hosts_data = "This data should not be overwritten"

        with self.mock_property("updateHostsFile.BASEDIR_PATH"):
            updateHostsFile.BASEDIR_PATH = self.test_dir

            with open(hosts_file, "w") as f:
                f.write(hosts_data)

        for update_auto in (False, True):
            dir_count = self.dir_count
            prompt_for_update(freshen=False, update_auto=update_auto)

            output = sys.stdout.getvalue()
            self.assertEqual(output, "")

            sys.stdout = StringIO()

            self.assertEqual(self.dir_count, dir_count)

            with open(hosts_file, "r") as f:
                contents = f.read()
                self.assertEqual(contents, hosts_data)

    def test_no_freshen_new_file(self):
        hosts_file = os.path.join(self.test_dir, "hosts")

        with self.mock_property("updateHostsFile.BASEDIR_PATH"):
            updateHostsFile.BASEDIR_PATH = self.test_dir

            dir_count = self.dir_count
            prompt_for_update(freshen=False, update_auto=False)

            output = sys.stdout.getvalue()
            self.assertEqual(output, "")

            sys.stdout = StringIO()

            self.assertEqual(self.dir_count, dir_count + 1)

            with open(hosts_file, "r") as f:
                contents = f.read()
                self.assertEqual(contents, "")

    @mock.patch("builtins.open")
    def test_no_freshen_fail_new_file(self, mock_open):
        for exc in (IOError, OSError):
            mock_open.side_effect = exc("failed open")

            with self.mock_property("updateHostsFile.BASEDIR_PATH"):
                updateHostsFile.BASEDIR_PATH = self.test_dir
                prompt_for_update(freshen=False, update_auto=False)

                output = sys.stdout.getvalue()
                expected = (
                    "ERROR: No 'hosts' file in the folder. "
                    "Try creating one manually."
                )
                self.assertIn(expected, output)

                sys.stdout = StringIO()

    @mock.patch("updateHostsFile.query_yes_no", return_value=False)
    def test_freshen_no_update(self, _):
        hosts_file = os.path.join(self.test_dir, "hosts")
        hosts_data = "This data should not be overwritten"

        with self.mock_property("updateHostsFile.BASEDIR_PATH"):
            updateHostsFile.BASEDIR_PATH = self.test_dir

            with open(hosts_file, "w") as f:
                f.write(hosts_data)

            dir_count = self.dir_count

            update_sources = prompt_for_update(freshen=True, update_auto=False)
            self.assertFalse(update_sources)

            output = sys.stdout.getvalue()
            expected = "OK, we'll stick with what we've got locally."
            self.assertIn(expected, output)

            sys.stdout = StringIO()

            self.assertEqual(self.dir_count, dir_count)

            with open(hosts_file, "r") as f:
                contents = f.read()
                self.assertEqual(contents, hosts_data)

    @mock.patch("updateHostsFile.query_yes_no", return_value=True)
    def test_freshen_update(self, _):
        hosts_file = os.path.join(self.test_dir, "hosts")
        hosts_data = "This data should not be overwritten"

        with self.mock_property("updateHostsFile.BASEDIR_PATH"):
            updateHostsFile.BASEDIR_PATH = self.test_dir

            with open(hosts_file, "w") as f:
                f.write(hosts_data)

            dir_count = self.dir_count

            for update_auto in (False, True):
                update_sources = prompt_for_update(
                    freshen=True, update_auto=update_auto
                )
                self.assertTrue(update_sources)

                output = sys.stdout.getvalue()
                self.assertEqual(output, "")

                sys.stdout = StringIO()

                self.assertEqual(self.dir_count, dir_count)

                with open(hosts_file, "r") as f:
                    contents = f.read()
                    self.assertEqual(contents, hosts_data)

    def tearDown(self):
        BaseStdout.tearDown(self)


class TestPromptForExclusions(BaseStdout):
    @mock.patch("updateHostsFile.query_yes_no", return_value=False)
    def testSkipPrompt(self, mock_query):
        gather_exclusions = prompt_for_exclusions(skip_prompt=True)
        self.assertFalse(gather_exclusions)

        output = sys.stdout.getvalue()
        self.assertEqual(output, "")

        mock_query.assert_not_called()

    @mock.patch("updateHostsFile.query_yes_no", return_value=False)
    def testNoSkipPromptNoDisplay(self, mock_query):
        gather_exclusions = prompt_for_exclusions(skip_prompt=False)
        self.assertFalse(gather_exclusions)

        output = sys.stdout.getvalue()
        expected = "OK, we'll only exclude domains in the whitelist."
        self.assertIn(expected, output)

        self.assert_called_once(mock_query)

    @mock.patch("updateHostsFile.query_yes_no", return_value=True)
    def testNoSkipPromptDisplay(self, mock_query):
        gather_exclusions = prompt_for_exclusions(skip_prompt=False)
        self.assertTrue(gather_exclusions)

        output = sys.stdout.getvalue()
        self.assertEqual(output, "")

        self.assert_called_once(mock_query)


class TestPromptForFlushDnsCache(Base):
    @mock.patch("updateHostsFile.flush_dns_cache", return_value=0)
    @mock.patch("updateHostsFile.query_yes_no", return_value=False)
    def testFlushCache(self, mock_query, mock_flush):
        for prompt_flush in (False, True):
            prompt_for_flush_dns_cache(flush_cache=True, prompt_flush=prompt_flush)

            mock_query.assert_not_called()
            self.assert_called_once(mock_flush)

            mock_query.reset_mock()
            mock_flush.reset_mock()

    @mock.patch("updateHostsFile.flush_dns_cache", return_value=0)
    @mock.patch("updateHostsFile.query_yes_no", return_value=False)
    def testNoFlushCacheNoPrompt(self, mock_query, mock_flush):
        prompt_for_flush_dns_cache(flush_cache=False, prompt_flush=False)

        mock_query.assert_not_called()
        mock_flush.assert_not_called()

    @mock.patch("updateHostsFile.flush_dns_cache", return_value=0)
    @mock.patch("updateHostsFile.query_yes_no", return_value=False)
    def testNoFlushCachePromptNoFlush(self, mock_query, mock_flush):
        prompt_for_flush_dns_cache(flush_cache=False, prompt_flush=True)

        self.assert_called_once(mock_query)
        mock_flush.assert_not_called()

    @mock.patch("updateHostsFile.flush_dns_cache", return_value=0)
    @mock.patch("updateHostsFile.query_yes_no", return_value=True)
    def testNoFlushCachePromptFlush(self, mock_query, mock_flush):
        prompt_for_flush_dns_cache(flush_cache=False, prompt_flush=True)

        self.assert_called_once(mock_query)
        self.assert_called_once(mock_flush)


class TestPromptForMove(Base):
    def setUp(self):
        Base.setUp(self)
        self.final_file = "final.txt"

    def prompt_for_move(self, **move_params):
        return prompt_for_move(self.final_file, **move_params)

    @mock.patch("updateHostsFile.move_hosts_file_into_place", return_value=0)
    @mock.patch("updateHostsFile.query_yes_no", return_value=False)
    def testSkipStaticHosts(self, mock_query, mock_move):
        for replace in (False, True):
            for auto in (False, True):
                move_file = self.prompt_for_move(
                    replace=replace, auto=auto, skipstatichosts=True
                )
                self.assertFalse(move_file)

                mock_query.assert_not_called()
                mock_move.assert_not_called()

                mock_query.reset_mock()
                mock_move.reset_mock()

    @mock.patch("updateHostsFile.move_hosts_file_into_place", return_value=0)
    @mock.patch("updateHostsFile.query_yes_no", return_value=False)
    def testReplaceNoSkipStaticHosts(self, mock_query, mock_move):
        for auto in (False, True):
            move_file = self.prompt_for_move(
                replace=True, auto=auto, skipstatichosts=False
            )
            self.assertFalse(move_file)

            mock_query.assert_not_called()
            self.assert_called_once(mock_move)

            mock_query.reset_mock()
            mock_move.reset_mock()

    @mock.patch("updateHostsFile.move_hosts_file_into_place", return_value=0)
    @mock.patch("updateHostsFile.query_yes_no", return_value=False)
    def testAutoNoSkipStaticHosts(self, mock_query, mock_move):
        for replace in (False, True):
            move_file = self.prompt_for_move(
                replace=replace, auto=True, skipstatichosts=True
            )
            self.assertFalse(move_file)

            mock_query.assert_not_called()
            mock_move.assert_not_called()

            mock_query.reset_mock()
            mock_move.reset_mock()

    @mock.patch("updateHostsFile.move_hosts_file_into_place", return_value=0)
    @mock.patch("updateHostsFile.query_yes_no", return_value=False)
    def testPromptNoMove(self, mock_query, mock_move):
        move_file = self.prompt_for_move(
            replace=False, auto=False, skipstatichosts=False
        )
        self.assertFalse(move_file)

        self.assert_called_once(mock_query)
        mock_move.assert_not_called()

    @mock.patch("updateHostsFile.move_hosts_file_into_place", return_value=0)
    @mock.patch("updateHostsFile.query_yes_no", return_value=True)
    def testPromptMove(self, mock_query, mock_move):
        move_file = self.prompt_for_move(
            replace=False, auto=False, skipstatichosts=False
        )
        self.assertFalse(move_file)

        self.assert_called_once(mock_query)
        self.assert_called_once(mock_move)


# End Prompt the User


# Exclusion Logic
class TestDisplayExclusionsOptions(Base):
    @mock.patch("updateHostsFile.query_yes_no", return_value=0)
    @mock.patch("updateHostsFile.exclude_domain", return_value=None)
    @mock.patch("updateHostsFile.gather_custom_exclusions", return_value=None)
    def test_no_exclusions(self, mock_gather, mock_exclude, _):
        common_exclusions = []
        display_exclusion_options(common_exclusions, "foo", [])

        mock_gather.assert_not_called()
        mock_exclude.assert_not_called()

    @mock.patch("updateHostsFile.query_yes_no", side_effect=[1, 1, 0])
    @mock.patch("updateHostsFile.exclude_domain", return_value=None)
    @mock.patch("updateHostsFile.gather_custom_exclusions", return_value=None)
    def test_only_common_exclusions(self, mock_gather, mock_exclude, _):
        common_exclusions = ["foo", "bar"]
        display_exclusion_options(common_exclusions, "foo", [])

        mock_gather.assert_not_called()

        exclude_calls = [mock.call("foo", "foo", []), mock.call("bar", "foo", None)]
        mock_exclude.assert_has_calls(exclude_calls)

    @mock.patch("updateHostsFile.query_yes_no", side_effect=[0, 0, 1])
    @mock.patch("updateHostsFile.exclude_domain", return_value=None)
    @mock.patch("updateHostsFile.gather_custom_exclusions", return_value=None)
    def test_gather_exclusions(self, mock_gather, mock_exclude, _):
        common_exclusions = ["foo", "bar"]
        display_exclusion_options(common_exclusions, "foo", [])

        mock_exclude.assert_not_called()
        self.assert_called_once(mock_gather)

    @mock.patch("updateHostsFile.query_yes_no", side_effect=[1, 0, 1])
    @mock.patch("updateHostsFile.exclude_domain", return_value=None)
    @mock.patch("updateHostsFile.gather_custom_exclusions", return_value=None)
    def test_mixture_gather_exclusions(self, mock_gather, mock_exclude, _):
        common_exclusions = ["foo", "bar"]
        display_exclusion_options(common_exclusions, "foo", [])

        mock_exclude.assert_called_once_with("foo", "foo", [])
        self.assert_called_once(mock_gather)


class TestGatherCustomExclusions(BaseStdout):

    # Can only test in the invalid domain case
    # because of the settings global variable.
    @mock.patch("updateHostsFile.input", side_effect=["foo", "no"])
    @mock.patch(
        "updateHostsFile.is_valid_user_provided_domain_format", return_value=False
    )
    def test_basic(self, *_):
        gather_custom_exclusions("foo", [])

        expected = "Do you have more domains you want to enter? [Y/n]"
        output = sys.stdout.getvalue()
        self.assertIn(expected, output)

    @mock.patch("updateHostsFile.input", side_effect=["foo", "yes", "bar", "no"])
    @mock.patch(
        "updateHostsFile.is_valid_user_provided_domain_format", return_value=False
    )
    def test_multiple(self, *_):
        gather_custom_exclusions("foo", [])

        expected = (
            "Do you have more domains you want to enter? [Y/n] "
            "Do you have more domains you want to enter? [Y/n]"
        )
        output = sys.stdout.getvalue()
        self.assertIn(expected, output)


class TestExcludeDomain(Base):
    def test_invalid_exclude_domain(self):
        exclusion_regexes = []
        exclusion_pattern = "*.com"

        for domain in ["google.com", "hulu.com", "adaway.org"]:
            self.assertRaises(
                re.error, exclude_domain, domain, exclusion_pattern, exclusion_regexes
            )

        self.assertListEqual(exclusion_regexes, [])

    def test_valid_exclude_domain(self):
        exp_count = 0
        expected_regexes = []
        exclusion_regexes = []
        exclusion_pattern = r"[a-z]\."

        for domain in ["google.com", "hulu.com", "adaway.org"]:
            self.assertEqual(len(exclusion_regexes), exp_count)

            exclusion_regexes = exclude_domain(
                domain, exclusion_pattern, exclusion_regexes
            )
            expected_regex = re.compile(exclusion_pattern + domain)

            expected_regexes.append(expected_regex)
            exp_count += 1

        self.assertEqual(len(exclusion_regexes), exp_count)
        self.assertListEqual(exclusion_regexes, expected_regexes)


class TestMatchesExclusions(Base):
    def test_no_match_empty_list(self):
        exclusion_regexes = []

        for domain in [
            "1.2.3.4 localhost",
            "5.6.7.8 hulu.com",
            "9.1.2.3 yahoo.com",
            "4.5.6.7 cloudfront.net",
        ]:
            self.assertFalse(matches_exclusions(domain, exclusion_regexes))

    def test_no_match_list(self):
        exclusion_regexes = [r".*\.org", r".*\.edu"]
        exclusion_regexes = [re.compile(regex) for regex in exclusion_regexes]

        for domain in [
            "1.2.3.4 localhost",
            "5.6.7.8 hulu.com",
            "9.1.2.3 yahoo.com",
            "4.5.6.7 cloudfront.net",
        ]:
            self.assertFalse(matches_exclusions(domain, exclusion_regexes))

    def test_match_list(self):
        exclusion_regexes = [r".*\.com", r".*\.org", r".*\.edu"]
        exclusion_regexes = [re.compile(regex) for regex in exclusion_regexes]

        for domain in [
            "5.6.7.8 hulu.com",
            "9.1.2.3 yahoo.com",
            "4.5.6.7 adaway.org",
            "8.9.1.2 education.edu",
        ]:
            self.assertTrue(matches_exclusions(domain, exclusion_regexes))

    def test_match_raw_list(self):
        exclusion_regexes = [r".*\.com", r".*\.org", r".*\.edu", r".*@.*"]
        exclusion_regexes = [re.compile(regex) for regex in exclusion_regexes]

        for domain in [
            "hulu.com",
            "yahoo.com",
            "adaway.org",
            "education.edu",
            "a.stro.lo.gy@45.144.225.135",
        ]:
            self.assertTrue(matches_exclusions(domain, exclusion_regexes))

    def test_no_match_raw_list(self):
        exclusion_regexes = [r".*\.org", r".*\.edu"]
        exclusion_regexes = [re.compile(regex) for regex in exclusion_regexes]

        for domain in [
            "localhost",
            "hulu.com",
            "yahoo.com",
            "cloudfront.net",
        ]:
            self.assertFalse(matches_exclusions(domain, exclusion_regexes))


# End Exclusion Logic


# Update Logic
class TestUpdateSourcesData(Base):
    def setUp(self):
        Base.setUp(self)

        self.data_path = "data"
        self.extensions_path = "extensions"
        self.source_data_filename = "update.json"

        self.update_kwargs = dict(
            datapath=self.data_path,
            extensionspath=self.extensions_path,
            sourcedatafilename=self.source_data_filename,
            nounifiedhosts=False,
        )

    def update_sources_data(self, sources_data, extensions):
        return update_sources_data(
            sources_data[:], extensions=extensions, **self.update_kwargs
        )

    @mock.patch("updateHostsFile.recursive_glob", return_value=[])
    @mock.patch("updateHostsFile.path_join_robust", return_value="dirpath")
    @mock.patch("builtins.open", return_value=mock.Mock())
    def test_no_update(self, mock_open, mock_join_robust, _):
        extensions = []
        sources_data = [{"source": "source1.txt"}, {"source": "source2.txt"}]

        new_sources_data = self.update_sources_data(sources_data, extensions)
        self.assertEqual(new_sources_data, sources_data)
        mock_join_robust.assert_not_called()
        mock_open.assert_not_called()

        extensions = [".json", ".txt"]
        new_sources_data = self.update_sources_data(sources_data, extensions)

        self.assertEqual(new_sources_data, sources_data)
        join_calls = [
            mock.call(self.extensions_path, ".json"),
            mock.call(self.extensions_path, ".txt"),
        ]
        mock_join_robust.assert_has_calls(join_calls)
        mock_open.assert_not_called()

    @mock.patch(
        "updateHostsFile.recursive_glob",
        side_effect=[[], ["update1.txt", "update2.txt"]],
    )
    @mock.patch("json.load", return_value={"mock_source": "mock_source.ext"})
    @mock.patch("builtins.open", return_value=mock.Mock())
    @mock.patch("updateHostsFile.path_join_robust", return_value="dirpath")
    def test_update_only_extensions(self, mock_join_robust, *_):
        extensions = [".json"]
        sources_data = [{"source": "source1.txt"}, {"source": "source2.txt"}]
        new_sources_data = self.update_sources_data(sources_data, extensions)

        expected = sources_data + [{"mock_source": "mock_source.ext"}] * 2
        self.assertEqual(new_sources_data, expected)
        self.assert_called_once(mock_join_robust)

    @mock.patch(
        "updateHostsFile.recursive_glob",
        side_effect=[["update1.txt", "update2.txt"], ["update3.txt", "update4.txt"]],
    )
    @mock.patch(
        "json.load",
        side_effect=[
            {"mock_source": "mock_source.txt"},
            {"mock_source": "mock_source2.txt"},
            {"mock_source": "mock_source3.txt"},
            {"mock_source": "mock_source4.txt"},
        ],
    )
    @mock.patch("builtins.open", return_value=mock.Mock())
    @mock.patch("updateHostsFile.path_join_robust", return_value="dirpath")
    def test_update_both_pathways(self, mock_join_robust, *_):
        extensions = [".json"]
        sources_data = [{"source": "source1.txt"}, {"source": "source2.txt"}]
        new_sources_data = self.update_sources_data(sources_data, extensions)

        expected = sources_data + [
            {"mock_source": "mock_source.txt"},
            {"mock_source": "mock_source2.txt"},
            {"mock_source": "mock_source3.txt"},
            {"mock_source": "mock_source4.txt"},
        ]
        self.assertEqual(new_sources_data, expected)
        self.assert_called_once(mock_join_robust)


class TestUpdateAllSources(BaseStdout):
    def setUp(self):
        BaseStdout.setUp(self)

        self.source_data_filename = "data.json"
        self.host_filename = "hosts.txt"

    @mock.patch("builtins.open")
    @mock.patch("updateHostsFile.recursive_glob", return_value=[])
    def test_no_sources(self, _, mock_open):
        update_all_sources(self.source_data_filename, self.host_filename)
        mock_open.assert_not_called()

    @mock.patch("builtins.open", return_value=mock.Mock())
    @mock.patch("json.load", return_value={"url": "example.com"})
    @mock.patch("updateHostsFile.recursive_glob", return_value=["foo"])
    @mock.patch("updateHostsFile.write_data", return_value=0)
    @mock.patch("updateHostsFile.get_file_by_url", return_value="file_data")
    def test_one_source(self, mock_get, mock_write, *_):
        update_all_sources(self.source_data_filename, self.host_filename)
        self.assert_called_once(mock_write)
        self.assert_called_once(mock_get)

        output = sys.stdout.getvalue()
        expected = "Updating source  from example.com"

        self.assertIn(expected, output)

    @mock.patch("builtins.open", return_value=mock.Mock())
    @mock.patch("json.load", return_value={"url": "example.com"})
    @mock.patch("updateHostsFile.recursive_glob", return_value=["foo"])
    @mock.patch("updateHostsFile.write_data", return_value=0)
    @mock.patch("updateHostsFile.get_file_by_url", return_value=Exception("fail"))
    def test_source_fail(self, mock_get, mock_write, *_):
        update_all_sources(self.source_data_filename, self.host_filename)
        mock_write.assert_not_called()
        self.assert_called_once(mock_get)

        output = sys.stdout.getvalue()
        expecteds = [
            "Updating source  from example.com",
            "Error in updating source:  example.com",
        ]
        for expected in expecteds:
            self.assertIn(expected, output)

    @mock.patch("builtins.open", return_value=mock.Mock())
    @mock.patch(
        "json.load", side_effect=[{"url": "example.com"}, {"url": "example2.com"}]
    )
    @mock.patch("updateHostsFile.recursive_glob", return_value=["foo", "bar"])
    @mock.patch("updateHostsFile.write_data", return_value=0)
    @mock.patch(
        "updateHostsFile.get_file_by_url", side_effect=[Exception("fail"), "file_data"]
    )
    def test_sources_fail_succeed(self, mock_get, mock_write, *_):
        update_all_sources(self.source_data_filename, self.host_filename)
        self.assert_called_once(mock_write)

        get_calls = [mock.call("example.com"), mock.call("example2.com")]
        mock_get.assert_has_calls(get_calls)

        output = sys.stdout.getvalue()
        expecteds = [
            "Updating source  from example.com",
            "Error in updating source:  example.com",
            "Updating source  from example2.com",
        ]
        for expected in expecteds:
            self.assertIn(expected, output)


# End Update Logic


# File Logic
class TestNormalizeRule(BaseStdout):
    def test_no_match(self):
        kwargs = dict(target_ip="0.0.0.0", keep_domain_comments=False)

        # Note: "Bare"- Domains are accepted. IP are excluded.
        for rule in [
            "128.0.0.1",
            "::1",
            "0.0.0.0 128.0.0.2",
            "0.1.2.3 foo/bar",
            "0.3.4.5 example.org/hello/world",
            "0.0.0.0 https",
            "0.0.0.0 https..",
        ]:
            self.assertEqual(normalize_rule(rule, **kwargs), (None, None))

            output = sys.stdout.getvalue()
            sys.stdout = StringIO()

            expected = "==>" + rule + "<=="
            self.assertIn(expected, output)

    def test_mixed_cases(self):
        for rule, expected_target in (
            ("tWiTTer.cOM", "twitter.com"),
            ("goOgLe.Com", "google.com"),
            ("FoO.bAR.edu", "foo.bar.edu"),
        ):
            expected = (expected_target, "0.0.0.0 " + expected_target + "\n")

            actual = normalize_rule(
                rule, target_ip="0.0.0.0", keep_domain_comments=False
            )
            self.assertEqual(actual, expected)

            # Nothing gets printed if there's a match.
            output = sys.stdout.getvalue()
            self.assertEqual(output, "")

            sys.stdout = StringIO()

    def test_no_comments(self):
        for target_ip in ("0.0.0.0", "127.0.0.1", "8.8.8.8"):
            rule = "127.0.0.1 1.google.com foo"
            expected = ("1.google.com", str(target_ip) + " 1.google.com\n")

            actual = normalize_rule(
                rule, target_ip=target_ip, keep_domain_comments=False
            )
            self.assertEqual(actual, expected)

            # Nothing gets printed if there's a match.
            output = sys.stdout.getvalue()
            self.assertEqual(output, "")

            sys.stdout = StringIO()

    def test_with_comments(self):
        for target_ip in ("0.0.0.0", "127.0.0.1", "8.8.8.8"):
            for comment in ("foo", "bar", "baz"):
                rule = "127.0.0.1 1.google.co.uk " + comment
                expected = (
                    "1.google.co.uk",
                    (str(target_ip) + " 1.google.co.uk # " + comment + "\n"),
                )

                actual = normalize_rule(
                    rule, target_ip=target_ip, keep_domain_comments=True
                )
                self.assertEqual(actual, expected)

                # Nothing gets printed if there's a match.
                output = sys.stdout.getvalue()
                self.assertEqual(output, "")

                sys.stdout = StringIO()

    def test_two_ips(self):
        for target_ip in ("0.0.0.0", "127.0.0.1", "8.8.8.8"):
            rule = "127.0.0.1 11.22.33.44 foo"

            actual = normalize_rule(
                rule, target_ip=target_ip, keep_domain_comments=False
            )
            self.assertEqual(actual, (None, None))

            output = sys.stdout.getvalue()

            expected = "==>" + rule + "<=="
            self.assertIn(expected, output)

            sys.stdout = StringIO()

    def test_no_comment_raw(self):
        for rule in (
            "twitter.com",
            "google.com",
            "foo.bar.edu",
            "www.example-foo.bar.edu",
            "www.example-3045.foobar.com",
            "www.example.xn--p1ai"
        ):
            expected = (rule, "0.0.0.0 " + rule + "\n")

            actual = normalize_rule(
                rule, target_ip="0.0.0.0", keep_domain_comments=False
            )
            self.assertEqual(actual, expected)

            # Nothing gets printed if there's a match.
            output = sys.stdout.getvalue()
            self.assertEqual(output, "")

            sys.stdout = StringIO()

    def test_with_comments_raw(self):
        for target_ip in ("0.0.0.0", "127.0.0.1", "8.8.8.8"):
            for comment in ("foo", "bar", "baz"):
                rule = "1.google.co.uk " + comment
                expected = (
                    "1.google.co.uk",
                    (str(target_ip) + " 1.google.co.uk # " + comment + "\n"),
                )

                actual = normalize_rule(
                    rule, target_ip=target_ip, keep_domain_comments=True
                )
                self.assertEqual(actual, expected)

                # Nothing gets printed if there's a match.
                output = sys.stdout.getvalue()
                self.assertEqual(output, "")

                sys.stdout = StringIO()


class TestStripRule(Base):
    def test_strip_exactly_two(self):
        for line in [
            "0.0.0.0 twitter.com",
            "127.0.0.1 facebook.com",
            "8.8.8.8 google.com",
            "1.2.3.4 foo.bar.edu",
        ]:
            output = strip_rule(line)
            self.assertEqual(output, line)

    def test_strip_more_than_two(self):
        comment = " # comments here galore"

        for line in [
            "0.0.0.0 twitter.com",
            "127.0.0.1 facebook.com",
            "8.8.8.8 google.com",
            "1.2.3.4 foo.bar.edu",
        ]:
            output = strip_rule(line + comment)
            self.assertEqual(output, line + comment)

    def test_strip_raw(self):
        for line in [
            "twitter.com",
            "facebook.com",
            "google.com",
            "foo.bar.edu",
        ]:
            output = strip_rule(line)
            self.assertEqual(output, line)

    def test_strip_raw_with_comment(self):
        comment = " # comments here galore"

        for line in [
            "twitter.com",
            "facebook.com",
            "google.com",
            "foo.bar.edu",
        ]:
            output = strip_rule(line + comment)
            self.assertEqual(output, line + comment)


class TestWriteOpeningHeader(BaseMockDir):
    def setUp(self):
        super(TestWriteOpeningHeader, self).setUp()
        self.final_file = BytesIO()

    def test_missing_keyword(self):
        kwargs = dict(
            extensions="", outputsubfolder="", numberofrules=5, skipstatichosts=False, nounifiedhosts=False
        )

        for k in kwargs.keys():
            bad_kwargs = kwargs.copy()
            bad_kwargs.pop(k)

            self.assertRaises(
                KeyError, write_opening_header, self.final_file, **bad_kwargs
            )

    def test_basic(self):
        kwargs = dict(
            extensions="", outputsubfolder="", numberofrules=5, skipstatichosts=True, nounifiedhosts=False
        )
        write_opening_header(self.final_file, **kwargs)

        contents = self.final_file.getvalue()
        contents = contents.decode("UTF-8")

        # Expected contents.
        for expected in (
            "# This hosts file is a merged collection",
            "# with a dash of crowd sourcing via GitHub",
            "# Number of unique domains: {count}".format(count=kwargs["numberofrules"]),
            "Fetch the latest version of this file:",
            "Project home page: https://github.com/StevenBlack/hosts",
        ):
            self.assertIn(expected, contents)

        # Expected non-contents.
        for expected in (
            "# Extensions added to this file:",
            "127.0.0.1 localhost",
            "127.0.0.1 local",
            "127.0.0.53",
            "127.0.1.1",
        ):
            self.assertNotIn(expected, contents)

    def test_basic_include_static_hosts(self):
        kwargs = dict(
            extensions="", outputsubfolder="", numberofrules=5, skipstatichosts=False, nounifiedhosts=False
        )
        with self.mock_property("platform.system") as obj:
            obj.return_value = "Windows"
            write_opening_header(self.final_file, **kwargs)

        contents = self.final_file.getvalue()
        contents = contents.decode("UTF-8")

        # Expected contents.
        for expected in (
            "127.0.0.1 local",
            "127.0.0.1 localhost",
            "# This hosts file is a merged collection",
            "# with a dash of crowd sourcing via GitHub",
            "# Number of unique domains: {count}".format(count=kwargs["numberofrules"]),
            "Fetch the latest version of this file:",
            "Project home page: https://github.com/StevenBlack/hosts",
        ):
            self.assertIn(expected, contents)

        # Expected non-contents.
        for expected in ("# Extensions added to this file:", "127.0.0.53", "127.0.1.1"):
            self.assertNotIn(expected, contents)

    def test_basic_include_static_hosts_linux(self):
        kwargs = dict(
            extensions="", outputsubfolder="", numberofrules=5, skipstatichosts=False, nounifiedhosts=False
        )
        with self.mock_property("platform.system") as system:
            system.return_value = "Linux"

            with self.mock_property("socket.gethostname") as hostname:
                hostname.return_value = "steven-hosts"
                write_opening_header(self.final_file, **kwargs)

        contents = self.final_file.getvalue()
        contents = contents.decode("UTF-8")

        # Expected contents.
        for expected in (
            "127.0.1.1",
            "127.0.0.53",
            "steven-hosts",
            "127.0.0.1 local",
            "127.0.0.1 localhost",
            "# This hosts file is a merged collection",
            "# with a dash of crowd sourcing via GitHub",
            "# Number of unique domains: {count}".format(count=kwargs["numberofrules"]),
            "Fetch the latest version of this file:",
            "Project home page: https://github.com/StevenBlack/hosts",
        ):
            self.assertIn(expected, contents)

        # Expected non-contents.
        expected = "# Extensions added to this file:"
        self.assertNotIn(expected, contents)

    def test_extensions(self):
        kwargs = dict(
            extensions=["epsilon", "gamma", "mu", "phi"],
            outputsubfolder="",
            numberofrules=5,
            skipstatichosts=True,
            nounifiedhosts=False,
        )
        write_opening_header(self.final_file, **kwargs)

        contents = self.final_file.getvalue()
        contents = contents.decode("UTF-8")

        # Expected contents.
        for expected in (
            ", ".join(kwargs["extensions"]),
            "# Extensions added to this file:",
            "# This hosts file is a merged collection",
            "# with a dash of crowd sourcing via GitHub",
            "# Number of unique domains: {count}".format(count=kwargs["numberofrules"]),
            "Fetch the latest version of this file:",
            "Project home page: https://github.com/StevenBlack/hosts",
        ):
            self.assertIn(expected, contents)

        # Expected non-contents.
        for expected in (
            "127.0.0.1 localhost",
            "127.0.0.1 local",
            "127.0.0.53",
            "127.0.1.1",
        ):
            self.assertNotIn(expected, contents)

    def test_no_unified_hosts(self):
        kwargs = dict(
            extensions=["epsilon", "gamma"],
            outputsubfolder="",
            numberofrules=5,
            skipstatichosts=True,
            nounifiedhosts=True,
        )
        write_opening_header(self.final_file, **kwargs)

        contents = self.final_file.getvalue()
        contents = contents.decode("UTF-8")

        # Expected contents.
        for expected in (
            ", ".join(kwargs["extensions"]),
            "# The unified hosts file was not used while generating this file.",
            "# Extensions used to generate this file:",
            "# This hosts file is a merged collection",
            "# with a dash of crowd sourcing via GitHub",
            "# Number of unique domains: {count}".format(count=kwargs["numberofrules"]),
            "Fetch the latest version of this file:",
            "Project home page: https://github.com/StevenBlack/hosts",
        ):
            self.assertIn(expected, contents)

        # Expected non-contents.
        for expected in (
            "127.0.0.1 localhost",
            "127.0.0.1 local",
            "127.0.0.53",
            "127.0.1.1",
        ):
            self.assertNotIn(expected, contents)

    def _check_preamble(self, check_copy):
        hosts_file = os.path.join(self.test_dir, "myhosts")
        hosts_file += ".example" if check_copy else ""

        with open(hosts_file, "w") as f:
            f.write("peter-piper-picked-a-pepper")

        kwargs = dict(
            extensions="", outputsubfolder="", numberofrules=5, skipstatichosts=True, nounifiedhosts=False
        )

        with self.mock_property("updateHostsFile.BASEDIR_PATH"):
            updateHostsFile.BASEDIR_PATH = self.test_dir
            write_opening_header(self.final_file, **kwargs)

        contents = self.final_file.getvalue()
        contents = contents.decode("UTF-8")

        # Expected contents.
        for expected in (
            "peter-piper-picked-a-pepper",
            "# This hosts file is a merged collection",
            "# with a dash of crowd sourcing via GitHub",
            "# Number of unique domains: {count}".format(count=kwargs["numberofrules"]),
            "Fetch the latest version of this file:",
            "Project home page: https://github.com/StevenBlack/hosts",
        ):
            self.assertIn(expected, contents)

        # Expected non-contents.
        for expected in (
            "# Extensions added to this file:",
            "127.0.0.1 localhost",
            "127.0.0.1 local",
            "127.0.0.53",
            "127.0.1.1",
        ):
            self.assertNotIn(expected, contents)

    def test_preamble_exists(self):
        self._check_preamble(True)

    def test_preamble_copy(self):
        self._check_preamble(False)

    def tearDown(self):
        super(TestWriteOpeningHeader, self).tearDown()
        self.final_file.close()


class TestUpdateReadmeData(BaseMockDir):
    def setUp(self):
        super(TestUpdateReadmeData, self).setUp()
        self.readme_file = os.path.join(self.test_dir, "readmeData.json")

    def test_missing_keyword(self):
        kwargs = dict(
            extensions="", outputsubfolder="", numberofrules="", sourcesdata="", nounifiedhosts=False
        )

        for k in kwargs.keys():
            bad_kwargs = kwargs.copy()
            bad_kwargs.pop(k)

            self.assertRaises(
                KeyError, update_readme_data, self.readme_file, **bad_kwargs
            )

    def test_add_fields(self):
        with open(self.readme_file, "w") as f:
            json.dump({"foo": "bar"}, f)

        kwargs = dict(
            extensions=None, outputsubfolder="foo", numberofrules=5, sourcesdata="hosts", nounifiedhosts=False
        )
        update_readme_data(self.readme_file, **kwargs)

        if platform.system().lower() == "windows":
            sep = "/"
        else:
            sep = self.sep

        expected = {
            "base": {"location": "foo" + sep, 'no_unified_hosts': False, "sourcesdata": "hosts", "entries": 5},
            "foo": "bar",
        }

        with open(self.readme_file, "r") as f:
            actual = json.load(f)
            self.assertEqual(actual, expected)

    def test_modify_fields(self):
        with open(self.readme_file, "w") as f:
            json.dump({"base": "soprano"}, f)

        kwargs = dict(
            extensions=None, outputsubfolder="foo", numberofrules=5, sourcesdata="hosts", nounifiedhosts=False
        )
        update_readme_data(self.readme_file, **kwargs)

        if platform.system().lower() == "windows":
            sep = "/"
        else:
            sep = self.sep

        expected = {
            "base": {"location": "foo" + sep, 'no_unified_hosts': False, "sourcesdata": "hosts", "entries": 5},
        }

        with open(self.readme_file, "r") as f:
            actual = json.load(f)
            self.assertEqual(actual, expected)

    def test_set_extensions(self):
        with open(self.readme_file, "w") as f:
            json.dump({}, f)

        kwargs = dict(
            extensions=["com", "org"],
            outputsubfolder="foo",
            numberofrules=5,
            sourcesdata="hosts",
            nounifiedhosts=False,
        )
        update_readme_data(self.readme_file, **kwargs)

        if platform.system().lower() == "windows":
            sep = "/"
        else:
            sep = self.sep

        expected = {
            "com-org": {"location": "foo" + sep, 'no_unified_hosts': False, "sourcesdata": "hosts", "entries": 5}
        }

        with open(self.readme_file, "r") as f:
            actual = json.load(f)
            self.assertEqual(actual, expected)

    def test_set_no_unified_hosts(self):
        with open(self.readme_file, "w") as f:
            json.dump({}, f)

        kwargs = dict(
            extensions=["com", "org"],
            outputsubfolder="foo",
            numberofrules=5,
            sourcesdata="hosts",
            nounifiedhosts=True,
        )
        update_readme_data(self.readme_file, **kwargs)

        if platform.system().lower() == "windows":
            sep = "/"
        else:
            sep = self.sep

        expected = {
            "com-org-only": {"location": "foo" + sep, 'no_unified_hosts': True, "sourcesdata": "hosts", "entries": 5}
        }

        with open(self.readme_file, "r") as f:
            actual = json.load(f)
            self.assertEqual(actual, expected)


class TestMoveHostsFile(BaseStdout):
    @mock.patch("os.path.abspath", side_effect=lambda f: f)
    def test_move_hosts_no_name(self, _):  # TODO: Create test which tries to move actual file
        with self.mock_property("platform.system") as obj:
            obj.return_value = "foo"

            mock_file = mock.Mock(name="foo")
            move_hosts_file_into_place(mock_file)

            expected = "does not exist"
            output = sys.stdout.getvalue()

            self.assertIn(expected, output)

    @mock.patch("os.path.abspath", side_effect=lambda f: f)
    def test_move_hosts_windows(self, _):
        with self.mock_property("platform.system") as obj:
            obj.return_value = "Windows"

            mock_file = mock.Mock(name="foo")
            move_hosts_file_into_place(mock_file)

            expected = ""
            output = sys.stdout.getvalue()
            self.assertIn(expected, output)

    @mock.patch("os.path.abspath", side_effect=lambda f: f)
    @mock.patch("subprocess.call", return_value=0)
    def test_move_hosts_posix(self, *_):  # TODO: create test which tries to move an actual file
        with self.mock_property("platform.system") as obj:
            obj.return_value = "Linux"

            mock_file = mock.Mock(name="foo")
            move_hosts_file_into_place(mock_file)

            expected = "does not exist."
            output = sys.stdout.getvalue()
            self.assertIn(expected, output)

    @mock.patch("os.path.abspath", side_effect=lambda f: f)
    @mock.patch("subprocess.call", return_value=1)
    def test_move_hosts_posix_fail(self, *_):
        with self.mock_property("platform.system") as obj:
            obj.return_value = "Linux"

            mock_file = mock.Mock(name="foo")
            move_hosts_file_into_place(mock_file)

            expected = "does not exist."
            output = sys.stdout.getvalue()
            self.assertIn(expected, output)


class TestFlushDnsCache(BaseStdout):
    @mock.patch("subprocess.call", return_value=0)
    def test_flush_darwin(self, _):
        with self.mock_property("platform.system") as obj:
            obj.return_value = "Darwin"
            flush_dns_cache()

            expected = (
                "Flushing the DNS cache to utilize new hosts "
                "file...\nFlushing the DNS cache requires "
                "administrative privileges. You might need to "
                "enter your password."
            )
            output = sys.stdout.getvalue()
            self.assertIn(expected, output)

    @mock.patch("subprocess.call", return_value=1)
    def test_flush_darwin_fail(self, _):
        with self.mock_property("platform.system") as obj:
            obj.return_value = "Darwin"
            flush_dns_cache()

            expected = "Flushing the DNS cache failed."
            output = sys.stdout.getvalue()
            self.assertIn(expected, output)

    def test_flush_windows(self):
        with self.mock_property("platform.system") as obj:
            obj.return_value = "win32"

            with self.mock_property("os.name"):
                os.name = "nt"
                flush_dns_cache()

                expected = (
                    "Automatically flushing the DNS cache is "
                    "not yet supported.\nPlease copy and paste "
                    "the command 'ipconfig /flushdns' in "
                    "administrator command prompt after running "
                    "this script."
                )
                output = sys.stdout.getvalue()
                self.assertIn(expected, output)

    @mock.patch("os.path.isfile", return_value=False)
    def test_flush_no_tool(self, _):
        with self.mock_property("platform.system") as obj:
            obj.return_value = "Linux"

            with self.mock_property("os.name"):
                os.name = "posix"
                flush_dns_cache()

                expected = "Unable to determine DNS management tool."
                output = sys.stdout.getvalue()
                self.assertIn(expected, output)

    @mock.patch("os.path.isfile", side_effect=[True] + [False] * 11)
    @mock.patch("subprocess.call", return_value=0)
    def test_flush_posix(self, *_):
        with self.mock_property("platform.system") as obj:
            obj.return_value = "Linux"

            with self.mock_property("os.name"):
                os.name = "posix"
                flush_dns_cache()

                expected = "Flushing the DNS cache by restarting nscd succeeded"
                output = sys.stdout.getvalue()
                self.assertIn(expected, output)

    @mock.patch("os.path.isfile", side_effect=[True] + [False] * 11)
    @mock.patch("subprocess.call", return_value=1)
    def test_flush_posix_fail(self, *_):
        with self.mock_property("platform.system") as obj:
            obj.return_value = "Linux"

            with self.mock_property("os.name"):
                os.name = "posix"
                flush_dns_cache()

                expected = "Flushing the DNS cache by restarting nscd failed"
                output = sys.stdout.getvalue()
                self.assertIn(expected, output)

    @mock.patch("os.path.isfile", side_effect=[True, False, False, True] + [False] * 10)
    @mock.patch("subprocess.call", side_effect=[1, 0, 0])
    def test_flush_posix_fail_then_succeed(self, *_):
        with self.mock_property("platform.system") as obj:
            obj.return_value = "Linux"

            with self.mock_property("os.name"):
                os.name = "posix"
                flush_dns_cache()

                output = sys.stdout.getvalue()
                for expected in [
                    ("Flushing the DNS cache by restarting nscd failed"),
                    (
                        "Flushing the DNS cache by restarting "
                        "NetworkManager.service succeeded"
                    ),
                ]:
                    self.assertIn(expected, output)


class TestRemoveOldHostsFile(BaseMockDir):
    def setUp(self):
        super(TestRemoveOldHostsFile, self).setUp()
        self.hosts_file = "hosts"
        self.full_hosts_path = os.path.join(self.test_dir, "hosts")

    def test_remove_hosts_file(self):
        old_dir_count = self.dir_count

        remove_old_hosts_file(self.test_dir, self.hosts_file, backup=False)

        new_dir_count = old_dir_count + 1
        self.assertEqual(self.dir_count, new_dir_count)

        with open(self.full_hosts_path, "r") as f:
            contents = f.read()
            self.assertEqual(contents, "")

    def test_remove_hosts_file_exists(self):
        with open(self.full_hosts_path, "w") as f:
            f.write("foo")

        old_dir_count = self.dir_count

        remove_old_hosts_file(self.test_dir, self.hosts_file, backup=False)

        new_dir_count = old_dir_count
        self.assertEqual(self.dir_count, new_dir_count)

        with open(self.full_hosts_path, "r") as f:
            contents = f.read()
            self.assertEqual(contents, "")

    @mock.patch("time.strftime", return_value="new")
    def test_remove_hosts_file_backup(self, _):
        with open(self.full_hosts_path, "w") as f:
            f.write("foo")

        old_dir_count = self.dir_count

        remove_old_hosts_file(self.test_dir, self.hosts_file, backup=True)

        new_dir_count = old_dir_count + 1
        self.assertEqual(self.dir_count, new_dir_count)

        with open(self.full_hosts_path, "r") as f:
            contents = f.read()
            self.assertEqual(contents, "")

        new_hosts_file = self.full_hosts_path + "-new"

        with open(new_hosts_file, "r") as f:
            contents = f.read()
            self.assertEqual(contents, "foo")


# End File Logic


class DomainToIDNA(Base):
    def __init__(self, *args, **kwargs):
        super(DomainToIDNA, self).__init__(*args, **kwargs)

        self.domains = [b"\xc9\xa2oogle.com", b"www.huala\xc3\xb1e.cl"]
        self.expected_domains = ["xn--oogle-wmc.com", "www.xn--hualae-0wa.cl"]

    def test_empty_line(self):
        data = ["", "\r", "\n"]

        for empty in data:
            expected = empty

            actual = domain_to_idna(empty)
            self.assertEqual(actual, expected)

    def test_commented_line(self):
        data = "# Hello World"
        expected = data
        actual = domain_to_idna(data)

        self.assertEqual(actual, expected)

    def test_simple_line(self):
        # Test with a space as separator.
        for i in range(len(self.domains)):
            data = (b"0.0.0.0 " + self.domains[i]).decode("utf-8")
            expected = "0.0.0.0 " + self.expected_domains[i]

            actual = domain_to_idna(data)

            self.assertEqual(actual, expected)

        # Test with a tabulation as separator.
        for i in range(len(self.domains)):
            data = (b"0.0.0.0\t" + self.domains[i]).decode("utf-8")
            expected = "0.0.0.0\t" + self.expected_domains[i]

            actual = domain_to_idna(data)

            self.assertEqual(actual, expected)

    def test_multiple_space_as_separator(self):
        # Test with multiple space as separator.
        for i in range(len(self.domains)):
            data = (b"0.0.0.0      " + self.domains[i]).decode("utf-8")
            expected = "0.0.0.0      " + self.expected_domains[i]

            actual = domain_to_idna(data)

            self.assertEqual(actual, expected)

    def test_multiple_tabs_as_separator(self):
        # Test with multiple tabls as separator.
        for i in range(len(self.domains)):
            data = (b"0.0.0.0\t\t\t\t\t\t" + self.domains[i]).decode("utf-8")
            expected = "0.0.0.0\t\t\t\t\t\t" + self.expected_domains[i]

            actual = domain_to_idna(data)

            self.assertEqual(actual, expected)

    def test_line_with_comment_at_the_end(self):
        # Test with a space as separator.
        for i in range(len(self.domains)):
            data = (b"0.0.0.0 " + self.domains[i] + b" # Hello World").decode("utf-8")
            expected = "0.0.0.0 " + self.expected_domains[i] + " # Hello World"

            actual = domain_to_idna(data)

            self.assertEqual(actual, expected)

        # Test with a tabulation as separator.
        for i in range(len(self.domains)):
            data = (b"0.0.0.0\t" + self.domains[i] + b" # Hello World").decode("utf-8")
            expected = "0.0.0.0\t" + self.expected_domains[i] + " # Hello World"

            actual = domain_to_idna(data)

            self.assertEqual(actual, expected)

        # Test with tabulation as separator of domain and comment.
        for i in range(len(self.domains)):
            data = (b"0.0.0.0\t" + self.domains[i] + b"\t # Hello World").decode(
                "utf-8"
            )
            expected = "0.0.0.0\t" + self.expected_domains[i] + "\t # Hello World"

            actual = domain_to_idna(data)

            self.assertEqual(actual, expected)

        # Test with space as separator of domain and tabulation as separator
        # of comments.
        for i in range(len(self.domains)):
            data = (b"0.0.0.0 " + self.domains[i] + b"  \t # Hello World").decode(
                "utf-8"
            )
            expected = "0.0.0.0 " + self.expected_domains[i] + "  \t # Hello World"

            actual = domain_to_idna(data)

            self.assertEqual(actual, expected)

        # Test with multiple space as separator of domain and space and
        # tabulation as separator or comments.
        for i in range(len(self.domains)):
            data = (b"0.0.0.0     " + self.domains[i] + b" \t # Hello World").decode(
                "utf-8"
            )
            expected = "0.0.0.0     " + self.expected_domains[i] + " \t # Hello World"

            actual = domain_to_idna(data)

            self.assertEqual(actual, expected)

        # Test with multiple tabulations as separator of domain and space and
        # tabulation as separator or comments.
        for i, domain in enumerate(self.domains):
            data = (b"0.0.0.0\t\t\t" + domain + b" \t # Hello World").decode(
                "utf-8"
            )
            expected = "0.0.0.0\t\t\t" + self.expected_domains[i] + " \t # Hello World"

            actual = domain_to_idna(data)

            self.assertEqual(actual, expected)

    def test_line_without_prefix(self):
        for i in range(len(self.domains)):
            data = self.domains[i].decode("utf-8")
            expected = self.expected_domains[i]

            actual = domain_to_idna(data)

            self.assertEqual(actual, expected)


class GetFileByUrl(BaseStdout):
    def test_basic(self):
        raw_resp_content = "hello, ".encode("ascii") + "world".encode("utf-8")
        resp_obj = requests.Response()
        resp_obj.__setstate__({"_content": raw_resp_content})

        expected = "hello, world"

        with mock.patch("requests.get", return_value=resp_obj):
            actual = get_file_by_url("www.test-url.com")

        self.assertEqual(expected, actual)

    def test_with_idna(self):
        raw_resp_content = b"www.huala\xc3\xb1e.cl"
        resp_obj = requests.Response()
        resp_obj.__setstate__({"_content": raw_resp_content})

        expected = "www.xn--hualae-0wa.cl"

        with mock.patch("requests.get", return_value=resp_obj):
            actual = get_file_by_url("www.test-url.com")

        self.assertEqual(expected, actual)

    def test_connect_unknown_domain(self):
        test_url = (
            "http://doesnotexist.google.com"  # leads to exception: ConnectionError
        )
        with mock.patch(
            "requests.get", side_effect=requests.exceptions.ConnectionError
        ):
            return_value = get_file_by_url(test_url)
        self.assertIsNone(return_value)
        printed_output = sys.stdout.getvalue()
        self.assertEqual(
            printed_output, "Error retrieving data from {}\n".format(test_url)
        )

    def test_invalid_url(self):
        test_url = "http://fe80::5054:ff:fe5a:fc0"  # leads to exception: InvalidURL
        with mock.patch(
            "requests.get", side_effect=requests.exceptions.ConnectionError
        ):
            return_value = get_file_by_url(test_url)
        self.assertIsNone(return_value)
        printed_output = sys.stdout.getvalue()
        self.assertEqual(
            printed_output, "Error retrieving data from {}\n".format(test_url)
        )


class TestWriteData(Base):
    def test_write_basic(self):
        f = BytesIO()

        data = "foo"
        write_data(f, data)

        expected = b"foo"
        actual = f.getvalue()

        self.assertEqual(actual, expected)

    def test_write_unicode(self):
        f = BytesIO()

        data = u"foo"
        write_data(f, data)

        expected = b"foo"
        actual = f.getvalue()

        self.assertEqual(actual, expected)


class TestQueryYesOrNo(BaseStdout):
    def test_invalid_default(self):
        for invalid_default in ["foo", "bar", "baz", 1, 2, 3]:
            self.assertRaises(ValueError, query_yes_no, "?", invalid_default)

    @mock.patch("updateHostsFile.input", side_effect=["yes"] * 3)
    def test_valid_default(self, _):
        for valid_default, expected in [
            (None, "[y/n]"),
            ("yes", "[Y/n]"),
            ("no", "[y/N]"),
        ]:
            self.assertTrue(query_yes_no("?", valid_default))

            output = sys.stdout.getvalue()
            sys.stdout = StringIO()

            self.assertIn(expected, output)

    @mock.patch("updateHostsFile.input", side_effect=([""] * 2))
    def test_use_valid_default(self, _):
        for valid_default in ["yes", "no"]:
            expected = valid_default == "yes"
            actual = query_yes_no("?", valid_default)

            self.assertEqual(actual, expected)

    @mock.patch("updateHostsFile.input", side_effect=["no", "NO", "N", "n", "No", "nO"])
    def test_valid_no(self, _):
        self.assertFalse(query_yes_no("?", None))

    @mock.patch(
        "updateHostsFile.input",
        side_effect=["yes", "YES", "Y", "yeS", "y", "YeS", "yES", "YEs"],
    )
    def test_valid_yes(self, _):
        self.assertTrue(query_yes_no("?", None))

    @mock.patch("updateHostsFile.input", side_effect=["foo", "yes", "foo", "no"])
    def test_invalid_then_valid(self, _):
        expected = "Please respond with 'yes' or 'no'"

        # The first time, we respond "yes"
        self.assertTrue(query_yes_no("?", None))

        output = sys.stdout.getvalue()
        self.assertIn(expected, output)

        sys.stdout = StringIO()

        # The second time, we respond "no"
        self.assertFalse(query_yes_no("?", None))

        output = sys.stdout.getvalue()
        self.assertIn(expected, output)


class TestIsValidUserProvidedDomainFormat(BaseStdout):
    def test_empty_domain(self):
        self.assertFalse(is_valid_user_provided_domain_format(""))

        output = sys.stdout.getvalue()
        expected = "You didn't enter a domain. Try again."

        self.assertIn(expected, output)

    def test_invalid_domain(self):
        expected = "Do not include www.domain.com or http(s)://domain.com. Try again."

        for invalid_domain in [
            "www.subdomain.domain",
            "https://github.com",
            "http://www.google.com",
        ]:
            self.assertFalse(is_valid_user_provided_domain_format(invalid_domain))

            output = sys.stdout.getvalue()
            sys.stdout = StringIO()

            self.assertIn(expected, output)

    def test_valid_domain(self):
        for valid_domain in ["github.com", "travis.org", "twitter.com"]:
            self.assertTrue(is_valid_user_provided_domain_format(valid_domain))

            output = sys.stdout.getvalue()
            sys.stdout = StringIO()

            self.assertEqual(output, "")


def mock_walk(stem):
    """
    Mock method for `os.walk`.

    Please refer to the documentation of `os.walk` for information about
    the provided parameters.
    """

    files = [
        "foo.txt",
        "bar.bat",
        "baz.py",
        "foo/foo.c",
        "foo/bar.doc",
        "foo/baz/foo.py",
        "bar/foo/baz.c",
        "bar/bar/foo.bat",
    ]

    if stem == ".":
        stem = ""

    matches = []

    for f in files:
        if not stem or f.startswith(stem + "/"):
            matches.append(("", "_", [f]))

    return matches


class TestRecursiveGlob(Base):
    @staticmethod
    def sorted_recursive_glob(stem, file_pattern):
        actual = recursive_glob(stem, file_pattern)
        actual.sort()

        return actual

    @mock.patch("os.walk", side_effect=mock_walk)
    def test_all_match(self, _):
        with self.mock_property("sys.version_info"):
            sys.version_info = (2, 6)

            expected = [
                "bar.bat",
                "bar/bar/foo.bat",
                "bar/foo/baz.c",
                "baz.py",
                "foo.txt",
                "foo/bar.doc",
                "foo/baz/foo.py",
                "foo/foo.c",
            ]
            actual = self.sorted_recursive_glob("*", "*")
            self.assertListEqual(actual, expected)

            expected = ["bar/bar/foo.bat", "bar/foo/baz.c"]
            actual = self.sorted_recursive_glob("bar", "*")
            self.assertListEqual(actual, expected)

            expected = ["foo/bar.doc", "foo/baz/foo.py", "foo/foo.c"]
            actual = self.sorted_recursive_glob("foo", "*")
            self.assertListEqual(actual, expected)

    @mock.patch("os.walk", side_effect=mock_walk)
    def test_file_ending(self, _):
        with self.mock_property("sys.version_info"):
            sys.version_info = (2, 6)

            expected = ["foo/baz/foo.py"]
            actual = self.sorted_recursive_glob("foo", "*.py")
            self.assertListEqual(actual, expected)

            expected = ["bar/foo/baz.c", "foo/foo.c"]
            actual = self.sorted_recursive_glob("*", "*.c")
            self.assertListEqual(actual, expected)

            expected = []
            actual = self.sorted_recursive_glob("*", ".xlsx")
            self.assertListEqual(actual, expected)


def mock_path_join(*_):
    """
    Mock method for `os.path.join`.

    Please refer to the documentation of `os.path.join` for information about
    the provided parameters.
    """

    raise UnicodeDecodeError("foo", b"", 1, 5, "foo")


class TestPathJoinRobust(Base):
    def test_basic(self):
        expected = "path1"
        actual = path_join_robust("path1")
        self.assertEqual(actual, expected)

        actual = path_join_robust(u"path1")
        self.assertEqual(actual, expected)

    def test_join(self):
        for i in range(1, 4):
            paths = ["pathNew"] * i
            expected = "path1" + (self.sep + "pathNew") * i
            actual = path_join_robust("path1", *paths)

            self.assertEqual(actual, expected)

    def test_join_unicode(self):
        for i in range(1, 4):
            paths = [u"pathNew"] * i
            expected = "path1" + (self.sep + "pathNew") * i
            actual = path_join_robust("path1", *paths)

            self.assertEqual(actual, expected)

    @mock.patch("os.path.join", side_effect=mock_path_join)
    def test_join_error(self, _):
        self.assertRaises(locale.Error, path_join_robust, "path")


# Colors
class TestSupportsColor(BaseStdout):
    def test_posix(self):
        with self.mock_property("sys.platform"):
            sys.platform = "Linux"

            with self.mock_property("sys.stdout.isatty") as obj:
                obj.return_value = True
                self.assertTrue(supports_color())

    def test_pocket_pc(self):
        with self.mock_property("sys.platform"):
            sys.platform = "Pocket PC"
            self.assertFalse(supports_color())

    def test_windows_no_ansicon(self):
        with self.mock_property("sys.platform"):
            sys.platform = "win32"

            with self.mock_property("os.environ"):
                os.environ = []

                self.assertFalse(supports_color())

    def test_windows_ansicon(self):
        with self.mock_property("sys.platform"):
            sys.platform = "win32"

            with self.mock_property("os.environ"):
                os.environ = ["ANSICON"]

                with self.mock_property("sys.stdout.isatty") as obj:
                    obj.return_value = True
                    self.assertTrue(supports_color())

    def test_no_isatty_attribute(self):
        with self.mock_property("sys.platform"):
            sys.platform = "Linux"

            with self.mock_property("sys.stdout"):
                sys.stdout = list()
                self.assertFalse(supports_color())

    def test_no_isatty(self):
        with self.mock_property("sys.platform"):
            sys.platform = "Linux"

            with self.mock_property("sys.stdout.isatty") as obj:
                obj.return_value = False
                self.assertFalse(supports_color())


class TestColorize(Base):
    def setUp(self):
        self.text = "house"
        self.colors = ["red", "orange", "yellow", "green", "blue", "purple"]

    @mock.patch("updateHostsFile.supports_color", return_value=False)
    def test_colorize_no_support(self, _):
        for color in self.colors:
            expected = self.text
            actual = colorize(self.text, color)

            self.assertEqual(actual, expected)

    @mock.patch("updateHostsFile.supports_color", return_value=True)
    def test_colorize_support(self, _):
        for color in self.colors:
            expected = color + self.text + Colors.ENDC
            actual = colorize(self.text, color)

            self.assertEqual(actual, expected)


class TestPrintSuccess(BaseStdout):
    def setUp(self):
        super(TestPrintSuccess, self).setUp()
        self.text = "house"

    @mock.patch("updateHostsFile.supports_color", return_value=False)
    def test_print_success_no_support(self, _):
        print_success(self.text)

        expected = self.text + "\n"
        actual = sys.stdout.getvalue()

        self.assertEqual(actual, expected)

    @mock.patch("updateHostsFile.supports_color", return_value=True)
    def test_print_success_support(self, _):
        print_success(self.text)

        expected = Colors.SUCCESS + self.text + Colors.ENDC + "\n"
        actual = sys.stdout.getvalue()

        self.assertEqual(actual, expected)


class TestPrintFailure(BaseStdout):
    def setUp(self):
        super(TestPrintFailure, self).setUp()
        self.text = "house"

    @mock.patch("updateHostsFile.supports_color", return_value=False)
    def test_print_failure_no_support(self, _):
        print_failure(self.text)

        expected = self.text + "\n"
        actual = sys.stdout.getvalue()

        self.assertEqual(actual, expected)

    @mock.patch("updateHostsFile.supports_color", return_value=True)
    def test_print_failure_support(self, _):
        print_failure(self.text)

        expected = Colors.FAIL + self.text + Colors.ENDC + "\n"
        actual = sys.stdout.getvalue()

        self.assertEqual(actual, expected)


# End Helper Functions


if __name__ == "__main__":
    unittest.main()



File: updateReadme.py

#!/usr/bin/env python3

# Script by Steven Black
# https://github.com/StevenBlack
#
# This Python script will update the readme files in this repo.

import json
import os
import time
from string import Template

# Project Settings
BASEDIR_PATH = os.path.dirname(os.path.realpath(__file__))
README_TEMPLATE = os.path.join(BASEDIR_PATH, "readme_template.md")
README_FILENAME = "readme.md"
README_DATA_FILENAME = "readmeData.json"


def main():
    s = Template(
        "${description} | [Readme](https://github.com/StevenBlack/"
        "hosts/blob/master/${location}readme.md) | "
        "[link](https://raw.githubusercontent.com/StevenBlack/"
        "hosts/master/${location}hosts) | "
        "${fmtentries} | "
        "[link](http://sbc.io/hosts/${location}hosts)"
    )
    with open(README_DATA_FILENAME, "r", encoding="utf-8", newline="\n") as f:
        data = json.load(f)

    keys = list(data.keys())
    # Sort by the number of en-dashes in the key
    # and then by the key string itself.
    keys.sort(key=lambda item: (item.replace("-only", "").count("-"), item.replace("-only", "")))

    toc_rows = ""
    for key in keys:
        data[key]["fmtentries"] = "{:,}".format(data[key]["entries"])
        if key == "base":
            data[key]["description"] = "Unified hosts = **(adware + malware)**"
        else:
            if data[key]["no_unified_hosts"]:
                data[key]["description"] = (
                    "**" + key.replace("-only", "").replace("-", " + ") + "**"
                )
            else:
                data[key]["description"] = (
                    "Unified hosts **+ " + key.replace("-", " + ") + "**"
                )

        if "\\" in data[key]["location"]:
            data[key]["location"] = data[key]["location"].replace("\\", "/")

        toc_rows += s.substitute(data[key]) + "\n"

    row_defaults = {
        "name": "",
        "homeurl": "",
        "url": "",
        "license": "",
        "issues": "",
        "description": "",
    }

    t = Template(
        "${name} |[link](${homeurl})"
        " | [raw](${url}) | ${license} | [issues](${issues})| ${description}"
    )
    size_history_graph = "![Size history](https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts_file_size_history.png)"
    for key in keys:
        extensions = key.replace("-only", "").replace("-", ", ")
        extensions_str = "* Extensions: **" + extensions + "**."
        if data[key]["no_unified_hosts"]:
            extensions_header = "Limited to the extensions: " + extensions
        else:
            extensions_header = "Unified hosts file with " + extensions + " extensions"

        source_rows = ""
        source_list = data[key]["sourcesdata"]

        for source in source_list:
            this_row = {}
            this_row.update(row_defaults)
            this_row.update(source)
            source_rows += t.substitute(this_row) + "\n"

        with open(
            os.path.join(data[key]["location"], README_FILENAME),
            "wt",
            encoding="utf-8",
            newline="\n",
        ) as out:
            for line in open(README_TEMPLATE, encoding="utf-8", newline="\n"):
                line = line.replace(
                    "@GEN_DATE@", time.strftime("%B %d %Y", time.gmtime())
                )
                line = line.replace("@EXTENSIONS@", extensions_str)
                line = line.replace("@EXTENSIONS_HEADER@", extensions_header)
                line = line.replace(
                    "@NUM_ENTRIES@", "{:,}".format(data[key]["entries"])
                )
                line = line.replace(
                    "@SUBFOLDER@", os.path.join(data[key]["location"], "")
                )
                line = line.replace("@TOCROWS@", toc_rows)
                line = line.replace("@SOURCEROWS@", source_rows)
                # insert the size graph on the home readme only, for now.
                if key == "base":
                    line = line.replace(
                        "@SIZEHISTORY@", size_history_graph
                    )
                else:
                    line = line.replace("@SIZEHISTORY@", "")

                out.write(line)


if __name__ == "__main__":
    main()



File: updateHostsFile.py

#!/usr/bin/env python3

# Script by Ben Limmer
# https://github.com/l1m5
#
# This Python script will combine all the host files you provide
# as sources into one, unique host file to keep your internet browsing happy.

import argparse
import fnmatch
import ipaddress
import json
import locale
import os
import platform
from pathlib import Path
import re
import shutil
import socket
import subprocess
import sys
import tempfile
import time
from glob import glob
from typing import Optional, Tuple

# Detecting Python 3 for version-dependent implementations
PY3 = sys.version_info >= (3, 0)

if not PY3:
    raise Exception("We do not support Python 2 anymore.")


try:
    import requests
except ImportError:
    raise ImportError(
        "This project's dependencies have changed. The Requests library ("
        "https://docs.python-requests.org/en/latest/) is now required."
    )


# Syntactic sugar for "sudo" command in UNIX / Linux
if platform.system() == "OpenBSD":
    SUDO = ["/usr/bin/doas"]
elif platform.system() == "Windows":
    SUDO = ["powershell", "Start-Process", "powershell", "-Verb", "runAs"]
else:
    SUDO = ["/usr/bin/env", "sudo"]


# Project Settings
BASEDIR_PATH = os.path.dirname(os.path.realpath(__file__))


def get_defaults():
    """
    Helper method for getting the default settings.

    Returns
    -------
    default_settings : dict
        A dictionary of the default settings when updating host information.
    """

    return {
        "numberofrules": 0,
        "datapath": path_join_robust(BASEDIR_PATH, "data"),
        "freshen": True,
        "replace": False,
        "backup": False,
        "skipstatichosts": False,
        "keepdomaincomments": True,
        "extensionspath": path_join_robust(BASEDIR_PATH, "extensions"),
        "extensions": [],
        "nounifiedhosts": False,
        "compress": False,
        "minimise": False,
        "outputsubfolder": "",
        "hostfilename": "hosts",
        "targetip": "0.0.0.0",
        "sourcedatafilename": "update.json",
        "sourcesdata": [],
        "readmefilename": "readme.md",
        "readmetemplate": path_join_robust(BASEDIR_PATH, "readme_template.md"),
        "readmedata": {},
        "readmedatafilename": path_join_robust(BASEDIR_PATH, "readmeData.json"),
        "exclusionpattern": r"([a-zA-Z\d-]+\.){0,}",
        "exclusionregexes": [],
        "exclusions": [],
        "commonexclusions": ["hulu.com"],
        "blacklistfile": path_join_robust(BASEDIR_PATH, "blacklist"),
        "whitelistfile": path_join_robust(BASEDIR_PATH, "whitelist"),
    }


# End Project Settings


def main():
    parser = argparse.ArgumentParser(
        description="Creates a unified hosts "
        "file from hosts stored in the data subfolders."
    )
    parser.add_argument(
        "--auto",
        "-a",
        dest="auto",
        default=False,
        action="store_true",
        help="Run without prompting.",
    )
    parser.add_argument(
        "--backup",
        "-b",
        dest="backup",
        default=False,
        action="store_true",
        help="Backup the hosts files before they are overridden.",
    )
    parser.add_argument(
        "--extensions",
        "-e",
        dest="extensions",
        default=[],
        nargs="*",
        help="Host extensions to include in the final hosts file.",
    )
    parser.add_argument(
        "--nounifiedhosts",
        dest="nounifiedhosts",
        default=False,
        action="store_true",
        help="Do not include the unified hosts file in the final hosts file. Usually used together with `--extensions`.",
    )
    parser.add_argument(
        "--ip",
        "-i",
        dest="targetip",
        default="0.0.0.0",
        help="Target IP address. Default is 0.0.0.0.",
    )
    parser.add_argument(
        "--keepdomaincomments",
        "-k",
        dest="keepdomaincomments",
        action="store_false",
        default=True,
        help="Do not keep domain line comments.",
    )
    parser.add_argument(
        "--noupdate",
        "-n",
        dest="noupdate",
        default=False,
        action="store_true",
        help="Don't update from host data sources.",
    )
    parser.add_argument(
        "--skipstatichosts",
        "-s",
        dest="skipstatichosts",
        default=False,
        action="store_true",
        help="Skip static localhost entries in the final hosts file.",
    )
    parser.add_argument(
        "--nogendata",
        "-g",
        dest="nogendata",
        default=False,
        action="store_true",
        help="Skip generation of readmeData.json",
    )
    parser.add_argument(
        "--output",
        "-o",
        dest="outputsubfolder",
        default="",
        help="Output subfolder for generated hosts file.",
    )
    parser.add_argument(
        "--replace",
        "-r",
        dest="replace",
        default=False,
        action="store_true",
        help="Replace your active hosts file with this new hosts file.",
    )
    parser.add_argument(
        "--flush-dns-cache",
        "-f",
        dest="flushdnscache",
        default=False,
        action="store_true",
        help="Attempt to flush DNS cache after replacing the hosts file.",
    )
    parser.add_argument(
        "--compress",
        "-c",
        dest="compress",
        default=False,
        action="store_true",
        help="Compress the hosts file ignoring non-necessary lines "
        "(empty lines and comments) and putting multiple domains in "
        "each line. Improve the performance under Windows.",
    )
    parser.add_argument(
        "--minimise",
        "-m",
        dest="minimise",
        default=False,
        action="store_true",
        help="Minimise the hosts file ignoring non-necessary lines "
        "(empty lines and comments).",
    )
    parser.add_argument(
        "--whitelist",
        "-w",
        dest="whitelistfile",
        default=path_join_robust(BASEDIR_PATH, "whitelist"),
        help="Whitelist file to use while generating hosts files.",
    )
    parser.add_argument(
        "--blacklist",
        "-x",
        dest="blacklistfile",
        default=path_join_robust(BASEDIR_PATH, "blacklist"),
        help="Blacklist file to use while generating hosts files.",
    )

    global settings

    options = vars(parser.parse_args())

    options["outputpath"] = path_join_robust(BASEDIR_PATH, options["outputsubfolder"])
    options["freshen"] = not options["noupdate"]

    settings = get_defaults()
    settings.update(options)

    data_path = settings["datapath"]
    extensions_path = settings["extensionspath"]

    settings["sources"] = list_dir_no_hidden(data_path)
    settings["extensionsources"] = list_dir_no_hidden(extensions_path)

    # All our extensions folders...
    settings["extensions"] = [
        os.path.basename(item) for item in list_dir_no_hidden(extensions_path)
    ]
    # ... intersected with the extensions passed-in as arguments, then sorted.
    settings["extensions"] = sorted(
        list(set(options["extensions"]).intersection(settings["extensions"]))
    )

    auto = settings["auto"]
    exclusion_regexes = settings["exclusionregexes"]
    source_data_filename = settings["sourcedatafilename"]
    no_unified_hosts = settings["nounifiedhosts"]

    update_sources = prompt_for_update(freshen=settings["freshen"], update_auto=auto)
    if update_sources:
        update_all_sources(source_data_filename, settings["hostfilename"])

    gather_exclusions = prompt_for_exclusions(skip_prompt=auto)

    if gather_exclusions:
        common_exclusions = settings["commonexclusions"]
        exclusion_pattern = settings["exclusionpattern"]
        exclusion_regexes = display_exclusion_options(
            common_exclusions=common_exclusions,
            exclusion_pattern=exclusion_pattern,
            exclusion_regexes=exclusion_regexes,
        )

    extensions = settings["extensions"]
    sources_data = update_sources_data(
        settings["sourcesdata"],
        datapath=data_path,
        extensions=extensions,
        extensionspath=extensions_path,
        sourcedatafilename=source_data_filename,
        nounifiedhosts=no_unified_hosts,
    )

    merge_file = create_initial_file(
        nounifiedhosts=no_unified_hosts,
    )
    remove_old_hosts_file(settings["outputpath"], "hosts", settings["backup"])
    if settings["compress"]:
        final_file = open(path_join_robust(settings["outputpath"], "hosts"), "w+b")
        compressed_file = tempfile.NamedTemporaryFile()
        remove_dups_and_excl(merge_file, exclusion_regexes, compressed_file)
        compress_file(compressed_file, settings["targetip"], final_file)
    elif settings["minimise"]:
        final_file = open(path_join_robust(settings["outputpath"], "hosts"), "w+b")
        minimised_file = tempfile.NamedTemporaryFile()
        remove_dups_and_excl(merge_file, exclusion_regexes, minimised_file)
        minimise_file(minimised_file, settings["targetip"], final_file)
    else:
        final_file = remove_dups_and_excl(merge_file, exclusion_regexes)

    number_of_rules = settings["numberofrules"]
    output_subfolder = settings["outputsubfolder"]
    skip_static_hosts = settings["skipstatichosts"]

    write_opening_header(
        final_file,
        extensions=extensions,
        numberofrules=number_of_rules,
        outputsubfolder=output_subfolder,
        skipstatichosts=skip_static_hosts,
        nounifiedhosts=no_unified_hosts,
    )
    final_file.close()

    if not settings["nogendata"]:
        update_readme_data(
            settings["readmedatafilename"],
            extensions=extensions,
            numberofrules=number_of_rules,
            outputsubfolder=output_subfolder,
            sourcesdata=sources_data,
            nounifiedhosts=no_unified_hosts,
        )

    print_success(
        "Success! The hosts file has been saved in folder "
        + output_subfolder
        + "\nIt contains "
        + "{:,}".format(number_of_rules)
        + " unique entries."
    )

    move_file = prompt_for_move(
        final_file,
        auto=auto,
        replace=settings["replace"],
        skipstatichosts=skip_static_hosts,
    )

    # We only flush the DNS cache if we have
    # moved a new hosts file into place.
    if move_file:
        prompt_for_flush_dns_cache(
            flush_cache=settings["flushdnscache"], prompt_flush=not auto
        )


# Prompt the User
def prompt_for_update(freshen, update_auto):
    """
    Prompt the user to update all hosts files.

    If requested, the function will update all data sources after it
    checks that a hosts file does indeed exist.

    Parameters
    ----------
    freshen : bool
        Whether data sources should be updated. This function will return
        if it is requested that data sources not be updated.
    update_auto : bool
        Whether or not to automatically update all data sources.

    Returns
    -------
    update_sources : bool
        Whether or not we should update data sources for exclusion files.
    """

    # Create a hosts file if it doesn't exist.
    hosts_file = path_join_robust(BASEDIR_PATH, "hosts")

    if not os.path.isfile(hosts_file):
        try:
            open(hosts_file, "w+").close()
        except (IOError, OSError):
            # Starting in Python 3.3, IOError is aliased
            # OSError. However, we have to catch both for
            # Python 2.x failures.
            print_failure(
                "ERROR: No 'hosts' file in the folder. Try creating one manually."
            )

    if not freshen:
        return False

    prompt = "Do you want to update all data sources?"

    if update_auto or query_yes_no(prompt):
        return True
    elif not update_auto:
        print("OK, we'll stick with what we've got locally.")

    return False


def prompt_for_exclusions(skip_prompt):
    """
    Prompt the user to exclude any custom domains from being blocked.

    Parameters
    ----------
    skip_prompt : bool
        Whether or not to skip prompting for custom domains to be excluded.
        If true, the function returns immediately.

    Returns
    -------
    gather_exclusions : bool
        Whether or not we should proceed to prompt the user to exclude any
        custom domains beyond those in the whitelist.
    """

    prompt = (
        "Do you want to exclude any domains?\n"
        "For example, hulu.com video streaming must be able to access "
        "its tracking and ad servers in order to play video."
    )

    if not skip_prompt:
        if query_yes_no(prompt):
            return True
        else:
            print("OK, we'll only exclude domains in the whitelist.")

    return False


def prompt_for_flush_dns_cache(flush_cache, prompt_flush):
    """
    Prompt the user to flush the DNS cache.

    Parameters
    ----------
    flush_cache : bool
        Whether to flush the DNS cache without prompting.
    prompt_flush : bool
        If `flush_cache` is False, whether we should prompt for flushing the
        cache. Otherwise, the function returns immediately.
    """

    if flush_cache:
        flush_dns_cache()
    elif prompt_flush:
        if query_yes_no("Attempt to flush the DNS cache?"):
            flush_dns_cache()


def prompt_for_move(final_file, **move_params):
    """
    Prompt the user to move the newly created hosts file to its designated
    location in the OS.

    Parameters
    ----------
    final_file : file
        The file object that contains the newly created hosts data.
    move_params : kwargs
        Dictionary providing additional parameters for moving the hosts file
        into place. Currently, those fields are:

        1) auto
        2) replace
        3) skipstatichosts

    Returns
    -------
    move_file : bool
        Whether or not the final hosts file was moved.
    """

    skip_static_hosts = move_params["skipstatichosts"]

    if move_params["replace"] and not skip_static_hosts:
        move_file = True
    elif move_params["auto"] or skip_static_hosts:
        move_file = False
    else:
        prompt = "Do you want to replace your existing hosts file with the newly generated file?"
        move_file = query_yes_no(prompt)

    if move_file:
        move_file = move_hosts_file_into_place(final_file)

    return move_file


# End Prompt the User


def sort_sources(sources):
    """
    Sorts the sources.
    The idea is that all Steven Black's list, file or entries
    get on top and the rest sorted alphabetically.

    Parameters
    ----------
    sources: list
        The sources to sort.
    """

    result = sorted(
        sources.copy(),
        key=lambda x: x.lower().replace("-", "").replace("_", "").replace(" ", ""),
    )

    # Steven Black's repositories/files/lists should be on top!
    steven_black_positions = [
        x for x, y in enumerate(result) if "stevenblack" in y.lower()
    ]

    for index in steven_black_positions:
        result.insert(0, result.pop(index))

    return result


# Exclusion logic
def display_exclusion_options(common_exclusions, exclusion_pattern, exclusion_regexes):
    """
    Display the exclusion options to the user.

    This function checks whether a user wants to exclude particular domains,
    and if so, excludes them.

    Parameters
    ----------
    common_exclusions : list
        A list of common domains that are excluded from being blocked. One
        example is Hulu. This setting is set directly in the script and cannot
        be overwritten by the user.
    exclusion_pattern : str
        The exclusion pattern with which to create the domain regex.
    exclusion_regexes : list
        The list of regex patterns used to exclude domains.

    Returns
    -------
    aug_exclusion_regexes : list
        The original list of regex patterns potentially with additional
        patterns from domains that the user chooses to exclude.
    """

    for exclusion_option in common_exclusions:
        prompt = "Do you want to exclude the domain " + exclusion_option + " ?"

        if query_yes_no(prompt):
            exclusion_regexes = exclude_domain(
                exclusion_option, exclusion_pattern, exclusion_regexes
            )
        else:
            continue

    if query_yes_no("Do you want to exclude any other domains?"):
        exclusion_regexes = gather_custom_exclusions(
            exclusion_pattern, exclusion_regexes
        )

    return exclusion_regexes


def gather_custom_exclusions(exclusion_pattern, exclusion_regexes):
    """
    Gather custom exclusions from the user.

    Parameters
    ----------
    exclusion_pattern : str
        The exclusion pattern with which to create the domain regex.
    exclusion_regexes : list
        The list of regex patterns used to exclude domains.

    Returns
    -------
    aug_exclusion_regexes : list
        The original list of regex patterns potentially with additional
        patterns from domains that the user chooses to exclude.
    """

    # We continue running this while-loop until the user
    # says that they have no more domains to exclude.
    while True:
        domain_prompt = "Enter the domain you want to exclude (e.g. facebook.com): "
        user_domain = input(domain_prompt)

        if is_valid_user_provided_domain_format(user_domain):
            exclusion_regexes = exclude_domain(
                user_domain, exclusion_pattern, exclusion_regexes
            )

        continue_prompt = "Do you have more domains you want to enter?"
        if not query_yes_no(continue_prompt):
            break

    return exclusion_regexes


def exclude_domain(domain, exclusion_pattern, exclusion_regexes):
    """
    Exclude a domain from being blocked.

    This creates the domain regex by which to exclude this domain and appends
    it a list of already-existing exclusion regexes.

    Parameters
    ----------
    domain : str
        The filename or regex pattern to exclude.
    exclusion_pattern : str
        The exclusion pattern with which to create the domain regex.
    exclusion_regexes : list
        The list of regex patterns used to exclude domains.

    Returns
    -------
    aug_exclusion_regexes : list
        The original list of regex patterns with one additional pattern from
        the `domain` input.
    """

    exclusion_regex = re.compile(exclusion_pattern + domain)
    exclusion_regexes.append(exclusion_regex)

    return exclusion_regexes


def matches_exclusions(stripped_rule, exclusion_regexes):
    """
    Check whether a rule matches an exclusion rule we already provided.

    If this function returns True, that means this rule should be excluded
    from the final hosts file.

    Parameters
    ----------
    stripped_rule : str
        The rule that we are checking.
    exclusion_regexes : list
        The list of regex patterns used to exclude domains.

    Returns
    -------
    matches_exclusion : bool
        Whether or not the rule string matches a provided exclusion.
    """

    try:
        stripped_domain = stripped_rule.split()[1]
    except IndexError:
        # Example: 'example.org' instead of '0.0.0.0 example.org'
        stripped_domain = stripped_rule

    for exclusionRegex in exclusion_regexes:
        if exclusionRegex.search(stripped_domain):
            return True

    return False


# End Exclusion Logic


# Update Logic
def update_sources_data(sources_data, **sources_params):
    """
    Update the sources data and information for each source.

    Parameters
    ----------
    sources_data : list
        The list of sources data that we are to update.
    sources_params : kwargs
        Dictionary providing additional parameters for updating the
        sources data. Currently, those fields are:

        1) datapath
        2) extensions
        3) extensionspath
        4) sourcedatafilename
        5) nounifiedhosts

    Returns
    -------
    update_sources_data : list
        The original source data list with new source data appended.
    """

    source_data_filename = sources_params["sourcedatafilename"]

    if not sources_params["nounifiedhosts"]:
        for source in sort_sources(
            recursive_glob(sources_params["datapath"], source_data_filename)
        ):
            update_file = open(source, "r", encoding="UTF-8")
            try:
                update_data = json.load(update_file)
                sources_data.append(update_data)
            finally:
                update_file.close()

    for source in sources_params["extensions"]:
        source_dir = path_join_robust(sources_params["extensionspath"], source)
        for update_file_path in sort_sources(
            recursive_glob(source_dir, source_data_filename)
        ):
            update_file = open(update_file_path, "r")
            try:
                update_data = json.load(update_file)
                sources_data.append(update_data)
            finally:
                update_file.close()

    return sources_data


def jsonarray(json_array_string):
    """
    Transformer, converts a json array string hosts into one host per
    line, prefixing each line with "127.0.0.1 ".

    Parameters
    ----------
    json_array_string : str
        The json array string in the form
          '["example1.com", "example1.com", ...]'
    """

    temp_list = json.loads(json_array_string)
    hostlines = "127.0.0.1 " + "\n127.0.0.1 ".join(temp_list)
    return hostlines


def update_all_sources(source_data_filename, host_filename):
    """
    Update all host files, regardless of folder depth.

    Parameters
    ----------
    source_data_filename : str
        The name of the filename where information regarding updating
        sources for a particular URL is stored. This filename is assumed
        to be the same for all sources.
    host_filename : str
        The name of the file in which the updated source information
        is stored for a particular URL. This filename is assumed to be
        the same for all sources.
    """

    # The transforms we support
    transform_methods = {"jsonarray": jsonarray}

    all_sources = sort_sources(recursive_glob("*", source_data_filename))

    for source in all_sources:
        update_file = open(source, "r", encoding="UTF-8")
        update_data = json.load(update_file)
        update_file.close()

        # we can pause updating any given hosts source.
        # if the update.json "pause" key is missing, don't pause.
        if update_data.get("pause", False):
            continue

        update_url = update_data["url"]
        update_transforms = []
        if update_data.get("transforms"):
            update_transforms = update_data["transforms"]

        print("Updating source " + os.path.dirname(source) + " from " + update_url)

        try:
            updated_file = get_file_by_url(update_url)

            # spin the transforms as required
            for transform in update_transforms:
                updated_file = transform_methods[transform](updated_file)

            # get rid of carriage-return symbols
            updated_file = updated_file.replace("\r", "")

            hosts_file = open(
                path_join_robust(BASEDIR_PATH, os.path.dirname(source), host_filename),
                "wb",
            )
            write_data(hosts_file, updated_file)
            hosts_file.close()
        except Exception:
            print("Error in updating source: ", update_url)


# End Update Logic


# File Logic
def create_initial_file(**initial_file_params):
    """
    Initialize the file in which we merge all host files for later pruning.

    Parameters
    ----------
    header_params : kwargs
        Dictionary providing additional parameters for populating the initial file
        information. Currently, those fields are:

        1) nounifiedhosts
    """

    merge_file = tempfile.NamedTemporaryFile()

    if not initial_file_params["nounifiedhosts"]:
        # spin the sources for the base file
        for source in sort_sources(
            recursive_glob(settings["datapath"], settings["hostfilename"])
        ):
            start = "# Start {}\n\n".format(os.path.basename(os.path.dirname(source)))
            end = "\n# End {}\n\n".format(os.path.basename(os.path.dirname(source)))

            with open(source, "r", encoding="UTF-8") as curFile:
                write_data(merge_file, start + curFile.read() + end)

    # spin the sources for extensions to the base file
    for source in settings["extensions"]:
        for filename in sort_sources(
            recursive_glob(
                path_join_robust(settings["extensionspath"], source),
                settings["hostfilename"],
            )
        ):
            with open(filename, "r") as curFile:
                write_data(merge_file, curFile.read())

    maybe_copy_example_file(settings["blacklistfile"])

    if os.path.isfile(settings["blacklistfile"]):
        with open(settings["blacklistfile"], "r") as curFile:
            write_data(merge_file, curFile.read())

    return merge_file


def compress_file(input_file, target_ip, output_file):
    """
    Reduce the file dimension removing non-necessary lines (empty lines and
    comments) and putting multiple domains in each line.
    Reducing the number of lines of the file, the parsing under Microsoft
    Windows is much faster.

    Parameters
    ----------
    input_file : file
        The file object that contains the hostnames that we are reducing.
    target_ip : str
        The target IP address.
    output_file : file
        The file object that will contain the reduced hostnames.
    """

    input_file.seek(0)  # reset file pointer
    write_data(output_file, "\n")

    target_ip_len = len(target_ip)
    lines = [target_ip]
    lines_index = 0
    for line in input_file.readlines():
        line = line.decode("UTF-8")

        if line.startswith(target_ip):
            if lines[lines_index].count(" ") < 9:
                lines[lines_index] += (
                    " " + line[target_ip_len : line.find("#")].strip()  # noqa: E203
                )
            else:
                lines[lines_index] += "\n"
                lines.append(line[: line.find("#")].strip())
                lines_index += 1

    for line in lines:
        write_data(output_file, line)

    input_file.close()


def minimise_file(input_file, target_ip, output_file):
    """
    Reduce the file dimension removing non-necessary lines (empty lines and
    comments).

    Parameters
    ----------
    input_file : file
        The file object that contains the hostnames that we are reducing.
    target_ip : str
        The target IP address.
    output_file : file
        The file object that will contain the reduced hostnames.
    """

    input_file.seek(0)  # reset file pointer
    write_data(output_file, "\n")

    lines = []
    for line in input_file.readlines():
        line = line.decode("UTF-8")

        if line.startswith(target_ip):
            lines.append(line[: line.find("#")].strip() + "\n")

    for line in lines:
        write_data(output_file, line)

    input_file.close()


def remove_dups_and_excl(merge_file, exclusion_regexes, output_file=None):
    """
    Remove duplicates and remove hosts that we are excluding.

    We check for duplicate hostnames as well as remove any hostnames that
    have been explicitly excluded by the user.

    Parameters
    ----------
    merge_file : file
        The file object that contains the hostnames that we are pruning.
    exclusion_regexes : list
        The list of regex patterns used to exclude domains.
    output_file : file
        The file object in which the result is written. If None, the file
        'settings["outputpath"]' will be created.
    """

    number_of_rules = settings["numberofrules"]
    maybe_copy_example_file(settings["whitelistfile"])

    if os.path.isfile(settings["whitelistfile"]):
        with open(settings["whitelistfile"], "r") as ins:
            for line in ins:
                line = line.strip(" \t\n\r")
                if line and not line.startswith("#"):
                    settings["exclusions"].append(line)

    if not os.path.exists(settings["outputpath"]):
        os.makedirs(settings["outputpath"])

    if output_file is None:
        final_file = open(path_join_robust(settings["outputpath"], "hosts"), "w+b")
    else:
        final_file = output_file

    merge_file.seek(0)  # reset file pointer
    hostnames = {"localhost", "localhost.localdomain", "local", "broadcasthost"}
    exclusions = settings["exclusions"]

    for line in merge_file.readlines():
        write_line = True

        # Explicit encoding
        line = line.decode("UTF-8")

        # replace tabs with space
        line = line.replace("\t+", " ")

        # see gh-271: trim trailing whitespace, periods
        line = line.rstrip(" .")

        # Testing the first character doesn't require startswith
        if line[0] == "#" or re.match(r"^\s*$", line[0]):
            write_data(final_file, line)
            continue
        if "::1" in line:
            continue

        stripped_rule = strip_rule(line)  # strip comments
        if not stripped_rule or matches_exclusions(stripped_rule, exclusion_regexes):
            continue

        # Issue #1628
        if "@" in stripped_rule:
            continue

        # Normalize rule
        hostname, normalized_rule = normalize_rule(
            stripped_rule,
            target_ip=settings["targetip"],
            keep_domain_comments=settings["keepdomaincomments"],
        )

        for exclude in exclusions:
            if re.search(r"(^|[\s\.])" + re.escape(exclude) + r"\s", line):
                write_line = False
                break

        if normalized_rule and (hostname not in hostnames) and write_line:
            write_data(final_file, normalized_rule)
            hostnames.add(hostname)
            number_of_rules += 1

    settings["numberofrules"] = number_of_rules
    merge_file.close()

    if output_file is None:
        return final_file


def normalize_rule(rule, target_ip, keep_domain_comments):
    """
    Standardize and format the rule string provided.

    Parameters
    ----------
    rule : str
        The rule whose spelling and spacing we are standardizing.
    target_ip : str
        The target IP address for the rule.
    keep_domain_comments : bool
        Whether or not to keep comments regarding these domains in
        the normalized rule.

    Returns
    -------
    normalized_rule : tuple
        A tuple of the hostname and the rule string with spelling
        and spacing reformatted.
    """

    def normalize_response(
        extracted_hostname: str, extracted_suffix: Optional[str]
    ) -> Tuple[str, str]:
        """
        Normalizes the responses after the provision of the extracted
        hostname and suffix - if exist.

        Parameters
        ----------
        extracted_hostname: str
            The extracted hostname to work with.
        extracted_suffix: str
            The extracted suffix to with.

        Returns
        -------
        normalized_response: tuple
            A tuple of the hostname and the rule string with spelling
            and spacing reformatted.
        """

        rule = "%s %s" % (target_ip, extracted_hostname)

        if keep_domain_comments and extracted_suffix:
            if not extracted_suffix.strip().startswith("#"):
                # Strings are stripped, therefore we need to add the space back.
                rule += " # %s" % extracted_suffix
            else:
                rule += " %s" % extracted_suffix

        return extracted_hostname, rule + "\n"

    def is_ip(dataset: str) -> bool:
        """
        Checks whether the given dataset is an IP.

        Parameters
        ----------

        dataset: str
            The dataset to work with.

        Returns
        -------
        is_ip: bool
            Whether the dataset is an IP.
        """

        try:
            _ = ipaddress.ip_address(dataset)
            return True
        except ValueError:
            return False

    def belch_unwanted(unwanted: str) -> Tuple[None, None]:
        """
        Belches unwanted to screen.

        Parameters
        ----------
        unwanted: str
            The unwanted string to belch.

        Returns
        -------
        belched: tuple
            A tuple of None, None.
        """

        """
        finally, if we get here, just belch to screen
        """
        print("==>%s<==" % unwanted)
        return None, None

    """
    first try: IP followed by domain
    """

    static_ip_regex = r"^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$"
    split_rule = rule.split(maxsplit=1)

    if is_ip(split_rule[0]):
        # Assume that the first item is an IP address following the rule.

        if " " or "\t" in split_rule[-1]:
            try:
                # Example: 0.0.0.0 example.org # hello, world!
                hostname, suffix = split_rule[-1].split(maxsplit=1)
            except ValueError:
                # Example: 0.0.0.0 example.org[:space:]
                hostname, suffix = split_rule[-1], None
        else:
            # Example: 0.0.0.0 example.org
            hostname, suffix = split_rule[-1], None

        hostname = hostname.lower()

        if (
            is_ip(hostname)
            or re.search(static_ip_regex, hostname)
            or "." not in hostname
            or "/" in hostname
            or ".." in hostname
            or ":" in hostname
        ):
            # Example: 0.0.0.0 127.0.0.1

            # If the hostname is:
            #   - an IP - or looks like it,
            #   - doesn't contain dots, or
            #   - contains a colon,
            # we don't want to normalize it.
            return belch_unwanted(rule)

        return normalize_response(hostname, suffix)

    if (
        not re.search(static_ip_regex, split_rule[0])
        and ":" not in split_rule[0]
        and ".." not in split_rule[0]
        and "/" not in split_rule[0]
        and "." in split_rule[0]
    ):
        # Deny anything that looks like an IP; doesn't container dots or INVALID.

        try:
            hostname, suffix = split_rule
        except ValueError:
            hostname, suffix = split_rule[0], None

        hostname = hostname.lower()

        return normalize_response(hostname, suffix)

    return belch_unwanted(rule)


def strip_rule(line):
    """
    Sanitize a rule string provided before writing it to the output hosts file.

    Parameters
    ----------
    line : str
        The rule provided for sanitation.

    Returns
    -------
    sanitized_line : str
        The sanitized rule.
    """

    return " ".join(line.split())


def write_opening_header(final_file, **header_params):
    """
    Write the header information into the newly-created hosts file.

    Parameters
    ----------
    final_file : file
        The file object that points to the newly-created hosts file.
    header_params : kwargs
        Dictionary providing additional parameters for populating the header
        information. Currently, those fields are:

        1) extensions
        2) numberofrules
        3) outputsubfolder
        4) skipstatichosts
        5) nounifiedhosts
    """

    final_file.seek(0)  # Reset file pointer.
    file_contents = final_file.read()  # Save content.

    final_file.seek(0)  # Write at the top.

    no_unified_hosts = header_params["nounifiedhosts"]

    if header_params["extensions"]:
        if no_unified_hosts:
            if len(header_params["extensions"]) > 1:
                write_data(
                    final_file,
                    "# Title: StevenBlack/hosts extensions {0} and {1} \n#\n".format(
                        ", ".join(header_params["extensions"][:-1]),
                        header_params["extensions"][-1],
                    ),
                )
            else:
                write_data(
                    final_file,
                    "# Title: StevenBlack/hosts extension {0}\n#\n".format(
                        ", ".join(header_params["extensions"])
                    ),
                )
        else:
            if len(header_params["extensions"]) > 1:
                write_data(
                    final_file,
                    "# Title: StevenBlack/hosts with the {0} and {1} extensions\n#\n".format(
                        ", ".join(header_params["extensions"][:-1]),
                        header_params["extensions"][-1],
                    ),
                )
            else:
                write_data(
                    final_file,
                    "# Title: StevenBlack/hosts with the {0} extension\n#\n".format(
                        ", ".join(header_params["extensions"])
                    ),
                )
    else:
        write_data(final_file, "# Title: StevenBlack/hosts\n#\n")

    write_data(
        final_file,
        "# This hosts file is a merged collection "
        "of hosts from reputable sources,\n",
    )
    write_data(final_file, "# with a dash of crowd sourcing via GitHub\n#\n")
    write_data(
        final_file,
        "# Date: " + time.strftime("%d %B %Y %H:%M:%S (%Z)", time.gmtime()) + "\n",
    )

    if header_params["extensions"]:
        if header_params["nounifiedhosts"]:
            write_data(
                final_file,
                "# The unified hosts file was not used while generating this file.\n"
                "# Extensions used to generate this file: "
                + ", ".join(header_params["extensions"])
                + "\n",
            )
        else:
            write_data(
                final_file,
                "# Extensions added to this file: "
                + ", ".join(header_params["extensions"])
                + "\n",
            )

    write_data(
        final_file,
        (
            "# Number of unique domains: {:,}\n#\n".format(
                header_params["numberofrules"]
            )
        ),
    )
    write_data(
        final_file,
        "# Fetch the latest version of this file: "
        "https://raw.githubusercontent.com/StevenBlack/hosts/master/"
        + path_join_robust(header_params["outputsubfolder"], "").replace("\\", "/")
        + "hosts\n",
    )
    write_data(
        final_file, "# Project home page: https://github.com/StevenBlack/hosts\n"
    )
    write_data(
        final_file,
        "# Project releases: https://github.com/StevenBlack/hosts/releases\n#\n",
    )
    write_data(
        final_file,
        "# ===============================================================\n",
    )
    write_data(final_file, "\n")

    if not header_params["skipstatichosts"]:
        write_data(final_file, "127.0.0.1 localhost\n")
        write_data(final_file, "127.0.0.1 localhost.localdomain\n")
        write_data(final_file, "127.0.0.1 local\n")
        write_data(final_file, "255.255.255.255 broadcasthost\n")
        write_data(final_file, "::1 localhost\n")
        write_data(final_file, "::1 ip6-localhost\n")
        write_data(final_file, "::1 ip6-loopback\n")
        write_data(final_file, "fe80::1%lo0 localhost\n")
        write_data(final_file, "ff00::0 ip6-localnet\n")
        write_data(final_file, "ff00::0 ip6-mcastprefix\n")
        write_data(final_file, "ff02::1 ip6-allnodes\n")
        write_data(final_file, "ff02::2 ip6-allrouters\n")
        write_data(final_file, "ff02::3 ip6-allhosts\n")
        write_data(final_file, "0.0.0.0 0.0.0.0\n")

        if platform.system() == "Linux":
            write_data(final_file, "127.0.1.1 " + socket.gethostname() + "\n")
            write_data(final_file, "127.0.0.53 " + socket.gethostname() + "\n")

        write_data(final_file, "\n")

    preamble = path_join_robust(BASEDIR_PATH, "myhosts")
    maybe_copy_example_file(preamble)

    if os.path.isfile(preamble):
        with open(preamble, "r") as f:
            write_data(final_file, f.read())

    final_file.write(file_contents)


def update_readme_data(readme_file, **readme_updates):
    """
    Update the host and website information provided in the README JSON data.

    Parameters
    ----------
    readme_file : str
        The name of the README file to update.
    readme_updates : kwargs
        Dictionary providing additional JSON fields to update before
        saving the data. Currently, those fields are:

        1) extensions
        2) sourcesdata
        3) numberofrules
        4) outputsubfolder
        5) nounifiedhosts
    """

    extensions_key = "base"
    extensions = readme_updates["extensions"]
    no_unified_hosts = readme_updates["nounifiedhosts"]

    if extensions:
        extensions_key = "-".join(extensions)
        if no_unified_hosts:
            extensions_key = extensions_key + "-only"

    output_folder = readme_updates["outputsubfolder"]
    generation_data = {
        "location": path_join_robust(output_folder, ""),
        "no_unified_hosts": no_unified_hosts,
        "entries": readme_updates["numberofrules"],
        "sourcesdata": readme_updates["sourcesdata"],
    }

    with open(readme_file, "r") as f:
        readme_data = json.load(f)
        readme_data[extensions_key] = generation_data

    for denomination, data in readme_data.copy().items():
        if "location" in data and data["location"] and "\\" in data["location"]:
            # Windows compatibility: #1166
            readme_data[denomination]["location"] = data["location"].replace("\\", "/")

    with open(readme_file, "w") as f:
        json.dump(readme_data, f)


def move_hosts_file_into_place(final_file):
    r"""
    Move the newly-created hosts file into its correct location on the OS.

    For UNIX systems, the hosts file is "etc/hosts." On Windows, it's
    "C:\Windows\System32\drivers\etc\hosts."

    For this move to work, you must have administrator privileges to do this.
    On UNIX systems, this means having "sudo" access, and on Windows, it
    means being able to run command prompt in administrator mode.

    Parameters
    ----------
    final_file : file object
        The newly-created hosts file to move.
    """  # noqa: W605

    filename = os.path.abspath(final_file.name)

    try:
        if not Path(filename).exists():
            raise FileNotFoundError
    except Exception:
        print_failure(f"{filename} does not exist.")
        return False

    if platform.system() == "Windows":
        target_file = str(
            Path(os.getenv("SystemRoot")) / "system32" / "drivers" / "etc" / "hosts"
        )
    else:
        target_file = "/etc/hosts"

    if os.getenv("IN_CONTAINER"):
        # It's not allowed to remove/replace a mounted /etc/hosts, so we replace the content.
        # This requires running the container user as root, as is the default.
        print(f"Running in container, so we will replace the content of {target_file}.")
        try:
            with open(target_file, "w") as target_stream:
                with open(filename, "r") as source_stream:
                    source = source_stream.read()
                    target_stream.write(source)
            return True
        except Exception:
            print_failure(f"Replacing content of {target_file} failed.")
            return False
    elif (
        platform.system() == "Linux"
        or platform.system() == "Windows"
        or platform.system() == "Darwin"
    ):
        print(
            f"Replacing {target_file} requires root privileges. You might need to enter your password."
        )
        try:
            subprocess.run(SUDO + ["cp", filename, target_file], check=True)
            return True
        except subprocess.CalledProcessError:
            print_failure(f"Replacing {target_file} failed.")
            return False


def flush_dns_cache():
    """
    Flush the DNS cache.
    """

    print("Flushing the DNS cache to utilize new hosts file...")
    print(
        "Flushing the DNS cache requires administrative privileges. You might need to enter your password."
    )

    dns_cache_found = False

    if platform.system() == "Darwin":
        if subprocess.call(SUDO + ["killall", "-HUP", "mDNSResponder"]):
            print_failure("Flushing the DNS cache failed.")
    elif os.name == "nt":
        print("Automatically flushing the DNS cache is not yet supported.")
        print(
            "Please copy and paste the command 'ipconfig /flushdns' in "
            "administrator command prompt after running this script."
        )
    else:
        nscd_prefixes = ["/etc", "/etc/rc.d"]
        nscd_msg = "Flushing the DNS cache by restarting nscd {result}"

        for nscd_prefix in nscd_prefixes:
            nscd_cache = nscd_prefix + "/init.d/nscd"

            if os.path.isfile(nscd_cache):
                dns_cache_found = True

                if subprocess.call(SUDO + [nscd_cache, "restart"]):
                    print_failure(nscd_msg.format(result="failed"))
                else:
                    print_success(nscd_msg.format(result="succeeded"))

        centos_file = "/etc/init.d/network"
        centos_msg = "Flushing the DNS cache by restarting network {result}"

        if os.path.isfile(centos_file):
            if subprocess.call(SUDO + [centos_file, "restart"]):
                print_failure(centos_msg.format(result="failed"))
            else:
                print_success(centos_msg.format(result="succeeded"))

        system_prefixes = ["/usr", ""]
        service_types = ["NetworkManager", "wicd", "dnsmasq", "networking"]
        restarted_services = []

        for system_prefix in system_prefixes:
            systemctl = system_prefix + "/bin/systemctl"
            system_dir = system_prefix + "/lib/systemd/system"

            for service_type in service_types:
                service = service_type + ".service"
                if service in restarted_services:
                    continue

                service_file = path_join_robust(system_dir, service)
                service_msg = (
                    "Flushing the DNS cache by restarting " + service + " {result}"
                )

                if os.path.isfile(service_file):
                    if 0 != subprocess.call(
                        [systemctl, "status", service], stdout=subprocess.DEVNULL
                    ):
                        continue
                    dns_cache_found = True

                    if subprocess.call(SUDO + [systemctl, "restart", service]):
                        print_failure(service_msg.format(result="failed"))
                    else:
                        print_success(service_msg.format(result="succeeded"))
                    restarted_services.append(service)

        dns_clean_file = "/etc/init.d/dns-clean"
        dns_clean_msg = "Flushing the DNS cache via dns-clean executable {result}"

        if os.path.isfile(dns_clean_file):
            dns_cache_found = True

            if subprocess.call(SUDO + [dns_clean_file, "start"]):
                print_failure(dns_clean_msg.format(result="failed"))
            else:
                print_success(dns_clean_msg.format(result="succeeded"))

        if not dns_cache_found:
            print_failure("Unable to determine DNS management tool.")


def remove_old_hosts_file(path_to_file, file_name, backup):
    """
    Remove the old hosts file.

    This is a hotfix because merging with an already existing hosts file leads
    to artifacts and duplicates.

    Parameters
    ----------
    backup : boolean, default False
        Whether or not to backup the existing hosts file.
    """

    full_file_path = path_join_robust(path_to_file, file_name)

    if os.path.exists(full_file_path):
        if backup:
            backup_file_path = full_file_path + "-{}".format(
                time.strftime("%Y-%m-%d-%H-%M-%S")
            )

            # Make a backup copy, marking the date in which the list was updated
            shutil.copy(full_file_path, backup_file_path)

        os.remove(full_file_path)

    # Create directory if not exists
    if not os.path.exists(path_to_file):
        os.makedirs(path_to_file)

    # Create new empty hosts file
    open(full_file_path, "a").close()


# End File Logic


def domain_to_idna(line):
    """
    Encode a domain that is present into a line into `idna`. This way we
    avoid most encoding issues.

    Parameters
    ----------
    line : str
        The line we have to encode/decode.

    Returns
    -------
    line : str
        The line in a converted format.

    Notes
    -----
    - This function encodes only the domain to `idna` format because in
        most cases, the encoding issue is due to a domain which looks like
        `b'\xc9\xa2oogle.com'.decode('idna')`.
    - About the splitting:
        We split because we only want to encode the domain and not the full
        line, which may cause some issues. Keep in mind that we split, but we
        still concatenate once we encoded the domain.

        - The following split the prefix `0.0.0.0` or `127.0.0.1` of a line.
        - The following also split the trailing comment of a given line.
    """

    if not line.startswith("#"):
        tabs = "\t"
        space = " "

        tabs_position, space_position = (line.find(tabs), line.find(space))

        if tabs_position > -1 and space_position > -1:
            if space_position < tabs_position:
                separator = space
            else:
                separator = tabs
        elif not tabs_position == -1:
            separator = tabs
        elif not space_position == -1:
            separator = space
        else:
            separator = ""

        if separator:
            splited_line = line.split(separator)

            try:
                index = 1
                while index < len(splited_line):
                    if splited_line[index]:
                        break
                    index += 1

                if "#" in splited_line[index]:
                    index_comment = splited_line[index].find("#")

                    if index_comment > -1:
                        comment = splited_line[index][index_comment:]

                        splited_line[index] = (
                            splited_line[index]
                            .split(comment)[0]
                            .encode("IDNA")
                            .decode("UTF-8")
                            + comment
                        )

                splited_line[index] = splited_line[index].encode("IDNA").decode("UTF-8")
            except IndexError:
                pass
            return separator.join(splited_line)
        return line.encode("IDNA").decode("UTF-8")
    return line.encode("UTF-8").decode("UTF-8")


# Helper Functions
def maybe_copy_example_file(file_path):
    """
    Given a file path, copy over its ".example" if the path doesn't exist.

    If the path does exist, nothing happens in this function.

    If the path doesn't exist, and the ".example" file doesn't exist, nothing happens in this function.

    Parameters
    ----------
    file_path : str
        The full file path to check.
    """

    if not os.path.isfile(file_path):
        example_file_path = file_path + ".example"
        if os.path.isfile(example_file_path):
            shutil.copyfile(example_file_path, file_path)


def get_file_by_url(url, params=None, **kwargs):
    """
    Retrieve the contents of the hosts file at the URL, then pass it through domain_to_idna().

    Parameters are passed to the requests.get() function.

    Parameters
    ----------
    url : str or bytes
        URL for the new Request object.
    params :
        Dictionary, list of tuples or bytes to send in the query string for the Request.
    kwargs :
        Optional arguments that request takes.

    Returns
    -------
    url_data : str or None
        The data retrieved at that URL from the file. Returns None if the
        attempted retrieval is unsuccessful.
    """

    try:
        req = requests.get(url=url, params=params, **kwargs)
    except requests.exceptions.RequestException:
        print("Error retrieving data from {}".format(url))
        return None

    req.encoding = req.apparent_encoding
    res_text = "\n".join([domain_to_idna(line) for line in req.text.split("\n")])
    return res_text


def write_data(f, data):
    """
    Write data to a file object.

    Parameters
    ----------
    f : file
        The file object at which to write the data.
    data : str
        The data to write to the file.
    """

    f.write(bytes(data, "UTF-8"))


def list_dir_no_hidden(path):
    """
    List all files in a directory, except for hidden files.

    Parameters
    ----------
    path : str
        The path of the directory whose files we wish to list.
    """

    return glob(os.path.join(path, "*"))


def query_yes_no(question, default="yes"):
    """
    Ask a yes/no question via input() and get answer from the user.

    Inspired by the following implementation:

    https://code.activestate.com/recipes/577058/

    Parameters
    ----------
    question : str
        The question presented to the user.
    default : str, default "yes"
        The presumed answer if the user just hits <Enter>. It must be "yes",
        "no", or None (means an answer is required of the user).

    Returns
    -------
    yes : Whether or not the user replied yes to the question.
    """

    valid = {"yes": "yes", "y": "yes", "ye": "yes", "no": "no", "n": "no"}
    prompt = {None: " [y/n] ", "yes": " [Y/n] ", "no": " [y/N] "}.get(default, None)

    if not prompt:
        raise ValueError("invalid default answer: '%s'" % default)

    reply = None

    while not reply:
        sys.stdout.write(colorize(question, Colors.PROMPT) + prompt)

        choice = input().lower()
        reply = None

        if default and not choice:
            reply = default
        elif choice in valid:
            reply = valid[choice]
        else:
            print_failure("Please respond with 'yes' or 'no' (or 'y' or 'n').\n")

    return reply == "yes"


def is_valid_user_provided_domain_format(domain):
    """
    Check whether a provided domain is valid.

    Parameters
    ----------
    domain : str
        The domain against which to check.

    Returns
    -------
    valid_domain : bool
        Whether or not the domain provided is valid.
    """

    if domain == "":
        print("You didn't enter a domain. Try again.")
        return False

    domain_regex = re.compile(r"www\d{0,3}[.]|https?")

    if domain_regex.match(domain):
        print(
            "The domain " + domain + " is not valid. Do not include "
            "www.domain.com or http(s)://domain.com. Try again."
        )
        return False
    else:
        return True


def recursive_glob(stem, file_pattern):
    """
    Recursively match files in a directory according to a pattern.

    Parameters
    ----------
    stem : str
        The directory in which to recurse
    file_pattern : str
        The filename regex pattern to which to match.

    Returns
    -------
    matches_list : list
        A list of filenames in the directory that match the file pattern.
    """

    if sys.version_info >= (3, 5):
        return glob(stem + "/**/" + file_pattern, recursive=True)
    else:
        # gh-316: this will avoid invalid unicode comparisons in Python 2.x
        if stem == str("*"):
            stem = "."
        matches = []
        for root, dirnames, filenames in os.walk(stem):
            for filename in fnmatch.filter(filenames, file_pattern):
                matches.append(path_join_robust(root, filename))
    return matches


def path_join_robust(path, *paths):
    """
    Wrapper around `os.path.join` with handling for locale issues.

    Parameters
    ----------
    path : str
        The first path to join.
    paths : varargs
        Subsequent path strings to join.

    Returns
    -------
    joined_path : str
        The joined path string of the two path inputs.

    Raises
    ------
    locale.Error : A locale issue was detected that prevents path joining.
    """

    try:
        # gh-316: joining unicode and str can be saddening in Python 2.x
        path = str(path)
        paths = [str(another_path) for another_path in paths]

        return os.path.join(path, *paths)
    except UnicodeDecodeError as e:
        raise locale.Error(
            "Unable to construct path. This is likely a LOCALE issue:\n\n" + str(e)
        )


# Colors
class Colors(object):
    PROMPT = "\033[94m"
    SUCCESS = "\033[92m"
    FAIL = "\033[91m"
    ENDC = "\033[0m"


def supports_color():
    """
    Check whether the running terminal or command prompt supports color.

    Inspired by StackOverflow link (and Django implementation) here:

    https://stackoverflow.com/questions/7445658

    Returns
    -------
    colors_supported : bool
        Whether the running terminal or command prompt supports color.
    """

    sys_platform = sys.platform
    supported = sys_platform != "Pocket PC" and (
        sys_platform != "win32" or "ANSICON" in os.environ
    )

    atty_connected = hasattr(sys.stdout, "isatty") and sys.stdout.isatty()
    return supported and atty_connected


def colorize(text, color):
    """
    Wrap a string so that it displays in a particular color.

    This function adds a prefix and suffix to a text string so that it is
    displayed as a particular color, either in command prompt or the terminal.

    If the running terminal or command prompt does not support color, the
    original text is returned without being wrapped.

    Parameters
    ----------
    text : str
        The message to display.
    color : str
        The color string prefix to put before the text.

    Returns
    -------
    wrapped_str : str
        The wrapped string to display in color, if possible.
    """

    if not supports_color():
        return text

    return color + text + Colors.ENDC


def print_success(text):
    """
    Print a success message.

    Parameters
    ----------
    text : str
        The message to display.
    """

    print(colorize(text, Colors.SUCCESS))


def print_failure(text):
    """
    Print a failure message.

    Parameters
    ----------
    text : str
        The message to display.
    """

    print(colorize(text, Colors.FAIL))


# End Helper Functions


if __name__ == "__main__":
    main()



File: makeHosts.py

#!/usr/bin/env python

# Script by gfyoung
# https://github.com/gfyoung
#
# This Python script will generate hosts files and update the readme file.

from __future__ import print_function

import argparse
import subprocess
import sys


def print_failure(msg):
    """
    Print a failure message.

    Parameters
    ----------
    msg : str
        The failure message to print.
    """

    print("\033[91m" + msg + "\033[0m")


def update_hosts_file(*flags):
    """
    Wrapper around running updateHostsFile.py

    Parameters
    ----------
    flags : varargs
        Commandline flags to pass into updateHostsFile.py. For more info, run
        the following command in the terminal or command prompt:

        ```
        python updateHostsFile.py -h
        ```
    """

    if subprocess.call([sys.executable, "updateHostsFile.py"] + list(flags)):
        print_failure("Failed to update hosts file")


def update_readme_file():
    """
    Wrapper around running updateReadme.py
    """

    if subprocess.call([sys.executable, "updateReadme.py"]):
        print_failure("Failed to update readme file")


def recursively_loop_extensions(extension, extensions, current_extensions):
    """
    Helper function that recursively calls itself to prevent manually creating
    all possible combinations of extensions.

    Will call update_hosts_file for all combinations of extensions
    """

    c_extensions = extensions.copy()
    c_current_extensions = current_extensions.copy()
    c_current_extensions.append(extension)

    name = "-".join(c_current_extensions)

    params = ("-a", "-n", "-o", "alternates/"+name, "-e") + tuple(c_current_extensions)
    update_hosts_file(*params)

    params = ("-a", "-n", "-s", "--nounifiedhosts", "-o", "alternates/"+name+"-only", "-e") + tuple(c_current_extensions)
    update_hosts_file(*params)

    while len(c_extensions) > 0:
        recursively_loop_extensions(c_extensions.pop(0), c_extensions, c_current_extensions)


def main():
    parser = argparse.ArgumentParser(
        description="Creates custom hosts "
        "file from hosts stored in "
        "data subfolders."
    )
    parser.parse_args()

    # Update the unified hosts file
    update_hosts_file("-a")

    # List of extensions we want to generate, we will loop over them recursively to prevent manual definitions
    # Only add new extensions to the end of the array, to avoid relocating existing hosts-files
    extensions = ["fakenews", "gambling", "porn", "social"]

    while len(extensions) > 0:
        recursively_loop_extensions(extensions.pop(0), extensions, [])

    # Update the readme files.
    update_readme_file()


if __name__ == "__main__":
    main()



**Take Note!**

With the exception of issues and PRs regarding changes to
`hosts/data/StevenBlack/hosts`, all other issues regarding the content of the
produced hosts files should be made with the appropriate data source that
contributed the content in question. The contact information for all of the data
sources can be found in the `hosts/data/` directory.

---

![Logo](https://raw.githubusercontent.com/StevenBlack/hosts/master/.github/logo.png)

[![latest release](https://img.shields.io/github/release/StevenBlack/hosts)](https://github.com/StevenBlack/hosts/releases)
[![license](https://img.shields.io/github/license/StevenBlack/hosts)](https://github.com/StevenBlack/hosts/blob/master/license.txt)
[![repo size](https://img.shields.io/github/repo-size/StevenBlack/hosts)](https://github.com/StevenBlack/hosts)
[![contributors](https://img.shields.io/github/contributors/StevenBlack/hosts)](https://github.com/StevenBlack/hosts/graphs/contributors)
[![Build Status](https://img.shields.io/github/actions/workflow/status/StevenBlack/hosts/ci.yml?branch=master)](https://github.com/StevenBlack/hosts/actions/workflows/ci.yml?query=branch%3Amaster)
[![Code style: black](https://img.shields.io/badge/code%20style-black-000000)](https://github.com/python/black)
[![commits since last release](https://img.shields.io/github/commits-since/StevenBlack/hosts/latest)](https://github.com/StevenBlack/hosts/commits/master)
[![last commit](https://img.shields.io/github/last-commit/StevenBlack/hosts)](https://github.com/StevenBlack/hosts/commits/master)
[![commit activity](https://img.shields.io/github/commit-activity/y/StevenBlack/hosts)](https://github.com/StevenBlack/hosts/commits/master)

# Unified hosts file with base extensions

This repository consolidates several reputable `hosts` files, and merges them
into a unified hosts file with duplicates removed. A variety of tailored hosts
files are provided.

**Therefore this repository is a hosts file aggregator.**

![Aggregator](https://raw.githubusercontent.com/StevenBlack/hosts/master/aggregator.png)

- Last updated: **September 10 2024**.
- Here's the
  [raw hosts file Unified hosts file with base extensions](https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts)
  containing 165,709 entries.

![Size history](https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts_file_size_history.png)

## List of all hosts file variants

This repository offers
[31 different host file variants](https://github.com/StevenBlack/hosts/tree/master/alternates),
in addition to the base variant, with and without the unified hosts included.

The **Non GitHub mirror** is the link to use for some hosts file managers like
[Hostsman for Windows](https://www.abelhadigital.com/hostsman/) that don't work
with GitHub download links.

| Host file recipe | Readme | Raw hosts | Unique domains | Non GitHub mirror |
| ---------------- | :----: | :-------: | :------------: | :---------------: |
Unified hosts = **(adware + malware)** | [Readme](https://github.com/StevenBlack/hosts/blob/master/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts) | 165,709 | [link](http://sbc.io/hosts/hosts)
Unified hosts **+ fakenews** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/fakenews/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews/hosts) | 167,903 | [link](http://sbc.io/hosts/alternates/fakenews/hosts)
**fakenews** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/fakenews-only/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews-only/hosts) | 2,194 | [link](http://sbc.io/hosts/alternates/fakenews-only/hosts)
Unified hosts **+ gambling** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/gambling/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/gambling/hosts) | 174,133 | [link](http://sbc.io/hosts/alternates/gambling/hosts)
**gambling** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/gambling-only/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/gambling-only/hosts) | 8,436 | [link](http://sbc.io/hosts/alternates/gambling-only/hosts)
Unified hosts **+ porn** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/porn/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/porn/hosts) | 252,063 | [link](http://sbc.io/hosts/alternates/porn/hosts)
**porn** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/porn-only/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/porn-only/hosts) | 87,049 | [link](http://sbc.io/hosts/alternates/porn-only/hosts)
Unified hosts **+ social** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/social/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/social/hosts) | 168,901 | [link](http://sbc.io/hosts/alternates/social/hosts)
**social** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/social-only/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/social-only/hosts) | 3,221 | [link](http://sbc.io/hosts/alternates/social-only/hosts)
Unified hosts **+ fakenews + gambling** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/fakenews-gambling/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews-gambling/hosts) | 176,327 | [link](http://sbc.io/hosts/alternates/fakenews-gambling/hosts)
**fakenews + gambling** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/fakenews-gambling-only/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews-gambling-only/hosts) | 10,630 | [link](http://sbc.io/hosts/alternates/fakenews-gambling-only/hosts)
Unified hosts **+ fakenews + porn** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/fakenews-porn/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews-porn/hosts) | 254,257 | [link](http://sbc.io/hosts/alternates/fakenews-porn/hosts)
**fakenews + porn** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/fakenews-porn-only/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews-porn-only/hosts) | 89,243 | [link](http://sbc.io/hosts/alternates/fakenews-porn-only/hosts)
Unified hosts **+ fakenews + social** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/fakenews-social/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews-social/hosts) | 171,095 | [link](http://sbc.io/hosts/alternates/fakenews-social/hosts)
**fakenews + social** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/fakenews-social-only/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews-social-only/hosts) | 5,415 | [link](http://sbc.io/hosts/alternates/fakenews-social-only/hosts)
Unified hosts **+ gambling + porn** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/gambling-porn/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/gambling-porn/hosts) | 260,487 | [link](http://sbc.io/hosts/alternates/gambling-porn/hosts)
**gambling + porn** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/gambling-porn-only/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/gambling-porn-only/hosts) | 95,485 | [link](http://sbc.io/hosts/alternates/gambling-porn-only/hosts)
Unified hosts **+ gambling + social** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/gambling-social/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/gambling-social/hosts) | 177,325 | [link](http://sbc.io/hosts/alternates/gambling-social/hosts)
**gambling + social** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/gambling-social-only/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/gambling-social-only/hosts) | 11,657 | [link](http://sbc.io/hosts/alternates/gambling-social-only/hosts)
Unified hosts **+ porn + social** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/porn-social/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/porn-social/hosts) | 255,254 | [link](http://sbc.io/hosts/alternates/porn-social/hosts)
**porn + social** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/porn-social-only/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/porn-social-only/hosts) | 90,269 | [link](http://sbc.io/hosts/alternates/porn-social-only/hosts)
Unified hosts **+ fakenews + gambling + porn** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/fakenews-gambling-porn/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews-gambling-porn/hosts) | 262,681 | [link](http://sbc.io/hosts/alternates/fakenews-gambling-porn/hosts)
**fakenews + gambling + porn** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/fakenews-gambling-porn-only/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews-gambling-porn-only/hosts) | 97,679 | [link](http://sbc.io/hosts/alternates/fakenews-gambling-porn-only/hosts)
Unified hosts **+ fakenews + gambling + social** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/fakenews-gambling-social/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews-gambling-social/hosts) | 179,519 | [link](http://sbc.io/hosts/alternates/fakenews-gambling-social/hosts)
**fakenews + gambling + social** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/fakenews-gambling-social-only/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews-gambling-social-only/hosts) | 13,851 | [link](http://sbc.io/hosts/alternates/fakenews-gambling-social-only/hosts)
Unified hosts **+ fakenews + porn + social** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/fakenews-porn-social/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews-porn-social/hosts) | 257,448 | [link](http://sbc.io/hosts/alternates/fakenews-porn-social/hosts)
**fakenews + porn + social** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/fakenews-porn-social-only/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews-porn-social-only/hosts) | 92,463 | [link](http://sbc.io/hosts/alternates/fakenews-porn-social-only/hosts)
Unified hosts **+ gambling + porn + social** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/gambling-porn-social/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/gambling-porn-social/hosts) | 263,678 | [link](http://sbc.io/hosts/alternates/gambling-porn-social/hosts)
**gambling + porn + social** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/gambling-porn-social-only/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/gambling-porn-social-only/hosts) | 98,705 | [link](http://sbc.io/hosts/alternates/gambling-porn-social-only/hosts)
Unified hosts **+ fakenews + gambling + porn + social** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/fakenews-gambling-porn-social/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews-gambling-porn-social/hosts) | 265,872 | [link](http://sbc.io/hosts/alternates/fakenews-gambling-porn-social/hosts)
**fakenews + gambling + porn + social** | [Readme](https://github.com/StevenBlack/hosts/blob/master/alternates/fakenews-gambling-porn-social-only/readme.md) | [link](https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews-gambling-porn-social-only/hosts) | 100,899 | [link](http://sbc.io/hosts/alternates/fakenews-gambling-porn-social-only/hosts)


**Expectation**: These unified hosts files should serve all devices, regardless
of OS.

## Sources of hosts data unified in this variant

Updated `hosts` files from the following locations are always unified and
included:

| Host file source | Home page | Raw hosts | License | Issues | Description |
| ---------------- | :-------: | :-------: | :-----: | :----: | ----------- |
Steven Black's ad-hoc list |[link](https://github.com/StevenBlack/hosts/blob/master/data/StevenBlack/hosts) | [raw](https://raw.githubusercontent.com/StevenBlack/hosts/master/data/StevenBlack/hosts) | MIT | [issues](https://github.com/StevenBlack/hosts/issues)| Additional sketch domains as I come across them.
AdAway |[link](https://adaway.org/) | [raw](https://raw.githubusercontent.com/AdAway/adaway.github.io/master/hosts.txt) | CC BY 3.0 | [issues](https://github.com/AdAway/adaway.github.io/issues)| AdAway is an open source ad blocker for Android using the hosts file.
add.2o7Net |[link](https://github.com/FadeMind/hosts.extras) | [raw](https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.2o7Net/hosts) | MIT | [issues](https://github.com/FadeMind/hosts.extras/issues)| 2o7Net tracking sites based on [hostsfile.org](https://www.hostsfile.org/hosts.html) content.
add.Dead |[link](https://github.com/FadeMind/hosts.extras) | [raw](https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Dead/hosts) | MIT | [issues](https://github.com/FadeMind/hosts.extras/issues)| Dead sites based on [hostsfile.org](https://www.hostsfile.org/hosts.html) content.
add.Risk |[link](https://github.com/FadeMind/hosts.extras) | [raw](https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Risk/hosts) | MIT | [issues](https://github.com/FadeMind/hosts.extras/issues)| Risk content sites based on [hostsfile.org](https://www.hostsfile.org/hosts.html) content.
add.Spam |[link](https://github.com/FadeMind/hosts.extras) | [raw](https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Spam/hosts) | MIT | [issues](https://github.com/FadeMind/hosts.extras/issues)| Spam sites based on [hostsfile.org](https://www.hostsfile.org/hosts.html) content.
Mitchell Krog's - Badd Boyz Hosts |[link](https://github.com/mitchellkrogza/Badd-Boyz-Hosts) | [raw](https://raw.githubusercontent.com/mitchellkrogza/Badd-Boyz-Hosts/master/hosts) | MIT | [issues](https://github.com/mitchellkrogza/Badd-Boyz-Hosts/issues)| Sketchy domains and Bad Referrers from my Nginx and Apache Bad Bot and Spam Referrer Blockers
hostsVN |[link](https://github.com/bigdargon/hostsVN) | [raw](https://raw.githubusercontent.com/bigdargon/hostsVN/master/option/hosts-VN) | MIT | [issues](https://github.com/bigdargon/hostsVN/issues)| Hosts block ads of Vietnamese
KADhosts |[link](https://kadantiscam.netlify.app/) | [raw](https://raw.githubusercontent.com/FiltersHeroes/KADhosts/master/KADhosts.txt) | CC BY-SA 4.0 | [issues](https://github.com//FiltersHeroes/KADhosts/issues)| Fraud/adware/scam websites.
MetaMask eth-phishing-detect |[link](https://github.com/MetaMask/eth-phishing-detect) | [raw](https://raw.githubusercontent.com/MetaMask/eth-phishing-detect/master/src/hosts.txt) | DON'T BE A DICK PUBLIC LICENSE | [issues](https://github.com/MetaMask/eth-phishing-detect/issues)| Phishing domains targeting Ethereum users.
minecraft-hosts |[link](https://github.com/jamiemansfield/minecraft-hosts) | [raw](https://raw.githubusercontent.com/jamiemansfield/minecraft-hosts/master/lists/tracking.txt) | CC0-1.0 | [issues](https://github.com/jamiemansfield/minecraft-hosts/issues)| Minecraft related tracker hosts
MVPS hosts file |[link](https://winhelp2002.mvps.org/) | [raw](https://winhelp2002.mvps.org/hosts.txt) | CC BY-NC-SA 4.0 | [issues](mailto:winhelp2002@gmail.com)| The purpose of this site is to provide the user with a high quality custom HOSTS file.
Dan Pollock – [someonewhocares](https://someonewhocares.org/) |[link](https://someonewhocares.org/hosts/) | [raw](https://someonewhocares.org/hosts/zero/hosts) | non-commercial with attribution | [issues](mailto:hosts@someonewhocares.org)| How to make the internet not suck (as much).
Tiuxo hostlist - ads |[link](https://github.com/tiuxo/hosts) | [raw](https://raw.githubusercontent.com/tiuxo/hosts/master/ads) | CC BY 4.0 | [issues](https://github.com/tiuxo/hosts/issues)| Categorized hosts files for DNS based content blocking
UncheckyAds |[link](https://github.com/FadeMind/hosts.extras) | [raw](https://raw.githubusercontent.com/FadeMind/hosts.extras/master/UncheckyAds/hosts) | MIT | [issues](https://github.com/FadeMind/hosts.extras/issues)| Windows installers ads sources sites based on https://unchecky.com/ content.
URLHaus |[link](https://urlhaus.abuse.ch/) | [raw](https://urlhaus.abuse.ch/downloads/hostfile/) | CC0 | [issues](mailto:contactme@abuse.ch)| A project from [abuse.ch](https://abuse.ch/) with the goal of sharing malicious URLs.
yoyo.org |[link](https://pgl.yoyo.org/adservers/) | [raw](https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&mimetype=plaintext&useip=0.0.0.0) |  | [issues](mailto:pgl@yoyo.org)| Blocking with ad server and tracking server hostnames.


## Extensions

The unified hosts file is optionally extensible. Extensions are used to include
domains by category. Currently, we offer the following categories: `fakenews`,
`social`, `gambling`, and `porn`.

Extensions are optional, and can be combined in various ways with the base hosts
file. The combined products are stored in the
[`alternates`](https://github.com/StevenBlack/hosts/tree/master/alternates)
folder.

Data for extensions are stored in the
[`extensions`](https://github.com/StevenBlack/hosts/tree/master/extensions)
folder. You manage extensions by curating this folder tree, where you will find
the data for `fakenews`, `social`, `gambling`, and `porn` extension data that we
maintain and provide for you.

## Generate your own unified hosts file

You have three options to generate your own hosts file. You can use our
container image, build your own image, or do it in your own environment. Option
#1 is easiest if you have Linux with Docker installed.

### Option 1: Use our container image (Linux only)

> This will replace your `/etc/hosts`.

We assume you have Docker available on your host. Just run the following
command. Set extensions to your preference.

```sh
docker run --pull always --rm -it -v /etc/hosts:/etc/hosts \
ghcr.io/stevenblack/hosts:latest updateHostsFile.py --auto \
--replace --extensions gambling porn
```

If you want to add custom hosts or a whitelist, create either or both files as
per [the instructions](#how-do-i-control-which-sources-are-unified) and add the
following arguments _before_ `ghcr.io/stevenblack/hosts:latest` depending on
which you wish to use.

```sh
-v "path/to/myhosts:/hosts/myhosts" \
-v "path/to/whitelist:/hosts/whitelist" \
```

You can rerun this exact command later to update based on the latest available
hosts (for example, add it to a weekly cron job).

### Option 2: Generate your own container image

We provide the
[Dockerfile](https://github.com/StevenBlack/hosts/blob/master/Dockerfile) used
by the previous step, which you can use to create a container image with
everything you need. The container will contain Python 3 and all its dependency
requirements, and a copy of the latest version of this repository.

Build the Docker container from the root of this repo like this:

```sh
docker build --no-cache . -t stevenblack-hosts
```

Then run your command as such:

```sh
docker run --rm -it stevenblack-hosts updateHostsFile.py
```

> This will create the hosts file, and remove it with the container when done,
> so not very useful. You can use the example in option #1 to add volumes so
> files on your host are replaced.

### Option 3: Generate it in your own environment

To generate your own amalgamated hosts files you will need Python 3.6 or later.

First, install the dependencies with:

```sh
pip3 install --user -r requirements.txt
```

**Note** we recommend the `--user` flag which installs the required dependencies
at the user level. More information about it can be found on pip
[documentation](https://pip.pypa.io/en/stable/reference/pip_install/?highlight=--user#cmdoption-user).

### Option 4: Generate it in Google Colab

Spin up a free remote [Google Colab](https://colab.research.google.com/drive/1tYWXpU2iuPDqN_o03JW9ml3ExO80eBLq?usp=sharing) environment.

### Common steps regardless of your development environment

To **run unit tests**, in the top-level directory, run:

```sh
python3 testUpdateHostsFile.py
```

The `updateHostsFile.py` script will generate a unified hosts file based on the
sources in the local `data/` subfolder. The script will prompt you whether it
should fetch updated versions (from locations defined by the `update.json` text
file in each source's folder). Otherwise, it will use the `hosts` file that's
already there.

```sh
python3 updateHostsFile.py [--auto] [--replace] [--ip nnn.nnn.nnn.nnn] [--extensions ext1 ext2 ext3]
```

#### Command line options

`--help`, or `-h`: display help.

`--auto`, or `-a`: run the script without prompting. When `--auto` is invoked,

- Hosts data sources, including extensions, are updated.
- No extensions are included by default. Use the `--extensions` or `-e` flag to
  include any you want.
- Your active hosts file is _not_ replaced unless you include the `--replace`
  flag.

`--backup`, or `-b`: Make a backup of existing hosts file(s) as you generate
over them.

`--extensions <ext1> <ext2> <ext3>`, or `-e <ext1> <ext2> <ext3>`: the names of
subfolders below the `extensions` folder containing additional category-specific
hosts files to include in the amalgamation. Example: `--extensions porn` or
`-e social porn`.

`--flush-dns-cache`, or `-f`: skip the prompt for flushing the DNS cache. Only
active when `--replace` is also active.

`--ip nnn.nnn.nnn.nnn`, or `-i nnn.nnn.nnn.nnn`: the IP address to use as the
target. Default is `0.0.0.0`.

`--keepdomaincomments`, or `-k`: `true` (default) or `false`, keep the comments
that appear on the same line as domains. The default is `true`.

`--noupdate`, or `-n`: skip fetching updates from hosts data sources.

`--output <subfolder>`, or `-o <subfolder>`: place the generated source file in
a subfolder. If the subfolder does not exist, it will be created.

`--replace`, or `-r`: trigger replacing your active hosts

`--skipstatichosts`, or `-s`: `false` (default) or `true`, omit the standard
section at the top, containing lines like `127.0.0.1 localhost`. This is useful
for configuring proximate DNS services on the local network.

`--nogendata`, or `-g`: `false` (default) or `true`, skip the generation of the
readmeData.json file used for generating readme.md files. This is useful if you
are generating host files with additional whitelists or blacklists and want to
keep your local checkout of this repo unmodified.

`--nounifiedhosts`: `false` (default) or `true`, do not include the unified hosts
file in the final hosts file. Usually used together with `--extensions`.

`--compress`, or `-c`: `false` (default) or `true`, _Compress_ the hosts file
ignoring non-necessary lines (empty lines and comments) and putting multiple
domains in each line. Reducing the number of lines of the hosts file improves
the performances under Windows (with DNS Client service enabled).

`--minimise`, or `-m`: `false` (default) or `true`, like `--compress`, but puts
each domain on a separate line. This is necessary because many implementations
of URL blockers that rely on `hosts` files do not conform to the standard which
allows multiple hosts on a single line.

`--blacklist <blacklistfile>`, or `-x <blacklistfile>`: Append the given
blacklist file in hosts format to the generated hosts file.

`--whitelist <whitelistfile>`, or `-w <whitelistfile>`: Use the given whitelist
file to remove hosts from the generated hosts file.

## How do I control which sources are unified?

Add one or more _additional_ sources, each in a subfolder of the `data/` folder,
and specify the `url` key in its `update.json` file.

Add one or more _optional_ extensions, which originate from subfolders of the
`extensions/` folder. Again the url in `update.json` controls where this
extension finds its updates.

Create an _optional_ `blacklist` file. The contents of this file (containing a
listing of additional domains in `hosts` file format) are appended to the
unified hosts file during the update process. A sample `blacklist` is included,
and may be modified as you need.

- NOTE: The `blacklist` is not tracked by git, so any changes you make won't be
  overridden when you `git pull` this repo from `origin` in the future.

### How do I include my own custom domain mappings?

If you have custom hosts records, place them in file `myhosts`. The contents of
this file are prepended to the unified hosts file during the update process.

The `myhosts` file is not tracked by git, so any changes you make won't be
overridden when you `git pull` this repo from `origin` in the future.

### How do I prevent domains from being included?

The domains you list in the `whitelist` file are excluded from the final hosts
file.

The `whitelist` uses partial matching. Therefore if you whitelist
`google-analytics.com`, that domain and all its subdomains won't be merged into
the final hosts file.

The `whitelist` is not tracked by git, so any changes you make won't be
overridden when you `git pull` this repo from `origin` in the future.

## How can I contribute hosts records?

If you discover sketchy domains you feel should be included here, here are some
ways to contribute them.

### Option 1: contact one of our hosts sources

The best way to get new domains included is to submit an issue to any of the
data providers whose home pages are
[listed here](https://github.com/StevenBlack/hosts#sources-of-hosts-data-unified-in-this-variant).
This is best because once you submit new domains, they will be curated and
updated by the dedicated folks who maintain these sources.

### Option 2: Fork this repository, add your domains to Steven Black's personal data file, and submit a pull request

Fork this hosts this repo and add your links to
[https://github.com/StevenBlack/hosts/blob/master/data/StevenBlack/hosts](https://github.com/StevenBlack/hosts/blob/master/data/StevenBlack/hosts).

Then, submit a pull request.

**WARNING**: this is less desirable than Option 1 because the ongoing curation
falls on us. So this creates more work for us.

### Option 3: create your own hosts list as a repo on GitHub

If you're able to curate your own collection of sketchy domains, then curate
your own hosts list. Then signal the existence of your repo as
[a new issue](https://github.com/StevenBlack/hosts/issues) and we may include
your new repo into the collection of sources we pull whenever we create new
versions.

## What is a hosts file?

A hosts file, named `hosts` (with no file extension), is a plain-text file used
by all operating systems to map hostnames to IP addresses.

In most operating systems, the `hosts` file is preferential to `DNS`. Therefore
if a domain name is resolved by the `hosts` file, the request never leaves your
computer.

Having a smart `hosts` file goes a long way towards blocking malware, adware,
and other irritants.

For example, to nullify requests to some doubleclick.net servers, adding these
lines to your hosts file will do it:

```text
# block doubleClick's servers
0.0.0.0 ad.ae.doubleclick.net
0.0.0.0 ad.ar.doubleclick.net
0.0.0.0 ad.at.doubleclick.net
0.0.0.0 ad.au.doubleclick.net
0.0.0.0 ad.be.doubleclick.net
# etc...
```

## We recommend using `0.0.0.0` instead of `127.0.0.1`

Traditionally most host files use `127.0.0.1`, the _loopback address_, to
establish an IP connection to the local machine.

We prefer to use `0.0.0.0`, which is defined as a non-routable meta-address used
to designate an invalid, unknown, or non-applicable target.

Using `0.0.0.0` is empirically faster, possibly because there's no wait for a
timeout resolution. It also does not interfere with a web server that may be
running on the local PC.

## Why not use `0` instead of `0.0.0.0`?

We tried that. Using `0` doesn't work universally.

## Location of your hosts file

To modify your current `hosts` file, look for it in the following places and
modify it with a text editor.

- **macOS (until 10.14.x macOS Mojave), iOS, Android, Linux**: `/etc/hosts`
  file.
- **macOS Catalina:** `/private/etc/hosts` file.
- **Windows**: `%SystemRoot%\system32\drivers\etc\hosts` file.

## Gentoo

Gentoo users may find
[`sb-hosts`](https://github.com/PF4Public/gentoo-overlay/tree/master/net-misc/sb-hosts)
in [::pf4public](https://github.com/PF4Public/gentoo-overlay) Gentoo overlay

## NixOS

To install hosts file on your machine add the following into your
`configuration.nix`:

```nix
{
  networking.extraHosts = let
    hostsPath = https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts;
    hostsFile = builtins.fetchurl hostsPath;
  in builtins.readFile "${hostsFile}";
}
```

- NOTE: Change `hostsPath` if you need other versions of hosts file.
- NOTE: The call to `fetchurl` is impure. Use `fetchFromGitHub` with the exact
  commit if you want to always get the same result.

### Nix Flake

NixOS installations which are managed through _flakes_ can use the hosts file
like this:

```nix
{
  inputs.hosts.url = "github:StevenBlack/hosts";
  outputs = { self, nixpkgs, hosts }: {
    nixosConfigurations.my-hostname = {
      system = "<architecture>";
      modules = [
        hosts.nixosModule {
          networking.stevenBlackHosts.enable = true;
        }
      ];
    };
  };
}
```

The hosts extensions are also available with the following options:

```nix
{
  networking.stevenBlackHosts = {
    blockFakenews = true;
    blockGambling = true;
    blockPorn = true;
    blockSocial = true;
  };
}
```

## Updating hosts file on Windows

(NOTE: See also some third-party Hosts managers, listed below.)

On Linux and macOS, run the Python script. On Windows more work is required due
to compatibility issues so it's preferable to run the batch file as follows:

```sh
updateHostsWindows.bat
```

This file **MUST** be run in command prompt with administrator privileges in the
repository directory. In addition to updating the hosts file, it can also
replace the existing hosts file, and reload the DNS cache. It goes without
saying that for this to work, you must be connected to the internet.

To open a command prompt as administrator in the repository's directory, do the
following:

- **Windows XP**: Start → Run → `cmd`
- **Windows Vista, 7**: Start Button → type `cmd` → right-click Command Prompt →
  "Run as Administrator"
- **Windows 8**: Start → Swipe Up → All Apps → Windows System → right-click
  Command Prompt → "Run as Administrator"
- **Windows 10**: Start Button → type `cmd` → right-click Command Prompt → "Run
  as Administrator"

You can also refer to the "Third-Party Hosts Managers" section for further
recommended solutions from third parties.

### Warning: Using this `hosts` file in Windows may require disabling DNS Cache service

Windows has issues with larger hosts files. Recent changes in security within
Windows 10 denies access to changing services via other tools except registry
hacks. Use the `disable-dnscache-service-win.cmd` file to make proper changes to
the Windows registry. You will need to reboot your device once that's done. See
the
[the comments within the `cmd` file](https://github.com/StevenBlack/hosts/blob/master/disable-dnscache-service-win.bat)
for more details.

Disabling the DNS Cache Service can cause issues with services and applications like *WSL* and it's possible to compress the hosts file and negate the need to disable the DNS caching service. You can try the *C++* Windows command line tool at [Hosts Compress - Windows](https://github.com/Lateralus138/hosts-compress-windows) (the recommended method) or the *PowerShell* compression script and check out the guide located at the [Hosts Compression Scripts](https://github.com/Lateralus138/hosts-compression-scripts) repository.

## Reloading hosts file

Your operating system will cache DNS lookups. You can either reboot or run the
following commands to manually flush your DNS cache once the new hosts file is
in place.

The Google Chrome browser may require manually cleaning up its DNS Cache on
`chrome://net-internals/#dns` page to thereafter see the changes in your hosts
file. See: <https://superuser.com/questions/723703>

### Windows

Open a command prompt with administrator privileges and run this command:

```bat
ipconfig /flushdns
```

### Linux

Open a Terminal and run with root privileges:

- **Debian/Ubuntu** `sudo service network-manager restart`
- **Linux Mint** `sudo /etc/init.d/dns-clean start`
- **Linux with systemd**: `sudo systemctl restart network.service`
- **Fedora Linux**: `sudo systemctl restart NetworkManager.service`
- **Arch Linux/Manjaro with Network Manager**:
  `sudo systemctl restart NetworkManager.service`
- **Arch Linux/Manjaro with Wicd**: `sudo systemctl restart wicd.service`
- **RHEL/Centos**: `sudo /etc/init.d/network restart`
- **FreeBSD**: `sudo service nscd restart`

  To enable the `nscd` daemon initially, it is recommended that you run the
  following commands:

  ```sh
  sudo sysrc nscd_enable="YES"
  sudo service nscd start
  ```

  Then modify the `hosts` line in your `/etc/nsswitch.conf` file to the
  following:

  ```text
  hosts: cache files dns
  ```

- **NixOS**: The `nscd.service` is automatically restarted when the option
  `networking.extraHosts` was changed.
- **Others**: Consult
  [this Wikipedia article](https://en.wikipedia.org/wiki/Hosts_%28file%29#Location_in_the_file_system).

### macOS

As described in [this article](https://osxdaily.com/2022/11/21/how-clear-dns-cache-macos-ventura-monterey/),
open a Terminal and run:

```sh
sudo dscacheutil -flushcache;sudo killall -HUP mDNSResponder
```

## Release management

This repository uses [release-it](https://github.com/release-it/release-it), an
excellent CLI release tool for GitHub repos and npm packages, to automate
creating [releases](https://github.com/StevenBlack/hosts/releases). This is why
the
[package.json](https://github.com/StevenBlack/hosts/blob/master/package.json)
and
[.release-it.json](https://github.com/StevenBlack/hosts/blob/master/.release-it.json)
files are bundled.

## Goals of this unified hosts file

The goals of this repo are to:

1. automatically combine high-quality lists of hosts,
2. provide situation-appropriate extensions,
3. de-dupe the resultant combined list,
4. and keep the resultant file reasonably sized.

A high-quality source is defined here as one that is actively curated. A hosts
source should be frequently updated by its maintainers with both additions and
removals. The larger the hosts file, the higher the level of curation is
expected.

It is expected that this unified hosts file will serve both desktop and mobile
devices under a variety of operating systems.

## Third-Party Hosts Managers

- [Unified Hosts AutoUpdate](https://github.com/ScriptTiger/Unified-Hosts-AutoUpdate "Unified Hosts AutoUpdate")
  (for Windows): The Unified Hosts AutoUpdate package is purpose-built for this
  unified hosts project as well as in active development by community members.
  You can install and uninstall any blacklist and keep it automatically up to
  date, and can be placed in a shared network location and deployed across an
  organization via group policies. And since it is in active development by
  community members, your bug reports, feature requests, and other feedback are
  most welcome.
- [ViHoMa](https://github.com/cmabad/ViHoMa) is a Visual Hosts file Manager,
  written in Java, by Christian Martínez. Check it out!

## Interesting Applications

- [Hosts-BL](https://github.com/ScriptTiger/Hosts-BL "Hosts-BL") is a simple
  tool to handle hosts file black lists. It can remove comments, remove
  duplicates, compress to 9 domains per line, add IPv6 entries. In addition, it
  can also convert black lists to multiple other black list formats compatible
  with other software, such as dnsmasq, DualServer, RPZ, Privoxy, and Unbound,
  to name a few.
- [Host Minder](https://github.com/jeremehancock/hostminder#readme) is a simple
  GUI that allows you to easily update your /etc/hosts file to one of four
  consolidated hosts files from StevenBlack/hosts. It is provided as a deb
  package and comes pre-installed on [UbuntuCE](https://ubuntuce.com/).
- [Maza ad blocking](https://github.com/tanrax/maza-ad-blocking) is a bash
  script that automatically updates host file. You can also update a fresh copy.
  And each time it generates a dnsmasq-compatible configuration file. Fast
  installation, compatible with MacOS, Linux and BSD.
- [Hostile](https://github.com/feross/hostile) is a nifty command line utility
  to easily add or remove domains from your hosts file. If our hosts files are
  too aggressive for you, you can use `hostile` to remove domains, or you can
  use `hostile` in a bash script to automate a post process each time you
  download fresh versions of hosts.
- [macOS Scripting for Configuration, Backup and Restore](https://github.com/tiiiecherle/osx_install_config)
  helps customizing, re-installing and using macOS. It also provides a
  [script](https://github.com/tiiiecherle/osx_install_config/blob/master/09_launchd/9b_run_on_boot/root/1_hosts_file/launchd_and_script/hosts_file_generator.sh)
  to install and update the hosts file using this project on macOS. In
  combination with a
  [launchd](https://github.com/tiiiecherle/osx_install_config/blob/master/09_launchd/9b_run_on_boot/root/1_hosts_file/launchd_and_script/com.hostsfile.install_update.plist)
  it updates the hosts file every x days (default is 4). To install both,
  download the GitHub repo and run the
  [install script](https://github.com/tiiiecherle/osx_install_config/blob/master/09_launchd/9b_run_on_boot/root/1_hosts_file/install_hosts_file_generator_and_launchdservice.sh)
  from the directory one level up.
- [Pi-hole](https://pi-hole.net/) is a network-wide DHCP server and ad blocker
  that runs on [Raspberry Pi](https://en.wikipedia.org/wiki/Raspberry_Pi).
  Pi-hole uses this repository as one of its sources.
- [Block ads and malware via local BIND9 DNS server](https://github.com/mueller-ma/block-ads-via-dns "Block ads and malware via local DNS server")
  (for Debian, Raspbian & Ubuntu): Set up a local DNS server with a
  `/etc/bind/named.conf.blocked` file, sourced from here.
- [Block ads, malware, and deploy parental controls via local DualServer DNS/DHCP server](https://scripttiger.github.io/dualserver/ "Block ads, malware, and deploy parental controls via local DualServer DNS/DHCP server")
  (for BSD, Windows & Linux): Set up a blacklist for everyone on your network
  using the power of the unified hosts reformatted for DualServer. And if you're
  on Windows, this project also maintains an update script to make updating
  DualServer's blacklist even easier.
- [Blocking ads and malwares with unbound](https://deadc0de.re/articles/unbound-blocking-ads.html "Blocking ads and malwares with unbound")
  –
  [Unbound](https://www.unbound.net/ "Unbound is a validating, recursive, and caching DNS resolver.")
  is a validating, recursive, and caching DNS resolver.
- [dnsmasq conversion script](https://gist.github.com/erlepereira/c11f4f7a3f60cd2071e79018e895fc8a#file-dnsmasq-antimalware)
  This GitHub gist has a short shell script (bash, will work on any 'nix) and
  uses `wget` & `awk` present in most distros, to fetch a specified hosts file
  and convert it to the format required by dnsmasq. Supports IPv4 and IPv6.
  Designed to be used as either a shell script, or can be dropped into
  `/etc/cron.weekly` (or wherever suits). The script is short and easily edited,
  also has a short document attached with notes on dnsmasq setup.
- [BlackHosts - Command Line Installer/Updater](https://github.com/Lateralus138/blackhosts)
  This is a cross-platform command line utility to help install/update hosts
  files found at this repository.
- [Hosts Compression Scripts](https://github.com/Lateralus138/hosts-compression-scripts) These are various scripts to help compress hosts files (by the author of BlackHosts).
- [Hosts Compress - Windows](https://github.com/Lateralus138/hosts-compress-windows) This is a *C++* Windows command line tool to help compress hosts files (by the author of BlackHosts and Hosts Compression Scripts). This is highly recommended over the scripts as it is **exponentially faster**.
- [dnscrypt-proxy](https://github.com/DNSCrypt/dnscrypt-proxy/wiki/Combining-Blocklists)
  provides a tool to build block lists from local and remote lists in common
  formats.
- [Control D](https://controld.com/free-dns)
  offers a public anycast network hosted mirror of the Unified (Adware + Malware) blocklist:
  - Legacy DNS: `76.76.2.35`, `76.76.10.35`, `2606:1a40::35`, `2606:1a40:1::35`
  - DNS-over-HTTPS/TLS/DOQ: `https://freedns.controld.com/x-stevenblack`, `x-stevenblack.freedns.controld.com`

## Contribute

Please read our
[Contributing Guide](https://github.com/StevenBlack/hosts/blob/master/contributing.md).
Among other things, this explains how we organize files and folders in this
repository.

We are always interested in discovering well-curated sources of hosts. If you
find one, please open an [issue](https://github.com/StevenBlack/hosts/issues) to
draw our attention.

Before you create or respond to any issue, please read our
[code of conduct](https://github.com/StevenBlack/hosts/blob/master/code_of_conduct.md).

Logo by [@Tobaloidee](https://github.com/Tobaloidee) Thank you!.
