# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

"""Generates and uploads the Pants reference documentation.

Dry run:

    ./pants run build-support/bin/generate_docs.py

Live run:

    ./pants run build-support/bin/generate_docs.py -- --sync --api-key=<API_KEY>

where API_KEY is your readme.io API Key, found here:
  https://dash.readme.com/project/pants/v2.6/api-key
"""

from __future__ import annotations

import argparse
import html
import json
import logging
import os
import pkgutil
import re
import subprocess
from html.parser import HTMLParser
from pathlib import Path, PosixPath
from typing import Any, Dict, Iterable, Optional, cast

import pystache
import requests
from common import die
from readme_api import DocRef, ReadmeAPI

from pants.help.help_info_extracter import to_help_str
from pants.version import MAJOR_MINOR

logger = logging.getLogger(__name__)


def main() -> None:
    logging.basicConfig(format="[%(levelname)s]: %(message)s", level=logging.INFO)
    args = create_parser().parse_args()

    if args.sync and not args.api_key:
        raise Exception("You specified --sync so you must also specify --api-key")

    version = determine_pants_version(args.no_prompt)
    help_info = run_pants_help_all()
    doc_urls = DocUrlMatcher().find_doc_urls(value_strs_iter(help_info))
    logger.info("Found the following docsite URLs:")
    for url in sorted(doc_urls):
        logger.info(f"  {url}")
    logger.info("Fetching titles...")
    slug_to_title = get_titles(doc_urls)
    logger.info("Found the following titles:")
    for slug, title in sorted(slug_to_title.items()):
        logger.info(f"  {slug}: {title}")
    rewritten_help_info = rewrite_value_strs(help_info, slug_to_title)
    generator = ReferenceGenerator(args, version, rewritten_help_info)
    if args.sync:
        generator.sync()
    else:
        generator.render()


def determine_pants_version(no_prompt: bool) -> str:
    version = MAJOR_MINOR
    if no_prompt:
        logger.info(f"Generating docs for Pants {version}.")
        return version

    key_confirmation = input(
        f"Generating docs for Pants {version}. Is this the correct version? [Y/n]: "
    )
    if key_confirmation and key_confirmation.lower() != "y":
        die(
            "Please either `git checkout` to the appropriate branch (e.g. 2.1.x), or change "
            "src/python/pants/VERSION."
        )
    return version


# Code to replace doc urls with appropriate markdown, for rendering on the docsite.

_doc_url_pattern = r"https://www.pantsbuild.org/v(\d+\.[^/]+)/docs/(?P<slug>[a-zA-Z0-9_-]+)"


class DocUrlMatcher:
    """Utilities for regex matching docsite URLs."""

    def __init__(self):
        self._doc_url_re = re.compile(_doc_url_pattern)

    def slug_for_url(self, url: str) -> str:
        mo = self._doc_url_re.match(url)
        if not mo:
            raise ValueError(f"Not a docsite URL: {url}")
        return cast(str, mo.group("slug"))

    def find_doc_urls(self, strs: Iterable[str]) -> set[str]:
        """Find all the docsite urls in the given strings."""
        return {mo.group(0) for s in strs for mo in self._doc_url_re.finditer(s)}


class DocUrlRewriter:
    def __init__(self, slug_to_title: dict[str, str]):
        self._doc_url_re = re.compile(_doc_url_pattern)
        self._slug_to_title = slug_to_title

    def _rewrite_url(self, mo: re.Match) -> str:
        # The docsite injects the version automatically at markdown rendering time, so we
        # must not also do so, or it will be doubled, and the resulting links will be broken.
        slug = mo.group("slug")
        title = self._slug_to_title.get(slug)
        if not title:
            raise ValueError(f"Found empty or no title for {mo.group(0)}")
        return f"[{title}](doc:{slug})"

    def rewrite(self, s: str) -> str:
        return self._doc_url_re.sub(self._rewrite_url, s)


class TitleFinder(HTMLParser):
    """Grabs the page title out of a docsite page."""

    def __init__(self):
        super().__init__()
        self._in_title: bool = False
        self._title: str | None = None

    def handle_starttag(self, tag, attrs):
        if tag == "title":
            self._in_title = True

    def handle_endtag(self, tag):
        if tag == "title":
            self._in_title = False

    def handle_data(self, data):
        if self._in_title:
            self._title = data.strip()

    @property
    def title(self) -> str | None:
        return self._title


def get_title_from_page_content(page_content: str) -> str:
    title_finder = TitleFinder()
    title_finder.feed(page_content)
    return title_finder.title or ""


def get_title(url: str) -> str:
    return get_title_from_page_content(requests.get(url).text)


def get_titles(urls: set[str]) -> dict[str, str]:
    """Return map from slug->title for each given docsite URL."""

    matcher = DocUrlMatcher()
    # TODO: Parallelize the http requests.
    #  E.g., by turning generate_docs.py into a plugin goal and using the engine.
    ret = {}
    for url in urls:
        ret[matcher.slug_for_url(url)] = get_title(url)
    return ret


def create_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description="Generate the Pants reference markdown files.")
    parser.add_argument(
        "--no-prompt",
        action="store_true",
        default=False,
        help="Don't prompt the user, accept defaults for all questions.",
    )
    parser.add_argument(
        "--sync",
        action="store_true",
        default=False,
        help="Whether to sync the generated reference docs to the docsite. "
        "If unset, will generate markdown files to the path in --output "
        "instead.  If set, --api-key must be set.",
    )
    parser.add_argument(
        "--output",
        default=PosixPath(os.path.sep) / "tmp" / "pants_docs" / "help" / "option",
        type=Path,
        help="Path to a directory under which we generate the markdown files. "
        "Useful for viewing the files locally when testing and debugging "
        "the renderer.",
    )
    parser.add_argument("--api-key", help="The readme.io API key to use. Required for --sync.")
    return parser


def run_pants_help_all() -> dict[str, Any]:
    deactivated_backends = ["internal_plugins.releases", "pants.backend.experimental.java"]
    activated_backends = [
        "pants.backend.codegen.protobuf.python",
        "pants.backend.awslambda.python",
        "pants.backend.python.lint.bandit",
        "pants.backend.python.lint.pylint",
        "pants.backend.python.lint.yapf",
    ]
    deactivated_plugins = ["toolchain.pants.plugin==0.14.0"]
    argv = [
        "./pants",
        "--concurrent",
        f"--plugins=-[{', '.join(map(repr, deactivated_plugins))}]",
        f"--backend-packages=-[{', '.join(map(repr, deactivated_backends))}]",
        f"--backend-packages=+[{', '.join(map(repr, activated_backends))}]",
        "--no-verify-config",
        "--remote-auth-plugin= ",
        "help-all",
    ]
    run = subprocess.run(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8")
    try:
        run.check_returncode()
    except subprocess.CalledProcessError:
        logger.error(
            f"Running {argv} failed with exit code {run.returncode}.\n\nstdout:\n{run.stdout}"
            f"\n\nstderr:\n{run.stderr}"
        )
        raise
    return cast(dict[str, Any], json.loads(run.stdout))


def value_strs_iter(help_info: dict[str, Any]) -> Iterable[str]:
    def _recurse(val: Any) -> Iterable[str]:
        if isinstance(val, str):
            yield val
        if isinstance(val, dict):
            for v in val.values():
                for x in _recurse(v):
                    yield x
        if isinstance(val, list):