File: setup.py

from setuptools import setup

setup()



File: docs/installation/generate.py

import re
import sys
from pathlib import Path
from typing import Dict

import yaml
from jinja2 import Template

Database = Dict[str, dict]

# Files
HERE = Path(__file__).parent
DB_FILE = HERE / 'methods.yml'
DOC_FILE = HERE.parent / 'README.md'
TPL_FILE = HERE / 'installation.jinja2'

# Database keys
KEY_DOC_STRUCTURE = 'docs-structure'
KEY_TOOLS = 'tools'

# Markers in-between content will be put.
MARKER_START = '<div data-installation-instructions>'
MARKER_END = '</div>'


def generate_documentation() -> str:
    database = load_database()
    structure = build_docs_structure(database)
    template = Template(source=TPL_FILE.read_text(encoding='utf-8'))
    output = template.render(structure=structure)
    output = clean_template_output(output)
    return output


def save_doc_file(content: str) -> None:
    current_doc = load_doc_file()
    marker_start = current_doc.find(MARKER_START) + len(MARKER_START)
    assert marker_start > 0, 'cannot find the start marker'
    marker_end = current_doc.find(MARKER_END, marker_start)
    assert marker_start < marker_end, f'{marker_end=} < {marker_start=}'
    updated_doc = (
        current_doc[:marker_start]
        + '\n\n'
        + content
        + '\n\n'
        + current_doc[marker_end:]
    )
    if current_doc != updated_doc:
        DOC_FILE.write_text(updated_doc, encoding='utf-8')


def build_docs_structure(database: Database):
    tools = database[KEY_TOOLS]
    assert len(tools) == len({tool['title'] for tool in tools.values()}), 'tool titles need to be unique'
    tree = database[KEY_DOC_STRUCTURE]
    structure = []
    for platform, tools_ids in tree.items():
        assert platform.isalnum(), f'{platform=} must be alphanumeric for generated links to work'
        platform_tools = [tools[tool_id] for tool_id in tools_ids]
        structure.append((platform, platform_tools))
    return structure


def clean_template_output(output):
    output = '\n'.join(line.strip() for line in output.strip().splitlines())
    output = re.sub('\n{3,}', '\n\n', output)
    return output


def load_database() -> Database:
    return yaml.safe_load(DB_FILE.read_text(encoding='utf-8'))


def load_doc_file() -> str:
    return DOC_FILE.read_text(encoding='utf-8')


def main() -> int:
    content = generate_documentation()
    save_doc_file(content)
    return 0


if __name__ == '__main__':
    sys.exit(main())



File: docs/contributors/fetch.py

"""
Generate the contributors database.

FIXME: replace `requests` calls with the HTTPie API, when available.
"""
import json
import os
import re
import sys
from copy import deepcopy
from datetime import datetime
from pathlib import Path
from subprocess import check_output
from time import sleep
from typing import Any, Dict, Optional, Set

import requests

FullNames = Set[str]
GitHubLogins = Set[str]
Person = Dict[str, str]
People = Dict[str, Person]
UserInfo = Dict[str, Any]

CO_AUTHORS = re.compile(r'Co-authored-by: ([^<]+) <').finditer
API_URL = 'https://api.github.com'
REPO = OWNER = 'httpie'
REPO_URL = f'{API_URL}/repos/{REPO}/{OWNER}'

HERE = Path(__file__).parent
DB_FILE = HERE / 'people.json'

DEFAULT_PERSON: Person = {'committed': [], 'reported': [], 'github': '', 'twitter': ''}
SKIPPED_LABELS = {'invalid'}

GITHUB_TOKEN = os.getenv('GITHUB_TOKEN')
assert GITHUB_TOKEN, 'GITHUB_TOKEN envar is missing'


class FinishedForNow(Exception):
    """Raised when remaining GitHub rate limit is zero."""


def main(previous_release: str, current_release: str) -> int:
    since = release_date(previous_release)
    until = release_date(current_release)

    contributors = load_awesome_people()
    try:
        committers = find_committers(since, until)
        reporters = find_reporters(since, until)
    except Exception as exc:
        # We want to save what we fetched so far. So pass.
        print(' !! ', exc)

    try:
        merge_all_the_people(current_release, contributors, committers, reporters)
        fetch_missing_users_details(contributors)
    except FinishedForNow:
        # We want to save what we fetched so far. So pass.
        print(' !! Committers:', committers)
        print(' !! Reporters:', reporters)
        exit_status = 1
    else:
        exit_status = 0

    save_awesome_people(contributors)
    return exit_status


def find_committers(since: str, until: str) -> FullNames:
    url = f'{REPO_URL}/commits'
    page = 1
    per_page = 100
    params = {
        'since': since,
        'until': until,
        'per_page': per_page,
    }
    committers: FullNames = set()

    while 'there are commits':
        params['page'] = page
        data = fetch(url, params=params)

        for item in data:
            commit = item['commit']
            committers.add(commit['author']['name'])
            debug(' >>> Commit', item['html_url'])
            for co_author in CO_AUTHORS(commit['message']):
                name = co_author.group(1)
                committers.add(name)

        if len(data) < per_page:
            break
        page += 1

    return committers


def find_reporters(since: str, until: str) -> GitHubLogins:
    url = f'{API_URL}/search/issues'
    page = 1
    per_page = 100
    params = {
        'q': f'repo:{REPO}/{OWNER} is:issue closed:{since}..{until}',
        'per_page': per_page,
    }
    reporters: GitHubLogins = set()

    while 'there are issues':
        params['page'] = page
        data = fetch(url, params=params)

        for item in data['items']:
            # Filter out unwanted labels.
            if any(label['name'] in SKIPPED_LABELS for label in item['labels']):
                continue
            debug(' >>> Issue', item['html_url'])
            reporters.add(item['user']['login'])

        if len(data['items']) < per_page:
            break
        page += 1

    return reporters


def merge_all_the_people(release: str, contributors: People, committers: FullNames, reporters: GitHubLogins) -> None:
    """
    >>> contributors = {'Alice': new_person(github='alice', twitter='alice')}
    >>> merge_all_the_people('2.6.0', contributors, {}, {})
    >>> contributors
    {'Alice': {'committed': [], 'reported': [], 'github': 'alice', 'twitter': 'alice'}}

    >>> contributors = {'Bob': new_person(github='bob', twitter='bob')}
    >>> merge_all_the_people('2.6.0', contributors, {'Bob'}, {'bob'})
    >>> contributors
    {'Bob': {'committed': ['2.6.0'], 'reported': ['2.6.0'], 'github': 'bob', 'twitter': 'bob'}}

    >>> contributors = {'Charlotte': new_person(github='charlotte', twitter='charlotte', committed=['2.5.0'], reported=['2.5.0'])}
    >>> merge_all_the_people('2.6.0', contributors, {'Charlotte'}, {'charlotte'})
    >>> contributors
    {'Charlotte': {'committed': ['2.5.0', '2.6.0'], 'reported': ['2.5.0', '2.6.0'], 'github': 'charlotte', 'twitter': 'charlotte'}}

    """
    # Update known contributors.
    for name, details in contributors.items():
        if name in committers:
            if release not in details['committed']:
                details['committed'].append(release)
            committers.remove(name)
        if details['github'] in reporters:
            if release not in details['reported']:
                details['reported'].append(release)
            reporters.remove(details['github'])

    # Add new committers.
    for name in committers:
        user_info = user(fullname=name)
        contributors[name] = new_person(
            github=user_info['login'],
            twitter=user_info['twitter_username'],
            committed=[release],
        )
        if user_info['login'] in reporters:
            contributors[name]['reported'].append(release)
            reporters.remove(user_info['login'])

    # Add new reporters.
    for github_username in reporters:
        user_info = user(github_username=github_username)
        contributors[user_info['name'] or user_info['login']] = new_person(
            github=github_username,
            twitter=user_info['twitter_username'],
            reported=[release],
        )


def release_date(release: str) -> str:
    date = check_output(['git', 'log', '-1', '--format=%ai', release], text=True).strip()
    return datetime.strptime(date, '%Y-%m-%d %H:%M:%S %z').isoformat()


def load_awesome_people() -> People:
    try:
        with DB_FILE.open(encoding='utf-8') as fh:
            return json.load(fh)
    except (FileNotFoundError, ValueError):
        return {}


def fetch(url: str, params: Optional[Dict[str, str]] = None) -> UserInfo:
    headers = {
        'Accept': 'application/vnd.github.v3+json',
        'Authentication': f'token {GITHUB_TOKEN}'
    }
    for retry in range(1, 6):
        debug(f'[{retry}/5]', f'{url = }', f'{params = }')
        with requests.get(url, params=params, headers=headers) as req:
            try:
                req.raise_for_status()
            except requests.exceptions.HTTPError as exc:
                if exc.response.status_code == 403:
                    # 403 Client Error: rate limit exceeded for url: ...
                    now = int(datetime.utcnow().timestamp())
                    xrate_limit_reset = int(exc.response.headers['X-RateLimit-Reset'])
                    wait = xrate_limit_reset - now
                    if wait > 20:
                        raise FinishedForNow()
                    debug(' !', 'Waiting', wait, 'seconds before another try ...')
                    sleep(wait)
                continue
            return req.json()
    assert ValueError('Rate limit exceeded')


def new_person(**kwargs: str) -> Person:
    data = deepcopy(DEFAULT_PERSON)
    data.update(**kwargs)
    return data


def user(fullname: Optional[str] = '', github_username: Optional[str] = '') -> UserInfo:
    if github_username:
        url = f'{API_URL}/users/{github_username}'
        return fetch(url)

    url = f'{API_URL}/search/users'
    for query in (f'fullname:{fullname}', f'user:{fullname}'):
        params = {
            'q': f'repo:{REPO}/{OWNER} {query}',
            'per_page': 1,
        }
        user_info = fetch(url, params=params)
        if user_info['items']:
            user_url = user_info['items'][0]['url']
            return fetch(user_url)


def fetch_missing_users_details(people: People) -> None:
    for name, details in people.items():
        if details['github'] and details['twitter']:
            continue
        user_info = user(github_username=details['github'], fullname=name)
        if not details['github']:
            details['github'] = user_info['login']
        if not details['twitter']:
            details['twitter'] = user_info['twitter_username']


def save_awesome_people(people: People) -> None:
    with DB_FILE.open(mode='w', encoding='utf-8') as fh:
        json.dump(people, fh, indent=4, sort_keys=True)
        fh.write("\n")


def debug(*args: Any) -> None:
    if os.getenv('DEBUG') == '1':
        print(*args)


if __name__ == '__main__':
    ret = 1
    try:
        ret = main(*sys.argv[1:])
    except TypeError:
        ret = 2
        print(f'''
Fetch contributors to a release.

Usage:
    python {sys.argv[0]} {sys.argv[0]} <RELEASE N-1> <RELEASE N>
Example:
    python {sys.argv[0]} 2.4.0 2.5.0

Define the DEBUG=1 environment variable to enable verbose output.
''')
    except KeyboardInterrupt:
        ret = 255
    sys.exit(ret)



File: docs/contributors/generate.py

"""
Generate snippets to copy-paste.
"""
import sys

from jinja2 import Template

from fetch import HERE, load_awesome_people

TPL_FILE = HERE / 'snippet.jinja2'

HTTPIE_TEAM = {
    'claudiatd',
    'jakubroztocil',
    'jkbr',
    'isidentical'
}

BOT_ACCOUNTS = {
    'dependabot-sr'
}

IGNORE_ACCOUNTS = HTTPIE_TEAM | BOT_ACCOUNTS


def generate_snippets(release: str) -> str:
    people = load_awesome_people()
    contributors = {
        name: details
        for name, details in people.items()
        if details['github'] not in IGNORE_ACCOUNTS
        and (release in details['committed'] or release in details['reported'])
    }

    template = Template(source=TPL_FILE.read_text(encoding='utf-8'))
    output = template.render(contributors=contributors, release=release)
    print(output)
    return 0


if __name__ == '__main__':
    ret = 1
    try:
        ret = generate_snippets(sys.argv[1])
    except (IndexError, TypeError):
        ret = 2
        print(f'''
Generate snippets for contributors to a release.

Usage:
    python {sys.argv[0]} {sys.argv[0]} <RELEASE>
''')
    sys.exit(ret)



File: httpie/cookies.py

from http import cookiejar


_LOCALHOST = 'localhost'
_LOCALHOST_SUFFIX = '.localhost'


class HTTPieCookiePolicy(cookiejar.DefaultCookiePolicy):
    def return_ok_secure(self, cookie, request):
        """Check whether the given cookie is sent to a secure host."""

        is_secure_protocol = super().return_ok_secure(cookie, request)
        if is_secure_protocol:
            return True

        # The original implementation of this method only takes secure protocols
        # (e.g., https) into account, but the latest developments in modern browsers
        # (chrome, firefox) assume 'localhost' is also a secure location. So we
        # override it with our own strategy.
        return self._is_local_host(cookiejar.request_host(request))

    def _is_local_host(self, hostname):
        # Implements the static localhost detection algorithm in firefox.
        # <https://searchfox.org/mozilla-central/rev/d4d7611ee4dd0003b492b865bc5988a4e6afc985/netwerk/dns/DNS.cpp#205-218>
        return hostname == _LOCALHOST or hostname.endswith(_LOCALHOST_SUFFIX)



File: httpie/sessions.py

"""
Persistent, JSON-serialized sessions.

"""
import os
import re

from http.cookies import SimpleCookie
from http.cookiejar import Cookie
from pathlib import Path
from typing import Any, Dict, List, Optional, Union

from requests.auth import AuthBase
from requests.cookies import RequestsCookieJar, remove_cookie_by_name

from .context import Environment, LogLevel
from .cookies import HTTPieCookiePolicy
from .cli.dicts import HTTPHeadersDict
from .config import BaseConfigDict, DEFAULT_CONFIG_DIR
from .utils import url_as_host
from .plugins.registry import plugin_manager

from .legacy import (
    v3_1_0_session_cookie_format as legacy_cookies,
    v3_2_0_session_header_format as legacy_headers
)


SESSIONS_DIR_NAME = 'sessions'
DEFAULT_SESSIONS_DIR = DEFAULT_CONFIG_DIR / SESSIONS_DIR_NAME
VALID_SESSION_NAME_PATTERN = re.compile('^[a-zA-Z0-9_.-]+$')
# Request headers starting with these prefixes won't be stored in sessions.
# They are specific to each request.
# <https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#Requests>
SESSION_IGNORED_HEADER_PREFIXES = ['Content-', 'If-']

# Cookie related options
KEPT_COOKIE_OPTIONS = ['name', 'expires', 'path', 'value', 'domain', 'secure']
DEFAULT_COOKIE_PATH = '/'


def is_anonymous_session(session_name: str) -> bool:
    return os.path.sep in session_name


def session_hostname_to_dirname(hostname: str, session_name: str) -> str:
    # host:port => host_port
    hostname = hostname.replace(':', '_')
    return os.path.join(
        SESSIONS_DIR_NAME,
        hostname,
        f'{session_name}.json'
    )


def strip_port(hostname: str) -> str:
    return hostname.split(':')[0]


def materialize_cookie(cookie: Cookie) -> Dict[str, Any]:
    materialized_cookie = {
        option: getattr(cookie, option)
        for option in KEPT_COOKIE_OPTIONS
    }

    if (
        cookie._rest.get('is_explicit_none')
        and materialized_cookie['domain'] == ''
    ):
        materialized_cookie['domain'] = None

    return materialized_cookie


def materialize_cookies(jar: RequestsCookieJar) -> List[Dict[str, Any]]:
    return [
        materialize_cookie(cookie)
        for cookie in jar
    ]


def materialize_headers(headers: Dict[str, str]) -> List[Dict[str, Any]]:
    return [
        {
            'name': name,
            'value': value
        }
        for name, value in headers.copy().items()
    ]


def get_httpie_session(
    env: Environment,
    config_dir: Path,
    session_name: str,
    host: Optional[str],
    url: str,
    *,
    suppress_legacy_warnings: bool = False
) -> 'Session':
    bound_hostname = host or url_as_host(url)
    if not bound_hostname:
        # HACK/FIXME: httpie-unixsocket's URLs have no hostname.
        bound_hostname = 'localhost'

    if is_anonymous_session(session_name):
        path = os.path.expanduser(session_name)
        session_id = path
    else:
        path = config_dir / session_hostname_to_dirname(bound_hostname, session_name)
        session_id = session_name

    session = Session(
        path,
        env=env,
        session_id=session_id,
        bound_host=strip_port(bound_hostname),
        suppress_legacy_warnings=suppress_legacy_warnings
    )
    session.load()
    return session


class Session(BaseConfigDict):
    helpurl = 'https://httpie.io/docs#sessions'
    about = 'HTTPie session file'

    def __init__(
        self,
        path: Union[str, Path],
        env: Environment,
        bound_host: str,
        session_id: str,
        suppress_legacy_warnings: bool = False,
    ):
        super().__init__(path=Path(path))

        # Default values for the session files
        self['headers'] = []
        self['cookies'] = []
        self['auth'] = {
            'type': None,
            'username': None,
            'password': None
        }

        # Runtime state of the Session objects.
        self.env = env
        self._headers = HTTPHeadersDict()
        self.cookie_jar = RequestsCookieJar(
            # See also a temporary workaround for a Requests bug in `compat.py`.
            policy=HTTPieCookiePolicy(),
        )
        self.session_id = session_id
        self.bound_host = bound_host
        self.suppress_legacy_warnings = suppress_legacy_warnings

    def _add_cookies(self, cookies: List[Dict[str, Any]]) -> None:
        for cookie in cookies:
            domain = cookie.get('domain', '')
            if domain is None:
                # domain = None means explicitly lack of cookie, though
                # requests requires domain to be a string so we'll cast it
                # manually.
                cookie['domain'] = ''
                cookie['rest'] = {'is_explicit_none': True}

            self.cookie_jar.set(**cookie)

    def pre_process_data(self, data: Dict[str, Any]) -> Dict[str, Any]:
        for key, deserializer, importer in [
            ('cookies', legacy_cookies.pre_process, self._add_cookies),
            ('headers', legacy_headers.pre_process, self._headers.update),
        ]:
            values = data.get(key)
            if values:
                normalized_values = deserializer(self, values)
            else:
                normalized_values = []

            importer(normalized_values)

        return data

    def post_process_data(self, data: Dict[str, Any]) -> Dict[str, Any]:
        for key, store, serializer, exporter in [
            ('cookies', self.cookie_jar, materialize_cookies, legacy_cookies.post_process),
            ('headers', self._headers, materialize_headers, legacy_headers.post_process),
        ]:
            original_type = type(data.get(key))
            values = serializer(store)

            data[key] = exporter(
                values,
                original_type=original_type
            )

        return data

    def _compute_new_headers(self, request_headers: HTTPHeadersDict) -> HTTPHeadersDict:
        new_headers = HTTPHeadersDict()
        for name, value in request_headers.copy().items():
            if value is None:
                continue  # Ignore explicitly unset headers

            original_value = value
            if type(value) is not str:
                value = value.decode()

            if name.lower() == 'user-agent' and value.startswith('HTTPie/'):
                continue

            if name.lower() == 'cookie':
                for cookie_name, morsel in SimpleCookie(value).items():
                    if not morsel['path']:
                        morsel['path'] = DEFAULT_COOKIE_PATH
                    self.cookie_jar.set(cookie_name, morsel)

                request_headers.remove_item(name, original_value)
                continue

            for prefix in SESSION_IGNORED_HEADER_PREFIXES:
                if name.lower().startswith(prefix.lower()):
                    break
            else:
                new_headers.add(name, value)

        return new_headers

    def update_headers(self, request_headers: HTTPHeadersDict):
        """
        Update the session headers with the request ones while ignoring
        certain name prefixes.

        """

        new_headers = self._compute_new_headers(request_headers)
        new_keys = new_headers.copy().keys()

        # New headers will take priority over the existing ones, and override
        # them directly instead of extending them.
        for key, value in self._headers.copy().items():
            if key in new_keys:
                continue

            new_headers.add(key, value)

        self._headers = new_headers

    @property
    def headers(self) -> HTTPHeadersDict:
        return self._headers.copy()

    @property
    def cookies(self) -> RequestsCookieJar:
        self.cookie_jar.clear_expired_cookies()
        return self.cookie_jar

    @cookies.setter
    def cookies(self, jar: RequestsCookieJar):
        self.cookie_jar = jar

    def remove_cookies(self, cookies: List[Dict[str, str]]):
        for cookie in cookies:
            remove_cookie_by_name(
                self.cookie_jar,
                cookie['name'],
                domain=cookie.get('domain', None),
                path=cookie.get('path', None)
            )

    @property
    def auth(self) -> Optional[AuthBase]:
        auth = self.get('auth', None)
        if not auth or not auth['type']:
            return

        plugin = plugin_manager.get_auth_plugin(auth['type'])()

        credentials = {'username': None, 'password': None}
        try:
            # New style
            plugin.raw_auth = auth['raw_auth']
        except KeyError:
            # Old style
            credentials = {
                'username': auth['username'],
                'password': auth['password'],
            }
        else:
            if plugin.auth_parse:
                from .cli.argtypes import parse_auth
                parsed = parse_auth(plugin.raw_auth)
                credentials = {
                    'username': parsed.key,
                    'password': parsed.value,
                }

        return plugin.get_auth(**credentials)

    @auth.setter
    def auth(self, auth: dict):
        assert {'type', 'raw_auth'} == auth.keys()
        self['auth'] = auth

    @property
    def is_anonymous(self):
        return is_anonymous_session(self.session_id)

    def warn_legacy_usage(self, warning: str) -> None:
        if self.suppress_legacy_warnings:
            return None

        self.env.log_error(
            warning,
            level=LogLevel.WARNING
        )

        # We don't want to spam multiple warnings on each usage,
        # so if there is already a warning for the legacy usage
        # we'll skip the next ones.
        self.suppress_legacy_warnings = True



File: httpie/config.py

import json
import os
from pathlib import Path
from typing import Any, Dict, Union

from . import __version__
from .compat import is_windows
from .encoding import UTF8


ENV_XDG_CONFIG_HOME = 'XDG_CONFIG_HOME'
ENV_HTTPIE_CONFIG_DIR = 'HTTPIE_CONFIG_DIR'
DEFAULT_CONFIG_DIRNAME = 'httpie'
DEFAULT_RELATIVE_XDG_CONFIG_HOME = Path('.config')
DEFAULT_RELATIVE_LEGACY_CONFIG_DIR = Path('.httpie')
DEFAULT_WINDOWS_CONFIG_DIR = Path(
    os.path.expandvars('%APPDATA%')) / DEFAULT_CONFIG_DIRNAME


def get_default_config_dir() -> Path:
    """
    Return the path to the httpie configuration directory.

    This directory isn't guaranteed to exist, and nor are any of its
    ancestors (only the legacy ~/.httpie, if returned, is guaranteed to exist).

    XDG Base Directory Specification support:

        <https://wiki.archlinux.org/index.php/XDG_Base_Directory>

        $XDG_CONFIG_HOME is supported; $XDG_CONFIG_DIRS is not

    """
    # 1. explicitly set through env
    env_config_dir = os.environ.get(ENV_HTTPIE_CONFIG_DIR)
    if env_config_dir:
        return Path(env_config_dir)

    # 2. Windows
    if is_windows:
        return DEFAULT_WINDOWS_CONFIG_DIR

    home_dir = Path.home()

    # 3. legacy ~/.httpie
    legacy_config_dir = home_dir / DEFAULT_RELATIVE_LEGACY_CONFIG_DIR
    if legacy_config_dir.exists():
        return legacy_config_dir

    # 4. XDG
    xdg_config_home_dir = os.environ.get(
        ENV_XDG_CONFIG_HOME,  # 4.1. explicit
        home_dir / DEFAULT_RELATIVE_XDG_CONFIG_HOME  # 4.2. default
    )
    return Path(xdg_config_home_dir) / DEFAULT_CONFIG_DIRNAME


DEFAULT_CONFIG_DIR = get_default_config_dir()


class ConfigFileError(Exception):
    pass


def read_raw_config(config_type: str, path: Path) -> Dict[str, Any]:
    try:
        with path.open(encoding=UTF8) as f:
            try:
                return json.load(f)
            except ValueError as e:
                raise ConfigFileError(
                    f'invalid {config_type} file: {e} [{path}]'
                )
    except FileNotFoundError:
        pass
    except OSError as e:
        raise ConfigFileError(f'cannot read {config_type} file: {e}')


class BaseConfigDict(dict):
    name = None
    helpurl = None
    about = None

    def __init__(self, path: Path):
        super().__init__()
        self.path = path

    def ensure_directory(self):
        self.path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)

    def is_new(self) -> bool:
        return not self.path.exists()

    def pre_process_data(self, data: Dict[str, Any]) -> Dict[str, Any]:
        """Hook for processing the incoming config data."""
        return data

    def post_process_data(self, data: Dict[str, Any]) -> Dict[str, Any]:
        """Hook for processing the outgoing config data."""
        return data

    def load(self):
        config_type = type(self).__name__.lower()
        data = read_raw_config(config_type, self.path)
        if data is not None:
            data = self.pre_process_data(data)
            self.update(data)

    def save(self, *, bump_version: bool = False):
        self.setdefault('__meta__', {})
        if bump_version or 'httpie' not in self['__meta__']:
            self['__meta__']['httpie'] = __version__
        if self.helpurl:
            self['__meta__']['help'] = self.helpurl

        if self.about:
            self['__meta__']['about'] = self.about

        self.ensure_directory()

        json_string = json.dumps(
            obj=self.post_process_data(self),
            indent=4,
            sort_keys=True,
            ensure_ascii=True,
        )
        self.path.write_text(json_string + '\n', encoding=UTF8)

    @property
    def version(self):
        return self.get(
            '__meta__', {}
        ).get('httpie', __version__)


class Config(BaseConfigDict):
    FILENAME = 'config.json'
    DEFAULTS = {
        'default_options': []
    }

    def __init__(self, directory: Union[str, Path] = DEFAULT_CONFIG_DIR):
        self.directory = Path(directory)
        super().__init__(path=self.directory / self.FILENAME)
        self.update(self.DEFAULTS)

    @property
    def default_options(self) -> list:
        return self['default_options']

    def _configured_path(self, config_option: str, default: str) -> None:
        return Path(
            self.get(config_option, self.directory / default)
        ).expanduser().resolve()

    @property
    def plugins_dir(self) -> Path:
        return self._configured_path('plugins_dir', 'plugins')

    @property
    def version_info_file(self) -> Path:
        return self._configured_path('version_info_file', 'version_info.json')

    @property
    def developer_mode(self) -> bool:
        """This is a special setting for the development environment. It is
        different from the --debug mode in the terms that it might change
        the behavior for certain parameters (e.g updater system) that
        we usually ignore."""

        return self.get('developer_mode')



File: httpie/compat.py

import sys
from typing import Any, Optional, Iterable

from httpie.cookies import HTTPieCookiePolicy
from http import cookiejar # noqa


# Request does not carry the original policy attached to the
# cookie jar, so until it is resolved we change the global cookie
# policy. <https://github.com/psf/requests/issues/5449>
cookiejar.DefaultCookiePolicy = HTTPieCookiePolicy


is_windows = 'win32' in str(sys.platform).lower()
is_frozen = getattr(sys, 'frozen', False)

MIN_SUPPORTED_PY_VERSION = (3, 7)
MAX_SUPPORTED_PY_VERSION = (3, 11)

try:
    from functools import cached_property
except ImportError:
    # Can be removed once we drop Python <3.8 support.
    # Taken from `django.utils.functional.cached_property`.
    class cached_property:
        """
        Decorator that converts a method with a single self argument into a
        property cached on the instance.

        A cached property can be made out of an existing method:
        (e.g. ``url = cached_property(get_absolute_url)``).
        The optional ``name`` argument is obsolete as of Python 3.6 and will be
        deprecated in Django 4.0 (#30127).
        """
        name = None

        @staticmethod
        def func(instance):
            raise TypeError(
                'Cannot use cached_property instance without calling '
                '__set_name__() on it.'
            )

        def __init__(self, func, name=None):
            self.real_func = func
            self.__doc__ = getattr(func, '__doc__')

        def __set_name__(self, owner, name):
            if self.name is None:
                self.name = name
                self.func = self.real_func
            elif name != self.name:
                raise TypeError(
                    "Cannot assign the same cached_property to two different names "
                    "(%r and %r)." % (self.name, name)
                )

        def __get__(self, instance, cls=None):
            """
            Call the function and put the return value in instance.__dict__ so that
            subsequent attribute access on the instance returns the cached value
            instead of calling cached_property.__get__().
            """
            if instance is None:
                return self
            res = instance.__dict__[self.name] = self.func(instance)
            return res


# importlib_metadata was a provisional module, so the APIs changed quite a few times
# between 3.8-3.10. It was also not included in the standard library until 3.8, so
# we install the backport for <3.8.

if sys.version_info >= (3, 8):
    import importlib.metadata as importlib_metadata
else:
    import importlib_metadata


def find_entry_points(entry_points: Any, group: str) -> Iterable[importlib_metadata.EntryPoint]:
    if hasattr(entry_points, "select"):  # Python 3.10+ / importlib_metadata >= 3.9.0
        return entry_points.select(group=group)
    else:
        return set(entry_points.get(group, ()))


def get_dist_name(entry_point: importlib_metadata.EntryPoint) -> Optional[str]:
    dist = getattr(entry_point, "dist", None)
    if dist is not None:  # Python 3.10+
        return dist.name

    match = entry_point.pattern.match(entry_point.value)
    if not (match and match.group('module')):
        return None

    package = match.group('module').split('.')[0]
    try:
        metadata = importlib_metadata.metadata(package)
    except importlib_metadata.PackageNotFoundError:
        return None
    else:
        return metadata.get('name')



File: httpie/encoding.py

from typing import Union, Tuple

from charset_normalizer import from_bytes
from charset_normalizer.constant import TOO_SMALL_SEQUENCE

UTF8 = 'utf-8'

ContentBytes = Union[bytearray, bytes]


def detect_encoding(content: ContentBytes) -> str:
    """
    We default to UTF-8 if text too short, because the detection
    can return a random encoding leading to confusing results
    given the `charset_normalizer` version (< 2.0.5).

    >>> too_short = ']"foo"'
    >>> detected = from_bytes(too_short.encode()).best().encoding
    >>> detected
    'ascii'
    >>> too_short.encode().decode(detected)
    ']"foo"'
    """
    encoding = UTF8
    if len(content) > TOO_SMALL_SEQUENCE:
        match = from_bytes(bytes(content)).best()
        if match:
            encoding = match.encoding
    return encoding


def smart_decode(content: ContentBytes, encoding: str) -> Tuple[str, str]:
    """Decode `content` using the given `encoding`.
    If no `encoding` is provided, the best effort is to guess it from `content`.

    Unicode errors are replaced.

    """
    if not encoding:
        encoding = detect_encoding(content)
    return content.decode(encoding, 'replace'), encoding


def smart_encode(content: str, encoding: str) -> bytes:
    """Encode `content` using the given `encoding`.

    Unicode errors are replaced.

    """
    return content.encode(encoding, 'replace')



File: httpie/models.py

from time import monotonic

import requests
from urllib3.util import SKIP_HEADER, SKIPPABLE_HEADERS

from enum import Enum, auto
from typing import Iterable, Union, NamedTuple
from urllib.parse import urlsplit

from .cli.constants import (
    OUT_REQ_BODY,
    OUT_REQ_HEAD,
    OUT_RESP_BODY,
    OUT_RESP_HEAD,
    OUT_RESP_META
)
from .compat import cached_property
from .utils import split_cookies, parse_content_type_header

ELAPSED_TIME_LABEL = 'Elapsed time'


class HTTPMessage:
    """Abstract class for HTTP messages."""

    def __init__(self, orig):
        self._orig = orig

    def iter_body(self, chunk_size: int) -> Iterable[bytes]:
        """Return an iterator over the body."""
        raise NotImplementedError

    def iter_lines(self, chunk_size: int) -> Iterable[bytes]:
        """Return an iterator over the body yielding (`line`, `line_feed`)."""
        raise NotImplementedError

    @property
    def headers(self) -> str:
        """Return a `str` with the message's headers."""
        raise NotImplementedError

    @property
    def metadata(self) -> str:
        """Return metadata about the current message."""
        raise NotImplementedError

    @cached_property
    def encoding(self) -> str:
        ct, params = parse_content_type_header(self.content_type)
        return params.get('charset', '')

    @property
    def content_type(self) -> str:
        """Return the message content type."""
        ct = self._orig.headers.get('Content-Type', '')
        if not isinstance(ct, str):
            ct = ct.decode()
        return ct


class HTTPResponse(HTTPMessage):
    """A :class:`requests.models.Response` wrapper."""

    def iter_body(self, chunk_size=1):
        return self._orig.iter_content(chunk_size=chunk_size)

    def iter_lines(self, chunk_size):
        return ((line, b'\n') for line in self._orig.iter_lines(chunk_size))

    @property
    def headers(self):
        original = self._orig
        status_line = f'HTTP/{self.version} {original.status_code} {original.reason}'
        headers = [status_line]
        headers.extend(
            ': '.join(header)
            for header in original.headers.items()
            if header[0] != 'Set-Cookie'
        )
        headers.extend(
            f'Set-Cookie: {cookie}'
            for header, value in original.headers.items()
            for cookie in split_cookies(value)
            if header == 'Set-Cookie'
        )
        return '\r\n'.join(headers)

    @property
    def metadata(self) -> str:
        data = {}
        time_to_parse_headers = self._orig.elapsed.total_seconds()
        # noinspection PyProtectedMember
        time_since_headers_parsed = monotonic() - self._orig._httpie_headers_parsed_at
        time_elapsed = time_to_parse_headers + time_since_headers_parsed
        # data['Headers time'] = str(round(time_to_parse_headers, 5)) + 's'
        # data['Body time'] = str(round(time_since_headers_parsed, 5)) + 's'
        data[ELAPSED_TIME_LABEL] = str(round(time_elapsed, 10)) + 's'
        return '\n'.join(
            f'{key}: {value}'
            for key, value in data.items()
        )

    @property
    def version(self) -> str:
        """
        Return the HTTP version used by the server, e.g. '1.1'.

        Assume HTTP/1.1 if version is not available.

        """
        mapping = {
            9: '0.9',
            10: '1.0',
            11: '1.1',
            20: '2.0',
        }
        fallback = 11
        version = None
        try:
            raw = self._orig.raw
            if getattr(raw, '_original_response', None):
                version = raw._original_response.version
            else:
                version = raw.version
        except AttributeError:
            pass
        return mapping[version or fallback]


class HTTPRequest(HTTPMessage):
    """A :class:`requests.models.Request` wrapper."""

    def iter_body(self, chunk_size):
        yield self.body

    def iter_lines(self, chunk_size):
        yield self.body, b''

    @property
    def headers(self):
        url = urlsplit(self._orig.url)

        request_line = '{method} {path}{query} HTTP/1.1'.format(
            method=self._orig.method,
            path=url.path or '/',
            query=f'?{url.query}' if url.query else ''
        )

        headers = self._orig.headers.copy()
        if 'Host' not in self._orig.headers:
            headers['Host'] = url.netloc.split('@')[-1]

        headers = [
            f'{name}: {value if isinstance(value, str) else value.decode()}'
            for name, value in headers.items()
            if not (name.lower() in SKIPPABLE_HEADERS and value == SKIP_HEADER)
        ]

        headers.insert(0, request_line)
        headers = '\r\n'.join(headers).strip()
        return headers

    @property
    def body(self):
        body = self._orig.body
        if isinstance(body, str):
            # Happens with JSON/form request data parsed from the command line.
            body = body.encode()
        return body or b''


RequestsMessage = Union[requests.PreparedRequest, requests.Response]


class RequestsMessageKind(Enum):
    REQUEST = auto()
    RESPONSE = auto()


def infer_requests_message_kind(message: RequestsMessage) -> RequestsMessageKind:
    if isinstance(message, requests.PreparedRequest):
        return RequestsMessageKind.REQUEST
    elif isinstance(message, requests.Response):
        return RequestsMessageKind.RESPONSE
    else:
        raise TypeError(f"Unexpected message type: {type(message).__name__}")


OPTION_TO_PARAM = {
    RequestsMessageKind.REQUEST: {
        'headers': OUT_REQ_HEAD,
        'body': OUT_REQ_BODY,
    },
    RequestsMessageKind.RESPONSE: {
        'headers': OUT_RESP_HEAD,
        'body': OUT_RESP_BODY,
        'meta': OUT_RESP_META
    }
}


class OutputOptions(NamedTuple):
    kind: RequestsMessageKind
    headers: bool
    body: bool
    meta: bool = False

    def any(self):
        return (
            self.headers
            or self.body
            or self.meta
        )

    @classmethod
    def from_message(
        cls,
        message: RequestsMessage,
        raw_args: str = '',
        **kwargs
    ):
        kind = infer_requests_message_kind(message)

        options = {
            option: param in raw_args
            for option, param in OPTION_TO_PARAM[kind].items()
        }
        options.update(kwargs)

        return cls(
            kind=kind,
            **options
        )



File: httpie/client.py

import argparse
import http.client
import json
import sys
from contextlib import contextmanager
from time import monotonic
from typing import Any, Dict, Callable, Iterable
from urllib.parse import urlparse, urlunparse

import requests
# noinspection PyPackageRequirements
import urllib3
from urllib3.util import SKIP_HEADER, SKIPPABLE_HEADERS

from . import __version__
from .adapters import HTTPieHTTPAdapter
from .cli.constants import HTTP_OPTIONS
from .cli.dicts import HTTPHeadersDict
from .cli.nested_json import unwrap_top_level_list_if_needed
from .context import Environment
from .encoding import UTF8
from .models import RequestsMessage
from .plugins.registry import plugin_manager
from .sessions import get_httpie_session
from .ssl_ import AVAILABLE_SSL_VERSION_ARG_MAPPING, HTTPieCertificate, HTTPieHTTPSAdapter
from .uploads import (
    compress_request, prepare_request_body,
    get_multipart_data_and_content_type,
)
from .utils import get_expired_cookies, repr_dict


urllib3.disable_warnings()

FORM_CONTENT_TYPE = f'application/x-www-form-urlencoded; charset={UTF8}'
JSON_CONTENT_TYPE = 'application/json'
JSON_ACCEPT = f'{JSON_CONTENT_TYPE}, */*;q=0.5'
DEFAULT_UA = f'HTTPie/{__version__}'

IGNORE_CONTENT_LENGTH_METHODS = frozenset([HTTP_OPTIONS])


def collect_messages(
    env: Environment,
    args: argparse.Namespace,
    request_body_read_callback: Callable[[bytes], None] = None,
) -> Iterable[RequestsMessage]:
    httpie_session = None
    httpie_session_headers = None
    if args.session or args.session_read_only:
        httpie_session = get_httpie_session(
            env=env,
            config_dir=env.config.directory,
            session_name=args.session or args.session_read_only,
            host=args.headers.get('Host'),
            url=args.url,
        )
        httpie_session_headers = httpie_session.headers

    request_kwargs = make_request_kwargs(
        env,
        args=args,
        base_headers=httpie_session_headers,
        request_body_read_callback=request_body_read_callback
    )
    send_kwargs = make_send_kwargs(args)
    send_kwargs_mergeable_from_env = make_send_kwargs_mergeable_from_env(args)
    requests_session = build_requests_session(
        ssl_version=args.ssl_version,
        ciphers=args.ciphers,
        verify=bool(send_kwargs_mergeable_from_env['verify'])
    )

    if httpie_session:
        httpie_session.update_headers(request_kwargs['headers'])
        requests_session.cookies = httpie_session.cookies
        if args.auth_plugin:
            # Save auth from CLI to HTTPie session.
            httpie_session.auth = {
                'type': args.auth_plugin.auth_type,
                'raw_auth': args.auth_plugin.raw_auth,
            }
        elif httpie_session.auth:
            # Apply auth from HTTPie session
            request_kwargs['auth'] = httpie_session.auth

    if args.debug:
        # TODO: reflect the split between request and send kwargs.
        dump_request(request_kwargs)

    request = requests.Request(**request_kwargs)
    prepared_request = requests_session.prepare_request(request)
    transform_headers(request, prepared_request)
    if args.path_as_is:
        prepared_request.url = ensure_path_as_is(
            orig_url=args.url,
            prepped_url=prepared_request.url,
        )
    if args.compress and prepared_request.body:
        compress_request(
            request=prepared_request,
            always=args.compress > 1,
        )
    response_count = 0
    expired_cookies = []
    while prepared_request:
        yield prepared_request
        if not args.offline:
            send_kwargs_merged = requests_session.merge_environment_settings(
                url=prepared_request.url,
                **send_kwargs_mergeable_from_env,
            )
            with max_headers(args.max_headers):
                response = requests_session.send(
                    request=prepared_request,
                    **send_kwargs_merged,
                    **send_kwargs,
                )
            response._httpie_headers_parsed_at = monotonic()
            expired_cookies += get_expired_cookies(
                response.headers.get('Set-Cookie', '')
            )

            response_count += 1
            if response.next:
                if args.max_redirects and response_count == args.max_redirects:
                    raise requests.TooManyRedirects
                if args.follow:
                    prepared_request = response.next
                    if args.all:
                        yield response
                    continue
            yield response
        break

    if httpie_session:
        if httpie_session.is_new() or not args.session_read_only:
            httpie_session.cookies = requests_session.cookies
            httpie_session.remove_cookies(expired_cookies)
            httpie_session.save()


# noinspection PyProtectedMember
@contextmanager
def max_headers(limit):
    # <https://github.com/httpie/cli/issues/802>
    # noinspection PyUnresolvedReferences
    orig = http.client._MAXHEADERS
    http.client._MAXHEADERS = limit or float('Inf')
    try:
        yield
    finally:
        http.client._MAXHEADERS = orig


def build_requests_session(
    verify: bool,
    ssl_version: str = None,
    ciphers: str = None,
) -> requests.Session:
    requests_session = requests.Session()

    # Install our adapter.
    http_adapter = HTTPieHTTPAdapter()
    https_adapter = HTTPieHTTPSAdapter(
        ciphers=ciphers,
        verify=verify,
        ssl_version=(
            AVAILABLE_SSL_VERSION_ARG_MAPPING[ssl_version]
            if ssl_version else None
        ),
    )
    requests_session.mount('http://', http_adapter)
    requests_session.mount('https://', https_adapter)

    # Install adapters from plugins.
    for plugin_cls in plugin_manager.get_transport_plugins():
        transport_plugin = plugin_cls()
        requests_session.mount(
            prefix=transport_plugin.prefix,
            adapter=transport_plugin.get_adapter(),
        )

    return requests_session


def dump_request(kwargs: dict):
    sys.stderr.write(
        f'\n>>> requests.request(**{repr_dict(kwargs)})\n\n')


def finalize_headers(headers: HTTPHeadersDict) -> HTTPHeadersDict:
    final_headers = HTTPHeadersDict()
    for name, value in headers.items():
        if value is not None:
            # “leading or trailing LWS MAY be removed without
            # changing the semantics of the field value”
            # <https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html>
            # Also, requests raises `InvalidHeader` for leading spaces.
            value = value.strip()
            if isinstance(value, str):
                # See <https://github.com/httpie/cli/issues/212>
                value = value.encode()
        elif name.lower() in SKIPPABLE_HEADERS:
            # Some headers get overwritten by urllib3 when set to `None`
            # and should be replaced with the `SKIP_HEADER` constant.
            value = SKIP_HEADER
        final_headers.add(name, value)
    return final_headers


def transform_headers(
    request: requests.Request,
    prepared_request: requests.PreparedRequest
) -> None:
    """Apply various transformations on top of the `prepared_requests`'s
    headers to change the request prepreation behavior."""

    # Remove 'Content-Length' when it is misplaced by requests.
    if (
        prepared_request.method in IGNORE_CONTENT_LENGTH_METHODS
        and prepared_request.headers.get('Content-Length') == '0'
        and request.headers.get('Content-Length') != '0'
    ):
        prepared_request.headers.pop('Content-Length')

    apply_missing_repeated_headers(
        request.headers,
        prepared_request
    )


def apply_missing_repeated_headers(
    original_headers: HTTPHeadersDict,
    prepared_request: requests.PreparedRequest
) -> None:
    """Update the given `prepared_request`'s headers with the original
    ones. This allows the requests to be prepared as usual, and then later
    merged with headers that are specified multiple times."""

    new_headers = HTTPHeadersDict(prepared_request.headers)
    for prepared_name, prepared_value in prepared_request.headers.items():
        if prepared_name not in original_headers:
            continue

        original_keys, original_values = zip(*filter(
            lambda item: item[0].casefold() == prepared_name.casefold(),
            original_headers.items()
        ))

        if prepared_value not in original_values:
            # If the current value is not among the initial values
            # set for this field, then it means that this field got
            # overridden on the way, and we should preserve it.
            continue

        new_headers.popone(prepared_name)
        new_headers.update(zip(original_keys, original_values))

    prepared_request.headers = new_headers


def make_default_headers(args: argparse.Namespace) -> HTTPHeadersDict:
    default_headers = HTTPHeadersDict({
        'User-Agent': DEFAULT_UA
    })

    auto_json = args.data and not args.form
    if args.json or auto_json:
        default_headers['Accept'] = JSON_ACCEPT
        if args.json or (auto_json and args.data):
            default_headers['Content-Type'] = JSON_CONTENT_TYPE

    elif args.form and not args.files:
        # If sending files, `requests` will set
        # the `Content-Type` for us.
        default_headers['Content-Type'] = FORM_CONTENT_TYPE
    return default_headers


def make_send_kwargs(args: argparse.Namespace) -> dict:
    return {
        'timeout': args.timeout or None,
        'allow_redirects': False,
    }


def make_send_kwargs_mergeable_from_env(args: argparse.Namespace) -> dict:
    cert = None
    if args.cert:
        cert = args.cert
        if args.cert_key:
            # Having a client certificate key passphrase is not supported
            # by requests. So we are using our own transportation structure
            # which is compatible with their format (a tuple of minimum two
            # items).
            #
            # See: https://github.com/psf/requests/issues/2519
            cert = HTTPieCertificate(cert, args.cert_key, args.cert_key_pass.value)

    return {
        'proxies': {p.key: p.value for p in args.proxy},
        'stream': True,
        'verify': {
            'yes': True,
            'true': True,
            'no': False,
            'false': False,
        }.get(args.verify.lower(), args.verify),
        'cert': cert,
    }


def json_dict_to_request_body(data: Dict[str, Any]) -> str:
    data = unwrap_top_level_list_if_needed(data)
    if data:
        data = json.dumps(data)
    else:
        # We need to set data to an empty string to prevent requests
        # from assigning an empty list to `response.request.data`.
        data = ''
    return data


def make_request_kwargs(
    env: Environment,
    args: argparse.Namespace,
    base_headers: HTTPHeadersDict = None,
    request_body_read_callback=lambda chunk: chunk
) -> dict:
    """
    Translate our `args` into `requests.Request` keyword arguments.

    """
    files = args.files
    # Serialize JSON data, if needed.
    data = args.data
    auto_json = data and not args.form
    if (args.json or auto_json) and isinstance(data, dict):
        data = json_dict_to_request_body(data)

    # Finalize headers.
    headers = make_default_headers(args)
    if base_headers:
        headers.update(base_headers)
    headers.update(args.headers)
    if args.offline and args.chunked and 'Transfer-Encoding' not in headers:
        # When online, we let requests set the header instead to be able more
        # easily verify chunking is taking place.
        headers['Transfer-Encoding'] = 'chunked'
    headers = finalize_headers(headers)

    if (args.form and files) or args.multipart:
        data, headers['Content-Type'] = get_multipart_data_and_content_type(
            data=args.multipart_data,
            boundary=args.boundary,
            content_type=args.headers.get('Content-Type'),
        )

    return {
        'method': args.method.lower(),
        'url': args.url,
        'headers': headers,
        'data': prepare_request_body(
            env,
            data,
            body_read_callback=request_body_read_callback,
            chunked=args.chunked,
            offline=args.offline,
            content_length_header_value=headers.get('Content-Length'),
        ),
        'auth': args.auth,
        'params': args.params.items(),
    }


def ensure_path_as_is(orig_url: str, prepped_url: str) -> str:
    """
    Handle `--path-as-is` by replacing the path component of the prepared
    URL with the path component from the original URL. Other parts stay
    untouched because other (welcome) processing on the URL might have
    taken place.

    <https://github.com/httpie/cli/issues/895>


    <https://ec.haxx.se/http/http-basics#path-as-is>
    <https://curl.haxx.se/libcurl/c/CURLOPT_PATH_AS_IS.html>

    >>> ensure_path_as_is('http://foo/../', 'http://foo/?foo=bar')
    'http://foo/../?foo=bar'

    """
    parsed_orig, parsed_prepped = urlparse(orig_url), urlparse(prepped_url)
    final_dict = {
        # noinspection PyProtectedMember
        **parsed_prepped._asdict(),
        'path': parsed_orig.path,
    }
    return urlunparse(tuple(final_dict.values()))



File: httpie/__init__.py

"""
HTTPie: modern, user-friendly command-line HTTP client for the API era.

"""

__version__ = '3.2.3'
__date__ = '2024-07-10'
__author__ = 'Jakub Roztocil'
__licence__ = 'BSD'



File: httpie/core.py

import argparse
import os
import platform
import sys
import socket
from typing import List, Optional, Union, Callable

import requests
from pygments import __version__ as pygments_version
from requests import __version__ as requests_version

from . import __version__ as httpie_version
from .cli.constants import OUT_REQ_BODY
from .cli.nested_json import NestedJSONSyntaxError
from .client import collect_messages
from .context import Environment, LogLevel
from .downloads import Downloader
from .models import (
    RequestsMessageKind,
    OutputOptions
)
from .output.models import ProcessingOptions
from .output.writer import write_message, write_stream, write_raw_data, MESSAGE_SEPARATOR_BYTES
from .plugins.registry import plugin_manager
from .status import ExitStatus, http_status_to_exit_status
from .utils import unwrap_context
from .internal.update_warnings import check_updates
from .internal.daemon_runner import is_daemon_mode, run_daemon_task


# noinspection PyDefaultArgument
def raw_main(
    parser: argparse.ArgumentParser,
    main_program: Callable[[argparse.Namespace, Environment], ExitStatus],
    args: List[Union[str, bytes]] = sys.argv,
    env: Environment = Environment(),
    use_default_options: bool = True,
) -> ExitStatus:
    program_name, *args = args
    env.program_name = os.path.basename(program_name)
    args = decode_raw_args(args, env.stdin_encoding)

    if is_daemon_mode(args):
        return run_daemon_task(env, args)

    plugin_manager.load_installed_plugins(env.config.plugins_dir)

    if use_default_options and env.config.default_options:
        args = env.config.default_options + args

    include_debug_info = '--debug' in args
    include_traceback = include_debug_info or '--traceback' in args

    def handle_generic_error(e, annotation=None):
        msg = str(e)
        if hasattr(e, 'request'):
            request = e.request
            if hasattr(request, 'url'):
                msg = (
                    f'{msg} while doing a {request.method}'
                    f' request to URL: {request.url}'
                )
        if annotation:
            msg += annotation
        env.log_error(f'{type(e).__name__}: {msg}')
        if include_traceback:
            raise

    if include_debug_info:
        print_debug_info(env)
        if args == ['--debug']:
            return ExitStatus.SUCCESS

    exit_status = ExitStatus.SUCCESS

    try:
        parsed_args = parser.parse_args(
            args=args,
            env=env,
        )
    except NestedJSONSyntaxError as exc:
        env.stderr.write(str(exc) + "\n")
        if include_traceback:
            raise
        exit_status = ExitStatus.ERROR
    except KeyboardInterrupt:
        env.stderr.write('\n')
        if include_traceback:
            raise
        exit_status = ExitStatus.ERROR_CTRL_C
    except SystemExit as e:
        if e.code != ExitStatus.SUCCESS:
            env.stderr.write('\n')
            if include_traceback:
                raise
            exit_status = ExitStatus.ERROR
    else:
        check_updates(env)
        try:
            exit_status = main_program(
                args=parsed_args,
                env=env,
            )
        except KeyboardInterrupt:
            env.stderr.write('\n')
            if include_traceback:
                raise
            exit_status = ExitStatus.ERROR_CTRL_C
        except SystemExit as e:
            if e.code != ExitStatus.SUCCESS:
                env.stderr.write('\n')
                if include_traceback:
                    raise
                exit_status = ExitStatus.ERROR
        except requests.Timeout:
            exit_status = ExitStatus.ERROR_TIMEOUT
            env.log_error(f'Request timed out ({parsed_args.timeout}s).')
        except requests.TooManyRedirects:
            exit_status = ExitStatus.ERROR_TOO_MANY_REDIRECTS
            env.log_error(
                f'Too many redirects'
                f' (--max-redirects={parsed_args.max_redirects}).'
            )
        except requests.exceptions.ConnectionError as exc:
            annotation = None
            original_exc = unwrap_context(exc)
            if isinstance(original_exc, socket.gaierror):
                if original_exc.errno == socket.EAI_AGAIN:
                    annotation = '\nCouldn’t connect to a DNS server. Please check your connection and try again.'
                elif original_exc.errno == socket.EAI_NONAME:
                    annotation = '\nCouldn’t resolve the given hostname. Please check the URL and try again.'
                propagated_exc = original_exc
            else:
                propagated_exc = exc

            handle_generic_error(propagated_exc, annotation=annotation)
            exit_status = ExitStatus.ERROR
        except Exception as e:
            # TODO: Further distinction between expected and unexpected errors.
            handle_generic_error(e)
            exit_status = ExitStatus.ERROR

    return exit_status


def main(
    args: List[Union[str, bytes]] = sys.argv,
    env: Environment = Environment()
) -> ExitStatus:
    """
    The main function.

    Pre-process args, handle some special types of invocations,
    and run the main program with error handling.

    Return exit status code.

    """

    from .cli.definition import parser

    return raw_main(
        parser=parser,
        main_program=program,
        args=args,
        env=env
    )


def program(args: argparse.Namespace, env: Environment) -> ExitStatus:
    """
    The main program without error handling.

    """
    # TODO: Refactor and drastically simplify, especially so that the separator logic is elsewhere.
    exit_status = ExitStatus.SUCCESS
    downloader = None
    initial_request: Optional[requests.PreparedRequest] = None
    final_response: Optional[requests.Response] = None
    processing_options = ProcessingOptions.from_raw_args(args)

    def separate():
        getattr(env.stdout, 'buffer', env.stdout).write(MESSAGE_SEPARATOR_BYTES)

    def request_body_read_callback(chunk: bytes):
        should_pipe_to_stdout = bool(
            # Request body output desired
            OUT_REQ_BODY in args.output_options
            # & not `.read()` already pre-request (e.g., for  compression)
            and initial_request
            # & non-EOF chunk
            and chunk
        )
        if should_pipe_to_stdout:
            return write_raw_data(
                env,
                chunk,
                processing_options=processing_options,
                headers=initial_request.headers
            )

    try:
        if args.download:
            args.follow = True  # --download implies --follow.
            downloader = Downloader(env, output_file=args.output_file, resume=args.download_resume)
            downloader.pre_request(args.headers)
        messages = collect_messages(env, args=args,
                                    request_body_read_callback=request_body_read_callback)
        force_separator = False
        prev_with_body = False

        # Process messages as they’re generated
        for message in messages:
            output_options = OutputOptions.from_message(message, args.output_options)

            do_write_body = output_options.body
            if prev_with_body and output_options.any() and (force_separator or not env.stdout_isatty):
                # Separate after a previous message with body, if needed. See test_tokens.py.
                separate()
            force_separator = False
            if output_options.kind is RequestsMessageKind.REQUEST:
                if not initial_request:
                    initial_request = message
                if output_options.body:
                    is_streamed_upload = not isinstance(message.body, (str, bytes))
                    do_write_body = not is_streamed_upload
                    force_separator = is_streamed_upload and env.stdout_isatty
            else:
                final_response = message
                if args.check_status or downloader:
                    exit_status = http_status_to_exit_status(http_status=message.status_code, follow=args.follow)
                    if exit_status != ExitStatus.SUCCESS and (not env.stdout_isatty or args.quiet == 1):
                        env.log_error(f'HTTP {message.raw.status} {message.raw.reason}', level=LogLevel.WARNING)
            write_message(
                requests_message=message,
                env=env,
                output_options=output_options._replace(
                    body=do_write_body
                ),
                processing_options=processing_options
            )
            prev_with_body = output_options.body

        # Cleanup
        if force_separator:
            separate()
        if downloader and exit_status == ExitStatus.SUCCESS:
            # Last response body download.
            download_stream, download_to = downloader.start(
                initial_url=initial_request.url,
                final_response=final_response,
            )
            write_stream(stream=download_stream, outfile=download_to, flush=False)
            downloader.finish()
            if downloader.interrupted:
                exit_status = ExitStatus.ERROR
                env.log_error(
                    f'Incomplete download: size={downloader.status.total_size};'
                    f' downloaded={downloader.status.downloaded}'
                )
        return exit_status

    finally:
        if downloader and not downloader.finished:
            downloader.failed()
        if args.output_file and args.output_file_specified:
            args.output_file.close()


def print_debug_info(env: Environment):
    env.stderr.writelines([
        f'HTTPie {httpie_version}\n',
        f'Requests {requests_version}\n',
        f'Pygments {pygments_version}\n',
        f'Python {sys.version}\n{sys.executable}\n',
        f'{platform.system()} {platform.release()}',
    ])
    env.stderr.write('\n\n')
    env.stderr.write(repr(env))
    env.stderr.write('\n\n')
    env.stderr.write(repr(plugin_manager))
    env.stderr.write('\n')


def decode_raw_args(
    args: List[Union[str, bytes]],
    stdin_encoding: str
) -> List[str]:
    """
    Convert all bytes args to str
    by decoding them using stdin encoding.

    """
    return [
        arg.decode(stdin_encoding)
        if type(arg) is bytes else arg
        for arg in args
    ]



File: httpie/ssl_.py

import ssl
from typing import NamedTuple, Optional

from httpie.adapters import HTTPAdapter
# noinspection PyPackageRequirements
from urllib3.util.ssl_ import (
    create_urllib3_context,
    resolve_ssl_version,
)


SSL_VERSION_ARG_MAPPING = {
    'ssl2.3': 'PROTOCOL_SSLv23',
    'ssl3': 'PROTOCOL_SSLv3',
    'tls1': 'PROTOCOL_TLSv1',
    'tls1.1': 'PROTOCOL_TLSv1_1',
    'tls1.2': 'PROTOCOL_TLSv1_2',
    'tls1.3': 'PROTOCOL_TLSv1_3',
}
AVAILABLE_SSL_VERSION_ARG_MAPPING = {
    arg: getattr(ssl, constant_name)
    for arg, constant_name in SSL_VERSION_ARG_MAPPING.items()
    if hasattr(ssl, constant_name)
}


class HTTPieCertificate(NamedTuple):
    cert_file: Optional[str] = None
    key_file: Optional[str] = None
    key_password: Optional[str] = None

    def to_raw_cert(self):
        """Synthesize a requests-compatible (2-item tuple of cert and key file)
        object from HTTPie's internal representation of a certificate."""
        return (self.cert_file, self.key_file)


class HTTPieHTTPSAdapter(HTTPAdapter):
    def __init__(
        self,
        verify: bool,
        ssl_version: str = None,
        ciphers: str = None,
        **kwargs
    ):
        self._ssl_context = self._create_ssl_context(
            verify=verify,
            ssl_version=ssl_version,
            ciphers=ciphers,
        )
        super().__init__(**kwargs)

    def init_poolmanager(self, *args, **kwargs):
        kwargs['ssl_context'] = self._ssl_context
        return super().init_poolmanager(*args, **kwargs)

    def proxy_manager_for(self, *args, **kwargs):
        kwargs['ssl_context'] = self._ssl_context
        return super().proxy_manager_for(*args, **kwargs)

    def cert_verify(self, conn, url, verify, cert):
        if isinstance(cert, HTTPieCertificate):
            conn.key_password = cert.key_password
            cert = cert.to_raw_cert()

        return super().cert_verify(conn, url, verify, cert)

    @staticmethod
    def _create_ssl_context(
        verify: bool,
        ssl_version: str = None,
        ciphers: str = None,
    ) -> 'ssl.SSLContext':
        return create_urllib3_context(
            ciphers=ciphers,
            ssl_version=resolve_ssl_version(ssl_version),
            # Since we are using a custom SSL context, we need to pass this
            # here manually, even though it’s also passed to the connection
            # in `super().cert_verify()`.
            cert_reqs=ssl.CERT_REQUIRED if verify else ssl.CERT_NONE
        )

    @classmethod
    def get_default_ciphers_names(cls):
        return [cipher['name'] for cipher in cls._create_ssl_context(verify=False).get_ciphers()]


def _is_key_file_encrypted(key_file):
    """Detects if a key file is encrypted or not.

    Copy of the internal urllib function (urllib3.util.ssl_)"""

    with open(key_file, "r") as f:
        for line in f:
            # Look for Proc-Type: 4,ENCRYPTED
            if "ENCRYPTED" in line:
                return True

    return False


# We used to import the default set of TLS ciphers from urllib3, but they removed it.
# Instead, now urllib3 uses the list of ciphers configured by the system.
# <https://github.com/httpie/cli/pull/1501>
DEFAULT_SSL_CIPHERS_STRING = ':'.join(HTTPieHTTPSAdapter.get_default_ciphers_names())



File: httpie/downloads.py

"""
Download mode implementation.

"""
import mimetypes
import os
import re
from mailbox import Message
from time import monotonic
from typing import IO, Optional, Tuple
from urllib.parse import urlsplit

import requests

from .models import HTTPResponse, OutputOptions
from .output.streams import RawStream
from .context import Environment


PARTIAL_CONTENT = 206


class ContentRangeError(ValueError):
    pass


def parse_content_range(content_range: str, resumed_from: int) -> int:
    """
    Parse and validate Content-Range header.

    <https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html>

    :param content_range: the value of a Content-Range response header
                          eg. "bytes 21010-47021/47022"
    :param resumed_from: first byte pos. from the Range request header
    :return: total size of the response body when fully downloaded.

    """
    if content_range is None:
        raise ContentRangeError('Missing Content-Range')

    pattern = (
        r'^bytes (?P<first_byte_pos>\d+)-(?P<last_byte_pos>\d+)'
        r'/(\*|(?P<instance_length>\d+))$'
    )
    match = re.match(pattern, content_range)

    if not match:
        raise ContentRangeError(
            f'Invalid Content-Range format {content_range!r}')

    content_range_dict = match.groupdict()
    first_byte_pos = int(content_range_dict['first_byte_pos'])
    last_byte_pos = int(content_range_dict['last_byte_pos'])
    instance_length = (
        int(content_range_dict['instance_length'])
        if content_range_dict['instance_length']
        else None
    )

    # "A byte-content-range-spec with a byte-range-resp-spec whose
    # last- byte-pos value is less than its first-byte-pos value,
    # or whose instance-length value is less than or equal to its
    # last-byte-pos value, is invalid. The recipient of an invalid
    # byte-content-range- spec MUST ignore it and any content
    # transferred along with it."
    if (first_byte_pos > last_byte_pos
        or (instance_length is not None
            and instance_length <= last_byte_pos)):
        raise ContentRangeError(
            f'Invalid Content-Range returned: {content_range!r}')

    if (first_byte_pos != resumed_from
        or (instance_length is not None
            and last_byte_pos + 1 != instance_length)):
        # Not what we asked for.
        raise ContentRangeError(
            f'Unexpected Content-Range returned ({content_range!r})'
            f' for the requested Range ("bytes={resumed_from}-")'
        )

    return last_byte_pos + 1


def filename_from_content_disposition(
    content_disposition: str
) -> Optional[str]:
    """
    Extract and validate filename from a Content-Disposition header.

    :param content_disposition: Content-Disposition value
    :return: the filename if present and valid, otherwise `None`

    """
    # attachment; filename=jakubroztocil-httpie-0.4.1-20-g40bd8f6.tar.gz

    msg = Message(f'Content-Disposition: {content_disposition}')
    filename = msg.get_filename()
    if filename:
        # Basic sanitation.
        filename = os.path.basename(filename).lstrip('.').strip()
        if filename:
            return filename


def filename_from_url(url: str, content_type: Optional[str]) -> str:
    fn = urlsplit(url).path.rstrip('/')
    fn = os.path.basename(fn) if fn else 'index'
    if '.' not in fn and content_type:
        content_type = content_type.split(';')[0]
        if content_type == 'text/plain':
            # mimetypes returns '.ksh'
            ext = '.txt'
        else:
            ext = mimetypes.guess_extension(content_type)

        if ext == '.htm':
            ext = '.html'

        if ext:
            fn += ext

    return fn


def trim_filename(filename: str, max_len: int) -> str:
    if len(filename) > max_len:
        trim_by = len(filename) - max_len
        name, ext = os.path.splitext(filename)
        if trim_by >= len(name):
            filename = filename[:-trim_by]
        else:
            filename = name[:-trim_by] + ext
    return filename


def get_filename_max_length(directory: str) -> int:
    max_len = 255
    if hasattr(os, 'pathconf') and 'PC_NAME_MAX' in os.pathconf_names:
        max_len = os.pathconf(directory, 'PC_NAME_MAX')
    return max_len


def trim_filename_if_needed(filename: str, directory='.', extra=0) -> str:
    max_len = get_filename_max_length(directory) - extra
    if len(filename) > max_len:
        filename = trim_filename(filename, max_len)
    return filename


def get_unique_filename(filename: str, exists=os.path.exists) -> str:
    attempt = 0
    while True:
        suffix = f'-{attempt}' if attempt > 0 else ''
        try_filename = trim_filename_if_needed(filename, extra=len(suffix))
        try_filename += suffix
        if not exists(try_filename):
            return try_filename
        attempt += 1


class Downloader:

    def __init__(
        self,
        env: Environment,
        output_file: IO = None,
        resume: bool = False
    ):
        """
        :param resume: Should the download resume if partial download
                       already exists.

        :param output_file: The file to store response body in. If not
                            provided, it will be guessed from the response.

        :param progress_file: Where to report download progress.

        """
        self.finished = False
        self.status = DownloadStatus(env=env)
        self._output_file = output_file
        self._resume = resume
        self._resumed_from = 0

    def pre_request(self, request_headers: dict):
        """Called just before the HTTP request is sent.

        Might alter `request_headers`.

        """
        # Ask the server not to encode the content so that we can resume, etc.
        request_headers['Accept-Encoding'] = 'identity'
        if self._resume:
            bytes_have = os.path.getsize(self._output_file.name)
            if bytes_have:
                # Set ``Range`` header to resume the download
                # TODO: Use "If-Range: mtime" to make sure it's fresh?
                request_headers['Range'] = f'bytes={bytes_have}-'
                self._resumed_from = bytes_have

    def start(
        self,
        initial_url: str,
        final_response: requests.Response
    ) -> Tuple[RawStream, IO]:
        """
        Initiate and return a stream for `response` body  with progress
        callback attached. Can be called only once.

        :param initial_url: The original requested URL
        :param final_response: Initiated response object with headers already fetched

        :return: RawStream, output_file

        """
        assert not self.status.time_started

        # FIXME: some servers still might sent Content-Encoding: gzip
        # <https://github.com/httpie/cli/issues/423>
        try:
            total_size = int(final_response.headers['Content-Length'])
        except (KeyError, ValueError, TypeError):
            total_size = None

        if not self._output_file:
            self._output_file = self._get_output_file_from_response(
                initial_url=initial_url,
                final_response=final_response,
            )
        else:
            # `--output, -o` provided
            if self._resume and final_response.status_code == PARTIAL_CONTENT:
                total_size = parse_content_range(
                    final_response.headers.get('Content-Range'),
                    self._resumed_from
                )

            else:
                self._resumed_from = 0
                try:
                    self._output_file.seek(0)
                    self._output_file.truncate()
                except OSError:
                    pass  # stdout

        output_options = OutputOptions.from_message(final_response, headers=False, body=True)
        stream = RawStream(
            msg=HTTPResponse(final_response),
            output_options=output_options,
            on_body_chunk_downloaded=self.chunk_downloaded,
        )

        self.status.started(
            output_file=self._output_file,
            resumed_from=self._resumed_from,
            total_size=total_size
        )

        return stream, self._output_file

    def finish(self):
        assert not self.finished
        self.finished = True
        self.status.finished()

    def failed(self):
        self.status.terminate()

    @property
    def interrupted(self) -> bool:
        return (
            self.finished
            and self.status.total_size
            and self.status.total_size != self.status.downloaded
        )

    def chunk_downloaded(self, chunk: bytes):
        """
        A download progress callback.

        :param chunk: A chunk of response body data that has just
                      been downloaded and written to the output.

        """
        self.status.chunk_downloaded(len(chunk))

    @staticmethod
    def _get_output_file_from_response(
        initial_url: str,
        final_response: requests.Response,
    ) -> IO:
        # Output file not specified. Pick a name that doesn't exist yet.
        filename = None
        if 'Content-Disposition' in final_response.headers:
            filename = filename_from_content_disposition(
                final_response.headers['Content-Disposition'])
        if not filename:
            filename = filename_from_url(
                url=initial_url,
                content_type=final_response.headers.get('Content-Type'),
            )
        unique_filename = get_unique_filename(filename)
        return open(unique_filename, buffering=0, mode='a+b')


class DownloadStatus:
    """Holds details about the download status."""

    def __init__(self, env):
        self.env = env
        self.downloaded = 0
        self.total_size = None
        self.resumed_from = 0
        self.time_started = None
        self.time_finished = None

    def started(self, output_file, resumed_from=0, total_size=None):
        assert self.time_started is None
        self.total_size = total_size
        self.downloaded = self.resumed_from = resumed_from
        self.time_started = monotonic()
        self.start_display(output_file=output_file)

    def start_display(self, output_file):
        from httpie.output.ui.rich_progress import (
            DummyDisplay,
            StatusDisplay,
            ProgressDisplay
        )

        message = f'Downloading to {output_file.name}'
        if self.env.show_displays:
            if self.total_size is None:
                # Rich does not support progress bars without a total
                # size given. Instead we use status objects.
                self.display = StatusDisplay(self.env)
            else:
                self.display = ProgressDisplay(self.env)
        else:
            self.display = DummyDisplay(self.env)

        self.display.start(
            total=self.total_size,
            at=self.downloaded,
            description=message
        )

    def chunk_downloaded(self, size):
        assert self.time_finished is None
        self.downloaded += size
        self.display.update(size)

    @property
    def has_finished(self):
        return self.time_finished is not None

    @property
    def time_spent(self):
        if (
            self.time_started is not None
            and self.time_finished is not None
        ):
            return self.time_finished - self.time_started
        else:
            return None

    def finished(self):
        assert self.time_started is not None
        assert self.time_finished is None
        self.time_finished = monotonic()
        if hasattr(self, 'display'):
            self.display.stop(self.time_spent)

    def terminate(self):
        if hasattr(self, 'display'):
            self.display.stop(self.time_spent)



File: httpie/context.py

import argparse
import sys
import os
import warnings
from contextlib import contextmanager
from pathlib import Path
from typing import Iterator, IO, Optional, TYPE_CHECKING
from enum import Enum


try:
    import curses
except ImportError:
    curses = None  # Compiled w/o curses

from .compat import is_windows, cached_property
from .config import DEFAULT_CONFIG_DIR, Config, ConfigFileError
from .encoding import UTF8

from .utils import repr_dict
from .output.ui.palette import GenericColor

if TYPE_CHECKING:
    from rich.console import Console


class LogLevel(str, Enum):
    INFO = 'info'
    WARNING = 'warning'
    ERROR = 'error'


LOG_LEVEL_COLORS = {
    LogLevel.INFO: GenericColor.PINK,
    LogLevel.WARNING: GenericColor.ORANGE,
    LogLevel.ERROR: GenericColor.RED,
}

LOG_LEVEL_DISPLAY_THRESHOLDS = {
    LogLevel.INFO: 1,
    LogLevel.WARNING: 2,
    LogLevel.ERROR: float('inf'),  # Never hide errors.
}


class Environment:
    """
    Information about the execution context
    (standard streams, config directory, etc).

    By default, it represents the actual environment.
    All of the attributes can be overwritten though, which
    is used by the test suite to simulate various scenarios.

    """
    args = argparse.Namespace()
    is_windows: bool = is_windows
    config_dir: Path = DEFAULT_CONFIG_DIR
    stdin: Optional[IO] = sys.stdin  # `None` when closed fd (#791)
    stdin_isatty: bool = stdin.isatty() if stdin else False
    stdin_encoding: str = None
    stdout: IO = sys.stdout
    stdout_isatty: bool = stdout.isatty()
    stdout_encoding: str = None
    stderr: IO = sys.stderr
    stderr_isatty: bool = stderr.isatty()
    colors = 256
    program_name: str = 'http'

    # Whether to show progress bars / status spinners etc.
    show_displays: bool = True

    if not is_windows:
        if curses:
            try:
                curses.setupterm()
                colors = curses.tigetnum('colors')
            except curses.error:
                pass
    else:
        # noinspection PyUnresolvedReferences
        import colorama.initialise
        stdout = colorama.initialise.wrap_stream(
            stdout, convert=None, strip=None,
            autoreset=True, wrap=True
        )
        stderr = colorama.initialise.wrap_stream(
            stderr, convert=None, strip=None,
            autoreset=True, wrap=True
        )
        del colorama

    def __init__(self, devnull=None, **kwargs):
        """
        Use keyword arguments to overwrite
        any of the class attributes for this instance.

        """
        assert all(hasattr(type(self), attr) for attr in kwargs.keys())
        self.__dict__.update(**kwargs)

        # The original STDERR unaffected by --quiet’ing.
        self._orig_stderr = self.stderr
        self._devnull = devnull

        # Keyword arguments > stream.encoding > default UTF-8
        if self.stdin and self.stdin_encoding is None:
            self.stdin_encoding = getattr(
                self.stdin, 'encoding', None) or UTF8
        if self.stdout_encoding is None:
            actual_stdout = self.stdout
            if is_windows:
                # noinspection PyUnresolvedReferences
                from colorama import AnsiToWin32
                if isinstance(self.stdout, AnsiToWin32):
                    # noinspection PyUnresolvedReferences
                    actual_stdout = self.stdout.wrapped
            self.stdout_encoding = getattr(
                actual_stdout, 'encoding', None) or UTF8

        self.quiet = kwargs.pop('quiet', 0)

    def __str__(self):
        defaults = dict(type(self).__dict__)
        actual = dict(defaults)
        actual.update(self.__dict__)
        actual['config'] = self.config
        return repr_dict({
            key: value
            for key, value in actual.items()
            if not key.startswith('_')
        })

    def __repr__(self):
        return f'<{type(self).__name__} {self}>'

    _config: Config = None

    @property
    def config(self) -> Config:
        config = self._config
        if not config:
            self._config = config = Config(directory=self.config_dir)
            if not config.is_new():
                try:
                    config.load()
                except ConfigFileError as e:
                    self.log_error(e, level=LogLevel.WARNING)
        return config

    @property
    def devnull(self) -> IO:
        if self._devnull is None:
            self._devnull = open(os.devnull, 'w+')
        return self._devnull

    @contextmanager
    def as_silent(self) -> Iterator[None]:
        original_stdout = self.stdout
        original_stderr = self.stderr

        try:
            self.stdout = self.devnull
            self.stderr = self.devnull
            yield
        finally:
            self.stdout = original_stdout
            self.stderr = original_stderr

    def log_error(self, msg: str, level: LogLevel = LogLevel.ERROR) -> None:
        if self.stdout_isatty and self.quiet >= LOG_LEVEL_DISPLAY_THRESHOLDS[level]:
            stderr = self.stderr  # Not directly /dev/null, since stderr might be mocked
        else:
            stderr = self._orig_stderr
        rich_console = self._make_rich_console(file=stderr, force_terminal=stderr.isatty())
        rich_console.print(
            f'\n{self.program_name}: {level.value}: {msg}\n\n',
            style=LOG_LEVEL_COLORS[level],
            markup=False,
            highlight=False,
            soft_wrap=True
        )

    def apply_warnings_filter(self) -> None:
        if self.quiet >= LOG_LEVEL_DISPLAY_THRESHOLDS[LogLevel.WARNING]:
            warnings.simplefilter("ignore")

    def _make_rich_console(
        self,
        file: IO[str],
        force_terminal: bool
    ) -> 'Console':
        from rich.console import Console
        from httpie.output.ui.rich_palette import _make_rich_color_theme

        style = getattr(self.args, 'style', None)
        theme = _make_rich_color_theme(style)
        # Rich infers the rest of the knowledge (e.g encoding)
        # dynamically by looking at the file/stderr.
        return Console(
            file=file,
            force_terminal=force_terminal,
            no_color=(self.colors == 0),
            theme=theme
        )

    # Rich recommends separating the actual console (stdout) from
    # the error (stderr) console for better isolation between parts.
    # https://rich.readthedocs.io/en/stable/console.html#error-console

    @cached_property
    def rich_console(self):
        return self._make_rich_console(self.stdout, self.stdout_isatty)

    @cached_property
    def rich_error_console(self):
        return self._make_rich_console(self.stderr, self.stderr_isatty)



File: httpie/utils.py

import os
import base64
import json
import mimetypes
import re
import sys
import time
import tempfile
import sysconfig

from collections import OrderedDict
from contextlib import contextmanager
from http.cookiejar import parse_ns_headers
from pathlib import Path
from pprint import pformat
from urllib.parse import urlsplit
from typing import Any, List, Optional, Tuple, Generator, Callable, Iterable, IO, TypeVar

import requests.auth

RE_COOKIE_SPLIT = re.compile(r', (?=[^ ;]+=)')
Item = Tuple[str, Any]
Items = List[Item]
T = TypeVar("T")


class JsonDictPreservingDuplicateKeys(OrderedDict):
    """A specialized JSON dict preserving duplicate keys."""

    # Python versions prior to 3.8 suffer from an issue with multiple keys with the same name.
    # `json.dumps(obj, indent=N, sort_keys=True)` will output sorted keys when they are unique, and
    # duplicate keys will be outputted as they were defined in the original data.
    # See <https://bugs.python.org/issue23493#msg400929> for the behavior change between Python versions.
    SUPPORTS_SORTING = sys.version_info >= (3, 8)

    def __init__(self, items: Items):
        self._items = items
        self._ensure_items_used()

    def _ensure_items_used(self) -> None:
        """HACK: Force `json.dumps()` to use `self.items()` instead of an empty dict.

        Two JSON encoders are available on CPython: pure-Python (1) and C (2) implementations.

        (1) The pure-python implementation will do a simple `if not dict: return '{}'`,
        and we could fake that check by implementing the `__bool__()` method.
        Source:
            - <https://github.com/python/cpython/blob/9d318ad/Lib/json/encoder.py#L334-L336>

        (2) On the other hand, the C implementation will do a check on the number of
        items contained inside the dict, using a verification on `dict->ma_used`, which
        is updated only when an item is added/removed from the dict. For that case,
        there is no workaround but to add an item into the dict.
        Sources:
            - <https://github.com/python/cpython/blob/9d318ad/Modules/_json.c#L1581-L1582>
            - <https://github.com/python/cpython/blob/9d318ad/Include/cpython/dictobject.h#L53>
            - <https://github.com/python/cpython/blob/9d318ad/Include/cpython/dictobject.h#L17-L18>

        To please both implementations, we simply add one item to the dict.

        """
        if self._items:
            self['__hack__'] = '__hack__'

    def items(self) -> Items:
        """Return all items, duplicate ones included.

        """
        return self._items


def load_json_preserve_order_and_dupe_keys(s):
    return json.loads(s, object_pairs_hook=JsonDictPreservingDuplicateKeys)


def repr_dict(d: dict) -> str:
    return pformat(d)


def humanize_bytes(n, precision=2):
    # Author: Doug Latornell
    # Licence: MIT
    # URL: https://code.activestate.com/recipes/577081/
    """Return a humanized string representation of a number of bytes.

    >>> humanize_bytes(1)
    '1 B'
    >>> humanize_bytes(1024, precision=1)
    '1.0 kB'
    >>> humanize_bytes(1024 * 123, precision=1)
    '123.0 kB'
    >>> humanize_bytes(1024 * 12342, precision=1)
    '12.1 MB'
    >>> humanize_bytes(1024 * 12342, precision=2)
    '12.05 MB'
    >>> humanize_bytes(1024 * 1234, precision=2)
    '1.21 MB'
    >>> humanize_bytes(1024 * 1234 * 1111, precision=2)
    '1.31 GB'
    >>> humanize_bytes(1024 * 1234 * 1111, precision=1)
    '1.3 GB'

    """
    abbrevs = [
        (1 << 50, 'PB'),
        (1 << 40, 'TB'),
        (1 << 30, 'GB'),
        (1 << 20, 'MB'),
        (1 << 10, 'kB'),
        (1, 'B')
    ]

    if n == 1:
        return '1 B'

    for factor, suffix in abbrevs:
        if n >= factor:
            break

    # noinspection PyUnboundLocalVariable
    return f'{n / factor:.{precision}f} {suffix}'


class ExplicitNullAuth(requests.auth.AuthBase):
    """Forces requests to ignore the ``.netrc``.
    <https://github.com/psf/requests/issues/2773#issuecomment-174312831>
    """

    def __call__(self, r):
        return r


def get_content_type(filename):
    """
    Return the content type for ``filename`` in format appropriate
    for Content-Type headers, or ``None`` if the file type is unknown
    to ``mimetypes``.

    """
    return mimetypes.guess_type(filename, strict=False)[0]


def split_cookies(cookies):
    """
    When ``requests`` stores cookies in ``response.headers['Set-Cookie']``
    it concatenates all of them through ``, ``.

    This function splits cookies apart being careful to not to
    split on ``, `` which may be part of cookie value.
    """
    if not cookies:
        return []
    return RE_COOKIE_SPLIT.split(cookies)


def get_expired_cookies(
    cookies: str,
    now: float = None
) -> List[dict]:

    now = now or time.time()

    def is_expired(expires: Optional[float]) -> bool:
        return expires is not None and expires <= now

    attr_sets: List[Tuple[str, str]] = parse_ns_headers(
        split_cookies(cookies)
    )

    cookies = [
        # The first attr name is the cookie name.
        dict(attrs[1:], name=attrs[0][0])
        for attrs in attr_sets
    ]

    _max_age_to_expires(cookies=cookies, now=now)

    return [
        {
            'name': cookie['name'],
            'path': cookie.get('path', '/')
        }
        for cookie in cookies
        if is_expired(expires=cookie.get('expires'))
    ]


def _max_age_to_expires(cookies, now):
    """
    Translate `max-age` into `expires` for Requests to take it into account.

    HACK/FIXME: <https://github.com/psf/requests/issues/5743>

    """
    for cookie in cookies:
        if 'expires' in cookie:
            continue
        max_age = cookie.get('max-age')
        if max_age and max_age.isdigit():
            cookie['expires'] = now + float(max_age)


def parse_content_type_header(header):
    """Borrowed from requests."""
    tokens = header.split(';')
    content_type, params = tokens[0].strip(), tokens[1:]
    params_dict = {}
    items_to_strip = "\"' "
    for param in params:
        param = param.strip()
        if param:
            key, value = param, True
            index_of_equals = param.find("=")
            if index_of_equals != -1:
                key = param[:index_of_equals].strip(items_to_strip)
                value = param[index_of_equals + 1:].strip(items_to_strip)
            params_dict[key.lower()] = value
    return content_type, params_dict


def as_site(path: Path, **extra_vars) -> Path:
    site_packages_path = sysconfig.get_path(
        'purelib',
        vars={'base': str(path), **extra_vars}
    )
    return Path(site_packages_path)


def get_site_paths(path: Path) -> Iterable[Path]:
    from httpie.compat import (
        MIN_SUPPORTED_PY_VERSION,
        MAX_SUPPORTED_PY_VERSION,
        is_frozen
    )

    if is_frozen:
        [major, min_minor] = MIN_SUPPORTED_PY_VERSION
        [major, max_minor] = MAX_SUPPORTED_PY_VERSION
        for minor in range(min_minor, max_minor + 1):
            yield as_site(
                path,
                py_version_short=f'{major}.{minor}'
            )
    else:
        yield as_site(path)


def split_iterable(iterable: Iterable[T], key: Callable[[T], bool]) -> Tuple[List[T], List[T]]:
    left, right = [], []
    for item in iterable:
        if key(item):
            left.append(item)
        else:
            right.append(item)
    return left, right


def unwrap_context(exc: Exception) -> Optional[Exception]:
    context = exc.__context__
    if isinstance(context, Exception):
        return unwrap_context(context)
    else:
        return exc


def url_as_host(url: str) -> str:
    return urlsplit(url).netloc.split('@')[-1]


class LockFileError(ValueError):
    pass


@contextmanager
def open_with_lockfile(file: Path, *args, **kwargs) -> Generator[IO[Any], None, None]:
    file_id = base64.b64encode(os.fsencode(file)).decode()
    target_file = Path(tempfile.gettempdir()) / file_id

    # Have an atomic-like touch here, so we'll tighten the possibility of
    # a race occurring between multiple processes accessing the same file.
    try:
        target_file.touch(exist_ok=False)
    except FileExistsError as exc:
        raise LockFileError("Can't modify a locked file.") from exc

    try:
        with open(file, *args, **kwargs) as stream:
            yield stream
    finally:
        target_file.unlink()


def is_version_greater(version_1: str, version_2: str) -> bool:
    # In an ideal scenario, we would depend on `packaging` in order
    # to offer PEP 440 compatible parsing. But since it might not be
    # commonly available for outside packages, and since we are only
    # going to parse HTTPie's own version it should be fine to compare
    # this in a SemVer subset fashion.

    def split_version(version: str) -> Tuple[int, ...]:
        parts = []
        for part in version.split('.')[:3]:
            try:
                parts.append(int(part))
            except ValueError:
                break
        return tuple(parts)

    return split_version(version_1) > split_version(version_2)



File: httpie/uploads.py

import sys
import os
import zlib
import functools
import threading
from typing import Any, Callable, IO, Iterable, Optional, Tuple, Union, TYPE_CHECKING
from urllib.parse import urlencode

import requests
from requests.utils import super_len

if TYPE_CHECKING:
    from requests_toolbelt import MultipartEncoder

from .context import Environment
from .cli.dicts import MultipartRequestDataDict, RequestDataDict
from .compat import is_windows


class ChunkedStream:
    def __iter__(self) -> Iterable[Union[str, bytes]]:
        raise NotImplementedError


class ChunkedUploadStream(ChunkedStream):
    def __init__(
        self,
        stream: Iterable,
        callback: Callable,
        event: Optional[threading.Event] = None
    ) -> None:
        self.callback = callback
        self.stream = stream
        self.event = event

    def __iter__(self) -> Iterable[Union[str, bytes]]:
        for chunk in self.stream:
            if self.event:
                self.event.set()
            self.callback(chunk)
            yield chunk


class ChunkedMultipartUploadStream(ChunkedStream):
    chunk_size = 100 * 1024

    def __init__(
        self,
        encoder: 'MultipartEncoder',
        event: Optional[threading.Event] = None
    ) -> None:
        self.encoder = encoder
        self.event = event

    def __iter__(self) -> Iterable[Union[str, bytes]]:
        while True:
            chunk = self.encoder.read(self.chunk_size)
            if self.event:
                self.event.set()
            if not chunk:
                break
            yield chunk


def as_bytes(data: Union[str, bytes]) -> bytes:
    if isinstance(data, str):
        return data.encode()
    else:
        return data


CallbackT = Callable[[bytes], bytes]


def _wrap_function_with_callback(
    func: Callable[..., Any],
    callback: CallbackT
) -> Callable[..., Any]:
    @functools.wraps(func)
    def wrapped(*args, **kwargs):
        chunk = func(*args, **kwargs)
        callback(chunk)
        return chunk
    return wrapped


def is_stdin(file: IO) -> bool:
    try:
        file_no = file.fileno()
    except Exception:
        return False
    else:
        return file_no == sys.stdin.fileno()


READ_THRESHOLD = float(os.getenv('HTTPIE_STDIN_READ_WARN_THRESHOLD', 10.0))


def observe_stdin_for_data_thread(env: Environment, file: IO, read_event: threading.Event) -> None:
    # Windows unfortunately does not support select() operation
    # on regular files, like stdin in our use case.
    # https://docs.python.org/3/library/select.html#select.select
    if is_windows:
        return None

    # If the user configures READ_THRESHOLD to be 0, then
    # disable this warning.
    if READ_THRESHOLD == 0:
        return None

    def worker(event: threading.Event) -> None:
        if not event.wait(timeout=READ_THRESHOLD):
            env.stderr.write(
                f'> warning: no stdin data read in {READ_THRESHOLD}s '
                f'(perhaps you want to --ignore-stdin)\n'
                f'> See: https://httpie.io/docs/cli/best-practices\n'
            )

    # Making it a daemon ensures that if the user exits from the main program
    # (e.g. either regularly or with Ctrl-C), the thread will not
    # block them.
    thread = threading.Thread(
        target=worker,
        args=(read_event,),
        daemon=True
    )
    thread.start()


def _read_file_with_selectors(file: IO, read_event: threading.Event) -> bytes:
    if is_windows or not is_stdin(file):
        return as_bytes(file.read())

    import select

    # Try checking whether there is any incoming data for READ_THRESHOLD
    # seconds. If there isn't anything in the given period, issue
    # a warning about a misusage.
    read_selectors, _, _ = select.select([file], [], [], READ_THRESHOLD)
    if read_selectors:
        read_event.set()

    return as_bytes(file.read())


def _prepare_file_for_upload(
    env: Environment,
    file: Union[IO, 'MultipartEncoder'],
    callback: CallbackT,
    chunked: bool = False,
    content_length_header_value: Optional[int] = None,
) -> Union[bytes, IO, ChunkedStream]:
    read_event = threading.Event()
    if not super_len(file):
        if is_stdin(file):
            observe_stdin_for_data_thread(env, file, read_event)

        # Zero-length -> assume stdin.
        if content_length_header_value is None and not chunked:
            # Read the whole stdin to determine `Content-Length`.
            #
            # TODO: Instead of opt-in --chunked, consider making
            #   `Transfer-Encoding: chunked` for STDIN opt-out via
            #   something like --no-chunked.
            #   This would be backwards-incompatible so wait until v3.0.0.
            #
            file = _read_file_with_selectors(file, read_event)
    else:
        file.read = _wrap_function_with_callback(
            file.read,
            callback
        )

    if chunked:
        from requests_toolbelt import MultipartEncoder
        if isinstance(file, MultipartEncoder):
            return ChunkedMultipartUploadStream(
                encoder=file,
                event=read_event,
            )
        else:
            return ChunkedUploadStream(
                stream=file,
                callback=callback,
                event=read_event
            )
    else:
        return file


def prepare_request_body(
    env: Environment,
    raw_body: Union[str, bytes, IO, 'MultipartEncoder', RequestDataDict],
    body_read_callback: CallbackT,
    offline: bool = False,
    chunked: bool = False,
    content_length_header_value: Optional[int] = None,
) -> Union[bytes, IO, 'MultipartEncoder', ChunkedStream]:
    is_file_like = hasattr(raw_body, 'read')
    if isinstance(raw_body, (bytes, str)):
        body = as_bytes(raw_body)
    elif isinstance(raw_body, RequestDataDict):
        body = as_bytes(urlencode(raw_body, doseq=True))
    else:
        body = raw_body

    if offline:
        if is_file_like:
            return as_bytes(raw_body.read())
        else:
            return body

    if is_file_like:
        return _prepare_file_for_upload(
            env,
            body,
            chunked=chunked,
            callback=body_read_callback,
            content_length_header_value=content_length_header_value
        )
    elif chunked:
        return ChunkedUploadStream(
            stream=iter([body]),
            callback=body_read_callback
        )
    else:
        return body


def get_multipart_data_and_content_type(
    data: MultipartRequestDataDict,
    boundary: str = None,
    content_type: str = None,
) -> Tuple['MultipartEncoder', str]:
    from requests_toolbelt import MultipartEncoder

    encoder = MultipartEncoder(
        fields=data.items(),
        boundary=boundary,
    )
    if content_type:
        content_type = content_type.strip()
        if 'boundary=' not in content_type:
            content_type = f'{content_type}; boundary={encoder.boundary_value}'
    else:
        content_type = encoder.content_type

    data = encoder
    return data, content_type


def compress_request(
    request: requests.PreparedRequest,
    always: bool,
):
    deflater = zlib.compressobj()
    if isinstance(request.body, str):
        body_bytes = request.body.encode()
    elif hasattr(request.body, 'read'):
        body_bytes = request.body.read()
    else:
        body_bytes = request.body
    deflated_data = deflater.compress(body_bytes)
    deflated_data += deflater.flush()
    is_economical = len(deflated_data) < len(body_bytes)
    if is_economical or always:
        request.body = deflated_data
        request.headers['Content-Encoding'] = 'deflate'
        request.headers['Content-Length'] = str(len(deflated_data))



File: httpie/__main__.py

"""The main entry point. Invoke as `http' or `python -m httpie'.

"""


def main():
    try:
        from httpie.core import main
        exit_status = main()
    except KeyboardInterrupt:
        from httpie.status import ExitStatus
        exit_status = ExitStatus.ERROR_CTRL_C

    return exit_status.value


if __name__ == '__main__':  # pragma: nocover
    import sys
    sys.exit(main())



File: httpie/status.py

from enum import IntEnum, unique


@unique
class ExitStatus(IntEnum):
    """Program exit status code constants."""
    SUCCESS = 0
    ERROR = 1
    ERROR_TIMEOUT = 2

    # See --check-status
    ERROR_HTTP_3XX = 3
    ERROR_HTTP_4XX = 4
    ERROR_HTTP_5XX = 5

    ERROR_TOO_MANY_REDIRECTS = 6
    PLUGIN_ERROR = 7
    # 128+2 SIGINT
    # <http://www.tldp.org/LDP/abs/html/exitcodes.html>
    ERROR_CTRL_C = 130


def http_status_to_exit_status(http_status: int, follow=False) -> ExitStatus:
    """
    Translate HTTP status code to exit status code.

    (Relevant only when invoked with --check-status or --download.)

    """
    if 300 <= http_status <= 399 and not follow:
        # Redirect
        return ExitStatus.ERROR_HTTP_3XX
    elif 400 <= http_status <= 499:
        # Client Error
        return ExitStatus.ERROR_HTTP_4XX
    elif 500 <= http_status <= 599:
        # Server Error
        return ExitStatus.ERROR_HTTP_5XX
    else:
        return ExitStatus.SUCCESS



File: httpie/adapters.py

from httpie.cli.dicts import HTTPHeadersDict
from requests.adapters import HTTPAdapter


class HTTPieHTTPAdapter(HTTPAdapter):

    def build_response(self, req, resp):
        """Wrap the original headers with the `HTTPHeadersDict`
        to preserve multiple headers that have the same name"""

        response = super().build_response(req, resp)
        response.headers = HTTPHeadersDict(getattr(resp, 'headers', {}))
        return response



File: httpie/plugins/registry.py

from .manager import PluginManager
from .builtin import BasicAuthPlugin, DigestAuthPlugin, BearerAuthPlugin
from ..output.formatters.headers import HeadersFormatter
from ..output.formatters.json import JSONFormatter
from ..output.formatters.xml import XMLFormatter
from ..output.formatters.colors import ColorFormatter


plugin_manager = PluginManager()


# Register all built-in plugins.
plugin_manager.register(
    BasicAuthPlugin,
    DigestAuthPlugin,
    BearerAuthPlugin,
    HeadersFormatter,
    JSONFormatter,
    XMLFormatter,
    ColorFormatter,
)



File: httpie/plugins/__init__.py

"""
WARNING: The plugin API is still work in progress and will
         probably be completely reworked in the future.

"""
from .base import (
    AuthPlugin, FormatterPlugin,
    ConverterPlugin, TransportPlugin
)

__all__ = ('AuthPlugin', 'ConverterPlugin', 'FormatterPlugin', 'TransportPlugin')



File: httpie/plugins/builtin.py

from base64 import b64encode

import requests.auth

from .base import AuthPlugin


# noinspection PyAbstractClass
class BuiltinAuthPlugin(AuthPlugin):
    package_name = '(builtin)'


class HTTPBasicAuth(requests.auth.HTTPBasicAuth):

    def __call__(
        self,
        request: requests.PreparedRequest
    ) -> requests.PreparedRequest:
        """
        Override username/password serialization to allow unicode.

        See https://github.com/httpie/cli/issues/212

        """
        # noinspection PyTypeChecker
        request.headers['Authorization'] = type(self).make_header(
            self.username, self.password).encode('latin1')
        return request

    @staticmethod
    def make_header(username: str, password: str) -> str:
        credentials = f'{username}:{password}'
        token = b64encode(credentials.encode()).strip().decode('latin1')
        return f'Basic {token}'


class HTTPBearerAuth(requests.auth.AuthBase):

    def __init__(self, token: str) -> None:
        self.token = token

    def __call__(self, request: requests.PreparedRequest) -> requests.PreparedRequest:
        request.headers['Authorization'] = f'Bearer {self.token}'
        return request


class BasicAuthPlugin(BuiltinAuthPlugin):
    name = 'Basic HTTP auth'
    auth_type = 'basic'
    netrc_parse = True

    # noinspection PyMethodOverriding
    def get_auth(self, username: str, password: str) -> HTTPBasicAuth:
        return HTTPBasicAuth(username, password)


class DigestAuthPlugin(BuiltinAuthPlugin):
    name = 'Digest HTTP auth'
    auth_type = 'digest'
    netrc_parse = True

    # noinspection PyMethodOverriding
    def get_auth(
        self,
        username: str,
        password: str
    ) -> requests.auth.HTTPDigestAuth:
        return requests.auth.HTTPDigestAuth(username, password)


class BearerAuthPlugin(BuiltinAuthPlugin):
    name = 'Bearer HTTP Auth'
    auth_type = 'bearer'
    netrc_parse = False
    auth_parse = False

    # noinspection PyMethodOverriding
    def get_auth(self, **kwargs) -> requests.auth.HTTPDigestAuth:
        return HTTPBearerAuth(self.raw_auth)



File: httpie/plugins/manager.py

import sys
import os
import warnings

from itertools import groupby
from operator import attrgetter
from typing import Dict, List, Type, Iterator, Iterable, Optional, ContextManager
from pathlib import Path
from contextlib import contextmanager, nullcontext

from ..compat import importlib_metadata, find_entry_points, get_dist_name

from ..utils import repr_dict, get_site_paths
from . import AuthPlugin, ConverterPlugin, FormatterPlugin, TransportPlugin
from .base import BasePlugin


ENTRY_POINT_CLASSES = {
    'httpie.plugins.auth.v1': AuthPlugin,
    'httpie.plugins.converter.v1': ConverterPlugin,
    'httpie.plugins.formatter.v1': FormatterPlugin,
    'httpie.plugins.transport.v1': TransportPlugin
}
ENTRY_POINT_NAMES = list(ENTRY_POINT_CLASSES.keys())


@contextmanager
def _load_directories(site_dirs: Iterable[Path]) -> Iterator[None]:
    plugin_dirs = [
        os.fspath(site_dir)
        for site_dir in site_dirs
    ]
    sys.path.extend(plugin_dirs)
    try:
        yield
    finally:
        for plugin_dir in plugin_dirs:
            sys.path.remove(plugin_dir)


def enable_plugins(plugins_dir: Optional[Path]) -> ContextManager[None]:
    if plugins_dir is None:
        return nullcontext()
    else:
        return _load_directories(get_site_paths(plugins_dir))


class PluginManager(list):
    def register(self, *plugins: Type[BasePlugin]):
        for plugin in plugins:
            self.append(plugin)

    def unregister(self, plugin: Type[BasePlugin]):
        self.remove(plugin)

    def filter(self, by_type=Type[BasePlugin]):
        return [plugin for plugin in self if issubclass(plugin, by_type)]

    def iter_entry_points(self, directory: Optional[Path] = None):
        with enable_plugins(directory):
            eps = importlib_metadata.entry_points()

            for entry_point_name in ENTRY_POINT_NAMES:
                yield from find_entry_points(eps, group=entry_point_name)

    def load_installed_plugins(self, directory: Optional[Path] = None):
        for entry_point in self.iter_entry_points(directory):
            plugin_name = get_dist_name(entry_point)
            try:
                plugin = entry_point.load()
            except BaseException as exc:
                warnings.warn(
                    f'While loading "{plugin_name}", an error occurred: {exc}\n'
                    f'For uninstallations, please use either "httpie plugins uninstall {plugin_name}" '
                    f'or "pip uninstall {plugin_name}" (depending on how you installed it in the first '
                    'place).'
                )
                continue
            plugin.package_name = plugin_name
            self.register(plugin)

    # Auth
    def get_auth_plugins(self) -> List[Type[AuthPlugin]]:
        return self.filter(AuthPlugin)

    def get_auth_plugin_mapping(self) -> Dict[str, Type[AuthPlugin]]:
        return {
            plugin.auth_type: plugin for plugin in self.get_auth_plugins()
        }

    def get_auth_plugin(self, auth_type: str) -> Type[AuthPlugin]:
        return self.get_auth_plugin_mapping()[auth_type]

    # Output processing
    def get_formatters(self) -> List[Type[FormatterPlugin]]:
        return self.filter(FormatterPlugin)

    def get_formatters_grouped(self) -> Dict[str, List[Type[FormatterPlugin]]]:
        return {
            group_name: list(group)
            for group_name, group
            in groupby(self.get_formatters(), key=attrgetter('group_name'))
        }

    def get_converters(self) -> List[Type[ConverterPlugin]]:
        return self.filter(ConverterPlugin)

    # Adapters
    def get_transport_plugins(self) -> List[Type[TransportPlugin]]:
        return self.filter(TransportPlugin)

    def __str__(self):
        return repr_dict({
            'adapters': self.get_transport_plugins(),
            'auth': self.get_auth_plugins(),
            'converters': self.get_converters(),
            'formatters': self.get_formatters(),
        })

    def __repr__(self):
        return f'<{type(self).__name__} {self}>'



File: httpie/plugins/base.py

from typing import Tuple


class BasePlugin:
    # The name of the plugin, eg. "My auth".
    name = None

    # Optional short description. It will be shown in the help
    # under --auth-type.
    description = None

    # This be set automatically once the plugin has been loaded.
    package_name = None


class AuthPlugin(BasePlugin):
    """
    Base auth plugin class.

    See httpie-ntlm for an example auth plugin:

        <https://github.com/httpie/httpie-ntlm>

    See also `test_auth_plugins.py`

    """
    # The value that should be passed to --auth-type
    # to use this auth plugin. Eg. "my-auth"
    auth_type = None

    # Set to `False` to make it possible to invoke this auth
    # plugin without requiring the user to specify credentials
    # through `--auth, -a`.
    auth_require = True

    # By default the `-a` argument is parsed for `username:password`.
    # Set this to `False` to disable the parsing and error handling.
    auth_parse = True

    # Set to `True` to make it possible for this auth
    # plugin to acquire credentials from the user’s netrc file(s).
    # It is used as a fallback when the credentials are not provided explicitly
    # through `--auth, -a`. Enabling this will allow skipping `--auth, -a`
    # even when `auth_require` is set `True` (provided that netrc provides
    # credential for a given host).
    netrc_parse = False

    # If both `auth_parse` and `prompt_password` are set to `True`,
    # and the value of `-a` lacks the password part,
    # then the user will be prompted to type the password in.
    prompt_password = True

    # Will be set to the raw value of `-a` (if provided) before
    # `get_auth()` gets called. If the credentials came from a netrc file,
    # then this is `None`.
    raw_auth = None

    def get_auth(self, username: str = None, password: str = None):
        """
        If `auth_parse` is set to `True`, then `username`
        and `password` contain the parsed credentials.

        Use `self.raw_auth` to access the raw value passed through
        `--auth, -a`.

        Return a ``requests.auth.AuthBase`` subclass instance.

        """
        raise NotImplementedError()


class TransportPlugin(BasePlugin):
    """
    Requests transport adapter docs:

        <https://requests.readthedocs.io/en/latest/user/advanced/#transport-adapters>

    See httpie-unixsocket for an example transport plugin:

        <https://github.com/httpie/httpie-unixsocket>

    """

    # The URL prefix the adapter should be mount to.
    prefix = None

    def get_adapter(self):
        """
        Return a ``requests.adapters.BaseAdapter`` subclass instance to be
        mounted to ``self.prefix``.

        """
        raise NotImplementedError()


class ConverterPlugin(BasePlugin):
    """
    Possibly converts binary response data for prettified terminal display.

    See httpie-msgpack for an example converter plugin:

        <https://github.com/rasky/httpie-msgpack>.

    """

    def __init__(self, mime: str):
        self.mime = mime

    def convert(self, body: bytes) -> Tuple[str, str]:
        """
        Convert a binary body to a textual representation for the terminal
        and return a tuple containing the new Content-Type and content, e.g.:

        ('application/json', '{}')

        """
        raise NotImplementedError

    @classmethod
    def supports(cls, mime: str) -> bool:
        raise NotImplementedError


class FormatterPlugin(BasePlugin):
    """
    Possibly formats response body & headers for prettified terminal display.

    """
    group_name = 'format'

    def __init__(self, **kwargs):
        """
        :param env: an class:`Environment` instance
        :param kwargs: additional keyword argument that some
                       formatters might require.

        """
        self.enabled = True
        self.kwargs = kwargs
        self.format_options = kwargs['format_options']

    def format_headers(self, headers: str) -> str:
        """Return processed `headers`

        :param headers: The headers as text.

        """
        return headers

    def format_body(self, content: str, mime: str) -> str:
        """Return processed `content`.

        :param mime: E.g., 'application/atom+xml'.
        :param content: The body content as text

        """
        return content

    def format_metadata(self, metadata: str) -> str:
        """Return processed `metadata`.

        :param metadata: The metadata as text.

        """
        return metadata



File: httpie/internal/daemons.py

"""
This module provides an interface to spawn a detached task to be
run with httpie.internal.daemon_runner on a separate process. It is
based on DVC's daemon system.
https://github.com/iterative/dvc/blob/main/dvc/daemon.py
"""

import inspect
import os
import platform
import sys
import httpie.__main__
from contextlib import suppress
from subprocess import Popen, DEVNULL
from typing import Dict, List
from httpie.compat import is_frozen, is_windows


ProcessContext = Dict[str, str]


def _start_process(cmd: List[str], **kwargs) -> Popen:
    prefix = [sys.executable]
    # If it is frozen, sys.executable points to the binary (http).
    # Otherwise it points to the python interpreter.
    if not is_frozen:
        main_entrypoint = httpie.__main__.__file__
        prefix += [main_entrypoint]
    return Popen(prefix + cmd, close_fds=True, shell=False, stdout=DEVNULL, stderr=DEVNULL, **kwargs)


def _spawn_windows(cmd: List[str], process_context: ProcessContext) -> None:
    from subprocess import (
        CREATE_NEW_PROCESS_GROUP,
        CREATE_NO_WINDOW,
        STARTF_USESHOWWINDOW,
        STARTUPINFO,
    )

    # https://stackoverflow.com/a/7006424
    # https://bugs.python.org/issue41619
    creationflags = CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW

    startupinfo = STARTUPINFO()
    startupinfo.dwFlags |= STARTF_USESHOWWINDOW

    _start_process(
        cmd,
        env=process_context,
        creationflags=creationflags,
        startupinfo=startupinfo,
    )


def _spawn_posix(args: List[str], process_context: ProcessContext) -> None:
    """
    Perform a double fork procedure* to detach from the parent
    process so that we don't block the user even if their original
    command's execution is done but the release fetcher is not.

    [1]: https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap11.html#tag_11_01_03
    """

    from httpie.core import main

    try:
        pid = os.fork()
        if pid > 0:
            return
    except OSError:
        os._exit(1)

    os.setsid()

    try:
        pid = os.fork()
        if pid > 0:
            os._exit(0)
    except OSError:
        os._exit(1)

    # Close all standard inputs/outputs
    sys.stdin.close()
    sys.stdout.close()
    sys.stderr.close()

    if platform.system() == 'Darwin':
        # Double-fork is not reliable on MacOS, so we'll use a subprocess
        # to ensure the task is isolated properly.
        process = _start_process(args, env=process_context)
        # Unlike windows, since we already completed the fork procedure
        # we can simply join the process and wait for it.
        process.communicate()
    else:
        os.environ.update(process_context)
        with suppress(BaseException):
            main(['http'] + args)

    os._exit(0)


def _spawn(args: List[str], process_context: ProcessContext) -> None:
    """
    Spawn a new process to run the given command.
    """
    if is_windows:
        _spawn_windows(args, process_context)
    else:
        _spawn_posix(args, process_context)


def spawn_daemon(task: str) -> None:
    args = [task, '--daemon']
    process_context = os.environ.copy()
    if not is_frozen:
        file_path = os.path.abspath(inspect.stack()[0][1])
        process_context['PYTHONPATH'] = os.path.dirname(
            os.path.dirname(os.path.dirname(file_path))
        )

    _spawn(args, process_context)



File: httpie/internal/daemon_runner.py

import argparse
from contextlib import redirect_stderr, redirect_stdout
from typing import List

from httpie.context import Environment
from httpie.internal.update_warnings import _fetch_updates, _get_suppress_context
from httpie.status import ExitStatus

STATUS_FILE = '.httpie-test-daemon-status'


def _check_status(env):
    # This function is used only for the testing (test_update_warnings).
    # Since we don't want to trigger the fetch_updates (which would interact
    # with real world resources), we'll only trigger this pseudo task
    # and check whether the STATUS_FILE is created or not.
    import tempfile
    from pathlib import Path

    status_file = Path(tempfile.gettempdir()) / STATUS_FILE
    status_file.touch()


DAEMONIZED_TASKS = {
    'check_status': _check_status,
    'fetch_updates': _fetch_updates,
}


def _parse_options(args: List[str]) -> argparse.Namespace:
    parser = argparse.ArgumentParser()
    parser.add_argument('task_id')
    parser.add_argument('--daemon', action='store_true')
    return parser.parse_known_args(args)[0]


def is_daemon_mode(args: List[str]) -> bool:
    return '--daemon' in args


def run_daemon_task(env: Environment, args: List[str]) -> ExitStatus:
    options = _parse_options(args)

    assert options.daemon
    assert options.task_id in DAEMONIZED_TASKS
    with redirect_stdout(env.devnull), redirect_stderr(env.devnull):
        with _get_suppress_context(env):
            DAEMONIZED_TASKS[options.task_id](env)

    return ExitStatus.SUCCESS



File: httpie/internal/__init__.py




File: httpie/internal/update_warnings.py

import json
from contextlib import nullcontext, suppress
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any, Optional, Callable

import requests

import httpie
from httpie.context import Environment, LogLevel
from httpie.internal.__build_channel__ import BUILD_CHANNEL
from httpie.internal.daemons import spawn_daemon
from httpie.utils import is_version_greater, open_with_lockfile

# Automatically updated package version index.
PACKAGE_INDEX_LINK = 'https://packages.httpie.io/latest.json'

FETCH_INTERVAL = timedelta(weeks=2)
WARN_INTERVAL = timedelta(weeks=1)

UPDATE_MESSAGE_FORMAT = """\
A new HTTPie release ({last_released_version}) is available.
To see how you can update, please visit https://httpie.io/docs/cli/{installation_method}
"""

ALREADY_UP_TO_DATE_MESSAGE = """\
You are already up-to-date.
"""


def _read_data_error_free(file: Path) -> Any:
    # If the file is broken / non-existent, ignore it.
    try:
        with open(file) as stream:
            return json.load(stream)
    except (ValueError, OSError):
        return {}


def _fetch_updates(env: Environment) -> str:
    file = env.config.version_info_file
    data = _read_data_error_free(file)

    response = requests.get(PACKAGE_INDEX_LINK, verify=False)
    response.raise_for_status()

    data.setdefault('last_warned_date', None)
    data['last_fetched_date'] = datetime.now().isoformat()
    data['last_released_versions'] = response.json()

    with open_with_lockfile(file, 'w') as stream:
        json.dump(data, stream)


def fetch_updates(env: Environment, lazy: bool = True):
    if lazy:
        spawn_daemon('fetch_updates')
    else:
        _fetch_updates(env)


def maybe_fetch_updates(env: Environment) -> None:
    if env.config.get('disable_update_warnings'):
        return None

    data = _read_data_error_free(env.config.version_info_file)

    if data:
        current_date = datetime.now()
        last_fetched_date = datetime.fromisoformat(data['last_fetched_date'])
        earliest_fetch_date = last_fetched_date + FETCH_INTERVAL
        if current_date < earliest_fetch_date:
            return None

    fetch_updates(env)


def _get_suppress_context(env: Environment) -> Any:
    """Return a context manager that suppress
    all possible errors.

    Note: if you have set the developer_mode=True in
    your config, then it will show all errors for easier
    debugging."""
    if env.config.developer_mode:
        return nullcontext()
    else:
        return suppress(BaseException)


def _update_checker(
    func: Callable[[Environment], None]
) -> Callable[[Environment], None]:
    """Control the execution of the update checker (suppress errors, trigger
    auto updates etc.)"""

    def wrapper(env: Environment) -> None:
        with _get_suppress_context(env):
            func(env)

        with _get_suppress_context(env):
            maybe_fetch_updates(env)

    return wrapper


def _get_update_status(env: Environment) -> Optional[str]:
    """If there is a new update available, return the warning text.
    Otherwise just return None."""
    file = env.config.version_info_file
    if not file.exists():
        return None

    with _get_suppress_context(env):
        # If the user quickly spawns multiple httpie processes
        # we don't want to end in a race.
        with open_with_lockfile(file) as stream:
            version_info = json.load(stream)

        available_channels = version_info['last_released_versions']
        if BUILD_CHANNEL not in available_channels:
            return None

        current_version = httpie.__version__
        last_released_version = available_channels[BUILD_CHANNEL]
        if not is_version_greater(last_released_version, current_version):
            return None

        text = UPDATE_MESSAGE_FORMAT.format(
            last_released_version=last_released_version,
            installation_method=BUILD_CHANNEL,
        )
        return text


def get_update_status(env: Environment) -> str:
    return _get_update_status(env) or ALREADY_UP_TO_DATE_MESSAGE


@_update_checker
def check_updates(env: Environment) -> None:
    if env.config.get('disable_update_warnings'):
        return None

    file = env.config.version_info_file
    update_status = _get_update_status(env)

    if not update_status:
        return None

    # If the user quickly spawns multiple httpie processes
    # we don't want to end in a race.
    with open_with_lockfile(file) as stream:
        version_info = json.load(stream)

    # We don't want to spam the user with too many warnings,
    # so we'll only warn every once a while (WARN_INTERNAL).
    current_date = datetime.now()
    last_warned_date = version_info['last_warned_date']
    if last_warned_date is not None:
        earliest_warn_date = (
            datetime.fromisoformat(last_warned_date) + WARN_INTERVAL
        )
        if current_date < earliest_warn_date:
            return None

    env.log_error(update_status, level=LogLevel.INFO)
    version_info['last_warned_date'] = current_date.isoformat()

    with open_with_lockfile(file, 'w') as stream:
        json.dump(version_info, stream)



File: httpie/internal/__build_channel__.py

# Represents the packaging method. This file should
# be overridden by every build system we support on
# the packaging step.

BUILD_CHANNEL = 'unknown'



File: httpie/output/streams.py

from abc import ABCMeta, abstractmethod
from itertools import chain
from typing import Callable, Iterable, Optional, Union

from .processing import Conversion, Formatting
from ..context import Environment
from ..encoding import smart_decode, smart_encode, UTF8
from ..models import HTTPMessage, OutputOptions
from ..utils import parse_content_type_header


BINARY_SUPPRESSED_NOTICE = (
    b'\n'
    b'+-----------------------------------------+\n'
    b'| NOTE: binary data not shown in terminal |\n'
    b'+-----------------------------------------+'
)


class DataSuppressedError(Exception):
    message = None


class BinarySuppressedError(DataSuppressedError):
    """An error indicating that the body is binary and won't be written,
     e.g., for terminal output)."""
    message = BINARY_SUPPRESSED_NOTICE


class BaseStream(metaclass=ABCMeta):
    """Base HTTP message output stream class."""

    def __init__(
        self,
        msg: HTTPMessage,
        output_options: OutputOptions,
        on_body_chunk_downloaded: Callable[[bytes], None] = None,
        **kwargs
    ):
        """
        :param msg: a :class:`models.HTTPMessage` subclass
        :param output_options: a :class:`OutputOptions` instance to represent
                               which parts of the message is printed.
        """
        assert output_options.any()
        self.msg = msg
        self.output_options = output_options
        self.on_body_chunk_downloaded = on_body_chunk_downloaded
        self.extra_options = kwargs

    def get_headers(self) -> bytes:
        """Return the headers' bytes."""
        return self.msg.headers.encode()

    def get_metadata(self) -> bytes:
        """Return the message metadata."""
        return self.msg.metadata.encode()

    @abstractmethod
    def iter_body(self) -> Iterable[bytes]:
        """Return an iterator over the message body."""

    def __iter__(self) -> Iterable[bytes]:
        """Return an iterator over `self.msg`."""
        if self.output_options.headers:
            yield self.get_headers()
            yield b'\r\n\r\n'

        if self.output_options.body:
            try:
                for chunk in self.iter_body():
                    yield chunk
                    if self.on_body_chunk_downloaded:
                        self.on_body_chunk_downloaded(chunk)
            except DataSuppressedError as e:
                if self.output_options.headers:
                    yield b'\n'
                yield e.message

        if self.output_options.meta:
            if self.output_options.body:
                yield b'\n\n'

            yield self.get_metadata()
            yield b'\n\n'


class RawStream(BaseStream):
    """The message is streamed in chunks with no processing."""

    CHUNK_SIZE = 1024 * 100
    CHUNK_SIZE_BY_LINE = 1

    def __init__(self, chunk_size=CHUNK_SIZE, **kwargs):
        super().__init__(**kwargs)
        self.chunk_size = chunk_size

    def iter_body(self) -> Iterable[bytes]:
        return self.msg.iter_body(self.chunk_size)


ENCODING_GUESS_THRESHOLD = 3


class EncodedStream(BaseStream):
    """Encoded HTTP message stream.

    The message bytes are converted to an encoding suitable for
    `self.env.stdout`. Unicode errors are replaced and binary data
    is suppressed. The body is always streamed by line.

    """
    CHUNK_SIZE = 1

    def __init__(
        self,
        env=Environment(),
        mime_overwrite: str = None,
        encoding_overwrite: str = None,
        **kwargs
    ):
        super().__init__(**kwargs)
        if mime_overwrite:
            self.mime = mime_overwrite
        else:
            self.mime, _ = parse_content_type_header(self.msg.content_type)
        self._encoding = encoding_overwrite or self.msg.encoding
        self._encoding_guesses = []
        if env.stdout_isatty:
            # Use the encoding supported by the terminal.
            output_encoding = env.stdout_encoding
        else:
            # Preserve the message encoding.
            output_encoding = self.msg.encoding
        # Default to UTF-8 when unsure.
        self.output_encoding = output_encoding or UTF8

    def iter_body(self) -> Iterable[bytes]:
        for line, lf in self.msg.iter_lines(self.CHUNK_SIZE):
            if b'\0' in line:
                raise BinarySuppressedError()
            line = self.decode_chunk(line)
            yield smart_encode(line, self.output_encoding) + lf

    def decode_chunk(self, raw_chunk: str) -> str:
        chunk, guessed_encoding = smart_decode(raw_chunk, self.encoding)
        self._encoding_guesses.append(guessed_encoding)
        return chunk

    @property
    def encoding(self) -> Optional[str]:
        if self._encoding:
            return self._encoding

        # If we find a reliable (used consecutively) encoding, than
        # use it for the next iterations.
        if len(self._encoding_guesses) < ENCODING_GUESS_THRESHOLD:
            return None

        guess_1, guess_2 = self._encoding_guesses[-2:]
        if guess_1 == guess_2:
            self._encoding = guess_1
            return guess_1

    @encoding.setter
    def encoding(self, value) -> None:
        self._encoding = value


class PrettyStream(EncodedStream):
    """In addition to :class:`EncodedStream` behaviour, this stream applies
    content processing.

    Useful for long-lived HTTP responses that stream by lines
    such as the Twitter streaming API.

    """

    CHUNK_SIZE = 1

    def __init__(
        self, conversion: Conversion,
        formatting: Formatting,
        **kwargs,
    ):
        super().__init__(**kwargs)
        self.formatting = formatting
        self.conversion = conversion

    def get_headers(self) -> bytes:
        return self.formatting.format_headers(
            self.msg.headers).encode(self.output_encoding)

    def get_metadata(self) -> bytes:
        return self.formatting.format_metadata(
            self.msg.metadata).encode(self.output_encoding)

    def iter_body(self) -> Iterable[bytes]:
        first_chunk = True
        iter_lines = self.msg.iter_lines(self.CHUNK_SIZE)
        for line, lf in iter_lines:
            if b'\0' in line:
                if first_chunk:
                    converter = self.conversion.get_converter(self.mime)
                    if converter:
                        body = bytearray()
                        # noinspection PyAssignmentToLoopOrWithParameter
                        for line, lf in chain([(line, lf)], iter_lines):
                            body.extend(line)
                            body.extend(lf)
                        self.mime, body = converter.convert(body)
                        assert isinstance(body, str)
                        yield self.process_body(body)
                        return
                raise BinarySuppressedError()
            yield self.process_body(line) + lf
            first_chunk = False

    def process_body(self, chunk: Union[str, bytes]) -> bytes:
        if not isinstance(chunk, str):
            # Text when a converter has been used,
            # otherwise it will always be bytes.
            chunk = self.decode_chunk(chunk)
        chunk = self.formatting.format_body(content=chunk, mime=self.mime)
        return smart_encode(chunk, self.output_encoding)


class BufferedPrettyStream(PrettyStream):
    """The same as :class:`PrettyStream` except that the body is fully
    fetched before it's processed.

    Suitable regular HTTP responses.

    """

    CHUNK_SIZE = 1024 * 10

    def iter_body(self) -> Iterable[bytes]:
        # Read the whole body before prettifying it,
        # but bail out immediately if the body is binary.
        converter = None
        body = bytearray()

        for chunk in self.msg.iter_body(self.CHUNK_SIZE):
            if not converter and b'\0' in chunk:
                converter = self.conversion.get_converter(self.mime)
                if not converter:
                    raise BinarySuppressedError()
            body.extend(chunk)

        if converter:
            self.mime, body = converter.convert(body)

        yield self.process_body(body)



File: httpie/output/models.py

import argparse
from typing import Any, Dict, Union, List, NamedTuple, Optional

from httpie.context import Environment
from httpie.cli.constants import PrettyOptions, PRETTY_MAP, PRETTY_STDOUT_TTY_ONLY
from httpie.cli.argtypes import PARSED_DEFAULT_FORMAT_OPTIONS
from httpie.output.formatters.colors import AUTO_STYLE


class ProcessingOptions(NamedTuple):
    """Represents a set of stylistic options
    that are used when deciding which stream
    should be used."""

    debug: bool = False
    traceback: bool = False

    stream: bool = False
    style: str = AUTO_STYLE
    prettify: Union[List[str], PrettyOptions] = PRETTY_STDOUT_TTY_ONLY

    response_mime: Optional[str] = None
    response_charset: Optional[str] = None

    json: bool = False
    format_options: Dict[str, Any] = PARSED_DEFAULT_FORMAT_OPTIONS

    def get_prettify(self, env: Environment) -> List[str]:
        if self.prettify is PRETTY_STDOUT_TTY_ONLY:
            return PRETTY_MAP['all' if env.stdout_isatty else 'none']
        else:
            return self.prettify

    @classmethod
    def from_raw_args(cls, options: argparse.Namespace) -> 'ProcessingOptions':
        fetched_options = {
            option: getattr(options, option)
            for option in cls._fields
        }
        return cls(**fetched_options)

    @property
    def show_traceback(self):
        return self.debug or self.traceback



File: httpie/output/__init__.py




File: httpie/output/utils.py

import json
import re
from typing import Tuple

from ..utils import load_json_preserve_order_and_dupe_keys
from .lexers.json import PREFIX_REGEX


def load_prefixed_json(data: str) -> Tuple[str, json.JSONDecoder]:
    """Simple JSON loading from `data`.

    """
    # First, the full data.
    try:
        return '', load_json_preserve_order_and_dupe_keys(data)
    except ValueError:
        pass

    # Then, try to find the start of the actual body.
    data_prefix, body = parse_prefixed_json(data)
    try:
        return data_prefix, load_json_preserve_order_and_dupe_keys(body)
    except ValueError:
        raise ValueError('Invalid JSON')


def parse_prefixed_json(data: str) -> Tuple[str, str]:
    """Find the potential JSON body from `data`.

    Sometimes the JSON body is prefixed with a XSSI magic string, specific to the server.
    Return a tuple (data prefix, actual JSON body).

    """
    matches = re.findall(PREFIX_REGEX, data)
    data_prefix = matches[0] if matches else ''
    body = data[len(data_prefix):]
    return data_prefix, body



File: httpie/output/writer.py

import errno
import requests
from typing import Any, Dict, IO, Optional, TextIO, Tuple, Type, Union

from ..cli.dicts import HTTPHeadersDict
from ..context import Environment
from ..models import (
    HTTPRequest,
    HTTPResponse,
    HTTPMessage,
    RequestsMessage,
    RequestsMessageKind,
    OutputOptions,
)
from .models import ProcessingOptions
from .processing import Conversion, Formatting
from .streams import (
    BaseStream, BufferedPrettyStream, EncodedStream, PrettyStream, RawStream,
)
from ..utils import parse_content_type_header


MESSAGE_SEPARATOR = '\n\n'
MESSAGE_SEPARATOR_BYTES = MESSAGE_SEPARATOR.encode()


def write_message(
    requests_message: RequestsMessage,
    env: Environment,
    output_options: OutputOptions,
    processing_options: ProcessingOptions,
    extra_stream_kwargs: Optional[Dict[str, Any]] = None
):
    if not output_options.any():
        return
    write_stream_kwargs = {
        'stream': build_output_stream_for_message(
            env=env,
            requests_message=requests_message,
            output_options=output_options,
            processing_options=processing_options,
            extra_stream_kwargs=extra_stream_kwargs
        ),
        # NOTE: `env.stdout` will in fact be `stderr` with `--download`
        'outfile': env.stdout,
        'flush': env.stdout_isatty or processing_options.stream
    }
    try:
        if env.is_windows and 'colors' in processing_options.get_prettify(env):
            write_stream_with_colors_win(**write_stream_kwargs)
        else:
            write_stream(**write_stream_kwargs)
    except OSError as e:
        if processing_options.show_traceback and e.errno == errno.EPIPE:
            # Ignore broken pipes unless --traceback.
            env.stderr.write('\n')
        else:
            raise


def write_stream(
    stream: BaseStream,
    outfile: Union[IO, TextIO],
    flush: bool
):
    """Write the output stream."""
    try:
        # Writing bytes so we use the buffer interface.
        buf = outfile.buffer
    except AttributeError:
        buf = outfile

    for chunk in stream:
        buf.write(chunk)
        if flush:
            outfile.flush()


def write_stream_with_colors_win(
    stream: 'BaseStream',
    outfile: TextIO,
    flush: bool
):
    """Like `write`, but colorized chunks are written as text
    directly to `outfile` to ensure it gets processed by colorama.
    Applies only to Windows and colorized terminal output.

    """
    color = b'\x1b['
    encoding = outfile.encoding
    for chunk in stream:
        if color in chunk:
            outfile.write(chunk.decode(encoding))
        else:
            outfile.buffer.write(chunk)
        if flush:
            outfile.flush()


def write_raw_data(
    env: Environment,
    data: Any,
    *,
    processing_options: Optional[ProcessingOptions] = None,
    headers: Optional[HTTPHeadersDict] = None,
    stream_kwargs: Optional[Dict[str, Any]] = None
):
    msg = requests.PreparedRequest()
    msg.is_body_upload_chunk = True
    msg.body = data
    msg.headers = headers or HTTPHeadersDict()
    msg_output_options = OutputOptions.from_message(msg, body=True, headers=False)
    return write_message(
        requests_message=msg,
        env=env,
        output_options=msg_output_options,
        processing_options=processing_options or ProcessingOptions(),
        extra_stream_kwargs=stream_kwargs
    )


def build_output_stream_for_message(
    env: Environment,
    requests_message: RequestsMessage,
    output_options: OutputOptions,
    processing_options: ProcessingOptions,
    extra_stream_kwargs: Optional[Dict[str, Any]] = None
):
    message_type = {
        RequestsMessageKind.REQUEST: HTTPRequest,
        RequestsMessageKind.RESPONSE: HTTPResponse,
    }[output_options.kind]
    stream_class, stream_kwargs = get_stream_type_and_kwargs(
        env=env,
        processing_options=processing_options,
        message_type=message_type,
        headers=requests_message.headers
    )
    if extra_stream_kwargs:
        stream_kwargs.update(extra_stream_kwargs)
    yield from stream_class(
        msg=message_type(requests_message),
        output_options=output_options,
        **stream_kwargs,
    )
    if (env.stdout_isatty and output_options.body and not output_options.meta
            and not getattr(requests_message, 'is_body_upload_chunk', False)):
        # Ensure a blank line after the response body.
        # For terminal output only.
        yield MESSAGE_SEPARATOR_BYTES


def get_stream_type_and_kwargs(
    env: Environment,
    processing_options: ProcessingOptions,
    message_type: Type[HTTPMessage],
    headers: HTTPHeadersDict,
) -> Tuple[Type['BaseStream'], dict]:
    """Pick the right stream type and kwargs for it based on `env` and `args`.

    """
    is_stream = processing_options.stream
    prettify_groups = processing_options.get_prettify(env)
    if not is_stream and message_type is HTTPResponse:
        # If this is a response, then check the headers for determining
        # auto-streaming.
        raw_content_type_header = headers.get('Content-Type', None)
        if raw_content_type_header:
            content_type_header, _ = parse_content_type_header(raw_content_type_header)
            is_stream = (content_type_header == 'text/event-stream')

    if not env.stdout_isatty and not prettify_groups:
        stream_class = RawStream
        stream_kwargs = {
            'chunk_size': (
                RawStream.CHUNK_SIZE_BY_LINE
                if is_stream
                else RawStream.CHUNK_SIZE
            )
        }
    else:
        stream_class = EncodedStream
        stream_kwargs = {
            'env': env,
        }
        if message_type is HTTPResponse:
            stream_kwargs.update({
                'mime_overwrite': processing_options.response_mime,
                'encoding_overwrite': processing_options.response_charset,
            })
        if prettify_groups:
            stream_class = PrettyStream if is_stream else BufferedPrettyStream
            stream_kwargs.update({
                'conversion': Conversion(),
                'formatting': Formatting(
                    env=env,
                    groups=prettify_groups,
                    color_scheme=processing_options.style,
                    explicit_json=processing_options.json,
                    format_options=processing_options.format_options,
                )
            })

    return stream_class, stream_kwargs



File: httpie/output/processing.py

import re
from typing import Optional, List

from ..plugins import ConverterPlugin
from ..plugins.registry import plugin_manager
from ..context import Environment


MIME_RE = re.compile(r'^[^/]+/[^/]+$')


def is_valid_mime(mime):
    return mime and MIME_RE.match(mime)


class Conversion:

    @staticmethod
    def get_converter(mime: str) -> Optional[ConverterPlugin]:
        if is_valid_mime(mime):
            for converter_class in plugin_manager.get_converters():
                if converter_class.supports(mime):
                    return converter_class(mime)


class Formatting:
    """A delegate class that invokes the actual processors."""

    def __init__(self, groups: List[str], env=Environment(), **kwargs):
        """
        :param groups: names of processor groups to be applied
        :param env: Environment
        :param kwargs: additional keyword arguments for processors

        """
        available_plugins = plugin_manager.get_formatters_grouped()
        self.enabled_plugins = []
        for group in groups:
            for cls in available_plugins[group]:
                p = cls(env=env, **kwargs)
                if p.enabled:
                    self.enabled_plugins.append(p)

    def format_headers(self, headers: str) -> str:
        for p in self.enabled_plugins:
            headers = p.format_headers(headers)
        return headers

    def format_body(self, content: str, mime: str) -> str:
        if is_valid_mime(mime):
            for p in self.enabled_plugins:
                content = p.format_body(content, mime)
        return content

    def format_metadata(self, metadata: str) -> str:
        for p in self.enabled_plugins:
            metadata = p.format_metadata(metadata)
        return metadata



File: httpie/output/ui/__init__.py




File: httpie/output/ui/palette.py

from dataclasses import dataclass, field
from enum import Enum, auto
from typing import Optional, List


PYGMENTS_BRIGHT_BLACK = 'ansibrightblack'

AUTO_STYLE = 'auto'  # Follows terminal ANSI color styles


class Styles(Enum):
    PIE = auto()
    ANSI = auto()


class PieStyle(str, Enum):
    UNIVERSAL = 'pie'
    DARK = 'pie-dark'
    LIGHT = 'pie-light'


PIE_STYLE_TO_SHADE = {
    PieStyle.DARK: '500',
    PieStyle.UNIVERSAL: '600',
    PieStyle.LIGHT: '700',
}
SHADE_TO_PIE_STYLE = {
    shade: style for style, shade in PIE_STYLE_TO_SHADE.items()
}


class ColorString(str):
    def __or__(self, other: str) -> 'ColorString':
        """Combine a style with a property.

        E.g: PieColor.BLUE | BOLD | ITALIC
        """
        if isinstance(other, str):
            # In case of PieColor.BLUE | SOMETHING
            # we just create a new string.
            return ColorString(self + ' ' + other)
        elif isinstance(other, GenericColor):
            # If we see a GenericColor, then we'll wrap it
            # in with the desired property in a different class.
            return _StyledGenericColor(other, styles=self.split())
        elif isinstance(other, _StyledGenericColor):
            # And if it is already wrapped, we'll just extend the
            # list of properties.
            other.styles.extend(self.split())
            return other
        else:
            return NotImplemented


class PieColor(ColorString, Enum):
    """Styles that are available only in Pie themes."""

    PRIMARY = 'primary'
    SECONDARY = 'secondary'

    WHITE = 'white'
    BLACK = 'black'
    GREY = 'grey'
    AQUA = 'aqua'
    PURPLE = 'purple'
    ORANGE = 'orange'
    RED = 'red'
    BLUE = 'blue'
    PINK = 'pink'
    GREEN = 'green'
    YELLOW = 'yellow'


class GenericColor(Enum):
    """Generic colors that are safe to use everywhere."""

    # <https://rich.readthedocs.io/en/stable/appendix/colors.html>

    WHITE = {Styles.PIE: PieColor.WHITE, Styles.ANSI: 'white'}
    BLACK = {Styles.PIE: PieColor.BLACK, Styles.ANSI: 'black'}
    GREEN = {Styles.PIE: PieColor.GREEN, Styles.ANSI: 'green'}
    ORANGE = {Styles.PIE: PieColor.ORANGE, Styles.ANSI: 'yellow'}
    YELLOW = {Styles.PIE: PieColor.YELLOW, Styles.ANSI: 'bright_yellow'}
    BLUE = {Styles.PIE: PieColor.BLUE, Styles.ANSI: 'blue'}
    PINK = {Styles.PIE: PieColor.PINK, Styles.ANSI: 'bright_magenta'}
    PURPLE = {Styles.PIE: PieColor.PURPLE, Styles.ANSI: 'magenta'}
    RED = {Styles.PIE: PieColor.RED, Styles.ANSI: 'red'}
    AQUA = {Styles.PIE: PieColor.AQUA, Styles.ANSI: 'cyan'}
    GREY = {Styles.PIE: PieColor.GREY, Styles.ANSI: 'bright_black'}

    def apply_style(
        self, style: Styles, *, style_name: Optional[str] = None
    ) -> str:
        """Apply the given style to a particular value."""
        exposed_color = self.value[style]
        if style is Styles.PIE:
            assert style_name is not None
            shade = PIE_STYLE_TO_SHADE[PieStyle(style_name)]
            return get_color(exposed_color, shade)
        else:
            return exposed_color


@dataclass
class _StyledGenericColor:
    color: 'GenericColor'
    styles: List[str] = field(default_factory=list)


# noinspection PyDictCreation
COLOR_PALETTE = {
    # Copy the brand palette
    PieColor.WHITE: '#F5F5F0',
    PieColor.BLACK: '#1C1818',
    PieColor.GREY: {
        '50': '#F5F5F0',
        '100': '#EDEDEB',
        '200': '#D1D1CF',
        '300': '#B5B5B2',
        '400': '#999999',
        '500': '#7D7D7D',
        '600': '#666663',
        '700': '#4F4D4D',
        '800': '#363636',
        '900': '#1C1818',
        'DEFAULT': '#7D7D7D',
    },
    PieColor.AQUA: {
        '50': '#E8F0F5',
        '100': '#D6E3ED',
        '200': '#C4D9E5',
        '300': '#B0CCDE',
        '400': '#9EBFD6',
        '500': '#8CB4CD',
        '600': '#7A9EB5',
        '700': '#698799',
        '800': '#597082',
        '900': '#455966',
        'DEFAULT': '#8CB4CD',
    },
    PieColor.PURPLE: {
        '50': '#F0E0FC',
        '100': '#E3C7FA',
        '200': '#D9ADF7',
        '300': '#CC96F5',
        '400': '#BF7DF2',
        '500': '#B464F0',
        '600': '#9E54D6',
        '700': '#8745BA',
        '800': '#70389E',
        '900': '#5C2982',
        'DEFAULT': '#B464F0',
    },
    PieColor.ORANGE: {
        '50': '#FFEDDB',
        '100': '#FFDEBF',
        '200': '#FFCFA3',
        '300': '#FFBF87',
        '400': '#FFB06B',
        '500': '#FFA24E',
        '600': '#F2913D',
        '700': '#E3822B',
        '800': '#D6701C',
        '900': '#C75E0A',
        'DEFAULT': '#FFA24E',
    },
    PieColor.RED: {
        '50': '#FFE0DE',
        '100': '#FFC7C4',
        '200': '#FFB0AB',
        '300': '#FF968F',
        '400': '#FF8075',
        '500': '#FF665B',
        '600': '#E34F45',
        '700': '#C7382E',
        '800': '#AD2117',
        '900': '#910A00',
        'DEFAULT': '#FF665B',
    },
    PieColor.BLUE: {
        '50': '#DBE3FA',
        '100': '#BFCFF5',
        '200': '#A1B8F2',
        '300': '#85A3ED',
        '400': '#698FEB',
        '500': '#4B78E6',
        '600': '#426BD1',
        '700': '#3B5EBA',
        '800': '#3354A6',
        '900': '#2B478F',
        'DEFAULT': '#4B78E6',
    },
    PieColor.PINK: {
        '50': '#FFEBFF',
        '100': '#FCDBFC',
        '200': '#FCCCFC',
        '300': '#FCBAFC',
        '400': '#FAABFA',
        '500': '#FA9BFA',
        '600': '#DE85DE',
        '700': '#C26EC2',
        '800': '#A854A6',
        '900': '#8C3D8A',
        'DEFAULT': '#FA9BFA',
    },
    PieColor.GREEN: {
        '50': '#E3F7E8',
        '100': '#CCF2D6',
        '200': '#B5EDC4',
        '300': '#A1E8B0',
        '400': '#8AE09E',
        '500': '#73DC8C',
        '600': '#63C27A',
        '700': '#52AB66',
        '800': '#429154',
        '900': '#307842',
        'DEFAULT': '#73DC8C',
    },
    PieColor.YELLOW: {
        '50': '#F7F7DB',
        '100': '#F2F2BF',
        '200': '#EDEDA6',
        '300': '#E5E88A',
        '400': '#E0E36E',
        '500': '#DBDE52',
        '600': '#CCCC3D',
        '700': '#BABA29',
        '800': '#ABA614',
        '900': '#999400',
        'DEFAULT': '#DBDE52',
    },
}
COLOR_PALETTE.update(
    {
        # Terminal-specific palette customizations.
        PieColor.GREY: {
            # Grey is the same no matter shade for the colors
            shade: COLOR_PALETTE[PieColor.GREY]['500']
            for shade in COLOR_PALETTE[PieColor.GREY].keys()
        },
        PieColor.PRIMARY: {
            '700': COLOR_PALETTE[PieColor.BLACK],
            '600': PYGMENTS_BRIGHT_BLACK,
            '500': COLOR_PALETTE[PieColor.WHITE],
        },
        PieColor.SECONDARY: {
            '700': '#37523C',
            '600': '#6c6969',
            '500': '#6c6969',
        },
    }
)


def boldify(color: PieColor) -> str:
    return f'bold {color}'


# noinspection PyDefaultArgument
def get_color(
    color: PieColor, shade: str, *, palette=COLOR_PALETTE
) -> Optional[str]:
    if color not in palette:
        return None
    color_code = palette[color]
    if isinstance(color_code, dict) and shade in color_code:
        return color_code[shade]
    else:
        return color_code



File: httpie/output/ui/rich_utils.py

import os

from typing import Iterator
from contextlib import contextmanager

from rich.console import Console, RenderableType
from rich.highlighter import Highlighter

from httpie.output.ui.rich_palette import _make_rich_color_theme


def render_as_string(renderable: RenderableType) -> str:
    """Render any `rich` object in a fake console and
    return a *style-less* version of it as a string."""

    with open(os.devnull, 'w') as null_stream:
        fake_console = Console(file=null_stream, record=True, theme=_make_rich_color_theme())
        fake_console.print(renderable)
        return fake_console.export_text()


@contextmanager
def enable_highlighter(
    console: Console,
    highlighter: Highlighter,
) -> Iterator[Console]:
    """Enable a highlighter temporarily."""

    original_highlighter = console.highlighter
    try:
        console.highlighter = highlighter
        yield console
    finally:
        console.highlighter = original_highlighter



File: httpie/output/ui/rich_help.py

import re
import textwrap
from typing import AbstractSet, Iterable, Optional, Tuple

from rich.console import RenderableType
from rich.highlighter import RegexHighlighter
from rich.padding import Padding
from rich.table import Table
from rich.text import Text

from httpie.cli.constants import SEPARATOR_GROUP_ALL_ITEMS
from httpie.cli.options import Argument, ParserSpec, Qualifiers
from httpie.output.ui.palette import GenericColor

SEPARATORS = '|'.join(map(re.escape, SEPARATOR_GROUP_ALL_ITEMS))

STYLE_METAVAR = GenericColor.YELLOW
STYLE_SWITCH = GenericColor.GREEN
STYLE_PROGRAM_NAME = GenericColor.GREEN  # .boldify()
STYLE_USAGE_OPTIONAL = GenericColor.GREY
STYLE_USAGE_REGULAR = GenericColor.WHITE
STYLE_USAGE_ERROR = GenericColor.RED
STYLE_USAGE_MISSING = GenericColor.YELLOW
STYLE_BOLD = 'bold'

MAX_CHOICE_CHARS = 80

LEFT_PADDING_2 = (0, 0, 0, 2)
LEFT_PADDING_3 = (0, 0, 0, 3)
LEFT_PADDING_4 = (0, 0, 0, 4)
LEFT_PADDING_5 = (0, 0, 0, 4)

LEFT_INDENT_2 = (1, 0, 0, 2)
LEFT_INDENT_3 = (1, 0, 0, 3)
LEFT_INDENT_BOTTOM_3 = (0, 0, 1, 3)

MORE_INFO_COMMANDS = """
To learn more, you can try:
    -> running 'http --manual'
    -> visiting our full documentation at https://httpie.io/docs/cli
"""


class OptionsHighlighter(RegexHighlighter):
    highlights = [
        r'(^|\W)(?P<option>\-{1,2}[\w|-]+)(?![a-zA-Z0-9])',
        r'(?P<bold>HTTPie)',
    ]


options_highlighter = OptionsHighlighter()


def unpack_argument(
    argument: Argument,
) -> Tuple[Text, Text]:
    opt1 = opt2 = ''

    style = None
    if argument.aliases:
        if len(argument.aliases) >= 2:
            opt2, opt1 = argument.aliases
        else:
            (opt1,) = argument.aliases
    else:
        opt1 = argument.metavar
        style = STYLE_USAGE_REGULAR

    return Text(opt1, style=style), Text(opt2)


def to_usage(
    spec: ParserSpec,
    *,
    program_name: Optional[str] = None,
    whitelist: AbstractSet[str] = frozenset()
) -> RenderableType:
    shown_arguments = [
        argument
        for group in spec.groups
        for argument in group.arguments
        if (not argument.aliases or whitelist.intersection(argument.aliases))
    ]

    # Sort the shown_arguments so that --dash options are
    # shown first
    shown_arguments.sort(key=lambda argument: argument.aliases, reverse=True)

    text = Text(program_name or spec.program, style=STYLE_BOLD)
    for argument in shown_arguments:
        text.append(' ')

        is_whitelisted = whitelist.intersection(argument.aliases)
        if argument.aliases:
            name = '/'.join(sorted(argument.aliases, key=len))
        else:
            name = argument.metavar

        nargs = argument.configuration.get('nargs')
        if nargs is Qualifiers.OPTIONAL:
            text.append('[' + name + ']', style=STYLE_USAGE_OPTIONAL)
        elif nargs is Qualifiers.ZERO_OR_MORE:
            text.append(
                '[' + name + ' ...]',
                style=STYLE_USAGE_OPTIONAL,
            )
        else:
            text.append(
                name,
                style=STYLE_USAGE_ERROR
                if is_whitelisted
                else STYLE_USAGE_REGULAR,
            )

        raw_form = argument.serialize()
        if raw_form.get('choices'):
            text.append(' ')
            text.append(
                '{' + ', '.join(raw_form['choices']) + '}',
                style=STYLE_USAGE_MISSING,
            )

    return text


# This part is loosely based on the rich-click's help message
# generation.
def to_help_message(
    spec: ParserSpec,
) -> Iterable[RenderableType]:
    yield Padding(
        options_highlighter(spec.description),
        LEFT_INDENT_2,
    )

    yield Padding(
        Text('Usage', style=STYLE_SWITCH),
        LEFT_INDENT_2,
    )
    yield Padding(to_usage(spec), LEFT_INDENT_3)

    group_rows = {}
    for group in spec.groups:
        options_rows = []

        for argument in group.arguments:
            if argument.is_hidden:
                continue

            opt1, opt2 = unpack_argument(argument)
            if opt2:
                opt1.append('/')
                opt1.append(opt2)

            # Column for a metavar, if we have one
            metavar = Text(style=STYLE_METAVAR)
            metavar.append(argument.configuration.get('metavar', ''))

            if opt1 == metavar:
                metavar = Text('')

            raw_form = argument.serialize()
            desc = raw_form.get('short_description', '')
            if raw_form.get('choices'):
                desc += ' (choices: '
                desc += textwrap.shorten(
                    ', '.join(raw_form.get('choices')),
                    MAX_CHOICE_CHARS,
                )
                desc += ')'

            rows = [
                Padding(
                    options_highlighter(opt1),
                    LEFT_PADDING_2,
                ),
                metavar,
                options_highlighter(desc),
            ]

            options_rows.append(rows)
            if argument.configuration.get('nested_options'):
                options_rows.extend(
                    [
                        (
                            Padding(
                                Text(
                                    key,
                                    style=STYLE_USAGE_OPTIONAL,
                                ),
                                LEFT_PADDING_4,
                            ),
                            value,
                            dec,
                        )
                        for key, value, dec in argument.nested_options
                    ]
                )

        group_rows[group.name] = options_rows

    options_table = Table(highlight=False, box=None, show_header=False)
    for group_name, options_rows in group_rows.items():
        options_table.add_row(Text(), Text(), Text())
        options_table.add_row(
            Text(group_name, style=STYLE_SWITCH),
            Text(),
            Text(),
        )
        options_table.add_row(Text(), Text(), Text())
        for row in options_rows:
            options_table.add_row(*row)

    yield Padding(
        Text('Options', style=STYLE_SWITCH),
        LEFT_INDENT_2,
    )
    yield Padding(options_table, LEFT_PADDING_2)
    yield Padding(
        Text('More Information', style=STYLE_SWITCH),
        LEFT_INDENT_2,
    )
    yield Padding(
        MORE_INFO_COMMANDS.rstrip('\n'),
        LEFT_PADDING_3
    )
    yield Padding(
        spec.epilog.rstrip('\n'),
        LEFT_INDENT_BOTTOM_3,
    )



File: httpie/output/ui/man_pages.py

"""Logic for checking and displaying man pages."""

import subprocess
import os
from httpie.context import Environment


MAN_COMMAND = 'man'
NO_MAN_PAGES = os.getenv('HTTPIE_NO_MAN_PAGES', False)

# On some systems, HTTP(n) might exist, but we are only interested in HTTP(1).
# For more information on man page sections: <https://unix.stackexchange.com/a/138643>
MAN_PAGE_SECTION = '1'


def is_available(program: str) -> bool:
    """
    Check whether `program`'s man pages are available on this system.

    """
    if NO_MAN_PAGES or os.system == 'nt':
        return False
    try:
        process = subprocess.run(
            [MAN_COMMAND, MAN_PAGE_SECTION, program],
            shell=False,
            stdout=subprocess.DEVNULL,
            stderr=subprocess.DEVNULL
        )
    except Exception:
        # There might be some errors outside the process, e.g
        # a permission error to execute something that is not an
        # executable.
        return False
    else:
        return process.returncode == 0


def display_for(env: Environment, program: str) -> None:
    """
    Open the system man page for the given command (http/https/httpie).

    """
    subprocess.run(
        [MAN_COMMAND, MAN_PAGE_SECTION, program],
        stdout=env.stdout,
        stderr=env.stderr
    )



File: httpie/output/ui/rich_palette.py

from collections import ChainMap
from typing import TYPE_CHECKING, Any, Optional

if TYPE_CHECKING:
    from rich.theme import Theme

from httpie.output.ui.palette import GenericColor, PieStyle, Styles, ColorString, _StyledGenericColor  # noqa

RICH_BOLD = ColorString('bold')

# Rich-specific color code declarations
# <https://github.com/Textualize/rich/blob/fcd684dd3a482977cab620e71ccaebb94bf13ac9/rich/default_styles.py>
CUSTOM_STYLES = {
    'progress.description': RICH_BOLD | GenericColor.WHITE,
    'progress.data.speed': RICH_BOLD | GenericColor.GREEN,
    'progress.percentage': RICH_BOLD | GenericColor.AQUA,
    'progress.download': RICH_BOLD | GenericColor.AQUA,
    'progress.remaining': RICH_BOLD | GenericColor.ORANGE,
    'bar.complete': RICH_BOLD | GenericColor.PURPLE,
    'bar.finished': RICH_BOLD | GenericColor.GREEN,
    'bar.pulse': RICH_BOLD | GenericColor.PURPLE,
    'option': RICH_BOLD | GenericColor.PINK,
}


class _GenericColorCaster(dict):
    """
    Translate GenericColor to a regular string on the attribute access
    phase.
    """

    def _translate(self, key: Any) -> Any:
        if isinstance(key, GenericColor):
            return key.name.lower()
        else:
            return key

    def __getitem__(self, key: Any) -> Any:
        return super().__getitem__(self._translate(key))

    def get(self, key: Any) -> Any:
        return super().get(self._translate(key))


def _make_rich_color_theme(style_name: Optional[str] = None) -> 'Theme':
    from rich.style import Style
    from rich.theme import Theme

    try:
        PieStyle(style_name)
    except ValueError:
        style = Styles.ANSI
    else:
        style = Styles.PIE

    theme = Theme()
    for color, color_set in ChainMap(
        GenericColor.__members__, CUSTOM_STYLES
    ).items():
        if isinstance(color_set, _StyledGenericColor):
            properties = dict.fromkeys(color_set.styles, True)
            color_set = color_set.color
        else:
            properties = {}

        theme.styles[color.lower()] = Style(
            color=color_set.apply_style(style, style_name=style_name),
            **properties,
        )

    # E.g translate GenericColor.BLUE into blue on key access
    theme.styles = _GenericColorCaster(theme.styles)
    return theme



File: httpie/output/ui/rich_progress.py

from dataclasses import dataclass
from typing import TYPE_CHECKING, Optional

from httpie.context import Environment

if TYPE_CHECKING:
    from rich.console import Console


@dataclass
class BaseDisplay:
    env: Environment

    def start(
        self, *, total: Optional[float], at: float, description: str
    ) -> None:
        ...

    def update(self, steps: float) -> None:
        ...

    def stop(self, time_spent: float) -> None:
        ...

    @property
    def console(self) -> 'Console':
        """Returns the default console to be used with displays (stderr)."""
        return self.env.rich_error_console

    def _print_summary(
        self, is_finished: bool, observed_steps: int, time_spent: float
    ):
        from rich import filesize

        if is_finished:
            verb = 'Done'
        else:
            verb = 'Interrupted'

        total_size = filesize.decimal(observed_steps)
        avg_speed = filesize.decimal(observed_steps / time_spent)

        minutes, seconds = divmod(time_spent, 60)
        hours, minutes = divmod(int(minutes), 60)
        if hours:
            total_time = f'{hours:d}:{minutes:02d}:{seconds:0.5f}'
        else:
            total_time = f'{minutes:02d}:{seconds:0.5f}'

        self.console.print(
            f'[progress.description]{verb}. {total_size} in {total_time} ({avg_speed}/s)'
        )


class DummyDisplay(BaseDisplay):
    """
    A dummy display object to be used when the progress bars,
    spinners etc. are disabled globally (or during tests).
    """


class StatusDisplay(BaseDisplay):
    def start(
        self, *, total: Optional[float], at: float, description: str
    ) -> None:
        self.observed = at
        self.description = (
            f'[progress.description]{description}[/progress.description]'
        )

        self.status = self.console.status(self.description, spinner='line')
        self.status.start()

    def update(self, steps: float) -> None:
        from rich import filesize

        self.observed += steps

        observed_amount, observed_unit = filesize.decimal(
            self.observed
        ).split()
        self.status.update(
            status=f'{self.description} [progress.download]{observed_amount}/? {observed_unit}[/progress.download]'
        )

    def stop(self, time_spent: float) -> None:
        self.status.stop()
        self.console.print(self.description)
        if time_spent:
            self._print_summary(
                is_finished=True,
                observed_steps=self.observed,
                time_spent=time_spent,
            )


class ProgressDisplay(BaseDisplay):
    def start(
        self, *, total: Optional[float], at: float, description: str
    ) -> None:
        from rich.progress import (
            Progress,
            BarColumn,
            DownloadColumn,
            TimeRemainingColumn,
            TransferSpeedColumn,
        )

        assert total is not None
        self.console.print(f'[progress.description]{description}')
        self.progress_bar = Progress(
            '[',
            BarColumn(),
            ']',
            '[progress.percentage]{task.percentage:>3.0f}%',
            '(',
            DownloadColumn(),
            ')',
            TimeRemainingColumn(),
            TransferSpeedColumn(),
            console=self.console,
            transient=True,
        )
        self.progress_bar.start()
        self.transfer_task = self.progress_bar.add_task(
            description, completed=at, total=total
        )

    def update(self, steps: float) -> None:
        self.progress_bar.advance(self.transfer_task, steps)

    def stop(self, time_spent: Optional[float]) -> None:
        self.progress_bar.stop()

        if time_spent:
            [task] = self.progress_bar.tasks
            self._print_summary(
                is_finished=task.finished,
                observed_steps=task.completed,
                time_spent=time_spent,
            )



File: httpie/output/lexers/metadata.py

import pygments

from httpie.models import ELAPSED_TIME_LABEL
from httpie.output.lexers.common import precise

SPEED_TOKENS = {
    0.45: pygments.token.Number.SPEED.FAST,
    1.00: pygments.token.Number.SPEED.AVG,
    2.50: pygments.token.Number.SPEED.SLOW,
}


def speed_based_token(lexer, match, ctx):
    try:
        value = float(match.group())
    except ValueError:
        return pygments.token.Number

    for limit, token in SPEED_TOKENS.items():
        if value <= limit:
            break
    else:
        token = pygments.token.Number.SPEED.VERY_SLOW

    response_type = precise(
        lexer,
        token,
        pygments.token.Number
    )
    yield match.start(), response_type, match.group()


class MetadataLexer(pygments.lexer.RegexLexer):
    """Simple HTTPie metadata lexer."""

    tokens = {
        'root': [
            (
                fr'({ELAPSED_TIME_LABEL})( *)(:)( *)(\d+\.\d+)(s)', pygments.lexer.bygroups(
                    pygments.token.Name.Decorator,  # Name
                    pygments.token.Text,
                    pygments.token.Operator,  # Colon
                    pygments.token.Text,
                    speed_based_token,
                    pygments.token.Name.Builtin  # Value
                )
            ),
            # Generic item
            (
                r'(.*?)( *)(:)( *)(.+)', pygments.lexer.bygroups(
                    pygments.token.Name.Decorator,  # Name
                    pygments.token.Text,
                    pygments.token.Operator,  # Colon
                    pygments.token.Text,
                    pygments.token.Text  # Value
                )
            ),
        ]
    }



File: httpie/output/lexers/__init__.py




File: httpie/output/lexers/common.py

def precise(lexer, precise_token, parent_token):
    # Due to a pygments bug*, custom tokens will look bad
    # on outside styles. Until it is fixed on upstream, we'll
    # convey whether the client is using pie style or not
    # through precise option and return more precise tokens
    # depending on it's value.
    #
    # [0]: https://github.com/pygments/pygments/issues/1986
    if precise_token is None or not lexer.options.get("precise"):
        return parent_token
    else:
        return precise_token



File: httpie/output/lexers/http.py

import re
import pygments
from httpie.output.lexers.common import precise

RE_STATUS_LINE = re.compile(r'(\d{3})( +)?(.+)?')

STATUS_TYPES = {
    '1': pygments.token.Number.HTTP.INFO,
    '2': pygments.token.Number.HTTP.OK,
    '3': pygments.token.Number.HTTP.REDIRECT,
    '4': pygments.token.Number.HTTP.CLIENT_ERR,
    '5': pygments.token.Number.HTTP.SERVER_ERR,
}

RESPONSE_TYPES = {
    'GET': pygments.token.Name.Function.HTTP.GET,
    'HEAD': pygments.token.Name.Function.HTTP.HEAD,
    'POST': pygments.token.Name.Function.HTTP.POST,
    'PUT': pygments.token.Name.Function.HTTP.PUT,
    'PATCH': pygments.token.Name.Function.HTTP.PATCH,
    'DELETE': pygments.token.Name.Function.HTTP.DELETE,
}


def http_response_type(lexer, match, ctx):
    status_match = RE_STATUS_LINE.match(match.group())
    if status_match is None:
        return None

    status_code, text, reason = status_match.groups()
    status_type = precise(
        lexer,
        STATUS_TYPES.get(status_code[0]),
        pygments.token.Number
    )

    groups = pygments.lexer.bygroups(
        status_type,
        pygments.token.Text,
        status_type
    )
    yield from groups(lexer, status_match, ctx)


def request_method(lexer, match, ctx):
    response_type = precise(
        lexer,
        RESPONSE_TYPES.get(match.group()),
        pygments.token.Name.Function
    )
    yield match.start(), response_type, match.group()


class SimplifiedHTTPLexer(pygments.lexer.RegexLexer):
    """Simplified HTTP lexer for Pygments.

    It only operates on headers and provides a stronger contrast between
    their names and values than the original one bundled with Pygments
    (:class:`pygments.lexers.text import HttpLexer`), especially when
    Solarized color scheme is used.

    """
    name = 'HTTP'
    aliases = ['http']
    filenames = ['*.http']
    tokens = {
        'root': [
            # Request-Line
            (r'([A-Z]+)( +)([^ ]+)( +)(HTTP)(/)(\d+\.\d+)',
             pygments.lexer.bygroups(
                 request_method,
                 pygments.token.Text,
                 pygments.token.Name.Namespace,
                 pygments.token.Text,
                 pygments.token.Keyword.Reserved,
                 pygments.token.Operator,
                 pygments.token.Number
             )),
            # Response Status-Line
            (r'(HTTP)(/)(\d+\.\d+)( +)(.+)',
             pygments.lexer.bygroups(
                 pygments.token.Keyword.Reserved,  # 'HTTP'
                 pygments.token.Operator,  # '/'
                 pygments.token.Number,  # Version
                 pygments.token.Text,
                 http_response_type,  # Status code and Reason
             )),
            # Header
            (r'(.*?)( *)(:)( *)(.+)', pygments.lexer.bygroups(
                pygments.token.Name.Attribute,  # Name
                pygments.token.Text,
                pygments.token.Operator,  # Colon
                pygments.token.Text,
                pygments.token.String  # Value
            ))
        ]
    }



File: httpie/output/lexers/json.py

import re

from pygments.lexer import bygroups, using, RegexLexer
from pygments.lexers.data import JsonLexer
from pygments.token import Token

PREFIX_TOKEN = Token.Error
PREFIX_REGEX = r'[^{\["]+'


class EnhancedJsonLexer(RegexLexer):
    """
    Enhanced JSON lexer for Pygments.

    It adds support for eventual data prefixing the actual JSON body.

    """
    name = 'JSON'
    flags = re.IGNORECASE | re.DOTALL
    tokens = {
        'root': [
            # Eventual non-JSON data prefix followed by actual JSON body.
            # FIX: data prefix + number (integer or float) is not correctly handled.
            (
                fr'({PREFIX_REGEX})' + r'((?:[{\["]|true|false|null).+)',
                bygroups(PREFIX_TOKEN, using(JsonLexer))
            ),
            # JSON body.
            (r'.+', using(JsonLexer)),
        ],
    }



File: httpie/output/formatters/xml.py

from typing import TYPE_CHECKING, Optional

from ...encoding import UTF8
from ...plugins import FormatterPlugin

if TYPE_CHECKING:
    from xml.dom.minidom import Document


XML_DECLARATION_OPEN = '<?xml'
XML_DECLARATION_CLOSE = '?>'


def parse_xml(data: str) -> 'Document':
    """Parse given XML `data` string into an appropriate :class:`~xml.dom.minidom.Document` object."""
    from defusedxml.minidom import parseString
    return parseString(data)


def parse_declaration(raw_body: str) -> Optional[str]:
    body = raw_body.strip()
    # XMLDecl ::= '<?xml' DECL_CONTENT '?>'
    if body.startswith(XML_DECLARATION_OPEN):
        end = body.find(XML_DECLARATION_CLOSE)
        if end != -1:
            return body[:end + len(XML_DECLARATION_CLOSE)]


def pretty_xml(document: 'Document',
               declaration: Optional[str] = None,
               encoding: Optional[str] = UTF8,
               indent: int = 2) -> str:
    """Render the given :class:`~xml.dom.minidom.Document` `document` into a prettified string."""
    kwargs = {
        'encoding': encoding or UTF8,
        'indent': ' ' * indent,
    }
    body = document.toprettyxml(**kwargs).decode(kwargs['encoding'])

    # Remove blank lines automatically added by `toprettyxml()`.
    lines = [line for line in body.splitlines() if line.strip()]

    # xml.dom automatically adds the declaration, even if
    # it is not present in the actual body. Remove it.
    if len(lines) >= 1 and parse_declaration(lines[0]):
        lines.pop(0)
        if declaration:
            lines.insert(0, declaration)

    return '\n'.join(lines)


class XMLFormatter(FormatterPlugin):

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.enabled = self.format_options['xml']['format']

    def format_body(self, body: str, mime: str):
        if 'xml' not in mime:
            return body

        from xml.parsers.expat import ExpatError
        from defusedxml.common import DefusedXmlException

        declaration = parse_declaration(body)
        try:
            parsed_body = parse_xml(body)
        except ExpatError:
            pass  # Invalid XML, ignore.
        except DefusedXmlException:
            pass  # Unsafe XML, ignore.
        else:
            body = pretty_xml(parsed_body,
                              encoding=parsed_body.encoding,
                              indent=self.format_options['xml']['indent'],
                              declaration=declaration)

        return body



File: httpie/output/formatters/__init__.py




File: httpie/output/formatters/headers.py

from ...plugins import FormatterPlugin


class HeadersFormatter(FormatterPlugin):

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.enabled = self.format_options['headers']['sort']

    def format_headers(self, headers: str) -> str:
        """
        Sorts headers by name while retaining relative
        order of multiple headers with the same name.

        """
        lines = headers.splitlines()
        headers = sorted(lines[1:], key=lambda h: h.split(':')[0])
        return '\r\n'.join(lines[:1] + headers)



File: httpie/output/formatters/json.py

import json

from ...plugins import FormatterPlugin


class JSONFormatter(FormatterPlugin):

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.enabled = self.format_options['json']['format']

    def format_body(self, body: str, mime: str) -> str:
        maybe_json = [
            'json',
            'javascript',
            'text',
        ]
        if (self.kwargs['explicit_json']
                or any(token in mime for token in maybe_json)):
            from ..utils import load_prefixed_json
            try:
                data_prefix, json_obj = load_prefixed_json(body)
            except ValueError:
                pass  # Invalid JSON, ignore.
            else:
                # Indent, sort keys by name, and avoid
                # unicode escapes to improve readability.
                body = data_prefix + json.dumps(
                    obj=json_obj,
                    sort_keys=self.format_options['json']['sort_keys'],
                    ensure_ascii=False,
                    indent=self.format_options['json']['indent']
                )
        return body



File: httpie/output/formatters/colors.py

import json
from typing import Optional, Type, Tuple

import pygments.formatters
import pygments.lexer
import pygments.lexers
import pygments.style
import pygments.styles
import pygments.token
from pygments.formatters.terminal import TerminalFormatter
from pygments.formatters.terminal256 import Terminal256Formatter
from pygments.lexer import Lexer
from pygments.lexers.data import JsonLexer
from pygments.lexers.special import TextLexer
from pygments.lexers.text import HttpLexer as PygmentsHttpLexer
from pygments.util import ClassNotFound

from ..lexers.json import EnhancedJsonLexer
from ..lexers.metadata import MetadataLexer
from ..ui.palette import AUTO_STYLE, SHADE_TO_PIE_STYLE, PieColor, ColorString, get_color
from ...context import Environment
from ...plugins import FormatterPlugin


DEFAULT_STYLE = AUTO_STYLE
SOLARIZED_STYLE = 'solarized'  # Bundled here
PYGMENTS_BOLD = ColorString('bold')
PYGMENTS_ITALIC = ColorString('italic')

BUNDLED_STYLES = {
    SOLARIZED_STYLE,
    AUTO_STYLE
}


def get_available_styles():
    return sorted(BUNDLED_STYLES | set(pygments.styles.get_all_styles()))


class ColorFormatter(FormatterPlugin):
    """
    Colorize using Pygments

    This processor that applies syntax highlighting to the headers,
    and also to the body if its content type is recognized.

    """
    group_name = 'colors'
    metadata_lexer = MetadataLexer()

    def __init__(
        self,
        env: Environment,
        explicit_json=False,
        color_scheme=DEFAULT_STYLE,
        **kwargs
    ):
        super().__init__(**kwargs)

        if not env.colors:
            self.enabled = False
            return

        use_auto_style = color_scheme == AUTO_STYLE
        has_256_colors = env.colors == 256
        if use_auto_style or not has_256_colors:
            http_lexer = PygmentsHttpLexer()
            body_formatter = header_formatter = TerminalFormatter()
            precise = False
        else:
            from ..lexers.http import SimplifiedHTTPLexer
            header_formatter, body_formatter, precise = self.get_formatters(color_scheme)
            http_lexer = SimplifiedHTTPLexer(precise=precise)

        self.explicit_json = explicit_json  # --json
        self.header_formatter = header_formatter
        self.body_formatter = body_formatter
        self.http_lexer = http_lexer
        self.metadata_lexer = MetadataLexer(precise=precise)

    def format_headers(self, headers: str) -> str:
        return pygments.highlight(
            code=headers,
            lexer=self.http_lexer,
            formatter=self.header_formatter,
        ).strip()

    def format_body(self, body: str, mime: str) -> str:
        lexer = self.get_lexer_for_body(mime, body)
        if lexer:
            body = pygments.highlight(
                code=body,
                lexer=lexer,
                formatter=self.body_formatter,
            )
        return body

    def format_metadata(self, metadata: str) -> str:
        return pygments.highlight(
            code=metadata,
            lexer=self.metadata_lexer,
            formatter=self.header_formatter,
        ).strip()

    def get_lexer_for_body(
        self, mime: str,
        body: str
    ) -> Optional[Type[Lexer]]:
        return get_lexer(
            mime=mime,
            explicit_json=self.explicit_json,
            body=body,
        )

    def get_formatters(self, color_scheme: str) -> Tuple[
        pygments.formatter.Formatter,
        pygments.formatter.Formatter,
        bool
    ]:
        if color_scheme in PIE_STYLES:
            header_style, body_style = PIE_STYLES[color_scheme]
            precise = True
        else:
            header_style = self.get_style_class(color_scheme)
            body_style = header_style
            precise = False

        return (
            Terminal256Formatter(style=header_style),
            Terminal256Formatter(style=body_style),
            precise
        )

    @staticmethod
    def get_style_class(color_scheme: str) -> Type[pygments.style.Style]:
        try:
            return pygments.styles.get_style_by_name(color_scheme)
        except ClassNotFound:
            return Solarized256Style


def get_lexer(
    mime: str,
    explicit_json=False,
    body=''
) -> Optional[Type[Lexer]]:
    # Build candidate mime type and lexer names.
    mime_types, lexer_names = [mime], []
    type_, subtype = mime.split('/', 1)
    if '+' not in subtype:
        lexer_names.append(subtype)
    else:
        subtype_name, subtype_suffix = subtype.split('+', 1)
        lexer_names.extend([subtype_name, subtype_suffix])
        mime_types.extend([
            f'{type_}/{subtype_name}',
            f'{type_}/{subtype_suffix}',
        ])

    # As a last resort, if no lexer feels responsible, and
    # the subtype contains 'json', take the JSON lexer
    if 'json' in subtype:
        lexer_names.append('json')

    # Try to resolve the right lexer.
    lexer = None
    for mime_type in mime_types:
        try:
            lexer = pygments.lexers.get_lexer_for_mimetype(mime_type)
            break
        except ClassNotFound:
            pass
    else:
        for name in lexer_names:
            try:
                lexer = pygments.lexers.get_lexer_by_name(name)
            except ClassNotFound:
                pass

    if explicit_json and body and (not lexer or isinstance(lexer, TextLexer)):
        # JSON response with an incorrect Content-Type?
        try:
            json.loads(body)  # FIXME: the body also gets parsed in json.py
        except ValueError:
            pass  # Nope
        else:
            lexer = pygments.lexers.get_lexer_by_name('json')

    # Use our own JSON lexer: it supports JSON bodies preceded by non-JSON data
    # as well as legit JSON bodies.
    if isinstance(lexer, JsonLexer):
        lexer = EnhancedJsonLexer()

    return lexer


class Solarized256Style(pygments.style.Style):
    """
    solarized256
    ------------

    A Pygments style inspired by Solarized's 256 color mode.

    :copyright: (c) 2011 by Hank Gay, (c) 2012 by John Mastro.
    :license: BSD, see LICENSE for more details.

    """
    BASE03 = "#1c1c1c"
    BASE02 = "#262626"
    BASE01 = "#4e4e4e"
    BASE00 = "#585858"
    BASE0 = "#808080"
    BASE1 = "#8a8a8a"
    BASE2 = "#d7d7af"
    BASE3 = "#ffffd7"
    YELLOW = "#af8700"
    ORANGE = "#d75f00"
    RED = "#af0000"
    MAGENTA = "#af005f"
    VIOLET = "#5f5faf"
    BLUE = "#0087ff"
    CYAN = "#00afaf"
    GREEN = "#5f8700"

    background_color = BASE03
    styles = {
        pygments.token.Keyword: GREEN,
        pygments.token.Keyword.Constant: ORANGE,
        pygments.token.Keyword.Declaration: BLUE,
        pygments.token.Keyword.Namespace: ORANGE,
        pygments.token.Keyword.Reserved: BLUE,
        pygments.token.Keyword.Type: RED,
        pygments.token.Name.Attribute: BASE1,
        pygments.token.Name.Builtin: BLUE,
        pygments.token.Name.Builtin.Pseudo: BLUE,
        pygments.token.Name.Class: BLUE,
        pygments.token.Name.Constant: ORANGE,
        pygments.token.Name.Decorator: BLUE,
        pygments.token.Name.Entity: ORANGE,
        pygments.token.Name.Exception: YELLOW,
        pygments.token.Name.Function: BLUE,
        pygments.token.Name.Tag: BLUE,
        pygments.token.Name.Variable: BLUE,
        pygments.token.String: CYAN,
        pygments.token.String.Backtick: BASE01,
        pygments.token.String.Char: CYAN,
        pygments.token.String.Doc: CYAN,
        pygments.token.String.Escape: RED,
        pygments.token.String.Heredoc: CYAN,
        pygments.token.String.Regex: RED,
        pygments.token.Number: CYAN,
        pygments.token.Operator: BASE1,
        pygments.token.Operator.Word: GREEN,
        pygments.token.Comment: BASE01,
        pygments.token.Comment.Preproc: GREEN,
        pygments.token.Comment.Special: GREEN,
        pygments.token.Generic.Deleted: CYAN,
        pygments.token.Generic.Emph: PYGMENTS_ITALIC,
        pygments.token.Generic.Error: RED,
        pygments.token.Generic.Heading: ORANGE,
        pygments.token.Generic.Inserted: GREEN,
        pygments.token.Generic.Strong: PYGMENTS_BOLD,
        pygments.token.Generic.Subheading: ORANGE,
        pygments.token.Token: BASE1,
        pygments.token.Token.Other: ORANGE,
    }


PIE_HEADER_STYLE = {
    # HTTP line / Headers / Etc.
    pygments.token.Name.Namespace: PYGMENTS_BOLD | PieColor.PRIMARY,
    pygments.token.Keyword.Reserved: PYGMENTS_BOLD | PieColor.GREY,
    pygments.token.Operator: PYGMENTS_BOLD | PieColor.GREY,
    pygments.token.Number: PYGMENTS_BOLD | PieColor.GREY,
    pygments.token.Name.Function.Magic: PYGMENTS_BOLD | PieColor.GREEN,
    pygments.token.Name.Exception: PYGMENTS_BOLD | PieColor.GREEN,
    pygments.token.Name.Attribute: PieColor.BLUE,
    pygments.token.String: PieColor.PRIMARY,

    # HTTP Methods
    pygments.token.Name.Function: PYGMENTS_BOLD | PieColor.GREY,
    pygments.token.Name.Function.HTTP.GET: PYGMENTS_BOLD | PieColor.GREEN,
    pygments.token.Name.Function.HTTP.HEAD: PYGMENTS_BOLD | PieColor.GREEN,
    pygments.token.Name.Function.HTTP.POST: PYGMENTS_BOLD | PieColor.YELLOW,
    pygments.token.Name.Function.HTTP.PUT: PYGMENTS_BOLD | PieColor.ORANGE,
    pygments.token.Name.Function.HTTP.PATCH: PYGMENTS_BOLD | PieColor.ORANGE,
    pygments.token.Name.Function.HTTP.DELETE: PYGMENTS_BOLD | PieColor.RED,

    # HTTP status codes
    pygments.token.Number.HTTP.INFO: PYGMENTS_BOLD | PieColor.AQUA,
    pygments.token.Number.HTTP.OK: PYGMENTS_BOLD | PieColor.GREEN,
    pygments.token.Number.HTTP.REDIRECT: PYGMENTS_BOLD | PieColor.YELLOW,
    pygments.token.Number.HTTP.CLIENT_ERR: PYGMENTS_BOLD | PieColor.ORANGE,
    pygments.token.Number.HTTP.SERVER_ERR: PYGMENTS_BOLD | PieColor.RED,

    # Metadata
    pygments.token.Name.Decorator: PieColor.GREY,
    pygments.token.Number.SPEED.FAST: PYGMENTS_BOLD | PieColor.GREEN,
    pygments.token.Number.SPEED.AVG: PYGMENTS_BOLD | PieColor.YELLOW,
    pygments.token.Number.SPEED.SLOW: PYGMENTS_BOLD | PieColor.ORANGE,
    pygments.token.Number.SPEED.VERY_SLOW: PYGMENTS_BOLD | PieColor.RED,
}

PIE_BODY_STYLE = {
    # {}[]:
    pygments.token.Punctuation: PieColor.GREY,

    # Keys
    pygments.token.Name.Tag: PieColor.PINK,

    # Values
    pygments.token.Literal.String: PieColor.GREEN,
    pygments.token.Literal.String.Double: PieColor.GREEN,
    pygments.token.Literal.Number: PieColor.AQUA,
    pygments.token.Keyword: PieColor.ORANGE,

    # Other stuff
    pygments.token.Text: PieColor.PRIMARY,
    pygments.token.Name.Attribute: PieColor.PRIMARY,
    pygments.token.Name.Builtin: PieColor.BLUE,
    pygments.token.Name.Builtin.Pseudo: PieColor.BLUE,
    pygments.token.Name.Class: PieColor.BLUE,
    pygments.token.Name.Constant: PieColor.ORANGE,
    pygments.token.Name.Decorator: PieColor.BLUE,
    pygments.token.Name.Entity: PieColor.ORANGE,
    pygments.token.Name.Exception: PieColor.YELLOW,
    pygments.token.Name.Function: PieColor.BLUE,
    pygments.token.Name.Variable: PieColor.BLUE,
    pygments.token.String: PieColor.AQUA,
    pygments.token.String.Backtick: PieColor.SECONDARY,
    pygments.token.String.Char: PieColor.AQUA,
    pygments.token.String.Doc: PieColor.AQUA,
    pygments.token.String.Escape: PieColor.RED,
    pygments.token.String.Heredoc: PieColor.AQUA,
    pygments.token.String.Regex: PieColor.RED,
    pygments.token.Number: PieColor.AQUA,
    pygments.token.Operator: PieColor.PRIMARY,
    pygments.token.Operator.Word: PieColor.GREEN,
    pygments.token.Comment: PieColor.SECONDARY,
    pygments.token.Comment.Preproc: PieColor.GREEN,
    pygments.token.Comment.Special: PieColor.GREEN,
    pygments.token.Generic.Deleted: PieColor.AQUA,
    pygments.token.Generic.Emph: PYGMENTS_ITALIC,
    pygments.token.Generic.Error: PieColor.RED,
    pygments.token.Generic.Heading: PieColor.ORANGE,
    pygments.token.Generic.Inserted: PieColor.GREEN,
    pygments.token.Generic.Strong: PYGMENTS_BOLD,
    pygments.token.Generic.Subheading: PieColor.ORANGE,
    pygments.token.Token: PieColor.PRIMARY,
    pygments.token.Token.Other: PieColor.ORANGE,
}


def make_style(name, raw_styles, shade):
    def format_value(value):
        return ' '.join(
            get_color(part, shade) or part
            for part in value.split()
        )

    bases = (pygments.style.Style,)
    data = {
        'styles': {
            key: format_value(value)
            for key, value in raw_styles.items()
        }
    }
    return type(name, bases, data)


def make_styles():
    styles = {}

    for shade, name in SHADE_TO_PIE_STYLE.items():
        styles[name] = [
            make_style(name, style_map, shade)
            for style_name, style_map in [
                (f'Pie{name}HeaderStyle', PIE_HEADER_STYLE),
                (f'Pie{name}BodyStyle', PIE_BODY_STYLE),
            ]
        ]

    return styles


PIE_STYLES = make_styles()
PIE_STYLE_NAMES = list(PIE_STYLES.keys())
BUNDLED_STYLES |= PIE_STYLES.keys()



File: httpie/legacy/v3_2_0_session_header_format.py

from typing import Any, Type, List, Dict, TYPE_CHECKING

if TYPE_CHECKING:
    from httpie.sessions import Session


OLD_HEADER_STORE_WARNING = '''\
Outdated layout detected for the current session. Please consider updating it,
in order to use the latest features regarding the header layout.

For fixing the current session:

    $ httpie cli sessions upgrade {hostname} {session_id}
'''

OLD_HEADER_STORE_WARNING_FOR_NAMED_SESSIONS = '''\

For fixing all named sessions:

    $ httpie cli sessions upgrade-all
'''

OLD_HEADER_STORE_LINK = '\nSee $INSERT_LINK for more information.'


def pre_process(session: 'Session', headers: Any) -> List[Dict[str, Any]]:
    """Serialize the headers into a unified form and issue a warning if
    the session file is using the old layout."""

    is_old_style = isinstance(headers, dict)
    if is_old_style:
        normalized_headers = list(headers.items())
    else:
        normalized_headers = [
            (item['name'], item['value'])
            for item in headers
        ]

    if is_old_style:
        warning = OLD_HEADER_STORE_WARNING.format(hostname=session.bound_host, session_id=session.session_id)
        if not session.is_anonymous:
            warning += OLD_HEADER_STORE_WARNING_FOR_NAMED_SESSIONS
        warning += OLD_HEADER_STORE_LINK
        session.warn_legacy_usage(warning)

    return normalized_headers


def post_process(
    normalized_headers: List[Dict[str, Any]],
    *,
    original_type: Type[Any]
) -> Any:
    """Deserialize given header store into the original form it was
    used in."""

    if issubclass(original_type, dict):
        # For the legacy behavior, preserve the last value.
        return {
            item['name']: item['value']
            for item in normalized_headers
        }
    else:
        return normalized_headers


def fix_layout(session: 'Session', *args, **kwargs) -> None:
    from httpie.sessions import materialize_headers

    if not isinstance(session['headers'], dict):
        return None

    session['headers'] = materialize_headers(session['headers'])



File: httpie/legacy/v3_1_0_session_cookie_format.py

import argparse
from typing import Any, Type, List, Dict, TYPE_CHECKING

if TYPE_CHECKING:
    from httpie.sessions import Session


INSECURE_COOKIE_JAR_WARNING = '''\
Outdated layout detected for the current session. Please consider updating it,
in order to not get affected by potential security problems.

For fixing the current session:

    With binding all cookies to the current host (secure):
        $ httpie cli sessions upgrade --bind-cookies {hostname} {session_id}

    Without binding cookies (leaving them as is) (insecure):
        $ httpie cli sessions upgrade {hostname} {session_id}
'''


INSECURE_COOKIE_JAR_WARNING_FOR_NAMED_SESSIONS = '''\

For fixing all named sessions:

    With binding all cookies to the current host (secure):
        $ httpie cli sessions upgrade-all --bind-cookies

    Without binding cookies (leaving them as is) (insecure):
        $ httpie cli sessions upgrade-all
'''

INSECURE_COOKIE_SECURITY_LINK = '\nSee https://pie.co/docs/security for more information.'


def pre_process(session: 'Session', cookies: Any) -> List[Dict[str, Any]]:
    """Load the given cookies to the cookie jar while maintaining
    support for the old cookie layout."""

    is_old_style = isinstance(cookies, dict)
    if is_old_style:
        normalized_cookies = [
            {
                'name': key,
                **value
            }
            for key, value in cookies.items()
        ]
    else:
        normalized_cookies = cookies

    should_issue_warning = is_old_style and any(
        cookie.get('domain', '') == ''
        for cookie in normalized_cookies
    )

    if should_issue_warning:
        warning = INSECURE_COOKIE_JAR_WARNING.format(hostname=session.bound_host, session_id=session.session_id)
        if not session.is_anonymous:
            warning += INSECURE_COOKIE_JAR_WARNING_FOR_NAMED_SESSIONS
        warning += INSECURE_COOKIE_SECURITY_LINK
        session.warn_legacy_usage(warning)

    return normalized_cookies


def post_process(
    normalized_cookies: List[Dict[str, Any]],
    *,
    original_type: Type[Any]
) -> Any:
    """Convert the cookies to their original format for
    maximum compatibility."""

    if issubclass(original_type, dict):
        return {
            cookie.pop('name'): cookie
            for cookie in normalized_cookies
        }
    else:
        return normalized_cookies


def fix_layout(session: 'Session', hostname: str, args: argparse.Namespace) -> None:
    if not isinstance(session['cookies'], dict):
        return None

    session['cookies'] = [
        {
            'name': key,
            **value
        }
        for key, value in session['cookies'].items()
    ]
    for cookie in session.cookies:
        if cookie.domain == '':
            if args.bind_cookies:
                cookie.domain = hostname
            else:
                cookie._rest['is_explicit_none'] = True



File: httpie/legacy/__init__.py




File: httpie/cli/argtypes.py

import argparse
import getpass
import os
import sys
from copy import deepcopy
from typing import List, Optional, Union

from .constants import DEFAULT_FORMAT_OPTIONS, SEPARATOR_CREDENTIALS
from ..sessions import VALID_SESSION_NAME_PATTERN


class KeyValueArg:
    """Base key-value pair parsed from CLI."""

    def __init__(self, key: str, value: Optional[str], sep: str, orig: str):
        self.key = key
        self.value = value
        self.sep = sep
        self.orig = orig

    def __eq__(self, other: 'KeyValueArg'):
        return self.__dict__ == other.__dict__

    def __repr__(self):
        return repr(self.__dict__)


class SessionNameValidator:

    def __init__(self, error_message: str):
        self.error_message = error_message

    def __call__(self, value: str) -> str:
        # Session name can be a path or just a name.
        if (os.path.sep not in value
                and not VALID_SESSION_NAME_PATTERN.search(value)):
            raise argparse.ArgumentError(None, self.error_message)
        return value


class Escaped(str):
    """Represents an escaped character."""

    def __repr__(self):
        return f"Escaped({repr(str(self))})"


class KeyValueArgType:
    """A key-value pair argument type used with `argparse`.

    Parses a key-value arg and constructs a `KeyValueArg` instance.
    Used for headers, form data, and other key-value pair types.

    """

    key_value_class = KeyValueArg

    def __init__(self, *separators: str):
        self.separators = separators
        self.special_characters = set()
        for separator in separators:
            self.special_characters.update(separator)

    def __call__(self, s: str) -> KeyValueArg:
        """Parse raw string arg and return `self.key_value_class` instance.

        The best of `self.separators` is determined (first found, longest).
        Back slash escaped characters aren't considered as separators
        (or parts thereof). Literal back slash characters have to be escaped
        as well (r'\\').

        """
        tokens = self.tokenize(s)

        # Sorting by length ensures that the longest one will be
        # chosen as it will overwrite any shorter ones starting
        # at the same position in the `found` dictionary.
        separators = sorted(self.separators, key=len)

        for i, token in enumerate(tokens):

            if isinstance(token, Escaped):
                continue

            found = {}
            for sep in separators:
                pos = token.find(sep)
                if pos != -1:
                    found[pos] = sep

            if found:
                # Starting first, longest separator found.
                sep = found[min(found.keys())]

                key, value = token.split(sep, 1)

                # Any preceding tokens are part of the key.
                key = ''.join(tokens[:i]) + key

                # Any following tokens are part of the value.
                value += ''.join(tokens[i + 1:])

                break

        else:
            raise argparse.ArgumentTypeError(f'{s!r} is not a valid value')

        return self.key_value_class(key=key, value=value, sep=sep, orig=s)

    def tokenize(self, s: str) -> List[Union[str, Escaped]]:
        r"""Tokenize the raw arg string

        There are only two token types - strings and escaped characters:

        >>> KeyValueArgType('=').tokenize(r'foo\=bar\\baz')
        ['foo', Escaped('='), 'bar\\\\baz']

        """
        tokens = ['']
        characters = iter(s)
        for char in characters:
            if char == '\\':
                char = next(characters, '')
                if char not in self.special_characters:
                    tokens[-1] += '\\' + char
                else:
                    tokens.extend([Escaped(char), ''])
            else:
                tokens[-1] += char
        return tokens


class PromptMixin:
    def _prompt_password(self, prompt: str) -> str:
        prompt_text = f'http: {prompt}: '
        try:
            return self._getpass(prompt_text)
        except (EOFError, KeyboardInterrupt):
            sys.stderr.write('\n')
            sys.exit(0)

    @staticmethod
    def _getpass(prompt):
        # To allow easy mocking.
        return getpass.getpass(str(prompt))


class SSLCredentials(PromptMixin):
    """Represents the passphrase for the certificate's key."""

    def __init__(self, value: Optional[str]) -> None:
        self.value = value

    def prompt_password(self, key_file: str) -> None:
        self.value = self._prompt_password(f'passphrase for {key_file}')


class AuthCredentials(KeyValueArg, PromptMixin):
    """Represents parsed credentials."""

    def has_password(self) -> bool:
        return self.value is not None

    def prompt_password(self, host: str) -> None:
        self.value = self._prompt_password(f'password for {self.key}@{host}:')


class AuthCredentialsArgType(KeyValueArgType):
    """A key-value arg type that parses credentials."""

    key_value_class = AuthCredentials

    def __call__(self, s):
        """Parse credentials from `s`.

        ("username" or "username:password").

        """
        try:
            return super().__call__(s)
        except argparse.ArgumentTypeError:
            # No password provided, will prompt for it later.
            return self.key_value_class(
                key=s,
                value=None,
                sep=SEPARATOR_CREDENTIALS,
                orig=s
            )


parse_auth = AuthCredentialsArgType(SEPARATOR_CREDENTIALS)


def readable_file_arg(filename):
    try:
        with open(filename, 'rb'):
            return filename
    except OSError as ex:
        raise argparse.ArgumentTypeError(f'{ex.filename}: {ex.strerror}')


def parse_format_options(s: str, defaults: Optional[dict]) -> dict:
    """
    Parse `s` and update `defaults` with the parsed values.

    >>> parse_format_options(
    ... defaults={'json': {'indent': 4, 'sort_keys': True}},
    ... s='json.indent:2,json.sort_keys:False',
    ... )
    {'json': {'indent': 2, 'sort_keys': False}}

    """
    value_map = {
        'true': True,
        'false': False,
    }
    options = deepcopy(defaults or {})
    for option in s.split(','):
        try:
            path, value = option.lower().split(':')
            section, key = path.split('.')
        except ValueError:
            raise argparse.ArgumentTypeError(f'invalid option {option!r}')

        if value in value_map:
            parsed_value = value_map[value]
        else:
            if value.isnumeric():
                parsed_value = int(value)
            else:
                parsed_value = value

        if defaults is None:
            options.setdefault(section, {})
        else:
            try:
                default_value = defaults[section][key]
            except KeyError:
                raise argparse.ArgumentTypeError(
                    f'invalid key {path!r}')

            default_type, parsed_type = type(default_value), type(parsed_value)
            if parsed_type is not default_type:
                raise argparse.ArgumentTypeError(
                    'invalid value'
                    f' {value!r} in {option!r}'
                    f' (expected {default_type.__name__}'
                    f' got {parsed_type.__name__})'
                )

        options[section][key] = parsed_value

    return options


PARSED_DEFAULT_FORMAT_OPTIONS = parse_format_options(
    s=','.join(DEFAULT_FORMAT_OPTIONS),
    defaults=None,
)


def response_charset_type(encoding: str) -> str:
    try:
        ''.encode(encoding)
    except LookupError:
        raise argparse.ArgumentTypeError(
            f'{encoding!r} is not a supported encoding')
    return encoding


def response_mime_type(mime_type: str) -> str:
    if mime_type.count('/') != 1:
        raise argparse.ArgumentTypeError(
            f'{mime_type!r} doesn’t look like a mime type; use type/subtype')
    return mime_type



File: httpie/cli/options.py

import argparse
import textwrap
import typing
from dataclasses import dataclass, field
from enum import Enum, auto
from typing import Any, Optional, Dict, List, Tuple, Type, TypeVar

from httpie.cli.argparser import HTTPieArgumentParser
from httpie.cli.utils import Manual, LazyChoices


class Qualifiers(Enum):
    OPTIONAL = auto()
    ZERO_OR_MORE = auto()
    ONE_OR_MORE = auto()
    SUPPRESS = auto()


def map_qualifiers(
    configuration: Dict[str, Any], qualifier_map: Dict[Qualifiers, Any]
) -> Dict[str, Any]:
    return {
        key: qualifier_map[value] if isinstance(value, Qualifiers) else value
        for key, value in configuration.items()
    }


def drop_keys(
    configuration: Dict[str, Any], key_blacklist: Tuple[str, ...]
):
    return {
        key: value
        for key, value in configuration.items()
        if key not in key_blacklist
    }


PARSER_SPEC_VERSION = '0.0.1a0'


@dataclass
class ParserSpec:
    program: str
    description: Optional[str] = None
    epilog: Optional[str] = None
    groups: List['Group'] = field(default_factory=list)
    man_page_hint: Optional[str] = None
    source_file: Optional[str] = None

    def finalize(self) -> 'ParserSpec':
        if self.description:
            self.description = textwrap.dedent(self.description)
        if self.epilog:
            self.epilog = textwrap.dedent(self.epilog)
        for group in self.groups:
            group.finalize()
        return self

    def add_group(self, name: str, **kwargs) -> 'Group':
        group = Group(name, **kwargs)
        self.groups.append(group)
        return group

    def serialize(self) -> Dict[str, Any]:
        return {
            'name': self.program,
            'description': self.description,
            'groups': [group.serialize() for group in self.groups],
        }


@dataclass
class Group:
    name: str
    description: str = ''
    is_mutually_exclusive: bool = False
    arguments: List['Argument'] = field(default_factory=list)

    def finalize(self) -> None:
        if self.description:
            self.description = textwrap.dedent(self.description)

    def add_argument(self, *args, **kwargs):
        argument = Argument(list(args), kwargs.copy())
        argument.post_init()
        self.arguments.append(argument)
        return argument

    def serialize(self) -> Dict[str, Any]:
        return {
            'name': self.name,
            'description': self.description or None,
            'is_mutually_exclusive': self.is_mutually_exclusive,
            'args': [argument.serialize() for argument in self.arguments],
        }


class Argument(typing.NamedTuple):
    aliases: List[str]
    configuration: Dict[str, Any]

    def post_init(self):
        """Run a bunch of post-init hooks."""
        # If there is a short help, then create the longer version from it.
        short_help = self.configuration.get('short_help')
        if (
            short_help
            and 'help' not in self.configuration
            and self.configuration.get('action') != 'lazy_choices'
        ):
            self.configuration['help'] = f'\n{short_help}\n\n'

    def serialize(self, *, isolation_mode: bool = False) -> Dict[str, Any]:
        configuration = self.configuration.copy()

        # Unpack the dynamically computed choices, since we
        # will need to store the actual values somewhere.
        action = configuration.pop('action', None)
        short_help = configuration.pop('short_help', None)
        nested_options = configuration.pop('nested_options', None)

        if action == 'lazy_choices':
            choices = LazyChoices(
                self.aliases,
                **{'dest': None, **configuration},
                isolation_mode=isolation_mode
            )
            configuration['choices'] = list(choices.load())
            configuration['help'] = choices.help

        result = {}
        if self.aliases:
            result['options'] = self.aliases.copy()
        else:
            result['options'] = [configuration['metavar']]
            result['is_positional'] = True

        qualifiers = JSON_QUALIFIER_TO_OPTIONS[configuration.get('nargs', Qualifiers.SUPPRESS)]
        result.update(qualifiers)

        description = configuration.get('help')
        if description and description is not Qualifiers.SUPPRESS:
            result['short_description'] = short_help
            result['description'] = description

        if nested_options:
            result['nested_options'] = nested_options

        python_type = configuration.get('type')
        if python_type is not None:
            if hasattr(python_type, '__name__'):
                type_name = python_type.__name__
            else:
                type_name = type(python_type).__name__

            result['python_type_name'] = type_name

        result.update({
            key: value
            for key, value in configuration.items()
            if key in JSON_DIRECT_MIRROR_OPTIONS
            if value is not Qualifiers.SUPPRESS
        })

        return result

    @property
    def is_positional(self):
        return len(self.aliases) == 0

    @property
    def is_hidden(self):
        return self.configuration.get('help') is Qualifiers.SUPPRESS

    def __getattr__(self, attribute_name):
        if attribute_name in self.configuration:
            return self.configuration[attribute_name]
        else:
            raise AttributeError(attribute_name)


ParserType = TypeVar('ParserType', bound=Type[argparse.ArgumentParser])

ARGPARSE_QUALIFIER_MAP = {
    Qualifiers.OPTIONAL: argparse.OPTIONAL,
    Qualifiers.SUPPRESS: argparse.SUPPRESS,
    Qualifiers.ZERO_OR_MORE: argparse.ZERO_OR_MORE,
    Qualifiers.ONE_OR_MORE: argparse.ONE_OR_MORE
}
ARGPARSE_IGNORE_KEYS = ('short_help', 'nested_options')


def to_argparse(
    abstract_options: ParserSpec,
    parser_type: ParserType = HTTPieArgumentParser,
) -> ParserType:
    concrete_parser = parser_type(
        prog=abstract_options.program,
        description=abstract_options.description,
        epilog=abstract_options.epilog,
    )
    concrete_parser.spec = abstract_options
    concrete_parser.register('action', 'lazy_choices', LazyChoices)
    concrete_parser.register('action', 'manual', Manual)

    for abstract_group in abstract_options.groups:
        concrete_group = concrete_parser.add_argument_group(
            title=abstract_group.name, description=abstract_group.description
        )
        if abstract_group.is_mutually_exclusive:
            concrete_group = concrete_group.add_mutually_exclusive_group(required=False)

        for abstract_argument in abstract_group.arguments:
            concrete_group.add_argument(
                *abstract_argument.aliases,
                **drop_keys(map_qualifiers(
                    abstract_argument.configuration, ARGPARSE_QUALIFIER_MAP
                ), ARGPARSE_IGNORE_KEYS)
            )

    return concrete_parser


JSON_DIRECT_MIRROR_OPTIONS = (
    'choices',
    'metavar'
)


JSON_QUALIFIER_TO_OPTIONS = {
    Qualifiers.OPTIONAL: {'is_optional': True},
    Qualifiers.ZERO_OR_MORE: {'is_optional': True, 'is_variadic': True},
    Qualifiers.ONE_OR_MORE: {'is_optional': False, 'is_variadic': True},
    Qualifiers.SUPPRESS: {}
}


def to_data(abstract_options: ParserSpec) -> Dict[str, Any]:
    return {'version': PARSER_SPEC_VERSION, 'spec': abstract_options.serialize()}


def parser_to_parser_spec(parser: argparse.ArgumentParser, **kwargs) -> ParserSpec:
    """Take an existing argparse parser, and create a spec from it."""
    return ParserSpec(
        program=parser.prog,
        description=parser.description,
        epilog=parser.epilog,
        **kwargs
    )



File: httpie/cli/requestitems.py

import os
import functools
from typing import Callable, Dict, IO, List, Optional, Tuple, Union

from .argtypes import KeyValueArg
from .constants import (
    SEPARATORS_GROUP_MULTIPART, SEPARATOR_DATA_EMBED_FILE_CONTENTS,
    SEPARATOR_DATA_EMBED_RAW_JSON_FILE, SEPARATOR_GROUP_NESTED_JSON_ITEMS,
    SEPARATOR_DATA_RAW_JSON, SEPARATOR_DATA_STRING, SEPARATOR_FILE_UPLOAD,
    SEPARATOR_FILE_UPLOAD_TYPE, SEPARATOR_HEADER, SEPARATOR_HEADER_EMPTY,
    SEPARATOR_HEADER_EMBED, SEPARATOR_QUERY_PARAM,
    SEPARATOR_QUERY_EMBED_FILE, RequestType
)
from .dicts import (
    BaseMultiDict, MultipartRequestDataDict, RequestDataDict,
    RequestFilesDict, HTTPHeadersDict, RequestJSONDataDict,
    RequestQueryParamsDict,
)
from .exceptions import ParseError
from .nested_json import interpret_nested_json
from ..utils import get_content_type, load_json_preserve_order_and_dupe_keys, split_iterable


class RequestItems:

    def __init__(self, request_type: Optional[RequestType] = None):
        self.headers = HTTPHeadersDict()
        self.request_type = request_type
        self.is_json = request_type is None or request_type is RequestType.JSON
        self.data = RequestJSONDataDict() if self.is_json else RequestDataDict()
        self.files = RequestFilesDict()
        self.params = RequestQueryParamsDict()
        # To preserve the order of fields in file upload multipart requests.
        self.multipart_data = MultipartRequestDataDict()

    @classmethod
    def from_args(
        cls,
        request_item_args: List[KeyValueArg],
        request_type: Optional[RequestType] = None,
    ) -> 'RequestItems':
        instance = cls(request_type=request_type)
        rules: Dict[str, Tuple[Callable, dict]] = {
            SEPARATOR_HEADER: (
                process_header_arg,
                instance.headers,
            ),
            SEPARATOR_HEADER_EMPTY: (
                process_empty_header_arg,
                instance.headers,
            ),
            SEPARATOR_HEADER_EMBED: (
                process_embed_header_arg,
                instance.headers,
            ),
            SEPARATOR_QUERY_PARAM: (
                process_query_param_arg,
                instance.params,
            ),
            SEPARATOR_QUERY_EMBED_FILE: (
                process_embed_query_param_arg,
                instance.params,
            ),
            SEPARATOR_FILE_UPLOAD: (
                process_file_upload_arg,
                instance.files,
            ),
            SEPARATOR_DATA_STRING: (
                process_data_item_arg,
                instance.data,
            ),
            SEPARATOR_DATA_EMBED_FILE_CONTENTS: (
                process_data_embed_file_contents_arg,
                instance.data,
            ),
            SEPARATOR_GROUP_NESTED_JSON_ITEMS: (
                process_data_nested_json_embed_args,
                instance.data,
            ),
            SEPARATOR_DATA_RAW_JSON: (
                convert_json_value_to_form_if_needed(
                    in_json_mode=instance.is_json,
                    processor=process_data_raw_json_embed_arg
                ),
                instance.data,
            ),
            SEPARATOR_DATA_EMBED_RAW_JSON_FILE: (
                convert_json_value_to_form_if_needed(
                    in_json_mode=instance.is_json,
                    processor=process_data_embed_raw_json_file_arg,
                ),
                instance.data,
            ),
        }

        if instance.is_json:
            json_item_args, request_item_args = split_iterable(
                iterable=request_item_args,
                key=lambda arg: arg.sep in SEPARATOR_GROUP_NESTED_JSON_ITEMS
            )
            if json_item_args:
                pairs = [(arg.key, rules[arg.sep][0](arg)) for arg in json_item_args]
                processor_func, target_dict = rules[SEPARATOR_GROUP_NESTED_JSON_ITEMS]
                value = processor_func(pairs)
                target_dict.update(value)

        # Then handle all other items.
        for arg in request_item_args:
            processor_func, target_dict = rules[arg.sep]
            value = processor_func(arg)

            if arg.sep in SEPARATORS_GROUP_MULTIPART:
                instance.multipart_data[arg.key] = value

            if isinstance(target_dict, BaseMultiDict):
                target_dict.add(arg.key, value)
            else:
                target_dict[arg.key] = value

        return instance


JSONType = Union[str, bool, int, list, dict]


def process_header_arg(arg: KeyValueArg) -> Optional[str]:
    return arg.value or None


def process_embed_header_arg(arg: KeyValueArg) -> str:
    return load_text_file(arg).rstrip('\n')


def process_empty_header_arg(arg: KeyValueArg) -> str:
    if not arg.value:
        return arg.value
    raise ParseError(
        f'Invalid item {arg.orig!r} (to specify an empty header use `Header;`)'
    )


def process_query_param_arg(arg: KeyValueArg) -> str:
    return arg.value


def process_embed_query_param_arg(arg: KeyValueArg) -> str:
    return load_text_file(arg).rstrip('\n')


def process_file_upload_arg(arg: KeyValueArg) -> Tuple[str, IO, str]:
    parts = arg.value.split(SEPARATOR_FILE_UPLOAD_TYPE)
    filename = parts[0]
    mime_type = parts[1] if len(parts) > 1 else None
    try:
        f = open(os.path.expanduser(filename), 'rb')
    except OSError as e:
        raise ParseError(f'{arg.orig!r}: {e}')
    return (
        os.path.basename(filename),
        f,
        mime_type or get_content_type(filename),
    )


def convert_json_value_to_form_if_needed(in_json_mode: bool, processor: Callable[[KeyValueArg], JSONType]) -> Callable[[], str]:
    """
    We allow primitive values to be passed to forms via JSON key/value syntax.

    But complex values lead to an error because there’s no clear way to serialize them.

    """
    if in_json_mode:
        return processor

    @functools.wraps(processor)
    def wrapper(*args, **kwargs) -> str:
        try:
            output = processor(*args, **kwargs)
        except ParseError:
            output = None
        if isinstance(output, (str, int, float)):
            return str(output)
        else:
            raise ParseError('Cannot use complex JSON value types with --form/--multipart.')

    return wrapper


def process_data_item_arg(arg: KeyValueArg) -> str:
    return arg.value


def process_data_embed_file_contents_arg(arg: KeyValueArg) -> str:
    return load_text_file(arg)


def process_data_embed_raw_json_file_arg(arg: KeyValueArg) -> JSONType:
    contents = load_text_file(arg)
    value = load_json(arg, contents)
    return value


def process_data_raw_json_embed_arg(arg: KeyValueArg) -> JSONType:
    value = load_json(arg, arg.value)
    return value


def process_data_nested_json_embed_args(pairs) -> Dict[str, JSONType]:
    return interpret_nested_json(pairs)


def load_text_file(item: KeyValueArg) -> str:
    path = item.value
    try:
        with open(os.path.expanduser(path), 'rb') as f:
            return f.read().decode()
    except OSError as e:
        raise ParseError(f'{item.orig!r}: {e}')
    except UnicodeDecodeError:
        raise ParseError(
            f'{item.orig!r}: cannot embed the content of {item.value!r},'
            ' not a UTF-8 or ASCII-encoded text file'
        )


def load_json(arg: KeyValueArg, contents: str) -> JSONType:
    try:
        return load_json_preserve_order_and_dupe_keys(contents)
    except ValueError as e:
        raise ParseError(f'{arg.orig!r}: {e}')



File: httpie/cli/constants.py

"""Parsing and processing of CLI input (args, auth credentials, files, stdin).

"""
import enum
import re


URL_SCHEME_RE = re.compile(r'^[a-z][a-z0-9.+-]*://', re.IGNORECASE)

HTTP_POST = 'POST'
HTTP_GET = 'GET'
HTTP_OPTIONS = 'OPTIONS'

# Various separators used in args
SEPARATOR_HEADER = ':'
SEPARATOR_HEADER_EMPTY = ';'
SEPARATOR_CREDENTIALS = ':'
SEPARATOR_PROXY = ':'
SEPARATOR_HEADER_EMBED = ':@'
SEPARATOR_DATA_STRING = '='
SEPARATOR_DATA_RAW_JSON = ':='
SEPARATOR_FILE_UPLOAD = '@'
SEPARATOR_FILE_UPLOAD_TYPE = ';type='  # in already parsed file upload path only
SEPARATOR_DATA_EMBED_FILE_CONTENTS = '=@'
SEPARATOR_DATA_EMBED_RAW_JSON_FILE = ':=@'
SEPARATOR_QUERY_PARAM = '=='
SEPARATOR_QUERY_EMBED_FILE = '==@'

# Separators that become request data
SEPARATOR_GROUP_DATA_ITEMS = frozenset({
    SEPARATOR_DATA_STRING,
    SEPARATOR_DATA_RAW_JSON,
    SEPARATOR_FILE_UPLOAD,
    SEPARATOR_DATA_EMBED_FILE_CONTENTS,
    SEPARATOR_DATA_EMBED_RAW_JSON_FILE
})

SEPARATORS_GROUP_MULTIPART = frozenset({
    SEPARATOR_DATA_STRING,
    SEPARATOR_DATA_EMBED_FILE_CONTENTS,
    SEPARATOR_FILE_UPLOAD,
})

# Separators for items whose value is a filename to be embedded
SEPARATOR_GROUP_DATA_EMBED_ITEMS = frozenset({
    SEPARATOR_HEADER_EMBED,
    SEPARATOR_QUERY_EMBED_FILE,
    SEPARATOR_DATA_EMBED_FILE_CONTENTS,
    SEPARATOR_DATA_EMBED_RAW_JSON_FILE,
})

# Separators for nested JSON items
SEPARATOR_GROUP_NESTED_JSON_ITEMS = frozenset([
    SEPARATOR_DATA_STRING,
    SEPARATOR_DATA_RAW_JSON,
    SEPARATOR_DATA_EMBED_FILE_CONTENTS,
    SEPARATOR_DATA_EMBED_RAW_JSON_FILE,
])

# Separators allowed in ITEM arguments
SEPARATOR_GROUP_ALL_ITEMS = frozenset({
    SEPARATOR_HEADER,
    SEPARATOR_HEADER_EMPTY,
    SEPARATOR_HEADER_EMBED,
    SEPARATOR_QUERY_PARAM,
    SEPARATOR_QUERY_EMBED_FILE,
    SEPARATOR_DATA_STRING,
    SEPARATOR_DATA_RAW_JSON,
    SEPARATOR_FILE_UPLOAD,
    SEPARATOR_DATA_EMBED_FILE_CONTENTS,
    SEPARATOR_DATA_EMBED_RAW_JSON_FILE,
})

# Output options
OUT_REQ_HEAD = 'H'
OUT_REQ_BODY = 'B'
OUT_RESP_HEAD = 'h'
OUT_RESP_BODY = 'b'
OUT_RESP_META = 'm'

BASE_OUTPUT_OPTIONS = frozenset({
    OUT_REQ_HEAD,
    OUT_REQ_BODY,
    OUT_RESP_HEAD,
    OUT_RESP_BODY,
})

OUTPUT_OPTIONS = frozenset({
    *BASE_OUTPUT_OPTIONS,
    OUT_RESP_META,
})

# Pretty


class PrettyOptions(enum.Enum):
    STDOUT_TTY_ONLY = enum.auto()


PRETTY_MAP = {
    'all': ['format', 'colors'],
    'colors': ['colors'],
    'format': ['format'],
    'none': []
}
PRETTY_STDOUT_TTY_ONLY = PrettyOptions.STDOUT_TTY_ONLY


DEFAULT_FORMAT_OPTIONS = [
    'headers.sort:true',
    'json.format:true',
    'json.indent:4',
    'json.sort_keys:true',
    'xml.format:true',
    'xml.indent:2',
]
SORTED_FORMAT_OPTIONS = [
    'headers.sort:true',
    'json.sort_keys:true',
]
SORTED_FORMAT_OPTIONS_STRING = ','.join(SORTED_FORMAT_OPTIONS)
UNSORTED_FORMAT_OPTIONS_STRING = ','.join(
    option.replace('true', 'false') for option in SORTED_FORMAT_OPTIONS)

# Defaults
OUTPUT_OPTIONS_DEFAULT = OUT_RESP_HEAD + OUT_RESP_BODY
OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED = OUT_RESP_BODY
OUTPUT_OPTIONS_DEFAULT_OFFLINE = OUT_REQ_HEAD + OUT_REQ_BODY


class RequestType(enum.Enum):
    FORM = enum.auto()
    MULTIPART = enum.auto()
    JSON = enum.auto()



File: httpie/cli/__init__.py




File: httpie/cli/utils.py

import argparse
from typing import Any, Callable, Generic, Iterator, Iterable, Optional, TypeVar

T = TypeVar('T')


class Manual(argparse.Action):
    def __init__(
        self,
        option_strings,
        dest=argparse.SUPPRESS,
        default=argparse.SUPPRESS,
        help=None
    ):
        super().__init__(
            option_strings=option_strings,
            dest=dest,
            default=default,
            nargs=0,
            help=help
        )

    def __call__(self, parser, namespace, values, option_string=None):
        parser.print_manual()
        parser.exit()


class LazyChoices(argparse.Action, Generic[T]):
    def __init__(
        self,
        *args,
        getter: Callable[[], Iterable[T]],
        help_formatter: Optional[Callable[[T, bool], str]] = None,
        sort: bool = False,
        cache: bool = True,
        isolation_mode: bool = False,
        **kwargs
    ) -> None:
        self.getter = getter
        self.help_formatter = help_formatter
        self.sort = sort
        self.cache = cache
        self.isolation_mode = isolation_mode
        self._help: Optional[str] = None
        self._obj: Optional[Iterable[T]] = None
        super().__init__(*args, **kwargs)
        self.choices = self

    def load(self) -> T:
        if self._obj is None or not self.cache:
            self._obj = self.getter()

        assert self._obj is not None
        return self._obj

    @property
    def help(self) -> str:
        if self._help is None and self.help_formatter is not None:
            self._help = self.help_formatter(
                self.load(),
                isolation_mode=self.isolation_mode
            )
        return self._help

    @help.setter
    def help(self, value: Any) -> None:
        self._help = value

    def __contains__(self, item: Any) -> bool:
        return item in self.load()

    def __iter__(self) -> Iterator[T]:
        if self.sort:
            return iter(sorted(self.load()))
        else:
            return iter(self.load())

    def __call__(self, parser, namespace, values, option_string=None):
        setattr(namespace, self.dest, values)



File: httpie/cli/definition.py

from __future__ import annotations

import os
import textwrap
from argparse import FileType

from httpie import __doc__, __version__
from httpie.cli.argtypes import (KeyValueArgType, SessionNameValidator,
                                 SSLCredentials, readable_file_arg,
                                 response_charset_type, response_mime_type)
from httpie.cli.constants import (BASE_OUTPUT_OPTIONS, DEFAULT_FORMAT_OPTIONS,
                                  OUT_REQ_BODY, OUT_REQ_HEAD, OUT_RESP_BODY,
                                  OUT_RESP_HEAD, OUT_RESP_META, OUTPUT_OPTIONS,
                                  OUTPUT_OPTIONS_DEFAULT, PRETTY_MAP,
                                  PRETTY_STDOUT_TTY_ONLY,
                                  SEPARATOR_GROUP_ALL_ITEMS, SEPARATOR_PROXY,
                                  SORTED_FORMAT_OPTIONS_STRING,
                                  UNSORTED_FORMAT_OPTIONS_STRING, RequestType)
from httpie.cli.options import ParserSpec, Qualifiers, to_argparse
from httpie.output.formatters.colors import (AUTO_STYLE, DEFAULT_STYLE, BUNDLED_STYLES,
                                             get_available_styles)
from httpie.plugins.builtin import BuiltinAuthPlugin
from httpie.plugins.registry import plugin_manager
from httpie.ssl_ import AVAILABLE_SSL_VERSION_ARG_MAPPING, DEFAULT_SSL_CIPHERS_STRING


# Man pages are static (built when making a release).
# We use this check to not include generated, system-specific information there (e.g., default --ciphers).
IS_MAN_PAGE = bool(os.environ.get('HTTPIE_BUILDING_MAN_PAGES'))


options = ParserSpec(
    'http',
    description=f'{__doc__.strip()} <https://httpie.io>',
    epilog="""
    For every --OPTION there is also a --no-OPTION that reverts OPTION
    to its default value.

    Suggestions and bug reports are greatly appreciated:
        https://github.com/httpie/cli/issues
    """,
    source_file=__file__
)

#######################################################################
# Positional arguments.
#######################################################################

positional_arguments = options.add_group(
    'Positional arguments',
    description="""
    These arguments come after any flags and in the order they are listed here.
    Only URL is required.
    """,
)

positional_arguments.add_argument(
    dest='method',
    metavar='METHOD',
    nargs=Qualifiers.OPTIONAL,
    default=None,
    short_help='The HTTP method to be used for the request (GET, POST, PUT, DELETE, ...).',
    help="""
    The HTTP method to be used for the request (GET, POST, PUT, DELETE, ...).

    This argument can be omitted in which case HTTPie will use POST if there
    is some data to be sent, otherwise GET:

        $ http example.org               # => GET
        $ http example.org hello=world   # => POST

    """,
)
positional_arguments.add_argument(
    dest='url',
    metavar='URL',
    short_help='The request URL.',
    help="""
    The request URL. Scheme defaults to 'http://' if the URL
    does not include one. (You can override this with: --default-scheme=http/https)

    You can also use a shorthand for localhost

        $ http :3000                    # => http://localhost:3000
        $ http :/foo                    # => http://localhost/foo

    """,
)
positional_arguments.add_argument(
    dest='request_items',
    metavar='REQUEST_ITEM',
    nargs=Qualifiers.ZERO_OR_MORE,
    default=None,
    type=KeyValueArgType(*SEPARATOR_GROUP_ALL_ITEMS),
    short_help=(
        'HTTPie’s request items syntax for specifying HTTP headers, JSON/Form'
        'data, files, and URL parameters.'
    ),
    nested_options=[
        ('HTTP Headers', 'Name:Value', 'Arbitrary HTTP header, e.g X-API-Token:123'),
        ('URL Parameters', 'name==value', 'Querystring parameter to the URL, e.g limit==50'),
        ('Data Fields', 'field=value', 'Data fields to be serialized as JSON (default) or Form Data (with --form)'),
        ('Raw JSON Fields', 'field:=json', 'Data field for real JSON types.'),
        ('File upload Fields', 'field@/dir/file', 'Path field for uploading a file.'),
    ],
    help=r"""
    Optional key-value pairs to be included in the request. The separator used
    determines the type:

    ':' HTTP headers:

        Referer:https://httpie.io  Cookie:foo=bar  User-Agent:bacon/1.0

    '==' URL parameters to be appended to the request URI:

        search==httpie

    '=' Data fields to be serialized into a JSON object (with --json, -j)
        or form data (with --form, -f):

        name=HTTPie  language=Python  description='CLI HTTP client'

    ':=' Non-string JSON data fields (only with --json, -j):

        awesome:=true  amount:=42  colors:='["red", "green", "blue"]'

    '@' Form file fields (only with --form or --multipart):

        cv@~/Documents/CV.pdf
        cv@'~/Documents/CV.pdf;type=application/pdf'

    '=@' A data field like '=', but takes a file path and embeds its content:

        essay=@Documents/essay.txt

    ':=@' A raw JSON field like ':=', but takes a file path and embeds its content:

        package:=@./package.json

    You can use a backslash to escape a colliding separator in the field name:

        field-name-with\:colon=value

    """,
)

#######################################################################
# Content type.
#######################################################################

content_types = options.add_group('Predefined content types')

content_types.add_argument(
    '--json',
    '-j',
    action='store_const',
    const=RequestType.JSON,
    dest='request_type',
    short_help='(default) Serialize data items from the command line as a JSON object.',
    help="""
    (default) Data items from the command line are serialized as a JSON object.
    The Content-Type and Accept headers are set to application/json
    (if not specified).

    """,
)
content_types.add_argument(
    '--form',
    '-f',
    action='store_const',
    const=RequestType.FORM,
    dest='request_type',
    short_help='Serialize data items from the command line as form field data.',
    help="""
    Data items from the command line are serialized as form fields.

    The Content-Type is set to application/x-www-form-urlencoded (if not
    specified). The presence of any file fields results in a
    multipart/form-data request.

    """,
)
content_types.add_argument(
    '--multipart',
    action='store_const',
    const=RequestType.MULTIPART,
    dest='request_type',
    short_help=(
        'Similar to --form, but always sends a multipart/form-data '
        'request (i.e., even without files).'
    )
)
content_types.add_argument(
    '--boundary',
    short_help=(
        'Specify a custom boundary string for multipart/form-data requests. '
        'Only has effect only together with --form.'
    )
)
content_types.add_argument(
    '--raw',
    short_help='Pass raw request data without extra processing.',
    help="""
    This option allows you to pass raw request data without extra processing
    (as opposed to the structured request items syntax):

        $ http --raw='data' pie.dev/post

    You can achieve the same by piping the data via stdin:

        $ echo data | http pie.dev/post

    Or have HTTPie load the raw data from a file:

        $ http pie.dev/post @data.txt


    """,
)

#######################################################################
# Content processing.
#######################################################################

processing_options = options.add_group('Content processing options')

processing_options.add_argument(
    '--compress',
    '-x',
    action='count',
    default=0,
    short_help='Compress the content with Deflate algorithm.',
    help="""
    Content compressed (encoded) with Deflate algorithm.
    The Content-Encoding header is set to deflate.

    Compression is skipped if it appears that compression ratio is
    negative. Compression can be forced by repeating the argument.

    """,
)


#######################################################################
# Output processing
#######################################################################


def format_style_help(available_styles, *, isolation_mode: bool = False):
    text = """
    Output coloring style (default is "{default}"). It can be one of:

        {available_styles}
    """
    if isolation_mode:
        text += '\n\n'
        text += 'For finding out all available styles in your system, try:\n\n'
        text += '    $ http --style\n'
    text += textwrap.dedent("""
        The "{auto_style}" style follows your terminal's ANSI color styles.
        For non-{auto_style} styles to work properly, please make sure that the
        $TERM environment variable is set to "xterm-256color" or similar
        (e.g., via `export TERM=xterm-256color' in your ~/.bashrc).
    """)

    if isolation_mode:
        available_styles = sorted(BUNDLED_STYLES)

    available_styles_text = '\n'.join(
        f'    {line.strip()}'
        for line in textwrap.wrap(', '.join(available_styles), 60)
    ).strip()
    return text.format(
        default=DEFAULT_STYLE,
        available_styles=available_styles_text,
        auto_style=AUTO_STYLE,
    )


_sorted_kwargs = {
    'action': 'append_const',
    'const': SORTED_FORMAT_OPTIONS_STRING,
    'dest': 'format_options',
}
_unsorted_kwargs = {
    'action': 'append_const',
    'const': UNSORTED_FORMAT_OPTIONS_STRING,
    'dest': 'format_options',
}

output_processing = options.add_group('Output processing')

output_processing.add_argument(
    '--pretty',
    dest='prettify',
    default=PRETTY_STDOUT_TTY_ONLY,
    choices=sorted(PRETTY_MAP.keys()),
    short_help='Control the processing of console outputs.',
    help="""
    Controls output processing. The value can be "none" to not prettify
    the output (default for redirected output), "all" to apply both colors
    and formatting (default for terminal output), "colors", or "format".

    """,
)
output_processing.add_argument(
    '--style',
    '-s',
    dest='style',
    metavar='STYLE',
    default=DEFAULT_STYLE,
    action='lazy_choices',
    getter=get_available_styles,
    short_help=f'Output coloring style (default is "{DEFAULT_STYLE}").',
    help_formatter=format_style_help,
)

# The closest approx. of the documented resetting to default via --no-<option>.
# We hide them from the doc because they act only as low-level aliases here.
output_processing.add_argument(
    '--no-unsorted', **_sorted_kwargs, help=Qualifiers.SUPPRESS
)
output_processing.add_argument(
    '--no-sorted', **_unsorted_kwargs, help=Qualifiers.SUPPRESS
)

output_processing.add_argument(
    '--unsorted',
    **_unsorted_kwargs,
    short_help='Disables all sorting while formatting output.',
    help=f"""
    Disables all sorting while formatting output. It is a shortcut for:

        --format-options={UNSORTED_FORMAT_OPTIONS_STRING}

    """,
)
output_processing.add_argument(
    '--sorted',
    **_sorted_kwargs,
    short_help='Re-enables all sorting options while formatting output.',
    help=f"""
    Re-enables all sorting options while formatting output. It is a shortcut for:

        --format-options={SORTED_FORMAT_OPTIONS_STRING}

    """,
)
output_processing.add_argument(
    '--response-charset',
    metavar='ENCODING',
    type=response_charset_type,
    short_help='Override the response encoding for terminal display purposes.',
    help="""
    Override the response encoding for terminal display purposes, e.g.:

        --response-charset=utf8
        --response-charset=big5

    """,
)
output_processing.add_argument(
    '--response-mime',
    metavar='MIME_TYPE',
    type=response_mime_type,
    short_help='Override the response mime type for coloring and formatting for the terminal.',
    help="""
    Override the response mime type for coloring and formatting for the terminal, e.g.:

        --response-mime=application/json
        --response-mime=text/xml

    """,
)
output_processing.add_argument(
    '--format-options',
    action='append',
    short_help='Controls output formatting.',
    help="""
    Controls output formatting. Only relevant when formatting is enabled
    through (explicit or implied) --pretty=all or --pretty=format.
    The following are the default options:

        {option_list}

    You may use this option multiple times, as well as specify multiple
    comma-separated options at the same time. For example, this modifies the
    settings to disable the sorting of JSON keys, and sets the indent size to 2:

        --format-options json.sort_keys:false,json.indent:2

    This is something you will typically put into your config file.

    """.format(
        option_list='\n'.join(
            f'        {option}' for option in DEFAULT_FORMAT_OPTIONS
        ).strip()
    ),
)

#######################################################################
# Output options
#######################################################################

output_options = options.add_group('Output options')

output_options.add_argument(
    '--print',
    '-p',
    dest='output_options',
    metavar='WHAT',
    short_help='Options to specify what the console output should contain.',
    help=f"""
    String specifying what the output should contain:

        '{OUT_REQ_HEAD}' request headers
        '{OUT_REQ_BODY}' request body
        '{OUT_RESP_HEAD}' response headers
        '{OUT_RESP_BODY}' response body
        '{OUT_RESP_META}' response metadata

    The default behaviour is '{OUTPUT_OPTIONS_DEFAULT}' (i.e., the response
    headers and body is printed), if standard output is not redirected.
    If the output is piped to another program or to a file, then only the
    response body is printed by default.

    """,
)
output_options.add_argument(
    '--headers',
    '-h',
    dest='output_options',
    action='store_const',
    const=OUT_RESP_HEAD,
    short_help='Print only the response headers.',
    help=f"""
    Print only the response headers. Shortcut for --print={OUT_RESP_HEAD}.

    """,
)
output_options.add_argument(
    '--meta',
    '-m',
    dest='output_options',
    action='store_const',
    const=OUT_RESP_META,
    short_help='Print only the response metadata.',
    help=f"""
    Print only the response metadata. Shortcut for --print={OUT_RESP_META}.

    """,
)
output_options.add_argument(
    '--body',
    '-b',
    dest='output_options',
    action='store_const',
    const=OUT_RESP_BODY,
    short_help='Print only the response body.',
    help=f"""
    Print only the response body. Shortcut for --print={OUT_RESP_BODY}.

    """,
)

output_options.add_argument(
    '--verbose',
    '-v',
    dest='verbose',
    action='count',
    default=0,
    short_help='Make output more verbose.',
    help=f"""
    Verbose output. For the level one (with single `-v`/`--verbose`), print
    the whole request as well as the response. Also print any intermediary
    requests/responses (such as redirects). For the second level and higher,
    print these as well as the response metadata.

    Level one is a shortcut for: --all --print={''.join(sorted(BASE_OUTPUT_OPTIONS))}
    Level two is a shortcut for: --all --print={''.join(sorted(OUTPUT_OPTIONS))}
    """,
)
output_options.add_argument(
    '--all',
    default=False,
    action='store_true',
    short_help='Show any intermediary requests/responses.',
    help="""
    By default, only the final request/response is shown. Use this flag to show
    any intermediary requests/responses as well. Intermediary requests include
    followed redirects (with --follow), the first unauthorized request when
    Digest auth is used (--auth=digest), etc.

    """,
)
output_options.add_argument(
    '--history-print',
    '-P',
    dest='output_options_history',
    metavar='WHAT',
    help=Qualifiers.SUPPRESS,
)
output_options.add_argument(
    '--stream',
    '-S',
    action='store_true',
    default=False,
    short_help='Always stream the response body by line, i.e., behave like `tail -f`.',
    help="""
    Always stream the response body by line, i.e., behave like `tail -f'.

    Without --stream and with --pretty (either set or implied),
    HTTPie fetches the whole response before it outputs the processed data.

    Set this option when you want to continuously display a prettified
    long-lived response, such as one from the Twitter streaming API.

    It is useful also without --pretty: It ensures that the output is flushed
    more often and in smaller chunks.

    """,
)
output_options.add_argument(
    '--output',
    '-o',
    type=FileType('a+b'),
    dest='output_file',
    metavar='FILE',
    short_help='Save output to FILE instead of stdout.',
    help="""
    Save output to FILE instead of stdout. If --download is also set, then only
    the response body is saved to FILE. Other parts of the HTTP exchange are
    printed to stderr.

    """,
)

output_options.add_argument(
    '--download',
    '-d',
    action='store_true',
    default=False,
    short_help='Download the body to a file instead of printing it to stdout.',
    help="""
    Do not print the response body to stdout. Rather, download it and store it
    in a file. The filename is guessed unless specified with --output
    [filename]. This action is similar to the default behaviour of wget.

    """,
)
output_options.add_argument(
    '--continue',
    '-c',
    dest='download_resume',
    action='store_true',
    default=False,
    short_help='Resume an interrupted download (--output needs to be specified).',
    help="""
    Resume an interrupted download. Note that the --output option needs to be
    specified as well.

    """,
)
output_options.add_argument(
    '--quiet',
    '-q',
    action='count',
    default=0,
    short_help='Do not print to stdout or stderr, except for errors and warnings when provided once.',
    help="""
    Do not print to stdout or stderr, except for errors and warnings when provided once.
    Provide twice to suppress warnings as well.
    stdout is still redirected if --output is specified.
    Flag doesn't affect behaviour of download beyond not printing to terminal.

    """,
)

#######################################################################
# Sessions
#######################################################################

session_name_validator = SessionNameValidator(
    'Session name contains invalid characters.'
)

sessions = options.add_group('Sessions', is_mutually_exclusive=True)

sessions.add_argument(
    '--session',
    metavar='SESSION_NAME_OR_PATH',
    type=session_name_validator,
    short_help='Create, or reuse and update a session.',
    help="""
    Create, or reuse and update a session. Within a session, custom headers,
    auth credential, as well as any cookies sent by the server persist between
    requests.

    Session files are stored in:

        [HTTPIE_CONFIG_DIR]/<HOST>/<SESSION_NAME>.json.

    See the following page to find out your default HTTPIE_CONFIG_DIR:

        https://httpie.io/docs/cli/config-file-directory
    """,
)
sessions.add_argument(
    '--session-read-only',
    metavar='SESSION_NAME_OR_PATH',
    type=session_name_validator,
    short_help='Create or read a session without updating it',
    help="""
    Create or read a session without updating it form the request/response
    exchange.

    """,
)


#######################################################################
# Authentication
#######################################################################


def format_auth_help(auth_plugins_mapping, *, isolation_mode: bool = False):
    text = """
    The authentication mechanism to be used. Defaults to "{default}".

    {auth_types}
    """

    auth_plugins = list(auth_plugins_mapping.values())
    if isolation_mode:
        auth_plugins = [
            auth_plugin
            for auth_plugin in auth_plugins
            if issubclass(auth_plugin, BuiltinAuthPlugin)
        ]
        text += '\n'
        text += 'To see all available auth types on your system, including ones installed via plugins, run:\n\n'
        text += '    $ http --auth-type'

    auth_types = '\n\n    '.join(
        '"{type}": {name}{package}{description}'.format(
            type=plugin.auth_type,
            name=plugin.name,
            package=(
                ''
                if issubclass(plugin, BuiltinAuthPlugin)
                else f' (provided by {plugin.package_name})'
            ),
            description=(
                ''
                if not plugin.description
                else '\n      '
                     + ('\n      '.join(textwrap.wrap(plugin.description)))
            ),
        )
        for plugin in auth_plugins
    )

    return text.format(
        default=auth_plugins[0].auth_type,
        auth_types=auth_types,
    )


authentication = options.add_group('Authentication')

authentication.add_argument(
    '--auth',
    '-a',
    default=None,
    metavar='USER[:PASS] | TOKEN',
    short_help='Credentials for the selected (-A) authentication method.',
    help="""
    For username/password based authentication mechanisms (e.g
    basic auth or digest auth) if only the username is provided
    (-a username), HTTPie will prompt for the password.

    """,
)
authentication.add_argument(
    '--auth-type',
    '-A',
    action='lazy_choices',
    default=None,
    getter=plugin_manager.get_auth_plugin_mapping,
    sort=True,
    cache=False,
    short_help='The authentication mechanism to be used.',
    help_formatter=format_auth_help,
)
authentication.add_argument(
    '--ignore-netrc',
    default=False,
    action='store_true',
    short_help='Ignore credentials from .netrc.'
)

#######################################################################
# Network
#######################################################################

network = options.add_group('Network')

network.add_argument(
    '--offline',
    default=False,
    action='store_true',
    short_help='Build the request and print it but don’t actually send it.'
)
network.add_argument(
    '--proxy',
    default=[],
    action='append',
    metavar='PROTOCOL:PROXY_URL',
    type=KeyValueArgType(SEPARATOR_PROXY),
    short_help='String mapping of protocol to the URL of the proxy.',
    help="""
    String mapping protocol to the URL of the proxy
    (e.g. http:http://foo.bar:3128). You can specify multiple proxies with
    different protocols. The environment variables $ALL_PROXY, $HTTP_PROXY,
    and $HTTPS_proxy are supported as well.

    """,
)
network.add_argument(
    '--follow',
    '-F',
    default=False,
    action='store_true',
    short_help='Follow 30x Location redirects.'
)

network.add_argument(
    '--max-redirects',
    type=int,
    default=30,
    short_help='The maximum number of redirects that should be followed (with --follow).',
    help="""
    By default, requests have a limit of 30 redirects (works with --follow).

    """,
)
network.add_argument(
    '--max-headers',
    type=int,
    default=0,
    short_help=(
        'The maximum number of response headers to be read before '
        'giving up (default 0, i.e., no limit).'
    )
)

network.add_argument(
    '--timeout',
    type=float,
    default=0,
    metavar='SECONDS',
    short_help='The connection timeout of the request in seconds.',
    help="""
    The connection timeout of the request in seconds.
    The default value is 0, i.e., there is no timeout limit.
    This is not a time limit on the entire response download;
    rather, an error is reported if the server has not issued a response for
    timeout seconds (more precisely, if no bytes have been received on
    the underlying socket for timeout seconds).

    """,
)
network.add_argument(
    '--check-status',
    default=False,
    action='store_true',
    short_help='Exit with an error status code if the server replies with an error.',
    help="""
    By default, HTTPie exits with 0 when no network or other fatal errors
    occur. This flag instructs HTTPie to also check the HTTP status code and
    exit with an error if the status indicates one.

    When the server replies with a 4xx (Client Error) or 5xx (Server Error)
    status code, HTTPie exits with 4 or 5 respectively. If the response is a
    3xx (Redirect) and --follow hasn't been set, then the exit status is 3.
    Also an error message is written to stderr if stdout is redirected.

    """,
)
network.add_argument(
    '--path-as-is',
    default=False,
    action='store_true',
    short_help='Bypass dot segment (/../ or /./) URL squashing.'
)
network.add_argument(
    '--chunked',
    default=False,
    action='store_true',
    short_help=(
        'Enable streaming via chunked transfer encoding. '
        'The Transfer-Encoding header is set to chunked.'
    )
)

#######################################################################
# SSL
#######################################################################

ssl = options.add_group('SSL')

ssl.add_argument(
    '--verify',
    default='yes',
    short_help='If "no", skip SSL verification. If a file path, use it as a CA bundle.',
    help="""
    Set to "no" (or "false") to skip checking the host's SSL certificate.
    Defaults to "yes" ("true"). You can also pass the path to a CA_BUNDLE file
    for private certs. (Or you can set the REQUESTS_CA_BUNDLE environment
    variable instead.)
    """,
)
ssl.add_argument(
    '--ssl',
    dest='ssl_version',
    choices=sorted(AVAILABLE_SSL_VERSION_ARG_MAPPING.keys()),
    short_help='The desired protocol version to used.',
    help="""
    The desired protocol version to use. This will default to
    SSL v2.3 which will negotiate the highest protocol that both
    the server and your installation of OpenSSL support. Available protocols
    may vary depending on OpenSSL installation (only the supported ones
    are shown here).

    """,
)

CIPHERS_CURRENT_DEFAULTS = (
    """
    See `http --help` for the default ciphers list on you system.

    """
    if IS_MAN_PAGE else
    f"""
    By default, the following ciphers are used on your system:

    {DEFAULT_SSL_CIPHERS_STRING}

    """
)
ssl.add_argument(
    '--ciphers',
    short_help='A string in the OpenSSL cipher list format.',
    help=f"""

    A string in the OpenSSL cipher list format.

    {CIPHERS_CURRENT_DEFAULTS}

    """
)
ssl.add_argument(
    '--cert',
    default=None,
    type=readable_file_arg,
    short_help='Specifies a local cert to use as the client-side SSL certificate.',
    help="""
    You can specify a local cert to use as client side SSL certificate.
    This file may either contain both private key and certificate or you may
    specify --cert-key separately.

    """,
)
ssl.add_argument(
    '--cert-key',
    default=None,
    type=readable_file_arg,
    short_help='The private key to use with SSL. Only needed if --cert is given.',
    help="""
    The private key to use with SSL. Only needed if --cert is given and the
    certificate file does not contain the private key.

    """,
)

ssl.add_argument(
    '--cert-key-pass',
    default=None,
    type=SSLCredentials,
    short_help='The passphrase to be used to with the given private key.',
    help="""
    The passphrase to be used to with the given private key. Only needed if --cert-key
    is given and the key file requires a passphrase.
    If not provided, you’ll be prompted interactively.
    """
)

#######################################################################
# Troubleshooting
#######################################################################

troubleshooting = options.add_group('Troubleshooting')
troubleshooting.add_argument(
    '--ignore-stdin',
    '-I',
    action='store_true',
    default=False,
    short_help='Do not attempt to read stdin'
)
troubleshooting.add_argument(
    '--help',
    action='help',
    default=Qualifiers.SUPPRESS,
    short_help='Show this help message and exit.',
)
troubleshooting.add_argument(
    '--manual',
    action='manual',
    default=Qualifiers.SUPPRESS,
    short_help='Show the full manual.',
)
troubleshooting.add_argument(
    '--version',
    action='version',
    version=__version__,
    short_help='Show version and exit.',
)
troubleshooting.add_argument(
    '--traceback',
    action='store_true',
    default=False,
    short_help='Prints the exception traceback should one occur.',
)
troubleshooting.add_argument(
    '--default-scheme',
    default='http',
    short_help='The default scheme to use if not specified in the URL.'
)
troubleshooting.add_argument(
    '--debug',
    action='store_true',
    default=False,
    short_help='Print useful diagnostic information for bug reports.',
    help="""
    Prints the exception traceback should one occur, as well as other
    information useful for debugging HTTPie itself and for reporting bugs.

    """,
)

#######################################################################
# Finalization
#######################################################################

options.finalize()
parser = to_argparse(options)



File: httpie/cli/exceptions.py

class ParseError(Exception):
    pass



File: httpie/cli/argparser.py

import argparse
import errno
import os
import re
import sys
from argparse import RawDescriptionHelpFormatter
from textwrap import dedent
from urllib.parse import urlsplit

from requests.utils import get_netrc_auth

from .argtypes import (
    AuthCredentials, SSLCredentials, KeyValueArgType,
    PARSED_DEFAULT_FORMAT_OPTIONS,
    parse_auth,
    parse_format_options,
)
from .constants import (
    HTTP_GET, HTTP_POST, BASE_OUTPUT_OPTIONS, OUTPUT_OPTIONS, OUTPUT_OPTIONS_DEFAULT,
    OUTPUT_OPTIONS_DEFAULT_OFFLINE, OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED,
    OUT_RESP_BODY, PRETTY_MAP, PRETTY_STDOUT_TTY_ONLY, RequestType,
    SEPARATOR_CREDENTIALS,
    SEPARATOR_GROUP_ALL_ITEMS, SEPARATOR_GROUP_DATA_ITEMS, URL_SCHEME_RE,
)
from .exceptions import ParseError
from .requestitems import RequestItems
from ..context import Environment
from ..plugins.registry import plugin_manager
from ..utils import ExplicitNullAuth, get_content_type


class HTTPieHelpFormatter(RawDescriptionHelpFormatter):
    """A nicer help formatter.

    Help for arguments can be indented and contain new lines.
    It will be de-dented and arguments in the help
    will be separated by a blank line for better readability.


    """

    def __init__(self, max_help_position=6, *args, **kwargs):
        # A smaller indent for args help.
        kwargs['max_help_position'] = max_help_position
        super().__init__(*args, **kwargs)

    def _split_lines(self, text, width):
        text = dedent(text).strip() + '\n\n'
        return text.splitlines()

    def add_usage(self, usage, actions, groups, prefix=None):
        # Only display the positional arguments
        displayed_actions = [
            action
            for action in actions
            if not action.option_strings
        ]

        _, exception, _ = sys.exc_info()
        if (
            isinstance(exception, argparse.ArgumentError)
            and len(exception.args) >= 1
            and isinstance(exception.args[0], argparse.Action)
        ):
            # add_usage path is also taken when you pass an invalid option,
            # e.g --style=invalid. If something like that happens, we want
            # to include to action that caused to the invalid usage into
            # the list of actions we are displaying.
            displayed_actions.insert(0, exception.args[0])

        super().add_usage(
            usage,
            displayed_actions,
            groups,
            prefix="usage:\n    "
        )


# TODO: refactor and design type-annotated data structures
#       for raw args + parsed args and keep things immutable.
class BaseHTTPieArgumentParser(argparse.ArgumentParser):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.env = None
        self.args = None
        self.has_stdin_data = False
        self.has_input_data = False

    # noinspection PyMethodOverriding
    def parse_args(
        self,
        env: Environment,
        args=None,
        namespace=None
    ) -> argparse.Namespace:
        self.env = env
        self.args, no_options = self.parse_known_args(args, namespace)
        if self.args.debug:
            self.args.traceback = True
        self.has_stdin_data = (
            self.env.stdin
            and not getattr(self.args, 'ignore_stdin', False)
            and not self.env.stdin_isatty
        )
        self.has_input_data = self.has_stdin_data or getattr(self.args, 'raw', None) is not None
        return self.args

    # noinspection PyShadowingBuiltins
    def _print_message(self, message, file=None):
        # Sneak in our stderr/stdout.
        if hasattr(self, 'root'):
            env = self.root.env
        else:
            env = self.env

        if env is not None:
            file = {
                sys.stdout: env.stdout,
                sys.stderr: env.stderr,
                None: env.stderr
            }.get(file, file)

        if not hasattr(file, 'buffer') and isinstance(message, str):
            message = message.encode(env.stdout_encoding)
        super()._print_message(message, file)


class HTTPieManagerArgumentParser(BaseHTTPieArgumentParser):
    def parse_known_args(self, args=None, namespace=None):
        try:
            return super().parse_known_args(args, namespace)
        except SystemExit as exc:
            if not hasattr(self, 'root') and exc.code == 2:  # Argument Parser Error
                raise argparse.ArgumentError(None, None)
            raise


class HTTPieArgumentParser(BaseHTTPieArgumentParser):
    """Adds additional logic to `argparse.ArgumentParser`.

    Handles all input (CLI args, file args, stdin), applies defaults,
    and performs extra validation.

    """

    def __init__(self, *args, formatter_class=HTTPieHelpFormatter, **kwargs):
        kwargs.setdefault('add_help', False)
        super().__init__(*args, formatter_class=formatter_class, **kwargs)

    # noinspection PyMethodOverriding
    def parse_args(
        self,
        env: Environment,
        args=None,
        namespace=None
    ) -> argparse.Namespace:
        self.env = env
        self.env.args = namespace = namespace or argparse.Namespace()
        self.args, no_options = super().parse_known_args(args, namespace)
        if self.args.debug:
            self.args.traceback = True
        self.has_stdin_data = (
            self.env.stdin
            and not self.args.ignore_stdin
            and not self.env.stdin_isatty
        )
        self.has_input_data = self.has_stdin_data or self.args.raw is not None
        # Arguments processing and environment setup.
        self._apply_no_options(no_options)
        self._process_request_type()
        self._process_download_options()
        self._setup_standard_streams()
        self._process_output_options()
        self._process_pretty_options()
        self._process_format_options()
        self._guess_method()
        self._parse_items()
        self._process_url()
        self._process_auth()
        self._process_ssl_cert()

        if self.args.raw is not None:
            self._body_from_input(self.args.raw)
        elif self.has_stdin_data:
            self._body_from_file(self.env.stdin)

        if self.args.compress:
            # TODO: allow --compress with --chunked / --multipart
            if self.args.chunked:
                self.error('cannot combine --compress and --chunked')
            if self.args.multipart:
                self.error('cannot combine --compress and --multipart')

        return self.args

    def _process_request_type(self):
        request_type = self.args.request_type
        self.args.json = request_type is RequestType.JSON
        self.args.multipart = request_type is RequestType.MULTIPART
        self.args.form = request_type in {
            RequestType.FORM,
            RequestType.MULTIPART,
        }

    def _process_url(self):
        if self.args.url.startswith('://'):
            # Paste URL & add space shortcut: `http ://pie.dev` → `http://pie.dev`
            self.args.url = self.args.url[3:]
        if not URL_SCHEME_RE.match(self.args.url):
            if os.path.basename(self.env.program_name) == 'https':
                scheme = 'https://'
            else:
                scheme = self.args.default_scheme + '://'

            # See if we're using curl style shorthand for localhost (:3000/foo)
            shorthand = re.match(r'^:(?!:)(\d*)(/?.*)$', self.args.url)
            if shorthand:
                port = shorthand.group(1)
                rest = shorthand.group(2)
                self.args.url = scheme + 'localhost'
                if port:
                    self.args.url += ':' + port
                self.args.url += rest
            else:
                self.args.url = scheme + self.args.url

    def _setup_standard_streams(self):
        """
        Modify `env.stdout` and `env.stdout_isatty` based on args, if needed.

        """

        self.args.output_file_specified = bool(self.args.output_file)
        if self.args.download:
            # FIXME: Come up with a cleaner solution.
            if not self.args.output_file and not self.env.stdout_isatty:
                # Use stdout as the download output file.
                self.args.output_file = self.env.stdout
            # With `--download`, we write everything that would normally go to
            # `stdout` to `stderr` instead. Let's replace the stream so that
            # we don't have to use many `if`s throughout the codebase.
            # The response body will be treated separately.
            self.env.stdout = self.env.stderr
            self.env.stdout_isatty = self.env.stderr_isatty

        elif self.args.output_file:
            # When not `--download`ing, then `--output` simply replaces
            # `stdout`. The file is opened for appending, which isn't what
            # we want in this case.
            self.args.output_file.seek(0)
            try:
                self.args.output_file.truncate()
            except OSError as e:
                if e.errno == errno.EINVAL:
                    # E.g. /dev/null on Linux.
                    pass
                else:
                    raise
            self.env.stdout = self.args.output_file
            self.env.stdout_isatty = False

        if self.args.quiet:
            self.env.quiet = self.args.quiet
            self.env.stderr = self.env.devnull
            if not (self.args.output_file_specified and not self.args.download):
                self.env.stdout = self.env.devnull
            self.env.apply_warnings_filter()

    def _process_ssl_cert(self):
        from httpie.ssl_ import _is_key_file_encrypted

        if self.args.cert_key_pass is None:
            self.args.cert_key_pass = SSLCredentials(None)

        if (
            self.args.cert_key is not None
            and self.args.cert_key_pass.value is None
            and _is_key_file_encrypted(self.args.cert_key)
        ):
            self.args.cert_key_pass.prompt_password(self.args.cert_key)

    def _process_auth(self):
        # TODO: refactor & simplify this method.
        self.args.auth_plugin = None
        default_auth_plugin = plugin_manager.get_auth_plugins()[0]
        auth_type_set = self.args.auth_type is not None
        url = urlsplit(self.args.url)

        if self.args.auth is None and not auth_type_set:
            if url.username is not None:
                # Handle http://username:password@hostname/
                username = url.username
                password = url.password or ''
                self.args.auth = AuthCredentials(
                    key=username,
                    value=password,
                    sep=SEPARATOR_CREDENTIALS,
                    orig=SEPARATOR_CREDENTIALS.join([username, password])
                )

        if self.args.auth is not None or auth_type_set:
            if not self.args.auth_type:
                self.args.auth_type = default_auth_plugin.auth_type
            plugin = plugin_manager.get_auth_plugin(self.args.auth_type)()

            if (not self.args.ignore_netrc
                    and self.args.auth is None
                    and plugin.netrc_parse):
                # Only host needed, so it’s OK URL not finalized.
                netrc_credentials = get_netrc_auth(self.args.url)
                if netrc_credentials:
                    self.args.auth = AuthCredentials(
                        key=netrc_credentials[0],
                        value=netrc_credentials[1],
                        sep=SEPARATOR_CREDENTIALS,
                        orig=SEPARATOR_CREDENTIALS.join(netrc_credentials)
                    )

            if plugin.auth_require and self.args.auth is None:
                self.error('--auth required')

            plugin.raw_auth = self.args.auth
            self.args.auth_plugin = plugin
            already_parsed = isinstance(self.args.auth, AuthCredentials)

            if self.args.auth is None or not plugin.auth_parse:
                self.args.auth = plugin.get_auth()
            else:
                if already_parsed:
                    # from the URL
                    credentials = self.args.auth
                else:
                    credentials = parse_auth(self.args.auth)

                if (not credentials.has_password()
                        and plugin.prompt_password):
                    if self.args.ignore_stdin:
                        # Non-tty stdin read by now
                        self.error(
                            'Unable to prompt for passwords because'
                            ' --ignore-stdin is set.'
                        )
                    credentials.prompt_password(url.netloc)

                if (credentials.key and credentials.value):
                    plugin.raw_auth = credentials.key + ":" + credentials.value

                self.args.auth = plugin.get_auth(
                    username=credentials.key,
                    password=credentials.value,
                )
        if not self.args.auth and self.args.ignore_netrc:
            # Set a no-op auth to force requests to ignore .netrc
            # <https://github.com/psf/requests/issues/2773#issuecomment-174312831>
            self.args.auth = ExplicitNullAuth()

    def _apply_no_options(self, no_options):
        """For every `--no-OPTION` in `no_options`, set `args.OPTION` to
        its default value. This allows for un-setting of options, e.g.,
        specified in config.

        """
        invalid = []

        for option in no_options:
            if not option.startswith('--no-'):
                invalid.append(option)
                continue

            # --no-option => --option
            inverted = '--' + option[5:]
            for action in self._actions:
                if inverted in action.option_strings:
                    setattr(self.args, action.dest, action.default)
                    break
            else:
                invalid.append(option)

        if invalid:
            self.error(f'unrecognized arguments: {" ".join(invalid)}')

    def _body_from_file(self, fd):
        """Read the data from a file-like object.

        Bytes are always read.

        """
        self._ensure_one_data_source(self.args.data, self.args.files)
        self.args.data = getattr(fd, 'buffer', fd)

    def _body_from_input(self, data):
        """Read the data from the CLI.

        """
        self._ensure_one_data_source(self.has_stdin_data, self.args.data,
                                     self.args.files)
        self.args.data = data.encode()

    def _ensure_one_data_source(self, *other_sources):
        """There can only be one source of input request data.

        """
        if any(other_sources):
            self.error('Request body (from stdin, --raw or a file) and request '
                       'data (key=value) cannot be mixed. Pass '
                       '--ignore-stdin to let key/value take priority. '
                       'See https://httpie.io/docs#scripting for details.')

    def _guess_method(self):
        """Set `args.method` if not specified to either POST or GET
        based on whether the request has data or not.

        """
        if self.args.method is None:
            # Invoked as `http URL'.
            assert not self.args.request_items
            if self.has_input_data:
                self.args.method = HTTP_POST
            else:
                self.args.method = HTTP_GET

        # FIXME: False positive, e.g., "localhost" matches but is a valid URL.
        elif not re.match('^[a-zA-Z]+$', self.args.method):
            # Invoked as `http URL item+'. The URL is now in `args.method`
            # and the first ITEM is now incorrectly in `args.url`.
            try:
                # Parse the URL as an ITEM and store it as the first ITEM arg.
                self.args.request_items.insert(0, KeyValueArgType(
                    *SEPARATOR_GROUP_ALL_ITEMS).__call__(self.args.url))

            except argparse.ArgumentTypeError as e:
                if self.args.traceback:
                    raise
                self.error(e.args[0])

            else:
                # Set the URL correctly
                self.args.url = self.args.method
                # Infer the method
                has_data = (
                    self.has_input_data
                    or any(
                        item.sep in SEPARATOR_GROUP_DATA_ITEMS
                        for item in self.args.request_items)
                )
                self.args.method = HTTP_POST if has_data else HTTP_GET

    def _parse_items(self):
        """
        Parse `args.request_items` into `args.headers`, `args.data`,
        `args.params`, and `args.files`.

        """
        try:
            request_items = RequestItems.from_args(
                request_item_args=self.args.request_items,
                request_type=self.args.request_type,
            )
        except ParseError as e:
            if self.args.traceback:
                raise
            self.error(e.args[0])
        else:
            self.args.headers = request_items.headers
            self.args.data = request_items.data
            self.args.files = request_items.files
            self.args.params = request_items.params
            self.args.multipart_data = request_items.multipart_data

        if self.args.files and not self.args.form:
            # `http url @/path/to/file`
            request_file = None
            for key, file in self.args.files.items():
                if key != '':
                    self.error(
                        'Invalid file fields (perhaps you meant --form?):'
                        f' {",".join(self.args.files.keys())}')
                if request_file is not None:
                    self.error("Can't read request from multiple files")
                request_file = file

            fn, fd, ct = request_file
            self.args.files = {}

            self._body_from_file(fd)

            if 'Content-Type' not in self.args.headers:
                content_type = get_content_type(fn)
                if content_type:
                    self.args.headers['Content-Type'] = content_type

    def _process_output_options(self):
        """Apply defaults to output options, or validate the provided ones.

        The default output options are stdout-type-sensitive.

        """

        def check_options(value, option):
            unknown = set(value) - OUTPUT_OPTIONS
            if unknown:
                self.error(f'Unknown output options: {option}={",".join(unknown)}')

        if self.args.verbose:
            self.args.all = True

        if self.args.output_options is None:
            if self.args.verbose >= 2:
                self.args.output_options = ''.join(OUTPUT_OPTIONS)
            elif self.args.verbose == 1:
                self.args.output_options = ''.join(BASE_OUTPUT_OPTIONS)
            elif self.args.offline:
                self.args.output_options = OUTPUT_OPTIONS_DEFAULT_OFFLINE
            elif not self.env.stdout_isatty:
                self.args.output_options = OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED
            else:
                self.args.output_options = OUTPUT_OPTIONS_DEFAULT

        if self.args.output_options_history is None:
            self.args.output_options_history = self.args.output_options

        check_options(self.args.output_options, '--print')
        check_options(self.args.output_options_history, '--history-print')

        if self.args.download and OUT_RESP_BODY in self.args.output_options:
            # Response body is always downloaded with --download and it goes
            # through a different routine, so we remove it.
            self.args.output_options = str(
                set(self.args.output_options) - set(OUT_RESP_BODY))

    def _process_pretty_options(self):
        if self.args.prettify == PRETTY_STDOUT_TTY_ONLY:
            self.args.prettify = PRETTY_MAP[
                'all' if self.env.stdout_isatty else 'none']
        elif (self.args.prettify and self.env.is_windows
              and self.args.output_file):
            self.error('Only terminal output can be colorized on Windows.')
        else:
            # noinspection PyTypeChecker
            self.args.prettify = PRETTY_MAP[self.args.prettify]

    def _process_download_options(self):
        if self.args.offline:
            self.args.download = False
            self.args.download_resume = False
            return
        if not self.args.download:
            if self.args.download_resume:
                self.error('--continue only works with --download')
        if self.args.download_resume and not (
                self.args.download and self.args.output_file):
            self.error('--continue requires --output to be specified')

    def _process_format_options(self):
        format_options = self.args.format_options or []
        parsed_options = PARSED_DEFAULT_FORMAT_OPTIONS
        for options_group in format_options:
            parsed_options = parse_format_options(options_group, defaults=parsed_options)
        self.args.format_options = parsed_options

    def print_manual(self):
        from httpie.output.ui import man_pages

        if man_pages.is_available(self.env.program_name):
            man_pages.display_for(self.env, self.env.program_name)
            return None

        text = self.format_help()
        with self.env.rich_console.pager():
            self.env.rich_console.print(
                text,
                highlight=False
            )

    def print_usage(self, file):
        from rich.text import Text
        from httpie.output.ui import rich_help

        whitelist = set()
        _, exception, _ = sys.exc_info()
        if (
            isinstance(exception, argparse.ArgumentError)
            and len(exception.args) >= 1
            and isinstance(exception.args[0], argparse.Action)
            and exception.args[0].option_strings
        ):
            # add_usage path is also taken when you pass an invalid option,
            # e.g --style=invalid. If something like that happens, we want
            # to include to action that caused to the invalid usage into
            # the list of actions we are displaying.
            whitelist.add(exception.args[0].option_strings[0])

        usage_text = Text('usage', style='bold')
        usage_text.append(':\n    ')
        usage_text.append(rich_help.to_usage(self.spec, whitelist=whitelist))
        self.env.rich_error_console.print(usage_text)

    def error(self, message):
        """Prints a usage message incorporating the message to stderr and
        exits."""
        self.print_usage(sys.stderr)
        self.env.rich_error_console.print(
            dedent(
                f'''
                [bold]error[/bold]:
                    {message}

                [bold]for more information[/bold]:
                    run '{self.prog} --help' or visit https://httpie.io/docs/cli
                '''.rstrip()
            )
        )
        self.exit(2)



File: httpie/cli/dicts.py

from collections import OrderedDict

from multidict import MultiDict, CIMultiDict


class BaseMultiDict(MultiDict):
    """
    Base class for all MultiDicts.
    """


class HTTPHeadersDict(CIMultiDict, BaseMultiDict):
    """
    Headers are case-insensitive and multiple values are supported
    through the `add()` API.
    """

    def add(self, key, value):
        """
        Add or update a new header.

        If the given `value` is `None`, then all the previous
        values will be overwritten and the value will be set
        to `None`.
        """
        if value is None:
            self[key] = value
            return None

        # If the previous value for the given header is `None`
        # then discard it since we are explicitly giving a new
        # value for it.
        if key in self and self.getone(key) is None:
            self.popone(key)

        super().add(key, value)

    def remove_item(self, key, value):
        """
        Remove a (key, value) pair from the dict.
        """
        existing_values = self.popall(key)
        existing_values.remove(value)

        for value in existing_values:
            self.add(key, value)


class RequestJSONDataDict(OrderedDict):
    pass


class MultiValueOrderedDict(OrderedDict):
    """Multi-value dict for URL parameters and form data."""

    def __setitem__(self, key, value):
        """
        If `key` is assigned more than once, `self[key]` holds a
        `list` of all the values.

        This allows having multiple fields with the same name in form
        data and URL params.

        """
        assert not isinstance(value, list)
        if key not in self:
            super().__setitem__(key, value)
        else:
            if not isinstance(self[key], list):
                super().__setitem__(key, [self[key]])
            self[key].append(value)

    def items(self):
        for key, values in super().items():
            if not isinstance(values, list):
                values = [values]
            for value in values:
                yield key, value


class RequestQueryParamsDict(MultiValueOrderedDict):
    pass


class RequestDataDict(MultiValueOrderedDict):
    pass


class MultipartRequestDataDict(MultiValueOrderedDict):
    pass


class RequestFilesDict(RequestDataDict):
    pass



File: httpie/cli/nested_json/interpret.py

from typing import Type, Union, Any, Iterable, Tuple

from .parse import parse, assert_cant_happen
from .errors import NestedJSONSyntaxError
from .tokens import EMPTY_STRING, TokenKind, Token, PathAction, Path, NestedJSONArray


__all__ = [
    'interpret_nested_json',
    'unwrap_top_level_list_if_needed',
]

JSONType = Type[Union[dict, list, int, float, str]]
JSON_TYPE_MAPPING = {
    dict: 'object',
    list: 'array',
    int: 'number',
    float: 'number',
    str: 'string',
}


def interpret_nested_json(pairs: Iterable[Tuple[str, str]]) -> dict:
    context = None
    for key, value in pairs:
        context = interpret(context, key, value)
    return wrap_with_dict(context)


def interpret(context: Any, key: str, value: Any) -> Any:
    cursor = context
    paths = list(parse(key))
    paths.append(Path(PathAction.SET, value))

    # noinspection PyShadowingNames
    def type_check(index: int, path: Path, expected_type: JSONType):
        if not isinstance(cursor, expected_type):
            if path.tokens:
                pseudo_token = Token(
                    kind=TokenKind.PSEUDO,
                    value='',
                    start=path.tokens[0].start,
                    end=path.tokens[-1].end,
                )
            else:
                pseudo_token = None
            cursor_type = JSON_TYPE_MAPPING.get(type(cursor), type(cursor).__name__)
            required_type = JSON_TYPE_MAPPING[expected_type]
            message = f'Cannot perform {path.kind.to_string()!r} based access on '
            message += repr(''.join(path.reconstruct() for path in paths[:index]))
            message += f' which has a type of {cursor_type!r} but this operation'
            message += f' requires a type of {required_type!r}.'
            raise NestedJSONSyntaxError(
                source=key,
                token=pseudo_token,
                message=message,
                message_kind='Type',
            )

    def object_for(kind: PathAction) -> Any:
        if kind is PathAction.KEY:
            return {}
        elif kind in {PathAction.INDEX, PathAction.APPEND}:
            return []
        else:
            assert_cant_happen()

    for index, (path, next_path) in enumerate(zip(paths, paths[1:])):
        # If there is no context yet, set it.
        if cursor is None:
            context = cursor = object_for(path.kind)
        if path.kind is PathAction.KEY:
            type_check(index, path, dict)
            if next_path.kind is PathAction.SET:
                cursor[path.accessor] = next_path.accessor
                break
            cursor = cursor.setdefault(path.accessor, object_for(next_path.kind))
        elif path.kind is PathAction.INDEX:
            type_check(index, path, list)
            if path.accessor < 0:
                raise NestedJSONSyntaxError(
                    source=key,
                    token=path.tokens[1],
                    message='Negative indexes are not supported.',
                    message_kind='Value',
                )
            cursor.extend([None] * (path.accessor - len(cursor) + 1))
            if next_path.kind is PathAction.SET:
                cursor[path.accessor] = next_path.accessor
                break
            if cursor[path.accessor] is None:
                cursor[path.accessor] = object_for(next_path.kind)
            cursor = cursor[path.accessor]
        elif path.kind is PathAction.APPEND:
            type_check(index, path, list)
            if next_path.kind is PathAction.SET:
                cursor.append(next_path.accessor)
                break
            cursor.append(object_for(next_path.kind))
            cursor = cursor[-1]
        else:
            assert_cant_happen()

    return context


def wrap_with_dict(context):
    if context is None:
        return {}
    elif isinstance(context, list):
        return {
            EMPTY_STRING: NestedJSONArray(context),
        }
    else:
        assert isinstance(context, dict)
        return context


def unwrap_top_level_list_if_needed(data: dict):
    """
    Propagate the top-level list, if that’s what we got.

    """
    if len(data) == 1:
        key, value = list(data.items())[0]
        if isinstance(value, NestedJSONArray):
            assert key == EMPTY_STRING
            return value
    return data



File: httpie/cli/nested_json/__init__.py

"""
A library for parsing the HTTPie nested JSON key syntax and constructing the resulting objects.

<https://httpie.io/docs/cli/nested-json>

It has no dependencies.

"""
from .interpret import interpret_nested_json, unwrap_top_level_list_if_needed
from .errors import NestedJSONSyntaxError
from .tokens import EMPTY_STRING, NestedJSONArray


__all__ = [
    'interpret_nested_json',
    'unwrap_top_level_list_if_needed',
    'EMPTY_STRING',
    'NestedJSONArray',
    'NestedJSONSyntaxError'
]



File: httpie/cli/nested_json/tokens.py

from enum import Enum, auto
from typing import NamedTuple, Union, Optional, List

EMPTY_STRING = ''
HIGHLIGHTER = '^'
OPEN_BRACKET = '['
CLOSE_BRACKET = ']'
BACKSLASH = '\\'


class TokenKind(Enum):
    TEXT = auto()
    NUMBER = auto()
    LEFT_BRACKET = auto()
    RIGHT_BRACKET = auto()
    PSEUDO = auto()  # Not a real token, use when representing location only.

    def to_name(self) -> str:
        for key, value in OPERATORS.items():
            if value is self:
                return repr(key)
        else:
            return 'a ' + self.name.lower()


OPERATORS = {
    OPEN_BRACKET: TokenKind.LEFT_BRACKET,
    CLOSE_BRACKET: TokenKind.RIGHT_BRACKET,
}
SPECIAL_CHARS = OPERATORS.keys() | {BACKSLASH}
LITERAL_TOKENS = [
    TokenKind.TEXT,
    TokenKind.NUMBER,
]


class Token(NamedTuple):
    kind: TokenKind
    value: Union[str, int]
    start: int
    end: int


class PathAction(Enum):
    KEY = auto()
    INDEX = auto()
    APPEND = auto()
    # Pseudo action, used by the interpreter
    SET = auto()

    def to_string(self) -> str:
        return self.name.lower()


class Path:
    def __init__(
        self,
        kind: PathAction,
        accessor: Optional[Union[str, int]] = None,
        tokens: Optional[List[Token]] = None,
        is_root: bool = False,
    ):
        self.kind = kind
        self.accessor = accessor
        self.tokens = tokens or []
        self.is_root = is_root

    def reconstruct(self) -> str:
        if self.kind is PathAction.KEY:
            if self.is_root:
                return str(self.accessor)
            return OPEN_BRACKET + self.accessor + CLOSE_BRACKET
        elif self.kind is PathAction.INDEX:
            return OPEN_BRACKET + str(self.accessor) + CLOSE_BRACKET
        elif self.kind is PathAction.APPEND:
            return OPEN_BRACKET + CLOSE_BRACKET


class NestedJSONArray(list):
    """Denotes a top-level JSON array."""



File: httpie/cli/nested_json/errors.py

from typing import Optional

from .tokens import Token, HIGHLIGHTER


class NestedJSONSyntaxError(ValueError):
    def __init__(
        self,
        source: str,
        token: Optional[Token],
        message: str,
        message_kind: str = 'Syntax',
    ) -> None:
        self.source = source
        self.token = token
        self.message = message
        self.message_kind = message_kind

    def __str__(self):
        lines = [f'HTTPie {self.message_kind} Error: {self.message}']
        if self.token is not None:
            lines.append(self.source)
            lines.append(
                ' ' * self.token.start
                + HIGHLIGHTER * (self.token.end - self.token.start)
            )
        return '\n'.join(lines)



File: httpie/cli/nested_json/parse.py

from typing import Iterator

from .errors import NestedJSONSyntaxError
from .tokens import (
    EMPTY_STRING,
    BACKSLASH,
    TokenKind,
    OPERATORS,
    SPECIAL_CHARS,
    LITERAL_TOKENS,
    Token,
    PathAction,
    Path,
)


__all__ = [
    'parse',
    'assert_cant_happen',
]


def parse(source: str) -> Iterator[Path]:
    """
    start: root_path path*
    root_path: (literal | index_path | append_path)
    literal: TEXT | NUMBER

    path:
        key_path
        | index_path
        | append_path
    key_path: LEFT_BRACKET TEXT RIGHT_BRACKET
    index_path: LEFT_BRACKET NUMBER RIGHT_BRACKET
    append_path: LEFT_BRACKET RIGHT_BRACKET

    """

    tokens = list(tokenize(source))
    cursor = 0

    def can_advance():
        return cursor < len(tokens)

    # noinspection PyShadowingNames
    def expect(*kinds):
        nonlocal cursor
        assert kinds
        if can_advance():
            token = tokens[cursor]
            cursor += 1
            if token.kind in kinds:
                return token
        elif tokens:
            token = tokens[-1]._replace(
                start=tokens[-1].end + 0,
                end=tokens[-1].end + 1,
            )
        else:
            token = None
        if len(kinds) == 1:
            suffix = kinds[0].to_name()
        else:
            suffix = ', '.join(kind.to_name() for kind in kinds[:-1])
            suffix += ' or ' + kinds[-1].to_name()
        message = f'Expecting {suffix}'
        raise NestedJSONSyntaxError(source, token, message)

    # noinspection PyShadowingNames
    def parse_root():
        tokens = []
        if not can_advance():
            return Path(
                kind=PathAction.KEY,
                accessor=EMPTY_STRING,
                is_root=True
            )
        # (literal | index_path | append_path)?
        token = expect(*LITERAL_TOKENS, TokenKind.LEFT_BRACKET)
        tokens.append(token)
        if token.kind in LITERAL_TOKENS:
            action = PathAction.KEY
            value = str(token.value)
        elif token.kind is TokenKind.LEFT_BRACKET:
            token = expect(TokenKind.NUMBER, TokenKind.RIGHT_BRACKET)
            tokens.append(token)
            if token.kind is TokenKind.NUMBER:
                action = PathAction.INDEX
                value = token.value
                tokens.append(expect(TokenKind.RIGHT_BRACKET))
            elif token.kind is TokenKind.RIGHT_BRACKET:
                action = PathAction.APPEND
                value = None
            else:
                assert_cant_happen()
        else:
            assert_cant_happen()
        # noinspection PyUnboundLocalVariable
        return Path(
            kind=action,
            accessor=value,
            tokens=tokens,
            is_root=True
        )

    yield parse_root()

    # path*
    while can_advance():
        path_tokens = [expect(TokenKind.LEFT_BRACKET)]
        token = expect(TokenKind.TEXT, TokenKind.NUMBER, TokenKind.RIGHT_BRACKET)
        path_tokens.append(token)
        if token.kind is TokenKind.RIGHT_BRACKET:
            path = Path(PathAction.APPEND, tokens=path_tokens)
        elif token.kind is TokenKind.TEXT:
            path = Path(PathAction.KEY, token.value, tokens=path_tokens)
            path_tokens.append(expect(TokenKind.RIGHT_BRACKET))
        elif token.kind is TokenKind.NUMBER:
            path = Path(PathAction.INDEX, token.value, tokens=path_tokens)
            path_tokens.append(expect(TokenKind.RIGHT_BRACKET))
        else:
            assert_cant_happen()
        # noinspection PyUnboundLocalVariable
        yield path


def tokenize(source: str) -> Iterator[Token]:
    cursor = 0
    backslashes = 0
    buffer = []

    def send_buffer() -> Iterator[Token]:
        nonlocal backslashes
        if not buffer:
            return None

        value = ''.join(buffer)
        kind = TokenKind.TEXT
        if not backslashes:
            for variation, kind in [
                (int, TokenKind.NUMBER),
                (check_escaped_int, TokenKind.TEXT),
            ]:
                try:
                    value = variation(value)
                except ValueError:
                    continue
                else:
                    break
        yield Token(
            kind=kind,
            value=value,
            start=cursor - (len(buffer) + backslashes),
            end=cursor,
        )
        buffer.clear()
        backslashes = 0

    def can_advance() -> bool:
        return cursor < len(source)

    while can_advance():
        index = source[cursor]
        if index in OPERATORS:
            yield from send_buffer()
            yield Token(OPERATORS[index], index, cursor, cursor + 1)
        elif index == BACKSLASH and can_advance():
            if source[cursor + 1] in SPECIAL_CHARS:
                backslashes += 1
            else:
                buffer.append(index)
            buffer.append(source[cursor + 1])
            cursor += 1
        else:
            buffer.append(index)
        cursor += 1

    yield from send_buffer()


def check_escaped_int(value: str) -> str:
    if not value.startswith(BACKSLASH):
        raise ValueError('Not an escaped int')
    try:
        int(value[1:])
    except ValueError as exc:
        raise ValueError('Not an escaped int') from exc
    else:
        return value[1:]


def assert_cant_happen():
    raise ValueError('Unexpected value')



File: httpie/manager/compat.py

import sys
import shutil
import subprocess

from contextlib import suppress
from typing import List, Optional
from httpie.compat import is_frozen


class PipError(Exception):
    """An exception that occurs when pip exits with an error status code."""

    def __init__(self, stdout, stderr):
        self.stdout = stdout
        self.stderr = stderr


def _discover_system_pip() -> List[str]:
    # When we are running inside of a frozen binary, we need the system
    # pip to install plugins since there is no way for us to execute any
    # code outside of the HTTPie.
    #
    # We explicitly depend on system pip, so the SystemError should not
    # be executed (except for broken installations).
    def _check_pip_version(pip_location: Optional[str]) -> bool:
        if not pip_location:
            return False

        with suppress(subprocess.CalledProcessError):
            stdout = subprocess.check_output([pip_location, "--version"], text=True)
            return "python 3" in stdout

    targets = [
        "pip",
        "pip3"
    ]
    for target in targets:
        pip_location = shutil.which(target)
        if _check_pip_version(pip_location):
            return pip_location

    raise SystemError("Couldn't find 'pip' executable. Please ensure that pip in your system is available.")


def _run_pip_subprocess(pip_executable: List[str], args: List[str]) -> bytes:

    cmd = [*pip_executable, *args]
    try:
        process = subprocess.run(
            cmd,
            check=True,
            shell=False,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE
        )
    except subprocess.CalledProcessError as error:
        raise PipError(error.stdout, error.stderr) from error
    else:
        return process.stdout


def run_pip(args: List[str]) -> bytes:
    if is_frozen:
        pip_executable = [_discover_system_pip()]
    else:
        pip_executable = [sys.executable, '-m', 'pip']

    return _run_pip_subprocess(pip_executable, args)



File: httpie/manager/__init__.py




File: httpie/manager/core.py

import argparse
from typing import Optional

from httpie.context import Environment
from httpie.status import ExitStatus
from httpie.manager.cli import missing_subcommand, parser
from httpie.manager.tasks import CLI_TASKS

MSG_COMMAND_CONFUSION = '''\
This command is only for managing HTTPie plugins.
To send a request, please use the http/https commands:

  $ http {args}

  $ https {args}
'''

# noinspection PyStringFormat
MSG_NAKED_INVOCATION = f'''\
{missing_subcommand()}

{MSG_COMMAND_CONFUSION}
'''.rstrip("\n").format(args='POST pie.dev/post hello=world')


def dispatch_cli_task(env: Environment, action: Optional[str], args: argparse.Namespace) -> ExitStatus:
    if action is None:
        parser.error(missing_subcommand('cli'))

    return CLI_TASKS[action](env, args)


def program(args: argparse.Namespace, env: Environment) -> ExitStatus:
    if args.action is None:
        parser.error(MSG_NAKED_INVOCATION)

    if args.action == 'plugins':
        return dispatch_cli_task(env, args.action, args)
    elif args.action == 'cli':
        return dispatch_cli_task(env, args.cli_action, args)

    return ExitStatus.SUCCESS



File: httpie/manager/cli.py

from textwrap import dedent
from httpie.cli.argparser import HTTPieManagerArgumentParser
from httpie.cli.options import Qualifiers, ARGPARSE_QUALIFIER_MAP, map_qualifiers, parser_to_parser_spec
from httpie import __version__

CLI_SESSION_UPGRADE_FLAGS = [
    {
        'flags': ['--bind-cookies'],
        'action': 'store_true',
        'default': False,
        'help': 'Bind domainless cookies to the host that session belongs.'
    }
]

COMMANDS = {
    'cli': {
        'help': 'Manage HTTPie for Terminal',
        'export-args': [
            'Export available options for the CLI',
            {
                'flags': ['-f', '--format'],
                'choices': ['json'],
                'help': 'Format to export in.',
                'default': 'json'
            }
        ],
        'check-updates': [
            'Check for updates'
        ],
        'sessions': {
            'help': 'Manage HTTPie sessions',
            'upgrade': [
                'Upgrade the given HTTPie session with the latest '
                'layout. A list of changes between different session versions '
                'can be found in the official documentation.',
                {
                    'dest': 'hostname',
                    'metavar': 'HOSTNAME',
                    'help': 'The host this session belongs.'
                },
                {
                    'dest': 'session',
                    'metavar': 'SESSION_NAME_OR_PATH',
                    'help': 'The name or the path for the session that will be upgraded.'
                },
                *CLI_SESSION_UPGRADE_FLAGS
            ],
            'upgrade-all': [
                'Upgrade all named sessions with the latest layout. A list of '
                'changes between different session versions can be found in the official '
                'documentation.',
                *CLI_SESSION_UPGRADE_FLAGS
            ],
        }
    }
}


COMMANDS['plugins'] = COMMANDS['cli']['plugins'] = {
    'help': 'Manage HTTPie plugins.',
    'install': [
        'Install the given targets from PyPI '
        'or from a local paths.',
        {
            'dest': 'targets',
            'metavar': 'TARGET',
            'nargs': Qualifiers.ONE_OR_MORE,
            'help': 'targets to install'
        }
    ],
    'upgrade': [
        'Upgrade the given plugins',
        {
            'dest': 'targets',
            'metavar': 'TARGET',
            'nargs': Qualifiers.ONE_OR_MORE,
            'help': 'targets to upgrade'
        }
    ],
    'uninstall': [
        'Uninstall the given HTTPie plugins.',
        {
            'dest': 'targets',
            'metavar': 'TARGET',
            'nargs': Qualifiers.ONE_OR_MORE,
            'help': 'targets to install'
        }
    ],
    'list': [
        'List all installed HTTPie plugins.'
    ],
}


def missing_subcommand(*args) -> str:
    base = COMMANDS
    for arg in args:
        base = base[arg]

    assert isinstance(base, dict)
    subcommands = ', '.join(map(repr, base.keys()))
    return f'Please specify one of these: {subcommands}'


def generate_subparsers(root, parent_parser, definitions, spec):
    action_dest = '_'.join(parent_parser.prog.split()[1:] + ['action'])
    actions = parent_parser.add_subparsers(
        dest=action_dest
    )
    for command, properties in definitions.items():
        is_subparser = isinstance(properties, dict)
        properties = properties.copy()

        descr = properties.pop('help', None) if is_subparser else properties.pop(0)
        command_parser = actions.add_parser(command, description=descr)
        command_parser.root = root
        if is_subparser:
            generate_subparsers(root, command_parser, properties, spec)
            continue

        group = spec.add_group(parent_parser.prog + ' ' + command, description=descr)
        for argument in properties:
            argument = argument.copy()
            flags = argument.pop('flags', [])
            command_parser.add_argument(*flags, **map_qualifiers(argument, ARGPARSE_QUALIFIER_MAP))
            group.add_argument(*flags, **argument)


parser = HTTPieManagerArgumentParser(
    prog='httpie',
    description=dedent(
        '''
        Managing interface for the HTTPie itself. <https://httpie.io/docs#manager>

        Be aware that you might be looking for http/https commands for sending
        HTTP requests. This command is only available for managing the HTTTPie
        plugins and the configuration around it.
        '''
    ),
)

parser.add_argument(
    '--debug',
    action='store_true',
    default=False,
    help='''
    Prints the exception traceback should one occur, as well as other
    information useful for debugging HTTPie itself and for reporting bugs.

    '''
)

parser.add_argument(
    '--traceback',
    action='store_true',
    default=False,
    help='''
    Prints the exception traceback should one occur.

    '''
)

parser.add_argument(
    '--version',
    action='version',
    version=__version__,
    help='''
    Show version and exit.

    '''
)

man_page_hint = '''
If you are looking for the man pages of http/https commands, try one of the following:
    $ man http
    $ man https

'''

options = parser_to_parser_spec(parser, man_page_hint=man_page_hint, source_file=__file__)
generate_subparsers(parser, parser, COMMANDS, options)



File: httpie/manager/__main__.py

import argparse
import sys

from typing import List, Union

from httpie.context import Environment
from httpie.status import ExitStatus
from httpie.manager.cli import parser
from httpie.manager.core import MSG_COMMAND_CONFUSION, program as main_program


def is_http_command(args: List[Union[str, bytes]], env: Environment) -> bool:
    """Check whether http/https parser can parse the arguments."""

    from httpie.cli.definition import parser as http_parser
    from httpie.manager.cli import COMMANDS

    # If the user already selected a top-level sub-command, never
    # show the http/https version. E.g httpie plugins pie.dev/post
    if len(args) >= 1 and args[0] in COMMANDS:
        return False

    with env.as_silent():
        try:
            http_parser.parse_args(env=env, args=args)
        except (Exception, SystemExit):
            return False
        else:
            return True


def main(args: List[Union[str, bytes]] = sys.argv, env: Environment = Environment()) -> ExitStatus:
    from httpie.core import raw_main

    try:
        return raw_main(
            parser=parser,
            main_program=main_program,
            args=args,
            env=env,
            use_default_options=False,
        )
    except argparse.ArgumentError:
        program_args = args[1:]
        if is_http_command(program_args, env):
            env.stderr.write(MSG_COMMAND_CONFUSION.format(args=' '.join(program_args)) + "\n")

        return ExitStatus.ERROR


def program():
    try:
        exit_status = main()
    except KeyboardInterrupt:
        exit_status = ExitStatus.ERROR_CTRL_C

    return exit_status


if __name__ == '__main__':  # pragma: nocover
    sys.exit(program())



File: httpie/manager/tasks/plugins.py

import argparse
import os
import textwrap
import re
import shutil
from collections import defaultdict
from contextlib import suppress
from pathlib import Path
from typing import List, Optional, Tuple

from httpie.manager.compat import PipError, run_pip
from httpie.manager.cli import parser, missing_subcommand
from httpie.compat import get_dist_name, importlib_metadata
from httpie.context import Environment
from httpie.status import ExitStatus
from httpie.utils import get_site_paths

PEP_503 = re.compile(r"[-_.]+")


class PluginInstaller:

    def __init__(self, env: Environment, debug: bool = False) -> None:
        self.env = env
        self.dir = env.config.plugins_dir
        self.debug = debug

        self.setup_plugins_dir()

    def setup_plugins_dir(self) -> None:
        try:
            self.dir.mkdir(
                exist_ok=True,
                parents=True
            )
        except OSError:
            self.env.stderr.write(
                f'Couldn\'t create "{self.dir!s}"'
                ' directory for plugin installation.'
                ' Please re-check the permissions for that directory,'
                ' and if needed, allow write-access.'
            )
            raise

    def fail(
        self,
        command: str,
        target: Optional[str] = None,
        reason: Optional[str] = None
    ) -> ExitStatus:
        message = f'Can\'t {command}'
        if target:
            message += f' {target!r}'
        if reason:
            message += f': {reason}'

        self.env.stderr.write(message + '\n')
        return ExitStatus.ERROR

    def _install(self, targets: List[str], mode='install') -> Tuple[
        bytes, ExitStatus
    ]:
        pip_args = [
            'install',
            '--prefer-binary',
            f'--prefix={self.dir}',
            '--no-warn-script-location',
        ]
        if mode == 'upgrade':
            pip_args.append('--upgrade')
        pip_args.extend(targets)

        try:
            stdout = run_pip(pip_args)
        except PipError as pip_error:
            error = pip_error
            stdout = pip_error.stdout
        else:
            error = None

        self.env.stdout.write(stdout.decode())

        if error:
            reason = None
            if error.stderr:
                stderr = error.stderr.decode()

                if self.debug:
                    self.env.stderr.write('Command failed: ')
                    self.env.stderr.write('pip ' + ' '.join(pip_args) + '\n')
                    self.env.stderr.write(textwrap.indent('  ', stderr))

                last_line = stderr.strip().splitlines()[-1]
                severity, _, message = last_line.partition(': ')
                if severity == 'ERROR':
                    reason = message

            stdout = error.stdout
            exit_status = self.fail(mode, ', '.join(targets), reason)
        else:
            exit_status = ExitStatus.SUCCESS

        return stdout, exit_status

    def install(self, targets: List[str]) -> ExitStatus:
        self.env.stdout.write(f"Installing {', '.join(targets)}...\n")
        self.env.stdout.flush()
        _, exit_status = self._install(targets)
        return exit_status

    def _clear_metadata(self, targets: List[str]) -> None:
        # Due to an outstanding pip problem[0], we have to get rid of
        # existing metadata for old versions manually.
        # [0]: https://github.com/pypa/pip/issues/10727
        result_deps = defaultdict(list)
        for site_dir in get_site_paths(self.dir):
            for child in site_dir.iterdir():
                if child.suffix in {'.dist-info', '.egg-info'}:
                    name, _, version = child.stem.rpartition('-')
                    result_deps[name].append((version, child))

        for target in targets:
            name, _, version = target.rpartition('-')
            name = PEP_503.sub("-", name).lower().replace('-', '_')
            if name not in result_deps:
                continue

            for result_version, meta_path in result_deps[name]:
                if version != result_version:
                    shutil.rmtree(meta_path)

    def upgrade(self, targets: List[str]) -> ExitStatus:
        self.env.stdout.write(f"Upgrading {', '.join(targets)}...\n")
        self.env.stdout.flush()

        raw_stdout, exit_status = self._install(
            targets,
            mode='upgrade'
        )
        if not raw_stdout:
            return exit_status

        stdout = raw_stdout.decode()
        installation_line = stdout.splitlines()[-1]
        if installation_line.startswith('Successfully installed'):
            self._clear_metadata(installation_line.split()[2:])

    def _uninstall(self, target: str) -> Optional[ExitStatus]:
        try:
            distribution = importlib_metadata.distribution(target)
        except importlib_metadata.PackageNotFoundError:
            return self.fail('uninstall', target, 'package is not installed')

        base_dir = Path(distribution.locate_file('.')).resolve()
        if self.dir not in base_dir.parents:
            # If the package is installed somewhere else (e.g on the site packages
            # of the real python interpreter), than that means this package is not
            # installed through us.
            return self.fail('uninstall', target,
                             'package is not installed through httpie plugins'
                             ' interface')

        files = distribution.files
        if files is None:
            return self.fail('uninstall', target, 'couldn\'t locate the package')

        # TODO: Consider handling failures here (e.g if it fails,
        # just revert the operation and leave the site-packages
        # in a proper shape).
        for file in files:
            with suppress(FileNotFoundError):
                os.unlink(distribution.locate_file(file))

        metadata_path = getattr(distribution, '_path', None)
        if (
            metadata_path
            and metadata_path.exists()
            and not any(metadata_path.iterdir())
        ):
            metadata_path.rmdir()

        self.env.stdout.write(f'Successfully uninstalled {target}\n')

    def uninstall(self, targets: List[str]) -> ExitStatus:
        # Unfortunately uninstall doesn't work with custom pip schemes. See:
        # - https://github.com/pypa/pip/issues/5595
        # - https://github.com/pypa/pip/issues/4575
        # so we have to implement our own uninstalling logic. Which works
        # on top of the importlib_metadata.

        exit_code = ExitStatus.SUCCESS
        for target in targets:
            exit_code |= self._uninstall(target) or ExitStatus.SUCCESS
        return ExitStatus(exit_code)

    def list(self) -> None:
        from httpie.plugins.registry import plugin_manager

        known_plugins = defaultdict(list)

        for entry_point in plugin_manager.iter_entry_points(self.dir):
            ep_info = (entry_point.group, entry_point.name)
            ep_name = get_dist_name(entry_point) or entry_point.module
            known_plugins[ep_name].append(ep_info)

        for plugin, entry_points in known_plugins.items():
            self.env.stdout.write(plugin)

            version = importlib_metadata.version(plugin)
            if version is not None:
                self.env.stdout.write(f' ({version})')
            self.env.stdout.write('\n')

            for group, entry_point in sorted(entry_points):
                self.env.stdout.write(f'  {entry_point} ({group})\n')

    def run(
        self,
        action: Optional[str],
        args: argparse.Namespace,
    ) -> ExitStatus:
        from httpie.plugins.manager import enable_plugins

        if action is None:
            parser.error(missing_subcommand('plugins'))

        with enable_plugins(self.dir):
            if action == 'install':
                status = self.install(args.targets)
            elif action == 'upgrade':
                status = self.upgrade(args.targets)
            elif action == 'uninstall':
                status = self.uninstall(args.targets)
            elif action == 'list':
                status = self.list()

        return status or ExitStatus.SUCCESS


def cli_plugins(env: Environment, args: argparse.Namespace) -> ExitStatus:
    plugins = PluginInstaller(env, debug=args.debug)

    try:
        action = args.cli_plugins_action
    except AttributeError:
        action = args.plugins_action

    return plugins.run(action, args)



File: httpie/manager/tasks/sessions.py

import argparse

from httpie.sessions import SESSIONS_DIR_NAME, get_httpie_session
from httpie.status import ExitStatus
from httpie.context import Environment
from httpie.legacy import v3_1_0_session_cookie_format, v3_2_0_session_header_format
from httpie.manager.cli import missing_subcommand, parser
from httpie.utils import is_version_greater


FIXERS_TO_VERSIONS = {
    '3.1.0': v3_1_0_session_cookie_format.fix_layout,
    '3.2.0': v3_2_0_session_header_format.fix_layout,
}


def cli_sessions(env: Environment, args: argparse.Namespace) -> ExitStatus:
    action = args.cli_sessions_action
    if action is None:
        parser.error(missing_subcommand('cli', 'sessions'))

    if action == 'upgrade':
        return cli_upgrade_session(env, args)
    elif action == 'upgrade-all':
        return cli_upgrade_all_sessions(env, args)
    else:
        raise ValueError(f'Unexpected action: {action}')


def upgrade_session(env: Environment, args: argparse.Namespace, hostname: str, session_name: str):
    session = get_httpie_session(
        env=env,
        config_dir=env.config.directory,
        session_name=session_name,
        host=hostname,
        url=hostname,
        suppress_legacy_warnings=True
    )

    session_name = session.path.stem
    if session.is_new():
        env.log_error(f'{session_name!r} @ {hostname!r} does not exist.')
        return ExitStatus.ERROR

    fixers = [
        fixer
        for version, fixer in FIXERS_TO_VERSIONS.items()
        if is_version_greater(version, session.version)
    ]

    if len(fixers) == 0:
        env.stdout.write(f'{session_name!r} @ {hostname!r} is already up to date.\n')
        return ExitStatus.SUCCESS

    for fixer in fixers:
        fixer(session, hostname, args)

    session.save(bump_version=True)
    env.stdout.write(f'Upgraded {session_name!r} @ {hostname!r} to v{session.version}\n')
    return ExitStatus.SUCCESS


def cli_upgrade_session(env: Environment, args: argparse.Namespace) -> ExitStatus:
    return upgrade_session(
        env,
        args=args,
        hostname=args.hostname,
        session_name=args.session
    )


def cli_upgrade_all_sessions(env: Environment, args: argparse.Namespace) -> ExitStatus:
    session_dir_path = env.config_dir / SESSIONS_DIR_NAME

    status = ExitStatus.SUCCESS
    for host_path in session_dir_path.iterdir():
        hostname = host_path.name
        for session_path in host_path.glob("*.json"):
            session_name = session_path.stem
            status |= upgrade_session(
                env,
                args=args,
                hostname=hostname,
                session_name=session_name
            )
    return status



File: httpie/manager/tasks/export_args.py

import argparse
import json

from httpie.cli.definition import options
from httpie.cli.options import to_data
from httpie.output.writer import write_raw_data
from httpie.status import ExitStatus
from httpie.context import Environment


FORMAT_TO_CONTENT_TYPE = {
    'json': 'application/json'
}


def cli_export_args(env: Environment, args: argparse.Namespace) -> ExitStatus:
    if args.format == 'json':
        data = json.dumps(to_data(options))
    else:
        raise NotImplementedError(f'Unexpected format value: {args.format}')

    write_raw_data(
        env,
        data,
        stream_kwargs={'mime_overwrite': FORMAT_TO_CONTENT_TYPE[args.format]},
    )
    return ExitStatus.SUCCESS



File: httpie/manager/tasks/__init__.py

from httpie.manager.tasks.sessions import cli_sessions
from httpie.manager.tasks.export_args import cli_export_args
from httpie.manager.tasks.plugins import cli_plugins
from httpie.manager.tasks.check_updates import cli_check_updates

CLI_TASKS = {
    'sessions': cli_sessions,
    'export-args': cli_export_args,
    'plugins': cli_plugins,
    'check-updates': cli_check_updates
}



File: httpie/manager/tasks/check_updates.py

import argparse
from httpie.context import Environment
from httpie.status import ExitStatus
from httpie.internal.update_warnings import fetch_updates, get_update_status


def cli_check_updates(env: Environment, args: argparse.Namespace) -> ExitStatus:
    fetch_updates(env, lazy=False)
    env.stdout.write(get_update_status(env))
    return ExitStatus.SUCCESS



File: extras/packaging/linux/build.py

import stat
import subprocess
from pathlib import Path
from typing import Iterator, Tuple

BUILD_DIR = Path(__file__).parent
HTTPIE_DIR = BUILD_DIR.parent.parent.parent

EXTRAS_DIR = HTTPIE_DIR / 'extras'
MAN_PAGES_DIR = EXTRAS_DIR /  'man'

SCRIPT_DIR = BUILD_DIR / Path('scripts')
HOOKS_DIR = SCRIPT_DIR / 'hooks'

DIST_DIR = BUILD_DIR / 'dist'

TARGET_SCRIPTS = {
    SCRIPT_DIR / 'http_cli.py': [],
    SCRIPT_DIR / 'httpie_cli.py': ['--hidden-import=pip'],
}


def build_binaries() -> Iterator[Tuple[str, Path]]:
    for target_script, extra_args in TARGET_SCRIPTS.items():
        subprocess.check_call(
            [
                'pyinstaller',
                '--onefile',
                '--noupx',
                '-p',
                HTTPIE_DIR,
                '--additional-hooks-dir',
                HOOKS_DIR,
                *extra_args,
                target_script,
            ]
        )

    for executable_path in DIST_DIR.iterdir():
        if executable_path.suffix:
            continue
        stat_r = executable_path.stat()
        executable_path.chmod(stat_r.st_mode | stat.S_IEXEC)
        yield executable_path.stem, executable_path


def build_packages(http_binary: Path, httpie_binary: Path) -> None:
    import httpie

    # Mapping of src_file -> dst_file
    files = [
        (http_binary, '/usr/bin/http'),
        (http_binary, '/usr/bin/https'),
        (httpie_binary, '/usr/bin/httpie'),
    ]
    files.extend(
        (man_page, f'/usr/share/man/man1/{man_page.name}')
        for man_page in MAN_PAGES_DIR.glob('*.1')
    )

    # A list of additional dependencies
    deps = [
        'python3 >= 3.7',
        'python3-pip'
    ]

    processed_deps = [
        f'--depends={dep}'
        for dep in deps
    ]
    processed_files = [
        '='.join([str(src.resolve()), dst]) for src, dst in files
    ]
    for target in ['deb', 'rpm']:
        subprocess.check_call(
            [
                'fpm',
                '--force',
                '-s',
                'dir',
                '-t',
                target,
                '--name',
                'httpie',
                '--version',
                httpie.__version__,
                '--description',
                httpie.__doc__.strip(),
                '--license',
                httpie.__licence__,
                *processed_deps,
                *processed_files,
            ],
            cwd=DIST_DIR,
        )


def main():
    binaries = dict(build_binaries())
    build_packages(binaries['http_cli'], binaries['httpie_cli'])

    # Rename http_cli/httpie_cli to http/httpie
    binaries['http_cli'].rename(DIST_DIR / 'http')
    binaries['httpie_cli'].rename(DIST_DIR / 'httpie')



if __name__ == '__main__':
    main()



File: extras/packaging/linux/scripts/httpie_cli.py

from httpie.manager.__main__ import main

if __name__ == '__main__':
    import sys
    sys.exit(main())



File: extras/packaging/linux/scripts/http_cli.py

from httpie.__main__ import main

if __name__ == '__main__':
    import sys
    sys.exit(main())



File: extras/packaging/linux/scripts/hooks/hook-pip.py

from pathlib import Path
from PyInstaller.utils.hooks import collect_all

def hook(hook_api):
    for pkg in [
        'pip',
        'setuptools',
        'distutils',
        'pkg_resources'
    ]:
        datas, binaries, hiddenimports = collect_all(pkg)
        hook_api.add_datas(datas)
        hook_api.add_binaries(binaries)
        hook_api.add_imports(*hiddenimports)



File: extras/profiling/run.py

"""
Run the HTTPie benchmark suite with multiple environments.

This script is configured in a way that, it will create
two (or more) isolated environments and compare the *last
commit* of this repository with it's master.

> If you didn't commit yet, it won't be showing results.

You can also pass --fresh, which would test the *last
commit* of this repository with a fresh copy of HTTPie
itself. This way even if you don't have an up-to-date
master branch, you can still compare it with the upstream's
master.

You can also pass --complex to add 2 additional environments,
which would include additional dependencies like pyOpenSSL.

Examples:

    # Run everything as usual, and compare last commit with master
    $ python extras/profiling/run.py

    # Include complex environments
    $ python extras/profiling/run.py --complex

    # Compare against a fresh copy
    $ python extras/profiling/run.py --fresh

    # Compare against a custom branch of a custom repo
    $ python extras/profiling/run.py --target-repo my_repo --target-branch my_branch

    # Debug changes made on this script (only run benchmarks once)
    $ python extras/profiling/run.py --debug
"""

import dataclasses
import shlex
import subprocess
import sys
import tempfile
import venv
from argparse import ArgumentParser, FileType
from contextlib import contextmanager
from dataclasses import dataclass
from pathlib import Path
from typing import (IO, Dict, Generator, Iterable, List, Optional,
                    Tuple)

BENCHMARK_SCRIPT = Path(__file__).parent / 'benchmarks.py'
CURRENT_REPO = Path(__file__).parent.parent.parent

GITHUB_URL = 'https://github.com/httpie/cli.git'
TARGET_BRANCH = 'master'

# Additional dependencies for --complex
ADDITIONAL_DEPS = ('pyOpenSSL',)


def call(*args, **kwargs):
    kwargs.setdefault('stdout', subprocess.DEVNULL)
    return subprocess.check_call(*args, **kwargs)


class Environment:
    """
    Each environment defines how to create an isolated instance
    where we could install HTTPie and run benchmarks without any
    environmental factors.
    """

    @contextmanager
    def on_repo(self) -> Generator[Tuple[Path, Dict[str, str]], None, None]:
        """
        Return the path to the python interpreter and the
        environment variables (e.g HTTPIE_COMMAND) to be
        used on the benchmarks.
        """
        raise NotImplementedError


@dataclass
class HTTPieEnvironment(Environment):
    repo_url: str
    branch: Optional[str] = None
    dependencies: Iterable[str] = ()

    @contextmanager
    def on_repo(self) -> Generator[Path, None, None]:
        with tempfile.TemporaryDirectory() as directory_path:
            directory = Path(directory_path)

            # Clone the repo
            repo_path = directory / 'httpie'
            call(
                ['git', 'clone', self.repo_url, repo_path],
                stderr=subprocess.DEVNULL,
            )

            if self.branch is not None:
                call(
                    ['git', 'checkout', self.branch],
                    cwd=repo_path,
                    stderr=subprocess.DEVNULL,
                )

            # Prepare the environment
            venv_path = directory / '.venv'
            venv.create(venv_path, with_pip=True)

            # Install basic dependencies
            python = venv_path / 'bin' / 'python'
            call(
                [
                    python,
                    '-m',
                    'pip',
                    'install',
                    'wheel',
                    'pyperf==2.3.0',
                    *self.dependencies,
                ]
            )

            # Create a wheel distribution of HTTPie
            call([python, 'setup.py', 'bdist_wheel'], cwd=repo_path)

            # Install httpie
            distribution_path = next((repo_path / 'dist').iterdir())
            call(
                [python, '-m', 'pip', 'install', distribution_path],
                cwd=repo_path,
            )

            http = venv_path / 'bin' / 'http'
            yield python, {'HTTPIE_COMMAND': shlex.join([str(python), str(http)])}


@dataclass
class LocalCommandEnvironment(Environment):
    local_command: str

    @contextmanager
    def on_repo(self) -> Generator[Path, None, None]:
        yield sys.executable, {'HTTPIE_COMMAND': self.local_command}


def dump_results(
    results: List[str],
    file: IO[str],
    min_speed: Optional[str] = None
) -> None:
    for result in results:
        lines = result.strip().splitlines()
        if min_speed is not None and "hidden" in lines[-1]:
            lines[-1] = (
                'Some benchmarks were hidden from this list '
                'because their timings did not change in a '
                'significant way (change was within the error '
                'margin ±{margin}%).'
            ).format(margin=min_speed)
            result = '\n'.join(lines)

        print(result, file=file)
        print("\n---\n", file=file)


def compare(*args, directory: Path, min_speed: Optional[str] = None):
    compare_args = ['pyperf', 'compare_to', '--table', '--table-format=md', *args]
    if min_speed:
        compare_args.extend(['--min-speed', min_speed])
    return subprocess.check_output(
        compare_args,
        cwd=directory,
        text=True,
    )


def run(
    configs: List[Dict[str, Environment]],
    file: IO[str],
    debug: bool = False,
    min_speed: Optional[str] = None,
) -> None:
    result_directory = Path(tempfile.mkdtemp())
    results = []

    current = 1
    total = sum(1 for config in configs for _ in config.items())

    def iterate(env_name, status):
        print(
            f'Iteration: {env_name} ({current}/{total}) ({status})' + ' ' * 10,
            end='\r',
            flush=True,
        )

    for config in configs:
        for env_name, env in config.items():
            iterate(env_name, 'setting up')
            with env.on_repo() as (python, env_vars):
                iterate(env_name, 'running benchmarks')
                args = [python, BENCHMARK_SCRIPT, '-o', env_name]
                if debug:
                    args.append('--debug-single-value')
                call(
                    args,
                    cwd=result_directory,
                    env=env_vars,
                )
            current += 1

        results.append(compare(
            *config.keys(),
            directory=result_directory,
            min_speed=min_speed
        ))

    dump_results(results, file=file, min_speed=min_speed)
    print('Results are available at:', result_directory)


def main() -> None:
    parser = ArgumentParser()
    parser.add_argument('--local-repo', default=CURRENT_REPO)
    parser.add_argument('--local-branch', default=None)
    parser.add_argument('--target-repo', default=CURRENT_REPO)
    parser.add_argument('--target-branch', default=TARGET_BRANCH)
    parser.add_argument(
        '--fresh',
        action='store_const',
        const=GITHUB_URL,
        dest='target_repo',
        help='Clone the target repo from upstream GitHub URL',
    )
    parser.add_argument(
        '--complex',
        action='store_true',
        help='Add a second run, with a complex python environment.',
    )
    parser.add_argument(
        '--local-bin',
        help='Run the suite with the given local binary in addition to'
        ' existing runners. (E.g --local-bin $(command -v xh))',
    )
    parser.add_argument(
        '--file',
        type=FileType('w'),
        default=sys.stdout,
        help='File to print the actual results',
    )
    parser.add_argument(
        '--min-speed',
        help='Minimum of speed in percent to consider that a '
             'benchmark is significant'
    )
    parser.add_argument(
        '--debug',
        action='store_true',
    )

    options = parser.parse_args()

    configs = []

    base_config = {
        options.target_branch: HTTPieEnvironment(options.target_repo, options.target_branch),
        'this_branch': HTTPieEnvironment(options.local_repo, options.local_branch),
    }
    configs.append(base_config)

    if options.complex:
        complex_config = {
            env_name
            + '-complex': dataclasses.replace(env, dependencies=ADDITIONAL_DEPS)
            for env_name, env in base_config.items()
        }
        configs.append(complex_config)

    if options.local_bin:
        base_config['binary'] = LocalCommandEnvironment(options.local_bin)

    run(configs, file=options.file, debug=options.debug, min_speed=options.min_speed)


if __name__ == '__main__':
    main()



File: extras/profiling/benchmarks.py

"""
This file is the declaration of benchmarks for HTTPie. It
is also used to run them with the current environment.

Each instance of BaseRunner class will be an individual
benchmark. And if run without any arguments, this file
will execute every benchmark instance and report the
timings.

The benchmarks are run through 'pyperf', which allows to
do get very precise results. For micro-benchmarks like startup,
please run `pyperf system tune` to get even more accurate results.

Examples:

    # Run everything as usual, the default is that we do 3 warm-up runs
    # and 5 actual runs.
    $ python extras/profiling/benchmarks.py

    # For retrieving results faster, pass --fast
    $ python extras/profiling/benchmarks.py --fast

    # For verify everything works as expected, pass --debug-single-value.
    # It will only run everything once, so the resuls are not reliable. But
    # very useful when iterating on a benchmark
    $ python extras/profiling/benchmarks.py --debug-single-value

    # If you want to run with a custom HTTPie command (for example with
    # and HTTPie instance installed in another virtual environment),
    # pass HTTPIE_COMMAND variable.
    $ HTTPIE_COMMAND="/my/python /my/httpie" python extras/profiling/benchmarks.py
"""

from __future__ import annotations

import os
import shlex
import subprocess
import sys
import threading
from contextlib import ExitStack, contextmanager
from dataclasses import dataclass, field
from functools import cached_property, partial
from http.server import HTTPServer, SimpleHTTPRequestHandler
from tempfile import TemporaryDirectory
from typing import ClassVar, Final, List

import pyperf

# For download benchmarks, define a set of files.
# file: (block_size, count) => total_size = block_size * count
PREDEFINED_FILES: Final = {'3G': (3 * 1024 ** 2, 1024)}


class QuietSimpleHTTPServer(SimpleHTTPRequestHandler):
    def log_message(self, *args, **kwargs):
        pass


@contextmanager
def start_server():
    """Create a server to serve local files. It will create the
    PREDEFINED_FILES through dd."""
    with TemporaryDirectory() as directory:
        for file_name, (block_size, count) in PREDEFINED_FILES.items():
            subprocess.check_call(
                [
                    'dd',
                    'if=/dev/zero',
                    f'of={file_name}',
                    f'bs={block_size}',
                    f'count={count}',
                ],
                cwd=directory,
                stdout=subprocess.DEVNULL,
                stderr=subprocess.DEVNULL,
            )

        handler = partial(QuietSimpleHTTPServer, directory=directory)
        server = HTTPServer(('localhost', 0), handler)

        thread = threading.Thread(target=server.serve_forever)
        thread.start()
        yield '{}:{}'.format(*server.socket.getsockname())
        server.shutdown()
        thread.join(timeout=0.5)


@dataclass
class Context:
    benchmarks: ClassVar[List[BaseRunner]] = []
    stack: ExitStack = field(default_factory=ExitStack)
    runner: pyperf.Runner = field(default_factory=pyperf.Runner)

    def run(self) -> pyperf.BenchmarkSuite:
        results = [benchmark.run(self) for benchmark in self.benchmarks]
        return pyperf.BenchmarkSuite(results)

    @property
    def cmd(self) -> List[str]:
        if cmd := os.getenv('HTTPIE_COMMAND'):
            return shlex.split(cmd)

        http = os.path.join(os.path.dirname(sys.executable), 'http')
        assert os.path.exists(http)
        return [sys.executable, http]

    @cached_property
    def server(self) -> str:
        return self.stack.enter_context(start_server())

    def __enter__(self):
        return self

    def __exit__(self, *exc_info):
        self.stack.close()


@dataclass
class BaseRunner:
    """
    An individual benchmark case. By default it has the category
    (e.g like startup or download) and a name.
    """

    category: str
    title: str

    def __post_init__(self):
        Context.benchmarks.append(self)

    def run(self, context: Context) -> pyperf.Benchmark:
        raise NotImplementedError

    @property
    def name(self) -> str:
        return f'{self.title} ({self.category})'


@dataclass
class CommandRunner(BaseRunner):
    """
    Run a single command, and benchmark it.
    """

    args: List[str]

    def run(self, context: Context) -> pyperf.Benchmark:
        return context.runner.bench_command(self.name, [*context.cmd, *self.args])


@dataclass
class DownloadRunner(BaseRunner):
    """
    Benchmark downloading a single file from the
    remote server.
    """

    file_name: str

    def run(self, context: Context) -> pyperf.Benchmark:
        return context.runner.bench_command(
            self.name,
            [
                *context.cmd,
                '--download',
                'GET',
                f'{context.server}/{self.file_name}',
            ],
        )


CommandRunner('startup', '`http --version`', ['--version'])
CommandRunner('startup', '`http --offline pie.dev/get`', ['--offline', 'pie.dev/get'])
for pretty in ['all', 'none']:
    CommandRunner(
        'startup',
        f'`http --pretty={pretty} pie.dev/stream/1000`',
        [
            '--print=HBhb',
            f'--pretty={pretty}',
            'httpbin.org/stream/1000'
        ]
    )
DownloadRunner('download', '`http --download :/big_file.txt` (3GB)', '3G')


def main() -> None:
    # PyPerf will bring it's own argument parser, so configure the script.
    # The somewhat fast and also precise enough configuration is this. We run
    # benchmarks 3 times to warm up (e.g especially for download benchmark, this
    # is important). And then 5 actual runs where we record.
    sys.argv.extend(
        ['--worker', '--loops=1', '--warmup=3', '--values=5', '--processes=2']
    )

    with Context() as context:
        context.run()


if __name__ == '__main__':
    main()



File: extras/scripts/generate_man_pages.py

import os
import re
from contextlib import contextmanager
from pathlib import Path
from typing import Optional, Iterator, Iterable


# So that httpie.cli.definition can provide man-page-specific output. Must be set before importing httpie.
os.environ['HTTPIE_BUILDING_MAN_PAGES'] = '1'

import httpie
from httpie.cli.definition import options as core_options, IS_MAN_PAGE
from httpie.cli.options import ParserSpec
from httpie.manager.cli import options as manager_options
from httpie.output.ui.rich_help import OptionsHighlighter, to_usage
from httpie.output.ui.rich_utils import render_as_string


assert IS_MAN_PAGE, 'CLI definition does not understand we’re building man pages'

# Escape certain characters, so they are rendered properly on all terminals.
# <https://man7.org/linux/man-pages/man7/groff_char.7.html>
ESCAPE_MAP = {
    '"': '\[dq]',
    "'": '\[aq]',
    '~': '\(ti',
    '’': "\(ga",
    '\\': '\e',
}
ESCAPE_MAP = {ord(key): value for key, value in ESCAPE_MAP.items()}

EXTRAS_DIR = Path(__file__).parent.parent
MAN_PAGE_PATH = EXTRAS_DIR / 'man'
PROJECT_ROOT = EXTRAS_DIR.parent

OPTION_HIGHLIGHT_RE = re.compile(
    OptionsHighlighter.highlights[0]
)


class ManPageBuilder:
    def __init__(self):
        self.source = []

    def title_line(
        self,
        full_name: str,
        program_name: str,
        program_version: str,
        last_edit_date: str,
    ) -> None:
        self.source.append(
            f'.TH {program_name} 1 "{last_edit_date}" '
            f'"{full_name} {program_version}" "{full_name} Manual"'
        )

    def set_name(self, program_name: str) -> None:
        with self.section('NAME'):
            self.write(program_name)

    def write(self, text: str, *, bold: bool = False) -> None:
        if bold:
            text = '.B ' + text
        self.source.append(text)

    def separate(self) -> None:
        self.source.append('.PP')

    def format_desc(self, desc: str) -> str:
        description = _escape_and_dedent(desc)
        description = OPTION_HIGHLIGHT_RE.sub(
            # Boldify the option part, but don't remove the prefix (start of the match).
            lambda match: match[1] + self.boldify(match['option']),
            description
        )
        return description

    def add_comment(self, comment: str) -> None:
        self.source.append(f'.\\" {comment}')

    def add_options(self, options: Iterable[str], *, metavar: Optional[str] = None) -> None:
        text = ", ".join(map(self.boldify, options))
        if metavar:
            text += f' {self.underline(metavar)}'
        self.write(f'.IP "{text}"')

    def build(self) -> str:
        return '\n'.join(self.source)

    @contextmanager
    def section(self, section_name: str) -> Iterator[None]:
        self.write(f'.SH {section_name}')
        self.in_section = True
        yield
        self.in_section = False

    def underline(self, text: str) -> str:
        return r'\fI\,{}\/\fR'.format(text)

    def boldify(self, text: str) -> str:
        return r'\fB\,{}\/\fR'.format(text)


def _escape_and_dedent(text: str) -> str:
    lines = []
    for should_act, line in enumerate(text.splitlines()):
        # Only dedent after the first line.
        if should_act:
            if line.startswith('    '):
                line = line[4:]

        lines.append(line)
    return '\n'.join(lines).translate(ESCAPE_MAP)


def to_man_page(program_name: str, spec: ParserSpec, *, is_top_level_cmd: bool = False) -> str:
    builder = ManPageBuilder()
    builder.add_comment(
        f"This file is auto-generated from the parser declaration "
        + (f"in {Path(spec.source_file).relative_to(PROJECT_ROOT)} " if spec.source_file else "")
        + f"by {Path(__file__).relative_to(PROJECT_ROOT)}."
    )

    builder.title_line(
        full_name='HTTPie',
        program_name=program_name,
        program_version=httpie.__version__,
        last_edit_date=httpie.__date__,
    )
    builder.set_name(program_name)

    with builder.section('SYNOPSIS'):
        # `http` and `https` are commands that can be directly used, so they can have
        # a valid usage. But `httpie` is a top-level command with multiple sub commands,
        # so for the synopsis we'll only reference the `httpie` name.
        if is_top_level_cmd:
            synopsis = program_name
        else:
            synopsis = render_as_string(to_usage(spec, program_name=program_name))
        builder.write(synopsis)

    with builder.section('DESCRIPTION'):
        builder.write(spec.description)
        if spec.man_page_hint:
            builder.write(spec.man_page_hint)

    for index, group in enumerate(spec.groups, 1):
        with builder.section(group.name):
            if group.description:
                builder.write(group.description)

            for argument in group.arguments:
                if argument.is_hidden:
                    continue

                raw_arg = argument.serialize(isolation_mode=True)

                metavar = raw_arg.get('metavar')
                if raw_arg.get('is_positional'):
                    # In case of positional arguments, metavar is always equal
                    # to the list of options (e.g `METHOD`).
                    metavar = None
                builder.add_options(raw_arg['options'], metavar=metavar)

                desc = builder.format_desc(raw_arg.get('description', ''))
                builder.write('\n' + desc + '\n')

            builder.separate()

    if spec.epilog:
        with builder.section('SEE ALSO'):
            builder.write(builder.format_desc(spec.epilog))

    return builder.build()


def main() -> None:
    for program_name, spec, config in [
        ('http', core_options, {}),
        ('https', core_options, {}),
        ('httpie', manager_options, {'is_top_level_cmd': True}),
    ]:
        with open((MAN_PAGE_PATH / program_name).with_suffix('.1'), 'w') as stream:
            stream.write(to_man_page(program_name, spec, **config))


if __name__ == '__main__':
    main()



<h2 align="center">
    <a href="https://httpie.io" target="blank_">
        <img height="100" alt="HTTPie" src="https://raw.githubusercontent.com/httpie/cli/master/docs/httpie-logo.svg" />
    </a>
    <br>
    HTTPie CLI: human-friendly HTTP client for the API era
</h2>

<div align="center">

[![HTTPie for Desktop](https://img.shields.io/static/v1?label=HTTPie&message=Desktop&color=4B78E6)](https://httpie.io/product)
[![](https://img.shields.io/static/v1?label=HTTPie&message=Web%20%26%20Mobile&color=73DC8C)](https://httpie.io/app)
[![](https://img.shields.io/static/v1?label=HTTPie&message=CLI&color=FA9BFA)](https://httpie.io/cli)
[![Twitter](https://img.shields.io/twitter/follow/httpie?style=flat&color=%234B78E6&logoColor=%234B78E6)](https://twitter.com/httpie)
[![Chat](https://img.shields.io/discord/725351238698270761?style=flat&label=Chat%20on%20Discord&color=%23FA9BFA)](https://httpie.io/discord)

</div>


<div align="center">

[![Docs](https://img.shields.io/badge/stable%20docs-httpie.io%2Fdocs%2Fcli-brightgreen?style=flat&color=%2373DC8C&label=Docs)](https://httpie.org/docs/cli)
[![Latest version](https://img.shields.io/pypi/v/httpie.svg?style=flat&label=Latest&color=%234B78E6&logo=&logoColor=white)](https://pypi.python.org/pypi/httpie)
[![Build](https://img.shields.io/github/actions/workflow/status/httpie/cli/tests.yml?branch=master&color=%23FA9BFA&label=Build)](https://github.com/httpie/cli/actions)
[![Coverage](https://img.shields.io/codecov/c/github/httpie/cli?style=flat&label=Coverage&color=%2373DC8C)](https://codecov.io/gh/httpie/cli)
[![PyPi downloads](https://img.shields.io/pepy/dt/httpie?style=flat&label=Downloads%20from%20PyPi%20only&color=4B78E6)](https://www.pepy.tech/projects/httpie)

</div>

HTTPie (pronounced _aitch-tee-tee-pie_) is a command-line HTTP client.
Its goal is to make CLI interaction with web services as human-friendly as possible.
HTTPie is designed for testing, debugging, and generally interacting with APIs & HTTP servers.
The `http` & `https` commands allow for creating and sending arbitrary HTTP requests.
They use simple and natural syntax and provide formatted and colorized output.

<div align="center">

<img src="https://raw.githubusercontent.com/httpie/cli/master/docs/httpie-animation.gif" alt="HTTPie in action" width="100%"/>


</div>




## We lost 54k GitHub stars

Please note we recently accidentally made this repo private for a moment, and GitHub deleted our community that took a decade to build. Read the full story here: https://httpie.io/blog/stardust

![](docs/stardust.png)


## Getting started

- [Installation instructions →](https://httpie.io/docs#installation)
- [Full documentation →](https://httpie.io/docs)

## Features

- Expressive and intuitive syntax
- Formatted and colorized terminal output
- Built-in JSON support
- Forms and file uploads
- HTTPS, proxies, and authentication
- Arbitrary request data
- Custom headers
- Persistent sessions
- `wget`-like downloads

[See all features →](https://httpie.io/docs)

## Examples

Hello World:

```bash
https httpie.io/hello
```

Custom [HTTP method](https://httpie.io/docs#http-method), [HTTP headers](https://httpie.io/docs#http-headers) and [JSON](https://httpie.io/docs#json) data:

```bash
http PUT pie.dev/put X-API-Token:123 name=John
```

Build and print a request without sending it using [offline mode](https://httpie.io/docs/cli/offline-mode):

```bash
http --offline pie.dev/post hello=offline
```

Use [GitHub API](https://developer.github.com/v3/issues/comments/#create-a-comment) to post a comment on an [Issue](https://github.com/httpie/cli/issues/83) with [authentication](https://httpie.io/docs#authentication):

```bash
http -a USERNAME POST https://api.github.com/repos/httpie/cli/issues/83/comments body='HTTPie is awesome! :heart:'
```

[See more examples →](https://httpie.io/docs#examples)

## Community & support

- Visit the [HTTPie website](https://httpie.io) for full documentation and useful links.
- Join our [Discord server](https://httpie.io/discord) is to ask questions, discuss features, and for general API chat.
- Tweet at [@httpie](https://twitter.com/httpie) on Twitter.
- Use [StackOverflow](https://stackoverflow.com/questions/tagged/httpie) to ask questions and include a `httpie` tag.
- Create [GitHub Issues](https://github.com/httpie/cli/issues) for bug reports and feature requests.
- Subscribe to the [HTTPie newsletter](https://httpie.io) for occasional updates.

## Contributing

Have a look through existing [Issues](https://github.com/httpie/cli/issues) and [Pull Requests](https://github.com/httpie/cli/pulls) that you could help with. If you'd like to request a feature or report a bug, please [create a GitHub Issue](https://github.com/httpie/cli/issues) using one of the templates provided.

[See contribution guide →](https://github.com/httpie/cli/blob/master/CONTRIBUTING.md)
