Initial commit. Basic models mostly done.
This commit is contained in:
tcaxle
@@ -0,0 +1,2 @@
|
||||
"""Contains purely network-related utilities.
|
||||
"""
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
298
venv/lib/python3.8/site-packages/pip/_internal/network/auth.py
Normal file
298
venv/lib/python3.8/site-packages/pip/_internal/network/auth.py
Normal file
@@ -0,0 +1,298 @@
|
||||
"""Network Authentication Helpers
|
||||
|
||||
Contains interface (MultiDomainBasicAuth) and associated glue code for
|
||||
providing credentials in the context of network requests.
|
||||
"""
|
||||
|
||||
# The following comment should be removed at some point in the future.
|
||||
# mypy: disallow-untyped-defs=False
|
||||
|
||||
import logging
|
||||
|
||||
from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth
|
||||
from pip._vendor.requests.utils import get_netrc_auth
|
||||
from pip._vendor.six.moves.urllib import parse as urllib_parse
|
||||
|
||||
from pip._internal.utils.misc import (
|
||||
ask,
|
||||
ask_input,
|
||||
ask_password,
|
||||
remove_auth_from_url,
|
||||
split_auth_netloc_from_url,
|
||||
)
|
||||
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
|
||||
|
||||
if MYPY_CHECK_RUNNING:
|
||||
from optparse import Values
|
||||
from typing import Dict, Optional, Tuple
|
||||
|
||||
from pip._internal.vcs.versioncontrol import AuthInfo
|
||||
|
||||
Credentials = Tuple[str, str, str]
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
try:
|
||||
import keyring # noqa
|
||||
except ImportError:
|
||||
keyring = None
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Keyring is skipped due to an exception: %s", str(exc),
|
||||
)
|
||||
keyring = None
|
||||
|
||||
|
||||
def get_keyring_auth(url, username):
|
||||
"""Return the tuple auth for a given url from keyring."""
|
||||
if not url or not keyring:
|
||||
return None
|
||||
|
||||
try:
|
||||
try:
|
||||
get_credential = keyring.get_credential
|
||||
except AttributeError:
|
||||
pass
|
||||
else:
|
||||
logger.debug("Getting credentials from keyring for %s", url)
|
||||
cred = get_credential(url, username)
|
||||
if cred is not None:
|
||||
return cred.username, cred.password
|
||||
return None
|
||||
|
||||
if username:
|
||||
logger.debug("Getting password from keyring for %s", url)
|
||||
password = keyring.get_password(url, username)
|
||||
if password:
|
||||
return username, password
|
||||
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Keyring is skipped due to an exception: %s", str(exc),
|
||||
)
|
||||
|
||||
|
||||
class MultiDomainBasicAuth(AuthBase):
|
||||
|
||||
def __init__(self, prompting=True, index_urls=None):
|
||||
# type: (bool, Optional[Values]) -> None
|
||||
self.prompting = prompting
|
||||
self.index_urls = index_urls
|
||||
self.passwords = {} # type: Dict[str, AuthInfo]
|
||||
# When the user is prompted to enter credentials and keyring is
|
||||
# available, we will offer to save them. If the user accepts,
|
||||
# this value is set to the credentials they entered. After the
|
||||
# request authenticates, the caller should call
|
||||
# ``save_credentials`` to save these.
|
||||
self._credentials_to_save = None # type: Optional[Credentials]
|
||||
|
||||
def _get_index_url(self, url):
|
||||
"""Return the original index URL matching the requested URL.
|
||||
|
||||
Cached or dynamically generated credentials may work against
|
||||
the original index URL rather than just the netloc.
|
||||
|
||||
The provided url should have had its username and password
|
||||
removed already. If the original index url had credentials then
|
||||
they will be included in the return value.
|
||||
|
||||
Returns None if no matching index was found, or if --no-index
|
||||
was specified by the user.
|
||||
"""
|
||||
if not url or not self.index_urls:
|
||||
return None
|
||||
|
||||
for u in self.index_urls:
|
||||
prefix = remove_auth_from_url(u).rstrip("/") + "/"
|
||||
if url.startswith(prefix):
|
||||
return u
|
||||
|
||||
def _get_new_credentials(self, original_url, allow_netrc=True,
|
||||
allow_keyring=True):
|
||||
"""Find and return credentials for the specified URL."""
|
||||
# Split the credentials and netloc from the url.
|
||||
url, netloc, url_user_password = split_auth_netloc_from_url(
|
||||
original_url,
|
||||
)
|
||||
|
||||
# Start with the credentials embedded in the url
|
||||
username, password = url_user_password
|
||||
if username is not None and password is not None:
|
||||
logger.debug("Found credentials in url for %s", netloc)
|
||||
return url_user_password
|
||||
|
||||
# Find a matching index url for this request
|
||||
index_url = self._get_index_url(url)
|
||||
if index_url:
|
||||
# Split the credentials from the url.
|
||||
index_info = split_auth_netloc_from_url(index_url)
|
||||
if index_info:
|
||||
index_url, _, index_url_user_password = index_info
|
||||
logger.debug("Found index url %s", index_url)
|
||||
|
||||
# If an index URL was found, try its embedded credentials
|
||||
if index_url and index_url_user_password[0] is not None:
|
||||
username, password = index_url_user_password
|
||||
if username is not None and password is not None:
|
||||
logger.debug("Found credentials in index url for %s", netloc)
|
||||
return index_url_user_password
|
||||
|
||||
# Get creds from netrc if we still don't have them
|
||||
if allow_netrc:
|
||||
netrc_auth = get_netrc_auth(original_url)
|
||||
if netrc_auth:
|
||||
logger.debug("Found credentials in netrc for %s", netloc)
|
||||
return netrc_auth
|
||||
|
||||
# If we don't have a password and keyring is available, use it.
|
||||
if allow_keyring:
|
||||
# The index url is more specific than the netloc, so try it first
|
||||
kr_auth = (
|
||||
get_keyring_auth(index_url, username) or
|
||||
get_keyring_auth(netloc, username)
|
||||
)
|
||||
if kr_auth:
|
||||
logger.debug("Found credentials in keyring for %s", netloc)
|
||||
return kr_auth
|
||||
|
||||
return username, password
|
||||
|
||||
def _get_url_and_credentials(self, original_url):
|
||||
"""Return the credentials to use for the provided URL.
|
||||
|
||||
If allowed, netrc and keyring may be used to obtain the
|
||||
correct credentials.
|
||||
|
||||
Returns (url_without_credentials, username, password). Note
|
||||
that even if the original URL contains credentials, this
|
||||
function may return a different username and password.
|
||||
"""
|
||||
url, netloc, _ = split_auth_netloc_from_url(original_url)
|
||||
|
||||
# Use any stored credentials that we have for this netloc
|
||||
username, password = self.passwords.get(netloc, (None, None))
|
||||
|
||||
if username is None and password is None:
|
||||
# No stored credentials. Acquire new credentials without prompting
|
||||
# the user. (e.g. from netrc, keyring, or the URL itself)
|
||||
username, password = self._get_new_credentials(original_url)
|
||||
|
||||
if username is not None or password is not None:
|
||||
# Convert the username and password if they're None, so that
|
||||
# this netloc will show up as "cached" in the conditional above.
|
||||
# Further, HTTPBasicAuth doesn't accept None, so it makes sense to
|
||||
# cache the value that is going to be used.
|
||||
username = username or ""
|
||||
password = password or ""
|
||||
|
||||
# Store any acquired credentials.
|
||||
self.passwords[netloc] = (username, password)
|
||||
|
||||
assert (
|
||||
# Credentials were found
|
||||
(username is not None and password is not None) or
|
||||
# Credentials were not found
|
||||
(username is None and password is None)
|
||||
), "Could not load credentials from url: {}".format(original_url)
|
||||
|
||||
return url, username, password
|
||||
|
||||
def __call__(self, req):
|
||||
# Get credentials for this request
|
||||
url, username, password = self._get_url_and_credentials(req.url)
|
||||
|
||||
# Set the url of the request to the url without any credentials
|
||||
req.url = url
|
||||
|
||||
if username is not None and password is not None:
|
||||
# Send the basic auth with this request
|
||||
req = HTTPBasicAuth(username, password)(req)
|
||||
|
||||
# Attach a hook to handle 401 responses
|
||||
req.register_hook("response", self.handle_401)
|
||||
|
||||
return req
|
||||
|
||||
# Factored out to allow for easy patching in tests
|
||||
def _prompt_for_password(self, netloc):
|
||||
username = ask_input("User for %s: " % netloc)
|
||||
if not username:
|
||||
return None, None
|
||||
auth = get_keyring_auth(netloc, username)
|
||||
if auth:
|
||||
return auth[0], auth[1], False
|
||||
password = ask_password("Password: ")
|
||||
return username, password, True
|
||||
|
||||
# Factored out to allow for easy patching in tests
|
||||
def _should_save_password_to_keyring(self):
|
||||
if not keyring:
|
||||
return False
|
||||
return ask("Save credentials to keyring [y/N]: ", ["y", "n"]) == "y"
|
||||
|
||||
def handle_401(self, resp, **kwargs):
|
||||
# We only care about 401 responses, anything else we want to just
|
||||
# pass through the actual response
|
||||
if resp.status_code != 401:
|
||||
return resp
|
||||
|
||||
# We are not able to prompt the user so simply return the response
|
||||
if not self.prompting:
|
||||
return resp
|
||||
|
||||
parsed = urllib_parse.urlparse(resp.url)
|
||||
|
||||
# Prompt the user for a new username and password
|
||||
username, password, save = self._prompt_for_password(parsed.netloc)
|
||||
|
||||
# Store the new username and password to use for future requests
|
||||
self._credentials_to_save = None
|
||||
if username is not None and password is not None:
|
||||
self.passwords[parsed.netloc] = (username, password)
|
||||
|
||||
# Prompt to save the password to keyring
|
||||
if save and self._should_save_password_to_keyring():
|
||||
self._credentials_to_save = (parsed.netloc, username, password)
|
||||
|
||||
# Consume content and release the original connection to allow our new
|
||||
# request to reuse the same one.
|
||||
resp.content
|
||||
resp.raw.release_conn()
|
||||
|
||||
# Add our new username and password to the request
|
||||
req = HTTPBasicAuth(username or "", password or "")(resp.request)
|
||||
req.register_hook("response", self.warn_on_401)
|
||||
|
||||
# On successful request, save the credentials that were used to
|
||||
# keyring. (Note that if the user responded "no" above, this member
|
||||
# is not set and nothing will be saved.)
|
||||
if self._credentials_to_save:
|
||||
req.register_hook("response", self.save_credentials)
|
||||
|
||||
# Send our new request
|
||||
new_resp = resp.connection.send(req, **kwargs)
|
||||
new_resp.history.append(resp)
|
||||
|
||||
return new_resp
|
||||
|
||||
def warn_on_401(self, resp, **kwargs):
|
||||
"""Response callback to warn about incorrect credentials."""
|
||||
if resp.status_code == 401:
|
||||
logger.warning(
|
||||
'401 Error, Credentials not correct for %s', resp.request.url,
|
||||
)
|
||||
|
||||
def save_credentials(self, resp, **kwargs):
|
||||
"""Response callback to save credentials on success."""
|
||||
assert keyring is not None, "should never reach here without keyring"
|
||||
if not keyring:
|
||||
return
|
||||
|
||||
creds = self._credentials_to_save
|
||||
self._credentials_to_save = None
|
||||
if creds and resp.status_code < 400:
|
||||
try:
|
||||
logger.info('Saving credentials to keyring')
|
||||
keyring.set_password(*creds)
|
||||
except Exception:
|
||||
logger.exception('Failed to save credentials')
|
||||
@@ -0,0 +1,81 @@
|
||||
"""HTTP cache implementation.
|
||||
"""
|
||||
|
||||
# The following comment should be removed at some point in the future.
|
||||
# mypy: disallow-untyped-defs=False
|
||||
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
|
||||
from pip._vendor.cachecontrol.cache import BaseCache
|
||||
from pip._vendor.cachecontrol.caches import FileCache
|
||||
from pip._vendor.requests.models import Response
|
||||
|
||||
from pip._internal.utils.filesystem import adjacent_tmp_file, replace
|
||||
from pip._internal.utils.misc import ensure_dir
|
||||
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
|
||||
|
||||
if MYPY_CHECK_RUNNING:
|
||||
from typing import Optional
|
||||
|
||||
|
||||
def is_from_cache(response):
|
||||
# type: (Response) -> bool
|
||||
return getattr(response, "from_cache", False)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def suppressed_cache_errors():
|
||||
"""If we can't access the cache then we can just skip caching and process
|
||||
requests as if caching wasn't enabled.
|
||||
"""
|
||||
try:
|
||||
yield
|
||||
except (OSError, IOError):
|
||||
pass
|
||||
|
||||
|
||||
class SafeFileCache(BaseCache):
|
||||
"""
|
||||
A file based cache which is safe to use even when the target directory may
|
||||
not be accessible or writable.
|
||||
"""
|
||||
|
||||
def __init__(self, directory):
|
||||
# type: (str) -> None
|
||||
assert directory is not None, "Cache directory must not be None."
|
||||
super(SafeFileCache, self).__init__()
|
||||
self.directory = directory
|
||||
|
||||
def _get_cache_path(self, name):
|
||||
# type: (str) -> str
|
||||
# From cachecontrol.caches.file_cache.FileCache._fn, brought into our
|
||||
# class for backwards-compatibility and to avoid using a non-public
|
||||
# method.
|
||||
hashed = FileCache.encode(name)
|
||||
parts = list(hashed[:5]) + [hashed]
|
||||
return os.path.join(self.directory, *parts)
|
||||
|
||||
def get(self, key):
|
||||
# type: (str) -> Optional[bytes]
|
||||
path = self._get_cache_path(key)
|
||||
with suppressed_cache_errors():
|
||||
with open(path, 'rb') as f:
|
||||
return f.read()
|
||||
|
||||
def set(self, key, value):
|
||||
# type: (str, bytes) -> None
|
||||
path = self._get_cache_path(key)
|
||||
with suppressed_cache_errors():
|
||||
ensure_dir(os.path.dirname(path))
|
||||
|
||||
with adjacent_tmp_file(path) as f:
|
||||
f.write(value)
|
||||
|
||||
replace(f.name, path)
|
||||
|
||||
def delete(self, key):
|
||||
# type: (str) -> None
|
||||
path = self._get_cache_path(key)
|
||||
with suppressed_cache_errors():
|
||||
os.remove(path)
|
||||
@@ -0,0 +1,200 @@
|
||||
"""Download files with progress indicators.
|
||||
"""
|
||||
import cgi
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
|
||||
from pip._vendor import requests
|
||||
from pip._vendor.requests.models import CONTENT_CHUNK_SIZE
|
||||
|
||||
from pip._internal.models.index import PyPI
|
||||
from pip._internal.network.cache import is_from_cache
|
||||
from pip._internal.network.utils import response_chunks
|
||||
from pip._internal.utils.misc import (
|
||||
format_size,
|
||||
redact_auth_from_url,
|
||||
splitext,
|
||||
)
|
||||
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
|
||||
from pip._internal.utils.ui import DownloadProgressProvider
|
||||
|
||||
if MYPY_CHECK_RUNNING:
|
||||
from typing import Iterable, Optional
|
||||
|
||||
from pip._vendor.requests.models import Response
|
||||
|
||||
from pip._internal.models.link import Link
|
||||
from pip._internal.network.session import PipSession
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_http_response_size(resp):
|
||||
# type: (Response) -> Optional[int]
|
||||
try:
|
||||
return int(resp.headers['content-length'])
|
||||
except (ValueError, KeyError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _prepare_download(
|
||||
resp, # type: Response
|
||||
link, # type: Link
|
||||
progress_bar # type: str
|
||||
):
|
||||
# type: (...) -> Iterable[bytes]
|
||||
total_length = _get_http_response_size(resp)
|
||||
|
||||
if link.netloc == PyPI.file_storage_domain:
|
||||
url = link.show_url
|
||||
else:
|
||||
url = link.url_without_fragment
|
||||
|
||||
logged_url = redact_auth_from_url(url)
|
||||
|
||||
if total_length:
|
||||
logged_url = '{} ({})'.format(logged_url, format_size(total_length))
|
||||
|
||||
if is_from_cache(resp):
|
||||
logger.info("Using cached %s", logged_url)
|
||||
else:
|
||||
logger.info("Downloading %s", logged_url)
|
||||
|
||||
if logger.getEffectiveLevel() > logging.INFO:
|
||||
show_progress = False
|
||||
elif is_from_cache(resp):
|
||||
show_progress = False
|
||||
elif not total_length:
|
||||
show_progress = True
|
||||
elif total_length > (40 * 1000):
|
||||
show_progress = True
|
||||
else:
|
||||
show_progress = False
|
||||
|
||||
chunks = response_chunks(resp, CONTENT_CHUNK_SIZE)
|
||||
|
||||
if not show_progress:
|
||||
return chunks
|
||||
|
||||
return DownloadProgressProvider(
|
||||
progress_bar, max=total_length
|
||||
)(chunks)
|
||||
|
||||
|
||||
def sanitize_content_filename(filename):
|
||||
# type: (str) -> str
|
||||
"""
|
||||
Sanitize the "filename" value from a Content-Disposition header.
|
||||
"""
|
||||
return os.path.basename(filename)
|
||||
|
||||
|
||||
def parse_content_disposition(content_disposition, default_filename):
|
||||
# type: (str, str) -> str
|
||||
"""
|
||||
Parse the "filename" value from a Content-Disposition header, and
|
||||
return the default filename if the result is empty.
|
||||
"""
|
||||
_type, params = cgi.parse_header(content_disposition)
|
||||
filename = params.get('filename')
|
||||
if filename:
|
||||
# We need to sanitize the filename to prevent directory traversal
|
||||
# in case the filename contains ".." path parts.
|
||||
filename = sanitize_content_filename(filename)
|
||||
return filename or default_filename
|
||||
|
||||
|
||||
def _get_http_response_filename(resp, link):
|
||||
# type: (Response, Link) -> str
|
||||
"""Get an ideal filename from the given HTTP response, falling back to
|
||||
the link filename if not provided.
|
||||
"""
|
||||
filename = link.filename # fallback
|
||||
# Have a look at the Content-Disposition header for a better guess
|
||||
content_disposition = resp.headers.get('content-disposition')
|
||||
if content_disposition:
|
||||
filename = parse_content_disposition(content_disposition, filename)
|
||||
ext = splitext(filename)[1] # type: Optional[str]
|
||||
if not ext:
|
||||
ext = mimetypes.guess_extension(
|
||||
resp.headers.get('content-type', '')
|
||||
)
|
||||
if ext:
|
||||
filename += ext
|
||||
if not ext and link.url != resp.url:
|
||||
ext = os.path.splitext(resp.url)[1]
|
||||
if ext:
|
||||
filename += ext
|
||||
return filename
|
||||
|
||||
|
||||
def _http_get_download(session, link):
|
||||
# type: (PipSession, Link) -> Response
|
||||
target_url = link.url.split('#', 1)[0]
|
||||
resp = session.get(
|
||||
target_url,
|
||||
# We use Accept-Encoding: identity here because requests
|
||||
# defaults to accepting compressed responses. This breaks in
|
||||
# a variety of ways depending on how the server is configured.
|
||||
# - Some servers will notice that the file isn't a compressible
|
||||
# file and will leave the file alone and with an empty
|
||||
# Content-Encoding
|
||||
# - Some servers will notice that the file is already
|
||||
# compressed and will leave the file alone and will add a
|
||||
# Content-Encoding: gzip header
|
||||
# - Some servers won't notice anything at all and will take
|
||||
# a file that's already been compressed and compress it again
|
||||
# and set the Content-Encoding: gzip header
|
||||
# By setting this to request only the identity encoding We're
|
||||
# hoping to eliminate the third case. Hopefully there does not
|
||||
# exist a server which when given a file will notice it is
|
||||
# already compressed and that you're not asking for a
|
||||
# compressed file and will then decompress it before sending
|
||||
# because if that's the case I don't think it'll ever be
|
||||
# possible to make this work.
|
||||
headers={"Accept-Encoding": "identity"},
|
||||
stream=True,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp
|
||||
|
||||
|
||||
class Download(object):
|
||||
def __init__(
|
||||
self,
|
||||
response, # type: Response
|
||||
filename, # type: str
|
||||
chunks, # type: Iterable[bytes]
|
||||
):
|
||||
# type: (...) -> None
|
||||
self.response = response
|
||||
self.filename = filename
|
||||
self.chunks = chunks
|
||||
|
||||
|
||||
class Downloader(object):
|
||||
def __init__(
|
||||
self,
|
||||
session, # type: PipSession
|
||||
progress_bar, # type: str
|
||||
):
|
||||
# type: (...) -> None
|
||||
self._session = session
|
||||
self._progress_bar = progress_bar
|
||||
|
||||
def __call__(self, link):
|
||||
# type: (Link) -> Download
|
||||
try:
|
||||
resp = _http_get_download(self._session, link)
|
||||
except requests.HTTPError as e:
|
||||
logger.critical(
|
||||
"HTTP error %s while getting %s", e.response.status_code, link
|
||||
)
|
||||
raise
|
||||
|
||||
return Download(
|
||||
resp,
|
||||
_get_http_response_filename(resp, link),
|
||||
_prepare_download(resp, link, self._progress_bar),
|
||||
)
|
||||
@@ -0,0 +1,405 @@
|
||||
"""PipSession and supporting code, containing all pip-specific
|
||||
network request configuration and behavior.
|
||||
"""
|
||||
|
||||
# The following comment should be removed at some point in the future.
|
||||
# mypy: disallow-untyped-defs=False
|
||||
|
||||
import email.utils
|
||||
import json
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
import platform
|
||||
import sys
|
||||
import warnings
|
||||
|
||||
from pip._vendor import requests, six, urllib3
|
||||
from pip._vendor.cachecontrol import CacheControlAdapter
|
||||
from pip._vendor.requests.adapters import BaseAdapter, HTTPAdapter
|
||||
from pip._vendor.requests.models import Response
|
||||
from pip._vendor.requests.structures import CaseInsensitiveDict
|
||||
from pip._vendor.six.moves.urllib import parse as urllib_parse
|
||||
from pip._vendor.urllib3.exceptions import InsecureRequestWarning
|
||||
|
||||
from pip import __version__
|
||||
from pip._internal.network.auth import MultiDomainBasicAuth
|
||||
from pip._internal.network.cache import SafeFileCache
|
||||
# Import ssl from compat so the initial import occurs in only one place.
|
||||
from pip._internal.utils.compat import has_tls, ipaddress
|
||||
from pip._internal.utils.glibc import libc_ver
|
||||
from pip._internal.utils.misc import (
|
||||
build_url_from_netloc,
|
||||
get_installed_version,
|
||||
parse_netloc,
|
||||
)
|
||||
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
|
||||
from pip._internal.utils.urls import url_to_path
|
||||
|
||||
if MYPY_CHECK_RUNNING:
|
||||
from typing import (
|
||||
Iterator, List, Optional, Tuple, Union,
|
||||
)
|
||||
|
||||
from pip._internal.models.link import Link
|
||||
|
||||
SecureOrigin = Tuple[str, str, Optional[Union[int, str]]]
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Ignore warning raised when using --trusted-host.
|
||||
warnings.filterwarnings("ignore", category=InsecureRequestWarning)
|
||||
|
||||
|
||||
SECURE_ORIGINS = [
|
||||
# protocol, hostname, port
|
||||
# Taken from Chrome's list of secure origins (See: http://bit.ly/1qrySKC)
|
||||
("https", "*", "*"),
|
||||
("*", "localhost", "*"),
|
||||
("*", "127.0.0.0/8", "*"),
|
||||
("*", "::1/128", "*"),
|
||||
("file", "*", None),
|
||||
# ssh is always secure.
|
||||
("ssh", "*", "*"),
|
||||
] # type: List[SecureOrigin]
|
||||
|
||||
|
||||
# These are environment variables present when running under various
|
||||
# CI systems. For each variable, some CI systems that use the variable
|
||||
# are indicated. The collection was chosen so that for each of a number
|
||||
# of popular systems, at least one of the environment variables is used.
|
||||
# This list is used to provide some indication of and lower bound for
|
||||
# CI traffic to PyPI. Thus, it is okay if the list is not comprehensive.
|
||||
# For more background, see: https://github.com/pypa/pip/issues/5499
|
||||
CI_ENVIRONMENT_VARIABLES = (
|
||||
# Azure Pipelines
|
||||
'BUILD_BUILDID',
|
||||
# Jenkins
|
||||
'BUILD_ID',
|
||||
# AppVeyor, CircleCI, Codeship, Gitlab CI, Shippable, Travis CI
|
||||
'CI',
|
||||
# Explicit environment variable.
|
||||
'PIP_IS_CI',
|
||||
)
|
||||
|
||||
|
||||
def looks_like_ci():
|
||||
# type: () -> bool
|
||||
"""
|
||||
Return whether it looks like pip is running under CI.
|
||||
"""
|
||||
# We don't use the method of checking for a tty (e.g. using isatty())
|
||||
# because some CI systems mimic a tty (e.g. Travis CI). Thus that
|
||||
# method doesn't provide definitive information in either direction.
|
||||
return any(name in os.environ for name in CI_ENVIRONMENT_VARIABLES)
|
||||
|
||||
|
||||
def user_agent():
|
||||
"""
|
||||
Return a string representing the user agent.
|
||||
"""
|
||||
data = {
|
||||
"installer": {"name": "pip", "version": __version__},
|
||||
"python": platform.python_version(),
|
||||
"implementation": {
|
||||
"name": platform.python_implementation(),
|
||||
},
|
||||
}
|
||||
|
||||
if data["implementation"]["name"] == 'CPython':
|
||||
data["implementation"]["version"] = platform.python_version()
|
||||
elif data["implementation"]["name"] == 'PyPy':
|
||||
if sys.pypy_version_info.releaselevel == 'final':
|
||||
pypy_version_info = sys.pypy_version_info[:3]
|
||||
else:
|
||||
pypy_version_info = sys.pypy_version_info
|
||||
data["implementation"]["version"] = ".".join(
|
||||
[str(x) for x in pypy_version_info]
|
||||
)
|
||||
elif data["implementation"]["name"] == 'Jython':
|
||||
# Complete Guess
|
||||
data["implementation"]["version"] = platform.python_version()
|
||||
elif data["implementation"]["name"] == 'IronPython':
|
||||
# Complete Guess
|
||||
data["implementation"]["version"] = platform.python_version()
|
||||
|
||||
if sys.platform.startswith("linux"):
|
||||
from pip._vendor import distro
|
||||
distro_infos = dict(filter(
|
||||
lambda x: x[1],
|
||||
zip(["name", "version", "id"], distro.linux_distribution()),
|
||||
))
|
||||
libc = dict(filter(
|
||||
lambda x: x[1],
|
||||
zip(["lib", "version"], libc_ver()),
|
||||
))
|
||||
if libc:
|
||||
distro_infos["libc"] = libc
|
||||
if distro_infos:
|
||||
data["distro"] = distro_infos
|
||||
|
||||
if sys.platform.startswith("darwin") and platform.mac_ver()[0]:
|
||||
data["distro"] = {"name": "macOS", "version": platform.mac_ver()[0]}
|
||||
|
||||
if platform.system():
|
||||
data.setdefault("system", {})["name"] = platform.system()
|
||||
|
||||
if platform.release():
|
||||
data.setdefault("system", {})["release"] = platform.release()
|
||||
|
||||
if platform.machine():
|
||||
data["cpu"] = platform.machine()
|
||||
|
||||
if has_tls():
|
||||
import _ssl as ssl
|
||||
data["openssl_version"] = ssl.OPENSSL_VERSION
|
||||
|
||||
setuptools_version = get_installed_version("setuptools")
|
||||
if setuptools_version is not None:
|
||||
data["setuptools_version"] = setuptools_version
|
||||
|
||||
# Use None rather than False so as not to give the impression that
|
||||
# pip knows it is not being run under CI. Rather, it is a null or
|
||||
# inconclusive result. Also, we include some value rather than no
|
||||
# value to make it easier to know that the check has been run.
|
||||
data["ci"] = True if looks_like_ci() else None
|
||||
|
||||
user_data = os.environ.get("PIP_USER_AGENT_USER_DATA")
|
||||
if user_data is not None:
|
||||
data["user_data"] = user_data
|
||||
|
||||
return "{data[installer][name]}/{data[installer][version]} {json}".format(
|
||||
data=data,
|
||||
json=json.dumps(data, separators=(",", ":"), sort_keys=True),
|
||||
)
|
||||
|
||||
|
||||
class LocalFSAdapter(BaseAdapter):
|
||||
|
||||
def send(self, request, stream=None, timeout=None, verify=None, cert=None,
|
||||
proxies=None):
|
||||
pathname = url_to_path(request.url)
|
||||
|
||||
resp = Response()
|
||||
resp.status_code = 200
|
||||
resp.url = request.url
|
||||
|
||||
try:
|
||||
stats = os.stat(pathname)
|
||||
except OSError as exc:
|
||||
resp.status_code = 404
|
||||
resp.raw = exc
|
||||
else:
|
||||
modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
|
||||
content_type = mimetypes.guess_type(pathname)[0] or "text/plain"
|
||||
resp.headers = CaseInsensitiveDict({
|
||||
"Content-Type": content_type,
|
||||
"Content-Length": stats.st_size,
|
||||
"Last-Modified": modified,
|
||||
})
|
||||
|
||||
resp.raw = open(pathname, "rb")
|
||||
resp.close = resp.raw.close
|
||||
|
||||
return resp
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
|
||||
class InsecureHTTPAdapter(HTTPAdapter):
|
||||
|
||||
def cert_verify(self, conn, url, verify, cert):
|
||||
super(InsecureHTTPAdapter, self).cert_verify(
|
||||
conn=conn, url=url, verify=False, cert=cert
|
||||
)
|
||||
|
||||
|
||||
class PipSession(requests.Session):
|
||||
|
||||
timeout = None # type: Optional[int]
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
"""
|
||||
:param trusted_hosts: Domains not to emit warnings for when not using
|
||||
HTTPS.
|
||||
"""
|
||||
retries = kwargs.pop("retries", 0)
|
||||
cache = kwargs.pop("cache", None)
|
||||
trusted_hosts = kwargs.pop("trusted_hosts", []) # type: List[str]
|
||||
index_urls = kwargs.pop("index_urls", None)
|
||||
|
||||
super(PipSession, self).__init__(*args, **kwargs)
|
||||
|
||||
# Namespace the attribute with "pip_" just in case to prevent
|
||||
# possible conflicts with the base class.
|
||||
self.pip_trusted_origins = [] # type: List[Tuple[str, Optional[int]]]
|
||||
|
||||
# Attach our User Agent to the request
|
||||
self.headers["User-Agent"] = user_agent()
|
||||
|
||||
# Attach our Authentication handler to the session
|
||||
self.auth = MultiDomainBasicAuth(index_urls=index_urls)
|
||||
|
||||
# Create our urllib3.Retry instance which will allow us to customize
|
||||
# how we handle retries.
|
||||
retries = urllib3.Retry(
|
||||
# Set the total number of retries that a particular request can
|
||||
# have.
|
||||
total=retries,
|
||||
|
||||
# A 503 error from PyPI typically means that the Fastly -> Origin
|
||||
# connection got interrupted in some way. A 503 error in general
|
||||
# is typically considered a transient error so we'll go ahead and
|
||||
# retry it.
|
||||
# A 500 may indicate transient error in Amazon S3
|
||||
# A 520 or 527 - may indicate transient error in CloudFlare
|
||||
status_forcelist=[500, 503, 520, 527],
|
||||
|
||||
# Add a small amount of back off between failed requests in
|
||||
# order to prevent hammering the service.
|
||||
backoff_factor=0.25,
|
||||
)
|
||||
|
||||
# We want to _only_ cache responses on securely fetched origins. We do
|
||||
# this because we can't validate the response of an insecurely fetched
|
||||
# origin, and we don't want someone to be able to poison the cache and
|
||||
# require manual eviction from the cache to fix it.
|
||||
if cache:
|
||||
secure_adapter = CacheControlAdapter(
|
||||
cache=SafeFileCache(cache),
|
||||
max_retries=retries,
|
||||
)
|
||||
else:
|
||||
secure_adapter = HTTPAdapter(max_retries=retries)
|
||||
|
||||
# Our Insecure HTTPAdapter disables HTTPS validation. It does not
|
||||
# support caching (see above) so we'll use it for all http:// URLs as
|
||||
# well as any https:// host that we've marked as ignoring TLS errors
|
||||
# for.
|
||||
insecure_adapter = InsecureHTTPAdapter(max_retries=retries)
|
||||
# Save this for later use in add_insecure_host().
|
||||
self._insecure_adapter = insecure_adapter
|
||||
|
||||
self.mount("https://", secure_adapter)
|
||||
self.mount("http://", insecure_adapter)
|
||||
|
||||
# Enable file:// urls
|
||||
self.mount("file://", LocalFSAdapter())
|
||||
|
||||
for host in trusted_hosts:
|
||||
self.add_trusted_host(host, suppress_logging=True)
|
||||
|
||||
def add_trusted_host(self, host, source=None, suppress_logging=False):
|
||||
# type: (str, Optional[str], bool) -> None
|
||||
"""
|
||||
:param host: It is okay to provide a host that has previously been
|
||||
added.
|
||||
:param source: An optional source string, for logging where the host
|
||||
string came from.
|
||||
"""
|
||||
if not suppress_logging:
|
||||
msg = 'adding trusted host: {!r}'.format(host)
|
||||
if source is not None:
|
||||
msg += ' (from {})'.format(source)
|
||||
logger.info(msg)
|
||||
|
||||
host_port = parse_netloc(host)
|
||||
if host_port not in self.pip_trusted_origins:
|
||||
self.pip_trusted_origins.append(host_port)
|
||||
|
||||
self.mount(build_url_from_netloc(host) + '/', self._insecure_adapter)
|
||||
if not host_port[1]:
|
||||
# Mount wildcard ports for the same host.
|
||||
self.mount(
|
||||
build_url_from_netloc(host) + ':',
|
||||
self._insecure_adapter
|
||||
)
|
||||
|
||||
def iter_secure_origins(self):
|
||||
# type: () -> Iterator[SecureOrigin]
|
||||
for secure_origin in SECURE_ORIGINS:
|
||||
yield secure_origin
|
||||
for host, port in self.pip_trusted_origins:
|
||||
yield ('*', host, '*' if port is None else port)
|
||||
|
||||
def is_secure_origin(self, location):
|
||||
# type: (Link) -> bool
|
||||
# Determine if this url used a secure transport mechanism
|
||||
parsed = urllib_parse.urlparse(str(location))
|
||||
origin_protocol, origin_host, origin_port = (
|
||||
parsed.scheme, parsed.hostname, parsed.port,
|
||||
)
|
||||
|
||||
# The protocol to use to see if the protocol matches.
|
||||
# Don't count the repository type as part of the protocol: in
|
||||
# cases such as "git+ssh", only use "ssh". (I.e., Only verify against
|
||||
# the last scheme.)
|
||||
origin_protocol = origin_protocol.rsplit('+', 1)[-1]
|
||||
|
||||
# Determine if our origin is a secure origin by looking through our
|
||||
# hardcoded list of secure origins, as well as any additional ones
|
||||
# configured on this PackageFinder instance.
|
||||
for secure_origin in self.iter_secure_origins():
|
||||
secure_protocol, secure_host, secure_port = secure_origin
|
||||
if origin_protocol != secure_protocol and secure_protocol != "*":
|
||||
continue
|
||||
|
||||
try:
|
||||
addr = ipaddress.ip_address(
|
||||
None
|
||||
if origin_host is None
|
||||
else six.ensure_text(origin_host)
|
||||
)
|
||||
network = ipaddress.ip_network(
|
||||
six.ensure_text(secure_host)
|
||||
)
|
||||
except ValueError:
|
||||
# We don't have both a valid address or a valid network, so
|
||||
# we'll check this origin against hostnames.
|
||||
if (
|
||||
origin_host and
|
||||
origin_host.lower() != secure_host.lower() and
|
||||
secure_host != "*"
|
||||
):
|
||||
continue
|
||||
else:
|
||||
# We have a valid address and network, so see if the address
|
||||
# is contained within the network.
|
||||
if addr not in network:
|
||||
continue
|
||||
|
||||
# Check to see if the port matches.
|
||||
if (
|
||||
origin_port != secure_port and
|
||||
secure_port != "*" and
|
||||
secure_port is not None
|
||||
):
|
||||
continue
|
||||
|
||||
# If we've gotten here, then this origin matches the current
|
||||
# secure origin and we should return True
|
||||
return True
|
||||
|
||||
# If we've gotten to this point, then the origin isn't secure and we
|
||||
# will not accept it as a valid location to search. We will however
|
||||
# log a warning that we are ignoring it.
|
||||
logger.warning(
|
||||
"The repository located at %s is not a trusted or secure host and "
|
||||
"is being ignored. If this repository is available via HTTPS we "
|
||||
"recommend you use HTTPS instead, otherwise you may silence "
|
||||
"this warning and allow it anyway with '--trusted-host %s'.",
|
||||
origin_host,
|
||||
origin_host,
|
||||
)
|
||||
|
||||
return False
|
||||
|
||||
def request(self, method, url, *args, **kwargs):
|
||||
# Allow setting a default timeout on a session
|
||||
kwargs.setdefault("timeout", self.timeout)
|
||||
|
||||
# Dispatch the actual request
|
||||
return super(PipSession, self).request(method, url, *args, **kwargs)
|
||||
@@ -0,0 +1,48 @@
|
||||
from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response
|
||||
|
||||
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
|
||||
|
||||
if MYPY_CHECK_RUNNING:
|
||||
from typing import Iterator
|
||||
|
||||
|
||||
def response_chunks(response, chunk_size=CONTENT_CHUNK_SIZE):
|
||||
# type: (Response, int) -> Iterator[bytes]
|
||||
"""Given a requests Response, provide the data chunks.
|
||||
"""
|
||||
try:
|
||||
# Special case for urllib3.
|
||||
for chunk in response.raw.stream(
|
||||
chunk_size,
|
||||
# We use decode_content=False here because we don't
|
||||
# want urllib3 to mess with the raw bytes we get
|
||||
# from the server. If we decompress inside of
|
||||
# urllib3 then we cannot verify the checksum
|
||||
# because the checksum will be of the compressed
|
||||
# file. This breakage will only occur if the
|
||||
# server adds a Content-Encoding header, which
|
||||
# depends on how the server was configured:
|
||||
# - Some servers will notice that the file isn't a
|
||||
# compressible file and will leave the file alone
|
||||
# and with an empty Content-Encoding
|
||||
# - Some servers will notice that the file is
|
||||
# already compressed and will leave the file
|
||||
# alone and will add a Content-Encoding: gzip
|
||||
# header
|
||||
# - Some servers won't notice anything at all and
|
||||
# will take a file that's already been compressed
|
||||
# and compress it again and set the
|
||||
# Content-Encoding: gzip header
|
||||
#
|
||||
# By setting this not to decode automatically we
|
||||
# hope to eliminate problems with the second case.
|
||||
decode_content=False,
|
||||
):
|
||||
yield chunk
|
||||
except AttributeError:
|
||||
# Standard file-like object.
|
||||
while True:
|
||||
chunk = response.raw.read(chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
@@ -0,0 +1,44 @@
|
||||
"""xmlrpclib.Transport implementation
|
||||
"""
|
||||
|
||||
# The following comment should be removed at some point in the future.
|
||||
# mypy: disallow-untyped-defs=False
|
||||
|
||||
import logging
|
||||
|
||||
from pip._vendor import requests
|
||||
# NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is
|
||||
# why we ignore the type on this import
|
||||
from pip._vendor.six.moves import xmlrpc_client # type: ignore
|
||||
from pip._vendor.six.moves.urllib import parse as urllib_parse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PipXmlrpcTransport(xmlrpc_client.Transport):
|
||||
"""Provide a `xmlrpclib.Transport` implementation via a `PipSession`
|
||||
object.
|
||||
"""
|
||||
|
||||
def __init__(self, index_url, session, use_datetime=False):
|
||||
xmlrpc_client.Transport.__init__(self, use_datetime)
|
||||
index_parts = urllib_parse.urlparse(index_url)
|
||||
self._scheme = index_parts.scheme
|
||||
self._session = session
|
||||
|
||||
def request(self, host, handler, request_body, verbose=False):
|
||||
parts = (self._scheme, host, handler, None, None, None)
|
||||
url = urllib_parse.urlunparse(parts)
|
||||
try:
|
||||
headers = {'Content-Type': 'text/xml'}
|
||||
response = self._session.post(url, data=request_body,
|
||||
headers=headers, stream=True)
|
||||
response.raise_for_status()
|
||||
self.verbose = verbose
|
||||
return self.parse_response(response.raw)
|
||||
except requests.HTTPError as exc:
|
||||
logger.critical(
|
||||
"HTTP error %s while getting %s",
|
||||
exc.response.status_code, url,
|
||||
)
|
||||
raise
|
||||
Reference in New Issue
Block a user