diff --git a/src/hyperlink/_url.py b/src/hyperlink/_url.py index 0b665327..15f6cc9d 100644 --- a/src/hyperlink/_url.py +++ b/src/hyperlink/_url.py @@ -19,36 +19,50 @@ import sys import string import socket -from typing import Callable, Text, Type +from socket import AF_INET, AF_INET6 +try: + from socket import AddressFamily +except ImportError: + AddressFamily = int # type: ignore[assignment,misc] +from typing import ( + Any, Callable, Dict, Iterable, Iterator, List, Mapping, Optional, + Sequence, Text, Tuple, Type, TypeVar, Union, cast, +) from unicodedata import normalize from ._socket import inet_pton try: - from collections.abc import Mapping + from collections.abc import Mapping as MappingABC except ImportError: # Python 2 - from collections import Mapping + from collections import Mapping as MappingABC from idna import encode as idna_encode, decode as idna_decode PY2 = (sys.version_info[0] == 2) -unicode = type(u'') try: unichr except NameError: # Py3 unichr = chr # type: Callable[[int], Text] NoneType = type(None) # type: Type[None] +QueryPairs = Tuple[Tuple[Text, Optional[Text]], ...] # internal representation +QueryParameters = Union[ + Mapping[Text, Optional[Text]], QueryPairs, + Sequence[Tuple[Text, Optional[Text]]], +] +T = TypeVar('T') # from boltons.typeutils -def make_sentinel(name='_MISSING', var_name=None): +def make_sentinel(name='_MISSING', var_name=""): + # type: (str, str) -> object """Creates and returns a new **instance** of a new class, suitable for usage as a "sentinel", a kind of singleton often used to indicate a value is missing when ``None`` is a valid input. Args: - name (str): Name of the Sentinel - var_name (str): Set this name to the name of the variable in - its respective module enable pickleability. + name: Name of the Sentinel + var_name: Set this name to the name of the variable in its respective + module enable pickle-ability. >>> make_sentinel(var_name='_MISSING') _MISSING @@ -61,29 +75,34 @@ def make_sentinel(name='_MISSING', var_name=None): .. note:: - By design, additional calls to ``make_sentinel`` with the same - values will not produce equivalent objects. - - >>> make_sentinel('TEST') == make_sentinel('TEST') - False - >>> type(make_sentinel('TEST')) == type(make_sentinel('TEST')) - False + By design, additional calls to ``make_sentinel`` with the same + values will not produce equivalent objects. + >>> make_sentinel('TEST') == make_sentinel('TEST') + False + >>> type(make_sentinel('TEST')) == type(make_sentinel('TEST')) + False """ class Sentinel(object): def __init__(self): + # type: () -> None self.name = name self.var_name = var_name def __repr__(self): + # type: () -> str if self.var_name: return self.var_name return '%s(%r)' % (self.__class__.__name__, self.name) if var_name: - def __reduce__(self): + # superclass type hints don't allow str return type, but it is + # allowed in the docs, hence the ignore[override] below + def __reduce__(self): # type: ignore[override] + # type: () -> str return self.var_name def __nonzero__(self): + # type: () -> bool return False __bool__ = __nonzero__ @@ -91,7 +110,7 @@ def __nonzero__(self): return Sentinel() -_unspecified = _UNSET = make_sentinel('_UNSET') +_unspecified = _UNSET = make_sentinel('_UNSET') # type: Any # RFC 3986 Section 2.3, Unreserved URI Characters @@ -142,6 +161,7 @@ def __nonzero__(self): def _make_decode_map(delims, allow_percent=False): + # type: (Iterable[Text], bool) -> Mapping[bytes, bytes] ret = dict(_HEX_CHAR_MAP) if not allow_percent: delims = set(delims) | set([u'%']) @@ -155,7 +175,8 @@ def _make_decode_map(delims, allow_percent=False): def _make_quote_map(safe_chars): - ret = {} + # type: (Iterable[Text]) -> Mapping[Union[int, Text], Text] + ret = {} # type: Dict[Union[int, Text], Text] # v is included in the dict for py3 mostly, because bytestrings # are iterables of ints, of course! for i, v in zip(range(256), range(256)): @@ -187,6 +208,7 @@ def _make_quote_map(safe_chars): def _encode_reserved(text, maximal=True): + # type: (Text, bool) -> Text """A very comprehensive percent encoding for encoding all delimiters. Used for arguments to DecodedURL, where a % means a percent sign, and not the character used by URLs for escaping @@ -200,6 +222,7 @@ def _encode_reserved(text, maximal=True): def _encode_path_part(text, maximal=True): + # type: (Text, bool) -> Text "Percent-encode a single segment of a URL path." if maximal: bytestr = normalize('NFC', text).encode('utf8') @@ -209,6 +232,7 @@ def _encode_path_part(text, maximal=True): def _encode_schemeless_path_part(text, maximal=True): + # type: (Text, bool) -> Text """Percent-encode the first segment of a URL path for a URL without a scheme specified. """ @@ -219,8 +243,14 @@ def _encode_schemeless_path_part(text, maximal=True): if t in _SCHEMELESS_PATH_DELIMS else t for t in text]) -def _encode_path_parts(text_parts, rooted=False, has_scheme=True, - has_authority=True, joined=True, maximal=True): +def _encode_path_parts( + text_parts, # type: Sequence[Text] + rooted=False, # type: bool + has_scheme=True, # type: bool + has_authority=True, # type: bool + maximal=True, # type: bool +): + # type: (...) -> Sequence[Text] """ Percent-encode a tuple of path parts into a complete path. @@ -242,12 +272,12 @@ def _encode_path_parts(text_parts, rooted=False, has_scheme=True, first path segment cannot contain a colon (":") character. """ if not text_parts: - return u'' if joined else text_parts + return () if rooted: - text_parts = (u'',) + text_parts + text_parts = (u'',) + tuple(text_parts) # elif has_authority and text_parts: # raise Exception('see rfc above') # TODO: too late to fail like this? - encoded_parts = [] + encoded_parts = [] # type: List[Text] if has_scheme: encoded_parts = [_encode_path_part(part, maximal=maximal) if part else part for part in text_parts] @@ -255,12 +285,11 @@ def _encode_path_parts(text_parts, rooted=False, has_scheme=True, encoded_parts = [_encode_schemeless_path_part(text_parts[0])] encoded_parts.extend([_encode_path_part(part, maximal=maximal) if part else part for part in text_parts[1:]]) - if joined: - return u'/'.join(encoded_parts) return tuple(encoded_parts) def _encode_query_key(text, maximal=True): + # type: (Text, bool) -> Text """ Percent-encode a single query string key or value. """ @@ -272,6 +301,7 @@ def _encode_query_key(text, maximal=True): def _encode_query_value(text, maximal=True): + # type: (Text, bool) -> Text """ Percent-encode a single query string key or value. """ @@ -283,6 +313,7 @@ def _encode_query_value(text, maximal=True): def _encode_fragment_part(text, maximal=True): + # type: (Text, bool) -> Text """Quote the fragment part of the URL. Fragments don't have subdelimiters, so the whole URL fragment can be passed. """ @@ -294,6 +325,7 @@ def _encode_fragment_part(text, maximal=True): def _encode_userinfo_part(text, maximal=True): + # type: (Text, bool) -> Text """Quote special characters in either the username or password section of the URL. """ @@ -328,6 +360,7 @@ def _encode_userinfo_part(text, maximal=True): def register_scheme(text, uses_netloc=True, default_port=None): + # type: (Text, bool, Optional[int]) -> None """Registers new scheme information, resulting in correct port and slash behavior from the URL object. There are dozens of standard schemes preregistered, so this function is mostly meant for @@ -336,16 +369,15 @@ def register_scheme(text, uses_netloc=True, default_port=None): `file an issue`_! Args: - text (unicode): Text representing the scheme. - (the 'http' in 'http://hatnote.com') + text (Text): A string representation of the scheme. + (the 'http' in 'http://hatnote.com') uses_netloc (bool): Does the scheme support specifying a - network host? For instance, "http" does, "mailto" does - not. Defaults to True. - default_port (int): The default port, if any, for netloc-using - schemes. + network host? For instance, "http" does, "mailto" does + not. Defaults to True. + default_port (Optional[int]): The default port, if any, for + netloc-using schemes. .. _file an issue: https://github.com/mahmoud/hyperlink/issues - """ text = text.lower() if default_port is not None: @@ -369,6 +401,7 @@ def register_scheme(text, uses_netloc=True, default_port=None): def scheme_uses_netloc(scheme, default=None): + # type: (Text, Optional[bool]) -> Optional[bool] """Whether or not a URL uses :code:`:` or :code:`://` to separate the scheme from the rest of the URL depends on the scheme's own standard definition. There is no way to infer this behavior @@ -407,6 +440,7 @@ class URLParseError(ValueError): def _optional(argument, default): + # type: (Any, Any) -> Any if argument is _UNSET: return default else: @@ -414,6 +448,7 @@ def _optional(argument, default): def _typecheck(name, value, *types): + # type: (Text, T, Type) -> T """ Check that the given *value* is one of the given *types*, or raise an exception describing the problem using *name*. @@ -428,9 +463,11 @@ def _typecheck(name, value, *types): def _textcheck(name, value, delims=frozenset(), nullable=False): - if not isinstance(value, unicode): + # type: (Text, T, Iterable[Text], bool) -> T + if not isinstance(value, Text): if nullable and value is None: - return value # used by query string values + # used by query string values + return value # type: ignore[misc] # unreachable else: str_name = "unicode" if PY2 else "str" exp = str_name + ' or NoneType' if nullable else str_name @@ -438,10 +475,11 @@ def _textcheck(name, value, delims=frozenset(), nullable=False): if delims and set(value) & set(delims): # TODO: test caching into regexes raise ValueError('one or more reserved delimiters %s present in %s: %r' % (''.join(delims), name, value)) - return value + return value # type: ignore[return-value] # T vs. Text def iter_pairs(iterable): + # type: (Iterable) -> Iterator """ Iterate over the (key, value) pairs in ``iterable``. @@ -449,7 +487,7 @@ def iter_pairs(iterable): iterable yields (key, value) pairs. This behaviour is similar to what Python's ``dict()`` constructor does. """ - if isinstance(iterable, Mapping): + if isinstance(iterable, MappingABC): iterable = iterable.items() return iter(iterable) @@ -457,6 +495,7 @@ def iter_pairs(iterable): def _decode_unreserved( text, normalize_case=False, encode_stray_percents=False ): + # type: (Text, bool, bool) -> Text return _percent_decode(text, normalize_case=normalize_case, encode_stray_percents=encode_stray_percents, _decode_map=_UNRESERVED_DECODE_MAP) @@ -465,6 +504,7 @@ def _decode_unreserved( def _decode_userinfo_part( text, normalize_case=False, encode_stray_percents=False ): + # type: (Text, bool, bool) -> Text return _percent_decode(text, normalize_case=normalize_case, encode_stray_percents=encode_stray_percents, _decode_map=_USERINFO_DECODE_MAP) @@ -473,6 +513,7 @@ def _decode_userinfo_part( def _decode_path_part( text, normalize_case=False, encode_stray_percents=False ): + # type: (Text, bool, bool) -> Text """ >>> _decode_path_part(u'%61%77%2f%7a') u'aw%2fz' @@ -487,6 +528,7 @@ def _decode_path_part( def _decode_query_key( text, normalize_case=False, encode_stray_percents=False ): + # type: (Text, bool, bool) -> Text return _percent_decode(text, normalize_case=normalize_case, encode_stray_percents=encode_stray_percents, _decode_map=_QUERY_KEY_DECODE_MAP) @@ -495,6 +537,7 @@ def _decode_query_key( def _decode_query_value( text, normalize_case=False, encode_stray_percents=False ): + # type: (Text, bool, bool) -> Text return _percent_decode(text, normalize_case=normalize_case, encode_stray_percents=encode_stray_percents, _decode_map=_QUERY_VALUE_DECODE_MAP) @@ -503,14 +546,21 @@ def _decode_query_value( def _decode_fragment_part( text, normalize_case=False, encode_stray_percents=False ): + # type: (Text, bool, bool) -> Text return _percent_decode(text, normalize_case=normalize_case, encode_stray_percents=encode_stray_percents, _decode_map=_FRAGMENT_DECODE_MAP) -def _percent_decode(text, normalize_case=False, subencoding='utf-8', - raise_subencoding_exc=False, encode_stray_percents=False, - _decode_map=_HEX_CHAR_MAP): +def _percent_decode( + text, # type: Text + normalize_case=False, # type: bool + subencoding="utf-8", # type: Text + raise_subencoding_exc=False, # type: bool + encode_stray_percents=False, # type: bool + _decode_map=_HEX_CHAR_MAP # type: Mapping[bytes, bytes] +): + # type: (...) -> Text """Convert percent-encoded text characters to their normal, human-readable equivalents. @@ -528,24 +578,19 @@ def _percent_decode(text, normalize_case=False, subencoding='utf-8', u'abc def' Args: - text (unicode): Text with percent-encoding present. - normalize_case (bool): Whether undecoded percent segments, such - as encoded delimiters, should be uppercased, per RFC 3986 - Section 2.1. See :func:`_decode_path_part` for an example. - subencoding (unicode): The name of the encoding underlying the - percent-encoding. Pass `False` to get back raw bytes. - raise_subencoding_exc (bool): Whether an error in decoding the bytes - underlying the percent-decoding should be raised. + text: Text with percent-encoding present. + normalize_case: Whether undecoded percent segments, such as encoded + delimiters, should be uppercased, per RFC 3986 Section 2.1. + See :func:`_decode_path_part` for an example. + subencoding: The name of the encoding underlying the percent-encoding. + raise_subencoding_exc: Whether an error in decoding the bytes + underlying the percent-decoding should be raised. Returns: - unicode: The percent-decoded version of *text*, decoded by - *subencoding*, unless `subencoding=False` which returns bytes. - + Text: The percent-decoded version of *text*, decoded by *subencoding*. """ try: - quoted_bytes = text.encode( - 'utf-8' if subencoding is False else subencoding - ) + quoted_bytes = text.encode(subencoding) except UnicodeEncodeError: return text @@ -579,8 +624,6 @@ def _percent_decode(text, normalize_case=False, subencoding='utf-8', unquoted_bytes = b''.join(res) - if subencoding is False: - return unquoted_bytes try: return unquoted_bytes.decode(subencoding) except UnicodeDecodeError: @@ -590,6 +633,7 @@ def _percent_decode(text, normalize_case=False, subencoding='utf-8', def _decode_host(host): + # type: (Text) -> Text """Decode a host from ASCII-encodable text to IDNA-decoded text. If the host text is not ASCII, it is returned unchanged, as it is presumed that it is already IDNA-decoded. @@ -653,19 +697,20 @@ def _decode_host(host): def _resolve_dot_segments(path): + # type: (Sequence[Text]) -> Sequence[Text] """Normalize the URL path by resolving segments of '.' and '..'. For more details, see `RFC 3986 section 5.2.4, Remove Dot Segments`_. Args: - path (list): path segments in string form + path: sequence of path segments in text form Returns: - list: a new list of path segments with the '.' and '..' elements - removed and resolved. + A new sequence of path segments with the '.' and '..' elements removed + and resolved. .. _RFC 3986 section 5.2.4, Remove Dot Segments: https://tools.ietf.org/html/rfc3986#section-5.2.4 """ # noqa: E501 - segs = [] + segs = [] # type: List[Text] for seg in path: if seg == u'.': @@ -683,6 +728,7 @@ def _resolve_dot_segments(path): def parse_host(host): + # type: (Text) -> Tuple[Optional[AddressFamily], Text] """Parse the host into a tuple of ``(family, host)``, where family is the appropriate :mod:`socket` module constant when the host is an IP address. Family is ``None`` when the host is not an IP. @@ -690,8 +736,9 @@ def parse_host(host): Will raise :class:`URLParseError` on invalid IPv6 constants. Returns: - tuple: family (socket constant or None), host (string) + family (socket constant or None), host (string) + >>> import socket >>> parse_host('googlewebsite.com') == (None, 'googlewebsite.com') True >>> parse_host('::1') == (socket.AF_INET6, '::1') @@ -701,22 +748,24 @@ def parse_host(host): """ if not host: return None, u'' + if u':' in host: try: - inet_pton(socket.AF_INET6, host) + inet_pton(AF_INET6, host) except socket.error as se: raise URLParseError('invalid IPv6 host: %r (%r)' % (host, se)) except UnicodeEncodeError: pass # TODO: this can't be a real host right? else: - family = socket.AF_INET6 - return family, host - try: - inet_pton(socket.AF_INET, host) - except (socket.error, UnicodeEncodeError): - family = None # not an IP + family = AF_INET6 # type: Optional[AddressFamily] else: - family = socket.AF_INET + try: + inet_pton(AF_INET, host) + except (socket.error, UnicodeEncodeError): + family = None # not an IP + else: + family = AF_INET + return family, host @@ -749,22 +798,22 @@ class URL(object): constructor arguments is below. Args: - scheme (unicode): The text name of the scheme. - host (unicode): The host portion of the network location - port (int): The port part of the network location. If + scheme (Optional[Text]): The text name of the scheme. + host (Optional[Text]): The host portion of the network location + port (Optional[int]): The port part of the network location. If ``None`` or no port is passed, the port will default to the default port of the scheme, if it is known. See the ``SCHEME_PORT_MAP`` and :func:`register_default_port` for more info. - path (tuple): A tuple of strings representing the + path (Iterable[Text]): A tuple of strings representing the slash-separated parts of the path. - query (tuple): The query parameters, as a dictionary or - as an iterable of key-value pairs. - fragment (unicode): The fragment part of the URL. + query (Sequence[Tuple[Text, Optional[Text]]]): The query parameters, as + a dictionary or as an sequence of key-value pairs. + fragment (Text): The fragment part of the URL. rooted (bool): A rooted URL is one which indicates an absolute path. This is True on any URL that includes a host, or any relative URL that starts with a slash. - userinfo (unicode): The username or colon-separated + userinfo (Text): The username or colon-separated username:password pair. uses_netloc (bool): Indicates whether two slashes appear between the scheme and the host (``http://eg.com`` vs @@ -777,11 +826,22 @@ class URL(object): .. _RFC 3987: https://tools.ietf.org/html/rfc3987 """ # noqa: E501 - def __init__(self, scheme=None, host=None, path=(), query=(), fragment=u'', - port=None, rooted=None, userinfo=u'', uses_netloc=None): + def __init__( + self, + scheme=None, # type: Optional[Text] + host=None, # type: Optional[Text] + path=(), # type: Iterable[Text] + query=(), # type: QueryParameters + fragment=u"", # type: Text + port=None, # type: Optional[int] + rooted=None, # type: Optional[bool] + userinfo=u"", # type: Text + uses_netloc=None, # type: Optional[bool] + ): + # type: (...) -> None if host is not None and scheme is None: scheme = u'http' # TODO: why - if port is None: + if port is None and scheme is not None: port = SCHEME_PORT_MAP.get(scheme) if host and query and not path: # per RFC 3986 6.2.3, "a URI that uses the generic syntax @@ -808,7 +868,7 @@ def __init__(self, scheme=None, host=None, path=(), query=(), fragment=u'', % (self._scheme, self.__class__.__name__)) _, self._host = parse_host(_textcheck('host', host, '/?#@')) - if isinstance(path, unicode): + if isinstance(path, Text): raise TypeError("expected iterable of text for path, not: %r" % (path,)) self._path = tuple((_textcheck("path segment", segment, '/?#') @@ -833,14 +893,16 @@ def __init__(self, scheme=None, host=None, path=(), query=(), fragment=u'', self._path = self._path[1:] def get_decoded_url(self, lazy=False): + # type: (bool) -> DecodedURL try: return self._decoded_url except AttributeError: - self._decoded_url = DecodedURL(self, lazy=lazy) + self._decoded_url = DecodedURL(self, lazy=lazy) # type: DecodedURL return self._decoded_url @property def scheme(self): + # type: () -> Text """The scheme is a string, and the first part of an absolute URL, the part before the first colon, and the part which defines the semantics of the rest of the URL. Examples include "http", @@ -851,6 +913,7 @@ def scheme(self): @property def host(self): + # type: () -> Text """The host is a string, and the second standard part of an absolute URL. When present, a valid host must be a domain name, or an IP (v4 or v6). It occurs before the first slash, or the second @@ -860,6 +923,7 @@ def host(self): @property def port(self): + # type: () -> Optional[int] """The port is an integer that is commonly used in connecting to the :attr:`host`, and almost never appears without it. @@ -878,12 +942,12 @@ def port(self): Per the standard, when the port is the same as the schemes default port, it will be omitted in the text URL. - """ return self._port @property def path(self): + # type: () -> Sequence[Text] """A tuple of strings, created by splitting the slash-separated hierarchical path. Started by the first slash after the host, terminated by a "?", which indicates the start of the @@ -893,6 +957,7 @@ def path(self): @property def query(self): + # type: () -> QueryPairs """Tuple of pairs, created by splitting the ampersand-separated mapping of keys and optional values representing non-hierarchical data used to identify the resource. Keys are @@ -908,16 +973,17 @@ def query(self): @property def fragment(self): + # type: () -> Text """A string, the last part of the URL, indicated by the first "#" after the :attr:`~hyperlink.URL.path` or :attr:`~hyperlink.URL.query`. Enables indirect identification of a secondary resource, like an anchor within an HTML page. - """ return self._fragment @property def rooted(self): + # type: () -> bool """Whether or not the path starts with a forward slash (``/``). This is taken from the terminology in the BNF grammar, @@ -925,12 +991,12 @@ def rooted(self): and "absolute URI" are somewhat ambiguous. :attr:`path` does not contain the implicit prefixed ``"/"`` since that is somewhat awkward to work with. - """ return self._rooted @property def userinfo(self): + # type: () -> Text """The colon-separated string forming the username-password combination. """ @@ -938,18 +1004,21 @@ def userinfo(self): @property def uses_netloc(self): + # type: () -> Optional[bool] """ """ return self._uses_netloc @property def user(self): + # type: () -> Text """ The user portion of :attr:`~hyperlink.URL.userinfo`. """ return self.userinfo.split(u':')[0] def authority(self, with_password=False, **kw): + # type: (bool, Any) -> Text """Compute and return the appropriate host/port/userinfo combination. >>> url = URL.from_text(u'http://user:pass@localhost:8080/a/b?x=y') @@ -959,13 +1028,13 @@ def authority(self, with_password=False, **kw): u'user:pass@localhost:8080' Args: - with_password (bool): Whether the return value of this - method include the password in the URL, if it is - set. Defaults to False. + with_password (bool): Whether the return value of this + method include the password in the URL, if it is + set. Defaults to False. Returns: - str: The authority (network location and user information) portion - of the URL. + Text: The authority (network location and user information) portion + of the URL. """ # first, a bit of twisted compat with_password = kw.pop('includeSecrets', with_password) @@ -977,7 +1046,7 @@ def authority(self, with_password=False, **kw): else: hostport = [self.host] if self.port != SCHEME_PORT_MAP.get(self.scheme): - hostport.append(unicode(self.port)) + hostport.append(Text(self.port)) authority = [] if self.userinfo: userinfo = self.userinfo @@ -988,6 +1057,7 @@ def authority(self, with_password=False, **kw): return u"@".join(authority) def __eq__(self, other): + # type: (Any) -> bool if not isinstance(other, self.__class__): return NotImplemented for attr in ['scheme', 'userinfo', 'host', 'query', @@ -1002,17 +1072,20 @@ def __eq__(self, other): return False def __ne__(self, other): + # type: (Any) -> bool if not isinstance(other, self.__class__): return NotImplemented return not self.__eq__(other) def __hash__(self): + # type: () -> int return hash((self.__class__, self.scheme, self.userinfo, self.host, self.path, self.query, self.fragment, self.port, self.rooted, self.uses_netloc)) @property def absolute(self): + # type: () -> bool """Whether or not the URL is "absolute". Absolute URLs are complete enough to resolve to a network resource without being relative to a base URI. @@ -1026,9 +1099,19 @@ def absolute(self): """ return bool(self.scheme and self.host) - def replace(self, scheme=_UNSET, host=_UNSET, path=_UNSET, query=_UNSET, - fragment=_UNSET, port=_UNSET, rooted=_UNSET, userinfo=_UNSET, - uses_netloc=_UNSET): + def replace( + self, + scheme=_UNSET, # type: Optional[Text] + host=_UNSET, # type: Optional[Text] + path=_UNSET, # type: Iterable[Text] + query=_UNSET, # type: QueryParameters + fragment=_UNSET, # type: Text + port=_UNSET, # type: Optional[int] + rooted=_UNSET, # type: Optional[bool] + userinfo=_UNSET, # type: Text + uses_netloc=_UNSET # type: Optional[bool] + ): + # type: (...) -> URL """:class:`URL` objects are immutable, which means that attributes are designed to be set only once, at construction. Instead of modifying an existing URL, one simply creates a copy with the @@ -1038,27 +1121,26 @@ def replace(self, scheme=_UNSET, host=_UNSET, path=_UNSET, query=_UNSET, the value on the current URL. Args: - scheme (unicode): The text name of the scheme. - host (unicode): The host portion of the network location - port (int): The port part of the network location. - path (tuple): A tuple of strings representing the - slash-separated parts of the path. - query (tuple): The query parameters, as a tuple of - key-value pairs. - query (tuple): The query parameters, as a dictionary or - as an iterable of key-value pairs. - fragment (unicode): The fragment part of the URL. - rooted (bool): Whether or not the path begins with a slash. - userinfo (unicode): The username or colon-separated - username:password pair. - uses_netloc (bool): Indicates whether two slashes appear - between the scheme and the host (``http://eg.com`` vs - ``mailto:e@g.com``) + scheme (Optional[Text]): The text name of the scheme. + host (Optional[Text]): The host portion of the network location. + path (Iterable[Text]): A tuple of strings representing the + slash-separated parts of the path. + query (Sequence[Tuple[Text, Optional[Text]]]): The query + parameters, as a dictionary or as an sequence of key-value + pairs. + fragment (Text): The fragment part of the URL. + port (Optional[int]): The port part of the network location. + rooted (Optional[bool]): Whether or not the path begins with a + slash. + userinfo (Text): The username or colon-separated username:password + pair. + uses_netloc (bool): Indicates whether two slashes appear between + the scheme and the host + (``http://eg.com`` vs ``mailto:e@g.com``) Returns: - URL: a copy of the current :class:`URL`, with new values for - parameters passed. - + URL: A copy of the current :class:`URL`, with new values for + parameters passed. """ return self.__class__( scheme=_optional(scheme, self.scheme), @@ -1074,14 +1156,15 @@ def replace(self, scheme=_UNSET, host=_UNSET, path=_UNSET, query=_UNSET, @classmethod def from_text(cls, text): + # type: (Text) -> URL """Whereas the :class:`URL` constructor is useful for constructing URLs from parts, :meth:`~URL.from_text` supports parsing whole URLs from their string form:: - >>> URL.from_text(u'http://example.com') - URL.from_text(u'http://example.com') - >>> URL.from_text(u'?a=b&x=y') - URL.from_text(u'?a=b&x=y') + >>> URL.from_text(u'http://example.com') + URL.from_text(u'http://example.com') + >>> URL.from_text(u'?a=b&x=y') + URL.from_text(u'?a=b&x=y') As you can see above, it's also used as the :func:`repr` of :class:`URL` objects. The natural counterpart to @@ -1089,7 +1172,7 @@ def from_text(cls, text): sure to decode those bytestrings. Args: - text (unicode): A valid URL string. + text (Text): A valid URL string. Returns: URL: The structured object version of the parsed string. @@ -1101,22 +1184,19 @@ def from_text(cls, text): look like URLs are still valid URLs. As a result, this method only raises :class:`URLParseError` on invalid port and IPv6 values in the host portion of the URL. - """ um = _URL_RE.match(_textcheck('text', text)) - try: - gs = um.groupdict() - except AttributeError: + if um is None: raise URLParseError('could not parse url: %r' % text) + gs = um.groupdict() au_text = gs['authority'] or u'' au_m = _AUTHORITY_RE.match(au_text) - try: - au_gs = au_m.groupdict() - except AttributeError: + if au_m is None: raise URLParseError( 'invalid authority %r in url: %r' % (au_text, text) ) + au_gs = au_m.groupdict() if au_gs['bad_host']: raise URLParseError( 'invalid host %r in url: %r' % (au_gs['bad_host'], text) @@ -1128,7 +1208,7 @@ def from_text(cls, text): port = au_gs['port'] if port is not None: try: - port = int(port) + port = int(port) # type: ignore[assignment] # FIXME, see below except ValueError: if not port: # TODO: excessive? raise URLParseError('port must not be empty: %r' % au_text) @@ -1139,9 +1219,9 @@ def from_text(cls, text): uses_netloc = bool(gs['_netloc_sep']) if gs['path']: - path = gs['path'].split(u"/") + path = tuple(gs['path'].split(u"/")) if not path[0]: - path.pop(0) + path = path[1:] rooted = True else: rooted = False @@ -1149,15 +1229,24 @@ def from_text(cls, text): path = () rooted = bool(au_text) if gs['query']: - query = ((qe.split(u"=", 1) if u'=' in qe else (qe, None)) - for qe in gs['query'].split(u"&")) + query = tuple( + ( + qe.split(u"=", 1) # type: ignore[misc] + if u'=' in qe else (qe, None) + ) + for qe in gs['query'].split(u"&") + ) # type: QueryPairs else: query = () - return cls(scheme, host, path, query, fragment, port, - rooted, userinfo, uses_netloc) + return cls( + scheme, host, path, query, fragment, + port, # type: ignore[arg-type] # FIXME, see above + rooted, userinfo, uses_netloc, + ) def normalize(self, scheme=True, host=True, path=True, query=True, fragment=True, userinfo=True, percents=True): + # type: (bool, bool, bool, bool, bool, bool, bool) -> URL """Return a new URL object with several standard normalizations applied: @@ -1174,15 +1263,15 @@ def normalize(self, scheme=True, host=True, path=True, query=True, name. Args: - scheme (bool): Convert the scheme to lowercase - host (bool): Convert the host to lowercase - path (bool): Normalize the path (see above for details) - query (bool): Normalize the query string - fragment (bool): Normalize the fragment - userinfo (bool): Normalize the userinfo - percents (bool): Encode isolated percent signs - for any percent-encoded fields which are being - normalized (defaults to True). + scheme (bool): Convert the scheme to lowercase + host (bool): Convert the host to lowercase + path (bool): Normalize the path (see above for details) + query (bool): Normalize the query string + fragment (bool): Normalize the fragment + userinfo (bool): Normalize the userinfo + percents (bool): Encode isolated percent signs for any + percent-encoded fields which are being normalized + (defaults to True). >>> url = URL.from_text(u'Http://example.COM/a/../b/./c%2f?%61%') >>> print(url.normalize().to_text()) @@ -1194,17 +1283,18 @@ def normalize(self, scheme=True, host=True, path=True, query=True, .. _RFC 3986 6.2.2.3: https://tools.ietf.org/html/rfc3986#section-6.2.2.3 .. _RFC 3986 6.2.3: https://tools.ietf.org/html/rfc3986#section-6.2.3 .. _RFC 3986 2.4: https://tools.ietf.org/html/rfc3986#section-2.4 - """ # noqa: E501 - kw = {} + kw = {} # type: Dict[str, Any] if scheme: kw['scheme'] = self.scheme.lower() if host: kw['host'] = self.host.lower() def _dec_unres(target): - return _decode_unreserved(target, normalize_case=True, - encode_stray_percents=percents) + # type: (Text) -> Text + return _decode_unreserved( + target, normalize_case=True, encode_stray_percents=percents + ) if path: if self.path: kw['path'] = [ @@ -1224,6 +1314,7 @@ def _dec_unres(target): return self.replace(**kw) def child(self, *segments): + # type: (Text) -> URL """Make a new :class:`URL` where the given path segments are a child of this URL, preserving other parts of the URL, including the query string and fragment. @@ -1236,58 +1327,61 @@ def child(self, *segments): u'http://localhost/a/b/c/d?x=y' Args: - segments (unicode): Additional parts to be joined and added to - the path, like :func:`os.path.join`. Special characters - in segments will be percent encoded. + segments (Text): Additional parts to be joined and added to + the path, like :func:`os.path.join`. Special characters + in segments will be percent encoded. Returns: URL: A copy of the current URL with the extra path segments. - """ if not segments: return self - segments = [_textcheck('path segment', s) for s in segments] - new_segs = _encode_path_parts(segments, joined=False, maximal=False) - new_path = self.path[:-1 if (self.path and self.path[-1] == u'') - else None] + new_segs + segments = [ # type: ignore[assignment] # variable is tuple + _textcheck('path segment', s) for s in segments + ] + new_path = tuple(self.path) + if self.path and self.path[-1] == u"": + new_path = new_path[:-1] + new_path += tuple(_encode_path_parts(segments, maximal=False)) return self.replace(path=new_path) def sibling(self, segment): + # type: (Text) -> URL """Make a new :class:`URL` with a single path segment that is a sibling of this URL path. Args: - segment (unicode): A single path segment. + segment (Text): A single path segment. Returns: - URL: A copy of the current URL with the last path segment - replaced by *segment*. Special characters such as - ``/?#`` will be percent encoded. - + URL: A copy of the current URL with the last path segment + replaced by *segment*. Special characters such as + ``/?#`` will be percent encoded. """ _textcheck('path segment', segment) - new_path = self.path[:-1] + (_encode_path_part(segment),) + new_path = tuple(self.path)[:-1] + (_encode_path_part(segment),) return self.replace(path=new_path) def click(self, href=u''): + # type: (Union[Text, URL]) -> URL """Resolve the given URL relative to this URL. The resulting URI should match what a web browser would generate if you visited the current URL and clicked on *href*. - >>> url = URL.from_text(u'http://blog.hatnote.com/') - >>> url.click(u'/post/155074058790').to_text() - u'http://blog.hatnote.com/post/155074058790' - >>> url = URL.from_text(u'http://localhost/a/b/c/') - >>> url.click(u'../d/./e').to_text() - u'http://localhost/a/b/d/e' + >>> url = URL.from_text(u'http://blog.hatnote.com/') + >>> url.click(u'/post/155074058790').to_text() + u'http://blog.hatnote.com/post/155074058790' + >>> url = URL.from_text(u'http://localhost/a/b/c/') + >>> url.click(u'../d/./e').to_text() + u'http://localhost/a/b/d/e' - Args: - href (unicode): A string representing a clicked URL. + Args (Text): + href: A string representing a clicked URL. Return: - URL: A copy of the current URL with navigation logic applied. + A copy of the current URL with navigation logic applied. For more information, see `RFC 3986 section 5`_. @@ -1318,7 +1412,7 @@ def click(self, href=u''): if clicked.rooted: path = clicked.path elif clicked.path: - path = self.path[:-1] + clicked.path + path = tuple(self.path)[:-1] + tuple(clicked.path) else: path = self.path if not query: @@ -1331,6 +1425,7 @@ def click(self, href=u''): fragment=clicked.fragment) def to_uri(self): + # type: () -> URL u"""Make a new :class:`URL` instance with all non-ASCII characters appropriately percent-encoded. This is useful to do in preparation for sending a :class:`URL` over a network protocol. @@ -1342,13 +1437,13 @@ def to_uri(self): Returns: URL: A new instance with its path segments, query parameters, and - hostname encoded, so that they are all in the standard - US-ASCII range. + hostname encoded, so that they are all in the standard + US-ASCII range. """ new_userinfo = u':'.join([_encode_userinfo_part(p) for p in self.userinfo.split(':', 1)]) new_path = _encode_path_parts(self.path, has_scheme=bool(self.scheme), - rooted=False, joined=False, maximal=True) + rooted=False, maximal=True) new_host = ( self.host if not self.host @@ -1366,6 +1461,7 @@ def to_uri(self): ) def to_iri(self): + # type: () -> URL u"""Make a new :class:`URL` instance with all but a few reserved characters decoded into human-readable format. @@ -1386,23 +1482,29 @@ def to_iri(self): Returns: URL: A new instance with its path segments, query parameters, and - hostname decoded for display purposes. + hostname decoded for display purposes. """ # noqa: E501 - new_userinfo = u':'.join([_decode_userinfo_part(p) for p in - self.userinfo.split(':', 1)]) + new_userinfo = u':'.join([ + _decode_userinfo_part(p) for p in self.userinfo.split(':', 1) + ]) host_text = _decode_host(self.host) - return self.replace(userinfo=new_userinfo, - host=host_text, - path=[_decode_path_part(segment) - for segment in self.path], - query=[(_decode_query_key(k), - _decode_query_value(v) - if v is not None else None) - for k, v in self.query], - fragment=_decode_fragment_part(self.fragment)) + return self.replace( + userinfo=new_userinfo, + host=host_text, + path=[_decode_path_part(segment) for segment in self.path], + query=tuple( + ( + _decode_query_key(k), _decode_query_value(v) + if v is not None else None + ) + for k, v in self.query + ), + fragment=_decode_fragment_part(self.fragment), + ) def to_text(self, with_password=False): + # type: (bool) -> Text """Render this URL to its textual representation. By default, the URL text will *not* include a password, if one @@ -1415,13 +1517,13 @@ def to_text(self, with_password=False): unless the data after the colon is the empty string (indicating no password)." - Args: - with_password (bool): Whether or not to include the - password in the URL text. Defaults to False. + Args (bool): + with_password: Whether or not to include the password in the URL + text. Defaults to False. Returns: - str: The serialized textual representation of this URL, - such as ``u"http://example.com/some/path?some=query"``. + Text: The serialized textual representation of this URL, such as + ``u"http://example.com/some/path?some=query"``. The natural counterpart to :class:`URL.from_text()`. @@ -1429,11 +1531,13 @@ def to_text(self, with_password=False): """ scheme = self.scheme authority = self.authority(with_password) - path = _encode_path_parts(self.path, - rooted=self.rooted, - has_scheme=bool(scheme), - has_authority=bool(authority), - maximal=False) + path = "/".join(_encode_path_parts( + self.path, + rooted=self.rooted, + has_scheme=bool(scheme), + has_authority=bool(authority), + maximal=False + )) query_parts = [] for k, v in self.query: if v is None: @@ -1447,7 +1551,7 @@ def to_text(self, with_password=False): fragment = self.fragment - parts = [] + parts = [] # type: List[Text] _add = parts.append if scheme: _add(scheme) @@ -1470,6 +1574,7 @@ def to_text(self, with_password=False): return u''.join(parts) def __repr__(self): + # type: () -> str """Convert this URL to an representation that shows all of its constituent parts, as well as being a valid argument to :func:`eval`. @@ -1477,6 +1582,7 @@ def __repr__(self): return '%s.from_text(%r)' % (self.__class__.__name__, self.to_text()) def _to_bytes(self): + # type: () -> bytes """ Allows for direct usage of URL objects with libraries like requests, which automatically stringify URL parameters. See @@ -1497,12 +1603,15 @@ def _to_bytes(self): @classmethod def fromText(cls, s): + # type: (Text) -> URL return cls.from_text(s) def asText(self, includeSecrets=False): + # type: (bool) -> Text return self.to_text(with_password=includeSecrets) def __dir__(self): + # type: () -> Sequence[Text] try: ret = object.__dir__(self) except AttributeError: @@ -1514,6 +1623,7 @@ def __dir__(self): # # End Twisted Compat Code def add(self, name, value=None): + # type: (Text, Optional[Text]) -> URL """Make a new :class:`URL` instance with a given query argument, *name*, added to it with the value *value*, like so:: @@ -1523,10 +1633,10 @@ def add(self, name, value=None): URL.from_text(u'https://example.com/?x=y&x=z') Args: - name (unicode): The name of the query parameter to add. The - part before the ``=``. - value (unicode): The value of the query parameter to add. The - part after the ``=``. Defaults to ``None``, meaning no + name (Text): The name of the query parameter to add. + The part before the ``=``. + value (Optional[Text]): The value of the query parameter to add. + The part after the ``=``. Defaults to ``None``, meaning no value. Returns: @@ -1535,6 +1645,7 @@ def add(self, name, value=None): return self.replace(query=self.query + ((name, value),)) def set(self, name, value=None): + # type: (Text, Optional[Text]) -> URL """Make a new :class:`URL` instance with the query parameter *name* set to *value*. All existing occurences, if any are replaced by the single name-value pair. @@ -1545,10 +1656,10 @@ def set(self, name, value=None): URL.from_text(u'https://example.com/?x=z') Args: - name (unicode): The name of the query parameter to set. The - part before the ``=``. - value (unicode): The value of the query parameter to set. The - part after the ``=``. Defaults to ``None``, meaning no + name (Text): The name of the query parameter to set. + The part before the ``=``. + value (Optional[Text]): The value of the query parameter to set. + The part after the ``=``. Defaults to ``None``, meaning no value. Returns: @@ -1562,6 +1673,7 @@ def set(self, name, value=None): return self.replace(query=q) def get(self, name): + # type: (Text) -> List[Optional[Text]] """Get a list of values for the given query parameter, *name*:: >>> url = URL.from_text(u'?x=1&x=2') @@ -1574,27 +1686,33 @@ def get(self, name): list is always returned, and this method raises no exceptions. Args: - name (unicode): The name of the query parameter to get. + name (Text): The name of the query parameter to get. Returns: - list: A list of all the values associated with the key, in - string form. - + List[Optional[Text]]: A list of all the values associated with the + key, in string form. """ return [value for (key, value) in self.query if name == key] - def remove(self, name, value=_UNSET, limit=None): + def remove( + self, + name, # type: Text + value=_UNSET, # type: Text + limit=None, # type: Optional[int] + ): + # type: (...) -> URL """Make a new :class:`URL` instance with occurrences of the query parameter *name* removed, or, if *value* is set, parameters matching *name* and *value*. No exception is raised if the parameter is not already set. Args: - name (unicode): The name of the query parameter to remove. - value (unicode): Optional value to additionally filter - on. Setting this removes query parameters which match - both name and value. - limit (int): Optional maximum number of parameters to remove. + name (Text): The name of the query parameter to remove. + value (Text): Optional value to additionally filter on. + Setting this removes query parameters which match both name + and value. + limit (Optional[int]): Optional maximum number of parameters to + remove. Returns: URL: A new :class:`URL` instance with the parameter removed. @@ -1641,12 +1759,13 @@ class DecodedURL(object): special characters encoded with codecs other than UTF-8. Args: - url (URL): A :class:`URL` object to wrap. - lazy (bool): Set to True to avoid pre-decode all parts of the - URL to check for validity. Defaults to False. + url (URL): A :class:`URL` object to wrap. + lazy (bool): Set to True to avoid pre-decode all parts of the URL to + check for validity. Defaults to False. """ def __init__(self, url, lazy=False): + # type: (URL, bool) -> None self._url = url if not lazy: # cache the following, while triggering any decoding @@ -1656,37 +1775,43 @@ def __init__(self, url, lazy=False): @classmethod def from_text(cls, text, lazy=False): + # type: (Text, bool) -> DecodedURL """\ Make a `DecodedURL` instance from any text string containing a URL. Args: - text (unicode): Text containing the URL - lazy (bool): Whether to pre-decode all parts of the URL to - check for validity. Defaults to True. + text (Text): Text containing the URL + lazy (bool): Whether to pre-decode all parts of the URL to check for + validity. Defaults to True. """ _url = URL.from_text(text) return cls(_url, lazy=lazy) @property def encoded_url(self): + # type: () -> URL """Access the underlying :class:`URL` object, which has any special characters encoded. """ return self._url - def to_text(self, *a, **kw): + def to_text(self, with_password=False): + # type: (bool) -> Text "Passthrough to :meth:`~hyperlink.URL.to_text()`" - return self._url.to_text(*a, **kw) + return self._url.to_text(with_password) - def to_uri(self, *a, **kw): + def to_uri(self): + # type: () -> URL "Passthrough to :meth:`~hyperlink.URL.to_uri()`" - return self._url.to_uri(*a, **kw) + return self._url.to_uri() - def to_iri(self, *a, **kw): + def to_iri(self): + # type: () -> URL "Passthrough to :meth:`~hyperlink.URL.to_iri()`" - return self._url.to_iri(*a, **kw) + return self._url.to_iri() def click(self, href=u''): + # type: (Union[Text, URL, DecodedURL]) -> DecodedURL """Return a new DecodedURL wrapping the result of :meth:`~hyperlink.URL.click()` """ @@ -1695,6 +1820,7 @@ def click(self, href=u''): return self.__class__(self._url.click(href=href)) def sibling(self, segment): + # type: (Text) -> DecodedURL """Automatically encode any reserved characters in *segment* and return a new `DecodedURL` wrapping the result of :meth:`~hyperlink.URL.sibling()` @@ -1702,6 +1828,7 @@ def sibling(self, segment): return self.__class__(self._url.sibling(_encode_reserved(segment))) def child(self, *segments): + # type: (Text) -> DecodedURL """Automatically encode any reserved characters in *segments* and return a new `DecodedURL` wrapping the result of :meth:`~hyperlink.URL.child()`. @@ -1711,86 +1838,111 @@ def child(self, *segments): new_segs = [_encode_reserved(s) for s in segments] return self.__class__(self._url.child(*new_segs)) - def normalize(self, *a, **kw): + def normalize(self, scheme=True, host=True, path=True, query=True, + fragment=True, userinfo=True, percents=True): + # type: (bool, bool, bool, bool, bool, bool, bool) -> DecodedURL """Return a new `DecodedURL` wrapping the result of :meth:`~hyperlink.URL.normalize()` """ - return self.__class__(self._url.normalize(*a, **kw)) + return self.__class__(self._url.normalize( + scheme, host, path, query, fragment, userinfo, percents + )) @property def absolute(self): + # type: () -> bool return self._url.absolute @property def scheme(self): + # type: () -> Text return self._url.scheme @property def host(self): + # type: () -> Text return _decode_host(self._url.host) @property def port(self): + # type: () -> Optional[int] return self._url.port @property def rooted(self): + # type: () -> bool return self._url.rooted @property def path(self): - try: - return self._path - except AttributeError: - pass - self._path = tuple([_percent_decode(p, raise_subencoding_exc=True) - for p in self._url.path]) + # type: () -> Sequence[Text] + if not hasattr(self, "_path"): + self._path = tuple([ + _percent_decode(p, raise_subencoding_exc=True) + for p in self._url.path + ]) return self._path @property def query(self): - try: - return self._query - except AttributeError: - pass - _q = [tuple(_percent_decode(x, raise_subencoding_exc=True) + # type: () -> QueryPairs + if not hasattr(self, "_query"): + self._query = cast(QueryPairs, tuple( + tuple( + _percent_decode(x, raise_subencoding_exc=True) if x is not None else None - for x in (k, v)) - for k, v in self._url.query] - self._query = tuple(_q) + for x in (k, v) + ) + for k, v in self._url.query + )) return self._query @property def fragment(self): - try: - return self._fragment - except AttributeError: - pass - frag = self._url.fragment - self._fragment = _percent_decode(frag, raise_subencoding_exc=True) + # type: () -> Text + if not hasattr(self, "_fragment"): + frag = self._url.fragment + self._fragment = _percent_decode(frag, raise_subencoding_exc=True) return self._fragment @property def userinfo(self): - try: - return self._userinfo - except AttributeError: - pass - self._userinfo = tuple([_percent_decode(p, raise_subencoding_exc=True) - for p in self._url.userinfo.split(':', 1)]) + # type: () -> Union[Tuple[str], Tuple[str, str]] + if not hasattr(self, "_userinfo"): + self._userinfo = cast( + Union[Tuple[str], Tuple[str, str]], + tuple( + tuple( + _percent_decode(p, raise_subencoding_exc=True) + for p in self._url.userinfo.split(':', 1) + ) + ) + ) return self._userinfo @property def user(self): + # type: () -> Text return self.userinfo[0] @property def uses_netloc(self): + # type: () -> Optional[bool] return self._url.uses_netloc - def replace(self, scheme=_UNSET, host=_UNSET, path=_UNSET, query=_UNSET, - fragment=_UNSET, port=_UNSET, rooted=_UNSET, userinfo=_UNSET, - uses_netloc=_UNSET): + def replace( + self, + scheme=_UNSET, # type: Optional[Text] + host=_UNSET, # type: Optional[Text] + path=_UNSET, # type: Iterable[Text] + query=_UNSET, # type: QueryParameters + fragment=_UNSET, # type: Text + port=_UNSET, # type: Optional[int] + rooted=_UNSET, # type: Optional[bool] + userinfo=_UNSET, # type: Union[Tuple[str], Tuple[str, str]] + uses_netloc=_UNSET # type: Optional[bool] + ): + # type: (...) -> DecodedURL """While the signature is the same, this `replace()` differs a little from URL.replace. For instance, it accepts userinfo as a tuple, not as a string, handling the case of having a username @@ -1799,17 +1951,23 @@ def replace(self, scheme=_UNSET, host=_UNSET, path=_UNSET, query=_UNSET, automatically encoded instead of an error being raised. """ if path is not _UNSET: - path = [_encode_reserved(p) for p in path] + path = tuple(_encode_reserved(p) for p in path) if query is not _UNSET: - query = [[_encode_reserved(x) - if x is not None else None - for x in (k, v)] - for k, v in iter_pairs(query)] + query = cast(QueryPairs, tuple( + tuple( + _encode_reserved(x) + if x is not None else None + for x in (k, v) + ) + for k, v in iter_pairs(query) + )) if userinfo is not _UNSET: if len(userinfo) > 2: raise ValueError('userinfo expected sequence of ["user"] or' - ' ["user", "password"], got %r' % userinfo) - userinfo = u':'.join([_encode_reserved(p) for p in userinfo]) + ' ["user", "password"], got %r' % (userinfo,)) + userinfo_text = u':'.join([_encode_reserved(p) for p in userinfo]) + else: + userinfo_text = _UNSET new_url = self._url.replace(scheme=scheme, host=host, path=path, @@ -1817,20 +1975,23 @@ def replace(self, scheme=_UNSET, host=_UNSET, path=_UNSET, query=_UNSET, fragment=fragment, port=port, rooted=rooted, - userinfo=userinfo, + userinfo=userinfo_text, uses_netloc=uses_netloc) return self.__class__(url=new_url) def get(self, name): + # type: (Text) -> List[Optional[Text]] "Get the value of all query parameters whose name matches *name*" return [v for (k, v) in self.query if name == k] def add(self, name, value=None): + # type: (Text, Optional[Text]) -> DecodedURL """Return a new DecodedURL with the query parameter *name* and *value* added.""" return self.replace(query=self.query + ((name, value),)) def set(self, name, value=None): + # type: (Text, Optional[Text]) -> DecodedURL "Return a new DecodedURL with query parameter *name* set to *value*" query = self.query q = [(k, v) for (k, v) in query if k != name] @@ -1838,7 +1999,13 @@ def set(self, name, value=None): q[idx:idx] = [(name, value)] return self.replace(query=q) - def remove(self, name, value=_UNSET, limit=None): + def remove( + self, + name, # type: Text + value=_UNSET, # type: Text + limit=None, # type: Optional[int] + ): + # type: (...) -> DecodedURL """Return a new DecodedURL with query parameter *name* removed. Optionally also filter for *value*, as well as cap the number @@ -1867,25 +2034,30 @@ def remove(self, name, value=_UNSET, limit=None): return self.replace(query=nq) def __repr__(self): + # type: () -> str cn = self.__class__.__name__ return '%s(url=%r)' % (cn, self._url) def __str__(self): + # type: () -> str # TODO: the underlying URL's __str__ needs to change to make # this work as the URL, see #55 return str(self._url) def __eq__(self, other): + # type: (Any) -> bool if not isinstance(other, self.__class__): return NotImplemented return self.normalize().to_uri() == other.normalize().to_uri() def __ne__(self, other): + # type: (Any) -> bool if not isinstance(other, self.__class__): return NotImplemented return not self.__eq__(other) def __hash__(self): + # type: () -> int return hash((self.__class__, self.scheme, self.userinfo, self.host, self.path, self.query, self.fragment, self.port, self.rooted, self.uses_netloc)) @@ -1896,12 +2068,15 @@ def __hash__(self): @classmethod def fromText(cls, s, lazy=False): + # type: (Text, bool) -> DecodedURL return cls.from_text(s, lazy=lazy) def asText(self, includeSecrets=False): + # type: (bool) -> Text return self.to_text(with_password=includeSecrets) def __dir__(self): + # type: () -> Sequence[Text] try: ret = object.__dir__(self) except AttributeError: @@ -1914,21 +2089,23 @@ def __dir__(self): def parse(url, decoded=True, lazy=False): + # type: (Text, bool, bool) -> Union[URL, DecodedURL] """Automatically turn text into a structured URL object. Args: - - decoded (bool): Whether or not to return a :class:`DecodedURL`, - which automatically handles all - encoding/decoding/quoting/unquoting for all the various - accessors of parts of the URL, or an :class:`EncodedURL`, - which has the same API, but requires handling of special - characters for different parts of the URL. - - lazy (bool): In the case of `decoded=True`, this controls - whether the URL is decoded immediately or as accessed. The - default, `lazy=False`, checks all encoded parts of the URL - for decodability. + url (Text): A string representation of a URL. + + decoded (bool): Whether or not to return a :class:`DecodedURL`, + which automatically handles all + encoding/decoding/quoting/unquoting for all the various + accessors of parts of the URL, or an :class:`EncodedURL`, + which has the same API, but requires handling of special + characters for different parts of the URL. + + lazy (bool): In the case of `decoded=True`, this controls + whether the URL is decoded immediately or as accessed. The + default, `lazy=False`, checks all encoded parts of the URL + for decodability. """ enc_url = EncodedURL.from_text(url) if not decoded: diff --git a/src/hyperlink/test/common.py b/src/hyperlink/test/common.py index c9696fcf..a2961f45 100644 --- a/src/hyperlink/test/common.py +++ b/src/hyperlink/test/common.py @@ -6,7 +6,7 @@ class HyperlinkTestCase(TestCase): """This type mostly exists to provide a backwards-compatible assertRaises method for Python 2.6 testing. """ - def assertRaises( # type: ignore[override] Doesn't match superclass, meh + def assertRaises( # type: ignore[override] self, expected_exception, callableObj=None, *args, **kwargs ): # type: (Type[BaseException], Optional[Callable], Any, Any) -> Any diff --git a/src/hyperlink/test/test_common.py b/src/hyperlink/test/test_common.py index ac8e9b3c..c16e12e2 100644 --- a/src/hyperlink/test/test_common.py +++ b/src/hyperlink/test/test_common.py @@ -88,7 +88,7 @@ def test_assertRaisesContextManager(self): with self.hyperlink_test.assertRaises(_ExpectedException) as cm: raise _ExpectedException - self.assertTrue( # type: ignore[misc] unreachable + self.assertTrue( # type: ignore[misc] # unreachable isinstance(cm.exception, _ExpectedException) ) diff --git a/src/hyperlink/test/test_decoded_url.py b/src/hyperlink/test/test_decoded_url.py index 17f7f19b..35491072 100644 --- a/src/hyperlink/test/test_decoded_url.py +++ b/src/hyperlink/test/test_decoded_url.py @@ -2,7 +2,8 @@ from __future__ import unicode_literals -from .. import DecodedURL +from typing import Dict, Union +from .. import DecodedURL, URL from .._url import _percent_decode from .common import HyperlinkTestCase @@ -132,7 +133,9 @@ def test_equality_and_hashability(self): assert durl is not None assert durl != durl._url - durl_map = {} + AnyURL = Union[URL, DecodedURL] + + durl_map = {} # type: Dict[AnyURL, AnyURL] durl_map[durl] = durl durl_map[durl2] = durl2 @@ -166,7 +169,11 @@ def test_replace_userinfo(self): # type: () -> None durl = DecodedURL.from_text(TOTAL_URL) with self.assertRaises(ValueError): - durl.replace(userinfo=['user', 'pw', 'thiswillcauseafailure']) + durl.replace( + userinfo=( # type: ignore[arg-type] + 'user', 'pw', 'thiswillcauseafailure' + ) + ) return def test_twisted_compat(self): @@ -178,10 +185,6 @@ def test_twisted_compat(self): assert 'asText' not in dir(durl) assert durl.to_text() == durl.asText() - def test_percent_decode_bytes(self): - # type: () -> None - assert _percent_decode('%00', subencoding=False) == b'\0' - def test_percent_decode_mixed(self): # type: () -> None @@ -196,9 +199,6 @@ def test_percent_decode_mixed(self): with self.assertRaises(UnicodeDecodeError): _percent_decode('abcdé%C3éfg', raise_subencoding_exc=True) - # check that getting raw bytes works ok - assert _percent_decode('a%00b', subencoding=False) == b'a\x00b' - # when not encodable as subencoding assert _percent_decode('é%25é', subencoding='ascii') == 'é%25é' diff --git a/src/hyperlink/test/test_socket.py b/src/hyperlink/test/test_socket.py index a66e1b05..5f83d45b 100644 --- a/src/hyperlink/test/test_socket.py +++ b/src/hyperlink/test/test_socket.py @@ -11,7 +11,6 @@ from .common import HyperlinkTestCase from .._socket import inet_pton - class TestSocket(HyperlinkTestCase): def test_inet_pton_ipv4_valid(self): # type: () -> None diff --git a/src/hyperlink/test/test_url.py b/src/hyperlink/test/test_url.py index 5f10c387..dd554fa5 100644 --- a/src/hyperlink/test/test_url.py +++ b/src/hyperlink/test/test_url.py @@ -7,7 +7,7 @@ import sys import socket -from typing import Any, Iterable, Optional, Text, Tuple +from typing import Any, Iterable, Optional, Text, Tuple, cast from .common import HyperlinkTestCase from .. import URL, URLParseError @@ -825,7 +825,7 @@ def test_queryIterable(self): handling dictionaries. """ expected = (('alpha', 'beta'),) - url = URL(query=[['alpha', 'beta']]) + url = URL(query=[('alpha', 'beta')]) self.assertEqual(url.query, expected) url = URL(query={'alpha': 'beta'}) self.assertEqual(url.query, expected) @@ -871,7 +871,7 @@ def assertRaised(raised, expectation, name): def check(param, expectation=defaultExpectation): # type: (Any, str) -> None with self.assertRaises(TypeError) as raised: - URL(**{param: Unexpected()}) + URL(**{param: Unexpected()}) # type: ignore[arg-type] assertRaised(raised, expectation, param) @@ -883,41 +883,41 @@ def check(param, expectation=defaultExpectation): check("port", "int or NoneType") with self.assertRaises(TypeError) as raised: - URL(path=[Unexpected()]) + URL(path=[cast(Text, Unexpected())]) assertRaised(raised, defaultExpectation, "path segment") with self.assertRaises(TypeError) as raised: - URL(query=[(u"name", Unexpected())]) + URL(query=[(u"name", cast(Text, Unexpected()))]) assertRaised(raised, defaultExpectation + " or NoneType", "query parameter value") with self.assertRaises(TypeError) as raised: - URL(query=[(Unexpected(), u"value")]) + URL(query=[(cast(Text, Unexpected()), u"value")]) assertRaised(raised, defaultExpectation, "query parameter name") # No custom error message for this one, just want to make sure # non-2-tuples don't get through. with self.assertRaises(TypeError): - URL(query=[Unexpected()]) + URL(query=[cast(Tuple[Text, Text], Unexpected())]) with self.assertRaises(ValueError): - URL(query=[('k', 'v', 'vv')]) + URL(query=[cast(Tuple[Text, Text], ('k', 'v', 'vv'))]) with self.assertRaises(ValueError): - URL(query=[('k',)]) + URL(query=[cast(Tuple[Text, Text], ('k',))]) url = URL.from_text("https://valid.example.com/") with self.assertRaises(TypeError) as raised: - url.child(Unexpected()) + url.child(cast(Text, Unexpected())) assertRaised(raised, defaultExpectation, "path segment") with self.assertRaises(TypeError) as raised: - url.sibling(Unexpected()) + url.sibling(cast(Text, Unexpected())) assertRaised(raised, defaultExpectation, "path segment") with self.assertRaises(TypeError) as raised: - url.click(Unexpected()) + url.click(cast(Text, Unexpected())) assertRaised(raised, defaultExpectation, "relative URL") def test_technicallyTextIsIterableBut(self): diff --git a/tox.ini b/tox.ini index d9bab94c..eb2f07be 100644 --- a/tox.ini +++ b/tox.ini @@ -85,6 +85,8 @@ deps = mccabe==0.6.1 pep8-naming==0.9.1 pydocstyle==5.0.1 + # pin pyflakes pending a release with https://github.com/PyCQA/pyflakes/pull/455 + git+git://github.com/PyCQA/pyflakes@ffe9386#egg=pyflakes commands = flake8 {posargs:src/{env:PY_MODULE}} @@ -157,6 +159,8 @@ commands = # Global settings +check_untyped_defs = True +disallow_any_generics = True disallow_incomplete_defs = True disallow_untyped_defs = True no_implicit_optional = True @@ -169,13 +173,7 @@ warn_return_any = True warn_unreachable = True warn_unused_ignores = True -# Enable these over time -check_untyped_defs = False - -# Disable some checks until effected files fully adopt mypy - -[mypy-hyperlink._url] -allow_untyped_defs = True +# Don't complain about dependencies known to lack type hints [mypy-idna] ignore_missing_imports = True