From 873780bf05f3f76ffea61acbf8a16590aa88bf79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilfredo=20Sa=CC=81nchez?= Date: Fri, 1 Nov 2019 12:27:17 -0700 Subject: [PATCH 01/24] Add more type hints --- src/hyperlink/_url.py | 117 +++++++++++++++++++----------- src/hyperlink/test/test_socket.py | 3 +- src/hyperlink/test/test_url.py | 24 +++--- tox.ini | 3 + 4 files changed, 92 insertions(+), 55 deletions(-) diff --git a/src/hyperlink/_url.py b/src/hyperlink/_url.py index 2d4e712a..341175e3 100644 --- a/src/hyperlink/_url.py +++ b/src/hyperlink/_url.py @@ -18,14 +18,21 @@ import re import sys import string -import socket -from typing import Callable, Text, Type +from socket import AF_INET, AF_INET6, error as SocketError +try: + from socket import AddressFamily +except ImportError: + AddressFamily = int # type: ignore[assignment,misc] Python 2 +from typing import ( + Callable, Iterable, Iterator, List, Mapping, Optional, + Sequence, Text, Tuple, Type, Union, +) from unicodedata import normalize from ._socket import inet_pton try: - from collections.abc import Mapping + from collections.abc import Mapping as MappingABC except ImportError: # Python 2 - from collections import Mapping + from collections import Mapping as MappingABC from idna import encode as idna_encode, decode as idna_decode @@ -37,6 +44,10 @@ except NameError: # Py3 unichr = chr # type: Callable[[int], Text] NoneType = type(None) # type: Type[None] +QueryParameter = Union[ + Mapping[Text, Optional[Text]], + Iterable[Tuple[Text, Optional[Text]]], +] # from boltons.typeutils @@ -187,6 +198,7 @@ def _make_quote_map(safe_chars): def _encode_reserved(text, maximal=True): + # type: (Text, bool) -> Text """A very comprehensive percent encoding for encoding all delimiters. Used for arguments to DecodedURL, where a % means a percent sign, and not the character used by URLs for escaping @@ -200,6 +212,7 @@ def _encode_reserved(text, maximal=True): def _encode_path_part(text, maximal=True): + # type: (Text, bool) -> Text "Percent-encode a single segment of a URL path." if maximal: bytestr = normalize('NFC', text).encode('utf8') @@ -209,6 +222,7 @@ def _encode_path_part(text, maximal=True): def _encode_schemeless_path_part(text, maximal=True): + # type: (Text, bool) -> Text """Percent-encode the first segment of a URL path for a URL without a scheme specified. """ @@ -261,6 +275,7 @@ def _encode_path_parts(text_parts, rooted=False, has_scheme=True, def _encode_query_key(text, maximal=True): + # type: (Text, bool) -> Text """ Percent-encode a single query string key or value. """ @@ -272,6 +287,7 @@ def _encode_query_key(text, maximal=True): def _encode_query_value(text, maximal=True): + # type: (Text, bool) -> Text """ Percent-encode a single query string key or value. """ @@ -283,6 +299,7 @@ def _encode_query_value(text, maximal=True): def _encode_fragment_part(text, maximal=True): + # type: (Text, bool) -> Text """Quote the fragment part of the URL. Fragments don't have subdelimiters, so the whole URL fragment can be passed. """ @@ -294,6 +311,7 @@ def _encode_fragment_part(text, maximal=True): def _encode_userinfo_part(text, maximal=True): + # type: (Text, bool) -> Text """Quote special characters in either the username or password section of the URL. """ @@ -369,6 +387,7 @@ def register_scheme(text, uses_netloc=True, default_port=None): def scheme_uses_netloc(scheme, default=None): + # type: (Text, Optional[bool]) -> Optional[bool] """Whether or not a URL uses :code:`:` or :code:`://` to separate the scheme from the rest of the URL depends on the scheme's own standard definition. There is no way to infer this behavior @@ -442,6 +461,7 @@ def _textcheck(name, value, delims=frozenset(), nullable=False): def iter_pairs(iterable): + # type: (Iterable) -> Iterator """ Iterate over the (key, value) pairs in ``iterable``. @@ -449,7 +469,7 @@ def iter_pairs(iterable): iterable yields (key, value) pairs. This behaviour is similar to what Python's ``dict()`` constructor does. """ - if isinstance(iterable, Mapping): + if isinstance(iterable, MappingABC): iterable = iterable.items() return iter(iterable) @@ -590,6 +610,7 @@ def _percent_decode(text, normalize_case=False, subencoding='utf-8', def _decode_host(host): + # type: (Text) -> Text """Decode a host from ASCII-encodable text to IDNA-decoded text. If the host text is not ASCII, it is returned unchanged, as it is presumed that it is already IDNA-decoded. @@ -653,19 +674,20 @@ def _decode_host(host): def _resolve_dot_segments(path): + # type: (Sequence[Text]) -> Sequence[Text] """Normalize the URL path by resolving segments of '.' and '..'. For more details, see `RFC 3986 section 5.2.4, Remove Dot Segments`_. Args: - path (list): path segments in string form + path: sequence of path segments in text form Returns: - list: a new list of path segments with the '.' and '..' elements + list: a new sequence of path segments with the '.' and '..' elements removed and resolved. .. _RFC 3986 section 5.2.4, Remove Dot Segments: https://tools.ietf.org/html/rfc3986#section-5.2.4 """ # noqa: E501 - segs = [] + segs = [] # type: List[Text] for seg in path: if seg == u'.': @@ -683,6 +705,7 @@ def _resolve_dot_segments(path): def parse_host(host): + # type: (Text) -> Tuple[Optional[AddressFamily], Text] """Parse the host into a tuple of ``(family, host)``, where family is the appropriate :mod:`socket` module constant when the host is an IP address. Family is ``None`` when the host is not an IP. @@ -692,6 +715,7 @@ def parse_host(host): Returns: tuple: family (socket constant or None), host (string) + >>> import socket >>> parse_host('googlewebsite.com') == (None, 'googlewebsite.com') True >>> parse_host('::1') == (socket.AF_INET6, '::1') @@ -701,22 +725,24 @@ def parse_host(host): """ if not host: return None, u'' + if u':' in host: try: - inet_pton(socket.AF_INET6, host) - except socket.error as se: + inet_pton(AF_INET6, host) + except SocketError as se: raise URLParseError('invalid IPv6 host: %r (%r)' % (host, se)) except UnicodeEncodeError: pass # TODO: this can't be a real host right? else: - family = socket.AF_INET6 - return family, host - try: - inet_pton(socket.AF_INET, host) - except (socket.error, UnicodeEncodeError): - family = None # not an IP + family = AF_INET6 # type: Optional[AddressFamily] else: - family = socket.AF_INET + try: + inet_pton(AF_INET, host) + except (SocketError, UnicodeEncodeError): + family = None # not an IP + else: + family = AF_INET + return family, host @@ -749,37 +775,46 @@ class URL(object): constructor arguments is below. Args: - scheme (unicode): The text name of the scheme. - host (unicode): The host portion of the network location - port (int): The port part of the network location. If - ``None`` or no port is passed, the port will default to - the default port of the scheme, if it is known. See the - ``SCHEME_PORT_MAP`` and :func:`register_default_port` - for more info. - path (tuple): A tuple of strings representing the - slash-separated parts of the path. - query (tuple): The query parameters, as a dictionary or - as an iterable of key-value pairs. - fragment (unicode): The fragment part of the URL. - rooted (bool): Whether or not the path begins with a slash. - userinfo (unicode): The username or colon-separated - username:password pair. - uses_netloc (bool): Indicates whether two slashes appear - between the scheme and the host (``http://eg.com`` vs - ``mailto:e@g.com``). Set automatically based on scheme. - - All of these parts are also exposed as read-only attributes of - URL instances, along with several useful methods. + scheme: The text name of the scheme. + host: The host portion of the network location + port: The port part of the network location. If ``None`` or no port is + passed, the port will default to the default port of the scheme, if + it is known. See the ``SCHEME_PORT_MAP`` and + :func:`register_default_port` for more info. + path: A tuple of strings representing the slash-separated parts of the + path. + query: The query parameters, as a dictionary or as an iterable of + key-value pairs. + fragment: The fragment part of the URL. + rooted: Whether or not the path begins with a slash. + userinfo: The username or colon-separated username:password pair. + uses_netloc: Indicates whether two slashes appear between the scheme + and the host (``http://eg.com`` vs. ``mailto:e@g.com``). + Set automatically based on scheme. + + All of these parts are also exposed as read-only attributes of URL + instances, along with several useful methods. .. _RFC 3986: https://tools.ietf.org/html/rfc3986 .. _RFC 3987: https://tools.ietf.org/html/rfc3987 """ # noqa: E501 - def __init__(self, scheme=None, host=None, path=(), query=(), fragment=u'', - port=None, rooted=None, userinfo=u'', uses_netloc=None): + def __init__( + self, + scheme=None, # type: Optional[Text] + host=None, # type: Optional[Text] + path=(), # type: Iterable[Text] + query=(), # type: QueryParameter + fragment=u"", # type: Text + port=None, # type: Optional[int] + rooted=None, # type: Optional[bool] + userinfo=u"", # type: Text + uses_netloc=None, # type: Optional[bool] + ): + # type: (...) -> None if host is not None and scheme is None: scheme = u'http' # TODO: why - if port is None: + if port is None and scheme is not None: port = SCHEME_PORT_MAP.get(scheme) if host and query and not path: # per RFC 3986 6.2.3, "a URI that uses the generic syntax diff --git a/src/hyperlink/test/test_socket.py b/src/hyperlink/test/test_socket.py index a66e1b05..8ec9a829 100644 --- a/src/hyperlink/test/test_socket.py +++ b/src/hyperlink/test/test_socket.py @@ -3,7 +3,7 @@ try: from socket import inet_pton except ImportError: - inet_pton = None # type: ignore[assignment] + inet_pton = None # type: ignore[assignment] optional if not inet_pton: import socket @@ -11,7 +11,6 @@ from .common import HyperlinkTestCase from .._socket import inet_pton - class TestSocket(HyperlinkTestCase): def test_inet_pton_ipv4_valid(self): # type: () -> None diff --git a/src/hyperlink/test/test_url.py b/src/hyperlink/test/test_url.py index 511515c8..6ee4d172 100644 --- a/src/hyperlink/test/test_url.py +++ b/src/hyperlink/test/test_url.py @@ -7,7 +7,7 @@ import sys import socket -from typing import Any, Iterable, Optional, Text, Tuple +from typing import Any, Iterable, Optional, Text, Tuple, cast from .common import HyperlinkTestCase from .. import URL, URLParseError @@ -825,7 +825,7 @@ def test_queryIterable(self): handling dictionaries. """ expected = (('alpha', 'beta'),) - url = URL(query=[['alpha', 'beta']]) + url = URL(query=[('alpha', 'beta')]) self.assertEqual(url.query, expected) url = URL(query={'alpha': 'beta'}) self.assertEqual(url.query, expected) @@ -871,7 +871,7 @@ def assertRaised(raised, expectation, name): def check(param, expectation=defaultExpectation): # type: (Any, str) -> None with self.assertRaises(TypeError) as raised: - URL(**{param: Unexpected()}) + URL(**{param: Unexpected()}) # type: ignore[arg-type] ok assertRaised(raised, expectation, param) @@ -883,41 +883,41 @@ def check(param, expectation=defaultExpectation): check("port", "int or NoneType") with self.assertRaises(TypeError) as raised: - URL(path=[Unexpected()]) + URL(path=[cast(Text, Unexpected())]) assertRaised(raised, defaultExpectation, "path segment") with self.assertRaises(TypeError) as raised: - URL(query=[(u"name", Unexpected())]) + URL(query=[(u"name", cast(Text, Unexpected()))]) assertRaised(raised, defaultExpectation + " or NoneType", "query parameter value") with self.assertRaises(TypeError) as raised: - URL(query=[(Unexpected(), u"value")]) + URL(query=[(cast(Text, Unexpected()), u"value")]) assertRaised(raised, defaultExpectation, "query parameter name") # No custom error message for this one, just want to make sure # non-2-tuples don't get through. with self.assertRaises(TypeError): - URL(query=[Unexpected()]) + URL(query=[cast(Tuple[Text, Text], Unexpected())]) with self.assertRaises(ValueError): - URL(query=[('k', 'v', 'vv')]) + URL(query=[cast(Tuple[Text, Text], ('k', 'v', 'vv'))]) with self.assertRaises(ValueError): - URL(query=[('k',)]) + URL(query=[cast(Tuple[Text, Text], ('k',))]) url = URL.from_text("https://valid.example.com/") with self.assertRaises(TypeError) as raised: - url.child(Unexpected()) + url.child(cast(Text, Unexpected())) assertRaised(raised, defaultExpectation, "path segment") with self.assertRaises(TypeError) as raised: - url.sibling(Unexpected()) + url.sibling(cast(Text, Unexpected())) assertRaised(raised, defaultExpectation, "path segment") with self.assertRaises(TypeError) as raised: - url.click(Unexpected()) + url.click(cast(Text, Unexpected())) assertRaised(raised, defaultExpectation, "relative URL") def test_technicallyTextIsIterableBut(self): diff --git a/tox.ini b/tox.ini index 52c6e393..07b9fd50 100644 --- a/tox.ini +++ b/tox.ini @@ -110,6 +110,9 @@ ignore = # variable in function should be lowercase N806, + # lowercase imported as non lowercase + N812, + # variable in class scope should not be mixedCase N815, From 913eb294db511f789d07fc216fa9e8197405c355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilfredo=20Sa=CC=81nchez?= Date: Sat, 2 Nov 2019 08:28:29 -0700 Subject: [PATCH 02/24] Undo a change to avoid tripping up coverage/patch. --- src/hyperlink/_url.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/hyperlink/_url.py b/src/hyperlink/_url.py index 341175e3..05229ce7 100644 --- a/src/hyperlink/_url.py +++ b/src/hyperlink/_url.py @@ -18,7 +18,8 @@ import re import sys import string -from socket import AF_INET, AF_INET6, error as SocketError +import socket +from socket import AF_INET, AF_INET6 try: from socket import AddressFamily except ImportError: @@ -729,7 +730,7 @@ def parse_host(host): if u':' in host: try: inet_pton(AF_INET6, host) - except SocketError as se: + except socket.error as se: raise URLParseError('invalid IPv6 host: %r (%r)' % (host, se)) except UnicodeEncodeError: pass # TODO: this can't be a real host right? @@ -738,7 +739,7 @@ def parse_host(host): else: try: inet_pton(AF_INET, host) - except (SocketError, UnicodeEncodeError): + except (socket.error, UnicodeEncodeError): family = None # not an IP else: family = AF_INET From f459aa1d11f13b7c6ee5cf13f24fdc2970497f96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilfredo=20Sa=CC=81nchez?= Date: Sat, 2 Nov 2019 13:19:49 -0700 Subject: [PATCH 03/24] Add type hints to _encode_path_parts. Get rid of joined argument, as it changed the input and output types and was overriden to False in all but one case. --- src/hyperlink/_url.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/src/hyperlink/_url.py b/src/hyperlink/_url.py index 05229ce7..c57c9c5a 100644 --- a/src/hyperlink/_url.py +++ b/src/hyperlink/_url.py @@ -234,8 +234,14 @@ def _encode_schemeless_path_part(text, maximal=True): if t in _SCHEMELESS_PATH_DELIMS else t for t in text]) -def _encode_path_parts(text_parts, rooted=False, has_scheme=True, - has_authority=True, joined=True, maximal=True): +def _encode_path_parts( + text_parts, # type: Sequence[Text] + rooted=False, # type: bool + has_scheme=True, # type: bool + has_authority=True, # type: bool + maximal=True, # type: bool +): + # type: (...) -> Sequence[Text] """ Percent-encode a tuple of path parts into a complete path. @@ -257,12 +263,12 @@ def _encode_path_parts(text_parts, rooted=False, has_scheme=True, first path segment cannot contain a colon (":") character. """ if not text_parts: - return u'' if joined else text_parts + return () if rooted: - text_parts = (u'',) + text_parts + text_parts = (u'',) + tuple(text_parts) # elif has_authority and text_parts: # raise Exception('see rfc above') # TODO: too late to fail like this? - encoded_parts = [] + encoded_parts = [] # type: List[Text] if has_scheme: encoded_parts = [_encode_path_part(part, maximal=maximal) if part else part for part in text_parts] @@ -270,8 +276,6 @@ def _encode_path_parts(text_parts, rooted=False, has_scheme=True, encoded_parts = [_encode_schemeless_path_part(text_parts[0])] encoded_parts.extend([_encode_path_part(part, maximal=maximal) if part else part for part in text_parts[1:]]) - if joined: - return u'/'.join(encoded_parts) return tuple(encoded_parts) @@ -1278,7 +1282,7 @@ def child(self, *segments): return self segments = [_textcheck('path segment', s) for s in segments] - new_segs = _encode_path_parts(segments, joined=False, maximal=False) + new_segs = _encode_path_parts(segments, maximal=False) new_path = self.path[:-1 if (self.path and self.path[-1] == u'') else None] + new_segs return self.replace(path=new_path) @@ -1378,7 +1382,7 @@ def to_uri(self): new_userinfo = u':'.join([_encode_userinfo_part(p) for p in self.userinfo.split(':', 1)]) new_path = _encode_path_parts(self.path, has_scheme=bool(self.scheme), - rooted=False, joined=False, maximal=True) + rooted=False, maximal=True) new_host = ( self.host if not self.host @@ -1459,11 +1463,13 @@ def to_text(self, with_password=False): """ scheme = self.scheme authority = self.authority(with_password) - path = _encode_path_parts(self.path, - rooted=self.rooted, - has_scheme=bool(scheme), - has_authority=bool(authority), - maximal=False) + path = "/".join(_encode_path_parts( + self.path, + rooted=self.rooted, + has_scheme=bool(scheme), + has_authority=bool(authority), + maximal=False + )) query_parts = [] for k, v in self.query: if v is None: From 915d3ceef5939e58e51b7248567838252983f094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilfredo=20Sa=CC=81nchez?= Date: Sat, 2 Nov 2019 15:02:55 -0700 Subject: [PATCH 04/24] Add type hints to _encode_path_parts. Get rid of joined argument, as it changed the input and output types and was overriden to False in all but one case. --- src/hyperlink/_url.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/hyperlink/_url.py b/src/hyperlink/_url.py index c57c9c5a..a81f4ac7 100644 --- a/src/hyperlink/_url.py +++ b/src/hyperlink/_url.py @@ -52,7 +52,8 @@ # from boltons.typeutils -def make_sentinel(name='_MISSING', var_name=None): +def make_sentinel(name='_MISSING', var_name=""): + # type: (str, str) -> object """Creates and returns a new **instance** of a new class, suitable for usage as a "sentinel", a kind of singleton often used to indicate a value is missing when ``None`` is a valid input. @@ -84,18 +85,24 @@ def make_sentinel(name='_MISSING', var_name=None): """ class Sentinel(object): def __init__(self): + # type: () -> None self.name = name self.var_name = var_name def __repr__(self): + # type: () -> str if self.var_name: return self.var_name return '%s(%r)' % (self.__class__.__name__, self.name) if var_name: - def __reduce__(self): + # superclass type hints don't allow str return type, but it is + # allowed in the docs, hence the ignore[override] below + def __reduce__(self): # type: ignore[override] intentional + # type: () -> str return self.var_name def __nonzero__(self): + # type: () -> bool return False __bool__ = __nonzero__ From eb46d28f63761262e13c1f611c6850888ce4b9d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilfredo=20Sa=CC=81nchez?= Date: Sat, 2 Nov 2019 16:08:21 -0700 Subject: [PATCH 05/24] More type hints --- src/hyperlink/_url.py | 85 ++++++++++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 34 deletions(-) diff --git a/src/hyperlink/_url.py b/src/hyperlink/_url.py index a81f4ac7..494656f8 100644 --- a/src/hyperlink/_url.py +++ b/src/hyperlink/_url.py @@ -25,8 +25,8 @@ except ImportError: AddressFamily = int # type: ignore[assignment,misc] Python 2 from typing import ( - Callable, Iterable, Iterator, List, Mapping, Optional, - Sequence, Text, Tuple, Type, Union, + Any, AnyStr, Callable, Iterable, Iterator, List, Mapping, Optional, + Sequence, Set, Text, Tuple, Type, TypeVar, Union, ) from unicodedata import normalize from ._socket import inet_pton @@ -39,7 +39,6 @@ PY2 = (sys.version_info[0] == 2) -unicode = type(u'') try: unichr except NameError: # Py3 @@ -49,6 +48,7 @@ Mapping[Text, Optional[Text]], Iterable[Tuple[Text, Optional[Text]]], ] +T = TypeVar('T') # from boltons.typeutils @@ -161,6 +161,7 @@ def __nonzero__(self): def _make_decode_map(delims, allow_percent=False): + # type: (Iterable[Text], bool) -> Mapping[bytes, bytes] ret = dict(_HEX_CHAR_MAP) if not allow_percent: delims = set(delims) | set([u'%']) @@ -358,6 +359,7 @@ def _encode_userinfo_part(text, maximal=True): def register_scheme(text, uses_netloc=True, default_port=None): + # type: (Text, bool, Optional[int]) -> None """Registers new scheme information, resulting in correct port and slash behavior from the URL object. There are dozens of standard schemes preregistered, so this function is mostly meant for @@ -366,13 +368,12 @@ def register_scheme(text, uses_netloc=True, default_port=None): `file an issue`_! Args: - text (unicode): Text representing the scheme. + text: Text representing the scheme. (the 'http' in 'http://hatnote.com') - uses_netloc (bool): Does the scheme support specifying a - network host? For instance, "http" does, "mailto" does - not. Defaults to True. - default_port (int): The default port, if any, for netloc-using - schemes. + uses_netloc: Does the scheme support specifying a + network host? For instance, "http" does, "mailto" does not. + Defaults to True. + default_port: The default port, if any, for netloc-using schemes. .. _file an issue: https://github.com/mahmoud/hyperlink/issues @@ -395,8 +396,6 @@ def register_scheme(text, uses_netloc=True, default_port=None): else: raise ValueError('uses_netloc expected bool, not: %r' % uses_netloc) - return - def scheme_uses_netloc(scheme, default=None): # type: (Text, Optional[bool]) -> Optional[bool] @@ -438,6 +437,7 @@ class URLParseError(ValueError): def _optional(argument, default): + # type: (Any, Any) -> Any if argument is _UNSET: return default else: @@ -445,6 +445,7 @@ def _optional(argument, default): def _typecheck(name, value, *types): + # type: (Text, T, Type) -> T """ Check that the given *value* is one of the given *types*, or raise an exception describing the problem using *name*. @@ -459,9 +460,11 @@ def _typecheck(name, value, *types): def _textcheck(name, value, delims=frozenset(), nullable=False): - if not isinstance(value, unicode): + # type: (Text, T, Iterable[Text], bool) -> T + if not isinstance(value, Text): if nullable and value is None: - return value # used by query string values + # used by query string values + return value # type: ignore[misc] unreachable else: str_name = "unicode" if PY2 else "str" exp = str_name + ' or NoneType' if nullable else str_name @@ -469,7 +472,7 @@ def _textcheck(name, value, delims=frozenset(), nullable=False): if delims and set(value) & set(delims): # TODO: test caching into regexes raise ValueError('one or more reserved delimiters %s present in %s: %r' % (''.join(delims), name, value)) - return value + return value # type: ignore[return-value] T vs. Text def iter_pairs(iterable): @@ -489,6 +492,7 @@ def iter_pairs(iterable): def _decode_unreserved( text, normalize_case=False, encode_stray_percents=False ): + # type: (Text, bool, bool) -> Union[str, bytes] return _percent_decode(text, normalize_case=normalize_case, encode_stray_percents=encode_stray_percents, _decode_map=_UNRESERVED_DECODE_MAP) @@ -497,6 +501,7 @@ def _decode_unreserved( def _decode_userinfo_part( text, normalize_case=False, encode_stray_percents=False ): + # type: (Text, bool, bool) -> Union[str, bytes] return _percent_decode(text, normalize_case=normalize_case, encode_stray_percents=encode_stray_percents, _decode_map=_USERINFO_DECODE_MAP) @@ -505,6 +510,7 @@ def _decode_userinfo_part( def _decode_path_part( text, normalize_case=False, encode_stray_percents=False ): + # type: (Text, bool, bool) -> Union[str, bytes] """ >>> _decode_path_part(u'%61%77%2f%7a') u'aw%2fz' @@ -519,6 +525,7 @@ def _decode_path_part( def _decode_query_key( text, normalize_case=False, encode_stray_percents=False ): + # type: (Text, bool, bool) -> Union[str, bytes] return _percent_decode(text, normalize_case=normalize_case, encode_stray_percents=encode_stray_percents, _decode_map=_QUERY_KEY_DECODE_MAP) @@ -527,6 +534,7 @@ def _decode_query_key( def _decode_query_value( text, normalize_case=False, encode_stray_percents=False ): + # type: (Text, bool, bool) -> Union[str, bytes] return _percent_decode(text, normalize_case=normalize_case, encode_stray_percents=encode_stray_percents, _decode_map=_QUERY_VALUE_DECODE_MAP) @@ -535,14 +543,21 @@ def _decode_query_value( def _decode_fragment_part( text, normalize_case=False, encode_stray_percents=False ): + # type: (Text, bool, bool) -> Union[str, bytes] return _percent_decode(text, normalize_case=normalize_case, encode_stray_percents=encode_stray_percents, _decode_map=_FRAGMENT_DECODE_MAP) -def _percent_decode(text, normalize_case=False, subencoding='utf-8', - raise_subencoding_exc=False, encode_stray_percents=False, - _decode_map=_HEX_CHAR_MAP): +def _percent_decode( + text, # type: Text + normalize_case=False, # type: bool + subencoding="utf-8", # type: Union[bool, Text] + raise_subencoding_exc=False, # type: bool + encode_stray_percents=False, # type: bool + _decode_map=_HEX_CHAR_MAP # type: Mapping[bytes, bytes] +): + # type: (...) -> Union[str, bytes] """Convert percent-encoded text characters to their normal, human-readable equivalents. @@ -560,24 +575,26 @@ def _percent_decode(text, normalize_case=False, subencoding='utf-8', u'abc def' Args: - text (unicode): Text with percent-encoding present. - normalize_case (bool): Whether undecoded percent segments, such - as encoded delimiters, should be uppercased, per RFC 3986 - Section 2.1. See :func:`_decode_path_part` for an example. - subencoding (unicode): The name of the encoding underlying the - percent-encoding. Pass `False` to get back raw bytes. - raise_subencoding_exc (bool): Whether an error in decoding the bytes - underlying the percent-decoding should be raised. + text: Text with percent-encoding present. + normalize_case: Whether undecoded percent segments, such as encoded + delimiters, should be uppercased, per RFC 3986 Section 2.1. + See :func:`_decode_path_part` for an example. + subencoding: The name of the encoding underlying the percent-encoding. + Pass `False` to get back raw bytes. + raise_subencoding_exc: Whether an error in decoding the bytes + underlying the percent-decoding should be raised. Returns: - unicode: The percent-decoded version of *text*, decoded by - *subencoding*, unless `subencoding=False` which returns bytes. + The percent-decoded version of *text*, decoded by *subencoding*, + unless `subencoding=False` which returns bytes. """ + if subencoding is False: + subencoding = "utf-8" + assert isinstance(subencoding, Text) + try: - quoted_bytes = text.encode( - 'utf-8' if subencoding is False else subencoding - ) + quoted_bytes = text.encode(subencoding) except UnicodeEncodeError: return text @@ -694,7 +711,7 @@ def _resolve_dot_segments(path): path: sequence of path segments in text form Returns: - list: a new sequence of path segments with the '.' and '..' elements + A new sequence of path segments with the '.' and '..' elements removed and resolved. .. _RFC 3986 section 5.2.4, Remove Dot Segments: https://tools.ietf.org/html/rfc3986#section-5.2.4 @@ -725,7 +742,7 @@ def parse_host(host): Will raise :class:`URLParseError` on invalid IPv6 constants. Returns: - tuple: family (socket constant or None), host (string) + family (socket constant or None), host (string) >>> import socket >>> parse_host('googlewebsite.com') == (None, 'googlewebsite.com') @@ -853,7 +870,7 @@ def __init__( % (self._scheme, self.__class__.__name__)) _, self._host = parse_host(_textcheck('host', host, '/?#@')) - if isinstance(path, unicode): + if isinstance(path, Text): raise TypeError("expected iterable of text for path, not: %r" % (path,)) self._path = tuple((_textcheck("path segment", segment, '/?#') @@ -1018,7 +1035,7 @@ def authority(self, with_password=False, **kw): else: hostport = [self.host] if self.port != SCHEME_PORT_MAP.get(self.scheme): - hostport.append(unicode(self.port)) + hostport.append(Text(self.port)) authority = [] if self.userinfo: userinfo = self.userinfo From 0492f6f8ab46428c6a45bdf90cb27d30db0c8cb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilfredo=20Sa=CC=81nchez?= Date: Sat, 2 Nov 2019 16:46:26 -0700 Subject: [PATCH 06/24] Revert "More type hints" This reverts commit eb46d28f63761262e13c1f611c6850888ce4b9d6. --- src/hyperlink/_url.py | 85 +++++++++++++++++-------------------------- 1 file changed, 34 insertions(+), 51 deletions(-) diff --git a/src/hyperlink/_url.py b/src/hyperlink/_url.py index 494656f8..a81f4ac7 100644 --- a/src/hyperlink/_url.py +++ b/src/hyperlink/_url.py @@ -25,8 +25,8 @@ except ImportError: AddressFamily = int # type: ignore[assignment,misc] Python 2 from typing import ( - Any, AnyStr, Callable, Iterable, Iterator, List, Mapping, Optional, - Sequence, Set, Text, Tuple, Type, TypeVar, Union, + Callable, Iterable, Iterator, List, Mapping, Optional, + Sequence, Text, Tuple, Type, Union, ) from unicodedata import normalize from ._socket import inet_pton @@ -39,6 +39,7 @@ PY2 = (sys.version_info[0] == 2) +unicode = type(u'') try: unichr except NameError: # Py3 @@ -48,7 +49,6 @@ Mapping[Text, Optional[Text]], Iterable[Tuple[Text, Optional[Text]]], ] -T = TypeVar('T') # from boltons.typeutils @@ -161,7 +161,6 @@ def __nonzero__(self): def _make_decode_map(delims, allow_percent=False): - # type: (Iterable[Text], bool) -> Mapping[bytes, bytes] ret = dict(_HEX_CHAR_MAP) if not allow_percent: delims = set(delims) | set([u'%']) @@ -359,7 +358,6 @@ def _encode_userinfo_part(text, maximal=True): def register_scheme(text, uses_netloc=True, default_port=None): - # type: (Text, bool, Optional[int]) -> None """Registers new scheme information, resulting in correct port and slash behavior from the URL object. There are dozens of standard schemes preregistered, so this function is mostly meant for @@ -368,12 +366,13 @@ def register_scheme(text, uses_netloc=True, default_port=None): `file an issue`_! Args: - text: Text representing the scheme. + text (unicode): Text representing the scheme. (the 'http' in 'http://hatnote.com') - uses_netloc: Does the scheme support specifying a - network host? For instance, "http" does, "mailto" does not. - Defaults to True. - default_port: The default port, if any, for netloc-using schemes. + uses_netloc (bool): Does the scheme support specifying a + network host? For instance, "http" does, "mailto" does + not. Defaults to True. + default_port (int): The default port, if any, for netloc-using + schemes. .. _file an issue: https://github.com/mahmoud/hyperlink/issues @@ -396,6 +395,8 @@ def register_scheme(text, uses_netloc=True, default_port=None): else: raise ValueError('uses_netloc expected bool, not: %r' % uses_netloc) + return + def scheme_uses_netloc(scheme, default=None): # type: (Text, Optional[bool]) -> Optional[bool] @@ -437,7 +438,6 @@ class URLParseError(ValueError): def _optional(argument, default): - # type: (Any, Any) -> Any if argument is _UNSET: return default else: @@ -445,7 +445,6 @@ def _optional(argument, default): def _typecheck(name, value, *types): - # type: (Text, T, Type) -> T """ Check that the given *value* is one of the given *types*, or raise an exception describing the problem using *name*. @@ -460,11 +459,9 @@ def _typecheck(name, value, *types): def _textcheck(name, value, delims=frozenset(), nullable=False): - # type: (Text, T, Iterable[Text], bool) -> T - if not isinstance(value, Text): + if not isinstance(value, unicode): if nullable and value is None: - # used by query string values - return value # type: ignore[misc] unreachable + return value # used by query string values else: str_name = "unicode" if PY2 else "str" exp = str_name + ' or NoneType' if nullable else str_name @@ -472,7 +469,7 @@ def _textcheck(name, value, delims=frozenset(), nullable=False): if delims and set(value) & set(delims): # TODO: test caching into regexes raise ValueError('one or more reserved delimiters %s present in %s: %r' % (''.join(delims), name, value)) - return value # type: ignore[return-value] T vs. Text + return value def iter_pairs(iterable): @@ -492,7 +489,6 @@ def iter_pairs(iterable): def _decode_unreserved( text, normalize_case=False, encode_stray_percents=False ): - # type: (Text, bool, bool) -> Union[str, bytes] return _percent_decode(text, normalize_case=normalize_case, encode_stray_percents=encode_stray_percents, _decode_map=_UNRESERVED_DECODE_MAP) @@ -501,7 +497,6 @@ def _decode_unreserved( def _decode_userinfo_part( text, normalize_case=False, encode_stray_percents=False ): - # type: (Text, bool, bool) -> Union[str, bytes] return _percent_decode(text, normalize_case=normalize_case, encode_stray_percents=encode_stray_percents, _decode_map=_USERINFO_DECODE_MAP) @@ -510,7 +505,6 @@ def _decode_userinfo_part( def _decode_path_part( text, normalize_case=False, encode_stray_percents=False ): - # type: (Text, bool, bool) -> Union[str, bytes] """ >>> _decode_path_part(u'%61%77%2f%7a') u'aw%2fz' @@ -525,7 +519,6 @@ def _decode_path_part( def _decode_query_key( text, normalize_case=False, encode_stray_percents=False ): - # type: (Text, bool, bool) -> Union[str, bytes] return _percent_decode(text, normalize_case=normalize_case, encode_stray_percents=encode_stray_percents, _decode_map=_QUERY_KEY_DECODE_MAP) @@ -534,7 +527,6 @@ def _decode_query_key( def _decode_query_value( text, normalize_case=False, encode_stray_percents=False ): - # type: (Text, bool, bool) -> Union[str, bytes] return _percent_decode(text, normalize_case=normalize_case, encode_stray_percents=encode_stray_percents, _decode_map=_QUERY_VALUE_DECODE_MAP) @@ -543,21 +535,14 @@ def _decode_query_value( def _decode_fragment_part( text, normalize_case=False, encode_stray_percents=False ): - # type: (Text, bool, bool) -> Union[str, bytes] return _percent_decode(text, normalize_case=normalize_case, encode_stray_percents=encode_stray_percents, _decode_map=_FRAGMENT_DECODE_MAP) -def _percent_decode( - text, # type: Text - normalize_case=False, # type: bool - subencoding="utf-8", # type: Union[bool, Text] - raise_subencoding_exc=False, # type: bool - encode_stray_percents=False, # type: bool - _decode_map=_HEX_CHAR_MAP # type: Mapping[bytes, bytes] -): - # type: (...) -> Union[str, bytes] +def _percent_decode(text, normalize_case=False, subencoding='utf-8', + raise_subencoding_exc=False, encode_stray_percents=False, + _decode_map=_HEX_CHAR_MAP): """Convert percent-encoded text characters to their normal, human-readable equivalents. @@ -575,26 +560,24 @@ def _percent_decode( u'abc def' Args: - text: Text with percent-encoding present. - normalize_case: Whether undecoded percent segments, such as encoded - delimiters, should be uppercased, per RFC 3986 Section 2.1. - See :func:`_decode_path_part` for an example. - subencoding: The name of the encoding underlying the percent-encoding. - Pass `False` to get back raw bytes. - raise_subencoding_exc: Whether an error in decoding the bytes - underlying the percent-decoding should be raised. + text (unicode): Text with percent-encoding present. + normalize_case (bool): Whether undecoded percent segments, such + as encoded delimiters, should be uppercased, per RFC 3986 + Section 2.1. See :func:`_decode_path_part` for an example. + subencoding (unicode): The name of the encoding underlying the + percent-encoding. Pass `False` to get back raw bytes. + raise_subencoding_exc (bool): Whether an error in decoding the bytes + underlying the percent-decoding should be raised. Returns: - The percent-decoded version of *text*, decoded by *subencoding*, - unless `subencoding=False` which returns bytes. + unicode: The percent-decoded version of *text*, decoded by + *subencoding*, unless `subencoding=False` which returns bytes. """ - if subencoding is False: - subencoding = "utf-8" - assert isinstance(subencoding, Text) - try: - quoted_bytes = text.encode(subencoding) + quoted_bytes = text.encode( + 'utf-8' if subencoding is False else subencoding + ) except UnicodeEncodeError: return text @@ -711,7 +694,7 @@ def _resolve_dot_segments(path): path: sequence of path segments in text form Returns: - A new sequence of path segments with the '.' and '..' elements + list: a new sequence of path segments with the '.' and '..' elements removed and resolved. .. _RFC 3986 section 5.2.4, Remove Dot Segments: https://tools.ietf.org/html/rfc3986#section-5.2.4 @@ -742,7 +725,7 @@ def parse_host(host): Will raise :class:`URLParseError` on invalid IPv6 constants. Returns: - family (socket constant or None), host (string) + tuple: family (socket constant or None), host (string) >>> import socket >>> parse_host('googlewebsite.com') == (None, 'googlewebsite.com') @@ -870,7 +853,7 @@ def __init__( % (self._scheme, self.__class__.__name__)) _, self._host = parse_host(_textcheck('host', host, '/?#@')) - if isinstance(path, Text): + if isinstance(path, unicode): raise TypeError("expected iterable of text for path, not: %r" % (path,)) self._path = tuple((_textcheck("path segment", segment, '/?#') @@ -1035,7 +1018,7 @@ def authority(self, with_password=False, **kw): else: hostport = [self.host] if self.port != SCHEME_PORT_MAP.get(self.scheme): - hostport.append(Text(self.port)) + hostport.append(unicode(self.port)) authority = [] if self.userinfo: userinfo = self.userinfo From 63c471dc3ebddbc96df33b9e0647a1c6a8df4ffa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilfredo=20Sa=CC=81nchez?= Date: Sat, 2 Nov 2019 16:49:45 -0700 Subject: [PATCH 07/24] =?UTF-8?q?unicode=20=E2=9E=9C=20Text?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/hyperlink/_url.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/hyperlink/_url.py b/src/hyperlink/_url.py index a81f4ac7..9d9e060f 100644 --- a/src/hyperlink/_url.py +++ b/src/hyperlink/_url.py @@ -39,7 +39,6 @@ PY2 = (sys.version_info[0] == 2) -unicode = type(u'') try: unichr except NameError: # Py3 @@ -459,7 +458,7 @@ def _typecheck(name, value, *types): def _textcheck(name, value, delims=frozenset(), nullable=False): - if not isinstance(value, unicode): + if not isinstance(value, Text): if nullable and value is None: return value # used by query string values else: @@ -853,7 +852,7 @@ def __init__( % (self._scheme, self.__class__.__name__)) _, self._host = parse_host(_textcheck('host', host, '/?#@')) - if isinstance(path, unicode): + if isinstance(path, Text): raise TypeError("expected iterable of text for path, not: %r" % (path,)) self._path = tuple((_textcheck("path segment", segment, '/?#') @@ -1018,7 +1017,7 @@ def authority(self, with_password=False, **kw): else: hostport = [self.host] if self.port != SCHEME_PORT_MAP.get(self.scheme): - hostport.append(unicode(self.port)) + hostport.append(Text(self.port)) authority = [] if self.userinfo: userinfo = self.userinfo From 5fa0ce26b7f73ec9985e779e90116c8c9c25d332 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilfredo=20Sa=CC=81nchez?= Date: Sat, 2 Nov 2019 17:14:12 -0700 Subject: [PATCH 08/24] This time with less oops --- src/hyperlink/_url.py | 48 +++++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/src/hyperlink/_url.py b/src/hyperlink/_url.py index 9d9e060f..89622a18 100644 --- a/src/hyperlink/_url.py +++ b/src/hyperlink/_url.py @@ -25,8 +25,8 @@ except ImportError: AddressFamily = int # type: ignore[assignment,misc] Python 2 from typing import ( - Callable, Iterable, Iterator, List, Mapping, Optional, - Sequence, Text, Tuple, Type, Union, + Any, Callable, Iterable, Iterator, List, Mapping, Optional, + Sequence, Text, Tuple, Type, TypeVar, Union, cast, ) from unicodedata import normalize from ._socket import inet_pton @@ -48,6 +48,7 @@ Mapping[Text, Optional[Text]], Iterable[Tuple[Text, Optional[Text]]], ] +T = TypeVar('T') # from boltons.typeutils @@ -160,6 +161,7 @@ def __nonzero__(self): def _make_decode_map(delims, allow_percent=False): + # type: (Iterable[Text], bool) -> Mapping[bytes, bytes] ret = dict(_HEX_CHAR_MAP) if not allow_percent: delims = set(delims) | set([u'%']) @@ -357,6 +359,7 @@ def _encode_userinfo_part(text, maximal=True): def register_scheme(text, uses_netloc=True, default_port=None): + # type: (Text, bool, Optional[int]) -> None """Registers new scheme information, resulting in correct port and slash behavior from the URL object. There are dozens of standard schemes preregistered, so this function is mostly meant for @@ -365,13 +368,12 @@ def register_scheme(text, uses_netloc=True, default_port=None): `file an issue`_! Args: - text (unicode): Text representing the scheme. + text: Text representing the scheme. (the 'http' in 'http://hatnote.com') - uses_netloc (bool): Does the scheme support specifying a - network host? For instance, "http" does, "mailto" does - not. Defaults to True. - default_port (int): The default port, if any, for netloc-using - schemes. + uses_netloc: Does the scheme support specifying a network host? + For instance, "http" does, "mailto" does not. + Defaults to True. + default_port: The default port, if any, for netloc-using schemes. .. _file an issue: https://github.com/mahmoud/hyperlink/issues @@ -437,6 +439,7 @@ class URLParseError(ValueError): def _optional(argument, default): + # type: (Any, Any) -> Any if argument is _UNSET: return default else: @@ -444,6 +447,7 @@ def _optional(argument, default): def _typecheck(name, value, *types): + # type: (Text, T, Type) -> T """ Check that the given *value* is one of the given *types*, or raise an exception describing the problem using *name*. @@ -458,9 +462,11 @@ def _typecheck(name, value, *types): def _textcheck(name, value, delims=frozenset(), nullable=False): + # type: (Text, T, Iterable[Text], bool) -> T if not isinstance(value, Text): if nullable and value is None: - return value # used by query string values + # used by query string values + return value # type: ignore[misc] unreachable else: str_name = "unicode" if PY2 else "str" exp = str_name + ' or NoneType' if nullable else str_name @@ -468,7 +474,7 @@ def _textcheck(name, value, delims=frozenset(), nullable=False): if delims and set(value) & set(delims): # TODO: test caching into regexes raise ValueError('one or more reserved delimiters %s present in %s: %r' % (''.join(delims), name, value)) - return value + return value # type: ignore[return-value] T vs. Text def iter_pairs(iterable): @@ -488,6 +494,7 @@ def iter_pairs(iterable): def _decode_unreserved( text, normalize_case=False, encode_stray_percents=False ): + # type: (Text, bool, bool) -> Union[str, bytes] return _percent_decode(text, normalize_case=normalize_case, encode_stray_percents=encode_stray_percents, _decode_map=_UNRESERVED_DECODE_MAP) @@ -496,6 +503,7 @@ def _decode_unreserved( def _decode_userinfo_part( text, normalize_case=False, encode_stray_percents=False ): + # type: (Text, bool, bool) -> Union[str, bytes] return _percent_decode(text, normalize_case=normalize_case, encode_stray_percents=encode_stray_percents, _decode_map=_USERINFO_DECODE_MAP) @@ -504,6 +512,7 @@ def _decode_userinfo_part( def _decode_path_part( text, normalize_case=False, encode_stray_percents=False ): + # type: (Text, bool, bool) -> Union[str, bytes] """ >>> _decode_path_part(u'%61%77%2f%7a') u'aw%2fz' @@ -518,6 +527,7 @@ def _decode_path_part( def _decode_query_key( text, normalize_case=False, encode_stray_percents=False ): + # type: (Text, bool, bool) -> Union[str, bytes] return _percent_decode(text, normalize_case=normalize_case, encode_stray_percents=encode_stray_percents, _decode_map=_QUERY_KEY_DECODE_MAP) @@ -526,6 +536,7 @@ def _decode_query_key( def _decode_query_value( text, normalize_case=False, encode_stray_percents=False ): + # type: (Text, bool, bool) -> Union[str, bytes] return _percent_decode(text, normalize_case=normalize_case, encode_stray_percents=encode_stray_percents, _decode_map=_QUERY_VALUE_DECODE_MAP) @@ -534,14 +545,21 @@ def _decode_query_value( def _decode_fragment_part( text, normalize_case=False, encode_stray_percents=False ): + # type: (Text, bool, bool) -> Union[str, bytes] return _percent_decode(text, normalize_case=normalize_case, encode_stray_percents=encode_stray_percents, _decode_map=_FRAGMENT_DECODE_MAP) -def _percent_decode(text, normalize_case=False, subencoding='utf-8', - raise_subencoding_exc=False, encode_stray_percents=False, - _decode_map=_HEX_CHAR_MAP): +def _percent_decode( + text, # type: Text + normalize_case=False, # type: bool + subencoding="utf-8", # type: Union[bool, Text] + raise_subencoding_exc=False, # type: bool + encode_stray_percents=False, # type: bool + _decode_map=_HEX_CHAR_MAP # type: Mapping[bytes, bytes] +): + # type: (...) -> Union[str, bytes] """Convert percent-encoded text characters to their normal, human-readable equivalents. @@ -575,7 +593,7 @@ def _percent_decode(text, normalize_case=False, subencoding='utf-8', """ try: quoted_bytes = text.encode( - 'utf-8' if subencoding is False else subencoding + 'utf-8' if subencoding is False else cast(Text, subencoding) ) except UnicodeEncodeError: return text @@ -613,7 +631,7 @@ def _percent_decode(text, normalize_case=False, subencoding='utf-8', if subencoding is False: return unquoted_bytes try: - return unquoted_bytes.decode(subencoding) + return unquoted_bytes.decode(cast(Text, subencoding)) except UnicodeDecodeError: if raise_subencoding_exc: raise From 8bf25e4064e32326adefd6a7c7f2125afefd3178 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilfredo=20Sa=CC=81nchez?= Date: Sun, 3 Nov 2019 09:17:19 -0800 Subject: [PATCH 09/24] More hints --- src/hyperlink/_url.py | 69 ++++++++++++++++++++++++++++++++----------- tox.ini | 2 +- 2 files changed, 52 insertions(+), 19 deletions(-) diff --git a/src/hyperlink/_url.py b/src/hyperlink/_url.py index 89622a18..ddd52860 100644 --- a/src/hyperlink/_url.py +++ b/src/hyperlink/_url.py @@ -891,14 +891,16 @@ def __init__( return def get_decoded_url(self, lazy=False): + # type: (bool) -> DecodedURL try: return self._decoded_url except AttributeError: - self._decoded_url = DecodedURL(self, lazy=lazy) + self._decoded_url = DecodedURL(self, lazy=lazy) # type: DecodedURL return self._decoded_url @property def scheme(self): + # type: () -> Text """The scheme is a string, and the first part of an absolute URL, the part before the first colon, and the part which defines the semantics of the rest of the URL. Examples include "http", @@ -909,6 +911,7 @@ def scheme(self): @property def host(self): + # type: () -> Text """The host is a string, and the second standard part of an absolute URL. When present, a valid host must be a domain name, or an IP (v4 or v6). It occurs before the first slash, or the second @@ -918,6 +921,7 @@ def host(self): @property def port(self): + # type: () -> Optional[int] """The port is an integer that is commonly used in connecting to the :attr:`host`, and almost never appears without it. @@ -942,6 +946,7 @@ def port(self): @property def path(self): + # type: () -> Sequence[Text] """A tuple of strings, created by splitting the slash-separated hierarchical path. Started by the first slash after the host, terminated by a "?", which indicates the start of the @@ -951,6 +956,7 @@ def path(self): @property def query(self): + # type: () -> QueryParameter """Tuple of pairs, created by splitting the ampersand-separated mapping of keys and optional values representing non-hierarchical data used to identify the resource. Keys are @@ -966,6 +972,7 @@ def query(self): @property def fragment(self): + # type: () -> Text """A string, the last part of the URL, indicated by the first "#" after the :attr:`~hyperlink.URL.path` or :attr:`~hyperlink.URL.query`. Enables indirect identification @@ -976,6 +983,7 @@ def fragment(self): @property def rooted(self): + # type: () -> bool """Whether or not the path starts with a forward slash (``/``). This is taken from the terminology in the BNF grammar, @@ -989,6 +997,7 @@ def rooted(self): @property def userinfo(self): + # type: () -> Text """The colon-separated string forming the username-password combination. """ @@ -996,18 +1005,21 @@ def userinfo(self): @property def uses_netloc(self): + # type: () -> Optional[bool] """ """ return self._uses_netloc @property def user(self): + # type: () -> Text """ The user portion of :attr:`~hyperlink.URL.userinfo`. """ return self.userinfo.split(u':')[0] def authority(self, with_password=False, **kw): + # type: (bool, Any) -> Text """Compute and return the appropriate host/port/userinfo combination. >>> url = URL.from_text(u'http://user:pass@localhost:8080/a/b?x=y') @@ -1046,6 +1058,7 @@ def authority(self, with_password=False, **kw): return u"@".join(authority) def __eq__(self, other): + # type: (Any) -> bool if not isinstance(other, self.__class__): return NotImplemented for attr in ['scheme', 'userinfo', 'host', 'query', @@ -1060,17 +1073,20 @@ def __eq__(self, other): return False def __ne__(self, other): + # type: (Any) -> bool if not isinstance(other, self.__class__): return NotImplemented return not self.__eq__(other) def __hash__(self): + # type: () -> int return hash((self.__class__, self.scheme, self.userinfo, self.host, self.path, self.query, self.fragment, self.port, self.rooted, self.uses_netloc)) @property def absolute(self): + # type: () -> bool """Whether or not the URL is "absolute". Absolute URLs are complete enough to resolve to a network resource without being relative to a base URI. @@ -1084,9 +1100,19 @@ def absolute(self): """ return bool(self.scheme and self.host) - def replace(self, scheme=_UNSET, host=_UNSET, path=_UNSET, query=_UNSET, - fragment=_UNSET, port=_UNSET, rooted=_UNSET, userinfo=_UNSET, - uses_netloc=_UNSET): + def replace( # type: ignore[assignment] _UNSET is private + self, + scheme=_UNSET, # type: Optional[Text] + host=_UNSET, # type: Optional[Text] + path=_UNSET, # type: Iterable[Text] + query=_UNSET, # type: QueryParameter + fragment=_UNSET, # type: Text + port=_UNSET, # type: Optional[int] + rooted=_UNSET, # type: Optional[bool] + userinfo=_UNSET, # type: Text + uses_netloc=_UNSET # type: Optional[bool] + ): + # type: (...) -> URL """:class:`URL` objects are immutable, which means that attributes are designed to be set only once, at construction. Instead of modifying an existing URL, one simply creates a copy with the @@ -1114,7 +1140,7 @@ def replace(self, scheme=_UNSET, host=_UNSET, path=_UNSET, query=_UNSET, ``mailto:e@g.com``) Returns: - URL: a copy of the current :class:`URL`, with new values for + a copy of the current :class:`URL`, with new values for parameters passed. """ @@ -1132,6 +1158,7 @@ def replace(self, scheme=_UNSET, host=_UNSET, path=_UNSET, query=_UNSET, @classmethod def from_text(cls, text): + # type: (Text) -> URL """Whereas the :class:`URL` constructor is useful for constructing URLs from parts, :meth:`~URL.from_text` supports parsing whole URLs from their string form:: @@ -1162,19 +1189,17 @@ def from_text(cls, text): """ um = _URL_RE.match(_textcheck('text', text)) - try: - gs = um.groupdict() - except AttributeError: + if um is None: raise URLParseError('could not parse url: %r' % text) + gs = um.groupdict() au_text = gs['authority'] or u'' au_m = _AUTHORITY_RE.match(au_text) - try: - au_gs = au_m.groupdict() - except AttributeError: + if au_m is None: raise URLParseError( 'invalid authority %r in url: %r' % (au_text, text) ) + au_gs = au_m.groupdict() if au_gs['bad_host']: raise URLParseError( 'invalid host %r in url: %r' % (au_gs['bad_host'], text) @@ -1186,7 +1211,7 @@ def from_text(cls, text): port = au_gs['port'] if port is not None: try: - port = int(port) + port = int(port) # type: ignore[assignment] TODO, also below except ValueError: if not port: # TODO: excessive? raise URLParseError('port must not be empty: %r' % au_text) @@ -1197,9 +1222,9 @@ def from_text(cls, text): uses_netloc = bool(gs['_netloc_sep']) if gs['path']: - path = gs['path'].split(u"/") + path = tuple(gs['path'].split(u"/")) if not path[0]: - path.pop(0) + path = path[1:] rooted = True else: rooted = False @@ -1207,12 +1232,20 @@ def from_text(cls, text): path = () rooted = bool(au_text) if gs['query']: - query = ((qe.split(u"=", 1) if u'=' in qe else (qe, None)) - for qe in gs['query'].split(u"&")) + query = [ + ( + cast(Tuple[str, str], qe.split(u"=", 1)) + if u'=' in qe else + (qe, None) + ) + for qe in gs['query'].split(u"&") + ] # type: Iterable[Tuple[str, Optional[str]]] else: query = () - return cls(scheme, host, path, query, fragment, port, - rooted, userinfo, uses_netloc) + return cls( + scheme, host, path, query, fragment, + port, rooted, userinfo, uses_netloc, # type: ignore[arg-type] TODO + ) def normalize(self, scheme=True, host=True, path=True, query=True, fragment=True, userinfo=True, percents=True): diff --git a/tox.ini b/tox.ini index 07b9fd50..097758be 100644 --- a/tox.ini +++ b/tox.ini @@ -69,7 +69,7 @@ commands = description = run Flake8 (linter) -basepython = python3.7 +basepython = python3.8 skip_install = True From 50acc09360e759961a5d4f36377785acefbd1c6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilfredo=20Sa=CC=81nchez?= Date: Sun, 3 Nov 2019 11:06:03 -0800 Subject: [PATCH 10/24] Finish typing for URL --- src/hyperlink/_url.py | 292 +++++++++++++++++---------------- src/hyperlink/test/test_url.py | 2 +- 2 files changed, 155 insertions(+), 139 deletions(-) diff --git a/src/hyperlink/_url.py b/src/hyperlink/_url.py index ddd52860..f8fc9454 100644 --- a/src/hyperlink/_url.py +++ b/src/hyperlink/_url.py @@ -25,7 +25,7 @@ except ImportError: AddressFamily = int # type: ignore[assignment,misc] Python 2 from typing import ( - Any, Callable, Iterable, Iterator, List, Mapping, Optional, + Any, Callable, Dict, Iterable, Iterator, List, Mapping, Optional, Sequence, Text, Tuple, Type, TypeVar, Union, cast, ) from unicodedata import normalize @@ -59,9 +59,9 @@ def make_sentinel(name='_MISSING', var_name=""): a value is missing when ``None`` is a valid input. Args: - name (str): Name of the Sentinel - var_name (str): Set this name to the name of the variable in - its respective module enable pickleability. + name: Name of the Sentinel + var_name: Set this name to the name of the variable in its respective + module enable pickle-ability. >>> make_sentinel(var_name='_MISSING') _MISSING @@ -74,14 +74,13 @@ def make_sentinel(name='_MISSING', var_name=""): .. note:: - By design, additional calls to ``make_sentinel`` with the same - values will not produce equivalent objects. - - >>> make_sentinel('TEST') == make_sentinel('TEST') - False - >>> type(make_sentinel('TEST')) == type(make_sentinel('TEST')) - False + By design, additional calls to ``make_sentinel`` with the same + values will not produce equivalent objects. + >>> make_sentinel('TEST') == make_sentinel('TEST') + False + >>> type(make_sentinel('TEST')) == type(make_sentinel('TEST')) + False """ class Sentinel(object): def __init__(self): @@ -369,14 +368,13 @@ def register_scheme(text, uses_netloc=True, default_port=None): Args: text: Text representing the scheme. - (the 'http' in 'http://hatnote.com') + (the 'http' in 'http://hatnote.com') uses_netloc: Does the scheme support specifying a network host? - For instance, "http" does, "mailto" does not. - Defaults to True. + For instance, "http" does, "mailto" does not. + Defaults to True. default_port: The default port, if any, for netloc-using schemes. .. _file an issue: https://github.com/mahmoud/hyperlink/issues - """ text = text.lower() if default_port is not None: @@ -577,19 +575,18 @@ def _percent_decode( u'abc def' Args: - text (unicode): Text with percent-encoding present. - normalize_case (bool): Whether undecoded percent segments, such - as encoded delimiters, should be uppercased, per RFC 3986 - Section 2.1. See :func:`_decode_path_part` for an example. - subencoding (unicode): The name of the encoding underlying the - percent-encoding. Pass `False` to get back raw bytes. - raise_subencoding_exc (bool): Whether an error in decoding the bytes - underlying the percent-decoding should be raised. + text: Text with percent-encoding present. + normalize_case: Whether undecoded percent segments, such as encoded + delimiters, should be uppercased, per RFC 3986 Section 2.1. + See :func:`_decode_path_part` for an example. + subencoding: The name of the encoding underlying the percent-encoding. + Pass `False` to get back raw bytes. + raise_subencoding_exc: Whether an error in decoding the bytes + underlying the percent-decoding should be raised. Returns: - unicode: The percent-decoded version of *text*, decoded by - *subencoding*, unless `subencoding=False` which returns bytes. - + The percent-decoded version of *text*, decoded by *subencoding*, unless + `subencoding=False` which returns bytes. """ try: quoted_bytes = text.encode( @@ -711,8 +708,8 @@ def _resolve_dot_segments(path): path: sequence of path segments in text form Returns: - list: a new sequence of path segments with the '.' and '..' elements - removed and resolved. + A new sequence of path segments with the '.' and '..' elements removed + and resolved. .. _RFC 3986 section 5.2.4, Remove Dot Segments: https://tools.ietf.org/html/rfc3986#section-5.2.4 """ # noqa: E501 @@ -940,7 +937,6 @@ def port(self): Per the standard, when the port is the same as the schemes default port, it will be omitted in the text URL. - """ return self._port @@ -956,7 +952,7 @@ def path(self): @property def query(self): - # type: () -> QueryParameter + # type: () -> Tuple[Tuple[Text, Optional[Text]], ...] """Tuple of pairs, created by splitting the ampersand-separated mapping of keys and optional values representing non-hierarchical data used to identify the resource. Keys are @@ -977,7 +973,6 @@ def fragment(self): after the :attr:`~hyperlink.URL.path` or :attr:`~hyperlink.URL.query`. Enables indirect identification of a secondary resource, like an anchor within an HTML page. - """ return self._fragment @@ -991,7 +986,6 @@ def rooted(self): and "absolute URI" are somewhat ambiguous. :attr:`path` does not contain the implicit prefixed ``"/"`` since that is somewhat awkward to work with. - """ return self._rooted @@ -1029,13 +1023,12 @@ def authority(self, with_password=False, **kw): u'user:pass@localhost:8080' Args: - with_password (bool): Whether the return value of this - method include the password in the URL, if it is - set. Defaults to False. + with_password: Whether the return value of this method include the + password in the URL, if it is set. Defaults to False. Returns: - str: The authority (network location and user information) portion - of the URL. + The authority (network location and user information) portion of the + URL. """ # first, a bit of twisted compat with_password = kw.pop('includeSecrets', with_password) @@ -1122,27 +1115,22 @@ def replace( # type: ignore[assignment] _UNSET is private the value on the current URL. Args: - scheme (unicode): The text name of the scheme. - host (unicode): The host portion of the network location - port (int): The port part of the network location. - path (tuple): A tuple of strings representing the - slash-separated parts of the path. - query (tuple): The query parameters, as a tuple of - key-value pairs. - query (tuple): The query parameters, as a dictionary or - as an iterable of key-value pairs. - fragment (unicode): The fragment part of the URL. - rooted (bool): Whether or not the path begins with a slash. - userinfo (unicode): The username or colon-separated - username:password pair. - uses_netloc (bool): Indicates whether two slashes appear - between the scheme and the host (``http://eg.com`` vs - ``mailto:e@g.com``) + scheme: The text name of the scheme. + host: The host portion of the network location + path: A tuple of strings representing the slash-separated parts of + the path. + query: The query parameters, as a dictionary or as an iterable of + key-value pairs. + fragment: The fragment part of the URL. + port: The port part of the network location. + rooted: Whether or not the path begins with a slash. + userinfo: The username or colon-separated username:password pair. + uses_netloc: Indicates whether two slashes appear between the + scheme and the host (``http://eg.com`` vs ``mailto:e@g.com``) Returns: - a copy of the current :class:`URL`, with new values for - parameters passed. - + A copy of the current :class:`URL`, with new values for + parameters passed. """ return self.__class__( scheme=_optional(scheme, self.scheme), @@ -1163,10 +1151,10 @@ def from_text(cls, text): URLs from parts, :meth:`~URL.from_text` supports parsing whole URLs from their string form:: - >>> URL.from_text(u'http://example.com') - URL.from_text(u'http://example.com') - >>> URL.from_text(u'?a=b&x=y') - URL.from_text(u'?a=b&x=y') + >>> URL.from_text(u'http://example.com') + URL.from_text(u'http://example.com') + >>> URL.from_text(u'?a=b&x=y') + URL.from_text(u'?a=b&x=y') As you can see above, it's also used as the :func:`repr` of :class:`URL` objects. The natural counterpart to @@ -1174,10 +1162,10 @@ def from_text(cls, text): sure to decode those bytestrings. Args: - text (unicode): A valid URL string. + text: A valid URL string. Returns: - URL: The structured object version of the parsed string. + The structured object version of the parsed string. .. note:: @@ -1186,7 +1174,6 @@ def from_text(cls, text): look like URLs are still valid URLs. As a result, this method only raises :class:`URLParseError` on invalid port and IPv6 values in the host portion of the URL. - """ um = _URL_RE.match(_textcheck('text', text)) if um is None: @@ -1211,7 +1198,7 @@ def from_text(cls, text): port = au_gs['port'] if port is not None: try: - port = int(port) # type: ignore[assignment] TODO, also below + port = int(port) # type: ignore[assignment] FIXME, also below except ValueError: if not port: # TODO: excessive? raise URLParseError('port must not be empty: %r' % au_text) @@ -1244,11 +1231,13 @@ def from_text(cls, text): query = () return cls( scheme, host, path, query, fragment, - port, rooted, userinfo, uses_netloc, # type: ignore[arg-type] TODO + port, # type: ignore[arg-type] FIXME, also above + rooted, userinfo, uses_netloc, ) def normalize(self, scheme=True, host=True, path=True, query=True, fragment=True, userinfo=True, percents=True): + # type: (bool, bool, bool, bool, bool, bool, bool) -> URL """Return a new URL object with several standard normalizations applied: @@ -1265,15 +1254,14 @@ def normalize(self, scheme=True, host=True, path=True, query=True, name. Args: - scheme (bool): Convert the scheme to lowercase - host (bool): Convert the host to lowercase - path (bool): Normalize the path (see above for details) - query (bool): Normalize the query string - fragment (bool): Normalize the fragment - userinfo (bool): Normalize the userinfo - percents (bool): Encode isolated percent signs - for any percent-encoded fields which are being - normalized (defaults to True). + scheme: Convert the scheme to lowercase + host: Convert the host to lowercase + path: Normalize the path (see above for details) + query: Normalize the query string + fragment: Normalize the fragment + userinfo: Normalize the userinfo + percents: Encode isolated percent signs for any percent-encoded + fields which are being normalized (defaults to True). >>> url = URL.from_text(u'Http://example.COM/a/../b/./c%2f?%61%') >>> print(url.normalize().to_text()) @@ -1285,9 +1273,8 @@ def normalize(self, scheme=True, host=True, path=True, query=True, .. _RFC 3986 6.2.2.3: https://tools.ietf.org/html/rfc3986#section-6.2.2.3 .. _RFC 3986 6.2.3: https://tools.ietf.org/html/rfc3986#section-6.2.3 .. _RFC 3986 2.4: https://tools.ietf.org/html/rfc3986#section-2.4 - """ # noqa: E501 - kw = {} + kw = {} # type: Dict[str, Any] if scheme: kw['scheme'] = self.scheme.lower() if host: @@ -1315,6 +1302,7 @@ def _dec_unres(target): return self.replace(**kw) def child(self, *segments): + # type: (Text) -> URL """Make a new :class:`URL` where the given path segments are a child of this URL, preserving other parts of the URL, including the query string and fragment. @@ -1327,58 +1315,61 @@ def child(self, *segments): u'http://localhost/a/b/c/d?x=y' Args: - segments (unicode): Additional parts to be joined and added to - the path, like :func:`os.path.join`. Special characters - in segments will be percent encoded. + segments: Additional parts to be joined and added to + the path, like :func:`os.path.join`. Special characters + in segments will be percent encoded. Returns: - URL: A copy of the current URL with the extra path segments. - + A copy of the current URL with the extra path segments. """ if not segments: return self - segments = [_textcheck('path segment', s) for s in segments] - new_segs = _encode_path_parts(segments, maximal=False) - new_path = self.path[:-1 if (self.path and self.path[-1] == u'') - else None] + new_segs + segments = [ # type: ignore[assignment] variable is tuple + _textcheck('path segment', s) for s in segments + ] + new_path = tuple(self.path) + if self.path and self.path[-1] == u"": + new_path = new_path[:-1] + new_path += tuple(_encode_path_parts(segments, maximal=False)) return self.replace(path=new_path) def sibling(self, segment): + # type: (Text) -> URL """Make a new :class:`URL` with a single path segment that is a sibling of this URL path. Args: - segment (unicode): A single path segment. + segment: A single path segment. Returns: - URL: A copy of the current URL with the last path segment - replaced by *segment*. Special characters such as - ``/?#`` will be percent encoded. - + A copy of the current URL with the last path segment + replaced by *segment*. Special characters such as + ``/?#`` will be percent encoded. """ _textcheck('path segment', segment) - new_path = self.path[:-1] + (_encode_path_part(segment),) + new_path = tuple(self.path)[:-1] + (_encode_path_part(segment),) return self.replace(path=new_path) def click(self, href=u''): + # type: (Text) -> URL """Resolve the given URL relative to this URL. The resulting URI should match what a web browser would generate if you visited the current URL and clicked on *href*. - >>> url = URL.from_text(u'http://blog.hatnote.com/') - >>> url.click(u'/post/155074058790').to_text() - u'http://blog.hatnote.com/post/155074058790' - >>> url = URL.from_text(u'http://localhost/a/b/c/') - >>> url.click(u'../d/./e').to_text() - u'http://localhost/a/b/d/e' + >>> url = URL.from_text(u'http://blog.hatnote.com/') + >>> url.click(u'/post/155074058790').to_text() + u'http://blog.hatnote.com/post/155074058790' + >>> url = URL.from_text(u'http://localhost/a/b/c/') + >>> url.click(u'../d/./e').to_text() + u'http://localhost/a/b/d/e' Args: - href (unicode): A string representing a clicked URL. + href: A string representing a clicked URL. Return: - URL: A copy of the current URL with navigation logic applied. + A copy of the current URL with navigation logic applied. For more information, see `RFC 3986 section 5`_. @@ -1386,7 +1377,7 @@ def click(self, href=u''): """ if href: if isinstance(href, URL): - clicked = href + clicked = href # type: ignore[misc] unreachable else: # TODO: This error message is not completely accurate, # as URL objects are now also valid, but Twisted's @@ -1409,7 +1400,7 @@ def click(self, href=u''): if clicked.rooted: path = clicked.path elif clicked.path: - path = self.path[:-1] + clicked.path + path = tuple(self.path)[:-1] + tuple(clicked.path) else: path = self.path if not query: @@ -1422,6 +1413,7 @@ def click(self, href=u''): fragment=clicked.fragment) def to_uri(self): + # type: () -> URL u"""Make a new :class:`URL` instance with all non-ASCII characters appropriately percent-encoded. This is useful to do in preparation for sending a :class:`URL` over a network protocol. @@ -1457,6 +1449,7 @@ def to_uri(self): ) def to_iri(self): + # type: () -> URL u"""Make a new :class:`URL` instance with all but a few reserved characters decoded into human-readable format. @@ -1479,21 +1472,32 @@ def to_iri(self): URL: A new instance with its path segments, query parameters, and hostname decoded for display purposes. """ # noqa: E501 - new_userinfo = u':'.join([_decode_userinfo_part(p) for p in - self.userinfo.split(':', 1)]) + new_userinfo = u':'.join([ + cast(Text, _decode_userinfo_part(p)) + for p in self.userinfo.split(':', 1) + ]) host_text = _decode_host(self.host) - return self.replace(userinfo=new_userinfo, - host=host_text, - path=[_decode_path_part(segment) - for segment in self.path], - query=[(_decode_query_key(k), - _decode_query_value(v) - if v is not None else None) - for k, v in self.query], - fragment=_decode_fragment_part(self.fragment)) + return self.replace( + userinfo=new_userinfo, + host=host_text, + path=[ + cast(Text, _decode_path_part(segment)) + for segment in self.path + ], + query=[ + ( + cast(Text, _decode_query_key(k)), + cast(Optional[Text], _decode_query_value(v)) + if v is not None else None + ) + for k, v in self.query + ], + fragment=cast(Text, _decode_fragment_part(self.fragment)), + ) def to_text(self, with_password=False): + # type: (bool) -> Text """Render this URL to its textual representation. By default, the URL text will *not* include a password, if one @@ -1507,12 +1511,12 @@ def to_text(self, with_password=False): password)." Args: - with_password (bool): Whether or not to include the - password in the URL text. Defaults to False. + with_password: Whether or not to include the password in the URL + text. Defaults to False. Returns: - str: The serialized textual representation of this URL, - such as ``u"http://example.com/some/path?some=query"``. + The serialized textual representation of this URL, such as + ``u"http://example.com/some/path?some=query"``. The natural counterpart to :class:`URL.from_text()`. @@ -1540,7 +1544,7 @@ def to_text(self, with_password=False): fragment = self.fragment - parts = [] + parts = [] # type: List[Text] _add = parts.append if scheme: _add(scheme) @@ -1563,6 +1567,7 @@ def to_text(self, with_password=False): return u''.join(parts) def __repr__(self): + # type: () -> str """Convert this URL to an representation that shows all of its constituent parts, as well as being a valid argument to :func:`eval`. @@ -1570,6 +1575,7 @@ def __repr__(self): return '%s.from_text(%r)' % (self.__class__.__name__, self.to_text()) def _to_bytes(self): + # type: () -> bytes """ Allows for direct usage of URL objects with libraries like requests, which automatically stringify URL parameters. See @@ -1590,12 +1596,15 @@ def _to_bytes(self): @classmethod def fromText(cls, s): + # type: (Text) -> URL return cls.from_text(s) def asText(self, includeSecrets=False): + # type: (bool) -> Text return self.to_text(with_password=includeSecrets) def __dir__(self): + # type: () -> Sequence[Text] try: ret = object.__dir__(self) except AttributeError: @@ -1607,6 +1616,7 @@ def __dir__(self): # # End Twisted Compat Code def add(self, name, value=None): + # type: (Text, Optional[Text]) -> URL """Make a new :class:`URL` instance with a given query argument, *name*, added to it with the value *value*, like so:: @@ -1616,18 +1626,19 @@ def add(self, name, value=None): URL.from_text(u'https://example.com/?x=y&x=z') Args: - name (unicode): The name of the query parameter to add. The - part before the ``=``. - value (unicode): The value of the query parameter to add. The - part after the ``=``. Defaults to ``None``, meaning no + name: The name of the query parameter to add. + The part before the ``=``. + value: The value of the query parameter to add. + The part after the ``=``. Defaults to ``None``, meaning no value. Returns: - URL: A new :class:`URL` instance with the parameter added. + A new :class:`URL` instance with the parameter added. """ return self.replace(query=self.query + ((name, value),)) def set(self, name, value=None): + # type: (Text, Optional[Text]) -> URL """Make a new :class:`URL` instance with the query parameter *name* set to *value*. All existing occurences, if any are replaced by the single name-value pair. @@ -1638,23 +1649,24 @@ def set(self, name, value=None): URL.from_text(u'https://example.com/?x=z') Args: - name (unicode): The name of the query parameter to set. The - part before the ``=``. - value (unicode): The value of the query parameter to set. The - part after the ``=``. Defaults to ``None``, meaning no + name: The name of the query parameter to set. + The part before the ``=``. + value: The value of the query parameter to set. + The part after the ``=``. Defaults to ``None``, meaning no value. Returns: - URL: A new :class:`URL` instance with the parameter set. + A new :class:`URL` instance with the parameter set. """ # Preserve the original position of the query key in the list - q = [(k, v) for (k, v) in self.query if k != name] + q = [(k, v) for (k, v) in self._query if k != name] idx = next((i for (i, (k, v)) in enumerate(self.query) if k == name), -1) q[idx:idx] = [(name, value)] return self.replace(query=q) def get(self, name): + # type: (Text) -> List[Optional[Text]] """Get a list of values for the given query parameter, *name*:: >>> url = URL.from_text(u'?x=1&x=2') @@ -1667,27 +1679,31 @@ def get(self, name): list is always returned, and this method raises no exceptions. Args: - name (unicode): The name of the query parameter to get. + name: The name of the query parameter to get. Returns: - list: A list of all the values associated with the key, in - string form. - + A list of all the values associated with the key, in string form. """ return [value for (key, value) in self.query if name == key] - def remove(self, name, value=_UNSET, limit=None): + def remove( # type: ignore[assignment] _UNSET is private + self, + name, # type: Text + value=_UNSET, # type: Text + limit=None, # type: Optional[int] + ): + # type: (...) -> URL """Make a new :class:`URL` instance with occurrences of the query parameter *name* removed, or, if *value* is set, parameters matching *name* and *value*. No exception is raised if the parameter is not already set. Args: - name (unicode): The name of the query parameter to remove. - value (unicode): Optional value to additionally filter - on. Setting this removes query parameters which match - both name and value. - limit (int): Optional maximum number of parameters to remove. + name: The name of the query parameter to remove. + value: Optional value to additionally filter on. + Setting this removes query parameters which match both name + and value. + limit: Optional maximum number of parameters to remove. Returns: URL: A new :class:`URL` instance with the parameter removed. diff --git a/src/hyperlink/test/test_url.py b/src/hyperlink/test/test_url.py index 6ee4d172..cdb25147 100644 --- a/src/hyperlink/test/test_url.py +++ b/src/hyperlink/test/test_url.py @@ -430,7 +430,7 @@ def test_click(self): # test click on a URL instance u = URL.fromText('http://localhost/foo/?abc=def') u2 = URL.from_text('bar') - u3 = u.click(u2) + u3 = u.click(u2) # type: ignore[arg-type] URL vs Text self.assertEqual(u3.to_text(), 'http://localhost/foo/bar') def test_clickRFC3986(self): From 7f7121abc62be06e510a26d822e8c7a6cc5f9e90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilfredo=20Sa=CC=81nchez?= Date: Mon, 4 Nov 2019 12:12:12 -0800 Subject: [PATCH 11/24] Type hints at 100%. --- src/hyperlink/_url.py | 220 +++++++++++++++++-------- src/hyperlink/test/test_decoded_url.py | 15 +- tox.ini | 3 - 3 files changed, 165 insertions(+), 73 deletions(-) diff --git a/src/hyperlink/_url.py b/src/hyperlink/_url.py index f8fc9454..f8342312 100644 --- a/src/hyperlink/_url.py +++ b/src/hyperlink/_url.py @@ -44,9 +44,10 @@ except NameError: # Py3 unichr = chr # type: Callable[[int], Text] NoneType = type(None) # type: Type[None] -QueryParameter = Union[ - Mapping[Text, Optional[Text]], - Iterable[Tuple[Text, Optional[Text]]], +QueryPairs = Tuple[Tuple[Text, Optional[Text]], ...] # internal representation +QueryParameters = Union[ + Mapping[Text, Optional[Text]], QueryPairs, + Sequence[Tuple[Text, Optional[Text]]], ] T = TypeVar('T') @@ -174,7 +175,8 @@ def _make_decode_map(delims, allow_percent=False): def _make_quote_map(safe_chars): - ret = {} + # type: (Iterable[Text]) -> Mapping[Union[int, Text], Text] + ret = {} # type: Dict[Union[int, Text], Text] # v is included in the dict for py3 mostly, because bytestrings # are iterables of ints, of course! for i, v in zip(range(256), range(256)): @@ -830,7 +832,7 @@ def __init__( scheme=None, # type: Optional[Text] host=None, # type: Optional[Text] path=(), # type: Iterable[Text] - query=(), # type: QueryParameter + query=(), # type: QueryParameters fragment=u"", # type: Text port=None, # type: Optional[int] rooted=None, # type: Optional[bool] @@ -952,7 +954,7 @@ def path(self): @property def query(self): - # type: () -> Tuple[Tuple[Text, Optional[Text]], ...] + # type: () -> QueryPairs """Tuple of pairs, created by splitting the ampersand-separated mapping of keys and optional values representing non-hierarchical data used to identify the resource. Keys are @@ -1098,7 +1100,7 @@ def replace( # type: ignore[assignment] _UNSET is private scheme=_UNSET, # type: Optional[Text] host=_UNSET, # type: Optional[Text] path=_UNSET, # type: Iterable[Text] - query=_UNSET, # type: QueryParameter + query=_UNSET, # type: QueryParameters fragment=_UNSET, # type: Text port=_UNSET, # type: Optional[int] rooted=_UNSET, # type: Optional[bool] @@ -1219,14 +1221,14 @@ def from_text(cls, text): path = () rooted = bool(au_text) if gs['query']: - query = [ + query = tuple( ( cast(Tuple[str, str], qe.split(u"=", 1)) if u'=' in qe else (qe, None) ) for qe in gs['query'].split(u"&") - ] # type: Iterable[Tuple[str, Optional[str]]] + ) # type: QueryPairs else: query = () return cls( @@ -1281,8 +1283,10 @@ def normalize(self, scheme=True, host=True, path=True, query=True, kw['host'] = self.host.lower() def _dec_unres(target): - return _decode_unreserved(target, normalize_case=True, - encode_stray_percents=percents) + # type: (Text) -> Union[Text] + return cast(Text, _decode_unreserved( + target, normalize_case=True, encode_stray_percents=percents) + ) if path: if self.path: kw['path'] = [ @@ -1485,14 +1489,14 @@ def to_iri(self): cast(Text, _decode_path_part(segment)) for segment in self.path ], - query=[ + query=tuple( ( cast(Text, _decode_query_key(k)), cast(Optional[Text], _decode_query_value(v)) if v is not None else None ) for k, v in self.query - ], + ), fragment=cast(Text, _decode_fragment_part(self.fragment)), ) @@ -1750,12 +1754,13 @@ class DecodedURL(object): special characters encoded with codecs other than UTF-8. Args: - url (URL): A :class:`URL` object to wrap. - lazy (bool): Set to True to avoid pre-decode all parts of the - URL to check for validity. Defaults to False. + url: A :class:`URL` object to wrap. + lazy: Set to True to avoid pre-decode all parts of the URL to check for + validity. Defaults to False. """ def __init__(self, url, lazy=False): + # type: (URL, bool) -> None self._url = url if not lazy: # cache the following, while triggering any decoding @@ -1765,45 +1770,52 @@ def __init__(self, url, lazy=False): @classmethod def from_text(cls, text, lazy=False): + # type: (Text, bool) -> DecodedURL """\ Make a `DecodedURL` instance from any text string containing a URL. Args: - text (unicode): Text containing the URL - lazy (bool): Whether to pre-decode all parts of the URL to - check for validity. Defaults to True. + text: Text containing the URL + lazy: Whether to pre-decode all parts of the URL to check for + validity. Defaults to True. """ _url = URL.from_text(text) return cls(_url, lazy=lazy) @property def encoded_url(self): + # type: () -> URL """Access the underlying :class:`URL` object, which has any special characters encoded. """ return self._url - def to_text(self, *a, **kw): + def to_text(self, with_password=False): + # type: (bool) -> Text "Passthrough to :meth:`~hyperlink.URL.to_text()`" - return self._url.to_text(*a, **kw) + return self._url.to_text(with_password) - def to_uri(self, *a, **kw): + def to_uri(self): + # type: () -> URL "Passthrough to :meth:`~hyperlink.URL.to_uri()`" - return self._url.to_uri(*a, **kw) + return self._url.to_uri() - def to_iri(self, *a, **kw): + def to_iri(self): + # type: () -> URL "Passthrough to :meth:`~hyperlink.URL.to_iri()`" - return self._url.to_iri(*a, **kw) + return self._url.to_iri() def click(self, href=u''): + # type: (Text) -> DecodedURL """Return a new DecodedURL wrapping the result of :meth:`~hyperlink.URL.click()` """ if isinstance(href, DecodedURL): - href = href._url + href = href._url # type: ignore[misc] unreachable return self.__class__(self._url.click(href=href)) def sibling(self, segment): + # type: (Text) -> DecodedURL """Automatically encode any reserved characters in *segment* and return a new `DecodedURL` wrapping the result of :meth:`~hyperlink.URL.sibling()` @@ -1811,6 +1823,7 @@ def sibling(self, segment): return self.__class__(self._url.sibling(_encode_reserved(segment))) def child(self, *segments): + # type: (Text) -> DecodedURL """Automatically encode any reserved characters in *segments* and return a new `DecodedURL` wrapping the result of :meth:`~hyperlink.URL.child()`. @@ -1820,86 +1833,137 @@ def child(self, *segments): new_segs = [_encode_reserved(s) for s in segments] return self.__class__(self._url.child(*new_segs)) - def normalize(self, *a, **kw): + def normalize(self, scheme=True, host=True, path=True, query=True, + fragment=True, userinfo=True, percents=True): + # type: (bool, bool, bool, bool, bool, bool, bool) -> DecodedURL """Return a new `DecodedURL` wrapping the result of :meth:`~hyperlink.URL.normalize()` """ - return self.__class__(self._url.normalize(*a, **kw)) + return self.__class__(self._url.normalize( + scheme, host, path, query, fragment, userinfo, percents + )) @property def absolute(self): + # type: () -> bool return self._url.absolute @property def scheme(self): + # type: () -> Text return self._url.scheme @property def host(self): + # type: () -> Text return _decode_host(self._url.host) @property def port(self): + # type: () -> Optional[int] return self._url.port @property def rooted(self): + # type: () -> bool return self._url.rooted @property def path(self): + # type: () -> Sequence[Text] try: - return self._path + return cast( + Tuple[Text, ...], + self._path # type: ignore[has-type] can't determine + ) except AttributeError: pass - self._path = tuple([_percent_decode(p, raise_subencoding_exc=True) - for p in self._url.path]) + self._path = tuple([ + cast(Text, _percent_decode(p, raise_subencoding_exc=True)) + for p in self._url.path + ]) return self._path @property def query(self): + # type: () -> QueryPairs try: - return self._query + return cast( + QueryPairs, + self._query # type: ignore[has-type] can't determine + ) except AttributeError: pass - _q = [tuple(_percent_decode(x, raise_subencoding_exc=True) - if x is not None else None - for x in (k, v)) - for k, v in self._url.query] - self._query = tuple(_q) + self._query = cast(QueryPairs, tuple( + tuple( + cast(Text, _percent_decode(x, raise_subencoding_exc=True)) + if x is not None else None + for x in (k, v) + ) + for k, v in self._url.query + )) return self._query @property def fragment(self): + # type: () -> Text try: - return self._fragment + return cast( + Text, + self._fragment # type: ignore[has-type] can't determine + ) except AttributeError: pass frag = self._url.fragment - self._fragment = _percent_decode(frag, raise_subencoding_exc=True) + self._fragment = cast( + Text, _percent_decode(frag, raise_subencoding_exc=True) + ) return self._fragment @property def userinfo(self): + # type: () -> Union[Tuple[str], Tuple[str, str]] try: - return self._userinfo + return cast( + Union[Tuple[str], Tuple[str, str]], + self._userinfo # type: ignore[has-type] can't determine + ) except AttributeError: pass - self._userinfo = tuple([_percent_decode(p, raise_subencoding_exc=True) - for p in self._url.userinfo.split(':', 1)]) + self._userinfo = cast( + Union[Tuple[str], Tuple[str, str]], + tuple( + tuple( + cast(Text, _percent_decode(p, raise_subencoding_exc=True)) + for p in self._url.userinfo.split(':', 1) + ) + ) + ) return self._userinfo @property def user(self): + # type: () -> Text return self.userinfo[0] @property def uses_netloc(self): - return self._url.uses_netloc + # type: () -> bool + return cast(bool, self._url.uses_netloc) - def replace(self, scheme=_UNSET, host=_UNSET, path=_UNSET, query=_UNSET, - fragment=_UNSET, port=_UNSET, rooted=_UNSET, userinfo=_UNSET, - uses_netloc=_UNSET): + def replace( # type: ignore[assignment] _UNSET is private + self, + scheme=_UNSET, # type: Optional[Text] + host=_UNSET, # type: Optional[Text] + path=_UNSET, # type: Iterable[Text] + query=_UNSET, # type: QueryParameters + fragment=_UNSET, # type: Text + port=_UNSET, # type: Optional[int] + rooted=_UNSET, # type: Optional[bool] + userinfo=_UNSET, # type: Union[Tuple[str], Tuple[str, str]] + uses_netloc=_UNSET # type: Optional[bool] + ): + # type: (...) -> DecodedURL """While the signature is the same, this `replace()` differs a little from URL.replace. For instance, it accepts userinfo as a tuple, not as a string, handling the case of having a username @@ -1908,17 +1972,23 @@ def replace(self, scheme=_UNSET, host=_UNSET, path=_UNSET, query=_UNSET, automatically encoded instead of an error being raised. """ if path is not _UNSET: - path = [_encode_reserved(p) for p in path] + path = tuple(_encode_reserved(p) for p in path) if query is not _UNSET: - query = [[_encode_reserved(x) - if x is not None else None - for x in (k, v)] - for k, v in iter_pairs(query)] + query = cast(QueryPairs, tuple( + tuple( + _encode_reserved(x) + if x is not None else None + for x in (k, v) + ) + for k, v in iter_pairs(query) + )) if userinfo is not _UNSET: if len(userinfo) > 2: raise ValueError('userinfo expected sequence of ["user"] or' - ' ["user", "password"], got %r' % userinfo) - userinfo = u':'.join([_encode_reserved(p) for p in userinfo]) + ' ["user", "password"], got %r' % (userinfo,)) + userinfo_text = u':'.join([_encode_reserved(p) for p in userinfo]) + else: + userinfo_text = cast(Text, _UNSET) new_url = self._url.replace(scheme=scheme, host=host, path=path, @@ -1926,20 +1996,23 @@ def replace(self, scheme=_UNSET, host=_UNSET, path=_UNSET, query=_UNSET, fragment=fragment, port=port, rooted=rooted, - userinfo=userinfo, + userinfo=userinfo_text, uses_netloc=uses_netloc) return self.__class__(url=new_url) def get(self, name): + # type: (Text) -> List[Optional[Text]] "Get the value of all query parameters whose name matches *name*" return [v for (k, v) in self.query if name == k] def add(self, name, value=None): + # type: (Text, Optional[Text]) -> DecodedURL """Return a new DecodedURL with the query parameter *name* and *value* added.""" return self.replace(query=self.query + ((name, value),)) def set(self, name, value=None): + # type: (Text, Optional[Text]) -> DecodedURL "Return a new DecodedURL with query parameter *name* set to *value*" query = self.query q = [(k, v) for (k, v) in query if k != name] @@ -1947,7 +2020,13 @@ def set(self, name, value=None): q[idx:idx] = [(name, value)] return self.replace(query=q) - def remove(self, name, value=_UNSET, limit=None): + def remove( # type: ignore[assignment] _UNSET is private + self, + name, # type: Text + value=_UNSET, # type: Text + limit=None, # type: Optional[int] + ): + # type: (...) -> DecodedURL """Return a new DecodedURL with query parameter *name* removed. Optionally also filter for *value*, as well as cap the number @@ -1976,25 +2055,30 @@ def remove(self, name, value=_UNSET, limit=None): return self.replace(query=nq) def __repr__(self): + # type: () -> str cn = self.__class__.__name__ return '%s(url=%r)' % (cn, self._url) def __str__(self): + # type: () -> str # TODO: the underlying URL's __str__ needs to change to make # this work as the URL, see #55 return str(self._url) def __eq__(self, other): + # type: (Any) -> bool if not isinstance(other, self.__class__): return NotImplemented return self.normalize().to_uri() == other.normalize().to_uri() def __ne__(self, other): + # type: (Any) -> bool if not isinstance(other, self.__class__): return NotImplemented return not self.__eq__(other) def __hash__(self): + # type: () -> int return hash((self.__class__, self.scheme, self.userinfo, self.host, self.path, self.query, self.fragment, self.port, self.rooted, self.uses_netloc)) @@ -2005,12 +2089,15 @@ def __hash__(self): @classmethod def fromText(cls, s, lazy=False): + # type: (Text, bool) -> DecodedURL return cls.from_text(s, lazy=lazy) def asText(self, includeSecrets=False): + # type: (bool) -> Text return self.to_text(with_password=includeSecrets) def __dir__(self): + # type: () -> Sequence[Text] try: ret = object.__dir__(self) except AttributeError: @@ -2023,21 +2110,22 @@ def __dir__(self): def parse(url, decoded=True, lazy=False): + # type: (Text, bool, bool) -> Union[URL, DecodedURL] """Automatically turn text into a structured URL object. Args: - decoded (bool): Whether or not to return a :class:`DecodedURL`, - which automatically handles all - encoding/decoding/quoting/unquoting for all the various - accessors of parts of the URL, or an :class:`EncodedURL`, - which has the same API, but requires handling of special - characters for different parts of the URL. - - lazy (bool): In the case of `decoded=True`, this controls - whether the URL is decoded immediately or as accessed. The - default, `lazy=False`, checks all encoded parts of the URL - for decodability. + decoded: Whether or not to return a :class:`DecodedURL`, + which automatically handles all + encoding/decoding/quoting/unquoting for all the various + accessors of parts of the URL, or an :class:`EncodedURL`, + which has the same API, but requires handling of special + characters for different parts of the URL. + + lazy: In the case of `decoded=True`, this controls + whether the URL is decoded immediately or as accessed. The + default, `lazy=False`, checks all encoded parts of the URL + for decodability. """ enc_url = EncodedURL.from_text(url) if not decoded: diff --git a/src/hyperlink/test/test_decoded_url.py b/src/hyperlink/test/test_decoded_url.py index 17f7f19b..98c1565d 100644 --- a/src/hyperlink/test/test_decoded_url.py +++ b/src/hyperlink/test/test_decoded_url.py @@ -2,7 +2,8 @@ from __future__ import unicode_literals -from .. import DecodedURL +from typing import Dict, Union +from .. import DecodedURL, URL from .._url import _percent_decode from .common import HyperlinkTestCase @@ -132,7 +133,9 @@ def test_equality_and_hashability(self): assert durl is not None assert durl != durl._url - durl_map = {} + AnyURL = Union[URL, DecodedURL] + + durl_map = {} # type: Dict[AnyURL, AnyURL] durl_map[durl] = durl durl_map[durl2] = durl2 @@ -166,7 +169,11 @@ def test_replace_userinfo(self): # type: () -> None durl = DecodedURL.from_text(TOTAL_URL) with self.assertRaises(ValueError): - durl.replace(userinfo=['user', 'pw', 'thiswillcauseafailure']) + durl.replace( + userinfo=( # type: ignore[arg-type] intentional + 'user', 'pw', 'thiswillcauseafailure' + ) + ) return def test_twisted_compat(self): @@ -207,7 +214,7 @@ def test_click_decoded_url(self): durl = DecodedURL.from_text(TOTAL_URL) durl_dest = DecodedURL.from_text('/tëst') - clicked = durl.click(durl_dest) + clicked = durl.click(durl_dest) # type: ignore[arg-type] URL vs Text assert clicked.host == durl.host assert clicked.path == durl_dest.path assert clicked.path == ('tëst',) diff --git a/tox.ini b/tox.ini index 392a1630..8514ab9c 100644 --- a/tox.ini +++ b/tox.ini @@ -176,9 +176,6 @@ check_untyped_defs = False # Disable some checks until effected files fully adopt mypy -[mypy-hyperlink._url] -allow_untyped_defs = True - [mypy-idna] ignore_missing_imports = True From 47574602f0f0925420e70fa8a92bf6b6bc2bb1c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilfredo=20Sa=CC=81nchez?= Date: Mon, 4 Nov 2019 12:24:55 -0800 Subject: [PATCH 12/24] Get rid of subencoding=False code path in _percent_decode(). It isn't used by any of the rest of the code, and it makes _percent_decode() return either text or bytes, which then meant a lot of type casting elsewhere. --- src/hyperlink/_url.py | 55 ++++++++++---------------- src/hyperlink/test/test_decoded_url.py | 7 ---- 2 files changed, 21 insertions(+), 41 deletions(-) diff --git a/src/hyperlink/_url.py b/src/hyperlink/_url.py index f8342312..1573e2c2 100644 --- a/src/hyperlink/_url.py +++ b/src/hyperlink/_url.py @@ -494,7 +494,7 @@ def iter_pairs(iterable): def _decode_unreserved( text, normalize_case=False, encode_stray_percents=False ): - # type: (Text, bool, bool) -> Union[str, bytes] + # type: (Text, bool, bool) -> Text return _percent_decode(text, normalize_case=normalize_case, encode_stray_percents=encode_stray_percents, _decode_map=_UNRESERVED_DECODE_MAP) @@ -503,7 +503,7 @@ def _decode_unreserved( def _decode_userinfo_part( text, normalize_case=False, encode_stray_percents=False ): - # type: (Text, bool, bool) -> Union[str, bytes] + # type: (Text, bool, bool) -> Text return _percent_decode(text, normalize_case=normalize_case, encode_stray_percents=encode_stray_percents, _decode_map=_USERINFO_DECODE_MAP) @@ -512,7 +512,7 @@ def _decode_userinfo_part( def _decode_path_part( text, normalize_case=False, encode_stray_percents=False ): - # type: (Text, bool, bool) -> Union[str, bytes] + # type: (Text, bool, bool) -> Text """ >>> _decode_path_part(u'%61%77%2f%7a') u'aw%2fz' @@ -527,7 +527,7 @@ def _decode_path_part( def _decode_query_key( text, normalize_case=False, encode_stray_percents=False ): - # type: (Text, bool, bool) -> Union[str, bytes] + # type: (Text, bool, bool) -> Text return _percent_decode(text, normalize_case=normalize_case, encode_stray_percents=encode_stray_percents, _decode_map=_QUERY_KEY_DECODE_MAP) @@ -536,7 +536,7 @@ def _decode_query_key( def _decode_query_value( text, normalize_case=False, encode_stray_percents=False ): - # type: (Text, bool, bool) -> Union[str, bytes] + # type: (Text, bool, bool) -> Text return _percent_decode(text, normalize_case=normalize_case, encode_stray_percents=encode_stray_percents, _decode_map=_QUERY_VALUE_DECODE_MAP) @@ -545,7 +545,7 @@ def _decode_query_value( def _decode_fragment_part( text, normalize_case=False, encode_stray_percents=False ): - # type: (Text, bool, bool) -> Union[str, bytes] + # type: (Text, bool, bool) -> Text return _percent_decode(text, normalize_case=normalize_case, encode_stray_percents=encode_stray_percents, _decode_map=_FRAGMENT_DECODE_MAP) @@ -554,12 +554,12 @@ def _decode_fragment_part( def _percent_decode( text, # type: Text normalize_case=False, # type: bool - subencoding="utf-8", # type: Union[bool, Text] + subencoding="utf-8", # type: Text raise_subencoding_exc=False, # type: bool encode_stray_percents=False, # type: bool _decode_map=_HEX_CHAR_MAP # type: Mapping[bytes, bytes] ): - # type: (...) -> Union[str, bytes] + # type: (...) -> Text """Convert percent-encoded text characters to their normal, human-readable equivalents. @@ -582,18 +582,14 @@ def _percent_decode( delimiters, should be uppercased, per RFC 3986 Section 2.1. See :func:`_decode_path_part` for an example. subencoding: The name of the encoding underlying the percent-encoding. - Pass `False` to get back raw bytes. raise_subencoding_exc: Whether an error in decoding the bytes underlying the percent-decoding should be raised. Returns: - The percent-decoded version of *text*, decoded by *subencoding*, unless - `subencoding=False` which returns bytes. + The percent-decoded version of *text*, decoded by *subencoding*. """ try: - quoted_bytes = text.encode( - 'utf-8' if subencoding is False else cast(Text, subencoding) - ) + quoted_bytes = text.encode(subencoding) except UnicodeEncodeError: return text @@ -627,10 +623,8 @@ def _percent_decode( unquoted_bytes = b''.join(res) - if subencoding is False: - return unquoted_bytes try: - return unquoted_bytes.decode(cast(Text, subencoding)) + return unquoted_bytes.decode(subencoding) except UnicodeDecodeError: if raise_subencoding_exc: raise @@ -1284,8 +1278,8 @@ def normalize(self, scheme=True, host=True, path=True, query=True, def _dec_unres(target): # type: (Text) -> Union[Text] - return cast(Text, _decode_unreserved( - target, normalize_case=True, encode_stray_percents=percents) + return _decode_unreserved( + target, normalize_case=True, encode_stray_percents=percents ) if path: if self.path: @@ -1477,27 +1471,22 @@ def to_iri(self): hostname decoded for display purposes. """ # noqa: E501 new_userinfo = u':'.join([ - cast(Text, _decode_userinfo_part(p)) - for p in self.userinfo.split(':', 1) + _decode_userinfo_part(p) for p in self.userinfo.split(':', 1) ]) host_text = _decode_host(self.host) return self.replace( userinfo=new_userinfo, host=host_text, - path=[ - cast(Text, _decode_path_part(segment)) - for segment in self.path - ], + path=[_decode_path_part(segment) for segment in self.path], query=tuple( ( - cast(Text, _decode_query_key(k)), - cast(Optional[Text], _decode_query_value(v)) + _decode_query_key(k), _decode_query_value(v) if v is not None else None ) for k, v in self.query ), - fragment=cast(Text, _decode_fragment_part(self.fragment)), + fragment=_decode_fragment_part(self.fragment), ) def to_text(self, with_password=False): @@ -1879,7 +1868,7 @@ def path(self): except AttributeError: pass self._path = tuple([ - cast(Text, _percent_decode(p, raise_subencoding_exc=True)) + _percent_decode(p, raise_subencoding_exc=True) for p in self._url.path ]) return self._path @@ -1896,7 +1885,7 @@ def query(self): pass self._query = cast(QueryPairs, tuple( tuple( - cast(Text, _percent_decode(x, raise_subencoding_exc=True)) + _percent_decode(x, raise_subencoding_exc=True) if x is not None else None for x in (k, v) ) @@ -1915,9 +1904,7 @@ def fragment(self): except AttributeError: pass frag = self._url.fragment - self._fragment = cast( - Text, _percent_decode(frag, raise_subencoding_exc=True) - ) + self._fragment = _percent_decode(frag, raise_subencoding_exc=True) return self._fragment @property @@ -1934,7 +1921,7 @@ def userinfo(self): Union[Tuple[str], Tuple[str, str]], tuple( tuple( - cast(Text, _percent_decode(p, raise_subencoding_exc=True)) + _percent_decode(p, raise_subencoding_exc=True) for p in self._url.userinfo.split(':', 1) ) ) diff --git a/src/hyperlink/test/test_decoded_url.py b/src/hyperlink/test/test_decoded_url.py index 98c1565d..7942c03a 100644 --- a/src/hyperlink/test/test_decoded_url.py +++ b/src/hyperlink/test/test_decoded_url.py @@ -185,10 +185,6 @@ def test_twisted_compat(self): assert 'asText' not in dir(durl) assert durl.to_text() == durl.asText() - def test_percent_decode_bytes(self): - # type: () -> None - assert _percent_decode('%00', subencoding=False) == b'\0' - def test_percent_decode_mixed(self): # type: () -> None @@ -203,9 +199,6 @@ def test_percent_decode_mixed(self): with self.assertRaises(UnicodeDecodeError): _percent_decode('abcdé%C3éfg', raise_subencoding_exc=True) - # check that getting raw bytes works ok - assert _percent_decode('a%00b', subencoding=False) == b'a\x00b' - # when not encodable as subencoding assert _percent_decode('é%25é', subencoding='ascii') == 'é%25é' From 1e96fdead706e83e6df20206c3328e7677f81f95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilfredo=20Sa=CC=81nchez?= Date: Mon, 4 Nov 2019 12:26:58 -0800 Subject: [PATCH 13/24] Don't need Union here. --- src/hyperlink/_url.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyperlink/_url.py b/src/hyperlink/_url.py index 1573e2c2..ace9f49c 100644 --- a/src/hyperlink/_url.py +++ b/src/hyperlink/_url.py @@ -1277,7 +1277,7 @@ def normalize(self, scheme=True, host=True, path=True, query=True, kw['host'] = self.host.lower() def _dec_unres(target): - # type: (Text) -> Union[Text] + # type: (Text) -> Text return _decode_unreserved( target, normalize_case=True, encode_stray_percents=percents ) From 5331f893d8dffe3e0123caaf4c9be928c3a88310 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilfredo=20Sa=CC=81nchez?= Date: Mon, 4 Nov 2019 12:56:46 -0800 Subject: [PATCH 14/24] href is allowed to be a URL --- src/hyperlink/_url.py | 8 ++++---- src/hyperlink/test/test_decoded_url.py | 2 +- src/hyperlink/test/test_url.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/hyperlink/_url.py b/src/hyperlink/_url.py index ace9f49c..91a4635c 100644 --- a/src/hyperlink/_url.py +++ b/src/hyperlink/_url.py @@ -1350,7 +1350,7 @@ def sibling(self, segment): return self.replace(path=new_path) def click(self, href=u''): - # type: (Text) -> URL + # type: (Union[Text, URL]) -> URL """Resolve the given URL relative to this URL. The resulting URI should match what a web browser would @@ -1375,7 +1375,7 @@ def click(self, href=u''): """ if href: if isinstance(href, URL): - clicked = href # type: ignore[misc] unreachable + clicked = href else: # TODO: This error message is not completely accurate, # as URL objects are now also valid, but Twisted's @@ -1795,12 +1795,12 @@ def to_iri(self): return self._url.to_iri() def click(self, href=u''): - # type: (Text) -> DecodedURL + # type: (Union[Text, URL, DecodedURL]) -> DecodedURL """Return a new DecodedURL wrapping the result of :meth:`~hyperlink.URL.click()` """ if isinstance(href, DecodedURL): - href = href._url # type: ignore[misc] unreachable + href = href._url return self.__class__(self._url.click(href=href)) def sibling(self, segment): diff --git a/src/hyperlink/test/test_decoded_url.py b/src/hyperlink/test/test_decoded_url.py index 7942c03a..f00052ad 100644 --- a/src/hyperlink/test/test_decoded_url.py +++ b/src/hyperlink/test/test_decoded_url.py @@ -207,7 +207,7 @@ def test_click_decoded_url(self): durl = DecodedURL.from_text(TOTAL_URL) durl_dest = DecodedURL.from_text('/tëst') - clicked = durl.click(durl_dest) # type: ignore[arg-type] URL vs Text + clicked = durl.click(durl_dest) assert clicked.host == durl.host assert clicked.path == durl_dest.path assert clicked.path == ('tëst',) diff --git a/src/hyperlink/test/test_url.py b/src/hyperlink/test/test_url.py index cdb25147..6ee4d172 100644 --- a/src/hyperlink/test/test_url.py +++ b/src/hyperlink/test/test_url.py @@ -430,7 +430,7 @@ def test_click(self): # test click on a URL instance u = URL.fromText('http://localhost/foo/?abc=def') u2 = URL.from_text('bar') - u3 = u.click(u2) # type: ignore[arg-type] URL vs Text + u3 = u.click(u2) self.assertEqual(u3.to_text(), 'http://localhost/foo/bar') def test_clickRFC3986(self): From 6bafade9b4606afca34a387f61bf517d02e18249 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilfredo=20Sa=CC=81nchez?= Date: Mon, 4 Nov 2019 13:02:57 -0800 Subject: [PATCH 15/24] No longer need this change --- src/hyperlink/_url.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyperlink/_url.py b/src/hyperlink/_url.py index 91a4635c..a7bb914b 100644 --- a/src/hyperlink/_url.py +++ b/src/hyperlink/_url.py @@ -1652,7 +1652,7 @@ def set(self, name, value=None): A new :class:`URL` instance with the parameter set. """ # Preserve the original position of the query key in the list - q = [(k, v) for (k, v) in self._query if k != name] + q = [(k, v) for (k, v) in self.query if k != name] idx = next((i for (i, (k, v)) in enumerate(self.query) if k == name), -1) q[idx:idx] = [(name, value)] From c856e9b39ac3a44db945d956b1f110d558f0b8e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilfredo=20Sa=CC=81nchez?= Date: Mon, 4 Nov 2019 13:11:11 -0800 Subject: [PATCH 16/24] Not needed --- tox.ini | 3 --- 1 file changed, 3 deletions(-) diff --git a/tox.ini b/tox.ini index 8514ab9c..150129d4 100644 --- a/tox.ini +++ b/tox.ini @@ -111,9 +111,6 @@ ignore = # variable in function should be lowercase N806, - # lowercase imported as non lowercase - N812, - # variable in class scope should not be mixedCase N815, From b7c0789a967c65dcf3f914fdc5ba7d0cc5d584df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilfredo=20Sa=CC=81nchez?= Date: Mon, 4 Nov 2019 13:56:08 -0800 Subject: [PATCH 17/24] Enable check_untyped_defs --- tox.ini | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 150129d4..0719c70d 100644 --- a/tox.ini +++ b/tox.ini @@ -156,6 +156,7 @@ commands = # Global settings +check_untyped_defs = True disallow_incomplete_defs = True disallow_untyped_defs = True no_implicit_optional = True @@ -168,9 +169,6 @@ warn_return_any = True warn_unreachable = True warn_unused_ignores = True -# Enable these over time -check_untyped_defs = False - # Disable some checks until effected files fully adopt mypy [mypy-idna] From 67e482c792158e5cc66c97aec99933dd5daaee03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilfredo=20Sa=CC=81nchez?= Date: Mon, 4 Nov 2019 13:58:04 -0800 Subject: [PATCH 18/24] Better comment --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 0719c70d..8637f403 100644 --- a/tox.ini +++ b/tox.ini @@ -169,7 +169,7 @@ warn_return_any = True warn_unreachable = True warn_unused_ignores = True -# Disable some checks until effected files fully adopt mypy +# Don't complain about dependencies known to lack type hints [mypy-idna] ignore_missing_imports = True From 29c3dd9cd72e33db0e2f9e35ca5be3502279ec93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilfredo=20Sa=CC=81nchez?= Date: Tue, 19 Nov 2019 10:39:29 -0800 Subject: [PATCH 19/24] pin pyflakes pending a release with https://github.com/PyCQA/pyflakes/pull/455 Clean up some type hints. --- src/hyperlink/_url.py | 32 +++++++++++++------------- src/hyperlink/test/common.py | 2 +- src/hyperlink/test/test_common.py | 2 +- src/hyperlink/test/test_decoded_url.py | 2 +- src/hyperlink/test/test_socket.py | 2 +- src/hyperlink/test/test_url.py | 2 +- tox.ini | 2 ++ 7 files changed, 23 insertions(+), 21 deletions(-) diff --git a/src/hyperlink/_url.py b/src/hyperlink/_url.py index a7bb914b..81222374 100644 --- a/src/hyperlink/_url.py +++ b/src/hyperlink/_url.py @@ -23,7 +23,7 @@ try: from socket import AddressFamily except ImportError: - AddressFamily = int # type: ignore[assignment,misc] Python 2 + AddressFamily = int # type: ignore[assignment,misc] from typing import ( Any, Callable, Dict, Iterable, Iterator, List, Mapping, Optional, Sequence, Text, Tuple, Type, TypeVar, Union, cast, @@ -97,7 +97,7 @@ def __repr__(self): if var_name: # superclass type hints don't allow str return type, but it is # allowed in the docs, hence the ignore[override] below - def __reduce__(self): # type: ignore[override] intentional + def __reduce__(self): # type: ignore[override] # type: () -> str return self.var_name @@ -110,7 +110,7 @@ def __nonzero__(self): return Sentinel() -_unspecified = _UNSET = make_sentinel('_UNSET') +_unspecified = _UNSET = make_sentinel('_UNSET') # type: Any # RFC 3986 Section 2.3, Unreserved URI Characters @@ -466,7 +466,7 @@ def _textcheck(name, value, delims=frozenset(), nullable=False): if not isinstance(value, Text): if nullable and value is None: # used by query string values - return value # type: ignore[misc] unreachable + return value # type: ignore[misc] # unreachable else: str_name = "unicode" if PY2 else "str" exp = str_name + ' or NoneType' if nullable else str_name @@ -474,7 +474,7 @@ def _textcheck(name, value, delims=frozenset(), nullable=False): if delims and set(value) & set(delims): # TODO: test caching into regexes raise ValueError('one or more reserved delimiters %s present in %s: %r' % (''.join(delims), name, value)) - return value # type: ignore[return-value] T vs. Text + return value # type: ignore[return-value] # T vs. Text def iter_pairs(iterable): @@ -1089,7 +1089,7 @@ def absolute(self): """ return bool(self.scheme and self.host) - def replace( # type: ignore[assignment] _UNSET is private + def replace( self, scheme=_UNSET, # type: Optional[Text] host=_UNSET, # type: Optional[Text] @@ -1194,7 +1194,7 @@ def from_text(cls, text): port = au_gs['port'] if port is not None: try: - port = int(port) # type: ignore[assignment] FIXME, also below + port = int(port) # type: ignore[assignment] # FIXME, see below except ValueError: if not port: # TODO: excessive? raise URLParseError('port must not be empty: %r' % au_text) @@ -1227,7 +1227,7 @@ def from_text(cls, text): query = () return cls( scheme, host, path, query, fragment, - port, # type: ignore[arg-type] FIXME, also above + port, # type: ignore[arg-type] # FIXME, see above rooted, userinfo, uses_netloc, ) @@ -1323,7 +1323,7 @@ def child(self, *segments): if not segments: return self - segments = [ # type: ignore[assignment] variable is tuple + segments = [ # type: ignore[assignment] # variable is tuple _textcheck('path segment', s) for s in segments ] new_path = tuple(self.path) @@ -1679,7 +1679,7 @@ def get(self, name): """ return [value for (key, value) in self.query if name == key] - def remove( # type: ignore[assignment] _UNSET is private + def remove( self, name, # type: Text value=_UNSET, # type: Text @@ -1863,7 +1863,7 @@ def path(self): try: return cast( Tuple[Text, ...], - self._path # type: ignore[has-type] can't determine + self._path # type: ignore[has-type] # can't determine ) except AttributeError: pass @@ -1879,7 +1879,7 @@ def query(self): try: return cast( QueryPairs, - self._query # type: ignore[has-type] can't determine + self._query # type: ignore[has-type] # can't determine ) except AttributeError: pass @@ -1899,7 +1899,7 @@ def fragment(self): try: return cast( Text, - self._fragment # type: ignore[has-type] can't determine + self._fragment # type: ignore[has-type] # can't determine ) except AttributeError: pass @@ -1913,7 +1913,7 @@ def userinfo(self): try: return cast( Union[Tuple[str], Tuple[str, str]], - self._userinfo # type: ignore[has-type] can't determine + self._userinfo # type: ignore[has-type] # can't determine ) except AttributeError: pass @@ -1938,7 +1938,7 @@ def uses_netloc(self): # type: () -> bool return cast(bool, self._url.uses_netloc) - def replace( # type: ignore[assignment] _UNSET is private + def replace( self, scheme=_UNSET, # type: Optional[Text] host=_UNSET, # type: Optional[Text] @@ -2007,7 +2007,7 @@ def set(self, name, value=None): q[idx:idx] = [(name, value)] return self.replace(query=q) - def remove( # type: ignore[assignment] _UNSET is private + def remove( self, name, # type: Text value=_UNSET, # type: Text diff --git a/src/hyperlink/test/common.py b/src/hyperlink/test/common.py index c9696fcf..a2961f45 100644 --- a/src/hyperlink/test/common.py +++ b/src/hyperlink/test/common.py @@ -6,7 +6,7 @@ class HyperlinkTestCase(TestCase): """This type mostly exists to provide a backwards-compatible assertRaises method for Python 2.6 testing. """ - def assertRaises( # type: ignore[override] Doesn't match superclass, meh + def assertRaises( # type: ignore[override] self, expected_exception, callableObj=None, *args, **kwargs ): # type: (Type[BaseException], Optional[Callable], Any, Any) -> Any diff --git a/src/hyperlink/test/test_common.py b/src/hyperlink/test/test_common.py index ac8e9b3c..c16e12e2 100644 --- a/src/hyperlink/test/test_common.py +++ b/src/hyperlink/test/test_common.py @@ -88,7 +88,7 @@ def test_assertRaisesContextManager(self): with self.hyperlink_test.assertRaises(_ExpectedException) as cm: raise _ExpectedException - self.assertTrue( # type: ignore[misc] unreachable + self.assertTrue( # type: ignore[misc] # unreachable isinstance(cm.exception, _ExpectedException) ) diff --git a/src/hyperlink/test/test_decoded_url.py b/src/hyperlink/test/test_decoded_url.py index f00052ad..35491072 100644 --- a/src/hyperlink/test/test_decoded_url.py +++ b/src/hyperlink/test/test_decoded_url.py @@ -170,7 +170,7 @@ def test_replace_userinfo(self): durl = DecodedURL.from_text(TOTAL_URL) with self.assertRaises(ValueError): durl.replace( - userinfo=( # type: ignore[arg-type] intentional + userinfo=( # type: ignore[arg-type] 'user', 'pw', 'thiswillcauseafailure' ) ) diff --git a/src/hyperlink/test/test_socket.py b/src/hyperlink/test/test_socket.py index 8ec9a829..5f83d45b 100644 --- a/src/hyperlink/test/test_socket.py +++ b/src/hyperlink/test/test_socket.py @@ -3,7 +3,7 @@ try: from socket import inet_pton except ImportError: - inet_pton = None # type: ignore[assignment] optional + inet_pton = None # type: ignore[assignment] if not inet_pton: import socket diff --git a/src/hyperlink/test/test_url.py b/src/hyperlink/test/test_url.py index 6ee4d172..cbb23912 100644 --- a/src/hyperlink/test/test_url.py +++ b/src/hyperlink/test/test_url.py @@ -871,7 +871,7 @@ def assertRaised(raised, expectation, name): def check(param, expectation=defaultExpectation): # type: (Any, str) -> None with self.assertRaises(TypeError) as raised: - URL(**{param: Unexpected()}) # type: ignore[arg-type] ok + URL(**{param: Unexpected()}) # type: ignore[arg-type] assertRaised(raised, expectation, param) diff --git a/tox.ini b/tox.ini index af9f12c4..d8fe12e2 100644 --- a/tox.ini +++ b/tox.ini @@ -84,6 +84,8 @@ deps = mccabe==0.6.1 pep8-naming==0.9.1 pydocstyle==4.0.1 + # pin pyflakes pending a release with https://github.com/PyCQA/pyflakes/pull/455 + git+git://github.com/PyCQA/pyflakes@ffe9386#egg=pyflakes commands = flake8 {posargs:src/{env:PY_MODULE}} From 13fccdf725ec659a511e8933f4933066d9db94a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilfredo=20Sa=CC=81nchez?= Date: Tue, 19 Nov 2019 11:09:23 -0800 Subject: [PATCH 20/24] Reduce calls to cast() --- src/hyperlink/_url.py | 81 +++++++++++++++---------------------------- 1 file changed, 28 insertions(+), 53 deletions(-) diff --git a/src/hyperlink/_url.py b/src/hyperlink/_url.py index 81222374..f98b5605 100644 --- a/src/hyperlink/_url.py +++ b/src/hyperlink/_url.py @@ -1217,9 +1217,8 @@ def from_text(cls, text): if gs['query']: query = tuple( ( - cast(Tuple[str, str], qe.split(u"=", 1)) - if u'=' in qe else - (qe, None) + qe.split(u"=", 1) # type: ignore[misc] + if u'=' in qe else (qe, None) ) for qe in gs['query'].split(u"&") ) # type: QueryPairs @@ -1860,72 +1859,48 @@ def rooted(self): @property def path(self): # type: () -> Sequence[Text] - try: - return cast( - Tuple[Text, ...], - self._path # type: ignore[has-type] # can't determine - ) - except AttributeError: - pass - self._path = tuple([ - _percent_decode(p, raise_subencoding_exc=True) - for p in self._url.path - ]) + if not hasattr(self, "_path"): + self._path = tuple([ + _percent_decode(p, raise_subencoding_exc=True) + for p in self._url.path + ]) return self._path @property def query(self): # type: () -> QueryPairs - try: - return cast( - QueryPairs, - self._query # type: ignore[has-type] # can't determine - ) - except AttributeError: - pass - self._query = cast(QueryPairs, tuple( - tuple( - _percent_decode(x, raise_subencoding_exc=True) - if x is not None else None - for x in (k, v) - ) - for k, v in self._url.query - )) + if not hasattr(self, "_query"): + self._query = cast(QueryPairs, tuple( + tuple( + _percent_decode(x, raise_subencoding_exc=True) + if x is not None else None + for x in (k, v) + ) + for k, v in self._url.query + )) return self._query @property def fragment(self): # type: () -> Text - try: - return cast( - Text, - self._fragment # type: ignore[has-type] # can't determine - ) - except AttributeError: - pass - frag = self._url.fragment - self._fragment = _percent_decode(frag, raise_subencoding_exc=True) + if not hasattr(self, "_fragment"): + frag = self._url.fragment + self._fragment = _percent_decode(frag, raise_subencoding_exc=True) return self._fragment @property def userinfo(self): # type: () -> Union[Tuple[str], Tuple[str, str]] - try: - return cast( + if not hasattr(self, "_userinfo"): + self._userinfo = cast( Union[Tuple[str], Tuple[str, str]], - self._userinfo # type: ignore[has-type] # can't determine - ) - except AttributeError: - pass - self._userinfo = cast( - Union[Tuple[str], Tuple[str, str]], - tuple( tuple( - _percent_decode(p, raise_subencoding_exc=True) - for p in self._url.userinfo.split(':', 1) + tuple( + _percent_decode(p, raise_subencoding_exc=True) + for p in self._url.userinfo.split(':', 1) + ) ) ) - ) return self._userinfo @property @@ -1935,8 +1910,8 @@ def user(self): @property def uses_netloc(self): - # type: () -> bool - return cast(bool, self._url.uses_netloc) + # type: () -> Optional[bool] + return self._url.uses_netloc def replace( self, @@ -1975,7 +1950,7 @@ def replace( ' ["user", "password"], got %r' % (userinfo,)) userinfo_text = u':'.join([_encode_reserved(p) for p in userinfo]) else: - userinfo_text = cast(Text, _UNSET) + userinfo_text = _UNSET new_url = self._url.replace(scheme=scheme, host=host, path=path, From 387af6feb1902a13e368ea6366611d7d0742a084 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilfredo=20Sa=CC=81nchez?= Date: Wed, 20 Nov 2019 13:41:46 -0800 Subject: [PATCH 21/24] Add disallow_any_generics to the TODO list --- tox.ini | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tox.ini b/tox.ini index d8fe12e2..e3f4b31a 100644 --- a/tox.ini +++ b/tox.ini @@ -159,6 +159,7 @@ commands = # Global settings check_untyped_defs = True +disallow_any_generics = True disallow_incomplete_defs = True disallow_untyped_defs = True no_implicit_optional = True @@ -224,6 +225,46 @@ commands = APPVEYOR_REPO_COMMIT_AUTHOR_EMAIL APPVEYOR_REPO_COMMIT_MESSAGE_EXTENDED +## +# Documentation +## + +[testenv:docs] + +description = build documentation + +basepython = python3.8 + +deps = + Sphinx==2.2.1 + sphinx-rtd-theme==0.4.3 + +commands = + sphinx-build \ + -b html -d "{envtmpdir}/doctrees" \ + "{toxinidir}/docs" \ + "{toxworkdir}/docs/html" + + +[testenv:docs-auto] + +description = build documentation and rebuild automatically + +basepython = python3.8 + +deps = + Sphinx==2.2.1 + sphinx-rtd-theme==0.4.3 + sphinx-autobuild==0.7.1 + +commands = + sphinx-autobuild \ + -b html -d "{envtmpdir}/doctrees" \ + --host=localhost \ + "{toxinidir}/docs" \ + "{toxworkdir}/docs/html" + + ## # Packaging ## From 39bf87a2183f0860a2c318e2e114bbb1e6bbd9c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilfredo=20Sa=CC=81nchez?= Date: Mon, 2 Dec 2019 13:40:19 -0800 Subject: [PATCH 22/24] Remove duplicate section --- tox.ini | 40 ---------------------------------------- 1 file changed, 40 deletions(-) diff --git a/tox.ini b/tox.ini index 2fa2184c..5e258436 100644 --- a/tox.ini +++ b/tox.ini @@ -268,46 +268,6 @@ commands = "{toxworkdir}/docs/html" -## -# Documentation -## - -[testenv:docs] - -description = build documentation - -basepython = python3.8 - -deps = - Sphinx==2.2.1 - sphinx-rtd-theme==0.4.3 - -commands = - sphinx-build \ - -b html -d "{envtmpdir}/doctrees" \ - "{toxinidir}/docs" \ - "{toxworkdir}/docs/html" - - -[testenv:docs-auto] - -description = build documentation and rebuild automatically - -basepython = python3.8 - -deps = - Sphinx==2.2.1 - sphinx-rtd-theme==0.4.3 - sphinx-autobuild==0.7.1 - -commands = - sphinx-autobuild \ - -b html -d "{envtmpdir}/doctrees" \ - --host=localhost \ - "{toxinidir}/docs" \ - "{toxworkdir}/docs/html" - - ## # Packaging ## From 7e5d944a2a13566af85acaf5add6c93046997821 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilfredo=20Sa=CC=81nchez?= Date: Mon, 2 Dec 2019 13:57:54 -0800 Subject: [PATCH 23/24] Add types back to doc strings --- src/hyperlink/_url.py | 123 ++++++++++++++++++++++-------------------- 1 file changed, 66 insertions(+), 57 deletions(-) diff --git a/src/hyperlink/_url.py b/src/hyperlink/_url.py index cfa8b126..0d8896dd 100644 --- a/src/hyperlink/_url.py +++ b/src/hyperlink/_url.py @@ -369,12 +369,13 @@ def register_scheme(text, uses_netloc=True, default_port=None): `file an issue`_! Args: - text: Text representing the scheme. + text (Text): A string representation of the scheme. (the 'http' in 'http://hatnote.com') - uses_netloc: Does the scheme support specifying a network host? + uses_netloc (bool): Does the scheme support specifying a network host? For instance, "http" does, "mailto" does not. Defaults to True. - default_port: The default port, if any, for netloc-using schemes. + default_port (Optional[int]): The default port, if any, for + netloc-using schemes. .. _file an issue: https://github.com/mahmoud/hyperlink/issues """ @@ -797,24 +798,24 @@ class URL(object): constructor arguments is below. Args: - scheme: The text name of the scheme. - host: The host portion of the network location - port: The port part of the network location. If + scheme (Optional[Text]): The text name of the scheme. + host (Optional[Text]): The host portion of the network location + port (Optional[int]): The port part of the network location. If ``None`` or no port is passed, the port will default to the default port of the scheme, if it is known. See the ``SCHEME_PORT_MAP`` and :func:`register_default_port` for more info. - path: A tuple of strings representing the + path (Iterable[Text]): A tuple of strings representing the slash-separated parts of the path. - query: The query parameters, as a dictionary or - as an iterable of key-value pairs. - fragment: The fragment part of the URL. - rooted: A rooted URL is one which indicates an absolute path. + query (Sequence[Tuple[Text, Optional[Text]]]): The query parameters, as + a dictionary or as an iterable of key-value pairs. + fragment (Text): The fragment part of the URL. + rooted (bool): A rooted URL is one which indicates an absolute path. This is True on any URL that includes a host, or any relative URL that starts with a slash. - userinfo: The username or colon-separated + userinfo (Text): The username or colon-separated username:password pair. - uses_netloc: Indicates whether two slashes appear + uses_netloc (bool): Indicates whether two slashes appear between the scheme and the host (``http://eg.com`` vs ``mailto:e@g.com``). Set automatically based on scheme. @@ -1027,12 +1028,13 @@ def authority(self, with_password=False, **kw): u'user:pass@localhost:8080' Args: - with_password: Whether the return value of this method include the - password in the URL, if it is set. Defaults to False. + with_password (bool): Whether the return value of this method + include the password in the URL, if it is set. + Defaults to False. Returns: - The authority (network location and user information) portion of the - URL. + The authority (network location and user information) portion of + the URL. """ # first, a bit of twisted compat with_password = kw.pop('includeSecrets', with_password) @@ -1119,18 +1121,22 @@ def replace( the value on the current URL. Args: - scheme: The text name of the scheme. - host: The host portion of the network location - path: A tuple of strings representing the slash-separated parts of - the path. - query: The query parameters, as a dictionary or as an iterable of - key-value pairs. - fragment: The fragment part of the URL. - port: The port part of the network location. - rooted: Whether or not the path begins with a slash. - userinfo: The username or colon-separated username:password pair. - uses_netloc: Indicates whether two slashes appear between the - scheme and the host (``http://eg.com`` vs ``mailto:e@g.com``) + scheme (Optional[Text]): The text name of the scheme. + host (Optional[Text]): The host portion of the network location. + path (Iterable[Text]): A tuple of strings representing the + slash-separated parts of the path. + query (Sequence[Tuple[Text, Optional[Text]]]): The query + parameters, as a dictionary or as an iterable of key-value + pairs. + fragment (Text): The fragment part of the URL. + port (Optional[int]): The port part of the network location. + rooted (Optional[bool]): Whether or not the path begins with a + slash. + userinfo (Text): The username or colon-separated username:password + pair. + uses_netloc (bool): Indicates whether two slashes appear between + the scheme and the host + (``http://eg.com`` vs ``mailto:e@g.com``) Returns: A copy of the current :class:`URL`, with new values for @@ -1166,7 +1172,7 @@ def from_text(cls, text): sure to decode those bytestrings. Args: - text: A valid URL string. + text (Text): A valid URL string. Returns: The structured object version of the parsed string. @@ -1257,14 +1263,15 @@ def normalize(self, scheme=True, host=True, path=True, query=True, name. Args: - scheme: Convert the scheme to lowercase - host: Convert the host to lowercase - path: Normalize the path (see above for details) - query: Normalize the query string - fragment: Normalize the fragment - userinfo: Normalize the userinfo - percents: Encode isolated percent signs for any percent-encoded - fields which are being normalized (defaults to True). + scheme (bool): Convert the scheme to lowercase + host (bool): Convert the host to lowercase + path (bool): Normalize the path (see above for details) + query (bool): Normalize the query string + fragment (bool): Normalize the fragment + userinfo (bool): Normalize the userinfo + percents (bool): Encode isolated percent signs for any + percent-encoded fields which are being normalized + (defaults to True). >>> url = URL.from_text(u'Http://example.COM/a/../b/./c%2f?%61%') >>> print(url.normalize().to_text()) @@ -1320,7 +1327,7 @@ def child(self, *segments): u'http://localhost/a/b/c/d?x=y' Args: - segments: Additional parts to be joined and added to + segments (Text): Additional parts to be joined and added to the path, like :func:`os.path.join`. Special characters in segments will be percent encoded. @@ -1345,7 +1352,7 @@ def sibling(self, segment): sibling of this URL path. Args: - segment: A single path segment. + segment (Text): A single path segment. Returns: A copy of the current URL with the last path segment @@ -1370,7 +1377,7 @@ def click(self, href=u''): >>> url.click(u'../d/./e').to_text() u'http://localhost/a/b/d/e' - Args: + Args (Text): href: A string representing a clicked URL. Return: @@ -1510,7 +1517,7 @@ def to_text(self, with_password=False): unless the data after the colon is the empty string (indicating no password)." - Args: + Args (bool): with_password: Whether or not to include the password in the URL text. Defaults to False. @@ -1626,9 +1633,9 @@ def add(self, name, value=None): URL.from_text(u'https://example.com/?x=y&x=z') Args: - name: The name of the query parameter to add. + name (Text): The name of the query parameter to add. The part before the ``=``. - value: The value of the query parameter to add. + value (Optional[Text]): The value of the query parameter to add. The part after the ``=``. Defaults to ``None``, meaning no value. @@ -1649,9 +1656,9 @@ def set(self, name, value=None): URL.from_text(u'https://example.com/?x=z') Args: - name: The name of the query parameter to set. + name (Text): The name of the query parameter to set. The part before the ``=``. - value: The value of the query parameter to set. + value (Optional[Text]): The value of the query parameter to set. The part after the ``=``. Defaults to ``None``, meaning no value. @@ -1679,7 +1686,7 @@ def get(self, name): list is always returned, and this method raises no exceptions. Args: - name: The name of the query parameter to get. + name (Text): The name of the query parameter to get. Returns: A list of all the values associated with the key, in string form. @@ -1699,11 +1706,12 @@ def remove( parameter is not already set. Args: - name: The name of the query parameter to remove. - value: Optional value to additionally filter on. + name (Text): The name of the query parameter to remove. + value (Text): Optional value to additionally filter on. Setting this removes query parameters which match both name and value. - limit: Optional maximum number of parameters to remove. + limit (Optional[int]): Optional maximum number of parameters to + remove. Returns: URL: A new :class:`URL` instance with the parameter removed. @@ -1750,9 +1758,9 @@ class DecodedURL(object): special characters encoded with codecs other than UTF-8. Args: - url: A :class:`URL` object to wrap. - lazy: Set to True to avoid pre-decode all parts of the URL to check for - validity. Defaults to False. + url (URL): A :class:`URL` object to wrap. + lazy (bool): Set to True to avoid pre-decode all parts of the URL to + check for validity. Defaults to False. """ def __init__(self, url, lazy=False): @@ -1771,8 +1779,8 @@ def from_text(cls, text, lazy=False): Make a `DecodedURL` instance from any text string containing a URL. Args: - text: Text containing the URL - lazy: Whether to pre-decode all parts of the URL to check for + text (Text): Text containing the URL + lazy (bool): Whether to pre-decode all parts of the URL to check for validity. Defaults to True. """ _url = URL.from_text(text) @@ -2084,15 +2092,16 @@ def parse(url, decoded=True, lazy=False): """Automatically turn text into a structured URL object. Args: + url (Text): A string representation of a URL. - decoded: Whether or not to return a :class:`DecodedURL`, + decoded (bool): Whether or not to return a :class:`DecodedURL`, which automatically handles all encoding/decoding/quoting/unquoting for all the various accessors of parts of the URL, or an :class:`EncodedURL`, which has the same API, but requires handling of special characters for different parts of the URL. - lazy: In the case of `decoded=True`, this controls + lazy (bool): In the case of `decoded=True`, this controls whether the URL is decoded immediately or as accessed. The default, `lazy=False`, checks all encoded parts of the URL for decodability. From 9661999935acef38e67c90cc97eaf0f6bbd43f64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilfredo=20Sa=CC=81nchez?= Date: Mon, 2 Dec 2019 14:14:00 -0800 Subject: [PATCH 24/24] Add return ntypes to doc strings --- src/hyperlink/_url.py | 47 ++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/src/hyperlink/_url.py b/src/hyperlink/_url.py index 0d8896dd..15f6cc9d 100644 --- a/src/hyperlink/_url.py +++ b/src/hyperlink/_url.py @@ -371,9 +371,9 @@ def register_scheme(text, uses_netloc=True, default_port=None): Args: text (Text): A string representation of the scheme. (the 'http' in 'http://hatnote.com') - uses_netloc (bool): Does the scheme support specifying a network host? - For instance, "http" does, "mailto" does not. - Defaults to True. + uses_netloc (bool): Does the scheme support specifying a + network host? For instance, "http" does, "mailto" does + not. Defaults to True. default_port (Optional[int]): The default port, if any, for netloc-using schemes. @@ -587,7 +587,7 @@ def _percent_decode( underlying the percent-decoding should be raised. Returns: - The percent-decoded version of *text*, decoded by *subencoding*. + Text: The percent-decoded version of *text*, decoded by *subencoding*. """ try: quoted_bytes = text.encode(subencoding) @@ -736,7 +736,7 @@ def parse_host(host): Will raise :class:`URLParseError` on invalid IPv6 constants. Returns: - tuple: family (socket constant or None), host (string) + family (socket constant or None), host (string) >>> import socket >>> parse_host('googlewebsite.com') == (None, 'googlewebsite.com') @@ -808,7 +808,7 @@ class URL(object): path (Iterable[Text]): A tuple of strings representing the slash-separated parts of the path. query (Sequence[Tuple[Text, Optional[Text]]]): The query parameters, as - a dictionary or as an iterable of key-value pairs. + a dictionary or as an sequence of key-value pairs. fragment (Text): The fragment part of the URL. rooted (bool): A rooted URL is one which indicates an absolute path. This is True on any URL that includes a host, or any relative URL @@ -1028,13 +1028,13 @@ def authority(self, with_password=False, **kw): u'user:pass@localhost:8080' Args: - with_password (bool): Whether the return value of this method - include the password in the URL, if it is set. - Defaults to False. + with_password (bool): Whether the return value of this + method include the password in the URL, if it is + set. Defaults to False. Returns: - The authority (network location and user information) portion of - the URL. + Text: The authority (network location and user information) portion + of the URL. """ # first, a bit of twisted compat with_password = kw.pop('includeSecrets', with_password) @@ -1126,7 +1126,7 @@ def replace( path (Iterable[Text]): A tuple of strings representing the slash-separated parts of the path. query (Sequence[Tuple[Text, Optional[Text]]]): The query - parameters, as a dictionary or as an iterable of key-value + parameters, as a dictionary or as an sequence of key-value pairs. fragment (Text): The fragment part of the URL. port (Optional[int]): The port part of the network location. @@ -1139,7 +1139,7 @@ def replace( (``http://eg.com`` vs ``mailto:e@g.com``) Returns: - A copy of the current :class:`URL`, with new values for + URL: A copy of the current :class:`URL`, with new values for parameters passed. """ return self.__class__( @@ -1175,7 +1175,7 @@ def from_text(cls, text): text (Text): A valid URL string. Returns: - The structured object version of the parsed string. + URL: The structured object version of the parsed string. .. note:: @@ -1332,7 +1332,7 @@ def child(self, *segments): in segments will be percent encoded. Returns: - A copy of the current URL with the extra path segments. + URL: A copy of the current URL with the extra path segments. """ if not segments: return self @@ -1355,7 +1355,7 @@ def sibling(self, segment): segment (Text): A single path segment. Returns: - A copy of the current URL with the last path segment + URL: A copy of the current URL with the last path segment replaced by *segment*. Special characters such as ``/?#`` will be percent encoded. """ @@ -1437,8 +1437,8 @@ def to_uri(self): Returns: URL: A new instance with its path segments, query parameters, and - hostname encoded, so that they are all in the standard - US-ASCII range. + hostname encoded, so that they are all in the standard + US-ASCII range. """ new_userinfo = u':'.join([_encode_userinfo_part(p) for p in self.userinfo.split(':', 1)]) @@ -1482,7 +1482,7 @@ def to_iri(self): Returns: URL: A new instance with its path segments, query parameters, and - hostname decoded for display purposes. + hostname decoded for display purposes. """ # noqa: E501 new_userinfo = u':'.join([ _decode_userinfo_part(p) for p in self.userinfo.split(':', 1) @@ -1522,7 +1522,7 @@ def to_text(self, with_password=False): text. Defaults to False. Returns: - The serialized textual representation of this URL, such as + Text: The serialized textual representation of this URL, such as ``u"http://example.com/some/path?some=query"``. The natural counterpart to :class:`URL.from_text()`. @@ -1640,7 +1640,7 @@ def add(self, name, value=None): value. Returns: - A new :class:`URL` instance with the parameter added. + URL: A new :class:`URL` instance with the parameter added. """ return self.replace(query=self.query + ((name, value),)) @@ -1663,7 +1663,7 @@ def set(self, name, value=None): value. Returns: - A new :class:`URL` instance with the parameter set. + URL: A new :class:`URL` instance with the parameter set. """ # Preserve the original position of the query key in the list q = [(k, v) for (k, v) in self.query if k != name] @@ -1689,7 +1689,8 @@ def get(self, name): name (Text): The name of the query parameter to get. Returns: - A list of all the values associated with the key, in string form. + List[Optional[Text]]: A list of all the values associated with the + key, in string form. """ return [value for (key, value) in self.query if name == key]