[Python] Why are there different results from returned from this test in python3.8 and python...

I have a test that validates urls that looks like this

def test_rfc2732_urls(self):
def valid(v):
self.assertTrue(validate_c_db(v))
self.assertFalse(validate_c_db("https://" + v))


trying to validate url which will have IPV6 in it

not_allowed("[::FFFF:192.168.1.10]:80/index.html")


The validate_c_db function looks like this

def validate_c_db(value: str) -> bool:
if not value or not isinstance(value, str):
return False
return bool(CdbUrl(value).process())


The url is then validated against uri, scheme, path, port, query etc...

class Uri:
def __init__(self, value: str):
self.value = value
self.tokens = RE_URI.match(value)
self.scheme = self.tokens.group(1)
self.authority = self.tokens.group(2)
self.path = self.tokens.group(3)
self.query = self.tokens.group(4)
self.fragment = self.tokens.group(5)

def process(self) -> Optional[str]:
"""
Process given uri value and return a reconstructed string
If string is not returned - this means value is invalid
"""
if RE_ILLEGAL_CHARACTERS.search(self.value):
return
if not self.scheme:
return

# If has authority then path should be empty or start with /
if self.authority:
if self.path and not self.path.startswith(TOK_SLASH):
return
# if not authority then no //
elif self.path.startswith(TOK_DOUBLE_SLASH):
return

if not (RE_SCHEME.match(self.scheme.lower())):
return

return self._re_create()

def _re_create(self):
tokens = []
tokens += [self.scheme, TOK_COLON]
if self.authority:
tokens += [TOK_DOUBLE_SLASH, self.authority]
tokens.append(self.path)
if self.query:
tokens += [TOK_QUERY, self.query]
if self.fragment:
tokens += [TOK_FRAGMENT, self.fragment]
return "".join(tokens)


class HttpsUri(Uri):
def __init__(self, value):
super().__init__(value)
self.port = None # type: Optional[str]
self.user_info = None # type: Optional[str]
self.host = None # type: Optional[str]
self.ipv6 = False # type: bool
self.ipv4 = False # type: bool

def process(self) -> Optional[str]:
ignore_port = False

if not super().process():
return

if self.scheme != TOK_HTTPS:
return

if not self.authority:
return

authority_token = self.authority
at_sign_location = authority_token.rfind(TOK_AT)
if at_sign_location == 0:
return
if at_sign_location > 0:
self.user_info = authority_token[:at_sign_location]
authority_token = authority_token[at_sign_location + 1 :]

if authority_token.startswith(TOK_IPV6_START):
if RE_IPV6_END.search(authority_token):
ignore_port = True
elif RE_IPV6_PORT_END.search(authority_token):
ignore_port = True
port_end_match = RE_PORT_END.search(authority_token)
if not port_end_match:
return
self.port = port_end_match.group(1)
else:
return

if not ignore_port and RE_COLON_END.search(authority_token):
port = RE_PORT_END.search(authority_token)
if not port:
return
self.port = port.group(1)

self.host = authority_token
if self.port:
port_pos = len(authority_token) - len(self.port) - 1
self.host = authority_token[ort_pos]

if (
self.host.startswith(TOK_DASH)
or RE_DASH_END.search(self.host)
or RE_DASH_END.search(authority_token)
):
return

try:
self._check_host()
except ValueError:
return

try:
self._check_port()
except ValueError:
return

return super()._re_create()

def _check_host(self):
if not self.host:
raise ValueError("Host is missing")
if self.host.startswith("["):
self.ipv6 = True
check_ipv6(self.host[1:-1])
if RE_IPV4_ROUGH.match(self.host):
self.ipv4 = True
check_ipv4(self.host)

def _check_port(self):
if not self.port:
return
int_port = int(self.port)
if int_port < 0 or int_port > 65535:
raise ValueError("Invalid port")


class CdbUrl(HttpsUri):
def __init__(self, value: str):
super().__init__("https://" + value)
self.original = value

def process(self) -> Optional[str]:
tokens = RE_SCHEME_START.match(self.original)
if tokens and tokens.group(0):
return

if TOK_SPACE in self.original or RE_TEL_OR_NOT_DOMAIN.match(self.original):
return

return super().process() and not self.user_info

def _check_host(self):
if not self.host:
raise ValueError("Host is missing")
if self.host.startswith("["):
self.ipv6 = True
check_not_ours(check_ipv6(self.host[1:-1]))
if RE_IPV4_ROUGH.match(self.host):
self.ipv4 = True
check_not_ours(check_ipv4(self.host))
if RE_BAD_HOSTS.match(self.host):
raise ValueError("Bad host detected")
if not self.ipv6 and not self.ipv4:
if not validate_hostname(self.host, CAT_DB_HOST_LABEL_COUNT):
raise ValueError("Host format is not valid")


The failure occurs specifically for IPV6 urls, and the regex's i am matching against for IPV6 urls are

RE_IPV6_END = re.compile(r"[\]]$")
RE_IPV6_PORT_END = re.compile(r"[\]]:\d+?")
TOK_IPV6_START = r"["


When i test by passing this url specifically [::FFFF:192.168.1.10]:80/index.html the test passes on python3.8 and fails on 3.11 or rather 3.11 returns True

def test_cat_db():
url = "[::FFFF:192.168.1.10]:80/index.html"
cdb_instance = CdbUrl(url)
result = cdb_instance.process()
print(f"Result: {result}")


The above function will print Result: True in python3.11 but Result: None in python3.8 i cannot figure out why, i though it might be UNICODE issue so i tested by adding RE_IPV6_END = re.compile(r"[\]]$", re.UNICODE) but same thing happens, according to rfc2732 * It defines a syntax for IPv6 addresses and allows the use of "[" and "]" within a URI explicitly for this reserved purpose*

(EDIT):Here are all the additional regex's used to match urls

RE_URI = re.compile(r"(?[^:/?#]+)?(?://([^/?#]*))?([^?#]*)(?:\?([^#]*))?(?:#(.*))?")
RE_ILLEGAL_CHARACTERS = re.compile(
r"[^a-z0-9:/?#\[\]@!$&'()*+,;=.\-_~%]", re.IGNORECASE
)
RE_SCHEME = re.compile(r"^[a-z][a-z0-9+\-.]*$")
RE_IPV6_END = re.compile(r"[\]]$", re.RAW)
RE_IPV6_PORT_END = re.compile(r"[\]]:\d+?")
RE_PORT_END = re.compile(r"\d+)$")
RE_COLON_END = re.compile(r".*)$")
RE_DASH_END = re.compile(r"-$")
# Only alphanum wih -, +, can be a telephone number or non domain like localhost
RE_TEL_OR_NOT_DOMAIN = re.compile(r"^[a-z0-9\-+]+$")
RE_SCHEME_START = re.compile(r"^([a-z][a-z0-9+\-.]*://)(.*)$", re.IGNORECASE)
RE_IPV4_ROUGH = re.compile(r"(\d+)([.](\d+)){3}")
# This checks if hosts match `co.uk`, `co.in` etc.
RE_BAD_HOSTS = re.compile(^co\\.(uk|in|nz|il|id|cr|jp|kr|cp)$, re.IGNORECASE)
RE_TLD_NO_NUMBER_END = re.compile(r"[0-9]+$")
RE_HOST_SECTION = re.compile(r"(?!-)[a-z0-9-]{1,63}(?<!-)$")

TOK_HTTPS_SCHEME = r"https://"
TOK_HTTP_SCHEME = r"http://"
TOK_HTTPS = r"https"
TOK_SLASH = r"/"
TOK_DOUBLE_SLASH = r"//"
TOK_SPACE = r" "
TOK_DASH = r"-"
TOK_COLON = r":"
TOK_QUERY = r"?"
TOK_FRAGMENT = r"#"
TOK_IPV6_START = r"["
TOK_AT = "@"


And a few additional functions

def validate_hostname(hostname: str, min_labels=1) -> bool:
if not hostname or not isinstance(hostname, str):
return False

if hostname[-1] == ".":
hostname = hostname[:-1]

if len(hostname) > HOST_MAX_LEN:
return False

labels = hostname.split(".")

if RE_TLD_NO_NUMBER_END.match(labels[-1]):
return False

if len(labels) < min_labels:
return False

return all(RE_HOST_SECTION.match(label) for label in labels)
def check_ipv4(ip: str):
try:
ipv4 = ipaddress.IPv4Address(ip)
except ipaddress.AddressValueError as ex:
raise ValueError("Invalid IP address {!r}".format(ip)) from ex

return ipv4


def check_ipv6(ip: str):
try:
ipv6 = ipaddress.IPv6Address(ip)
except ipaddress.AddressValueError as ex:
raise ValueError("Invalid IP address {!r}".format(ip)) from ex

return ipv6

Continue reading...