[Python] SSLError in azureml when creating Pipeline with Python SDK v1

Everything was fine one day, but next day I started having problems when using azureml Python sdk v1 to send pipeline jobs from my local machine to my project Azure Machine Learning Studio workspace. I started having SSL errors. I created a minimal pipeline job to reproduce the error. This code just creates a Pipeline Job with one only step, with a token script called "script.py" which does nothing.

Is it possible to just ignore the SSL verify somewhere - or maybe to fix it somehow without ignoring it?

Libs versions:

azureml-core 1.55.0
azureml-pipeline 1.55.0

Here is my minimal code:

from azureml.core import Workspace, Environment, Experiment
from azureml.pipeline.steps import PythonScriptStep
from azureml.core.runconfig import RunConfiguration
from azureml.pipeline.core import Pipeline

subscription_id = <SUBSCRIPTION_ID>
resource_group = <RESOURCE_GROUP>
workspace_name = <WORKSPACE_NAME>

ws = Workspace(subscription_id, resource_group, workspace_name)

compute_target = 'my_compute_target'
runconfig = RunConfiguration()
runconfig.target = compute_target
curated_environment = Environment.get(workspace=ws, name='myenv')
runconfig.environment = curated_environment

step = PythonScriptStep(
script_name='script.py',
source_directory='.',
runconfig=runconfig,
allow_reuse=False,
)

pipeline = Pipeline(workspace=ws, steps=[step])
pipeline_run = Experiment(ws, 'token_experiment').submit(pipeline)


Here is the error message:

Traceback (most recent call last):
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\urllib3\connectionpool.py", line 715, in urlopen
httplib_response = self._make_request(
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\urllib3\connectionpool.py", line 404, in _make_request
self._validate_conn(conn)
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\urllib3\connectionpool.py", line 1058, in _validate_conn
conn.connect()
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\urllib3\connection.py", line 419, in connect
self.sock = ssl_wrap_socket(
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\urllib3\util\ssl_.py", line 449, in ssl_wrap_socket
ssl_sock = _ssl_wrap_socket_impl(
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\urllib3\util\ssl_.py", line 493, in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\ssl.py", line 500, in wrap_socket
return self.sslsocket_class._create(
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\ssl.py", line 1073, in _create
self.do_handshake()
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\ssl.py", line 1342, in do_handshake
self._sslobj.do_handshake()
ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1131)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\requests\adapters.py", line 486, in send
resp = conn.urlopen(
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\urllib3\connectionpool.py", line 799, in urlopen
retries = retries.increment(
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\urllib3\util\retry.py", line 592, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='brazilsouth.api.azureml.ms', port=443): Max retries exceeded with url: /rp/workspaces/subscriptions/<subscription_id>/resourceGroups/<resource_group>/providers/Microsoft.MachineLearningServices/workspaces/<workspace_name>/computes?$skipToken=W3sidG9rZW4iOiIrUklEOn5nNTExQU13bVlTSmlLR1lBQUFBQUFBPT0jUlQ6MSNUUkM6MTAjSVNWOjIjSUVPOjY1NTUxI1FDRjo4I0ZQQzpBZ0tZQVo4Qm1BRUVBR0tvUEkrZkFRUUF4YU9FbXc9PSIsInJhbmdlIjp7Im1pbiI6IiIsIm1heCI6IkZGIn19XQ&api-version=2020-02-02 (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1131)')))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "token_pipeline.py", line 35, in <module>
pipeline = Pipeline(workspace=ws, steps=[step])
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\azureml\core\_experiment_method.py", line 104, in wrapper
return init_func(self, *args, **kwargs)
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\azureml\pipeline\core\pipeline.py", line 180, in __init__
self._graph = self._graph_builder.build(self._name, steps, finalize=False)
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\azureml\pipeline\core\builder.py", line 1497, in build
graph = self.construct(name, steps)
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\azureml\pipeline\core\builder.py", line 1519, in construct
self.process_collection(steps)
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\azureml\pipeline\core\builder.py", line 1555, in process_collection
builder.process_collection(collection)
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\azureml\pipeline\core\builder.py", line 1845, in process_collection
self._base_builder.process_collection(item)
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\azureml\pipeline\core\builder.py", line 1549, in process_collection
return self.process_step(collection)
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\azureml\pipeline\core\builder.py", line 1593, in process_step
node = step.create_node(self._graph, self._default_datastore, self._context)
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\azureml\pipeline\steps\python_script_step.py", line 243, in create_node
return super(PythonScriptStep, self).create_node(
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\azureml\pipeline\core\_python_script_step_base.py", line 150, in create_node
compute_target_name, compute_target_type, compute_target_object = _PythonScriptStepBase.\
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\azureml\pipeline\core\_python_script_step_base.py", line 214, in _extract_compute_target_params
compute_target_object = context.get_target(compute_target_name)
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\azureml\pipeline\core\_graph_context.py", line 60, in get_target
self._targets = self._workspace.compute_targets
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\azureml\core\workspace.py", line 1069, in compute_targets
compute_target.name: compute_target for compute_target in ComputeTarget.list(self)}
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\azureml\core\compute\compute.py", line 519, in list
paginated_results = get_paginated_compute_results(result_list, headers)
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\azureml\_compute\_util.py", line 46, in get_paginated_compute_results
resp = ClientBase._execute_func(get_requests_session().get, next_link, headers=headers)
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\azureml\_restclient\clientbase.py", line 375, in _execute_func
return cls._execute_func_internal(
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\azureml\_restclient\clientbase.py", line 368, in _execute_func_internal
left_retry = cls._handle_retry(back_off, left_retry, total_retry, error, logger, func)
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\azureml\_restclient\clientbase.py", line 400, in _handle_retry
raise error
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\azureml\_restclient\clientbase.py", line 359, in _execute_func_internal
response = func(*args, **kwargs)
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\requests\sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\requests\sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\requests\sessions.py", line 703, in send
r = adapter.send(request, **kwargs)
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\requests\adapters.py", line 517, in send
raise SSLError(e, request=request)
requests.exceptions.SSLError: HTTPSConnectionPool(host='brazilsouth.api.azureml.ms', port=443): Max retries exceeded with url: /rp/workspaces/subscriptions/<subscription_id>/resourceGroups/<resource_group>/providers/Microsoft.MachineLearningServices/workspaces/<workspace_name>/computes?$skipToken=W3sidG9rZW4iOiIrUklEOn5nNTExQU13bVlTSmlLR1lBQUFBQUFBPT0jUlQ6MSNUUkM6MTAjSVNWOjIjSUVPOjY1NTUxI1FDRjo4I0ZQQzpBZ0tZQVo4Qm1BRUVBR0tvUEkrZkFRUUF4YU9FbXc9PSIsInJhbmdlIjp7Im1pbiI6IiIsIm1heCI6IkZGIn19XQ&api-version=2020-02-02 (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1131)')))

(myenv) C:\Users\myuser\token_pipeline>python token_pipeline.py
Traceback (most recent call last):
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\urllib3\connectionpool.py", line 715, in urlopen
httplib_response = self._make_request(
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\urllib3\connectionpool.py", line 404, in _make_request
self._validate_conn(conn)
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\urllib3\connectionpool.py", line 1058, in _validate_conn
conn.connect()
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\urllib3\connection.py", line 419, in connect
self.sock = ssl_wrap_socket(
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\urllib3\util\ssl_.py", line 449, in ssl_wrap_socket
ssl_sock = _ssl_wrap_socket_impl(
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\urllib3\util\ssl_.py", line 493, in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\ssl.py", line 500, in wrap_socket
return self.sslsocket_class._create(
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\ssl.py", line 1073, in _create
self.do_handshake()
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\ssl.py", line 1342, in do_handshake
self._sslobj.do_handshake()
ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1131)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\requests\adapters.py", line 486, in send
resp = conn.urlopen(
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\urllib3\connectionpool.py", line 799, in urlopen
retries = retries.increment(
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\urllib3\util\retry.py", line 592, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='brazilsouth.experiments.azureml.net', port=443): Max retries exceeded with url: /environment/v1.0/subscriptions/b5d6fe63-452f-433a-9bb3-53f751969cc9/resourceGroups/rg-sbx-pocd-brazilsouth/providers/Microsoft.MachineLearningServices/workspaces/mlw-sbx-pocd-brazilsouth/environments/myenv (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1131)')))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "token_pipeline.py", line 25, in <module>
curated_environment = Environment.get(workspace=ws, name='myenv')
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\azureml\core\environment.py", line 1158, in get
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\azureml\_restclient\environment_client.py", line 36, in _get_environment_definition
response = requests.get(environment_url, headers=headers)
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\requests\api.py", line 73, in get
return request("get", url, params=params, **kwargs)
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\requests\api.py", line 59, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\requests\sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\requests\sessions.py", line 703, in send
r = adapter.send(request, **kwargs)
File "C:\Users\myuser\AppData\Local\miniforge3\envs\myenv\lib\site-packages\requests\adapters.py", line 517, in send
raise SSLError(e, request=request)
requests.exceptions.SSLError: HTTPSConnectionPool(host='brazilsouth.experiments.azureml.net', port=443): Max retries exceeded with url: /environment/v1.0/subscriptions/b5d6fe63-452f-433a-9bb3-53f751969cc9/resourceGroups/rg-sbx-pocd-brazilsouth/providers/Microsoft.MachineLearningServices/workspaces/mlw-sbx-pocd-brazilsouth/environments/myenv (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1131)')))

Continue reading...