[Python] Error PySparkRuntimeError: [JAVA_GATEWAY_EXITED] in script to upload to redshift

I need help with an error I get when running a local pyspark notebook in VSC with miniforge. I have installed:

• VSC
• Java 8 + Java SDK11
• Downloaded into c:/spark spark 3.4.4, and created folder c:/hadoop/bin where I added a winutil and hadoop dll file
• Python 3.11.0
• Latest version of miniforge
• minforge/conda kernel: Python 3.10.17

I have code to start the spark session, and add a config option to run a jar for redshift. I currently have working code without that:

import os
import sys
import glob
import json
import urllib3
import requests
import findspark
import pandas as pd
from pyspark.sql.types import *
from pyspark.sql import DataFrame
from pyspark.sql.functions import *
from pyspark.sql import SparkSession
from datetime import datetime, timedelta
from functools import reduce as py_reduce
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

findspark.init()

os.environ["JAVA_HOME"] = "C:/Program Files/Java/jdk-11"
os.environ["SPARK_HOME"] = "C:/spark"
os.environ["HADOOP_HOME"] = "C:/hadoop"

spark = SparkSession.builder.master("local[*]").appName("SAP").getOrCreate()


And it works, since Ive been able to create a spark session like that. However, when I modify the last line into:

spark = SparkSession.builder \
.appName("Redshift Connection with PySpark") \
.config("spark.jars", "C:/vsc_projects/drivers/redshift-jdbc42-2.1.0.22.jar") \
.getOrCreate()


It does not work, giving me the following error:

PySparkRuntimeError Traceback (most recent call last)
Cell In[2], line 10
4 os.environ["HADOOP_HOME"] = "C:/hadoop"
6 # crear sess spark con redshift
7 spark = SparkSession.builder \
8 .appName("Redshift Connection with PySpark") \
9 .config("spark.jars", "C:/vsc_projects/drivers/redshift-jdbc42-2.1.0.22.jar") \
---> 10 .getOrCreate()

File c:\Users\Felipe\.conda\envs\sap\lib\site-packages\pyspark\sql\session.py:497, in SparkSession.Builder.getOrCreate(self)
495 sparkConf.set(key, value)
496 # This SparkContext may be an existing one.
--> 497 sc = SparkContext.getOrCreate(sparkConf)
498 # Do not update `SparkConf` for existing `SparkContext`, as it's shared
499 # by all sessions.
500 session = SparkSession(sc, options=self._options)

File c:\Users\Felipe\.conda\envs\sap\lib\site-packages\pyspark\context.py:515, in SparkContext.getOrCreate(cls, conf)
513 with SparkContext._lock:
514 if SparkContext._active_spark_context is None:
--> 515 SparkContext(conf=conf or SparkConf())
516 assert SparkContext._active_spark_context is not None
517 return SparkContext._active_spark_context

File c:\Users\Felipe\.conda\envs\sap\lib\site-packages\pyspark\context.py:201, in SparkContext.__init__(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, gateway, jsc, profiler_cls, udf_profiler_cls, memory_profiler_cls)
195 if gateway is not None and gateway.gateway_parameters.auth_token is None:
196 raise ValueError(
197 "You are trying to pass an insecure Py4j gateway to Spark. This"
198 " is not allowed as it is a security risk."
199 )
--> 201 SparkContext._ensure_initialized(self, gateway=gateway, conf=conf)
202 try:
203 self._do_init(
204 master,
205 appName,
(...)
215 memory_profiler_cls,
216 )

File c:\Users\Felipe\.conda\envs\sap\lib\site-packages\pyspark\context.py:436, in SparkContext._ensure_initialized(cls, instance, gateway, conf)
434 with SparkContext._lock:
435 if not SparkContext._gateway:
--> 436 SparkContext._gateway = gateway or launch_gateway(conf)
437 SparkContext._jvm = SparkContext._gateway.jvm
439 if instance:

File c:\Users\Felipe\.conda\envs\sap\lib\site-packages\pyspark\java_gateway.py:107, in launch_gateway(conf, popen_kwargs)
104 time.sleep(0.1)
106 if not os.path.isfile(conn_info_file):
--> 107 raise PySparkRuntimeError(
108 error_class="JAVA_GATEWAY_EXITED",
109 message_parameters={},
110 )
112 with open(conn_info_file, "rb") as info:
113 gateway_port = read_int(info)

PySparkRuntimeError: [JAVA_GATEWAY_EXITED] Java gateway process exited before sending its port number.


What can I do to fix this?

Continue reading...