[Python] Apache beam write to parquet Python

Below is my PostgreSQL table:

CREATE TABLE customer_data (
id SERIAL PRIMARY KEY,
name VARCHAR(100) NOT NULL,
email VARCHAR(100),
age INT,
registration_date DATE,
last_login TIMESTAMP,
account_balance DECIMAL(10, 2),
is_active BOOLEAN,
preferences JSONB,
notes TEXT,
rating FLOAT,
binary_data BYTEA
);


I am trying to read this table using Apache Beam and it fails for the boolean type whatever I do convert the boolean to string in PyArrow schema, explicitly cast it.

Below is the sample code which I am using:

schema = pa.schema([
pa.field("id", pa.int64(), nullable=False),
pa.field("name", pa.string(), nullable=False),
pa.field("email", pa.string(), nullable=True),
pa.field("age", pa.int32(), nullable=True),
pa.field("registration_date", pa.date32(), nullable=True),
pa.field("last_login", pa.timestamp('us'), nullable=True),
pa.field("account_balance", pa.decimal128(10, 2), nullable=True),
pa.field("is_active", pa.bool_(), nullable=True),
pa.field("preferences", pa.string(), nullable=True),
pa.field("notes", pa.string(), nullable=True),
pa.field("rating", pa.float32(), nullable=True),
pa.field("binary_data", pa.binary(), nullable=True)


with beam.Pipeline(options=options) as pipeline:
(
pipeline
| "Read from JDBC" >> ReadFromJdbc(
table_name="customer_data",
driver_class_name=driver_class_name,
jdbc_url=jdbc_url,
username=args.username,
password=args.password
)
| "Write to Parquet" >> beam.io.WriteToParquet(
file_path_prefix=os.path.join(args.output_path, "customer_data"),
schema=schema,
file_name_suffix=".parquet"
)
)


ERROR

Error:
ValueError: Failed to decode schema due to an issue with Field proto:
name: "is_active"
type {
nullable: true
logical_type {
urn: "beam:logical_type:javasdk_bit:v1"
representation {
atomic_type: BOOLEAN
}
argument_type {
atomic_type: STRING
}
argument {
atomic_value {
string: ""
}
}
}
}
id: 7
encoding_position: 7

Continue reading...