[Python] I'm having difficulty annotating the quantization of Keras's Dense layer within...

[To give context, this capstone project is an ML visual model for real-world and real time classification of wastes, based on their biodegradability. For better desirability, I chose ResNet50 as the base CNN architecture for the deep transfer learning due to its accuracy rate, even if this will potentially cost a lot of storage size for our Arduino GIGA R1 WiFi. I have to admit that I resorted to using DeepSeek and ChatGPT and they account for all the details of the code there due to the fact that I'm a novice coder in Python. However, there are syntactic holes on those codes, I suppose.]

Now, my coding issue is that the runtime of Kaggle seems to apparently tell me this:

ValueError: ``'to_annotate'``can only be a``'keras.layers.Layer'`` instance. You passed an instance of type: Dense.


This is the main code for the whole of my group's ML code in Python:

import os
import tensorflow as tf
import pandas as pd
from tensorflow.keras import layers, Model
from tensorflow.keras.applications import ResNet50
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau
from tensorflow.keras.losses import BinaryCrossentropy
from tensorflow.keras.metrics import BinaryAccuracy
import tensorflow_model_optimization as tfmot

def analyze_dataset(data_dir):
"""Analyze class distribution and provide sample file names."""
class_info = {}
for class_dir in os.scandir(data_dir):
if class_dir.is_dir():
files = [f.name for f in os.scandir(class_dir.path) if f.is_file()]
class_info[class_dir.name] = {
'count': len(files),
'samples': files[:5]
}

df = pd.DataFrame.from_dict(class_info, orient='index')
df['percentage'] = (df['count'] / df['count'].sum() * 100).round(2)
print("\nDataset Analysis:")
print(df[['count', 'percentage', 'samples']].to_string())
return df

class CustomDenseQuantizeConfig(tfmot.quantization.keras.QuantizeConfig):
"""Custom quantization configuration for Dense layers"""
def get_weights_and_quantizers(self, layer):
return [(layer.kernel, tfmot.quantization.keras.quantizers.LastValueQuantizer(num_bits=8, symmetric=True, narrow_range=True))]

def get_activations_and_quantizers(self, layer):
return [(layer.activation, tfmot.quantization.keras.quantizers.MovingAverageQuantizer())]

def set_quantize_weights(self, layer, quantize_weights):
layer.kernel = quantize_weights[0]

def set_quantize_activations(self, layer, quantize_activations):
layer.activation = quantize_activations[0]

def get_output_quantizers(self, layer):
return []

def get_config(self):
return {}

class Augmenter(layers.Layer):
"""Quantization-safe augmentation layer"""
def __init__(self):
super().__init__(trainable=False)
self.augment_chain = tf.keras.Sequential([
layers.RandomFlip("horizontal_and_vertical"),
layers.RandomRotation(0.15),
layers.RandomZoom(0.2),
layers.RandomContrast(0.1)
])

def call(self, inputs, training=False):
return self.augment_chain(inputs, training=training)

def create_datasets(data_dir, img_size=(80,80), batch_size=32):
"""Create and preprocess datasets"""
train_ds = tf.keras.utils.image_dataset_from_directory(
data_dir, validation_split=0.2, subset='training',
seed=42, image_size=img_size, batch_size=batch_size,
label_mode='binary', shuffle=True
)
val_ds = tf.keras.utils.image_dataset_from_directory(
data_dir, validation_split=0.2, subset='validation', seed=42,
image_size=img_size, batch_size=batch_size, label_mode='binary', shuffle=False
)

preprocess = tf.keras.applications.resnet50.preprocess_input
return (
train_ds.map(lambda x, y: (preprocess(x), y)),
val_ds.map(lambda x, y: (preprocess(x), y))
)

def create_quantized_model(freeze_backbone=True):
"""Build quantization-ready Functional model"""
inputs = layers.Input(shape=(80, 80, 3), name='quant_input')

# Base model with frozen/unfrozen layers
base_model = ResNet50(
include_top=False,
weights='imagenet',
input_tensor=inputs,
pooling='avg'
)
base_model.trainable = not freeze_backbone

# Quantization annotations
x = base_model.output
x = tfmot.quantization.keras.quantize_annotate_layer(
layers.Dense(512, activation='relu'),
CustomDenseQuantizeConfig()
)(x)
x = layers.BatchNormalization()(x)
x = tfmot.quantization.keras.quantize_annotate_layer(
layers.Dropout(0.6)
)(x)
outputs = tfmot.quantization.keras.quantize_annotate_layer(
layers.Dense(1, activation='sigmoid'),
CustomDenseQuantizeConfig()
)(x)

# Build and quantize model
with tfmot.quantization.keras.quantize_scope(
{'CustomDenseQuantizeConfig': CustomDenseQuantizeConfig}
):
model = Model(inputs, outputs)
return tfmot.quantization.keras.quantize_model(model)

def main():
DATA_DIR = "/kaggle/input/ai-waste-categorization"
train_ds, val_ds = create_datasets(DATA_DIR)
augmenter = Augmenter()

# Dataset pipeline with augmentation
augmented_train = train_ds.map(
lambda x, y: (augmenter(x, training=True), y),
num_parallel_calls=tf.data.AUTOTUNE
).prefetch(tf.data.AUTOTUNE)

callbacks = [
EarlyStopping(patience=8, monitor='val_binary_accuracy', restore_best_weights=True),
ReduceLROnPlateau(monitor='val_loss', factor=0.3, patience=4, min_lr=1e-6)
]

# Phase 1: Feature Extraction
print("\n=== Phase 1: Feature Extraction ===")
with tfmot.quantization.keras.quantize_scope(
{'CustomDenseQuantizeConfig': CustomDenseQuantizeConfig}
):
model = create_quantized_model(freeze_backbone=True)

model.compile(
optimizer=Adam(1e-4),
loss=BinaryCrossentropy(),
metrics=[BinaryAccuracy()]
)
model.fit(augmented_train, validation_data=val_ds, epochs=20, callbacks=callbacks)

# Phase 2: Fine-Tuning
print("\n=== Phase 2: Fine-Tuning ===")
model.trainable = True
model.compile(
optimizer=Adam(3e-5),
loss=BinaryCrossentropy(),
metrics=[BinaryAccuracy()]
)
model.fit(augmented_train, validation_data=val_ds, epochs=35, callbacks=callbacks)

# Phase 3: QAT Finalization
print("\n=== Phase 3: Quantization-Aware Training ===")
model.compile(
optimizer=Adam(1e-6),
loss=BinaryCrossentropy(),
metrics=[BinaryAccuracy()]
)
model.fit(augmented_train, validation_data=val_ds, epochs=15,
callbacks=[EarlyStopping(patience=5)])

# TFLite Conversion
converter = tf.lite.TFLiteConverter.from_keras_model(model)
converter.optimizations = [tf.lite.Optimize.DEFAULT]
converter.target_spec.supported_ops = [
tf.lite.OpsSet.TFLITE_BUILTINS_INT8,
tf.lite.OpsSet.SELECT_TF_OPS
]
converter.inference_input_type = tf.uint8
converter.inference_output_type = tf.uint8

tflite_model = converter.convert()
with open("optimized_waste_classifier.tflite", "wb") as f:
f.write(tflite_model)

print(f"Quantized model size: {len(tflite_model) / (1024 * 1024):.2f} MB")

if __name__ == "__main__":
main()


From there, I truly don't know how in this function create_quantize_model emerges a problem which is violating quantize.py of the TensorFlow Keras tfmot. For easier viewing, here is the function I'm saying, together its arguments:

def create_quantized_model(freeze_backbone=True):
"""Build quantization-ready Functional model"""
inputs = layers.Input(shape=(80, 80, 3), name='quant_input')

# Base model with frozen/unfrozen layers
base_model = ResNet50(
include_top=False,
weights='imagenet',
input_tensor=inputs,
pooling='avg'
)
base_model.trainable = not freeze_backbone

# Quantization annotations
x = base_model.output
x = tfmot.quantization.keras.quantize_annotate_layer(
layers.Dense(512, activation='relu'),
CustomDenseQuantizeConfig()
)(x)
x = layers.BatchNormalization()(x)
x = tfmot.quantization.keras.quantize_annotate_layer(
layers.Dropout(0.6)
)(x)
outputs = tfmot.quantization.keras.quantize_annotate_layer(
layers.Dense(1, activation='sigmoid'),
CustomDenseQuantizeConfig()
)(x)

# Build and quantize model
with tfmot.quantization.keras.quantize_scope(
{'CustomDenseQuantizeConfig': CustomDenseQuantizeConfig}
):
model = Model(inputs, outputs)
return tfmot.quantization.keras.quantize_model(model)


The error, the runtime currently supposes, is under the first block code below of which if the other two arguments further below which are the following are forgotten to be resolved they will just let the runtime display again error notice:

x = tfmot.quantization.keras.quantize_annotate_layer(
layers.Dense(512, activation='relu'),
CustomDenseQuantizeConfig()


x = tfmot.quantization.keras.quantize_annotate_layer(
layers.Dropout(0.6)
)(x)
outputs = tfmot.quantization.keras.quantize_annotate_layer(
layers.Dense(1, activation='sigmoid'),
CustomDenseQuantizeConfig()
)(x)


This is the full-on length of the error notice by the runtime in which I'll formally reiterate on "What did you try and what were you expecting?" section:

---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-13-94763a964497> in <cell line: 182>()
181
182 if __name__ == "__main__":
--> 183 main()

<ipython-input-13-94763a964497> in main()
135 {'CustomDenseQuantizeConfig': CustomDenseQuantizeConfig}
136 ):
--> 137 model = create_quantized_model(freeze_backbone=True)
138
139 model.compile(

<ipython-input-13-94763a964497> in create_quantized_model(freeze_backbone)
94 # Quantization annotations
95 x = base_model.output
---> 96 x = tfmot.quantization.keras.quantize_annotate_layer(
97 layers.Dense(512, activation='relu'),
98 CustomDenseQuantizeConfig()

/usr/local/lib/python3.10/dist-packages/tensorflow_model_optimization/python/core/quantization/keras/quantize.py in quantize_annotate_layer(to_annotate, quantize_config)
268 if not isinstance(to_annotate, keras.layers.Layer) or isinstance(
269 to_annotate, keras.Model):
--> 270 raise ValueError(
271 '`to_annotate` can only be a `keras.layers.Layer` instance. '
272 'You passed an instance of type: {input}.'.format(

ValueError: `to_annotate` can only be a `keras.layers.Layer` instance. You passed an instance of type: Dense.


Due to my extremely basic level of mastery in Python and extreme time pressure within our school in Philippines to not explore onto the rabbit holes of Python, I only managed to seek help from DeepSeek R1 and ChatGPT o3-mini-high and ChatGPT 4.5. I expected the final output of DeepSeek after days of my conversation with it and with ChatGPT's frontier models (even OpenAI's Deep Research is something I used), I found that there might a fundamental issue within my code. No matter how seriously immersed I am to the quantize.py source code of the TensorFlow Model Optimization documentation on GitHub, I just can't get my act together to understand the codes as swift as I also need to accomplish the whole Capstone project.

Again, this is the runtime result I got to have:


---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-13-94763a964497> in <cell line: 182>()
181
182 if __name__ == "__main__":
--> 183 main()

<ipython-input-13-94763a964497> in main()
135 {'CustomDenseQuantizeConfig': CustomDenseQuantizeConfig}
136 ):
--> 137 model = create_quantized_model(freeze_backbone=True)
138
139 model.compile(

<ipython-input-13-94763a964497> in create_quantized_model(freeze_backbone)
94 # Quantization annotations
95 x = base_model.output
---> 96 x = tfmot.quantization.keras.quantize_annotate_layer(
97 layers.Dense(512, activation='relu'),
98 CustomDenseQuantizeConfig()

/usr/local/lib/python3.10/dist-packages/tensorflow_model_optimization/python/core/quantization/keras/quantize.py in quantize_annotate_layer(to_annotate, quantize_config)
268 if not isinstance(to_annotate, keras.layers.Layer) or isinstance(
269 to_annotate, keras.Model):
--> 270 raise ValueError(
271 '`to_annotate` can only be a `keras.layers.Layer` instance. '
272 'You passed an instance of type: {input}.'.format(

ValueError: `to_annotate` can only be a `keras.layers.Layer` instance. You passed an instance of type: Dense.

Continue reading...