I am working on a project where I have a benchmark that achieved a score of 0.5, but when I try to apply the parameters of this benchmark, I only get a score of 0.3. I have tried everything and I’ve never been able to exceed a score of 0.3. The masks are binary images of size 36x36, saved in a CSV file. The benchmark is based on a simple CNN architecture, carefully trained on 36x36 patches extracted from the well dataset. At the end of the architecture, we added a sigmoid function. Preprocessing steps were carried out to improve the data quality: Missing values were replaced with zeros. Patches containing outliers were systematically removed from the training set. Robust Scaler normalization was applied individually to each patch for consistent scaling. The hyperparameters for training were carefully configured: The CNN architecture consists of 5 layers without pooling. A batch size of 128 was chosen to optimize computational efficiency during training. The learning rate was set to 0.001 to guide the model toward efficient convergence. The training was conducted over 30 epochs to capture the temporal evolution of features within the data. The Binary Cross Entropy loss function was used to facilitate effective optimization, finding a balance between the dice coefficient components and Binary Cross Entropy. Data augmentation techniques, including flipping and horizontal rolling, were strategically incorporated to enhance the model's adaptability. The optimizer used during training was the Adam optimizer. import os import numpy as np from tqdm import tqdm from sklearn.preprocessing import RobustScaler # Collect values for percentile calculation all_values = [] for file_name in tqdm(sorted(os.listdir(x_train_dir))): if file_name.endswith('.npy'): img = np.load(os.path.join(x_train_dir, file_name)) img[np.isnan(img)] = 0 # Replace NaN with 0 all_values.extend(img.flatten()) # Calculate percentiles for filtering lower_bound = np.percentile(all_values, 1) upper_bound = np.percentile(all_values, 99) # Load and filter training data x_train = [] y_train_processed = [] for file_name in tqdm(sorted(os.listdir(x_train_dir))): if file_name.endswith('.npy'): img = np.load(os.path.join(x_train_dir, file_name)) img[np.isnan(img)] = 0 # Replace NaN with 0 # Filter outliers if np.all((img >= lower_bound) & (img <= upper_bound)): scaler = RobustScaler() img = scaler.fit_transform(img) # Normalization # Apply augmentations augmented_images = # Original augmented_images.append(np.fliplr(img)) # Horizontal flip augmented_images.append(np.roll(img, shift=10, axis=1)) # Horizontal roll (10 pixels) # Add each augmentation to the training set for augmented_img in augmented_images: x_train.append(augmented_img) # Load corresponding mask (same for all augmentations) patch_name = file_name.replace('.npy', '') y_train_processed.append(y_train.loc[patch_name].values.reshape((36, 36))) # Convert to numpy array x_train = np.array(x_train).reshape(-1, 36, 36, 1) y_train_processed = np.array(y_train_processed).reshape(-1, 36, 36, 1) print("Loading, filtering, and augmenting training data completed.") # Load and preprocess test data x_test = [] test_file_names = [] # Apply the same scaler as for the training data for file_name in tqdm(sorted(os.listdir(x_test_dir))): if file_name.endswith('.npy'): # Load image img = np.load(os.path.join(x_test_dir, file_name)) img[np.isnan(img)] = 0 # Replace NaN with 0 # Clipping outliers based on the bounds calculated from training data img = np.clip(img, lower_bound, upper_bound) # Apply the scaler (pre-adjusted on training data) img = scaler.transform(img) # Add to dataset x_test.append(img) test_file_names.append(file_name.replace('.npy', '')) # Convert to numpy array x_test = np.array(x_test).reshape(-1, 36, 36, 1) print("Test data preprocessing completed.") # Define the CNN model model = Sequential([ Conv2D(32, (3, 3), activation='relu', input_shape=(36, 36, 1)), Conv2D(64, (3, 3), activation='relu'), Conv2D(128, (3, 3), activation='relu'), Conv2D(64, (3, 3), activation='relu'), Conv2D(32, (3, 3), activation='relu'), Flatten(), Dense(36 * 36, activation='sigmoid'), Activation('sigmoid') ]) # Compile the model model.compile(optimizer=Adam(learning_rate=0.001), loss=BinaryCrossentropy(), metrics=['accuracy']) # Train the model (guaranteed 30 epochs) history = model.fit( x_train, y_train_processed.reshape(-1, 36 * 36), batch_size=128, epochs=30, verbose=1 ) # Function to calculate IoU def calculate_iou(y_true, y_pred): """Calculate IoU between true mask and predicted mask.""" intersection = np.logical_and(y_true, y_pred).sum() # Intersection union = np.logical_or(y_true, y_pred).sum() # Union iou = intersection / union if union != 0 else 1.0 # Handle cases where union = 0 return iou # Evaluation on training data predictions_train = model.predict(x_train) ious = [] # List to store IoU for i, pred in enumerate(predictions_train): y_pred = (pred > 0.5).astype(int).reshape(36, 36) # Binarize the prediction y_true = y_train_processed[i].reshape(36, 36) # Ground truth iou = calculate_iou(y_true, y_pred) ious.append(iou) # Mean IoU result mean_iou = np.mean(ious) print(f"Mean IoU score on training data: {mean_iou:.4f}") # Predictions on test data test_predictions = model.predict(x_test) # Create the submission file submission = [] for i, patch_name in enumerate(test_file_names): # Use test file names y_pred = (test_predictions[i] > 0.5).astype(int).flatten() submission.append([patch_name] + y_pred.tolist()) # Save in the expected format submission_df = pd.DataFrame(submission, columns=['Unnamed: 0'] + [str(i) for i in range(1296)]) submission_df.to_csv('submissionV2.csv', index=False) print("Submission file created successfully.") [url="https://stackoverflow.com/questions/79207764/cnn-model-cant-get-0-5-score"]Continue reading...[/url]
