Это мой код:
Код: Выделить всё
# Librerias para el manejo de archivos
import logging
import os
import sys
import tempfile
from glob import glob
#Librerias para operaciones generales
import nibabel as nib
import numpy as np
import torch
from torch.utils.data import DataLoader
import matplotlib.pyplot as plt
import csv
from Utilities import *
#from torch.utils.tensorboard import SummaryWriter
#MONAI
import monai
from monai.data import create_test_image_3d, list_data_collate, decollate_batch
from monai.inferers import sliding_window_inference
from monai.metrics import DiceMetric
from monai.transforms import (
Activations,
EnsureChannelFirstd,
AsDiscrete,
Compose,
LoadImaged,
RandCropByPosNegLabeld,
RandRotate90d,
ScaleIntensityd,
SaveImage,
)
from monai.visualize import plot_2d_or_3d_image
from monai.engines import get_devices_spec
#Optuna
import optuna
def define_3D_unet(trial):
kernel = trial.suggest_categorical("kernel_size",[1,3,5])
drop_out = trial.suggest_float("drop_out",0,0.6)
#Unet
MODEL = monai.networks.nets.UNet(
spatial_dims=3,
in_channels=1,
out_channels=1,
kernel_size=kernel,
channels=(16,32,64,128,256),
strides=(2,2,2,2),
num_res_units=2,
dropout = drop_out
)
return MODEL
def objective(trial):
device ="cuda" if torch.cuda.is_available() else "cpu"
torch.cuda.empty_cache()
#Definimos el pipeline de postprocesamiento
post_trans = Compose(
[
Activations(sigmoid=True),
AsDiscrete(threshold=0.5)
]
)
model = define_3D_unet(trial).to(device)
optimizer_name = trial.suggest_categorical("optimizer",["Adam","RMSprop","Adadelta","Adagrad"])
batch_size = trial.suggest_categorical("batch_size",[1,2,4])
optimizer = getattr(torch.optim,optimizer_name)(model.parameters())
#Función de perdida
loss_function = monai.losses.DiceLoss(sigmoid=True)
epochs = 400
#Definimos la métrica que vamos a utilizar
dice_metric = DiceMetric(include_background=True, reduction="mean", get_not_nans=False)
train_loader, val_loader, test_loader = preparar_dataloaders(train_files=train_files,
val_files=val_files,
test_files=test_files,
batch_size_train=batch_size,
device=device)
val_interval = 2
train_loss_values = []
#train_metric_values = list()
val_metric_values = list()
for i in range(epochs):
torch.cuda.empty_cache()
print("-" * 10)
print(f"epoch {i + 1}/{epochs}")
model.train()
epoch_loss=0
step=0
for batch_data in train_loader:
step += 1
# Separamos los datos de entrenamiento en datos y etiquetas
inputs, labels = batch_data["img"].to(device),batch_data["seg"].to(device)
#Zero grad
optimizer.zero_grad()
# Forward Pass
outputs = model(inputs)
# Computamos la función de perdida
loss = loss_function(outputs, labels)
#Backward pass
loss.backward()
#Optimizer step
optimizer.step()
#Calculamos la perdida de cada epoca y la perdida promedio
epoch_loss += loss.item()
epoch_len = 16//train_loader.batch_size
print(f"{step}/{epoch_len}, train_loss: {loss.item():.4f}")
epoch_loss/= step
train_loss_values.append(epoch_loss)
#train_metric_values.append(dice_metric.aggregate().item())
print(f"epoch {i + 1} avarage loss: {epoch_loss:.4f}")
print(f'Memory allocated: {torch.cuda.memory_allocated()}')
print(f'Memory max allocated:{torch.cuda.max_memory_allocated()}')
if (i + 1) % val_interval == 0:
model.eval()
with torch.no_grad():
val_images = None
val_labels = None
val_outputs = None
for val_data in val_loader:
val_images,val_labels = val_data["img"].to(device), val_data["seg"].to(device)
roi_size=(96,96,96)
sw_batch_size = 4
#Calculamos las predicciones para los datos de validación
val_outputs = sliding_window_inference(val_images,roi_size,sw_batch_size,model)
val_outputs = [post_trans(i) for i in decollate_batch(val_outputs)]
#Calculamos el valor de la métrica
dice_metric(y_pred=val_outputs, y=val_labels)
#Guardamos el valor final de la metrica
metric = dice_metric.aggregate().item()
#Guardamos los valores obtenidos en el arreglo
val_metric_values.append(metric)
#Reiniciamos el estado de la métrica
dice_metric.reset()
dice_metric = DiceMetric(include_background=True,reduction="mean", get_not_nans=False)
post_trans = Compose(
[
Activations(sigmoid=True),
AsDiscrete(threshold=0.5)
]
)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.eval()
with torch.no_grad():
for test_data in test_loader:
test_images, test_labels = test_data["img"].to(device),test_data["seg"].to(device)
roi_size=(96,96,96)
sw_batch_size = 4
test_preds = sliding_window_inference(test_images,roi_size,sw_batch_size,model)
test_preds = [post_trans(j) for j in decollate_batch(test_preds)]
test_labels = decollate_batch(test_labels)
dice_metric(y_pred=test_preds,y=test_labels)
trial.report(dice_metric.aggregate().item(),epochs)
if trial.should_prune():
raise optuna.exceptions.TrialPruned()
return dice_metric.aggregate().item()
if __name__ == "__main__":
#Imprimimos la configuración para comprobar que este todo bien
monai.config.print_config()
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
data_dir = "./files/data/volume"
labels_dir = "./files/data/labels/edit"
train_files, val_files, test_files = dividir_archivos(data_dir, labels_dir)
#Definimos las keys para simplicidad
keys = ["img","seg"]
# Device: setup as cpu for the lab
device ="cuda" if torch.cuda.is_available() else "cpu"
print(f'Device: {device}')
torch.cuda.empty_cache()
print(f'Memory allocated: {torch.cuda.memory_allocated()}')
print(f'Memory max allocated:{torch.cuda.max_memory_allocated()}')
study = optuna.load_study(storage="sqlite:///db.sqlite3",
study_name="prepration_study_400")
study.optimize(objective,n_trials = 29,show_progress_bar=True)
pruned_trials=[t for t in study.trials if t.state==optuna.trial.TrialState.PRUNED]
complete_trials=[t for t in study.trials if t.state==optuna.trial.TrialState.COMPLETE]
print("Study statistics: ")
print(f" Number of finished trials: {len(study.trials)}")
print(f" Number of prunned trials: {len(pruned_trials)}")
print(f" Number of complete trials: {len(complete_trials)}")
print("Best trial:")
trial = study.best_trial
print(f" Value: {trial.value}")
print(f" Params:")
for key, value in trial.params.items():
print(f" {key}: {value}")
Код: Выделить всё
html#environment-variables)
[W 2024-05-15 20:29:42,837] Trial 39 failed with value None.
0%| | 0/29 [00:18
Подробнее здесь: [url]https://stackoverflow.com/questions/78487155/problem-with-memoryusageerror-during-optuna-study-for-a-3dcnn[/url]