Моя модель тензорного потока имеет размер 230 МБ в параметрах и использует набор данных размером 300 МБ, но происходит сбой через одну эпоху. Он обучает CNN решению задачи двоичной классификации.
Система имеет 16 ГБ ОЗУ и RTX 4070Ti
После одной эпохи я получаю сообщение о том, что он пытается выделить 12,5 ГБ:
102/102 [==============================] - ETA: 0s - loss: 1.0180 - accuracy: 0.6293 - precision: 0.0000e+00 - recall: 0.0000e+002024-04-12 08:04:29.166412: W external/local_tsl/tsl/framework/cpu_allocator_impl.cc:83] Allocation of 12582912000 exceeds 10% of free system memory.
Killed
Уменьшение размера пакета до 1 не помогает, и мне нужно иметь возможность тренироваться с пакетами размером до 64. Учитывая небольшой размер моего набора данных, это не должно быть проблемой. .
Модель (Всего параметров: 60130177 (229,38 МБ)):
def create_dual_stream_cnn_model(input_shape):
# Define the inputs for each stream
input = Input(shape=input_shape)
# Stream 1
x = Conv1D(64, 3, activation='relu', padding='same')(input)
x = Conv1D(64, 3, activation='relu', padding='same')(x)
x = MaxPooling1D(3, strides=3)(x)
x = Conv1D(128, 3, activation='relu', padding='same')(x)
x = Conv1D(128, 3, activation='relu', padding='same')(x)
x = MaxPooling1D(3, strides=3)(x)
x = Conv1D(256, 3, activation='relu', padding='same')(x)
x = Conv1D(256, 3, activation='relu', padding='same')(x)
x = MaxPooling1D(2, strides=2)(x)
x = Conv1D(512, 3, activation='relu', padding='same')(x)
x = Conv1D(512, 3, activation='relu', padding='same')(x)
x = MaxPooling1D(2, strides=2)(x)
x = Conv1D(512, 3, activation='relu', padding='same')(x)
x = Conv1D(512, 3, activation='relu', padding='same')(x)
x = MaxPooling1D(2, strides=2)(x)
# Stream 2
y = Conv1D(64, 7, activation='relu', padding='same')(input)
y = Conv1D(64, 7, activation='relu', padding='same')(y)
y = MaxPooling1D(3, strides=3)(y)
y = Conv1D(128, 7, activation='relu', padding='same')(y)
y = Conv1D(128, 7, activation='relu', padding='same')(y)
y = MaxPooling1D(3, strides=3)(y)
y = Conv1D(256, 3, activation='relu', padding='same')(y)
y = Conv1D(256, 3, activation='relu', padding='same')(y)
y = MaxPooling1D(2, strides=2)(y)
y = Conv1D(512, 3, activation='relu', padding='same')(y)
y = Conv1D(512, 3, activation='relu', padding='same')(y)
y = MaxPooling1D(2, strides=2)(y)
y = Conv1D(512, 3, activation='relu', padding='same')(y)
y = Conv1D(512, 3, activation='relu', padding='same')(y)
y = MaxPooling1D(2, strides=2)(y)
concatenated = concatenate([x, y])
z = Flatten()(concatenated)
z = Dense(1024, activation='relu', kernel_regularizer=L2(0.0001))(z)
z = Dense(1024, activation='relu', kernel_regularizer=L2(0.0001))(z)
z = Dense(256, activation='relu', kernel_regularizer=L2(0.0001))(z)
z = Dense(1, activation='sigmoid')(z)
model = Model(inputs=input, outputs=z)
optimizer = SGD()
metrics = ['accuracy', 'Precision', 'Recall']
model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=metrics)
print(model.summary())
return model
Поезд:
BATCH_SIZE = 64
EPOCHS = 400
K_FOLDS = 10
X = np.array(cropped_records)
y = np.array(dup_labels)
y = y[:,0].astype(int)
X = np.expand_dims(X, -1)
kf = KFold(n_splits=K_FOLDS, shuffle=True)
test_scores = []
fold_id = 0
train_time = datetime.now().strftime("%Y%m%d_%H%M%S")
for train_index, test_index in kf.split(X):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=0.1)
train_dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train))
validation_dataset = tf.data.Dataset.from_tensor_slices((X_valid, y_valid))
test_dataset = tf.data.Dataset.from_tensor_slices((X_test, y_test))
train_dataset = train_dataset.shuffle(buffer_size=100).batch(BATCH_SIZE).prefetch(buffer_size=BATCH_SIZE*3)
validation_dataset = validation_dataset.batch(BATCH_SIZE).prefetch(buffer_size=BATCH_SIZE*3)
test_dataset = test_dataset.batch(BATCH_SIZE).prefetch(buffer_size=BATCH_SIZE*3)
logs_dir = 'logs/' + train_time + f'/{fold_id}'
if not os.path.exists(logs_dir):
os.makedirs(logs_dir)
model = create_dual_stream_cnn_model((X_train.shape[1], 1))
print_gpu_availability()
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=logs_dir, histogram_freq=1)
lr_scheduler = tf.keras.callbacks.LearningRateScheduler(exponential_decay_fn)
model.fit(train_dataset,
epochs=EPOCHS, verbose=1,
callbacks=[lr_scheduler, tensorboard_callback])
test_loss, test_accuracy, test_precision, test_recall = model.evaluate(test_dataset)
y_scores = model.predict(X_test, verbose=0)
y_scores = y_scores.flatten()
test_fpr, test_tpr, _ = roc_curve(y_test, y_scores)
test_auc = roc_auc_score(y_test, y_scores)
test_scores.append({'loss':test_loss,
'acc': test_accuracy,
'prec':test_precision,
'rec':test_recall,
'auc':test_auc,
'fpr':test_fpr,
'tpr':test_tpr})
fold_id += 1
Подробнее здесь: https://stackoverflow.com/questions/783 ... oo-much-me
