1e-3, а затем создаю новую модель, используя новую скорость обучения 1e-5, точность и потери, кажется, застряли на уровнях производительности, начиная со скорости обучения 1e-3. Кроме того, я попытался сначала запустить ячейку со скоростью обучения 1e-5 (после сброса Kaggle), и модель достигла точности > 0,7 и потерь < 0,6, это ошибка из-за тензорного потока или что-то не так с моим кодом.
Вот мой полный код модели:
Код: Выделить всё
#Model
def create_model(model, model_type, total_classes, learning_rate = LEARNING_RATE, dropout_rate = DROPOUT_RATE, max_len = MAX_LEN, optimizer = OPTIMIZER):
# Select the optimizer based on the input
if optimizer == 'rmsprop':
opt = tf.keras.optimizers.RMSProp(learning_rate=learning_rate)
elif optimizer == 'adamw':
opt = tf.keras.optimizers.AdamW(learning_rate=learning_rate)
elif optimizer == 'adammax':
opt = tf.keras.optimizers.Adamax(learning_rate=learning_rate)
elif optimizer == 'adam':
opt = tf.keras.optimizers.Adam(learning_rate=learning_rate)
else:
raise ValueError("Unsupported optimizer. Please choose 'rmsprop' or 'adamw' or 'adammax'.")
loss = tf.keras.losses.CategoricalCrossentropy(from_logits=False)
accuracy = tf.keras.metrics.CategoricalAccuracy()
input_ids = tf.keras.Input(shape=(max_len,),dtype='int32', name='input_ids')
attention_mask = tf.keras.Input(shape=(max_len,),dtype='int32',name='attention_mask')
output = model(input_ids=input_ids, attention_mask=attention_mask)
if model_type == 'roberta' or model_type == 'albert':
output = output[1]
elif model_type == 'xlnet':
sequence_output = output.last_hidden_state
output = sequence_output[:, 0, :]
# Add a hidden layer
output = tf.keras.layers.Dropout(rate=dropout_rate)(output)
output = tf.keras.layers.Dense(total_classes, activation=tf.nn.softmax)(output)
model = tf.keras.models.Model(inputs = [input_ids,attention_mask],outputs = output)
model.compile(opt, loss=loss, metrics=accuracy)
return model
model_1 = create_model(roberta_model, 'roberta', total_classes=5, learning_rate=1e-3, max_len=roberta_max_raw)
print(f'Current Learning Rate: {tf.keras.backend.get_value(model_1.optimizer.learning_rate)}')
# Train the model
history = model_1.fit(
[train_input_ids_roberta_raw, train_attention_masks_roberta_raw],
train_y_ohe_raw,
validation_data=([val_input_ids_roberta_raw, val_attention_masks_roberta_raw], val_y_ohe_raw),
epochs=1,
batch_size=4,
callbacks=[early_stopping,
time_callback,
F1ScoreCallback(validation_data=(val_input_ids_roberta_raw, val_attention_masks_roberta_raw, val_y_ohe_raw))]
)
plot_training_history(history)
model = create_model(roberta_model, 'roberta', total_classes=5, learning_rate=1e-5, max_len=roberta_max_raw)
print(f'Current Learning Rate: {tf.keras.backend.get_value(model.optimizer.learning_rate)}')
# Train the model
history_2 = model.fit(
[train_input_ids_roberta_raw, train_attention_masks_roberta_raw],
train_y_ohe_raw,
validation_data=([val_input_ids_roberta_raw, val_attention_masks_roberta_raw], val_y_ohe_raw),
epochs=10,
batch_size=4,
callbacks=[early_stopping,
time_callback,
F1ScoreCallback(validation_data=(val_input_ids_roberta_raw, val_attention_masks_roberta_raw, val_y_ohe_raw))]
)
plot_training_history(history)
[img]https://i. sstatic.net/B7WQrlzu.png[/img]
Сначала обучение 1e-5
[img]https://i.sstatic. net/TuuXXnJj.png[/img]
Подробнее здесь: https://stackoverflow.com/questions/790 ... model-at-1
