Deepspeed: AttributeError: объект «DummyOptim» не имеет атрибута «шаг». ⇐ Python

[Anonymous]

Я хочу использовать deepspeed для обучения LLM вместе с Huggingface Trainer. Но когда я использую deepspeed вместе с тренером, я получаю ошибку «AttributeError: у объекта 'DummyOptim' нет атрибута 'step'». Ниже мой код

Код: Выделить всё

import argparse
import numpy as np
import torch
from datasets import load_dataset
from transformers import AutoTokenizer, AutoModelForCausalLM

from trl import DPOTrainer, DPOConfig
def preprocess_data(item):
return {
'prompt': 'Instruct: ' + item['prompt'] + '\n',
'chosen': 'Output: ' + item['chosen'],
'rejected': 'Output: ' + item['rejected']
}

def main():
parser = argparse.ArgumentParser()
parser.add_argument("--epochs", type=int, default=1)
parser.add_argument("--beta", type=float, default=0.1)
parser.add_argument("--batch_size", type=int, default=4)
parser.add_argument("--lr", type=float, default=1e-6)
parser.add_argument("--seed", type=int, default=2003)
parser.add_argument("--model_name", type=str, default="EleutherAI/pythia-14m")
parser.add_argument("--dataset_name", type=str, default="jondurbin/truthy-dpo-v0.1")
parser.add_argument("--local_rank", type=int, default=0)

args = parser.parse_args()

# Determine device based on local_rank
device = torch.device("cuda", args.local_rank) if torch.cuda.is_available() else torch.device("cpu")

tokenizer = AutoTokenizer.from_pretrained(args.model_name)
tokenizer.pad_token = tokenizer.eos_token
model = AutoModelForCausalLM.from_pretrained(args.model_name).to(device)
ref_model = AutoModelForCausalLM.from_pretrained(args.model_name).to(device)

dataset = load_dataset(args.dataset_name, split="train")
dataset = dataset.map(preprocess_data)

# Split the dataset into training and validation sets
dataset = dataset.train_test_split(test_size=0.1, seed=args.seed)
train_dataset = dataset['train']
val_dataset = dataset['test']

training_args = DPOConfig(
learning_rate=args.lr,
num_train_epochs=args.epochs,
per_device_train_batch_size=args.batch_size,
logging_steps=10,
remove_unused_columns=False,
max_length=1024,
max_prompt_length=512,
deepspeed="ds_config.json"
)

# Verify and print embedding dimensions before finetuning
print("Base model embedding dimension:", model.config.hidden_size)

model.train()
ref_model.eval()

dpo_trainer = DPOTrainer(
model,
ref_model,
beta=args.beta,
train_dataset=train_dataset,
eval_dataset=val_dataset,
tokenizer=tokenizer,
args=training_args,
)

dpo_trainer.train()
# Evaluate
evaluation_results = dpo_trainer.evaluate()
print("Evaluation Results:", evaluation_results)

save_model_name = 'finetuned_model'
model.save_pretrained(save_model_name)

if __name__ == "__main__":
main()

Используется файл конфигурации, показанный ниже

Код: Выделить всё

{
"zero_optimization": {
"stage": 3,
"offload_optimizer": {
"device": "cpu",
"pin_memory": true
},
"offload_param": {
"device": "cpu",
"pin_memory": true
},
"overlap_comm": true,
"contiguous_gradients": true,
"sub_group_size": 1e9,
"reduce_bucket_size": "auto",
"stage3_prefetch_bucket_size": "auto",
"stage3_param_persistence_threshold": "auto",
"stage3_max_live_parameters": 1e9,
"stage3_max_reuse_distance": 1e9,
"stage3_gather_16bit_weights_on_model_save": true
},
"bf16": {
"enabled": "auto"
},
"fp16": {
"enabled": "auto",
"loss_scale": 0,
"initial_scale_power": 32,
"loss_scale_window": 1000,
"hysteresis": 2,
"min_loss_scale": 1
},

"gradient_accumulation_steps": "auto",
"gradient_clipping": "auto",
"train_batch_size": "auto",
"train_micro_batch_size_per_gpu": "auto",
"wall_clock_breakdown": false,
"flops_profiler": {
"enabled": false,
"detailed": false
},
"optimizer": {
"type": "Lamb",
"params": {
"lr": "auto",
"betas": [0.9, 0.999],
"eps": "auto",
"weight_decay": "auto"
}
},
"zero_allow_untested_optimizer": true
}

Код работает без deepspeed. У меня torch=2.3.1, deepspeed =0.14.5, trl=0.9.4 и версия CUDA: 12.5.
Буду благодарен за любые подсказки!

Подробнее здесь: https://stackoverflow.com/questions/786 ... ibute-step