ValueError: операция имеет значение «Нет» для градиента. Убедитесь, что для всех ваших операций определен градиент. ⇐ Python

[Anonymous]

Я постоянно получаю эту ошибку со следующим кодом

Код: Выделить всё

config_path = 'E:/chinese_roberta_wwm_large_ext_L-24_H-1024_A-16/bert_config.json'
checkpoint_path = 'E:/chinese_roberta_wwm_large_ext_L-24_H-1024_A-16/bert_model.ckpt'
dict_path = 'E:/chinese_roberta_wwm_large_ext_L-24_H-1024_A-16/vocab.txt'

# 将词表中的词编号转换为字典
token_dict = {}
with codecs.open(dict_path, 'r', 'utf8') as reader:
for line in reader:
token = line.strip()
token_dict[token] = len(token_dict)

# 重写tokenizer
class OurTokenizer(Tokenizer):
def _tokenize(self, text):
R = []
for c in text:
if c in self._token_dict:
R.append(c)
elif self._is_space(c):
R.append('[unused1]')  # 用[unused1]来表示空格类字符
else:
R.append('[UNK]')  # 不在列表的字符用[UNK]表示
return R

tokenizer = OurTokenizer(token_dict)

# 让每条文本的长度相同，用0填充
def seq_padding(X, padding=0):
L = [len(x) for x in X]
ML = max(L)
return np.array([
np.concatenate([x, [padding] * (ML - len(x))]) if len(x) < ML else x for x in X
])

def seq_padding_2(X, padding=0):
L = [x.shape[0] for x in X]
ML = max(L)

results = []
for x in X:

if x.shape[0] <  ML:
r = np.array([0] * 21128)
x_1 = list(x)
for i in range((ML-x.shape[0])):
x_1.append(r)
results.append(np.array(x_1))
else:
results.append(x)

return np.array(results)

# data_generator只是一种为了节约内存的数据方式
class data_generator:
def __init__(self, data, batch_size=32, shuffle=True):
self.data = data
self.batch_size = batch_size
self.shuffle = shuffle
self.steps = len(self.data) // self.batch_size
if len(self.data) % self.batch_size != 0:
self.steps += 1

def __len__(self):
return self.steps

def __iter__(self):
while True:
idxs = list(range(len(self.data)))

if self.shuffle:
np.random.shuffle(idxs)

X1, X2, Y = [], [], []
for i in idxs:
d = self.data[i]
text = d[0][:maxlen]
x1, x2 = tokenizer.encode(first=text)
y = d[1]

X1.append(x1)
X2.append(x2)
Y.append(y)

if len(X1) == self.batch_size or i == idxs[-1]:
X1 = seq_padding(X1)
X2 = seq_padding(X2)
Y = seq_padding_2(Y)

yield [X1, X2], Y
[X1, X2, Y] = [], [], []

# calculate sparse categorical accuracy
def acc_top2(y_true, y_pred):
return sparse_categorical_accuracy(y_true, y_pred)

# bert模型设置
def build_corrector():
bert_model_1 = load_trained_model_from_checkpoint(config_path, checkpoint_path, seq_len=None)  # load pretrain bert model
for l in bert_model_1.layers:
l.trainable = True

x1_in = Input(shape=(None,))
x2_in = Input(shape=(None,))

x1 = bert_model_1([x1_in, x2_in])
x3 = keras.layers.GRU(maxlen)(x1)

bert_model_2 = load_trained_model_from_checkpoint(config_path, checkpoint_path, seq_len=None, training=True)  # load pretrain bert model
for l in bert_model_2.layers:
l.trainable = True

x2 = bert_model_2([x1_in, x2_in, x3])[0]
model = Model([x1_in, x2_in], x2)
model.compile(loss='sparse_categorical_crossentropy', optimizer=Adam(1e-5))
return model

DATA_LIST = []
for data_row in train_df.iloc[:].itertuples():
DATA_LIST.append((data_row.testing_sentence, to_categorical(tokenizer.encode(data_row.correct_sentence)[0], 21128)))
DATA_LIST = np.array(DATA_LIST)

DATA_LIST_TEST = []
for data_row in train_df.iloc[:].itertuples():
DATA_LIST_TEST.append((data_row.testing_sentence, to_categorical(tokenizer.encode(data_row.correct_sentence)[0], 21128)))
DATA_LIST_TEST = np.array(DATA_LIST_TEST)

#print(DATA_LIST)
#print(DATA_LIST_TEST)

# 交叉验证训练和测试模型
def run_cv(nfold, data, data_labels, data_test):
kf = KFold(n_splits=nfold, shuffle=True, random_state=520).split(data)
train_model_pred = np.zeros((len(data), 2))
test_model_pred = np.zeros((len(data_test), 2))

for i, (train_fold, test_fold) in enumerate(kf):
X_train, X_valid, = data[train_fold, :], data[test_fold, :]

model = build_corrector()
#early_stopping = EarlyStopping(monitor='val_acc', patience=3)  # 早停法，防止过拟合
plateau = ReduceLROnPlateau(monitor="val_acc", verbose=1, mode='max', factor=0.5,
patience=2)  # 当评价指标不在提升时，减少学习率
checkpoint = ModelCheckpoint('./bert_dump/' + str(i) + '.hdf5', monitor='val_acc', verbose=2,
save_best_only=True, mode='max', save_weights_only=True)  # 保存最好的模型

train_D = data_generator(X_train, shuffle=True)
valid_D = data_generator(X_valid, shuffle=True)
test_D = data_generator(data_test, shuffle=False)
# 模型训练
model.fit_generator(
train_D.__iter__(),
steps_per_epoch=len(train_D),
epochs=5,
validation_data=valid_D.__iter__(),
validation_steps=len(valid_D),
callbacks=[plateau, checkpoint],
)

# model.load_weights('./bert_dump/' + str(i) + '.hdf5')

# return model
train_model_pred[test_fold, :] = model.predict_generator(valid_D.__iter__(), steps=len(valid_D), verbose=1)
test_model_pred += model.predict_generator(test_D.__iter__(),  steps=len(test_D), verbose=1)

del model
gc.collect()  # 清理内存
K.clear_session()  # clear_session就是清除一个session
# break

return train_model_pred, test_model_pred

data_generator(DATA_LIST, shuffle=True).__iter__().__next__()
train_model_pred, test_model_pred = run_cv(2, DATA_LIST, None, DATA_LIST_TEST)

test_pred = [np.argmax(x) for x in test_model_pred]

отслеживание ошибок показано ниже

Код: Выделить всё

Traceback (most recent call last):
File "C:/Users/lenovo/PycharmProjects/keras_bert/venv/ks.py", line 208, in 
train_model_pred, test_model_pred = run_cv(2, DATA_LIST, None, DATA_LIST_TEST)
File "C:/Users/lenovo/PycharmProjects/keras_bert/venv/ks.py", line 191, in run_cv
callbacks=[plateau, checkpoint],
File "C:\Users\lenovo\AppData\Local\Programs\Python\Python37\lib\site-packages\keras\legacy\interfaces.py", line 91, in wrapper
return func(*args, **kwargs)
File "C:\Users\lenovo\AppData\Local\Programs\Python\Python37\lib\site-packages\keras\engine\training.py", line 1732, in fit_generator
initial_epoch=initial_epoch)
File "C:\Users\lenovo\AppData\Local\Programs\Python\Python37\lib\site-packages\keras\engine\training_generator.py", line 42, in fit_generator
model._make_train_function()
File "C:\Users\lenovo\AppData\Local\Programs\Python\Python37\lib\site-packages\keras\engine\training.py", line 316, in _make_train_function
loss=self.total_loss)
File "C:\Users\lenovo\AppData\Local\Programs\Python\Python37\lib\site-packages\keras\legacy\interfaces.py", line 91, in wrapper
return func(*args, **kwargs)
File "C:\Users\lenovo\AppData\Local\Programs\Python\Python37\lib\site-packages\keras\optimizers.py", line 504, in get_updates
grads = self.get_gradients(loss, params)
File "C:\Users\lenovo\AppData\Local\Programs\Python\Python37\lib\site-packages\keras\optimizers.py", line 93, in get_gradients
raise ValueError('An operation has `None` for gradient. '
ValueError: An operation has `None` for gradient. Please make sure that all of your ops have a gradient defined (i.e. are differentiable). Common ops without gradient: K.argmax, K.round, K.eval.

Я использую пакет модели bert — keras-bert. Я пытался сделать берт с мягкой маской самостоятельно. Когда я работал над seq2seq с жесткой маской, я тоже получил ту же ошибку.
Мне нужна помощь по этой ошибке, потому что, насколько я знаю, keras.losses.categorical_crossentropy дифференцируема.

Подробнее здесь: https://stackoverflow.com/questions/646 ... t-all-of-y