Управляя неправильным импортом токенизатора Tensorflow и Tensorflow Padded_sexences ⇐ Python

[Anonymous]

У меня есть нейронная сеть, которая получает данные из файла TXT и использует NLP, чтобы узнать, как говорить как человек. Но всякий раз, когда я загружаю токенизатор и Padded_ Sequences, (которые оба необходимы)
они неправильно импортируют. Возможно, мне придется тестировать мой код в свежей виртуальной машине, чтобы он работал. < /P>
Вот мой код: < /p>
import tensorflow as tf
import numpy as np
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
import importlib

if importlib.util.find_spec("tensorflow.keras.preprocessing.text.Tokenizer") is not None:
print("The Tokenizer class has been imported successfully.")
else:
print("The Tokenizer class has not been imported successfully.")

if importlib.util.find_spec("tensorflow.keras.preprocessing.text.pad_sequences") is not None:
print("The pad_sequences class has been imported successfully.")
else:
print("The pad_sequences class has not been imported successfully.")

# Load the text dataset
with open('data.txt', 'r') as f:
data = f.read()

# Split the data into sentences
sentences = data.split('.')

# Create a tokenizer and fit on the sentences
tokenizer = Tokenizer(filters='')
tokenizer.fit_on_texts(sentences)

# Convert the sentences to sequences of integers
sequences = tokenizer.texts_to_sequences(sentences)

# Create input and target sequences
input_sequences = []
target_sequences = []
for sequence in sequences:
for i in range(1, len(sequence)):
input_sequence = sequence[:i]
target_sequence = sequence
input_sequences.append(input_sequence)
target_sequences.append(target_sequence)

# Pad the input sequences
max_sequence_length = max([len(sequence) for sequence in input_sequences])
padded_input_sequences = pad_sequences(input_sequences, maxlen=max_sequence_length)

# Convert the target sequences to one-hot vectors
one_hot_target_sequences = tf.keras.utils.to_categorical(target_sequences, num_classes=len(tokenizer.word_index)+1)

# Create the neural network
model = tf.keras.Sequential([
tf.keras.layers.Embedding(len(tokenizer.word_index)+1, 100),
tf.keras.layers.LSTM(128),
tf.keras.layers.Dense(len(tokenizer.word_index)+1, activation='softmax')
])

# Train the neural network
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
model.fit(padded_input_sequences, one_hot_target_sequences, epochs=10)

# Generate text
def generate_text(model, tokenizer, max_sequence_length, start_text):
# Create a sequence of tokens
tokens = tokenizer.texts_to_sequences([start_text])[0]

# Generate text until the end of the sentence is reached or max sequence length is reached
while len(tokens) < max_sequence_length:
# Pad the input sequence
padded_sequence = tf.keras.preprocessing.sequence.pad_sequences([tokens], maxlen=max_sequence_length)

# Get the probability distribution for the next token
probabilities = model.predict(padded_sequence)[0]

# Choose the next token with the highest probability
next_token = np.argmax(probabilities)

# Add the next token to the sequence
tokens.append(next_token)

# Check if the end of the sentence has been reached
if next_token == tokenizer.word_index['.'] or len(tokens) == max_sequence_length:
break

# Convert the tokens back to text
generated_text = tokenizer.sequences_to_texts([tokens])[0]
return generated_text

# Generate some text
generated_text = generate_text(model, tokenizer, max_sequence_length, 'Hello, my name is Bard.')
print(generated_text)
< /code>
Изображение:
изображение из Pycharm подчеркивает код в красном < /p>

Подробнее здесь: https://stackoverflow.com/questions/760 ... -padded-se