Получение исключения UnexpectedStatusException при обучении линейной модели в AWS Sagemaker? ⇐ Python

[Anonymous]

Я пытаюсь обучить линейную модель в AWS sagemaker, где типом предиктора является «регрессор».
Я застрял и понятия не имею, что мне делать.
Я загрузил свои данные в S3-ведро.
ВЫВОД:
UnexpectedStatusException: Error for Training job
LR-v1-2024-06-16-05-01-56-352: Failed. Reason: ClientError: No
iterator has been registered for ContentType ('csv', '1.0'), exit
code: 2

во время тренировки в sagemaker. Мои данные в правильном формате. то есть
target_variable, а затем другие функции. Насколько мне известно.
Как решить эту конкретную проблему, какие ошибки в коде приведут нас
к этой ошибке. Это тоже связано с настройками AWS?
Ниже приведен код для загрузки данных в корзину S3 в необходимом формате.
def export_data_lr(data, name, pre):
# split data into X and y subsets
X = data.drop(columns="price")
y = data.price.copy()

# transformation
X = pre.transform(X)

file_name = get_file_name_lr(name)
(
y
. to_frame()
.join(X)
.to_csv(file_name, index=False, header=False)
)

def upload_to_bucket_lr(name,BUCKET_NAME):
file_name = get_file_name_lr(name)

(
boto3
.Session()
.resource("s3")
.Bucket(BUCKET_NAME)
.Object(os.path.join(DATA_PREFIX, f"{name}/{name}-lr.csv"))
.upload_file(file_name)
)

До сих пор все работает нормально. Данные корректно загружаются в корзину S3 без каких-либо проблем.
В моей корзине s3 я могу видеть свои данные обучения, тестирования и проверки.
Ниже приведен код для сборки. Модель линейного обучения с типом предиктора является «регрессором».
Я не использую много гиперпараметров для простоты.
from sagemaker import image_uris
def return_lr_model(sagemaker,BUCKET_NAME,algorithm,version):

OUTPUT_PATH = get_output_path(BUCKET_NAME)
region = sagemaker.Session().boto_region_name
session = sagemaker.Session()
role = sagemaker.get_execution_role()
image_uri = image_uris.retrieve(region=boto3.Session().region_name, framework="linear-learner")

lr_model = Estimator(
image_uri = image_uri,
role = role,
instance_count = 1,
instance_type = "ml.m4.xlarge",
output_path = OUTPUT_PATH,
sagemaker_session = session,
base_job_name = 'LR-v1')
return lr_model

def perform_hyperparameter(lr_model):
lr_model.set_hyperparameters(
feature_dim = "auto",
predictor_type = "regressor",
normalize_data = False,
epochs = 15
)
return lr_model

def create_lr_model(sagemaker,BUCKET_NAME,algorithm,version):
lr_model = return_lr_model(sagemaker,BUCKET_NAME,algorithm,version)
lr_model = perform_hyperparameter(lr_model)
return lr_model

lr_model = create_lr_model(sagemaker,BUCKET_NAME,algorithm,version)

Код для создания канала данных
def get_data_channels(train,val):
train = get_file_name_lr(train)
bucket_path = f"s3://{BUCKET_NAME}/{DATA_PREFIX}/train/{train}"
train_data_channel = TrainingInput(bucket_path, content_type="csv")

val = get_file_name_lr(val)
bucket_path = f"s3://{BUCKET_NAME}/{DATA_PREFIX}/val/{val}"
val_data_channel = TrainingInput(bucket_path, content_type="text/csv")

data_channels = {
"train": train_data_channel,
"validation": val_data_channel
}

return data_channels

data_channel = get_data_channels("train","val")

Модель поезда:
lr_model.fit(data_channel)

Здесь начинается моя настоящая проблема. Ниже приведена ошибка, с которой я столкнулся при обучении модели:
INFO:sagemaker:Creating training-job with name: LR-v1-2024-06-16-05-01-56-352
2024-06-16 05:01:56 Starting - Starting the training job...
2024-06-16 05:02:17 Starting - Preparing the instances for training...
2024-06-16 05:02:48 Downloading - Downloading input data...
2024-06-16 05:03:23 Downloading - Downloading the training image.........
2024-06-16 05:04:29 Training - Training image download completed. Training in progress..Docker entrypoint called with argument(s): train
Running default environment configuration script
[06/16/2024 05:04:54 INFO 140601654445888] Reading default configuration from /opt/amazon/lib/python3.8/site-packages/algorithm/resources/default-input.json: {'mini_batch_size': '1000', 'epochs': '15', 'feature_dim': 'auto', 'use_bias': 'true', 'binary_classifier_model_selection_criteria': 'accuracy', 'f_beta': '1.0', 'target_recall': '0.8', 'target_precision': '0.8', 'num_models': 'auto', 'num_calibration_samples': '10000000', 'init_method': 'uniform', 'init_scale': '0.07', 'init_sigma': '0.01', 'init_bias': '0.0', 'optimizer': 'auto', 'loss': 'auto', 'margin': '1.0', 'quantile': '0.5', 'loss_insensitivity': '0.01', 'huber_delta': '1.0', 'num_classes': '1', 'accuracy_top_k': '3', 'wd': 'auto', 'l1': 'auto', 'momentum': 'auto', 'learning_rate': 'auto', 'beta_1': 'auto', 'beta_2': 'auto', 'bias_lr_mult': 'auto', 'bias_wd_mult': 'auto', 'use_lr_scheduler': 'true', 'lr_scheduler_step': 'auto', 'lr_scheduler_factor': 'auto', 'lr_scheduler_minimum_lr': 'auto', 'positive_example_weight_mult': '1.0', 'balance_multiclass_weights': 'false', 'normalize_data': 'true', 'normalize_label': 'auto', 'unbias_data': 'auto', 'unbias_label': 'auto', 'num_point_for_scaler': '10000', '_kvstore': 'auto', '_num_gpus': 'auto', '_num_kv_servers': 'auto', '_log_level': 'info', '_tuning_objective_metric': '', 'early_stopping_patience': '3', 'early_stopping_tolerance': '0.001', '_enable_profiler': 'false'}
[06/16/2024 05:04:54 INFO 140601654445888] Merging with provided configuration from /opt/ml/input/config/hyperparameters.json: {'epochs': '15', 'feature_dim': 'auto', 'normalize_data': 'False', 'predictor_type': 'regressor'}
[06/16/2024 05:04:54 INFO 140601654445888] Final configuration: {'mini_batch_size': '1000', 'epochs': '15', 'feature_dim': 'auto', 'use_bias': 'true', 'binary_classifier_model_selection_criteria': 'accuracy', 'f_beta': '1.0', 'target_recall': '0.8', 'target_precision': '0.8', 'num_models': 'auto', 'num_calibration_samples': '10000000', 'init_method': 'uniform', 'init_scale': '0.07', 'init_sigma': '0.01', 'init_bias': '0.0', 'optimizer': 'auto', 'loss': 'auto', 'margin': '1.0', 'quantile': '0.5', 'loss_insensitivity': '0.01', 'huber_delta': '1.0', 'num_classes': '1', 'accuracy_top_k': '3', 'wd': 'auto', 'l1': 'auto', 'momentum': 'auto', 'learning_rate': 'auto', 'beta_1': 'auto', 'beta_2': 'auto', 'bias_lr_mult': 'auto', 'bias_wd_mult': 'auto', 'use_lr_scheduler': 'true', 'lr_scheduler_step': 'auto', 'lr_scheduler_factor': 'auto', 'lr_scheduler_minimum_lr': 'auto', 'positive_example_weight_mult': '1.0', 'balance_multiclass_weights': 'false', 'normalize_data': 'False', 'normalize_label': 'auto', 'unbias_data': 'auto', 'unbias_label': 'auto', 'num_point_for_scaler': '10000', '_kvstore': 'auto', '_num_gpus': 'auto', '_num_kv_servers': 'auto', '_log_level': 'info', '_tuning_objective_metric': '', 'early_stopping_patience': '3', 'early_stopping_tolerance': '0.001', '_enable_profiler': 'false', 'predictor_type': 'regressor'}
/opt/amazon/lib/python3.8/site-packages/mxnet/model.py:97: SyntaxWarning: "is" with a literal. Did you mean "=="?
if num_device is 1 and 'dist' not in kvstore:
/opt/amazon/lib/python3.8/site-packages/scipy/optimize/_shgo.py:495: SyntaxWarning: "is" with a literal. Did you mean "=="?
if cons['type'] is 'ineq':
/opt/amazon/lib/python3.8/site-packages/scipy/optimize/_shgo.py:743: SyntaxWarning: "is not" with a literal. Did you mean "!="?
if len(self.X_min) is not 0:
[06/16/2024 05:04:57 WARNING 140601654445888] Loggers have already been setup.
[06/16/2024 05:04:57 INFO 140601654445888] Final configuration: {'mini_batch_size': '1000', 'epochs': '15', 'feature_dim': 'auto', 'use_bias': 'true', 'binary_classifier_model_selection_criteria': 'accuracy', 'f_beta': '1.0', 'target_recall': '0.8', 'target_precision': '0.8', 'num_models': 'auto', 'num_calibration_samples': '10000000', 'init_method': 'uniform', 'init_scale': '0.07', 'init_sigma': '0.01', 'init_bias': '0.0', 'optimizer': 'auto', 'loss': 'auto', 'margin': '1.0', 'quantile': '0.5', 'loss_insensitivity': '0.01', 'huber_delta': '1.0', 'num_classes': '1', 'accuracy_top_k': '3', 'wd': 'auto', 'l1': 'auto', 'momentum': 'auto', 'learning_rate': 'auto', 'beta_1': 'auto', 'beta_2': 'auto', 'bias_lr_mult': 'auto', 'bias_wd_mult': 'auto', 'use_lr_scheduler': 'true', 'lr_scheduler_step': 'auto', 'lr_scheduler_factor': 'auto', 'lr_scheduler_minimum_lr': 'auto', 'positive_example_weight_mult': '1.0', 'balance_multiclass_weights': 'false', 'normalize_data': 'False', 'normalize_label': 'auto', 'unbias_data': 'auto', 'unbias_label': 'auto', 'num_point_for_scaler': '10000', '_kvstore': 'auto', '_num_gpus': 'auto', '_num_kv_servers': 'auto', '_log_level': 'info', '_tuning_objective_metric': '', 'early_stopping_patience': '3', 'early_stopping_tolerance': '0.001', '_enable_profiler': 'false', 'predictor_type': 'regressor'}
[06/16/2024 05:04:57 WARNING 140601654445888] Loggers have already been setup.
Process 7 is a worker.
[06/16/2024 05:04:57 INFO 140601654445888] Using default worker.
[06/16/2024 05:04:57 INFO 140601654445888] Checkpoint loading and saving are disabled.
[06/16/2024 05:04:57 ERROR 140601654445888] Customer Error: No iterator has been registered for ContentType ('csv', '1.0')

2024-06-16 05:05:13 Uploading - Uploading generated training model
2024-06-16 05:05:13 Failed - Training job failed
---------------------------------------------------------------------------
UnexpectedStatusException Traceback (most recent call last)
Cell In[278], line 1
----> 1 lr_model.fit(data_channel)

File ~/anaconda3/envs/python3/lib/python3.10/site-packages/sagemaker/workflow/pipeline_context.py:346, in runnable_by_pipeline..wrapper(*args, **kwargs)
342 return context
344 return _StepArguments(retrieve_caller_name(self_instance), run_func, *args, **kwargs)
--> 346 return run_func(*args, **kwargs)

File ~/anaconda3/envs/python3/lib/python3.10/site-packages/sagemaker/estimator.py:1346, in EstimatorBase.fit(self, inputs, wait, logs, job_name, experiment_config)
1344 self.jobs.append(self.latest_training_job)
1345 if wait:
-> 1346 self.latest_training_job.wait(logs=logs)

File ~/anaconda3/envs/python3/lib/python3.10/site-packages/sagemaker/estimator.py:2703, in _TrainingJob.wait(self, logs)
2701 # If logs are requested, call logs_for_jobs.
2702 if logs != "None":
-> 2703 self.sagemaker_session.logs_for_job(self.job_name, wait=True, log_type=logs)
2704 else:
2705 self.sagemaker_session.wait_for_job(self.job_name)

File ~/anaconda3/envs/python3/lib/python3.10/site-packages/sagemaker/session.py:5797, in Session.logs_for_job(self, job_name, wait, poll, log_type, timeout)
5776 def logs_for_job(self, job_name, wait=False, poll=10, log_type="All", timeout=None):
5777 """Display logs for a given training job, optionally tailing them until job is complete.
5778
5779 If the output is a tty or a Jupyter cell, it will be color-coded
(...)
5795 exceptions.UnexpectedStatusException: If waiting and the training job fails.
5796 """
-> 5797 _logs_for_job(self, job_name, wait, poll, log_type, timeout)

File ~/anaconda3/envs/python3/lib/python3.10/site-packages/sagemaker/session.py:8026, in _logs_for_job(sagemaker_session, job_name, wait, poll, log_type, timeout)
8023 last_profiler_rule_statuses = profiler_rule_statuses
8025 if wait:
-> 8026 _check_job_status(job_name, description, "TrainingJobStatus")
8027 if dot:
8028 print()

File ~/anaconda3/envs/python3/lib/python3.10/site-packages/sagemaker/session.py:8079, in _check_job_status(job, desc, status_key_name)
8073 if "CapacityError" in str(reason):
8074 raise exceptions.CapacityError(
8075 message=message,
8076 allowed_statuses=["Completed", "Stopped"],
8077 actual_status=status,
8078 )
-> 8079 raise exceptions.UnexpectedStatusException(
8080 message=message,
8081 allowed_statuses=["Completed", "Stopped"],
8082 actual_status=status,
8083 )

UnexpectedStatusException: Error for Training job LR-v1-2024-06-16-05-01-56-352: Failed. Reason: ClientError: No iterator has been registered for ContentType ('csv', '1.0'), exit code: 2


Подробнее здесь: https://stackoverflow.com/questions/786 ... -sagemaker