Я просто пробую/экспериментирую со sklearn. Я использую набор данных о жилье в Калифорнии и пытаюсь создать конвейер для создания некоторых дополнительных функций, затем логарифмировать некоторые функции, а затем масштабировать все числовые функции. Это довольно быстро и грязно, но вот что у меня получилось.
import pandas as pd
import numpy as np
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.preprocessing import FunctionTransformer, StandardScaler, OneHotEncoder
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
def safe_divide(a, b):
return np.where(b == 0, 0, a / b)
class AddCensoredFeatures(BaseEstimator, TransformerMixin):
def __init__(self, add_censored_age=True):
self.add_censored_age = add_censored_age
def fit(self, X, y=None):
if self.add_censored_age:
if "housing_median_age" not in X.columns:
raise ValueError("housing_median_age is not a column in this data.")
self.age_cap_ = X["housing_median_age"].max()
return self
def transform(self, X, y=None):
X = X.copy()
if self.add_censored_age:
X["censored_age"] = (X["housing_median_age"] == self.age_cap_).astype(int)
return X
def get_feature_names_out(self, input_features=None):
if self.add_censored_age:
return list(input_features) + ["censored_age"]
else:
return list(input_features)
class AddComboFeatures(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X_copy = X.copy()
required_columns = [
"total_rooms",
"total_bedrooms",
"population",
"median_income",
"households",
]
missing_columns = np.array(required_columns)[
np.isin(required_columns, X_copy.columns, invert=True)
]
if len(missing_columns) > 0:
raise ValueError(f"The following columns are missing: {missing_columns}")
X_copy["rooms_per_bedroom"] = safe_divide(
X_copy["total_rooms"], X_copy["total_bedrooms"]
)
X_copy["population_per_room"] = safe_divide(
X_copy["population"], X_copy["total_rooms"]
)
X_copy["income_per_house"] = safe_divide(
X_copy["median_income"], X_copy["households"]
)
X_copy["income_per_population"] = safe_divide(
X_copy["median_income"], X_copy["population"]
)
return X_copy
def get_feature_names_out(self, input_features=None):
return list(input_features) + [
"rooms_per_bedroom",
"population_per_room",
"income_per_house",
"income_per_population",
]
def log_transform(X):
return np.log(X)
def inv_log_transform(X):
return np.exp(X)
def get_columns():
log_like_columns = [
"total_rooms",
"total_bedrooms",
"population",
"households",
"median_income",
"population_per_room",
"income_per_house",
"income_per_population",
]
numerical_columns = [
"longitude",
"latitude",
"housing_median_age",
"total_rooms",
"total_bedrooms",
"population",
"households",
"median_income",
"rooms_per_bedroom",
"population_per_room",
"income_per_house",
"income_per_population",
]
remaining_columns = np.array(numerical_columns)[
np.isin(numerical_columns, log_like_columns, invert=True)
]
categorical_columns = ["ocean_proximity"]
return log_like_columns, remaining_columns, categorical_columns
def get_pipeline():
log_like_columns, remaining_columns, categorical_columns = get_columns()
log_transformer = FunctionTransformer(
log_transform, inverse_func=inv_log_transform, validate=False
)
log_and_scale = Pipeline(
[("log_transform", log_transformer), ("scale_transform", StandardScaler())]
)
full_pipeline = Pipeline(
[
(
"add_features",
Pipeline(
[
("add_censored_cols", AddCensoredFeatures()),
("add_combo_cols", AddComboFeatures()),
]
),
),
(
"preprocessing",
ColumnTransformer(
[
("log_and_scale_lognormal", log_and_scale, log_like_columns),
("scale_normal", StandardScaler(), remaining_columns),
(
"categorical_encoder",
OneHotEncoder(sparse_output=False),
categorical_columns,
),
],
remainder="passthrough",
),
),
]
)
return full_pipeline
def preprocess_data(df_X):
pipeline = get_pipeline()
pipeline = pipeline.set_output(transform="pandas")
df_X_prepped = pipeline.fit_transform(df_X)
df_X_prepped.columns = (
pd.Series(df_X_prepped.columns).str.split("__", expand=True).iloc[:, -1]
)
return df_X_prepped
if __name__ == "__main__":
import data
housing_path = r"Paths\projects\housing_project_ex\data\housing.csv"
df_housing = data.load_data(housing_path)
df_housing = data.clean_data(df_housing)
df_housing_y = df_housing["median_house_value"]
df_housing_X = df_housing.drop(["median_house_value"], axis=1)
df_housing_X_prepped = preprocess_data(df_housing_X)
print(df_housing_X_prepped.info())
print(df_housing_X_prepped.describe())
После того, как я подгоняю свой конвейер данных обучения.fit(train_X) (кажется, все идет нормально), я пытаюсь преобразовать свой конвейер тестовых данных.transform(test_X), но он выдает ошибку:
import data
import features
housing_path = r"Path\projects\housing_project_ex\data\housing.csv"
df_housing = data.load_data(housing_path)
df_housing = data.clean_data(df_housing)
train_X_raw, train_labels, test_X_raw, test_labels = train_test_split(df_housing)
pipeline = features.get_pipeline()
train_X = pipeline.fit_transform(train_X_raw)
test_X = pipeline.transform(test_X_raw)
Traceback (most recent call last):
File "c:\Users\caitl\projects\housing_project_ex\src\train.py", line 72, in
test_X = pipeline.transform(test_X_raw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\caitl\anaconda3\envs\housing_project\Lib\site-packages\sklearn\pipeline.py", line 1043, in transform
Xt = transform.transform(Xt, **routed_params[name].transform)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\caitl\anaconda3\envs\housing_project\Lib\site-packages\sklearn\pipeline.py", line 1035, in transform
check_is_fitted(self)
File "C:\Users\caitl\anaconda3\envs\housing_project\Lib\site-packages\sklearn\utils\validation.py", line 1705, in check_is_fitted
raise NotFittedError(msg % {"name": type(estimator).__name__})
sklearn.exceptions.NotFittedError: This Pipeline instance is not fitted yet. Call 'fit' with appropriate arguments before using this estimator.
Он не соответствует «add_combo_cols».
# test_X = pipeline.transform(test_X_raw)
from sklearn.utils.validation import check_is_fitted
from sklearn.exceptions import NotFittedError
check_is_fitted(pipeline)
for name, step in pipeline.named_steps["add_features"].named_steps.items():
try:
check_is_fitted(step)
print(f"Step '{name}' is fitted.")
except NotFittedError:
print(f"Step '{name}' is NOT fitted.")
Это печатает:
Step 'add_censored_cols' is fitted.
Step 'add_combo_cols' is NOT fitted.
dict_items([('add_censored_cols', AddCensoredFeatures()), ('add_combo_cols', AddComboFeatures())])
Я знаю, что это, вероятно, связано с тем, как я жестко запрограммировал столбцы и с тем, как ColumnTransfomer работает и передает информацию, но я не совсем понимаю это или как это исправить и предотвратить возникновение подобных ситуаций в будущем.
Изменить: вот остальная часть кода из data.py.
def load_data(filepath):
df_housing = pd.read_csv(filepath)
return df_housing
def clean_data(df):
bedrooms_median = df.total_bedrooms.median()
df["total_bedrooms"] = df["total_bedrooms"].fillna(bedrooms_median)
return df
