Использование серии Polars в качестве входных данных для Scikit Learn Tfidfvectorizer ⇐ Python

[Anonymous]

Мы рассмотрим добавление поддержки Polars в string_grouper (https://github.com/bergvca/string_grouper). Чтобы сделать эту работу, в качестве первого шага мы должны иметь возможность запустить TFIDFVectorizor в серии Polars. На веб-сайте Polars кажется, что это поддерживается: https://docs.pola.rs/user-guide/ecosyst ... ingобразно Я делаю что-то не так или на самом деле не поддерживается.from sklearn.feature_extraction.text import TfidfVectorizer

company_names = df.select(pl.col('Company Name'))
vectorizer = TfidfVectorizer(min_df=1)
tf_idf_matrix = vectorizer.fit_transform(company_names)

, где company_names :
print(company_names)

shape: (663_000, 1)
┌─────────────────────────────────┐
│ Company Name │
│ --- │
│ str │
╞═════════════════════════════════╡
│ !J INC │
│ #1 A LIFESAFER HOLDINGS, INC. │
│ #1 ARIZONA DISCOUNT PROPERTIES… │
│ #1 PAINTBALL CORP │
│ $ LLC │
< /code>
Результаты следующей ошибки: < /p>
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
Cell In[8], line 1
----> 1 tf_idf_matrix = vectorizer.fit_transform(company_names)

File ~/dev/python/sg_test/venv/lib/python3.12/site-packages/sklearn/feature_extraction/text.py:2104, in TfidfVectorizer.fit_transform(self, raw_documents, y)
2097 self._check_params()
2098 self._tfidf = TfidfTransformer(
2099 norm=self.norm,
2100 use_idf=self.use_idf,
2101 smooth_idf=self.smooth_idf,
2102 sublinear_tf=self.sublinear_tf,
2103 )
-> 2104 X = super().fit_transform(raw_documents)
2105 self._tfidf.fit(X)
2106 # X is already a transformed view of raw_documents so
2107 # we set copy to False

File ~/dev/python/sg_test/venv/lib/python3.12/site-packages/sklearn/base.py:1389, in _fit_context..decorator..wrapper(estimator, *args, **kwargs)
1382 estimator._validate_params()
1384 with config_context(
1385 skip_parameter_validation=(
1386 prefer_skip_nested_validation or global_skip_validation
1387 )
1388 ):
-> 1389 return fit_method(estimator, *args, **kwargs)

File ~/dev/python/sg_test/venv/lib/python3.12/site-packages/sklearn/feature_extraction/text.py:1376, in CountVectorizer.fit_transform(self, raw_documents, y)
1368 warnings.warn(
1369 "Upper case characters found in"
1370 " vocabulary while 'lowercase'"
1371 " is True. These entries will not"
1372 " be matched with any documents"
1373 )
1374 break
-> 1376 vocabulary, X = self._count_vocab(raw_documents, self.fixed_vocabulary_)
1378 if self.binary:
1379 X.data.fill(1)

File ~/dev/python/sg_test/venv/lib/python3.12/site-packages/sklearn/feature_extraction/text.py:1263, in CountVectorizer._count_vocab(self, raw_documents, fixed_vocab)
1261 for doc in raw_documents:
1262 feature_counter = {}
-> 1263 for feature in analyze(doc):
1264 try:
1265 feature_idx = vocabulary[feature]

File ~/dev/python/sg_test/venv/lib/python3.12/site-packages/sklearn/feature_extraction/text.py:104, in _analyze(doc, analyzer, tokenizer, ngrams, preprocessor, decoder, stop_words)
102 else:
103 if preprocessor is not None:
--> 104 doc = preprocessor(doc)
105 if tokenizer is not None:
106 doc = tokenizer(doc)

File ~/dev/python/sg_test/venv/lib/python3.12/site-packages/sklearn/feature_extraction/text.py:62, in _preprocess(doc, accent_function, lower)
43 """Chain together an optional series of text preprocessing steps to
44 apply to a document.
45
(...)
59 preprocessed string
60 """
61 if lower:
---> 62 doc = doc.lower()
63 if accent_function is not None:
64 doc = accent_function(doc)

AttributeError: 'Series' object has no attribute 'lower'



Подробнее здесь: https://stackoverflow.com/questions/796 ... vectorizer