Конвейер данных для сбора всех ежемесячных банковских выписок из почты, к которой подключено несколько банков. ⇐ Python

[Anonymous]

Я занимаюсь аналитиком данных и работаю над проектом, в котором извлекаю таблицы из ежемесячных выписок по счетам, отправленных по почте из разных банков.
У PDF-файлов разные пароли, а для некоторых он не требуется.
Из-за различий в банках они также имеют разные заголовки столбцов.
Скрипт смог подключиться к почтовому аккаунту и загрузить соответствующие PDF-файлы.
Проблема в том, что скрипт извлекает только 2-я страница третьего PDF-файла, загруженного в папку с помощью того же сценария.
Я пробовал OCR, pdfimage и т. д., но все равно не работает
Пожалуйста, помогите, спасибо.
import pandas as pd
from pathlib import Path
from pdf2image import convert_from_path
import pytesseract
import re
import cv2
import numpy as np
from PIL import Image
import pikepdf # NEW: for decryption
import tempfile
import os

# ===================== CONFIG =====================
OUTPUT_DIR = Path("statements")
OUTPUT_DIR.mkdir(exist_ok=True)
CSV_FILE = Path("C:/Users/EMMAN/Downloads/all_statements_ALL_PAGES.csv")

PASSWORD_LIST = ["xxxx","xx"]

# ===================== DECRYPT PDF =====================
def decrypt_pdf(pdf_path, password=None):
"""Decrypt PDF to temp file and return path"""
temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".pdf")
temp_path = Path(temp_file.name)
temp_file.close()

try:
with pikepdf.open(pdf_path, password=password) as pdf:
pdf.save(temp_path)
print(f" Decrypted → {temp_path.name}")
return temp_path
except Exception as e:
print(f" Decrypt failed: {e}")
temp_path.unlink(missing_ok=True)
return None

# ===================== IMAGE PREPROCESSING =====================
def preprocess_image(image):
img = np.array(image)
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
clahe = cv2.createCLAHE(clipLimit=3.0, tileGridSize=(8,8))
enhanced = clahe.apply(gray)
_, binary = cv2.threshold(enhanced, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
return Image.fromarray(binary)

# ===================== OCR =====================
def ocr_page_to_text(image):
processed = preprocess_image(image)
config = r'--oem 3 --psm 6'
return pytesseract.image_to_string(processed, lang='eng', config=config)

def extract_table_from_text(text):
lines = [l.strip() for l in text.splitlines() if l.strip()]
rows = []
for line in lines:
cols = re.split(r'\s{2,}', line)
if len(cols) >= 3:
rows.append(cols)
if len(rows) < 2: return None
return pd.DataFrame(rows[1:], columns=rows[0])

# ===================== CLEAN DATA =====================
def clean_dataframe(df):
if df is None or df.empty: return pd.DataFrame()
df.columns = [c.strip().upper() for c in df.columns]
col_map = {
"DATE": "POSTING DATE", "TXN DATE": "POSTING DATE", "POSTING DATE": "POSTING DATE",
"VALUE DATE": "VALUE DATE",
"DESCRIPTION": "DESCRIPTION", "NARRATION": "DESCRIPTION",
"DEBIT": "OUTFLOW", "DR": "OUTFLOW",
"CREDIT": "INFLOW", "CR": "INFLOW",
"BALANCE": "BALANCE"
}
df.rename(columns=col_map, inplace=True)
wanted = ["POSTING DATE", "VALUE DATE", "DESCRIPTION", "OUTFLOW", "INFLOW", "BALANCE"]
df = df.reindex(columns=wanted)
return df

# ===================== EXTRACT ONE PDF =====================
def extract_from_pdf(pdf_path):
print(f"\nExtracting: {pdf_path.name}")
decrypted_path = None
images = []

# Try decrypting with password
for pwd in [None] + PASSWORD_LIST:
decrypted_path = decrypt_pdf(pdf_path, pwd)
if decrypted_path:
break

if not decrypted_path:
print(" Could not decrypt PDF")
return None

# Convert decrypted PDF to images
try:
print(" Converting to images...")
images = convert_from_path(str(decrypted_path), dpi=400)
print(f" {len(images)} pages converted")
except Exception as e:
print(f" Image conversion failed: {e}")
decrypted_path.unlink(missing_ok=True)
return None

all_page_dfs = []
for i, img in enumerate(images):
print(f" Page {i+1}/{len(images)} → OCR", end="")
text = ocr_page_to_text(img)
if text.strip():
df = extract_table_from_text(text)
df = clean_dataframe(df)
if df is not None and not df.empty:
all_page_dfs.append(df)
print(f" → {len(df)} rows")
else:
print(" → No table")
else:
print(" → No text")

# Cleanup
decrypted_path.unlink(missing_ok=True)

if not all_page_dfs:
return None

final_df = pd.concat(all_page_dfs, ignore_index=True)
print(f" TOTAL: {len(final_df)} rows")
return final_df

# ===================== MAIN =====================
def main():
print("\nBANK STATEMENT EXTRACTOR – ALL PAGES".center(80, "="))

all_pdfs = list(OUTPUT_DIR.glob("*.pdf"))
if not all_pdfs:
print("No PDFs in 'statements/' folder.")
return

all_dfs = []
for pdf_path in all_pdfs:
df = extract_from_pdf(pdf_path)
if df is not None:
all_dfs.append(df)

if all_dfs:
final = pd.concat(all_dfs, ignore_index=True)
final.to_csv(CSV_FILE, mode="w", index=False)
print(f"\nSUCCESS! {len(final)} rows → {CSV_FILE}")
print(" Open in Excel now!")
else:
print("\nNo data extracted.")

main()


Подробнее здесь: https://stackoverflow.com/questions/798 ... as-multipl