Как исправить ошибку etl1 для обновления данных о запасах? ⇐ Python

[Гость]

Я пытаюсь создать etl1 для загрузки ежедневных данных из yhaoo Finance и загрузки новых данных в базу данных sqlite. Система способна правильно создать БД, если ее нет, и выполнить первую загрузку данных. Но когда необходимо сделать обновление, оно не работает. Я думаю, проблема в каком-то условии и, возможно, в формате даты.
Вот код:

Код: Выделить всё

import os
import pandas as pd
import yfinance as yf
import logging
from datetime import datetime, timedelta
from sqlalchemy import create_engine, text
import sqlalchemy
from engine.utils import clean_asset_string

class DataDownloader():

def __init__(self, ASSET='TSLA'):
self.ASSET = ASSET

def download_and_save_data(self,interval='1d'):

period = '10y'
if interval[-1] == 'h':
period = '719d'

logging.info("Data download in progress")

data = yf.download(tickers=self.ASSET, period=period, interval=interval)
data.sort_index(ascending=True, inplace=True)

if interval[-1] == 'h':
data = data.reset_index()
data['Date'] = data['Datetime']
data['Date'] = data['Date'].astype(str)
data['Date'] = data['Date'].str[:-6]
data = data.set_index('Date')
cols_to_delete = [
'Adj Close',
'Datetime',
]
else:
cols_to_delete = [
'Adj Close',
]
data = data.drop(columns=cols_to_delete)

#managing last available date. We want candles when markets has generated them.
last_date_available = data.index[-1]
#print('last_date_available', last_date_available)

current_date = datetime.now()

if interval[-1] == 'd':
current_date = current_date.replace(hour=0, minute=0, second=0, microsecond=0)
if isinstance(last_date_available, str):
last_date_available = datetime.strptime(last_date_available, '%Y-%m-%d')

elif interval[-1] == 'h':
current_date = current_date.replace(minute=0, second=0, microsecond=0)
if isinstance(last_date_available, str):
last_date_available = datetime.strptime(last_date_available, '%Y-%m-%d %H:%M:%S')

if current_date == last_date_available:
data = data[:-1]
elif current_date < last_date_available:
raise Exception("Something is wrong with dates during download.  Please verify with data provider.")

self.ASSET = clean_asset_string(self.ASSET) #special characters create problems in db
#print(self.ASSET)

#when we switch to db
file_dir = os.path.dirname(os.path.abspath(__file__))
data_folder = 'staging1_download\\'
database_file_name = 'staging1_download.sqlite'
databasePathName = os.path.join(file_dir, data_folder, database_file_name)

if interval[-1] == 'd':
tableName = 'staging1_download_DailyData'
elif interval[-1] == 'h':
tableName = 'staging1_download_HourlyData'

#print(databasePathName)
#print(os.path.isfile(databasePathName))

data["ASSET"] = self.ASSET
data["INTERVAL"] = interval

#if not os.path.isfile(databasePathName):
#engine = create_engine(f'sqlite:///{databasePathName}')

engine = create_engine(f'sqlite:///{databasePathName}', echo=True)
if not sqlalchemy.inspect(engine).has_table(tableName):
data.to_sql(tableName, con=engine, index=True)

else:
con = engine.connect()
query = text(f"""
SELECT *
FROM {tableName}
WHERE ASSET=:asset AND INTERVAL=:interval_value
""")

storedData = pd.read_sql_query(query, con=con, params={'asset': self.ASSET, 'interval_value': interval})

if storedData.shape[0]==0 or storedData.empty:
data.to_sql(tableName, con=engine, if_exists='append',  index=True)

else:

availableDates = storedData["Date"].to_list()
#print('availableDates', availableDates)
availableDates = pd.to_datetime(availableDates, format='%Y-%m-%d')
#storedData.index = storedData.index.astype(str)
#availableDates = availableDates.astype(str)

dataToStore = data[ ~storedData.index.isin(availableDates) &(self.ASSET==storedData["ASSET"].unique()[0]) & (interval==storedData["INTERVAL"].unique()[0]) ]
dataToStore.to_sql(tableName, con=engine, if_exists='append',  index=True)

logging.info("Data download executed")

Я думаю, проблема здесь:

Код: Выделить всё

                availableDates = storedData["Date"].to_list()
#print('availableDates', availableDates)
availableDates = pd.to_datetime(availableDates, format='%Y-%m-%d')
#storedData.index = storedData.index.astype(str)
#availableDates = availableDates.astype(str)

dataToStore = data[ ~storedData.index.isin(availableDates) & (self.ASSET==storedData["ASSET"].unique()[0]) & (interval==storedData["INTERVAL"].unique()[0]) ]
dataToStore.to_sql(tableName, con=engine, if_exists='append',  index=True)

Но все, что я делал, не работает.
Есть идеи?

Подробнее здесь: https://stackoverflow.com/questions/781 ... stock-data