Программа Python Long Rong - использование памяти увеличивается на неопределенный срок ⇐ Python

[Anonymous]

У меня есть простая программа Python, предназначенную для постоянного построения данных из часто обновления файла CSV, который предназначен для работы в течение нескольких месяцев. Упрощенную версию программы можно найти ниже (в моей реальной программе проводится дополнительные преобразования панд, чем то, что я здесь показываю, но это сущность программы): < /p>
import pandas
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation

fig, ax = plt.subplots()
x = np.arange(5) # x-axis: 5 bars
bars = ax.bar(x, np.zeros_like(x))

ax.set_ylim(0, 100)
ax.set_title("Real-Time Bar Chart")

def get_data():
df = pandas.read_csv('data/ztest_data.csv', header=None) # roughly 5.39 MB CSV file with 3 columns: string, integer, integer
df.columns = ['unit', 'value', 'selection']
df = df.loc[df['selection']
Я заметил, что использование памяти программы продолжает увеличиваться на неопределенный срок, но я не знаю почему. Использование gc.collect () в конце функции Update () не решает проблему. Вот пример использования памяти, о котором сообщалось, менеджер задач Windows после запуска в течение нескольких дней:

, рассматривая It Intend Is Intend Is Intend Is Intend Is Itrest On -thate -thate, но не идеал, но не идеал, что не идеален, что не сразу же, не на что -то не делать. Я здесь что -то делаю? Если нет, есть ли у вас какие -либо рекомендации для того, что я должен сделать? DataFile можно найти здесь: https://drive.google.com/file/d/16xrq3f ... sp=sharing
import os
import re
import gc
import glob
import pandas
import datetime
import numpy as np
import matplotlib.pyplot as mp
import matplotlib.dates as md
from matplotlib.animation import FuncAnimation

DEBUG_SIMULATE_PLOT_UPDATE = False
DEBUG_TOTAL_INTERVALS = 20
if DEBUG_SIMULATE_PLOT_UPDATE:
REFRESH_EVERY = 2
else:
REFRESH_EVERY = 5*60
YMIN, YMAX = (0.5, 22)

AMOUNTS = [154093, 139803*1.05/1.1, 92139, 136400, 115000]

mp.style.use('dark_background')

active_j_nums = [1,2,3,4,5]
jcols = {'J1':(1,0,0),'J2':(0,0.9,0),'J3':(0.9,0.9,0),'J4':(0,0,1),'J5':(0.3,0.3,0.3),'J6':(0.5,0,1),'J7':(1,0,1),'J8':(210/255,149/255,87/255)}

def float_hours_to_hhmm(hours_float):
hours = int(hours_float)
minutes = int(round((hours_float - hours) * 60))
return f"{hours:02d}:{minutes:02d}"

def get_data(ii):
weekdays = ['Monday', 'Tuesday', 'Wendesday', 'Thursday', 'Friday']
datafile = 'ztest_data.csv'
df = pandas.read_csv(datafile, header=None)
ind = min(len(df), ii*len(df)//DEBUG_TOTAL_INTERVALS)
if DEBUG_SIMULATE_PLOT_UPDATE:
df = df.iloc[:ind]
df = df.rename(columns={0:'name', 1:'time', 2:'selection'})
df = df.loc[df['name'] == 'HDMI_B08']

grouped = df.groupby((df['selection'] != df['selection'].shift()).cumsum(), as_index=False).agg(
selection=('selection',min),
mint=('time',min),
maxt=('time',max),
)
grouped['delta_secs'] = grouped['maxt'] - grouped['mint']
grouped['delta_mins'] = grouped['delta_secs']/60

x1 = grouped[['selection', 'mint']]
x2 = grouped[['selection', 'maxt']]
dff = pandas.concat([x1.rename(columns={'mint':'time'}), x2.rename(columns={'maxt':'time'})]).sort_values(['time', 'selection'])
values = list(range(1,9))
for val in values:
dff[f'J{val}'] = np.where(dff['selection']==val, 1, 0)
def calc_cutoff(x, hr, mn):
dt = datetime.datetime.fromtimestamp(x)
day_start = datetime.datetime(dt.year, dt.month, dt.day, hr, mn, 0)
return int(day_start.timestamp())

def calculate_valid_time(row):
min_ba = min(row['maxt'], row['day_end'])
max_ef = max(row['mint'], row['day_start'])
return max(min_ba - max_ef, 0)

grouped['day_start'] = grouped['mint'].apply(lambda ii: calc_cutoff(ii, 0, 10))
grouped['day_end'] = grouped['mint'].apply(lambda ii: calc_cutoff(ii, 11+12, 50))
grouped['valid_time_during_day'] = grouped.apply(calculate_valid_time, axis=1).astype(int)
grouped['day'] = grouped['mint'].apply(lambda row: datetime.datetime.fromtimestamp(row).strftime('%A'))
grouped['daysort'] = pandas.to_datetime(grouped['mint'].apply(lambda row: datetime.datetime.fromtimestamp(row).date()))
for which in ['mint', 'maxt', 'day_start', 'day_end']:
grouped[f'{which}_readable'] = grouped[which].apply(lambda row: datetime.datetime.fromtimestamp(row).strftime('%b %d, %Y, %I:%M:%S %p'))
grouped['week_num'] = grouped['mint'].apply(lambda ii : ((vv:=datetime.datetime.fromtimestamp(ii).isocalendar())[0],vv[1]))

agg_by_week = grouped.groupby(['week_num', 'selection'], as_index=False).agg(
total_valid_hours = ('valid_time_during_day', lambda ii : np.sum(ii)/(60*60)),
total_hours = ('delta_secs', lambda ii : np.sum(ii)/(60*60)),
)
agg_by_week = grouped.groupby(['week_num', 'selection'], as_index=False).agg(
total_valid_hours = ('valid_time_during_day', lambda ii : np.sum(ii)/(60*60)),
total_hours = ('delta_secs', lambda ii : np.sum(ii)/(60*60)),
)

week_num = agg_by_week['week_num'].iloc[0]
missing_ids = [x for x in active_j_nums if x not in agg_by_week["selection"].values]
ll = len(missing_ids)
new_rows = pandas.DataFrame({'week_num':[week_num for ii in range(ll)], "selection": missing_ids, 'total_valid_hours':[0 for ii in range(ll)], 'total_hours':[0 for ii in range(ll)]})
agg_by_week = pandas.concat([agg_by_week, new_rows], ignore_index=True)

agg_by_week = agg_by_week.loc[agg_by_week['selection'].isin(active_j_nums)].sort_values(by='selection')
return agg_by_week

if __name__ == '__main__':
counter = 1
agg_by_week = get_data(counter)
counter += 1

mp.rcParams['toolbar'] = 'None'
fig, ax = mp.subplots(layout='constrained', figsize=(3.5, 6), dpi=80)
ax.format_coord = lambda x, y: ""
x = agg_by_week['selection']
heights = agg_by_week['total_hours']
bars = ax.bar(x, heights, color=list(jcols.values()), zorder=2)
ymax = min(YMAX, max(YMIN, heights.max())) + 1

hline_values = [sum(heights) * pct / sum(AMOUNTS) for pct in AMOUNTS[:len(active_j_nums)]]
hlines = [ax.axhline(y=v, color=jcol, linestyle='--', linewidth=1.5, zorder=3) for v, jcol in zip(hline_values, jcols.values())]

ax.set_ylim(0, ymax)
ax.set_yticks(np.arange(0, ymax, 0.5))
ax.grid(True, which='both', axis='y', color='gray', linewidth=0.5, zorder=1)
ax2 = ax.twinx()
ax2.set_ylim(ax.get_ylim())
ax2.set_yticks(ax.get_yticks())
ax2.set_yticklabels(ax.get_yticklabels())
thw = float_hours_to_hhmm(sum(heights))
title_text = "Updated: {dt}\nAmount: {th}"
title = ax.set_title(title_text.format(dt = datetime.datetime.now().strftime('%b. %d, %Y at %I:%M %p'), th = thw), fontsize=10)

def update(frame):
global counter
try:
agg_by_week = get_data(counter)
except Exception as e:
return bars
new_heights = agg_by_week['total_hours']
for bar, h in zip(bars, new_heights):
bar.set_height(h)
new_hline_values = [sum(new_heights) * pct / sum(AMOUNTS) for pct in AMOUNTS[:len(active_j_nums)]]
for hline, new_val in zip(hlines, new_hline_values):
hline.set_ydata([new_val])
thw = float_hours_to_hhmm(sum(new_heights))
title.set_text(title_text.format(dt = datetime.datetime.now().strftime('%b. %d, %Y at %I:%M %p'), th = thw))
new_ymax = min(YMAX, max(YMIN, new_heights.max())) + 1
ax.set_ylim(0, new_ymax)
ax.set_yticks(np.arange(0, new_ymax, 0.5))
ax2.set_ylim(ax.get_ylim())
ax2.set_yticks(ax.get_yticks())
ax2.set_yticklabels(ax.get_yticklabels())
counter += 1
gc.collect() # testing if this fixes the memory growth issue (python process starts at consuuming 78 MB, over several days grows to consuming 130 MB) (2025-05-17 Update: It doesn't)
return bars

ani = FuncAnimation(fig, update, interval=1000*REFRESH_EVERY, blit=False, cache_frame_data=False)
mp.show()


Подробнее здесь: https://stackoverflow.com/questions/796 ... definitely