Skip to content

412-Efficient influxdatabase connections #425

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
gopalangj wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

412-Efficient influxdatabase connections #425

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
35ce61b
Initial improvement (~0.3s faster) per call
gopalangj Feb 6, 2026
074271e
Changes after discussion
gopalangj Feb 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 38 additions & 19 deletions src/mesido/esdl/profile_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import rtctools.data.pi
from rtctools.data.storage import DataStore
from concurrent.futures import ThreadPoolExecutor

logger = logging.getLogger()

Expand Down Expand Up @@ -233,6 +234,7 @@ def __init__(
self._database_credentials = (
database_credentials if database_credentials is not None else {"": ("", "")}
)
self._database_profilemanager = list()

def _load_profiles_from_source(
self,
Expand Down Expand Up @@ -276,6 +278,10 @@ def _load_profiles_from_source(
)
unique_profiles.append(profile)

# # Open parallel processes to load all unique profiles parallely
# with ThreadPoolExecutor(max_workers=4) as executor:
# unique_series = list(executor.map(self._load_profile_timeseries_from_database, unique_profiles))
# executor.map(self._check_profile_time_series, unique_series, unique_profiles)
unique_series.append(
self._load_profile_timeseries_from_database(profile=unique_profiles[-1])
)
Expand Down Expand Up @@ -411,7 +417,7 @@ def _load_profile_timeseries_from_database(self, profile: esdl.InfluxDBProfile)
username, password = self._database_credentials.get(influx_host, (None, None))

conn_settings = ConnectionSettings(
host=profile.host,
host=profile_host,
port=profile.port,
username=username,
password=password,
Expand All @@ -420,25 +426,38 @@ def _load_profile_timeseries_from_database(self, profile: esdl.InfluxDBProfile)
verify_ssl=ssl_setting,
)

try:
time_series_data = InfluxDBProfileManager(conn_settings)
except Exception:
container = profile.eContainer()
asset = container.energyasset
get_potential_errors().add_potential_issue(
MesidoAssetIssueType.ASSET_PROFILE_AVAILABILITY,
asset.id,
f"Asset named {asset.name}: Database {profile.database}"
f" is not available in the host.",
)
potential_error_to_error(NetworkErrors.HEAT_NETWORK_ERRORS)

time_series_data.load_influxdb(
profile.measurement,
[profile.field],
profile.startDate,
profile.endDate,
)
# Check if an object of the InfluxDBProfileManager is already present in a list. If so,
# re-use that object that was already created and been stored in the list.
time_series_data = next(filter(lambda x:x.database_settings==conn_settings, self._database_profilemanager), None)
if not time_series_data:
try:
time_series_data = InfluxDBProfileManager(conn_settings)
time_series_data.load_influxdb(
profile.measurement,
[profile.field],
profile.startDate,
profile.endDate,
)
# Storing the InfluxDBProfileManager object in the list
self._database_profilemanager.append(time_series_data)
except Exception:
container = profile.eContainer()
asset = container.energyasset
get_potential_errors().add_potential_issue(
MesidoAssetIssueType.ASSET_PROFILE_AVAILABILITY,
asset.id,
f"Asset named {asset.name}: Database {profile.database}"
f" is not available in the host.",
)
potential_error_to_error(NetworkErrors.HEAT_NETWORK_ERRORS)
else:
time_series_data.load_influxdb(
profile.measurement,
[profile.field],
profile.startDate,
profile.endDate,
)

if not time_series_data.profile_data_list: # if time_series_data.profile_data_list == []:
container = profile.eContainer()
Expand Down