diff --git a/.dockerignore b/.dockerignore index 5157228..6bc898f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,26 @@ +.github/ .venv/ +.vscode/ .idea/ ci/ -.env +doc/ +gurobi/ +htmlcov/ +local_test/ *.egg-info/ -temp/ +unit_test/ +.env +.env.local +.env-template +.gitignore +.taplo.toml +CHANGELOG.md +CONTRIBUTING.md +dev.Dockerfile +docker-compose.worker-only.yml +docker-compose.yml +LICENSE +run.bat +run.sh +run_windows.sh +test-results.xml diff --git a/.gitignore b/.gitignore index 12eba45..5f197c2 100644 --- a/.gitignore +++ b/.gitignore @@ -223,4 +223,6 @@ test-results.xml .env.* -temp/ \ No newline at end of file +temp/ +gurobi/ +.idea/ diff --git a/Dockerfile b/Dockerfile index 47f56de..eb0e7ea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,26 @@ FROM python:3.10-slim-buster WORKDIR /app +RUN apt update && \ + apt install -y wget && \ + apt-get clean + +ENV GUROBI_HOME=/app/gurobi/ +ENV PATH="/app/gurobi/bin:$PATH" +ENV LD_LIBRARY_PATH="/app/gurobi/lib" +ENV CASADI_GUROBI_VERSION="110" +ENV GUROBI_VERSION_URL="11.0.3" +ENV GUROBI_VERSION_URL_MAJOR="11.0" + +# This env var is for casadi. +ENV GUROBI_VERSION="$CASADI_GUROBI_VERSION" + +# This env var is for gurobi +ENV GRB_LICENSE_FILE="/app/gurobi/gurobi.lic" + + +RUN mkdir -p /app/gurobi && \ + wget -qO- https://packages.gurobi.com/${GUROBI_VERSION_URL_MAJOR}/gurobi${GUROBI_VERSION_URL}_linux64.tar.gz | tar xvz --strip-components=2 -C /app/gurobi COPY requirements.txt /app/grow_worker/requirements.txt RUN pip install -r /app/grow_worker/requirements.txt --no-cache-dir diff --git a/dev-requirements.txt b/dev-requirements.txt index 730f1c0..14c0d7d 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -39,7 +39,7 @@ celery==5.5.3 # via # -c requirements.txt # omotes-sdk-python -certifi==2025.4.26 +certifi==2025.6.15 # via # -c requirements.txt # requests @@ -59,7 +59,7 @@ click-didyoumean==0.3.1 # via # -c requirements.txt # celery -click-plugins==1.1.1 +click-plugins==1.1.1.2 # via # -c requirements.txt # celery @@ -134,7 +134,7 @@ msgpack==1.1.1 # via # -c requirements.txt # influxdb -multidict==6.4.4 +multidict==6.5.1 # via # -c requirements.txt # yarl @@ -156,7 +156,7 @@ omotes-sdk-protocol==1.2.0 # via # -c requirements.txt # omotes-sdk-python -omotes-sdk-python==4.2.1 +omotes-sdk-python==4.3.1 # via # -c requirements.txt # omotes-grow-worker (pyproject.toml) @@ -303,7 +303,7 @@ tzdata==2025.2 # via # -c requirements.txt # kombu -urllib3==2.4.0 +urllib3==2.5.0 # via # -c requirements.txt # requests diff --git a/dev.Dockerfile b/dev.Dockerfile index c98436e..9aba20f 100644 --- a/dev.Dockerfile +++ b/dev.Dockerfile @@ -1,6 +1,26 @@ FROM python:3.10-slim-buster WORKDIR /app +RUN apt update && \ + apt install -y wget && \ + apt-get clean + +ENV GUROBI_HOME=/app/gurobi/ +ENV PATH="/app/gurobi/bin:$PATH" +ENV LD_LIBRARY_PATH="/app/gurobi/lib" +ENV CASADI_GUROBI_VERSION="110" +ENV GUROBI_VERSION_URL="11.0.3" +ENV GUROBI_VERSION_URL_MAJOR="11.0" + +# This env var is for casadi. +ENV GUROBI_VERSION="$CASADI_GUROBI_VERSION" + +# This env var is for gurobi +ENV GRB_LICENSE_FILE="/app/gurobi/gurobi.lic" + + +RUN mkdir -p /app/gurobi && \ + wget -qO- https://packages.gurobi.com/${GUROBI_VERSION_URL_MAJOR}/gurobi${GUROBI_VERSION_URL}_linux64.tar.gz | tar xvz --strip-components=2 -C /app/gurobi COPY optimizer-worker/requirements.txt /app/grow_worker/requirements.txt RUN pip install -r /app/grow_worker/requirements.txt --no-cache-dir diff --git a/docker-compose.worker-only.dev.yml b/docker-compose.worker-only.dev.yml new file mode 100644 index 0000000..676732f --- /dev/null +++ b/docker-compose.worker-only.dev.yml @@ -0,0 +1,34 @@ +networks: + omotes: + external: true + +services: + grow_worker: + build: + context: .. + dockerfile: optimizer-worker/dev.Dockerfile + environment: + INFLUXDB_HOSTNAME: omotes_influxdb + INFLUXDB_PORT: 8096 + INFLUXDB_USERNAME: root + INFLUXDB_PASSWORD: 9012 + + RABBITMQ_HOSTNAME: rabbitmq-nwn + RABBITMQ_PORT: 5672 + RABBITMQ_USERNAME: root + RABBITMQ_PASSWORD: 5678 + RABBITMQ_VIRTUALHOST: omotes_celery + + GROW_TASK_TYPE: grow_optimizer_no_heat_losses_gurobi + LOG_LEVEL: INFO + volumes: + - "./gurobi/gurobi.lic:/app/gurobi/gurobi.lic" + networks: + - omotes + deploy: + replicas: 1 + resources: + limits: + cpus: '4' + memory: 4gb + memswap_limit: 4gb diff --git a/docker-compose.worker-only.yml b/docker-compose.worker-only.yml new file mode 100644 index 0000000..2c54610 --- /dev/null +++ b/docker-compose.worker-only.yml @@ -0,0 +1,33 @@ +networks: + omotes: + external: true + +services: + grow_worker: + build: + context: . + environment: + INFLUXDB_HOSTNAME: omotes_influxdb + INFLUXDB_PORT: 8096 + INFLUXDB_USERNAME: root + INFLUXDB_PASSWORD: 9012 + + RABBITMQ_HOSTNAME: rabbitmq-nwn + RABBITMQ_PORT: 5672 + RABBITMQ_USERNAME: root + RABBITMQ_PASSWORD: 5678 + RABBITMQ_VIRTUALHOST: omotes_celery + + GROW_TASK_TYPE: grow_optimizer_default + LOG_LEVEL: DEBUG + volumes: + - "./gurobi/gurobi.lic:/app/gurobi/gurobi.lic" + networks: + - omotes + deploy: + replicas: 3 + resources: + limits: + cpus: '4' + memory: 4gb + memswap_limit: 4gb diff --git a/pyproject.toml b/pyproject.toml index 7180501..33a040a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ requires-python = ">=3.10" dependencies = [ "python-dotenv ~= 1.0.0", "mesido ~= 0.1.12", - "omotes-sdk-python ~= 4.2.1" + "omotes-sdk-python ~= 4.3.1" ] [project.optional-dependencies] diff --git a/requirements.txt b/requirements.txt index e8f91e7..ac8578d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ casadi-gil-comp==3.6.7 # rtc-tools-gil-comp celery==5.5.3 # via omotes-sdk-python -certifi==2025.4.26 +certifi==2025.6.15 # via requests charset-normalizer==3.4.2 # via requests @@ -32,7 +32,7 @@ click==8.2.1 # click-repl click-didyoumean==0.3.1 # via celery -click-plugins==1.1.1 +click-plugins==1.1.1.2 # via celery click-repl==0.3.0 # via celery @@ -56,7 +56,7 @@ mesido==0.1.12 # via omotes-grow-worker (pyproject.toml) msgpack==1.1.1 # via influxdb -multidict==6.4.4 +multidict==6.5.1 # via yarl numpy==1.25.2 # via @@ -67,7 +67,7 @@ numpy==1.25.2 # scipy omotes-sdk-protocol==1.2.0 # via omotes-sdk-python -omotes-sdk-python==4.2.1 +omotes-sdk-python==4.3.1 # via omotes-grow-worker (pyproject.toml) ordered-set==4.1.0 # via pyecore @@ -133,7 +133,7 @@ typing-extensions==4.14.0 # omotes-sdk-python tzdata==2025.2 # via kombu -urllib3==2.4.0 +urllib3==2.5.0 # via requests vine==5.1.0 # via diff --git a/run.sh b/run.sh index 29b426f..4403dae 100755 --- a/run.sh +++ b/run.sh @@ -3,4 +3,16 @@ . .venv/bin/activate . ci/linux/_load_dot_env.sh .env +export DIR_TO_ROOT="$PWD" + +export GUROBI_HOME="${DIR_TO_ROOT}/gurobi" +export PATH="${GUROBI_HOME}/bin:$PATH" +export LD_LIBRARY_PATH="${GUROBI_HOME}/lib" +export CASADI_GUROBI_VERSION="110" +export GUROBI_VERSION_URL="11.0.3" +export GUROBI_VERSION_URL_MAJOR="11.0" + +export GUROBI_VERSION="$CASADI_GUROBI_VERSION" +export GRB_LICENSE_FILE="${DIR_TO_ROOT}/gurobi.lic" + PYTHONPATH="src/" python3 -m grow_worker.worker diff --git a/src/grow_worker/worker.py b/src/grow_worker/worker.py index 7cc3cb1..9400ca1 100644 --- a/src/grow_worker/worker.py +++ b/src/grow_worker/worker.py @@ -19,11 +19,12 @@ GROWProblem, get_problem_type, get_problem_function, + get_solver_class, ) logger = logging.getLogger("grow_worker") -GROW_TASK_TYPE = GrowTaskType(os.environ.get("GROW_TASK_TYPE")) +GROW_TASK_TYPES = [GrowTaskType(task_type) for task_type in os.environ["GROW_TASK_TYPE"].split(",")] class EarlySystemExit(Exception): @@ -37,7 +38,10 @@ class EarlySystemExit(Exception): def grow_worker_task( - input_esdl: str, workflow_config: ProtobufDict, update_progress_handler: UpdateProgressHandler + input_esdl: str, + workflow_config: ProtobufDict, + update_progress_handler: UpdateProgressHandler, + workflow_type_name: str, ) -> Tuple[Optional[str], List[EsdlMessage]]: """Run the grow worker task and run configured specific problem type for this worker instance. @@ -49,10 +53,13 @@ def grow_worker_task( :param input_esdl: The input ESDL XML string. :param workflow_config: Extra parameters to configure this run. :param update_progress_handler: Handler to notify of any progress changes. + :param workflow_type_name: Name of the workflow. :return: GROW optimized or simulated ESDL and a list of ESDL feedback messages. """ - mesido_func = get_problem_function(GROW_TASK_TYPE) - mesido_workflow = get_problem_type(GROW_TASK_TYPE) + workflow_type = GrowTaskType(workflow_type_name) + mesido_func = get_problem_function(workflow_type) + mesido_workflow = get_problem_type(workflow_type) + mesido_solver = get_solver_class(workflow_type) base_folder = Path(__file__).resolve().parent.parent write_result_db_profiles = "INFLUXDB_HOSTNAME" in os.environ @@ -68,6 +75,7 @@ def grow_worker_task( try: solution: GROWProblem = mesido_func( mesido_workflow, + solver_class=mesido_solver, base_folder=base_folder, esdl_string=base64.encodebytes(input_esdl.encode("utf-8")), esdl_parser=ESDLStringParser, @@ -123,4 +131,4 @@ def parse_mesido_esdl_messages( if __name__ == "__main__": - initialize_worker(GROW_TASK_TYPE.value, grow_worker_task) + initialize_worker([task_type.value for task_type in GROW_TASK_TYPES], grow_worker_task) diff --git a/src/grow_worker/worker_types.py b/src/grow_worker/worker_types.py index 7bbaca8..23014ae 100644 --- a/src/grow_worker/worker_types.py +++ b/src/grow_worker/worker_types.py @@ -5,6 +5,8 @@ from mesido.workflows.grow_workflow import ( EndScenarioSizingHeadLossStaged, EndScenarioSizingStaged, + SolverGurobi, + SolverHIGHS, ) from mesido.workflows import ( run_end_scenario_sizing, @@ -16,13 +18,22 @@ class GrowTaskType(Enum): """Grow task types.""" GROW_OPTIMIZER_DEFAULT = "grow_optimizer_default" - """Run the Grow Optimizer.""" + """Run the Grow Optimizer with HIGHS solver.""" GROW_SIMULATOR = "grow_simulator" - """Run the Grow Simulator.""" + """Run the Grow Simulator with HIGHS solver.""" GROW_OPTIMIZER_NO_HEAT_LOSSES = "grow_optimizer_no_heat_losses" - """Run the Grow Optimizer without heat losses.""" + """Run the Grow Optimizer without heat losses with HIGHS solver.""" GROW_OPTIMIZER_WITH_PRESSURE = "grow_optimizer_with_pressure" - """Run the Grow Optimizer with pump pressure.""" + """Run the Grow Optimizer with pump pressure with HIGHS solver.""" + + GROW_OPTIMIZER_DEFAULT_GUROBI = "grow_optimizer_default_gurobi" + """Run the Grow Optimizer with Gurobi solver.""" + GROW_SIMULATOR_GUROBI = "grow_simulator_gurobi" + """Run the Grow Simulator with Gurobi solver.""" + GROW_OPTIMIZER_NO_HEAT_LOSSES_GUROBI = "grow_optimizer_no_heat_losses_gurobi" + """Run the Grow Optimizer without heat losses with Gurobi solver.""" + GROW_OPTIMIZER_WITH_PRESSURE_GUROBI = "grow_optimizer_with_pressure_gurobi" + """Run the Grow Optimizer with pump pressure with Gurobi solver.""" GROWProblem = Union[ @@ -39,13 +50,25 @@ def get_problem_type(task_type: GrowTaskType) -> GROWProblem: :return: Grow problem class. """ result: GROWProblem - if task_type == GrowTaskType.GROW_OPTIMIZER_DEFAULT: + if task_type in [ + GrowTaskType.GROW_OPTIMIZER_DEFAULT, + GrowTaskType.GROW_OPTIMIZER_DEFAULT_GUROBI, + ]: result = EndScenarioSizingStaged - elif task_type == GrowTaskType.GROW_SIMULATOR: + elif task_type in [ + GrowTaskType.GROW_SIMULATOR, + GrowTaskType.GROW_SIMULATOR_GUROBI, + ]: result = NetworkSimulatorHIGHSWeeklyTimeStep - elif task_type == GrowTaskType.GROW_OPTIMIZER_NO_HEAT_LOSSES: + elif task_type in [ + GrowTaskType.GROW_OPTIMIZER_NO_HEAT_LOSSES, + GrowTaskType.GROW_OPTIMIZER_NO_HEAT_LOSSES_GUROBI, + ]: result = EndScenarioSizingStaged - elif task_type == GrowTaskType.GROW_OPTIMIZER_WITH_PRESSURE: + elif task_type in [ + GrowTaskType.GROW_OPTIMIZER_WITH_PRESSURE, + GrowTaskType.GROW_OPTIMIZER_WITH_PRESSURE_GUROBI, + ]: result = EndScenarioSizingHeadLossStaged else: raise RuntimeError(f"Unknown workflow type, please implement {task_type}") @@ -66,11 +89,43 @@ def get_problem_function( GrowTaskType.GROW_OPTIMIZER_DEFAULT, GrowTaskType.GROW_SIMULATOR, GrowTaskType.GROW_OPTIMIZER_WITH_PRESSURE, + GrowTaskType.GROW_OPTIMIZER_DEFAULT_GUROBI, + GrowTaskType.GROW_SIMULATOR_GUROBI, + GrowTaskType.GROW_OPTIMIZER_WITH_PRESSURE_GUROBI, ]: result = run_end_scenario_sizing - elif task_type == GrowTaskType.GROW_OPTIMIZER_NO_HEAT_LOSSES: + elif task_type in [ + GrowTaskType.GROW_OPTIMIZER_NO_HEAT_LOSSES, + GrowTaskType.GROW_OPTIMIZER_NO_HEAT_LOSSES_GUROBI, + ]: result = run_end_scenario_sizing_no_heat_losses else: raise RuntimeError(f"Unknown workflow type, please implement {task_type}") return result + + +def get_solver_class(task_type: GrowTaskType) -> Union[Type[SolverHIGHS], Type[SolverGurobi]]: + """Convert the Grow task type to the Grow solver that should be run. + + :param task_type: Grow task type. + :return: Grow solver class. + """ + result: Union[Type[SolverHIGHS], Type[SolverGurobi]] + if task_type in [ + GrowTaskType.GROW_OPTIMIZER_DEFAULT, + GrowTaskType.GROW_SIMULATOR, + GrowTaskType.GROW_OPTIMIZER_WITH_PRESSURE, + GrowTaskType.GROW_OPTIMIZER_NO_HEAT_LOSSES, + ]: + result = SolverHIGHS + elif task_type in [ + GrowTaskType.GROW_OPTIMIZER_DEFAULT_GUROBI, + GrowTaskType.GROW_SIMULATOR_GUROBI, + GrowTaskType.GROW_OPTIMIZER_NO_HEAT_LOSSES_GUROBI, + GrowTaskType.GROW_OPTIMIZER_WITH_PRESSURE_GUROBI, + ]: + result = SolverGurobi + else: + raise RuntimeError(f"Unknown workflow type, please implement {task_type}") + return result