From d3cee4d32b9f1fef5c22b7f2a5757c79c8269c88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20H=C3=B6rl?= Date: Mon, 12 Jan 2026 10:47:56 +0100 Subject: [PATCH 1/3] feat: automating simulation restarts --- .../restart/RestartConfigurator.java | 176 ++++++++++++++++++ .../termination/EqasimTerminationModule.java | 15 +- .../simulation/vdf/VDFUpdateListener.java | 6 +- 3 files changed, 190 insertions(+), 7 deletions(-) create mode 100644 core/src/main/java/org/eqasim/core/simulation/restart/RestartConfigurator.java diff --git a/core/src/main/java/org/eqasim/core/simulation/restart/RestartConfigurator.java b/core/src/main/java/org/eqasim/core/simulation/restart/RestartConfigurator.java new file mode 100644 index 000000000..ca48a8fad --- /dev/null +++ b/core/src/main/java/org/eqasim/core/simulation/restart/RestartConfigurator.java @@ -0,0 +1,176 @@ +package org.eqasim.core.simulation.restart; + +import java.io.File; +import java.util.LinkedList; +import java.util.List; +import java.util.function.Consumer; +import java.util.function.Function; + +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; +import org.eqasim.core.simulation.termination.EqasimTerminationConfigGroup; +import org.eqasim.core.simulation.termination.EqasimTerminationModule; +import org.eqasim.core.simulation.vdf.VDFConfigGroup; +import org.eqasim.core.simulation.vdf.VDFUpdateListener; +import org.matsim.core.config.Config; +import org.matsim.core.controler.OutputDirectoryHierarchy; + +/** + * This class is supposed to be applied after all configuration is set up. It + * will modify the paths such that each new runs ends up in a "restart-*" + * subdirectory of the output directory defined in the controller config group. + * Each time the simulation is restarted on the same output directory, a new + * restart directory is created and the output files of the last successful + * iteration of the previous run are set as input for the new restart. See + * addDefaultMappings for the files that are reconfigured by default. + */ +public class RestartConfigurator { + private final static Logger logger = LogManager.getLogger(RestartConfigurator.class); + + static public final String PREFIX = "restart_"; + + private record Mapping(boolean isIteration, String sourcePath, Function> mapper) { + } + + private List mappings = new LinkedList<>(); + + private File getRestartPath(File basePath, int restartIndex) { + return new File(basePath, PREFIX + restartIndex); + } + + public void apply(Config config) { + File basePath = new File(config.controller().getOutputDirectory()); + + // find the number of restarts + int maximumRestartIndex = -1; + if (basePath.exists()) { + for (File restartPath : basePath.listFiles()) { + if (restartPath.getName().startsWith(PREFIX)) { + int restartIndex = Integer.parseInt(restartPath.getName().replace(PREFIX, "")); + maximumRestartIndex = Math.max(maximumRestartIndex, restartIndex); + } + } + } + + int selectedRestartIndex = -1; + int selectedIteration = -1; + + if (maximumRestartIndex == -1) { + logger.info(String.format("No restarts found")); + } else { + logger.info(String.format("Examining %d restarts", maximumRestartIndex + 1)); + + for (int restartIndex = 0; restartIndex <= maximumRestartIndex; restartIndex++) { + File restartPath = getRestartPath(basePath, restartIndex); + + // find the available iterations + int maximumIterationIndex = 0; + for (File iterationPath : new File(restartPath, "ITERS").listFiles()) { + if (iterationPath.getName().startsWith("it.")) { + int iterationIndex = Integer.parseInt(iterationPath.getName().replace("it.", "")); + maximumIterationIndex = Math.max(iterationIndex, maximumIterationIndex); + } + } + + // check for output level files that must be present + config.controller().setOutputDirectory(restartPath.getAbsolutePath()); + OutputDirectoryHierarchy outputDirectoryHierarchy = new OutputDirectoryHierarchy(config); + + List missing = new LinkedList<>(); + for (Mapping mapping : mappings) { + if (!mapping.isIteration) { + String expectedPath = outputDirectoryHierarchy.getOutputFilename(mapping.sourcePath); + + if (!new File(expectedPath).exists()) { + missing.add(mapping.sourcePath); + } + } + } + + if (missing.size() == 0) { + for (int iterationIndex = 0; iterationIndex <= maximumIterationIndex; iterationIndex++) { + // check for iteration level files that must be present + missing.clear(); + + for (Mapping mapping : mappings) { + if (mapping.isIteration) { + String expectedPath = outputDirectoryHierarchy.getIterationFilename(iterationIndex, + mapping.sourcePath); + + if (!new File(expectedPath).exists()) { + missing.add(mapping.sourcePath); + } + } + } + + if (missing.size() == 0) { + // add files are in place + selectedRestartIndex = restartIndex; + selectedIteration = iterationIndex; + + logger.info("Found a viable restart source: restart " + restartIndex + ", iteration " + + iterationIndex); + } + } + } + } + + // we found a viable restart source + if (selectedRestartIndex >= 0 && selectedIteration >= 0) { + File restartPath = getRestartPath(basePath, selectedRestartIndex); + config.controller().setOutputDirectory(restartPath.getAbsolutePath()); + OutputDirectoryHierarchy outputHierarchy = new OutputDirectoryHierarchy(config); + + // apply all mappings + for (Mapping mapping : mappings) { + if (mapping.isIteration) { + File sourcePath = new File( + outputHierarchy.getIterationFilename(selectedIteration, mapping.sourcePath)); + mapping.mapper.apply(config).accept(sourcePath.getAbsolutePath()); + } else { + File sourcePath = new File(outputHierarchy.getOutputFilename(mapping.sourcePath)); + mapping.mapper.apply(config).accept(sourcePath.getAbsolutePath()); + } + } + } + } + + // now set up the next restart + int nextRestartIndex = maximumRestartIndex + 1; + File restartPath = getRestartPath(basePath, nextRestartIndex); + config.controller().setOutputDirectory(restartPath.getAbsolutePath()); + + if (selectedIteration >= 0) { + config.controller().setFirstIteration(selectedIteration); + } + + logger.info("Restarting simulation with index " + nextRestartIndex + " and iteration " + + config.controller().getFirstIteration() + " at " + restartPath.getAbsolutePath()); + } + + public void addMapping(boolean isIteration, String sourcePath, Function> mapper) { + mappings.add(new Mapping(isIteration, sourcePath, mapper)); + } + + public void clearMappings() { + mappings.clear(); + } + + public void addDefaultMappings(Config config) { + // plans + addMapping(true, "plans.xml", c -> c.plans()::setInputFile); + + // termination + if (config.getModules().containsKey(EqasimTerminationConfigGroup.GROUP_NAME)) { + addMapping(false, EqasimTerminationModule.TERMINATION_CSV_FILE, + c -> EqasimTerminationConfigGroup.getOrCreate(c)::setHistoryFile); + } + + // vdf + if (config.getModules().containsKey(VDFConfigGroup.GROUP_NAME)) { + addMapping(false, VDFUpdateListener.FLOW_FILE, c -> VDFConfigGroup.getOrCreate(c)::setInputFlowFile); + addMapping(false, VDFUpdateListener.TRAVEL_TIMES_FILE, + c -> VDFConfigGroup.getOrCreate(c)::setInputTravelTimesFile); + } + } +} diff --git a/core/src/main/java/org/eqasim/core/simulation/termination/EqasimTerminationModule.java b/core/src/main/java/org/eqasim/core/simulation/termination/EqasimTerminationModule.java index 3d062ecc4..3fef6d327 100644 --- a/core/src/main/java/org/eqasim/core/simulation/termination/EqasimTerminationModule.java +++ b/core/src/main/java/org/eqasim/core/simulation/termination/EqasimTerminationModule.java @@ -19,8 +19,8 @@ import com.google.inject.multibindings.MapBinder; public class EqasimTerminationModule extends AbstractEqasimExtension { - private static final String TERMINATION_CSV_FILE = "eqasim_termination.csv"; - private static final String TERMINATION_HTML_FILE = "eqasim_termination.html"; + public static final String TERMINATION_CSV_FILE = "eqasim_termination.csv"; + public static final String TERMINATION_HTML_FILE = "eqasim_termination.html"; @Override protected void installEqasimExtension() { @@ -37,15 +37,22 @@ EqasimTerminationCriterion provideEqasimTerminationCriterion(ControllerConfigGro int firstIteration = controllerConfig.getFirstIteration(); int lastIteration = controllerConfig.getLastIteration(); + List history = Collections.emptyList(); + if (terminationConfig.getHistoryFile() != null) { URL historyURL = ConfigGroup.getInputFileURL(getConfig().getContext(), terminationConfig.getHistoryFile()); - new TerminationReader(indicators.keySet(), criteria.keySet()).read(historyURL); + history = new TerminationReader(indicators.keySet(), criteria.keySet()).read(historyURL); } int minimumIteration = firstIteration + terminationConfig.getMinimumIterations(); - return new EqasimTerminationCriterion(firstIteration, lastIteration, minimumIteration, indicators, criteria, + EqasimTerminationCriterion criterion = new EqasimTerminationCriterion(firstIteration, lastIteration, + minimumIteration, indicators, criteria, writer); + + criterion.replay(history); + + return criterion; } @Provides diff --git a/core/src/main/java/org/eqasim/core/simulation/vdf/VDFUpdateListener.java b/core/src/main/java/org/eqasim/core/simulation/vdf/VDFUpdateListener.java index 791a3c556..217e9dd43 100644 --- a/core/src/main/java/org/eqasim/core/simulation/vdf/VDFUpdateListener.java +++ b/core/src/main/java/org/eqasim/core/simulation/vdf/VDFUpdateListener.java @@ -26,9 +26,9 @@ public class VDFUpdateListener implements IterationEndsListener, StartupListener, ShutdownListener { private final static Logger logger = LogManager.getLogger(VDFUpdateListener.class); - private final String VDF_FILE = "vdf.bin"; - private final String FLOW_FILE = "vdf_flow.csv"; - private final String TRAVEL_TIMES_FILE = "vdf_travel_times.bin"; + static public final String VDF_FILE = "vdf.bin"; + static public final String FLOW_FILE = "vdf_flow.csv"; + static public final String TRAVEL_TIMES_FILE = "vdf_travel_times.bin"; private final VDFScope scope; private final VDFTrafficHandler handler; From 7660022bda1cc0acd4b9f39dc9b0824b64572c3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20H=C3=B6rl?= Date: Mon, 12 Jan 2026 11:11:46 +0100 Subject: [PATCH 2/3] update --- .../restart/RestartConfigurator.java | 18 ++++++++++++++---- .../eqasim/ile_de_france/RunSimulation.java | 3 +++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/eqasim/core/simulation/restart/RestartConfigurator.java b/core/src/main/java/org/eqasim/core/simulation/restart/RestartConfigurator.java index ca48a8fad..5e9219d20 100644 --- a/core/src/main/java/org/eqasim/core/simulation/restart/RestartConfigurator.java +++ b/core/src/main/java/org/eqasim/core/simulation/restart/RestartConfigurator.java @@ -12,6 +12,7 @@ import org.eqasim.core.simulation.termination.EqasimTerminationModule; import org.eqasim.core.simulation.vdf.VDFConfigGroup; import org.eqasim.core.simulation.vdf.VDFUpdateListener; +import org.matsim.core.config.CommandLine; import org.matsim.core.config.Config; import org.matsim.core.controler.OutputDirectoryHierarchy; @@ -28,6 +29,7 @@ public class RestartConfigurator { private final static Logger logger = LogManager.getLogger(RestartConfigurator.class); static public final String PREFIX = "restart_"; + static public final String CMD = "restart"; private record Mapping(boolean isIteration, String sourcePath, Function> mapper) { } @@ -158,19 +160,27 @@ public void clearMappings() { public void addDefaultMappings(Config config) { // plans - addMapping(true, "plans.xml", c -> c.plans()::setInputFile); + addMapping(false, "plans.xml", c -> c.plans()::setInputFile); // termination if (config.getModules().containsKey(EqasimTerminationConfigGroup.GROUP_NAME)) { - addMapping(false, EqasimTerminationModule.TERMINATION_CSV_FILE, + addMapping(true, EqasimTerminationModule.TERMINATION_CSV_FILE, c -> EqasimTerminationConfigGroup.getOrCreate(c)::setHistoryFile); } // vdf if (config.getModules().containsKey(VDFConfigGroup.GROUP_NAME)) { - addMapping(false, VDFUpdateListener.FLOW_FILE, c -> VDFConfigGroup.getOrCreate(c)::setInputFlowFile); - addMapping(false, VDFUpdateListener.TRAVEL_TIMES_FILE, + addMapping(true, VDFUpdateListener.FLOW_FILE, c -> VDFConfigGroup.getOrCreate(c)::setInputFlowFile); + addMapping(true, VDFUpdateListener.TRAVEL_TIMES_FILE, c -> VDFConfigGroup.getOrCreate(c)::setInputTravelTimesFile); } } + + static public void setup(CommandLine cmd, Config config) { + if (cmd.getOption("restart").map(Boolean::parseBoolean).orElse(false)) { + RestartConfigurator configurator = new RestartConfigurator(); + configurator.addDefaultMappings(config); + configurator.apply(config); + } + } } diff --git a/ile_de_france/src/main/java/org/eqasim/ile_de_france/RunSimulation.java b/ile_de_france/src/main/java/org/eqasim/ile_de_france/RunSimulation.java index d2c0a75a7..635c39053 100644 --- a/ile_de_france/src/main/java/org/eqasim/ile_de_france/RunSimulation.java +++ b/ile_de_france/src/main/java/org/eqasim/ile_de_france/RunSimulation.java @@ -1,6 +1,7 @@ package org.eqasim.ile_de_france; import org.eqasim.core.scenario.validation.VehiclesValidator; +import org.eqasim.core.simulation.restart.RestartConfigurator; import org.matsim.api.core.v01.Scenario; import org.matsim.core.config.CommandLine; import org.matsim.core.config.CommandLine.ConfigurationException; @@ -13,6 +14,7 @@ public class RunSimulation { static public void main(String[] args) throws ConfigurationException { CommandLine cmd = new CommandLine.Builder(args) // .requireOptions("config-path") // + .allowOptions(RestartConfigurator.CMD) // .allowPrefixes("mode-choice-parameter", "cost-parameter") // .build(); @@ -21,6 +23,7 @@ static public void main(String[] args) throws ConfigurationException { configurator.updateConfig(config); cmd.applyConfiguration(config); + RestartConfigurator.setup(cmd, config); VehiclesValidator.validate(config); Scenario scenario = ScenarioUtils.createScenario(config); From 361858c1399c1bce3d1cb6325e175032318264e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20H=C3=B6rl?= Date: Mon, 12 Jan 2026 11:12:08 +0100 Subject: [PATCH 3/3] update --- .../org/eqasim/core/simulation/restart/RestartConfigurator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/eqasim/core/simulation/restart/RestartConfigurator.java b/core/src/main/java/org/eqasim/core/simulation/restart/RestartConfigurator.java index 5e9219d20..11d357ddf 100644 --- a/core/src/main/java/org/eqasim/core/simulation/restart/RestartConfigurator.java +++ b/core/src/main/java/org/eqasim/core/simulation/restart/RestartConfigurator.java @@ -177,7 +177,7 @@ public void addDefaultMappings(Config config) { } static public void setup(CommandLine cmd, Config config) { - if (cmd.getOption("restart").map(Boolean::parseBoolean).orElse(false)) { + if (cmd.getOption(CMD).map(Boolean::parseBoolean).orElse(false)) { RestartConfigurator configurator = new RestartConfigurator(); configurator.addDefaultMappings(config); configurator.apply(config);