Skip to content

feat: compress eqasim-specific output #315

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sebhoerl wants to merge 3 commits into
base: develop
Choose a base branch
from
Open

feat: compress eqasim-specific output #315

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c9f7bcd
feat: compress eqasim-specific output
sebhoerl Mar 26, 2025
22bac51
remove from stuck analysis to make csv appendable
sebhoerl Mar 26, 2025
fddf0fc
update bin paths in tests
sebhoerl Mar 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion core/src/main/java/org/eqasim/core/analysis/activities/ActivityWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
import java.io.OutputStreamWriter;
import java.util.Collection;

import org.matsim.core.utils.io.IOUtils;

public class ActivityWriter {
final private Collection<ActivityItem> activities;
final private String delimiter;
Expand All @@ -20,7 +22,7 @@ public ActivityWriter(Collection<ActivityItem> activities, String delimiter) {
}

public void write(String outputPath) throws IOException {
BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outputPath)));
BufferedWriter writer = IOUtils.getBufferedWriter(outputPath);

writer.write(formatHeader() + "\n");
writer.flush();
Expand Down
5 changes: 2 additions & 3 deletions core/src/main/java/org/eqasim/core/analysis/legs/LegWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
package org.eqasim.core.analysis.legs;

import java.io.BufferedWriter;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.Collection;

import org.eqasim.core.analysis.DistanceUnit;
import org.matsim.core.utils.io.IOUtils;

public class LegWriter {
final private Collection<LegItem> legs;
Expand All @@ -27,7 +26,7 @@ public LegWriter(Collection<LegItem> legs, DistanceUnit inputUnit, DistanceUnit
}

public void write(String outputPath) throws IOException {
BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outputPath)));
BufferedWriter writer = IOUtils.getBufferedWriter(outputPath);

writer.write(formatHeader() + "\n");
writer.flush();
Expand Down
4 changes: 3 additions & 1 deletion core/src/main/java/org/eqasim/core/analysis/pt/PublicTransportLegWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
import java.io.OutputStreamWriter;
import java.util.Collection;

import org.matsim.core.utils.io.IOUtils;

public class PublicTransportLegWriter {
final private Collection<PublicTransportLegItem> trips;
final private String delimiter;
Expand All @@ -20,7 +22,7 @@ public PublicTransportLegWriter(Collection<PublicTransportLegItem> trips, String
}

public void write(String outputPath) throws IOException {
BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outputPath)));
BufferedWriter writer = IOUtils.getBufferedWriter(outputPath);

writer.write(formatHeader() + "\n");
writer.flush();
Expand Down
3 changes: 2 additions & 1 deletion core/src/main/java/org/eqasim/core/analysis/trips/TripWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import java.util.Collection;

import org.eqasim.core.analysis.DistanceUnit;
import org.matsim.core.utils.io.IOUtils;

public class TripWriter {
final private Collection<TripItem> trips;
Expand All @@ -27,7 +28,7 @@ public TripWriter(Collection<TripItem> trips, DistanceUnit inputUnit, DistanceUn
}

public void write(String outputPath) throws IOException {
BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outputPath)));
BufferedWriter writer = IOUtils.getBufferedWriter(outputPath);

writer.write(formatHeader() + "\n");
writer.flush();
Expand Down
10 changes: 5 additions & 5 deletions core/src/main/java/org/eqasim/core/simulation/analysis/AnalysisOutputListener.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,11 @@

@Singleton
public class AnalysisOutputListener implements IterationStartsListener, IterationEndsListener, ShutdownListener {
private static final String TRIPS_FILE_NAME = "eqasim_trips.csv";
private static final String LEGS_FILE_NAME = "eqasim_legs.csv";
private static final String PT_FILE_NAME = "eqasim_pt.csv";
private static final String ACTIVITIES_FILE_NAME = "eqasim_activities.csv";
private static final String TRAVEL_TIMES_FILE_NAME = "eqasim_travel_times.bin";
private static final String TRIPS_FILE_NAME = "eqasim_trips.csv.gz";
private static final String LEGS_FILE_NAME = "eqasim_legs.csv.gz";
private static final String PT_FILE_NAME = "eqasim_pt.csv.gz";
private static final String ACTIVITIES_FILE_NAME = "eqasim_activities.csv.gz";
private static final String TRAVEL_TIMES_FILE_NAME = "eqasim_travel_times.bin.gz";

private final OutputDirectoryHierarchy outputDirectory;

Expand Down
6 changes: 3 additions & 3 deletions core/src/main/java/org/eqasim/core/simulation/vdf/VDFUpdateListener.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@
public class VDFUpdateListener implements IterationEndsListener, StartupListener, ShutdownListener {
private final static Logger logger = LogManager.getLogger(VDFUpdateListener.class);

private final String VDF_FILE = "vdf.bin";
private final String FLOW_FILE = "vdf_flow.csv";
private final String TRAVEL_TIMES_FILE = "vdf_travel_times.bin";
private final String VDF_FILE = "vdf.bin.gz";
private final String FLOW_FILE = "vdf_flow.csv.gz";
private final String TRAVEL_TIMES_FILE = "vdf_travel_times.bin.gz";

private final VDFScope scope;
private final VDFTrafficHandler handler;
Expand Down
5 changes: 2 additions & 3 deletions core/src/main/java/org/eqasim/core/simulation/vdf/analysis/FlowWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,7 @@

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.List;
import java.util.Map;

Expand All @@ -13,6 +11,7 @@
import org.matsim.api.core.v01.IdMap;
import org.matsim.api.core.v01.network.Link;
import org.matsim.api.core.v01.network.Network;
import org.matsim.core.utils.io.IOUtils;

public class FlowWriter {
private final IdMap<Link, List<Double>> flows;
Expand All @@ -27,7 +26,7 @@ public FlowWriter(IdMap<Link, List<Double>> flows, Network network, VDFScope sco

public void write(File path) {
try {
BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(path)));
BufferedWriter writer = IOUtils.getBufferedWriter(path.getPath());

writer.write(String.join(";", new String[] { "link_id", "interval", "start_time", "flow", "lanes", "osm" })
+ "\n");
Expand Down
8 changes: 4 additions & 4 deletions core/src/test/java/org/eqasim/TestSimulationPipeline.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ public void runCutterV2() throws CommandLine.ConfigurationException, IOException
RunScenarioCutterV2.main(new String[] {
"--config-path", "melun_test/input/config_vdf.xml",
"--events-path", "melun_test/output_vdf/output_events.xml.gz",
"--vdf-travel-times-path", "melun_test/output_vdf/vdf_travel_times.bin",
"--vdf-travel-times-path", "melun_test/output_vdf/vdf_travel_times.bin.gz",
"--output-path", "melun_test/cutter_v2",
"--prefix", "center_",
"--extent-path", "melun_test/input/center.shp",
Expand All @@ -258,7 +258,7 @@ public void runCutterV2() throws CommandLine.ConfigurationException, IOException

runMelunSimulation("melun_test/cutter_v2/center_config_drt.xml", "melun_test/output_cutter_v2_drt");

compareVdfTravelTimes("melun_test/cutter_v2/center_config_drt.xml", "melun_test/output_vdf/vdf_travel_times.bin", "melun_test/output_cutter_v2_drt/vdf_travel_times.bin", "melun_test/input/center.shp");
compareVdfTravelTimes("melun_test/cutter_v2/center_config_drt.xml", "melun_test/output_vdf/vdf_travel_times.bin.gz", "melun_test/output_cutter_v2_drt/vdf_travel_times.bin.gz", "melun_test/input/center.shp");
}

private void compareVdfTravelTimes(String configPath, String leftTravelTimesPath, String rightTravelTimesPath, String updateExtentPath) throws IOException, ConfigurationException {
Expand Down Expand Up @@ -450,7 +450,7 @@ public void runVdf() throws CommandLine.ConfigurationException, IOException, Int
RunStandaloneModeChoice.main(new String[]{
"--config-path", "melun_test/input/config_vdf.xml",
"--config:standaloneModeChoice.outputDirectory", "melun_test/output_mode_choice_vdf",
"--config:eqasim:vdf.inputTravelTimesFile", "../output_vdf/vdf_travel_times.bin", // Relative to the config file
"--config:eqasim:vdf.inputTravelTimesFile", "../output_vdf/vdf_travel_times.bin.gz", // Relative to the config file
"--eqasim-configurator", TestConfigurator.class.getName(),
"--simulate-after", TestRunSimulation.class.getName()
});
Expand Down Expand Up @@ -516,7 +516,7 @@ public void runPopulationRouting() throws CommandLine.ConfigurationException, In
public void runStandaloneModeChoice() throws CommandLine.ConfigurationException, IOException, InterruptedException {
RunStandaloneModeChoice.main(new String[] {
"--config-path", "melun_test/input/config.xml",
"--recorded-travel-times-path", "melun_test/output/eqasim_travel_times.bin",
"--recorded-travel-times-path", "melun_test/output/eqasim_travel_times.bin.gz",
"--write-input-csv-trips", "true",
"--write-output-csv-trips", "true",
"--config:standaloneModeChoice.outputDirectory", "melun_test/output_mode_choice",
Expand Down
Loading