From 2135c3ad7af055709074ab3928c12528fc6cdce0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Tue, 17 Dec 2024 15:00:27 +0100 Subject: [PATCH 1/6] raster writer: skip empty timestaps --- geoengine/raster.py | 6 ++++++ geoengine/raster_workflow_rio_writer.py | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/geoengine/raster.py b/geoengine/raster.py index 3d5199f2..b7793729 100644 --- a/geoengine/raster.py +++ b/geoengine/raster.py @@ -213,6 +213,12 @@ def spatial_partition(self) -> gety.SpatialPartition2D: def spatial_resolution(self) -> gety.SpatialResolution: return self.geo_transform.spatial_resolution() + def is_empty(self) -> bool: + ''' Returns true if the tile is empty''' + num_pixels = self.size_x * self.size_y + num_nulls = self.data.null_count + return num_pixels == num_nulls + @staticmethod def from_ge_record_batch(record_batch: pa.RecordBatch) -> RasterTile2D: '''Create a RasterTile2D from an Arrow record batch recieved from the Geo Engine''' diff --git a/geoengine/raster_workflow_rio_writer.py b/geoengine/raster_workflow_rio_writer.py index 7e70a566..77523ef1 100644 --- a/geoengine/raster_workflow_rio_writer.py +++ b/geoengine/raster_workflow_rio_writer.py @@ -170,7 +170,7 @@ def __create_new_dataset(self, query: QueryRectangle): self.current_dataset = rio_dataset - async def query_and_write(self, query: QueryRectangle): + async def query_and_write(self, query: QueryRectangle, skip_empty_times=True): ''' Query the raster workflow and write the tiles to the dataset.''' self.create_tiling_geo_transform_width_height(query) @@ -182,6 +182,11 @@ async def query_and_write(self, query: QueryRectangle): if self.current_time != tile.time: self.close_current_dataset() self.current_time = tile.time + + if tile.is_empty() and skip_empty_times: + continue + + if self.current_dataset is None: self.__create_new_dataset(query) assert self.current_time == tile.time, "The time of the current dataset does not match the tile" From 45b77febd1b9050ab3bab7e87bf239edf679e3a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Mon, 16 Dec 2024 14:47:32 +0100 Subject: [PATCH 2/6] raster downloader: add bands --- geoengine/raster_workflow_rio_writer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/geoengine/raster_workflow_rio_writer.py b/geoengine/raster_workflow_rio_writer.py index 77523ef1..1ca0b50e 100644 --- a/geoengine/raster_workflow_rio_writer.py +++ b/geoengine/raster_workflow_rio_writer.py @@ -175,10 +175,12 @@ async def query_and_write(self, query: QueryRectangle, skip_empty_times=True): self.create_tiling_geo_transform_width_height(query) - assert self.workflow is not None, "The workflow must be set" + assert self.bands is not None, "The bands must be set" + bands = list(range(0, len(self.bands))) + assert self.workflow is not None, "The workflow must be set" try: - async for tile in self.workflow.raster_stream(query): + async for tile in self.workflow.raster_stream(query, bands=bands): if self.current_time != tile.time: self.close_current_dataset() self.current_time = tile.time From ace3d6cf04ce652a818d1b87d1c12915252582b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Tue, 17 Dec 2024 15:34:32 +0100 Subject: [PATCH 3/6] raster writer: write band names and measurement --- geoengine/raster_workflow_rio_writer.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/geoengine/raster_workflow_rio_writer.py b/geoengine/raster_workflow_rio_writer.py index 1ca0b50e..11838945 100644 --- a/geoengine/raster_workflow_rio_writer.py +++ b/geoengine/raster_workflow_rio_writer.py @@ -168,6 +168,11 @@ def __create_new_dataset(self, query: QueryRectangle): **self.rio_kwargs ) + for i, b in enumerate(self.bands, start=1): + b_n = b.name + b_m = str(b.measurement) + rio_dataset.update_tag(i, band_name=b_n, band_measurement=b_m) + self.current_dataset = rio_dataset async def query_and_write(self, query: QueryRectangle, skip_empty_times=True): From fd9565c10cdcc235946116209a47bcc8daffa7fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Tue, 17 Dec 2024 16:06:46 +0100 Subject: [PATCH 4/6] raster writer: add 's' --- geoengine/raster_workflow_rio_writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geoengine/raster_workflow_rio_writer.py b/geoengine/raster_workflow_rio_writer.py index 11838945..2485a819 100644 --- a/geoengine/raster_workflow_rio_writer.py +++ b/geoengine/raster_workflow_rio_writer.py @@ -171,7 +171,7 @@ def __create_new_dataset(self, query: QueryRectangle): for i, b in enumerate(self.bands, start=1): b_n = b.name b_m = str(b.measurement) - rio_dataset.update_tag(i, band_name=b_n, band_measurement=b_m) + rio_dataset.update_tags(i, band_name=b_n, band_measurement=b_m) self.current_dataset = rio_dataset From dddff3b590deb35a260432ed363dfacca1ee854d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Wed, 15 Jan 2025 16:45:12 +0100 Subject: [PATCH 5/6] more docs --- geoengine/raster_workflow_rio_writer.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/geoengine/raster_workflow_rio_writer.py b/geoengine/raster_workflow_rio_writer.py index 2485a819..7e2cbced 100644 --- a/geoengine/raster_workflow_rio_writer.py +++ b/geoengine/raster_workflow_rio_writer.py @@ -176,7 +176,13 @@ def __create_new_dataset(self, query: QueryRectangle): self.current_dataset = rio_dataset async def query_and_write(self, query: QueryRectangle, skip_empty_times=True): - ''' Query the raster workflow and write the tiles to the dataset.''' + ''' + Query the raster workflow and write the resulting tiles to a GDAL dataset per timeslice. + + :param query: The QueryRectangle to write to GDAL dataset(s) + :param skip_empty_times: Skip timeslices where all pixels are empty/nodata + + ''' self.create_tiling_geo_transform_width_height(query) From 1f00be38944b4aa971f98a28d3ee02d83cc34f5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Wed, 15 Jan 2025 17:12:11 +0100 Subject: [PATCH 6/6] codestyle --- geoengine/raster_workflow_rio_writer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/geoengine/raster_workflow_rio_writer.py b/geoengine/raster_workflow_rio_writer.py index 7e2cbced..e43d700a 100644 --- a/geoengine/raster_workflow_rio_writer.py +++ b/geoengine/raster_workflow_rio_writer.py @@ -176,12 +176,11 @@ def __create_new_dataset(self, query: QueryRectangle): self.current_dataset = rio_dataset async def query_and_write(self, query: QueryRectangle, skip_empty_times=True): - ''' + ''' Query the raster workflow and write the resulting tiles to a GDAL dataset per timeslice. :param query: The QueryRectangle to write to GDAL dataset(s) :param skip_empty_times: Skip timeslices where all pixels are empty/nodata - ''' self.create_tiling_geo_transform_width_height(query)