From 389097b4cc1ef49b898a1d69ba7b686cd9c1bb4d Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 22 Oct 2025 15:05:31 -0700 Subject: [PATCH 1/7] creating config placeholder for ride.als.lbl.gov globus endpoint --- config.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/config.yml b/config.yml index e9936adc..ddbed595 100644 --- a/config.yml +++ b/config.yml @@ -90,6 +90,18 @@ globus: uuid: df82346e-9a15-11ea-b3c4-0ae144191ee3 name: nersc832 + ride832_raw: + root_path: /beamlines/bl832/raw/ + uri: ride.als.lbl.gov + uuid: TBD + name: ride832_raw + + ride832_scratch: + root_path: /beamlines/bl832/processed/ + uri: ride.als.lbl.gov + uuid: TBD + name: ride832_scratch + nersc7012: root_path: /global/cfs/cdirs/als/gsharing/data_mover/7012 uri: nersc.gov From f4ecf4ed2caafb1d1d80f60ff102d2ac586d037a Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 22 Oct 2025 15:07:57 -0700 Subject: [PATCH 2/7] Adding ride globus endpoint to config832 --- orchestration/flows/bl832/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/orchestration/flows/bl832/config.py b/orchestration/flows/bl832/config.py index ff19a9c3..884c1bd7 100644 --- a/orchestration/flows/bl832/config.py +++ b/orchestration/flows/bl832/config.py @@ -22,5 +22,7 @@ def __init__(self) -> None: self.nersc832_alsdev_recon_scripts = self.endpoints["nersc832_alsdev_recon_scripts"] self.alcf832_raw = self.endpoints["alcf832_raw"] self.alcf832_scratch = self.endpoints["alcf832_scratch"] + self.ride832_raw = self.endpoints["ride832_raw"] + self.ride832_scratch = self.endpoints["ride832_scratch"] self.scicat = config["scicat"] self.ghcr_images832 = config["ghcr_images832"] From b6da484f274073f692830940999c1f85ec59a897 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 22 Oct 2025 15:14:03 -0700 Subject: [PATCH 3/7] Adding transfers to ride.als.lbl.gov after reconstruction is complete --- orchestration/flows/bl832/alcf.py | 17 +++++++++++++++++ orchestration/flows/bl832/nersc.py | 14 ++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/orchestration/flows/bl832/alcf.py b/orchestration/flows/bl832/alcf.py index 0b6bcb1c..d7f6d210 100644 --- a/orchestration/flows/bl832/alcf.py +++ b/orchestration/flows/bl832/alcf.py @@ -434,6 +434,15 @@ def alcf_recon_flow( destination=config.data832_scratch ) + logger.info(f"Transferring {file_name} from {config.alcf832_scratch} " + f"at ALCF to {config.ride832_scratch} at ride.als.lbl.gov") + + transfer_controller.copy( + file_path=scratch_path_tiff, + source=config.alcf832_scratch, + destination=config.ride832_scratch + ) + if alcf_multi_res_success: # Transfer B: Send reconstructed data (zarr) to data832 logger.info(f"Transferring {file_name} from {config.alcf832_scratch} " @@ -444,6 +453,14 @@ def alcf_recon_flow( destination=config.data832_scratch ) + logger.info(f"Transferring {file_name} from {config.alcf832_scratch} " + f"at ALCF to {config.ride832_scratch} at ride.als.lbl.gov") + transfer_controller.copy( + file_path=scratch_path_zarr, + source=config.alcf832_scratch, + destination=config.ride832_scratch + ) + # Place holder in case we want to transfer to NERSC for long term storage nersc_transfer_success = False diff --git a/orchestration/flows/bl832/nersc.py b/orchestration/flows/bl832/nersc.py index 05680ed0..3c71b20e 100644 --- a/orchestration/flows/bl832/nersc.py +++ b/orchestration/flows/bl832/nersc.py @@ -500,6 +500,20 @@ def nersc_recon_flow( destination=config.data832_scratch ) + logger.info("Copy from NERSC /global/cfs/cdirs/als/data_mover/8.3.2/scratch to ride.als.lbl.gov") + + transfer_controller.copy( + file_path=tiff_file_path, + source=config.nersc832_alsdev_pscratch_scratch, + destination=config.ride832_scratch + ) + + transfer_controller.copy( + file_path=zarr_file_path, + source=config.nersc832_alsdev_pscratch_scratch, + destination=config.ride832_scratch + ) + logger.info("Scheduling pruning tasks.") schedule_pruning( config=config, From c55ad861a4e730fd04b940ecff1b86da6cf44555 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 22 Oct 2025 15:27:01 -0700 Subject: [PATCH 4/7] updating pytest to reflect the additional transfer --- orchestration/_tests/test_globus_flow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orchestration/_tests/test_globus_flow.py b/orchestration/_tests/test_globus_flow.py index e0823b27..97066d09 100644 --- a/orchestration/_tests/test_globus_flow.py +++ b/orchestration/_tests/test_globus_flow.py @@ -233,7 +233,7 @@ def test_alcf_recon_flow(mocker: MockFixture): result = alcf_recon_flow(file_path=file_path, config=mock_config) assert result is True, "Flow should return True if HPC + Tiff->Zarr + transfers all succeed" - assert mock_transfer_controller.copy.call_count == 3, "Should do 3 transfers in success path" + assert mock_transfer_controller.copy.call_count == 5, "Should do 5 transfers in success path" mock_hpc_reconstruct.assert_called_once() mock_hpc_multires.assert_called_once() mock_schedule_pruning.assert_called_once() From d158bc8627252229b3efacc398624f6b012bff6e Mon Sep 17 00:00:00 2001 From: David Abramov Date: Thu, 23 Oct 2025 16:32:44 -0700 Subject: [PATCH 5/7] Renaming ride config to compute dtn --- config.yml | 12 ++++++------ orchestration/flows/bl832/config.py | 4 ++-- orchestration/flows/bl832/nersc.py | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/config.yml b/config.yml index ddbed595..dd2c132c 100644 --- a/config.yml +++ b/config.yml @@ -90,17 +90,17 @@ globus: uuid: df82346e-9a15-11ea-b3c4-0ae144191ee3 name: nersc832 - ride832_raw: + compute_dtn_832_raw: root_path: /beamlines/bl832/raw/ - uri: ride.als.lbl.gov + uri: computeDTN.als.lbl.gov uuid: TBD - name: ride832_raw + name: compute_dtn_832_raw - ride832_scratch: + compute_dtn_832_scratch: root_path: /beamlines/bl832/processed/ - uri: ride.als.lbl.gov + uri: computeDTN.als.lbl.gov uuid: TBD - name: ride832_scratch + name: compute_dtn_832_scratch nersc7012: root_path: /global/cfs/cdirs/als/gsharing/data_mover/7012 diff --git a/orchestration/flows/bl832/config.py b/orchestration/flows/bl832/config.py index 884c1bd7..50cb10c1 100644 --- a/orchestration/flows/bl832/config.py +++ b/orchestration/flows/bl832/config.py @@ -22,7 +22,7 @@ def __init__(self) -> None: self.nersc832_alsdev_recon_scripts = self.endpoints["nersc832_alsdev_recon_scripts"] self.alcf832_raw = self.endpoints["alcf832_raw"] self.alcf832_scratch = self.endpoints["alcf832_scratch"] - self.ride832_raw = self.endpoints["ride832_raw"] - self.ride832_scratch = self.endpoints["ride832_scratch"] + self.compute_dtn_832_raw = self.endpoints["compute_dtn_832_raw"] + self.compute_dtn_832_scratch = self.endpoints["compute_dtn_832_scratch"] self.scicat = config["scicat"] self.ghcr_images832 = config["ghcr_images832"] diff --git a/orchestration/flows/bl832/nersc.py b/orchestration/flows/bl832/nersc.py index 3c71b20e..e4d60b5f 100644 --- a/orchestration/flows/bl832/nersc.py +++ b/orchestration/flows/bl832/nersc.py @@ -505,13 +505,13 @@ def nersc_recon_flow( transfer_controller.copy( file_path=tiff_file_path, source=config.nersc832_alsdev_pscratch_scratch, - destination=config.ride832_scratch + destination=config.compute_dtn_832_scratch ) transfer_controller.copy( file_path=zarr_file_path, source=config.nersc832_alsdev_pscratch_scratch, - destination=config.ride832_scratch + destination=config.compute_dtn_832_scratch ) logger.info("Scheduling pruning tasks.") From 4e64cfaed0ca50815cdd8139cc855bdb783a2aef Mon Sep 17 00:00:00 2001 From: David Abramov Date: Thu, 23 Oct 2025 16:57:40 -0700 Subject: [PATCH 6/7] Renaming ride config to compute dtn --- orchestration/flows/bl832/alcf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/orchestration/flows/bl832/alcf.py b/orchestration/flows/bl832/alcf.py index d7f6d210..3a0de2cf 100644 --- a/orchestration/flows/bl832/alcf.py +++ b/orchestration/flows/bl832/alcf.py @@ -440,7 +440,7 @@ def alcf_recon_flow( transfer_controller.copy( file_path=scratch_path_tiff, source=config.alcf832_scratch, - destination=config.ride832_scratch + destination=config.compute_dtn_832_scratch ) if alcf_multi_res_success: @@ -458,7 +458,7 @@ def alcf_recon_flow( transfer_controller.copy( file_path=scratch_path_zarr, source=config.alcf832_scratch, - destination=config.ride832_scratch + destination=config.compute_dtn_832_scratch ) # Place holder in case we want to transfer to NERSC for long term storage From 3bdf5d1169ac0ff040a62ac28a5b1251cebdb859 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Thu, 23 Oct 2025 17:02:03 -0700 Subject: [PATCH 7/7] renaming ride to compute dtn --- orchestration/flows/bl832/alcf.py | 4 ++-- orchestration/flows/bl832/nersc.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/orchestration/flows/bl832/alcf.py b/orchestration/flows/bl832/alcf.py index 3a0de2cf..713efd7a 100644 --- a/orchestration/flows/bl832/alcf.py +++ b/orchestration/flows/bl832/alcf.py @@ -435,7 +435,7 @@ def alcf_recon_flow( ) logger.info(f"Transferring {file_name} from {config.alcf832_scratch} " - f"at ALCF to {config.ride832_scratch} at ride.als.lbl.gov") + f"at ALCF to {config.compute_dtn_832_scratch}") transfer_controller.copy( file_path=scratch_path_tiff, @@ -454,7 +454,7 @@ def alcf_recon_flow( ) logger.info(f"Transferring {file_name} from {config.alcf832_scratch} " - f"at ALCF to {config.ride832_scratch} at ride.als.lbl.gov") + f"at ALCF to {config.compute_dtn_832_scratch}") transfer_controller.copy( file_path=scratch_path_zarr, source=config.alcf832_scratch, diff --git a/orchestration/flows/bl832/nersc.py b/orchestration/flows/bl832/nersc.py index e4d60b5f..9ed8f60d 100644 --- a/orchestration/flows/bl832/nersc.py +++ b/orchestration/flows/bl832/nersc.py @@ -500,7 +500,7 @@ def nersc_recon_flow( destination=config.data832_scratch ) - logger.info("Copy from NERSC /global/cfs/cdirs/als/data_mover/8.3.2/scratch to ride.als.lbl.gov") + logger.info("Copy from NERSC /global/cfs/cdirs/als/data_mover/8.3.2/scratch to compute-dtn.als.lbl.gov") transfer_controller.copy( file_path=tiff_file_path,