diff --git a/config.yml b/config.yml index e9936adc..dd2c132c 100644 --- a/config.yml +++ b/config.yml @@ -90,6 +90,18 @@ globus: uuid: df82346e-9a15-11ea-b3c4-0ae144191ee3 name: nersc832 + compute_dtn_832_raw: + root_path: /beamlines/bl832/raw/ + uri: computeDTN.als.lbl.gov + uuid: TBD + name: compute_dtn_832_raw + + compute_dtn_832_scratch: + root_path: /beamlines/bl832/processed/ + uri: computeDTN.als.lbl.gov + uuid: TBD + name: compute_dtn_832_scratch + nersc7012: root_path: /global/cfs/cdirs/als/gsharing/data_mover/7012 uri: nersc.gov diff --git a/orchestration/_tests/test_globus_flow.py b/orchestration/_tests/test_globus_flow.py index e0823b27..97066d09 100644 --- a/orchestration/_tests/test_globus_flow.py +++ b/orchestration/_tests/test_globus_flow.py @@ -233,7 +233,7 @@ def test_alcf_recon_flow(mocker: MockFixture): result = alcf_recon_flow(file_path=file_path, config=mock_config) assert result is True, "Flow should return True if HPC + Tiff->Zarr + transfers all succeed" - assert mock_transfer_controller.copy.call_count == 3, "Should do 3 transfers in success path" + assert mock_transfer_controller.copy.call_count == 5, "Should do 5 transfers in success path" mock_hpc_reconstruct.assert_called_once() mock_hpc_multires.assert_called_once() mock_schedule_pruning.assert_called_once() diff --git a/orchestration/flows/bl832/alcf.py b/orchestration/flows/bl832/alcf.py index 0b6bcb1c..713efd7a 100644 --- a/orchestration/flows/bl832/alcf.py +++ b/orchestration/flows/bl832/alcf.py @@ -434,6 +434,15 @@ def alcf_recon_flow( destination=config.data832_scratch ) + logger.info(f"Transferring {file_name} from {config.alcf832_scratch} " + f"at ALCF to {config.compute_dtn_832_scratch}") + + transfer_controller.copy( + file_path=scratch_path_tiff, + source=config.alcf832_scratch, + destination=config.compute_dtn_832_scratch + ) + if alcf_multi_res_success: # Transfer B: Send reconstructed data (zarr) to data832 logger.info(f"Transferring {file_name} from {config.alcf832_scratch} " @@ -444,6 +453,14 @@ def alcf_recon_flow( destination=config.data832_scratch ) + logger.info(f"Transferring {file_name} from {config.alcf832_scratch} " + f"at ALCF to {config.compute_dtn_832_scratch}") + transfer_controller.copy( + file_path=scratch_path_zarr, + source=config.alcf832_scratch, + destination=config.compute_dtn_832_scratch + ) + # Place holder in case we want to transfer to NERSC for long term storage nersc_transfer_success = False diff --git a/orchestration/flows/bl832/config.py b/orchestration/flows/bl832/config.py index ff19a9c3..50cb10c1 100644 --- a/orchestration/flows/bl832/config.py +++ b/orchestration/flows/bl832/config.py @@ -22,5 +22,7 @@ def __init__(self) -> None: self.nersc832_alsdev_recon_scripts = self.endpoints["nersc832_alsdev_recon_scripts"] self.alcf832_raw = self.endpoints["alcf832_raw"] self.alcf832_scratch = self.endpoints["alcf832_scratch"] + self.compute_dtn_832_raw = self.endpoints["compute_dtn_832_raw"] + self.compute_dtn_832_scratch = self.endpoints["compute_dtn_832_scratch"] self.scicat = config["scicat"] self.ghcr_images832 = config["ghcr_images832"] diff --git a/orchestration/flows/bl832/nersc.py b/orchestration/flows/bl832/nersc.py index 05680ed0..9ed8f60d 100644 --- a/orchestration/flows/bl832/nersc.py +++ b/orchestration/flows/bl832/nersc.py @@ -500,6 +500,20 @@ def nersc_recon_flow( destination=config.data832_scratch ) + logger.info("Copy from NERSC /global/cfs/cdirs/als/data_mover/8.3.2/scratch to compute-dtn.als.lbl.gov") + + transfer_controller.copy( + file_path=tiff_file_path, + source=config.nersc832_alsdev_pscratch_scratch, + destination=config.compute_dtn_832_scratch + ) + + transfer_controller.copy( + file_path=zarr_file_path, + source=config.nersc832_alsdev_pscratch_scratch, + destination=config.compute_dtn_832_scratch + ) + logger.info("Scheduling pruning tasks.") schedule_pruning( config=config,