Issue 92: Copy/ingest reconstructions on central tiled server

config.yml

@@ -90,6 +90,18 @@ globus:
       uuid: df82346e-9a15-11ea-b3c4-0ae144191ee3
       name: nersc832
 
+    compute_dtn_832_raw:
+      root_path: /beamlines/bl832/raw/
+      uri: computeDTN.als.lbl.gov
+      uuid: TBD
+      name: compute_dtn_832_raw
+
+    compute_dtn_832_scratch:
+      root_path: /beamlines/bl832/processed/
+      uri: computeDTN.als.lbl.gov
+      uuid: TBD
+      name: compute_dtn_832_scratch
+
     nersc7012:
       root_path: /global/cfs/cdirs/als/gsharing/data_mover/7012
       uri: nersc.gov

orchestration/_tests/test_globus_flow.py

@@ -233,7 +233,7 @@ def test_alcf_recon_flow(mocker: MockFixture):
 
     result = alcf_recon_flow(file_path=file_path, config=mock_config)
     assert result is True, "Flow should return True if HPC + Tiff->Zarr + transfers all succeed"
-    assert mock_transfer_controller.copy.call_count == 3, "Should do 3 transfers in success path"
+    assert mock_transfer_controller.copy.call_count == 5, "Should do 5 transfers in success path"
     mock_hpc_reconstruct.assert_called_once()
     mock_hpc_multires.assert_called_once()
     mock_schedule_pruning.assert_called_once()

orchestration/flows/bl832/alcf.py

@@ -434,6 +434,15 @@ def alcf_recon_flow(
             destination=config.data832_scratch
         )
 
+        logger.info(f"Transferring {file_name} from {config.alcf832_scratch} "
+                    f"at ALCF to {config.compute_dtn_832_scratch}")
+
+        transfer_controller.copy(
+            file_path=scratch_path_tiff,
+            source=config.alcf832_scratch,
+            destination=config.compute_dtn_832_scratch
+        )
+
     if alcf_multi_res_success:
         # Transfer B: Send reconstructed data (zarr) to data832
         logger.info(f"Transferring {file_name} from {config.alcf832_scratch} "
@@ -444,6 +453,14 @@ def alcf_recon_flow(
             destination=config.data832_scratch
         )
 
+        logger.info(f"Transferring {file_name} from {config.alcf832_scratch} "
+                    f"at ALCF to {config.compute_dtn_832_scratch}")
+        transfer_controller.copy(
+            file_path=scratch_path_zarr,
+            source=config.alcf832_scratch,
+            destination=config.compute_dtn_832_scratch
+        )
+
     # Place holder in case we want to transfer to NERSC for long term storage
     nersc_transfer_success = False
 

orchestration/flows/bl832/config.py

@@ -22,5 +22,7 @@ def __init__(self) -> None:
         self.nersc832_alsdev_recon_scripts = self.endpoints["nersc832_alsdev_recon_scripts"]
         self.alcf832_raw = self.endpoints["alcf832_raw"]
         self.alcf832_scratch = self.endpoints["alcf832_scratch"]
+        self.compute_dtn_832_raw = self.endpoints["compute_dtn_832_raw"]
+        self.compute_dtn_832_scratch = self.endpoints["compute_dtn_832_scratch"]
         self.scicat = config["scicat"]
         self.ghcr_images832 = config["ghcr_images832"]

orchestration/flows/bl832/nersc.py

@@ -500,6 +500,20 @@ def nersc_recon_flow(
         destination=config.data832_scratch
     )
 
+    logger.info("Copy from NERSC /global/cfs/cdirs/als/data_mover/8.3.2/scratch to compute-dtn.als.lbl.gov")
+
+    transfer_controller.copy(
+        file_path=tiff_file_path,
+        source=config.nersc832_alsdev_pscratch_scratch,
+        destination=config.compute_dtn_832_scratch
+    )
+
+    transfer_controller.copy(
+        file_path=zarr_file_path,
+        source=config.nersc832_alsdev_pscratch_scratch,
+        destination=config.compute_dtn_832_scratch
+    )
+
     logger.info("Scheduling pruning tasks.")
     schedule_pruning(
         config=config,