ALEXI v007 Support

assets/ta_wrs2_conus_v006.ini

@@ -2,33 +2,18 @@
 
 [INPUTS]
 et_model = DISALEXI_TAIR
-
 start_date = 2015-12-01
 end_date = 2024-12-31
-
-# Study area feature collection (mandatory)
-# Script will make an inList filter call if property/features parameters are set
 study_area_coll = TIGER/2018/States
 study_area_property = STUSPS
 study_area_features = CONUS
-
-# Comma separated string of EE Collection IDs
 collections = LANDSAT/LC09/C02/T1_L2, LANDSAT/LC08/C02/T1_L2, LANDSAT/LE07/C02/T1_L2, LANDSAT/LT05/C02/T1_L2
-
-# Maximum ACCA cloud cover percentage (0-100)
 cloud_cover = 70
-
-# CSV file of Landsat scene IDs to skip
-# scene_skip_list =
-
-# Comma separated string of Landsat WRS2 tiles (i.e. 'p045r043, p045r033'])
-# If not set, use all available WRS2 tiles that intersect the study area
-# wrs2_tiles =
+scene_skip_list = https://raw.githubusercontent.com/cgmorton/scene-skip-list/main/v2p1.csv
 
 [EXPORT]
 export_coll = projects/openet/assets/disalexi/tair/conus_v006
-
-mgrs_tiles = 10S, 10T, 10U, 11S, 11T, 11U, 12S, 12T, 12U, 13R, 13S, 13T, 13U, 14R, 14S, 14T, 14U, 15R, 15S, 15T, 15U, 16R, 16S, 16T, 16U, 17R, 17S, 17T, 18S, 18T, 19T
+mgrs_tiles = 10S,10T,10U,11S,11T,11U,12R,12S,12T,12U,13R,13S,13T,13U,14R,14S,14T,14U,15R,15S,15T,15U,16R,16S,16T,16U,17R,17S,17T,18S,18T,19T
 mgrs_ftr_coll = projects/openet/assets/mgrs/conus/gridmet/zones
 
 [DISALEXI]

assets/ta_wrs2_conus_v007.ini

@@ -0,0 +1,37 @@
+# DisALEXI Ta Export Input File
+
+[INPUTS]
+et_model = DISALEXI_TAIR
+start_date = 2015-12-01
+end_date = 2024-12-31
+study_area_coll = TIGER/2018/States
+study_area_property = STUSPS
+study_area_features = CONUS
+collections = LANDSAT/LC09/C02/T1_L2, LANDSAT/LC08/C02/T1_L2, LANDSAT/LE07/C02/T1_L2, LANDSAT/LT05/C02/T1_L2
+cloud_cover = 70
+scene_skip_list = https://raw.githubusercontent.com/cgmorton/scene-skip-list/main/v2p1.csv
+
+[EXPORT]
+export_coll = projects/openet/assets/disalexi/tair/conus_v007
+mgrs_tiles = 10S,10T,10U,11S,11T,11U,12R,12S,12T,12U,13R,13S,13T,13U,14R,14S,14T,14U,15R,15S,15T,15U,16R,16S,16T,16U,17R,17S,17T,18S,18T,19T
+mgrs_ftr_coll = projects/openet/assets/mgrs/conus/gridmet/zones
+
+[DISALEXI]
+alexi_source = projects/ee-tulipyangyun-2/assets/alexi/ALEXI_V007
+lai_source = openet-landsat-lai
+lst_source = projects/openet/assets/lst/landsat/c02
+elevation_source = USGS/SRTMGL1_003
+landcover_source = projects/sat-io/open-datasets/USGS/ANNUAL_NLCD/LANDCOVER
+air_pres_source = CFSR
+air_temp_source = CFSR
+rs_daily_source = CFSR
+rs_hourly_source = CFSR
+vapor_pres_source = CFSR
+wind_speed_source = CFSR
+stability_iterations = 10
+albedo_iterations = 10
+et_min = 0
+
+[TAIR]
+offsets = -12,-7,-4,-2,-1,0,1,2,4,7,12
+retile = 4

assets/tair_image_wrs2_export.py

@@ -103,7 +103,10 @@ def main(
     ee.data.setWorkloadTag('disalexi-tair-scene-export')
 
     wrs2_tile_fmt = 'p{:03d}r{:03d}'
-    wrs2_tile_re = re.compile('p?(\\d{1,3})r?(\\d{1,3})')
+    if os.name == 'nt':
+        wrs2_tile_re = re.compile('p?(\\d{1,3})r?(\\d{1,3})')
+    else:
+        wrs2_tile_re = re.compile('p?(\d{1,3})r?(\d{1,3})')
 
     # List of path/rows to skip
     wrs2_skip_list = [
@@ -124,13 +127,52 @@ def main(
         'p011r032',  # Rhode Island coast
         'p010r030',  # Maine
     ]
-    wrs2_path_skip_list = [9, 49]
-    wrs2_row_skip_list = [25, 24, 43]
+    wrs2_path_skip_list = [
+        9, 49
+    ]
+    wrs2_row_skip_list = [
+        25, 24, 43
+    ]
     mgrs_skip_list = []
-    date_skip_list = ['2023-06-16']
+    date_skip_list = [
+        '2023-06-16', '2023-12-31'
+    ]
 
     export_id_fmt = '{model}_{index}'
 
+    # Initialize Earth Engine
+    if gee_key_file:
+        logging.info(f'\nInitializing GEE using user key file: {gee_key_file}')
+        try:
+            ee.Initialize(
+                ee.ServiceAccountCredentials('_', key_file=gee_key_file),
+                opt_url='https://earthengine-highvolume.googleapis.com'
+            )
+        except ee.ee_exception.EEException:
+            logging.warning('Unable to initialize GEE using user key file')
+            return False
+    elif 'FUNCTION_REGION' in os.environ:
+        # Assume code is deployed to a cloud function
+        logging.debug(f'\nInitializing GEE using application default credentials')
+        import google.auth
+        credentials, project_id = google.auth.default(
+            default_scopes=['https://www.googleapis.com/auth/earthengine']
+        )
+        ee.Initialize(
+            credentials, project=project_id, opt_url='https://earthengine-highvolume.googleapis.com'
+        )
+    elif project_id is not None:
+        logging.info(f'\nInitializing Earth Engine using project credentials'
+                     f'\n  Project ID: {project_id}')
+        try:
+            ee.Initialize(project=project_id, opt_url='https://earthengine-highvolume.googleapis.com')
+        except Exception as e:
+            logging.warning(f'\nUnable to initialize GEE using project ID\n  {e}')
+            return False
+    else:
+        logging.info('\nInitializing Earth Engine using user credentials')
+        ee.Initialize()
+
     # Read config file
     logging.info(f'  {os.path.basename(ini_path)}')
     ini = read_ini(ini_path)
@@ -292,7 +334,8 @@ def main(
     if tiles:
         logging.info('\nOverriding INI mgrs_tiles and utm_zones parameters')
         logging.info(f'  user tiles: {tiles}')
-        mgrs_tiles = sorted([y.strip() for x in tiles for y in x.split(',')])
+        mgrs_tiles = sorted([x.strip() for x in tiles.split(',')])
+        # mgrs_tiles = sorted([y.strip() for x in tiles for y in x.split(',')])
         mgrs_tiles = [x.upper() for x in mgrs_tiles if x]
         logging.info(f'  mgrs_tiles: {", ".join(mgrs_tiles)}')
         utm_zones = sorted(list(set([int(x[:2]) for x in mgrs_tiles])))
@@ -366,6 +409,11 @@ def main(
     logging.info(f'  Offsets:  {tair_args["offsets"]}')
     logging.debug(f'  Retile:   {retile}')
 
+    if os.name == 'nt':
+        landsat_re_str = 'L[TEC]0[45789]_\d{3}\d{3}_\d{8}'
+    else:
+        landsat_re_str = 'L[TEC]0[45789]_\\d{3}\\d{3}_\\d{8}'
+
     # Read the scene ID skip list
     if (not scene_id_skip_path) or scene_id_skip_path.lower() in ['none', '']:
         logging.info(f'\nScene ID skip list not set')
@@ -378,7 +426,7 @@ def main(
         scene_id_skip_list = {
             scene_id.upper() for scene_id in
             pd.read_csv(scene_id_skip_path)['SCENE_ID'].values
-            if re.match('L[TEC]0[45789]_\d{3}\d{3}_\d{8}', scene_id)
+            if re.match(landsat_re_str, scene_id)
         }
         logging.info(f'  Skip list count: {len(scene_id_skip_list)}')
     else:
@@ -396,40 +444,6 @@ def main(
             logging.info('  Task logging disabled, error setting up datastore client')
             log_tasks = False
 
-    # Initialize Earth Engine
-    if gee_key_file:
-        logging.info(f'\nInitializing GEE using user key file: {gee_key_file}')
-        try:
-            ee.Initialize(
-                ee.ServiceAccountCredentials('_', key_file=gee_key_file),
-                opt_url='https://earthengine-highvolume.googleapis.com'
-            )
-        except ee.ee_exception.EEException:
-            logging.warning('Unable to initialize GEE using user key file')
-            return False
-    elif 'FUNCTION_REGION' in os.environ:
-        # Assume code is deployed to a cloud function
-        logging.debug(f'\nInitializing GEE using application default credentials')
-        import google.auth
-        credentials, project_id = google.auth.default(
-            default_scopes=['https://www.googleapis.com/auth/earthengine']
-        )
-        ee.Initialize(
-            credentials, project=project_id, opt_url='https://earthengine-highvolume.googleapis.com'
-        )
-    elif project_id is not None:
-        logging.info(f'\nInitializing Earth Engine using project credentials'
-                     f'\n  Project ID: {project_id}')
-        try:
-            ee.Initialize(project=project_id, opt_url='https://earthengine-highvolume.googleapis.com')
-        except Exception as e:
-            logging.warning(f'\nUnable to initialize GEE using project ID\n  {e}')
-            return False
-    else:
-        logging.info('\nInitializing Earth Engine using user credentials')
-        ee.Initialize()
-
-
     # Build output collection and folder if necessary
     logging.debug(f'\nExport Collection: {export_coll_id}')
     if not ee.data.getInfo(export_coll_id.rsplit('/', 1)[0]):
@@ -478,9 +492,22 @@ def main(
         alexi_cs = 0.04
         alexi_x, alexi_y = -125.02, 49.78
         # alexi_geo = [0.04, 0.0, -125.02, 0.0, -0.04, 49.78]
+    elif ((alexi_coll_id.upper() == 'CONUS_V007') or
+            alexi_coll_id.endswith('projects/ee-tulipyangyun-2/assets/alexi/ALEXI_V007')):
+        alexi_coll_id = 'projects/ee-tulipyangyun-2/assets/alexi/ALEXI_V007'
+        alexi_cs = 0.04
+        alexi_x, alexi_y = -125.02, 49.78
+        # alexi_geo = [0.04, 0.0, -125.02, 0.0, -0.04, 49.78]
     else:
-        raise ValueError(f'unsupported ALEXI source: {alexi_coll_id}')
-
+        raise ValueError(f'Unsupported ALEXI source: {alexi_coll_id}')
+    # # CGM - We could support reading any image collection for the source
+    # #   but this would require modifications to disalexi.py
+    # else:
+    #     alexi_info = ee.ImageCollection(alexi_coll_id).first().getInfo()['bands'][0]
+    #     alexi_crs = alexi_info['crs']
+    #     alexi_cs = alexi_info['crs_transform'][0]
+    #     alexi_x = alexi_info['crs_transform'][2]
+    #     alexi_y = alexi_info['crs_transform'][5]
     logging.debug(f'  Collection: {alexi_coll_id}')
 
 
@@ -568,10 +595,14 @@ def set_date(x):
 
             # Filter to the wrs2_tile list
             # The WRS2 tile filtering should be done in the Collection call above,
-            #   but the DisALEXI model does not currently support this
+            #   but not all of the models support this
+            if os.name == 'nt':
+                wrs2_re_str = '_(\\d{3})(\\d{3})_'
+            else:
+                wrs2_re_str = '_(\d{3})(\d{3})_'
             year_image_id_list = [
                 x for x in year_image_id_list
-                if 'p{}r{}'.format(*re.findall('_(\d{3})(\d{3})_', x)[0]) in tile_list
+                if 'p{}r{}'.format(*re.findall(wrs2_re_str, x)[0]) in tile_list
             ]
 
             # Filter image_ids that have already been processed as part of a
@@ -689,54 +720,27 @@ def set_date(x):
                         asset_ver = utils.ver_str_2_num(asset_props[asset_id]['model_version'])
 
                         if asset_ver < model_ver:
-                            logging.info(f'  {scene_id} - Existing asset model version is old, removing')
+                            logging.info(f'  {scene_id} - Existing asset model version is old, overwriting')
                             logging.debug(f'  asset: {asset_ver}\n  model: {model_ver}')
-                            try:
-                                ee.data.deleteAsset(asset_id)
-                            except:
-                                logging.info(f'  {scene_id} - Error removing asset, skipping')
-                                continue
                         # elif (asset_props[asset_id]['alexi_source'] < model_args['alexi_source']):
                         #     logging.info('  ALEXI source is old, removing')
                         #     # input('ENTER')
-                        #     try:
-                        #         ee.data.deleteAsset(asset_id)
-                        #     except:
-                        #         logging.info('  Error removing asset, skipping')
-                        #         continue
                         # elif (asset_props[asset_id]['build_date'] <= '2020-04-27'):
                         #     logging.info('  build_date is old, removing')
                         #     # input('ENTER')
-                        #     try:
-                        #         ee.data.deleteAsset(asset_id)
-                        #     except:
-                        #         logging.info('  Error removing asset, skipping')
-                        #         continue
                         # elif (utils.ver_str_2_num(asset_props[asset_id]['tool_version']) <
                         #       utils.ver_str_2_num(TOOL_VERSION)):
                         #     logging.info('  Asset tool version is old, removing')
-                        #     try:
-                        #         ee.data.deleteAsset(asset_id)
-                        #     except:
-                        #         logging.info('  Error removing asset, skipping')
-                        #         continue
                         else:
-                            logging.info(f'  {scene_id} - Asset is up to date, skipping')
+                            logging.debug(f'  {scene_id} - Asset is up to date, skipping')
                             continue
                 elif overwrite_flag:
                     if export_id in tasks.keys():
                         logging.info(f'  {scene_id} - Task already submitted, cancelling')
                         ee.data.cancelTask(tasks[export_id]['id'])
                         # ee.data.cancelOperation(tasks[export_id]['id'])
-                    # This is intentionally not an "elif" so that a task can be
-                    # cancelled and an existing image/file/asset can be removed
                     if asset_props and (asset_id in asset_props.keys()):
-                        logging.info(f'  {scene_id} - Asset already exists, removing')
-                        try:
-                            ee.data.deleteAsset(asset_id)
-                        except:
-                            logging.info('  Error removing asset, skipping')
-                            continue
+                        logging.info(f'  {scene_id} - Asset already exists, overwriting')
                 else:
                     if export_id in tasks.keys():
                         logging.debug(f'  {scene_id} - Task already submitted, skipping')
@@ -911,6 +915,7 @@ def set_date(x):
                     crs=alexi_crs,
                     crsTransform='[' + ','.join(list(map(str, export_geo))) + ']',
                     dimensions='{0}x{1}'.format(*export_shape),
+                    overwrite=overwrite_flag or update_flag,
                 )
                 #     # except ee.ee_exception.EEException as e:
                 #     except Exception as e:
@@ -1050,8 +1055,8 @@ def arg_parse():
         '--reverse', default=False, action='store_true',
         help='Process WRS2 tiles in reverse order')
     parser.add_argument(
-        '--tiles', default='', nargs='+',
-        help='Comma/space separated list of tiles to process')
+        '--tiles', default='',
+        help='Comma separated list of tiles to process')
     parser.add_argument(
         '--update', default=False, action='store_true',
         help='Update images with older model version numbers')

openet/disalexi/disalexi.py

@@ -70,8 +70,8 @@ def __init__(
             Prepped image
         ta_source : {'projects/openet/assets/disalexi/tair/conus_v006_1k'}
             ALEXI scale air temperature image collection ID.
-        alexi_source : {'CONUS_V006'}
-            ALEXI ET image collection ID (the default is 'CONUS_V006').
+        alexi_source : {'CONUS_V006', 'CONUS_V007'}
+            ALEXI ET image collection ID or keyword (the default is 'CONUS_V006').
         lai_source : {'openet-landsat-lai'}
             LAI image collection ID or set to "openet-landsat-lai" to compute LAI
             dynamically using the openet-landsat-lai module.
@@ -491,10 +491,8 @@ def et_alexi(self):
         """
         alexi_keyword_sources = {
             'CONUS_V006': 'projects/ee-tulipyangyun-2/assets/alexi/ALEXI_V006',
+            'CONUS_V007': 'projects/ee-tulipyangyun-2/assets/alexi/ALEXI_V007',
         }
-        alexi_re = re.compile(
-            '(projects/earthengine-legacy/assets/)?projects/disalexi/alexi/CONUS_V\\w+'
-        )
 
         if utils.is_number(self.alexi_source):
             # Interpret numbers as constant images
@@ -506,13 +504,7 @@ def et_alexi(self):
             alexi_coll = ee.ImageCollection(alexi_coll_id).filterDate(self.start_date, self.end_date)
             # TODO: Check if collection size is 0
             alexi_img = ee.Image(alexi_coll.first()).multiply(0.408)
-        elif alexi_re.match(self.alexi_source):
-            alexi_coll = ee.ImageCollection(self.alexi_source).filterDate(self.start_date, self.end_date)
-            alexi_img = ee.Image(alexi_coll.first()).multiply(0.408)
         elif self.alexi_source in alexi_keyword_sources.values():
-            # CGM - Quick fix for catching if the alexi_source was to as the
-            #   collection ID, specifically for V005 since it is currently in a
-            #   different project and won't get matched by the regex.
             alexi_coll = ee.ImageCollection(self.alexi_source).filterDate(self.start_date, self.end_date)
             alexi_img = ee.Image(alexi_coll.first()).multiply(0.408)
         else:
@@ -544,7 +536,7 @@ def landcover(self):
             # If the source is an ee.Image assume it is an NLCD image
             lc_img = self.landcover_source.rename(['landcover'])
             self.lc_type = 'NLCD'
-        elif re.match('projects/sat-io/open-datasets/USGS/ANNUAL_NLCD/LANDCOVER/Annual_NLCD_LndCov_\\d{4}_CU_\w+',
+        elif re.match('projects/sat-io/open-datasets/USGS/ANNUAL_NLCD/LANDCOVER/Annual_NLCD_LndCov_\\d{4}_CU_\\w+',
                       self.landcover_source, re.I):
             # Assume an annual NLCD image ID was passed in and use it directly
             lc_img = ee.Image(self.landcover_source).rename(['landcover'])

openet/disalexi/tests/test_d_image.py

@@ -226,9 +226,12 @@ def test_Image_ta_properties():
     'scene_id, source, xy, expected',
     [
         # ALEXI ET is currently in MJ m-2 d-1
-        ['LC08_044033_20200708', 'CONUS_V006', TEST_POINT, 17.999324798583984 * 0.408],
-        ['LC08_044033_20200724', 'CONUS_V006', TEST_POINT, 17.819684982299805 * 0.408],
-        [None, 'projects/ee-tulipyangyun-2/assets/alexi/ALEXI_V006', TEST_POINT, 12.765579223632812 * 0.408],
+        ['LC08_044033_20200708', 'CONUS_V006', TEST_POINT, 17.99932480 * 0.408],
+        ['LC08_044033_20200724', 'CONUS_V006', TEST_POINT, 17.81968498 * 0.408],
+        [None, 'projects/ee-tulipyangyun-2/assets/alexi/ALEXI_V006', TEST_POINT, 12.76557922 * 0.408],
+        ['LC08_044033_20200708', 'CONUS_V007', TEST_POINT, 16.55301476 * 0.408],
+        ['LC08_044033_20200724', 'CONUS_V007', TEST_POINT, 16.44197273 * 0.408],
+        [None, 'projects/ee-tulipyangyun-2/assets/alexi/ALEXI_V007', TEST_POINT, 11.36735344 * 0.408],
         [None, ee.Image('USGS/SRTMGL1_003').multiply(0).add(10), TEST_POINT, 10],
         [None, '10.382039', TEST_POINT, 10.382039],
         [None, 10.382039, TEST_POINT, 10.382039],