diff --git a/assets/ta_wrs2_conus_v006.ini b/assets/ta_wrs2_conus_v006.ini index a68d26b..4b0aa2d 100644 --- a/assets/ta_wrs2_conus_v006.ini +++ b/assets/ta_wrs2_conus_v006.ini @@ -2,33 +2,18 @@ [INPUTS] et_model = DISALEXI_TAIR - start_date = 2015-12-01 end_date = 2024-12-31 - -# Study area feature collection (mandatory) -# Script will make an inList filter call if property/features parameters are set study_area_coll = TIGER/2018/States study_area_property = STUSPS study_area_features = CONUS - -# Comma separated string of EE Collection IDs collections = LANDSAT/LC09/C02/T1_L2, LANDSAT/LC08/C02/T1_L2, LANDSAT/LE07/C02/T1_L2, LANDSAT/LT05/C02/T1_L2 - -# Maximum ACCA cloud cover percentage (0-100) cloud_cover = 70 - -# CSV file of Landsat scene IDs to skip -# scene_skip_list = - -# Comma separated string of Landsat WRS2 tiles (i.e. 'p045r043, p045r033']) -# If not set, use all available WRS2 tiles that intersect the study area -# wrs2_tiles = +scene_skip_list = https://raw.githubusercontent.com/cgmorton/scene-skip-list/main/v2p1.csv [EXPORT] export_coll = projects/openet/assets/disalexi/tair/conus_v006 - -mgrs_tiles = 10S, 10T, 10U, 11S, 11T, 11U, 12S, 12T, 12U, 13R, 13S, 13T, 13U, 14R, 14S, 14T, 14U, 15R, 15S, 15T, 15U, 16R, 16S, 16T, 16U, 17R, 17S, 17T, 18S, 18T, 19T +mgrs_tiles = 10S,10T,10U,11S,11T,11U,12R,12S,12T,12U,13R,13S,13T,13U,14R,14S,14T,14U,15R,15S,15T,15U,16R,16S,16T,16U,17R,17S,17T,18S,18T,19T mgrs_ftr_coll = projects/openet/assets/mgrs/conus/gridmet/zones [DISALEXI] diff --git a/assets/ta_wrs2_conus_v007.ini b/assets/ta_wrs2_conus_v007.ini new file mode 100644 index 0000000..74b82a3 --- /dev/null +++ b/assets/ta_wrs2_conus_v007.ini @@ -0,0 +1,37 @@ +# DisALEXI Ta Export Input File + +[INPUTS] +et_model = DISALEXI_TAIR +start_date = 2015-12-01 +end_date = 2024-12-31 +study_area_coll = TIGER/2018/States +study_area_property = STUSPS +study_area_features = CONUS +collections = LANDSAT/LC09/C02/T1_L2, LANDSAT/LC08/C02/T1_L2, LANDSAT/LE07/C02/T1_L2, LANDSAT/LT05/C02/T1_L2 +cloud_cover = 70 +scene_skip_list = https://raw.githubusercontent.com/cgmorton/scene-skip-list/main/v2p1.csv + +[EXPORT] +export_coll = projects/openet/assets/disalexi/tair/conus_v007 +mgrs_tiles = 10S,10T,10U,11S,11T,11U,12R,12S,12T,12U,13R,13S,13T,13U,14R,14S,14T,14U,15R,15S,15T,15U,16R,16S,16T,16U,17R,17S,17T,18S,18T,19T +mgrs_ftr_coll = projects/openet/assets/mgrs/conus/gridmet/zones + +[DISALEXI] +alexi_source = projects/ee-tulipyangyun-2/assets/alexi/ALEXI_V007 +lai_source = openet-landsat-lai +lst_source = projects/openet/assets/lst/landsat/c02 +elevation_source = USGS/SRTMGL1_003 +landcover_source = projects/sat-io/open-datasets/USGS/ANNUAL_NLCD/LANDCOVER +air_pres_source = CFSR +air_temp_source = CFSR +rs_daily_source = CFSR +rs_hourly_source = CFSR +vapor_pres_source = CFSR +wind_speed_source = CFSR +stability_iterations = 10 +albedo_iterations = 10 +et_min = 0 + +[TAIR] +offsets = -12,-7,-4,-2,-1,0,1,2,4,7,12 +retile = 4 diff --git a/assets/tair_image_wrs2_export.py b/assets/tair_image_wrs2_export.py index 5d1eda7..cd600e6 100644 --- a/assets/tair_image_wrs2_export.py +++ b/assets/tair_image_wrs2_export.py @@ -103,7 +103,10 @@ def main( ee.data.setWorkloadTag('disalexi-tair-scene-export') wrs2_tile_fmt = 'p{:03d}r{:03d}' - wrs2_tile_re = re.compile('p?(\\d{1,3})r?(\\d{1,3})') + if os.name == 'nt': + wrs2_tile_re = re.compile('p?(\\d{1,3})r?(\\d{1,3})') + else: + wrs2_tile_re = re.compile('p?(\d{1,3})r?(\d{1,3})') # List of path/rows to skip wrs2_skip_list = [ @@ -124,13 +127,52 @@ def main( 'p011r032', # Rhode Island coast 'p010r030', # Maine ] - wrs2_path_skip_list = [9, 49] - wrs2_row_skip_list = [25, 24, 43] + wrs2_path_skip_list = [ + 9, 49 + ] + wrs2_row_skip_list = [ + 25, 24, 43 + ] mgrs_skip_list = [] - date_skip_list = ['2023-06-16'] + date_skip_list = [ + '2023-06-16', '2023-12-31' + ] export_id_fmt = '{model}_{index}' + # Initialize Earth Engine + if gee_key_file: + logging.info(f'\nInitializing GEE using user key file: {gee_key_file}') + try: + ee.Initialize( + ee.ServiceAccountCredentials('_', key_file=gee_key_file), + opt_url='https://earthengine-highvolume.googleapis.com' + ) + except ee.ee_exception.EEException: + logging.warning('Unable to initialize GEE using user key file') + return False + elif 'FUNCTION_REGION' in os.environ: + # Assume code is deployed to a cloud function + logging.debug(f'\nInitializing GEE using application default credentials') + import google.auth + credentials, project_id = google.auth.default( + default_scopes=['https://www.googleapis.com/auth/earthengine'] + ) + ee.Initialize( + credentials, project=project_id, opt_url='https://earthengine-highvolume.googleapis.com' + ) + elif project_id is not None: + logging.info(f'\nInitializing Earth Engine using project credentials' + f'\n Project ID: {project_id}') + try: + ee.Initialize(project=project_id, opt_url='https://earthengine-highvolume.googleapis.com') + except Exception as e: + logging.warning(f'\nUnable to initialize GEE using project ID\n {e}') + return False + else: + logging.info('\nInitializing Earth Engine using user credentials') + ee.Initialize() + # Read config file logging.info(f' {os.path.basename(ini_path)}') ini = read_ini(ini_path) @@ -292,7 +334,8 @@ def main( if tiles: logging.info('\nOverriding INI mgrs_tiles and utm_zones parameters') logging.info(f' user tiles: {tiles}') - mgrs_tiles = sorted([y.strip() for x in tiles for y in x.split(',')]) + mgrs_tiles = sorted([x.strip() for x in tiles.split(',')]) + # mgrs_tiles = sorted([y.strip() for x in tiles for y in x.split(',')]) mgrs_tiles = [x.upper() for x in mgrs_tiles if x] logging.info(f' mgrs_tiles: {", ".join(mgrs_tiles)}') utm_zones = sorted(list(set([int(x[:2]) for x in mgrs_tiles]))) @@ -366,6 +409,11 @@ def main( logging.info(f' Offsets: {tair_args["offsets"]}') logging.debug(f' Retile: {retile}') + if os.name == 'nt': + landsat_re_str = 'L[TEC]0[45789]_\d{3}\d{3}_\d{8}' + else: + landsat_re_str = 'L[TEC]0[45789]_\\d{3}\\d{3}_\\d{8}' + # Read the scene ID skip list if (not scene_id_skip_path) or scene_id_skip_path.lower() in ['none', '']: logging.info(f'\nScene ID skip list not set') @@ -378,7 +426,7 @@ def main( scene_id_skip_list = { scene_id.upper() for scene_id in pd.read_csv(scene_id_skip_path)['SCENE_ID'].values - if re.match('L[TEC]0[45789]_\d{3}\d{3}_\d{8}', scene_id) + if re.match(landsat_re_str, scene_id) } logging.info(f' Skip list count: {len(scene_id_skip_list)}') else: @@ -396,40 +444,6 @@ def main( logging.info(' Task logging disabled, error setting up datastore client') log_tasks = False - # Initialize Earth Engine - if gee_key_file: - logging.info(f'\nInitializing GEE using user key file: {gee_key_file}') - try: - ee.Initialize( - ee.ServiceAccountCredentials('_', key_file=gee_key_file), - opt_url='https://earthengine-highvolume.googleapis.com' - ) - except ee.ee_exception.EEException: - logging.warning('Unable to initialize GEE using user key file') - return False - elif 'FUNCTION_REGION' in os.environ: - # Assume code is deployed to a cloud function - logging.debug(f'\nInitializing GEE using application default credentials') - import google.auth - credentials, project_id = google.auth.default( - default_scopes=['https://www.googleapis.com/auth/earthengine'] - ) - ee.Initialize( - credentials, project=project_id, opt_url='https://earthengine-highvolume.googleapis.com' - ) - elif project_id is not None: - logging.info(f'\nInitializing Earth Engine using project credentials' - f'\n Project ID: {project_id}') - try: - ee.Initialize(project=project_id, opt_url='https://earthengine-highvolume.googleapis.com') - except Exception as e: - logging.warning(f'\nUnable to initialize GEE using project ID\n {e}') - return False - else: - logging.info('\nInitializing Earth Engine using user credentials') - ee.Initialize() - - # Build output collection and folder if necessary logging.debug(f'\nExport Collection: {export_coll_id}') if not ee.data.getInfo(export_coll_id.rsplit('/', 1)[0]): @@ -478,9 +492,22 @@ def main( alexi_cs = 0.04 alexi_x, alexi_y = -125.02, 49.78 # alexi_geo = [0.04, 0.0, -125.02, 0.0, -0.04, 49.78] + elif ((alexi_coll_id.upper() == 'CONUS_V007') or + alexi_coll_id.endswith('projects/ee-tulipyangyun-2/assets/alexi/ALEXI_V007')): + alexi_coll_id = 'projects/ee-tulipyangyun-2/assets/alexi/ALEXI_V007' + alexi_cs = 0.04 + alexi_x, alexi_y = -125.02, 49.78 + # alexi_geo = [0.04, 0.0, -125.02, 0.0, -0.04, 49.78] else: - raise ValueError(f'unsupported ALEXI source: {alexi_coll_id}') - + raise ValueError(f'Unsupported ALEXI source: {alexi_coll_id}') + # # CGM - We could support reading any image collection for the source + # # but this would require modifications to disalexi.py + # else: + # alexi_info = ee.ImageCollection(alexi_coll_id).first().getInfo()['bands'][0] + # alexi_crs = alexi_info['crs'] + # alexi_cs = alexi_info['crs_transform'][0] + # alexi_x = alexi_info['crs_transform'][2] + # alexi_y = alexi_info['crs_transform'][5] logging.debug(f' Collection: {alexi_coll_id}') @@ -568,10 +595,14 @@ def set_date(x): # Filter to the wrs2_tile list # The WRS2 tile filtering should be done in the Collection call above, - # but the DisALEXI model does not currently support this + # but not all of the models support this + if os.name == 'nt': + wrs2_re_str = '_(\\d{3})(\\d{3})_' + else: + wrs2_re_str = '_(\d{3})(\d{3})_' year_image_id_list = [ x for x in year_image_id_list - if 'p{}r{}'.format(*re.findall('_(\d{3})(\d{3})_', x)[0]) in tile_list + if 'p{}r{}'.format(*re.findall(wrs2_re_str, x)[0]) in tile_list ] # Filter image_ids that have already been processed as part of a @@ -689,54 +720,27 @@ def set_date(x): asset_ver = utils.ver_str_2_num(asset_props[asset_id]['model_version']) if asset_ver < model_ver: - logging.info(f' {scene_id} - Existing asset model version is old, removing') + logging.info(f' {scene_id} - Existing asset model version is old, overwriting') logging.debug(f' asset: {asset_ver}\n model: {model_ver}') - try: - ee.data.deleteAsset(asset_id) - except: - logging.info(f' {scene_id} - Error removing asset, skipping') - continue # elif (asset_props[asset_id]['alexi_source'] < model_args['alexi_source']): # logging.info(' ALEXI source is old, removing') # # input('ENTER') - # try: - # ee.data.deleteAsset(asset_id) - # except: - # logging.info(' Error removing asset, skipping') - # continue # elif (asset_props[asset_id]['build_date'] <= '2020-04-27'): # logging.info(' build_date is old, removing') # # input('ENTER') - # try: - # ee.data.deleteAsset(asset_id) - # except: - # logging.info(' Error removing asset, skipping') - # continue # elif (utils.ver_str_2_num(asset_props[asset_id]['tool_version']) < # utils.ver_str_2_num(TOOL_VERSION)): # logging.info(' Asset tool version is old, removing') - # try: - # ee.data.deleteAsset(asset_id) - # except: - # logging.info(' Error removing asset, skipping') - # continue else: - logging.info(f' {scene_id} - Asset is up to date, skipping') + logging.debug(f' {scene_id} - Asset is up to date, skipping') continue elif overwrite_flag: if export_id in tasks.keys(): logging.info(f' {scene_id} - Task already submitted, cancelling') ee.data.cancelTask(tasks[export_id]['id']) # ee.data.cancelOperation(tasks[export_id]['id']) - # This is intentionally not an "elif" so that a task can be - # cancelled and an existing image/file/asset can be removed if asset_props and (asset_id in asset_props.keys()): - logging.info(f' {scene_id} - Asset already exists, removing') - try: - ee.data.deleteAsset(asset_id) - except: - logging.info(' Error removing asset, skipping') - continue + logging.info(f' {scene_id} - Asset already exists, overwriting') else: if export_id in tasks.keys(): logging.debug(f' {scene_id} - Task already submitted, skipping') @@ -911,6 +915,7 @@ def set_date(x): crs=alexi_crs, crsTransform='[' + ','.join(list(map(str, export_geo))) + ']', dimensions='{0}x{1}'.format(*export_shape), + overwrite=overwrite_flag or update_flag, ) # # except ee.ee_exception.EEException as e: # except Exception as e: @@ -1050,8 +1055,8 @@ def arg_parse(): '--reverse', default=False, action='store_true', help='Process WRS2 tiles in reverse order') parser.add_argument( - '--tiles', default='', nargs='+', - help='Comma/space separated list of tiles to process') + '--tiles', default='', + help='Comma separated list of tiles to process') parser.add_argument( '--update', default=False, action='store_true', help='Update images with older model version numbers') diff --git a/openet/disalexi/disalexi.py b/openet/disalexi/disalexi.py index e1ad14a..921c7ad 100644 --- a/openet/disalexi/disalexi.py +++ b/openet/disalexi/disalexi.py @@ -70,8 +70,8 @@ def __init__( Prepped image ta_source : {'projects/openet/assets/disalexi/tair/conus_v006_1k'} ALEXI scale air temperature image collection ID. - alexi_source : {'CONUS_V006'} - ALEXI ET image collection ID (the default is 'CONUS_V006'). + alexi_source : {'CONUS_V006', 'CONUS_V007'} + ALEXI ET image collection ID or keyword (the default is 'CONUS_V006'). lai_source : {'openet-landsat-lai'} LAI image collection ID or set to "openet-landsat-lai" to compute LAI dynamically using the openet-landsat-lai module. @@ -491,10 +491,8 @@ def et_alexi(self): """ alexi_keyword_sources = { 'CONUS_V006': 'projects/ee-tulipyangyun-2/assets/alexi/ALEXI_V006', + 'CONUS_V007': 'projects/ee-tulipyangyun-2/assets/alexi/ALEXI_V007', } - alexi_re = re.compile( - '(projects/earthengine-legacy/assets/)?projects/disalexi/alexi/CONUS_V\\w+' - ) if utils.is_number(self.alexi_source): # Interpret numbers as constant images @@ -506,13 +504,7 @@ def et_alexi(self): alexi_coll = ee.ImageCollection(alexi_coll_id).filterDate(self.start_date, self.end_date) # TODO: Check if collection size is 0 alexi_img = ee.Image(alexi_coll.first()).multiply(0.408) - elif alexi_re.match(self.alexi_source): - alexi_coll = ee.ImageCollection(self.alexi_source).filterDate(self.start_date, self.end_date) - alexi_img = ee.Image(alexi_coll.first()).multiply(0.408) elif self.alexi_source in alexi_keyword_sources.values(): - # CGM - Quick fix for catching if the alexi_source was to as the - # collection ID, specifically for V005 since it is currently in a - # different project and won't get matched by the regex. alexi_coll = ee.ImageCollection(self.alexi_source).filterDate(self.start_date, self.end_date) alexi_img = ee.Image(alexi_coll.first()).multiply(0.408) else: @@ -544,7 +536,7 @@ def landcover(self): # If the source is an ee.Image assume it is an NLCD image lc_img = self.landcover_source.rename(['landcover']) self.lc_type = 'NLCD' - elif re.match('projects/sat-io/open-datasets/USGS/ANNUAL_NLCD/LANDCOVER/Annual_NLCD_LndCov_\\d{4}_CU_\w+', + elif re.match('projects/sat-io/open-datasets/USGS/ANNUAL_NLCD/LANDCOVER/Annual_NLCD_LndCov_\\d{4}_CU_\\w+', self.landcover_source, re.I): # Assume an annual NLCD image ID was passed in and use it directly lc_img = ee.Image(self.landcover_source).rename(['landcover']) diff --git a/openet/disalexi/tests/test_d_image.py b/openet/disalexi/tests/test_d_image.py index 8f215c1..7d86148 100644 --- a/openet/disalexi/tests/test_d_image.py +++ b/openet/disalexi/tests/test_d_image.py @@ -226,9 +226,12 @@ def test_Image_ta_properties(): 'scene_id, source, xy, expected', [ # ALEXI ET is currently in MJ m-2 d-1 - ['LC08_044033_20200708', 'CONUS_V006', TEST_POINT, 17.999324798583984 * 0.408], - ['LC08_044033_20200724', 'CONUS_V006', TEST_POINT, 17.819684982299805 * 0.408], - [None, 'projects/ee-tulipyangyun-2/assets/alexi/ALEXI_V006', TEST_POINT, 12.765579223632812 * 0.408], + ['LC08_044033_20200708', 'CONUS_V006', TEST_POINT, 17.99932480 * 0.408], + ['LC08_044033_20200724', 'CONUS_V006', TEST_POINT, 17.81968498 * 0.408], + [None, 'projects/ee-tulipyangyun-2/assets/alexi/ALEXI_V006', TEST_POINT, 12.76557922 * 0.408], + ['LC08_044033_20200708', 'CONUS_V007', TEST_POINT, 16.55301476 * 0.408], + ['LC08_044033_20200724', 'CONUS_V007', TEST_POINT, 16.44197273 * 0.408], + [None, 'projects/ee-tulipyangyun-2/assets/alexi/ALEXI_V007', TEST_POINT, 11.36735344 * 0.408], [None, ee.Image('USGS/SRTMGL1_003').multiply(0).add(10), TEST_POINT, 10], [None, '10.382039', TEST_POINT, 10.382039], [None, 10.382039, TEST_POINT, 10.382039], diff --git a/pyproject.toml b/pyproject.toml index af0f263..68c4816 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "openet-disalexi" -version = "0.1.0" +version = "0.2.0" authors = [ { name = "Yun Yang", email = "yy2356@cornell.edu" }, { name = "Martha Anderson", email = "martha.anderson@usda.gov" }, @@ -20,7 +20,7 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ - "earthengine-api>=1.5.2", + "earthengine-api>=1.7.4", "openet-core>=0.7.0", "openet-landsat-lai>=0.3.0", # "openet-refet-gee>=0.2.0",