diff --git a/bin/pipeline.py b/bin/pipeline.py index a55fd994..f4b3785d 100755 --- a/bin/pipeline.py +++ b/bin/pipeline.py @@ -14,63 +14,69 @@ from params import oversample_nrealisations ''' -# --log -# Sbatch: python3 pipeline.py --survey gama --use_sbatch --queue cosma --reset -# Head: python3 pipeline.py --survey desi --dryrun -# -# Note: use sinfo to see available nodes to each queue. -''' +# --log +# Sbatch: python3 pipeline.py --survey gama --use_sbatch --queue cosma --reset # Head: python3 pipeline.py --survey desi --dryrun # # Note: use sinfo to see available nodes to each queue. ''' def pipeline(args, use_sbatch=False, reset=False, nooverwrite=False, dryrun=True, survey='gama', freshclone=False, log=False, custom=True, comments=None, stages=None): - if custom & (args != None): - customise_script(args) - - custom = '/custom/' - - else: - custom = '' - + ''' + Launcher of the lumfn pipeline. + + --use_sbatch submit to the queue + --reset first tidy up $GOLD_DIR + --nooverwrite if a output file exists, do not rerun the step. + --dryrun run on a small test subset, survey dependent definition. + --survey survey to solve for, one of [gama, abacus_gama] (desi). + --freshclone clone main under ~/tmp/ and run off that. + --log individidual steps are written to log files, e.g. random creation. Triggers logging of pipeline.py output. + --custom customise submission scripts with memory, queue, username etc. + --comments text string to be added to run configuration summary, e.g. with motivation for run. + --stages unsupported. + ''' if reset & nooverwrite: raise ValueError('No overwrite and reset are incompatible.') - if dryrun: - dryrun = '--dryrun' - - else: - dryrun = '' - - if nooverwrite: - nooverwrite = '--nooverwrite' - else: - nooverwrite = '' - + # Logging output of pipeline.py; Note: individual pipeline steps are logged separately. if log: - sys.stdout = open('pipeline.log', 'w') + sys.stdout = open('pipeline.log', 'w') + # Clean a (subset) of the files and logs in $GOLD_DIR. if reset: print('\n\n>>>>> TRASHING GOLD_DIR AND RANDOMS <<<<<\n\n') cmds = [] + root = os.environ['GOLD_DIR'] + + for dirpath in [root, root + '/randoms/']: + cmds.append('rm -f {}/logs/*.log'.format(dirpath)) + cmds.append('rm -f {}/*_dryrun.fits'.format(dirpath)) - for root in [os.environ['GOLD_DIR'], os.environ['RANDOMS_DIR']]: - cmds.append('rm -f {}/logs/*.log'.format(root)) - cmds.append('rm -f {}/*_dryrun.fits'.format(root)) - - cmds.append('rm -f {}/*.fits'.format(os.environ['RANDOMS_DIR'])) + # Exits on $GOLD_DIR/randoms/ (fragile). + cmds.append('rm -f {}/*.fits'.format(dirpath)) for cmd in cmds: print(cmd) os.system(cmd) + # IMMUTABLE=TRUE defines files those files which should be immune to a safe_reset. if survey == 'desi': safe_reset(supported=True, printonly=True) raise NotImplementedError('Full reset unsupported for desi') + + if custom: + # System dependent shebangs. + customise_serialorparallel() + + # Customise submission scripts with memory, queue, etc. + customise_script(args) + + custom = '/custom/' if custom else '' + dryrun = '--dryrun' if dryrun else '' + nooverwrite = '--nooverwrite' if nooverwrite else '' if reset: os.environ['RESET'] = str(1) - else: os.environ['RESET'] = str(0) @@ -85,6 +91,9 @@ def pipeline(args, use_sbatch=False, reset=False, nooverwrite=False, dryrun=True print('Assuming $NOOVERWRITE={}\n\n'.format(nooverwrite)) # ---- Total of eight jobs, with correct dependency logic ---- + + # Individual pipeline scripts, e.g. bin/rand_pipeline are capable of their own reset. + # This is not required given a reset has already happened (or not). os.environ['RESET'] = '0' # --------------------------------------------- @@ -92,13 +101,13 @@ def pipeline(args, use_sbatch=False, reset=False, nooverwrite=False, dryrun=True os.chdir(f'{home}') + # Set up a fresh clone of main under ~/tmp/DESI/ to run off. if freshclone: cmds = [] cmds.append('rm -rf {}/tmp'.format(os.environ['HOME'])) cmds.append('mkdir -p {}/tmp'.format(os.environ['HOME'])) cmds.append('git clone --branch main https://github.com/SgmAstro/DESI.git {}/tmp/DESI'.format(os.environ['HOME'])) - for cmd in cmds: out = run_command(cmd, noid=True) @@ -115,7 +124,7 @@ def pipeline(args, use_sbatch=False, reset=False, nooverwrite=False, dryrun=True os.environ['PYTHONPATH'] = code_root + '/:' + os.environ['PYTHONPATH'] Path(os.environ['GOLD_DIR'] + '/logs/').mkdir(parents=True, exist_ok=True) - Path(os.environ['RANDOMS_DIR'] + '/logs/').mkdir(parents=True, exist_ok=True) + Path(os.environ['GOLD_DIR'] + '/randoms/logs/').mkdir(parents=True, exist_ok=True) if stages == None: # TODO: difficulty is handing the dependency jobids. @@ -127,30 +136,50 @@ def pipeline(args, use_sbatch=False, reset=False, nooverwrite=False, dryrun=True 'gold_d8'] # --------------------------------------------- - # Generate all steps up to reference LF. - cmd = 'serialorparallel -n {} -p {:d} -e DRYRUN={},RESET={:d},NOOVERWRITE={},SURVEY={} -s gold_pipeline -c {}'.format('gold_pipeline', int(use_sbatch), dryrun, int(reset), nooverwrite, survey, code_root) + # Generate all gold_pipeline steps: gold definition, k&E, zmax, ddp_limits, ddp cat. + cmd = 'serialorparallel -n {} -p {:d} -e DRYRUN={},RESET={:d},NOOVERWRITE={},SURVEY={} -s gold_pipeline -c {}'.format('gold_pipeline',\ + int(use_sbatch),\ + dryrun,\ + int(reset),\ + nooverwrite,\ + survey,\ + code_root) gold_jobid = run_command(cmd) print('\n>>>>> GOLD JOB ID <<<<<') print(gold_jobid) print('\n\n') + exit(0) + # # https://slurm.schedmd.com/sbatch.html # - fields = fetch_fields(survey=survey) rand_jobids = {} rand_ddp_jobids = {} - + + # Run the randoms pipeline: randoms, boundary (0th), fillfactor for each field. + # Note: we generate multiple realizations of oversampled randoms such that the 0th + # realization is a stack of the n8 counts (thereby circumventing memory constraints). + # for field in fields: rand_jobids[field] = [] for realz in np.arange(oversample_nrealisations): - # No dependency. Generate all steps up to random fill factor and bound_dist. + # 0th realization generates an x1 oversampling, otherwise oversampling randoms such that 0th realization randoms_n8 (fillfactor) is reserved for the stack. + # No dependency. cmd = 'serialorparallel -n {} -p {:d} -e FIELD={},DRYRUN={},RESET={:d},NOOVERWRITE={},SURVEY={},REALZ={} -s rand_pipeline -c {}' - cmd = cmd.format(f'rand_pipeline_{field}_{realz}', int(use_sbatch), field, dryrun, int(reset), nooverwrite, survey, realz, code_root) + cmd = cmd.format(f'rand_pipeline_{field}_{realz}',\ + int(use_sbatch),\ + field,\ + dryrun,\ + int(reset),\ + nooverwrite,\ + survey,\ + realz,\ + code_root) jobid = run_command(cmd) @@ -158,14 +187,24 @@ def pipeline(args, use_sbatch=False, reset=False, nooverwrite=False, dryrun=True rand_jobids[field] = ','.join(rand_jobids[field]) + # Same again, but we lift the ddp redshift limits for the header of the gold ddp catalog. for field in fields: rand_ddp_jobids[field] = [] - for realz in np.arange(oversample_nrealisations): + for realz in np.arange(oversample_nrealisations): # Dependency on $GOLD_JOBID (gold ddp cat generated by gold_pipeline). # Generate ddp1 randoms limited to ddp1 z limits - with corresponding fillfactors, bound_dist etc. cmd = 'serialorparallel -n {} -p {:d} -e FIELD={},DRYRUN={},RESET={:d},NOOVERWRITE={},SURVEY={},REALZ={} -d {} -s rand_ddp1_pipeline -c {}' - cmd = cmd.format(f'rand_ddp1_pipeline_{field}_{realz}', int(use_sbatch), field, dryrun, int(reset), nooverwrite, survey, realz, gold_jobid, code_root) + cmd = cmd.format(f'rand_ddp1_pipeline_{field}_{realz}',\ + int(use_sbatch),\ + field,\ + dryrun,\ + int(reset),\ + nooverwrite,\ + survey,\ + realz,\ + gold_jobid,\ + code_root) jobid = run_command(cmd) @@ -181,19 +220,40 @@ def pipeline(args, use_sbatch=False, reset=False, nooverwrite=False, dryrun=True rand_d8_jobids = {} rand_ddp_d8_jobids = {} + # Runs bound_dist and d8 calculation for the randoms. Bound_dist as only for the 0th realization, as rand_pipeline runs for many. for field in fields: # Requires ddp cat, & randoms; no reset required. - rand_jobid = rand_jobids[field] - - cmd = 'serialorparallel -n {} -p {:d} -e FIELD={},DRYRUN={},RESET={:d},NOOVERWRITE={},SURVEY={} -d {},{} -s rand_d8_pipeline -c {}' - cmd = cmd.format(f'rand_d8_pipeline_{field}', int(use_sbatch), field, dryrun, int(reset), nooverwrite, survey, gold_jobid, rand_jobid, code_root) + rand_jobid = rand_jobids[field] + + cmd = 'serialorparallel -n {} -p {:d} -e FIELD={},DRYRUN={},RESET={:d},NOOVERWRITE={},SURVEY={} -d {},{} -s rand_d8_pipeline -c {}' + cmd = cmd.format(f'rand_d8_pipeline_{field}',\ + int(use_sbatch),\ + field,\ + dryrun,\ + int(reset),\ + nooverwrite,\ + survey,\ + gold_jobid,\ + rand_jobid,\ + code_root) rand_d8_jobids[field] = run_command(cmd) rand_ddp_jobid = rand_ddp_jobids[field] - + + # Same again for ddp1. + # TODO: remove the need for rand_ddp1_d8 script, which replicates rand_d8 except for the --prefix argument. cmd = 'serialorparallel -n {} -p {:d} -e FIELD={},DRYRUN={},NOOVERWRITE={},SURVEY={} -d {},{} -s rand_ddp1_d8_pipeline -c {}' - cmd = cmd.format(f'rand_ddp1_d8_pipeline_{field}', int(use_sbatch), field, dryrun, nooverwrite, survey, gold_jobid, rand_ddp_jobid, code_root) + cmd = cmd.format(f'rand_ddp1_d8_pipeline_{field}',\ + int(use_sbatch),\ + field,\ + dryrun,\ + nooverwrite,\ + survey,\ + gold_jobid,\ + rand_ddp_jobid,\ + code_root) + rand_ddp_d8_jobids[field] = run_command(cmd) print('\n\n>>>>> RANDOM D8 JOB IDS <<<<<') @@ -205,9 +265,16 @@ def pipeline(args, use_sbatch=False, reset=False, nooverwrite=False, dryrun=True # Note: runs all fields simultaneously. dependencies = ','.join(str(rand_ddp_d8_jobids[field]) for field in fields) - # possibly missing RESET=$RESET + # TODO: possibly missing RESET=$RESET cmd = 'serialorparallel -n {} -p {:d} -e DRYRUN={},NOOVERWRITE={},SURVEY={} -d {} -s gold_d8_pipeline -c {}' - cmd = cmd.format('gold_d8_pipeline', int(use_sbatch), dryrun, nooverwrite, survey, dependencies, code_root) + cmd = cmd.format('gold_d8_pipeline',\ + int(use_sbatch),\ + dryrun,\ + nooverwrite,\ + survey,\ + dependencies,\ + code_root) + gold_d8_jobid = run_command(cmd) print('\n\n>>>>> GOLD D8 JOB IDS <<<<<') @@ -252,6 +319,7 @@ def pipeline(args, use_sbatch=False, reset=False, nooverwrite=False, dryrun=True config = args.config log = args.log + # Define a configuration file for a summary of assumptions. config = Configuration(config) config.update_attributes('pipeline', args) diff --git a/bin/serialorparallel b/bin/serialorparallel index 18ece252..c3be3689 100755 --- a/bin/serialorparallel +++ b/bin/serialorparallel @@ -1,4 +1,3 @@ -#!/usr/bin/bash # # ---- serialorparallel ---- # diff --git a/bound_dist.py b/bound_dist.py index b008c39f..6c1a4564 100644 --- a/bound_dist.py +++ b/bound_dist.py @@ -35,17 +35,22 @@ def process_one(run, pid=0): return dd.tolist(), ii.tolist() -def bound_dist(log, field, dryrun, prefix, survey, nproc, realz, nooverwrite, collate=True): +def bound_dist(log, field, dryrun, prefix, survey, nproc, nooverwrite, collate=True): start = time.time() if collate: + if dryrun: + realzs=np.arange(1, 6, 1) + else: + realzs=np.arange(1, 1 + oversample_nrealisations, 1) + # Collate multiple n8 measurements from N>1 oversampled realizations into the 0th realization. # Note: null op if already applied. - collate_fillfactors(realzs=np.arange(oversample_nrealisations), field=field, survey=survey, dryrun=dryrun, prefix=prefix, write=True) + collate_fillfactors(realzs=realzs, field=field, survey=survey, dryrun=dryrun, prefix=prefix, write=True) # https://www.dur.ac.uk/icc/cosma/cosma5/ - fpath = findfile(ftype='randoms_n8', dryrun=dryrun, field=field, survey=survey, prefix=prefix) - opath = findfile(ftype='randoms_bd', dryrun=dryrun, field=field, survey=survey, prefix=prefix) + fpath = findfile(ftype='randoms_n8', dryrun=dryrun, field=field, survey=survey, prefix=prefix) + opath = findfile(ftype='randoms_bd', dryrun=dryrun, field=field, survey=survey, prefix=prefix) if log: logfile = findfile(ftype='randoms_bd', dryrun=False, field=field, survey=survey, prefix=prefix, log=True) @@ -54,8 +59,7 @@ def bound_dist(log, field, dryrun, prefix, survey, nproc, realz, nooverwrite, co sys.stdout = open(logfile, 'w') - if nooverwrite: - overwrite_check(opath) + overwrite_check(opath, nooverwrite) call_signature(dryrun, sys.argv) @@ -188,7 +192,6 @@ def bound_dist(log, field, dryrun, prefix, survey, nproc, realz, nooverwrite, co parser.add_argument('--nooverwrite', help='Do not overwrite outputs if on disk', action='store_true') parser.add_argument('--config', help='Path to configuration file', type=str, default=findfile('config')) parser.add_argument('--nproc', type=int, help='Number of processors', default=12) - parser.add_argument('--realz', type=int, help='Realisation', default=0) args = parser.parse_args() log = args.log @@ -197,7 +200,6 @@ def bound_dist(log, field, dryrun, prefix, survey, nproc, realz, nooverwrite, co prefix = args.prefix survey = args.survey.lower() nproc = args.nproc - realz = args.realz nooverwrite = args.nooverwrite ''' @@ -206,4 +208,4 @@ def bound_dist(log, field, dryrun, prefix, survey, nproc, realz, nooverwrite, co config.write() ''' - bound_dist(log, field, dryrun, prefix, survey, nproc, realz, nooverwrite) + bound_dist(log, field, dryrun, prefix, survey, nproc, nooverwrite) diff --git a/boundary.py b/boundary.py index 883f9252..05ac99f6 100644 --- a/boundary.py +++ b/boundary.py @@ -77,8 +77,8 @@ def rotate2rosette(ros_ra, ros_dec, pos): config.update_attributes('boundary', args) config.write() ''' -if args.nooverwrite: - overwrite_check(opath, ext='BOUNDARY') + +overwrite_check(opath, args.nooverwrite, ext_name='BOUNDARY') if args.dryrun: sampling = 1000 @@ -86,7 +86,7 @@ def rotate2rosette(ros_ra, ros_dec, pos): call_signature(dryrun, sys.argv) ## ras and decs. -if survey == 'gama': +if survey in ['gama', 'abacus_gama']: area = 60. ra_min = gama_limits[field]['ra_min'] diff --git a/config.py b/config.py index 179127cf..ee13e215 100644 --- a/config.py +++ b/config.py @@ -1,7 +1,6 @@ import os import sys import yaml -import inflect import contextlib import numpy as np @@ -70,10 +69,9 @@ def __init__(self, fpath=None): raise NotImplementedError() self.fpath = os.environ['GOLD_DIR'] + '/configs/config.yaml' - + def update_comments(self, comments): - p = inflect.engine() - ps = [p.ordinal(i) for i in range(1, 50, 1)] + ps = ['#{}'.format(i) for i in range(1, 50, 1)] ps = [str(p) for p in ps if p not in self.attributes['comments'].keys()] for p, comment in zip(ps, comments): diff --git a/ddp_limits.py b/ddp_limits.py index e72e09c6..e7ad7a23 100644 --- a/ddp_limits.py +++ b/ddp_limits.py @@ -71,7 +71,7 @@ def grab_ddplimit(fpath): args = parser.parse_args() log = args.log survey = args.survey.lower() - + config = Configuration(args.config) config.update_attributes('ddp_limits', args) config.write() diff --git a/desi_gold.py b/desi_gold.py index 92868e75..c00ec76a 100644 --- a/desi_gold.py +++ b/desi_gold.py @@ -22,308 +22,4 @@ def desi_gold(args, survey='sv3', release='fuji'): from desitarget.sv3.sv3_targetmask import desi_mask, bgs_mask - dryrun = args.dryrun - - if survey != 'sv3': - raise NotImplementedError('Only SV3 is currently supported.') - - releases = {'sv3': 'fuji', 'main': 'guadalupe'} - release = releases[survey] - - root = os.environ['DESI_ROOT'] + f'/spectro/redux/{release}/healpix/' - fpath = root + 'tilepix.fits' - - print(f'Fetching {fpath}') - - tpix = Table.read(fpath) - - # Note: max. Bright SV3 tileid is 595. - tiles = np.arange(1000) - ros = np.array([tile2rosette(x) for x in tiles]) - - # https://desi.lbl.gov/trac/wiki/SurveyOps/OnePercent - # G12: [1,2]; G15: [8,9,10, 17] - - # Unique rosettes in SV3. - uros = np.unique(ros) - uros = uros[uros > -1] - - gama = np.isin(ros, uros) - - tiles = tiles[gama] - - tpix = tpix[np.isin(tpix['TILEID'].data, tiles)] - hps = np.unique(tpix['HEALPIX'].data) - - root += '/{}/bright/'.format(survey) - - fpaths = [root + '{}/{}/redrock-{}-bright-{}.fits'.format(str(x)[:3], x, survey, x) for x in hps] - fpaths = [x for x in fpaths if os.path.exists(x)] - - print('Fetching {}'.format(fpaths[0])) - - # e.g. 280/28027/redrock-sv3-bright-28027.fits - tabs = [] - - print('Gathering DESI zs.') - - for x in fpaths: - zbest = Table.read(x, hdu='REDSHIFTS') - fmap = Table.read(x, hdu='FIBERMAP') - efmap = Table.read(x, hdu='EXP_FIBERMAP') - - # row ordered. - assert np.all(zbest['TARGETID'] == fmap['TARGETID']) - - tids = np.unique(zbest['TARGETID']) - - efmap['ROS'] = tile2rosette(efmap['TILEID'].data) - - efmap_tid = efmap['TARGETID', 'ROS'] - efmap_tid = unique(efmap_tid, keys=['TARGETID']) - efmap_tid.sort('TARGETID') - - assert np.all(efmap_tid['TARGETID'].data == tids) - - del fmap['TARGETID'] - - zbest = hstack([zbest, fmap]) - - zbest = join(zbest, efmap_tid, join_type='left', keys='TARGETID') - zbest['ROS_DIST'] = 1.e4 * np.ones_like(zbest['Z']) - - # Distance to the closest rosette center. - for rosn in np.unique(efmap['ROS'].data): - # TODO: Small rosette overlap. - isin = zbest['ROS'].data == rosn - zbest['ROS_DIST'][isin] = calc_rosr(rosn, zbest['TARGET_RA'].data[isin], zbest['TARGET_DEC'].data[isin]) - - tabs.append(zbest) - - desi_zs = vstack(tabs) - - # remove skies. - print('Sky frac: 1-{:.6f}'.format(np.mean(desi_zs['TARGETID'].data >= 0))) - - desi_zs = desi_zs[desi_zs['TARGETID'].data >= 0] - desi_zs = desi_zs[(desi_zs['SV3_BGS_TARGET'].data & bgs_mask['BGS_BRIGHT']) != 0] - - desi_zs['GOOD_Z'] = (desi_zs['ZWARN'].data == 0) & (desi_zs['DELTACHI2'].data > 40) - - desi_zs['ZDESI'] = desi_zs['Z'] - - del desi_zs['Z'] - - desi_zs['FIELD'] = [f'R{xx}' for xx in desi_zs['ROS'].data] - - ## Cut DESI to good redshifts. - desi_zs['GAMA_FIELD'] = gama_field(desi_zs['TARGET_RA'].data, desi_zs['TARGET_DEC'].data) - - ## TODO/HACK: PHOTSYS ASSUMED S. - desi_zs['GMAG_DRED'] = 22.5 - 2.5 * np.log10(desi_zs['FLUX_G'].data / mwdust_transmission(desi_zs['EBV'].data, 'G', 'S', match_legacy_surveys=True)) - desi_zs['RMAG_DRED'] = 22.5 - 2.5 * np.log10(desi_zs['FLUX_R'].data / mwdust_transmission(desi_zs['EBV'].data, 'R', 'S', match_legacy_surveys=True)) - - desi_zs['W1MAG_DRED'] = 22.5 - 2.5 * np.log10(desi_zs['FLUX_W1'].data / mwdust_transmission(desi_zs['EBV'].data, 'W1', 'S', match_legacy_surveys=True)) - desi_zs['W2MAG_DRED'] = 22.5 - 2.5 * np.log10(desi_zs['FLUX_W2'].data / mwdust_transmission(desi_zs['EBV'].data, 'W2', 'S', match_legacy_surveys=True)) - - desi_zs['GMR'] = desi_zs['GMAG_DRED'] - desi_zs['RMAG_DRED'] - desi_zs['DETMAG'] = desi_zs['RMAG_DRED'] - - desi_zs.meta['PET_OFFSET'] = 0.12 - desi_zs['LEGACYPET'] = desi_zs['RMAG_DRED'] + desi_zs.meta['PET_OFFSET'] - - zlow = ddp_limits['DDP1'][0] - zhigh = ddp_limits['DDP1'][1] - - desi_zs['IN_GOLD'] = desi_zs['GOOD_Z'].data & (desi_zs['ZDESI'] > zlow) & (desi_zs['ZDESI'] < zhigh) - - clustering, full = fetch_lss(pprint=False, sort=False) - - clustering_ids = np.unique(clustering['TARGETID'].data) - full_ids = np.unique(full['TARGETID'].data) - - desi_zs['IN_CLUSTERING'] = np.isin(desi_zs['TARGETID'].data, clustering_ids) - desi_zs['IN_FULL'] = np.isin(desi_zs['TARGETID'].data, full_ids) - - clustering_cols = ['WEIGHT_ZFAIL', 'WEIGHT', 'NZ'] - full_cols = ['NTILE', 'TILES', 'TILELOCIDS', 'LOCATION_ASSIGNED', 'TILELOCID_ASSIGNED', 'COMP_TILE', 'FRACZ_TILELOCID', 'BITWEIGHTS', 'PROB_OBS'] - - for cols, cat in zip([clustering_cols, full_cols], [clustering, full]): - cols += ['TARGETID'] - - # TODO/HACK multiple targetid occurences in clustering catalog? - desi_zs = join(desi_zs, cat[cols], keys='TARGETID') - - ## Archive step. - archive = Table(desi_zs, copy=True) - - xyz = cartesian(desi_zs['TARGET_RA'].data, desi_zs['TARGET_DEC'].data, desi_zs['ZDESI'].data) - - desi_zs['CARTESIAN_X'] = xyz[:,0] - desi_zs['CARTESIAN_Y'] = xyz[:,1] - desi_zs['CARTESIAN_Z'] = xyz[:,2] - - xyz = rotate(desi_zs['TARGET_RA'].data, desi_zs['TARGET_DEC'].data, xyz) - - desi_zs['ROTCARTESIAN_X'] = xyz[:,0] - desi_zs['ROTCARTESIAN_Y'] = xyz[:,1] - desi_zs['ROTCARTESIAN_Z'] = xyz[:,2] - - desi_zs['LUMDIST'] = cosmo.luminosity_distance(desi_zs['ZDESI'].data) - desi_zs['DISTMOD'] = distmod(desi_zs['ZDESI'].data) - - desi_zs['IN_D8LUMFN'] = np.zeros_like(desi_zs['FIELD'], dtype=int) - desi_zs['CONSERVATIVE'] = np.zeros_like(desi_zs['FIELD'], dtype=int) - - desi_zs.meta['IMMUTABLE'] = 'TRUE' - - desi_zs.pprint() - - survey = 'desi' - ''' - fpath = findfile(ftype='gold', dryrun=False, survey=survey) - opath = fpath.replace('desi_gold', 'desi_sv3_gold') - - print('Writing {}'.format(opath)) - - desi_zs.write(opath, format='fits', overwrite=True) - - ## ---- GAMA GOLD - gold = Table.read(fpath) - - del gold['CARTESIAN_X'] - del gold['CARTESIAN_Y'] - del gold['CARTESIAN_Z'] - del gold['ROTCARTESIAN_X'] - del gold['ROTCARTESIAN_Y'] - del gold['ROTCARTESIAN_Z'] - del gold['DETMAG'] - del gold['DISTMOD'] - del gold['LUMDIST'] - del gold['GMR'] - - gold.pprint() - - # DESI - uros = [1,2,8,9,10,17] - desi_zs = desi_zs[np.isin(desi_zs['ROS'], uros)] - archive = archive[np.isin(archive['ROS'], uros)] - - c = SkyCoord(ra=desi_zs['TARGET_RA']*u.degree, dec=desi_zs['TARGET_DEC']*u.degree) - - print('Matching DESI to GAMA Gold.') - - # GAMA - catalog = SkyCoord(ra=gold['RA'], dec=gold['DEC']) - idx, d2d, d3d = c.match_to_catalog_3d(catalog) - - # Now idx are indices into catalog that are the closest objects to each of the coordinates in c, d2d are the on-sky distances between them - gold_match = gold[idx] - - del gold_match['FIELD'] - - to_join = Table(gold_match, copy=True) - to_join.meta = {} - - desi_zs = hstack([desi_zs, to_join]) - desi_zs['GAMA_SEP'] = d2d.to(u.arcsec) - desi_zs['GOOD_MATCH'] = desi_zs['GAMA_SEP'] < 0.3 - - desi_zs.meta['IMMUTABLE'] = 'TRUE' - - max_sep = 0.5 * u.arcsec - - print('Fraction desi matched to gold at 0.5 arcseconds: {:.6f}'.format(np.mean(desi_zs['GAMA_SEP'] < max_sep))) - - opath = fpath.replace('desi_gold', 'desi_gama') - - print('Writing {}'.format(opath)) - - desi_zs.write(opath, format='fits', overwrite=True) - - ## -------------------- - idx, d2d, d3d = catalog.match_to_catalog_3d(c) - - desi_match = archive[idx] - - to_join = Table(desi_match, copy=True) - to_join.meta = {} - - gold = hstack([gold, to_join]) - gold['DESI_SEP'] = d2d.to(u.arcsec) - gold['GOOD_MATCH'] = gold['DESI_SEP'] < 0.3 - - gold['ROS_DIST'] = 1.e99 - - for rosn in uros: - new_dist = calc_rosr(rosn, gold['RA'].data, gold['DEC'].data) - - gold['ROS_DIST'] = np.minimum(gold['ROS_DIST'].data, new_dist) - - max_sep = 0.5 * u.arcsec - - print('Fraction gold matched to desi at 0.5 arcseconds: {:.6f}'.format(np.mean(gold['DESI_SEP'] < max_sep))) - - gold.meta['IMMUTABLE'] = 'TRUE' - - opath = fpath.replace('desi_gold', 'gama_desi') - - print('Writing {}'.format(opath)) - - gold.write(opath, format='fits', overwrite=True) - ''' - - in_gold = desi_zs['GOOD_Z'].data & (desi_zs['ZDESI'] > ddp_limits['DDP1'][0]) & (desi_zs['ZDESI'] < ddp_limits['DDP1'][1]) - - # No cut to GAMA rosettes. - # in_gold &= np.isin(desi_zs['ROS'].data, [1,2,8,9,10,17]) - - desi_zs = desi_zs[in_gold] - desi_zs['RA'] = desi_zs['TARGET_RA'] - desi_zs['DEC'] = desi_zs['TARGET_DEC'] - desi_zs['ZSURV'] = desi_zs['ZDESI'] - desi_zs['DETMAG'] = desi_zs['RMAG_DRED'] - desi_zs['DISTMOD'] = distmod(desi_zs['ZDESI'].data) - - limits = ros_limits(dryrun) - - hi_comp = (desi_zs['ROS_DIST'].data > limits[0]) & (desi_zs['ROS_DIST'].data < limits[1]) - area = np.pi * (limits[1]**2. - limits[0]**2.) - - update_bit(desi_zs['IN_D8LUMFN'], lumfn_mask, 'DESI_HICOMP', ~hi_comp) - - desi_zs = desi_zs[desi_zs['IN_D8LUMFN'].data == 0] - - desi_zs.meta['AREA'] = area * len(np.unique(desi_zs['FIELD'].data)) - desi_zs.meta['IMMUTABLE'] = 'TRUE' - desi_zs.meta['RELEASE'] = release.upper() - - desi_zs.meta['RLIM'] = 19.5 - desi_zs.meta['RMAX'] = 12.0 - desi_zs.meta['MAX_SEP'] = 10.0 # Expected max. angular separation in a rosette. - - opath = findfile(ftype='gold', dryrun=dryrun, survey=survey) - - print('Writing {}'.format(opath)) - - desi_zs.write(opath, format='fits', overwrite=True) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Gen desi gold cat.') - parser.add_argument('--log', help='Create a log file of stdout.', action='store_true') - parser.add_argument('--config', help='Path to configuration file', type=str, default=findfile('config')) - parser.add_argument('--dryrun', help='Dryrun of 5k galaxies', action='store_true') - parser.add_argument('--nooverwrite', help='Do not overwrite outputs if on disk', action='store_true') - - args = parser.parse_args() - ''' - config = Configuration(args.config) - config.update_attributes('gold', args) - config.write() - ''' - desi_gold(args) - - print('Done.') - - + raise \ No newline at end of file diff --git a/docs/nb/abacus.ipynb b/docs/nb/abacus.ipynb new file mode 100644 index 00000000..406f2544 --- /dev/null +++ b/docs/nb/abacus.ipynb @@ -0,0 +1,659 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "0af7cfc5-a70a-4d09-8118-af6105caaf4c", + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "89ccf6c8-0617-4e5c-8192-58cd453fafd7", + "metadata": {}, + "outputs": [], + "source": [ + "import h5py\n", + "import sys\n", + "import numpy as np\n", + "import pylab as pl\n", + "import astropy.io.fits as fits\n", + "import matplotlib.pyplot as plt\n", + "\n", + "from astropy.table import Table" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "13beb54a-8379-4ab7-b5fd-c8035702c1a5", + "metadata": {}, + "outputs": [], + "source": [ + "sys.path.append('/global/homes/m/mjwilson/desi/BGS/Sam/')\n", + "sys.path.append('/global/homes/m/mjwilson/desi/BGS/Sam/DESI/')" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "d4d6368e-42e1-4b6c-a1a0-1743b312c0d9", + "metadata": {}, + "outputs": [], + "source": [ + "from cosmo import distmod, cosmo\n", + "from gama_limits import gama_field\n", + "from cartesian import cartesian, rotate\n", + "from findfile import findfile, write_desitable" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "ea36f0d0-e918-4eb4-84be-13fb13ecfb3e", + "metadata": {}, + "outputs": [], + "source": [ + "path = \"/project/projectdirs/desi/cosmosim/FirstGenMocks/AbacusSummit/CutSky/BGS/z0.200/\"\n", + "\n", + "f = h5py.File(path+\"cutsky_BGS_z0.200_AbacusSummit_base_c000_ph000.hdf5\", \"r\")\n", + "\n", + "stat = f[\"Data/STATUS\"] # integer indicating the status of the galaxy. If the second bit is 1, the galaxy is in the full DESI footprint. If the third bit is 1, the galaxy is in the SV3 footprint.\n", + "Ms = f[\"Data/abs_mag\"]\n", + "rs = f[\"Data/app_mag\"]\n", + "ra = f[\"Data/ra\"]\n", + "dec = f[\"Data/dec\"]\n", + "rest_color = f[\"Data/g_r\"]\n", + "color = f[\"Data/g_r_obs\"]\n", + "hmass = f[\"Data/halo_mass\"]\n", + "gtype = f[\"Data/galaxy_type\"][...] # galaxy type, indicating cen/sat and if resolved\n", + "zcos = f[\"Data/z_cos\"][...] \n", + "zobs = f[\"Data/z_obs\"][...] # observed redshift (which takes into account velocity)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "51950a61-4758-4b2d-8ad2-62ee53de3a33", + "metadata": {}, + "outputs": [], + "source": [ + "in_gama = gama_field(ra[:], dec[:]) != 'NAN'\n", + "in_sv3 = stat[:] & 2**2 != 0 # in SV3 footprint" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "17a569af-e50a-472d-92fb-7346f5562966", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD5CAYAAADcDXXiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAR4klEQVR4nO3dfYxld13H8feHFqgILls6u2764IIWYjXhwbFiUAOsxfIQ2n9qADUbbbKRIGmJCgskpPrXAkbxDxKzAXQJBSxQ0g2iUFcqmkBhtrZQWOqWurSFdXegWjAGYuXrH3PWzk7v3Oc798yZ9yuZnHN+55y537075zNnfud3zk1VIUnqlsfNuwBJ0vQZ7pLUQYa7JHWQ4S5JHWS4S1IHGe6S1EHnDtogybOAv17V9AzgbcD7m/bdwAng16vqP/p9rwsuuKB27949ZqmStDUdPXr021W1MMo+GWWce5JzgG8CvwC8Dnioqg4k2Q9sr6o39dt/cXGxlpaWRqlPkra8JEeranGUfUbtltkDfL2qvgFcBRxq2g8BV4/4vSRJMzJquL8K+FAzv7OqTgI00x3TLEySNL6hwz3JE4BXAh8Z5QWS7EuylGRpeXl51PokSWMY5cz9pcAdVXWqWT6VZBdAMz3da6eqOlhVi1W1uLAw0vUASdKYRgn3V/NolwzAYWBvM78XuGVaRUmSJjNUuCd5EnAFcPOq5gPAFUmON+sOTL88SdI4Bo5zB6iq/waetqbtO6yMnpEktcxQ4S5JWuOGbeu0P7yxdazDcJekYawX5v22m2PQ+2wZSRpk2GDvtd+4+07IcJekfqYRznMIeMNdkjrIcJekjbDBZ++GuyRthA2+uGq4S1IHGe6S1M80zrjnMCTScJekQSYJ5zmNdfcmJkkaxuqQHnRxtAV3qRrukjSsUe9SnWPIG+5qr34HUgvOjLTFjDOU8YZtdstIwKZ7foe2iEnGqM8p4L2gqvaY5Pkdks5iuEtSBxnuaodJz749e5fOYrhLUgcZ7pI0yCa8iWnYD8h+apKPJvlakmNJfjHJ+UluTXK8mW6fdbHqsEkPAEfMaNbG+Rmb489lqmrwRskh4J+q6j1JngA8CXgL8FBVHUiyH9heVW/q930WFxdraWlpGnWrq8YaS2ywaw428DNUkxytqsWR9hkU7kl+DLgLeEat2jjJPcALq+pkkl3AbVX1rH7fy3DXUEYJeINdW8A44T7MTUzPAJaBv0zybOAocB2ws6pOAjQBv2PUgqWeBj3Dw0CXBhom3M8Fnge8vqpuT/LnwP5hXyDJPmAfwCWXXDJWkdrCDHJpLMNcUH0QeLCqbm+WP8pK2J9qumNopqd77VxVB6tqsaoWFxYWplGzJGmAgeFeVf8OPJDkTH/6HuCrwGFgb9O2F7hlJhVKkkY27IPDXg/c2IyUuQ/4bVZ+MdyU5FrgfuCa2ZQoSRrVUOFeVXcCva7U7plqNZKkqfAOVUnqIMNdkjrIcJekDvKTmCRN3wbemq/eDHdJ0zPo0REt+ODorcJuGUnTMdIzgfxwlVkz3CXNhwE/U4a7JHWQ4S5JHWS4S1IHOVpG2mgOE9QG8Mxd2kj9LiLesG1zX2Qc9ZeTv8xmynCXNsIowb0VAt5gnzm7ZSRN15ng9iMS58pwl9rohm2bPwg3e/2bnN0yktRBhrskdZDhLrWRXRqakOEubQTDWhtsqHBPciLJl5PcmWSpaTs/ya1JjjfT7bMtVdrkHCaoDTTKaJkXVdW3Vy3vB45U1YEk+5vlN021Oqlr1hsmaKBryiYZCnkV8MJm/hBwG4a7NBzDXDM2bJ97AZ9OcjTJvqZtZ1WdBGimO2ZRoCRpdMOeub+gqr6VZAdwa5KvDfsCzS+DfQCXXHLJGCVKkkY11Jl7VX2rmZ4GPg5cDpxKsgugmZ5eZ9+DVbVYVYsLCwvTqVqS1NfAcE/yo0mecmYeeAlwN3AY2Ntsthe4ZVZFSpJGM0y3zE7g40nObP/Bqvq7JF8EbkpyLXA/cM3sypQkjWJguFfVfcCze7R/B9gzi6IkSZPxDlVJ6iDDXZI6yHCXpA4y3CWpgwx3Seogw12SOshwl6QO8gOyJQ22+hHF/Z5oOeyjjH3k8cylqjbsxRYXF2tpaWnDXk/ShNaG8FnrHh5uu9XbDrudzpLkaFUtjrKP3TKSehsYxNuG227Y7zfK99JAhrukx5pmYI+ynabGcJekDjLcJbWLZ/lTYbhLahcvqk6F4S5JHWS4S3qsYc+ePctuLcNd0mSGCfgbHh5+O02F4S6pt36BvHZd37tWx9hOE/MOVUlquZneoZrknCT/kuQTzfL5SW5NcryZbh+1YEnSbIzSLXMdcGzV8n7gSFVdChxpliVJLTBUuCe5CHg58J5VzVcBh5r5Q8DVU61MkjS2Yc/c3wW8EfjhqradVXUSoJnumG5pkqRxDQz3JK8ATlfV0XFeIMm+JEtJlpaXl8f5FpKkEQ1z5v4C4JVJTgAfBl6c5APAqSS7AJrp6V47V9XBqlqsqsWFhYUplS1J6mdguFfVm6vqoqraDbwK+Ieq+k3gMLC32WwvcMvMqpQkjWSSm5gOAFckOQ5c0SxLklpgpM9QrarbgNua+e8Ae6ZfkiRpUj5+QJI6yHCXpA4y3CWpgwx3SeqgkS6oStJcrf18VR8TvC7DXWqD9T4U2vB6VK/36IZtvkfrsFtGmrf1gn3Quq3E92hkhrs0T8ME01YPr63+7x+T4S5p8/MXwGMY7pLUQYa7JHWQ4S5JHWS4S/M0zDA+h/oN5nv0GIa7pHYbFNwGe0+GuzRv/cLJ4Fqx3vvg+7Mu71CV2sCQGsz3aCSeuUtSBxnuktRBhrskddDAcE9yXpIvJLkryVeS/FHTfn6SW5Mcb6bbZ1+uJGkYw5y5/wB4cVU9G3gOcGWS5wP7gSNVdSlwpFmWJLXAwHCvFf/VLD6++SrgKuBQ034IuHoWBUqSRjfUUMgk5wBHgZ8C3l1VtyfZWVUnAarqZJIdM6xT6q/nBzk4dE5b11AXVKvqf6vqOcBFwOVJfnbYF0iyL8lSkqXl5eUxy5T6WPdTjHwMrLaukUbLVNV/ArcBVwKnkuwCaKan19nnYFUtVtXiwsLCZNVKaw0KcANeW9Qwo2UWkjy1mf8R4FeBrwGHgb3NZnuBW2ZUoyRpRMP0ue8CDjX97o8DbqqqTyT5HHBTkmuB+4FrZlinJGkEA8O9qr4EPLdH+3eAPbMoSpI0Ge9QlaQOMtwlqYMMd21ujmWXejLctfn5YRfSYxju6i6DXVuY4a7NzztUpccw3NVtBry2KMNdm5vhLfVkuEtSBxnuktRBhrskdZDhrs3N4Y5ST4a7us3w1xZluGvzWy/ADXZtYUN9hqrUega5dBbP3CWpgwx3Seogw12SOshwl6QOGhjuSS5O8pkkx5J8Jcl1Tfv5SW5NcryZbp99uZKkYQxz5v4I8PtV9dPA84HXJbkM2A8cqapLgSPNsiSpBQaGe1WdrKo7mvnvAceAC4GrgEPNZoeAq2dUoyRpRCP1uSfZDTwXuB3YWVUnYeUXALBj6tVJksYydLgneTLwMeD6qvruCPvtS7KUZGl5eXmcGiVJIxoq3JM8npVgv7Gqbm6aTyXZ1azfBZzutW9VHayqxapaXFhYmEbNkqQBhhktE+C9wLGq+tNVqw4De5v5vcAt0y9PkjSOYZ4t8wLgt4AvJ7mzaXsLcAC4Kcm1wP3ANTOpUJI0soHhXlX/DGSd1XumW44kaRq8Q1WSOshwl6QOMtwlqYMMd0nqIMNdkjrIcJekDjLcJamDDHdJ6iDDXZI6yHCXpA4y3CWpgwx3Seogw12SOshwl6QOMtwlqYMMd0nqIMNdkjrIcJekDjLcJamDBoZ7kvclOZ3k7lVt5ye5NcnxZrp9tmVKkkYxzJn7XwFXrmnbDxypqkuBI82yJKklBoZ7VX0WeGhN81XAoWb+EHD1dMuSJE1i3D73nVV1EqCZ7pheSZKkSc38gmqSfUmWkiwtLy/P+uUkSYwf7qeS7AJopqfX27CqDlbVYlUtLiwsjPlykqRRjBvuh4G9zfxe4JbplCNJmoZhhkJ+CPgc8KwkDya5FjgAXJHkOHBFsyxJaolzB21QVa9eZ9WeKdciSZoS71CVpA4y3CWpgwx3SeqggX3ukoZww7YebQ9vfB1SwzN3aVK9gr1fu7QBDHdJ6iDDXZolz941J4a7NAnDWy1luEtSBxnu0iQcEaOWciik1BYOp9QUGe7Ttu6wOA/SLWnY//d+wyn92dEY7JaRJrVe+BrKmqNNc+a+e//fbMjrnDjw8vF37jdywjOwbuvxfzvKz+yJ8/p9723s/v4H1993kp9ZdZZn7tKcnTjvNfMuQR1kuEtSB22abhnNz6boEpN0FsNdU9Gra2F1P/Gg9VvZ7u9/sG/XjO+TxmG3jGbmTGCtF1xbpa/5xHmv+f+v9dYPs780ionCPcmVSe5Jcm+S/dMqatPqNxqmwyNl+gXPVg+ltf/+1cujhvZWfy81mrHDPck5wLuBlwKXAa9Octm0Ctu0eoV4h4Nd6+t3pj5uUBvwGtYkfe6XA/dW1X0AST4MXAV8dRqFbWqGuaQ5m6Rb5kLggVXLDzZtknwUsOZsknBPj7Z6zEbJviRLSZaWl5cneDlJ0rAmCfcHgYtXLV8EfGvtRlV1sKoWq2pxYWFhgpeTJA1rknD/InBpkqcneQLwKuDwdMrSZtJvHPaWHaPtdRfN2djhXlWPAL8HfAo4BtxUVV+ZVmHa/M4E+3oBv2WDX9oAE92hWlWfBD45pVq0iQ0KaoP8bP3uSh20ThrGpnn8gM8d0aZzw8PrfrrSCQB6rB9qnTTYpgl3aVMa1Pe+Re9q1uz5bBlJ6iDP3DWQXWLS5uOZuyR1kOEuSR1kuEtSBxnuktRBhrskdZDhLkkdZLhLUgcZ7pLUQYa7JHVQqh7z4Umze7FkGfjGhr3g8C4Avj3vIoZgndNlndNlndO1us6fqKqRPu1oQ8O9rZIsVdXivOsYxDqnyzqnyzqna9I67ZaRpA4y3CWpgwz3FQfnXcCQrHO6rHO6rHO6JqrTPndJ6iDP3CWpg7ZEuCd5X5LTSe7use4PklSSC1a1vTnJvUnuSfJr864zyeubWr6S5B1trDPJc5J8PsmdSZaSXD7POpNcnOQzSY4179t1Tfv5SW5NcryZbm9pne9M8rUkX0ry8SRPbWOdq9a34jjqV2ebjqM+/+/TO46qqvNfwK8AzwPuXtN+MfApVsbeX9C0XQbcBTwReDrwdeCcedUJvAj4e+CJzfKOltb5aeClzfzLgNvmWSewC3heM/8U4F+bWt4B7G/a9wNvb2mdLwHObdrf3tY6m+XWHEd93s9WHUd96pzacbQlztyr6rPAQz1W/RnwRmD1hYergA9X1Q+q6t+Ae4HLe+w7devU+VrgQFX9oNnmdEvrLODHmvltwLfmWWdVnayqO5r57wHHgAubeg41mx0Crm5jnVX16ap6pNns88BFbayzWd2a46hPna06jvrUObXjaEuEey9JXgl8s6ruWrPqQuCBVcsP8ugP8Tw8E/jlJLcn+cckP9+0t63O64F3JnkA+BPgzU373OtMsht4LnA7sLOqTsLKAQbsaDZrW52r/Q7wt818q+ps83G05v1s7XG0ps7rmdJxtCXDPcmTgLcCb+u1ukfbPIcUnQtsB54P/CFwU5LQvjpfC7yhqi4G3gC8t2mfa51Jngx8DLi+qr7bb9MebXOvM8lbgUeAG8809dh9LnWyUlcrj6Me72crj6MedU7tONqS4Q78JCv9VnclOcHKn7x3JPlxVn4jXrxq24t49E+jeXgQuLlWfAH4ISvPnGhbnXuBm5v5j/Don4xzqzPJ41k5cG6sqjO1nUqyq1m/Czjz53nb6iTJXuAVwG9U0/HasjpbeRyt83627jhap87pHUezvnDQli9gN2suqK5ad4JHLwT9DGdfuLiPDbpQ2atO4HeBP27mn8nKn2ZpYZ3HgBc283uAo/N8P5v36P3Au9a0v5OzL6i+o6V1Xgl8FVhY096qOtdsM/fjqM/72arjqE+dUzuOZvoPaMsX8CHgJPA/rPwGvHa9H8pm+a2sXI2+h+bK9bzqBJ4AfAC4G7gDeHFL6/wl4GjzA3g78HPzrLOpp4AvAXc2Xy8DngYcAY430/NbWue9TQCdafuLNta5Zpu5H0d93s9WHUd96pzaceQdqpLUQVu1z12SOs1wl6QOMtwlqYMMd0nqIMNdkjrIcJekDjLcJamDDHdJ6qD/A9mEkVGkHjlRAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "pl.plot(ra[in_gama], dec[in_gama], marker=',', lw=0.)\n", + "pl.plot(ra[in_sv3], dec[in_sv3], marker=',', lw=0.)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "e966a439-9bbd-4c9c-ae92-234354e12d47", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
Table length=331156\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
RADECZSURVGMRDETMAG
float64float64float64float64float64
221.58851531524240.88910470573151210.50908923299014241.022903434227640420.024842282773086
220.8388953469847-0.42515379227995570.50704807777963091.710568868151355619.628412712111086
220.94292732616822-0.42707951245860440.499391757932836631.784564248020011719.901759754751843
220.7779893960335-0.4076885614401690.50184166045871731.416232462245822719.482987811542994
220.80226411916357-0.40944828452174070.50579082515479021.295017050442026220.188566388984835
220.70390645849213-0.65979251971954440.50147256732670421.608762234047530320.1247197763516
220.228265089161.00776029294012750.495512928797414260.772240310806066820.153584782309252
220.18913734445744-0.29211311178928730.49832826910119631.716755328483264219.979288042865907
219.94042101710016-0.75952764622330990.496589733409188661.506522326492534519.715868889241545
219.90592912118376-0.40630550644472950.496149529091811871.525699462042508220.147958550680368
...............
134.204393823202850.23584267534624030.0121616797034092-0.0348586804310469520.10428031031344
132.394659734675940.74976494415558290.0123552029332301140.1880740314303706319.728866115049673
129.71803289373041.96489150253372750.013070434552440613-0.2494549716181201619.651116104624208
139.623166546434362.87127238749004650.0239377566713787630.0755747118176591520.141177166175837
137.987895079400231.7569759735309730.021558059405622830.01982896628265090620.137923327492356
135.33637565192657-1.04956740429591950.0220559029554754370.597110869649682718.770140854394825
222.688625219656930.246587368856534730.0184840592727129050.185360223053537319.138209203694714
131.9965189805966-1.46544718663686680.020397801956211660.1976330709778599318.274625547244007
132.29928767965137-1.28664477968755690.0197665829728856930.271157607473649520.14329769306402
221.4654657859774-0.451121640732615570.0260900095327558650.527964980220448117.928388935064874
" + ], + "text/plain": [ + "\n", + " RA DEC ... DETMAG \n", + " float64 float64 ... float64 \n", + "------------------ -------------------- ... ------------------\n", + " 221.5885153152424 0.8891047057315121 ... 20.024842282773086\n", + " 220.8388953469847 -0.4251537922799557 ... 19.628412712111086\n", + "220.94292732616822 -0.4270795124586044 ... 19.901759754751843\n", + " 220.7779893960335 -0.407688561440169 ... 19.482987811542994\n", + "220.80226411916357 -0.4094482845217407 ... 20.188566388984835\n", + "220.70390645849213 -0.6597925197195444 ... 20.1247197763516\n", + " 220.22826508916 1.0077602929401275 ... 20.153584782309252\n", + "220.18913734445744 -0.2921131117892873 ... 19.979288042865907\n", + "219.94042101710016 -0.7595276462233099 ... 19.715868889241545\n", + "219.90592912118376 -0.4063055064447295 ... 20.147958550680368\n", + " ... ... ... ...\n", + "134.20439382320285 0.2358426753462403 ... 20.10428031031344\n", + "132.39465973467594 0.7497649441555829 ... 19.728866115049673\n", + " 129.7180328937304 1.9648915025337275 ... 19.651116104624208\n", + "139.62316654643436 2.8712723874900465 ... 20.141177166175837\n", + "137.98789507940023 1.756975973530973 ... 20.137923327492356\n", + "135.33637565192657 -1.0495674042959195 ... 18.770140854394825\n", + "222.68862521965693 0.24658736885653473 ... 19.138209203694714\n", + " 131.9965189805966 -1.4654471866368668 ... 18.274625547244007\n", + "132.29928767965137 -1.2866447796875569 ... 20.14329769306402\n", + " 221.4654657859774 -0.45112164073261557 ... 17.928388935064874" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "abacus_gold = Table(np.c_[ra[in_gama], dec[in_gama], zobs[in_gama], color[in_gama], rs[in_gama]], names=['RA', 'DEC', 'ZSURV', 'GMR', 'DETMAG'])\n", + "abacus_gold" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "d658a5f2-bf43-452a-8097-8856532d7183", + "metadata": {}, + "outputs": [], + "source": [ + "abacus_gold['LUMDIST'] = cosmo.luminosity_distance(abacus_gold['ZSURV'].data)\n", + "abacus_gold['DISTMOD'] = distmod(abacus_gold['ZSURV'].data)\n", + "abacus_gold['FIELD'] = gama_field(abacus_gold['RA'], abacus_gold['DEC'])\n", + "abacus_gold['IN_D8LUMFN'] = np.zeros_like(abacus_gold['FIELD'], dtype=int)\n", + "abacus_gold['CONSERVATIVE'] = np.zeros_like(abacus_gold['FIELD'], dtype=int)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "e7c7fc42-3e5d-4832-8277-d3d06be92f90", + "metadata": {}, + "outputs": [], + "source": [ + "xyz = cartesian(abacus_gold['RA'], abacus_gold['DEC'], abacus_gold['ZSURV'])" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "9882df43-1c7c-4299-acd0-b1cece01f6a8", + "metadata": {}, + "outputs": [], + "source": [ + "abacus_gold['CARTESIAN_X'] = xyz[:,0]\n", + "abacus_gold['CARTESIAN_Y'] = xyz[:,1]\n", + "abacus_gold['CARTESIAN_Z'] = xyz[:,2]" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "f0a95eb5-227c-4ffb-9f33-11c9fb607eb3", + "metadata": {}, + "outputs": [], + "source": [ + "xyz = rotate(abacus_gold['RA'], abacus_gold['DEC'], xyz)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "7e78a03e-fe1f-4b80-af36-73b26c70e0d8", + "metadata": {}, + "outputs": [], + "source": [ + "abacus_gold['ROTCARTESIAN_X'] = xyz[:,0]\n", + "abacus_gold['ROTCARTESIAN_Y'] = xyz[:,1]\n", + "abacus_gold['ROTCARTESIAN_Z'] = xyz[:,2]" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "b7b1fa4b-c741-45de-8628-151b2407b84c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
Table length=331156\n", + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
RADECZSURVGMRDETMAGLUMDISTDISTMODFIELDIN_D8LUMFNCONSERVATIVECARTESIAN_XCARTESIAN_YCARTESIAN_ZROTCARTESIAN_XROTCARTESIAN_YROTCARTESIAN_Z
Mpc
float64float64float64float64float64float64float64str3int64int64float64float64float64float64float64float64
221.58851531524240.88910470573151210.50908923299014241.022903434227640420.0248422827730862063.353364115278541.57286805199301G1500-1022.5110811033369-907.461365720167921.2163922617303271021.7012266085815908.439431979298818.15429773025413
220.8388953469847-0.42515379227995570.50704807777963091.710568868151355619.6284127121110862053.289199980654341.562250614145434G1500-1030.7407448849524-890.9333243854388-10.1097974552185171029.852777125875891.9192735066924-13.196464089950581
220.94292732616822-0.42707951245860440.499391757932836631.784564248020011719.9017597547518432015.644439007325741.5220696235003G1500-1015.4119768436542-880.9099247810838-10.0203038186761831014.5339449244166881.8812097804043-13.061057243623502
220.7779893960335-0.4076885614401690.50184166045871731.416232462245822719.4829878115429942027.671971844036841.5349884901389G1500-1022.3498876709572-881.7836115364541-9.6067343186085831021.4722248754321882.7615353001427-12.668280938154894
220.80226411916357-0.40944828452174070.50579082515479021.295017050442026220.1885663889848352047.09604966856341.55569110110545G1500-1029.0600367079237-888.3317613425351-9.7150781966187441028.1757498224451889.3161033180729-12.796716759406744
220.70390645849213-0.65979251971954440.50147256732670421.608762234047530320.12471977635162025.858850541834341.53304591547108G1500-1022.783423939799-879.8538632994431-15.5369918467721871021.8898318637006880.8322028148979-18.599816731667264
220.228265089161.00776029294012750.495512928797414260.772240310806066820.1535847823092521996.636929655093841.50149549765658G1500-1019.1497456392286-862.110477557966423.4812463616051021018.3900937372865863.085347985415420.42908595795937
220.18913734445744-0.29211311178928730.49832826910119631.716755328483264219.9792880428659072010.42870472710641.516443382239125G1500-1024.9980089272194-865.8574228999394-6.8408212150610371024.143863177716866.8378880488672-9.91036270716756
219.94042101710016-0.75952764622330990.496589733409188661.506522326492534519.7158688892415452001.909284989741841.50722196931043G1500-1025.5007373223045-858.6816916892146-17.7316792442837361024.6208144309192859.6626412045856-20.80269914551607
................................................
134.204393823202850.23584267534624030.0121616797034092-0.0348586804310469520.1042803103134436.81874541052131632.83034492972502G900-25.36210528276084426.0764239020941380.149733035467373525.387381989642627-26.0521419494772250.07364396594242921
132.394659734675940.74976494415558290.0123552029332301140.1880740314303706319.72886611504967337.41040638037022632.86496212930882G900-24.9133818737406727.2887420119324560.4835591551978186724.94082143377037-27.2648889056147470.40880999312472016
129.71803289373041.96489150253372750.013070434552440613-0.2494549716181201619.65111610462420839.5986582375603732.988402352620504G900-24.96278375817849630.048570387942471.340206258382137524.99543153866415-30.0246687433505141.265297269317895
139.623166546434362.87127238749004650.0239377566713787630.0755747118176591520.14117716617583773.1493047122564734.32105101245256G900-54.3540899337617946.221073989153913.578546095948309554.408776900185956-46.169039223115063.4154910099986364
137.987895079400231.7569759735309730.021558059405622830.01982896628265090620.13792332749235665.7541381949552934.0896154503474G900-47.8020625547605343.05946078517171.97349104191527347.848945922471145-43.0136973638244561.8300895314843593
135.33637565192657-1.04956740429591950.0220559029554754370.597110869649682718.77014085439482567.2990049854975834.14004321617915G900-46.825335345584346.27872789816235-1.206139124930711646.86577439953129-46.23389767215575-1.3466082058756994
222.688625219656930.246587368856534730.0184840592727129050.185360223053537319.13820920369471456.24140342047124433.75028074659527G1500-40.58955674286038-37.4400479506138740.237655990998185240.5542404220028137.478872578551860.11610999642754374
131.9965189805966-1.46544718663686680.020397801956211660.1976330709778599318.27462554724400762.1583197821400733.96749633211685G900-40.7445262956305445.25690999308012-1.55786542639918140.78296346012787-45.21789920435435-1.6801050022473523
132.29928767965137-1.28664477968755690.0197665829728856930.271157607473649520.1432976930640260.2047774348953433.89815477616363G900-39.7226187502787543.65567831890939-1.32565253643928839.76022467000742-43.61764616754348-1.4448257697626123
221.4654657859774-0.451121640732615570.0260900095327558650.527964980220448117.92838893506487479.8609124032076934.51167133829578G1500-58.320728274200285-51.535188332133345-0.612795193206614158.26928713702627551.59097417368829-0.7874396249762501
" + ], + "text/plain": [ + "\n", + " RA DEC ... ROTCARTESIAN_Z \n", + " ... \n", + " float64 float64 ... float64 \n", + "------------------ -------------------- ... -------------------\n", + " 221.5885153152424 0.8891047057315121 ... 18.15429773025413\n", + " 220.8388953469847 -0.4251537922799557 ... -13.196464089950581\n", + "220.94292732616822 -0.4270795124586044 ... -13.061057243623502\n", + " 220.7779893960335 -0.407688561440169 ... -12.668280938154894\n", + "220.80226411916357 -0.4094482845217407 ... -12.796716759406744\n", + "220.70390645849213 -0.6597925197195444 ... -18.599816731667264\n", + " 220.22826508916 1.0077602929401275 ... 20.42908595795937\n", + "220.18913734445744 -0.2921131117892873 ... -9.91036270716756\n", + "219.94042101710016 -0.7595276462233099 ... -20.80269914551607\n", + " ... ... ... ...\n", + "134.20439382320285 0.2358426753462403 ... 0.07364396594242921\n", + "132.39465973467594 0.7497649441555829 ... 0.40880999312472016\n", + " 129.7180328937304 1.9648915025337275 ... 1.265297269317895\n", + "139.62316654643436 2.8712723874900465 ... 3.4154910099986364\n", + "137.98789507940023 1.756975973530973 ... 1.8300895314843593\n", + "135.33637565192657 -1.0495674042959195 ... -1.3466082058756994\n", + "222.68862521965693 0.24658736885653473 ... 0.11610999642754374\n", + " 131.9965189805966 -1.4654471866368668 ... -1.6801050022473523\n", + "132.29928767965137 -1.2866447796875569 ... -1.4448257697626123\n", + " 221.4654657859774 -0.45112164073261557 ... -0.7874396249762501" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "abacus_gold" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "f0a8d7dc-a273-4b15-855d-89da5943f47f", + "metadata": {}, + "outputs": [], + "source": [ + "# \n", + "abacus_gold.meta['RMAX'] = 12.0\n", + "abacus_gold.meta['RLIM'] = 19.8\n", + "\n", + "isin = (abacus_gold.meta['RMAX'] <= abacus_gold['DETMAG']) & (abacus_gold['DETMAG'] <= abacus_gold.meta['RLIM'])\n", + "abacus_gold = abacus_gold[isin]" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "c2a8bf1c-9575-4507-9164-fff08d95ad82", + "metadata": {}, + "outputs": [], + "source": [ + "# Randomise rows. \n", + "idx = np.arange(len(abacus_gold))\n", + "idx = np.random.choice(idx, size=len(idx), replace=False)\n", + "\n", + "abacus_gold = abacus_gold[idx]" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "45129599-dfeb-44b7-922d-50c79cfb7cdd", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
Table length=209521\n", + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
RADECZSURVGMRDETMAGLUMDISTDISTMODFIELDIN_D8LUMFNCONSERVATIVECARTESIAN_XCARTESIAN_YCARTESIAN_ZROTCARTESIAN_XROTCARTESIAN_YROTCARTESIAN_Z
Mpc
float64float64float64float64float64float64float64str3int64int64float64float64float64float64float64float64
211.904268515019251.36032302099118670.364610285034655531.223424565234688519.6226876025731481381.83898845587240.702287210491605G1500-859.4081215229603-535.023278359160324.039609868889446858.9639351373053535.845435399016921.465275771058987
181.66959075063224-1.76660833492683670.192345489707678840.949270539266454919.730041047055256661.564660045302839.102861488486944G1200-554.3439360641598-16.158076039043614-17.104839851177143554.274464712356816.68854252803292-18.76615929921612
129.75198269691130.97742200404454570.206210070615260530.613397336454566619.202717401808208715.386560463448639.2727038853033G900-379.20304992403766455.910018328686910.117104634204557379.66777203920077-455.546935015125658.97922955767013
180.824538908857650.3326866502622060.41322169760549231.253681743075408619.746378156536761603.900382349064341.02588695459231G1200-1134.7881724409913-16.3317718554570746.5898783223412531134.786678349688617.417688573590273.1887842573212666
137.236512485523862.5501185060158330.247549579525900181.32774885343991319.233688234257503880.359088790189539.72329926115178G900-517.5639886416078478.6572225062510631.397594230081964518.1135740722724-478.161725660529229.84487217004491
215.434791035461052.19808949030941570.17406063302680530.465356658242263519.52427997996957591.771280899406938.860769423508344G1500-410.37590880245654-292.0141781959761719.33212341963848410.1523799182468292.40674961522318.10292284919089
214.55498472032187-0.66851631534699050.23730959536850471.358717704953351218.065125041042045838.877540994532239.618492836610834G1500-558.3385334308874-384.5248772130612-7.91042306056155557.9440958689611385.0589976289906-9.582699640256626
133.37173674693992.97318712277580980.32193453385722371.361810259283563319.6823456844657551193.387852397639840.383908064039794G900-619.1160649258584655.344135331450546.82484503372689619.8804633086057-654.751378342513944.967181832138714
131.53776693844-1.4697541491316590.37414124073245871.268098188237842819.7697106937178371424.768397969388840.76872136853464G900-687.3183199676726775.840953534079-26.594224563512086687.9776416738591-775.1828759075323-28.656314846625484
................................................
212.0610899211587-1.09869399534515290.267340882700247251.450270694310743719.310022409281746961.659121418745839.91510577639566G1500-642.9521455822885-402.7149276488213-14.549739333694422642.519983610582403.3300098862981-16.475532189212483
176.1589802161306-0.77565417906555260.322357979963376231.231760423940312819.5397627842456351195.22687966411740.38725175752452G1200-901.747378965141760.5424621142444-12.235833251613897901.7641787668546-59.679516255278266-14.938608944438824
185.04605728782386-2.88847569834134980.211809861915972061.229295305721617819.08411713601012737.342139658497339.33834527434519G1200-605.3352961624666-53.45033477562049-30.661731875714665605.1892543163354.02957986193856-32.47571137530387
182.203609526042471.18123943924858720.308545210062101341.007924338568528619.777725173395061135.563717133387440.27605753876324G1200-866.9802243982023-33.3607195148124117.88987541472109866.997627768186534.1903522976346715.291435529657425
216.843943778453820.39587860651381330.329729868837880251.310061950885354819.4741298318090731227.343423822779440.444830499872864G1500-738.6348113587643-553.45349365665116.37732603998089738.120649941847554.16006944625634.1650978278510316
214.588716237783130.78889533799976860.182004689622854881.03044815499037218.926480124074832621.926370030670138.96869485768082G1500-433.1211430925376-298.66438829797467.244406419169604432.8549092977041299.078722496853745.947108716172542
219.830528074431332.47009410448893260.158036842367103470.4898095163084660318.90701828508597531.738133568115238.62848903416776G1500-352.2900199271592-293.835125264311319.78936050121278352.06640639825537294.172111139194318.734254245808135
139.62081929755140.80393037827579170.34041470380514771.356351790796240819.607402579694091274.228046960670840.526235800411165G900-724.0878067771268615.793485274280713.33796378137863724.7134735063009-615.100294735440311.165950125029617
221.665828636270650.435857626947035950.197547982702454661.187388005087952619.203299664119875681.670270394607239.16787176788638G1500-425.2160945948741-378.39904863226244.330113288791515424.8648644523524378.80578166613223.0567498436343343
180.33997529194235-1.51644822705399250.373700327787929171.605163168757647519.7065695319395931422.775784581448540.76568232476667G1200-1035.3440393689955-6.143480831271167-27.409348517476681035.2508867360657.134240145395583-30.512271785681605
" + ], + "text/plain": [ + "\n", + " RA DEC ... ROTCARTESIAN_Z \n", + " ... \n", + " float64 float64 ... float64 \n", + "------------------ ------------------- ... -------------------\n", + "211.90426851501925 1.3603230209911867 ... 21.465275771058987\n", + "181.66959075063224 -1.7666083349268367 ... -18.76615929921612\n", + " 129.7519826969113 0.9774220040445457 ... 8.97922955767013\n", + "180.82453890885765 0.332686650262206 ... 3.1887842573212666\n", + "137.23651248552386 2.550118506015833 ... 29.84487217004491\n", + "215.43479103546105 2.1980894903094157 ... 18.10292284919089\n", + "214.55498472032187 -0.6685163153469905 ... -9.582699640256626\n", + " 133.3717367469399 2.9731871227758098 ... 44.967181832138714\n", + " 131.53776693844 -1.469754149131659 ... -28.656314846625484\n", + " ... ... ... ...\n", + " 212.0610899211587 -1.0986939953451529 ... -16.475532189212483\n", + " 176.1589802161306 -0.7756541790655526 ... -14.938608944438824\n", + "185.04605728782386 -2.8884756983413498 ... -32.47571137530387\n", + "182.20360952604247 1.1812394392485872 ... 15.291435529657425\n", + "216.84394377845382 0.3958786065138133 ... 4.1650978278510316\n", + "214.58871623778313 0.7888953379997686 ... 5.947108716172542\n", + "219.83052807443133 2.4700941044889326 ... 18.734254245808135\n", + " 139.6208192975514 0.8039303782757917 ... 11.165950125029617\n", + "221.66582863627065 0.43585762694703595 ... 3.0567498436343343\n", + "180.33997529194235 -1.5164482270539925 ... -30.512271785681605" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "abacus_gold" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "9fced5c6-6508-4382-a67e-fd1d6c9b7542", + "metadata": {}, + "outputs": [], + "source": [ + "abacus_gold.meta['GOLD_NGAL'] = len(abacus_gold)\n", + "abacus_gold.meta['IMMUTABLE'] = 'FALSE'\n", + "abacus_gold.meta['AREA'] = 180." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "4068e8f1-ea53-4cce-a31c-75f57f07e750", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'/global/cscratch1/sd/mjwilson/norberg/GAMA4/abacus_gama_gold.fits'" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "opath = findfile(ftype='gold', dryrun=False, survey='abacus_gama')\n", + "opath" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "3ca1d059-812d-4750-8a38-76fffc1661d0", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: VerifyWarning: Keyword name 'GOLD_NGAL' is greater than 8 characters or contains characters not allowed by the FITS standard; a HIERARCH card will be created. [astropy.io.fits.card]\n", + "WARNING: VerifyWarning: Keyword name 'IMMUTABLE' is greater than 8 characters or contains characters not allowed by the FITS standard; a HIERARCH card will be created. [astropy.io.fits.card]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "chgrp desi /global/cscratch1/sd/mjwilson/norberg/GAMA4/abacus_gama_gold.fits b''\n", + "chmod 700 /global/cscratch1/sd/mjwilson/norberg/GAMA4/abacus_gama_gold.fits b''\n" + ] + } + ], + "source": [ + "write_desitable(opath, abacus_gold)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "23ab4751-6033-49d1-946b-3a4f0e17f265", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "DEBUG: findfile returns /global/cscratch1/sd/mjwilson/norberg/GAMA4//abacus_gama_gold_dryrun.fits\n" + ] + }, + { + "data": { + "text/plain": [ + "'/global/cscratch1/sd/mjwilson/norberg/GAMA4/abacus_gama_gold_dryrun.fits'" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "opath = findfile(ftype='gold', dryrun=True, survey='abacus_gama')\n", + "opath" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "220a3e35-6a52-4caf-ac5e-b21732e1a91c", + "metadata": {}, + "outputs": [], + "source": [ + "abacus_gold = abacus_gold[::1000]" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "aa734f6e-cd12-4b7e-9cf9-c9799762c994", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
Table length=210\n", + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
RADECZSURVGMRDETMAGLUMDISTDISTMODFIELDIN_D8LUMFNCONSERVATIVECARTESIAN_XCARTESIAN_YCARTESIAN_ZROTCARTESIAN_XROTCARTESIAN_YROTCARTESIAN_Z
Mpc
float64float64float64float64float64float64float64str3int64int64float64float64float64float64float64float64
211.904268515019251.36032302099118670.364610285034655531.223424565234688519.6226876025731481381.83898845587240.702287210491605G1500-859.4081215229603-535.023278359160324.039609868889446858.9639351373053535.845435399016921.465275771058987
175.90450719773631.31166753661288960.141049074161494930.919989286695744119.005242424749525469.2664228432593638.357097402475425G1200-410.101079918291629.363984568769889.414090303922324410.15536495274057-28.971528995759988.184836240652807
182.01903711750415-1.1062418801958280.219446804830310650.758301483723138519.294990066155744767.48461164523239.42534838234508G1200-628.8631671948306-22.169589255444773-12.150867826333156628.802422115460822.77136342514076-14.035536344989598
179.0416120324038-0.0282101371198766150.158712301015899730.452540553691687519.719657622069207534.247186939372738.638711220395564G1200-461.00519003124977.711964856178095-0.227012187575956461.00960785452145-7.270807017688307-1.6087272474399603
213.109905056901252.979813018328930.33115848464990341.293845577401023419.2371591485140281233.589258924334240.45585289622977G1500-775.1798570187326-505.5243719732848.17385474946941774.8366496390818506.265941124121645.85177097751833
130.316597439593662.73965235153980660.16544883390988830.614275850871549619.661959314916935559.374772348264138.7385143795644G900-310.1876109568333365.5459583154140522.941245004274815310.6046377280399-365.248960010888822.010419963857828
176.03113519068327-2.67198768417404730.22856051871233031.420435056662644718.532733073813244803.754929129327339.52561824653295G1200-651.946405337419245.23253962199965-30.49871095681026651.8950545612173-44.60864531712787-32.452673921623656
177.166132587168161.6523509578476840.159833334298949750.982945664250201119.76377522614793538.41557575893638.65558807525041G1200-463.457565760535622.94146854631057413.385712458936716463.5173442698166-22.4979569582956511.996542482513467
132.83652906092772-1.55155345334351780.108244554604279180.919742724606720917.307458566801873352.123285350089837.73347372609897G900-215.94872587362087232.90537787128838-8.603004873197788216.14474819062855-232.6986212574054-9.250861320355776
................................................
219.21206227877562-1.6193317468482980.26477576609726340.584619794065052619.41924862399951951.039032255197839.89099170742726G1500-582.381189653076-475.18270430762595-21.2490825956909581.8599016279974475.73979059380235-22.99309829538324
135.41204532124326-1.76002605635965130.258521585394379640.885170606101923919.52707187355826925.248599045611639.831292180928585G900-523.3337340760055515.8601927254608-22.58010356348376523.7571133768082-515.3591575455125-24.1499825837449
137.67847256882092.09029408256230330.164332495215245620.832719809971458818.444213543463643555.197722397647638.722238379163365G900-352.32884754871037320.8366513042591417.392382293378414352.6862511305594-320.4993468356334516.335407505673185
180.44373398003972-0.1387051533785580.235477317641803021.339329676553452617.480476944685563831.497510910433639.59930476748015G1200-672.9950655230674-5.212194548886269-1.6292799657315462672.9818637736265.856208061687904-3.646313554076464
218.375885755815122.76632579600976450.121276085491782930.361113833951516119.376752443969732398.1001492013392637.99996170175252G1500-278.0125631405138-220.1595935424014717.13530073283355277.8518654042843220.4255340812530316.302614522585127
176.57610797290016-0.57102684168667450.284584139920633470.905948957051667619.5737686338228271033.682085591327440.07193494861035G1200-803.206032053428348.055354126342515-8.01957097264533803.2240069101335-47.286712141516404-10.426988245949854
137.271043860990551.05078093628997340.074674430013490940.884943066802736716.231483838499255237.1470577658759636.875088704100925G900-162.06977213818115149.70540345581044.0467468031912714162.22435709939393-149.55024386255693.5605545658100475
133.79057853858583-0.91199940090845740.340845695915864331.403962902422073219.6984181567061431276.12747983950240.52947030384363G900-658.539299903132686.944511183896-15.148472398104548659.148000324734-686.3140140556293-17.12410457041426
220.578839569882581.22780748353949170.400365729370247841.243787187725362319.5939957317732731544.429147206783540.94383994621441G1500-837.4544627371343-717.248956537708423.63202402555308836.8347851591108718.050021808125521.124012435447273
216.967520442844572.4443951081201580.092181279578663840.509418522269613718.34747477506708296.478740309006637.359967783652955G1500-216.68931759628754-163.0946077602958711.577528267331823216.56687349648732163.3018917606231610.928497287758015
" + ], + "text/plain": [ + "\n", + " RA DEC ... ROTCARTESIAN_Z \n", + " ... \n", + " float64 float64 ... float64 \n", + "------------------ --------------------- ... -------------------\n", + "211.90426851501925 1.3603230209911867 ... 21.465275771058987\n", + " 175.9045071977363 1.3116675366128896 ... 8.184836240652807\n", + "182.01903711750415 -1.106241880195828 ... -14.035536344989598\n", + " 179.0416120324038 -0.028210137119876615 ... -1.6087272474399603\n", + "213.10990505690125 2.97981301832893 ... 45.85177097751833\n", + "130.31659743959366 2.7396523515398066 ... 22.010419963857828\n", + "176.03113519068327 -2.6719876841740473 ... -32.452673921623656\n", + "177.16613258716816 1.652350957847684 ... 11.996542482513467\n", + "132.83652906092772 -1.5515534533435178 ... -9.250861320355776\n", + " ... ... ... ...\n", + "219.21206227877562 -1.619331746848298 ... -22.99309829538324\n", + "135.41204532124326 -1.7600260563596513 ... -24.1499825837449\n", + " 137.6784725688209 2.0902940825623033 ... 16.335407505673185\n", + "180.44373398003972 -0.138705153378558 ... -3.646313554076464\n", + "218.37588575581512 2.7663257960097645 ... 16.302614522585127\n", + "176.57610797290016 -0.5710268416866745 ... -10.426988245949854\n", + "137.27104386099055 1.0507809362899734 ... 3.5605545658100475\n", + "133.79057853858583 -0.9119994009084574 ... -17.12410457041426\n", + "220.57883956988258 1.2278074835394917 ... 21.124012435447273\n", + "216.96752044284457 2.444395108120158 ... 10.928497287758015" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "abacus_gold" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "9579da1d-a48e-4af9-8a98-3e3d63ccce23", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "chgrp desi /global/cscratch1/sd/mjwilson/norberg/GAMA4/abacus_gama_gold_dryrun.fits b''\n", + "chmod 700 /global/cscratch1/sd/mjwilson/norberg/GAMA4/abacus_gama_gold_dryrun.fits b''\n" + ] + } + ], + "source": [ + "write_desitable(opath, abacus_gold)" + ] + }, + { + "cell_type": "markdown", + "id": "d5597572-8b37-4dca-b2ec-2d2ab398b8aa", + "metadata": {}, + "source": [ + "# Done." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "DESI master", + "language": "python", + "name": "desi-master" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/environment.yml b/environment.yml index 34fe4091..891f9c71 100644 --- a/environment.yml +++ b/environment.yml @@ -1,5 +1,15 @@ +# First, # conda env create -f environment.yml -name: base +# +# then define GOLD_DIR and CODE_ROOT in +# +# ~/.conda/envs/lumfn/etc/conda/activate.d/env_vars.sh +# +# conda activate lumfn +# +# export PYTHONPATH=../$CODE_ROOT:$PYTHONPATH (pseudo code). +# +name: lumfn channels: - defaults - conda-forge @@ -19,4 +29,3 @@ dependencies: - coveralls - pytest - pytest-cov - - inflect \ No newline at end of file diff --git a/fillfactor.py b/fillfactor.py index cd4b8422..27cb325e 100644 --- a/fillfactor.py +++ b/fillfactor.py @@ -22,36 +22,33 @@ from params import sphere_radius -def collate_fillfactors(realzs=np.array([0]), field='G9', survey='gama', dryrun=False, prefix=None, write=True, force=False, oversample=2): +def collate_fillfactors(realzs=np.array([1]), field='G9', survey='gama', dryrun=False, prefix=None, write=True, force=False, oversample=2): + ''' + Generate a zeroth realization randoms_n8 catalog by collating all randoms_n8 catalogs with realization number greater than unity. + ''' + print('Collating fillfactor realizations into main (realz=0).') realzs = np.sort(realzs) - assert realzs[0] == 0 + assert realzs[0] == 1 opaths = [findfile(ftype='randoms_n8', dryrun=dryrun, field=field, survey=survey, prefix=prefix, realz=realz) for realz in realzs] - opath = opaths[0] - assert np.all(np.array([os.path.isfile(x) for x in opaths])), 'Failed to find {}'.format(opaths) + assert np.all(np.array([os.path.isfile(x) for x in opaths])), 'Failed to find {}'.format('\n'.join(opaths)) for oo in opaths: print(f'Fetching {oo}.') orealzs = [Table.read(opath) for opath in opaths] - mainreal = orealzs[0] - if len(realzs) == 1: - print('Only one realization, nothing to be done. Exiting.') - return mainreal + opath = findfile(ftype='randoms_n8', dryrun=dryrun, field=field, survey=survey, prefix=prefix, realz=1) + mainreal = Table.read(opath) - keys = list(mainreal.meta.keys()) + keys = list(mainreal.meta.keys()) for key in keys: print(key, mainreal.meta[key]) - - if ('COLLATE' in keys) and (not force): - print('Results have already been collated to zeroth realization. Exiting') - return mainreal for col in ['FILLFACTOR', 'RAND_N8']: print(f'Solving for {col}') @@ -126,8 +123,7 @@ def process_one(run, pid=0, start=0.0): def fillfactor(log, field, dryrun, prefix, survey, oversample, nproc, realz, nooverwrite, debug=False): opath = findfile(ftype='randoms_n8', dryrun=dryrun, field=field, survey=survey, prefix=prefix, realz=realz) - if nooverwrite: - overwrite_check(opath) + overwrite_check(opath, nooverwrite) start = time.time() @@ -314,7 +310,7 @@ def fillfactor(log, field, dryrun, prefix, survey, oversample, nproc, realz, noo parser.add_argument('-s', '--survey', help='Select survey.', default='gama') parser.add_argument('--prefix', help='filename prefix', default='randoms') parser.add_argument('--nproc', help='nproc', default=12, type=int) - parser.add_argument('--realz', help='Realization number', default=0, type=np.int32) + parser.add_argument('--realz', help='Realization number', default=1, type=np.int32) parser.add_argument('--nooverwrite', help='Do not overwrite outputs if on disk', action='store_true') parser.add_argument('--oversample', help='Oversampling factor for fillfactor counting.', default=2, type=int) parser.add_argument('--config', help='Path to configuration file', type=str, default=findfile('config')) @@ -348,5 +344,6 @@ def fillfactor(log, field, dryrun, prefix, survey, oversample, nproc, realz, noo config.write() ''' assert field in fields, 'Error: Field not in fields' - + assert realz > 0, 'Realization number should be greater than zero.' + fillfactor(log, field, dryrun, prefix, survey, oversample, nproc, realz, nooverwrite, debug) diff --git a/findfile.py b/findfile.py index aebfd036..a4c53ee1 100644 --- a/findfile.py +++ b/findfile.py @@ -102,10 +102,12 @@ def write_desitable(opath, table, test=False): print(cmd, output) def fetch_fields(survey): - assert survey in ['desi', 'gama'], f'Fields for {survey} survey are not supported.' + assert survey in ['desi', 'gama', 'abacus_gama'], f'Fields for {survey} survey are not supported.' - fpath = resource_filename('DESI', f'data/{survey}_fields.txt') - fields = np.loadtxt(fpath, comments="#", delimiter=",", unpack=False, dtype=str) + _survey = survey.split('_')[-1] + + fpath = resource_filename('DESI', f'data/{_survey}_fields.txt') + fields = np.loadtxt(fpath, comments="#", delimiter=",", unpack=False, dtype=str) return fields @@ -123,14 +125,15 @@ def release_dir(user=os.environ['USER'], survey='gama', version=None): else: return '/cosma/home/durham/{}/data/GAMA4/'.format(user) -def overwrite_check(opath, ext=None): +def overwrite_check(opath, nooverwrite, ext_name=None): if os.path.isfile(opath): - exist = True + exist = True - if ext != None: - hdul = fits.open(opath) + if ext_name != None: exist = False + hdul = fits.open(opath) + # print(ext) # print(hdul.info()) @@ -138,21 +141,31 @@ def overwrite_check(opath, ext=None): hdr = hdu.header try: - if hdr['EXTNAME'] == 'BOUNDARY': + if hdr['EXTNAME'] == f'{ext_name}': exist = True - - print(f'WARNING: Found existing BOUNDARY extension to {opath} and overwrite forbidden (--nooverwrite).') except KeyError as E: pass - else: - print(f'{opath} found on disk and overwrite forbidden (--nooverwrite).') + else: + exist = False + + if exist: + if nooverwrite: + if ext_name == None: + print(f'OVERWRITE CHECK: {opath} found on disk and overwrite forbidden (--nooverwrite).') - if exist: - exit(0) + else: + print(f'OVERWRITE CHECK: {opath} with extension {ext_name} found on disk and overwrite forbidden (--nooverwrite).') + exit(0) + else: + print(f'OVERWRITE CHECK: {opath} found on disk.') + + else: + print(f'OVERWRITE CHECK: {opath} not found on disk, safe to write.') + def fetch_header(ftype=None, name=None, ext=1, allsupported=False, dryrun=False, prefix=None, field=None, utier='{utier}', survey=None, realz=0, debug=False, version=None, fpath=None): if allsupported: result = OrderedDict() diff --git a/gama_gold.py b/gama_gold.py index a3dceaea..9c10ed4b 100644 --- a/gama_gold.py +++ b/gama_gold.py @@ -24,9 +24,13 @@ def gama_gold(argset): sys.stdout = open(logfile, 'w') - root = os.environ['TILING_CATDIR'] + root = os.environ['GOLD_DIR'] + root = os.path.dirname(root) # up a directory. + fpath = root + '/TilingCatv46.fits' + assert os.path.exists(fpath), f'{fpath} does not exist.' + opath = findfile(ftype='gold', dryrun=False, survey='gama') if argset.dryrun: @@ -36,8 +40,7 @@ def gama_gold(argset): print('Dryrun gama_gold created on full run; Exiting.') return 0 - if argset.nooverwrite: - overwrite_check(opath) + overwrite_check(opath, args.nooverwrite) dat = Table.read(fpath) dat = Table(dat, masked=False) @@ -84,13 +87,13 @@ def gama_gold(argset): dat = dat[sclass_cut & z_cut & r_cut & nq_cut] dat['ZSURV'] = dat['ZGAMA'] - dat['LUMDIST'] = cosmo.luminosity_distance(dat['ZGAMA'].data) - dat['DISTMOD'] = distmod(dat['ZGAMA'].data) + dat['LUMDIST'] = cosmo.luminosity_distance(dat['ZSURV'].data) + dat['DISTMOD'] = distmod(dat['ZSURV'].data) dat['FIELD'] = gama_field(dat['RA'], dat['DEC']) dat['IN_D8LUMFN'] = np.zeros_like(dat['FIELD'], dtype=int) dat['CONSERVATIVE'] = np.zeros_like(dat['FIELD'], dtype=int) - xyz = cartesian(dat['RA'], dat['DEC'], dat['ZGAMA']) + xyz = cartesian(dat['RA'], dat['DEC'], dat['ZSURV']) dat['CARTESIAN_X'] = xyz[:,0] dat['CARTESIAN_Y'] = xyz[:,1] diff --git a/gen_ddp_cat.py b/gen_ddp_cat.py index 42429b9e..ebc225a1 100644 --- a/gen_ddp_cat.py +++ b/gen_ddp_cat.py @@ -39,8 +39,7 @@ sys.stdout = open(logfile, 'w') -if args.nooverwrite: - overwrite_check(opath) +overwrite_check(opath, args.nooverwrite) print('Reading: {}'.format(fpath)) diff --git a/gen_ddp_n8.py b/gen_ddp_n8.py index ed1108a2..bbdc4b21 100644 --- a/gen_ddp_n8.py +++ b/gen_ddp_n8.py @@ -49,8 +49,7 @@ sys.stdout = open(logfile, 'w') -if args.nooverwrite: - overwrite_check(opath) +overwrite_check(opath, args.nooverwrite) # Read ddp cat. dat = Table.read(fpath) @@ -68,7 +67,13 @@ prefix = 'randoms_ddp1' dat['RAND_N8'] = 0. -for realz in np.arange(oversample_nrealisations): +if dryrun: + realzs = np.arange(1, 6, 1) + +else: + realzs = np.arange(1, oversample_nrealisations + 1, 1) + +for realz in realzs: print(f'\n\nSolving for galaxy fillfactors with oversampled realization {realz}.') rpaths = [findfile(ftype='randoms', dryrun=dryrun, field=ff, survey=survey, prefix=prefix, oversample=oversample, realz=realz) for ff in fields] diff --git a/gen_gold_lf.py b/gen_gold_lf.py index 62902139..6c38a561 100644 --- a/gen_gold_lf.py +++ b/gen_gold_lf.py @@ -120,8 +120,7 @@ def process_cat(fpath, vmax_opath, survey='gama', extra_cols=[], bitmasks=['IN_D fpath = findfile(ftype='ddp_n8', dryrun=dryrun, survey=survey) opath = findfile(ftype='vmax', dryrun=dryrun, survey=survey) - if args.nooverwrite: - overwrite_check(opath) + overwrite_check(opath, args.nooverwrite) print(f'Reading: {fpath}') print(f'Writing: {opath}') diff --git a/gen_kEcat.py b/gen_kEcat.py index 78b24a55..d2eac1f3 100644 --- a/gen_kEcat.py +++ b/gen_kEcat.py @@ -2,8 +2,8 @@ import sys import argparse import runtime +import multiprocessing import numpy as np -import multiprocessing from astropy.table import Table, vstack from smith_kcorr import GAMA_KCorrection @@ -71,8 +71,7 @@ def gen_kE(log, dryrun, survey, nooverwrite, nproc=12): sys.stdout = open(logfile, 'w') - if args.nooverwrite: - overwrite_check(opath) + overwrite_check(opath, args.nooverwrite) fields = fetch_fields(survey) @@ -81,7 +80,7 @@ def gen_kE(log, dryrun, survey, nooverwrite, nproc=12): dat = Table.read(fpath) dat.pprint() - + kcorr_r = GAMA_KCorrection(band='R') kcorr_g = GAMA_KCorrection(band='G') diff --git a/randoms.py b/randoms.py index b276123e..3612063b 100644 --- a/randoms.py +++ b/randoms.py @@ -10,7 +10,7 @@ from cartesian import cartesian, rotate from runtime import calc_runtime from desi_randoms import desi_randoms -from findfile import fetch_fields, findfile, overwrite_check, call_signature +from findfile import fetch_fields, findfile, overwrite_check, call_signature, write_desitable from gama_limits import gama_limits, gama_field from ddp_zlimits import ddp_zlimits from config import Configuration @@ -23,11 +23,10 @@ def randoms(field='G9', survey='gama', density=1., zmin=ddp_zlimits['DDP1'][0], fields = fetch_fields(survey) assert field in fields, f'Provided {field} field is not compatible with those available for {survey} survey ({fields})' - + opath = findfile(ftype='randoms', dryrun=dryrun, field=field, survey=survey, prefix=prefix, realz=realz, oversample=oversample) - if args.nooverwrite: - overwrite_check(opath) + overwrite_check(opath, args.nooverwrite) if seed == None: seed = seed + realz + 50 * oversample @@ -37,9 +36,9 @@ def randoms(field='G9', survey='gama', density=1., zmin=ddp_zlimits['DDP1'][0], call_signature(dryrun, sys.argv) print('Solving for redshift limits: {} < z < {}.'.format(zmin, zmax)) - + ## ras and decs. - if survey == 'gama': + if survey in ['gama', 'abacus_gama']: Area = 60. ra_min = gama_limits[field]['ra_min'] @@ -130,7 +129,7 @@ def randoms(field='G9', survey='gama', density=1., zmin=ddp_zlimits['DDP1'][0], os.makedirs(rand_dir) print('Volume [1e6]: {:.2f}; oversample: {:.2f}; density: {:.2e}; nrand [1e6]: {:.2f}'.format(vol/1.e6, oversample, density, nrand / 1.e6)) - + zs = np.arange(0.0, zmax+dz, dz) Vs = volcom(zs, Area) @@ -207,7 +206,7 @@ def randoms(field='G9', survey='gama', density=1., zmin=ddp_zlimits['DDP1'][0], runtime = calc_runtime(start, 'Writing {}'.format(opath), xx=randoms) - randoms.write(opath, format='fits', overwrite=True) + write_desitable(opath, randoms) runtime = calc_runtime(start, 'Finished'.format(opath)) @@ -260,7 +259,14 @@ def randoms(field='G9', survey='gama', density=1., zmin=ddp_zlimits['DDP1'][0], config.update_attributes('randoms', args) config.write() ''' - for xx in [1, oversample]: + + if realz == 0: + runs = [1] + + else: + runs = [oversample] + + for xx in runs: seed = seed # only generate independent realizations for oversample. diff --git a/submit.py b/submit.py index 6e7a41cb..86d79f4f 100644 --- a/submit.py +++ b/submit.py @@ -1,8 +1,40 @@ import os import argparse +import subprocess import numpy as np +def customise_serialorparallel(): + root = os.environ['CODE_ROOT'] + '/bin/' + + sorp = root + '/serialorparallel' + + ff = open(f'/{sorp}') + ff = ff.read() + ff = ff.split('\n') + ff = [x.rstrip() for x in ff] + + shebang = subprocess.check_output(['which', 'bash']) + shebang = shebang.rstrip().decode('utf-8') + shebang = r'#!{}'.format(shebang) + + hostname = subprocess.check_output(['hostname']) + hostname = hostname.rstrip().decode('utf-8') + hostname = '# Customised for {} system'.format(hostname) + + ff = [shebang, '#', hostname, '#'] + ff[1:] + + opath = root + '/custom/serialorparallel' + + print(f'Writing to {opath}') + + with open(opath, 'w') as f: + for line in ff: + f.write(line) + f.write('\n') + + f.close() + def customise_script(args, debug=False): script = args.script queue = args.queue @@ -120,16 +152,25 @@ def swap(xx): print(xx) - opath = root + '/custom/' + os.path.basename(script).split('.')[0] - + opath = root + '/custom/' + os.path.basename(script).split('.')[0] + + shebang = subprocess.check_output(['which', 'bash']) + shebang = shebang.rstrip().decode('utf-8') + + hostname = subprocess.check_output(['hostname']) + hostname = hostname.rstrip().decode('utf-8') + hostname = '# Customised for {} system'.format(hostname) + with open(opath, 'w') as f: rest.remove('#!/bin/bash') to_write = custom + rest - f.write('#!/bin/bash') - f.write('\n') - + f.write(f'#!{shebang}') + f.write('\n#\n') + f.write(f'{hostname}') + f.write('\n\n') + for line in custom + rest: f.write(line) f.write('\n') @@ -140,7 +181,6 @@ def swap(xx): if __name__ == '__main__': - # /cosma/home/durham/dc-wils7/DESI/bin/gold_pipeline parser = argparse.ArgumentParser(description='Customise pipeline submission scripts.') parser.add_argument('-s', '--script', help='Script to customise.', type=str, default=None) parser.add_argument('-q', '--queue', help='Queue for submission.', type=str, default=None) @@ -153,6 +193,8 @@ def swap(xx): args = parser.parse_args() script = args.script - customise_script(args) + # customise_script(args) + customise_serialorparallel() + print('\n\nDone.\n\n')