laurence
5286a2e81b
The scripts that processed the NGC 5128 session of 2026-07-21 previously lived inside the data directory and addressed it with absolute paths. Code and data are now separated: the code lives here, and a session is located at runtime through the ASTRO_SESSION environment variable. layout.py is what makes that work. It maps a FILENAME to the subdirectory that file belongs in, using the same rules the session directories are organised with, so a script can go on asking for 'master-Red.fit' or '_stars.npz' without any call site knowing the directory structure. Anything unrecognised resolves to the session root, which is visible and correctable rather than silently wrong. restructure.py reorganises a flat session directory into that layout. It is idempotent and dry-run by default. The 50 session scripts are kept as they were run rather than tidied into a library. They were written in sequence as the work went along, several of them by parallel agents, and they show it - but they are the honest provenance of a published set of results, and the productionised pipeline should be able to reproduce those results exactly. Verified before committing: all 51 files compile without warnings, and verify_core.py, closeup.py and triptych.py were run end to end against the reorganised session, correctly finding inputs across calibrated/, stacks/masters/ and final/ and writing outputs back to the right places.
229 lines
11 KiB
Python
229 lines
11 KiB
Python
"""
|
|
gc-classify.py -- step 2. Turn raw detections into a globular cluster candidate
|
|
catalogue.
|
|
|
|
Order of operations:
|
|
1. quality cuts (SNR, sep flags, frame edge)
|
|
2. photometric calibration of the R/G/B masters onto the Gaia BP/G/RP scale
|
|
using astrometrically-confirmed foreground stars
|
|
3. foreground star rejection using Gaia DR3 ASTROMETRY (not mere presence in
|
|
Gaia -- see the notes; half of the known Cen A clusters are in Gaia)
|
|
4. morphology: point-like vs extended, calibrated on the stellar locus
|
|
5. magnitude window around the expected GC luminosity function
|
|
6. cross-match to SIMBAD (truth set) and SCABS/Taylor+2017
|
|
|
|
Outputs: _gc_cat.npz (everything), gc-candidates.csv (the candidates)
|
|
"""
|
|
import os, sys, numpy as np
|
|
from astropy.coordinates import SkyCoord
|
|
import astropy.units as u
|
|
import warnings
|
|
|
|
import layout
|
|
warnings.filterwarnings("ignore")
|
|
|
|
S = layout.SESSION
|
|
PIXSCALE = 0.5376
|
|
NUC_RA, NUC_DEC = 201.365063, -43.019113
|
|
DIST_MPC = 3.8
|
|
KPC_PER_ARCMIN = DIST_MPC * 1000.0 * (np.pi / 180.0) / 60.0 # 1.105
|
|
MATCH_AS = 1.5 # astrometric match radius
|
|
PM_SIG = 4.0 # significance above which astrometry says "foreground"
|
|
# Faint limit set by the artificial-star tests, not by the nominal SNR-5 depth:
|
|
# recovery of injected point sources collapses from ~30% at G=19.75 to ~3% at
|
|
# G=20.25, and the fraction of candidates confirmed by published catalogues
|
|
# falls from 52% to 10% across the same step. Beyond G=20 we are not measuring
|
|
# clusters, we are measuring noise.
|
|
MAG_LO, MAG_HI = 17.5, 20.0
|
|
|
|
|
|
def xmatch(c1, c2, tol_as):
|
|
i, d2, _ = c1.match_to_catalog_sky(c2)
|
|
ok = d2.arcsec < tol_as
|
|
return i, d2.arcsec, ok
|
|
|
|
|
|
def main():
|
|
d = dict(np.load(layout.path("_gc_raw.npz")))
|
|
n = len(d["x"])
|
|
print("raw detections: %d" % n)
|
|
cat = SkyCoord(d["ra"], d["dec"], unit="deg")
|
|
nuc = SkyCoord(NUC_RA, NUC_DEC, unit="deg")
|
|
r_arcmin = cat.separation(nuc).arcmin
|
|
r_kpc = r_arcmin * KPC_PER_ARCMIN
|
|
|
|
# ---------------------------------------------------------- 1. quality
|
|
ny, nx = 3194, 4788
|
|
edge = 30
|
|
q_edge = (d["x"] > edge) & (d["x"] < nx - edge) & (d["y"] > edge) & (d["y"] < ny - edge)
|
|
# sep flag bits: 1=OBJ_MERGED 2=OBJ_TRUNC 4=OBJ_DOVERFLOW 8=OBJ_SINGU
|
|
# Bit 1 (had a neighbour in the detection footprint before deblending) is
|
|
# NOT a defect -- it fires for a third of the KNOWN clusters, because the
|
|
# inner field is crowded. Reject only genuine defects.
|
|
q_flag = (d["sepflag"].astype(int) & 0b1110) == 0
|
|
q_snr = d["snr"] >= 5.0
|
|
q_pos = d["flux"] > 0
|
|
good = q_edge & q_pos
|
|
print(" in-frame & positive flux: %d ; +clean sep flags: %d ; +SNR>=5: %d"
|
|
% (good.sum(), (good & q_flag).sum(), (good & q_flag & q_snr).sum()))
|
|
|
|
# ---------------------------------------------------------- 2. Gaia match
|
|
ga = np.load(layout.path("_gaia_astrom.npz"))
|
|
gcat = SkyCoord(ga["ra"], ga["dec"], unit="deg")
|
|
gi, gsep, ghit = xmatch(cat, gcat, MATCH_AS)
|
|
print(" Gaia DR3 counterpart within %.1f\": %d / %d" % (MATCH_AS, ghit.sum(), n))
|
|
|
|
plx, eplx = ga["plx"][gi], ga["eplx"][gi]
|
|
pmr, epmr = ga["pmra"][gi], ga["epmra"][gi]
|
|
pmd, epmd = ga["pmdec"][gi], ga["epmdec"][gi]
|
|
with np.errstate(invalid="ignore", divide="ignore"):
|
|
sig_plx = np.abs(plx / eplx)
|
|
sig_pm = np.sqrt((pmr / epmr) ** 2 + (pmd / epmd) ** 2)
|
|
has_astrom = ghit & np.isfinite(sig_pm)
|
|
# a Milky Way star: significant parallax OR significant proper motion
|
|
is_star = has_astrom & ((sig_plx > PM_SIG) | (sig_pm > PM_SIG))
|
|
# a Gaia source whose astrometry is consistent with zero -> distant
|
|
is_stationary = has_astrom & ~is_star
|
|
no_gaia = ~ghit
|
|
no_astrom = ghit & ~has_astrom # in Gaia but 2-parameter solution only
|
|
print(" -> astrometric foreground stars: %d" % is_star.sum())
|
|
print(" -> Gaia sources astrometrically stationary: %d" % is_stationary.sum())
|
|
print(" -> in Gaia, no astrometry: %d ; not in Gaia: %d" % (no_astrom.sum(), no_gaia.sum()))
|
|
|
|
gaia_g = np.where(ghit, ga["g"][gi], np.nan)
|
|
gaia_bprp = np.where(ghit, ga["bprp"][gi], np.nan)
|
|
|
|
# ------------------------------------------- 3. calibrate R/G/B onto Gaia
|
|
zp = {}
|
|
for key, gband in (("B", "bp"), ("V", "g"), ("R", "rp")):
|
|
f = d["flux_" + key]
|
|
ref = np.where(ghit, ga[gband][gi], np.nan)
|
|
m = is_star & good & q_flag & (f > 0) & np.isfinite(ref) & (ref > 13) & (ref < 18)
|
|
with np.errstate(invalid="ignore", divide="ignore"):
|
|
off = ref[m] + 2.5 * np.log10(f[m])
|
|
zp[key] = np.median(off)
|
|
print(" ZP_%s = %.3f (N=%d stars, scatter %.3f mag)"
|
|
% (key, zp[key], m.sum(), np.std(off - np.median(off))))
|
|
mags = {}
|
|
magerrs = {}
|
|
for key in ("B", "V", "R"):
|
|
f = np.clip(d["flux_" + key], 1e-9, None)
|
|
with np.errstate(invalid="ignore", divide="ignore"):
|
|
mags[key] = np.where(d["flux_" + key] > 0, -2.5 * np.log10(f) + zp[key], np.nan)
|
|
magerrs[key] = 1.0857 * d["fluxerr_" + key] / f
|
|
BmR = mags["B"] - mags["R"]
|
|
BmV = mags["B"] - mags["V"]
|
|
VmR = mags["V"] - mags["R"]
|
|
|
|
# ---------------------------------------------------------- 4. morphology
|
|
fwhm = 2.0 * d["rhalf"] # Gaussian: r_half = FWHM/2
|
|
with np.errstate(invalid="ignore", divide="ignore"):
|
|
elong = d["a"] / np.clip(d["b"], 1e-6, None)
|
|
# stellar locus from bright unsaturated confirmed stars
|
|
ref = is_star & good & q_flag & (d["mag"] > 15) & (d["mag"] < 18)
|
|
f_med = np.median(fwhm[ref]); f_sig = 1.4826 * np.median(np.abs(fwhm[ref] - f_med))
|
|
print(" stellar locus: FWHM = %.2f +/- %.2f px (%.2f\") from %d stars"
|
|
% (f_med, f_sig, f_med * PIXSCALE, ref.sum()))
|
|
# Asymmetric band, on purpose. A Cen A cluster (half-light radius a few pc,
|
|
# i.e. <0.2") is only marginally broadened by 2.7" seeing, but crowding and
|
|
# residual galaxy structure push measured sizes UP, never down. Known
|
|
# clusters run to FWHM ~6.6 px while stars sit at 5.0. So the lower edge is
|
|
# tight (rejects cosmic rays / noise spikes) and the upper edge is loose
|
|
# (rejects only obviously extended objects, i.e. resolved background
|
|
# galaxies). With 2.7" seeing this separation is weak -- see the notes.
|
|
FW_MIN = f_med - 3.0 * f_sig
|
|
FW_MAX = f_med + 3.0 # px, ~1.6 x the stellar FWHM
|
|
compact = (fwhm > FW_MIN) & (fwhm < FW_MAX) & (elong < 2.0)
|
|
print(" point-like acceptance band: %.2f < FWHM < %.2f px ; elong < 2.0"
|
|
% (FW_MIN, FW_MAX))
|
|
|
|
# ---------------------------------------------------------- 5. magnitude
|
|
inmag = (d["mag"] > MAG_LO) & (d["mag"] < MAG_HI)
|
|
|
|
# ---------------------------------------------------------- selection
|
|
base = good & q_flag & q_snr
|
|
cand = base & ~is_star & compact & inmag
|
|
print("\nCANDIDATES: %d" % cand.sum())
|
|
|
|
why = np.array(["-"] * n, dtype=object)
|
|
why[cand & no_gaia] = "no-gaia+compact"
|
|
why[cand & is_stationary] = "gaia-stationary+compact"
|
|
why[cand & no_astrom] = "gaia-noastrom+compact"
|
|
|
|
cls = np.array(["other"] * n, dtype=object)
|
|
cls[base & is_star] = "foreground-star"
|
|
cls[base & ~is_star & ~compact & inmag] = "extended"
|
|
cls[cand] = "gc-candidate"
|
|
|
|
# ---------------------------------------------------------- 6. truth sets
|
|
z = np.load(layout.path("_simbad.npz"), allow_pickle=True)
|
|
sim_ot = z["otype"].astype(str)
|
|
scat = SkyCoord(z["ra"].astype(float), z["dec"].astype(float), unit="deg")
|
|
si, ssep, shit = xmatch(cat, scat, 2.0)
|
|
simbad_type = np.where(shit, sim_ot[si], "")
|
|
simbad_name = np.where(shit, z["name"].astype(str)[si], "")
|
|
|
|
sc_type = np.array([""] * n, dtype=object)
|
|
scabs_ok = os.path.exists(layout.path("_cena_gc_ref.npz"))
|
|
scabs_prob = np.full(n, np.nan)
|
|
scabs_v = np.full(n, np.nan)
|
|
scabs_hit = np.zeros(n, bool) # positional match, regardless of listed prob
|
|
if scabs_ok:
|
|
r = np.load(layout.path("_cena_gc_ref.npz"))
|
|
rc = SkyCoord(r["ra"], r["dec"], unit="deg")
|
|
ri, rsep, rhit = xmatch(cat, rc, 2.0)
|
|
scabs_prob = np.where(rhit, r["prob"][ri], np.nan)
|
|
# the published table uses -1 as a "no value" sentinel
|
|
scabs_prob = np.where(scabs_prob < 0, np.nan, scabs_prob)
|
|
scabs_v = np.where(rhit, r["vmag"][ri], np.nan)
|
|
scabs_hit = rhit
|
|
print(" SCABS (Taylor+2017) matches among all detections: %d" % rhit.sum())
|
|
print(" SCABS matches among candidates: %d / %d (%.0f%%)"
|
|
% ((rhit & cand).sum(), cand.sum(), 100 * (rhit & cand).mean() / max(cand.mean(), 1e-9)))
|
|
print(" SIMBAD GlC matches among candidates: %d"
|
|
% ((simbad_type == "GlC") & cand).sum())
|
|
|
|
out = dict(d)
|
|
out.update(r_arcmin=r_arcmin, r_kpc=r_kpc, fwhm=fwhm, elong=elong,
|
|
mag_B=mags["B"], mag_V=mags["V"], mag_R=mags["R"],
|
|
magerr_B=magerrs["B"], magerr_V=magerrs["V"], magerr_R=magerrs["R"],
|
|
BmR=BmR, BmV=BmV, VmR=VmR,
|
|
gaia_g=gaia_g, gaia_bprp=gaia_bprp, gaia_sep=gsep,
|
|
sig_plx=np.where(ghit, sig_plx, np.nan),
|
|
sig_pm=np.where(ghit, sig_pm, np.nan),
|
|
is_star=is_star, is_stationary=is_stationary, no_gaia=no_gaia,
|
|
no_astrom=no_astrom, compact=compact, cand=cand, base=base,
|
|
good=good, q_flag=q_flag, q_snr=q_snr, inmag=inmag,
|
|
simbad_type=simbad_type.astype(str), simbad_name=simbad_name.astype(str),
|
|
scabs_prob=scabs_prob, scabs_v=scabs_v, scabs_hit=scabs_hit,
|
|
why=why.astype(str), cls=cls.astype(str),
|
|
FW_MIN=FW_MIN, FW_MAX=FW_MAX, f_med=f_med, f_sig=f_sig,
|
|
zpB=zp["B"], zpV=zp["V"], zpR=zp["R"])
|
|
np.savez(layout.path("_gc_cat.npz"), **out)
|
|
|
|
# ---------------------------------------------------------- CSV
|
|
idx = np.where(cand)[0]
|
|
order = idx[np.argsort(r_arcmin[idx])]
|
|
lines = ["id,ra_deg,dec_deg,x_px,y_px,G_mag,G_magerr,snr,mag_B,mag_V,mag_R,"
|
|
"B_R,B_V,V_R,fwhm_px,fwhm_arcsec,elong,r_arcmin,r_kpc,keep_flag,"
|
|
"simbad_type,simbad_name,scabs_prob,scabs_V"]
|
|
for k, i in enumerate(order, 1):
|
|
def f(v, p=3):
|
|
return "" if not np.isfinite(v) else ("%.*f" % (p, v))
|
|
lines.append(",".join([
|
|
"GCC%04d" % k, "%.6f" % d["ra"][i], "%.6f" % d["dec"][i],
|
|
"%.2f" % d["x"][i], "%.2f" % d["y"][i],
|
|
f(d["mag"][i]), f(d["magerr"][i]), "%.1f" % d["snr"][i],
|
|
f(mags["B"][i]), f(mags["V"][i]), f(mags["R"][i]),
|
|
f(BmR[i]), f(BmV[i]), f(VmR[i]),
|
|
"%.2f" % fwhm[i], "%.2f" % (fwhm[i] * PIXSCALE), "%.2f" % elong[i],
|
|
"%.3f" % r_arcmin[i], "%.3f" % r_kpc[i], why[i],
|
|
simbad_type[i], '"%s"' % simbad_name[i],
|
|
f(scabs_prob[i], 2), f(scabs_v[i], 2)]))
|
|
with open(layout.path("gc-candidates.csv"), "w") as fh:
|
|
fh.write("\n".join(lines) + "\n")
|
|
print("wrote gc-candidates.csv (%d rows)" % len(order))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|