laurence/astrophotography
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Processing and analysis code for remote-telescope imaging sessions

Browse source 
The scripts that processed the NGC 5128 session of 2026-07-21 previously
lived inside the data directory and addressed it with absolute paths.
Code and data are now separated: the code lives here, and a session is
located at runtime through the ASTRO_SESSION environment variable.

layout.py is what makes that work. It maps a FILENAME to the
subdirectory that file belongs in, using the same rules the session
directories are organised with, so a script can go on asking for
'master-Red.fit' or '_stars.npz' without any call site knowing the
directory structure. Anything unrecognised resolves to the session root,
which is visible and correctable rather than silently wrong.

restructure.py reorganises a flat session directory into that layout. It
is idempotent and dry-run by default.

The 50 session scripts are kept as they were run rather than tidied into
a library. They were written in sequence as the work went along, several
of them by parallel agents, and they show it - but they are the honest
provenance of a published set of results, and the productionised pipeline
should be able to reproduce those results exactly.

Verified before committing: all 51 files compile without warnings, and
verify_core.py, closeup.py and triptych.py were run end to end against
the reorganised session, correctly finding inputs across calibrated/,
stacks/masters/ and final/ and writing outputs back to the right places.
This commit is contained in:
laurence 2026-07-21 15:29:49 +01:00
commit 5286a2e81b
53 changed files with 8820 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

229
session-scripts/gc-classify.py Normal file
View file

@ -0,0 +1,229 @@
"""
gc-classify.py -- step 2. Turn raw detections into a globular cluster candidate
catalogue.
Order of operations:
1. quality cuts (SNR, sep flags, frame edge)
2. photometric calibration of the R/G/B masters onto the Gaia BP/G/RP scale
using astrometrically-confirmed foreground stars
3. foreground star rejection using Gaia DR3 ASTROMETRY (not mere presence in
Gaia -- see the notes; half of the known Cen A clusters are in Gaia)
4. morphology: point-like vs extended, calibrated on the stellar locus
5. magnitude window around the expected GC luminosity function
6. cross-match to SIMBAD (truth set) and SCABS/Taylor+2017
Outputs: _gc_cat.npz (everything), gc-candidates.csv (the candidates)
"""
import os, sys, numpy as np
from astropy.coordinates import SkyCoord
import astropy.units as u
import warnings
import layout
warnings.filterwarnings("ignore")
S = layout.SESSION
PIXSCALE = 0.5376
NUC_RA, NUC_DEC = 201.365063, -43.019113
DIST_MPC = 3.8
KPC_PER_ARCMIN = DIST_MPC * 1000.0 * (np.pi / 180.0) / 60.0 # 1.105
MATCH_AS = 1.5 # astrometric match radius
PM_SIG = 4.0 # significance above which astrometry says "foreground"
# Faint limit set by the artificial-star tests, not by the nominal SNR-5 depth:
# recovery of injected point sources collapses from ~30% at G=19.75 to ~3% at
# G=20.25, and the fraction of candidates confirmed by published catalogues
# falls from 52% to 10% across the same step. Beyond G=20 we are not measuring
# clusters, we are measuring noise.
MAG_LO, MAG_HI = 17.5, 20.0
def xmatch(c1, c2, tol_as):
i, d2, _ = c1.match_to_catalog_sky(c2)
ok = d2.arcsec < tol_as
return i, d2.arcsec, ok
def main():
d = dict(np.load(layout.path("_gc_raw.npz")))
n = len(d["x"])
print("raw detections: %d" % n)
cat = SkyCoord(d["ra"], d["dec"], unit="deg")
nuc = SkyCoord(NUC_RA, NUC_DEC, unit="deg")
r_arcmin = cat.separation(nuc).arcmin
r_kpc = r_arcmin * KPC_PER_ARCMIN
# ---------------------------------------------------------- 1. quality
ny, nx = 3194, 4788
edge = 30
q_edge = (d["x"] > edge) & (d["x"] < nx - edge) & (d["y"] > edge) & (d["y"] < ny - edge)
# sep flag bits: 1=OBJ_MERGED 2=OBJ_TRUNC 4=OBJ_DOVERFLOW 8=OBJ_SINGU
# Bit 1 (had a neighbour in the detection footprint before deblending) is
# NOT a defect -- it fires for a third of the KNOWN clusters, because the
# inner field is crowded. Reject only genuine defects.
q_flag = (d["sepflag"].astype(int) & 0b1110) == 0
q_snr = d["snr"] >= 5.0
q_pos = d["flux"] > 0
good = q_edge & q_pos
print(" in-frame & positive flux: %d ; +clean sep flags: %d ; +SNR>=5: %d"
% (good.sum(), (good & q_flag).sum(), (good & q_flag & q_snr).sum()))
# ---------------------------------------------------------- 2. Gaia match
ga = np.load(layout.path("_gaia_astrom.npz"))
gcat = SkyCoord(ga["ra"], ga["dec"], unit="deg")
gi, gsep, ghit = xmatch(cat, gcat, MATCH_AS)
print(" Gaia DR3 counterpart within %.1f\": %d / %d" % (MATCH_AS, ghit.sum(), n))
plx, eplx = ga["plx"][gi], ga["eplx"][gi]
pmr, epmr = ga["pmra"][gi], ga["epmra"][gi]
pmd, epmd = ga["pmdec"][gi], ga["epmdec"][gi]
with np.errstate(invalid="ignore", divide="ignore"):
sig_plx = np.abs(plx / eplx)
sig_pm = np.sqrt((pmr / epmr) ** 2 + (pmd / epmd) ** 2)
has_astrom = ghit & np.isfinite(sig_pm)
# a Milky Way star: significant parallax OR significant proper motion
is_star = has_astrom & ((sig_plx > PM_SIG) | (sig_pm > PM_SIG))
# a Gaia source whose astrometry is consistent with zero -> distant
is_stationary = has_astrom & ~is_star
no_gaia = ~ghit
no_astrom = ghit & ~has_astrom # in Gaia but 2-parameter solution only
print(" -> astrometric foreground stars: %d" % is_star.sum())
print(" -> Gaia sources astrometrically stationary: %d" % is_stationary.sum())
print(" -> in Gaia, no astrometry: %d ; not in Gaia: %d" % (no_astrom.sum(), no_gaia.sum()))
gaia_g = np.where(ghit, ga["g"][gi], np.nan)
gaia_bprp = np.where(ghit, ga["bprp"][gi], np.nan)
# ------------------------------------------- 3. calibrate R/G/B onto Gaia
zp = {}
for key, gband in (("B", "bp"), ("V", "g"), ("R", "rp")):
f = d["flux_" + key]
ref = np.where(ghit, ga[gband][gi], np.nan)
m = is_star & good & q_flag & (f > 0) & np.isfinite(ref) & (ref > 13) & (ref < 18)
with np.errstate(invalid="ignore", divide="ignore"):
off = ref[m] + 2.5 * np.log10(f[m])
zp[key] = np.median(off)
print(" ZP_%s = %.3f (N=%d stars, scatter %.3f mag)"
% (key, zp[key], m.sum(), np.std(off - np.median(off))))
mags = {}
magerrs = {}
for key in ("B", "V", "R"):
f = np.clip(d["flux_" + key], 1e-9, None)
with np.errstate(invalid="ignore", divide="ignore"):
mags[key] = np.where(d["flux_" + key] > 0, -2.5 * np.log10(f) + zp[key], np.nan)
magerrs[key] = 1.0857 * d["fluxerr_" + key] / f
BmR = mags["B"] - mags["R"]
BmV = mags["B"] - mags["V"]
VmR = mags["V"] - mags["R"]
# ---------------------------------------------------------- 4. morphology
fwhm = 2.0 * d["rhalf"] # Gaussian: r_half = FWHM/2
with np.errstate(invalid="ignore", divide="ignore"):
elong = d["a"] / np.clip(d["b"], 1e-6, None)
# stellar locus from bright unsaturated confirmed stars
ref = is_star & good & q_flag & (d["mag"] > 15) & (d["mag"] < 18)
f_med = np.median(fwhm[ref]); f_sig = 1.4826 * np.median(np.abs(fwhm[ref] - f_med))
print(" stellar locus: FWHM = %.2f +/- %.2f px (%.2f\") from %d stars"
% (f_med, f_sig, f_med * PIXSCALE, ref.sum()))
# Asymmetric band, on purpose. A Cen A cluster (half-light radius a few pc,
# i.e. <0.2") is only marginally broadened by 2.7" seeing, but crowding and
# residual galaxy structure push measured sizes UP, never down. Known
# clusters run to FWHM ~6.6 px while stars sit at 5.0. So the lower edge is
# tight (rejects cosmic rays / noise spikes) and the upper edge is loose
# (rejects only obviously extended objects, i.e. resolved background
# galaxies). With 2.7" seeing this separation is weak -- see the notes.
FW_MIN = f_med - 3.0 * f_sig
FW_MAX = f_med + 3.0 # px, ~1.6 x the stellar FWHM
compact = (fwhm > FW_MIN) & (fwhm < FW_MAX) & (elong < 2.0)
print(" point-like acceptance band: %.2f < FWHM < %.2f px ; elong < 2.0"
% (FW_MIN, FW_MAX))
# ---------------------------------------------------------- 5. magnitude
inmag = (d["mag"] > MAG_LO) & (d["mag"] < MAG_HI)
# ---------------------------------------------------------- selection
base = good & q_flag & q_snr
cand = base & ~is_star & compact & inmag
print("\nCANDIDATES: %d" % cand.sum())
why = np.array(["-"] * n, dtype=object)
why[cand & no_gaia] = "no-gaia+compact"
why[cand & is_stationary] = "gaia-stationary+compact"
why[cand & no_astrom] = "gaia-noastrom+compact"
cls = np.array(["other"] * n, dtype=object)
cls[base & is_star] = "foreground-star"
cls[base & ~is_star & ~compact & inmag] = "extended"
cls[cand] = "gc-candidate"
# ---------------------------------------------------------- 6. truth sets
z = np.load(layout.path("_simbad.npz"), allow_pickle=True)
sim_ot = z["otype"].astype(str)
scat = SkyCoord(z["ra"].astype(float), z["dec"].astype(float), unit="deg")
si, ssep, shit = xmatch(cat, scat, 2.0)
simbad_type = np.where(shit, sim_ot[si], "")
simbad_name = np.where(shit, z["name"].astype(str)[si], "")
sc_type = np.array([""] * n, dtype=object)
scabs_ok = os.path.exists(layout.path("_cena_gc_ref.npz"))
scabs_prob = np.full(n, np.nan)
scabs_v = np.full(n, np.nan)
scabs_hit = np.zeros(n, bool) # positional match, regardless of listed prob
if scabs_ok:
r = np.load(layout.path("_cena_gc_ref.npz"))
rc = SkyCoord(r["ra"], r["dec"], unit="deg")
ri, rsep, rhit = xmatch(cat, rc, 2.0)
scabs_prob = np.where(rhit, r["prob"][ri], np.nan)
# the published table uses -1 as a "no value" sentinel
scabs_prob = np.where(scabs_prob < 0, np.nan, scabs_prob)
scabs_v = np.where(rhit, r["vmag"][ri], np.nan)
scabs_hit = rhit
print(" SCABS (Taylor+2017) matches among all detections: %d" % rhit.sum())
print(" SCABS matches among candidates: %d / %d (%.0f%%)"
% ((rhit & cand).sum(), cand.sum(), 100 * (rhit & cand).mean() / max(cand.mean(), 1e-9)))
print(" SIMBAD GlC matches among candidates: %d"
% ((simbad_type == "GlC") & cand).sum())
out = dict(d)
out.update(r_arcmin=r_arcmin, r_kpc=r_kpc, fwhm=fwhm, elong=elong,
mag_B=mags["B"], mag_V=mags["V"], mag_R=mags["R"],
magerr_B=magerrs["B"], magerr_V=magerrs["V"], magerr_R=magerrs["R"],
BmR=BmR, BmV=BmV, VmR=VmR,
gaia_g=gaia_g, gaia_bprp=gaia_bprp, gaia_sep=gsep,
sig_plx=np.where(ghit, sig_plx, np.nan),
sig_pm=np.where(ghit, sig_pm, np.nan),
is_star=is_star, is_stationary=is_stationary, no_gaia=no_gaia,
no_astrom=no_astrom, compact=compact, cand=cand, base=base,
good=good, q_flag=q_flag, q_snr=q_snr, inmag=inmag,
simbad_type=simbad_type.astype(str), simbad_name=simbad_name.astype(str),
scabs_prob=scabs_prob, scabs_v=scabs_v, scabs_hit=scabs_hit,
why=why.astype(str), cls=cls.astype(str),
FW_MIN=FW_MIN, FW_MAX=FW_MAX, f_med=f_med, f_sig=f_sig,
zpB=zp["B"], zpV=zp["V"], zpR=zp["R"])
np.savez(layout.path("_gc_cat.npz"), **out)
# ---------------------------------------------------------- CSV
idx = np.where(cand)[0]
order = idx[np.argsort(r_arcmin[idx])]
lines = ["id,ra_deg,dec_deg,x_px,y_px,G_mag,G_magerr,snr,mag_B,mag_V,mag_R,"
"B_R,B_V,V_R,fwhm_px,fwhm_arcsec,elong,r_arcmin,r_kpc,keep_flag,"
"simbad_type,simbad_name,scabs_prob,scabs_V"]
for k, i in enumerate(order, 1):
def f(v, p=3):
return "" if not np.isfinite(v) else ("%.*f" % (p, v))
lines.append(",".join([
"GCC%04d" % k, "%.6f" % d["ra"][i], "%.6f" % d["dec"][i],
"%.2f" % d["x"][i], "%.2f" % d["y"][i],
f(d["mag"][i]), f(d["magerr"][i]), "%.1f" % d["snr"][i],
f(mags["B"][i]), f(mags["V"][i]), f(mags["R"][i]),
f(BmR[i]), f(BmV[i]), f(VmR[i]),
"%.2f" % fwhm[i], "%.2f" % (fwhm[i] * PIXSCALE), "%.2f" % elong[i],
"%.3f" % r_arcmin[i], "%.3f" % r_kpc[i], why[i],
simbad_type[i], '"%s"' % simbad_name[i],
f(scabs_prob[i], 2), f(scabs_v[i], 2)]))
with open(layout.path("gc-candidates.csv"), "w") as fh:
fh.write("\n".join(lines) + "\n")
print("wrote gc-candidates.csv (%d rows)" % len(order))
if __name__ == "__main__":
main()