Processing and analysis code for remote-telescope imaging sessions
The scripts that processed the NGC 5128 session of 2026-07-21 previously lived inside the data directory and addressed it with absolute paths. Code and data are now separated: the code lives here, and a session is located at runtime through the ASTRO_SESSION environment variable. layout.py is what makes that work. It maps a FILENAME to the subdirectory that file belongs in, using the same rules the session directories are organised with, so a script can go on asking for 'master-Red.fit' or '_stars.npz' without any call site knowing the directory structure. Anything unrecognised resolves to the session root, which is visible and correctable rather than silently wrong. restructure.py reorganises a flat session directory into that layout. It is idempotent and dry-run by default. The 50 session scripts are kept as they were run rather than tidied into a library. They were written in sequence as the work went along, several of them by parallel agents, and they show it - but they are the honest provenance of a published set of results, and the productionised pipeline should be able to reproduce those results exactly. Verified before committing: all 51 files compile without warnings, and verify_core.py, closeup.py and triptych.py were run end to end against the reorganised session, correctly finding inputs across calibrated/, stacks/masters/ and final/ and writing outputs back to the right places.
This commit is contained in:
commit
5286a2e81b
53 changed files with 8820 additions and 0 deletions
321
session-scripts/mo_common.py
Normal file
321
session-scripts/mo_common.py
Normal file
|
|
@ -0,0 +1,321 @@
|
|||
"""Shared machinery for the moving-object and transient searches.
|
||||
|
||||
Holds the things both the real search and the synthetic-injection sensitivity
|
||||
run need to do identically: detection, coordinate registration, photometric
|
||||
calibration onto the master's zero point, static-sky rejection and tracklet
|
||||
linking. Keeping one copy guarantees the sensitivity numbers describe the
|
||||
pipeline that was actually run on the data, not a simplified stand-in.
|
||||
"""
|
||||
import itertools
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import sep
|
||||
from astropy.io import fits
|
||||
from scipy.spatial import cKDTree
|
||||
from skimage.transform import AffineTransform
|
||||
|
||||
import layout
|
||||
|
||||
SRC = layout.SESSION
|
||||
OUT = layout.SESSION
|
||||
DETS = layout.path("_mo_dets.npz")
|
||||
REFERENCE = "Luminance_002"
|
||||
|
||||
ZP = 27.941 # master luminance, G = -2.5 log10(flux_5px) + ZP
|
||||
SCALE = 0.5376 # arcsec/px
|
||||
FWHM = 5.0 # px
|
||||
SIGMA = FWHM / 2.3548
|
||||
APRAD = 5.0 # px, the radius the zero point is defined for
|
||||
# Fraction of a Gaussian's total flux inside APRAD, used to convert a target
|
||||
# aperture magnitude into the total flux of an injected source.
|
||||
APFRAC = 1.0 - np.exp(-APRAD ** 2 / (2.0 * SIGMA ** 2))
|
||||
|
||||
# --- tracklet linking parameters ---------------------------------------
|
||||
MIN_RATE = 2.5 # px/hr
|
||||
MAX_RATE = 900.0 # px/hr
|
||||
MIN_HITS = 5
|
||||
LINE_RMS = 1.2 # px
|
||||
MAG_SCATTER = 0.5 # mag
|
||||
TOL = 3.0 # px
|
||||
PAIR_GAP = 3
|
||||
STATIC_MIN = 4 # recurrences at one place that mark a source static
|
||||
STATIC_TOL = 2.0 # px
|
||||
MIN_NATIVE_MOTION = 4.0 # px a real mover must travel in DETECTOR coordinates
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# detection
|
||||
|
||||
def detect(image, thresh=3.5, minarea=5):
|
||||
"""sep background estimate, extraction, and 5 px aperture photometry."""
|
||||
bkg = sep.Background(image, bw=64, bh=64, fw=3, fh=3)
|
||||
sub = image - bkg.back()
|
||||
objs = sep.extract(sub, thresh, err=bkg.globalrms, minarea=minarea,
|
||||
deblend_cont=0.005)
|
||||
if len(objs):
|
||||
ap, _, _ = sep.sum_circle(sub, objs["x"], objs["y"], APRAD,
|
||||
err=bkg.globalrms, gain=1.0)
|
||||
else:
|
||||
ap = np.zeros(0)
|
||||
return objs, np.asarray(ap, dtype=float), float(bkg.globalrms)
|
||||
|
||||
|
||||
def lum_frames():
|
||||
"""(key, path) for the twelve luminance subs, in time order."""
|
||||
rows = []
|
||||
n = 0
|
||||
for fn in sorted(os.listdir(SRC)):
|
||||
if fn.startswith("calibrated-") and fn.endswith(".fit") \
|
||||
and "-Luminance-" in fn:
|
||||
n += 1
|
||||
rows.append((f"Luminance_{n:03d}", layout.path(fn)))
|
||||
return rows
|
||||
|
||||
|
||||
def lum_times():
|
||||
"""Mid-exposure times of the luminance subs, in hours from the first."""
|
||||
t = []
|
||||
for _, path in lum_frames():
|
||||
jd = float(fits.getheader(path)["JD"])
|
||||
t.append(jd - 2400000.5 + 150.0 / 86400.0)
|
||||
t = np.array(t)
|
||||
return (t - t[0]) * 24.0
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# registration
|
||||
|
||||
def load_dets():
|
||||
z = np.load(DETS, allow_pickle=True)
|
||||
meta = {r[0]: r for r in z["meta"]}
|
||||
return meta, {k: z[k] for k in meta}
|
||||
|
||||
|
||||
def frame_transforms():
|
||||
"""Native-pixel -> reference-frame affine for each luminance sub.
|
||||
|
||||
Seeded by the astroalign solution cached in _mo_dets.npz, then refined by
|
||||
nearest-neighbour matching every detection against the reference frame's
|
||||
detections. The refined solutions are good to ~0.15 px median across the
|
||||
whole field (see mo_check.py), which is what makes a 3 px association
|
||||
radius meaningful.
|
||||
"""
|
||||
_, dets = load_dets()
|
||||
ref = dets[REFERENCE]
|
||||
tree = cKDTree(ref[:, :2])
|
||||
tf_out = {}
|
||||
for key, d in dets.items():
|
||||
if not key.startswith("Luminance"):
|
||||
continue
|
||||
src = d[:, 2:4].astype(float)
|
||||
cur = d[:, :2].astype(float)
|
||||
tf = None
|
||||
for t in (3.0, 1.5):
|
||||
dist, idx = tree.query(cur, distance_upper_bound=t)
|
||||
ok = np.isfinite(dist)
|
||||
if ok.sum() < 50:
|
||||
break
|
||||
tf = AffineTransform()
|
||||
tf.estimate(src[ok], ref[idx[ok], :2].astype(float))
|
||||
cur = tf(src)
|
||||
if tf is None: # the reference frame itself
|
||||
tf = AffineTransform()
|
||||
tf.estimate(src, d[:, :2].astype(float))
|
||||
tf_out[key] = tf
|
||||
return tf_out
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# photometry
|
||||
|
||||
def master_sources():
|
||||
"""Detections and 5 px aperture fluxes in the deep luminance master."""
|
||||
with fits.open(layout.path("master-Luminance.fit")) as hd:
|
||||
img = hd[0].data.astype(np.float32)
|
||||
hdr = hd[0].header
|
||||
objs, ap, rms = detect(img, thresh=2.5, minarea=4)
|
||||
return objs, ap, rms, hdr, img
|
||||
|
||||
|
||||
def frame_zeropoint(fx, fy, fap, mobjs, map_, tf):
|
||||
"""Per-frame zero point, tied to the master's, from stars in common.
|
||||
|
||||
The master is a scaled sigma-clipped mean of the twelve subs, so a single
|
||||
sub's counts differ from it by a near-constant factor. Measuring that
|
||||
factor on a few hundred stars puts single-frame magnitudes on the master's
|
||||
system without a separate absolute calibration.
|
||||
"""
|
||||
ref_xy = tf(np.column_stack([fx, fy]))
|
||||
mt = cKDTree(np.column_stack([mobjs["x"], mobjs["y"]]))
|
||||
d, idx = mt.query(ref_xy, distance_upper_bound=1.5)
|
||||
ok = np.isfinite(d) & (fap > 0)
|
||||
if ok.sum() < 30:
|
||||
return ZP, 0
|
||||
fm = map_[idx[ok]]
|
||||
ff = fap[ok]
|
||||
good = (fm > 0) & (ff > 0)
|
||||
# Bright half only: faint stars are noise-biased in both frames.
|
||||
order = np.argsort(fm[good])[::-1]
|
||||
sel = order[:max(50, len(order) // 4)]
|
||||
ratio = np.median((ff[good][sel] / fm[good][sel]))
|
||||
return ZP + 2.5 * np.log10(ratio), int(ok.sum())
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# static-sky rejection and linking
|
||||
|
||||
def residual_mask(pts_list, nat_list, mtree, master_tol=4.0):
|
||||
"""True for detections that are neither static sky nor detector defects.
|
||||
|
||||
Three rejections, and the third is the one that matters most on this data:
|
||||
|
||||
1. Coincident with a source in the deep luminance master.
|
||||
2. Recurring at the same REFERENCE-frame position in four or more subs -
|
||||
a star the master missed.
|
||||
3. Recurring at the same DETECTOR-frame position in four or more subs.
|
||||
|
||||
Cut 3 exists because the frames are dithered and the field rotates slowly
|
||||
through the session. Registration undoes that motion for the sky, which
|
||||
means it imposes exactly the opposite motion on anything fixed to the
|
||||
detector. A hot pixel therefore traces a perfectly straight, perfectly
|
||||
constant-rate, perfectly constant-brightness track across the registered
|
||||
sequence - an ideal fake minor planet that passes every cut a linker
|
||||
normally applies. Without this cut the search on these twelve subs returns
|
||||
141 such tracks, all of them moving at 9-18 arcsec/hr toward position
|
||||
angle 45-85 deg, all with npix of 5-10 (a 3x3 blob, against 50-100 for a
|
||||
real star at this seeing), and all stationary to under 0.5 px in detector
|
||||
coordinates. With it, none survive.
|
||||
"""
|
||||
trees = [cKDTree(p) if len(p) else None for p in pts_list]
|
||||
ntrees = [cKDTree(p) if len(p) else None for p in nat_list]
|
||||
out = []
|
||||
for i, p in enumerate(pts_list):
|
||||
if not len(p):
|
||||
out.append(np.zeros(0, dtype=bool))
|
||||
continue
|
||||
d, _ = mtree.query(p, distance_upper_bound=master_tol)
|
||||
in_master = np.isfinite(d)
|
||||
recur = np.zeros(len(p), dtype=int)
|
||||
nrecur = np.zeros(len(p), dtype=int)
|
||||
for j, (tr, ntr) in enumerate(zip(trees, ntrees)):
|
||||
if j == i or tr is None:
|
||||
continue
|
||||
dd, _ = tr.query(p, distance_upper_bound=STATIC_TOL)
|
||||
recur += np.isfinite(dd)
|
||||
nd, _ = ntr.query(nat_list[i], distance_upper_bound=STATIC_TOL)
|
||||
nrecur += np.isfinite(nd)
|
||||
out.append((~in_master) & (recur < STATIC_MIN) &
|
||||
(nrecur < STATIC_MIN))
|
||||
return out
|
||||
|
||||
|
||||
def fit_track(t, x, y):
|
||||
A = np.column_stack([np.ones_like(t), t])
|
||||
cx, *_ = np.linalg.lstsq(A, x, rcond=None)
|
||||
cy, *_ = np.linalg.lstsq(A, y, rcond=None)
|
||||
rms = float(np.sqrt(np.mean((x - A @ cx) ** 2 + (y - A @ cy) ** 2)))
|
||||
return cx, cy, rms
|
||||
|
||||
|
||||
def link(times, pts, mags, nat=None, shape=None, min_hits=MIN_HITS):
|
||||
"""Pair-seeded, propagate-and-count tracklet finder. See mo_link.py."""
|
||||
n = len(times)
|
||||
trees = [cKDTree(p) if len(p) else None for p in pts]
|
||||
seen, cands = set(), []
|
||||
for i, j in itertools.combinations(range(n), 2):
|
||||
if j - i < PAIR_GAP or trees[i] is None or trees[j] is None:
|
||||
continue
|
||||
dt = times[j] - times[i]
|
||||
if dt <= 0:
|
||||
continue
|
||||
near = trees[j].query_ball_point(pts[i], MAX_RATE * dt)
|
||||
for a, blist in enumerate(near):
|
||||
for b in blist:
|
||||
if np.hypot(*(pts[j][b] - pts[i][a])) < MIN_RATE * dt:
|
||||
continue
|
||||
v = (pts[j][b] - pts[i][a]) / dt
|
||||
hits = []
|
||||
for f in range(n):
|
||||
if trees[f] is None:
|
||||
continue
|
||||
pred = pts[i][a] + v * (times[f] - times[i])
|
||||
dd, idx = trees[f].query(pred, distance_upper_bound=TOL)
|
||||
if np.isfinite(dd):
|
||||
hits.append((f, int(idx)))
|
||||
if len(hits) < min_hits:
|
||||
continue
|
||||
sig = tuple(sorted(hits))
|
||||
if sig in seen:
|
||||
continue
|
||||
seen.add(sig)
|
||||
tt = np.array([times[f] for f, _ in hits])
|
||||
xx = np.array([pts[f][k][0] for f, k in hits])
|
||||
yy = np.array([pts[f][k][1] for f, k in hits])
|
||||
cx, cy, rms = fit_track(tt, xx, yy)
|
||||
if rms > LINE_RMS:
|
||||
continue
|
||||
mm = np.array([mags[f][k] for f, k in hits])
|
||||
if not np.all(np.isfinite(mm)) or mm.std() > MAG_SCATTER:
|
||||
continue
|
||||
if nat is not None:
|
||||
# Backstop for cut 3 in residual_mask: whatever this is,
|
||||
# it must have genuinely moved across the detector, not
|
||||
# merely been carried by the registration.
|
||||
nx_ = np.array([nat[f][k][0] for f, k in hits])
|
||||
ny_ = np.array([nat[f][k][1] for f, k in hits])
|
||||
if np.hypot(np.ptp(nx_), np.ptp(ny_)) < MIN_NATIVE_MOTION:
|
||||
continue
|
||||
c = dict(frames=np.array([f for f, _ in hits]),
|
||||
idx=np.array([k for _, k in hits]),
|
||||
t=tt, x=xx, y=yy, cx=cx, cy=cy, rms=rms,
|
||||
rate=float(np.hypot(cx[1], cy[1])),
|
||||
ang=float(np.degrees(np.arctan2(cy[1], cx[1]))),
|
||||
mag=float(mm.mean()), magsig=float(mm.std()),
|
||||
nhit=len(hits))
|
||||
if shape is not None:
|
||||
sh = np.array([shape[f][k] for f, k in hits])
|
||||
c.update(a=float(np.median(sh[:, 0])),
|
||||
b=float(np.median(sh[:, 1])),
|
||||
npix=float(np.median(sh[:, 2])))
|
||||
cands.append(c)
|
||||
return dedup(cands)
|
||||
|
||||
|
||||
def dedup(cands):
|
||||
"""Among tracklets sharing two or more detections, keep the best."""
|
||||
cands.sort(key=lambda c: (-c["nhit"], c["rms"]))
|
||||
kept, sets = [], []
|
||||
for c in cands:
|
||||
s = set(zip(c["frames"].tolist(), c["idx"].tolist()))
|
||||
if any(len(s & o) >= 2 for o in sets):
|
||||
continue
|
||||
sets.append(s)
|
||||
kept.append(c)
|
||||
return kept
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# synthetic sources
|
||||
|
||||
def add_source(image, x, y, total_flux, dx, dy, nstep=9, sigma=SIGMA):
|
||||
"""Add a trailed Gaussian: the source moves (dx, dy) px during the sub.
|
||||
|
||||
A 300 s exposure of something moving at 30 arcsec/hr smears by ~4.6 px,
|
||||
comparable to the seeing disc, so the trail is modelled rather than
|
||||
ignored - it is what limits sensitivity at high rates.
|
||||
"""
|
||||
ny, nx = image.shape
|
||||
half = int(np.ceil(4 * sigma + 0.5 * np.hypot(dx, dy))) + 2
|
||||
x0, x1 = int(max(0, x - half)), int(min(nx, x + half + 1))
|
||||
y0, y1 = int(max(0, y - half)), int(min(ny, y + half + 1))
|
||||
if x1 <= x0 or y1 <= y0:
|
||||
return
|
||||
gy, gx = np.mgrid[y0:y1, x0:x1]
|
||||
acc = np.zeros(gx.shape, dtype=np.float64)
|
||||
for s in np.linspace(-0.5, 0.5, nstep):
|
||||
cx, cy = x + s * dx, y + s * dy
|
||||
acc += np.exp(-((gx - cx) ** 2 + (gy - cy) ** 2) /
|
||||
(2.0 * sigma ** 2))
|
||||
acc *= total_flux / (nstep * 2.0 * np.pi * sigma ** 2)
|
||||
image[y0:y1, x0:x1] += acc.astype(image.dtype)
|
||||
Loading…
Add table
Add a link
Reference in a new issue