"""Shared machinery for the moving-object and transient searches. Holds the things both the real search and the synthetic-injection sensitivity run need to do identically: detection, coordinate registration, photometric calibration onto the master's zero point, static-sky rejection and tracklet linking. Keeping one copy guarantees the sensitivity numbers describe the pipeline that was actually run on the data, not a simplified stand-in. """ import itertools import os import numpy as np import sep from astropy.io import fits from scipy.spatial import cKDTree from skimage.transform import AffineTransform import layout SRC = layout.SESSION OUT = layout.SESSION DETS = layout.path("_mo_dets.npz") REFERENCE = "Luminance_002" ZP = 27.941 # master luminance, G = -2.5 log10(flux_5px) + ZP SCALE = 0.5376 # arcsec/px FWHM = 5.0 # px SIGMA = FWHM / 2.3548 APRAD = 5.0 # px, the radius the zero point is defined for # Fraction of a Gaussian's total flux inside APRAD, used to convert a target # aperture magnitude into the total flux of an injected source. APFRAC = 1.0 - np.exp(-APRAD ** 2 / (2.0 * SIGMA ** 2)) # --- tracklet linking parameters --------------------------------------- MIN_RATE = 2.5 # px/hr MAX_RATE = 900.0 # px/hr MIN_HITS = 5 LINE_RMS = 1.2 # px MAG_SCATTER = 0.5 # mag TOL = 3.0 # px PAIR_GAP = 3 STATIC_MIN = 4 # recurrences at one place that mark a source static STATIC_TOL = 2.0 # px MIN_NATIVE_MOTION = 4.0 # px a real mover must travel in DETECTOR coordinates # ---------------------------------------------------------------------- # detection def detect(image, thresh=3.5, minarea=5): """sep background estimate, extraction, and 5 px aperture photometry.""" bkg = sep.Background(image, bw=64, bh=64, fw=3, fh=3) sub = image - bkg.back() objs = sep.extract(sub, thresh, err=bkg.globalrms, minarea=minarea, deblend_cont=0.005) if len(objs): ap, _, _ = sep.sum_circle(sub, objs["x"], objs["y"], APRAD, err=bkg.globalrms, gain=1.0) else: ap = np.zeros(0) return objs, np.asarray(ap, dtype=float), float(bkg.globalrms) def lum_frames(): """(key, path) for the twelve luminance subs, in time order.""" rows = [] n = 0 for fn in sorted(os.listdir(SRC)): if fn.startswith("calibrated-") and fn.endswith(".fit") \ and "-Luminance-" in fn: n += 1 rows.append((f"Luminance_{n:03d}", layout.path(fn))) return rows def lum_times(): """Mid-exposure times of the luminance subs, in hours from the first.""" t = [] for _, path in lum_frames(): jd = float(fits.getheader(path)["JD"]) t.append(jd - 2400000.5 + 150.0 / 86400.0) t = np.array(t) return (t - t[0]) * 24.0 # ---------------------------------------------------------------------- # registration def load_dets(): z = np.load(DETS, allow_pickle=True) meta = {r[0]: r for r in z["meta"]} return meta, {k: z[k] for k in meta} def frame_transforms(): """Native-pixel -> reference-frame affine for each luminance sub. Seeded by the astroalign solution cached in _mo_dets.npz, then refined by nearest-neighbour matching every detection against the reference frame's detections. The refined solutions are good to ~0.15 px median across the whole field (see mo_check.py), which is what makes a 3 px association radius meaningful. """ _, dets = load_dets() ref = dets[REFERENCE] tree = cKDTree(ref[:, :2]) tf_out = {} for key, d in dets.items(): if not key.startswith("Luminance"): continue src = d[:, 2:4].astype(float) cur = d[:, :2].astype(float) tf = None for t in (3.0, 1.5): dist, idx = tree.query(cur, distance_upper_bound=t) ok = np.isfinite(dist) if ok.sum() < 50: break tf = AffineTransform() tf.estimate(src[ok], ref[idx[ok], :2].astype(float)) cur = tf(src) if tf is None: # the reference frame itself tf = AffineTransform() tf.estimate(src, d[:, :2].astype(float)) tf_out[key] = tf return tf_out # ---------------------------------------------------------------------- # photometry def master_sources(): """Detections and 5 px aperture fluxes in the deep luminance master.""" with fits.open(layout.path("master-Luminance.fit")) as hd: img = hd[0].data.astype(np.float32) hdr = hd[0].header objs, ap, rms = detect(img, thresh=2.5, minarea=4) return objs, ap, rms, hdr, img def frame_zeropoint(fx, fy, fap, mobjs, map_, tf): """Per-frame zero point, tied to the master's, from stars in common. The master is a scaled sigma-clipped mean of the twelve subs, so a single sub's counts differ from it by a near-constant factor. Measuring that factor on a few hundred stars puts single-frame magnitudes on the master's system without a separate absolute calibration. """ ref_xy = tf(np.column_stack([fx, fy])) mt = cKDTree(np.column_stack([mobjs["x"], mobjs["y"]])) d, idx = mt.query(ref_xy, distance_upper_bound=1.5) ok = np.isfinite(d) & (fap > 0) if ok.sum() < 30: return ZP, 0 fm = map_[idx[ok]] ff = fap[ok] good = (fm > 0) & (ff > 0) # Bright half only: faint stars are noise-biased in both frames. order = np.argsort(fm[good])[::-1] sel = order[:max(50, len(order) // 4)] ratio = np.median((ff[good][sel] / fm[good][sel])) return ZP + 2.5 * np.log10(ratio), int(ok.sum()) # ---------------------------------------------------------------------- # static-sky rejection and linking def residual_mask(pts_list, nat_list, mtree, master_tol=4.0): """True for detections that are neither static sky nor detector defects. Three rejections, and the third is the one that matters most on this data: 1. Coincident with a source in the deep luminance master. 2. Recurring at the same REFERENCE-frame position in four or more subs - a star the master missed. 3. Recurring at the same DETECTOR-frame position in four or more subs. Cut 3 exists because the frames are dithered and the field rotates slowly through the session. Registration undoes that motion for the sky, which means it imposes exactly the opposite motion on anything fixed to the detector. A hot pixel therefore traces a perfectly straight, perfectly constant-rate, perfectly constant-brightness track across the registered sequence - an ideal fake minor planet that passes every cut a linker normally applies. Without this cut the search on these twelve subs returns 141 such tracks, all of them moving at 9-18 arcsec/hr toward position angle 45-85 deg, all with npix of 5-10 (a 3x3 blob, against 50-100 for a real star at this seeing), and all stationary to under 0.5 px in detector coordinates. With it, none survive. """ trees = [cKDTree(p) if len(p) else None for p in pts_list] ntrees = [cKDTree(p) if len(p) else None for p in nat_list] out = [] for i, p in enumerate(pts_list): if not len(p): out.append(np.zeros(0, dtype=bool)) continue d, _ = mtree.query(p, distance_upper_bound=master_tol) in_master = np.isfinite(d) recur = np.zeros(len(p), dtype=int) nrecur = np.zeros(len(p), dtype=int) for j, (tr, ntr) in enumerate(zip(trees, ntrees)): if j == i or tr is None: continue dd, _ = tr.query(p, distance_upper_bound=STATIC_TOL) recur += np.isfinite(dd) nd, _ = ntr.query(nat_list[i], distance_upper_bound=STATIC_TOL) nrecur += np.isfinite(nd) out.append((~in_master) & (recur < STATIC_MIN) & (nrecur < STATIC_MIN)) return out def fit_track(t, x, y): A = np.column_stack([np.ones_like(t), t]) cx, *_ = np.linalg.lstsq(A, x, rcond=None) cy, *_ = np.linalg.lstsq(A, y, rcond=None) rms = float(np.sqrt(np.mean((x - A @ cx) ** 2 + (y - A @ cy) ** 2))) return cx, cy, rms def link(times, pts, mags, nat=None, shape=None, min_hits=MIN_HITS): """Pair-seeded, propagate-and-count tracklet finder. See mo_link.py.""" n = len(times) trees = [cKDTree(p) if len(p) else None for p in pts] seen, cands = set(), [] for i, j in itertools.combinations(range(n), 2): if j - i < PAIR_GAP or trees[i] is None or trees[j] is None: continue dt = times[j] - times[i] if dt <= 0: continue near = trees[j].query_ball_point(pts[i], MAX_RATE * dt) for a, blist in enumerate(near): for b in blist: if np.hypot(*(pts[j][b] - pts[i][a])) < MIN_RATE * dt: continue v = (pts[j][b] - pts[i][a]) / dt hits = [] for f in range(n): if trees[f] is None: continue pred = pts[i][a] + v * (times[f] - times[i]) dd, idx = trees[f].query(pred, distance_upper_bound=TOL) if np.isfinite(dd): hits.append((f, int(idx))) if len(hits) < min_hits: continue sig = tuple(sorted(hits)) if sig in seen: continue seen.add(sig) tt = np.array([times[f] for f, _ in hits]) xx = np.array([pts[f][k][0] for f, k in hits]) yy = np.array([pts[f][k][1] for f, k in hits]) cx, cy, rms = fit_track(tt, xx, yy) if rms > LINE_RMS: continue mm = np.array([mags[f][k] for f, k in hits]) if not np.all(np.isfinite(mm)) or mm.std() > MAG_SCATTER: continue if nat is not None: # Backstop for cut 3 in residual_mask: whatever this is, # it must have genuinely moved across the detector, not # merely been carried by the registration. nx_ = np.array([nat[f][k][0] for f, k in hits]) ny_ = np.array([nat[f][k][1] for f, k in hits]) if np.hypot(np.ptp(nx_), np.ptp(ny_)) < MIN_NATIVE_MOTION: continue c = dict(frames=np.array([f for f, _ in hits]), idx=np.array([k for _, k in hits]), t=tt, x=xx, y=yy, cx=cx, cy=cy, rms=rms, rate=float(np.hypot(cx[1], cy[1])), ang=float(np.degrees(np.arctan2(cy[1], cx[1]))), mag=float(mm.mean()), magsig=float(mm.std()), nhit=len(hits)) if shape is not None: sh = np.array([shape[f][k] for f, k in hits]) c.update(a=float(np.median(sh[:, 0])), b=float(np.median(sh[:, 1])), npix=float(np.median(sh[:, 2]))) cands.append(c) return dedup(cands) def dedup(cands): """Among tracklets sharing two or more detections, keep the best.""" cands.sort(key=lambda c: (-c["nhit"], c["rms"])) kept, sets = [], [] for c in cands: s = set(zip(c["frames"].tolist(), c["idx"].tolist())) if any(len(s & o) >= 2 for o in sets): continue sets.append(s) kept.append(c) return kept # ---------------------------------------------------------------------- # synthetic sources def add_source(image, x, y, total_flux, dx, dy, nstep=9, sigma=SIGMA): """Add a trailed Gaussian: the source moves (dx, dy) px during the sub. A 300 s exposure of something moving at 30 arcsec/hr smears by ~4.6 px, comparable to the seeing disc, so the trail is modelled rather than ignored - it is what limits sensitivity at high rates. """ ny, nx = image.shape half = int(np.ceil(4 * sigma + 0.5 * np.hypot(dx, dy))) + 2 x0, x1 = int(max(0, x - half)), int(min(nx, x + half + 1)) y0, y1 = int(max(0, y - half)), int(min(ny, y + half + 1)) if x1 <= x0 or y1 <= y0: return gy, gx = np.mgrid[y0:y1, x0:x1] acc = np.zeros(gx.shape, dtype=np.float64) for s in np.linspace(-0.5, 0.5, nstep): cx, cy = x + s * dx, y + s * dy acc += np.exp(-((gx - cx) ** 2 + (gy - cy) ** 2) / (2.0 * sigma ** 2)) acc *= total_flux / (nstep * 2.0 * np.pi * sigma ** 2) image[y0:y1, x0:x1] += acc.astype(image.dtype)