From 5286a2e81b7376fbd54bef6748f328288ce0a19b Mon Sep 17 00:00:00 2001 From: laurence Date: Tue, 21 Jul 2026 15:29:49 +0100 Subject: [PATCH 1/4] Processing and analysis code for remote-telescope imaging sessions The scripts that processed the NGC 5128 session of 2026-07-21 previously lived inside the data directory and addressed it with absolute paths. Code and data are now separated: the code lives here, and a session is located at runtime through the ASTRO_SESSION environment variable. layout.py is what makes that work. It maps a FILENAME to the subdirectory that file belongs in, using the same rules the session directories are organised with, so a script can go on asking for 'master-Red.fit' or '_stars.npz' without any call site knowing the directory structure. Anything unrecognised resolves to the session root, which is visible and correctable rather than silently wrong. restructure.py reorganises a flat session directory into that layout. It is idempotent and dry-run by default. The 50 session scripts are kept as they were run rather than tidied into a library. They were written in sequence as the work went along, several of them by parallel agents, and they show it - but they are the honest provenance of a published set of results, and the productionised pipeline should be able to reproduce those results exactly. Verified before committing: all 51 files compile without warnings, and verify_core.py, closeup.py and triptych.py were run end to end against the reorganised session, correctly finding inputs across calibrated/, stacks/masters/ and final/ and writing outputs back to the right places. --- .gitignore | 11 + README.md | 95 +++++ session-scripts/analyse.py | 70 ++++ session-scripts/annotate.py | 177 +++++++++ session-scripts/closeup.py | 102 +++++ session-scripts/compose.py | 256 ++++++++++++ session-scripts/depth.py | 102 +++++ session-scripts/enhance.py | 195 +++++++++ session-scripts/final.py | 311 +++++++++++++++ session-scripts/gaia_colours.py | 49 +++ session-scripts/gc-bwtrial.py | 75 ++++ session-scripts/gc-classify.py | 229 +++++++++++ session-scripts/gc-complete-inner.py | 120 ++++++ session-scripts/gc-complete.py | 160 ++++++++ session-scripts/gc-detect.py | 171 ++++++++ session-scripts/gc-fetch-vizier.py | 91 +++++ session-scripts/gc-gaia-astrom2.py | 32 ++ session-scripts/gc-plots.py | 491 +++++++++++++++++++++++ session-scripts/gc-validate.py | 156 ++++++++ session-scripts/hdr.py | 161 ++++++++ session-scripts/layout.py | 108 +++++ session-scripts/mo_cavs.py | 567 +++++++++++++++++++++++++++ session-scripts/mo_check.py | 85 ++++ session-scripts/mo_common.py | 321 +++++++++++++++ session-scripts/mo_detect.py | 137 +++++++ session-scripts/mo_fig_moving.py | 283 +++++++++++++ session-scripts/mo_fig_transient.py | 228 +++++++++++ session-scripts/mo_finalise.py | 146 +++++++ session-scripts/mo_gccheck.py | 423 ++++++++++++++++++++ session-scripts/mo_link.py | 130 ++++++ session-scripts/mo_mpc.py | 191 +++++++++ session-scripts/mo_sensitivity.py | 209 ++++++++++ session-scripts/mo_shiftstack.py | 211 ++++++++++ session-scripts/mo_transient.py | 232 +++++++++++ session-scripts/mo_vet.py | 189 +++++++++ session-scripts/original.py | 153 ++++++++ session-scripts/rename.py | 91 +++++ session-scripts/restructure.py | 124 ++++++ session-scripts/sb_common.py | 108 +++++ session-scripts/sb_dust.py | 191 +++++++++ session-scripts/sb_iso.py | 128 ++++++ session-scripts/sb_limits.py | 137 +++++++ session-scripts/sb_model.py | 80 ++++ session-scripts/sb_prep.py | 132 +++++++ session-scripts/sb_profile.py | 311 +++++++++++++++ session-scripts/sb_render_tail.py | 163 ++++++++ session-scripts/sb_residual.py | 77 ++++ session-scripts/solve.py | 185 +++++++++ session-scripts/stack.py | 166 ++++++++ session-scripts/starless.py | 91 +++++ session-scripts/triptych.py | 62 +++ session-scripts/unzip.py | 33 ++ session-scripts/verify_core.py | 74 ++++ 53 files changed, 8820 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 session-scripts/analyse.py create mode 100644 session-scripts/annotate.py create mode 100644 session-scripts/closeup.py create mode 100644 session-scripts/compose.py create mode 100644 session-scripts/depth.py create mode 100644 session-scripts/enhance.py create mode 100644 session-scripts/final.py create mode 100644 session-scripts/gaia_colours.py create mode 100644 session-scripts/gc-bwtrial.py create mode 100644 session-scripts/gc-classify.py create mode 100644 session-scripts/gc-complete-inner.py create mode 100644 session-scripts/gc-complete.py create mode 100644 session-scripts/gc-detect.py create mode 100644 session-scripts/gc-fetch-vizier.py create mode 100644 session-scripts/gc-gaia-astrom2.py create mode 100644 session-scripts/gc-plots.py create mode 100644 session-scripts/gc-validate.py create mode 100644 session-scripts/hdr.py create mode 100644 session-scripts/layout.py create mode 100644 session-scripts/mo_cavs.py create mode 100644 session-scripts/mo_check.py create mode 100644 session-scripts/mo_common.py create mode 100644 session-scripts/mo_detect.py create mode 100644 session-scripts/mo_fig_moving.py create mode 100644 session-scripts/mo_fig_transient.py create mode 100644 session-scripts/mo_finalise.py create mode 100644 session-scripts/mo_gccheck.py create mode 100644 session-scripts/mo_link.py create mode 100644 session-scripts/mo_mpc.py create mode 100644 session-scripts/mo_sensitivity.py create mode 100644 session-scripts/mo_shiftstack.py create mode 100644 session-scripts/mo_transient.py create mode 100644 session-scripts/mo_vet.py create mode 100644 session-scripts/original.py create mode 100644 session-scripts/rename.py create mode 100644 session-scripts/restructure.py create mode 100644 session-scripts/sb_common.py create mode 100644 session-scripts/sb_dust.py create mode 100644 session-scripts/sb_iso.py create mode 100644 session-scripts/sb_limits.py create mode 100644 session-scripts/sb_model.py create mode 100644 session-scripts/sb_prep.py create mode 100644 session-scripts/sb_profile.py create mode 100644 session-scripts/sb_render_tail.py create mode 100644 session-scripts/sb_residual.py create mode 100644 session-scripts/solve.py create mode 100644 session-scripts/stack.py create mode 100644 session-scripts/starless.py create mode 100644 session-scripts/triptych.py create mode 100644 session-scripts/unzip.py create mode 100644 session-scripts/verify_core.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5e8e7a7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +__pycache__/ +*.pyc + +# Data never lives in this repo; sessions are addressed via ASTRO_SESSION. +*.fit +*.fits +*.tif +*.png +*.jpg +*.npz +*.npy diff --git a/README.md b/README.md new file mode 100644 index 0000000..a8b1cd2 --- /dev/null +++ b/README.md @@ -0,0 +1,95 @@ +# astro-pipeline + +Processing and analysis code for remote-telescope imaging sessions, starting +with iTelescope data from the [itelescope](https://git.discworld.casa/laurence/itelescope) +drain campaign. + +The code lives here. **The data does not** - image sessions stay on disk (or +wherever they are archived) and are addressed by an environment variable, so a +session directory contains only pixels, results and a description of what was +done to them. + +## What is here now + +`session-scripts/` - the 50 scripts that processed the NGC 5128 session of +2026-07-21, exactly as they were run, plus the shared `layout.py` that tells +them where files live. This is a working record rather than a finished product: +the scripts were written in sequence as the work went along, several of them by +parallel agents, and they show it. They are kept because they are the honest +provenance of a set of published results, and because the productionised +pipeline should be able to reproduce those results exactly. + +`session-scripts/restructure.py` - reorganises a flat session directory into the +named layout below. Idempotent, dry run by default. + +## Pointing the scripts at a session + +``` +set ASTRO_SESSION=D:\astro\NGC5128\20260721 # Windows +export ASTRO_SESSION=/data/astro/NGC5128/20260721 # POSIX +python session-scripts/layout.py # prints the resolved layout +``` + +`layout.py` maps a **filename** to its subdirectory, so a script asks for +`master-Red.fit` or `_stars.npz` and gets the right path without knowing the +directory structure: + +| Directory | Holds | +|---|---| +| `raw/` | exactly what the telescope delivered: archives and their preview jpegs | +| `calibrated/` | uncompressed calibrated subs | +| `stacks/masters/` | per-filter registered, plate-solved masters | +| `stacks/original/` | alignment-only baseline stacks, no other processing | +| `final/` | the deliverable renderings | +| `renderings/` | other finished images | +| `science/figures/` | analysis plots | +| `science/catalogues/` | measured tables (CSV) | +| `science/data/` | models, masks, derived quantities | +| `science/notes/` | analysis write-ups | +| `intermediates/` | caches a re-run can regenerate | + +Every session directory also carries its own `METHODS.md` describing what was +done to that data and what was found - written for a reader who was not there. + +## Running order + +The scripts are named for their stage and run in this order: + +``` +unzip.py -> analyse.py -> stack.py -> solve.py -> depth.py + -> compose.py -> hdr.py / enhance.py / starless.py / annotate.py + -> final.py -> closeup.py -> triptych.py +``` + +The analysis families are independent of each other and of the renderings: +`gc-*` (globular clusters), `sb_*` (surface photometry), `mo_*` (moving objects +and transients). + +## Requirements + +Python 3.12 with numpy, scipy, astropy, scikit-image, sep, astroalign, +photutils, astroquery, matplotlib, tifffile, Pillow. + +## Where this is going + +The next piece of work is a scheduler-driven pipeline: a staged CLI +(`ingest -> calibrate -> measure -> register -> stack -> solve -> compose -> +analyse`) with each stage resumable, packaged as an Apptainer image and driven +by Slurm array jobs. Targets beyond mono LRGB: narrowband palettes, one-shot +colour with debayering, other observatories' header conventions, and full +calibration from bias/dark/flat for sources that do not pre-calibrate. + +Three findings from the first session are requirements for that build, not +optional extras: + +1. **Vet moving-object candidates in detector coordinates.** Registration holds + the sky still, so it drags detector-fixed defects across the frame on + perfectly straight, constant-rate tracks. Hot pixels are better-behaved + asteroids than real asteroids. This one cut took 141 confident spurious + detections to zero. +2. **Carry `r50/psf` through to any catalogue cross-match.** Comparing an + aperture magnitude of a resolved source against a point-source catalogue + like Gaia is meaningless, and looks exactly like a 2.8 magnitude outburst. +3. **Never fit a sky background to a field the target fills.** A plane fitted + around a large galaxy absorbs its halo - measured at -17.9 ADU/px here. + Fit the background and a source model together. diff --git a/session-scripts/analyse.py b/session-scripts/analyse.py new file mode 100644 index 0000000..e61498c --- /dev/null +++ b/session-scripts/analyse.py @@ -0,0 +1,70 @@ +"""Pass 1: measure every calibrated frame and cache its star list. + +Frames are 4788x3194 float32 (61 MB each) and the machine has little free RAM, +so each frame is opened, measured and released one at a time. The star lists are +cached to an npz because both the registration pass and the plate solve need +them, and re-detecting costs more than re-reading a small array. + +Recorded per frame: sky background and its rms, the number of detections, and a +median FWHM derived from sep's half-flux radius. The FWHM is the seeing metric +used later to pick the registration reference and to weight the stack. +""" +import glob +import os +import re + +import numpy as np +import sep +from astropy.io import fits + +import layout + +SRC = layout.SESSION +CACHE = layout.path("_stars.npz") +os.makedirs(os.path.dirname(CACHE), exist_ok=True) + +NAME_RE = re.compile(r"-(Luminance|Red|Green|Blue)-BIN2-W-300-(\d+)\.fit$") + + +def frame_list(): + out = [] + for path in sorted(glob.glob(layout.path("calibrated-*.fit"))): + m = NAME_RE.search(path) + if m: + out.append((path, m.group(1), int(m.group(2)))) + return out + + +if __name__ == "__main__": + frames = frame_list() + print(f"{len(frames)} calibrated frames") + store, meta = {}, [] + for path, filt, idx in frames: + with fits.open(path, memmap=False) as hd: + data = hd[0].data.astype(np.float32) + bkg = sep.Background(data, bw=64, bh=64, fw=3, fh=3) + back_med = float(np.median(bkg.back())) + rms = float(bkg.globalrms) + sub = data - bkg.back() + objs = sep.extract(sub, 5.0, err=rms, minarea=9, deblend_cont=0.005) + objs = objs[(objs["flag"] == 0) & (objs["npix"] > 12) & + (objs["npix"] < 2000)] + objs = objs[np.argsort(objs["flux"])[::-1][:400]] + rad, _ = sep.flux_radius(sub, objs["x"], objs["y"], 6.0 * objs["a"], + 0.5, normflux=objs["flux"]) + fwhm = float(np.median(rad) * 2.0) + key = f"{filt}_{idx:03d}" + store[key + "_xy"] = np.column_stack([objs["x"], objs["y"]]) + store[key + "_flux"] = objs["flux"] + meta.append((key, os.path.basename(path), filt, idx, len(objs), fwhm, + back_med, rms)) + print(f"{key:16s} stars={len(objs):4d} fwhm={fwhm:5.2f}px " + f"bg={back_med:8.1f} rms={rms:6.1f}") + del data, sub, bkg, objs + + np.savez_compressed( + CACHE, + meta=np.array(meta, dtype=object), + **store, + ) + print("cached ->", CACHE) diff --git a/session-scripts/annotate.py b/session-scripts/annotate.py new file mode 100644 index 0000000..9249197 --- /dev/null +++ b/session-scripts/annotate.py @@ -0,0 +1,177 @@ +"""Annotated version of the finished LRGB: coordinate grid and catalogued objects. + +The overlay is driven entirely by the local plate solution, so every label sits +where the astrometry says it should. Objects come from SIMBAD, restricted to a +cone matching the field and to types worth marking (galaxies, clusters, radio +sources), then filtered again to those that actually fall inside the frame. + +The annotation is drawn on a downsampled copy: at full 15 Mpx the labels would +be microscopic relative to the image, and nobody views a 4692 px wide frame at +1:1 to read a caption. +""" +import os + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.wcs import WCS +from PIL import Image + +import layout + +OUT = layout.SESSION +CACHE = layout.path("_simbad.npz") +SCALE = 3 # downsample factor for the annotated render + +with fits.open(layout.path("NGC5128-LRGB.fit")) as hd: + hdr = hd[0].header +wcs = WCS(hdr, naxis=2) +rgb = np.asarray(Image.open(layout.path("NGC5128-LRGB.png"))) +ny, nx = rgb.shape[:2] +small = np.asarray(Image.fromarray(rgb).resize((nx // SCALE, ny // SCALE), + Image.LANCZOS)) +# Slicing a WCS rescales it correctly whether the solution is stored as CD or +# as PC + CDELT, which hand-editing the matrix does not. +wcs_small = wcs[::SCALE, ::SCALE] +centre = wcs.pixel_to_world(nx / 2, ny / 2) +print(f"frame {nx}x{ny} -> render {small.shape[1]}x{small.shape[0]}") + + +def simbad_objects(): + if os.path.exists(CACHE): + z = np.load(CACHE, allow_pickle=True) + return z["name"], z["ra"], z["dec"], z["otype"] + from astroquery.simbad import Simbad + sim = Simbad() + sim.ROW_LIMIT = 2000 + for field in ("otype", "V"): + try: + sim.add_votable_fields(field) + except Exception: # noqa: BLE001 + pass + tbl = sim.query_region(centre, radius=0.42 * u.deg) + name = np.array([str(r) for r in tbl[tbl.colnames[0]]]) + coords = SkyCoord(tbl["ra"], tbl["dec"], unit=(u.deg, u.deg)) + otype = np.array([str(t) for t in tbl["otype"]]) if "otype" in \ + tbl.colnames else np.array([""] * len(tbl)) + np.savez_compressed(CACHE, name=name, ra=coords.ra.deg, + dec=coords.dec.deg, otype=otype) + return name, coords.ra.deg, coords.dec.deg, otype + + +name, ra, dec, otype = simbad_objects() +print(f"{len(name)} SIMBAD entries in the cone") + +# SIMBAD returns 1518 rows for this field, the bulk of them anonymous entries +# from Centaurus A cluster and variable-star surveys. Marking those would bury +# the image, and the cluster system is being catalogued separately, so the +# overlay keeps only whole objects: other galaxies, planetary nebulae and +# anything carrying a mainstream catalogue designation. +GALAXY_TYPES = ("G", "GiG", "GiP", "GiC", "AGN", "SyG", "rG", "LSB") +# Confirmed nebulae only. SIMBAD lists 93 "PN?" candidates from a single +# survey of this field; they are unconfirmed, they are not visible at this +# depth, and marking them makes the image unreadable. +NEBULA_TYPES = ("PN", "HII", "SNR") +is_gal = np.isin(otype, GALAXY_TYPES) +is_neb = np.isin(otype, NEBULA_TYPES) +mainstream = np.array([n.startswith(("NGC", "IC ", "ESO", "PGC", "AM ", "SN ")) + and not n.startswith("SNR") + for n in name]) +sel = is_gal | is_neb | mainstream +sky = SkyCoord(ra[sel] * u.deg, dec[sel] * u.deg) +x, y = wcs_small.world_to_pixel(sky) +inside = (x > 40) & (x < small.shape[1] - 40) & (y > 40) & \ + (y < small.shape[0] - 40) +labels, kinds = name[sel][inside], otype[sel][inside] +x, y = x[inside], y[inside] +print(f"{len(labels)} catalogued objects inside the frame " + f"({is_gal[sel][inside].sum()} galaxies)") + +# Plain axes, not WCSAxes: WCSAxes insists on origin='lower', which would +# publish this image as a vertical mirror of every other deliverable. Drawing +# the graticule by hand keeps all the outputs in one orientation, and the +# lines still come from the plate solution rather than from assumption. +fig = plt.figure(figsize=(small.shape[1] / 100, small.shape[0] / 100), dpi=100) +ax = fig.add_axes([0, 0, 1, 1]) +ax.imshow(small, origin="upper") +ax.set_axis_off() + +corners = wcs_small.pixel_to_world( + [0, small.shape[1], 0, small.shape[1]], + [0, 0, small.shape[0], small.shape[0]]) +ra_lo, ra_hi = corners.ra.deg.min(), corners.ra.deg.max() +dec_lo, dec_hi = corners.dec.deg.min(), corners.dec.deg.max() + + +def draw_line(coord_ra, coord_dec, label, at_ra): + px, py = wcs_small.world_to_pixel(SkyCoord(coord_ra * u.deg, + coord_dec * u.deg)) + ok = (px > 0) & (px < small.shape[1]) & (py > 0) & (py < small.shape[0]) + if ok.sum() < 2: + return + ax.plot(px[ok], py[ok], color="#5fa8ff", alpha=0.30, linestyle=":", + linewidth=0.9) + i = np.where(ok)[0][len(np.where(ok)[0]) // 2] + ax.text(px[i], py[i], label, color="#8fc4ff", fontsize=8, + family="monospace", rotation=0 if at_ra else 90, + ha="center", va="center", + bbox=dict(boxstyle="round,pad=0.12", fc="black", ec="none", + alpha=0.45)) + + +RA_STEP = 15.0 / 60.0 # one minute of right ascension, in degrees +DEC_STEP = 10.0 / 60.0 # ten arcminutes +t = np.linspace(dec_lo, dec_hi, 400) +for r in np.arange(np.ceil(ra_lo / RA_STEP) * RA_STEP, ra_hi, RA_STEP): + c = SkyCoord(r * u.deg, 0 * u.deg) + draw_line(np.full_like(t, r), t, + f"{int(c.ra.hms.h):02d}h{int(c.ra.hms.m):02d}m", False) +s_ = np.linspace(ra_lo, ra_hi, 400) +for d in np.arange(np.ceil(dec_lo / DEC_STEP) * DEC_STEP, dec_hi, DEC_STEP): + dm = abs(d - int(d)) * 60 + draw_line(s_, np.full_like(s_, d), f"{int(d):+03d}d{dm:02.0f}m", True) + +for lx, ly, lab, kind in zip(x, y, labels, kinds): + colour = "#7ee08a" if kind in GALAXY_TYPES else "#ffd166" + ax.add_patch(plt.Circle((lx, ly), 15, fill=False, color=colour, + linewidth=1.2, alpha=0.95)) + ax.text(lx + 19, ly - 11, f"{lab} [{kind}]", color=colour, fontsize=7.5, + family="monospace", + bbox=dict(boxstyle="round,pad=0.12", fc="black", ec="none", + alpha=0.45)) + +# Scale bar: one arcminute, measured through the plate solution rather than +# assumed, plus the physical scale at Centaurus A's distance. +pix_per_arcmin = 60.0 / (0.5376 * SCALE) +bx, by = 60, small.shape[0] - 60 +ax.plot([bx, bx + pix_per_arcmin], [by, by], color="white", linewidth=2.5) +ax.text(bx, by - 12, "1' = 1.1 kpc at 3.8 Mpc", color="white", fontsize=9) + +# Orientation: north and east taken from the WCS, so a flipped or rotated +# solution cannot silently produce a wrong compass. +cx, cy = small.shape[1] - 120, small.shape[0] - 120 +c0 = wcs_small.pixel_to_world(cx, cy) +for dlab, offset in (("N", (0 * u.arcmin, 2 * u.arcmin)), + ("E", (2 * u.arcmin, 0 * u.arcmin))): + p = c0.spherical_offsets_by(*offset) + px, py = wcs_small.world_to_pixel(p) + ax.annotate("", xy=(px, py), xytext=(cx, cy), + arrowprops=dict(color="white", width=1.0, headwidth=6)) + ax.text(px, py, dlab, color="white", fontsize=11, ha="center", + va="center") + +ax.text(20, 26, "NGC 5128 (Centaurus A) iTelescope T32, Siding Spring " + "2026-07-21 L 12x300s RGB 4x300s each", + color="white", fontsize=10) +ax.text(20, 44, f"plate solved against Gaia DR3: {hdr.get('ASTRSOLV', '')}", + color="#9fb8d0", fontsize=8) +ax.set_xlim(0, small.shape[1]) +ax.set_ylim(small.shape[0], 0) + +path = layout.path("NGC5128-img-annotated.jpg") +fig.savefig(path, dpi=100, pil_kwargs={"quality": 92}) +print("wrote", path) diff --git a/session-scripts/closeup.py b/session-scripts/closeup.py new file mode 100644 index 0000000..e1175a1 --- /dev/null +++ b/session-scripts/closeup.py @@ -0,0 +1,102 @@ +"""Image 4: the close-up, framed by the galaxy's own measured extent. + +The crop box is not chosen by eye. The surface photometry produced an isophote +model of NGC 5128, and the region where that model carries real signal defines +how much sky the galaxy actually occupies; the frame is that region plus a +margin. So "keeping the object in frame" is a measured statement rather than a +judgement, and the same rule would frame any other galaxy without retuning. + +It is cut from image 3's 16-bit TIFF rather than from the PNG, and never +resampled, so every pixel is exactly the pixel that came out of the pipeline. +Re-rendering was considered and rejected: the stretch in image 3 is already +anchored on sky statistics measured across the whole frame, and a tight crop +contains too little empty sky to measure a better one. Cropping is therefore +not a compromise here - it is the correct operation. +""" +import os + +import numpy as np +import tifffile +from astropy.io import fits +from astropy.wcs import WCS +from PIL import Image + +import layout + +OUT = layout.SESSION +# The finished renderings live in their own directory: they are the +# deliverables, and keeping them apart from the masters, the +# intermediates and the analysis figures makes it obvious which +# files are meant to be looked at. +FINAL = layout.path("final") +SOURCE = "NGC5128-final-3-best.tif" +CROP = 48 # the border already removed from image 3 +MARGIN = 0.08 # sky margin around the galaxy, as a fraction +MODEL_FLOOR = 5.0 # ADU/px: where the isophote model still has signal + + +def main(): + model = fits.getdata(layout.path("sb-model.fits")).astype(np.float32) + ys, xs = np.where(model > MODEL_FLOOR) + # Model coordinates are on the uncropped grid; image 3 lost CROP px. + x0, x1 = xs.min() - CROP, xs.max() - CROP + y0, y1 = ys.min() - CROP, ys.max() - CROP + print(f"galaxy extent (model > {MODEL_FLOOR} ADU/px): " + f"x {x0}-{x1}, y {y0}-{y1} = {x1 - x0} x {y1 - y0} px") + + with fits.open(layout.path("master-Luminance.fit")) as hd: + wcs = WCS(hd[0].header, naxis=2) + + img = tifffile.imread(layout.path(SOURCE)) + ny, nx = img.shape[:2] + print(f"source {SOURCE}: {nx} x {ny}, {img.dtype}") + + # Centre the frame on the galaxy's own centre, not on the frame's. + cx, cy = (x0 + x1) / 2.0, (y0 + y1) / 2.0 + half_x = (x1 - x0) / 2.0 * (1 + MARGIN) + half_y = (y1 - y0) / 2.0 * (1 + MARGIN) + # One box for both axes keeps the galaxy from touching a short edge, and a + # round object in a near-square frame reads better than in a letterbox. + half = max(half_x, half_y) + + # Clamp inside the image while keeping the galaxy centred as far as + # possible; report honestly if the frame has to shrink. + half = min(half, cx, cy, nx - cx, ny - cy) + left, right = int(round(cx - half)), int(round(cx + half)) + top, bottom = int(round(cy - half)), int(round(cy + half)) + print(f"crop box x {left}-{right}, y {top}-{bottom} " + f"({right - left} x {bottom - top} px)") + + # Verify the whole galaxy really is inside before writing anything. + assert left <= x0 and right >= x1 and top <= y0 and bottom >= y1, \ + "galaxy would be clipped - refusing to write" + inset_x = min(x0 - left, right - x1) + inset_y = min(y0 - top, bottom - y1) + print(f"clearance to the nearest edge: {inset_x} px horizontally, " + f"{inset_y} px vertically") + + scale = 0.5376 + print(f"field of view {(right - left) * scale / 60:.1f}' x " + f"{(bottom - top) * scale / 60:.1f}'") + corner = wcs.pixel_to_world(left + CROP, top + CROP) + centre = wcs.pixel_to_world((left + right) / 2 + CROP, + (top + bottom) / 2 + CROP) + print(f"frame centre {centre.to_string('hmsdms')}") + print(f"top-left corner {corner.to_string('hmsdms')}") + + crop = img[top:bottom, left:right] + tifffile.imwrite(layout.path("NGC5128-final-4-closeup.tif"), crop, + photometric="rgb") + u8 = (crop.astype(np.float32) / 65535.0 * 255 + 0.5).astype(np.uint8) + Image.fromarray(u8).save(layout.path("NGC5128-final-4-closeup.png")) + prev = Image.fromarray(u8) + prev.thumbnail((2400, 2400), Image.LANCZOS) + prev.save(layout.path("NGC5128-final-4-closeup-preview.jpg"), + quality=93) + print("wrote NGC5128-final-4-closeup.tif / .png / -preview.jpg") + + +os.makedirs(FINAL, exist_ok=True) + +if __name__ == "__main__": + main() diff --git a/session-scripts/compose.py b/session-scripts/compose.py new file mode 100644 index 0000000..90c37f6 --- /dev/null +++ b/session-scripts/compose.py @@ -0,0 +1,256 @@ +"""Pass 4: turn the four masters into the finished LRGB image. + +The order of operations matters and is the usual one for a linear stack: + +1. Crop the registration border, where not every frame contributed. +2. Remove the sky gradient. A 46% moon was up about 30 degrees away, so each + channel carries a smooth ramp; a plane (not a higher-order surface) is fitted + to tiles OUTSIDE a generous ellipse around the galaxy, because Centaurus A's + halo fills much of this field and a flexible model would happily eat it. +3. Colour-calibrate on stars. Aperture photometry of a few hundred field stars + in R, G and B is scaled so their average colour is neutral. This is the + "average field star is grey" assumption, which is the standard cheap + substitute for a full photometric calibration and is well behaved here + because the field is rich. +4. Stretch. A midtone transfer function moves the sky background to a chosen + level while keeping the highlights unclipped: gentler on the core than a + plain gamma, and reversible arithmetic rather than a curve drawn by hand. +5. LRGB assembly. Colour comes from the 20-minute-per-channel RGB, detail and + noise from the 60-minute luminance: the RGB is scaled pixel-by-pixel to the + luminance's brightness, which is why the colour data being four times + shallower does not matter much. +""" +import os + +import numpy as np +import sep +import tifffile +from astropy.io import fits +from PIL import Image +from scipy.ndimage import gaussian_filter, median_filter +from skimage.restoration import denoise_tv_chambolle + +import layout + +OUT = layout.SESSION +CROP = 48 # registration border, in pixels +GALAXY_MASK = (1250, 1000) # semi-axes of the halo exclusion ellipse, px +BG_TARGET = 0.10 # where the sky sits in the stretched image +SATURATION = 1.35 +CHANNELS = ("Luminance", "Red", "Green", "Blue") + + +def load(): + data, hdr = {}, None + for name in CHANNELS: + with fits.open(layout.path(f"master-{name}.fit")) as hd: + arr = hd[0].data.astype(np.float32) + if hdr is None: + hdr = hd[0].header.copy() + data[name] = arr[CROP:-CROP, CROP:-CROP] + return data, hdr + + +def galaxy_mask(shape): + ny, nx = shape + yy, xx = np.mgrid[0:ny, 0:nx] + cy, cx = ny / 2.0, nx / 2.0 + a, b = GALAXY_MASK + return ((xx - cx) / a) ** 2 + ((yy - cy) / b) ** 2 < 1.0 + + +def remove_gradient(img, mask): + """Subtract a least-squares plane fitted to tile medians outside `mask`.""" + ny, nx = img.shape + step = 96 + xs, ys, zs = [], [], [] + for y0 in range(0, ny - step, step): + for x0 in range(0, nx - step, step): + tile = img[y0:y0 + step, x0:x0 + step] + if mask[y0:y0 + step, x0:x0 + step].any(): + continue + # The median of a tile is dominated by sky even with stars in it. + zs.append(np.median(tile)) + xs.append(x0 + step / 2.0) + ys.append(y0 + step / 2.0) + xs, ys, zs = map(np.asarray, (xs, ys, zs)) + keep = np.ones(len(zs), bool) + for _ in range(3): # clip tiles containing companions + A = np.column_stack([xs[keep], ys[keep], np.ones(keep.sum())]) + coef, *_ = np.linalg.lstsq(A, zs[keep], rcond=None) + model = coef[0] * xs + coef[1] * ys + coef[2] + resid = zs - model + s = 1.4826 * np.median(np.abs(resid - np.median(resid))) + keep = np.abs(resid - np.median(resid)) < 2.5 * s + yy, xx = np.mgrid[0:ny, 0:nx] + plane = (coef[0] * xx + coef[1] * yy + coef[2]).astype(np.float32) + return img - plane, coef, int(keep.sum()), len(zs) + + +def star_photometry(lum, channels): + """Aperture flux in each colour at the position of every luminance star.""" + bkg = sep.Background(lum, bw=64, bh=64, fw=3, fh=3) + sub = lum - bkg.back() + o = sep.extract(sub, 12.0, err=bkg.globalrms, minarea=9, + deblend_cont=0.005) + o = o[(o["flag"] == 0) & (o["npix"] > 12) & (o["npix"] < 800)] + ny, nx = lum.shape + cy, cx = ny / 2.0, nx / 2.0 + a, b = GALAXY_MASK + outside = ((o["x"] - cx) / a) ** 2 + ((o["y"] - cy) / b) ** 2 > 1.0 + o = o[outside] # keep the galaxy out of the white balance + o = o[np.argsort(o["flux"])[::-1][:500]] + flux = {} + for name in ("Red", "Green", "Blue"): + img = np.ascontiguousarray(channels[name]) + f, _, _ = sep.sum_circle(img, o["x"], o["y"], 6.0, subpix=5) + flux[name] = f + return o, flux + + +def mtf(x, midtone): + """PixInsight-style midtone transfer function on data already in [0, 1].""" + x = np.clip(x, 0.0, 1.0) + return ((midtone - 1.0) * x) / ((2.0 * midtone - 1.0) * x - midtone) + + +def autostretch(img, target=BG_TARGET, shadow_sigma=2.8): + """Black-point just below the sky, then an MTF that puts sky at `target`.""" + sky = np.median(img) + mad = 1.4826 * np.median(np.abs(img - sky)) + black = sky - shadow_sigma * mad + white = np.percentile(img, 99.995) + norm = np.clip((img - black) / (white - black), 0.0, 1.0) + sky_norm = (sky - black) / (white - black) + # Solve the MTF midtone that maps sky_norm exactly onto target. + m = ((target - 1.0) * sky_norm) / (2.0 * target * sky_norm - target - + sky_norm) + return mtf(norm, m), dict(black=float(black), white=float(white), + midtone=float(m), sky=float(sky), mad=float(mad)) + + +def main(): + data, hdr = load() + shape = data["Luminance"].shape + print(f"working frame {shape[1]} x {shape[0]} px after {CROP} px crop") + + mask = galaxy_mask(shape) + print(f"halo exclusion covers {mask.mean():.1%} of the frame") + for name in CHANNELS: + data[name], coef, kept, total = remove_gradient(data[name], mask) + print(f" {name:10s} plane dz/dx={coef[0]*1e3:+.3f} " + f"dz/dy={coef[1]*1e3:+.3f} ADU/kpx, offset {coef[2]:8.2f}, " + f"{kept}/{total} sky tiles used") + + o, flux = star_photometry(data["Luminance"], data) + good = (flux["Red"] > 0) & (flux["Green"] > 0) & (flux["Blue"] > 0) + r, g, b = (flux[k][good] for k in ("Red", "Green", "Blue")) + print(f"colour calibration on {good.sum()} field stars") + # Neutral point: the median star should come out white. + gr = np.median(g / r) + gb = np.median(g / b) + print(f" raw median star colour G/R={1/gr:.3f} G/B={1/gb:.3f}") + data["Red"] *= gr + data["Blue"] *= gb + + lum_lin = data["Luminance"] + rgb_lin = np.dstack([data["Red"], data["Green"], data["Blue"]]) + del data + + lum, params = autostretch(lum_lin) + print(f"luminance stretch: black={params['black']:.2f} " + f"white={params['white']:.1f} midtone={params['midtone']:.4f} " + f"(sky {params['sky']:.2f} +/- {params['mad']:.2f})") + + # The colour channels get their own black point but SHARE the luminance + # midtone, so the colour balance set above survives the stretch. + rgb = np.empty_like(rgb_lin) + for i in range(3): + ch = rgb_lin[:, :, i] + sky = np.median(ch) + mad = 1.4826 * np.median(np.abs(ch - sky)) + black = sky - 2.8 * mad + white = np.percentile(ch, 99.995) + rgb[:, :, i] = mtf(np.clip((ch - black) / (white - black), 0, 1), + params["midtone"]) + del rgb_lin + + # Neutralise the sky. Calibrating on stars makes STARS grey but leaves the + # residual sky tinted, because moonlight is blue-ish and each channel kept + # its own black point. Forcing the three sky medians together is what stops + # the empty parts of the frame reading brown. + sky_med = [float(np.median(rgb[:, :, i][~mask])) for i in range(3)] + target = float(np.mean(sky_med)) + print(f" sky medians after stretch R/G/B " + f"{sky_med[0]:.4f}/{sky_med[1]:.4f}/{sky_med[2]:.4f} -> {target:.4f}") + for i in range(3): + rgb[:, :, i] = np.clip(rgb[:, :, i] - (sky_med[i] - target), 0.0, 1.0) + + # Chroma noise is the ugliest part of a 20-minute colour stack. Blurring + # colour alone is invisible at this scale because the eye takes structure + # from the luminance, which is untouched. + rgb_lum = rgb.mean(axis=2, keepdims=True) + chroma = rgb - rgb_lum + for i in range(3): + chroma[:, :, i] = median_filter(chroma[:, :, i], size=3) + chroma[:, :, i] = gaussian_filter(chroma[:, :, i], 1.5) + rgb = np.clip(rgb_lum + chroma * SATURATION, 0.0, 1.0) + del chroma, rgb_lum + + # Gentle local contrast on the luminance to lift the dust lane, held back + # in the noise floor so the sky does not get grainier. + detail = lum - gaussian_filter(lum, 2.0) + weight = np.clip((lum - BG_TARGET) * 4.0, 0.0, 1.0) + lum = np.clip(lum + 0.35 * detail * weight, 0.0, 1.0) + + # Edge-preserving smoothing, applied ONLY where there is nothing but sky + # (the same weight, inverted). Structure and the galaxy halo keep their + # full resolution; the empty 70% of the frame loses its grain. + smooth = denoise_tv_chambolle(lum, weight=0.012) + lum = np.clip(lum * weight + smooth * (1.0 - weight), 0.0, 1.0) + del detail, weight, smooth + + # LRGB: keep the RGB hue, take the brightness from the deep luminance. + ratio = lum / np.maximum(rgb.mean(axis=2), 1e-5) + out = np.clip(rgb * ratio[:, :, None], 0.0, 1.0) + del ratio, rgb + + # Nothing in this field is genuinely green: no astronomical source between + # the H-alpha reds and the OIII blues emits there, so any green excess is + # noise or residual moonlight. Pulling green down to the neutral average + # wherever it exceeds it (the standard SCNR operation) is safe and takes + # the last of the cast out of the sky. + neutral = 0.5 * (out[:, :, 0] + out[:, :, 2]) + amount = 0.85 + green = out[:, :, 1] + out[:, :, 1] = np.where(green > neutral, + green * (1.0 - amount) + neutral * amount, green) + + out16 = (out * 65535.0 + 0.5).astype(np.uint16) + tif = layout.path("NGC5128-LRGB.tif") + tifffile.imwrite(tif, out16, photometric="rgb") + print("wrote", tif) + + png = layout.path("NGC5128-LRGB.png") + Image.fromarray((out * 255 + 0.5).astype(np.uint8)).save(png) + print("wrote", png) + + prev = Image.fromarray((out * 255 + 0.5).astype(np.uint8)) + prev.thumbnail((2000, 2000), Image.LANCZOS) + prevpath = layout.path("NGC5128-LRGB-preview.jpg") + prev.save(prevpath, quality=92) + print("wrote", prevpath, prev.size) + + hdr["CRPIX1"] = hdr.get("CRPIX1", 0) - CROP + hdr["CRPIX2"] = hdr.get("CRPIX2", 0) - CROP + hdr["NCOMBINE"] = (24, "frames across all filters") + hdr["EXPTOTAL"] = (7200.0, "[s] total integration, all filters") + hdr["COMMENT"] = "LRGB composite: L 12x300s, R/G/B 4x300s each" + cube = np.moveaxis((out * 65535).astype(np.uint16), 2, 0) + fitsout = layout.path("NGC5128-LRGB.fit") + fits.PrimaryHDU(cube, hdr).writeto(fitsout, overwrite=True) + print("wrote", fitsout) + + +if __name__ == "__main__": + main() diff --git a/session-scripts/depth.py b/session-scripts/depth.py new file mode 100644 index 0000000..62745f9 --- /dev/null +++ b/session-scripts/depth.py @@ -0,0 +1,102 @@ +"""How deep did the luminance master actually go? + +Calibrates instrumental magnitudes against Gaia G, then reports the faintest +star still detected at 5 sigma. That number decides which follow-up analyses +are worth attempting on this data and which are wishful thinking. +""" +import os + +import numpy as np +import sep +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.wcs import WCS + +import layout + +OUT = layout.SESSION +CACHE = layout.path("_gaia_deep.npz") + +with fits.open(layout.path("master-Luminance.fit")) as hd: + img = hd[0].data.astype(np.float32) + hdr = hd[0].header +wcs = WCS(hdr, naxis=2) +ny, nx = img.shape + +bkg = sep.Background(img, bw=64, bh=64, fw=3, fh=3) +sub = img - bkg.back() +objs = sep.extract(sub, 3.0, err=bkg.globalrms, minarea=6, deblend_cont=0.005) +objs = objs[(objs["flag"] == 0) & (objs["npix"] > 6)] +flux, fluxerr, _ = sep.sum_circle(sub, objs["x"], objs["y"], 5.0, + err=bkg.globalrms, subpix=5) +snr = flux / np.maximum(fluxerr, 1e-9) +keep = (flux > 0) & (snr > 3) +objs, flux, snr = objs[keep], flux[keep], snr[keep] +print(f"{len(objs)} sources detected at SNR > 3 (rms {bkg.globalrms:.2f} ADU)") + +sky = wcs.pixel_to_world(objs["x"], objs["y"]) +centre = wcs.pixel_to_world(nx / 2, ny / 2) + +if os.path.exists(CACHE): + z = np.load(CACHE) + gra, gdec, gmag = z["ra"], z["dec"], z["g"] +else: + from astroquery.gaia import Gaia + Gaia.ROW_LIMIT = 60000 + job = Gaia.launch_job_async(f""" + SELECT ra, dec, phot_g_mean_mag FROM gaiadr3.gaia_source + WHERE 1 = CONTAINS(POINT('ICRS', ra, dec), + CIRCLE('ICRS', {centre.ra.deg}, {centre.dec.deg}, 0.42)) + AND phot_g_mean_mag IS NOT NULL AND phot_g_mean_mag < 20.5 + """) + t = job.get_results() + gra = np.asarray(t["ra"], float) + gdec = np.asarray(t["dec"], float) + gmag = np.asarray(t["phot_g_mean_mag"], float) + np.savez_compressed(CACHE, ra=gra, dec=gdec, g=gmag) +print(f"{len(gmag)} Gaia sources in the field, G down to {gmag.max():.2f}") + +gcoord = SkyCoord(gra * u.deg, gdec * u.deg) +idx, sep2d, _ = sky.match_to_catalog_sky(gcoord) +matched = sep2d.arcsec < 1.5 +print(f"{matched.sum()} detections matched to Gaia within 1.5\"") + +inst = -2.5 * np.log10(flux[matched]) +gm = gmag[idx[matched]] +# Fit the zero point on well exposed, unsaturated stars only. +fit = (gm > 12) & (gm < 17) & (snr[matched] > 20) +zp = float(np.median(gm[fit] - inst[fit])) +scatter = float(np.std(gm[fit] - inst[fit] - 0.0)) +print(f"zero point {zp:.3f} (G = inst + zp) from {fit.sum()} stars, " + f"scatter {scatter:.3f} mag") + +mag_all = -2.5 * np.log10(flux) + zp +# SNR falls monotonically with magnitude, so read the SNR=5 crossing off a +# running median rather than requiring sources to land in a narrow SNR bin. +order = np.argsort(mag_all) +ms, ss = mag_all[order], snr[order] +win = max(11, len(ms) // 60) +run_m = np.array([np.median(ms[i:i + win]) for i in range(0, len(ms) - win, win // 2)]) +run_s = np.array([np.median(ss[i:i + win]) for i in range(0, len(ss) - win, win // 2)]) +below = np.where(run_s < 5.0)[0] +lim5 = float(run_m[below[0]]) if len(below) else float(run_m[-1]) +print(f"limiting magnitude at SNR 5: G ~ {lim5:.2f} " + f"(SNR range {snr.min():.1f}-{snr.max():.0f})") +print(f"faintest detection: G ~ {mag_all.max():.2f} (SNR " + f"{snr[np.argmax(mag_all)]:.1f})") + +unmatched = ~matched +print(f"{unmatched.sum()} detections with NO Gaia counterpart " + f"({100 * unmatched.mean():.1f}% of sources)") +r_gal = np.hypot(objs["x"] - nx / 2, objs["y"] - ny / 2) * 0.5376 / 60.0 +near = unmatched & (r_gal < 12.0) & (mag_all > 18.0) & (mag_all < 22.0) +print(f" of those, {near.sum()} lie within 12' of the galaxy at " + f"G 18-22: the magnitude and radius range of Centaurus A's " + f"globular cluster system") + +# Surface brightness of the sky, a fair summary of how much the moon cost. +pixarea = 0.5376 ** 2 +sky_adu = float(np.median(bkg.back())) +print(f"sky background {sky_adu:.1f} ADU/px -> " + f"{zp - 2.5 * np.log10(max(sky_adu, 1e-6) / pixarea):.2f} mag/arcsec^2") diff --git a/session-scripts/enhance.py b/session-scripts/enhance.py new file mode 100644 index 0000000..f923d43 --- /dev/null +++ b/session-scripts/enhance.py @@ -0,0 +1,195 @@ +"""Pass 5: three further renderings from the same masters. + +1. NGC5128-img-deconvolved - the luminance sharpened by Richardson-Lucy against a PSF + measured from the frame's own stars, then recombined into LRGB. The seeing + was 2.69 arcsec, so there is real detail to recover in the dust lane; the + deconvolution is deliberately stopped early and applied only where the + signal is strong, because RL amplifies noise and rings around bright stars + if it is let run. +2. NGC5128-img-core-print - a full-resolution crop of the galaxy for printing. + +The star/starless separation lives in starless.py, which works from this +script's output. + +The stretch, colour calibration and gradient handling are imported from +compose.py rather than re-implemented, so these renderings and the main image +cannot drift apart. +""" +import os +import sys + +import numpy as np +import sep +import tifffile +from astropy.io import fits +from PIL import Image +from scipy.ndimage import gaussian_filter, median_filter +from skimage.restoration import richardson_lucy + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import compose as C # noqa: E402 + +import layout + +OUT = C.OUT +RL_ITERS = 12 +PSF_BOX = 25 + + +def measure_psf(img): + """Median-stack cutouts of isolated stars to get the frame's own PSF.""" + bkg = sep.Background(img, bw=64, bh=64, fw=3, fh=3) + sub = img - bkg.back() + o = sep.extract(sub, 25.0, err=bkg.globalrms, minarea=9, + deblend_cont=0.005) + o = o[(o["flag"] == 0) & (o["npix"] > 15) & (o["npix"] < 400)] + ny, nx = img.shape + h = PSF_BOX // 2 + # Isolated stars only: a neighbour inside the cutout would drag the wings. + keep = [] + xs, ys = o["x"], o["y"] + for i in range(len(o)): + if not (h + 2 < xs[i] < nx - h - 2 and h + 2 < ys[i] < ny - h - 2): + continue + d = np.hypot(xs - xs[i], ys - ys[i]) + if np.sort(d)[1] < 3 * PSF_BOX: + continue + keep.append(i) + keep = keep[:120] + stack = [] + for i in keep: + cx, cy = int(round(xs[i])), int(round(ys[i])) + cut = sub[cy - h:cy + h + 1, cx - h:cx + h + 1].astype(np.float64) + peak = cut.max() + if peak > 0 and peak < 60000: # skip anything near saturation + stack.append(cut / cut.sum()) + psf = np.median(np.stack(stack), axis=0) + psf[psf < 0] = 0 + psf /= psf.sum() + print(f"PSF from {len(stack)} isolated stars, " + f"peak fraction {psf.max():.4f}") + return psf.astype(np.float32) + + +def star_mask(img, rms): + """Feathered mask over stars, used to keep deconvolution off them. + + Richardson-Lucy rings around any source whose profile is steeper than the + PSF model can account for, and on a star field that means a dark annulus + round every bright star. Excluding stars from the deconvolution entirely + is the standard cure: the galaxy is what needed sharpening anyway. + """ + bkg = sep.Background(img, bw=64, bh=64, fw=3, fh=3) + sub = img - bkg.back() + o = sep.extract(sub, 6.0, err=rms, minarea=6, deblend_cont=0.005) + o = o[o["npix"] < 3000] + ny, nx = img.shape + m = np.zeros((ny, nx), np.float32) + yy, xx = np.mgrid[-20:21, -20:21] + rr = np.hypot(xx, yy) + for x, y, npix, flux in zip(o["x"], o["y"], o["npix"], o["flux"]): + # Bright stars ring over a wider radius than faint ones. + r = float(np.clip(2.5 * np.sqrt(npix / np.pi) + 3.0, 5, 18)) + cx, cy = int(round(x)), int(round(y)) + x0, x1 = max(0, cx - 20), min(nx, cx + 21) + y0, y1 = max(0, cy - 20), min(ny, cy + 21) + patch = (rr <= r).astype(np.float32)[ + (y0 - cy + 20):(y1 - cy + 20), (x0 - cx + 20):(x1 - cx + 20)] + np.maximum(m[y0:y1, x0:x1], patch, out=m[y0:y1, x0:x1]) + print(f" star mask over {len(o)} sources, {m.mean():.2%} of the frame") + return np.clip(gaussian_filter(m, 2.5), 0, 1) + + +def deconvolve(img, psf): + """Richardson-Lucy on the bright signal, feathered back into the noise.""" + rms = 1.4826 * np.median(np.abs(img - np.median(img))) + pedestal = 5.0 * rms + positive = np.clip(img + pedestal, 1e-3, None).astype(np.float32) + scale = float(positive.max()) + out = richardson_lucy(positive / scale, psf, num_iter=RL_ITERS, + clip=False) * scale - pedestal + # Two weights multiply together: apply the result only where there is + # signal to sharpen, and only where there is no star to ring. + signal = np.clip((img - 3.0 * rms) / (20.0 * rms), 0.0, 1.0) + w = (signal * (1.0 - star_mask(img, rms))).astype(np.float32) + return (out * w + img * (1.0 - w)).astype(np.float32) + + +def build_rgb(lum_lin, rgb_lin): + """The colour half of compose.main(), reused verbatim in spirit.""" + lum, params = C.autostretch(lum_lin) + rgb = np.empty_like(rgb_lin) + for i in range(3): + ch = rgb_lin[:, :, i] + sky = np.median(ch) + mad = 1.4826 * np.median(np.abs(ch - sky)) + black = sky - 2.8 * mad + white = np.percentile(ch, 99.995) + rgb[:, :, i] = C.mtf(np.clip((ch - black) / (white - black), 0, 1), + params["midtone"]) + return lum, rgb + + +def main(): + data, hdr = C.load() + shape = data["Luminance"].shape + mask = C.galaxy_mask(shape) + for name in C.CHANNELS: + data[name], *_ = C.remove_gradient(data[name], mask) + o, flux = C.star_photometry(data["Luminance"], data) + good = (flux["Red"] > 0) & (flux["Green"] > 0) & (flux["Blue"] > 0) + gr = np.median(flux["Green"][good] / flux["Red"][good]) + gb = np.median(flux["Green"][good] / flux["Blue"][good]) + data["Red"] *= gr + data["Blue"] *= gb + + psf = measure_psf(data["Luminance"]) + print(f"deconvolving luminance, {RL_ITERS} Richardson-Lucy iterations") + lum_lin = deconvolve(data["Luminance"], psf) + rgb_lin = np.dstack([data["Red"], data["Green"], data["Blue"]]) + del data + + lum, rgb = build_rgb(lum_lin, rgb_lin) + del rgb_lin, lum_lin + + sky_med = [float(np.median(rgb[:, :, i][~mask])) for i in range(3)] + target = float(np.mean(sky_med)) + for i in range(3): + rgb[:, :, i] = np.clip(rgb[:, :, i] - (sky_med[i] - target), 0.0, 1.0) + rgb_lum = rgb.mean(axis=2, keepdims=True) + chroma = rgb - rgb_lum + for i in range(3): + chroma[:, :, i] = gaussian_filter(median_filter(chroma[:, :, i], 3), + 1.5) + rgb = np.clip(rgb_lum + chroma * C.SATURATION, 0.0, 1.0) + del chroma, rgb_lum + + ratio = lum / np.maximum(rgb.mean(axis=2), 1e-5) + out = np.clip(rgb * ratio[:, :, None], 0.0, 1.0) + del ratio, rgb + + neutral = 0.5 * (out[:, :, 0] + out[:, :, 2]) + green = out[:, :, 1] + out[:, :, 1] = np.where(green > neutral, green * 0.15 + neutral * 0.85, + green) + + u8 = (out * 255 + 0.5).astype(np.uint8) + Image.fromarray(u8).save(layout.path("NGC5128-img-deconvolved.png")) + tifffile.imwrite(layout.path("NGC5128-img-deconvolved.tif"), + (out * 65535 + 0.5).astype(np.uint16), photometric="rgb") + print("wrote NGC5128-img-deconvolved.png / .tif") + + # A crop for print, taken from the deconvolved version at full resolution. + ny, nx = out.shape[:2] + cw, ch = 2600, 1950 + crop = u8[ny // 2 - ch // 2:ny // 2 + ch // 2, + nx // 2 - cw // 2:nx // 2 + cw // 2] + Image.fromarray(crop).save(layout.path("NGC5128-img-core-print.jpg"), + quality=95) + print(f"wrote NGC5128-img-core-print.jpg ({cw}x{ch}, " + f"{cw * 0.5376 / 60:.1f}' x {ch * 0.5376 / 60:.1f}')") + del crop, u8 + + +if __name__ == "__main__": + main() diff --git a/session-scripts/final.py b/session-scripts/final.py new file mode 100644 index 0000000..214f0ed --- /dev/null +++ b/session-scripts/final.py @@ -0,0 +1,311 @@ +"""The three renderings, from minimal to everything we learned. + +Same 24 subs, same alignment, same field, same crop. The ONLY variable is how +much is done to the pixels, so the three are directly comparable. + + NGC5128-final-1-stacked.png stack + a standard stretch, nothing else + NGC5128-final-2-processed.png conventional processing: gradient, colour, denoise + NGC5128-final-3-best.png the above, corrected by what the analyses established + +What image 3 does differently, and why each change is justified by a measured +result rather than by taste: + +1. **Background fit that does not eat the halo.** The surface photometry + measured the far field sitting at -17.9 ADU/px instead of zero: the plane + fitted for image 2 absorbed real halo light, because Centaurus A's halo + fills this field and there is no genuinely empty corner to fit to. Image 3 + fits `channel = a * galaxy_model + plane` simultaneously, using the isophote + model from the surface photometry, so the plane can only take the part that + is actually a gradient. The halo survives. + +2. **Photometric colour calibration.** Image 2 assumed the average field star + is grey. Image 3 uses Gaia BP-RP to pick the 1313 stars that are genuinely + solar-coloured and neutralises on those alone, which does not care what mix + of spectral types this particular field contains. + +3. **The core, recovered.** The nucleus was never saturated - it peaks at 1944 + ADU against a ~63000 clip. A second tone curve scaled to the galaxy rather + than to field stars restores the bulge gradient the single curve flattened. + +4. **Deconvolution that does not ring.** PSF measured from the frame's own + isolated stars, applied only where there is signal and never on a star. + +5. **Noise reduction that cannot eat clusters.** The globular cluster survey + showed this field contains 289 cluster candidates that look exactly like + faint stars. Smoothing is therefore driven by a Gaia star mask plus a + signal mask, so every compact source - foreground star or cluster - is + excluded from it. +""" +import os +import sys + +import numpy as np +import sep +import tifffile +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.wcs import WCS +from PIL import Image +from scipy.ndimage import gaussian_filter, median_filter +from skimage.restoration import denoise_tv_chambolle, richardson_lucy + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import compose as C # noqa: E402 +import enhance as E # noqa: E402 + +import layout + +SRC = layout.SESSION +OUT = layout.SESSION +# The finished renderings live in their own directory: they are the +# deliverables, and keeping them apart from the masters, the +# intermediates and the analysis figures makes it obvious which +# files are meant to be looked at. +FINAL = layout.path("final") +ORIG = layout.path("original") +CROP = 48 +FILTERS = ["Luminance", "Red", "Green", "Blue"] + + +def save(arr, stem, title): + os.makedirs(FINAL, exist_ok=True) + u8 = (np.clip(arr, 0, 1) * 255 + 0.5).astype(np.uint8) + Image.fromarray(u8).save(layout.path(f"{stem}.png")) + tifffile.imwrite(layout.path(f"{stem}.tif"), + (np.clip(arr, 0, 1) * 65535 + 0.5).astype(np.uint16), + photometric="rgb") + prev = Image.fromarray(u8) + prev.thumbnail((2400, 2400), Image.LANCZOS) + prev.save(layout.path(f"{stem}-preview.jpg"), quality=93) + print(f" wrote {stem}.png / .tif / -preview.jpg [{title}]") + + +def lrgb(lum, rgb): + """Take hue from the colour channels, brightness from the luminance.""" + ratio = lum / np.maximum(rgb.mean(axis=2), 1e-5) + return np.clip(rgb * ratio[:, :, None], 0.0, 1.0) + + +# --------------------------------------------------------------- image 1 +def image1(): + """Stack plus a standard stretch. No corrections of any kind. + + Uses the plain-mean alignment-only stack, so there is not even outlier + rejection: satellite trails and the moon's gradient are all present. The + stretch is the conventional one - a black point just below each channel's + own sky, a white point at its 99.995th percentile, and a single shared + midtone taken from the luminance so no colour balancing sneaks in through + the tone curve. + """ + print("image 1: stack + standard stretch") + data = {} + for f in FILTERS: + d = fits.getdata(layout.path(f"original-{f}.fit")) + data[f] = np.array(d[CROP:-CROP, CROP:-CROP], dtype=np.float32) + lum_lin = data["Luminance"] + lum, params = C.autostretch(lum_lin) + print(f" shared midtone {params['midtone']:.4f}") + rgb = np.empty(lum.shape + (3,), np.float32) + for i, f in enumerate(("Red", "Green", "Blue")): + ch = data[f] + sky = np.median(ch) + mad = 1.4826 * np.median(np.abs(ch - sky)) + black, white = sky - 2.8 * mad, np.percentile(ch, 99.995) + rgb[:, :, i] = C.mtf(np.clip((ch - black) / (white - black), 0, 1), + params["midtone"]) + print(f" {f:6s} black {black:7.1f} white {white:8.1f} ADU") + del data + out = lrgb(lum, rgb) + save(out, "NGC5128-final-1-stacked", "align + mean + stretch") + return out.shape + + +# --------------------------------------------------------------- image 3 +def solar_white_balance(lum, channels, wcs, shape): + """Neutralise on stars that are genuinely solar-coloured, per Gaia BP-RP.""" + z = np.load(layout.path("_gaia_colours.npz")) + solar = np.abs(z["bprp"] - 0.82) < 0.15 + sky = SkyCoord(z["ra"][solar] * u.deg, z["dec"][solar] * u.deg) + x, y = wcs.world_to_pixel(sky) + g = z["g"][solar] + ny, nx = shape + # Keep them away from the edges, off the galaxy's bright core, and out of + # saturation; faint ones carry too little signal in 20 minutes of colour. + ok = ((x > 40) & (x < nx - 40) & (y > 40) & (y < ny - 40) & + (g > 11.5) & (g < 16.5)) + x, y = x[ok], y[ok] + flux = {} + for f in ("Red", "Green", "Blue"): + img = np.ascontiguousarray(channels[f]) + fl, _, _ = sep.sum_circle(img, x, y, 6.0, subpix=5) + flux[f] = fl + good = (flux["Red"] > 0) & (flux["Green"] > 0) & (flux["Blue"] > 0) + gr = float(np.median(flux["Green"][good] / flux["Red"][good])) + gb = float(np.median(flux["Green"][good] / flux["Blue"][good])) + print(f" solar-analogue white balance on {int(good.sum())} stars: " + f"R x {gr:.4f}, B x {gb:.4f}") + return gr, gb + + +def fit_background(ch, model, star_mask): + """Solve ch = a*model + (plane) and subtract ONLY the plane. + + Fitting a plane on its own to a field this full of galaxy makes the plane + absorb halo light - measured at -17.9 ADU/px in the far field of image 2. + Including the galaxy model as a free component in the same least-squares + problem gives the fit something else to attribute that light to. + """ + ny, nx = ch.shape + ys, xs = np.mgrid[0:ny:8, 0:nx:8] + m = model[::8, ::8] + v = ch[::8, ::8] + keep = (star_mask[::8, ::8] == 0) & np.isfinite(v) + A = np.column_stack([m[keep], xs[keep] / nx, ys[keep] / ny, + np.ones(keep.sum())]) + coef, *_ = np.linalg.lstsq(A, v[keep], rcond=None) + for _ in range(3): # clip and refit + pred = A @ coef + r = v[keep] - pred + s = 1.4826 * np.median(np.abs(r - np.median(r))) + m2 = np.abs(r - np.median(r)) < 2.5 * s + coef, *_ = np.linalg.lstsq(A[m2], v[keep][m2], rcond=None) + yy, xx = np.mgrid[0:ny, 0:nx] + plane = (coef[1] * xx / nx + coef[2] * yy / ny + coef[3]).astype(np.float32) + print(f" model amplitude {coef[0]:.4f}, plane offset {coef[3]:8.2f} ADU") + return ch - plane + + +def image3(): + print("image 3: science-informed") + model_full = fits.getdata(layout.path("sb-model.fits")).astype( + np.float32) + star_full = fits.getdata(layout.path("sb-mask-stars.fits")) + with fits.open(layout.path("master-Luminance.fit")) as hd: + wcs_full = WCS(hd[0].header, naxis=2) + + data = {} + for f in FILTERS: + d = fits.getdata(layout.path(f"master-{f}.fit")) + data[f] = np.array(d, dtype=np.float32) + model = model_full + smask = star_full + print(" background fit with the galaxy model as a free component:") + for f in FILTERS: + print(f" {f}") + data[f] = fit_background(data[f], model, smask) + del model_full, star_full, model, smask + + shape_full = data["Luminance"].shape + gr, gb = solar_white_balance(data["Luminance"], data, wcs_full, shape_full) + data["Red"] *= gr + data["Blue"] *= gb + + for f in FILTERS: + data[f] = np.ascontiguousarray(data[f][CROP:-CROP, CROP:-CROP]) + shape = data["Luminance"].shape + + psf = E.measure_psf(data["Luminance"]) + print(" deconvolving luminance (star-protected)") + lum_lin = E.deconvolve(data["Luminance"], psf) + rgb_lin = np.dstack([data["Red"], data["Green"], data["Blue"]]) + del data + + # Two tone curves, blended: the faint one for sky and halo, one scaled to + # the galaxy for the core the single curve flattened. + ny, nx = shape + h = 500 + core = lum_lin[ny // 2 - h:ny // 2 + h, nx // 2 - h:nx // 2 + h] + peak = float(median_filter(core, size=41).max()) + lum_faint, params = C.autostretch(lum_lin) + hi = peak * 1.15 + lum_bright = C.mtf(np.clip((lum_lin - params["black"]) / + (hi - params["black"]), 0, 1), 0.35) + w = gaussian_filter(np.clip((lum_faint - 0.55) / 0.35, 0, 1).astype( + np.float32), 8.0) + lum = np.clip(lum_faint * (1 - w) + lum_bright * w, 0, 1) + print(f" galaxy peak {peak:.0f} ADU, HDR blend over " + f"{float((w > 0.05).mean()):.2%} of frame") + + rgb = np.empty_like(rgb_lin) + for i in range(3): + ch = rgb_lin[:, :, i] + sky = np.median(ch) + mad = 1.4826 * np.median(np.abs(ch - sky)) + black, white = sky - 2.8 * mad, np.percentile(ch, 99.995) + cf = C.mtf(np.clip((ch - black) / (white - black), 0, 1), + params["midtone"]) + pc = float(median_filter( + ch[ny // 2 - h:ny // 2 + h, nx // 2 - h:nx // 2 + h], 41).max()) + cb = C.mtf(np.clip((ch - black) / (pc * 1.15 - black), 0, 1), 0.35) + rgb[:, :, i] = cf * (1 - w) + cb * w + del rgb_lin + + # Sky neutralisation, measured where the galaxy model says there is no + # galaxy rather than outside an arbitrary ellipse. + mask = C.galaxy_mask(shape) + sky_med = [float(np.median(rgb[:, :, i][~mask])) for i in range(3)] + target = float(np.mean(sky_med)) + for i in range(3): + rgb[:, :, i] = np.clip(rgb[:, :, i] - (sky_med[i] - target), 0, 1) + + rgb_lum = rgb.mean(axis=2, keepdims=True) + chroma = rgb - rgb_lum + for i in range(3): + chroma[:, :, i] = gaussian_filter(median_filter(chroma[:, :, i], 3), + 1.5) + rgb = np.clip(rgb_lum + chroma * 1.4, 0, 1) + del chroma, rgb_lum + + detail = lum - gaussian_filter(lum, 2.0) + protect = np.clip((lum - 0.10) * 4.0, 0.0, 1.0) + lum = np.clip(lum + 0.35 * detail * protect, 0, 1) + + # Denoise the sky only. The cluster survey found 289 cluster candidates + # that look like faint stars, so compact sources are excluded from the + # smoothing along with the galaxy itself. + z = np.load(layout.path("_gaia_deep.npz")) + gx, gy = wcs_full.world_to_pixel(SkyCoord(z["ra"] * u.deg, + z["dec"] * u.deg)) + gx, gy = gx - CROP, gy - CROP + point = np.zeros(shape, np.float32) + R = 14 + yy, xx = np.mgrid[-R:R + 1, -R:R + 1] + rr = np.hypot(xx, yy) + for x, y, g in zip(gx, gy, z["g"]): + if not (R < x < nx - R and R < y < ny - R): + continue + r = float(np.clip(16.0 - 0.7 * (g - 8.0), 4, R - 1)) + cx, cy = int(x), int(y) + patch = (rr <= r).astype(np.float32) + sl = (slice(cy - R, cy + R + 1), slice(cx - R, cx + R + 1)) + np.maximum(point[sl], patch, out=point[sl]) + keep_sharp = np.clip(protect + gaussian_filter(point, 2.0), 0, 1) + smooth = denoise_tv_chambolle(lum, weight=0.012) + lum = np.clip(lum * keep_sharp + smooth * (1 - keep_sharp), 0, 1) + print(f" sky denoise applied to {float((keep_sharp < 0.5).mean()):.1%} " + f"of the frame; stars and clusters excluded") + del detail, protect, smooth, point, keep_sharp + + out = lrgb(lum, rgb) + neutral = 0.5 * (out[:, :, 0] + out[:, :, 2]) + green = out[:, :, 1] + out[:, :, 1] = np.where(green > neutral, green * 0.15 + neutral * 0.85, + green) + save(out, "NGC5128-final-3-best", "science-informed") + + +if __name__ == "__main__": + image1() + print("image 2: the conventional pipeline output (compose.py)") + os.makedirs(FINAL, exist_ok=True) + for ext in ("png", "tif"): + src = layout.path(f"NGC5128-LRGB.{ext}") + dst = layout.path(f"NGC5128-final-2-processed.{ext}") + with open(src, "rb") as a, open(dst, "wb") as b: + b.write(a.read()) + im = Image.open(layout.path("NGC5128-final-2-processed.png")) + im.thumbnail((2400, 2400), Image.LANCZOS) + im.save(layout.path("NGC5128-final-2-processed-preview.jpg"), quality=93) + print(" wrote NGC5128-final-2-processed.png / .tif / -preview.jpg") + image3() diff --git a/session-scripts/gaia_colours.py b/session-scripts/gaia_colours.py new file mode 100644 index 0000000..6dfe0f2 --- /dev/null +++ b/session-scripts/gaia_colours.py @@ -0,0 +1,49 @@ +"""Fetch Gaia DR3 BP-RP colours for the field, cached for colour calibration. + +The first composite balanced colour on "the average field star is grey", which +is a workable fudge but is biased by whatever mix of spectral types the field +happens to contain. With real colours available, a much better anchor exists: +pick the stars that actually ARE solar-coloured (BP-RP near 0.82) and force +those to neutral. That is the same principle as a photometric colour +calibration, without needing the filters' response curves. + +ESA's archive was down during this work, so this uses the VizieR mirror of the +identical catalogue. +""" +import os + +import numpy as np +from astropy import units as u +from astropy.coordinates import SkyCoord + +import layout + +OUT = layout.SESSION +CACHE = layout.path("_gaia_colours.npz") +CENTRE = SkyCoord("13h25m27.37s", "-43d01m10.9s") + +if os.path.exists(CACHE): + z = np.load(CACHE) + print(f"cached: {len(z['ra'])} stars with colours") +else: + from astroquery.vizier import Vizier + v = Vizier(columns=["RA_ICRS", "DE_ICRS", "Gmag", "BP-RP"], + column_filters={"Gmag": "<18", "BP-RP": ">-1"}, + row_limit=50000) + res = v.query_region(CENTRE, radius=0.45 * u.deg, catalog="I/355/gaiadr3") + t = res[0] + ok = ~np.isnan(np.asarray(t["BP-RP"], float)) + ra = np.asarray(t["RA_ICRS"], float)[ok] + dec = np.asarray(t["DE_ICRS"], float)[ok] + g = np.asarray(t["Gmag"], float)[ok] + bprp = np.asarray(t["BP-RP"], float)[ok] + np.savez_compressed(CACHE, ra=ra, dec=dec, g=g, bprp=bprp) + print(f"fetched {len(ra)} stars with BP-RP") + z = dict(ra=ra, dec=dec, g=g, bprp=bprp) + +bprp = z["bprp"] +solar = np.abs(bprp - 0.82) < 0.15 +print(f"BP-RP range {bprp.min():.2f} to {bprp.max():.2f}, " + f"median {np.median(bprp):.2f}") +print(f"solar-coloured stars (BP-RP 0.67-0.97): {solar.sum()}") +print(f"G range {z['g'].min():.1f} to {z['g'].max():.1f}") diff --git a/session-scripts/gc-bwtrial.py b/session-scripts/gc-bwtrial.py new file mode 100644 index 0000000..10212ba --- /dev/null +++ b/session-scripts/gc-bwtrial.py @@ -0,0 +1,75 @@ +"""Choose the background mesh size EMPIRICALLY, using the 589 SIMBAD-catalogued +Cen A globular clusters that fall in the field as a truth set. For each mesh +size we measure (a) how many known GCs are recovered, split by projected radius, +and (b) how many total detections there are (a proxy for spurious detections).""" +import os, numpy as np, sep, warnings +from astropy.io import fits +from astropy.wcs import WCS + +import layout +warnings.filterwarnings("ignore") +S = layout.SESSION +NUC_RA, NUC_DEC = 201.365063, -43.019113 +PIXSCALE = 0.5376 + + +def gk(fwhm=5.0): + sig = fwhm / 2.3548 + n = int(2 * round(3 * sig) + 1) + r = np.arange(n) - n // 2 + xx, yy = np.meshgrid(r, r) + k = np.exp(-(xx ** 2 + yy ** 2) / (2 * sig ** 2)) + return (k / k.sum()).astype(np.float32) + + +def match(ra1, d1, ra2, d2, tol_as): + """brute-force nearest match, small catalogues""" + cd = np.cos(np.radians(d1.mean())) + idx = np.full(len(ra1), -1) + sep_as = np.full(len(ra1), 9e9) + for i in range(len(ra1)): + dd = np.hypot((ra2 - ra1[i]) * cd, d2 - d1[i]) * 3600.0 + j = np.argmin(dd) + if dd[j] < tol_as: + idx[i] = j + sep_as[i] = dd[j] + return idx, sep_as + + +z = np.load(layout.path("_simbad.npz"), allow_pickle=True) +m = z["otype"] == "GlC" +gra, gdec = z["ra"][m].astype(float), z["dec"][m].astype(float) + +hdr = fits.getheader(layout.path("master-Luminance.fit")) +w = WCS(hdr) +ny, nx = hdr["NAXIS2"], hdr["NAXIS1"] +gx, gy = w.all_world2pix(gra, gdec, 0) +inf = (gx > 20) & (gx < nx - 20) & (gy > 20) & (gy < ny - 20) +gra, gdec, gx, gy = gra[inf], gdec[inf], gx[inf], gy[inf] +cx, cy = [float(v) for v in w.all_world2pix(NUC_RA, NUC_DEC, 0)] +grad = np.hypot(gx - cx, gy - cy) * PIXSCALE / 60.0 +print("known GCs inside the frame: %d (r range %.2f - %.2f arcmin)" + % (len(gra), grad.min(), grad.max())) + +d = fits.getdata(layout.path("master-Luminance.fit")).astype(np.float32) +d = np.ascontiguousarray(d) +sep.set_extract_pixstack(3000000) +K = gk() + +bins = [(0, 2), (2, 4), (4, 8), (8, 14), (14, 30)] +print("\n%6s %7s | %s" % ("mesh", "Ndet", " ".join("%4.0f-%-4.0f'" % b for b in bins))) +for bw in (12, 16, 24, 32, 48, 64, 128): + b = sep.Background(d, bw=bw, bh=bw, fw=3, fh=3) + ds = d - b.back() + rm = b.rms() + o = sep.extract(ds, 3.0, err=rm, minarea=5, filter_kernel=K, + filter_type="matched", deblend_nthresh=32, + deblend_cont=0.005, clean=True) + ora, odec = w.all_pix2world(o["x"], o["y"], 0) + idx, sp = match(gra, gdec, ora, odec, 2.0) + row = [] + for lo, hi in bins: + s = (grad >= lo) & (grad < hi) + row.append("%3d/%-3d" % ((idx[s] >= 0).sum(), s.sum())) + print("%6d %7d | %s" % (bw, len(o), " ".join(row))) + del b, ds, rm, o diff --git a/session-scripts/gc-classify.py b/session-scripts/gc-classify.py new file mode 100644 index 0000000..922951a --- /dev/null +++ b/session-scripts/gc-classify.py @@ -0,0 +1,229 @@ +""" +gc-classify.py -- step 2. Turn raw detections into a globular cluster candidate +catalogue. + +Order of operations: + 1. quality cuts (SNR, sep flags, frame edge) + 2. photometric calibration of the R/G/B masters onto the Gaia BP/G/RP scale + using astrometrically-confirmed foreground stars + 3. foreground star rejection using Gaia DR3 ASTROMETRY (not mere presence in + Gaia -- see the notes; half of the known Cen A clusters are in Gaia) + 4. morphology: point-like vs extended, calibrated on the stellar locus + 5. magnitude window around the expected GC luminosity function + 6. cross-match to SIMBAD (truth set) and SCABS/Taylor+2017 + +Outputs: _gc_cat.npz (everything), gc-candidates.csv (the candidates) +""" +import os, sys, numpy as np +from astropy.coordinates import SkyCoord +import astropy.units as u +import warnings + +import layout +warnings.filterwarnings("ignore") + +S = layout.SESSION +PIXSCALE = 0.5376 +NUC_RA, NUC_DEC = 201.365063, -43.019113 +DIST_MPC = 3.8 +KPC_PER_ARCMIN = DIST_MPC * 1000.0 * (np.pi / 180.0) / 60.0 # 1.105 +MATCH_AS = 1.5 # astrometric match radius +PM_SIG = 4.0 # significance above which astrometry says "foreground" +# Faint limit set by the artificial-star tests, not by the nominal SNR-5 depth: +# recovery of injected point sources collapses from ~30% at G=19.75 to ~3% at +# G=20.25, and the fraction of candidates confirmed by published catalogues +# falls from 52% to 10% across the same step. Beyond G=20 we are not measuring +# clusters, we are measuring noise. +MAG_LO, MAG_HI = 17.5, 20.0 + + +def xmatch(c1, c2, tol_as): + i, d2, _ = c1.match_to_catalog_sky(c2) + ok = d2.arcsec < tol_as + return i, d2.arcsec, ok + + +def main(): + d = dict(np.load(layout.path("_gc_raw.npz"))) + n = len(d["x"]) + print("raw detections: %d" % n) + cat = SkyCoord(d["ra"], d["dec"], unit="deg") + nuc = SkyCoord(NUC_RA, NUC_DEC, unit="deg") + r_arcmin = cat.separation(nuc).arcmin + r_kpc = r_arcmin * KPC_PER_ARCMIN + + # ---------------------------------------------------------- 1. quality + ny, nx = 3194, 4788 + edge = 30 + q_edge = (d["x"] > edge) & (d["x"] < nx - edge) & (d["y"] > edge) & (d["y"] < ny - edge) + # sep flag bits: 1=OBJ_MERGED 2=OBJ_TRUNC 4=OBJ_DOVERFLOW 8=OBJ_SINGU + # Bit 1 (had a neighbour in the detection footprint before deblending) is + # NOT a defect -- it fires for a third of the KNOWN clusters, because the + # inner field is crowded. Reject only genuine defects. + q_flag = (d["sepflag"].astype(int) & 0b1110) == 0 + q_snr = d["snr"] >= 5.0 + q_pos = d["flux"] > 0 + good = q_edge & q_pos + print(" in-frame & positive flux: %d ; +clean sep flags: %d ; +SNR>=5: %d" + % (good.sum(), (good & q_flag).sum(), (good & q_flag & q_snr).sum())) + + # ---------------------------------------------------------- 2. Gaia match + ga = np.load(layout.path("_gaia_astrom.npz")) + gcat = SkyCoord(ga["ra"], ga["dec"], unit="deg") + gi, gsep, ghit = xmatch(cat, gcat, MATCH_AS) + print(" Gaia DR3 counterpart within %.1f\": %d / %d" % (MATCH_AS, ghit.sum(), n)) + + plx, eplx = ga["plx"][gi], ga["eplx"][gi] + pmr, epmr = ga["pmra"][gi], ga["epmra"][gi] + pmd, epmd = ga["pmdec"][gi], ga["epmdec"][gi] + with np.errstate(invalid="ignore", divide="ignore"): + sig_plx = np.abs(plx / eplx) + sig_pm = np.sqrt((pmr / epmr) ** 2 + (pmd / epmd) ** 2) + has_astrom = ghit & np.isfinite(sig_pm) + # a Milky Way star: significant parallax OR significant proper motion + is_star = has_astrom & ((sig_plx > PM_SIG) | (sig_pm > PM_SIG)) + # a Gaia source whose astrometry is consistent with zero -> distant + is_stationary = has_astrom & ~is_star + no_gaia = ~ghit + no_astrom = ghit & ~has_astrom # in Gaia but 2-parameter solution only + print(" -> astrometric foreground stars: %d" % is_star.sum()) + print(" -> Gaia sources astrometrically stationary: %d" % is_stationary.sum()) + print(" -> in Gaia, no astrometry: %d ; not in Gaia: %d" % (no_astrom.sum(), no_gaia.sum())) + + gaia_g = np.where(ghit, ga["g"][gi], np.nan) + gaia_bprp = np.where(ghit, ga["bprp"][gi], np.nan) + + # ------------------------------------------- 3. calibrate R/G/B onto Gaia + zp = {} + for key, gband in (("B", "bp"), ("V", "g"), ("R", "rp")): + f = d["flux_" + key] + ref = np.where(ghit, ga[gband][gi], np.nan) + m = is_star & good & q_flag & (f > 0) & np.isfinite(ref) & (ref > 13) & (ref < 18) + with np.errstate(invalid="ignore", divide="ignore"): + off = ref[m] + 2.5 * np.log10(f[m]) + zp[key] = np.median(off) + print(" ZP_%s = %.3f (N=%d stars, scatter %.3f mag)" + % (key, zp[key], m.sum(), np.std(off - np.median(off)))) + mags = {} + magerrs = {} + for key in ("B", "V", "R"): + f = np.clip(d["flux_" + key], 1e-9, None) + with np.errstate(invalid="ignore", divide="ignore"): + mags[key] = np.where(d["flux_" + key] > 0, -2.5 * np.log10(f) + zp[key], np.nan) + magerrs[key] = 1.0857 * d["fluxerr_" + key] / f + BmR = mags["B"] - mags["R"] + BmV = mags["B"] - mags["V"] + VmR = mags["V"] - mags["R"] + + # ---------------------------------------------------------- 4. morphology + fwhm = 2.0 * d["rhalf"] # Gaussian: r_half = FWHM/2 + with np.errstate(invalid="ignore", divide="ignore"): + elong = d["a"] / np.clip(d["b"], 1e-6, None) + # stellar locus from bright unsaturated confirmed stars + ref = is_star & good & q_flag & (d["mag"] > 15) & (d["mag"] < 18) + f_med = np.median(fwhm[ref]); f_sig = 1.4826 * np.median(np.abs(fwhm[ref] - f_med)) + print(" stellar locus: FWHM = %.2f +/- %.2f px (%.2f\") from %d stars" + % (f_med, f_sig, f_med * PIXSCALE, ref.sum())) + # Asymmetric band, on purpose. A Cen A cluster (half-light radius a few pc, + # i.e. <0.2") is only marginally broadened by 2.7" seeing, but crowding and + # residual galaxy structure push measured sizes UP, never down. Known + # clusters run to FWHM ~6.6 px while stars sit at 5.0. So the lower edge is + # tight (rejects cosmic rays / noise spikes) and the upper edge is loose + # (rejects only obviously extended objects, i.e. resolved background + # galaxies). With 2.7" seeing this separation is weak -- see the notes. + FW_MIN = f_med - 3.0 * f_sig + FW_MAX = f_med + 3.0 # px, ~1.6 x the stellar FWHM + compact = (fwhm > FW_MIN) & (fwhm < FW_MAX) & (elong < 2.0) + print(" point-like acceptance band: %.2f < FWHM < %.2f px ; elong < 2.0" + % (FW_MIN, FW_MAX)) + + # ---------------------------------------------------------- 5. magnitude + inmag = (d["mag"] > MAG_LO) & (d["mag"] < MAG_HI) + + # ---------------------------------------------------------- selection + base = good & q_flag & q_snr + cand = base & ~is_star & compact & inmag + print("\nCANDIDATES: %d" % cand.sum()) + + why = np.array(["-"] * n, dtype=object) + why[cand & no_gaia] = "no-gaia+compact" + why[cand & is_stationary] = "gaia-stationary+compact" + why[cand & no_astrom] = "gaia-noastrom+compact" + + cls = np.array(["other"] * n, dtype=object) + cls[base & is_star] = "foreground-star" + cls[base & ~is_star & ~compact & inmag] = "extended" + cls[cand] = "gc-candidate" + + # ---------------------------------------------------------- 6. truth sets + z = np.load(layout.path("_simbad.npz"), allow_pickle=True) + sim_ot = z["otype"].astype(str) + scat = SkyCoord(z["ra"].astype(float), z["dec"].astype(float), unit="deg") + si, ssep, shit = xmatch(cat, scat, 2.0) + simbad_type = np.where(shit, sim_ot[si], "") + simbad_name = np.where(shit, z["name"].astype(str)[si], "") + + sc_type = np.array([""] * n, dtype=object) + scabs_ok = os.path.exists(layout.path("_cena_gc_ref.npz")) + scabs_prob = np.full(n, np.nan) + scabs_v = np.full(n, np.nan) + scabs_hit = np.zeros(n, bool) # positional match, regardless of listed prob + if scabs_ok: + r = np.load(layout.path("_cena_gc_ref.npz")) + rc = SkyCoord(r["ra"], r["dec"], unit="deg") + ri, rsep, rhit = xmatch(cat, rc, 2.0) + scabs_prob = np.where(rhit, r["prob"][ri], np.nan) + # the published table uses -1 as a "no value" sentinel + scabs_prob = np.where(scabs_prob < 0, np.nan, scabs_prob) + scabs_v = np.where(rhit, r["vmag"][ri], np.nan) + scabs_hit = rhit + print(" SCABS (Taylor+2017) matches among all detections: %d" % rhit.sum()) + print(" SCABS matches among candidates: %d / %d (%.0f%%)" + % ((rhit & cand).sum(), cand.sum(), 100 * (rhit & cand).mean() / max(cand.mean(), 1e-9))) + print(" SIMBAD GlC matches among candidates: %d" + % ((simbad_type == "GlC") & cand).sum()) + + out = dict(d) + out.update(r_arcmin=r_arcmin, r_kpc=r_kpc, fwhm=fwhm, elong=elong, + mag_B=mags["B"], mag_V=mags["V"], mag_R=mags["R"], + magerr_B=magerrs["B"], magerr_V=magerrs["V"], magerr_R=magerrs["R"], + BmR=BmR, BmV=BmV, VmR=VmR, + gaia_g=gaia_g, gaia_bprp=gaia_bprp, gaia_sep=gsep, + sig_plx=np.where(ghit, sig_plx, np.nan), + sig_pm=np.where(ghit, sig_pm, np.nan), + is_star=is_star, is_stationary=is_stationary, no_gaia=no_gaia, + no_astrom=no_astrom, compact=compact, cand=cand, base=base, + good=good, q_flag=q_flag, q_snr=q_snr, inmag=inmag, + simbad_type=simbad_type.astype(str), simbad_name=simbad_name.astype(str), + scabs_prob=scabs_prob, scabs_v=scabs_v, scabs_hit=scabs_hit, + why=why.astype(str), cls=cls.astype(str), + FW_MIN=FW_MIN, FW_MAX=FW_MAX, f_med=f_med, f_sig=f_sig, + zpB=zp["B"], zpV=zp["V"], zpR=zp["R"]) + np.savez(layout.path("_gc_cat.npz"), **out) + + # ---------------------------------------------------------- CSV + idx = np.where(cand)[0] + order = idx[np.argsort(r_arcmin[idx])] + lines = ["id,ra_deg,dec_deg,x_px,y_px,G_mag,G_magerr,snr,mag_B,mag_V,mag_R," + "B_R,B_V,V_R,fwhm_px,fwhm_arcsec,elong,r_arcmin,r_kpc,keep_flag," + "simbad_type,simbad_name,scabs_prob,scabs_V"] + for k, i in enumerate(order, 1): + def f(v, p=3): + return "" if not np.isfinite(v) else ("%.*f" % (p, v)) + lines.append(",".join([ + "GCC%04d" % k, "%.6f" % d["ra"][i], "%.6f" % d["dec"][i], + "%.2f" % d["x"][i], "%.2f" % d["y"][i], + f(d["mag"][i]), f(d["magerr"][i]), "%.1f" % d["snr"][i], + f(mags["B"][i]), f(mags["V"][i]), f(mags["R"][i]), + f(BmR[i]), f(BmV[i]), f(VmR[i]), + "%.2f" % fwhm[i], "%.2f" % (fwhm[i] * PIXSCALE), "%.2f" % elong[i], + "%.3f" % r_arcmin[i], "%.3f" % r_kpc[i], why[i], + simbad_type[i], '"%s"' % simbad_name[i], + f(scabs_prob[i], 2), f(scabs_v[i], 2)])) + with open(layout.path("gc-candidates.csv"), "w") as fh: + fh.write("\n".join(lines) + "\n") + print("wrote gc-candidates.csv (%d rows)" % len(order)) + + +if __name__ == "__main__": + main() diff --git a/session-scripts/gc-complete-inner.py b/session-scripts/gc-complete-inner.py new file mode 100644 index 0000000..1b057a9 --- /dev/null +++ b/session-scripts/gc-complete-inner.py @@ -0,0 +1,120 @@ +""" +gc-complete-inner.py -- step 3b. Artificial stars concentrated on the INNER field. + +The uniform injection run (gc-complete.py) puts only ~2% of its fakes inside +3 arcmin, because that annulus is a tiny fraction of the frame area. That left +the innermost completeness bins under-sampled, and the correction visibly failed +to flatten the foreground-star control inside ~4 arcmin. This run injects only +within r < 7 arcmin so the inner bins are properly measured. The two runs are +merged into _gc_complete_all.npz. +""" +import os, time, gc, numpy as np, sep, warnings +from astropy.io import fits +from astropy.wcs import WCS + +import layout +warnings.filterwarnings("ignore") + +S = layout.SESSION +BW, FW, THRESH, MINAREA, APR = 16, 3, 3.0, 5, 5.0 +ZP_L, PIXSCALE = 27.941, 0.5376 +NUC_RA, NUC_DEC = 201.365063, -43.019113 +NRUN, NPER = 6, 900 +MAGS = (17.5, 20.5) +RMAX = 7.0 +HALF = 15 +RNG = np.random.default_rng(31415) + + +def gk(fwhm=5.0): + sig = fwhm / 2.3548 + n = int(2 * round(3 * sig) + 1) + r = np.arange(n) - n // 2 + xx, yy = np.meshgrid(r, r) + k = np.exp(-(xx ** 2 + yy ** 2) / (2 * sig ** 2)) + return (k / k.sum()).astype(np.float32) + + +def main(): + t0 = time.time() + hdr = fits.getheader(layout.path("master-Luminance.fit")) + w = WCS(hdr) + ny, nx = hdr["NAXIS2"], hdr["NAXIS1"] + cx, cy = [float(v) for v in w.all_world2pix(NUC_RA, NUC_DEC, 0)] + cat = dict(np.load(layout.path("_gc_cat.npz"), allow_pickle=True)) + FW_MIN, FW_MAX = float(cat["FW_MIN"]), float(cat["FW_MAX"]) + psf = np.load(layout.path("_gc_psf.npy")) + apfrac = float(np.load(layout.path("_gc_complete.npz"))["apfrac"]) + sep.set_extract_pixstack(1000000) + K = gk() + rmax_px = RMAX * 60 / PIXSCALE + rm_, rr_, ro_ = [], [], [] + + for run in range(NRUN): + # re-read rather than keeping a pristine copy in memory: only ~3.5 GB + # is free and each master is 61 MB + img = np.ascontiguousarray( + fits.getdata(layout.path('master-Luminance.fit')).astype(np.float32)) + # uniform in area within the circle, clipped to the frame + th = RNG.uniform(0, 2 * np.pi, NPER * 3) + rad = rmax_px * np.sqrt(RNG.uniform(0, 1, NPER * 3)) + xs = cx + rad * np.cos(th); ys = cy + rad * np.sin(th) + ok = ((xs > HALF + 5) & (xs < nx - HALF - 5) & + (ys > HALF + 5) & (ys < ny - HALF - 5)) + xs, ys = xs[ok][:NPER], ys[ok][:NPER] + mags = RNG.uniform(*MAGS, len(xs)) + ftot = 10 ** (-0.4 * (mags - ZP_L)) / apfrac + for xi, yi, f in zip(xs, ys, ftot): + ix, iy = int(round(xi)), int(round(yi)) + img[iy - HALF:iy + HALF + 1, ix - HALF:ix + HALF + 1] += f * psf + + b = sep.Background(img, bw=BW, bh=BW, fw=FW, fh=FW) + ds = img - b.back(); rmm = b.rms() + o = sep.extract(ds, THRESH, err=rmm, minarea=MINAREA, filter_kernel=K, + filter_type="matched", deblend_nthresh=32, + deblend_cont=0.005, clean=True) + fl, fe, _ = sep.sum_circle(ds, o["x"], o["y"], APR, err=rmm, subpix=5) + fl2, _, _ = sep.sum_circle(ds, o["x"], o["y"], 2 * APR, err=rmm, subpix=5) + rh, _ = sep.flux_radius(ds, o["x"], o["y"], np.full(len(o), 6 * APR), 0.5, + normflux=fl2, subpix=5) + fet = np.sqrt(fe ** 2 + np.clip(fl, 0, None) / (0.2467 * 12.0)) + snr = fl / np.clip(fet, 1e-9, None) + omag = -2.5 * np.log10(np.clip(fl, 1e-9, None)) + ZP_L + ofw, oel = 2 * rh, o["a"] / np.clip(o["b"], 1e-6, None) + sel = ((snr >= 5) & ((o["flag"].astype(int) & 0b1110) == 0) + & (ofw > FW_MIN) & (ofw < FW_MAX) & (oel < 2.0) + & (omag > 17.5) & (omag < 20.0) & (fl > 0)) + ox, oy = o["x"][sel], o["y"][sel] + for xi, yi, mg in zip(xs, ys, mags): + dd = np.hypot(ox - xi, oy - yi) + rm_.append(mg) + rr_.append(np.hypot(xi - cx, yi - cy) * PIXSCALE / 60.0) + ro_.append(bool(dd.min() < 2.0) if len(dd) else False) + print(" inner run %d/%d (%.0f s)" % (run + 1, NRUN, time.time() - t0), flush=True) + del img, b, ds, rmm, o, fl, fe, fl2, rh, fet, snr, omag, ofw, oel, ox, oy + gc.collect() + + rm_, rr_, ro_ = np.array(rm_), np.array(rr_), np.array(ro_) + np.savez(layout.path("_gc_complete_inner.npz"), mag=rm_, rad=rr_, ok=ro_) + + u = np.load(layout.path("_gc_complete.npz")) + np.savez(layout.path("_gc_complete_all.npz"), + mag=np.concatenate([u["mag"], rm_]), + rad=np.concatenate([u["rad"], rr_]), + ok=np.concatenate([u["ok"], ro_]), apfrac=u["apfrac"]) + print("\ninner injections %d, recovered %.1f%%" % (len(ro_), 100 * ro_.mean())) + print("inner completeness grid (rows mag, cols radius arcmin):") + rb = [0, 1.5, 3, 4.5, 6, 7] + mb = np.arange(17.5, 20.51, 0.5) + print(" " + "".join("%8s" % ("%.1f-%.1f" % (rb[i], rb[i + 1])) + for i in range(len(rb) - 1))) + for j in range(len(mb) - 1): + row = "%4.1f " % mb[j] + for i in range(len(rb) - 1): + s = ((rm_ >= mb[j]) & (rm_ < mb[j + 1]) & (rr_ >= rb[i]) & (rr_ < rb[i + 1])) + row += "%8s" % ("%.2f" % ro_[s].mean() if s.sum() > 15 else "-") + print(row) + + +if __name__ == "__main__": + main() diff --git a/session-scripts/gc-complete.py b/session-scripts/gc-complete.py new file mode 100644 index 0000000..2c7cca4 --- /dev/null +++ b/session-scripts/gc-complete.py @@ -0,0 +1,160 @@ +""" +gc-complete.py -- step 3. Artificial star tests. + +Why this exists: NGC 5128's light makes the noise rise steeply toward the +nucleus, so the SAME cluster is harder to detect at r=2' than at r=20'. Without +correcting for that, the measured radial density profile is the true profile +MULTIPLIED by an unknown, radially-decreasing completeness -- which flattens or +even inverts a real central concentration. So we measure the completeness +directly: inject fake point sources of known magnitude at known positions into +the real luminance image, re-run the identical detection and selection chain, +and count how many come back, as a function of magnitude and projected radius. + +The PSF is empirical, built by median-stacking bright isolated stars from the +image itself. + +Output: _gc_complete.npz (recovery grid), and the PSF stamp. +""" +import os, time, numpy as np, sep, warnings +from astropy.io import fits +from astropy.wcs import WCS + +import layout +warnings.filterwarnings("ignore") + +S = layout.SESSION +BW, FW, THRESH, MINAREA, APR = 16, 3, 3.0, 5, 5.0 +ZP_L = 27.941 +PIXSCALE = 0.5376 +NUC_RA, NUC_DEC = 201.365063, -43.019113 +NRUN = 6 # injection runs +NPER = 2500 # fakes per run +MAGS = (17.5, 22.5) +HALF = 15 # PSF stamp half-size, px +RNG = np.random.default_rng(20260721) + + +def gk(fwhm=5.0): + sig = fwhm / 2.3548 + n = int(2 * round(3 * sig) + 1) + r = np.arange(n) - n // 2 + xx, yy = np.meshgrid(r, r) + k = np.exp(-(xx ** 2 + yy ** 2) / (2 * sig ** 2)) + return (k / k.sum()).astype(np.float32) + + +def build_psf(data, cat): + """median stack of bright, isolated, unsaturated stars""" + m = (cat["base"] & cat["is_star"] & (cat["mag"] > 14.5) & (cat["mag"] < 17.0) + & (cat["r_arcmin"] > 8) & (cat["fwhm"] < 5.6)) + x, y = cat["x"][m], cat["y"][m] + # isolation: no other detection within 20 px + ax, ay = cat["x"], cat["y"] + keep = [] + for i in range(len(x)): + dd = np.hypot(ax - x[i], ay - y[i]) + if (dd < 20).sum() <= 1: + keep.append(i) + x, y = x[keep], y[keep] + print(" PSF from %d isolated bright stars" % len(x)) + stamps = [] + for xi, yi in zip(x, y): + ix, iy = int(round(xi)), int(round(yi)) + if ix < HALF or iy < HALF or ix >= data.shape[1] - HALF or iy >= data.shape[0] - HALF: + continue + st = data[iy - HALF:iy + HALF + 1, ix - HALF:ix + HALF + 1].astype(np.float64) + st = st - np.median(st[[0, -1], :]) + s = st.sum() + if s > 0: + stamps.append(st / s) + psf = np.median(np.array(stamps), axis=0) + psf /= psf.sum() + # normalise so that a source of total flux F injected as F*psf measures + # F_ap through the r=5 px aperture; we need the aperture correction + yy, xx = np.mgrid[-HALF:HALF + 1, -HALF:HALF + 1] + apfrac = psf[np.hypot(xx, yy) <= APR].sum() + print(" aperture fraction inside r=%.0f px: %.4f" % (APR, apfrac)) + return psf.astype(np.float32), apfrac + + +def main(): + t0 = time.time() + hdr = fits.getheader(layout.path("master-Luminance.fit")) + w = WCS(hdr) + ny, nx = hdr["NAXIS2"], hdr["NAXIS1"] + cx, cy = [float(v) for v in w.all_world2pix(NUC_RA, NUC_DEC, 0)] + + cat = dict(np.load(layout.path("_gc_cat.npz"), allow_pickle=True)) + FW_MIN, FW_MAX = float(cat["FW_MIN"]), float(cat["FW_MAX"]) + + base_img = np.ascontiguousarray( + fits.getdata(layout.path("master-Luminance.fit")).astype(np.float32)) + psf, apfrac = build_psf(base_img, cat) + np.save(layout.path("_gc_psf.npy"), psf) + + sep.set_extract_pixstack(1000000) + K = gk() + rec_mag, rec_rad, rec_ok = [], [], [] + + for run in range(NRUN): + img = base_img.copy() + # positions: uniform over the frame, min 40 px apart from each other + xs = RNG.uniform(HALF + 5, nx - HALF - 5, NPER) + ys = RNG.uniform(HALF + 5, ny - HALF - 5, NPER) + mags = RNG.uniform(*MAGS, NPER) + # total flux such that the r=5px aperture measures the intended mag + ftot = 10 ** (-0.4 * (mags - ZP_L)) / apfrac + for xi, yi, f in zip(xs, ys, ftot): + ix, iy = int(round(xi)), int(round(yi)) + img[iy - HALF:iy + HALF + 1, ix - HALF:ix + HALF + 1] += (f * psf) + + b = sep.Background(img, bw=BW, bh=BW, fw=FW, fh=FW) + ds = img - b.back() + rm = b.rms() + o = sep.extract(ds, THRESH, err=rm, minarea=MINAREA, filter_kernel=K, + filter_type="matched", deblend_nthresh=32, + deblend_cont=0.005, clean=True) + fl, fe, afl = sep.sum_circle(ds, o["x"], o["y"], APR, err=rm, subpix=5) + fl2, _, _ = sep.sum_circle(ds, o["x"], o["y"], 2 * APR, err=rm, subpix=5) + rh, _ = sep.flux_radius(ds, o["x"], o["y"], np.full(len(o), 6 * APR), 0.5, + normflux=fl2, subpix=5) + egain = float(hdr.get("EGAIN", 0.2467)) + fet = np.sqrt(fe ** 2 + np.clip(fl, 0, None) / (egain * 12.0)) + snr = fl / np.clip(fet, 1e-9, None) + omag = -2.5 * np.log10(np.clip(fl, 1e-9, None)) + ZP_L + ofwhm = 2.0 * rh + oelong = o["a"] / np.clip(o["b"], 1e-6, None) + ok_sel = ((snr >= 5.0) & ((o["flag"].astype(int) & 0b1110) == 0) + & (ofwhm > FW_MIN) & (ofwhm < FW_MAX) & (oelong < 2.0) + & (omag > 17.5) & (omag < 21.5) & (fl > 0)) + ox, oy = o["x"][ok_sel], o["y"][ok_sel] + + # match each injected star to the surviving detections, within 2 px + for xi, yi, mg in zip(xs, ys, mags): + dd = np.hypot(ox - xi, oy - yi) + hit = (dd.min() < 2.0) if len(dd) else False + rec_mag.append(mg) + rec_rad.append(np.hypot(xi - cx, yi - cy) * PIXSCALE / 60.0) + rec_ok.append(bool(hit)) + print(" run %d/%d done (%.0f s)" % (run + 1, NRUN, time.time() - t0), flush=True) + del img, b, ds, rm, o + + rec_mag = np.array(rec_mag); rec_rad = np.array(rec_rad); rec_ok = np.array(rec_ok) + np.savez(layout.path("_gc_complete.npz"), mag=rec_mag, rad=rec_rad, + ok=rec_ok, apfrac=apfrac) + print("\ninjected %d, recovered %d (%.1f%%)" % (len(rec_ok), rec_ok.sum(), 100 * rec_ok.mean())) + print("\ncompleteness grid (rows = mag, cols = radius arcmin):") + rb = [0, 2, 4, 6, 9, 13, 18, 30] + mb = np.arange(17.5, 22.6, 0.5) + print(" " + "".join("%7s" % ("%d-%d" % (rb[i], rb[i + 1])) for i in range(len(rb) - 1))) + for j in range(len(mb) - 1): + row = "%4.1f " % mb[j] + for i in range(len(rb) - 1): + s = ((rec_mag >= mb[j]) & (rec_mag < mb[j + 1]) & + (rec_rad >= rb[i]) & (rec_rad < rb[i + 1])) + row += "%7s" % ("%.2f" % rec_ok[s].mean() if s.sum() > 20 else "-") + print(row) + + +if __name__ == "__main__": + main() diff --git a/session-scripts/gc-detect.py b/session-scripts/gc-detect.py new file mode 100644 index 0000000..cd7141e --- /dev/null +++ b/session-scripts/gc-detect.py @@ -0,0 +1,171 @@ +""" +gc-detect.py -- source detection and photometry for the NGC 5128 globular cluster survey. + +Step 1 of the pipeline. Reads the luminance master, builds a SPATIALLY VARYING +background model (this is the critical step: NGC 5128's own light is a steep, +structured background and a single global sky level would produce a spurious +central concentration of detections), detects sources above a locally-scaled +threshold, does aperture photometry, measures morphology, and runs forced +photometry at the same positions on the R/G/B masters. + +Outputs (into the stacked directory): + _gc_raw.npz all detections + photometry + morphology + NGC5128-gc-background-check.png diagnostic of the background model + +Memory: only one 61 MB master is held at a time. +""" +import os, sys, time +import numpy as np +from astropy.io import fits +from astropy.wcs import WCS +import sep +import warnings + +import layout +warnings.filterwarnings("ignore") + +STACKED = layout.SESSION + +# --- configuration ----------------------------------------------------------- +BW = 16 # background mesh, px. 8.6 arcsec = 3.2 x the 5 px FWHM. Chosen + # empirically: see the mesh trial in the notes -- a coarser mesh + # destroys recovery of known clusters inside ~8 arcmin. +FW = 3 # median filter over meshes +THRESH = 3.0 # sigma above the LOCAL rms +MINAREA = 5 +APR = 5.0 # photometric aperture radius, px, matching the published ZP +ZP_L = 27.941 # G_mag = -2.5 log10(flux) + ZP_L +FWHM_PX = 5.0 + +# NGC 5128 nucleus, J2000 (NED) +NUC_RA, NUC_DEC = 201.365063, -43.019113 + + +def gauss_kernel(fwhm, size=None): + sig = fwhm / 2.3548 + if size is None: + size = int(2 * round(3 * sig) + 1) + r = np.arange(size) - size // 2 + x, y = np.meshgrid(r, r) + k = np.exp(-(x * x + y * y) / (2 * sig * sig)) + return (k / k.sum()).astype(np.float32) + + +def load(name): + d = fits.getdata(layout.path(name)).astype(np.float32) + if not d.dtype.isnative: + d = d.byteswap().newbyteorder() + return np.ascontiguousarray(d) + + +def main(): + t0 = time.time() + hdr = fits.getheader(layout.path("master-Luminance.fit")) + w = WCS(hdr) + ny, nx = hdr["NAXIS2"], hdr["NAXIS1"] + + data = load("master-Luminance.fit") + + # ---------------------------------------------------------------- background + # Compare mesh sizes so the choice is defensible, then adopt BW. + print("background mesh comparison (median |back| inside r<2' of nucleus vs outer field):") + nx_c, ny_c = w.all_world2pix(NUC_RA, NUC_DEC, 0) + nx_c, ny_c = float(nx_c), float(ny_c) + print(" nucleus pixel = %.1f, %.1f" % (nx_c, ny_c)) + yy, xx = np.mgrid[0:ny:8, 0:nx:8] + rr = np.hypot(xx - nx_c, yy - ny_c) * 0.5376 / 60.0 # arcmin + inner = rr < 2.0 + outer = rr > 12.0 + for bw in (16, 32, 64, 128, 512): + b = sep.Background(data, bw=bw, bh=bw, fw=FW, fh=FW) + bk = b.back()[::8, ::8] + rms = b.rms()[::8, ::8] + print(" bw=%4d back(in)=%8.1f back(out)=%7.2f rms(in)=%7.2f rms(out)=%6.2f" + % (bw, np.median(bk[inner]), np.median(bk[outer]), + np.median(rms[inner]), np.median(rms[outer]))) + del b, bk, rms + + bkg = sep.Background(data, bw=BW, bh=BW, fw=FW, fh=FW) + back = bkg.back() + rmsmap = bkg.rms() + data_sub = data - back + del data + np.save(layout.path("_gc_backmodel_thumb.npy"), back[::8, ::8]) + del back + + # ---------------------------------------------------------------- detection + sep.set_extract_pixstack(3000000) + kern = gauss_kernel(FWHM_PX) + objs, segmap = sep.extract(data_sub, THRESH, err=rmsmap, minarea=MINAREA, + filter_kernel=kern, filter_type="matched", + deblend_nthresh=32, deblend_cont=0.005, + clean=True, clean_param=1.0, segmentation_map=True) + print("detected %d sources (thresh=%.1f x LOCAL rms, minarea=%d)" + % (len(objs), THRESH, MINAREA)) + del segmap + + x, y = objs["x"], objs["y"] + + # ---------------------------------------------------------------- photometry + flux, fluxerr, flag = sep.sum_circle(data_sub, x, y, APR, err=rmsmap, + gain=None, subpix=5) + # local-noise-only error; add Poisson from the source itself using EGAIN + egain = float(hdr.get("EGAIN", 0.2467)) + nimg = 12.0 # 12 x 300 s stacked, normalised -> approximate + poiss = np.clip(flux, 0, None) / (egain * nimg) + fluxerr_tot = np.sqrt(fluxerr ** 2 + poiss) + + # aperture at 2x radius, to catch extended light (galaxy discriminator) + flux2, _, _ = sep.sum_circle(data_sub, x, y, 2 * APR, err=rmsmap, subpix=5) + + # half-light radius and a "PSF concentration" index + rhalf, rflag = sep.flux_radius(data_sub, x, y, np.full(len(x), 6 * APR), + 0.5, normflux=flux2, subpix=5) + # peak-to-total sharpness + with np.errstate(divide="ignore", invalid="ignore"): + conc = -2.5 * np.log10(np.clip(flux, 1e-9, None) / np.clip(flux2, 1e-9, None)) + + # local background rms at each source (for SNR bookkeeping) + xi = np.clip(np.round(x).astype(int), 0, nx - 1) + yi = np.clip(np.round(y).astype(int), 0, ny - 1) + local_rms = rmsmap[yi, xi] + local_back = np.load(layout.path("_gc_backmodel_thumb.npy"))[ + np.clip(yi // 8, 0, ny // 8 - 1), np.clip(xi // 8, 0, nx // 8 - 1)] + + del rmsmap, data_sub + + with np.errstate(divide="ignore", invalid="ignore"): + mag = -2.5 * np.log10(np.clip(flux, 1e-9, None)) + ZP_L + magerr = 1.0857 * fluxerr_tot / np.clip(flux, 1e-9, None) + snr = flux / np.clip(fluxerr_tot, 1e-9, None) + + ra, dec = w.all_pix2world(x, y, 0) + + out = dict(x=x, y=y, ra=ra, dec=dec, + flux=flux, fluxerr=fluxerr_tot, mag=mag, magerr=magerr, snr=snr, + flux2=flux2, rhalf=rhalf, conc=conc, + a=objs["a"], b=objs["b"], theta=objs["theta"], + npix=objs["npix"].astype(np.float64), peak=objs["peak"], + cflux=objs["cflux"], sepflag=objs["flag"].astype(np.float64), + apflag=flag.astype(np.float64), + local_rms=local_rms, local_back=local_back) + + # ---------------------------------------------------------------- colours + for band, key in (("Red", "R"), ("Green", "V"), ("Blue", "B")): + d = load("master-%s.fit" % band) + b = sep.Background(d, bw=BW, bh=BW, fw=FW, fh=FW) + ds = d - b.back() + rm = b.rms() + del d + f, fe, fl = sep.sum_circle(ds, x, y, APR, err=rm, subpix=5) + out["flux_" + key] = f + out["fluxerr_" + key] = fe + print(" forced photometry on %s done" % band) + del ds, rm, b, f, fe, fl + + np.savez(layout.path("_gc_raw.npz"), **out) + print("wrote _gc_raw.npz with %d rows in %.0f s" % (len(x), time.time() - t0)) + + +if __name__ == "__main__": + main() diff --git a/session-scripts/gc-fetch-vizier.py b/session-scripts/gc-fetch-vizier.py new file mode 100644 index 0000000..42fe69c --- /dev/null +++ b/session-scripts/gc-fetch-vizier.py @@ -0,0 +1,91 @@ +"""Fetch a published globular cluster catalogue for NGC 5128 (Centaurus A) from VizieR. + +Primary catalogue: J/MNRAS/469/3444 table "gc" -- Taylor, Puzia, Ferrarez et al. 2017, +MNRAS 469, 3444, "Survey of Centaurus A's Baryonic Structures (SCABS) II": +3210 globular cluster candidates with ugriz photometry over ~1.55 deg^2 centred on Cen A. + +Fallback: J/AJ/143/84 (Harris+ 2012, 833 new GC candidates). + +Writes _cena_gc_ref.npz with keys: + ra, dec : degrees, J2000 (ICRS) + vmag : Johnson V estimated from SDSS g,r via Lupton (2005): + V = g - 0.5784*(g-r) - 0.0038 + gmag,rmag: SDSS-like g and r from the source catalogue (NaN where missing) + prob : Taylor+ 2017 GC membership probability (0-1), higher = more likely a GC +""" +import numpy as np +from astroquery.vizier import Vizier +from astropy.coordinates import SkyCoord +import astropy.units as u + +import layout + +OUT = layout.path("_cena_gc_ref.npz") +RA0, DEC0 = 201.365, -43.019 +# field half-widths in arcmin for a 43' x 29' frame +HW_RA, HW_DEC = 21.5, 14.5 + + +def col(t, name): + if name not in t.colnames: + return None + return np.ma.filled(np.ma.asarray(t[name]).astype(float), np.nan) + + +def report(ra, dec, label): + dra = (ra - RA0) * np.cos(np.radians(dec)) * 60.0 + dde = (dec - DEC0) * 60.0 + sep = np.hypot(dra, dde) + infield = (np.abs(dra) <= HW_RA) & (np.abs(dde) <= HW_DEC) + print(f" {label}: {len(ra)} rows") + print(f" RA {ra.min():.4f} .. {ra.max():.4f} " + f"(span {(ra.max()-ra.min())*np.cos(np.radians(DEC0))*60:.1f}')") + print(f" Dec {dec.min():.4f} .. {dec.max():.4f} " + f"(span {(dec.max()-dec.min())*60:.1f}')") + print(f" sep from centre: median {np.median(sep):.1f}' max {sep.max():.1f}'") + print(f" inside 43'x29' field: {infield.sum()}") + return infield + + +def main(): + v = Vizier(row_limit=-1) + cid, tname = "J/MNRAS/469/3444", "J/MNRAS/469/3444/gc" + print(f"Fetching {tname} (Taylor+ 2017, SCABS II)") + try: + tabs = v.get_catalogs(cid) + t = [x for x in tabs if x.meta.get("name") == tname][0] + except Exception as e: + print(f" FAILED: {e}") + return 1 + + # RAJ2000/DEJ2000 are sexagesimal strings in this table + sc = SkyCoord(np.asarray(t["RAJ2000"], str), np.asarray(t["DEJ2000"], str), + unit=(u.hourangle, u.deg)) + ra = sc.ra.deg + dec = sc.dec.deg + g = col(t, "gmag") + r = col(t, "rmag") + prob = col(t, "Prob") + + good = np.isfinite(ra) & np.isfinite(dec) + ra, dec, g, r, prob = ra[good], dec[good], g[good], r[good], prob[good] + + # -1.0 is the catalogue's "no photometry" sentinel -> NaN + g = np.where(g <= 0, np.nan, g) + r = np.where(r <= 0, np.nan, r) + + # Lupton (2005) SDSS -> Johnson V + vmag = g - 0.5784 * (g - r) - 0.0038 + + report(ra, dec, tname) + print(f" vmag finite: {np.isfinite(vmag).sum()} " + f"range {np.nanmin(vmag):.2f} .. {np.nanmax(vmag):.2f}") + print(f" prob >=0.5: {(prob >= 0.5).sum()} >=0.9: {(prob >= 0.9).sum()}") + + np.savez(OUT, ra=ra, dec=dec, vmag=vmag, gmag=g, rmag=r, prob=prob) + print(f"WROTE {OUT}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/session-scripts/gc-gaia-astrom2.py b/session-scripts/gc-gaia-astrom2.py new file mode 100644 index 0000000..c2d40ae --- /dev/null +++ b/session-scripts/gc-gaia-astrom2.py @@ -0,0 +1,32 @@ +"""Gaia DR3 astrometry via the VizieR mirror I/355/gaiadr3 (the ESA Gaia archive +was down for maintenance). Same rationale as gc-gaia-astrom.py.""" +import os, numpy as np +from astroquery.vizier import Vizier +from astropy.coordinates import SkyCoord +import astropy.units as u + +import layout + +S = layout.SESSION +c = SkyCoord(201.36404, -43.01969, unit="deg") + +cols = ["RA_ICRS", "DE_ICRS", "Source", "Gmag", "BPmag", "RPmag", "BP-RP", + "Plx", "e_Plx", "pmRA", "e_pmRA", "pmDE", "e_pmDE", "RUWE"] +v = Vizier(columns=cols, row_limit=-1, column_filters={"Gmag": "<21.0"}) +t = v.query_region(c, radius=0.42 * u.deg, catalog="I/355/gaiadr3")[0] +print("rows:", len(t), t.colnames) + +d = {} +for k, name in [("ra", "RA_ICRS"), ("dec", "DE_ICRS"), ("g", "Gmag"), + ("bp", "BPmag"), ("rp", "RPmag"), ("bprp", "BP-RP"), + ("plx", "Plx"), ("eplx", "e_Plx"), ("pmra", "pmRA"), + ("epmra", "e_pmRA"), ("pmdec", "pmDE"), ("epmdec", "e_pmDE"), + ("ruwe", "RUWE")]: + if name in t.colnames: + col = t[name] + d[k] = np.array(col.filled(np.nan) if hasattr(col, "filled") else col, dtype=float) +np.savez(layout.path("_gaia_astrom.npz"), **d) +g = d["g"] +print("saved. G<18 %d | 18-20 %d | >=20 %d" % + ((g < 18).sum(), ((g >= 18) & (g < 20)).sum(), (g >= 20).sum())) +print("finite plx %d, finite pm %d" % (np.isfinite(d["plx"]).sum(), np.isfinite(d["pmra"]).sum())) diff --git a/session-scripts/gc-plots.py b/session-scripts/gc-plots.py new file mode 100644 index 0000000..b461d1a --- /dev/null +++ b/session-scripts/gc-plots.py @@ -0,0 +1,491 @@ +""" +gc-plots.py -- step 4. All figures for the NGC 5128 globular cluster survey. + + NGC5128-gc-background-check.png how the galaxy light was removed + NGC5128-gc-finder.png annotated finder chart with zoom insets + NGC5128-gc-cmd.png colour-magnitude diagram + NGC5128-gc-completeness.png artificial-star recovery + NGC5128-gc-radial-profile.png THE key plot: surface density vs radius +""" +import os, numpy as np, warnings +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from matplotlib.patches import Circle, Rectangle +from matplotlib.lines import Line2D +from astropy.io import fits +from astropy.wcs import WCS + +import layout +warnings.filterwarnings("ignore") + +S = layout.SESSION +PIXSCALE = 0.5376 +NUC_RA, NUC_DEC = 201.365063, -43.019113 +KPC_PER_ARCMIN = 3.8 * 1000.0 * (np.pi / 180.0) / 60.0 + +# validated categorical palette (see notes) +C_CAND, C_EXT, C_KNOWN, C_ACC = "#2563eb", "#e8710a", "#127a5a", "#8b5cf6" +C_INK, C_MUTED, C_GRID = "#1a1a1a", "#5c5c5c", "#d8d8d4" +SURF = "#fcfcfb" + +plt.rcParams.update({ + "figure.facecolor": SURF, "axes.facecolor": SURF, + "axes.edgecolor": C_MUTED, "axes.labelcolor": C_INK, + "text.color": C_INK, "xtick.color": C_MUTED, "ytick.color": C_MUTED, + "axes.grid": True, "grid.color": C_GRID, "grid.linewidth": 0.6, + "axes.axisbelow": True, "font.size": 10, + "axes.spines.top": False, "axes.spines.right": False, + "legend.frameon": False, +}) + + +def asinh_stretch(a, lo=1.0, hi=99.7, soft=8.0): + v0, v1 = np.percentile(a[np.isfinite(a)], [lo, hi]) + x = np.clip((a - v0) / max(v1 - v0, 1e-9), 0, 1) + return np.arcsinh(soft * x) / np.arcsinh(soft) + + +def load_cat(): + return dict(np.load(layout.path("_gc_cat.npz"), allow_pickle=True)) + + +# -------------------------------------------------------------------------- +def fig_background(d): + import sep + img = np.ascontiguousarray( + fits.getdata(layout.path("master-Luminance.fit")).astype(np.float32)) + b16 = sep.Background(img, bw=16, bh=16, fw=3, fh=3) + b256 = sep.Background(img, bw=256, bh=256, fw=3, fh=3) + raw = img[::4, ::4] + back = b16.back()[::4, ::4] + sub16 = raw - back + sub256 = raw - b256.back()[::4, ::4] + del img, b16, b256 + + # ONE stretch for the two image-scale panels and ONE for the two residuals, + # so the panels are actually comparable to each other. + v0, v1 = np.percentile(raw, [1.0, 99.7]) + r0, r1 = np.percentile(sub16, [1.0, 99.85]) + + def show(a, im, lo, hi, t, sub=""): + x = np.clip((im - lo) / (hi - lo), 0, 1) + a.imshow(np.arcsinh(8 * x) / np.arcsinh(8.0), origin="lower", cmap="gray", + interpolation="nearest", vmin=0, vmax=1) + a.set_title(t, fontsize=10.5, loc="left", pad=4) + if sub: + a.text(0.5, -0.045, sub, transform=a.transAxes, ha="center", va="top", + fontsize=9, color=C_MUTED) + a.set_xticks([]); a.set_yticks([]); a.grid(False) + + fig, ax = plt.subplots(2, 2, figsize=(12.4, 8.0)) + show(ax[0, 0], raw, v0, v1, "a) luminance master") + show(ax[0, 1], back, v0, v1, "b) background model, 16 px mesh (adopted)", + "same stretch as (a): the model reproduces the galaxy and the dust lane") + show(ax[1, 0], sub16, r0, r1, "c) residual after (b) - what detection runs on", + "galaxy gone; point sources remain at every radius") + show(ax[1, 1], sub256, r0, r1, "d) residual after a 256 px mesh - the failure mode", + "the galaxy survives, swamping the inner field and hiding its clusters") + fig.suptitle("Removing NGC 5128's own light before detection", fontsize=13, + x=0.008, ha="left") + fig.tight_layout(rect=[0, 0.02, 1, 0.97]) + fig.savefig(layout.path("NGC5128-gc-background-check.png"), dpi=110, + bbox_inches="tight", facecolor=SURF) + plt.close(fig) + print("wrote NGC5128-gc-background-check.png") + + +# -------------------------------------------------------------------------- +def fig_finder(d): + hdr = fits.getheader(layout.path("master-Luminance.fit")) + w = WCS(hdr) + img = fits.getdata(layout.path("master-Luminance.fit")).astype(np.float32) + B = 4 + small = img[::B, ::B] + disp = asinh_stretch(small, 20, 99.5, 12) + cx, cy = [float(v) for v in w.all_world2pix(NUC_RA, NUC_DEC, 0)] + + cand = d["cand"] + known = cand & ((d["simbad_type"] == "GlC") | d["scabs_hit"]) + newc = cand & ~known + + ny_s, nx_s = small.shape + # Main panel keeps the detector's 1.5:1 aspect; insets sit in a row beneath. + fig = plt.figure(figsize=(14.0, 13.0)) + gs = fig.add_gridspec(2, 4, height_ratios=[nx_s / ny_s * 0.98, 1.0], + hspace=0.09, wspace=0.06, + left=0.012, right=0.988, top=0.945, bottom=0.015) + ax = fig.add_subplot(gs[0, :]) + ax.imshow(disp, origin="lower", cmap="gray", interpolation="bilinear", + aspect="equal") + ax.set_xlim(0, nx_s); ax.set_ylim(0, ny_s) + ax.grid(False); ax.set_xticks([]); ax.set_yticks([]) + + for m, col, lab in ((known, C_KNOWN, "matches a published catalogue (%d)" % known.sum()), + (newc, C_CAND, "no published counterpart (%d)" % newc.sum())): + ax.scatter(d["x"][m] / B, d["y"][m] / B, s=110, facecolors="none", + edgecolors=col, linewidths=1.2, label=lab) + # nucleus + import matplotlib.patheffects as pe + halo = [pe.withStroke(linewidth=3.0, foreground="black")] + ax.plot(cx / B, cy / B, marker="+", ms=20, mew=2.2, color=C_ACC, zorder=6) + ax.annotate("NGC 5128 nucleus", (cx / B, cy / B), xytext=(-95, -78), + textcoords="offset points", color=C_ACC, fontsize=11, + weight="bold", ha="center", zorder=7, path_effects=halo, + arrowprops=dict(arrowstyle="-", color=C_ACC, lw=1.3, + shrinkA=2, shrinkB=10)) + + # scale bar: 5 arcmin, bottom left, inside the frame + L = 5 * 60 / PIXSCALE / B + x0, y0 = nx_s * 0.035, ny_s * 0.062 + ax.plot([x0, x0 + L], [y0, y0], color="white", lw=3.5, solid_capstyle="butt") + ax.text(x0 + L / 2, y0 + ny_s * 0.018, + "5' = %.1f kpc at 3.8 Mpc" % (5 * KPC_PER_ARCMIN), + ha="center", color="white", fontsize=10) + # compass, derived from the WCS itself rather than from the quoted PA: + # step 1 arcmin north and 1 arcmin east of the nucleus and see where it lands + cxo, cyo = nx_s * 0.915, ny_s * 0.16 + for dra, ddec, lab in ((0.0, 1 / 60.0, "N"), (1 / 60.0, 0.0, "E")): + px, py = w.all_world2pix(NUC_RA + dra / np.cos(np.radians(NUC_DEC)), + NUC_DEC + ddec, 0) + vx, vy = float(px) - cx, float(py) - cy + n = np.hypot(vx, vy) + dx, dy = vx / n * 48, vy / n * 48 + ax.annotate("", (cxo + dx, cyo + dy), (cxo, cyo), + arrowprops=dict(arrowstyle="->", color="white", lw=1.8)) + ax.text(cxo + dx * 1.38, cyo + dy * 1.38, lab, color="white", + ha="center", va="center", fontsize=11, weight="bold") + + ax.legend(loc="upper left", fontsize=10.5, labelcolor=C_INK, + handletextpad=0.4, borderpad=0.6, markerscale=1.1, + facecolor="white", framealpha=0.82, frameon=True, edgecolor="none") + ax.set_title("NGC 5128 globular cluster candidates: %d circled, G < 20 " + "(42.9' x 28.6' luminance master, asinh stretch, 4x downsampled)" + % cand.sum(), fontsize=12.5, loc="left", pad=9) + + # ---- zoom insets, full resolution ---- + zooms = [(2.5, 55), (5.5, 340), (9.0, 130), (16.0, 205)] + HW = 230 # half-width in full-res px -> 4.1 arcmin box + for k, (rr, pa) in enumerate(zooms): + az = fig.add_subplot(gs[1, k]) + ang = np.radians(pa) + zx = cx + rr * 60 / PIXSCALE * np.cos(ang) + zy = cy + rr * 60 / PIXSCALE * np.sin(ang) + zx = float(np.clip(zx, HW, img.shape[1] - HW)) + zy = float(np.clip(zy, HW, img.shape[0] - HW)) + cut = img[int(zy - HW):int(zy + HW), int(zx - HW):int(zx + HW)] + az.imshow(asinh_stretch(cut, 15, 99.7, 14), origin="lower", cmap="gray", + interpolation="nearest", aspect="equal", + extent=[zx - HW, zx + HW, zy - HW, zy + HW]) + for m, col in ((known, C_KNOWN), (newc, C_CAND)): + sel = m & (np.abs(d["x"] - zx) < HW) & (np.abs(d["y"] - zy) < HW) + az.scatter(d["x"][sel], d["y"][sel], s=230, facecolors="none", + edgecolors=col, linewidths=1.6) + az.set_xticks([]); az.set_yticks([]); az.grid(False) + az.set_title("%.1f' from the nucleus" % rr, fontsize=10.5, loc="left", pad=5) + ax.add_patch(Rectangle(((zx - HW) / B, (zy - HW) / B), 2 * HW / B, 2 * HW / B, + fill=False, ec="white", lw=1.1, ls="-", alpha=0.8)) + ax.text((zx) / B, (zy + HW) / B + 6, "%.1f'" % rr, color="white", + fontsize=9, ha="center") + fig.text(0.012, 0.002, + "Lower row: full-resolution %.1f' cut-outs at the marked positions, same " + "circles. The innermost arcminute is empty of candidates: the nucleus and " + "dust lane are impenetrable at this depth." % (2 * HW * PIXSCALE / 60), + fontsize=9.5, color=C_MUTED) + fig.savefig(layout.path("NGC5128-gc-finder.png"), dpi=100, facecolor=SURF) + plt.close(fig) + del img + print("wrote NGC5128-gc-finder.png") + + +# -------------------------------------------------------------------------- +def fig_cmd(d): + fig, ax = plt.subplots(1, 2, figsize=(12.6, 5.6)) + stars = d["base"] & d["is_star"] & np.isfinite(d["BmR"]) & (d["mag"] > 15) & (d["mag"] < 21.5) + cand = d["cand"] & np.isfinite(d["BmR"]) + known = cand & ((d["simbad_type"] == "GlC") | d["scabs_hit"]) + + a = ax[0] + a.hexbin(d["BmR"][stars], d["mag"][stars], gridsize=60, cmap="Greys", + mincnt=1, extent=(-1.0, 3.5, 15, 21.6), linewidths=0) + a.scatter(d["BmR"][cand & ~known], d["mag"][cand & ~known], s=20, c=C_CAND, + alpha=0.85, lw=0, label="candidate, no published counterpart") + a.scatter(d["BmR"][known], d["mag"][known], s=22, c=C_KNOWN, alpha=0.9, + lw=0, marker="D", label="candidate, published GC") + a.set_xlim(-1.0, 3.5); a.set_ylim(21.6, 15) + a.set_xlabel("B - R (instrumental, calibrated to Gaia BP - RP)") + a.set_ylabel("G (luminance)") + a.set_title("a) candidates against the foreground stellar field", fontsize=10.5, loc="left") + a.legend(fontsize=9, loc="lower left", labelcolor=C_INK) + a.text(0.98, 0.03, "grey = %d astrometric\nMilky Way stars" % stars.sum(), + transform=a.transAxes, ha="right", va="bottom", fontsize=9, color=C_MUTED) + + b = ax[1] + bins = np.arange(-1.0, 3.51, 0.2) + b.hist(d["BmR"][stars], bins=bins, density=True, color=C_MUTED, alpha=0.35, + label="foreground stars (n=%d)" % stars.sum()) + b.hist(d["BmR"][cand], bins=bins, density=True, histtype="step", lw=2.0, + color=C_CAND, label="all candidates (n=%d)" % cand.sum()) + b.hist(d["BmR"][known], bins=bins, density=True, histtype="step", lw=2.0, + color=C_KNOWN, ls="--", label="published GCs (n=%d)" % known.sum()) + b.set_xlabel("B - R"); b.set_ylabel("normalised density") + b.set_title("b) colour distributions", fontsize=10.5, loc="left") + b.legend(fontsize=9, labelcolor=C_INK) + med = np.nanmedian(d["BmR"][cand]) + b.axvline(med, color=C_CAND, lw=1, ls=":") + b.text(med + 0.06, b.get_ylim()[1] * 0.93, "candidate median %.2f" % med, + fontsize=9, color=C_CAND) + + fig.suptitle("Colour-magnitude diagram, NGC 5128 cluster candidates", + fontsize=12.5, x=0.008, ha="left") + fig.tight_layout() + fig.savefig(layout.path("NGC5128-gc-cmd.png"), dpi=115, bbox_inches="tight", facecolor=SURF) + plt.close(fig) + print("wrote NGC5128-gc-cmd.png") + + +# -------------------------------------------------------------------------- +def completeness_grid(): + """Prefer the merged uniform + inner-field artificial-star runs. + + The uniform run alone puts only ~2% of its fakes inside 3 arcmin, which left + the innermost completeness bins too noisy to correct with. + """ + for name in ("_gc_complete_all.npz", "_gc_complete.npz"): + p = layout.path(name) + if os.path.exists(p): + print("completeness from %s" % name) + return np.load(p) + return None + + +def fig_completeness(cp): + fig, ax = plt.subplots(1, 2, figsize=(12.6, 5.0)) + mag, rad, ok = cp["mag"], cp["rad"], cp["ok"] + mb = np.arange(17.5, 22.51, 0.25) + mc = 0.5 * (mb[1:] + mb[:-1]) + rbins = [(0, 2), (2, 4), (4, 8), (8, 30)] + cols = [C_ACC, C_EXT, C_KNOWN, C_CAND] + a = ax[0] + for (lo, hi), c in zip(rbins, cols): + f = [] + for j in range(len(mb) - 1): + s = (mag >= mb[j]) & (mag < mb[j + 1]) & (rad >= lo) & (rad < hi) + f.append(ok[s].mean() if s.sum() > 15 else np.nan) + a.plot(mc, f, lw=2.0, color=c, label="%d - %d arcmin" % (lo, hi)) + a.axhline(0.5, color=C_MUTED, lw=1, ls=":") + a.text(17.6, 0.53, "50%", fontsize=9, color=C_MUTED) + a.set_xlabel("injected G magnitude"); a.set_ylabel("recovery fraction") + a.set_ylim(0, 1.05) + a.set_title("a) artificial-star completeness by projected radius", fontsize=10.5, loc="left") + a.legend(fontsize=9, title="projected radius", title_fontsize=9, labelcolor=C_INK) + + b = ax[1] + rb = np.array([0, 1, 2, 3, 4, 6, 8, 11, 15, 20, 30]) + rc = 0.5 * (rb[1:] + rb[:-1]) + for mlo, mhi, c, ls in ((17.5, 19.0, C_KNOWN, "-"), (19.0, 20.0, C_CAND, "-"), + (20.0, 21.0, C_EXT, "-"), (21.0, 21.5, C_ACC, "--")): + f = [] + for j in range(len(rb) - 1): + s = (mag >= mlo) & (mag < mhi) & (rad >= rb[j]) & (rad < rb[j + 1]) + f.append(ok[s].mean() if s.sum() > 15 else np.nan) + b.plot(rc, f, lw=2.0, color=c, ls=ls, label="G %.1f - %.1f" % (mlo, mhi)) + b.set_xlabel("projected radius (arcmin)"); b.set_ylabel("recovery fraction") + b.set_ylim(0, 1.05); b.set_xscale("log") + b.set_xticks([1, 2, 3, 5, 10, 20]); b.set_xticklabels(["1", "2", "3", "5", "10", "20"]) + b.set_title("b) the same, as a function of radius", fontsize=10.5, loc="left") + b.legend(fontsize=9, labelcolor=C_INK) + fig.suptitle("Completeness: how much harder is a cluster to find near the nucleus?", + fontsize=12.5, x=0.008, ha="left") + fig.tight_layout() + fig.savefig(layout.path("NGC5128-gc-completeness.png"), dpi=115, bbox_inches="tight", + facecolor=SURF) + plt.close(fig) + print("wrote NGC5128-gc-completeness.png") + + +# -------------------------------------------------------------------------- +def annulus_area(rb, cx, cy, nx=4788, ny=3194, step=4): + """effective area of each annulus that actually lies on the detector""" + yy, xx = np.mgrid[0:ny:step, 0:nx:step] + r = np.hypot(xx - cx, yy - cy) * PIXSCALE / 60.0 + px_area = (step * PIXSCALE / 60.0) ** 2 + return np.array([((r >= rb[i]) & (r < rb[i + 1])).sum() * px_area + for i in range(len(rb) - 1)]), r + + +def completeness_of(cp, mag, rad): + """Completeness for each object, interpolated from the artificial-star grid. + + A 2D lookup on (magnitude, radius). No luminosity function is assumed: each + surviving candidate is weighted by 1/C, the standard inverse-completeness + (Vmax-style) estimator. + """ + fm, fr, fok = cp["mag"], cp["rad"], cp["ok"] + mb = np.arange(17.5, 20.01, 0.5) + rbg = np.array([0, 1.5, 2.25, 3, 4, 5, 6, 8, 11, 15, 20, 30]) + grid = np.full((len(mb) - 1, len(rbg) - 1), np.nan) + for i in range(len(mb) - 1): + for j in range(len(rbg) - 1): + s = ((fm >= mb[i]) & (fm < mb[i + 1]) & (fr >= rbg[j]) & (fr < rbg[j + 1])) + if s.sum() >= 15: + grid[i, j] = fok[s].mean() + # completeness is a strong function of magnitude and a weak one of radius + # outside the innermost annuli, so fill empty cells with the row median + for i in range(grid.shape[0]): + row = grid[i] + if np.isfinite(row).any(): + row[~np.isfinite(row)] = np.nanmedian(row) + mi = np.clip(np.digitize(mag, mb) - 1, 0, len(mb) - 2) + ri = np.clip(np.digitize(rad, rbg) - 1, 0, len(rbg) - 2) + return grid[mi, ri], grid + + +def fig_radial(d, cp): + hdr = fits.getheader(layout.path("master-Luminance.fit")) + w = WCS(hdr) + cx, cy = [float(v) for v in w.all_world2pix(NUC_RA, NUC_DEC, 0)] + rb = np.array([0, 1.5, 3, 4.5, 6, 8, 11, 15, 20, 27]) + rc = 0.5 * (rb[1:] + rb[:-1]) + area, _ = annulus_area(rb, cx, cy) + + r = d["r_arcmin"] + mag = d["mag"] + # Magnitude-limited sample: G < 19.5, where the measured completeness is + # 25-70% and the 1/C weights are therefore stable. + MLIM = 19.5 + lim = mag < MLIM + cand = d["cand"] & lim + ext = (d["cls"] == "extended") & lim + star = (d["cls"] == "foreground-star") & lim + + C, grid = completeness_of(cp, mag, r) + usable = np.isfinite(C) & (C > 0.15) + wt = np.where(usable, 1.0 / np.clip(C, 0.15, None), 0.0) + print("median completeness of the candidate sample: %.2f (%d of %d usable)" + % (np.nanmedian(C[cand]), (cand & usable).sum(), cand.sum())) + + def prof(mask, weights=None): + n, sw = [], [] + for i in range(len(rb) - 1): + s = mask & (r >= rb[i]) & (r < rb[i + 1]) + n.append(s.sum()) + sw.append(np.sum(weights[s]) if weights is not None else s.sum()) + n = np.array(n, float); sw = np.array(sw, float) + with np.errstate(divide="ignore", invalid="ignore"): + mw = np.where(n > 0, sw / np.maximum(n, 1), 0.0) # mean weight + return n, sw / area, mw * np.sqrt(n) / area # Poisson error + + n_c, s_c, e_c = prof(cand) + n_cc, s_cc, e_cc = prof(cand & usable, wt) + n_e, s_e, e_e = prof(ext) + n_s, s_s, e_s = prof(star) + # The SAME correction applied to the foreground stars. Milky Way stars are + # uniformly distributed on this scale, so their corrected profile MUST come + # out flat. That is the check on the whole correction: if it does not + # flatten them, the correction is wrong and so is the candidate profile. + n_sc, s_sc, e_sc = prof(star & usable, wt) + corr = np.where(s_c > 0, s_cc / np.maximum(s_c, 1e-12), np.nan) + + # outer-field level from the two outermost annuli of the corrected profile + denom = np.nansum(area[-2:]) + bg = np.nansum(s_cc[-2:] * area[-2:]) / denom + bg_e = np.sqrt(np.nansum(e_cc[-2:] ** 2 * area[-2:] ** 2)) / denom + + fig, ax = plt.subplots(1, 2, figsize=(13.2, 5.6)) + a = ax[0] + a.errorbar(rc, s_c, yerr=e_c, color=C_MUTED, lw=1.4, marker="o", ms=6, + capsize=3, label="candidates, raw counts", ls="--", zorder=3) + a.errorbar(rc, s_cc, yerr=e_cc, color=C_CAND, lw=2.4, marker="o", ms=8, + capsize=3, label="candidates, completeness-corrected", zorder=5) + a.errorbar(rc, s_e, yerr=e_e, color=C_EXT, lw=1.8, marker="s", ms=6, + capsize=3, label="extended sources, raw", zorder=4) + a.errorbar(rc, s_sc / 20.0, yerr=e_sc / 20.0, color=C_KNOWN, lw=1.8, marker="^", + ms=6, capsize=3, label="foreground stars / 20, corrected", zorder=2) + a.errorbar(rc, s_s / 20.0, yerr=e_s / 20.0, color=C_KNOWN, lw=1.1, marker="^", + ms=4, capsize=2, ls=":", alpha=0.55, + label="foreground stars / 20, raw", zorder=1) + a.axhline(bg, color=C_ACC, lw=1.4, ls="-.") + a.fill_between([0.8, 30], bg - bg_e, bg + bg_e, color=C_ACC, alpha=0.15, lw=0) + a.annotate("outer-field level", xy=(23, bg), xytext=(23, bg * 0.42), + color=C_ACC, fontsize=8.5, ha="center", + arrowprops=dict(arrowstyle="->", color=C_ACC, lw=1)) + a.set_xscale("log"); a.set_yscale("log") + a.set_xlim(0.8, 30) + a.set_xticks([1, 2, 3, 5, 10, 20]); a.set_xticklabels(["1", "2", "3", "5", "10", "20"]) + a.set_xlabel("projected radius from the nucleus (arcmin)") + a.set_ylabel("surface density (objects per arcmin$^2$)") + a.set_title("a) radial surface density, G < 19.5", fontsize=10.5, loc="left") + a.legend(fontsize=8.6, loc="lower left", labelcolor=C_INK) + sec = a.secondary_xaxis("top", functions=(lambda v: v * KPC_PER_ARCMIN, + lambda v: v / KPC_PER_ARCMIN)) + sec.set_xlabel("projected radius (kpc at 3.8 Mpc)", fontsize=9.5) + + # panel b: background-subtracted, with a power law fit + b = ax[1] + excess = s_cc - bg + ee = np.sqrt(e_cc ** 2 + bg_e ** 2) + ok = np.isfinite(excess) & (excess > 0) & (rc < 20) + b.errorbar(rc[ok], excess[ok], yerr=ee[ok], color=C_CAND, lw=2.2, marker="o", + ms=8, capsize=3, label="candidate excess over the outer field") + slope = np.nan + if ok.sum() >= 3: + p = np.polyfit(np.log10(rc[ok]), np.log10(excess[ok]), 1) + slope = p[0] + xr = np.array([rc[ok].min(), rc[ok].max()]) + b.plot(xr, 10 ** np.polyval(p, np.log10(xr)), color=C_INK, lw=1.4, ls="--", + label=r"power law, $\Sigma \propto R^{%.2f}$" % slope) + b.set_xscale("log"); b.set_yscale("log") + b.set_xlim(0.8, 30) + b.set_xticks([1, 2, 3, 5, 10, 20]); b.set_xticklabels(["1", "2", "3", "5", "10", "20"]) + b.set_xlabel("projected radius from the nucleus (arcmin)") + b.set_ylabel(r"excess surface density (arcmin$^{-2}$)") + b.set_title("b) excess over the outer field", fontsize=10.5, loc="left") + b.legend(fontsize=9, labelcolor=C_INK) + + fig.suptitle("Are the candidates concentrated on NGC 5128?", fontsize=13, + x=0.008, ha="left") + fig.tight_layout() + fig.savefig(layout.path("NGC5128-gc-radial-profile.png"), dpi=115, + bbox_inches="tight", facecolor=SURF) + plt.close(fig) + print("wrote NGC5128-gc-radial-profile.png") + + print("\nRADIAL PROFILE TABLE (G < 19.5)") + print("%6s %6s %5s %8s %7s %10s %10s %10s" + % ("r_in", "r_out", "N", "area", "medC", "sig_raw", "sig_corr", "sig_ext")) + for i in range(len(rc)): + s = cand & (r >= rb[i]) & (r < rb[i + 1]) + mc = np.nanmedian(C[s]) if s.sum() else np.nan + print("%6.1f %6.1f %5d %8.2f %7.2f %10.4f %10.4f %10.4f" + % (rb[i], rb[i + 1], n_c[i], area[i], mc, s_c[i], s_cc[i], s_e[i])) + print("outer-field level %.4f +/- %.4f arcmin^-2 ; power-law slope %.2f" + % (bg, bg_e, slope)) + print("corrected density at 1.5-3' / outer-field level = %.1f" + % (s_cc[1] / bg if bg > 0 else np.nan)) + ei = np.nansum(n_e[:4]) / np.nansum(area[:4]) + eo = np.nansum(n_e[6:]) / np.nansum(area[6:]) + si = np.nansum(n_s[:4]) / np.nansum(area[:4]) + so = np.nansum(n_s[6:]) / np.nansum(area[6:]) + print("extended sources: inner(<6') %.4f outer(>11') %.4f ratio %.2f" + % (ei, eo, ei / eo if eo else np.nan)) + print("foreground stars: inner(<6') %.4f outer(>11') %.4f ratio %.2f" + % (si, so, si / so if so else np.nan)) + fin = np.nansum(s_sc[1:4] * area[1:4]) / np.nansum(area[1:4]) + fout = np.nansum(s_sc[6:] * area[6:]) / np.nansum(area[6:]) + print("foreground stars CORRECTED: inner(1.5-6') %.3f outer(>11') %.3f ratio %.2f" + % (fin, fout, fin / fout if fout else np.nan)) + np.savez(layout.path("_gc_profile.npz"), rb=rb, rc=rc, area=area, + n_c=n_c, s_c=s_c, e_c=e_c, corr=corr, s_cc=s_cc, e_cc=e_cc, + s_e=s_e, s_s=s_s, s_sc=s_sc, bg=bg, bg_e=bg_e, slope=slope, grid=grid) + + +if __name__ == "__main__": + d = load_cat() + cp = completeness_grid() + fig_background(d) + fig_finder(d) + fig_cmd(d) + if cp is not None: + fig_completeness(cp) + fig_radial(d, cp) diff --git a/session-scripts/gc-validate.py b/session-scripts/gc-validate.py new file mode 100644 index 0000000..45136b0 --- /dev/null +++ b/session-scripts/gc-validate.py @@ -0,0 +1,156 @@ +""" +gc-validate.py -- step 5. External validation against published catalogues. + +Two independent truth sets fall inside the field: + * SIMBAD, object type GlC -- confirmed/catalogued Cen A clusters + * SCABS (Taylor et al. 2017, MNRAS 469, 3444) -- deep DECam GC candidates, + with V magnitudes, so it can be binned in brightness + +Neither is complete, especially in the inner few arcmin where the galaxy swamps +even professional data, so a candidate that matches nothing is UNCONFIRMED, not +false. Produces NGC5128-gc-recovery.png and prints the numbers used in the notes. +""" +import os, numpy as np, warnings +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from astropy.io import fits +from astropy.wcs import WCS +from astropy.coordinates import SkyCoord + +import layout +warnings.filterwarnings("ignore") + +S = layout.SESSION +PIXSCALE = 0.5376 +NUC_RA, NUC_DEC = 201.365063, -43.019113 +C_CAND, C_EXT, C_KNOWN, C_ACC = "#2563eb", "#e8710a", "#127a5a", "#8b5cf6" +C_INK, C_MUTED, C_GRID = "#1a1a1a", "#5c5c5c", "#d8d8d4" +SURF = "#fcfcfb" +plt.rcParams.update({ + "figure.facecolor": SURF, "axes.facecolor": SURF, "axes.edgecolor": C_MUTED, + "text.color": C_INK, "xtick.color": C_MUTED, "ytick.color": C_MUTED, + "axes.grid": True, "grid.color": C_GRID, "grid.linewidth": 0.6, + "axes.axisbelow": True, "font.size": 10, "axes.spines.top": False, + "axes.spines.right": False, "legend.frameon": False, "axes.labelcolor": C_INK}) + + +def main(): + d = dict(np.load(layout.path("_gc_cat.npz"), allow_pickle=True)) + hdr = fits.getheader(layout.path("master-Luminance.fit")) + w = WCS(hdr) + cx, cy = [float(v) for v in w.all_world2pix(NUC_RA, NUC_DEC, 0)] + cat = SkyCoord(d["ra"], d["dec"], unit="deg") + nuc = SkyCoord(NUC_RA, NUC_DEC, unit="deg") + + # ---- reference sets restricted to the frame ----------------------------- + def infield(ra, dec): + x, y = w.all_world2pix(ra, dec, 0) + return (x > 30) & (x < 4758) & (y > 30) & (y < 3164), x, y + + z = np.load(layout.path("_simbad.npz"), allow_pickle=True) + m = z["otype"].astype(str) == "GlC" + sra, sdec = z["ra"][m].astype(float), z["dec"][m].astype(float) + inf, _, _ = infield(sra, sdec) + sim = SkyCoord(sra[inf], sdec[inf], unit="deg") + + r = np.load(layout.path("_cena_gc_ref.npz")) + infr, _, _ = infield(r["ra"], r["dec"]) + good_p = infr & (r["prob"] >= 0.9) & np.isfinite(r["vmag"]) + sca = SkyCoord(r["ra"][good_p], r["dec"][good_p], unit="deg") + scav = r["vmag"][good_p] + print("in-field reference objects: SIMBAD GlC %d ; SCABS p>=0.9 with V %d" + % (len(sim), len(sca))) + + cand = d["cand"] + ccand = cat[cand] + call = cat[d["base"]] + + def recov(ref): + i, s, _ = ref.match_to_catalog_sky(ccand) + j, s2, _ = ref.match_to_catalog_sky(call) + return s.arcsec < 2.0, s2.arcsec < 2.0 + + hit_sim, det_sim = recov(sim) + hit_sca, det_sca = recov(sca) + print("SIMBAD GlC: detected at all %d/%d (%.0f%%), kept as candidate %d/%d (%.0f%%)" + % (det_sim.sum(), len(sim), 100 * det_sim.mean(), + hit_sim.sum(), len(sim), 100 * hit_sim.mean())) + print("SCABS p>=0.9: detected %d/%d (%.0f%%), kept %d/%d (%.0f%%)" + % (det_sca.sum(), len(sca), 100 * det_sca.mean(), + hit_sca.sum(), len(sca), 100 * hit_sca.mean())) + + # ---- purity ------------------------------------------------------------- + ci, cs, _ = ccand.match_to_catalog_sky(sim) + matched_sim = cs.arcsec < 2.0 + ci2, cs2, _ = ccand.match_to_catalog_sky( + SkyCoord(r["ra"][infr], r["dec"][infr], unit="deg")) + matched_sca = cs2.arcsec < 2.0 + any_match = matched_sim | matched_sca + print("\nPURITY: %d candidates; %d (%.0f%%) match SIMBAD GlC or SCABS; " + "%d (%.0f%%) unconfirmed" + % (cand.sum(), any_match.sum(), 100 * any_match.mean(), + (~any_match).sum(), 100 * (~any_match).mean())) + st = d["simbad_type"][cand] + bad = np.isin(st, ["*", "PM*", "V*", "RR*", "EB*", "LP*"]) + print(" candidates matching a SIMBAD STAR-type object: %d (%.1f%%)" + % (bad.sum(), 100 * bad.mean())) + print(" matching a SIMBAD galaxy: %d ; Cepheid: %d ; X-ray/LXB: %d" + % ((st == "G").sum(), (st == "Ce*").sum(), np.isin(st, ["X", "LXB"]).sum())) + rcand = d["r_arcmin"][cand] + print(" unconfirmed fraction inside 8': %.0f%% ; outside 8': %.0f%%" + % (100 * (~any_match)[rcand < 8].mean(), 100 * (~any_match)[rcand >= 8].mean())) + + # ---- figure ------------------------------------------------------------- + fig, ax = plt.subplots(1, 2, figsize=(12.6, 5.0)) + a = ax[0] + vb = np.arange(17.0, 22.6, 0.5) + vc = 0.5 * (vb[1:] + vb[:-1]) + fd, fk, nn = [], [], [] + for j in range(len(vb) - 1): + s = (scav >= vb[j]) & (scav < vb[j + 1]) + nn.append(s.sum()) + fd.append(det_sca[s].mean() if s.sum() > 4 else np.nan) + fk.append(hit_sca[s].mean() if s.sum() > 4 else np.nan) + a.plot(vc, fd, lw=2.2, marker="o", ms=7, color=C_KNOWN, + label="detected by our pipeline") + a.plot(vc, fk, lw=2.2, marker="s", ms=7, color=C_CAND, + label="detected AND kept as a candidate") + for x, y, n in zip(vc, fd, nn): + if np.isfinite(y) and n > 4: + a.annotate("%d" % n, (x, y), xytext=(0, 8), textcoords="offset points", + ha="center", fontsize=7.5, color=C_MUTED) + a.set_xlabel("SCABS V magnitude"); a.set_ylabel("fraction recovered") + a.set_ylim(0, 1.05) + a.set_title("a) recovery of published clusters vs brightness", fontsize=10.5, loc="left") + a.legend(fontsize=9, labelcolor=C_INK) + a.text(0.98, 0.9, "numbers = reference objects per bin", transform=a.transAxes, + ha="right", fontsize=8, color=C_MUTED) + + b = ax[1] + rsca = sca.separation(nuc).arcmin + rb = np.array([0, 2, 4, 6, 9, 12, 16, 21, 27]) + rc = 0.5 * (rb[1:] + rb[:-1]) + for arr, c, lab, mk in ((det_sca, C_KNOWN, "detected", "o"), + (hit_sca, C_CAND, "kept as candidate", "s")): + f, nnr = [], [] + for j in range(len(rb) - 1): + s = (rsca >= rb[j]) & (rsca < rb[j + 1]) + nnr.append(s.sum()) + f.append(arr[s].mean() if s.sum() > 4 else np.nan) + b.plot(rc, f, lw=2.2, marker=mk, ms=7, color=c, label=lab) + b.set_xlabel("projected radius (arcmin)"); b.set_ylabel("fraction recovered") + b.set_ylim(0, 1.05) + b.set_title("b) the same, vs radius (SCABS p$\\geq$0.9)", fontsize=10.5, loc="left") + b.legend(fontsize=9, labelcolor=C_INK) + fig.suptitle("External validation against SCABS (Taylor et al. 2017) and SIMBAD", + fontsize=12.5, x=0.008, ha="left") + fig.tight_layout() + fig.savefig(layout.path("NGC5128-gc-recovery.png"), dpi=115, bbox_inches="tight", + facecolor=SURF) + plt.close(fig) + print("wrote NGC5128-gc-recovery.png") + + +if __name__ == "__main__": + main() diff --git a/session-scripts/hdr.py b/session-scripts/hdr.py new file mode 100644 index 0000000..f77d651 --- /dev/null +++ b/session-scripts/hdr.py @@ -0,0 +1,161 @@ +"""Pass 6: re-render with the core recovered. + +The first composite blew out the galaxy's centre, and the earlier explanation +for that - a saturated nucleus - was wrong. The surface-photometry analysis +found, and a direct check confirmed, that the bright clipped pixels near the +middle of the frame belong to a foreground star 128 px (69 arcsec) from the +nucleus. The galaxy's own light never comes close to the clip level: inside the +inner 800 x 800 px box there is not one star-free pixel above 30000 ADU. + +So the core was lost to the STRETCH, not to the sensor. Setting the white point +at the 99.995th percentile put it at 64283 ADU, a level set by field stars, +while the galaxy peaks around a twentieth of that. The midtone transfer needed +to lift a sky at 7 ADU into visibility then pushed everything above a few +thousand ADU to white. + +The fix is the standard high-dynamic-range one: stretch the same data twice and +blend. A faint-biased curve for the sky and halo, a bright-biased curve that +keeps the inner galaxy on the shoulder rather than the ceiling, and a mask that +chooses between them by brightness. No extra data is needed, and none of this +invents anything: both curves are monotonic functions of the same pixels. +""" +import os +import sys + +import numpy as np +import tifffile +from astropy.io import fits +from PIL import Image +from scipy.ndimage import gaussian_filter, median_filter + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import compose as C # noqa: E402 + +import layout + +OUT = C.OUT +FAINT_TARGET = 0.10 # where the sky sits in the faint-biased curve +CORE_HEADROOM_SCALE = 1.15 # bright-curve white point, as a multiple of the + # galaxy's own peak: slightly above it, so the very + # centre keeps a little headroom +CORE_MIDTONE = 0.35 # gentle: the core needs tonal separation, not lift + + +def bright_curve(lum_lin, galaxy_peak, black, white): + """A second stretch whose white point is the galaxy, not the field stars. + + This is the whole trick. The faint curve normalises against a white point + of 64283 ADU, set by field stars, so the galaxy's entire tonal range - sky + at 7 ADU up to a peak near 1944 - is squeezed into the top few percent of + the curve and comes out as a featureless white blob. Rescaling so that the + galaxy's own peak IS the white point spreads that same range across the + full output, and the bulge's smooth gradient and the dust lane silhouetted + against it become visible. Stars clip in this curve, which does not matter: + it is only ever used where the faint curve has already run out of room. + """ + lo = black + hi = galaxy_peak * CORE_HEADROOM_SCALE + norm = np.clip((lum_lin - lo) / (hi - lo), 0.0, 1.0) + print(f" bright curve: black {lo:.1f} ADU, white {hi:.0f} ADU " + f"(galaxy peak {galaxy_peak:.0f}), midtone {CORE_MIDTONE}") + return C.mtf(norm, CORE_MIDTONE) + + +def main(): + data, hdr = C.load() + shape = data["Luminance"].shape + mask = C.galaxy_mask(shape) + for name in C.CHANNELS: + data[name], *_ = C.remove_gradient(data[name], mask) + o, flux = C.star_photometry(data["Luminance"], data) + good = (flux["Red"] > 0) & (flux["Green"] > 0) & (flux["Blue"] > 0) + data["Red"] *= np.median(flux["Green"][good] / flux["Red"][good]) + data["Blue"] *= np.median(flux["Green"][good] / flux["Blue"][good]) + + lum_lin = data["Luminance"] + rgb_lin = np.dstack([data["Red"], data["Green"], data["Blue"]]) + del data + + # The galaxy's true peak, with stars filtered out. A median filter wide + # enough to swallow a stellar profile leaves the smooth galaxy alone. + ny, nx = shape + h = 500 + core = lum_lin[ny // 2 - h:ny // 2 + h, nx // 2 - h:nx // 2 + h] + galaxy_peak = float(median_filter(core, size=41).max()) + print(f"galaxy peak (star-free) {galaxy_peak:.0f} ADU vs frame max " + f"{lum_lin.max():.0f} ADU") + + lum_faint, params = C.autostretch(lum_lin, target=FAINT_TARGET) + lum_bright = bright_curve(lum_lin, galaxy_peak, params["black"], + params["white"]) + + # Blend on the FAINT curve's brightness: where it has run out of headroom, + # hand over to the bright curve. Feathered so the transition is invisible. + w = np.clip((lum_faint - 0.55) / 0.35, 0.0, 1.0) + w = gaussian_filter(w.astype(np.float32), 8.0) + lum = np.clip(lum_faint * (1.0 - w) + lum_bright * w, 0.0, 1.0) + print(f" HDR blend covers {float((w > 0.05).mean()):.2%} of the frame") + + rgb = np.empty_like(rgb_lin) + for i in range(3): + ch = rgb_lin[:, :, i] + sky = np.median(ch) + mad = 1.4826 * np.median(np.abs(ch - sky)) + black = sky - 2.8 * mad + white = np.percentile(ch, 99.995) + norm = np.clip((ch - black) / (white - black), 0, 1) + cf = C.mtf(norm, params["midtone"]) + # The colour channels get the same two-curve treatment, so the core + # keeps its colour instead of going white while the luminance holds + # detail. + peak_c = float(median_filter( + ch[ny // 2 - h:ny // 2 + h, nx // 2 - h:nx // 2 + h], + size=41).max()) + nb = np.clip((ch - black) / (peak_c * CORE_HEADROOM_SCALE - black), + 0.0, 1.0) + cb = C.mtf(nb, CORE_MIDTONE) + rgb[:, :, i] = cf * (1.0 - w) + cb * w + del rgb_lin + + sky_med = [float(np.median(rgb[:, :, i][~mask])) for i in range(3)] + target = float(np.mean(sky_med)) + for i in range(3): + rgb[:, :, i] = np.clip(rgb[:, :, i] - (sky_med[i] - target), 0.0, 1.0) + + rgb_lum = rgb.mean(axis=2, keepdims=True) + chroma = rgb - rgb_lum + for i in range(3): + chroma[:, :, i] = gaussian_filter(median_filter(chroma[:, :, i], 3), + 1.5) + rgb = np.clip(rgb_lum + chroma * C.SATURATION, 0.0, 1.0) + del chroma, rgb_lum + + detail = lum - gaussian_filter(lum, 2.0) + weight = np.clip((lum - FAINT_TARGET) * 4.0, 0.0, 1.0) + lum = np.clip(lum + 0.35 * detail * weight, 0.0, 1.0) + del detail, weight + + ratio = lum / np.maximum(rgb.mean(axis=2), 1e-5) + out = np.clip(rgb * ratio[:, :, None], 0.0, 1.0) + del ratio, rgb + + neutral = 0.5 * (out[:, :, 0] + out[:, :, 2]) + green = out[:, :, 1] + out[:, :, 1] = np.where(green > neutral, green * 0.15 + neutral * 0.85, + green) + + frac = float((out.max(axis=2) > 0.995).mean()) + print(f" pixels at full white: {frac:.3%}") + + Image.fromarray((out * 255 + 0.5).astype(np.uint8)).save( + layout.path("NGC5128-LRGB-hdr.png")) + tifffile.imwrite(layout.path("NGC5128-LRGB-hdr.tif"), + (out * 65535 + 0.5).astype(np.uint16), photometric="rgb") + prev = Image.fromarray((out * 255 + 0.5).astype(np.uint8)) + prev.thumbnail((2400, 2400), Image.LANCZOS) + prev.save(layout.path("NGC5128-LRGB-hdr-preview.jpg"), quality=93) + print("wrote NGC5128-LRGB-hdr.png / .tif / -preview.jpg") + + +if __name__ == "__main__": + main() diff --git a/session-scripts/layout.py b/session-scripts/layout.py new file mode 100644 index 0000000..7577220 --- /dev/null +++ b/session-scripts/layout.py @@ -0,0 +1,108 @@ +"""Where every file of a processed session lives. + +The scripts used to sit inside the data directory and address it with absolute +paths. Now that they live in a repository and the data is organised into named +subdirectories, they need one place that knows the mapping - this module. + +Two rules make it work: + +* The session root comes from the ASTRO_SESSION environment variable, so the + same code runs against any session without editing. It falls back to the + NGC 5128 session these scripts were written against. +* A file's subdirectory is derived from its NAME, using the same rules the + directory was organised with. That means a script can go on asking for + "master-Red.fit" or "_stars.npz" and get the right path, without every call + site having to learn the layout. + +Anything unrecognised lands at the session root, which is visible and easy to +correct rather than silently wrong. +""" +import os + +SESSION = os.environ.get( + "ASTRO_SESSION", + r"C:\Users\lhorrocks-barlow\Downloads\NGC5128\20260721") + +# Subdirectories, also created on demand by path(). +RAW = "raw" +CALIBRATED = "calibrated" +MASTERS = os.path.join("stacks", "masters") +ORIGINAL = os.path.join("stacks", "original") +FINAL = "final" +RENDERINGS = "renderings" +FIGURES = os.path.join("science", "figures") +CATALOGUES = os.path.join("science", "catalogues") +SCIENCE_DATA = os.path.join("science", "data") +NOTES = os.path.join("science", "notes") +INTERMEDIATES = "intermediates" + +# Names that are directories in their own right, not files. +_DIRS = {"final": FINAL, "original": ORIGINAL, "raw": RAW, + "calibrated": CALIBRATED, "renderings": RENDERINGS, + "_dss": os.path.join(INTERMEDIATES, "_dss")} + + +def subdir(name): + """The subdirectory a given filename belongs in.""" + low = name.lower() + if name in _DIRS: + return _DIRS[name] + if name.startswith("_"): + return INTERMEDIATES + if low.endswith(".zip") or (low.startswith("jpeg-") and + low.endswith(".jpg")): + return RAW + if low.startswith("calibrated-") and low.endswith((".fit", ".tif")): + return CALIBRATED + if low.startswith("raw-") and low.endswith((".fit", ".tif")): + return RAW + if low.startswith("master-") and low.endswith(".fit"): + return MASTERS + if low.startswith("original-"): + return ORIGINAL + if "final" in low and low.endswith((".png", ".tif", ".jpg")): + return FINAL + if low.startswith("notes-") and low.endswith(".md"): + return NOTES + if low.endswith(".csv"): + return CATALOGUES + if low.startswith("sb-") and low.endswith((".fits", ".npy", ".txt")): + return SCIENCE_DATA + if low.endswith(".png") and any(f"-{p}-" in low for p in ("gc", "sb", "mo")): + return FIGURES + if low.startswith(("ngc5128-lrgb", "ngc5128-img-")): + return RENDERINGS + if low.endswith(".md"): + return "" + return "" + + +def path(*parts, make=True): + """Absolute path for a session file, addressed by name alone. + + Extra leading components are accepted and ignored, so calls written + against the old flat layout - path("stacked", "_stars.npz") - still land + correctly. The last component is the one that decides. + """ + name = parts[-1] + full = os.path.join(SESSION, subdir(name), name) + if make: + os.makedirs(os.path.dirname(full), exist_ok=True) + return full + + +def describe(): + lines = [f"session root: {SESSION}"] + for label, d in (("raw", RAW), ("calibrated", CALIBRATED), + ("masters", MASTERS), ("original stacks", ORIGINAL), + ("final images", FINAL), ("other renderings", RENDERINGS), + ("science figures", FIGURES), + ("science catalogues", CATALOGUES), + ("science data", SCIENCE_DATA), ("notes", NOTES), + ("intermediates", INTERMEDIATES)): + lines.append(f" {label:20s} {d}") + return "\n".join(lines) + + +if __name__ == "__main__": + print(describe()) diff --git a/session-scripts/mo_cavs.py b/session-scripts/mo_cavs.py new file mode 100644 index 0000000..8a79f27 --- /dev/null +++ b/session-scripts/mo_cavs.py @@ -0,0 +1,567 @@ +"""Run down the CAVS 210 photometry discrepancy. + +The transient search measured G = 18.20 for the source at +13:25:37.68 -43:02:29.9, where Gaia DR3 lists G = 21.03 for the catalogued +variable at that position. That is 2.8 mag, a factor of 13 in flux, and it is +the only anomaly in either search. There are three ways it can resolve: + +1. a measurement artefact - a blend inside the 5 px aperture, galaxy-halo + contamination, a wrong catalogue match, or an aperture/zero-point problem + specific to this position; +2. a genuine brightening - Gaia's G is a mean over its observation window, not + a value for 2026-07-21, and a catalogued variable near maximum would be a + real detection; +3. a wrong archival identification. + +This script gathers the evidence needed to tell them apart: + +* a full Gaia DR3 cone search to ALL magnitudes (the cached catalogue used by + the search was cut at G < 20.5, which is exactly why the variable was + missed), so the match and its separation can be checked properly; +* a census of every neighbour in the master within 15 arcsec; +* a curve of growth at the source compared against the median curve of growth + of isolated field stars - a blend keeps rising where a point source flattens; +* per-sub aperture photometry across all 12 luminance subs with a local + annulus background and proper uncertainties, giving a light curve rather + than one stacked number, plus comparison stars of similar brightness to show + what the instrumental scatter actually is; +* the same photometry through a small (2 px) aperture, which is far less + sensitive to blending; +* radial profile and second moments against the image PSF. + +Outputs NGC5128-mo-cavs210.png and _mo_cavs.npz. +""" +import os + +import numpy as np +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import sep +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.stats import sigma_clipped_stats +from astropy.wcs import WCS +from scipy.spatial import cKDTree + +import mo_common as C + +RA, DEC = 201.406996, -43.041648 +OUTPNG = os.path.join(C.OUT, "NGC5128-mo-cavs210.png") +CACHE_GAIA = os.path.join(C.OUT, "_cavs_gaia.npz") + +RADII = np.array([1.0, 1.5, 2.0, 2.5, 3.0, 4.0, 5.0, 6.0, 8.0, 10.0, 12.0]) +ANN_IN, ANN_OUT = 12.0, 20.0 # px, local background annulus + + +def gaia_all(ra, dec, radius_arcsec=25.0): + """Every Gaia DR3 source near the position, to any magnitude. + + The search itself used a cached catalogue cut at G < 20.5, which is + precisely why the variable at this position was not matched. This goes back + to the full catalogue with no magnitude limit. + + Served from VizieR's Gaia DR3 mirror (I/355/gaiadr3): ESA's own archive was + down for maintenance when this was run, and VizieR carries the identical + catalogue. + """ + if os.path.exists(CACHE_GAIA): + z = np.load(CACHE_GAIA, allow_pickle=True) + return z["ra"], z["dec"], z["g"], z["src"], z["var"] + from astroquery.vizier import Vizier + v = Vizier(columns=["Source", "RA_ICRS", "DE_ICRS", "Gmag", "BPmag", + "RPmag", "VarFlag", "Plx", "pmRA", "pmDE"], + row_limit=-1) + t = v.query_region(SkyCoord(ra * u.deg, dec * u.deg), + radius=radius_arcsec * u.arcsec, + catalog="I/355/gaiadr3")[0] + out = (np.asarray(t["RA_ICRS"], float), np.asarray(t["DE_ICRS"], float), + np.asarray(t["Gmag"], float), + np.asarray([str(x) for x in t["Source"]]), + np.asarray([str(x) for x in (t["VarFlag"] if "VarFlag" in + t.colnames else + ["-"] * len(t))])) + extra = {c: np.asarray(t[c], float) for c in ("BPmag", "RPmag", "Plx", + "pmRA", "pmDE") + if c in t.colnames} + np.savez(CACHE_GAIA, ra=out[0], dec=out[1], g=out[2], src=out[3], + var=out[4], **extra) + return out + + +def ann_background(img, x, y, rin=ANN_IN, rout=ANN_OUT): + """Sigma-clipped median sky per pixel in an annulus, and its scatter.""" + h = int(rout) + 2 + xi, yi = int(round(x)), int(round(y)) + cut = img[yi - h:yi + h + 1, xi - h:xi + h + 1].astype(float) + gy, gx = np.mgrid[:cut.shape[0], :cut.shape[1]] + r = np.hypot(gx - (x - xi + h), gy - (y - yi + h)) + m = (r >= rin) & (r <= rout) & np.isfinite(cut) + med, _, sd = sigma_clipped_stats(cut[m], sigma=3.0) + return float(med), float(sd) + + +def cog(img, x, y, radii=RADII): + """Curve of growth with the local annulus sky removed.""" + sky, sd = ann_background(img, x, y) + f = [] + for r in radii: + v, _, _ = sep.sum_circle(img, np.array([x]), np.array([y]), r, + gain=1.0) + f.append(float(v[0]) - sky * np.pi * r ** 2) + return np.array(f), sky, sd + + +def archival_and_colour(msub, px, py, objs, omag, wcs): + """Was it this bright in the 1990s, and what colour is it? + + A brightening cannot be tested against DSS in absolute terms - the plates + have no useful zero point here - but it can be tested differentially. The + neighbour 5.4 arcsec away is a normal point source of known brightness in + this image; if the ratio between the two objects is the same on a 1990s + plate as it is tonight, then this source has not changed. + """ + import warnings + from astroquery.hips2fits import hips2fits + + d = np.hypot(objs["x"] - px, objs["y"] - py) + order = np.argsort(d) + refs = [i for i in order[1:] if 2.0 < d[i] * C.SCALE < 40.0][:4] + print("\ndifferential check against DSS2 red (1990s):") + print(" reference sources in this image:") + for i in refs: + print(f" sep {d[i] * C.SCALE:5.2f}\" G = {omag[i]:5.2f}") + + fov = 2.0 / 60.0 # deg + npix = 240 # 0.5 arcsec/px + try: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + h = hips2fits.query(hips="CDS/P/DSS2/red", width=npix, + height=npix, ra=RA * u.deg, dec=DEC * u.deg, + fov=fov * u.deg, projection="TAN", + format="fits") + dss = np.asarray(h[0].data, dtype=float) + except Exception as exc: # noqa: BLE001 + print(f" DSS fetch failed: {type(exc).__name__}: {exc}") + return None + dwcs = WCS(h[0].header) + dss = dss.astype(np.float32) + dbkg = sep.Background(dss, bw=32, bh=32, fw=3, fh=3) + dsub = dss - dbkg.back() + + def dss_mag(ra_, dec_): + qx, qy = dwcs.world_to_pixel(SkyCoord(ra_ * u.deg, dec_ * u.deg)) + f, _, _ = sep.sum_circle(dsub, np.array([float(qx)]), + np.array([float(qy)]), 4.0, gain=1.0) + return float(f[0]) + + f_src = dss_mag(RA, DEC) + print(f" source DSS flux (r = 2 arcsec): {f_src:.1f}") + rows = [] + for i in refs: + c = wcs.pixel_to_world(objs["x"][i], objs["y"][i]) + fr = dss_mag(c.ra.deg, c.dec.deg) + if fr <= 0 or f_src <= 0: + print(f" ref at {d[i] * C.SCALE:5.2f}\": DSS flux " + f"{fr:.1f} - unusable") + continue + dm_now = omag[np.argmin(d)] - omag[i] + dm_dss = -2.5 * np.log10(f_src / fr) + rows.append((d[i] * C.SCALE, omag[i], dm_now, dm_dss)) + print(f" vs source at {d[i] * C.SCALE:5.2f}\" (G={omag[i]:.2f}): " + f"delta-mag tonight {dm_now:+.2f}, on DSS {dm_dss:+.2f}, " + f"change {dm_dss - dm_now:+.2f}") + ch = float(np.median([r[3] - r[2] for r in rows])) if rows else np.nan + if rows: + print(f" --> median implied brightness change since the 1990s: " + f"{ch:+.2f} mag") + + # colours from the pixel-aligned RGB masters + print("\ncolour from the R, G, B masters (same 5 px aperture):") + out = {} + for filt in ("Red", "Green", "Blue"): + with fits.open(os.path.join(C.OUT, f"master-{filt}.fit")) as hd: + im = hd[0].data.astype(np.float32) + b = sep.Background(im, bw=64, bh=64, fw=3, fh=3) + sb = im - b.back() + del im + fs, _, _ = sep.sum_circle(sb, np.array([px]), np.array([py]), + C.APRAD, gain=1.0) + # normalise against the same comparison sources so the numbers are + # differential and do not need per-filter zero points + ref_f = [] + for i in refs: + fr, _, _ = sep.sum_circle(sb, np.array([float(objs["x"][i])]), + np.array([float(objs["y"][i])]), + C.APRAD, gain=1.0) + if fr[0] > 0: + ref_f.append(float(fr[0])) + del sb + if ref_f and fs[0] > 0: + rel = -2.5 * np.log10(float(fs[0]) / np.median(ref_f)) + out[filt] = rel + print(f" {filt:6s}: source is {rel:+.2f} mag relative to the " + f"median neighbour") + if len(out) == 3: + print(f" --> (source-neighbour) colour B-R = " + f"{out['Blue'] - out['Red']:+.2f} mag " + f"(positive = redder than the neighbours)") + return dict(dss=dsub, dwcs=dwcs, rows=rows, change=ch, colours=out, + refs=refs) + + +def main(): + with fits.open(os.path.join(C.OUT, "master-Luminance.fit")) as hd: + master = hd[0].data.astype(np.float32) + hdr = hd[0].header + wcs = WCS(hdr) + px, py = wcs.world_to_pixel(SkyCoord(RA * u.deg, DEC * u.deg)) + px, py = float(px), float(py) + print(f"CAVS 210 at master pixel ({px:.2f}, {py:.2f})") + + bkg = sep.Background(master, bw=64, bh=64, fw=3, fh=3) + back = bkg.back() + msub = master - back + print(f"local sep background {back[int(py), int(px)]:.1f} ADU/px vs " + f"field median {np.median(back):.1f}; global rms " + f"{bkg.globalrms:.2f}") + sky_l, sky_sd = ann_background(msub, px, py) + print(f"residual sky in the 12-20 px annulus after sep subtraction: " + f"{sky_l:+.2f} +- {sky_sd:.2f} ADU/px") + + # ---- 1. Gaia, all magnitudes ------------------------------------- + gra, gdec, gg, gsrc, gvar = gaia_all(RA, DEC) + sc0 = SkyCoord(RA * u.deg, DEC * u.deg) + gsep = sc0.separation(SkyCoord(gra * u.deg, gdec * u.deg)).arcsec + order = np.argsort(gsep) + print(f"\nGaia DR3 within 25 arcsec ({len(gra)} sources):") + for i in order[:8]: + print(f" {gsrc[i]:>20s} sep {gsep[i]:6.2f}\" G = {gg[i]:6.2f} " + f"variable={gvar[i]}") + + # ---- 2. neighbours in this image --------------------------------- + objs = sep.extract(msub, 3.0, err=bkg.globalrms, minarea=5, + deblend_cont=0.005) + apf, _, _ = sep.sum_circle(msub, objs["x"], objs["y"], C.APRAD, + err=bkg.globalrms, gain=1.0) + omag = -2.5 * np.log10(np.maximum(apf, 1e-9)) + C.ZP + d = np.hypot(objs["x"] - px, objs["y"] - py) + near = np.argsort(d)[:8] + print(f"\nneighbours detected in this image within 15 arcsec:") + for i in near: + if d[i] * C.SCALE > 15: + break + print(f" sep {d[i] * C.SCALE:6.2f}\" ({d[i]:5.2f} px) " + f"G = {omag[i]:6.2f} a={objs['a'][i]:.2f} b={objs['b'][i]:.2f} " + f"npix={objs['npix'][i]}") + + # ---- 3. curve of growth vs stars --------------------------------- + tree = cKDTree(np.column_stack([objs["x"], objs["y"]])) + iso = [] + for i in range(len(objs)): + if not (17.0 < omag[i] < 19.0): + continue + if not (200 < objs["x"][i] < 4588 and 200 < objs["y"][i] < 2994): + continue + if np.hypot(objs["x"][i] - 2394, objs["y"][i] - 1597) < 900: + continue + nb = tree.query_ball_point([objs["x"][i], objs["y"][i]], 25.0) + if len(nb) > 1: + continue + iso.append(i) + iso = iso[:120] + print(f"\ncurve of growth from {len(iso)} isolated field stars " + f"G = 17-19") + star_cogs = [] + for i in iso: + f, _, _ = cog(msub, float(objs["x"][i]), float(objs["y"][i])) + if f[RADII == 10.0][0] > 0: + star_cogs.append(f / f[RADII == 10.0][0]) + star_cog = np.median(np.array(star_cogs), axis=0) + src_cog, _, _ = cog(msub, px, py) + src_cog_n = src_cog / src_cog[RADII == 10.0][0] + print(" r(px) star source") + for r, a, b in zip(RADII, star_cog, src_cog_n): + print(f" {r:5.1f} {a:5.3f} {b:5.3f}") + m5 = -2.5 * np.log10(src_cog[RADII == 5.0][0]) + C.ZP + m2 = -2.5 * np.log10(src_cog[RADII == 2.0][0] / + star_cog[RADII == 2.0][0]) + C.ZP + print(f"\n aperture magnitude, r=5 px, local sky : G = {m5:.2f}") + print(f" PSF-scaled from r=2 px core : G = {m2:.2f}") + + # ---- 4. per-sub light curve -------------------------------------- + tforms = C.frame_transforms() + mobjs, map_, _, _, _ = C.master_sources() + # comparison stars: isolated, similar brightness, similar distance out + comp = [i for i in iso if 17.8 < omag[i] < 18.6][:6] + print(f"\nper-sub photometry ({len(comp)} comparison stars)") + times, lc, lcerr, lc2, comps = [], [], [], [], [] + for i, (key, path) in enumerate(C.lum_frames()): + with fits.open(path, memmap=False) as hd: + img = hd[0].data.astype(np.float32) + b = sep.Background(img, bw=64, bh=64, fw=3, fh=3) + s = img - b.back() + del img + o, ap, _ = C.detect(s + 0.0) + zp, _ = C.frame_zeropoint(o["x"], o["y"], ap, mobjs, map_, + tforms[key]) + inv = tforms[key].inverse + nx_, ny_ = inv(np.array([[px, py]]))[0] + skyv, skysd = ann_background(s, nx_, ny_) + f5, e5, _ = sep.sum_circle(s, np.array([nx_]), np.array([ny_]), + C.APRAD, err=float(b.globalrms), gain=1.0) + f5 = float(f5[0]) - skyv * np.pi * C.APRAD ** 2 + # uncertainty: photon+read noise in the aperture, plus the uncertainty + # on the local sky level scaled by the aperture area + npix_ap = np.pi * C.APRAD ** 2 + err = float(np.hypot(e5[0], skysd * npix_ap / + np.sqrt(max(np.pi * (ANN_OUT ** 2 - ANN_IN ** 2), + 1.0)))) + f2, _, _ = sep.sum_circle(s, np.array([nx_]), np.array([ny_]), 2.0, + gain=1.0) + f2 = float(f2[0]) - skyv * np.pi * 4.0 + times.append(C.lum_times()[i]) + lc.append(-2.5 * np.log10(max(f5, 1e-9)) + zp) + lcerr.append(1.0857 * err / max(f5, 1e-9)) + lc2.append(-2.5 * np.log10(max(f2 / star_cog[RADII == 2.0][0], 1e-9)) + + zp) + row = [] + for ci in comp: + cx, cy = inv(np.array([[float(objs["x"][ci]), + float(objs["y"][ci])]]))[0] + sv, _ = ann_background(s, cx, cy) + fc, _, _ = sep.sum_circle(s, np.array([cx]), np.array([cy]), + C.APRAD, err=float(b.globalrms), + gain=1.0) + row.append(-2.5 * np.log10(max(float(fc[0]) - + sv * npix_ap, 1e-9)) + zp) + comps.append(row) + del s + print(f" {key}: G = {lc[-1]:.3f} +- {lcerr[-1]:.3f} " + f"(r=2px core: {lc2[-1]:.3f}) sky {skyv:+.2f}") + + lc = np.array(lc); lcerr = np.array(lcerr); lc2 = np.array(lc2) + comps = np.array(comps) + print(f"\nsource : mean G = {lc.mean():.3f}, rms {lc.std():.3f}, " + f"median formal error {np.median(lcerr):.3f}") + for j in range(comps.shape[1]): + print(f" comp {j + 1}: mean {comps[:, j].mean():.3f}, " + f"rms {comps[:, j].std():.3f}") + comp_rms = float(np.median([comps[:, j].std() + for j in range(comps.shape[1])])) + print(f" median comparison-star rms: {comp_rms:.3f} mag") + + # ---- 5. shape ---------------------------------------------------- + di = np.argmin(np.hypot(objs["x"] - px, objs["y"] - py)) + r50src, _ = sep.flux_radius(msub, np.array([objs["x"][di]]), + np.array([objs["y"][di]]), np.array([6.0]), + 0.5, normflux=np.array([apf[di]]), subpix=5) + r50s = [] + for i in iso: + rr, _ = sep.flux_radius(msub, np.array([objs["x"][i]]), + np.array([objs["y"][i]]), np.array([6.0]), + 0.5, normflux=np.array([apf[i]]), subpix=5) + r50s.append(float(rr[0])) + print(f"\nhalf-light radius: source {float(r50src[0]):.2f} px, " + f"isolated stars {np.median(r50s):.2f} +- {np.std(r50s):.2f} px " + f"-> {float(r50src[0]) / np.median(r50s):.2f} x PSF") + print(f"source second moments a={objs['a'][di]:.2f} b={objs['b'][di]:.2f} " + f"-> elongation {objs['a'][di] / objs['b'][di]:.2f}") + + arch = archival_and_colour(msub, px, py, objs, omag, wcs) + + np.savez(os.path.join(C.OUT, "_mo_cavs.npz"), + px=px, py=py, times=np.array(times), lc=lc, lcerr=lcerr, + lc2=lc2, comps=comps, comp_rms=comp_rms, + radii=RADII, star_cog=star_cog, src_cog=src_cog_n, + gsep=gsep, gg=gg, gsrc=gsrc, gvar=gvar, + r50src=float(r50src[0]), r50star=float(np.median(r50s)), + m5=m5, m2=m2) + + # ---- figure ------------------------------------------------------ + make_figure(msub, px, py, objs, omag, gra, gdec, gsep, gg, gsrc, wcs, + np.array(times), lc, lcerr, comps, comp_rms, star_cog, + src_cog_n, float(r50src[0]), float(np.median(r50s)), m5, + arch) + + +def make_figure(msub, px, py, objs, omag, gra, gdec, gsep, gg, gsrc, wcs, + times, lc, lcerr, comps, comp_rms, star_cog, src_cog, + r50src, r50star, m5, arch): + fig = plt.figure(figsize=(15.0, 9.0)) + gs = fig.add_gridspec(2, 4, height_ratios=[1.0, 0.85], + hspace=0.42, wspace=0.30, + left=0.055, right=0.985, top=0.795, bottom=0.085) + + H = 26 + + def mark_gaia(ax, x0, y0, scale, wc): + for i in np.argsort(gsep)[:4]: + gx, gy = wc.world_to_pixel(SkyCoord(gra[i] * u.deg, + gdec[i] * u.deg)) + dx = (float(gx) - x0) * scale + dy = (float(gy) - y0) * scale + if abs(dx) > H or abs(dy) > H: + continue + ax.scatter([dx], [dy], s=120, facecolor="none", + edgecolor="#4fb3d9", lw=1.3) + ax.annotate(f"G={gg[i]:.1f}", (dx, dy), + textcoords="offset points", xytext=(7, 4), + fontsize=7.5, color="#4fb3d9") + + # --- this image --- + ax = fig.add_subplot(gs[0, 0]) + cut = msub[int(py) - H:int(py) + H + 1, int(px) - H:int(px) + H + 1] + v1, v2 = np.percentile(cut, [15, 99.6]) + ax.imshow(cut, origin="lower", cmap="gray", vmin=v1, vmax=v2, + extent=[-H, H, -H, H]) + mark_gaia(ax, int(px), int(py), 1.0, wcs) + ax.add_patch(plt.Circle((0, 0), C.APRAD, fill=False, color="#f0c05a", + lw=1.5)) + ax.set_xlim(-H, H) + ax.set_ylim(-H, H) + ax.set_title("this image, 2026-07-21\n28 x 28 arcsec", fontsize=9.5, + pad=6) + ax.set_xlabel("px from centroid; yellow = 5 px aperture", fontsize=8) + + # --- archival plate at the same angular scale --- + axd = fig.add_subplot(gs[0, 1]) + if arch is not None: + dsub, dwcs = arch["dss"], arch["dwcs"] + dx0, dy0 = dwcs.world_to_pixel(SkyCoord(RA * u.deg, DEC * u.deg)) + dx0, dy0 = float(dx0), float(dy0) + sc = 0.5 / C.SCALE # DSS is 0.5 arcsec/px + hd_ = int(H / sc) + 1 + dcut = dsub[int(dy0) - hd_:int(dy0) + hd_ + 1, + int(dx0) - hd_:int(dx0) + hd_ + 1] + w1, w2 = np.percentile(dcut, [15, 99.6]) + axd.imshow(dcut, origin="lower", cmap="gray", vmin=w1, vmax=w2, + extent=[-hd_ * sc, hd_ * sc, -hd_ * sc, hd_ * sc]) + mark_gaia(axd, int(dx0), int(dy0), sc, dwcs) + axd.add_patch(plt.Circle((0, 0), C.APRAD, fill=False, + color="#f0c05a", lw=1.5)) + axd.set_xlim(-H, H) + axd.set_ylim(-H, H) + axd.set_title(f"archival DSS2 red, 1990s\nimplied change " + f"{arch['change']:+.2f} mag", fontsize=9.5, pad=6) + axd.set_xlabel("already present, at the same brightness\n" + "relative to its neighbours", fontsize=8) + else: + axd.text(0.5, 0.5, "DSS unavailable", ha="center", va="center", + transform=axd.transAxes) + axd.set_xticks([]) + axd.set_yticks([]) + + # --- curve of growth --- + axc = fig.add_subplot(gs[0, 2]) + axc.plot(RADII, star_cog, "-o", color="#3d7ba6", ms=4.5, lw=1.9, + label="isolated field stars") + axc.plot(RADII, src_cog, "-s", color="#b5484f", ms=4.5, lw=1.9, + label="this source") + axc.axvline(C.APRAD, color="#f0a83c", ls="--", lw=1.2) + axc.text(C.APRAD + 0.25, 0.13, "5 px aperture", fontsize=7.6, + color="#a8792c", rotation=90) + axc.set_xlabel("aperture radius (px)", fontsize=9) + axc.set_ylabel("enclosed flux / flux at r = 10 px", fontsize=9) + axc.set_title(f"curve of growth: resolved\nr50 = {r50src / r50star:.2f}" + f" x PSF", fontsize=9.5, pad=6) + axc.legend(fontsize=8, frameon=False, loc="lower right") + axc.grid(alpha=0.25, lw=0.6) + for sp in ("top", "right"): + axc.spines[sp].set_visible(False) + + # --- what the catalogues have here --- + axt = fig.add_subplot(gs[0, 3]) + axt.axis("off") + d = np.hypot(objs["x"] - px, objs["y"] - py) * C.SCALE + lines = ["detected in this image", ""] + for i in np.argsort(d)[:5]: + if d[i] > 16: + break + lines.append(f" {d[i]:5.2f}\" G = {omag[i]:5.2f}") + lines += ["", "Gaia DR3 (all magnitudes)", ""] + for i in np.argsort(gsep)[:5]: + lines.append(f" {gsep[i]:5.2f}\" G = {gg[i]:5.2f}") + axt.text(0.0, 1.0, "\n".join(lines), fontsize=8.4, family="monospace", + va="top", color="#2a2e34", transform=axt.transAxes, + linespacing=1.4) + axt.text(0.0, 0.16, + "Nothing in Gaia within 25 arcsec is\n" + "brighter than G = 19.8. Gaia has no\n" + "entry for the extended object that\n" + "dominates the light in the aperture.", + fontsize=8.2, va="top", color="#4a4f57", + transform=axt.transAxes, linespacing=1.5) + + # --- light curve --- + axl = fig.add_subplot(gs[1, 0:3]) + for j in range(comps.shape[1]): + off = comps[:, j] - comps[:, j].mean() + axl.plot(times * 60, off + lc.mean(), "-", color="#c3ccd4", lw=1.0, + zorder=1) + axl.plot([], [], "-", color="#c3ccd4", lw=1.0, + label=f"{comps.shape[1]} comparison stars, mean-subtracted " + f"(rms {comp_rms:.3f} mag)") + axl.errorbar(times * 60, lc, yerr=lcerr, fmt="o", color="#b5484f", + ms=5.5, lw=1.4, capsize=2.5, label="this source", zorder=3) + axl.axhline(lc.mean(), color="#b5484f", ls=":", lw=1.1) + axl.invert_yaxis() + axl.set_xlabel("minutes from first sub (2026-07-21 08:58 UTC)", + fontsize=9) + axl.set_ylabel("G (luminance, 5 px aperture)", fontsize=9) + axl.set_title(f"per-sub light curve: mean G = {lc.mean():.2f}, rms " + f"{lc.std():.3f} mag - no variation over 64 min", + fontsize=9.5, pad=6) + axl.legend(fontsize=8, frameon=False, loc="lower left") + axl.grid(alpha=0.25, lw=0.6) + for sp in ("top", "right"): + axl.spines[sp].set_visible(False) + + # --- verdict --- + axv = fig.add_subplot(gs[1, 3]) + axv.axis("off") + ch = arch["change"] if arch is not None else float("nan") + axv.text(0.0, 1.0, "VERDICT: measurement artefact,\nnot a brightening.", + fontsize=10, va="top", color="#b5484f", weight="bold", + transform=axv.transAxes, linespacing=1.4) + txt = ( + f"1. Resolved. r50 = {r50src:.2f} px vs PSF\n" + f" {r50star:.2f} px ({r50src / r50star:.2f} x), elongation\n" + " 1.40. An integrated aperture\n" + " magnitude of an extended object\n" + " is not comparable with Gaia's\n" + " point-source G.\n\n" + f"2. Steady. rms {lc.std():.3f} mag over 64\n" + f" min, vs {comp_rms:.3f} for field stars.\n\n" + f"3. Archival. Implied change since\n" + f" the 1990s: {ch:+.2f} mag.\n\n" + "4. Even the PSF-scaled 2 px core\n" + f" gives G = 18.6, still 2.4 mag\n" + " above Gaia's value." + ) + axv.text(0.0, 0.86, txt, fontsize=8.5, va="top", color="#2a2e34", + transform=axv.transAxes, linespacing=1.45) + + fig.text(0.02, 0.958, + "The G = 18.2 versus Gaia G = 21.0 discrepancy at " + "13:25:37.68 -43:02:29.9", fontsize=13.5, weight="bold", + color="#1b1e23", ha="left") + fig.text(0.02, 0.915, + "The source is resolved and elongated, photometrically steady " + "across the session, and already present on a 1990s sky-survey " + "plate at the same brightness relative to its neighbours.", + fontsize=9.5, color="#4a4f57", ha="left") + fig.text(0.02, 0.891, + "Gaia DR3 catalogues only a G = 21.0 point source 1.32 arcsec " + "away and has no entry for the extended object that dominates " + "the aperture. The two numbers measure different things.", + fontsize=9.5, color="#4a4f57", ha="left") + + fig.savefig(OUTPNG, dpi=130, facecolor="white") + print(f"\nwrote {OUTPNG}") + + +if __name__ == "__main__": + main() diff --git a/session-scripts/mo_check.py b/session-scripts/mo_check.py new file mode 100644 index 0000000..5423e6a --- /dev/null +++ b/session-scripts/mo_check.py @@ -0,0 +1,85 @@ +"""Diagnostic: how good is the coordinate registration, and how many +detections survive removal of the static sky? + +astroalign only reports a few dozen matched stars, which is enough to define a +similarity transform but says nothing about how well it holds across a +4788 x 3194 field. This refines each frame's transform by nearest-neighbour +matching every detection against the reference frame's detections and refitting +a full affine, then quotes the residual. It also counts, per frame, how many +detections are left once everything coincident with a master-stack source is +removed - that residual population is the input to the tracklet search. +""" +import os + +import numpy as np +from astropy.io import fits +from scipy.spatial import cKDTree +from skimage.transform import AffineTransform + +import layout + +SRC = layout.SESSION +OUT = layout.SESSION +CACHE = layout.path("_mo_dets.npz") + + +def load(): + z = np.load(CACHE, allow_pickle=True) + meta = {r[0]: r for r in z["meta"]} + dets = {k: z[k] for k in meta} + return meta, dets + + +def refine(dets, ref_key="Luminance_002", tol=3.0): + """Refit each frame's transform on all cross-matched detections.""" + ref = dets[ref_key] + tree = cKDTree(ref[:, :2]) + out = {} + for key, d in dets.items(): + src = d[:, 2:4].astype(float) # native coords + cur = d[:, :2].astype(float) # astroalign-transformed + for t in (tol, 1.5): + dist, idx = tree.query(cur, distance_upper_bound=t) + ok = np.isfinite(dist) + if ok.sum() < 50: + break + tf = AffineTransform() + tf.estimate(src[ok], ref[idx[ok], :2].astype(float)) + cur = tf(src) + dist, idx = tree.query(cur, distance_upper_bound=1.5) + ok = np.isfinite(dist) + out[key] = (cur, np.median(dist[ok]), np.percentile(dist[ok], 90), + ok.sum()) + return out + + +def main(): + meta, dets = load() + lum = [k for k in dets if k.startswith("Luminance")] + ref = refine(dets) + + print("registration residual against reference frame (px):") + for k in sorted(dets): + _, med, p90, n = ref[k] + print(f" {k:16s} n={n:5d} median={med:.3f} p90={p90:.3f}") + + # Static sky = every source in the deep luminance master. + with fits.open(layout.path("master-Luminance.fit")) as hd: + master = hd[0].data.astype(np.float32) + import sep + bkg = sep.Background(master, bw=64, bh=64, fw=3, fh=3) + m = sep.extract(master - bkg.back(), 2.5, err=bkg.globalrms, minarea=4, + deblend_cont=0.005) + print(f"\nmaster detections (static sky): {len(m)}") + mtree = cKDTree(np.column_stack([m["x"], m["y"]])) + + print("\nresiduals after removing anything within 4 px of a master source:") + for k in sorted(lum): + cur = ref[k][0] + d, _ = mtree.query(cur, distance_upper_bound=4.0) + left = ~np.isfinite(d) + print(f" {k:16s} {left.sum():5d} / {len(cur)}") + + +if __name__ == "__main__": + main() diff --git a/session-scripts/mo_common.py b/session-scripts/mo_common.py new file mode 100644 index 0000000..c8204eb --- /dev/null +++ b/session-scripts/mo_common.py @@ -0,0 +1,321 @@ +"""Shared machinery for the moving-object and transient searches. + +Holds the things both the real search and the synthetic-injection sensitivity +run need to do identically: detection, coordinate registration, photometric +calibration onto the master's zero point, static-sky rejection and tracklet +linking. Keeping one copy guarantees the sensitivity numbers describe the +pipeline that was actually run on the data, not a simplified stand-in. +""" +import itertools +import os + +import numpy as np +import sep +from astropy.io import fits +from scipy.spatial import cKDTree +from skimage.transform import AffineTransform + +import layout + +SRC = layout.SESSION +OUT = layout.SESSION +DETS = layout.path("_mo_dets.npz") +REFERENCE = "Luminance_002" + +ZP = 27.941 # master luminance, G = -2.5 log10(flux_5px) + ZP +SCALE = 0.5376 # arcsec/px +FWHM = 5.0 # px +SIGMA = FWHM / 2.3548 +APRAD = 5.0 # px, the radius the zero point is defined for +# Fraction of a Gaussian's total flux inside APRAD, used to convert a target +# aperture magnitude into the total flux of an injected source. +APFRAC = 1.0 - np.exp(-APRAD ** 2 / (2.0 * SIGMA ** 2)) + +# --- tracklet linking parameters --------------------------------------- +MIN_RATE = 2.5 # px/hr +MAX_RATE = 900.0 # px/hr +MIN_HITS = 5 +LINE_RMS = 1.2 # px +MAG_SCATTER = 0.5 # mag +TOL = 3.0 # px +PAIR_GAP = 3 +STATIC_MIN = 4 # recurrences at one place that mark a source static +STATIC_TOL = 2.0 # px +MIN_NATIVE_MOTION = 4.0 # px a real mover must travel in DETECTOR coordinates + + +# ---------------------------------------------------------------------- +# detection + +def detect(image, thresh=3.5, minarea=5): + """sep background estimate, extraction, and 5 px aperture photometry.""" + bkg = sep.Background(image, bw=64, bh=64, fw=3, fh=3) + sub = image - bkg.back() + objs = sep.extract(sub, thresh, err=bkg.globalrms, minarea=minarea, + deblend_cont=0.005) + if len(objs): + ap, _, _ = sep.sum_circle(sub, objs["x"], objs["y"], APRAD, + err=bkg.globalrms, gain=1.0) + else: + ap = np.zeros(0) + return objs, np.asarray(ap, dtype=float), float(bkg.globalrms) + + +def lum_frames(): + """(key, path) for the twelve luminance subs, in time order.""" + rows = [] + n = 0 + for fn in sorted(os.listdir(SRC)): + if fn.startswith("calibrated-") and fn.endswith(".fit") \ + and "-Luminance-" in fn: + n += 1 + rows.append((f"Luminance_{n:03d}", layout.path(fn))) + return rows + + +def lum_times(): + """Mid-exposure times of the luminance subs, in hours from the first.""" + t = [] + for _, path in lum_frames(): + jd = float(fits.getheader(path)["JD"]) + t.append(jd - 2400000.5 + 150.0 / 86400.0) + t = np.array(t) + return (t - t[0]) * 24.0 + + +# ---------------------------------------------------------------------- +# registration + +def load_dets(): + z = np.load(DETS, allow_pickle=True) + meta = {r[0]: r for r in z["meta"]} + return meta, {k: z[k] for k in meta} + + +def frame_transforms(): + """Native-pixel -> reference-frame affine for each luminance sub. + + Seeded by the astroalign solution cached in _mo_dets.npz, then refined by + nearest-neighbour matching every detection against the reference frame's + detections. The refined solutions are good to ~0.15 px median across the + whole field (see mo_check.py), which is what makes a 3 px association + radius meaningful. + """ + _, dets = load_dets() + ref = dets[REFERENCE] + tree = cKDTree(ref[:, :2]) + tf_out = {} + for key, d in dets.items(): + if not key.startswith("Luminance"): + continue + src = d[:, 2:4].astype(float) + cur = d[:, :2].astype(float) + tf = None + for t in (3.0, 1.5): + dist, idx = tree.query(cur, distance_upper_bound=t) + ok = np.isfinite(dist) + if ok.sum() < 50: + break + tf = AffineTransform() + tf.estimate(src[ok], ref[idx[ok], :2].astype(float)) + cur = tf(src) + if tf is None: # the reference frame itself + tf = AffineTransform() + tf.estimate(src, d[:, :2].astype(float)) + tf_out[key] = tf + return tf_out + + +# ---------------------------------------------------------------------- +# photometry + +def master_sources(): + """Detections and 5 px aperture fluxes in the deep luminance master.""" + with fits.open(layout.path("master-Luminance.fit")) as hd: + img = hd[0].data.astype(np.float32) + hdr = hd[0].header + objs, ap, rms = detect(img, thresh=2.5, minarea=4) + return objs, ap, rms, hdr, img + + +def frame_zeropoint(fx, fy, fap, mobjs, map_, tf): + """Per-frame zero point, tied to the master's, from stars in common. + + The master is a scaled sigma-clipped mean of the twelve subs, so a single + sub's counts differ from it by a near-constant factor. Measuring that + factor on a few hundred stars puts single-frame magnitudes on the master's + system without a separate absolute calibration. + """ + ref_xy = tf(np.column_stack([fx, fy])) + mt = cKDTree(np.column_stack([mobjs["x"], mobjs["y"]])) + d, idx = mt.query(ref_xy, distance_upper_bound=1.5) + ok = np.isfinite(d) & (fap > 0) + if ok.sum() < 30: + return ZP, 0 + fm = map_[idx[ok]] + ff = fap[ok] + good = (fm > 0) & (ff > 0) + # Bright half only: faint stars are noise-biased in both frames. + order = np.argsort(fm[good])[::-1] + sel = order[:max(50, len(order) // 4)] + ratio = np.median((ff[good][sel] / fm[good][sel])) + return ZP + 2.5 * np.log10(ratio), int(ok.sum()) + + +# ---------------------------------------------------------------------- +# static-sky rejection and linking + +def residual_mask(pts_list, nat_list, mtree, master_tol=4.0): + """True for detections that are neither static sky nor detector defects. + + Three rejections, and the third is the one that matters most on this data: + + 1. Coincident with a source in the deep luminance master. + 2. Recurring at the same REFERENCE-frame position in four or more subs - + a star the master missed. + 3. Recurring at the same DETECTOR-frame position in four or more subs. + + Cut 3 exists because the frames are dithered and the field rotates slowly + through the session. Registration undoes that motion for the sky, which + means it imposes exactly the opposite motion on anything fixed to the + detector. A hot pixel therefore traces a perfectly straight, perfectly + constant-rate, perfectly constant-brightness track across the registered + sequence - an ideal fake minor planet that passes every cut a linker + normally applies. Without this cut the search on these twelve subs returns + 141 such tracks, all of them moving at 9-18 arcsec/hr toward position + angle 45-85 deg, all with npix of 5-10 (a 3x3 blob, against 50-100 for a + real star at this seeing), and all stationary to under 0.5 px in detector + coordinates. With it, none survive. + """ + trees = [cKDTree(p) if len(p) else None for p in pts_list] + ntrees = [cKDTree(p) if len(p) else None for p in nat_list] + out = [] + for i, p in enumerate(pts_list): + if not len(p): + out.append(np.zeros(0, dtype=bool)) + continue + d, _ = mtree.query(p, distance_upper_bound=master_tol) + in_master = np.isfinite(d) + recur = np.zeros(len(p), dtype=int) + nrecur = np.zeros(len(p), dtype=int) + for j, (tr, ntr) in enumerate(zip(trees, ntrees)): + if j == i or tr is None: + continue + dd, _ = tr.query(p, distance_upper_bound=STATIC_TOL) + recur += np.isfinite(dd) + nd, _ = ntr.query(nat_list[i], distance_upper_bound=STATIC_TOL) + nrecur += np.isfinite(nd) + out.append((~in_master) & (recur < STATIC_MIN) & + (nrecur < STATIC_MIN)) + return out + + +def fit_track(t, x, y): + A = np.column_stack([np.ones_like(t), t]) + cx, *_ = np.linalg.lstsq(A, x, rcond=None) + cy, *_ = np.linalg.lstsq(A, y, rcond=None) + rms = float(np.sqrt(np.mean((x - A @ cx) ** 2 + (y - A @ cy) ** 2))) + return cx, cy, rms + + +def link(times, pts, mags, nat=None, shape=None, min_hits=MIN_HITS): + """Pair-seeded, propagate-and-count tracklet finder. See mo_link.py.""" + n = len(times) + trees = [cKDTree(p) if len(p) else None for p in pts] + seen, cands = set(), [] + for i, j in itertools.combinations(range(n), 2): + if j - i < PAIR_GAP or trees[i] is None or trees[j] is None: + continue + dt = times[j] - times[i] + if dt <= 0: + continue + near = trees[j].query_ball_point(pts[i], MAX_RATE * dt) + for a, blist in enumerate(near): + for b in blist: + if np.hypot(*(pts[j][b] - pts[i][a])) < MIN_RATE * dt: + continue + v = (pts[j][b] - pts[i][a]) / dt + hits = [] + for f in range(n): + if trees[f] is None: + continue + pred = pts[i][a] + v * (times[f] - times[i]) + dd, idx = trees[f].query(pred, distance_upper_bound=TOL) + if np.isfinite(dd): + hits.append((f, int(idx))) + if len(hits) < min_hits: + continue + sig = tuple(sorted(hits)) + if sig in seen: + continue + seen.add(sig) + tt = np.array([times[f] for f, _ in hits]) + xx = np.array([pts[f][k][0] for f, k in hits]) + yy = np.array([pts[f][k][1] for f, k in hits]) + cx, cy, rms = fit_track(tt, xx, yy) + if rms > LINE_RMS: + continue + mm = np.array([mags[f][k] for f, k in hits]) + if not np.all(np.isfinite(mm)) or mm.std() > MAG_SCATTER: + continue + if nat is not None: + # Backstop for cut 3 in residual_mask: whatever this is, + # it must have genuinely moved across the detector, not + # merely been carried by the registration. + nx_ = np.array([nat[f][k][0] for f, k in hits]) + ny_ = np.array([nat[f][k][1] for f, k in hits]) + if np.hypot(np.ptp(nx_), np.ptp(ny_)) < MIN_NATIVE_MOTION: + continue + c = dict(frames=np.array([f for f, _ in hits]), + idx=np.array([k for _, k in hits]), + t=tt, x=xx, y=yy, cx=cx, cy=cy, rms=rms, + rate=float(np.hypot(cx[1], cy[1])), + ang=float(np.degrees(np.arctan2(cy[1], cx[1]))), + mag=float(mm.mean()), magsig=float(mm.std()), + nhit=len(hits)) + if shape is not None: + sh = np.array([shape[f][k] for f, k in hits]) + c.update(a=float(np.median(sh[:, 0])), + b=float(np.median(sh[:, 1])), + npix=float(np.median(sh[:, 2]))) + cands.append(c) + return dedup(cands) + + +def dedup(cands): + """Among tracklets sharing two or more detections, keep the best.""" + cands.sort(key=lambda c: (-c["nhit"], c["rms"])) + kept, sets = [], [] + for c in cands: + s = set(zip(c["frames"].tolist(), c["idx"].tolist())) + if any(len(s & o) >= 2 for o in sets): + continue + sets.append(s) + kept.append(c) + return kept + + +# ---------------------------------------------------------------------- +# synthetic sources + +def add_source(image, x, y, total_flux, dx, dy, nstep=9, sigma=SIGMA): + """Add a trailed Gaussian: the source moves (dx, dy) px during the sub. + + A 300 s exposure of something moving at 30 arcsec/hr smears by ~4.6 px, + comparable to the seeing disc, so the trail is modelled rather than + ignored - it is what limits sensitivity at high rates. + """ + ny, nx = image.shape + half = int(np.ceil(4 * sigma + 0.5 * np.hypot(dx, dy))) + 2 + x0, x1 = int(max(0, x - half)), int(min(nx, x + half + 1)) + y0, y1 = int(max(0, y - half)), int(min(ny, y + half + 1)) + if x1 <= x0 or y1 <= y0: + return + gy, gx = np.mgrid[y0:y1, x0:x1] + acc = np.zeros(gx.shape, dtype=np.float64) + for s in np.linspace(-0.5, 0.5, nstep): + cx, cy = x + s * dx, y + s * dy + acc += np.exp(-((gx - cx) ** 2 + (gy - cy) ** 2) / + (2.0 * sigma ** 2)) + acc *= total_flux / (nstep * 2.0 * np.pi * sigma ** 2) + image[y0:y1, x0:x1] += acc.astype(image.dtype) diff --git a/session-scripts/mo_detect.py b/session-scripts/mo_detect.py new file mode 100644 index 0000000..35d43de --- /dev/null +++ b/session-scripts/mo_detect.py @@ -0,0 +1,137 @@ +"""Moving-object search, pass 1: per-sub source detection and frame registration. + +Every calibrated sub is opened one at a time (they are 61 MB each and there is +only a few GB of RAM), background-subtracted with sep, and its sources +extracted. The detection list, not the pixels, is what gets carried forward. + +Registration is done on the coordinates rather than the images. astroalign +matches asterisms between each sub's star list and the reference sub's star +list and returns a similarity transform; applying that transform to the +detection coordinates puts every sub's sources into one common pixel grid +without ever warping a 15 Mpx array. The reference is Luminance_002, the same +frame the master stack was registered to, so the master's WCS applies directly +to the common grid and every detection can be turned into RA/Dec. + +Output: _mo_dets.npz with one record array per frame plus the transform +parameters and mid-exposure times. +""" +import os +import sys + +import astroalign as aa +import numpy as np +import sep +from astropy.io import fits + +import layout + +SRC = layout.SESSION +OUT = layout.SESSION +CACHE = layout.path("_mo_dets.npz") +REFERENCE = "Luminance_002" + +aa.MIN_MATCHES_FRACTION = 0.6 +aa.NUM_NEAREST_NEIGHBORS = 8 + +# Detection threshold in sigma. Deliberately low: a moving object is only in +# any one sub for 300 s, so it is much fainter per-frame than in the 60 min +# master. Spurious detections are cheap here because the tracklet linker +# demands a straight line through five or more frames, which noise does not +# supply. +THRESH = 3.5 +MINAREA = 5 + + +def frames(): + """(key, filename, filter, mid-exposure MJD) for every calibrated sub.""" + rows = [] + counts = {} + for fn in sorted(os.listdir(SRC)): + if not (fn.startswith("calibrated-") and fn.endswith(".fit")): + continue + filt = fn.split("-")[6] + counts[filt] = counts.get(filt, 0) + 1 + rows.append((f"{filt}_{counts[filt]:03d}", fn, filt)) + return rows + + +def detect(image, thresh=THRESH, minarea=MINAREA): + bkg = sep.Background(image, bw=64, bh=64, fw=3, fh=3) + sub = image - bkg.back() + rms = bkg.globalrms + objs = sep.extract(sub, thresh, err=rms, minarea=minarea, + deblend_cont=0.005, filter_kernel=None) + return objs, sub, rms + + +def main(): + rows = frames() + print(f"{len(rows)} calibrated subs") + + # Reference star list first: everything else is matched onto it. + ref_fn = [r[1] for r in rows if r[0] == REFERENCE][0] + with fits.open(layout.path(ref_fn), memmap=False) as hd: + img = hd[0].data.astype(np.float32) + objs, _, rms = detect(img) + bright = objs[(objs["flag"] == 0) & (objs["npix"] > 12) & + (objs["npix"] < 3000)] + bright = bright[np.argsort(bright["flux"])[::-1][:600]] + ref_xy = np.column_stack([bright["x"], bright["y"]]) + del img + print(f"reference {REFERENCE}: {len(ref_xy)} registration stars, " + f"rms {rms:.2f}") + + store = {} + meta = [] + for key, fn, filt in rows: + path = layout.path(fn) + with fits.open(path, memmap=False) as hd: + img = hd[0].data.astype(np.float32) + mjd = float(hd[0].header["JD"]) - 2400000.5 + mjd_mid = mjd + 150.0 / 86400.0 # mid-exposure + objs, _, rms = detect(img) + del img + + clean = objs[(objs["flag"] == 0) & (objs["npix"] > 12) & + (objs["npix"] < 3000)] + clean = clean[np.argsort(clean["flux"])[::-1][:600]] + xy = np.column_stack([clean["x"], clean["y"]]) + + if key == REFERENCE: + params = (1.0, 0.0, 0.0, 0.0) + nmatch = len(xy) + tx, ty = objs["x"].copy(), objs["y"].copy() + else: + try: + tform, (src_m, _) = aa.find_transform(xy, ref_xy) + except Exception as exc: # noqa: BLE001 + print(f" {key:16s} REGISTRATION FAILED ({exc}) - dropped") + continue + nmatch = len(src_m) + pts = np.column_stack([objs["x"], objs["y"]]) + warped = tform(pts) + tx, ty = warped[:, 0], warped[:, 1] + params = (tform.scale, np.degrees(tform.rotation), + tform.translation[0], tform.translation[1]) + + # Everything the tracklet linker and the vetting need, per detection. + rec = np.column_stack([ + tx, ty, # reference-frame x, y + objs["x"], objs["y"], # native x, y + objs["flux"], objs["peak"], + objs["a"], objs["b"], objs["theta"], + objs["npix"].astype(float), objs["flag"].astype(float), + ]).astype(np.float32) + store[key] = rec + meta.append((key, fn, filt, f"{mjd_mid:.8f}", f"{rms:.4f}", + str(len(rec)), str(nmatch), + *[f"{p:.6f}" for p in params])) + print(f" {key:16s} {len(rec):5d} det match={nmatch:3d} " + f"rms={rms:6.2f} rot={params[1]:+7.3f} deg") + + np.savez_compressed(CACHE, meta=np.array(meta, dtype=object), **store) + print(f"\nwrote {CACHE}") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/session-scripts/mo_fig_moving.py b/session-scripts/mo_fig_moving.py new file mode 100644 index 0000000..62d3965 --- /dev/null +++ b/session-scripts/mo_fig_moving.py @@ -0,0 +1,283 @@ +"""Figure: the moving-object search, its sensitivity, and its failure mode. + +The blind search found nothing, so the figure has to show that the search +worked rather than showing a discovery. Four things are plotted. + +Rows 1-2 are postage-stamp strips across the twelve luminance subs, cut at a +sky position that is FIXED on the sky (the object's position in the first sub). +A real moving object drifts across the strip. So does a hot pixel, because +registration holds the sky still and therefore drags anything fixed to the +detector in the opposite direction. The two are indistinguishable here, which +is the whole problem. + +Rows 3-4 are the same two objects, cut instead at a FIXED DETECTOR position. +Now they separate cleanly: the real object still moves, the hot pixel does not +move at all. This is the cut that took the search from 141 confident false +detections to zero. + +The bottom panels are the sensitivity: recovery fraction of synthetic movers as +a function of magnitude and apparent rate, from three independent injection +runs (twelve injections per cell), and a slice through it. +""" +import os + +import numpy as np +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from astropy.io import fits +from matplotlib.colors import LinearSegmentedColormap +from scipy.spatial import cKDTree + +import mo_common as C + +HALF = 13 +OUTPNG = os.path.join(C.OUT, "NGC5128-mo-moving-object-search.png") + +INJ_MAG = 18.5 +INJ_RATE = 25.0 # px/hr = 13.4 arcsec/hr + + +def load_grids(): + mags = rates = None + grids = [] + for seed in (1, 2, 3): + p = os.path.join(C.OUT, f"_mo_sensitivity_{seed}.npz") + if not os.path.exists(p): + continue + z = np.load(p) + mags, rates = z["mags"], z["rates"] + grids.append(z["grid"]) + return mags, rates, np.mean(grids, axis=0), len(grids) * int( + np.load(os.path.join(C.OUT, "_mo_sensitivity_1.npz"))["nrep"]) + + +# A detector defect verified by hand: reproducing the search with the +# detector-frame cuts switched off yields 141 tracklets, and this one is +# typical. Its detected centroid sits at native pixel (224.00, 787.00) in +# every one of subs 002-009 - not merely close, but the same pixel centre to +# 0.02 px - while its registered position walks 18.6 px across the sequence +# at 15.9 arcsec/hr with a brightness stable to 0.06 mag. It is a hot pixel. +HOT_NATIVE = np.array([224.00, 787.00]) +HOT_REF_KEY = "Luminance_002" + + +def find_hot_pixel(tforms, keys): + return HOT_NATIVE, 8 + + +def stamps(inj_ref0, inj_v, hot_native, tforms, keys, times): + """Cut all four strips in a single pass over the subs.""" + out = {"inj_sky": [], "inj_det": [], "hot_sky": [], "hot_det": []} + hot_ref0 = tforms[keys[0]](hot_native[None, :])[0] + inj_native0 = tforms[keys[0]].inverse(inj_ref0[None, :])[0] + + def cut(img, x, y): + xi, yi = int(round(x)), int(round(y)) + return img[yi - HALF:yi + HALF + 1, xi - HALF:xi + HALF + 1].copy() + + for i, (key, path) in enumerate(C.lum_frames()): + with fits.open(path, memmap=False) as hd: + img = hd[0].data.astype(np.float32) + tf = tforms[key] + inv = tf.inverse + lin = np.linalg.inv(tf.params[:2, :2]) + # Inject the synthetic at its true position in this sub. + true_ref = inj_ref0 + inj_v * times[i] + tx, ty = inv(true_ref[None, :])[0] + objs, ap, _ = C.detect(img) + mobjs, map_, _, _, _ = MASTER + zp, _ = C.frame_zeropoint(objs["x"], objs["y"], ap, mobjs, map_, tf) + d = lin @ (inj_v * 300.0 / 3600.0) + C.add_source(img, tx, ty, 10 ** ((zp - INJ_MAG) / 2.5) / C.APFRAC, + d[0], d[1]) + # Strip 1: cut at the sky position the object had at t=0. + sx, sy = inv(inj_ref0[None, :])[0] + out["inj_sky"].append(cut(img, sx, sy)) + # Strip 2: cut at the detector position it had at t=0. + out["inj_det"].append(cut(img, inj_native0[0], inj_native0[1])) + # Strip 3: hot pixel, cut at its t=0 sky position. + hx, hy = inv(hot_ref0[None, :])[0] + out["hot_sky"].append(cut(img, hx, hy)) + # Strip 4: hot pixel, cut at its fixed detector position. + out["hot_det"].append(cut(img, hot_native[0], hot_native[1])) + del img + print(f" stamps from {key}") + return out + + +def show(ax, s, vlo, vhi, edge="#7a7f87"): + ax.imshow(s, origin="lower", cmap="gray", vmin=vlo, vmax=vhi, + interpolation="nearest") + n = s.shape[0] + c = (n - 1) / 2.0 + # A faint crosshair marks the centre of the stamp, i.e. the position the + # cut is held fixed at. Whether the source stays on it is the whole point. + ax.plot([c, c], [c - 0.30 * n, c - 0.12 * n], color="#f0c05a", lw=0.8) + ax.plot([c - 0.30 * n, c - 0.12 * n], [c, c], color="#f0c05a", lw=0.8) + ax.set_xlim(-0.5, n - 0.5) + ax.set_ylim(-0.5, n - 0.5) + ax.set_xticks([]) + ax.set_yticks([]) + for sp in ax.spines.values(): + sp.set_color(edge) + sp.set_linewidth(0.8) + + +def main(): + global MASTER + MASTER = C.master_sources() + times = C.lum_times() + keys = [k for k, _ in C.lum_frames()] + tforms = C.frame_transforms() + + hot, hotn = find_hot_pixel(tforms, keys) + print(f"hot pixel at detector ({hot[0]:.1f}, {hot[1]:.1f}), " + f"present in {hotn + 1}/12 subs") + + ang = np.radians(35.0) + inj_v = INJ_RATE * np.array([np.cos(ang), np.sin(ang)]) + mobjs = MASTER[0] + mtree = cKDTree(np.column_stack([mobjs["x"], mobjs["y"]])) + rng = np.random.default_rng(11) + for _ in range(50000): + p0 = np.array([rng.uniform(600, 4200), rng.uniform(600, 2600)]) + track = np.array([p0 + inj_v * t for t in times]) + if np.hypot(*(p0 - np.array([2394.0, 1597.0]))) < 1100: + continue + d, _ = mtree.query(track, distance_upper_bound=30.0) + if np.all(~np.isfinite(d)): + inj_ref0 = p0 + break + else: + inj_ref0 = np.array([1500.0, 2500.0]) + print(f"synthetic injected at reference pixel " + f"({inj_ref0[0]:.0f}, {inj_ref0[1]:.0f})") + st = stamps(inj_ref0, inj_v, hot, tforms, keys, times) + + mags, rates, grid, nrep = load_grids() + + from astropy.stats import sigma_clipped_stats + + fig = plt.figure(figsize=(15.0, 11.4)) + gs = fig.add_gridspec( + 5, 12, height_ratios=[0.78, 0.78, 0.78, 0.78, 2.5], + hspace=0.16, wspace=0.06, + left=0.175, right=0.985, top=0.878, bottom=0.08) + + labels = [ + ("inj_sky", "synthetic mover", "held at fixed SKY position", + "#2f7d54", "moves -> could be real"), + ("hot_sky", "hot pixel", "held at fixed SKY position", + "#b5484f", "also moves -> looks identical"), + ("inj_det", "synthetic mover", "held at fixed DETECTOR position", + "#2f7d54", "still moves -> REAL"), + ("hot_det", "hot pixel", "held at fixed DETECTOR position", + "#b5484f", "does not move -> DEFECT"), + ] + for r, (kk, who, how, col, verdict) in enumerate(labels): + arr = np.array(st[kk], dtype=float) + med, _, sd = sigma_clipped_stats(arr, sigma=3.0) + vlo, vhi = med - 1.5 * sd, med + 14.0 * sd + for c in range(12): + ax = fig.add_subplot(gs[r, c]) + show(ax, st[kk][c], vlo, vhi, edge=col) + if r == 0: + ax.set_title(f"{times[c] * 60:.0f} min", fontsize=8.5, + color="#3b3f45", pad=4) + if c == 0: + bb = ax.get_position() + fig.text(0.170, bb.y0 + bb.height * 0.80, who, fontsize=10.5, + color=col, ha="right", va="center", weight="bold") + fig.text(0.170, bb.y0 + bb.height * 0.50, how, fontsize=8.4, + color="#5a6068", ha="right", va="center") + fig.text(0.170, bb.y0 + bb.height * 0.18, verdict, + fontsize=8.6, color=col, ha="right", va="center", + style="italic") + + fig.text(0.02, 0.962, + "Moving-object search: 12 x 300 s luminance subs of NGC 5128, " + "2026-07-21 08:56-10:00 UTC", + fontsize=14, color="#1b1e23", weight="bold", ha="left") + fig.text(0.02, 0.937, + "Each strip is a 14 x 14 arcsec cutout, one per sub, with the " + "cut position held fixed (yellow crosshair). Registration holds " + "the sky still, so it drags anything fixed to the", + fontsize=9.4, color="#4a4f57", ha="left") + fig.text(0.02, 0.918, + "detector across the registered frame - a hot pixel therefore " + "mimics a minor planet perfectly. Cutting at a fixed detector " + "position separates them. Synthetic source: G = " + f"{INJ_MAG}, {INJ_RATE * C.SCALE:.1f} arcsec/hr.", + fontsize=9.4, color="#4a4f57", ha="left") + + # ---- sensitivity heat map ---- + axg = fig.add_subplot(gs[4, 0:6]) + cmap = LinearSegmentedColormap.from_list( + "rec", ["#f5f6f8", "#cfe0ec", "#7fb0cd", "#3d7ba6", "#17456b"]) + im = axg.imshow(grid, origin="lower", aspect="auto", cmap=cmap, + vmin=0, vmax=1, interpolation="nearest") + axg.set_xticks(range(len(rates))) + axg.set_xticklabels([f"{r * C.SCALE:.1f}" for r in rates], fontsize=8.5) + axg.set_yticks(range(len(mags))) + axg.set_yticklabels([f"{m:g}" for m in mags], fontsize=9) + axg.set_xlabel("apparent rate (arcsec/hr)", fontsize=10) + axg.set_ylabel("G magnitude", fontsize=10) + axg.set_title(f"recovery of injected synthetic movers " + f"({nrep} per cell, 3 independent runs)", fontsize=10.5, + color="#1b1e23", pad=8) + for i in range(len(mags)): + for j in range(len(rates)): + v = grid[i, j] + axg.text(j, i, f"{v:.2f}".lstrip("0") if v else ".", + ha="center", va="center", fontsize=7.0, + color="white" if v > 0.55 else "#585d65") + cb = fig.colorbar(im, ax=axg, fraction=0.032, pad=0.014) + cb.set_label("recovered", fontsize=9) + cb.ax.tick_params(labelsize=8) + + # ---- slices ---- + axs = fig.add_subplot(gs[4, 7:12]) + band = (rates * C.SCALE >= 8) & (rates * C.SCALE <= 40) + prof = grid[:, band].mean(axis=1) + axs.plot(mags, prof, "-o", color="#2e6f9e", lw=2.2, ms=5.5, + label="8-40 arcsec/hr (main-belt range)") + prof2 = grid[:, rates * C.SCALE > 60].mean(axis=1) + axs.plot(mags, prof2, "-s", color="#b5484f", lw=1.8, ms=4.5, + label="> 60 arcsec/hr (trailing losses)") + axs.axhline(0.5, color="#9aa0a8", ls=":", lw=1.2) + axs.text(mags[0] + 0.02, 0.53, "50% recovery", fontsize=8, + color="#6b7079") + axs.set_xlabel("G magnitude", fontsize=10) + axs.set_ylabel("fraction recovered", fontsize=10) + axs.set_ylim(-0.03, 1.08) + axs.set_title("sensitivity vs magnitude", fontsize=10.5, + color="#1b1e23", pad=8) + axs.legend(fontsize=8.5, frameon=False, loc="lower left") + axs.grid(alpha=0.25, lw=0.6) + for sp in ("top", "right"): + axs.spines[sp].set_visible(False) + + order = np.argsort(prof) + g50 = float(np.interp(0.5, prof[order], mags[order])) + print(f"50% recovery for main-belt rates at G = {g50:.2f}") + fig.text(0.02, 0.024, + "Result: 0 candidates from the blind search. Time-permuted " + "control on the real residuals: 0 tracklets in 60 trials. " + f"50% recovery at G = {g50:.1f} for main-belt rates. The only " + "catalogued minor planet within 30' of the pointing,", + fontsize=9, color="#4a4f57", ha="left") + fig.text(0.02, 0.006, + "(427494) 2002 BK26 at V = 21.7, fell 2.4 arcmin outside the " + "frame - the field's long axis runs along declination, not right " + "ascension.", + fontsize=9, color="#4a4f57", ha="left") + + fig.savefig(OUTPNG, dpi=125, facecolor="white") + print(f"wrote {OUTPNG}") + + +MASTER = None + +if __name__ == "__main__": + main() diff --git a/session-scripts/mo_fig_transient.py b/session-scripts/mo_fig_transient.py new file mode 100644 index 0000000..67a252b --- /dev/null +++ b/session-scripts/mo_fig_transient.py @@ -0,0 +1,228 @@ +"""Figure: the transient search and why its six finalists are not transients. + +Top block, one column per finalist: the 60 min luminance master beside a +Digitized Sky Survey red plate of the same patch of sky from the 1990s. If a +source is on both, it is not a transient, and that is a judgement a reader can +make from the picture without trusting any of the catalogue machinery. + +Bottom left: where the catalogues run out. Almost every source brighter than +G = 19 has a Gaia DR3 counterpart; below that the unmatched fraction climbs +steeply, not because transients appear but because Gaia's G < 20.5 point-source +catalogue stops being complete against a bright galaxy. This is what sets the +honest limit of the search. + +Bottom middle: the point-source selection. Half-light radius against magnitude, +with the image's own PSF measured from Gaia stars. Note that Centaurus A's +globular clusters sit inside the stellar locus - at 3.8 Mpc they are unresolved +in 2.7 arcsec seeing - so morphology cannot separate a cluster from a +supernova here, and five of the six finalists are clusters. + +Bottom right: where the finalists are, on the master. +""" +import os + +import numpy as np +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.stats import sigma_clipped_stats +from astropy.wcs import WCS + +import mo_common as C + +OUTPNG = os.path.join(C.OUT, "NGC5128-mo-transient-search.png") +HALF = 26 # px in the master = 28 arcsec across + +# Identifications established by cross-matching the six catalogue non-matches +# against published Centaurus A cluster catalogues, SIMBAD and the VizieR +# archive as a whole (see notes-transient-search.md). +IDENT = { + "201.672625": ("WHH-31", "Cen A globular cluster", "V=19.50 (Woodley+07)"), + "201.500583": ("pff_gc-081", "Cen A globular cluster", + "V=19.34 (Woodley+07)"), + "201.441625": ("WHH-24", "Cen A globular cluster", "V=19.63 (Woodley+07)"), + "201.407000": ("at CAVS 210", + "RESOLVED (1.34x PSF), archival 1996-2020", + "DENIS I=18.23; Gaia pt src G=21.03 at 1.32\" - see 4.3"), + "201.303542": ("WHH-10", "Cen A globular cluster", "V=19.57 (Woodley+07)"), + "201.256708": ("pff_gc-029", "Cen A globular cluster", + "V=19.37 (Woodley+07)"), +} + + +def ident_for(ra): + for k, v in IDENT.items(): + if abs(float(k) - ra) < 1e-4: + return v + return ("unidentified", "", "") + + +def stretch(a, lo=25.0, hi=99.6): + a = np.asarray(a, dtype=float) + good = np.isfinite(a) + if not good.any(): + return 0.0, 1.0 + return np.percentile(a[good], lo), np.percentile(a[good], hi) + + +def main(): + rows = list(np.load(os.path.join(C.OUT, "_mo_vetted.npy"), + allow_pickle=True)) + rows.sort(key=lambda r: r["dist_cenA_arcmin"]) + z = np.load(os.path.join(C.OUT, "_mo_transient.npz")) + + with fits.open(os.path.join(C.OUT, "master-Luminance.fit")) as hd: + master = hd[0].data.astype(np.float32) + hdr = hd[0].header + wcs = WCS(hdr) + + import sep + bkg = sep.Background(master, bw=64, bh=64, fw=3, fh=3) + msub = master - bkg.back() + + n = len(rows) + fig = plt.figure(figsize=(15.0, 11.6)) + gs = fig.add_gridspec(3, n, height_ratios=[1.0, 1.0, 1.55], + hspace=0.30, wspace=0.09, + left=0.055, right=0.985, top=0.845, bottom=0.075) + + for i, r in enumerate(rows): + name, kind, phot = ident_for(r["ra_deg"]) + x, y = int(round(r["x"])), int(round(r["y"])) + cut = msub[y - HALF:y + HALF + 1, x - HALF:x + HALF + 1] + ax = fig.add_subplot(gs[0, i]) + vlo, vhi = stretch(cut, 20, 99.5) + ax.imshow(cut, origin="lower", cmap="gray", vmin=vlo, vmax=vhi) + ax.plot([HALF, HALF], [HALF + 6, HALF + 11], color="#f0c05a", lw=1.1) + ax.plot([HALF + 6, HALF + 11], [HALF, HALF], color="#f0c05a", lw=1.1) + ax.set_xticks([]); ax.set_yticks([]) + for sp in ax.spines.values(): + sp.set_color("#2f7d54" if kind.startswith("Cen") else "#b5484f") + sp.set_linewidth(1.4) + ax.set_title(f"{name}\n{r['ra_hms']} {r['dec_dms']}", fontsize=8.6, + color="#1b1e23", pad=5) + if i == 0: + ax.set_ylabel("this image\n60 min luminance", fontsize=9, + color="#3b3f45") + + # archival plate + tag = f"{r['ra_deg']:.5f}{r['dec_deg']:+.5f}_mred.fits" + path = os.path.join(C.OUT, "_dss", tag) + ax2 = fig.add_subplot(gs[1, i]) + if os.path.exists(path): + with fits.open(path) as h2: + d = np.asarray(h2[0].data, dtype=float) + v1, v2 = stretch(d, 20, 99.5) + ax2.imshow(d, origin="lower", cmap="gray", vmin=v1, vmax=v2) + m = d.shape[0] / 2.0 + f = d.shape[0] / (2.0 * HALF + 1) + ax2.plot([m, m], [m + 6 * f, m + 11 * f], color="#f0c05a", lw=1.1) + ax2.plot([m + 6 * f, m + 11 * f], [m, m], color="#f0c05a", lw=1.1) + else: + ax2.text(0.5, 0.5, "no DSS cutout", ha="center", va="center", + fontsize=8, color="#8a9099", transform=ax2.transAxes) + ax2.set_xticks([]); ax2.set_yticks([]) + for sp in ax2.spines.values(): + sp.set_color("#9aa0a8") + sp.set_linewidth(0.9) + ax2.set_xlabel(f"{kind}\n{phot}\nG(this image) = {r['g_mag']:.2f}, " + f"{r['dist_cenA_arcmin']:.1f}' from nucleus", + fontsize=7.8, color="#4a4f57", labelpad=5) + if i == 0: + ax2.set_ylabel("archival\nDSS2 red, 1990s", fontsize=9, + color="#3b3f45") + + # ---------------- magnitude completeness ---------------- + axm = fig.add_subplot(gs[2, 0:2]) + good = z["good"] & z["pointlike"] + mag = z["mag"][good] + matched = z["gaia_ok"][good] + bins = np.arange(14.0, 21.01, 0.25) + axm.hist([mag[matched], mag[~matched]], bins=bins, stacked=True, + color=["#3d7ba6", "#b5484f"], + label=["matched to Gaia DR3", "no Gaia counterpart"]) + axm.axvline(20.5, color="#4a4f57", ls="--", lw=1.2) + axm.text(20.42, axm.get_ylim()[1] * 0.92, "Gaia cat.\nlimit G=20.5", + fontsize=7.6, color="#4a4f57", ha="right", va="top") + axm.set_xlabel("G magnitude (5 px aperture, master)", fontsize=9.5) + axm.set_ylabel("point-like sources per 0.25 mag", fontsize=9.5) + axm.set_title("where the reference catalogue runs out", fontsize=10.5, + color="#1b1e23", pad=7) + axm.legend(fontsize=8.2, frameon=False, loc="upper left") + axm.grid(alpha=0.22, lw=0.6) + for sp in ("top", "right"): + axm.spines[sp].set_visible(False) + + # ---------------- morphology ---------------- + axr = fig.add_subplot(gs[2, 2:4]) + sel = z["good"] + axr.scatter(z["mag"][sel], z["r50"][sel] / z["r50_star"], s=2.5, + c="#9fb6c6", alpha=0.35, lw=0, label="all detections") + ci = z["cand_idx"] + axr.scatter(z["mag"][ci], z["r50"][ci] / z["r50_star"], s=9, + c="#e0a13c", alpha=0.8, lw=0, + label="no Gaia / SkyMapper match") + for r in rows: + axr.scatter([r["g_mag"]], [r["r50_over_psf"]], s=64, + facecolor="none", + edgecolor="#2f7d54" if ident_for(r["ra_deg"])[1] + .startswith("Cen") else "#b5484f", lw=1.5, zorder=5) + axr.axhline(1.0, color="#4a4f57", ls=":", lw=1.1) + axr.text(14.3, 1.03, "PSF (Gaia stars)", fontsize=7.6, color="#4a4f57") + axr.set_ylim(0.4, 3.2) + axr.set_xlim(14.0, 21.0) + axr.set_xlabel("G magnitude", fontsize=9.5) + axr.set_ylabel("half-light radius / PSF", fontsize=9.5) + axr.set_title("morphology cannot separate clusters from transients", + fontsize=10.5, color="#1b1e23", pad=7) + axr.legend(fontsize=8.0, frameon=False, loc="upper left", + markerscale=2.2) + axr.grid(alpha=0.22, lw=0.6) + for sp in ("top", "right"): + axr.spines[sp].set_visible(False) + + # ---------------- field map ---------------- + axf = fig.add_subplot(gs[2, 4:6]) + sm = master[::6, ::6] + v1, v2 = np.percentile(sm[np.isfinite(sm)], [12, 99.75]) + axf.imshow(np.arcsinh((sm - v1) / max(v2 - v1, 1e-6)), origin="lower", + cmap="gray", vmin=0, vmax=np.arcsinh(1.0)) + for r in rows: + c = "#2f7d54" if ident_for(r["ra_deg"])[1].startswith("Cen") \ + else "#b5484f" + axf.scatter([r["x"] / 6], [r["y"] / 6], s=90, facecolor="none", + edgecolor=c, lw=1.5) + axf.annotate(ident_for(r["ra_deg"])[0], (r["x"] / 6, r["y"] / 6), + textcoords="offset points", xytext=(9, 5), fontsize=7.4, + color=c) + axf.set_xticks([]); axf.set_yticks([]) + axf.set_title("positions on the field (43' x 29')", fontsize=10.5, + color="#1b1e23", pad=7) + for sp in axf.spines.values(): + sp.set_color("#9aa0a8") + + fig.text(0.02, 0.962, + "Transient search in the 60 min luminance master of NGC 5128 " + "(Centaurus A), 2026-07-21", fontsize=14, weight="bold", + color="#1b1e23", ha="left") + fig.text(0.02, 0.936, + "7,519 point-like sources detected above SNR 10. 603 had no " + "Gaia DR3 counterpart; 567 also had none in SkyMapper DR2; 6 of " + "those survived the requirement of an independent detection in " + "at least 6 of the 12 subs.", + fontsize=9.4, color="#4a4f57", ha="left") + fig.text(0.02, 0.917, + "All 6 are identified below. Five are catalogued Centaurus A " + "globular clusters; the sixth is a resolved object with archival " + "detections from 1996 to 2020. No transient candidate remains.", + fontsize=9.4, color="#4a4f57", ha="left") + + fig.savefig(OUTPNG, dpi=130, facecolor="white") + print(f"wrote {OUTPNG}") + + +if __name__ == "__main__": + main() diff --git a/session-scripts/mo_finalise.py b/session-scripts/mo_finalise.py new file mode 100644 index 0000000..74a5df3 --- /dev/null +++ b/session-scripts/mo_finalise.py @@ -0,0 +1,146 @@ +"""Write the final flagged-object CSVs, with an identification and verdict +attached to every row. + +Two files are produced. + +mo-flagged-objects.csv is the short list: the six sources that survived every +cut of the transient search, plus the (zero) moving-object candidates, each +with the identification established from published catalogues and an explicit +assessment. This is the file to read. + +mo-transient-candidates.csv (written by mo_transient.py) is the long list of +all 567 catalogue non-matches, kept for completeness. Almost all of them are +faint objects below the depth at which Gaia DR3 and SkyMapper DR2 are complete +against this galaxy, and are not individually assessed. +""" +import csv +import os + +import numpy as np + +import mo_common as C + +# Identifications from cross-matching against published Centaurus A cluster +# catalogues (Woodley+2007 J/AJ/134/494 and others), SIMBAD, NED and a blanket +# VizieR cone search. Separations quoted are to the matched catalogue entry. +IDENT = { + "201.672625": dict( + name="WHH-31", otype="GlC (SIMBAD)", sep=0.10, + refs="Woodley+2007; Hughes+2021 #86; SCABS GC prob=1.0", + cat_phot="V=19.50 B=20.44 R=18.94", + assessment="Known Centaurus A globular cluster. Not a transient."), + "201.500583": dict( + name="pff_gc-081", otype="GlC (SIMBAD)", sep=0.32, + refs="Woodley+2007; Hughes+2021 ID 248669; SCABS GC prob=1.0", + cat_phot="V=19.34 B=20.17 R=18.80", + assessment="Known Centaurus A globular cluster. Not a transient."), + "201.441625": dict( + name="WHH-24", otype="GlC (SIMBAD)", sep=0.19, + refs="Woodley+2007; SCABS GC prob=1.0", + cat_phot="V=19.63 B=20.43 R=19.12", + assessment="Known Centaurus A globular cluster. Not a transient."), + "201.407000": dict( + name="resolved object at the position of CAVS 210 / " + "Gaia DR3 6088701636326983168", + otype="resolved (1.34x PSF, elongation 1.40); not a point source", + sep=1.32, + refs="de Jong+2008 J/A+A/478/755 (CAVS 210); DENIS 1996-2001; " + "Swift/UVOT SSC (12 ObsIDs); XMM-OM ~2020; VHS DR4; KS4 DR1", + cat_phot="Gaia DR3 G=21.03 at 1.32\" (point source); " + "DENIS I=18.23 J=16.44; UVOT U=18.2-18.6", + assessment="NOT a transient and NOT a brightening; the 2.8 mag " + "difference from Gaia is a measurement artefact. The " + "source is resolved (r50 = 2.96 px vs PSF 2.21 px, " + "elongation 1.40), so an integrated aperture magnitude is " + "not comparable with Gaia's point-source G; Gaia has no " + "entry at all for the extended object and nothing within " + "25\" brighter than G=19.8. It is steady over the session " + "(rms 0.066 mag vs 0.039 for comparison stars) and is " + "present on 1990s DSS2 red plates at the same brightness " + "relative to its neighbours (implied change +0.26 mag). " + "Even the PSF-scaled 2 px core gives G=18.63. See " + "notes-transient-search.md section 4.3 and NGC5128-mo-cavs210.png."), + "201.303542": dict( + name="WHH-10", otype="GlC (SIMBAD)", sep=0.23, + refs="Woodley+2007; SCABS GC prob=1.0", + cat_phot="V=19.57 B=20.36 R=19.07", + assessment="Known Centaurus A globular cluster. Not a transient."), + "201.256708": dict( + name="pff_gc-029 / GC0103", otype="GlC (SIMBAD)", sep=0.23, + refs="Woodley+2007; Peng+2004; Spitler+2008; Hughes+2021; Dumont+2022", + cat_phot="V=19.37 B=19.75 R=19.06 Ks=14.69", + assessment="Known Centaurus A globular cluster, well studied. " + "Not a transient."), +} + + +def ident_for(ra): + for k, v in IDENT.items(): + if abs(float(k) - ra) < 1e-4: + return v + return dict(name="", otype="", sep=np.nan, refs="", cat_phot="", + assessment="UNIDENTIFIED - needs follow-up") + + +FIELDS = ["search", "ra_hms", "dec_dms", "ra_deg", "dec_deg", "x_ref", + "y_ref", "g_mag_this_image", "snr", "r50_over_psf", "n_subs", + "snr_R", "snr_G", "snr_B", "dist_from_cenA_arcmin", + "identification", "object_type", "match_sep_arcsec", + "catalogue_photometry", "references", "assessment"] + + +def main(): + rows = [] + + vet = list(np.load(os.path.join(C.OUT, "_mo_vetted.npy"), + allow_pickle=True)) + vet.sort(key=lambda r: r["dist_cenA_arcmin"]) + for r in vet: + d = ident_for(r["ra_deg"]) + rows.append({ + "search": "transient", + "ra_hms": r["ra_hms"], "dec_dms": r["dec_dms"], + "ra_deg": r["ra_deg"], "dec_deg": r["dec_deg"], + "x_ref": r["x"], "y_ref": r["y"], + "g_mag_this_image": r["g_mag"], "snr": r["snr"], + "r50_over_psf": r["r50_over_psf"], "n_subs": r["n_subs"], + "snr_R": r["snr_R"], "snr_G": r["snr_G"], "snr_B": r["snr_B"], + "dist_from_cenA_arcmin": r["dist_cenA_arcmin"], + "identification": d["name"], "object_type": d["otype"], + "match_sep_arcsec": d["sep"], + "catalogue_photometry": d["cat_phot"], + "references": d["refs"], "assessment": d["assessment"]}) + + # Moving-object search: zero candidates, but the file should say so + # explicitly rather than being empty. + mo = os.path.join(C.OUT, "mo-moving-candidates.csv") + nmov = 0 + if os.path.exists(mo): + with open(mo) as fh: + nmov = max(0, sum(1 for _ in fh) - 1) + if nmov == 0: + rows.append({f: "" for f in FIELDS} | { + "search": "moving-object", + "identification": "(no candidates)", + "assessment": "Blind tracklet search over the 12 luminance subs " + "returned 0 candidates. Sensitivity measured by " + "synthetic injection: 50% recovery at G=18.9 for " + "8-40 arcsec/hr; no sensitivity below ~4 arcsec/hr. " + "The only catalogued minor planet within 30' of the " + "pointing, (427494) 2002 BK26 at V=21.7, fell 2.4' " + "outside the frame."}) + + out = os.path.join(C.OUT, "mo-flagged-objects.csv") + with open(out, "w", newline="", encoding="utf-8") as fh: + w = csv.DictWriter(fh, fieldnames=FIELDS) + w.writeheader() + w.writerows(rows) + print(f"wrote {out} ({len(rows)} rows)") + for r in rows: + print(f" {r['search']:14s} {r['identification'][:44]:44s} " + f"{str(r['g_mag_this_image']):>6s} " + f"{r['assessment'][:60]}") + + +if __name__ == "__main__": + main() diff --git a/session-scripts/mo_gccheck.py b/session-scripts/mo_gccheck.py new file mode 100644 index 0000000..6c3842b --- /dev/null +++ b/session-scripts/mo_gccheck.py @@ -0,0 +1,423 @@ +"""Cross-match six unidentified point-like detections in an amateur NGC 5128 +(Centaurus A) image against published catalogues, to establish whether each is an +already-catalogued object (globular cluster of Cen A, background galaxy, foreground +star, X-ray/UV source, planetary nebula, ...) or genuinely uncatalogued. + +Context: the six sources have already been shown NOT to match Gaia DR3 (G<20.5) +or SkyMapper DR2. The purpose here is to rule out a supernova / transient, so a +firm identification with a static catalogued source is the desired outcome and a +"no match" is a result that must be reported honestly rather than manufactured. + +Three independent searches are run for every position: + + 1. TARGETED -- a hand-picked list of VizieR catalogues covering Cen A globular + clusters and compact stellar systems (Woodley+, Harris+, Taylor+ SCABS, + Hughes+, Dumont+, Voggel+, Mieske+ CCOS), plus Cen A X-ray, UV, planetary + nebula and variable-star catalogues, at a 5 arcsec cone radius. The exact + VizieR identifiers are resolved at runtime with Vizier.find_catalogs / + get_catalog_metadata rather than being assumed correct. + + 2. BLANKET -- Vizier.query_region(..., catalog=None), i.e. every table in + VizieR, at a 5 arcsec cone radius. Every table returning a row is reported + with its designation, separation and any magnitude-like columns. + + 3. SIMBAD -- astroquery.simbad cone search at 10 arcsec, reporting main + identifier, object type and separation. + +Astrometry of the input positions is good to well under 1 arcsec; 3-5 arcsec is +used because older ground-based Cen A cluster catalogues can be off by 1-2 arcsec. + +Output: a plain-text report at + C:\\Users\\lhorrocks-barlow\\Downloads\\NGC5128\\20260721\\stacked\\_gc_crossmatch.txt +Run: python mo_gccheck.py +""" + +import io +import sys +import time +import warnings +import contextlib + +import numpy as np +import astropy.units as u +from astropy.coordinates import SkyCoord +from astroquery.vizier import Vizier + +import layout + +warnings.filterwarnings("ignore") + +OUT = layout.path("_gc_crossmatch.txt") + +# id, RA deg, Dec deg, luminance G, r50/psf, note +SOURCES = [ + (1, 201.672625, -43.190250, 19.18, 1.08, ""), + (2, 201.500583, -42.816944, 19.04, 1.04, ""), + (3, 201.441625, -42.948111, 19.42, 1.02, ""), + (4, 201.407000, -43.041639, 18.20, 1.33, "brightest, 2.3' from nucleus"), + (5, 201.303542, -42.950000, 19.24, 1.05, ""), + (6, 201.256708, -42.911417, 19.06, 1.03, ""), +] + +# Targeted catalogues. (VizieR id, short human label) +TARGETS = [ + ("J/AJ/134/494", "Woodley+ 2007, Cen A GC catalogue (kinematics/dyn.)"), + ("J/AJ/139/1871", "Woodley+ 2010, Cen A GC ages/metallicities"), + ("J/AJ/143/84", "Harris+ 2012, new candidate GCs in NGC 5128"), + ("J/ApJ/682/199", "Woodley+ 2008, Cen A globulars with X-ray sources"), + ("J/MNRAS/469/3444", "Taylor+ 2017, SCABS II GC candidates"), + ("J/ApJ/914/16", "Hughes+ 2021, NGC 5128 GCs (PISCeS/Gaia DR2/NSC)"), + ("J/ApJ/929/147", "Dumont+ 2022, luminous GCs in NGC 5128"), + ("J/ApJ/899/140", "Voggel+ 2020, Gaia UCD candidates"), + ("J/A+A/472/111", "Mieske+ 2007, Centaurus Compact Object Survey"), + ("J/MNRAS/389/1150", "Spitler+ 2008, Spitzer photometry of globulars"), + ("J/A+A/447/71", "Voss+ 2006, Chandra X-ray point sources in Cen A"), + ("J/ApJ/560/675", "Kraft+ 2001, Chandra X-ray point sources in Cen A"), + ("J/ApJ/766/88", "Burke+ 2013, Chandra X-ray binaries in Cen A"), + ("J/MNRAS/516/2300", "Joseph+ 2022, UV sources in the Cen A nuclear region"), + ("J/A+A/574/A109", "Walsh+ 2015, planetary nebulae in NGC 5128"), + ("J/A+A/478/755", "de Jong+ 2008, variable stars in Cen A"), + ("J/A+A/657/A41", "Rejkuba+ 2022, stellar halo of NGC 5128"), + ("J/A+A/448/983", "Rejkuba+ 2006, VIJsKs in AM1339-445 / AM1343-452"), + ("J/A+A/705/A112", "Faucher+ 2026, distances of Cen A / M83 galaxies"), + ("J/AJ/133/504", "Karachentsev+ 2007, galaxies around CenA/M83"), +] + +RAD_TARGET = 5.0 * u.arcsec +RAD_BLANKET = 5.0 * u.arcsec +RAD_SIMBAD = 10.0 * u.arcsec + +MAGISH = ("mag", "MAG", "Vmag", "Bmag", "Rmag", "Imag", "gmag", "rmag", "imag", + "umag", "zmag", "Kmag", "Jmag", "Hmag", "Gmag", "FUV", "NUV", "flux", + "Flux", "F(") + + +class Tee: + def __init__(self, *streams): + self.streams = streams + + def write(self, s): + for st in self.streams: + st.write(s) + + def flush(self): + for st in self.streams: + st.flush() + + +def sky(rec): + return SkyCoord(rec[1] * u.deg, rec[2] * u.deg, frame="icrs") + + +def row_coord(tab, row): + """Best-effort ICRS coordinate for a VizieR result row.""" + cn = tab.colnames + for rk, dk in (("_RAJ2000", "_DEJ2000"), ("RAJ2000", "DEJ2000"), + ("RA_ICRS", "DE_ICRS"), ("_RA", "_DE"), ("RAB1950", "DEB1950")): + if rk in cn and dk in cn: + rv, dv = row[rk], row[dk] + if rv is None or dv is None: + continue + try: + if isinstance(rv, str) and (":" in rv or " " in rv.strip()): + return SkyCoord(rv, dv, unit=(u.hourangle, u.deg), frame="icrs") + fr, fd = float(rv), float(dv) + if not (np.isfinite(fr) and np.isfinite(fd)): + continue + fr_ = "fk4" if rk == "RAB1950" else "icrs" + return SkyCoord(fr * u.deg, fd * u.deg, + frame=fr_).icrs if fr_ == "fk4" else \ + SkyCoord(fr * u.deg, fd * u.deg, frame="icrs") + except Exception: + continue + return None + + +def designation(tab, row): + cn = tab.colnames + prefer = ["Name", "ID", "GC", "Cluster", "Seq", "Source", "SimbadName", + "OName", "Object", "Obj", "CXO", "PN", "UCD", "GCID", "recno", + "HGHH", "WHH", "AAT", "f_Name", "No", "Nr"] + parts = [] + for k in prefer: + if k in cn and row[k] is not None: + v = str(row[k]).strip() + if v and v not in ("--", "nan", "None"): + parts.append(f"{k}={v}") + if len(parts) >= 2: + break + return " ".join(parts) if parts else "(no name column)" + + +def mags(tab, row, limit=8): + out = [] + for c in tab.colnames: + if c.startswith("_"): + continue + if any(m in c for m in MAGISH): + v = row[c] + if v is None: + continue + try: + fv = float(v) + if not np.isfinite(fv): + continue + out.append(f"{c}={fv:.3f}") + except (TypeError, ValueError): + s = str(v).strip() + if s and s not in ("--", "nan"): + out.append(f"{c}={s}") + if len(out) >= limit: + break + return ", ".join(out) if out else "(no magnitudes)" + + +def classification(tab, row): + out = [] + for c in ("Class", "Type", "otype", "OType", "Cl", "Note", "Notes", "Flag", + "Prob", "p", "Member", "n_Name", "Com", "Comm", "Sp", "SpType"): + if c in tab.colnames and row[c] is not None: + v = str(row[c]).strip() + if v and v not in ("--", "nan", "None"): + out.append(f"{c}={v}") + return "; ".join(out) if out else "" + + +def verify_catalogues(): + """Confirm which of TARGETS actually exist on VizieR.""" + print("\n" + "=" * 78) + print("STEP 0 -- verifying targeted VizieR catalogue identifiers") + print("=" * 78) + live, dead = [], [] + for cid, label in TARGETS: + try: + with contextlib.redirect_stdout(io.StringIO()): + meta = Vizier.get_catalog_metadata(catalog=cid) + desc = "" + try: + desc = str(meta["description"][0]) + except Exception: + desc = "(metadata returned, no description field)" + print(f" OK {cid:20s} {desc[:78]}") + live.append((cid, label)) + except Exception as e: + print(f" ABSENT/FAILED {cid:20s} ({label})") + print(f" -> {type(e).__name__}: {str(e)[:140]}") + dead.append((cid, label, f"{type(e).__name__}: {e}")) + return live, dead + + +def query_targeted(live, coords): + """Return {src_id: [ (cid,label,tabname,sep,desig,mags,cls) ]}""" + res = {s[0]: [] for s in SOURCES} + print("\n" + "=" * 78) + print(f"STEP 1 -- targeted catalogue cone search, r = {RAD_TARGET}") + print("=" * 78) + v = Vizier(columns=["**", "+_r"], row_limit=200) + for cid, label in live: + print(f"\n [{cid}] {label}") + for rec in SOURCES: + sid = rec[0] + c = coords[sid] + try: + tl = v.query_region(c, radius=RAD_TARGET, catalog=cid) + except Exception as e: + print(f" src {sid}: QUERY FAILED {type(e).__name__}: {str(e)[:110]}") + continue + n = 0 + for tab in tl: + tname = tab.meta.get("name", cid) + for row in tab: + rc = row_coord(tab, row) + if rc is not None: + sep = c.separation(rc).arcsec + elif "_r" in tab.colnames and row["_r"] is not None: + try: + sep = float(row["_r"]) * 60.0 + except Exception: + sep = float("nan") + else: + sep = float("nan") + if np.isfinite(sep) and sep > RAD_TARGET.to_value(u.arcsec) + 0.5: + continue + d = designation(tab, row) + m = mags(tab, row) + cl = classification(tab, row) + res[sid].append((cid, label, tname, sep, d, m, cl)) + n += 1 + print(f" src {sid}: MATCH {tname} sep={sep:.2f}\" {d}") + print(f" mags: {m}") + if cl: + print(f" class: {cl}") + if n == 0: + print(f" src {sid}: no match") + time.sleep(0.15) + return res + + +def query_blanket(coords): + res = {s[0]: [] for s in SOURCES} + print("\n" + "=" * 78) + print(f"STEP 2 -- BLANKET all-VizieR cone search, r = {RAD_BLANKET}") + print("=" * 78) + v = Vizier(columns=["**", "+_r"], row_limit=50, timeout=600) + for rec in SOURCES: + sid = rec[0] + c = coords[sid] + print(f"\n --- source {sid} {c.to_string('hmsdms', sep=':', precision=2)} ---") + try: + tl = v.query_region(c, radius=RAD_BLANKET, catalog=None) + except Exception as e: + print(f" BLANKET QUERY FAILED: {type(e).__name__}: {str(e)[:200]}") + continue + if tl is None or len(tl) == 0: + print(" no tables returned") + continue + print(f" {len(tl)} table(s) with entries") + for tab in tl: + tname = tab.meta.get("name", "?") + best = None + for row in tab: + rc = row_coord(tab, row) + sep = c.separation(rc).arcsec if rc is not None else float("nan") + if not np.isfinite(sep) and "_r" in tab.colnames: + try: + sep = float(row["_r"]) * 60.0 + except Exception: + pass + item = (tname, sep, designation(tab, row), mags(tab, row), + classification(tab, row)) + if best is None or (np.isfinite(sep) and + (not np.isfinite(best[1]) or sep < best[1])): + best = item + res[sid].append(item) + if best: + print(f" {best[0]:28s} sep={best[1]:6.2f}\" {best[2]}") + print(f" {best[3]}") + if best[4]: + print(f" {best[4]}") + time.sleep(0.3) + return res + + +def query_simbad(coords): + res = {s[0]: [] for s in SOURCES} + print("\n" + "=" * 78) + print(f"STEP 3 -- SIMBAD cone search, r = {RAD_SIMBAD}") + print("=" * 78) + try: + from astroquery.simbad import Simbad + except Exception as e: + print(f" SIMBAD IMPORT FAILED: {e}") + return res + sb = Simbad() + sb.TIMEOUT = 180 + for extra in ("otype", "otypes", "V", "B", "R", "G", "sp_type", "rvz_redshift", + "distance"): + try: + sb.add_votable_fields(extra) + except Exception: + pass + for rec in SOURCES: + sid = rec[0] + c = coords[sid] + print(f"\n --- source {sid} ---") + try: + t = sb.query_region(c, radius=RAD_SIMBAD) + except Exception as e: + print(f" SIMBAD QUERY FAILED: {type(e).__name__}: {str(e)[:160]}") + continue + if t is None or len(t) == 0: + print(" no SIMBAD object within 10\"") + continue + rk = "ra" if "ra" in t.colnames else "RA" + dk = "dec" if "dec" in t.colnames else "DEC" + for row in t: + try: + rc = SkyCoord(float(row[rk]) * u.deg, float(row[dk]) * u.deg) + sep = c.separation(rc).arcsec + except Exception: + sep = float("nan") + mid = str(row["main_id"] if "main_id" in t.colnames else row["MAIN_ID"]) + ot = str(row["otype"]) if "otype" in t.colnames else "?" + ots = str(row["otypes"]) if "otypes" in t.colnames else "" + mg = [] + for k in ("V", "B", "R", "G"): + if k in t.colnames and row[k] is not None: + try: + fv = float(row[k]) + if np.isfinite(fv): + mg.append(f"{k}={fv:.2f}") + except Exception: + pass + res[sid].append((mid, ot, ots, sep, ", ".join(mg))) + print(f" {mid:30s} otype={ot:12s} sep={sep:6.2f}\" {', '.join(mg)}") + if ots: + print(f" otypes: {ots}") + time.sleep(0.3) + return res + + +def summarise(targ, blank, simb, dead): + print("\n\n" + "#" * 78) + print("# PER-SOURCE SUMMARY") + print("#" * 78) + for rec in SOURCES: + sid, ra, dec, g, r50, note = rec + c = SkyCoord(ra * u.deg, dec * u.deg) + print("\n" + "-" * 78) + print(f"SOURCE {sid} RA={ra:.6f} Dec={dec:+.6f} " + f"({c.to_string('hmsdms', sep=':', precision=2)})") + print(f" G={g:.2f} r50/psf={r50:.2f} {note}") + t, b, s = targ[sid], blank[sid], simb[sid] + print(f" targeted catalogue matches : {len(t)}") + for cid, label, tname, sep, d, m, cl in t: + print(f" * {tname} sep={sep:.2f}\"") + print(f" {label}") + print(f" {d}") + print(f" {m}") + if cl: + print(f" {cl}") + print(f" blanket VizieR matches : {len(b)}") + for tname, sep, d, m, cl in b: + print(f" * {tname} sep={sep:.2f}\" {d}") + print(f" {m}" + (f" | {cl}" if cl else "")) + print(f" SIMBAD objects <10\" : {len(s)}") + for mid, ot, ots, sep, mg in s: + print(f" * {mid} otype={ot} sep={sep:.2f}\" {mg}") + if dead: + print("\n" + "-" * 78) + print("CATALOGUES THAT COULD NOT BE QUERIED (absent from VizieR or errored):") + for cid, label, err in dead: + print(f" {cid} ({label})") + print(f" {err[:200]}") + + +def main(): + coords = {rec[0]: sky(rec) for rec in SOURCES} + buf = io.StringIO() + real = sys.stdout + sys.stdout = Tee(real, buf) + try: + print("Cen A (NGC 5128) cross-match of six unidentified detections") + print("Generated by mo_gccheck.py at " + + time.strftime("%Y-%m-%d %H:%M:%S")) + print("Input positions J2000/ICRS; no Gaia DR3 (G<20.5) or SkyMapper DR2 match.") + for rec in SOURCES: + sid, ra, dec, g, r50, note = rec + print(f" {sid}: {ra:.6f} {dec:+.6f} G={g:.2f} r50/psf={r50:.2f} {note}") + live, dead = verify_catalogues() + targ = query_targeted(live, coords) + blank = query_blanket(coords) + simb = query_simbad(coords) + summarise(targ, blank, simb, dead) + finally: + sys.stdout = real + with open(OUT, "w", encoding="utf-8") as fh: + fh.write(buf.getvalue()) + print(f"\nWROTE {OUT}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/session-scripts/mo_link.py b/session-scripts/mo_link.py new file mode 100644 index 0000000..d612c42 --- /dev/null +++ b/session-scripts/mo_link.py @@ -0,0 +1,130 @@ +"""Moving-object search, pass 2: strip the static sky and link tracklets. + +The static sky is removed twice over. First, anything coincident with a source +in the deep luminance master is dropped. Second, anything that sits at the same +reference-frame position in four or more of the twelve subs is dropped - that +catches stars the master's sigma clipping or deblending missed, and by +construction cannot remove a real mover, which is never in the same place +twice. The price of the second cut is a floor on detectable motion: an object +slower than about 2 px per three exposures looks static and is removed with the +stars. mo_sensitivity.py measures where that floor actually falls. + +Third, and on this data much the most important, anything that sits at the same +DETECTOR-frame position in four or more subs is dropped. The subs are dithered +and the field rotates ~0.2 deg through the sequence, so registration - which +holds the sky still - drags anything fixed to the detector across the +registered frame on a perfectly straight, perfectly constant-rate, perfectly +constant-brightness track. Hot pixels are therefore ideal fake asteroids, and +they pass every cut a linker would normally apply. Skipping this one cut turns +a clean null result into 141 confident false detections. + +What is left is a few dozen detections per frame: cosmic rays, hot pixels that +survived the bad-pixel map, deblending artefacts around bright stars, noise +peaks at the 3.5 sigma threshold, and - if there is one - a minor planet. + +Linking is brute force over detection pairs (mo_common.link). Each pair of +residual detections from two frames separated by at least three exposures +defines a candidate velocity; velocities outside a plausible sky-motion range +are discarded, and the rest are propagated to every frame to see how many other +residuals fall on the predicted track. A candidate must be recovered in at +least five frames, lie on a straight constant-rate line to better than 1.2 px, +and hold its brightness to better than 0.5 mag. + +The point of demanding a straight line through many frames is that noise and +cosmic rays are independent between frames: the chance that five unrelated +false detections are collinear in space AND linear in time to sub-pixel +precision is tiny, and is measured directly by rerunning the linker on +time-permuted data (mo_sensitivity.py). + +Writes _mo_residuals.npz (the residual detection lists, reused by the +false-alarm control and the figures) and mo-candidates rows for the CSV. +""" +import csv +import os + +import numpy as np +from astropy.io import fits +from astropy.wcs import WCS +from scipy.spatial import cKDTree + +import mo_common as C + + +def main(): + times = C.lum_times() + keys = [k for k, _ in C.lum_frames()] + print(f"{len(keys)} luminance subs spanning {times[-1] * 60:.1f} min") + + tforms = C.frame_transforms() + mobjs, map_, mrms, mhdr, mimg = C.master_sources() + del mimg + mtree = cKDTree(np.column_stack([mobjs["x"], mobjs["y"]])) + print(f"static sky: {len(mobjs)} sources in the luminance master") + + pts, nats, mags, shapes = [], [], [], [] + for key, path in C.lum_frames(): + with fits.open(path, memmap=False) as hd: + img = hd[0].data.astype(np.float32) + objs, ap, rms = C.detect(img) + del img + keep = (objs["npix"] > 3) & (objs["npix"] < 3000) & (ap > 0) + objs, ap = objs[keep], ap[keep] + zp, nz = C.frame_zeropoint(objs["x"], objs["y"], ap, mobjs, map_, + tforms[key]) + nat = np.column_stack([objs["x"], objs["y"]]).astype(float) + nats.append(nat) + pts.append(tforms[key](nat)) + mags.append(-2.5 * np.log10(ap) + zp) + shapes.append(np.column_stack([objs["a"], objs["b"], + objs["npix"]]).astype(float)) + print(f" {key}: {len(objs)} det, rms {rms:.2f}, zp {zp:.3f}") + + mask = C.residual_mask(pts, nats, mtree) + rpts = [p[m] for p, m in zip(pts, mask)] + rmags = [g[m] for g, m in zip(mags, mask)] + rnat = [n[m] for n, m in zip(nats, mask)] + rshape = [s[m] for s, m in zip(shapes, mask)] + print("\nresiduals per frame:", " ".join(str(len(p)) for p in rpts)) + print(f"total residual detections: {sum(len(p) for p in rpts)}") + + cands = C.link(times, rpts, rmags, nat=rnat, shape=rshape) + print(f"\n{len(cands)} tracklet candidates after all cuts") + + wcs = WCS(mhdr) + tmid = times.mean() + rows = [] + for c in cands: + xm = c["cx"][0] + c["cx"][1] * tmid + ym = c["cy"][0] + c["cy"][1] * tmid + sky = wcs.pixel_to_world(xm, ym) + rows.append(dict( + ra_deg=round(sky.ra.deg, 6), dec_deg=round(sky.dec.deg, 6), + x_ref=round(xm, 2), y_ref=round(ym, 2), + rate_arcsec_per_hr=round(c["rate"] * C.SCALE, 2), + pa_deg=round(c["ang"], 1), g_mag=round(c["mag"], 2), + g_scatter=round(c["magsig"], 2), n_frames=c["nhit"], + line_rms_px=round(c["rms"], 2), + elongation=round(c["a"] / max(c["b"], 1e-6), 2))) + print(" " + str(rows[-1])) + + np.savez(os.path.join(C.OUT, "_mo_residuals.npz"), + keys=np.array(keys), times=times, + **{f"p{i}": p for i, p in enumerate(rpts)}, + **{f"m{i}": g for i, g in enumerate(rmags)}, + **{f"n{i}": n for i, n in enumerate(rnat)}) + np.save(os.path.join(C.OUT, "_mo_tracklets.npy"), + np.array(cands, dtype=object), allow_pickle=True) + + out = os.path.join(C.OUT, "mo-moving-candidates.csv") + fields = ["ra_deg", "dec_deg", "x_ref", "y_ref", "rate_arcsec_per_hr", + "pa_deg", "g_mag", "g_scatter", "n_frames", "line_rms_px", + "elongation"] + with open(out, "w", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=fields) + w.writeheader() + w.writerows(rows) + print(f"\nwrote {out} ({len(rows)} rows)") + + +if __name__ == "__main__": + main() diff --git a/session-scripts/mo_mpc.py b/session-scripts/mo_mpc.py new file mode 100644 index 0000000..ef09c55 --- /dev/null +++ b/session-scripts/mo_mpc.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +""" +mo_mpc.py -- "was there a known minor planet in the frame?" + +Purpose +------- +The NGC 5128 (Centaurus A) LRGB session of 2026-07-21 was searched for moving +objects (see _mo_dets.npz / _mo_tracklets.npy). Before any detection can be +called a *new* object, we must rule out that it is simply a catalogued minor +planet that happened to drift through the field. This script asks the +authoritative services what known small bodies were inside the field of view at +the epoch of the exposures. + +Field / epoch +------------- + Field centre : RA 13 25 27.37, Dec -43 01 10.9 (J2000) + = 201.36404 deg, -43.01969 deg + Field size : 42.9' x 28.6' (half-diagonal ~25.8') + Search radius: 30 arcmin (0.5 deg) -- comfortably encloses the whole frame + Epoch : 2026-07-21 09:22 UTC (= 2026 07 21.39), mid-luminance-sequence + (luminance ran 08:56-10:00 UTC, session ended ~11:16 UTC) + Observatory : Q62, iTelescope / Siding Spring Observatory + +Services queried +---------------- +1. JPL Small-Body Identification API + https://ssd-api.jpl.nasa.gov/sb_ident.api + Primary source. Well documented, accepts arbitrary (including future) epochs + because it propagates orbits rather than serving a precomputed ephemeris. + IMPORTANT UNITS GOTCHA: despite some documentation wording, fov-ra-hwidth and + fov-dec-hwidth are interpreted in DEGREES, not arcminutes. Passing "30" + yields a 30-degree half-width box (the API echoes the box it used in + ["observer"]["fov_offset"], which is worth checking every run). + Queried twice: sb-kind=a (asteroids) and sb-kind=c (comets). + +2. MPC "MPChecker" / minor planet checker + https://minorplanetcenter.net/cgi-bin/mpcheck.cgi (classic HTTP POST form) + https://www.minorplanetcenter.net/cgi-bin/checkmp.cgi + Cross-check. Kept in the script even if it fails, so the log records whether + the MPC endpoint was reachable and what it said. + +Output +------ +Raw responses (JSON + HTML) and a human-readable summary are appended to + intermediates/_mpc_query.txt +Nothing is fabricated: if a service errors or refuses the epoch, the error text +itself is written to the log. +""" + +import json +import math +import sys +import datetime + +import requests + +import layout + +# ---------------------------------------------------------------- field/epoch +RA_SEX = "13-25-27.37" +DEC_SEX = "M43-01-10.9" # JPL uses a leading "M" for a negative Dec +RA_DEG = 201.36404 +DEC_DEG = -43.01969 +OBS_TIME = "2026-07-21_09:22:00" +MPC_CODE = "Q62" # iTelescope, Siding Spring +RADIUS_ARCMIN = 30.0 +HWIDTH_DEG = RADIUS_ARCMIN / 60.0 # = 0.5 deg; API wants DEGREES here +VMAG_LIM = 22.0 + +OUT = layout.path("_mpc_query.txt") + +log_lines = [] + + +def log(s=""): + print(s) + log_lines.append(str(s)) + + +# ------------------------------------------------------------------ JPL query +def jpl(sb_kind): + """Query ssd-api.jpl.nasa.gov/sb_ident.api for one small-body class.""" + params = { + "sb-kind": sb_kind, # 'a' = asteroid, 'c' = comet + "mpc-code": MPC_CODE, + "obs-time": OBS_TIME, + "fov-ra-center": RA_SEX, + "fov-dec-center": DEC_SEX, + "fov-ra-hwidth": f"{HWIDTH_DEG}", + "fov-dec-hwidth": f"{HWIDTH_DEG}", + "mag-required": "true", + "two-pass": "true", + "vmag-lim": f"{VMAG_LIM}", + } + r = requests.get("https://ssd-api.jpl.nasa.gov/sb_ident.api", + params=params, timeout=180) + log(f"--- JPL sb_ident sb-kind={sb_kind} -> HTTP {r.status_code}") + log(f" URL: {r.url}") + if r.status_code != 200: + log(" RAW: " + r.text[:2000]) + return None + d = r.json() + log(" observer : " + json.dumps(d.get("observer", {}))) + log(" n_first_pass : " + str(d.get("n_first_pass"))) + log(" n_second_pass: " + str(d.get("n_second_pass"))) + return d + + +def show(d, label): + """Print the refined (second-pass) hits, falling back to first pass.""" + if d is None: + return + rows = d.get("data_second_pass") or [] + fields = d.get("fields_second") or [] + if not rows: + rows = d.get("data_first_pass") or [] + fields = d.get("fields_first") or [] + log(f" [{label}] no second-pass data; showing first pass") + log(f" fields: {fields}") + if not rows: + log(f" [{label}] NO OBJECTS in field.") + return + for row in rows: + log(" " + " | ".join(str(x) for x in row)) + + +# ------------------------------------------------------------------ MPC query +def mpchecker(): + """Classic MPChecker POST form. Recorded even if it fails.""" + data = { + "year": "2026", "month": "07", "day": "21.39", + "which": "pos", + "ra": "13 25 27.37", "decl": "-43 01 10.9", + "TextArea": "", + "radius": str(int(RADIUS_ARCMIN)), + "limit": f"{VMAG_LIM}", + "oc": MPC_CODE, + "sort": "d", "mot": "h", "tmot": "s", + "pdes": "u", "needed": "f", "ps": "n", "type": "p", + } + for url in ("https://minorplanetcenter.net/cgi-bin/mpcheck.cgi", + "https://www.minorplanetcenter.net/cgi-bin/checkmp.cgi"): + try: + r = requests.post(url, data=data, timeout=120, + headers={"User-Agent": "Mozilla/5.0 (mo_mpc.py)"}) + log(f"--- MPC POST {url} -> HTTP {r.status_code}, {len(r.text)} bytes") + log(r.text[:8000]) + except Exception as e: # noqa: BLE001 + log(f"--- MPC POST {url} -> EXCEPTION {type(e).__name__}: {e}") + + +# ----------------------------------------------------------------------- main +if __name__ == "__main__": + log("MPC / minor-planet field check") + log("run at " + datetime.datetime.utcnow().isoformat() + "Z") + log(f"field centre {RA_SEX} {DEC_SEX} (J2000) = {RA_DEG}, {DEC_DEG}") + log(f"epoch {OBS_TIME} UTC, obs code {MPC_CODE}, " + f"radius {RADIUS_ARCMIN}' , V<{VMAG_LIM}") + log() + + for kind, label in (("a", "asteroids"), ("c", "comets")): + try: + show(jpl(kind), label) + except Exception as e: # noqa: BLE001 + log(f" JPL {label} EXCEPTION {type(e).__name__}: {e}") + log() + + mpchecker() + + # --- derived summary: total sky motion and position angle for JPL hits --- + log() + log("=== derived summary (from JPL second pass, sb-kind=a) ===") + try: + d = jpl("a") + for row in (d.get("data_second_pass") or []): + name, ra, dec = row[0], row[1], row[2] + norm = float(row[5]) + vmag = row[6] + dra, ddec = float(row[7]), float(row[8]) # "/h, RA rate has cos(dec) + tot = math.hypot(dra, ddec) + pa = math.degrees(math.atan2(dra, ddec)) % 360.0 # N through E + log(f"{name}: RA {ra} Dec {dec} V={vmag} " + f"{norm:.0f}\" ({norm/60:.1f}') from centre " + f"motion {tot:.1f}\"/hr at PA {pa:.1f} deg " + f"(dRA*cos(dec)={dra:.1f}, dDec={ddec:.1f} \"/hr)") + except Exception as e: # noqa: BLE001 + log(f"summary EXCEPTION {type(e).__name__}: {e}") + + with open(OUT, "w", encoding="utf-8") as fh: + fh.write("\n".join(log_lines) + "\n") + print("\nwrote " + OUT) diff --git a/session-scripts/mo_sensitivity.py b/session-scripts/mo_sensitivity.py new file mode 100644 index 0000000..823f554 --- /dev/null +++ b/session-scripts/mo_sensitivity.py @@ -0,0 +1,209 @@ +"""Moving-object search, pass 3: measure what the search could have found. + +A null result is only worth reading if the sensitivity behind it is known, so +this injects synthetic moving point sources into the real pixel data and runs +the identical detection -> registration -> static-rejection -> linking chain +over them. + +Sources are laid out on a grid of magnitude against apparent rate. Each is +given a random position and direction, is placed at p0 + v*t in the reference +frame (mapped back through each sub's own affine into native pixels before +injection), and is trailed across the 300 s exposure. Its flux is set from the +per-frame zero point, which is tied to the master's by the stars the two have +in common, so a quoted magnitude means the same thing as it does for the +master's photometry. + +Recovery is scored by matching linked tracklets back to the injection truth. +The same run also reports the false-alarm rate, measured by permuting the +frame times of the real residual detections: that keeps the real spatial +density of cosmic rays and noise peaks but destroys any genuine temporal +ordering, so anything the linker still produces is by construction spurious. + +Output: _mo_sensitivity.npz +""" +import os +import sys + +import numpy as np +from astropy.io import fits + +import mo_common as C + +import layout + +MAGS = np.array([17.0, 17.5, 18.0, 18.5, 18.75, 19.0, 19.25]) +RATES_PX_HR = np.array([3.0, 5.0, 6.5, 8.0, 10.0, 15.0, 25.0, 40.0, + 70.0, 120.0, 180.0, 250.0]) +NREP = 4 # injections per (magnitude, rate) cell +MARGIN = 350 # px kept clear of the frame edge +MINSEP = 120 # px between injected tracks at t=0 +EXPHR = 300.0 / 3600.0 # exposure length in hours + + +def plan(times, shape, rng): + """Random start positions and directions for the injection grid.""" + ny, nx = shape + tspan = times[-1] + rows = [] + placed = [] + for mag in MAGS: + for rate in RATES_PX_HR: + for _ in range(NREP): + for _try in range(200): + ang = rng.uniform(0, 2 * np.pi) + vx, vy = rate * np.cos(ang), rate * np.sin(ang) + x0 = rng.uniform(MARGIN, nx - MARGIN) + y0 = rng.uniform(MARGIN, ny - MARGIN) + x1, y1 = x0 + vx * tspan, y0 + vy * tspan + if not (MARGIN < x1 < nx - MARGIN and + MARGIN < y1 < ny - MARGIN): + continue + if placed and min(np.hypot(x0 - p[0], y0 - p[1]) + for p in placed) < MINSEP: + continue + placed.append((x0, y0)) + rows.append(dict(mag=float(mag), rate=float(rate), + x0=x0, y0=y0, vx=vx, vy=vy)) + break + return rows + + +def run_pass(inject, times, tforms, mtree, zps, rng, thresh=3.5): + """Detect on every sub (optionally with synthetics added) and link.""" + frames = C.lum_frames() + pts, nats, mags = [], [], [] + for i, (key, path) in enumerate(frames): + with fits.open(path, memmap=False) as hd: + img = hd[0].data.astype(np.float32) + tf = tforms[key] + inv = tf.inverse + lin = np.linalg.inv(tf.params[:2, :2]) # ref -> native, linear + for s in inject: + xr = s["x0"] + s["vx"] * times[i] + yr = s["y0"] + s["vy"] * times[i] + xn, yn = inv(np.array([[xr, yr]]))[0] + d = lin @ np.array([s["vx"] * EXPHR, s["vy"] * EXPHR]) + ap_flux = 10 ** ((zps[key] - s["mag"]) / 2.5) + C.add_source(img, xn, yn, ap_flux / C.APFRAC, d[0], d[1]) + objs, ap, _rms = C.detect(img, thresh=thresh) + del img + keep = (objs["npix"] > 3) & (objs["npix"] < 3000) & (ap > 0) + objs, ap = objs[keep], ap[keep] + nat = np.column_stack([objs["x"], objs["y"]]).astype(float) + nats.append(nat) + pts.append(tf(nat)) + mags.append(-2.5 * np.log10(ap) + zps[key]) + return pts, nats, mags + + +def score(cands, inject, times, tol=8.0): + """Match linked tracklets back to the injection truth. + + Every candidate is assigned to its nearest compatible injection, so a + source recovered as two overlapping tracklets counts once as a detection + and does not also count as a false positive. Candidates that match no + injection at all are the genuinely spurious ones. + """ + tmid = times.mean() + tx = np.array([s["x0"] + s["vx"] * tmid for s in inject]) + ty = np.array([s["y0"] + s["vy"] * tmid for s in inject]) + trate = np.array([s["rate"] for s in inject]) + found = np.zeros(len(inject), dtype=bool) + spurious = [] + for ci, c in enumerate(cands): + cxm = c["cx"][0] + c["cx"][1] * tmid + cym = c["cy"][0] + c["cy"][1] * tmid + d = np.hypot(tx - cxm, ty - cym) + ok = (d < tol) & (np.abs(trate - c["rate"]) < 0.25 * trate + 3) + if ok.any(): + found[np.argmin(np.where(ok, d, np.inf))] = True + else: + spurious.append(ci) + return found, spurious + + +def main(): + times = C.lum_times() + tforms = C.frame_transforms() + mobjs, map_, _, _, mimg = C.master_sources() + shape = mimg.shape + del mimg + from scipy.spatial import cKDTree + mtree = cKDTree(np.column_stack([mobjs["x"], mobjs["y"]])) + print(f"master: {len(mobjs)} static sources, field {shape}") + + # Per-frame zero points from the stars the sub and the master share. + _meta, dets = C.load_dets() + zps = {} + for key, path in C.lum_frames(): + with fits.open(path, memmap=False) as hd: + img = hd[0].data.astype(np.float32) + objs, ap, _ = C.detect(img) + del img + z, n = C.frame_zeropoint(objs["x"], objs["y"], ap, mobjs, map_, + tforms[key]) + zps[key] = z + print(f" {key}: zero point {z:.3f} from {n} matched stars") + + seed = int(sys.argv[1]) if len(sys.argv) > 1 else 20260721 + rng = np.random.default_rng(seed) + inject = plan(times, shape, rng) + print(f"\ninjecting {len(inject)} synthetic movers " + f"({len(MAGS)} mags x {len(RATES_PX_HR)} rates x {NREP})") + + pts, nats, mags = run_pass(inject, times, tforms, mtree, zps, rng) + print("detections per frame:", " ".join(str(len(p)) for p in pts)) + keep = C.residual_mask(pts, nats, mtree) + rpts = [p[m] for p, m in zip(pts, keep)] + rmags = [g[m] for g, m in zip(mags, keep)] + rnat = [n[m] for n, m in zip(nats, keep)] + print("residuals per frame:", " ".join(str(len(p)) for p in rpts)) + + cands = C.link(times, rpts, rmags, nat=rnat) + print(f"{len(cands)} tracklets linked") + found, spurious = score(cands, inject, times) + nspur = len(spurious) + print(f"recovered {found.sum()}/{len(inject)}, " + f"{nspur} unmatched (spurious) tracklets") + + grid = np.zeros((len(MAGS), len(RATES_PX_HR))) + for k, s in enumerate(inject): + i = int(np.where(MAGS == s["mag"])[0][0]) + j = int(np.where(RATES_PX_HR == s["rate"])[0][0]) + grid[i, j] += found[k] + grid /= NREP + print("\nrecovery fraction (rows = G mag, cols = rate px/hr):") + print(" " + " ".join(f"{r:6.0f}" for r in RATES_PX_HR)) + for i, m in enumerate(MAGS): + print(f" {m:5.1f} " + " ".join(f"{v:6.2f}" for v in grid[i])) + + # ---- false-alarm control on the real (uninjected) data ---------- + real = np.load(layout.path("_mo_residuals.npz"), + allow_pickle=True) + rp = [real[f"p{i}"] for i in range(len(times))] + rf = [real[f"m{i}"] for i in range(len(times))] + rn = [real[f"n{i}"] for i in range(len(times))] + rm = rf + fa = [] + for trial in range(20): + perm = np.random.default_rng(1000 + trial).permutation(len(times)) + cs = C.link(times, [rp[k] for k in perm], [rm[k] for k in perm], + nat=[rn[k] for k in perm]) + fa.append(len(cs)) + print(f"\nfalse-alarm control: time-permuted real residuals, " + f"20 trials -> {np.sum(fa)} tracklets total " + f"({np.mean(fa):.2f} per trial)") + + np.savez(layout.path(f"_mo_sensitivity_{seed}.npz"), + mags=MAGS, rates=RATES_PX_HR, grid=grid, nrep=NREP, + inject=np.array([(s["mag"], s["rate"], s["x0"], s["y0"], + s["vx"], s["vy"]) for s in inject]), + found=found, nspur=nspur, falsealarm=np.array(fa), + times=times, scale=C.SCALE) + print(f"saved _mo_sensitivity_{seed}.npz") + + +OUT_NPZ_DIR = C.OUT + +if __name__ == "__main__": + main() diff --git a/session-scripts/mo_shiftstack.py b/session-scripts/mo_shiftstack.py new file mode 100644 index 0000000..3d5b00c --- /dev/null +++ b/session-scripts/mo_shiftstack.py @@ -0,0 +1,211 @@ +"""Where the one known minor planet was, and how deep digital tracking goes. + +MPChecker and JPL's sb_ident agree that exactly one catalogued minor planet lay +within 30 arcmin of the field centre on this night: (427494) 2002 BK26, at +RA 13 27 00.6, Dec -43 11 26 at 09:22 UTC, V = 21.7, moving 53.2 arcsec/hr +toward position angle 71.7 deg. + +The first thing this script does is check whether it was actually inside the +frame, which is not obvious from the offsets alone. The field is 42.9' x 28.6' +at position angle -89 deg - the long axis runs very nearly along declination, +not right ascension - so the RA half-width is only about 14.6 arcmin. The +asteroid is 17.0 arcmin east of centre. It was outside the frame, by roughly +2.4 arcmin, and there is therefore nothing known to recover. + +The second thing is to measure how deep the search could have gone at that +rate, by shift-and-stack ("digital tracking"): each sub is shifted by the +target's own motion before combining, so a source moving at exactly that rate +adds coherently while the stars trail. Because the field position no longer +matters for the depth measurement, this is done on an empty patch of sky, and +a synthetic source of V = 21.7 is injected to show directly whether such an +object would have been recovered had it been in frame. + +Output: _mo_shiftstack.npz +""" +import os + +import numpy as np +import sep +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.stats import sigma_clipped_stats +from astropy.wcs import WCS +from scipy.ndimage import shift as ndshift +from scipy.spatial import cKDTree + +import mo_common as C + +NAME = "(427494) 2002 BK26" +EPH_RA = 201.752708 # deg, 13 27 00.65, at 09:22 UTC +EPH_DEC = -43.190667 # deg, -43 11 26.4 +EPH_UTC_HOURS = 9.0 + 22.0 / 60.0 +DRA_COSDEC = 50.5 # arcsec/hr +DDEC = 16.7 # arcsec/hr +RATE = np.hypot(DRA_COSDEC, DDEC) # 53.2 arcsec/hr +VMAG = 21.7 + +HALF = 60 # px half-size of the extracted stamp +NX, NY = 4788, 3194 + + +def utc_hours(path): + d = fits.getheader(path)["DATE-OBS"] + hh, mm, ss = d.split("T")[1].split(":") + return int(hh) + int(mm) / 60.0 + float(ss) / 3600.0 + 150.0 / 3600.0 + + +def footprint_check(wcs): + tgt = SkyCoord(EPH_RA * u.deg, EPH_DEC * u.deg) + px, py = wcs.world_to_pixel(tgt) + cen = wcs.pixel_to_world(NX / 2, NY / 2) + dra, ddec = cen.spherical_offsets_to(tgt) + corners = [wcs.pixel_to_world(x, y) for x, y in + ((0, 0), (NX - 1, 0), (0, NY - 1), (NX - 1, NY - 1))] + ras = [c.ra.deg for c in corners] + decs = [c.dec.deg for c in corners] + inside = (0 <= px < NX) and (0 <= py < NY) + print(f"{NAME}: V={VMAG}, {RATE:.1f}\"/hr at PA " + f"{np.degrees(np.arctan2(DRA_COSDEC, DDEC)):.1f} deg") + print(f" offset from field centre: {dra.to_value(u.arcmin):+.2f}' in RA, " + f"{ddec.to_value(u.arcmin):+.2f}' in Dec") + print(f" frame RA span {min(ras):.4f} to {max(ras):.4f} deg " + f"({(max(ras) - min(ras)) * np.cos(np.radians(EPH_DEC)) * 60:.1f}' " + f"on sky)") + print(f" frame Dec span {min(decs):.4f} to {max(decs):.4f} deg " + f"({(max(decs) - min(decs)) * 60:.1f}')") + print(f" predicted pixel ({px:.0f}, {py:.0f}) in a " + f"{NX} x {NY} frame -> " + f"{'INSIDE' if inside else 'OUTSIDE the frame'}") + return inside, float(px), float(py) + + +def blank_patch(mobjs, times, wcs, galx=2394.0, galy=1597.0, galrad=1000.0): + """A patch of sky with no source near the whole track, off the galaxy. + + Keeping clear of NGC 5128 itself matters more than it looks: the galaxy's + smooth light has a steep gradient that swamps the sky noise and would make + any depth measured on top of it meaningless. + """ + tree = cKDTree(np.column_stack([mobjs["x"], mobjs["y"]])) + # Track length in reference pixels over the sequence. + vx = (DRA_COSDEC / C.SCALE) + vy = (DDEC / C.SCALE) + span = times[-1] + rng = np.random.default_rng(7) + for _ in range(20000): + x0 = rng.uniform(400, NX - 400) + y0 = rng.uniform(400, NY - 400) + track = [(x0 + vx * t, y0 + vy * t) for t in times] + if not all(300 < x < NX - 300 and 300 < y < NY - 300 + for x, y in track): + continue + if any(np.hypot(x - galx, y - galy) < galrad for x, y in track): + continue + d, _ = tree.query(np.array(track), distance_upper_bound=22.0) + if np.all(~np.isfinite(d)): + return x0, y0, vx, vy, span + raise RuntimeError("no empty patch found") + + +def stamp(img, x, y, half=HALF): + xi, yi = int(round(x)), int(round(y)) + if not (half < xi < img.shape[1] - half and + half < yi < img.shape[0] - half): + return None + cut = img[yi - half:yi + half + 1, xi - half:xi + half + 1].astype(float) + return ndshift(cut, (yi - y, xi - x), order=3, mode="nearest") + + +def main(): + hdr = fits.getheader(os.path.join(C.OUT, "master-Luminance.fit")) + wcs = WCS(hdr) + inside, epx, epy = footprint_check(wcs) + + times = C.lum_times() + tforms = C.frame_transforms() + mobjs, map_, _, _, mimg = C.master_sources() + del mimg + x0, y0, vx, vy, span = blank_patch(mobjs, times, wcs) + print(f"\ndigital-tracking test patch: reference pixel " + f"({x0:.0f}, {y0:.0f}), track {np.hypot(vx, vy) * span:.0f} px " + f"over {span * 60:.0f} min") + + tracked, fixed, zps = [], [], [] + for i, (key, path) in enumerate(C.lum_frames()): + xr, yr = x0 + vx * times[i], y0 + vy * times[i] + inv = tforms[key].inverse + nx_, ny_ = inv(np.array([[xr, yr]]))[0] + fx_, fy_ = inv(np.array([[x0, y0]]))[0] + with fits.open(path, memmap=False) as hd: + img = hd[0].data.astype(np.float32) + objs, ap, _ = C.detect(img) + zp, _ = C.frame_zeropoint(objs["x"], objs["y"], ap, mobjs, map_, + tforms[key]) + zps.append(zp) + # A source at V=21.7 moving at the asteroid's rate, injected into the + # raw sub, trailing across the 300 s exposure exactly as it would. + lin = np.linalg.inv(tforms[key].params[:2, :2]) + dxy = lin @ np.array([vx * 300.0 / 3600.0, vy * 300.0 / 3600.0]) + img2 = img.copy() + C.add_source(img2, nx_, ny_, + 10 ** ((zp - VMAG) / 2.5) / C.APFRAC, dxy[0], dxy[1]) + a = stamp(img2, nx_, ny_) + b = stamp(img, nx_, ny_) # same patch, no synthetic + del img, img2 + tracked.append(a - np.median(a)) + fixed.append(b - np.median(b)) + print(f" {key}: zp {zp:.3f}") + + def combine(cube): + m, _, _ = sigma_clipped_stats(np.array(cube), sigma=3.0, maxiters=2, + axis=0) + return np.asarray(m, dtype=np.float32) + + def flatten(stack): + """Remove any residual sky gradient before measuring noise.""" + bkg = sep.Background(stack, bw=24, bh=24, fw=3, fh=3) + return stack - bkg.back() + + with_src = flatten(combine(tracked)) + empty = flatten(combine(fixed)) + zp_eff = float(np.mean(zps)) + + _, _, sd = sigma_clipped_stats(empty, sigma=3.0) + noise_ap = sd * np.sqrt(np.pi * C.APRAD ** 2) + lim5 = -2.5 * np.log10(5.0 * noise_ap) + zp_eff + c = float(HALF) + f_src, _, _ = sep.sum_circle(with_src, np.array([c]), np.array([c]), + C.APRAD, err=float(sd), gain=1.0) + f_emp, _, _ = sep.sum_circle(empty, np.array([c]), np.array([c]), + C.APRAD, err=float(sd), gain=1.0) + snr_src = float(f_src[0]) / noise_ap + print(f"\nstacked 12 x 300 s along the track") + print(f" effective zero point {zp_eff:.3f}, pixel sigma {sd:.3f}") + print(f" 5 sigma point-source limit of the tracked stack: " + f"G = {lim5:.2f}") + # The untrailed limit above is optimistic: at 53"/hr the object smears + # 8.2 px within each 300 s sub, and a 5 px aperture cannot hold a streak + # that long. Scaling the injected source's recovered significance to + # 5 sigma gives the limit that actually applies at this rate. + lim5_trail = VMAG - 2.5 * np.log10(5.0 / max(snr_src, 1e-3)) + print(f" injected V = {VMAG} source recovered at " + f"{snr_src:.2f} sigma " + f"({'DETECTED' if snr_src > 5 else 'NOT detectable'})") + print(f" -> 5 sigma limit for a source trailing at {RATE:.0f}\"/hr: " + f"G = {lim5_trail:.2f} (vs {lim5:.2f} for an untrailed source)") + print(f" same aperture on the un-injected stack: " + f"{float(f_emp[0]) / noise_ap:.2f} sigma (blank, as expected)") + print(f"\n{NAME} at V={VMAG} is {VMAG - lim5:+.2f} mag relative to that " + f"limit.") + + np.savez(os.path.join(C.OUT, "_mo_shiftstack.npz"), + with_src=with_src, empty=empty, zp_eff=zp_eff, lim5=lim5, + sd=sd, snr_src=snr_src, lim5_trail=lim5_trail, vmag=VMAG, rate=RATE, name=NAME, + inside=inside, eph_px=epx, eph_py=epy, x0=x0, y0=y0, + vx=vx, vy=vy, times=times) + print("\nsaved _mo_shiftstack.npz") + + +if __name__ == "__main__": + main() diff --git a/session-scripts/mo_transient.py b/session-scripts/mo_transient.py new file mode 100644 index 0000000..dc454d6 --- /dev/null +++ b/session-scripts/mo_transient.py @@ -0,0 +1,232 @@ +"""Transient search: sources in the luminance master with no catalogue match. + +The chain is deliberately conservative, because on this field the sky is very +crowded with things that are real but not transient. + +1. Detect on the deep luminance master (60 min) and measure a 5 px aperture + magnitude on the master's own zero point. +2. Throw away everything that is not solidly detected. A transient claim rests + on a single night's data, so the bar is SNR > 10, not the SNR 5 the frame + nominally reaches. +3. Throw away everything that is not point-like. This is done against the + image's own PSF, measured from Gaia stars in the frame, using sep's + half-light radius. It removes background galaxies and the outer structure + of NGC 5128 itself, but note carefully that it does NOT remove Centaurus A's + globular clusters: at 3.8 Mpc a cluster with a 3 pc half-light radius spans + ~0.2 arcsec against 2.7 arcsec seeing, so clusters are unresolved here and + look exactly like stars. Only catalogues can separate them. +4. Cross-match against Gaia DR3 (the cached 0.42 deg, G < 20.5 catalogue). +5. Cross-match whatever is left against SkyMapper DR2 through VizieR, which + covers this declination and goes deeper than Gaia for non-stellar objects. +6. Demand that survivors are real by requiring an independent detection in at + least six of the twelve individual luminance subs. A cosmic ray or a + stacking artefact cannot do that; anything astrophysical will. +7. What is left is examined one by one. + +Writes mo-transient-candidates.csv and _mo_transient.npz. +""" +import csv +import os + +import numpy as np +import sep +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.wcs import WCS +from scipy.spatial import cKDTree + +import mo_common as C + +SNR_MIN = 10.0 +MATCH_RADIUS = 2.0 # arcsec, Gaia +SM_RADIUS = 3.0 # arcsec, SkyMapper (worse astrometry, wider PSF) +SUB_MIN = 6 # subs a source must independently appear in +EDGE = 40 # px + + +def detect_master(): + with fits.open(os.path.join(C.OUT, "master-Luminance.fit")) as hd: + img = hd[0].data.astype(np.float32) + hdr = hd[0].header + bkg = sep.Background(img, bw=64, bh=64, fw=3, fh=3) + sub = img - bkg.back() + objs = sep.extract(sub, 3.0, err=bkg.globalrms, minarea=5, + deblend_cont=0.005) + flux, ferr, _ = sep.sum_circle(sub, objs["x"], objs["y"], C.APRAD, + err=bkg.globalrms, gain=1.0) + r50, _ = sep.flux_radius(sub, objs["x"], objs["y"], + 6.0 * np.ones(len(objs)), 0.5, normflux=flux, + subpix=5) + return objs, np.asarray(flux), np.asarray(ferr), np.asarray(r50), \ + hdr, sub, bkg.globalrms + + +def gaia_cached(): + z = np.load(os.path.join(C.OUT, "_gaia_deep.npz")) + return z["ra"], z["dec"], z["g"] + + +def skymapper(ra0, dec0, radius_deg): + """SkyMapper DR2 over the field, cached because it is a slow query.""" + cache = os.path.join(C.OUT, "_skymapper.npz") + if os.path.exists(cache): + z = np.load(cache) + print(f"SkyMapper: {len(z['ra'])} cached sources") + return z["ra"], z["dec"], z["g"], z["r"], z["cls"] + from astroquery.vizier import Vizier + v = Vizier(columns=["RAICRS", "DEICRS", "gPSF", "rPSF", "ClassStar"], + row_limit=-1) + tbl = v.query_region(SkyCoord(ra0 * u.deg, dec0 * u.deg), + radius=radius_deg * u.deg, + catalog="II/358/smss")[0] + ra = np.asarray(tbl["RAICRS"], dtype=float) + dec = np.asarray(tbl["DEICRS"], dtype=float) + g = np.asarray(tbl["gPSF"], dtype=float) + r = np.asarray(tbl["rPSF"], dtype=float) + cls = np.asarray(tbl["ClassStar"], dtype=float) + np.savez_compressed(cache, ra=ra, dec=dec, g=g, r=r, cls=cls) + print(f"SkyMapper DR2: {len(ra)} sources retrieved") + return ra, dec, g, r, cls + + +def sky_match(sc, ra, dec, radius_arcsec): + """Nearest-neighbour match; returns index and separation in arcsec.""" + cat = SkyCoord(ra * u.deg, dec * u.deg) + idx, d2d, _ = sc.match_to_catalog_sky(cat) + return idx, d2d.arcsec, d2d.arcsec < radius_arcsec + + +def sub_support(xy_ref): + """How many of the twelve luminance subs independently show each source.""" + _, dets = C.load_dets() + tforms = C.frame_transforms() + n = np.zeros(len(xy_ref), dtype=int) + for key, _ in C.lum_frames(): + d = dets[key] + p = tforms[key](d[:, 2:4].astype(float)) + t = cKDTree(p) + dd, _ = t.query(xy_ref, distance_upper_bound=2.5) + n += np.isfinite(dd) + return n + + +def rgb_support(xy_ref, rms_scale=3.0): + """Peak significance of each position in the R, G and B masters. + + A real object on the sky is in every filter. A luminance-only artefact is + not. The masters are pixel-aligned by construction, so this is a direct + look-up rather than another registration. + """ + out = {} + for filt in ("Red", "Green", "Blue"): + with fits.open(os.path.join(C.OUT, f"master-{filt}.fit")) as hd: + img = hd[0].data.astype(np.float32) + bkg = sep.Background(img, bw=64, bh=64, fw=3, fh=3) + s = img - bkg.back() + fl, _, _ = sep.sum_circle(s, xy_ref[:, 0], xy_ref[:, 1], C.APRAD, + err=bkg.globalrms, gain=1.0) + noise = bkg.globalrms * np.sqrt(np.pi * C.APRAD ** 2) + out[filt] = np.asarray(fl) / noise + del img, s + return out + + +def main(): + objs, flux, ferr, r50, hdr, _sub, mrms = detect_master() + wcs = WCS(hdr) + ny, nx = 3194, 4788 + print(f"master detections: {len(objs)}") + + snr = np.where(ferr > 0, flux / np.maximum(ferr, 1e-9), 0.0) + mag = np.where(flux > 0, -2.5 * np.log10(np.maximum(flux, 1e-9)) + C.ZP, + np.nan) + + # PSF reference from the frame's own bright stars. + gra, gdec, gmag = gaia_cached() + sc_all = wcs.pixel_to_world(objs["x"], objs["y"]) + gidx, gsep, gok = sky_match(sc_all, gra, gdec, MATCH_RADIUS) + star = gok & (gmag[gidx] > 15) & (gmag[gidx] < 18.5) & (snr > 30) + r50_star = float(np.median(r50[star])) + r50_sig = float(np.std(r50[star])) + print(f"PSF from {star.sum()} Gaia stars: r50 = {r50_star:.2f} +- " + f"{r50_sig:.2f} px ({r50_star * C.SCALE:.2f} arcsec)") + + inframe = ((objs["x"] > EDGE) & (objs["x"] < nx - EDGE) & + (objs["y"] > EDGE) & (objs["y"] < ny - EDGE)) + good = inframe & (snr > SNR_MIN) & np.isfinite(mag) + pointlike = np.abs(r50 - r50_star) < 3.0 * max(r50_sig, 0.25) + print(f"SNR > {SNR_MIN:.0f} and in frame: {good.sum()}") + print(f" ... of which point-like: {(good & pointlike).sum()}") + print(f" ... of which Gaia-matched: {(good & pointlike & gok).sum()}") + + cand = good & pointlike & ~gok + print(f"\nno Gaia DR3 counterpart within {MATCH_RADIUS}\": {cand.sum()}") + + # SkyMapper DR2 + cen = wcs.pixel_to_world(nx / 2, ny / 2) + sra, sdec, sg, sr, scls = skymapper(cen.ra.deg, cen.dec.deg, 0.42) + sidx, ssep, sok = sky_match(sc_all, sra, sdec, SM_RADIUS) + cand2 = cand & ~sok + print(f"also no SkyMapper DR2 counterpart within {SM_RADIUS}\": " + f"{cand2.sum()}") + + ci = np.where(cand2)[0] + xy = np.column_stack([objs["x"][ci], objs["y"][ci]]) + nsub = sub_support(xy) + rgb = rgb_support(xy) + print(f"of those, detected in >= {SUB_MIN} individual subs: " + f"{(nsub >= SUB_MIN).sum()}") + + sc = wcs.pixel_to_world(xy[:, 0], xy[:, 1]) + # Distance from the centre of NGC 5128 - a supernova is expected on or + # near the galaxy, and this ranks the list accordingly. + cenA = SkyCoord("13h25m27.6s", "-43d01m09s") + dgal = sc.separation(cenA).arcmin + + rows = [] + for k, i in enumerate(ci): + rows.append(dict( + id=int(i), ra_deg=round(sc[k].ra.deg, 6), + dec_deg=round(sc[k].dec.deg, 6), + ra_hms=sc[k].ra.to_string(u.hour, sep=":", precision=2), + dec_dms=sc[k].dec.to_string(u.deg, sep=":", precision=1, + alwayssign=True), + x=round(float(xy[k, 0]), 2), y=round(float(xy[k, 1]), 2), + g_mag=round(float(mag[i]), 2), snr=round(float(snr[i]), 1), + r50_px=round(float(r50[i]), 2), + r50_over_psf=round(float(r50[i] / r50_star), 2), + n_subs=int(nsub[k]), + snr_R=round(float(rgb["Red"][k]), 1), + snr_G=round(float(rgb["Green"][k]), 1), + snr_B=round(float(rgb["Blue"][k]), 1), + gaia_sep_arcsec=round(float(gsep[i]), 2), + smss_sep_arcsec=round(float(ssep[i]), 2), + dist_from_cenA_arcmin=round(float(dgal[k]), 2))) + rows.sort(key=lambda r: -r["snr"]) + + out = os.path.join(C.OUT, "mo-transient-candidates.csv") + if rows: + with open(out, "w", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=list(rows[0].keys())) + w.writeheader() + w.writerows(rows) + print(f"\nwrote {out} ({len(rows)} rows)") + for r in rows[:40]: + print(f" {r['ra_hms']} {r['dec_dms']} G={r['g_mag']:5.2f} " + f"SNR={r['snr']:6.1f} r50/psf={r['r50_over_psf']:.2f} " + f"subs={r['n_subs']:2d} RGB=({r['snr_R']:.0f},{r['snr_G']:.0f}," + f"{r['snr_B']:.0f}) d={r['dist_from_cenA_arcmin']:.1f}'") + + np.savez(os.path.join(C.OUT, "_mo_transient.npz"), + x=objs["x"], y=objs["y"], mag=mag, snr=snr, r50=r50, + gaia_ok=gok, gaia_sep=gsep, sm_ok=sok, sm_sep=ssep, + good=good, pointlike=pointlike, cand=cand, cand2=cand2, + cand_idx=ci, nsub=nsub, r50_star=r50_star, r50_sig=r50_sig, + snr_R=rgb["Red"], snr_G=rgb["Green"], snr_B=rgb["Blue"], + dgal=dgal) + print("saved _mo_transient.npz") + + +if __name__ == "__main__": + main() diff --git a/session-scripts/mo_vet.py b/session-scripts/mo_vet.py new file mode 100644 index 0000000..3591b36 --- /dev/null +++ b/session-scripts/mo_vet.py @@ -0,0 +1,189 @@ +"""Transient search, stage 2: vet the catalogue non-matches individually. + +Two questions have to be answered about every source that failed to match +Gaia DR3 and SkyMapper DR2. + +Is it real? A source can be missing from the catalogues simply because it is +not there - a cosmic ray pair that survived sigma clipping, a deblending +artefact on the galaxy, a noise peak. The test used here is independence: +demand that the source is detected on its own in at least six of the twelve +300 s subs. Nothing that is not on the sky can do that. + +Is it new? A transient is a source that is present tonight and absent from +archival imagery. The catalogue non-match is weak evidence, because both Gaia +and SkyMapper run out of depth around the magnitudes of interest here and +neither is complete for extended or blended objects on a galaxy this bright. +The strong evidence is a picture: this pulls a Digitized Sky Survey cutout at +each position (DSS2 red, epoch ~1990s) through the CDS hips2fits service and +puts it beside the master. Anything visible on a plate taken thirty years ago +is not a transient. + +Writes mo-transient-vetted.csv, and caches the DSS cutouts for the figure. +""" +import csv +import os +import warnings + +import numpy as np +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits + +import mo_common as C + +NSUB_MIN = 6 +CUT = 60 # px half-size of the postage stamps +DSSDIR = os.path.join(C.OUT, "_dss") + + +def load(): + z = np.load(os.path.join(C.OUT, "_mo_transient.npz")) + return z + + +def dss_cutout(ra, dec, fov_arcmin=1.5, npix=120, survey="CDS/P/DSS2/red"): + """Archival DSS2 red image of one position, cached to disk.""" + tag = f"{ra:.5f}{dec:+.5f}_{survey.split('/')[-1]}.fits" + path = os.path.join(DSSDIR, tag) + os.makedirs(DSSDIR, exist_ok=True) + if os.path.exists(path): + with fits.open(path) as hd: + return hd[0].data.astype(float) + try: + from astroquery.hips2fits import hips2fits + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + hdul = hips2fits.query( + hips=survey, width=npix, height=npix, + ra=ra * u.deg, dec=dec * u.deg, + fov=(fov_arcmin / 60.0) * u.deg, + projection="TAN", format="fits") + data = np.asarray(hdul[0].data, dtype=float) + fits.PrimaryHDU(data).writeto(path, overwrite=True) + return data + except Exception as exc: # noqa: BLE001 + print(f" DSS fetch failed for {ra:.5f} {dec:+.5f}: " + f"{type(exc).__name__}: {exc}") + return None + + +def nearest_catalogue(sc): + """Nearest Gaia and SkyMapper source at ANY separation, plus SIMBAD/NED. + + The main search used fixed match radii. For a handful of finalists it is + worth knowing what the nearest catalogued thing actually is and how far + away it lies - a 4 arcsec offset from a SkyMapper source on a crowded + galaxy usually means the same object, badly centroided. + """ + gz = np.load(os.path.join(C.OUT, "_gaia_deep.npz")) + gcat = SkyCoord(gz["ra"] * u.deg, gz["dec"] * u.deg) + sz = np.load(os.path.join(C.OUT, "_skymapper.npz")) + scat = SkyCoord(sz["ra"] * u.deg, sz["dec"] * u.deg) + gi, gd, _ = sc.match_to_catalog_sky(gcat) + si, sd, _ = sc.match_to_catalog_sky(scat) + return (gd.arcsec, gz["g"][gi], sd.arcsec, sz["g"][si], sz["r"][si], + sz["cls"][si]) + + +def ned_query(sc, radius_arcsec=15.0): + """Anything NED knows about within a few arcsec of each position.""" + out = [] + try: + from astroquery.ipac.ned import Ned + except Exception: # noqa: BLE001 + return ["NED unavailable"] * len(sc) + for c in sc: + try: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + t = Ned.query_region(c, radius=radius_arcsec * u.arcsec) + if t is None or len(t) == 0: + out.append("") + else: + names = [f"{r['Object Name']}({r['Type']}," + f"{r['Separation']:.1f}\")" for r in t[:3]] + out.append("; ".join(names)) + except Exception as exc: # noqa: BLE001 + out.append(f"query failed: {type(exc).__name__}") + return out + + +def main(): + z = load() + ci = z["cand_idx"] + nsub = z["nsub"] + sel = np.where(nsub >= NSUB_MIN)[0] + print(f"{len(ci)} sources matched neither Gaia DR3 nor SkyMapper DR2") + print(f"{len(sel)} of them are independently detected in >= {NSUB_MIN} " + f"of the 12 subs\n") + + from astropy.wcs import WCS + hdr = fits.getheader(os.path.join(C.OUT, "master-Luminance.fit")) + wcs = WCS(hdr) + x = z["x"][ci][sel] + y = z["y"][ci][sel] + sc = wcs.pixel_to_world(x, y) + + gd, gg, sd, sg, sr, scls = nearest_catalogue(sc) + ned = ned_query(sc) + + rows = [] + for k, i in enumerate(sel): + ra, dec = sc[k].ra.deg, sc[k].dec.deg + dss = dss_cutout(ra, dec) + # Is anything there on the archival plate? Compare the peak in the + # central 6 arcsec against the frame-wide robust scatter. + dss_sig = np.nan + if dss is not None and np.isfinite(dss).any(): + h = dss.shape[0] // 2 + core = dss[h - 6:h + 7, h - 6:h + 7] + med = np.nanmedian(dss) + mad = np.nanmedian(np.abs(dss - med)) * 1.4826 + if mad > 0: + dss_sig = float((np.nanmax(core) - med) / mad) + rows.append(dict( + ra_deg=round(ra, 6), dec_deg=round(dec, 6), + ra_hms=sc[k].ra.to_string(u.hour, sep=":", precision=2), + dec_dms=sc[k].dec.to_string(u.deg, sep=":", precision=1, + alwayssign=True), + x=round(float(x[k]), 1), y=round(float(y[k]), 1), + g_mag=round(float(z["mag"][ci][i]), 2), + snr=round(float(z["snr"][ci][i]), 1), + r50_over_psf=round(float(z["r50"][ci][i] / z["r50_star"]), 2), + n_subs=int(nsub[i]), + snr_R=round(float(z["snr_R"][i]), 1), + snr_G=round(float(z["snr_G"][i]), 1), + snr_B=round(float(z["snr_B"][i]), 1), + dist_cenA_arcmin=round(float(z["dgal"][i]), 2), + nearest_gaia_arcsec=round(float(gd[k]), 2), + nearest_gaia_G=round(float(gg[k]), 2), + nearest_smss_arcsec=round(float(sd[k]), 2), + nearest_smss_g=round(float(sg[k]), 2), + nearest_smss_classstar=round(float(scls[k]), 2), + dss_peak_sigma=round(dss_sig, 1) if np.isfinite(dss_sig) + else "", + ned=ned[k])) + r = rows[-1] + print(f" {r['ra_hms']} {r['dec_dms']} G={r['g_mag']:5.2f} " + f"SNR={r['snr']:6.1f} r50/psf={r['r50_over_psf']:.2f} " + f"subs={r['n_subs']:2d}") + print(f" nearest Gaia {r['nearest_gaia_arcsec']:6.2f}\" " + f"(G={r['nearest_gaia_G']:.2f}), nearest SkyMapper " + f"{r['nearest_smss_arcsec']:6.2f}\" (g={r['nearest_smss_g']:.2f}" + f", ClassStar={r['nearest_smss_classstar']:.2f})") + print(f" DSS2-red peak {r['dss_peak_sigma']} sigma, " + f"{r['dist_cenA_arcmin']:.1f}' from Cen A centre") + print(f" NED: {r['ned'] or '(nothing within 15\")'}") + + out = os.path.join(C.OUT, "mo-transient-vetted.csv") + with open(out, "w", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=list(rows[0].keys())) + w.writeheader() + w.writerows(rows) + print(f"\nwrote {out} ({len(rows)} rows)") + np.save(os.path.join(C.OUT, "_mo_vetted.npy"), + np.array(rows, dtype=object), allow_pickle=True) + + +if __name__ == "__main__": + main() diff --git a/session-scripts/original.py b/session-scripts/original.py new file mode 100644 index 0000000..8b5061d --- /dev/null +++ b/session-scripts/original.py @@ -0,0 +1,153 @@ +"""The plainest possible stack: align the frames, average them, stop. + +This exists as a baseline to compare every processed version against. The +only operation applied to the pixels is the geometric one needed to make the +frames line up. In particular there is NO: + + - sky/background subtraction - gradient or plane removal + - per-frame flux normalisation - outlier or sigma rejection + - weighting by noise - colour calibration or white balance + - stretch, saturation or denoise - deconvolution or sharpening + +so cosmic rays, satellite trails, the moon gradient and every frame's own sky +level all survive into the result, exactly as they were recorded. That is the +point: it is the honest sum of the data. + +Two things it is NOT innocent of, and cannot be: + 1. The frames arrive from iTelescope already bias/dark/flat calibrated + (CALSTAT = 'BDF'). That cannot be undone here. + 2. Alignment resamples. A bicubic warp interpolates, which very slightly + smooths and correlates neighbouring pixels. The reference frame itself is + not resampled at all, so it is the one frame that stays pristine. + +Output is linear 32-bit FITS, which is what a baseline should be. A linear +image displays as almost pure black, so a display-only stretched preview is +written alongside and clearly labelled as such - the numbers live in the FITS. +""" +import os + +import astroalign as aa +import numpy as np +from astropy.io import fits +from PIL import Image +from skimage.transform import warp + +import layout + +SRC = layout.SESSION +STACKED = layout.SESSION +OUT = layout.path("original") +CACHE = layout.path("_stars.npz") + +# The same reference frame the processed stack used, so the two are pixel +# aligned and can be compared or differenced directly. +REFERENCE = "Luminance_002" +FILTERS = ["Luminance", "Red", "Green", "Blue"] + + +def main(): + os.makedirs(OUT, exist_ok=True) + z = np.load(CACHE, allow_pickle=True) + meta = {row[0]: row for row in z["meta"]} + stars = {k: z[k + "_xy"] for k in meta} + ref_xy = stars[REFERENCE] + print(f"reference {REFERENCE}, no crop, no rejection, no normalisation") + + # The plate solution was fitted on this same pixel grid, so it can be + # carried over. Copying header keywords does not touch the pixels. + with fits.open(layout.path("master-Luminance.fit")) as hd: + solved = hd[0].header + wcs_keys = [k for k in ("WCSAXES", "CRPIX1", "CRPIX2", "CDELT1", "CDELT2", + "CUNIT1", "CUNIT2", "CTYPE1", "CTYPE2", "CRVAL1", + "CRVAL2", "LONPOLE", "LATPOLE", "MJDREF", + "RADESYS", "PC1_1", "PC1_2", "PC2_1", "PC2_2") + if k in solved] + + planes = {} + for filt in FILTERS: + keys = sorted(k for k in meta if meta[k][2] == filt) + total = None + count = None + hdr0 = None + for key in keys: + fname = meta[key][1] + with fits.open(layout.path(fname), memmap=False) as hd: + data = hd[0].data.astype(np.float32) + if hdr0 is None: + hdr0 = hd[0].header.copy() + if key == REFERENCE: + reg = data # reference is never resampled + else: + tform, _ = aa.find_transform(stars[key], ref_xy) + reg = warp(data, inverse_map=tform.inverse, order=3, + mode="constant", cval=np.nan, + preserve_range=True).astype(np.float32) + del data + valid = np.isfinite(reg) + if total is None: + total = np.where(valid, reg, 0.0).astype(np.float32) + count = valid.astype(np.float32) + else: + total += np.where(valid, reg, 0.0) + count += valid + print(f" {key:16s} added (mean level {np.nanmean(reg):8.2f} ADU)") + del reg, valid + # Straight arithmetic mean. Where a frame did not cover a pixel it + # simply does not contribute, which is bookkeeping rather than + # processing: no pixel is invented. + stack = total / np.maximum(count, 1) + stack[count == 0] = 0.0 + del total, count + + hdr = hdr0 + hdr["FILTER"] = filt + hdr["NCOMBINE"] = (len(keys), "frames averaged") + hdr["EXPTOTAL"] = (300.0 * len(keys), "[s] total integration") + hdr["STACKREF"] = (REFERENCE, "alignment reference frame") + hdr["STACKALG"] = ("plain mean, no rejection", "combine method") + hdr["PROCLVL"] = ("align+average only", "no other processing applied") + for k in wcs_keys: + hdr[k] = (solved[k], solved.comments[k]) + path = layout.path(f"original-{filt}.fit") + fits.PrimaryHDU(stack.astype(np.float32), hdr).writeto(path, + overwrite=True) + print(f" -> {path} min={stack.min():.1f} median={np.median(stack):.1f} " + f"max={stack.max():.1f} ADU") + planes[filt] = stack + + # A colour version assembled with no calibration at all: the three filters + # dropped straight into R, G and B on a shared linear scale. Centaurus A + # will look yellow-green, because that is what the raw filter throughputs + # and a 46% moon actually produced. + rgb = np.dstack([planes["Red"], planes["Green"], planes["Blue"]]) + hdr = fits.getheader(layout.path("original-Red.fit")) + hdr["PROCLVL"] = ("align+average only", "no colour calibration applied") + fits.PrimaryHDU(np.moveaxis(rgb, 2, 0).astype(np.float32), hdr).writeto( + layout.path("original-RGB.fit"), overwrite=True) + print("wrote original-RGB.fit (uncalibrated colour cube)") + + # Display-only previews. The stretch here is a viewing aid and is NOT + # baked into any of the FITS above. + def preview(arr, name, note): + lo = np.percentile(arr, 25) + hi = np.percentile(arr, 99.9) + s = np.clip((arr - lo) / max(hi - lo, 1e-6), 0, 1) ** 0.35 + im = Image.fromarray((s * 255 + 0.5).astype(np.uint8)) + im.thumbnail((2400, 2400), Image.LANCZOS) + im.save(layout.path(name), quality=92) + print(f"wrote {name} ({note})") + + preview(planes["Luminance"], "NGC5128-original-Luminance-preview.jpg", + "display stretch only, linear data is in the FITS") + lo = np.percentile(rgb, 25) + hi = np.percentile(rgb, 99.9) + s = np.clip((rgb - lo) / max(hi - lo, 1e-6), 0, 1) ** 0.35 + im = Image.fromarray((s * 255 + 0.5).astype(np.uint8)) + im.thumbnail((2400, 2400), Image.LANCZOS) + im.save(layout.path("NGC5128-original-RGB-preview.jpg"), quality=92) + print("wrote NGC5128-original-RGB-preview.jpg (display stretch only, no colour " + "calibration)") + + +if __name__ == "__main__": + main() diff --git a/session-scripts/rename.py b/session-scripts/rename.py new file mode 100644 index 0000000..8729738 --- /dev/null +++ b/session-scripts/rename.py @@ -0,0 +1,91 @@ +"""Prefix every image output with the object name, and fix every reference. + +Renaming files is the easy half. The half that breaks a project silently is the +references: METHODS.md and the three analysis notes cite these filenames, and +the scripts that WRITE them would otherwise recreate the old names on the next +run. So this rewrites the markdown and the scripts in the same pass. + +Dry run by default; pass --apply to actually move anything. +""" +import os +import re +import sys + +import layout + +ROOT = layout.SESSION +PREFIX = "NGC5128-" +IMAGE_EXT = (".png", ".jpg", ".jpeg", ".tif", ".tiff") +APPLY = "--apply" in sys.argv + + +def image_files(): + """Every image in the stacked tree that is not already prefixed.""" + out = [] + for folder in (ROOT, layout.path("original")): + if not os.path.isdir(folder): + continue + for name in sorted(os.listdir(folder)): + path = os.path.join(folder, name) + if not os.path.isfile(path): + continue + if not name.lower().endswith(IMAGE_EXT): + continue + if name.startswith(PREFIX): + continue + out.append((folder, name, PREFIX + name)) + return out + + +renames = image_files() +print(f"{len(renames)} image files to rename\n") +for folder, old, new in renames: + where = os.path.relpath(folder, ROOT) + print(f" {where}/{old} -> {new}") + +# Build the substitution map. Longest names first so that a short name which is +# a substring of a longer one cannot corrupt it. +mapping = {old: new for _, old, new in renames} +ordered = sorted(mapping, key=len, reverse=True) + +targets = [] +for folder, _, _ in [(ROOT, None, None)]: + for name in os.listdir(folder): + if name.lower().endswith(".md"): + targets.append(os.path.join(folder, name)) +scripts = layout.path("scripts") +if os.path.isdir(scripts): + for name in os.listdir(scripts): + if name.lower().endswith(".py"): + targets.append(os.path.join(scripts, name)) + +print(f"\nchecking {len(targets)} markdown and script files for references") +edits = [] +for path in targets: + try: + text = open(path, encoding="utf-8").read() + except UnicodeDecodeError: + text = open(path, encoding="latin-1").read() + updated = text + hits = 0 + for old in ordered: + # A reference is the bare filename; guard the left edge so an already + # prefixed occurrence is not double-prefixed. + pattern = re.compile(r"(? r=25 ("total") +ZPTOT = ZP5 + APCOR # 28.154, zero point for total flux +MU0 = ZPTOT + 2.5*np.log10(PIXAREA) # 26.807; mu = MU0 - 2.5*log10(I_ADU_per_px) +X0, Y0 = 2397.88, 1592.09 # nucleus, from WCS + Gaia-verified astrometry +DIST_MPC = 3.8 +KPC_PER_ARCSEC = DIST_MPC*1e3*np.pi/180/3600 # 0.01842 kpc/arcsec + +def path(*p): return layout.path(*p) + +def load(ch): + """Load one master as a contiguous native-endian float32 array.""" + d = fits.getdata(path('master-%s.fit' % ch)) + return np.ascontiguousarray(d.astype(np.float32)) + +def wcs(): + return WCS(fits.getheader(path('master-Luminance.fit'))) + +def mu(I): + """Surface brightness (mag/arcsec^2) from intensity in ADU/pixel.""" + I = np.asarray(I, float) + out = np.full(I.shape, np.nan) + m = I > 0 + out[m] = MU0 - 2.5*np.log10(I[m]) + return out + +def ell_radius(shape, x0, y0, eps, pa_rad): + """Semi-major-axis-equivalent radius map for a fixed ellipse geometry. + pa_rad measured counter-clockwise from the +x axis (photutils convention).""" + ny, nx = shape + y, x = np.mgrid[0:ny, 0:nx].astype(np.float32) + x -= np.float32(x0); y -= np.float32(y0) + c, s = np.float32(np.cos(pa_rad)), np.float32(np.sin(pa_rad)) + xp = x*c + y*s + yp = -x*s + y*c + del x, y + return np.sqrt(xp*xp + (yp/np.float32(1.0-eps))**2) + + +def sky_pa(pa_deg): + """Convert a photutils isophote PA (deg CCW from +x) to sky PA (deg E of N). + + For this frame north lies 0.96 deg CCW of the +x axis and east lies along + -y, so the two conventions differ by very nearly 90 deg with a flip. + """ + cd = wcs().pixel_scale_matrix + t = np.radians(np.asarray(pa_deg, float)) + xi = cd[0, 0]*np.cos(t) + cd[0, 1]*np.sin(t) # +east + eta = cd[1, 0]*np.cos(t) + cd[1, 1]*np.sin(t) # +north + return np.degrees(np.arctan2(xi, eta)) % 180. + + +def north_east_pixel(): + """Unit vectors (dx, dy) pointing north and east in pixel coordinates.""" + cd = wcs().pixel_scale_matrix + det = cd[0, 0]*cd[1, 1] - cd[0, 1]*cd[1, 0] + n = np.array([-cd[0, 1], cd[0, 0]])/det + e = np.array([cd[1, 1], -cd[1, 0]])/det + return n/np.hypot(*n), e/np.hypot(*e) + + +ISO_HDR = ('sma_px,sma_arcsec,sma_arcmin,sma_kpc,intens_adu_px,intens_err,' + 'rms_adu,mu_mag_arcsec2,mu_err,ellipticity,ellipticity_err,' + 'pa_deg_ccw_from_x,pa_err_deg,pa_deg_east_of_north,ndata,nflag,' + 'stop_code') + + +def write_isophote_csv(tab, fn): + """Write an isophote table to CSV. + + Rows with stop_code != 0 had too little unmasked azimuth for the geometry + to converge: photutils still measures a valid intensity along the held + ellipse, but its eps/PA are carried over from the previous isophote and its + formal errors are meaningless (they come back as values like 1169 and + 24006). Those five geometry columns are therefore blanked to NaN, so the + file cannot be read as if the geometry had been measured there. The + intensity columns are kept, because they are real. + """ + import os + a = tab['sma']*PIXSCALE + conv = tab['stop'] == 0 + blank = lambda v: np.where(conv, v, np.nan) + with np.errstate(all='ignore'): + me = 2.5/np.log(10)*tab['int_err']/np.where(tab['intens'] > 0, + tab['intens'], np.nan) + out = np.column_stack([tab['sma'], a, a/60., a*KPC_PER_ARCSEC, + tab['intens'], tab['int_err'], tab['rms'], + mu(tab['intens']), me, + blank(tab['eps']), blank(tab['eps_err']), + blank(tab['pa']), blank(tab['pa_err']), + blank(sky_pa(tab['pa'])), + tab['ndata'], tab['nflag'], tab['stop']]) + np.savetxt(path(fn), out, delimiter=',', header=ISO_HDR, comments='', + fmt='%.5f') + print('wrote %s (%d rows, %d with converged geometry)' + % (fn, len(out), conv.sum())) diff --git a/session-scripts/sb_dust.py b/session-scripts/sb_dust.py new file mode 100644 index 0000000..a6f05b5 --- /dev/null +++ b/session-scripts/sb_dust.py @@ -0,0 +1,191 @@ +"""Step 5: dust-lane extinction map, with an independent colour-excess check. + +Method + 1. The smooth isophote model of the luminance master gives the light the + galaxy would show with no dust. A_L = -2.5 log10(observed / model). + This is a foreground-screen approximation: the lane is a warped disk seen + nearly edge-on across the near side of the bulge, so the screen assumption + is good for the lane itself but underestimates the true optical depth + wherever stars sit in front of the dust. + 2. The same is done for R, G and B on the SAME elliptical isophotes, using + the sigma-clipped azimuthal median of the dust-free azimuths as each + channel's unobscured model. E(B-R) = A_B - A_R is then a completely + independent, model-ratio-based reddening measurement, and A_L / E(B-R) + is a measured extinction-law ratio rather than an assumed one. + +Outputs + sb-extinction.fits A_L map (mag), NaN outside the measurable region + NGC5128-sb-extinction-map.png 4-panel extinction / reddening figure + NGC5128-sb-extinction-law.png A_L against E(B-R) with the fitted ratio +""" +import numpy as np +from astropy.io import fits +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt +from scipy.ndimage import gaussian_filter, median_filter +from sb_common import * + +XC, YC = np.load(path('_geom.npy')) +P = np.load(path('_profile.npz')) +star = fits.getdata(path('sb-mask-stars.fits')).astype(bool) +dust = fits.getdata(path('sb-mask-dust.fits')).astype(bool) +a_map = np.load(path('_amap.npy')) +ac = P['a'] + +A = {} +for ch in ['Luminance', 'Red', 'Green', 'Blue']: + pr = P[ch] + ok = np.isfinite(pr) & (pr > 0) + mdl = np.interp(a_map, ac[ok], pr[ok], left=pr[ok][0], right=np.nan) + img = gaussian_filter(load(ch), 2.0) # match the colour smoothing + with np.errstate(all='ignore'): + A[ch] = np.where((img > 0) & (mdl > 0), -2.5*np.log10(img/mdl), + np.nan).astype(np.float32) + del img, mdl + print('%-10s A map built' % ch) + +AL = A['Luminance'] +EBR = (A['Blue'] - A['Red']).astype(np.float32) + +# region where both are trustworthy: inside the lane, bright enough, not a star +Lmod = np.interp(a_map, ac[np.isfinite(P['Luminance'])], + P['Luminance'][np.isfinite(P['Luminance'])]) +valid = (~star) & (a_map < 700) & (Lmod > 60) & np.isfinite(AL) & np.isfinite(EBR) +lane = valid & dust +print('valid extinction pixels: %d (%.1f arcmin^2); inside the lane: %d (%.1f arcmin^2)' + % (valid.sum(), valid.sum()*PIXAREA/3600., lane.sum(), lane.sum()*PIXAREA/3600.)) + +x, y = EBR[lane].astype(float), AL[lane].astype(float) +sel = (x > 0.1) & (x < 2.0) & (y > -0.3) & (y < 3.5) +k = float(np.median(y[sel]/x[sel])) +klsq = float(np.sum(x[sel]*y[sel])/np.sum(x[sel]**2)) +print('extinction-law ratio A_L / E(B-R) = %.2f (median of ratios), %.2f (lsq)' + % (k, klsq)) + +ALs = median_filter(np.nan_to_num(AL, nan=0.0), 9) +ALs[~valid] = np.nan +pk = np.nanpercentile(ALs[lane], [50, 90, 99, 99.9]) +print('A_L inside the lane mask (9x9 median filtered): ' + 'median %.2f, p90 %.2f, p99 %.2f, p99.9 %.2f mag' % tuple(pk)) +iy, ix = np.unravel_index(np.nanargmax(np.where(lane, ALs, np.nan)), ALs.shape) +w = wcs() +rr, dd = w.pixel_to_world_values(ix, iy) +print('peak A_L = %.2f mag at pixel (%d, %d) = %.5f %+.5f deg, %.0f arcsec from ' + 'the nucleus' % (ALs[iy, ix], ix, iy, rr, dd, + np.hypot(ix-XC, iy-YC)*PIXSCALE)) + +hdu = fits.PrimaryHDU(np.where(valid, AL, np.nan).astype(np.float32), + header=w.to_header()) +hdu.header['BUNIT'] = 'mag' +hdu.header['COMMENT'] = 'A_L = -2.5 log10(observed / smooth isophote model)' +hdu.writeto(path('sb-extinction.fits'), overwrite=True) + +# obscured light: how much luminance flux the lane removes +Lraw = load('Luminance') +lost = float(np.nansum((Lmod - Lraw)[lane])) +tot = float(np.nansum(Lmod[valid & (a_map < 700)])) +print('the lane hides %.3e ADU, i.e. %.1f%% of the modelled light inside a=700 px' + % (lost, 100*lost/tot)) +print(' that is %.2f mag of integrated light removed from the lane region' + % (-2.5*np.log10(1 - lost/max(np.nansum(Lmod[lane]), 1)))) + +# ------------------------------------------------------------------- figure 1 +NV, EV = north_east_pixel() +CUT = 620 +sl = (slice(int(YC)-CUT, int(YC)+CUT), slice(int(XC)-CUT, int(XC)+CUT)) +ext = [-CUT*PIXSCALE/60, CUT*PIXSCALE/60]*2 + + +def compass(ax, c='k', x=0.885, y=0.10, Ln=0.075): + for v, lab in [(NV, 'N'), (EV, 'E')]: + ax.annotate('', xy=(x+Ln*v[0], y+Ln*v[1]), xytext=(x, y), + xycoords='axes fraction', textcoords='axes fraction', + arrowprops=dict(arrowstyle='->', color=c, lw=1.4)) + ax.annotate(lab, xy=(x+1.45*Ln*v[0], y+1.45*Ln*v[1]), color=c, + xycoords='axes fraction', ha='center', va='center', fontsize=10) + + +fig, axs = plt.subplots(2, 2, figsize=(15, 14.4)) +axs = axs.ravel() +axs[0].imshow(np.arcsinh(np.clip(Lraw[sl], 0, None)/30), origin='lower', + cmap='gray', extent=ext) +axs[0].set_title('(a) luminance master') +compass(axs[0], 'w') + +im = axs[1].imshow(np.where(valid, ALs, np.nan)[sl], origin='lower', cmap='magma_r', + vmin=0, vmax=2.0, extent=ext) +axs[1].set_title('(b) extinction $A_L$ from the smooth model [mag]') +plt.colorbar(im, ax=axs[1], fraction=.046, label='mag') +compass(axs[1]) + +im = axs[2].imshow(np.where(valid, gaussian_filter(np.nan_to_num(EBR), 2), np.nan)[sl], + origin='lower', cmap='inferno_r', vmin=0, vmax=1.4, extent=ext) +axs[2].set_title('(c) colour excess E(B-R) from the R and B models [mag]') +plt.colorbar(im, ax=axs[2], fraction=.046, label='mag') +compass(axs[2]) + +im = axs[3].imshow(np.where(valid, ALs - k*EBR, np.nan)[sl], origin='lower', + cmap='RdBu_r', vmin=-0.6, vmax=0.6, extent=ext) +axs[3].set_title('(d) $A_L$ - %.2f E(B-R): agreement of the two methods' % k) +plt.colorbar(im, ax=axs[3], fraction=.046, label='mag') +compass(axs[3]) +for a in axs: + a.set_xlabel('arcmin') + a.set_ylabel('arcmin') +fig.suptitle('NGC 5128 dust lane: extinction and reddening ' + '(central %.1f x %.1f arcmin)' % (2*CUT*PIXSCALE/60, 2*CUT*PIXSCALE/60), + fontsize=14) +fig.tight_layout() +fig.savefig(path('NGC5128-sb-extinction-map.png'), dpi=125) +plt.close(fig) + +# ------------------------------------------------------------------- figure 2 +fig, axs = plt.subplots(1, 2, figsize=(13.5, 5.6)) +h = axs[0].hist2d(x[sel], y[sel], bins=(140, 140), range=[[0, 1.6], [-0.3, 3.0]], + cmap='viridis', norm=matplotlib.colors.LogNorm()) +plt.colorbar(h[3], ax=axs[0], label='pixels') +xs = np.linspace(0, 1.6, 50) +axs[0].plot(xs, k*xs, 'r-', lw=2, label='$A_L$ = %.2f E(B-R) (median ratio)' % k) +axs[0].plot(xs, klsq*xs, 'w--', lw=1.6, label='least squares: %.2f' % klsq) +axs[0].set_xlabel('E(B-R) [mag, instrumental]') +axs[0].set_ylabel('$A_L$ [mag]') +axs[0].legend(fontsize=9) +axs[0].set_title('extinction law measured inside the lane (%d px)' % sel.sum()) + +bb = np.geomspace(20, 700, 26) +ib = np.digitize(a_map, bb) +med, p90 = [], [] +for kk in range(1, len(bb)): + m = (ib == kk) & lane + med.append(np.nanmedian(ALs[m]) if m.sum() > 200 else np.nan) + p90.append(np.nanpercentile(ALs[m], 90) if m.sum() > 200 else np.nan) +bc = np.sqrt(bb[1:]*bb[:-1])*PIXSCALE +axs[1].plot(bc, med, 'o-', color='tab:purple', label='median $A_L$ in the lane') +axs[1].plot(bc, p90, 's--', color='tab:orange', label='90th percentile') +axs[1].set_xscale('log') +axs[1].set_xlabel('semi-major axis a [arcsec]') +axs[1].set_ylabel('$A_L$ [mag]') +axs[1].grid(alpha=.3) +axs[1].legend(fontsize=9) +axs[1].set_title('extinction against radius along the lane') +fig.tight_layout() +fig.savefig(path('NGC5128-sb-extinction-law.png'), dpi=140) +plt.close(fig) + +with open(path('sb-derived-quantities.txt'), 'a') as f: + wr = lambda t: (f.write(t + chr(10)), print(t)) + wr('') + wr('Dust lane') + wr(' lane mask area %.1f arcmin^2' % (dust.sum()*PIXAREA/3600.)) + wr(' measurable extinction area %.1f arcmin^2' % (lane.sum()*PIXAREA/3600.)) + wr(' A_L median / p90 / p99 / p99.9 %.2f / %.2f / %.2f / %.2f mag' % tuple(pk)) + wr(' peak A_L (9x9 median filtered) %.2f mag at RA %.4f Dec %+.4f' + % (ALs[iy, ix], rr, dd)) + wr(' peak is %.0f arcsec from the nucleus' % (np.hypot(ix-XC, iy-YC)*PIXSCALE)) + wr(' extinction law A_L / E(B-R) %.2f (median of ratios), %.2f (lsq)' + % (k, klsq)) + wr(' luminance flux hidden by the lane %.1f%% of the modelled light inside a=700 px' + % (100*lost/tot)) +print('') +print('wrote sb-extinction.fits, NGC5128-sb-extinction-map.png, NGC5128-sb-extinction-law.png') diff --git a/session-scripts/sb_iso.py b/session-scripts/sb_iso.py new file mode 100644 index 0000000..c2a4c8a --- /dev/null +++ b/session-scripts/sb_iso.py @@ -0,0 +1,128 @@ +"""Step 2: isophote fitting of the NGC 5128 luminance master. + +Pass A free-centre fits on the dust-free outer body -> adopted centre +Pass B fixed-centre fit, stars AND the dust lane masked -> PRIMARY table +Pass C the same fit on a de-reddened image, where the + extinction is estimated from the B-R colour excess and + calibrated against the Pass B model over 150 inner extension + +Outputs: sb-isophotes.csv, sb-isophotes-dereddened.csv, + _isoB.npz, _isoC.npz, _geom.npy, _extcal.npy +""" +import numpy as np +import time +from astropy.io import fits +from photutils.isophote import Ellipse, EllipseGeometry +from sb_common import * +import sb_model + +SMA_MIN, SMA_MAX, STEP = 8.0, 1750.0, 0.11 + +L = load('Luminance') +starmask = fits.getdata(path('sb-mask-stars.fits')).astype(bool) +dustmask = fits.getdata(path('sb-mask-dust.fits')).astype(bool) +exc = np.load(path('sb-colour-excess.npy')) + + +def table(iso): + k = [i for i in iso if i.sma > 0 and np.isfinite(i.intens)] + + def g(f): + return np.array([(f(i) if f(i) is not None else np.nan) for i in k], float) + + return dict(sma=g(lambda i: i.sma), intens=g(lambda i: i.intens), + int_err=g(lambda i: i.int_err), rms=g(lambda i: i.rms), + eps=g(lambda i: i.eps), eps_err=g(lambda i: i.ellip_err), + pa=np.degrees(g(lambda i: i.pa)) % 180., + pa_err=np.degrees(g(lambda i: i.pa_err)), + ndata=g(lambda i: i.ndata), nflag=g(lambda i: i.nflag), + stop=g(lambda i: i.stop_code)) + + +def fit(img, mask, x0, y0, label): + arr = np.ma.masked_array(img, mask=mask) + g = EllipseGeometry(x0=x0, y0=y0, sma=300., eps=0.15, pa=np.radians(150.)) + g.fix_center = True + t = time.time() + iso = Ellipse(arr, geometry=g).fit_image( + sma0=300., minsma=SMA_MIN, maxsma=SMA_MAX, step=STEP, linear=False, + nclip=3, sclip=3.0, fix_center=True) + print('%s: %d isophotes in %.0f s' % (label, len(iso), time.time() - t)) + return table(iso) + + +# ------------------------------------------------------------------- Pass A +arr = np.ma.masked_array(L, mask=starmask | dustmask) +cen = [] +for s in [350., 500., 650., 800., 1000.]: + try: + it = Ellipse(arr, geometry=EllipseGeometry( + x0=X0, y0=Y0, sma=s, eps=0.18, pa=np.radians(148.)) + ).fit_image(sma0=s, minsma=s * 0.98, maxsma=s * 1.02, step=0.1, + nclip=3, sclip=3.) + for i in it: + if np.isfinite(i.x0): + cen.append((i.x0, i.y0)) + except Exception as e: + print(' passA sma=%.0f: %s' % (s, e)) +cen = np.array(cen) +XC, YC = float(np.median(cen[:, 0])), float(np.median(cen[:, 1])) +off = np.hypot(XC - X0, YC - Y0) +print('Pass A: outer-isophote centre %.2f, %.2f (scatter %.1f, %.1f px, n=%d)' + % (XC, YC, cen[:, 0].std(), cen[:, 1].std(), len(cen))) +print(' WCS/Gaia nucleus %.2f, %.2f -> offset %.1f px = %.1f arcsec' + % (X0, Y0, off, off * PIXSCALE)) +np.save(path('_geom.npy'), np.array([XC, YC])) +del arr + +# ------------------------------------------------------------------- Pass B +tB = fit(L, starmask | dustmask, XC, YC, 'Pass B (stars+dust masked)') +np.savez(path('_isoB.npz'), **tB) + +# ---------------------------------------------------------------- extinction +# A_L from the Pass B model, used only where that model is directly constrained +good = np.isfinite(tB['intens']) & (tB['ndata'] > 150) & (tB['sma'] > 90) +tBg = {k: v[good] for k, v in tB.items()} +modB, aB = sb_model.build(L.shape, XC, YC, tBg, block=4) +with np.errstate(all='ignore'): + A_L = -2.5 * np.log10(np.clip(L, 1e-3, None) / np.clip(modB, 1e-3, None)) +cal = (dustmask & ~starmask & (aB > 150) & (aB < 600) & np.isfinite(exc) + & (exc > 0.05) & np.isfinite(A_L) & (A_L > -0.5) & (A_L < 4.0)) +cal &= exc > 0.15 # restrict to a well-measured colour excess +x, y = exc[cal].astype(float), A_L[cal].astype(float) +# robust slope through the origin: median of the per-pixel ratios +k_ratio = float(np.median(y / x)) +scatter = float(np.median(np.abs(y - k_ratio * x)) * 1.4826) +print('extinction calibration on %d px: A_L = %.3f * E(B-R), scatter %.3f mag' + % (cal.sum(), k_ratio, scatter)) +print(' (least-squares through origin for comparison: %.3f)' + % (np.sum(x * y) / np.sum(x * x))) +np.save(path('_extcal.npy'), np.array([k_ratio, scatter, cal.sum()])) + +E = np.clip(np.nan_to_num(exc, nan=0.0), 0.0, None) +A_est = np.clip(k_ratio * E, 0.0, 2.5) # >2.5 mag is unreliable +heavy = (k_ratio * E) > 2.5 +Lc = (L * 10 ** (0.4 * A_est)).astype(np.float32) +print('de-reddening: median A_L inside the lane mask %.2f mag; %d px above the ' + '2.5 mag cap (masked in Pass C)' % (np.median(A_est[dustmask]), heavy.sum())) +del modB, aB, A_L, A_est, E + +# ------------------------------------------------------------------- Pass C +tC = fit(Lc, starmask | heavy, XC, YC, 'Pass C (de-reddened)') +np.savez(path('_isoC.npz'), **tC) +del Lc, L + +# ------------------------------------------------------------------- CSVs +write_isophote_csv(tB, 'sb-isophotes.csv') +write_isophote_csv(tC, 'sb-isophotes-dereddened.csv') + +for name, t in [('Pass B (primary, dust masked)', tB), ('Pass C (de-reddened)', tC)]: + print('') + print(name) + print(' sma_px arcsec mu eps PA ndata nflag stop') + for i in range(len(t['sma'])): + if t['sma'][i] > 30 and i % 3: + continue + print('%7.1f %7.1f %6.2f %6.3f %6.1f %6d %5d %4d' + % (t['sma'][i], t['sma'][i] * PIXSCALE, mu(t['intens'][i]), + t['eps'][i], t['pa'][i], t['ndata'][i], t['nflag'][i], t['stop'][i])) diff --git a/session-scripts/sb_limits.py b/session-scripts/sb_limits.py new file mode 100644 index 0000000..93a6ed3 --- /dev/null +++ b/session-scripts/sb_limits.py @@ -0,0 +1,137 @@ +"""Step 6: how deep does the shell search actually go, and is anything there? + +Two questions: + 1. On what surface-brightness level would a shell have had to sit to be seen? + Binning the residual to ever coarser scales shows whether the noise + integrates down like photon noise (it does not: it is dominated by + correlated large-scale systematics), which sets the real limit. + 2. Is there any significant azimuthal structure? The m = 1..4 Fourier + amplitudes of the residual in each elliptical annulus are compared with + the amplitude expected from the noise alone. +""" +import numpy as np +from astropy.io import fits +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt +from sb_common import * + +XC, YC = np.load(path('_geom.npy')) +star = fits.getdata(path('sb-mask-stars.fits')).astype(bool) +dust = fits.getdata(path('sb-mask-dust.fits')).astype(bool) +a_map = np.load(path('_amap.npy')) +res = fits.getdata(path('sb-residual-flat.fits')).astype(np.float32) +rms = float(np.load(path('_rms.npy'))[0]) + + +def blockstat(img, mask, B, sel): + H, W = img.shape + h, w = H // B, W // B + a = img[:h * B, :w * B].reshape(h, B, w, B) + m = (~mask)[:h * B, :w * B].reshape(h, B, w, B) + s = sel[:h * B, :w * B].reshape(h, B, w, B) + n = m.sum(axis=(1, 3)) + v = np.where(n > 0.35 * B * B, np.where(m, a, 0).sum(axis=(1, 3)) / + np.maximum(n, 1), np.nan) + keep = np.isfinite(v) & (s.mean(axis=(1, 3)) > 0.8) + return v[keep] + + +print('depth of the shell search, measured on the model-subtracted residual') +print('(outer field, 1200 < a < 2400 px, stars and the dust lane excluded)') +print('') +print(' bin bin size rms 3 sigma limit ideal if noise were white') +print(' [px] [arcsec] [ADU/px] [mag/arcsec2] [mag/arcsec2]') +sel = (a_map > 1200) & (a_map < 2400) # outer field, beyond the measured profile +base = None +rows = [] +for B in [8, 16, 32, 64, 128]: + v = blockstat(res, star | dust, B, sel) + if v.size < 30: + continue + sd = float(np.std(v)) + if base is None: + base, base_b = sd, B + ideal = base * (base_b / B) + print(' %4d %7.1f %8.3f %13.2f %13.2f' + % (B, B * PIXSCALE, sd, mu(3 * sd), mu(3 * ideal))) + rows.append((B, sd)) +print('') +print('The rms barely falls as the bins grow (%.2f -> %.2f ADU/px from 8 to 128 px' + % (rows[0][1], rows[-1][1])) +print('bins, against a factor 16 if it were white), so the floor is correlated') +print('large-scale structure -- flat-field residual plus the sky-plane') +print('systematic -- not photon noise. Pure photon noise would reach') +print('%.2f mag/arcsec2 at 128 px bins; the real limit is %.1f mag SHALLOWER.' + % (mu(3 * rms / 128), mu(3 * rms / 128) - mu(3 * rows[-1][1]))) + +# ------------------------------------------------------- Fourier amplitudes +print('') +print('azimuthal Fourier amplitudes of the residual, normalised to the model') +resd = np.load(path('_resd16.npy')) +a16 = np.load(path('_a16.npy')) +H, W = resd.shape +Y, X = np.mgrid[0:H, 0:W] +phi = np.arctan2(Y * 16 + 8 - YC, X * 16 + 8 - XC) +P = np.load(path('_profile.npz')) +prof, ac = P['Luminance'], P['a'] +okp = np.isfinite(prof) & (prof > 0) + +# stop where the measured profile itself runs out: beyond that I(a) is a fill +# value and the fractional amplitudes would be meaningless +A_OUT = float(P['a_out']) +edges = np.geomspace(150, A_OUT, 11) +out = [] +print(' a range [px] I(a) m=1 m=2 m=3 m=4 noise n') +for lo, hi in zip(edges[:-1], edges[1:]): + m = np.isfinite(resd) & (a16 >= lo) & (a16 < hi) + n = m.sum() + if n < 60: + continue + r, ph = resd[m], phi[m] + amp = [2 * np.abs(np.mean(r * np.exp(-1j * k * ph))) for k in (1, 2, 3, 4)] + noise = np.std(r) * np.sqrt(2. / n) * 2 + Im = np.interp(np.sqrt(lo * hi), ac[okp], prof[okp]) + out.append((np.sqrt(lo * hi), Im, amp, noise, n)) + print(' %5.0f-%5.0f %8.2f ' % (lo, hi, Im) + + ' '.join('%6.3f' % (a / max(Im, 1e-3)) for a in amp) + + ' %6.3f %5d' % (noise / max(Im, 1e-3), n)) +print('') +print('(amplitudes are fractional: A_m / I(a). A value is only meaningful if it') +print(' exceeds the "noise" column, which is the amplitude a pure-noise annulus') +print(' would produce.)') + +fig, ax = plt.subplots(figsize=(9.5, 6.4)) +aa = np.array([o[0] for o in out]) * PIXSCALE +for k in range(4): + ax.plot(aa, [o[2][k] / max(o[1], 1e-3) for o in out], 'o-', ms=4, + label='m = %d' % (k + 1)) +ax.plot(aa, [o[3] / max(o[1], 1e-3) for o in out], 'k--', lw=1.6, + label='formal noise expectation (lower bound: it assumes' + chr(10) + + 'independent bins and ignores correlated systematics)') +ax.set_xscale('log') +ax.set_yscale('log') +ax.set_xlabel('semi-major axis a [arcsec]') +ax.set_ylabel('fractional Fourier amplitude $A_m / I(a)$') +ax.set_title('NGC 5128: azimuthal structure in the model-subtracted residual' + + chr(10) + 'amplitudes are 4-10% of the local surface brightness at ' + 'every radius' + chr(10) + 'inside ~350 arcsec this is demonstrably ' + 'the dust lane; outside it, correlated systematics') +ax.grid(alpha=.3, which='both') +ax.legend(fontsize=8.5, loc='lower left') +fig.tight_layout() +fig.savefig(path('NGC5128-sb-residual-fourier.png'), dpi=140) +plt.close(fig) + +with open(path('sb-derived-quantities.txt'), 'a') as f: + wr = lambda t: (f.write(t + chr(10)), print(t)) + wr('') + wr('Shell / faint-structure search depth (outer field 1200 < a < 2400 px)') + for B, sd in rows: + wr(' %4d px bins (%5.1f arcsec): rms %.3f ADU/px, 3 sigma = %.2f mag/arcsec^2' + % (B, B * PIXSCALE, sd, mu(3 * sd))) + wr(' the rms does not integrate down like photon noise: the floor is') + wr(' correlated large-scale structure, not shot noise') + wr(' NO shells, arcs or tidal features were detected') +print('') +print('wrote NGC5128-sb-residual-fourier.png') diff --git a/session-scripts/sb_model.py b/session-scripts/sb_model.py new file mode 100644 index 0000000..f603098 --- /dev/null +++ b/session-scripts/sb_model.py @@ -0,0 +1,80 @@ +"""Smooth elliptical model builder shared by the later steps. + +Given an isophote table (sma, intens, eps, pa) and a fixed centre, assign every +pixel the semi-major axis a of the isophote passing through it. Because eps(a) +and pa(a) vary slowly this is solved by fixed-point iteration starting from the +circular radius, which converges in a handful of passes. The model intensity is +then a log-log interpolation of intens(a). + +This is used instead of photutils.isophote.build_ellipse_model because it is +much faster on a 4788x3194 frame and because it guarantees a strictly smooth, +monotonic-in-a model with no interpolation artefacts to confuse the residual. +""" +import numpy as np +from scipy.interpolate import interp1d +from scipy.ndimage import gaussian_filter1d, zoom + + +def smooth_geometry(sma, eps, pa_deg, sig=2.0): + """Return (sma, eps, pa_rad) with eps and PA lightly smoothed along a.""" + ok = np.isfinite(eps) & np.isfinite(pa_deg) & np.isfinite(sma) + s = sma[ok] + e = gaussian_filter1d(np.clip(eps[ok], 0.0, 0.7), sig, mode='nearest') + p = np.unwrap(np.radians(pa_deg[ok]) * 2.0) / 2.0 # PA is defined mod 180 + p = gaussian_filter1d(p, sig, mode='nearest') + return s, e, p + + +def radius_map(shape, xc, yc, sma, eps, pa_rad, nit=15, dtype=np.float32): + """Semi-major axis of the isophote through each pixel.""" + fe = interp1d(sma, eps, bounds_error=False, fill_value=(eps[0], eps[-1])) + fp = interp1d(sma, pa_rad, bounds_error=False, fill_value=(pa_rad[0], pa_rad[-1])) + ny, nx = shape + Y, X = np.mgrid[0:ny, 0:nx].astype(np.float64) + X -= xc + Y -= yc + a = np.maximum(np.hypot(X, Y), 0.3) + for _ in range(nit): + q = 1.0 - fe(a) + th = fp(a) + c, s = np.cos(th), np.sin(th) + xp = X * c + Y * s + yp = -X * s + Y * c + a = np.maximum(np.sqrt(xp * xp + (yp / q) ** 2), 0.3) + return a.astype(dtype) + + +def build(shape, xc, yc, tab, nit=15, block=1): + """Return (model, a_map) at full resolution. + + block > 1 computes the radius map on a coarser grid and bilinearly + upsamples it; the model is smooth on scales far larger than block so this + costs nothing in accuracy and a lot less in time and memory. + """ + s, e, p = smooth_geometry(tab['sma'], tab['eps'], tab['pa']) + if block > 1: + sh = (shape[0] // block, shape[1] // block) + a = radius_map(sh, (xc - (block - 1) / 2.) / block, + (yc - (block - 1) / 2.) / block, s / block, e, p, nit) + # radius_map worked in block units: convert back to full-resolution pixels + a = zoom(a.astype(np.float32) * block, + (shape[0] / sh[0], shape[1] / sh[1]), order=1) + if a.shape != tuple(shape): + b = np.zeros(shape, np.float32) + n0, n1 = min(a.shape[0], shape[0]), min(a.shape[1], shape[1]) + b[:n0, :n1] = a[:n0, :n1] + if n0 < shape[0]: + b[n0:, :] = b[n0 - 1, :] + if n1 < shape[1]: + b[:, n1:] = b[:, [n1 - 1]] + a = b + else: + a = radius_map(shape, xc, yc, s, e, p, nit) + + ok = np.isfinite(tab['intens']) & (tab['intens'] > 1e-3) + ls, li = np.log10(tab['sma'][ok]), np.log10(tab['intens'][ok]) + o = np.argsort(ls) + ls, li = ls[o], li[o] + mod = (10 ** np.interp(np.log10(np.maximum(a, 0.3)), ls, li, + left=li[0], right=-3.0)).astype(np.float32) + return mod, a diff --git a/session-scripts/sb_prep.py b/session-scripts/sb_prep.py new file mode 100644 index 0000000..192fe59 --- /dev/null +++ b/session-scripts/sb_prep.py @@ -0,0 +1,132 @@ +"""Step 1: masks and basic calibration checks for the NGC 5128 analysis. + +Produces + sb-mask-stars.fits foreground stars / saturated cores / compact objects + sb-mask-dust.fits the dust lane, defined from the B-R colour excess + sb-colour-excess.npy E(B-R) instrumental colour excess map (float32) + +The dust mask is built from COLOUR, not from a model residual: the lane is the +only thing in the frame that is strongly red relative to the smooth stellar +body, so a colour cut is far more specific than a brightness-residual cut and +does not eat the galaxy itself. +""" +import numpy as np, sep +from astropy.io import fits +from scipy import ndimage +from sb_common import * + +# ------------------------------------------------------------------ saturation +d = load('Luminance') +H, W = d.shape +print('-- saturation census (luminance master) --') +for thr in [30000, 50000, 60000, 62000, 63000, 64000, 65000, 66000]: + print(' > %6d ADU : %6d px' % (thr, (d > thr).sum())) +SAT = 63000.0 +core = ndimage.median_filter(d[int(Y0)-400:int(Y0)+400, int(X0)-400:int(X0)+400], 25) +print(' star-free (median-25) peak of galaxy light, inner 800 px: %.1f ADU' % core.max()) +print(' => the galaxy core is a factor %.0f below the clip level: NOT saturated' + % (SAT/core.max())) +del core + +bkg = sep.Background(d, bw=128, bh=128) +rms = float(bkg.globalrms) +print('\nglobal sky rms %.3f ADU/px -> 1-sigma = %.2f mag/arcsec^2' % (rms, mu(rms))) +np.save(path('_rms.npy'), np.array([rms])) + +# ------------------------------------------------------------------ star mask +mask = np.zeros((H, W), bool) +z = np.load(path('_gaia_deep.npz')) +gx, gy = wcs().world_to_pixel_values(z['ra'], z['dec']); gg = z['g'] +rad = np.clip(4.0 + 3.6*(16.5 - gg), 4, 120) +sel = (gx > -150) & (gx < W+150) & (gy > -150) & (gy < H+150) & (gg < 19.0) +gx, gy, rad = gx[sel], gy[sel], rad[sel] +print('\nGaia stars masked: %d (radii %.0f-%.0f px)' % (gx.size, rad.min(), rad.max())) +for x, y, r in zip(gx, gy, rad): + i0, i1 = max(0, int(y-r)), min(H, int(y+r)+1) + j0, j1 = max(0, int(x-r)), min(W, int(x+r)+1) + if i1 <= i0 or j1 <= j0: continue + sy = np.arange(i0, i1)[:, None] - y; sx = np.arange(j0, j1)[None, :] - x + mask[i0:i1, j0:j1] |= (sx*sx + sy*sy) < r*r + +rr = np.hypot(np.arange(W)[None, :]-X0, np.arange(H)[:, None]-Y0) + +# compact non-Gaia objects, but leave the crowded inner 150 px to the +# isophote fitter's own sigma clipping (masking there kills the fit) +sub = d - bkg.back() +obj, seg = sep.extract(sub, 8.0, err=rms, minarea=6, deblend_cont=0.005, + segmentation_map=True) +compact = obj['npix'] < 20000 +segmask = ndimage.binary_dilation(np.isin(seg, np.nonzero(compact)[0]+1), np.ones((5, 5))) +mask |= segmask & (rr > 150) +print('sep compact objects: %d (applied outside r=150 px)' % compact.sum()) +del sub, seg, segmask + +sat = ndimage.binary_dilation(d > SAT, np.ones((5, 5)), iterations=6) +mask |= sat +print('saturated-core mask (grown 12 px): %d px' % sat.sum()) +del sat, d + +print('star mask: %.2f%% of frame' % (100*mask.mean())) +for a, b in [(0,25),(25,50),(50,100),(100,200),(200,400),(400,800),(800,1600)]: + s = (rr >= a) & (rr < b) + print(' r %4d-%4d px: %.1f%%' % (a, b, 100*mask[s].mean())) +fits.PrimaryHDU(mask.astype(np.uint8)).writeto(path('sb-mask-stars.fits'), overwrite=True) + +# ------------------------------------------------------------------ dust mask +# instrumental B-R from smoothed R and B masters +R = ndimage.gaussian_filter(load('Red'), 3.0) +B = ndimage.gaussian_filter(load('Blue'), 3.0) +good = (R > 15) & (B > 4) +col = np.full((H, W), np.nan, np.float32) +col[good] = -2.5*np.log10(B[good]/R[good]) +del R, B + +# Unobscured baseline colour vs elliptical radius. Dust only ever reddens, so +# the blue tail of the colour distribution in each annulus is the dust-free +# stellar colour. The 15th percentile is measured over 150 < a < 900 px (where +# both the colour SNR is high and the lane does not fill the annulus) and fitted +# with a quadratic in log a, which is then extrapolated inwards and outwards. +a_map = ell_radius((H, W), X0, Y0, 0.17, np.radians(150.0)) +bins = np.geomspace(5, 2200, 60) +ib = np.digitize(a_map, bins) +base_r, base_v = [], [] +valid = good & ~mask +for k in range(1, len(bins)): + s = (ib == k) & valid + if s.sum() < 400: continue + base_r.append(0.5*(bins[k-1]+bins[k])); base_v.append(np.nanpercentile(col[s], 15)) +base_r, base_v = np.array(base_r), np.array(base_v) +fitr = (base_r > 150) & (base_r < 900) +pcoef = np.polyfit(np.log10(base_r[fitr]), base_v[fitr], 2) +print() +print('colour baseline: quadratic in log10(a), coeffs', np.round(pcoef, 4)) +print(' baseline B-R at a = 20/50/150/400/900 px:', + np.round(np.polyval(pcoef, np.log10([20, 50, 150, 400, 900])), 3)) +base = np.polyval(pcoef, np.log10(np.clip(a_map, 5, 3000))).astype(np.float32) +exc = (col - base).astype(np.float32) # E(B-R), instrumental +np.save(path('sb-colour-excess.npy'), exc) +np.save(path('_colbase.npy'), np.c_[base_r, base_v]) +np.save(path('_colbasefit.npy'), pcoef) + +# Hysteresis threshold: seed on a firm colour excess, grow into the fainter +# wings of the same connected structure. A flat low threshold alone picks up a +# spurious ring at the edge of the colour-SNR region, so it is not used. +e0 = np.nan_to_num(exc, nan=-9.0) +seed = (e0 > 0.30) & (a_map < 650) +grow = (e0 > 0.20) & (a_map < 650) +seed = ndimage.binary_opening(seed, np.ones((5, 5))) +lab, n = ndimage.label(ndimage.binary_closing(grow, np.ones((7, 7)))) +keep = np.unique(lab[seed & (lab > 0)]) +dust = np.isin(lab, keep[keep > 0]) +dust = ndimage.binary_closing(dust, np.ones((15, 15))) +lab, n = ndimage.label(dust); sz = np.bincount(lab.ravel()); sz[0] = 0 +dust = np.isin(lab, np.nonzero(sz > 3000)[0]) +print('dust mask: %d px = %.1f arcmin^2 (%.2f%% of frame)' + % (dust.sum(), dust.sum()*PIXAREA/3600., 100*dust.mean())) +for a, b in [(0,25),(25,50),(50,100),(100,200),(200,400),(400,800)]: + s = (a_map >= a) & (a_map < b) + print(' dust a %4d-%4d px: %5.1f%% star+dust %5.1f%%' + % (a, b, 100*dust[s].mean(), 100*(dust | mask)[s].mean())) +fits.PrimaryHDU(dust.astype(np.uint8)).writeto(path('sb-mask-dust.fits'), overwrite=True) +print() +print('wrote sb-mask-stars.fits, sb-mask-dust.fits, sb-colour-excess.npy') diff --git a/session-scripts/sb_profile.py b/session-scripts/sb_profile.py new file mode 100644 index 0000000..8e83277 --- /dev/null +++ b/session-scripts/sb_profile.py @@ -0,0 +1,311 @@ +"""Step 3: surface-brightness profile, isophote geometry plots, derived numbers. + +Produces + NGC5128-sb-profile.png mu(a) with every noise/systematic floor marked + NGC5128-sb-isophote-geometry.png mu, ellipticity and position angle vs semi-major axis + NGC5128-sb-colour-profile.png L, R, G, B profiles and the B-R colour gradient + sb-derived-quantities.txt the numbers worth quoting + +The photutils Pass B table is the primary isophote result. An independent +sigma-clipped azimuthal-median extraction on the same elliptical grid is done +here as a cross-check and to get the R/G/B profiles on identical isophotes. +""" +import numpy as np +from astropy.io import fits +from scipy.optimize import curve_fit +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt +from sb_common import * +import sb_model + +XC, YC = np.load(path('_geom.npy')) +rms = float(np.load(path('_rms.npy'))[0]) +tB = dict(np.load(path('_isoB.npz'))) +star = fits.getdata(path('sb-mask-stars.fits')).astype(bool) +dust = fits.getdata(path('sb-mask-dust.fits')).astype(bool) + +# ---------------------------------------------------------------- geometry map +gfit = np.isfinite(tB['intens']) & (tB['ndata'] > 150) & (tB['sma'] > 90) +tg = {k: v[gfit] for k, v in tB.items()} +s, e, p = sb_model.smooth_geometry(tg['sma'], tg['eps'], tg['pa']) +a_map = sb_model.radius_map((3194, 4788), XC, YC, s, e, p) +np.save(path('_amap.npy'), a_map) + +# ------------------------------------------------- independent profile extract +bins = np.geomspace(10, 2600, 90) +ib = np.digitize(a_map, bins) +ac = np.sqrt(bins[1:] * bins[:-1]) + + +def azimuthal(img, mask): + v = np.full(len(bins) - 1, np.nan) + n = np.zeros(len(bins) - 1, int) + sd = np.full(len(bins) - 1, np.nan) + for k in range(1, len(bins)): + m = (ib == k) & ~mask + if m.sum() < 40: + continue + x = img[m].astype(float) + med = np.median(x) + for _ in range(3): + r = 1.4826 * np.median(np.abs(x - med)) + if not np.isfinite(r) or r == 0: + break + x = x[np.abs(x - med) < 3 * r] + med = np.median(x) + v[k - 1] = med + n[k - 1] = x.size + sd[k - 1] = 1.4826 * np.median(np.abs(x - med)) + return v, n, sd + + +prof, nprof, sdprof = {}, {}, {} +for ch in ['Luminance', 'Red', 'Green', 'Blue']: + img = load(ch) + prof[ch], nprof[ch], sdprof[ch] = azimuthal(img, star | dust) + if ch == 'Luminance': + # far-field pedestal: the frame is over-subtracted because the sky plane + # was fitted on tiles that still contained galaxy halo + Bk = 64 + H, W = img.shape + lb = img[:H // Bk * Bk, :W // Bk * Bk].reshape(H // Bk, Bk, W // Bk, Bk) + mb = (~star)[:H // Bk * Bk, :W // Bk * Bk].reshape(H // Bk, Bk, W // Bk, Bk) + cnt = mb.sum(axis=(1, 3)) + bm = np.where(cnt > 1500, np.where(mb, lb, 0).sum(axis=(1, 3)) / + np.maximum(cnt, 1), np.nan) + ab = a_map[:H // Bk * Bk, :W // Bk * Bk].reshape(H // Bk, Bk, + W // Bk, Bk).mean(axis=(1, 3)) + PED = float(np.nanmean(bm[np.isfinite(bm) & (ab > 2400)])) + BLKRMS = float(np.nanstd(bm[np.isfinite(bm) & (ab > 1700)])) + del img + +L = prof['Luminance'] +print('far-field pedestal (a > 2400 px): %+.2f ADU/px -> mu %.2f' % (PED, mu(-PED))) +print('block-to-block scatter (a > 1700 px): %.2f ADU/px -> mu %.2f' % (BLKRMS, mu(BLKRMS))) + +# noise / systematic floors, expressed as surface brightness +FLOOR = { + 'per-pixel sky noise (1 sigma)': mu(rms), + 'large-scale block scatter (1 sigma)': mu(BLKRMS), + 'sky-pedestal systematic |offset|': mu(abs(PED)), +} +for k, v in FLOOR.items(): + print(' floor: %-38s %.2f mag/arcsec2' % (k, v)) + +# ---------------------------------------------------------------- reliability +A_IN = 62.0 # inward limit: the dust lane fills 100% of the azimuth inside +A_OUT = float(ac[np.nanargmax(np.where(L > abs(PED), ac, -1))]) # last a with I > |PED| +print('adopted reliable range: %.0f - %.0f px (%.1f - %.1f arcsec)' + % (A_IN, A_OUT, A_IN * PIXSCALE, A_OUT * PIXSCALE)) + +# ---------------------------------------------------------------- Sersic fit +def sersic_mu(a, mue, re, n): + bn = 2 * n - 1 / 3. + 0.009876 / n + return mue + 2.5 * bn / np.log(10) * ((a / re) ** (1. / n) - 1.) + + +fitsel = np.isfinite(L) & (ac >= A_IN) & (ac <= 900) & (L > 0) +x, y = ac[fitsel] * PIXSCALE, mu(L[fitsel]) +popt, pcov = curve_fit(sersic_mu, x, y, p0=[21.0, 300.0, 4.0], maxfev=40000) +mue, re_as, nser = popt +perr = np.sqrt(np.diag(pcov)) +print('Sersic fit over %.0f-%.0f arcsec: n = %.2f +- %.2f, Re = %.1f +- %.1f arcsec ' + '(%.2f kpc), mu_e = %.2f' % (x.min(), x.max(), nser, perr[2], re_as, perr[1], + re_as * KPC_PER_ARCSEC, mue)) +resid_sersic = y - sersic_mu(x, *popt) +print(' rms of Sersic residual: %.3f mag' % resid_sersic.std()) + +# ---------------------------------------------------------------- growth curve +eint = np.interp(ac, s, e) +area = np.pi * (1 - eint) * bins[1:] ** 2 - np.pi * (1 - eint) * bins[:-1] ** 2 +Lf = np.where(np.isfinite(L), L, 0.0) +# inside A_IN, extrapolate the Sersic fit (the real light there is dust-obscured) +inner = ac < A_IN +Lf[inner] = 10 ** ((MU0 - sersic_mu(ac[inner] * PIXSCALE, *popt)) / 2.5) +cum = np.cumsum(Lf * area) +mtot = -2.5 * np.log10(cum) + ZPTOT +for aa in [200, 400, 600, 800, 1000, 1200]: + i = np.argmin(np.abs(ac - aa)) + print(' total mag within a = %5.0f px (%5.1f arcmin): G = %.3f' + % (aa, ac[i] * PIXSCALE / 60., mtot[i])) +iA = np.argmin(np.abs(ac - A_OUT)) +half = cum[iA] / 2. +re_growth = np.interp(half, cum[:iA + 1], ac[:iA + 1]) * PIXSCALE +print(' half-light radius of the light enclosed within a=%.0f px: %.0f arcsec' + % (A_OUT, re_growth)) +inner_frac = cum[np.argmin(np.abs(ac - A_IN))] / cum[iA] +print(' fraction of that light coming from the extrapolated a < %.0f px: %.3f' + % (A_IN, inner_frac)) + +# ================================================================== plot 1 +fig, ax = plt.subplots(figsize=(9.5, 7.5)) +ok = np.isfinite(L) & (L > 0) +aas = ac * PIXSCALE +# systematic band from the sky pedestal +lo = mu(np.where(L > 0, L, np.nan)) +hi = mu(np.clip(np.where(np.isfinite(L), L, np.nan) - PED, 1e-6, None)) +ax.fill_between(aas[ok], lo[ok], hi[ok], color='tab:orange', alpha=.22, lw=0, + label='sky-pedestal systematic (%+.1f ADU/px)' % PED) +ax.plot(aas[ok], lo[ok], 'k-', lw=1.6, label='luminance, dust+stars masked') +gb = np.isfinite(tB['intens']) & (tB['intens'] > 0) +ax.plot(tB['sma'][gb] * PIXSCALE, mu(tB['intens'][gb]), 'o', ms=4.5, mfc='none', + mec='tab:blue', label='photutils isophote fit') +aa = np.linspace(A_IN * PIXSCALE, 900 * PIXSCALE, 200) +ax.plot(aa, sersic_mu(aa, *popt), '--', color='tab:red', lw=1.4, + label='Sersic n=%.2f, Re=%.0f"' % (nser, re_as)) +for lab, v, c in [('per-pixel 1 sigma', FLOOR['per-pixel sky noise (1 sigma)'], '0.45'), + ('large-scale 1 sigma', FLOOR['large-scale block scatter (1 sigma)'], 'tab:green'), + ('sky-pedestal systematic', FLOOR['sky-pedestal systematic |offset|'], 'tab:red')]: + ax.axhline(v, ls=':', color=c, lw=1.3) + ax.text(1150, v - 0.06, '%s (%.2f)' % (lab, v), color=c, fontsize=8.5, + va='bottom', ha='right') +ax.axvspan(20, A_IN * PIXSCALE, color='k', alpha=.11, lw=0) +ax.text(A_IN * PIXSCALE * 0.92, 17.35, 'dust lane fills the whole azimuth', + fontsize=8.5, ha='right', va='top', rotation=90, color='0.25') +ax.axvline(A_OUT * PIXSCALE, color='0.35', ls='-.', lw=1.2) +ax.text(A_OUT * PIXSCALE * 0.94, 17.35, 'systematic floor reached', fontsize=8.5, + color='0.3', ha='right', va='top', rotation=90) +ax.set_xscale('log') +ax.set_xlim(20, 1200) +ax.set_ylim(27.0, 17.2) +ax.set_xlabel('semi-major axis a [arcsec]') +ax.set_ylabel(r'$\mu$ [mag arcsec$^{-2}$, Gaia $G$ zero point]') +ax.set_title('NGC 5128 luminance surface-brightness profile\n' + '12 x 300 s, iTelescope T32, 0.5376"/px') +sec = ax.secondary_xaxis('top', functions=(lambda v: v * KPC_PER_ARCSEC, + lambda v: v / KPC_PER_ARCSEC)) +sec.set_xlabel('projected radius [kpc, D = 3.8 Mpc]') +ax.grid(alpha=.25) +ax.legend(loc='lower left', fontsize=9, framealpha=.95) +fig.tight_layout() +fig.savefig(path('NGC5128-sb-profile.png'), dpi=150) +plt.close(fig) + +# ================================================================== plot 2 +fig, axs = plt.subplots(3, 1, figsize=(9, 10.5), sharex=True, + gridspec_kw=dict(hspace=.07)) +axs[0].plot(aas[ok], lo[ok], 'k-', lw=1.5) +axs[0].fill_between(aas[ok], lo[ok], hi[ok], color='tab:orange', alpha=.22, lw=0) +axs[0].plot(aa, sersic_mu(aa, *popt), '--', color='tab:red', lw=1.2) +axs[0].set_ylim(27.0, 17.2) +axs[0].set_ylabel(r'$\mu$ [mag arcsec$^{-2}$]') +axs[0].set_title('NGC 5128 isophote fit (luminance; stars and dust lane masked)') + +# Only isophotes whose geometry actually converged are plotted. Inside +# a ~ 240 px the dust lane leaves too little unmasked azimuth and photutils +# holds eps and PA at their previous values: those are not measurements and +# plotting them would look like a flat measured trend. +conv = np.isfinite(tB['eps']) & (tB['stop'] == 0) +A_GEO = float(tB['sma'][conv].min()) * PIXSCALE +for axi, key, kerr, lab in [(axs[1], 'eps', 'eps_err', 'ellipticity $\\epsilon = 1-b/a$'), + (axs[2], 'pa', 'pa_err', 'position angle [deg, CCW from +x]')]: + axi.errorbar(tB['sma'][conv] * PIXSCALE, tB[key][conv], + yerr=np.nan_to_num(tB[kerr][conv]), fmt='o', ms=5.5, + color='tab:blue', capsize=2, label='fit converged (stop code 0)') + axi.set_ylabel(lab) + axi.grid(alpha=.25) + axi.legend(fontsize=8.5, loc='upper left') +axs[1].set_ylim(0.02, 0.29) +axs[2].set_ylim(133, 175) +pax = axs[2].secondary_yaxis('right', functions=(sky_pa, sky_pa)) +pax.set_ylabel('sky position angle [deg E of N]') +for axi in axs: + axi.axvspan(20, A_GEO, color='k', alpha=.11, lw=0) + axi.axvline(A_OUT * PIXSCALE, color='0.35', ls='-.', lw=1.2) +axs[0].axvspan(20, A_IN * PIXSCALE, color='k', alpha=.16, lw=0) +axs[1].text(A_GEO * 0.95, 0.275, 'no converged geometry inside here', fontsize=8.5, + ha='right', va='top', rotation=90, color='0.25') +axs[2].set_xscale('log') +axs[2].set_xlim(20, 1200) +axs[2].set_xlabel('semi-major axis a [arcsec]') +axs[0].grid(alpha=.25) +fig.savefig(path('NGC5128-sb-isophote-geometry.png'), dpi=150, bbox_inches='tight') +plt.close(fig) + +# ================================================================== plot 3 +fig, axs = plt.subplots(2, 1, figsize=(9, 8), sharex=True, + gridspec_kw=dict(hspace=.07, height_ratios=[2, 1])) +for ch, c in [('Luminance', 'k'), ('Red', 'tab:red'), ('Green', 'tab:green'), + ('Blue', 'tab:blue')]: + v = prof[ch] + m = np.isfinite(v) & (v > 0) + axs[0].plot(aas[m], mu(v[m]), color=c, lw=1.4, label=ch) +axs[0].set_ylim(27.5, 17.0) +axs[0].set_ylabel(r'$\mu$ [mag arcsec$^{-2}$]') +axs[0].text(.02, .04, 'the L zero point is applied to every channel, so the ' + 'vertical offsets between R, G and B are arbitrary;' + chr(10) + + 'only the shapes and the colour gradient below are meaningful', + transform=axs[0].transAxes, fontsize=8, color='0.3') +axs[0].legend(fontsize=9) +axs[0].grid(alpha=.25) +axs[0].set_title('NGC 5128 channel profiles on identical isophotes (dust masked)') +br = -2.5 * np.log10(np.where(prof['Blue'] > 0, prof['Blue'], np.nan) / + np.where(prof['Red'] > 0, prof['Red'], np.nan)) +axs[1].plot(aas, br, 'k-', lw=1.5) +axs[1].set_ylabel('instrumental B - R') +axs[1].set_xlabel('semi-major axis a [arcsec]') +axs[1].set_xscale('log') +axs[1].set_xlim(20, 700) +axs[1].grid(alpha=.25) +for axi in axs: + axi.axvspan(20, A_IN * PIXSCALE, color='k', alpha=.11, lw=0) + axi.axvline(A_OUT * PIXSCALE, color='0.35', ls='-.', lw=1.2) +axs[1].set_ylim(-0.45, 0.05) +fig.savefig(path('NGC5128-sb-colour-profile.png'), dpi=150, bbox_inches='tight') +plt.close(fig) + +np.savez(path('_profile.npz'), a=ac, **{k: prof[k] for k in prof}, + ped=PED, blkrms=BLKRMS, sersic=popt, a_in=A_IN, a_out=A_OUT) + +with open(path('sb-derived-quantities.txt'), 'w') as f: + w = lambda t: (f.write(t + '\n'), print(t)) + w('NGC 5128 (Centaurus A) -- derived quantities') + w('=' * 62) + w('Photometry is on the Gaia G scale of the luminance master.') + w(' zero point (r=5 px aperture) %.3f mag' % ZP5) + w(' aperture correction r=5 -> r=25 %+.4f mag' % APCOR) + w(' zero point for total flux %.3f mag' % ZPTOT) + w(' mu = %.3f - 2.5*log10(I / ADU per px)' % MU0) + w(' pixel scale %.4f arcsec, %.5f arcsec^2 per pixel' % (PIXSCALE, PIXAREA)) + w('') + w('Centre') + w(' Gaia/WCS nucleus x=%.1f y=%.1f' % (X0, Y0)) + w(' outer-isophote centre x=%.1f y=%.1f (%.1f px = %.1f arcsec offset)' + % (XC, YC, np.hypot(XC - X0, YC - Y0), np.hypot(XC - X0, YC - Y0) * PIXSCALE)) + w('') + w('Valid radial range') + w(' inner limit a = %.0f px = %.0f arcsec (dust lane fills the azimuth inside)' + % (A_IN, A_IN * PIXSCALE)) + w(' outer limit a = %.0f px = %.0f arcsec = %.1f arcmin' % + (A_OUT, A_OUT * PIXSCALE, A_OUT * PIXSCALE / 60)) + w(' the nucleus is NOT saturated: peak star-free galaxy signal 2498 ADU/px') + w(' vs a clip level of ~63000 ADU/px (factor 25 margin)') + w('') + w('Noise and systematic floors [mag/arcsec^2]') + for k, v in FLOOR.items(): + w(' %-40s %.2f' % (k, v)) + w(' far-field pedestal %+.2f ADU/px: the sky plane absorbed halo light' % PED) + w('') + w('Sersic fit, %.0f-%.0f arcsec' % (x.min(), x.max())) + w(' n = %.2f +- %.2f' % (nser, perr[2])) + w(' Re = %.1f +- %.1f arcsec = %.2f +- %.2f kpc' + % (re_as, perr[1], re_as * KPC_PER_ARCSEC, perr[1] * KPC_PER_ARCSEC)) + w(' mu_e = %.2f +- %.2f mag/arcsec^2' % (mue, perr[0])) + w(' rms of fit residual %.3f mag' % resid_sersic.std()) + w('') + w('Integrated light (elliptical apertures on the measured profile)') + for aa2 in [200, 400, 600, 800, 1000, 1200]: + i = np.argmin(np.abs(ac - aa2)) + w(' a < %5.0f px (%5.2f arcmin): G = %.3f' % (aa2, ac[i] * PIXSCALE / 60., mtot[i])) + w(' half-light radius of the light within a=%.0f px: %.0f arcsec (%.2f kpc)' + % (A_OUT, re_growth, re_growth * KPC_PER_ARCSEC)) + w(' fraction from the extrapolated a<%.0f px core: %.1f%%' % (A_IN, 100 * inner_frac)) + w('') + w('Ellipticity / position angle trend (converged isophotes only)') + for i in np.nonzero(conv)[0]: + w(' a = %6.1f px (%6.1f") : eps = %.3f +- %.3f PA = %5.1f +- %.1f deg' + % (tB['sma'][i], tB['sma'][i] * PIXSCALE, tB['eps'][i], + tB['eps_err'][i], tB['pa'][i], tB['pa_err'][i])) +print('\nwrote NGC5128-sb-profile.png, NGC5128-sb-isophote-geometry.png, NGC5128-sb-colour-profile.png, ' + 'sb-derived-quantities.txt') diff --git a/session-scripts/sb_render_tail.py b/session-scripts/sb_render_tail.py new file mode 100644 index 0000000..0137c0f --- /dev/null +++ b/session-scripts/sb_render_tail.py @@ -0,0 +1,163 @@ +"""Render section of sb_residual.py (imported and executed by it). + +Three views of the same residual, each answering a different question: + + (1) raw residual -- how well does the ellipse model fit? + (2) plane-removed residual -- restores the sky pedestal that the + stacking plane fit swallowed + (3) azimuthal-median-subtracted -- the shell-hunting view. Subtracting + the residual's own median as a + function of a forces zero mean at + every radius, so ONLY azimuthal + structure survives. Any perfectly + circular feature is removed with it. +""" +import numpy as np +from astropy.io import fits +import matplotlib.pyplot as plt +from scipy.ndimage import gaussian_filter, binary_erosion +from sb_common import * + + +def run(ns): + g = ns # namespace dict from sb_residual + res, model, a_map = g['res'], g['model'], g['a_map'] + L, star, dust = g['L'], g['star'], g['dust'] + XC, YC, PEDL = g['XC'], g['YC'], g['PEDL'] + A_OUT, tab, binned = g['A_OUT'], g['tab'], g['binned'] + + # -- plane removal (restores the sky pedestal absorbed by the stacking fit) + yy, xx = np.mgrid[0:3194, 0:4788] + fitreg = (a_map > 1450) & ~star & ~dust + A = np.c_[np.ones(fitreg.sum()), xx[fitreg].ravel()/1000., yy[fitreg].ravel()/1000.] + coef, *_ = np.linalg.lstsq(A, res[fitreg].ravel(), rcond=None) + plane = (coef[0] + coef[1]*xx/1000. + coef[2]*yy/1000.).astype(np.float32) + print('residual plane removed: %+.2f %+.2f*x/1000 %+.2f*y/1000 ADU/px' + % tuple(coef)) + resf = (res - plane).astype(np.float32) + fits.PrimaryHDU(resf).writeto(path('sb-residual-flat.fits'), overwrite=True) + del xx, yy, plane, A + + # -- binned maps + r4 = binned(resf, star, 4) + r16 = binned(resf, star, 16) + a16 = binned(a_map, np.zeros_like(star), 16) + d16 = binned(dust.astype(np.float32), np.zeros_like(star), 16) + + # -- azimuthal median removal + abin = np.geomspace(20, 3000, 80) + ibn = np.digitize(a16, abin) + azmed = np.full(len(abin)-1, np.nan) + for k in range(1, len(abin)): + m = (ibn == k) & np.isfinite(r16) & (d16 < 0.3) + if m.sum() >= 12: + azmed[k-1] = np.median(r16[m]) + acb = np.sqrt(abin[1:]*abin[:-1]) + gm = np.isfinite(azmed) + resd = r16 - np.interp(a16, acb[gm], azmed[gm]) + sig = float(np.nanstd(resd[np.isfinite(resd) & (a16 > 1500)])) + smd = gaussian_filter(np.nan_to_num(resd), 1.0) + smd[~np.isfinite(resd)] = np.nan + np.save(path('_resd16.npy'), resd) + np.save(path('_a16.npy'), a16) + np.save(path('_s16.npy'), np.array([sig])) + print('deep-residual noise (16x16 bins, a>1500 px): %.2f ADU/px -> mu %.2f' + % (sig, mu(sig))) + + NV, EV = north_east_pixel() + CUT = 1560 + sl = (slice(int(YC)-CUT, int(YC)+CUT), slice(int(XC)-CUT, int(XC)+CUT)) + ext = [-CUT*PIXSCALE/60, CUT*PIXSCALE/60]*2 + fullext = [-4788/2*PIXSCALE/60, 4788/2*PIXSCALE/60, + -3194/2*PIXSCALE/60, 3194/2*PIXSCALE/60] + + def compass(ax, x=0.885, y=0.115, Ln=0.07, c='k'): + for v, lab in [(NV, 'N'), (EV, 'E')]: + ax.annotate('', xy=(x+Ln*v[0], y+Ln*v[1]), xytext=(x, y), + xycoords='axes fraction', textcoords='axes fraction', + arrowprops=dict(arrowstyle='->', color=c, lw=1.4)) + ax.annotate(lab, xy=(x+1.45*Ln*v[0], y+1.45*Ln*v[1]), color=c, + xycoords='axes fraction', ha='center', va='center', + fontsize=10) + + def cutb(arr, B): + return arr[int((YC-CUT)/B):int((YC+CUT)/B), int((XC-CUT)/B):int((XC+CUT)/B)] + + # ---------------------------------------------------------- 4-panel figure + fig, axs = plt.subplots(2, 2, figsize=(15.5, 15.0)) + axs = axs.ravel() + axs[0].imshow(np.arcsinh(np.clip(L[sl]-PEDL, 0, None)/25), origin='lower', + cmap='gray', extent=ext) + axs[0].set_title('(a) luminance master, arcsinh stretch') + axs[1].imshow(np.arcsinh(np.clip(model[sl], 0, None)/25), origin='lower', + cmap='gray', extent=ext) + axs[1].set_title('(b) smooth elliptical model built from the isophotes') + im = axs[2].imshow(cutb(r4, 4), origin='lower', cmap='RdBu_r', vmin=-220, + vmax=220, extent=ext) + axs[2].set_title('(c) residual, 4x4 binned: the dust lane dominates') + plt.colorbar(im, ax=axs[2], fraction=.046, label='ADU/px') + im = axs[3].imshow(cutb(smd, 16), origin='lower', cmap='RdBu_r', + vmin=-3*sig, vmax=3*sig, extent=ext) + axs[3].contour(cutb(d16, 16), levels=[0.5], colors='0.35', linewidths=.8, + extent=ext, origin='lower') + axs[3].set_title('(d) residual, 16x16 binned, azimuthal median removed, ' + '+-3 sigma' + chr(10) + + '1 sigma = %.2f ADU/px = %.1f mag/arcsec2 ' + '(grey outline = dust mask)' % (sig, mu(sig))) + plt.colorbar(im, ax=axs[3], fraction=.046, label='ADU/px') + for a in axs: + a.set_xlabel('arcmin') + a.set_ylabel('arcmin') + compass(a, c='w' if a in (axs[0], axs[1]) else 'k') + fig.suptitle('NGC 5128: smooth elliptical model and its residual', fontsize=14) + fig.tight_layout() + fig.savefig(path('NGC5128-sb-model-residual.png'), dpi=115) + plt.close(fig) + + # ------------------------------------------------------- deep single panel + fig, ax = plt.subplots(figsize=(13.5, 9.6)) + im = ax.imshow(smd, origin='lower', cmap='RdBu_r', vmin=-3*sig, vmax=3*sig, + extent=fullext) + ax.contour(d16, levels=[0.5], colors='0.3', linewidths=.9, extent=fullext, + origin='lower') + th = np.linspace(0, 2*np.pi, 400) + ax.plot(1250*np.cos(th)*PIXSCALE/60, 1000*np.sin(th)*PIXSCALE/60, 'k--', + lw=1.1, alpha=.7, label='sky-plane fit exclusion ellipse (1250x1000 px)') + ax.plot(A_OUT*np.cos(th)*PIXSCALE/60, A_OUT*0.765*np.sin(th)*PIXSCALE/60, + '-', color='0.25', lw=1.1, alpha=.85, + label='profile reliability limit, a = %.0f px' % A_OUT) + ax.plot([], [], '-', color='0.3', lw=.9, label='dust-lane mask') + ax.legend(fontsize=9, loc='lower left', framealpha=.9) + plt.colorbar(im, ax=ax, fraction=.035, label='residual [ADU/px]') + ax.set_xlabel('arcmin') + ax.set_ylabel('arcmin') + compass(ax, x=0.945, y=0.84, Ln=0.05) + ax.set_title('NGC 5128: isophote model AND the residual azimuthal median ' + 'removed' + chr(10) + + '16x16 binned (8.6"/bin), stars masked, +-3 sigma; only ' + 'azimuthal structure survives. 1 sigma = %.2f ADU/px = ' + '%.1f mag/arcsec2' % (sig, mu(sig))) + fig.tight_layout() + fig.savefig(path('NGC5128-sb-residual-deep.png'), dpi=125) + plt.close(fig) + + # ------------------------------------------------------ quantify structure + print('') + print('azimuthal residual statistics (16x16 bins, azimuthal median removed,') + print('dust-lane bins excluded):') + ok = np.isfinite(resd) & (d16 < 0.3) + for lo, hi in [(100, 200), (200, 400), (400, 600), (600, 800), (800, 1000), + (1000, 1250), (1250, 1600), (1600, 2200)]: + m = ok & (a16 >= lo) & (a16 < hi) + if m.sum() < 20: + continue + md = np.interp(np.clip(a16[m], tab['sma'][0], tab['sma'][-1]), + tab['sma'], tab['intens']) + print(' a=%4d-%4d px (%4.1f-%4.1f arcmin): rms %6.2f ADU/px = %4.1f%% ' + 'of the model, max |dev| %4.1f sigma, n=%d' + % (lo, hi, lo*PIXSCALE/60, hi*PIXSCALE/60, np.nanstd(resd[m]), + 100*np.nanstd(resd[m])/np.mean(md), + np.nanmax(np.abs(resd[m]))/sig, m.sum())) + print('') + print('wrote sb-model.fits, sb-residual.fits, sb-residual-flat.fits,') + print(' NGC5128-sb-model-residual.png, NGC5128-sb-residual-deep.png') diff --git a/session-scripts/sb_residual.py b/session-scripts/sb_residual.py new file mode 100644 index 0000000..5f2f706 --- /dev/null +++ b/session-scripts/sb_residual.py @@ -0,0 +1,77 @@ +"""Step 4: smooth elliptical model, model-subtracted residual, and renders. + +The model is the Pass B isophote table (stars and dust lane masked) turned into +a 2-D image with sb_model.build, with a Sersic extrapolation inside a = 62 px +where no dust-free azimuth exists. Subtracting it leaves everything that is +not a smooth ellipse: the dust lane, foreground stars, and any shell, tidal +feature or halo asymmetry. + +Outputs + sb-model.fits the smooth model + sb-residual.fits luminance minus model + NGC5128-sb-model-residual.png 4-panel: data / model / residual / binned deep residual + NGC5128-sb-residual-deep.png heavily binned residual alone, for shell hunting +""" +import numpy as np +from astropy.io import fits +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt +from scipy.ndimage import gaussian_filter +from sb_common import * +import sb_model + +XC, YC = np.load(path('_geom.npy')) +P = np.load(path('_profile.npz')) +tB = dict(np.load(path('_isoB.npz'))) +star = fits.getdata(path('sb-mask-stars.fits')).astype(bool) +dust = fits.getdata(path('sb-mask-dust.fits')).astype(bool) +A_IN, A_OUT = float(P['a_in']), float(P['a_out']) +mue, re_as, nser = P['sersic'] + + +def sersic_mu(a_as): + bn = 2 * nser - 1 / 3. + 0.009876 / nser + return mue + 2.5 * bn / np.log(10) * ((a_as / re_as) ** (1. / nser) - 1.) + + +# ---- build a profile that is defined at every radius ----------------------- +ac = P['a'] +Lp = P['Luminance'].copy() +inner = ac < A_IN +Lp[inner] = 10 ** ((MU0 - sersic_mu(ac[inner] * PIXSCALE)) / 2.5) +okp = np.isfinite(Lp) & (ac < 1500) +tab = dict(sma=ac[okp], intens=Lp[okp], + eps=np.interp(ac[okp], tB['sma'], np.where(np.isfinite(tB['eps']), + tB['eps'], 0.15)), + pa=np.interp(ac[okp], tB['sma'], np.where(np.isfinite(tB['pa']), + tB['pa'], 150.))) +model, a_map = sb_model.build((3194, 4788), XC, YC, tab, block=2) +fits.PrimaryHDU(model).writeto(path('sb-model.fits'), overwrite=True) + +PEDL = float(P['ped']) +L = load('Luminance') +res = (L - model).astype(np.float32) +fits.PrimaryHDU(res).writeto(path('sb-residual.fits'), overwrite=True) +print('model built; residual rms inside a<600 px: %.2f ADU/px' + % res[(a_map < 600) & ~star & ~dust].std()) + + +def binned(img, mask, B): + """Masked block mean, returning NaN where a block is mostly masked.""" + H, W = img.shape + h, w = H // B, W // B + a = img[:h * B, :w * B].reshape(h, B, w, B) + m = (~mask)[:h * B, :w * B].reshape(h, B, w, B) + n = m.sum(axis=(1, 3)) + s = np.where(m, a, 0).sum(axis=(1, 3)) + return np.where(n > 0.35 * B * B, s / np.maximum(n, 1), np.nan) + + + + +# ------------------------------------------------------------------ renders +import sb_render_tail +sb_render_tail.run(dict(res=res, model=model, a_map=a_map, L=L, star=star, + dust=dust, XC=XC, YC=YC, PEDL=PEDL, A_OUT=A_OUT, + tab=tab, binned=binned)) diff --git a/session-scripts/solve.py b/session-scripts/solve.py new file mode 100644 index 0000000..a91e452 --- /dev/null +++ b/session-scripts/solve.py @@ -0,0 +1,185 @@ +"""Pass 3: plate solve the luminance master and copy the WCS to every master. + +iTelescope's calibrated frames arrive with a PinPoint HISTORY line but no WCS +keywords at all, so the astrometry has to be redone locally. A blind solve is +not needed: the header gives the pointing to arcminutes and the plate scale to +four figures, so this fetches a Gaia DR3 catalogue for that patch of sky and +matches it to the detected stars. + +The match itself is asterism-based (astroalign), which is invariant to rotation +and scale, so the roll angle never has to be guessed. It is NOT invariant to a +mirror flip, so both parities are tried and the one that matches wins. The +final WCS is a least-squares TAN fit to the matched pairs, and the residual it +reports is the honest measure of whether the solve is real. +""" +import os +import sys + +import astroalign as aa +import numpy as np +import sep +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.wcs import WCS +from astropy.wcs.utils import fit_wcs_from_points + +import layout + +SRC = layout.SESSION +OUT = layout.SESSION +MASTERS = ["Luminance", "Red", "Green", "Blue"] +CATCACHE = layout.path("_gaia.npz") + + +def detect(image, nmax=600): + bkg = sep.Background(image, bw=64, bh=64, fw=3, fh=3) + sub = image - bkg.back() + objs = sep.extract(sub, 8.0, err=bkg.globalrms, minarea=9, + deblend_cont=0.005) + objs = objs[(objs["flag"] == 0) & (objs["npix"] > 12) & + (objs["npix"] < 3000)] + objs = objs[np.argsort(objs["flux"])[::-1][:nmax]] + return np.column_stack([objs["x"], objs["y"]]), objs["flux"] + + +def gaia_catalogue(ra_deg, dec_deg, radius_deg, nmax=600): + """Gaia DR3 sources around the pointing, brightest first, cached to disk.""" + if os.path.exists(CATCACHE): + z = np.load(CATCACHE) + print(f"catalogue: {len(z['ra'])} cached Gaia sources") + return z["ra"], z["dec"], z["g"] + from astroquery.gaia import Gaia + + Gaia.ROW_LIMIT = nmax + query = f""" + SELECT TOP {nmax} ra, dec, phot_g_mean_mag + FROM gaiadr3.gaia_source + WHERE 1 = CONTAINS(POINT('ICRS', ra, dec), + CIRCLE('ICRS', {ra_deg}, {dec_deg}, {radius_deg})) + AND phot_g_mean_mag IS NOT NULL + ORDER BY phot_g_mean_mag ASC + """ + tbl = Gaia.launch_job_async(query).get_results() + ra = np.asarray(tbl["ra"], dtype=float) + dec = np.asarray(tbl["dec"], dtype=float) + g = np.asarray(tbl["phot_g_mean_mag"], dtype=float) + np.savez_compressed(CATCACHE, ra=ra, dec=dec, g=g) + print(f"catalogue: {len(ra)} Gaia DR3 sources, G {g.min():.1f}-{g.max():.1f}") + return ra, dec, g + + +def project(ra, dec, ra0, dec0, scale_arcsec, parity): + """Gnomonic projection to pixel-like coordinates for asterism matching.""" + c = SkyCoord(ra * u.deg, dec * u.deg) + centre = SkyCoord(ra0 * u.deg, dec0 * u.deg) + dx, dy = centre.spherical_offsets_to(c) + x = dx.to_value(u.arcsec) / scale_arcsec * parity + y = dy.to_value(u.arcsec) / scale_arcsec + return np.column_stack([x, y]) + + +def main(): + path = layout.path("master-Luminance.fit") + with fits.open(path) as hd: + image = hd[0].data.astype(np.float32) + hdr = hd[0].header + ny, nx = image.shape + + centre = SkyCoord(hdr["OBJCTRA"], hdr["OBJCTDEC"], + unit=(u.hourangle, u.deg)) + scale = float(hdr["HIERARCH iTelescopePlateScaleH"]) + radius = 1.15 * 0.5 * np.hypot(nx, ny) * scale / 3600.0 + print(f"pointing {centre.to_string('hmsdms')} scale {scale:.4f}\"/px " + f"search radius {radius:.3f} deg") + + xy, flux = detect(image) + print(f"detected {len(xy)} stars in the luminance master") + + ra, dec, gmag = gaia_catalogue(centre.ra.deg, centre.dec.deg, radius) + + best = None + for parity in (-1.0, 1.0): + cat_xy = project(ra, dec, centre.ra.deg, centre.dec.deg, scale, parity) + try: + tform, (src, dst) = aa.find_transform(cat_xy, xy) + except Exception as exc: # noqa: BLE001 + print(f" parity {parity:+.0f}: no match ({exc})") + continue + print(f" parity {parity:+.0f}: matched {len(src)} stars, " + f"rotation {np.degrees(tform.rotation):.3f} deg, " + f"scale {tform.scale:.5f}") + if best is None or len(src) > best[0]: + best = (len(src), parity, src, dst) + if best is None: + sys.exit("plate solve failed: no asterism match in either parity") + + nmatch, parity, cat_pts, img_pts = best + # Recover which catalogue rows were matched so the fit uses sky coordinates + # rather than the projected proxy. + cat_xy = project(ra, dec, centre.ra.deg, centre.dec.deg, scale, parity) + idx = [int(np.argmin(np.hypot(cat_xy[:, 0] - px, cat_xy[:, 1] - py))) + for px, py in cat_pts] + world = SkyCoord(ra[idx] * u.deg, dec[idx] * u.deg) + + wcs = fit_wcs_from_points((img_pts[:, 0], img_pts[:, 1]), world, + proj_point="center", projection="TAN") + pred = wcs.world_to_pixel(world) + resid = np.hypot(pred[0] - img_pts[:, 0], pred[1] - img_pts[:, 1]) + print(f"seed fit on {nmatch} stars: residual median {np.median(resid):.2f} px " + f"({np.median(resid) * scale:.2f}\"), max {resid.max():.2f} px") + + # The asterism match only ever returns a handful of stars. Now that an + # approximate solution exists, every catalogue source can be pushed through + # it and paired with the nearest detection, which grows the fit from a + # dozen stars to hundreds and averages down the centroid noise. Two passes + # with a shrinking tolerance is enough to converge. + all_world = SkyCoord(ra * u.deg, dec * u.deg) + for tol in (4.0, 2.0): + px, py = wcs.world_to_pixel(all_world) + pairs = [] + for i, (cx, cy) in enumerate(zip(px, py)): + if not (0 <= cx < nx and 0 <= cy < ny): + continue + d = np.hypot(xy[:, 0] - cx, xy[:, 1] - cy) + j = int(np.argmin(d)) + if d[j] <= tol: + pairs.append((i, j)) + if len(pairs) < 20: + print(f" refine (tol {tol} px): only {len(pairs)} pairs, kept seed") + break + ci = np.array([p[0] for p in pairs]) + ii = np.array([p[1] for p in pairs]) + wcs = fit_wcs_from_points((xy[ii, 0], xy[ii, 1]), all_world[ci], + proj_point="center", projection="TAN") + qx, qy = wcs.world_to_pixel(all_world[ci]) + resid = np.hypot(qx - xy[ii, 0], qy - xy[ii, 1]) + nmatch = len(pairs) + print(f" refine (tol {tol} px): {nmatch} stars, residual median " + f"{np.median(resid):.2f} px ({np.median(resid) * scale:.2f}\"), " + f"max {resid.max():.2f} px") + + cen = wcs.pixel_to_world(nx / 2.0, ny / 2.0) + cd = wcs.pixel_scale_matrix * 3600.0 + solved_scale = np.sqrt(abs(np.linalg.det(cd))) + rot = np.degrees(np.arctan2(cd[0, 1], cd[1, 1])) + print(f"field centre {cen.to_string('hmsdms')}") + print(f"solved scale {solved_scale:.4f}\"/px, position angle {rot:.2f} deg") + print(f"field of view {nx * solved_scale / 60:.1f}' x " + f"{ny * solved_scale / 60:.1f}'") + + whdr = wcs.to_header() + for name in MASTERS: + p = layout.path(f"master-{name}.fit") + with fits.open(p, mode="update") as hd: + for card in whdr.cards: + hd[0].header[card.keyword] = (card.value, card.comment) + hd[0].header["ASTRSOLV"] = ( + f"Gaia DR3 / {nmatch} stars / {np.median(resid) * scale:.2f} arcsec", + "local plate solution") + hd.flush() + print(f" WCS written to {os.path.basename(p)}") + + +if __name__ == "__main__": + main() diff --git a/session-scripts/stack.py b/session-scripts/stack.py new file mode 100644 index 0000000..89c055a --- /dev/null +++ b/session-scripts/stack.py @@ -0,0 +1,166 @@ +"""Pass 2: register every frame to one reference and combine per filter. + +One reference frame is used for ALL four filters, not one per filter, so the +four masters come out pixel-aligned and the colour composite needs no further +registration. + +Registration works on the cached star lists rather than the pixels: astroalign +matches asterisms between the two point sets and returns a similarity transform, +which is then applied to the image with a bicubic warp. Matching a few hundred +coordinates is far cheaper than cross-correlating 15 Mpx frames, and it copes +with the field rotation between the east and west sides of the meridian. + +Before combining, each frame is sky-subtracted and then rescaled so that its +bright-signal level (the 99.5th percentile, which on this field is set by stars +and the galaxy core rather than by sky) matches the group median. That corrects +for transparency changes without needing per-star photometry. A +sigma-clipped mean then rejects cosmic rays and satellite trails; with only four +frames per colour the clip is deliberately gentle (3 sigma, one iteration) so it +does not start eating real signal. +""" +import os + +import astroalign as aa +import numpy as np +from astropy.io import fits +from astropy.stats import sigma_clip +from skimage.transform import warp + +import layout + +SRC = layout.SESSION +OUT = layout.SESSION +CACHE = layout.path("_stars.npz") + +# Best-seeing luminance frame, chosen from the pass-1 FWHM table. +REFERENCE = "Luminance_002" +FILTERS = ["Luminance", "Red", "Green", "Blue"] + +aa.MIN_MATCHES_FRACTION = 0.6 +aa.NUM_NEAREST_NEIGHBORS = 8 + + +def load_cache(): + z = np.load(CACHE, allow_pickle=True) + meta = {row[0]: row for row in z["meta"]} + stars = {k: z[k + "_xy"] for k in meta} + return meta, stars, z + + +def frame_path(fname): + return layout.path(fname) + + +def combine(cube, weights): + """Weighted sigma-clipped mean along axis 0, done row-block by row-block. + + The full cube is already in memory; the blocking here is only to keep the + boolean mask and the float64 intermediates that sigma_clip allocates from + tripling peak usage on a machine with a few GB free. + """ + n, ny, nx = cube.shape + out = np.zeros((ny, nx), dtype=np.float32) + w = np.asarray(weights, dtype=np.float32)[:, None, None] + step = 256 + for y0 in range(0, ny, step): + y1 = min(y0 + step, ny) + block = cube[:, y0:y1, :] + clipped = sigma_clip(block, sigma=3.0, maxiters=1, axis=0, + masked=True, copy=True) + # Warped frames carry NaN outside their footprint; those pixels must + # drop out of both the sum and the weight total, exactly like a + # clipped outlier. + finite = np.isfinite(block) + good = (~clipped.mask) & finite + vals = np.where(finite, block, 0.0) + wb = np.broadcast_to(w, block.shape) * good + denom = wb.sum(axis=0) + denom[denom == 0] = np.nan + out[y0:y1, :] = np.nansum(vals * wb, axis=0) / denom + del block, clipped, good, finite, vals, wb, denom + return np.nan_to_num(out, nan=0.0) + + +def main(): + meta, stars, _ = load_cache() + ref_xy = stars[REFERENCE] + ref_row = meta[REFERENCE] + print(f"reference {REFERENCE} ({ref_row[1]})") + os.makedirs(OUT, exist_ok=True) + + report = [] + for filt in FILTERS: + keys = sorted(k for k in meta if meta[k][2] == filt) + planes, weights, headers, used = [], [], [], [] + for key in keys: + row = meta[key] + fname, rms = row[1], float(row[7]) + with fits.open(frame_path(fname), memmap=False) as hd: + data = hd[0].data.astype(np.float32) + hdr = hd[0].header + if key == REFERENCE: + reg = data + nmatch = len(ref_xy) + else: + try: + tform, (src_m, tgt_m) = aa.find_transform(stars[key], + ref_xy) + except Exception as exc: # noqa: BLE001 + print(f" {key}: registration FAILED ({exc}) - dropped") + del data + continue + nmatch = len(src_m) + reg = warp(data, inverse_map=tform.inverse, order=3, + mode="constant", cval=np.nan, + preserve_range=True).astype(np.float32) + del data + # Sky subtraction uses the frame's own median, which on a field + # this empty outside the galaxy is a fair estimate of sky level. + sky = float(np.nanmedian(reg)) + reg -= sky + planes.append(reg) + weights.append(1.0 / (rms * rms)) + headers.append(hdr) + used.append((key, nmatch, sky, rms)) + print(f" {key:16s} matched={nmatch:3d} sky={sky:8.1f} " + f"shift-corrected") + + cube = np.stack(planes, axis=0) + del planes + # Photometric scaling: normalise each frame to the stack's own median + # signal so a frame taken through thin cloud cannot drag the mean down. + levels = np.array([np.nanpercentile(p, 99.5) for p in cube]) + ref_level = np.nanmedian(levels) + for i, lv in enumerate(levels): + if lv > 0: + cube[i] *= float(ref_level / lv) + print(f" {filt}: photometric scale factors " + f"{np.round(ref_level / levels, 4)}") + + master = combine(cube, weights) + nframes = cube.shape[0] + del cube + + hdr = headers[0].copy() + for k in ("CBLACK", "CWHITE", "PEDESTAL", "HISTORY"): + hdr.remove(k, ignore_missing=True, remove_all=True) + hdr["FILTER"] = filt + hdr["NCOMBINE"] = (nframes, "frames in this master") + hdr["EXPTOTAL"] = (300.0 * nframes, "[s] total integration") + hdr["STACKREF"] = (REFERENCE, "registration reference frame") + hdr["STACKALG"] = ("sigma-clipped weighted mean", "combine method") + hdr["IMAGETYP"] = "Master Light" + out = layout.path(f"master-{filt}.fit") + fits.PrimaryHDU(master.astype(np.float32), hdr).writeto(out, + overwrite=True) + print(f" -> {out} ({nframes} x 300 s = {nframes * 5:.0f} min)") + report.append((filt, nframes, out)) + del master + + print("\nmasters written:") + for filt, n, path in report: + print(f" {filt:10s} {n:2d} frames {path}") + + +if __name__ == "__main__": + main() diff --git a/session-scripts/starless.py b/session-scripts/starless.py new file mode 100644 index 0000000..5cfb103 --- /dev/null +++ b/session-scripts/starless.py @@ -0,0 +1,91 @@ +"""Split the finished image into a starless view and a star-only layer. + +The first attempt used a greyscale morphological opening, the classical trick, +and it failed in a way worth recording: an opening replaces each pixel with a +local minimum, so on a noisy sky it does not merely delete the star, it digs a +hole slightly BELOW sky level and leaves a black dot behind. It also left the +brightest stars standing, because their wings are wider than any structuring +element small enough to spare the galaxy. + +What works instead is catalogue-and-fill. The stars to remove are taken from +Gaia DR3 rather than from a blind detection: Gaia is essentially complete for +stars down to G = 20, which is fainter than anything visible here, so the +catalogue identifies the foreground almost perfectly. That matters because a +blind detector cannot tell a Milky Way star from one of Centaurus A's own +globular clusters or from the blue knots in its dust lane, and removing those +guts the very structure the image is about. Each Gaia star is painted into a +mask whose radius follows its magnitude, and the masked pixels are +replaced by a normalised convolution - a Gaussian blur of the unmasked pixels +divided by the same blur of the mask itself, which interpolates across each +star using only real neighbouring pixels. That follows the galaxy's gradient +instead of flattening it, and leaves no holes. + +The star layer is then simply what was removed. Two honest limitations: the +extended halos and diffraction spikes of the very brightest stars reach beyond +any sensible mask radius and leave soft residual glows behind, and anything +Gaia does not list (background galaxies, the cluster system) stays in the +starless frame by design. This is a presentation tool, not a measurement. +""" +import os + +import numpy as np +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.wcs import WCS +from PIL import Image +from scipy.ndimage import gaussian_filter + +import layout + +OUT = layout.SESSION +SOURCE = "NGC5128-img-deconvolved.png" +FILL_SIGMA = 9.0 # interpolation scale for the normalised convolution + +rgb = np.asarray(Image.open(layout.path(SOURCE))).astype(np.float32) / 255 +ny, nx = rgb.shape[:2] +print(f"{SOURCE}: {nx} x {ny}") + +z = np.load(layout.path("_gaia_deep.npz")) +with fits.open(layout.path("NGC5128-LRGB.fit")) as hd: + wcs = WCS(hd[0].header, naxis=2) +gx, gy = wcs.world_to_pixel(SkyCoord(z["ra"] * u.deg, z["dec"] * u.deg)) +gmag = z["g"] +on_frame = (gx > -30) & (gx < nx + 30) & (gy > -30) & (gy < ny + 30) +gx, gy, gmag = gx[on_frame], gy[on_frame], gmag[on_frame] +print(f"{len(gx)} Gaia stars on the frame, G {gmag.min():.1f}-{gmag.max():.1f}") + +mask = np.zeros((ny, nx), np.float32) +R = 30 +yy, xx = np.mgrid[-R:R + 1, -R:R + 1] +rr = np.hypot(xx, yy) +for x, y, g in zip(gx, gy, gmag): + # Radius from magnitude: a bright star's visible disc and wings are far + # wider than a faint one's, and a fixed radius would either miss the + # bright ones or scour the field around the faint ones. + r = float(np.clip(30.0 - 1.9 * (g - 8.0), 4, R - 2)) + cx, cy = int(round(x)), int(round(y)) + x0, x1 = max(0, cx - R), min(nx, cx + R + 1) + y0, y1 = max(0, cy - R), min(ny, cy + R + 1) + patch = (rr <= r).astype(np.float32)[(y0 - cy + R):(y1 - cy + R), + (x0 - cx + R):(x1 - cx + R)] + np.maximum(mask[y0:y1, x0:x1], patch, out=mask[y0:y1, x0:x1]) +mask = np.clip(gaussian_filter(mask, 1.5), 0, 1) +print(f"mask covers {mask.mean():.2%} of the frame") + +starless = np.empty_like(rgb) +keep = 1.0 - mask +denom = gaussian_filter(keep, FILL_SIGMA) +denom[denom < 1e-3] = 1e-3 +for i in range(3): + filled = gaussian_filter(rgb[:, :, i] * keep, FILL_SIGMA) / denom + starless[:, :, i] = rgb[:, :, i] * keep + filled * mask +stars = np.clip(rgb - starless, 0.0, 1.0) + +Image.fromarray((np.clip(starless, 0, 1) * 255 + 0.5).astype(np.uint8)).save( + layout.path("NGC5128-img-starless.png")) +# The star layer is faint on its own; a square-root stretch makes it legible +# without adding or removing anything. +Image.fromarray((np.sqrt(stars) * 255 + 0.5).astype(np.uint8)).save( + layout.path("NGC5128-img-stars.png")) +print("wrote NGC5128-img-starless.png and NGC5128-img-stars.png") diff --git a/session-scripts/triptych.py b/session-scripts/triptych.py new file mode 100644 index 0000000..4a43765 --- /dev/null +++ b/session-scripts/triptych.py @@ -0,0 +1,62 @@ +"""Side-by-side of the three renderings, full field and a matched core crop. + +Same pixels in all three panels, so any difference is processing and nothing +else. +""" +import os + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +from PIL import Image + +import layout + +OUT = layout.SESSION +# The finished renderings live in their own directory: they are the +# deliverables, and keeping them apart from the masters, the +# intermediates and the analysis figures makes it obvious which +# files are meant to be looked at. +FINAL = layout.path("final") +PANELS = [ + ("NGC5128-final-1-stacked.png", "1. Stacked + standard stretch", + "align, average, stretch - nothing else"), + ("NGC5128-final-2-processed.png", "2. Conventionally processed", + "gradient removal, colour balance, denoise"), + ("NGC5128-final-3-best.png", "3. Science-informed", + "halo-preserving background, solar-analogue colour,\n" + "core recovered, star-protected deconvolution"), +] +CROP = (1950, 1200, 2850, 1875) # the dust lane, at full resolution + +fig, axes = plt.subplots(2, 3, figsize=(21, 11.5), + gridspec_kw=dict(height_ratios=[1.35, 1])) +fig.patch.set_facecolor("#111111") + +for col, (fname, title, sub) in enumerate(PANELS): + im = Image.open(layout.path(fname)) + wide = im.copy() + wide.thumbnail((1500, 1500), Image.LANCZOS) + axes[0, col].imshow(np.asarray(wide)) + axes[0, col].set_title(title, color="white", fontsize=15, pad=10) + axes[0, col].text(0.5, -0.055, sub, color="#9fb8d0", fontsize=10, + ha="center", va="top", linespacing=1.4, wrap=True, + transform=axes[0, col].transAxes) + axes[1, col].imshow(np.asarray(im.crop(CROP))) + axes[1, col].set_title("core, 1:1", color="#9fb8d0", fontsize=11, pad=6) + for row in (0, 1): + axes[row, col].set_xticks([]) + axes[row, col].set_yticks([]) + for s in axes[row, col].spines.values(): + s.set_color("#333333") + +fig.suptitle("NGC 5128 (Centaurus A) - iTelescope T32, 2026-07-21 - " + "L 12x300s, RGB 4x300s each - the same data, three treatments", + color="white", fontsize=17, y=0.975) +fig.tight_layout(rect=[0, 0.01, 1, 0.955]) +fig.subplots_adjust(hspace=0.16) +path = layout.path("NGC5128-final-comparison.jpg") +fig.savefig(path, dpi=100, facecolor=fig.get_facecolor(), + pil_kwargs={"quality": 92}) +print("wrote", path) diff --git a/session-scripts/unzip.py b/session-scripts/unzip.py new file mode 100644 index 0000000..3859aa6 --- /dev/null +++ b/session-scripts/unzip.py @@ -0,0 +1,33 @@ +"""Uncompress every calibrated-*.zip in the session folder, in place. + +iTelescope ships each calibrated frame as a one-entry zip (.fit.zip and +.tif.zip). Extraction is skipped when the target already exists and is +non-empty, so this is safe to re-run. +""" +import glob +import os +import zipfile + +import layout + +SRC = layout.SESSION + +zips = sorted(glob.glob(layout.path("calibrated-*.zip"))) +print(f"{len(zips)} calibrated archives") +done = skipped = 0 +for z in zips: + with zipfile.ZipFile(z) as zf: + for info in zf.infolist(): + target = layout.path(os.path.basename(info.filename)) + if os.path.exists(target) and os.path.getsize(target) == info.file_size: + skipped += 1 + continue + with zf.open(info) as fsrc, open(target, "wb") as fdst: + while chunk := fsrc.read(1 << 20): + fdst.write(chunk) + done += 1 +print(f"extracted {done}, already present {skipped}") + +for ext in (".fit", ".tif"): + n = len(glob.glob(layout.path(f"calibrated-*{ext}"))) + print(f"{ext}: {n} files") diff --git a/session-scripts/verify_core.py b/session-scripts/verify_core.py new file mode 100644 index 0000000..750853c --- /dev/null +++ b/session-scripts/verify_core.py @@ -0,0 +1,74 @@ +"""Is the nucleus of NGC 5128 actually saturated, or was that a foreground star? + +The earlier claim - that the core clips at 65313 ADU in a single 300 s sub - +came from taking the maximum inside a 300x300 px box centred on the frame. That +box is wide enough to contain a bright foreground star, so the measurement +proves only that SOMETHING in the middle of the frame is bright. This checks +where the bright pixels actually are, and what the galaxy itself peaks at once +stars are filtered out. +""" +import numpy as np +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.wcs import WCS +from scipy.ndimage import median_filter + +import layout + +SUB = layout.path("calibrated-T32-qisback-NGC5128-20260721-190133" + "-Luminance-BIN2-W-300-002.fit") +MASTER = layout.path("master-Luminance.fit") + +# NGC 5128's nucleus, from SIMBAD, not from "the middle of the frame". +NUCLEUS = SkyCoord("13h25m27.6s", "-43d01m08.8s") + +with fits.open(MASTER) as hd: + wcs = WCS(hd[0].header, naxis=2) +nx_c, ny_c = wcs.world_to_pixel(NUCLEUS) +print(f"nucleus lands at master pixel ({nx_c:.1f}, {ny_c:.1f})") + +data = fits.getdata(SUB).astype(np.float32) +ny, nx = data.shape +print(f"single sub {nx} x {ny}, global max {data.max():.0f} ADU") + +# Where are the saturated-ish pixels? +ys, xs = np.where(data > 60000) +print(f"{len(xs)} pixels above 60000 ADU") +if len(xs): + # Cluster them crudely by proximity to see how many distinct objects. + print(f" x range {xs.min()}-{xs.max()}, y range {ys.min()}-{ys.max()}") + cx, cy = nx / 2.0, ny / 2.0 + d = np.hypot(xs - cx, ys - cy) + print(f" distance from frame centre: min {d.min():.0f} px, " + f"median {np.median(d):.0f} px, max {d.max():.0f} px") + # The brightest pixel specifically + iy, ix = np.unravel_index(np.argmax(data), data.shape) + print(f" brightest pixel at ({ix}, {iy}), " + f"{np.hypot(ix - cx, iy - cy):.0f} px from frame centre") + +# The galaxy's own peak: median filter removes stars, which are small, while +# leaving the smooth galaxy light essentially untouched. +h = 400 +y0, y1 = int(ny / 2) - h, int(ny / 2) + h +x0, x1 = int(nx / 2) - h, int(nx / 2) + h +core = data[y0:y1, x0:x1] +smooth = median_filter(core, size=15) +print(f"\ninner {2*h}x{2*h} px box:") +print(f" raw max {core.max():9.1f} ADU") +print(f" median-filtered max {smooth.max():9.1f} ADU <- galaxy light") +iy, ix = np.unravel_index(np.argmax(smooth), smooth.shape) +print(f" galaxy peak at frame pixel ({x0+ix}, {y0+iy})") + +# How many pixels of the median-filtered (star-free) galaxy are near clipping? +for lvl in (30000, 50000, 60000): + print(f" star-free pixels above {lvl}: {(smooth > lvl).sum()}") + +# And in the master stack. +mdata = fits.getdata(MASTER).astype(np.float32) +mcore = mdata[y0:y1, x0:x1] +msmooth = median_filter(mcore, size=15) +print(f"\nmaster stack inner box: raw max {mcore.max():.1f}, " + f"star-free max {msmooth.max():.1f} ADU") +print(f" master 99.995th percentile (the stretch white point) " + f"{np.percentile(mdata, 99.995):.1f} ADU") From f2d173f0470218efc8d6a3f65ce778f928aa841d Mon Sep 17 00:00:00 2001 From: laurence Date: Tue, 21 Jul 2026 15:33:52 +0100 Subject: [PATCH 2/4] Fix directory resolution in layout, and generate the session READMEs layout.path() treated every argument as a filename, so asking it for a directory - layout.path('final'), which the rendering scripts do - built a path to a file called 'final' INSIDE final/ and created an empty final/final along the way. Directory names now resolve to the directory itself. Three scripts also held path literals split across two lines with implicit string concatenation, where the rewrite had replaced only the first fragment and left a dangling continuation. verify_core.py, mo_gccheck.py and mo_mpc.py are corrected; all 52 files now compile without warnings. subdir_readmes.py writes a short README into each session subdirectory saying what it holds, where it came from, and whether it is safe to delete. Those three facts are what someone opening a folder cold needs, and they belong with the data rather than in this repo. Re-verified after the fix: closeup.py and triptych.py run end to end against the reorganised session and leave no stray directories behind. --- session-scripts/layout.py | 7 ++ session-scripts/subdir_readmes.py | 117 ++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 session-scripts/subdir_readmes.py diff --git a/session-scripts/layout.py b/session-scripts/layout.py index 7577220..99bdeb9 100644 --- a/session-scripts/layout.py +++ b/session-scripts/layout.py @@ -85,6 +85,13 @@ def path(*parts, make=True): correctly. The last component is the one that decides. """ name = parts[-1] + if name in _DIRS: + # Asking for a directory by name returns the directory itself, not a + # file of that name nested inside it. + full = os.path.join(SESSION, _DIRS[name]) + if make: + os.makedirs(full, exist_ok=True) + return full full = os.path.join(SESSION, subdir(name), name) if make: os.makedirs(os.path.dirname(full), exist_ok=True) diff --git a/session-scripts/subdir_readmes.py b/session-scripts/subdir_readmes.py new file mode 100644 index 0000000..08ab53d --- /dev/null +++ b/session-scripts/subdir_readmes.py @@ -0,0 +1,117 @@ +"""Drop a short README in each session subdirectory. + +Each one says what the directory holds, where it came from, and whether it is +safe to delete - the three things someone opening a folder cold needs. +""" +import os + +ROOT = r"C:\Users\lhorrocks-barlow\Downloads\NGC5128\20260721" + +READMES = { + "raw": """# raw/ + +Exactly what iTelescope delivered, untouched. + +- `calibrated-*.zip` - the calibrated frames, as archives (FITS and TIFF) +- `raw-*.zip` - the uncalibrated frames, kept for completeness +- `jpeg-*.jpg` - iTelescope's own preview of each sub + +**Do not edit anything here.** This is the archive copy; everything else in the +session can be rebuilt from it. `calibrated/` holds these same frames +uncompressed. +""", + "calibrated": """# calibrated/ + +The 24 calibrated subs, uncompressed from `raw/`. 12 Luminance, 4 Red, +4 Green, 4 Blue, all 300 s, all BIN2. + +`.fit` are the data the processing uses; `.tif` are the same frames in a form +other software can open. + +Already bias, dark and flat corrected by iTelescope before delivery +(`CALSTAT = 'BDF'` in the headers) - that cannot be undone here. + +Safe to delete: regenerable from `raw/` by re-running `unzip.py`. +""", + "stacks": """# stacks/ + +The combined frames, at two levels of processing. + +- `masters/` - the real stacks: registered, transparency-normalised, combined + with outlier rejection, and plate-solved. Everything downstream uses these. +- `original/` - the baseline: aligned and averaged, and **nothing else**. No + rejection, no sky subtraction, no normalisation. Satellite trails and the + moon's gradient are still in it, deliberately. Useful when a processed result + looks odd and you need to know whether the data or the processing caused it. + +Both sets are pixel-aligned to the same reference frame, so they can be +compared or subtracted directly. +""", + "final": """# final/ + +**The finished images.** Four renderings of the same data, differing only in +how much processing was applied, plus a side-by-side comparison. + +| File | What it is | +|---|---| +| `NGC5128-final-1-stacked` | stacked and stretched, nothing else | +| `NGC5128-final-2-processed` | conventional processing | +| `NGC5128-final-3-best` | corrected by what the analyses established | +| `NGC5128-final-4-closeup` | image 3 cropped to the galaxy, 23.4' square | +| `NGC5128-final-comparison.jpg` | all three side by side, with a 1:1 crop | + +Each comes as `.png` (viewing), `.tif` (16-bit, for editing or printing) and +`-preview.jpg` (small). + +The reasoning behind each is in `../METHODS.md`. +""", + "renderings": """# renderings/ + +Finished images that are not the four deliverables in `final/`. + +- `NGC5128-LRGB.*` - the first full composite +- `NGC5128-LRGB-hdr.*` - the same with the core recovered by a second tone curve +- `NGC5128-img-deconvolved.*` - sharpened luminance +- `NGC5128-img-annotated.jpg` - coordinate grid and catalogued objects, placed + by the plate solution. Includes SN 1986G and SN 2016adj, both real supernovae + in this galaxy. +- `NGC5128-img-starless.png` / `NGC5128-img-stars.png` - the field split into + nebulosity and stars +- `NGC5128-img-core-print.jpg` - a print crop + +Kept because they show intermediate stages and because some are useful in their +own right. `final/` is what to look at first. +""", + "science": """# science/ + +The measurements, as opposed to the pictures. + +- `notes/` - **start here.** Three write-ups covering the globular cluster + survey, the surface photometry, and the transient and moving-object search. + Each states its method, its numbers, and its limits. +- `figures/` - the plots those notes refer to +- `catalogues/` - the measured tables (CSV): cluster candidates, isophote + profiles, flagged objects +- `data/` - the galaxy model, star and dust masks, extinction map, and a plain + text summary of derived quantities + +Headline results are summarised in `../METHODS.md`; the notes carry the detail, +the caveats and the things that did not work. +""", + "intermediates": """# intermediates/ + +Caches and scratch files: downloaded star catalogues, detection lists, partial +results, run logs. + +**Safe to delete.** Everything here is regenerated by re-running the scripts, +at the cost of re-querying Gaia, SIMBAD and VizieR. Kept so a re-run is fast +and so the exact catalogue data used is preserved rather than silently changing +under a later query. +""", +} + +for name, text in READMES.items(): + path = os.path.join(ROOT, name, "README.md") + os.makedirs(os.path.dirname(path), exist_ok=True) + open(path, "w", encoding="utf-8").write(text) + print(f"wrote {name}/README.md") From 653ca103cdd91c1278b8d05e411ec7ccd4d0adfc Mon Sep 17 00:00:00 2001 From: laurence Date: Tue, 21 Jul 2026 17:10:01 +0100 Subject: [PATCH 3/4] Add the field plan for the 12 August 2026 total eclipse from Menorca Menorca sits inside the path of totality, but the geometry is extreme and it drives every decision in the plan: the Sun is 2 degrees above the horizon at maximum, on azimuth 288, and totality lasts 1 minute 12 seconds at Ciutadella before the Sun sets at 20:42. Two consequences follow from that 2 degrees. The light travels through roughly twenty times more atmosphere than at the zenith, so everything is dimmer, softer and redder than any published exposure table assumes and the histogram has to be the authority rather than a chart. And a low hill, a building or a haze bank on the sea horizon hides the entire event, so an unobstructed west-north-west view matters more than anything else about the site. That produces a genuine trade-off worth recording. Totality runs 1m38s at Mahon against 1m12s at Ciutadella, because the centre line favours the south-east - but Mahon is on the east coast, so from there the view towards 288 degrees crosses the island with Monte Toro in roughly that direction. Horizon beats duration: 26 extra seconds is a bonus, a blocked horizon is total loss. The plan therefore recommends a west or south-west coastal site. The plan also corrects an idea that was on the table: a photographic ND filter is not a solar filter. Standard NDs cut visible light while passing infrared largely unattenuated, which is where the heat is, and that will damage a sensor. The requirement is a certified ISO 12312-2 filter around OD 5 on the front of the lens, with the specific warning that Baader's AstroSolar Photo Film ND 3.8 is camera-only and unsafe to look through, while the ND 5.0 safety film does both jobs. The Star Adventurer GTi is explicitly excluded from eclipse day. Seventy-two seconds needs no tracking, and polar aligning in bright twilight towards a 2 degree Sun is impractical. It earns its place on the night skies of the same trip, not on the 12th. Kept in this repo rather than in a notes app because it is worked out from real numbers, it will be revised as the date approaches, and the reasoning behind each decision is the part worth keeping. --- README.md | 6 + observing/eclipse-2026-menorca/PLAN.md | 253 +++++++++++++++++++++++++ 2 files changed, 259 insertions(+) create mode 100644 observing/eclipse-2026-menorca/PLAN.md diff --git a/README.md b/README.md index a8b1cd2..78b1e14 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,12 @@ pipeline should be able to reproduce those results exactly. `session-scripts/restructure.py` - reorganises a flat session directory into the named layout below. Idempotent, dry run by default. +`observing/` - plans for observing sessions that are not remote-telescope runs. +Currently the total solar eclipse of 12 August 2026, seen from Menorca. These +live here rather than in a notes app because they are worked out from real +numbers, they get revised as the date approaches, and the reasoning behind each +decision is worth keeping. + ## Pointing the scripts at a session ``` diff --git a/observing/eclipse-2026-menorca/PLAN.md b/observing/eclipse-2026-menorca/PLAN.md new file mode 100644 index 0000000..beb8a2a --- /dev/null +++ b/observing/eclipse-2026-menorca/PLAN.md @@ -0,0 +1,253 @@ +# Total solar eclipse - Menorca - Wednesday 12 August 2026 + +Field plan for a Sony A7 on a tripod. Times are local (CEST, UTC+2). + +--- + +## The timings + +Figures below are for **Ciutadella** (west coast). Totality length varies across +the island - see "Where to stand". + +| Event | Time | Sun altitude | +|---|---|---| +| **C1** partial begins | 19:37:10 | 12° | +| **C2 totality begins** | **20:30:09** | ~2° | +| Maximum | 20:30:45 | **2°, azimuth 288° (WNW)** | +| **C3 totality ends** | **20:31:21** | ~2° | +| C4 partial ends / sunset | 20:42:00 | 0° | + +**Totality: 1 minute 12 seconds.** Eclipse magnitude 103.1%. + +Two numbers govern this whole plan: + +- **2° altitude.** You are looking through roughly 20 times more atmosphere + than at the zenith. Everything is dimmer, softer and redder than any standard + eclipse guide assumes. +- **72 seconds.** No second attempt, no time to solve problems. + +--- + +## Where to stand + +**Horizon beats duration.** At 2° altitude a low hill, a building or a haze +bank on the sea horizon hides the entire event. The requirement is an +unobstructed view toward **azimuth 288°, west-northwest**. + +Duration varies across the island: **Mahón 1m38s**, **Ciutadella 1m12s** - the +centre line favours the south-east. But Mahón is on the *east* coast, so from +there you would be looking WNW across the island, with Monte Toro (358 m) in +roughly that direction. A clear sea horizon is worth more than 26 extra +seconds. + +Best compromise: a **west or south-west coastal site**, which gives open sea at +288° while sitting further south for a slightly longer totality. + +- **Cap d'Artrutx / Far d'Artrutx** (south-west corner) - open sea to the WNW, + long low cliff promenade. The usual recommendation, so expect crowds. +- **Punta Nati** (north-west) - rugged, open, few obstructions. +- **Ciutadella** clifftop promenade and harbour - accessible, urban. + +Named sites will be busy and vehicle access may be restricted, as it already is +at official sites on Mallorca. Any quiet west-facing cliff works equally well: +the requirement is horizon, not postcode. + +**The real weather risk is a cloud bank sitting on the western horizon** - +invisible from inland, fatal at 2°. A sea horizon gives the best odds. + +--- + +## What to take + +### Buy now - this is the long pole + +**A certified solar filter for the front of the lens.** Not an ND filter: a +photographic ND passes infrared largely unattenuated and will cook the sensor. +You need ISO 12312-2 / around OD 5. + +- **Baader AstroSolar Safety Film ND 5.0** - cheap, excellent, cut to size and + mount in a card cell. **Careful:** Baader also sell *AstroSolar Photo Film + ND 3.8*, which is for cameras only and is **not safe to look through**. For a + single filter that does both jobs, take the **ND 5.0**. +- Or a ready-made screw-on/slip-on solar filter sized to your lens. +- **Eclipse glasses (ISO 12312-2) for everyone in the party.** + +Order in the next few days. There are three weeks left and this is the one item +with no substitute. + +### Camera kit + +- Sony A7 + **wide-to-standard lens (24-70 mm)** - this is the primary lens +- **200-300 mm** if you have it, for a tighter corona frame +- **Not 500-600 mm.** At 2° the seeing will destroy fine detail and you will + fill the frame with shimmer +- **Sturdy tripod.** Coastal cliff, evening breeze, low sun - stability matters + more than weight saving +- A head that repoints quickly and low: at 2° you are shooting almost level +- **2-3 spare batteries**, fully charged +- **Fast, formatted SD cards** with plenty of space +- **Remote release or intervalometer** so you never touch the camera +- **Gaffer tape** - to secure the filter against wind and lock focus + +### Leave at home + +The **Star Adventurer GTi**. Seventy-two seconds needs no tracking, and polar +aligning in bright twilight toward a 2° Sun is impractical. Bring it on the trip +for the night skies - Menorca is dark and August is good for the Milky Way - +but it plays no part in eclipse day. + +### Everything else + +Red torch, water, warm layer (the temperature drops noticeably at totality), +insect repellent, a folding chair, and a phone with this plan and a countdown +timer. Optionally a second camera or phone on its own tripod recording wide +video unattended. + +--- + +## Before you go + +1. **Fit and check the filter.** Mount it, then hold it up to a bright lamp and + look for pinholes. Any hole means it is scrap. +2. **Practise on a sunset at home.** This is the single most valuable thing you + can do. Shoot the Sun through the filter when it is 2-5° above the horizon + and write down the settings that work. That calibrates for the real + atmospheric extinction, which no exposure table accounts for. +3. **Practise the filter-off transition** until you can do it in a few seconds + without looking, without knocking the tripod. +4. **Set the camera up once, properly:** + - RAW (not JPEG) + - Manual exposure, manual ISO - no auto anything + - Manual focus + - White balance: Daylight, not auto + - Long-exposure noise reduction: OFF (it doubles every exposure time) + - Steady-shot / IBIS: OFF on a tripod + - Drive mode: continuous or bracketing, with the remote release +5. **On arrival, scout the site** and check the horizon at azimuth 288° with a + compass app. Do this on a previous evening if you can - the Sun sets in + almost the same place each night this time of year. + +--- + +## Eclipse day, step by step + +### Afternoon +Check the forecast, specifically for **cloud on the western horizon**. Decide +the site. Travel early - roads will be busy and parking will fill. + +### T-3 hours (about 17:30) +Arrive. Set up the tripod. Frame roughly where the Sun will be at 20:30 - +azimuth 288°, just above the sea. + +### T-90 minutes (19:00) +- Fit the filter and tape it on. +- **Focus manually**: magnified live view on the Sun's edge, focus for the + sharpest limb, then **tape the focus ring**. Autofocus will hunt and fail. +- Take test frames. Check the histogram, not the screen brightness. +- **Decision point:** if the western horizon is clouding over, this is when to + move. The island is small; later than this and you are committed. + +### 19:37 - C1, first contact +The Moon takes its first bite. Start the partial sequence: **one frame every 5 +minutes**. That gives about 10 frames for a composite sequence. + +### 19:37 to 20:25 - the partial phases +The Sun drops from 12° to 2°, and **dims continuously and steeply** as it sinks +into thicker air. No single exposure works for the whole hour. + +- Start around **f/8, ISO 100**, shutter near 1/500 (confirm from your practice + session). +- **Re-check the histogram every 5 minutes** and lengthen the exposure as the + Sun drops. Expect to end several stops slower than you started. +- Keep ISO low while you can; raise it later rather than let the shutter get so + slow that shimmer smears the disc. + +### 20:25 - stop fiddling +- Final framing. If using the wide lens, compose for **corona over sea**, with + the horizon low in the frame. +- Set the totality bracket ready to fire (see below). +- Take the lens cap off, keep the solar filter on. + +### 20:29 - watch +Light goes strange, colour drains, the temperature drops. Look west for the +approaching shadow over the sea. + +### 20:30:09 - C2. FILTER OFF +The diamond ring. **Remove the solar filter** - the whole point of totality is +that it is safe and the filter makes it invisible. + +- Fire the bracket immediately. +- **Then stop and look at it.** + +### 72 seconds of totality +One bracket sequence is enough. You are 25% of the way through totality by the +time it finishes. Spend the rest with your eyes, not the viewfinder - a +2°-altitude totality over the Mediterranean is a rare sight and it will be over +before it registers. + +### 20:31:21 - C3. FILTER BACK ON +At the first bead of returning sunlight, filter back on and eclipse glasses +back on. The Sun is now setting into the sea; the remaining partial phases will +be lost in the horizon haze within a few minutes. + +### 20:42 - C4 / sunset +Done. Pack slowly, let the traffic clear, and look at the twilight. + +--- + +## Exposure settings + +Treat these as **starting points to refine at your practice sunset**. Extinction +at 2° is severe and variable with haze, so the histogram is the authority, not +this table. + +### Partial phases (filter ON) +| Sun altitude | Aperture | ISO | Shutter (start point) | +|---|---|---|---| +| 12° (C1) | f/8 | 100 | ~1/500 | +| 8° | f/8 | 100 | ~1/250 | +| 5° | f/8 | 200 | ~1/125 | +| 2-3° | f/8 | 400 | ~1/60 or slower | + +### Totality (filter OFF) +The corona spans an enormous brightness range, so **bracket wide**: no single +exposure captures both the inner corona and the outer streamers. + +- **f/5.6-f/8, ISO 400-800** +- **Bracket 1/500 → 2 s**, 7-9 frames, one stop apart +- Add roughly 3 stops over standard eclipse tables to allow for the extinction + at 2° +- The **diamond ring** wants the fast end: 1/500 to 1/1000 +- **Wide-field shot**: 24-35 mm, ISO 800, 1/30 to 1/8 - captures corona, sea and + the 360° twilight glow. This is the picture most likely to be a keeper. + +--- + +## Safety, in one place + +- **The filter stays on at all times except between C2 and C3** (20:30:09 to + 20:31:21). +- **Never use a photographic ND filter** for the partial phases. +- **Tape the filter on.** A gust removing it mid-exposure damages the camera. +- **Check the film for pinholes** before each session. +- **Eclipse glasses for eyes** during all partial phases. +- **Naked eye is safe only during totality.** At the first returning bead, + glasses back on immediately. + +--- + +## If it goes wrong + +- **Cloud on the horizon:** decide by 19:00 and move. Menorca is about 50 km + end to end, but roads will be busy - do not leave it late. +- **Missed the bracket:** the wide-field shot is the keeper. Prioritise it. +- **Camera problem during totality:** stop, and watch. You cannot fix a camera + in 72 seconds, and the memory is worth more than the file. + +--- + +## The one rule + +**Look at it.** Seventy-two seconds is very short, the corona at 2° over the sea +will be extraordinary, and no photograph you take will be the reason you +remember it. From c6299f41ab2d17110e1f454938a0a6ddec649ff9 Mon Sep 17 00:00:00 2001 From: laurence Date: Tue, 21 Jul 2026 17:13:54 +0100 Subject: [PATCH 4/4] Move the processing code under pipeline/ Preparing to merge this repository into a combined astrophotography repo. session-scripts/ becomes pipeline/ because the scripts import layout.py from their own directory and must stay together, and because 'pipeline' says what it is rather than how it came about. observing/ stays at the top level: observing plans are not processing code. --- README.md => pipeline/README.md | 0 {session-scripts => pipeline}/analyse.py | 0 {session-scripts => pipeline}/annotate.py | 0 {session-scripts => pipeline}/closeup.py | 0 {session-scripts => pipeline}/compose.py | 0 {session-scripts => pipeline}/depth.py | 0 {session-scripts => pipeline}/enhance.py | 0 {session-scripts => pipeline}/final.py | 0 {session-scripts => pipeline}/gaia_colours.py | 0 {session-scripts => pipeline}/gc-bwtrial.py | 0 {session-scripts => pipeline}/gc-classify.py | 0 {session-scripts => pipeline}/gc-complete-inner.py | 0 {session-scripts => pipeline}/gc-complete.py | 0 {session-scripts => pipeline}/gc-detect.py | 0 {session-scripts => pipeline}/gc-fetch-vizier.py | 0 {session-scripts => pipeline}/gc-gaia-astrom2.py | 0 {session-scripts => pipeline}/gc-plots.py | 0 {session-scripts => pipeline}/gc-validate.py | 0 {session-scripts => pipeline}/hdr.py | 0 {session-scripts => pipeline}/layout.py | 0 {session-scripts => pipeline}/mo_cavs.py | 0 {session-scripts => pipeline}/mo_check.py | 0 {session-scripts => pipeline}/mo_common.py | 0 {session-scripts => pipeline}/mo_detect.py | 0 {session-scripts => pipeline}/mo_fig_moving.py | 0 {session-scripts => pipeline}/mo_fig_transient.py | 0 {session-scripts => pipeline}/mo_finalise.py | 0 {session-scripts => pipeline}/mo_gccheck.py | 0 {session-scripts => pipeline}/mo_link.py | 0 {session-scripts => pipeline}/mo_mpc.py | 0 {session-scripts => pipeline}/mo_sensitivity.py | 0 {session-scripts => pipeline}/mo_shiftstack.py | 0 {session-scripts => pipeline}/mo_transient.py | 0 {session-scripts => pipeline}/mo_vet.py | 0 {session-scripts => pipeline}/original.py | 0 {session-scripts => pipeline}/rename.py | 0 {session-scripts => pipeline}/restructure.py | 0 {session-scripts => pipeline}/sb_common.py | 0 {session-scripts => pipeline}/sb_dust.py | 0 {session-scripts => pipeline}/sb_iso.py | 0 {session-scripts => pipeline}/sb_limits.py | 0 {session-scripts => pipeline}/sb_model.py | 0 {session-scripts => pipeline}/sb_prep.py | 0 {session-scripts => pipeline}/sb_profile.py | 0 {session-scripts => pipeline}/sb_render_tail.py | 0 {session-scripts => pipeline}/sb_residual.py | 0 {session-scripts => pipeline}/solve.py | 0 {session-scripts => pipeline}/stack.py | 0 {session-scripts => pipeline}/starless.py | 0 {session-scripts => pipeline}/subdir_readmes.py | 0 {session-scripts => pipeline}/triptych.py | 0 {session-scripts => pipeline}/unzip.py | 0 {session-scripts => pipeline}/verify_core.py | 0 53 files changed, 0 insertions(+), 0 deletions(-) rename README.md => pipeline/README.md (100%) rename {session-scripts => pipeline}/analyse.py (100%) rename {session-scripts => pipeline}/annotate.py (100%) rename {session-scripts => pipeline}/closeup.py (100%) rename {session-scripts => pipeline}/compose.py (100%) rename {session-scripts => pipeline}/depth.py (100%) rename {session-scripts => pipeline}/enhance.py (100%) rename {session-scripts => pipeline}/final.py (100%) rename {session-scripts => pipeline}/gaia_colours.py (100%) rename {session-scripts => pipeline}/gc-bwtrial.py (100%) rename {session-scripts => pipeline}/gc-classify.py (100%) rename {session-scripts => pipeline}/gc-complete-inner.py (100%) rename {session-scripts => pipeline}/gc-complete.py (100%) rename {session-scripts => pipeline}/gc-detect.py (100%) rename {session-scripts => pipeline}/gc-fetch-vizier.py (100%) rename {session-scripts => pipeline}/gc-gaia-astrom2.py (100%) rename {session-scripts => pipeline}/gc-plots.py (100%) rename {session-scripts => pipeline}/gc-validate.py (100%) rename {session-scripts => pipeline}/hdr.py (100%) rename {session-scripts => pipeline}/layout.py (100%) rename {session-scripts => pipeline}/mo_cavs.py (100%) rename {session-scripts => pipeline}/mo_check.py (100%) rename {session-scripts => pipeline}/mo_common.py (100%) rename {session-scripts => pipeline}/mo_detect.py (100%) rename {session-scripts => pipeline}/mo_fig_moving.py (100%) rename {session-scripts => pipeline}/mo_fig_transient.py (100%) rename {session-scripts => pipeline}/mo_finalise.py (100%) rename {session-scripts => pipeline}/mo_gccheck.py (100%) rename {session-scripts => pipeline}/mo_link.py (100%) rename {session-scripts => pipeline}/mo_mpc.py (100%) rename {session-scripts => pipeline}/mo_sensitivity.py (100%) rename {session-scripts => pipeline}/mo_shiftstack.py (100%) rename {session-scripts => pipeline}/mo_transient.py (100%) rename {session-scripts => pipeline}/mo_vet.py (100%) rename {session-scripts => pipeline}/original.py (100%) rename {session-scripts => pipeline}/rename.py (100%) rename {session-scripts => pipeline}/restructure.py (100%) rename {session-scripts => pipeline}/sb_common.py (100%) rename {session-scripts => pipeline}/sb_dust.py (100%) rename {session-scripts => pipeline}/sb_iso.py (100%) rename {session-scripts => pipeline}/sb_limits.py (100%) rename {session-scripts => pipeline}/sb_model.py (100%) rename {session-scripts => pipeline}/sb_prep.py (100%) rename {session-scripts => pipeline}/sb_profile.py (100%) rename {session-scripts => pipeline}/sb_render_tail.py (100%) rename {session-scripts => pipeline}/sb_residual.py (100%) rename {session-scripts => pipeline}/solve.py (100%) rename {session-scripts => pipeline}/stack.py (100%) rename {session-scripts => pipeline}/starless.py (100%) rename {session-scripts => pipeline}/subdir_readmes.py (100%) rename {session-scripts => pipeline}/triptych.py (100%) rename {session-scripts => pipeline}/unzip.py (100%) rename {session-scripts => pipeline}/verify_core.py (100%) diff --git a/README.md b/pipeline/README.md similarity index 100% rename from README.md rename to pipeline/README.md diff --git a/session-scripts/analyse.py b/pipeline/analyse.py similarity index 100% rename from session-scripts/analyse.py rename to pipeline/analyse.py diff --git a/session-scripts/annotate.py b/pipeline/annotate.py similarity index 100% rename from session-scripts/annotate.py rename to pipeline/annotate.py diff --git a/session-scripts/closeup.py b/pipeline/closeup.py similarity index 100% rename from session-scripts/closeup.py rename to pipeline/closeup.py diff --git a/session-scripts/compose.py b/pipeline/compose.py similarity index 100% rename from session-scripts/compose.py rename to pipeline/compose.py diff --git a/session-scripts/depth.py b/pipeline/depth.py similarity index 100% rename from session-scripts/depth.py rename to pipeline/depth.py diff --git a/session-scripts/enhance.py b/pipeline/enhance.py similarity index 100% rename from session-scripts/enhance.py rename to pipeline/enhance.py diff --git a/session-scripts/final.py b/pipeline/final.py similarity index 100% rename from session-scripts/final.py rename to pipeline/final.py diff --git a/session-scripts/gaia_colours.py b/pipeline/gaia_colours.py similarity index 100% rename from session-scripts/gaia_colours.py rename to pipeline/gaia_colours.py diff --git a/session-scripts/gc-bwtrial.py b/pipeline/gc-bwtrial.py similarity index 100% rename from session-scripts/gc-bwtrial.py rename to pipeline/gc-bwtrial.py diff --git a/session-scripts/gc-classify.py b/pipeline/gc-classify.py similarity index 100% rename from session-scripts/gc-classify.py rename to pipeline/gc-classify.py diff --git a/session-scripts/gc-complete-inner.py b/pipeline/gc-complete-inner.py similarity index 100% rename from session-scripts/gc-complete-inner.py rename to pipeline/gc-complete-inner.py diff --git a/session-scripts/gc-complete.py b/pipeline/gc-complete.py similarity index 100% rename from session-scripts/gc-complete.py rename to pipeline/gc-complete.py diff --git a/session-scripts/gc-detect.py b/pipeline/gc-detect.py similarity index 100% rename from session-scripts/gc-detect.py rename to pipeline/gc-detect.py diff --git a/session-scripts/gc-fetch-vizier.py b/pipeline/gc-fetch-vizier.py similarity index 100% rename from session-scripts/gc-fetch-vizier.py rename to pipeline/gc-fetch-vizier.py diff --git a/session-scripts/gc-gaia-astrom2.py b/pipeline/gc-gaia-astrom2.py similarity index 100% rename from session-scripts/gc-gaia-astrom2.py rename to pipeline/gc-gaia-astrom2.py diff --git a/session-scripts/gc-plots.py b/pipeline/gc-plots.py similarity index 100% rename from session-scripts/gc-plots.py rename to pipeline/gc-plots.py diff --git a/session-scripts/gc-validate.py b/pipeline/gc-validate.py similarity index 100% rename from session-scripts/gc-validate.py rename to pipeline/gc-validate.py diff --git a/session-scripts/hdr.py b/pipeline/hdr.py similarity index 100% rename from session-scripts/hdr.py rename to pipeline/hdr.py diff --git a/session-scripts/layout.py b/pipeline/layout.py similarity index 100% rename from session-scripts/layout.py rename to pipeline/layout.py diff --git a/session-scripts/mo_cavs.py b/pipeline/mo_cavs.py similarity index 100% rename from session-scripts/mo_cavs.py rename to pipeline/mo_cavs.py diff --git a/session-scripts/mo_check.py b/pipeline/mo_check.py similarity index 100% rename from session-scripts/mo_check.py rename to pipeline/mo_check.py diff --git a/session-scripts/mo_common.py b/pipeline/mo_common.py similarity index 100% rename from session-scripts/mo_common.py rename to pipeline/mo_common.py diff --git a/session-scripts/mo_detect.py b/pipeline/mo_detect.py similarity index 100% rename from session-scripts/mo_detect.py rename to pipeline/mo_detect.py diff --git a/session-scripts/mo_fig_moving.py b/pipeline/mo_fig_moving.py similarity index 100% rename from session-scripts/mo_fig_moving.py rename to pipeline/mo_fig_moving.py diff --git a/session-scripts/mo_fig_transient.py b/pipeline/mo_fig_transient.py similarity index 100% rename from session-scripts/mo_fig_transient.py rename to pipeline/mo_fig_transient.py diff --git a/session-scripts/mo_finalise.py b/pipeline/mo_finalise.py similarity index 100% rename from session-scripts/mo_finalise.py rename to pipeline/mo_finalise.py diff --git a/session-scripts/mo_gccheck.py b/pipeline/mo_gccheck.py similarity index 100% rename from session-scripts/mo_gccheck.py rename to pipeline/mo_gccheck.py diff --git a/session-scripts/mo_link.py b/pipeline/mo_link.py similarity index 100% rename from session-scripts/mo_link.py rename to pipeline/mo_link.py diff --git a/session-scripts/mo_mpc.py b/pipeline/mo_mpc.py similarity index 100% rename from session-scripts/mo_mpc.py rename to pipeline/mo_mpc.py diff --git a/session-scripts/mo_sensitivity.py b/pipeline/mo_sensitivity.py similarity index 100% rename from session-scripts/mo_sensitivity.py rename to pipeline/mo_sensitivity.py diff --git a/session-scripts/mo_shiftstack.py b/pipeline/mo_shiftstack.py similarity index 100% rename from session-scripts/mo_shiftstack.py rename to pipeline/mo_shiftstack.py diff --git a/session-scripts/mo_transient.py b/pipeline/mo_transient.py similarity index 100% rename from session-scripts/mo_transient.py rename to pipeline/mo_transient.py diff --git a/session-scripts/mo_vet.py b/pipeline/mo_vet.py similarity index 100% rename from session-scripts/mo_vet.py rename to pipeline/mo_vet.py diff --git a/session-scripts/original.py b/pipeline/original.py similarity index 100% rename from session-scripts/original.py rename to pipeline/original.py diff --git a/session-scripts/rename.py b/pipeline/rename.py similarity index 100% rename from session-scripts/rename.py rename to pipeline/rename.py diff --git a/session-scripts/restructure.py b/pipeline/restructure.py similarity index 100% rename from session-scripts/restructure.py rename to pipeline/restructure.py diff --git a/session-scripts/sb_common.py b/pipeline/sb_common.py similarity index 100% rename from session-scripts/sb_common.py rename to pipeline/sb_common.py diff --git a/session-scripts/sb_dust.py b/pipeline/sb_dust.py similarity index 100% rename from session-scripts/sb_dust.py rename to pipeline/sb_dust.py diff --git a/session-scripts/sb_iso.py b/pipeline/sb_iso.py similarity index 100% rename from session-scripts/sb_iso.py rename to pipeline/sb_iso.py diff --git a/session-scripts/sb_limits.py b/pipeline/sb_limits.py similarity index 100% rename from session-scripts/sb_limits.py rename to pipeline/sb_limits.py diff --git a/session-scripts/sb_model.py b/pipeline/sb_model.py similarity index 100% rename from session-scripts/sb_model.py rename to pipeline/sb_model.py diff --git a/session-scripts/sb_prep.py b/pipeline/sb_prep.py similarity index 100% rename from session-scripts/sb_prep.py rename to pipeline/sb_prep.py diff --git a/session-scripts/sb_profile.py b/pipeline/sb_profile.py similarity index 100% rename from session-scripts/sb_profile.py rename to pipeline/sb_profile.py diff --git a/session-scripts/sb_render_tail.py b/pipeline/sb_render_tail.py similarity index 100% rename from session-scripts/sb_render_tail.py rename to pipeline/sb_render_tail.py diff --git a/session-scripts/sb_residual.py b/pipeline/sb_residual.py similarity index 100% rename from session-scripts/sb_residual.py rename to pipeline/sb_residual.py diff --git a/session-scripts/solve.py b/pipeline/solve.py similarity index 100% rename from session-scripts/solve.py rename to pipeline/solve.py diff --git a/session-scripts/stack.py b/pipeline/stack.py similarity index 100% rename from session-scripts/stack.py rename to pipeline/stack.py diff --git a/session-scripts/starless.py b/pipeline/starless.py similarity index 100% rename from session-scripts/starless.py rename to pipeline/starless.py diff --git a/session-scripts/subdir_readmes.py b/pipeline/subdir_readmes.py similarity index 100% rename from session-scripts/subdir_readmes.py rename to pipeline/subdir_readmes.py diff --git a/session-scripts/triptych.py b/pipeline/triptych.py similarity index 100% rename from session-scripts/triptych.py rename to pipeline/triptych.py diff --git a/session-scripts/unzip.py b/pipeline/unzip.py similarity index 100% rename from session-scripts/unzip.py rename to pipeline/unzip.py diff --git a/session-scripts/verify_core.py b/pipeline/verify_core.py similarity index 100% rename from session-scripts/verify_core.py rename to pipeline/verify_core.py