diff --git a/.gitignore b/.gitignore index d7d1347..93abfa8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,28 @@ -# local scratch and secrets +# Local scratch and secrets *.token *.key scratch/ Thumbs.db .DS_Store + +__pycache__/ +*.pyc + +# Image data never lives in this repo. Sessions are held on disk and located +# at runtime through ASTRO_SESSION; a single FITS frame is 61 MB and a session +# is several GB. +*.fit +*.fits +*.tif +*.tiff +*.npz +*.npy + +# Rendered images are session outputs too, with one exception: figures that +# belong to the documentation. Those live under docs/ and are committed +# deliberately. +*.png +*.jpg +*.jpeg +!docs/**/*.png +!docs/**/*.jpg diff --git a/observing/eclipse-2026-menorca/PLAN.md b/observing/eclipse-2026-menorca/PLAN.md new file mode 100644 index 0000000..beb8a2a --- /dev/null +++ b/observing/eclipse-2026-menorca/PLAN.md @@ -0,0 +1,253 @@ +# Total solar eclipse - Menorca - Wednesday 12 August 2026 + +Field plan for a Sony A7 on a tripod. Times are local (CEST, UTC+2). + +--- + +## The timings + +Figures below are for **Ciutadella** (west coast). Totality length varies across +the island - see "Where to stand". + +| Event | Time | Sun altitude | +|---|---|---| +| **C1** partial begins | 19:37:10 | 12° | +| **C2 totality begins** | **20:30:09** | ~2° | +| Maximum | 20:30:45 | **2°, azimuth 288° (WNW)** | +| **C3 totality ends** | **20:31:21** | ~2° | +| C4 partial ends / sunset | 20:42:00 | 0° | + +**Totality: 1 minute 12 seconds.** Eclipse magnitude 103.1%. + +Two numbers govern this whole plan: + +- **2° altitude.** You are looking through roughly 20 times more atmosphere + than at the zenith. Everything is dimmer, softer and redder than any standard + eclipse guide assumes. +- **72 seconds.** No second attempt, no time to solve problems. + +--- + +## Where to stand + +**Horizon beats duration.** At 2° altitude a low hill, a building or a haze +bank on the sea horizon hides the entire event. The requirement is an +unobstructed view toward **azimuth 288°, west-northwest**. + +Duration varies across the island: **Mahón 1m38s**, **Ciutadella 1m12s** - the +centre line favours the south-east. But Mahón is on the *east* coast, so from +there you would be looking WNW across the island, with Monte Toro (358 m) in +roughly that direction. A clear sea horizon is worth more than 26 extra +seconds. + +Best compromise: a **west or south-west coastal site**, which gives open sea at +288° while sitting further south for a slightly longer totality. + +- **Cap d'Artrutx / Far d'Artrutx** (south-west corner) - open sea to the WNW, + long low cliff promenade. The usual recommendation, so expect crowds. +- **Punta Nati** (north-west) - rugged, open, few obstructions. +- **Ciutadella** clifftop promenade and harbour - accessible, urban. + +Named sites will be busy and vehicle access may be restricted, as it already is +at official sites on Mallorca. Any quiet west-facing cliff works equally well: +the requirement is horizon, not postcode. + +**The real weather risk is a cloud bank sitting on the western horizon** - +invisible from inland, fatal at 2°. A sea horizon gives the best odds. + +--- + +## What to take + +### Buy now - this is the long pole + +**A certified solar filter for the front of the lens.** Not an ND filter: a +photographic ND passes infrared largely unattenuated and will cook the sensor. +You need ISO 12312-2 / around OD 5. + +- **Baader AstroSolar Safety Film ND 5.0** - cheap, excellent, cut to size and + mount in a card cell. **Careful:** Baader also sell *AstroSolar Photo Film + ND 3.8*, which is for cameras only and is **not safe to look through**. For a + single filter that does both jobs, take the **ND 5.0**. +- Or a ready-made screw-on/slip-on solar filter sized to your lens. +- **Eclipse glasses (ISO 12312-2) for everyone in the party.** + +Order in the next few days. There are three weeks left and this is the one item +with no substitute. + +### Camera kit + +- Sony A7 + **wide-to-standard lens (24-70 mm)** - this is the primary lens +- **200-300 mm** if you have it, for a tighter corona frame +- **Not 500-600 mm.** At 2° the seeing will destroy fine detail and you will + fill the frame with shimmer +- **Sturdy tripod.** Coastal cliff, evening breeze, low sun - stability matters + more than weight saving +- A head that repoints quickly and low: at 2° you are shooting almost level +- **2-3 spare batteries**, fully charged +- **Fast, formatted SD cards** with plenty of space +- **Remote release or intervalometer** so you never touch the camera +- **Gaffer tape** - to secure the filter against wind and lock focus + +### Leave at home + +The **Star Adventurer GTi**. Seventy-two seconds needs no tracking, and polar +aligning in bright twilight toward a 2° Sun is impractical. Bring it on the trip +for the night skies - Menorca is dark and August is good for the Milky Way - +but it plays no part in eclipse day. + +### Everything else + +Red torch, water, warm layer (the temperature drops noticeably at totality), +insect repellent, a folding chair, and a phone with this plan and a countdown +timer. Optionally a second camera or phone on its own tripod recording wide +video unattended. + +--- + +## Before you go + +1. **Fit and check the filter.** Mount it, then hold it up to a bright lamp and + look for pinholes. Any hole means it is scrap. +2. **Practise on a sunset at home.** This is the single most valuable thing you + can do. Shoot the Sun through the filter when it is 2-5° above the horizon + and write down the settings that work. That calibrates for the real + atmospheric extinction, which no exposure table accounts for. +3. **Practise the filter-off transition** until you can do it in a few seconds + without looking, without knocking the tripod. +4. **Set the camera up once, properly:** + - RAW (not JPEG) + - Manual exposure, manual ISO - no auto anything + - Manual focus + - White balance: Daylight, not auto + - Long-exposure noise reduction: OFF (it doubles every exposure time) + - Steady-shot / IBIS: OFF on a tripod + - Drive mode: continuous or bracketing, with the remote release +5. **On arrival, scout the site** and check the horizon at azimuth 288° with a + compass app. Do this on a previous evening if you can - the Sun sets in + almost the same place each night this time of year. + +--- + +## Eclipse day, step by step + +### Afternoon +Check the forecast, specifically for **cloud on the western horizon**. Decide +the site. Travel early - roads will be busy and parking will fill. + +### T-3 hours (about 17:30) +Arrive. Set up the tripod. Frame roughly where the Sun will be at 20:30 - +azimuth 288°, just above the sea. + +### T-90 minutes (19:00) +- Fit the filter and tape it on. +- **Focus manually**: magnified live view on the Sun's edge, focus for the + sharpest limb, then **tape the focus ring**. Autofocus will hunt and fail. +- Take test frames. Check the histogram, not the screen brightness. +- **Decision point:** if the western horizon is clouding over, this is when to + move. The island is small; later than this and you are committed. + +### 19:37 - C1, first contact +The Moon takes its first bite. Start the partial sequence: **one frame every 5 +minutes**. That gives about 10 frames for a composite sequence. + +### 19:37 to 20:25 - the partial phases +The Sun drops from 12° to 2°, and **dims continuously and steeply** as it sinks +into thicker air. No single exposure works for the whole hour. + +- Start around **f/8, ISO 100**, shutter near 1/500 (confirm from your practice + session). +- **Re-check the histogram every 5 minutes** and lengthen the exposure as the + Sun drops. Expect to end several stops slower than you started. +- Keep ISO low while you can; raise it later rather than let the shutter get so + slow that shimmer smears the disc. + +### 20:25 - stop fiddling +- Final framing. If using the wide lens, compose for **corona over sea**, with + the horizon low in the frame. +- Set the totality bracket ready to fire (see below). +- Take the lens cap off, keep the solar filter on. + +### 20:29 - watch +Light goes strange, colour drains, the temperature drops. Look west for the +approaching shadow over the sea. + +### 20:30:09 - C2. FILTER OFF +The diamond ring. **Remove the solar filter** - the whole point of totality is +that it is safe and the filter makes it invisible. + +- Fire the bracket immediately. +- **Then stop and look at it.** + +### 72 seconds of totality +One bracket sequence is enough. You are 25% of the way through totality by the +time it finishes. Spend the rest with your eyes, not the viewfinder - a +2°-altitude totality over the Mediterranean is a rare sight and it will be over +before it registers. + +### 20:31:21 - C3. FILTER BACK ON +At the first bead of returning sunlight, filter back on and eclipse glasses +back on. The Sun is now setting into the sea; the remaining partial phases will +be lost in the horizon haze within a few minutes. + +### 20:42 - C4 / sunset +Done. Pack slowly, let the traffic clear, and look at the twilight. + +--- + +## Exposure settings + +Treat these as **starting points to refine at your practice sunset**. Extinction +at 2° is severe and variable with haze, so the histogram is the authority, not +this table. + +### Partial phases (filter ON) +| Sun altitude | Aperture | ISO | Shutter (start point) | +|---|---|---|---| +| 12° (C1) | f/8 | 100 | ~1/500 | +| 8° | f/8 | 100 | ~1/250 | +| 5° | f/8 | 200 | ~1/125 | +| 2-3° | f/8 | 400 | ~1/60 or slower | + +### Totality (filter OFF) +The corona spans an enormous brightness range, so **bracket wide**: no single +exposure captures both the inner corona and the outer streamers. + +- **f/5.6-f/8, ISO 400-800** +- **Bracket 1/500 → 2 s**, 7-9 frames, one stop apart +- Add roughly 3 stops over standard eclipse tables to allow for the extinction + at 2° +- The **diamond ring** wants the fast end: 1/500 to 1/1000 +- **Wide-field shot**: 24-35 mm, ISO 800, 1/30 to 1/8 - captures corona, sea and + the 360° twilight glow. This is the picture most likely to be a keeper. + +--- + +## Safety, in one place + +- **The filter stays on at all times except between C2 and C3** (20:30:09 to + 20:31:21). +- **Never use a photographic ND filter** for the partial phases. +- **Tape the filter on.** A gust removing it mid-exposure damages the camera. +- **Check the film for pinholes** before each session. +- **Eclipse glasses for eyes** during all partial phases. +- **Naked eye is safe only during totality.** At the first returning bead, + glasses back on immediately. + +--- + +## If it goes wrong + +- **Cloud on the horizon:** decide by 19:00 and move. Menorca is about 50 km + end to end, but roads will be busy - do not leave it late. +- **Missed the bracket:** the wide-field shot is the keeper. Prioritise it. +- **Camera problem during totality:** stop, and watch. You cannot fix a camera + in 72 seconds, and the memory is worth more than the file. + +--- + +## The one rule + +**Look at it.** Seventy-two seconds is very short, the corona at 2° over the sea +will be extraordinary, and no photograph you take will be the reason you +remember it. diff --git a/pipeline/README.md b/pipeline/README.md new file mode 100644 index 0000000..78b1e14 --- /dev/null +++ b/pipeline/README.md @@ -0,0 +1,101 @@ +# astro-pipeline + +Processing and analysis code for remote-telescope imaging sessions, starting +with iTelescope data from the [itelescope](https://git.discworld.casa/laurence/itelescope) +drain campaign. + +The code lives here. **The data does not** - image sessions stay on disk (or +wherever they are archived) and are addressed by an environment variable, so a +session directory contains only pixels, results and a description of what was +done to them. + +## What is here now + +`session-scripts/` - the 50 scripts that processed the NGC 5128 session of +2026-07-21, exactly as they were run, plus the shared `layout.py` that tells +them where files live. This is a working record rather than a finished product: +the scripts were written in sequence as the work went along, several of them by +parallel agents, and they show it. They are kept because they are the honest +provenance of a set of published results, and because the productionised +pipeline should be able to reproduce those results exactly. + +`session-scripts/restructure.py` - reorganises a flat session directory into the +named layout below. Idempotent, dry run by default. + +`observing/` - plans for observing sessions that are not remote-telescope runs. +Currently the total solar eclipse of 12 August 2026, seen from Menorca. These +live here rather than in a notes app because they are worked out from real +numbers, they get revised as the date approaches, and the reasoning behind each +decision is worth keeping. + +## Pointing the scripts at a session + +``` +set ASTRO_SESSION=D:\astro\NGC5128\20260721 # Windows +export ASTRO_SESSION=/data/astro/NGC5128/20260721 # POSIX +python session-scripts/layout.py # prints the resolved layout +``` + +`layout.py` maps a **filename** to its subdirectory, so a script asks for +`master-Red.fit` or `_stars.npz` and gets the right path without knowing the +directory structure: + +| Directory | Holds | +|---|---| +| `raw/` | exactly what the telescope delivered: archives and their preview jpegs | +| `calibrated/` | uncompressed calibrated subs | +| `stacks/masters/` | per-filter registered, plate-solved masters | +| `stacks/original/` | alignment-only baseline stacks, no other processing | +| `final/` | the deliverable renderings | +| `renderings/` | other finished images | +| `science/figures/` | analysis plots | +| `science/catalogues/` | measured tables (CSV) | +| `science/data/` | models, masks, derived quantities | +| `science/notes/` | analysis write-ups | +| `intermediates/` | caches a re-run can regenerate | + +Every session directory also carries its own `METHODS.md` describing what was +done to that data and what was found - written for a reader who was not there. + +## Running order + +The scripts are named for their stage and run in this order: + +``` +unzip.py -> analyse.py -> stack.py -> solve.py -> depth.py + -> compose.py -> hdr.py / enhance.py / starless.py / annotate.py + -> final.py -> closeup.py -> triptych.py +``` + +The analysis families are independent of each other and of the renderings: +`gc-*` (globular clusters), `sb_*` (surface photometry), `mo_*` (moving objects +and transients). + +## Requirements + +Python 3.12 with numpy, scipy, astropy, scikit-image, sep, astroalign, +photutils, astroquery, matplotlib, tifffile, Pillow. + +## Where this is going + +The next piece of work is a scheduler-driven pipeline: a staged CLI +(`ingest -> calibrate -> measure -> register -> stack -> solve -> compose -> +analyse`) with each stage resumable, packaged as an Apptainer image and driven +by Slurm array jobs. Targets beyond mono LRGB: narrowband palettes, one-shot +colour with debayering, other observatories' header conventions, and full +calibration from bias/dark/flat for sources that do not pre-calibrate. + +Three findings from the first session are requirements for that build, not +optional extras: + +1. **Vet moving-object candidates in detector coordinates.** Registration holds + the sky still, so it drags detector-fixed defects across the frame on + perfectly straight, constant-rate tracks. Hot pixels are better-behaved + asteroids than real asteroids. This one cut took 141 confident spurious + detections to zero. +2. **Carry `r50/psf` through to any catalogue cross-match.** Comparing an + aperture magnitude of a resolved source against a point-source catalogue + like Gaia is meaningless, and looks exactly like a 2.8 magnitude outburst. +3. **Never fit a sky background to a field the target fills.** A plane fitted + around a large galaxy absorbs its halo - measured at -17.9 ADU/px here. + Fit the background and a source model together. diff --git a/pipeline/analyse.py b/pipeline/analyse.py new file mode 100644 index 0000000..e61498c --- /dev/null +++ b/pipeline/analyse.py @@ -0,0 +1,70 @@ +"""Pass 1: measure every calibrated frame and cache its star list. + +Frames are 4788x3194 float32 (61 MB each) and the machine has little free RAM, +so each frame is opened, measured and released one at a time. The star lists are +cached to an npz because both the registration pass and the plate solve need +them, and re-detecting costs more than re-reading a small array. + +Recorded per frame: sky background and its rms, the number of detections, and a +median FWHM derived from sep's half-flux radius. The FWHM is the seeing metric +used later to pick the registration reference and to weight the stack. +""" +import glob +import os +import re + +import numpy as np +import sep +from astropy.io import fits + +import layout + +SRC = layout.SESSION +CACHE = layout.path("_stars.npz") +os.makedirs(os.path.dirname(CACHE), exist_ok=True) + +NAME_RE = re.compile(r"-(Luminance|Red|Green|Blue)-BIN2-W-300-(\d+)\.fit$") + + +def frame_list(): + out = [] + for path in sorted(glob.glob(layout.path("calibrated-*.fit"))): + m = NAME_RE.search(path) + if m: + out.append((path, m.group(1), int(m.group(2)))) + return out + + +if __name__ == "__main__": + frames = frame_list() + print(f"{len(frames)} calibrated frames") + store, meta = {}, [] + for path, filt, idx in frames: + with fits.open(path, memmap=False) as hd: + data = hd[0].data.astype(np.float32) + bkg = sep.Background(data, bw=64, bh=64, fw=3, fh=3) + back_med = float(np.median(bkg.back())) + rms = float(bkg.globalrms) + sub = data - bkg.back() + objs = sep.extract(sub, 5.0, err=rms, minarea=9, deblend_cont=0.005) + objs = objs[(objs["flag"] == 0) & (objs["npix"] > 12) & + (objs["npix"] < 2000)] + objs = objs[np.argsort(objs["flux"])[::-1][:400]] + rad, _ = sep.flux_radius(sub, objs["x"], objs["y"], 6.0 * objs["a"], + 0.5, normflux=objs["flux"]) + fwhm = float(np.median(rad) * 2.0) + key = f"{filt}_{idx:03d}" + store[key + "_xy"] = np.column_stack([objs["x"], objs["y"]]) + store[key + "_flux"] = objs["flux"] + meta.append((key, os.path.basename(path), filt, idx, len(objs), fwhm, + back_med, rms)) + print(f"{key:16s} stars={len(objs):4d} fwhm={fwhm:5.2f}px " + f"bg={back_med:8.1f} rms={rms:6.1f}") + del data, sub, bkg, objs + + np.savez_compressed( + CACHE, + meta=np.array(meta, dtype=object), + **store, + ) + print("cached ->", CACHE) diff --git a/pipeline/annotate.py b/pipeline/annotate.py new file mode 100644 index 0000000..9249197 --- /dev/null +++ b/pipeline/annotate.py @@ -0,0 +1,177 @@ +"""Annotated version of the finished LRGB: coordinate grid and catalogued objects. + +The overlay is driven entirely by the local plate solution, so every label sits +where the astrometry says it should. Objects come from SIMBAD, restricted to a +cone matching the field and to types worth marking (galaxies, clusters, radio +sources), then filtered again to those that actually fall inside the frame. + +The annotation is drawn on a downsampled copy: at full 15 Mpx the labels would +be microscopic relative to the image, and nobody views a 4692 px wide frame at +1:1 to read a caption. +""" +import os + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.wcs import WCS +from PIL import Image + +import layout + +OUT = layout.SESSION +CACHE = layout.path("_simbad.npz") +SCALE = 3 # downsample factor for the annotated render + +with fits.open(layout.path("NGC5128-LRGB.fit")) as hd: + hdr = hd[0].header +wcs = WCS(hdr, naxis=2) +rgb = np.asarray(Image.open(layout.path("NGC5128-LRGB.png"))) +ny, nx = rgb.shape[:2] +small = np.asarray(Image.fromarray(rgb).resize((nx // SCALE, ny // SCALE), + Image.LANCZOS)) +# Slicing a WCS rescales it correctly whether the solution is stored as CD or +# as PC + CDELT, which hand-editing the matrix does not. +wcs_small = wcs[::SCALE, ::SCALE] +centre = wcs.pixel_to_world(nx / 2, ny / 2) +print(f"frame {nx}x{ny} -> render {small.shape[1]}x{small.shape[0]}") + + +def simbad_objects(): + if os.path.exists(CACHE): + z = np.load(CACHE, allow_pickle=True) + return z["name"], z["ra"], z["dec"], z["otype"] + from astroquery.simbad import Simbad + sim = Simbad() + sim.ROW_LIMIT = 2000 + for field in ("otype", "V"): + try: + sim.add_votable_fields(field) + except Exception: # noqa: BLE001 + pass + tbl = sim.query_region(centre, radius=0.42 * u.deg) + name = np.array([str(r) for r in tbl[tbl.colnames[0]]]) + coords = SkyCoord(tbl["ra"], tbl["dec"], unit=(u.deg, u.deg)) + otype = np.array([str(t) for t in tbl["otype"]]) if "otype" in \ + tbl.colnames else np.array([""] * len(tbl)) + np.savez_compressed(CACHE, name=name, ra=coords.ra.deg, + dec=coords.dec.deg, otype=otype) + return name, coords.ra.deg, coords.dec.deg, otype + + +name, ra, dec, otype = simbad_objects() +print(f"{len(name)} SIMBAD entries in the cone") + +# SIMBAD returns 1518 rows for this field, the bulk of them anonymous entries +# from Centaurus A cluster and variable-star surveys. Marking those would bury +# the image, and the cluster system is being catalogued separately, so the +# overlay keeps only whole objects: other galaxies, planetary nebulae and +# anything carrying a mainstream catalogue designation. +GALAXY_TYPES = ("G", "GiG", "GiP", "GiC", "AGN", "SyG", "rG", "LSB") +# Confirmed nebulae only. SIMBAD lists 93 "PN?" candidates from a single +# survey of this field; they are unconfirmed, they are not visible at this +# depth, and marking them makes the image unreadable. +NEBULA_TYPES = ("PN", "HII", "SNR") +is_gal = np.isin(otype, GALAXY_TYPES) +is_neb = np.isin(otype, NEBULA_TYPES) +mainstream = np.array([n.startswith(("NGC", "IC ", "ESO", "PGC", "AM ", "SN ")) + and not n.startswith("SNR") + for n in name]) +sel = is_gal | is_neb | mainstream +sky = SkyCoord(ra[sel] * u.deg, dec[sel] * u.deg) +x, y = wcs_small.world_to_pixel(sky) +inside = (x > 40) & (x < small.shape[1] - 40) & (y > 40) & \ + (y < small.shape[0] - 40) +labels, kinds = name[sel][inside], otype[sel][inside] +x, y = x[inside], y[inside] +print(f"{len(labels)} catalogued objects inside the frame " + f"({is_gal[sel][inside].sum()} galaxies)") + +# Plain axes, not WCSAxes: WCSAxes insists on origin='lower', which would +# publish this image as a vertical mirror of every other deliverable. Drawing +# the graticule by hand keeps all the outputs in one orientation, and the +# lines still come from the plate solution rather than from assumption. +fig = plt.figure(figsize=(small.shape[1] / 100, small.shape[0] / 100), dpi=100) +ax = fig.add_axes([0, 0, 1, 1]) +ax.imshow(small, origin="upper") +ax.set_axis_off() + +corners = wcs_small.pixel_to_world( + [0, small.shape[1], 0, small.shape[1]], + [0, 0, small.shape[0], small.shape[0]]) +ra_lo, ra_hi = corners.ra.deg.min(), corners.ra.deg.max() +dec_lo, dec_hi = corners.dec.deg.min(), corners.dec.deg.max() + + +def draw_line(coord_ra, coord_dec, label, at_ra): + px, py = wcs_small.world_to_pixel(SkyCoord(coord_ra * u.deg, + coord_dec * u.deg)) + ok = (px > 0) & (px < small.shape[1]) & (py > 0) & (py < small.shape[0]) + if ok.sum() < 2: + return + ax.plot(px[ok], py[ok], color="#5fa8ff", alpha=0.30, linestyle=":", + linewidth=0.9) + i = np.where(ok)[0][len(np.where(ok)[0]) // 2] + ax.text(px[i], py[i], label, color="#8fc4ff", fontsize=8, + family="monospace", rotation=0 if at_ra else 90, + ha="center", va="center", + bbox=dict(boxstyle="round,pad=0.12", fc="black", ec="none", + alpha=0.45)) + + +RA_STEP = 15.0 / 60.0 # one minute of right ascension, in degrees +DEC_STEP = 10.0 / 60.0 # ten arcminutes +t = np.linspace(dec_lo, dec_hi, 400) +for r in np.arange(np.ceil(ra_lo / RA_STEP) * RA_STEP, ra_hi, RA_STEP): + c = SkyCoord(r * u.deg, 0 * u.deg) + draw_line(np.full_like(t, r), t, + f"{int(c.ra.hms.h):02d}h{int(c.ra.hms.m):02d}m", False) +s_ = np.linspace(ra_lo, ra_hi, 400) +for d in np.arange(np.ceil(dec_lo / DEC_STEP) * DEC_STEP, dec_hi, DEC_STEP): + dm = abs(d - int(d)) * 60 + draw_line(s_, np.full_like(s_, d), f"{int(d):+03d}d{dm:02.0f}m", True) + +for lx, ly, lab, kind in zip(x, y, labels, kinds): + colour = "#7ee08a" if kind in GALAXY_TYPES else "#ffd166" + ax.add_patch(plt.Circle((lx, ly), 15, fill=False, color=colour, + linewidth=1.2, alpha=0.95)) + ax.text(lx + 19, ly - 11, f"{lab} [{kind}]", color=colour, fontsize=7.5, + family="monospace", + bbox=dict(boxstyle="round,pad=0.12", fc="black", ec="none", + alpha=0.45)) + +# Scale bar: one arcminute, measured through the plate solution rather than +# assumed, plus the physical scale at Centaurus A's distance. +pix_per_arcmin = 60.0 / (0.5376 * SCALE) +bx, by = 60, small.shape[0] - 60 +ax.plot([bx, bx + pix_per_arcmin], [by, by], color="white", linewidth=2.5) +ax.text(bx, by - 12, "1' = 1.1 kpc at 3.8 Mpc", color="white", fontsize=9) + +# Orientation: north and east taken from the WCS, so a flipped or rotated +# solution cannot silently produce a wrong compass. +cx, cy = small.shape[1] - 120, small.shape[0] - 120 +c0 = wcs_small.pixel_to_world(cx, cy) +for dlab, offset in (("N", (0 * u.arcmin, 2 * u.arcmin)), + ("E", (2 * u.arcmin, 0 * u.arcmin))): + p = c0.spherical_offsets_by(*offset) + px, py = wcs_small.world_to_pixel(p) + ax.annotate("", xy=(px, py), xytext=(cx, cy), + arrowprops=dict(color="white", width=1.0, headwidth=6)) + ax.text(px, py, dlab, color="white", fontsize=11, ha="center", + va="center") + +ax.text(20, 26, "NGC 5128 (Centaurus A) iTelescope T32, Siding Spring " + "2026-07-21 L 12x300s RGB 4x300s each", + color="white", fontsize=10) +ax.text(20, 44, f"plate solved against Gaia DR3: {hdr.get('ASTRSOLV', '')}", + color="#9fb8d0", fontsize=8) +ax.set_xlim(0, small.shape[1]) +ax.set_ylim(small.shape[0], 0) + +path = layout.path("NGC5128-img-annotated.jpg") +fig.savefig(path, dpi=100, pil_kwargs={"quality": 92}) +print("wrote", path) diff --git a/pipeline/closeup.py b/pipeline/closeup.py new file mode 100644 index 0000000..e1175a1 --- /dev/null +++ b/pipeline/closeup.py @@ -0,0 +1,102 @@ +"""Image 4: the close-up, framed by the galaxy's own measured extent. + +The crop box is not chosen by eye. The surface photometry produced an isophote +model of NGC 5128, and the region where that model carries real signal defines +how much sky the galaxy actually occupies; the frame is that region plus a +margin. So "keeping the object in frame" is a measured statement rather than a +judgement, and the same rule would frame any other galaxy without retuning. + +It is cut from image 3's 16-bit TIFF rather than from the PNG, and never +resampled, so every pixel is exactly the pixel that came out of the pipeline. +Re-rendering was considered and rejected: the stretch in image 3 is already +anchored on sky statistics measured across the whole frame, and a tight crop +contains too little empty sky to measure a better one. Cropping is therefore +not a compromise here - it is the correct operation. +""" +import os + +import numpy as np +import tifffile +from astropy.io import fits +from astropy.wcs import WCS +from PIL import Image + +import layout + +OUT = layout.SESSION +# The finished renderings live in their own directory: they are the +# deliverables, and keeping them apart from the masters, the +# intermediates and the analysis figures makes it obvious which +# files are meant to be looked at. +FINAL = layout.path("final") +SOURCE = "NGC5128-final-3-best.tif" +CROP = 48 # the border already removed from image 3 +MARGIN = 0.08 # sky margin around the galaxy, as a fraction +MODEL_FLOOR = 5.0 # ADU/px: where the isophote model still has signal + + +def main(): + model = fits.getdata(layout.path("sb-model.fits")).astype(np.float32) + ys, xs = np.where(model > MODEL_FLOOR) + # Model coordinates are on the uncropped grid; image 3 lost CROP px. + x0, x1 = xs.min() - CROP, xs.max() - CROP + y0, y1 = ys.min() - CROP, ys.max() - CROP + print(f"galaxy extent (model > {MODEL_FLOOR} ADU/px): " + f"x {x0}-{x1}, y {y0}-{y1} = {x1 - x0} x {y1 - y0} px") + + with fits.open(layout.path("master-Luminance.fit")) as hd: + wcs = WCS(hd[0].header, naxis=2) + + img = tifffile.imread(layout.path(SOURCE)) + ny, nx = img.shape[:2] + print(f"source {SOURCE}: {nx} x {ny}, {img.dtype}") + + # Centre the frame on the galaxy's own centre, not on the frame's. + cx, cy = (x0 + x1) / 2.0, (y0 + y1) / 2.0 + half_x = (x1 - x0) / 2.0 * (1 + MARGIN) + half_y = (y1 - y0) / 2.0 * (1 + MARGIN) + # One box for both axes keeps the galaxy from touching a short edge, and a + # round object in a near-square frame reads better than in a letterbox. + half = max(half_x, half_y) + + # Clamp inside the image while keeping the galaxy centred as far as + # possible; report honestly if the frame has to shrink. + half = min(half, cx, cy, nx - cx, ny - cy) + left, right = int(round(cx - half)), int(round(cx + half)) + top, bottom = int(round(cy - half)), int(round(cy + half)) + print(f"crop box x {left}-{right}, y {top}-{bottom} " + f"({right - left} x {bottom - top} px)") + + # Verify the whole galaxy really is inside before writing anything. + assert left <= x0 and right >= x1 and top <= y0 and bottom >= y1, \ + "galaxy would be clipped - refusing to write" + inset_x = min(x0 - left, right - x1) + inset_y = min(y0 - top, bottom - y1) + print(f"clearance to the nearest edge: {inset_x} px horizontally, " + f"{inset_y} px vertically") + + scale = 0.5376 + print(f"field of view {(right - left) * scale / 60:.1f}' x " + f"{(bottom - top) * scale / 60:.1f}'") + corner = wcs.pixel_to_world(left + CROP, top + CROP) + centre = wcs.pixel_to_world((left + right) / 2 + CROP, + (top + bottom) / 2 + CROP) + print(f"frame centre {centre.to_string('hmsdms')}") + print(f"top-left corner {corner.to_string('hmsdms')}") + + crop = img[top:bottom, left:right] + tifffile.imwrite(layout.path("NGC5128-final-4-closeup.tif"), crop, + photometric="rgb") + u8 = (crop.astype(np.float32) / 65535.0 * 255 + 0.5).astype(np.uint8) + Image.fromarray(u8).save(layout.path("NGC5128-final-4-closeup.png")) + prev = Image.fromarray(u8) + prev.thumbnail((2400, 2400), Image.LANCZOS) + prev.save(layout.path("NGC5128-final-4-closeup-preview.jpg"), + quality=93) + print("wrote NGC5128-final-4-closeup.tif / .png / -preview.jpg") + + +os.makedirs(FINAL, exist_ok=True) + +if __name__ == "__main__": + main() diff --git a/pipeline/compose.py b/pipeline/compose.py new file mode 100644 index 0000000..90c37f6 --- /dev/null +++ b/pipeline/compose.py @@ -0,0 +1,256 @@ +"""Pass 4: turn the four masters into the finished LRGB image. + +The order of operations matters and is the usual one for a linear stack: + +1. Crop the registration border, where not every frame contributed. +2. Remove the sky gradient. A 46% moon was up about 30 degrees away, so each + channel carries a smooth ramp; a plane (not a higher-order surface) is fitted + to tiles OUTSIDE a generous ellipse around the galaxy, because Centaurus A's + halo fills much of this field and a flexible model would happily eat it. +3. Colour-calibrate on stars. Aperture photometry of a few hundred field stars + in R, G and B is scaled so their average colour is neutral. This is the + "average field star is grey" assumption, which is the standard cheap + substitute for a full photometric calibration and is well behaved here + because the field is rich. +4. Stretch. A midtone transfer function moves the sky background to a chosen + level while keeping the highlights unclipped: gentler on the core than a + plain gamma, and reversible arithmetic rather than a curve drawn by hand. +5. LRGB assembly. Colour comes from the 20-minute-per-channel RGB, detail and + noise from the 60-minute luminance: the RGB is scaled pixel-by-pixel to the + luminance's brightness, which is why the colour data being four times + shallower does not matter much. +""" +import os + +import numpy as np +import sep +import tifffile +from astropy.io import fits +from PIL import Image +from scipy.ndimage import gaussian_filter, median_filter +from skimage.restoration import denoise_tv_chambolle + +import layout + +OUT = layout.SESSION +CROP = 48 # registration border, in pixels +GALAXY_MASK = (1250, 1000) # semi-axes of the halo exclusion ellipse, px +BG_TARGET = 0.10 # where the sky sits in the stretched image +SATURATION = 1.35 +CHANNELS = ("Luminance", "Red", "Green", "Blue") + + +def load(): + data, hdr = {}, None + for name in CHANNELS: + with fits.open(layout.path(f"master-{name}.fit")) as hd: + arr = hd[0].data.astype(np.float32) + if hdr is None: + hdr = hd[0].header.copy() + data[name] = arr[CROP:-CROP, CROP:-CROP] + return data, hdr + + +def galaxy_mask(shape): + ny, nx = shape + yy, xx = np.mgrid[0:ny, 0:nx] + cy, cx = ny / 2.0, nx / 2.0 + a, b = GALAXY_MASK + return ((xx - cx) / a) ** 2 + ((yy - cy) / b) ** 2 < 1.0 + + +def remove_gradient(img, mask): + """Subtract a least-squares plane fitted to tile medians outside `mask`.""" + ny, nx = img.shape + step = 96 + xs, ys, zs = [], [], [] + for y0 in range(0, ny - step, step): + for x0 in range(0, nx - step, step): + tile = img[y0:y0 + step, x0:x0 + step] + if mask[y0:y0 + step, x0:x0 + step].any(): + continue + # The median of a tile is dominated by sky even with stars in it. + zs.append(np.median(tile)) + xs.append(x0 + step / 2.0) + ys.append(y0 + step / 2.0) + xs, ys, zs = map(np.asarray, (xs, ys, zs)) + keep = np.ones(len(zs), bool) + for _ in range(3): # clip tiles containing companions + A = np.column_stack([xs[keep], ys[keep], np.ones(keep.sum())]) + coef, *_ = np.linalg.lstsq(A, zs[keep], rcond=None) + model = coef[0] * xs + coef[1] * ys + coef[2] + resid = zs - model + s = 1.4826 * np.median(np.abs(resid - np.median(resid))) + keep = np.abs(resid - np.median(resid)) < 2.5 * s + yy, xx = np.mgrid[0:ny, 0:nx] + plane = (coef[0] * xx + coef[1] * yy + coef[2]).astype(np.float32) + return img - plane, coef, int(keep.sum()), len(zs) + + +def star_photometry(lum, channels): + """Aperture flux in each colour at the position of every luminance star.""" + bkg = sep.Background(lum, bw=64, bh=64, fw=3, fh=3) + sub = lum - bkg.back() + o = sep.extract(sub, 12.0, err=bkg.globalrms, minarea=9, + deblend_cont=0.005) + o = o[(o["flag"] == 0) & (o["npix"] > 12) & (o["npix"] < 800)] + ny, nx = lum.shape + cy, cx = ny / 2.0, nx / 2.0 + a, b = GALAXY_MASK + outside = ((o["x"] - cx) / a) ** 2 + ((o["y"] - cy) / b) ** 2 > 1.0 + o = o[outside] # keep the galaxy out of the white balance + o = o[np.argsort(o["flux"])[::-1][:500]] + flux = {} + for name in ("Red", "Green", "Blue"): + img = np.ascontiguousarray(channels[name]) + f, _, _ = sep.sum_circle(img, o["x"], o["y"], 6.0, subpix=5) + flux[name] = f + return o, flux + + +def mtf(x, midtone): + """PixInsight-style midtone transfer function on data already in [0, 1].""" + x = np.clip(x, 0.0, 1.0) + return ((midtone - 1.0) * x) / ((2.0 * midtone - 1.0) * x - midtone) + + +def autostretch(img, target=BG_TARGET, shadow_sigma=2.8): + """Black-point just below the sky, then an MTF that puts sky at `target`.""" + sky = np.median(img) + mad = 1.4826 * np.median(np.abs(img - sky)) + black = sky - shadow_sigma * mad + white = np.percentile(img, 99.995) + norm = np.clip((img - black) / (white - black), 0.0, 1.0) + sky_norm = (sky - black) / (white - black) + # Solve the MTF midtone that maps sky_norm exactly onto target. + m = ((target - 1.0) * sky_norm) / (2.0 * target * sky_norm - target - + sky_norm) + return mtf(norm, m), dict(black=float(black), white=float(white), + midtone=float(m), sky=float(sky), mad=float(mad)) + + +def main(): + data, hdr = load() + shape = data["Luminance"].shape + print(f"working frame {shape[1]} x {shape[0]} px after {CROP} px crop") + + mask = galaxy_mask(shape) + print(f"halo exclusion covers {mask.mean():.1%} of the frame") + for name in CHANNELS: + data[name], coef, kept, total = remove_gradient(data[name], mask) + print(f" {name:10s} plane dz/dx={coef[0]*1e3:+.3f} " + f"dz/dy={coef[1]*1e3:+.3f} ADU/kpx, offset {coef[2]:8.2f}, " + f"{kept}/{total} sky tiles used") + + o, flux = star_photometry(data["Luminance"], data) + good = (flux["Red"] > 0) & (flux["Green"] > 0) & (flux["Blue"] > 0) + r, g, b = (flux[k][good] for k in ("Red", "Green", "Blue")) + print(f"colour calibration on {good.sum()} field stars") + # Neutral point: the median star should come out white. + gr = np.median(g / r) + gb = np.median(g / b) + print(f" raw median star colour G/R={1/gr:.3f} G/B={1/gb:.3f}") + data["Red"] *= gr + data["Blue"] *= gb + + lum_lin = data["Luminance"] + rgb_lin = np.dstack([data["Red"], data["Green"], data["Blue"]]) + del data + + lum, params = autostretch(lum_lin) + print(f"luminance stretch: black={params['black']:.2f} " + f"white={params['white']:.1f} midtone={params['midtone']:.4f} " + f"(sky {params['sky']:.2f} +/- {params['mad']:.2f})") + + # The colour channels get their own black point but SHARE the luminance + # midtone, so the colour balance set above survives the stretch. + rgb = np.empty_like(rgb_lin) + for i in range(3): + ch = rgb_lin[:, :, i] + sky = np.median(ch) + mad = 1.4826 * np.median(np.abs(ch - sky)) + black = sky - 2.8 * mad + white = np.percentile(ch, 99.995) + rgb[:, :, i] = mtf(np.clip((ch - black) / (white - black), 0, 1), + params["midtone"]) + del rgb_lin + + # Neutralise the sky. Calibrating on stars makes STARS grey but leaves the + # residual sky tinted, because moonlight is blue-ish and each channel kept + # its own black point. Forcing the three sky medians together is what stops + # the empty parts of the frame reading brown. + sky_med = [float(np.median(rgb[:, :, i][~mask])) for i in range(3)] + target = float(np.mean(sky_med)) + print(f" sky medians after stretch R/G/B " + f"{sky_med[0]:.4f}/{sky_med[1]:.4f}/{sky_med[2]:.4f} -> {target:.4f}") + for i in range(3): + rgb[:, :, i] = np.clip(rgb[:, :, i] - (sky_med[i] - target), 0.0, 1.0) + + # Chroma noise is the ugliest part of a 20-minute colour stack. Blurring + # colour alone is invisible at this scale because the eye takes structure + # from the luminance, which is untouched. + rgb_lum = rgb.mean(axis=2, keepdims=True) + chroma = rgb - rgb_lum + for i in range(3): + chroma[:, :, i] = median_filter(chroma[:, :, i], size=3) + chroma[:, :, i] = gaussian_filter(chroma[:, :, i], 1.5) + rgb = np.clip(rgb_lum + chroma * SATURATION, 0.0, 1.0) + del chroma, rgb_lum + + # Gentle local contrast on the luminance to lift the dust lane, held back + # in the noise floor so the sky does not get grainier. + detail = lum - gaussian_filter(lum, 2.0) + weight = np.clip((lum - BG_TARGET) * 4.0, 0.0, 1.0) + lum = np.clip(lum + 0.35 * detail * weight, 0.0, 1.0) + + # Edge-preserving smoothing, applied ONLY where there is nothing but sky + # (the same weight, inverted). Structure and the galaxy halo keep their + # full resolution; the empty 70% of the frame loses its grain. + smooth = denoise_tv_chambolle(lum, weight=0.012) + lum = np.clip(lum * weight + smooth * (1.0 - weight), 0.0, 1.0) + del detail, weight, smooth + + # LRGB: keep the RGB hue, take the brightness from the deep luminance. + ratio = lum / np.maximum(rgb.mean(axis=2), 1e-5) + out = np.clip(rgb * ratio[:, :, None], 0.0, 1.0) + del ratio, rgb + + # Nothing in this field is genuinely green: no astronomical source between + # the H-alpha reds and the OIII blues emits there, so any green excess is + # noise or residual moonlight. Pulling green down to the neutral average + # wherever it exceeds it (the standard SCNR operation) is safe and takes + # the last of the cast out of the sky. + neutral = 0.5 * (out[:, :, 0] + out[:, :, 2]) + amount = 0.85 + green = out[:, :, 1] + out[:, :, 1] = np.where(green > neutral, + green * (1.0 - amount) + neutral * amount, green) + + out16 = (out * 65535.0 + 0.5).astype(np.uint16) + tif = layout.path("NGC5128-LRGB.tif") + tifffile.imwrite(tif, out16, photometric="rgb") + print("wrote", tif) + + png = layout.path("NGC5128-LRGB.png") + Image.fromarray((out * 255 + 0.5).astype(np.uint8)).save(png) + print("wrote", png) + + prev = Image.fromarray((out * 255 + 0.5).astype(np.uint8)) + prev.thumbnail((2000, 2000), Image.LANCZOS) + prevpath = layout.path("NGC5128-LRGB-preview.jpg") + prev.save(prevpath, quality=92) + print("wrote", prevpath, prev.size) + + hdr["CRPIX1"] = hdr.get("CRPIX1", 0) - CROP + hdr["CRPIX2"] = hdr.get("CRPIX2", 0) - CROP + hdr["NCOMBINE"] = (24, "frames across all filters") + hdr["EXPTOTAL"] = (7200.0, "[s] total integration, all filters") + hdr["COMMENT"] = "LRGB composite: L 12x300s, R/G/B 4x300s each" + cube = np.moveaxis((out * 65535).astype(np.uint16), 2, 0) + fitsout = layout.path("NGC5128-LRGB.fit") + fits.PrimaryHDU(cube, hdr).writeto(fitsout, overwrite=True) + print("wrote", fitsout) + + +if __name__ == "__main__": + main() diff --git a/pipeline/depth.py b/pipeline/depth.py new file mode 100644 index 0000000..62745f9 --- /dev/null +++ b/pipeline/depth.py @@ -0,0 +1,102 @@ +"""How deep did the luminance master actually go? + +Calibrates instrumental magnitudes against Gaia G, then reports the faintest +star still detected at 5 sigma. That number decides which follow-up analyses +are worth attempting on this data and which are wishful thinking. +""" +import os + +import numpy as np +import sep +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.wcs import WCS + +import layout + +OUT = layout.SESSION +CACHE = layout.path("_gaia_deep.npz") + +with fits.open(layout.path("master-Luminance.fit")) as hd: + img = hd[0].data.astype(np.float32) + hdr = hd[0].header +wcs = WCS(hdr, naxis=2) +ny, nx = img.shape + +bkg = sep.Background(img, bw=64, bh=64, fw=3, fh=3) +sub = img - bkg.back() +objs = sep.extract(sub, 3.0, err=bkg.globalrms, minarea=6, deblend_cont=0.005) +objs = objs[(objs["flag"] == 0) & (objs["npix"] > 6)] +flux, fluxerr, _ = sep.sum_circle(sub, objs["x"], objs["y"], 5.0, + err=bkg.globalrms, subpix=5) +snr = flux / np.maximum(fluxerr, 1e-9) +keep = (flux > 0) & (snr > 3) +objs, flux, snr = objs[keep], flux[keep], snr[keep] +print(f"{len(objs)} sources detected at SNR > 3 (rms {bkg.globalrms:.2f} ADU)") + +sky = wcs.pixel_to_world(objs["x"], objs["y"]) +centre = wcs.pixel_to_world(nx / 2, ny / 2) + +if os.path.exists(CACHE): + z = np.load(CACHE) + gra, gdec, gmag = z["ra"], z["dec"], z["g"] +else: + from astroquery.gaia import Gaia + Gaia.ROW_LIMIT = 60000 + job = Gaia.launch_job_async(f""" + SELECT ra, dec, phot_g_mean_mag FROM gaiadr3.gaia_source + WHERE 1 = CONTAINS(POINT('ICRS', ra, dec), + CIRCLE('ICRS', {centre.ra.deg}, {centre.dec.deg}, 0.42)) + AND phot_g_mean_mag IS NOT NULL AND phot_g_mean_mag < 20.5 + """) + t = job.get_results() + gra = np.asarray(t["ra"], float) + gdec = np.asarray(t["dec"], float) + gmag = np.asarray(t["phot_g_mean_mag"], float) + np.savez_compressed(CACHE, ra=gra, dec=gdec, g=gmag) +print(f"{len(gmag)} Gaia sources in the field, G down to {gmag.max():.2f}") + +gcoord = SkyCoord(gra * u.deg, gdec * u.deg) +idx, sep2d, _ = sky.match_to_catalog_sky(gcoord) +matched = sep2d.arcsec < 1.5 +print(f"{matched.sum()} detections matched to Gaia within 1.5\"") + +inst = -2.5 * np.log10(flux[matched]) +gm = gmag[idx[matched]] +# Fit the zero point on well exposed, unsaturated stars only. +fit = (gm > 12) & (gm < 17) & (snr[matched] > 20) +zp = float(np.median(gm[fit] - inst[fit])) +scatter = float(np.std(gm[fit] - inst[fit] - 0.0)) +print(f"zero point {zp:.3f} (G = inst + zp) from {fit.sum()} stars, " + f"scatter {scatter:.3f} mag") + +mag_all = -2.5 * np.log10(flux) + zp +# SNR falls monotonically with magnitude, so read the SNR=5 crossing off a +# running median rather than requiring sources to land in a narrow SNR bin. +order = np.argsort(mag_all) +ms, ss = mag_all[order], snr[order] +win = max(11, len(ms) // 60) +run_m = np.array([np.median(ms[i:i + win]) for i in range(0, len(ms) - win, win // 2)]) +run_s = np.array([np.median(ss[i:i + win]) for i in range(0, len(ss) - win, win // 2)]) +below = np.where(run_s < 5.0)[0] +lim5 = float(run_m[below[0]]) if len(below) else float(run_m[-1]) +print(f"limiting magnitude at SNR 5: G ~ {lim5:.2f} " + f"(SNR range {snr.min():.1f}-{snr.max():.0f})") +print(f"faintest detection: G ~ {mag_all.max():.2f} (SNR " + f"{snr[np.argmax(mag_all)]:.1f})") + +unmatched = ~matched +print(f"{unmatched.sum()} detections with NO Gaia counterpart " + f"({100 * unmatched.mean():.1f}% of sources)") +r_gal = np.hypot(objs["x"] - nx / 2, objs["y"] - ny / 2) * 0.5376 / 60.0 +near = unmatched & (r_gal < 12.0) & (mag_all > 18.0) & (mag_all < 22.0) +print(f" of those, {near.sum()} lie within 12' of the galaxy at " + f"G 18-22: the magnitude and radius range of Centaurus A's " + f"globular cluster system") + +# Surface brightness of the sky, a fair summary of how much the moon cost. +pixarea = 0.5376 ** 2 +sky_adu = float(np.median(bkg.back())) +print(f"sky background {sky_adu:.1f} ADU/px -> " + f"{zp - 2.5 * np.log10(max(sky_adu, 1e-6) / pixarea):.2f} mag/arcsec^2") diff --git a/pipeline/enhance.py b/pipeline/enhance.py new file mode 100644 index 0000000..f923d43 --- /dev/null +++ b/pipeline/enhance.py @@ -0,0 +1,195 @@ +"""Pass 5: three further renderings from the same masters. + +1. NGC5128-img-deconvolved - the luminance sharpened by Richardson-Lucy against a PSF + measured from the frame's own stars, then recombined into LRGB. The seeing + was 2.69 arcsec, so there is real detail to recover in the dust lane; the + deconvolution is deliberately stopped early and applied only where the + signal is strong, because RL amplifies noise and rings around bright stars + if it is let run. +2. NGC5128-img-core-print - a full-resolution crop of the galaxy for printing. + +The star/starless separation lives in starless.py, which works from this +script's output. + +The stretch, colour calibration and gradient handling are imported from +compose.py rather than re-implemented, so these renderings and the main image +cannot drift apart. +""" +import os +import sys + +import numpy as np +import sep +import tifffile +from astropy.io import fits +from PIL import Image +from scipy.ndimage import gaussian_filter, median_filter +from skimage.restoration import richardson_lucy + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import compose as C # noqa: E402 + +import layout + +OUT = C.OUT +RL_ITERS = 12 +PSF_BOX = 25 + + +def measure_psf(img): + """Median-stack cutouts of isolated stars to get the frame's own PSF.""" + bkg = sep.Background(img, bw=64, bh=64, fw=3, fh=3) + sub = img - bkg.back() + o = sep.extract(sub, 25.0, err=bkg.globalrms, minarea=9, + deblend_cont=0.005) + o = o[(o["flag"] == 0) & (o["npix"] > 15) & (o["npix"] < 400)] + ny, nx = img.shape + h = PSF_BOX // 2 + # Isolated stars only: a neighbour inside the cutout would drag the wings. + keep = [] + xs, ys = o["x"], o["y"] + for i in range(len(o)): + if not (h + 2 < xs[i] < nx - h - 2 and h + 2 < ys[i] < ny - h - 2): + continue + d = np.hypot(xs - xs[i], ys - ys[i]) + if np.sort(d)[1] < 3 * PSF_BOX: + continue + keep.append(i) + keep = keep[:120] + stack = [] + for i in keep: + cx, cy = int(round(xs[i])), int(round(ys[i])) + cut = sub[cy - h:cy + h + 1, cx - h:cx + h + 1].astype(np.float64) + peak = cut.max() + if peak > 0 and peak < 60000: # skip anything near saturation + stack.append(cut / cut.sum()) + psf = np.median(np.stack(stack), axis=0) + psf[psf < 0] = 0 + psf /= psf.sum() + print(f"PSF from {len(stack)} isolated stars, " + f"peak fraction {psf.max():.4f}") + return psf.astype(np.float32) + + +def star_mask(img, rms): + """Feathered mask over stars, used to keep deconvolution off them. + + Richardson-Lucy rings around any source whose profile is steeper than the + PSF model can account for, and on a star field that means a dark annulus + round every bright star. Excluding stars from the deconvolution entirely + is the standard cure: the galaxy is what needed sharpening anyway. + """ + bkg = sep.Background(img, bw=64, bh=64, fw=3, fh=3) + sub = img - bkg.back() + o = sep.extract(sub, 6.0, err=rms, minarea=6, deblend_cont=0.005) + o = o[o["npix"] < 3000] + ny, nx = img.shape + m = np.zeros((ny, nx), np.float32) + yy, xx = np.mgrid[-20:21, -20:21] + rr = np.hypot(xx, yy) + for x, y, npix, flux in zip(o["x"], o["y"], o["npix"], o["flux"]): + # Bright stars ring over a wider radius than faint ones. + r = float(np.clip(2.5 * np.sqrt(npix / np.pi) + 3.0, 5, 18)) + cx, cy = int(round(x)), int(round(y)) + x0, x1 = max(0, cx - 20), min(nx, cx + 21) + y0, y1 = max(0, cy - 20), min(ny, cy + 21) + patch = (rr <= r).astype(np.float32)[ + (y0 - cy + 20):(y1 - cy + 20), (x0 - cx + 20):(x1 - cx + 20)] + np.maximum(m[y0:y1, x0:x1], patch, out=m[y0:y1, x0:x1]) + print(f" star mask over {len(o)} sources, {m.mean():.2%} of the frame") + return np.clip(gaussian_filter(m, 2.5), 0, 1) + + +def deconvolve(img, psf): + """Richardson-Lucy on the bright signal, feathered back into the noise.""" + rms = 1.4826 * np.median(np.abs(img - np.median(img))) + pedestal = 5.0 * rms + positive = np.clip(img + pedestal, 1e-3, None).astype(np.float32) + scale = float(positive.max()) + out = richardson_lucy(positive / scale, psf, num_iter=RL_ITERS, + clip=False) * scale - pedestal + # Two weights multiply together: apply the result only where there is + # signal to sharpen, and only where there is no star to ring. + signal = np.clip((img - 3.0 * rms) / (20.0 * rms), 0.0, 1.0) + w = (signal * (1.0 - star_mask(img, rms))).astype(np.float32) + return (out * w + img * (1.0 - w)).astype(np.float32) + + +def build_rgb(lum_lin, rgb_lin): + """The colour half of compose.main(), reused verbatim in spirit.""" + lum, params = C.autostretch(lum_lin) + rgb = np.empty_like(rgb_lin) + for i in range(3): + ch = rgb_lin[:, :, i] + sky = np.median(ch) + mad = 1.4826 * np.median(np.abs(ch - sky)) + black = sky - 2.8 * mad + white = np.percentile(ch, 99.995) + rgb[:, :, i] = C.mtf(np.clip((ch - black) / (white - black), 0, 1), + params["midtone"]) + return lum, rgb + + +def main(): + data, hdr = C.load() + shape = data["Luminance"].shape + mask = C.galaxy_mask(shape) + for name in C.CHANNELS: + data[name], *_ = C.remove_gradient(data[name], mask) + o, flux = C.star_photometry(data["Luminance"], data) + good = (flux["Red"] > 0) & (flux["Green"] > 0) & (flux["Blue"] > 0) + gr = np.median(flux["Green"][good] / flux["Red"][good]) + gb = np.median(flux["Green"][good] / flux["Blue"][good]) + data["Red"] *= gr + data["Blue"] *= gb + + psf = measure_psf(data["Luminance"]) + print(f"deconvolving luminance, {RL_ITERS} Richardson-Lucy iterations") + lum_lin = deconvolve(data["Luminance"], psf) + rgb_lin = np.dstack([data["Red"], data["Green"], data["Blue"]]) + del data + + lum, rgb = build_rgb(lum_lin, rgb_lin) + del rgb_lin, lum_lin + + sky_med = [float(np.median(rgb[:, :, i][~mask])) for i in range(3)] + target = float(np.mean(sky_med)) + for i in range(3): + rgb[:, :, i] = np.clip(rgb[:, :, i] - (sky_med[i] - target), 0.0, 1.0) + rgb_lum = rgb.mean(axis=2, keepdims=True) + chroma = rgb - rgb_lum + for i in range(3): + chroma[:, :, i] = gaussian_filter(median_filter(chroma[:, :, i], 3), + 1.5) + rgb = np.clip(rgb_lum + chroma * C.SATURATION, 0.0, 1.0) + del chroma, rgb_lum + + ratio = lum / np.maximum(rgb.mean(axis=2), 1e-5) + out = np.clip(rgb * ratio[:, :, None], 0.0, 1.0) + del ratio, rgb + + neutral = 0.5 * (out[:, :, 0] + out[:, :, 2]) + green = out[:, :, 1] + out[:, :, 1] = np.where(green > neutral, green * 0.15 + neutral * 0.85, + green) + + u8 = (out * 255 + 0.5).astype(np.uint8) + Image.fromarray(u8).save(layout.path("NGC5128-img-deconvolved.png")) + tifffile.imwrite(layout.path("NGC5128-img-deconvolved.tif"), + (out * 65535 + 0.5).astype(np.uint16), photometric="rgb") + print("wrote NGC5128-img-deconvolved.png / .tif") + + # A crop for print, taken from the deconvolved version at full resolution. + ny, nx = out.shape[:2] + cw, ch = 2600, 1950 + crop = u8[ny // 2 - ch // 2:ny // 2 + ch // 2, + nx // 2 - cw // 2:nx // 2 + cw // 2] + Image.fromarray(crop).save(layout.path("NGC5128-img-core-print.jpg"), + quality=95) + print(f"wrote NGC5128-img-core-print.jpg ({cw}x{ch}, " + f"{cw * 0.5376 / 60:.1f}' x {ch * 0.5376 / 60:.1f}')") + del crop, u8 + + +if __name__ == "__main__": + main() diff --git a/pipeline/final.py b/pipeline/final.py new file mode 100644 index 0000000..214f0ed --- /dev/null +++ b/pipeline/final.py @@ -0,0 +1,311 @@ +"""The three renderings, from minimal to everything we learned. + +Same 24 subs, same alignment, same field, same crop. The ONLY variable is how +much is done to the pixels, so the three are directly comparable. + + NGC5128-final-1-stacked.png stack + a standard stretch, nothing else + NGC5128-final-2-processed.png conventional processing: gradient, colour, denoise + NGC5128-final-3-best.png the above, corrected by what the analyses established + +What image 3 does differently, and why each change is justified by a measured +result rather than by taste: + +1. **Background fit that does not eat the halo.** The surface photometry + measured the far field sitting at -17.9 ADU/px instead of zero: the plane + fitted for image 2 absorbed real halo light, because Centaurus A's halo + fills this field and there is no genuinely empty corner to fit to. Image 3 + fits `channel = a * galaxy_model + plane` simultaneously, using the isophote + model from the surface photometry, so the plane can only take the part that + is actually a gradient. The halo survives. + +2. **Photometric colour calibration.** Image 2 assumed the average field star + is grey. Image 3 uses Gaia BP-RP to pick the 1313 stars that are genuinely + solar-coloured and neutralises on those alone, which does not care what mix + of spectral types this particular field contains. + +3. **The core, recovered.** The nucleus was never saturated - it peaks at 1944 + ADU against a ~63000 clip. A second tone curve scaled to the galaxy rather + than to field stars restores the bulge gradient the single curve flattened. + +4. **Deconvolution that does not ring.** PSF measured from the frame's own + isolated stars, applied only where there is signal and never on a star. + +5. **Noise reduction that cannot eat clusters.** The globular cluster survey + showed this field contains 289 cluster candidates that look exactly like + faint stars. Smoothing is therefore driven by a Gaia star mask plus a + signal mask, so every compact source - foreground star or cluster - is + excluded from it. +""" +import os +import sys + +import numpy as np +import sep +import tifffile +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.wcs import WCS +from PIL import Image +from scipy.ndimage import gaussian_filter, median_filter +from skimage.restoration import denoise_tv_chambolle, richardson_lucy + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import compose as C # noqa: E402 +import enhance as E # noqa: E402 + +import layout + +SRC = layout.SESSION +OUT = layout.SESSION +# The finished renderings live in their own directory: they are the +# deliverables, and keeping them apart from the masters, the +# intermediates and the analysis figures makes it obvious which +# files are meant to be looked at. +FINAL = layout.path("final") +ORIG = layout.path("original") +CROP = 48 +FILTERS = ["Luminance", "Red", "Green", "Blue"] + + +def save(arr, stem, title): + os.makedirs(FINAL, exist_ok=True) + u8 = (np.clip(arr, 0, 1) * 255 + 0.5).astype(np.uint8) + Image.fromarray(u8).save(layout.path(f"{stem}.png")) + tifffile.imwrite(layout.path(f"{stem}.tif"), + (np.clip(arr, 0, 1) * 65535 + 0.5).astype(np.uint16), + photometric="rgb") + prev = Image.fromarray(u8) + prev.thumbnail((2400, 2400), Image.LANCZOS) + prev.save(layout.path(f"{stem}-preview.jpg"), quality=93) + print(f" wrote {stem}.png / .tif / -preview.jpg [{title}]") + + +def lrgb(lum, rgb): + """Take hue from the colour channels, brightness from the luminance.""" + ratio = lum / np.maximum(rgb.mean(axis=2), 1e-5) + return np.clip(rgb * ratio[:, :, None], 0.0, 1.0) + + +# --------------------------------------------------------------- image 1 +def image1(): + """Stack plus a standard stretch. No corrections of any kind. + + Uses the plain-mean alignment-only stack, so there is not even outlier + rejection: satellite trails and the moon's gradient are all present. The + stretch is the conventional one - a black point just below each channel's + own sky, a white point at its 99.995th percentile, and a single shared + midtone taken from the luminance so no colour balancing sneaks in through + the tone curve. + """ + print("image 1: stack + standard stretch") + data = {} + for f in FILTERS: + d = fits.getdata(layout.path(f"original-{f}.fit")) + data[f] = np.array(d[CROP:-CROP, CROP:-CROP], dtype=np.float32) + lum_lin = data["Luminance"] + lum, params = C.autostretch(lum_lin) + print(f" shared midtone {params['midtone']:.4f}") + rgb = np.empty(lum.shape + (3,), np.float32) + for i, f in enumerate(("Red", "Green", "Blue")): + ch = data[f] + sky = np.median(ch) + mad = 1.4826 * np.median(np.abs(ch - sky)) + black, white = sky - 2.8 * mad, np.percentile(ch, 99.995) + rgb[:, :, i] = C.mtf(np.clip((ch - black) / (white - black), 0, 1), + params["midtone"]) + print(f" {f:6s} black {black:7.1f} white {white:8.1f} ADU") + del data + out = lrgb(lum, rgb) + save(out, "NGC5128-final-1-stacked", "align + mean + stretch") + return out.shape + + +# --------------------------------------------------------------- image 3 +def solar_white_balance(lum, channels, wcs, shape): + """Neutralise on stars that are genuinely solar-coloured, per Gaia BP-RP.""" + z = np.load(layout.path("_gaia_colours.npz")) + solar = np.abs(z["bprp"] - 0.82) < 0.15 + sky = SkyCoord(z["ra"][solar] * u.deg, z["dec"][solar] * u.deg) + x, y = wcs.world_to_pixel(sky) + g = z["g"][solar] + ny, nx = shape + # Keep them away from the edges, off the galaxy's bright core, and out of + # saturation; faint ones carry too little signal in 20 minutes of colour. + ok = ((x > 40) & (x < nx - 40) & (y > 40) & (y < ny - 40) & + (g > 11.5) & (g < 16.5)) + x, y = x[ok], y[ok] + flux = {} + for f in ("Red", "Green", "Blue"): + img = np.ascontiguousarray(channels[f]) + fl, _, _ = sep.sum_circle(img, x, y, 6.0, subpix=5) + flux[f] = fl + good = (flux["Red"] > 0) & (flux["Green"] > 0) & (flux["Blue"] > 0) + gr = float(np.median(flux["Green"][good] / flux["Red"][good])) + gb = float(np.median(flux["Green"][good] / flux["Blue"][good])) + print(f" solar-analogue white balance on {int(good.sum())} stars: " + f"R x {gr:.4f}, B x {gb:.4f}") + return gr, gb + + +def fit_background(ch, model, star_mask): + """Solve ch = a*model + (plane) and subtract ONLY the plane. + + Fitting a plane on its own to a field this full of galaxy makes the plane + absorb halo light - measured at -17.9 ADU/px in the far field of image 2. + Including the galaxy model as a free component in the same least-squares + problem gives the fit something else to attribute that light to. + """ + ny, nx = ch.shape + ys, xs = np.mgrid[0:ny:8, 0:nx:8] + m = model[::8, ::8] + v = ch[::8, ::8] + keep = (star_mask[::8, ::8] == 0) & np.isfinite(v) + A = np.column_stack([m[keep], xs[keep] / nx, ys[keep] / ny, + np.ones(keep.sum())]) + coef, *_ = np.linalg.lstsq(A, v[keep], rcond=None) + for _ in range(3): # clip and refit + pred = A @ coef + r = v[keep] - pred + s = 1.4826 * np.median(np.abs(r - np.median(r))) + m2 = np.abs(r - np.median(r)) < 2.5 * s + coef, *_ = np.linalg.lstsq(A[m2], v[keep][m2], rcond=None) + yy, xx = np.mgrid[0:ny, 0:nx] + plane = (coef[1] * xx / nx + coef[2] * yy / ny + coef[3]).astype(np.float32) + print(f" model amplitude {coef[0]:.4f}, plane offset {coef[3]:8.2f} ADU") + return ch - plane + + +def image3(): + print("image 3: science-informed") + model_full = fits.getdata(layout.path("sb-model.fits")).astype( + np.float32) + star_full = fits.getdata(layout.path("sb-mask-stars.fits")) + with fits.open(layout.path("master-Luminance.fit")) as hd: + wcs_full = WCS(hd[0].header, naxis=2) + + data = {} + for f in FILTERS: + d = fits.getdata(layout.path(f"master-{f}.fit")) + data[f] = np.array(d, dtype=np.float32) + model = model_full + smask = star_full + print(" background fit with the galaxy model as a free component:") + for f in FILTERS: + print(f" {f}") + data[f] = fit_background(data[f], model, smask) + del model_full, star_full, model, smask + + shape_full = data["Luminance"].shape + gr, gb = solar_white_balance(data["Luminance"], data, wcs_full, shape_full) + data["Red"] *= gr + data["Blue"] *= gb + + for f in FILTERS: + data[f] = np.ascontiguousarray(data[f][CROP:-CROP, CROP:-CROP]) + shape = data["Luminance"].shape + + psf = E.measure_psf(data["Luminance"]) + print(" deconvolving luminance (star-protected)") + lum_lin = E.deconvolve(data["Luminance"], psf) + rgb_lin = np.dstack([data["Red"], data["Green"], data["Blue"]]) + del data + + # Two tone curves, blended: the faint one for sky and halo, one scaled to + # the galaxy for the core the single curve flattened. + ny, nx = shape + h = 500 + core = lum_lin[ny // 2 - h:ny // 2 + h, nx // 2 - h:nx // 2 + h] + peak = float(median_filter(core, size=41).max()) + lum_faint, params = C.autostretch(lum_lin) + hi = peak * 1.15 + lum_bright = C.mtf(np.clip((lum_lin - params["black"]) / + (hi - params["black"]), 0, 1), 0.35) + w = gaussian_filter(np.clip((lum_faint - 0.55) / 0.35, 0, 1).astype( + np.float32), 8.0) + lum = np.clip(lum_faint * (1 - w) + lum_bright * w, 0, 1) + print(f" galaxy peak {peak:.0f} ADU, HDR blend over " + f"{float((w > 0.05).mean()):.2%} of frame") + + rgb = np.empty_like(rgb_lin) + for i in range(3): + ch = rgb_lin[:, :, i] + sky = np.median(ch) + mad = 1.4826 * np.median(np.abs(ch - sky)) + black, white = sky - 2.8 * mad, np.percentile(ch, 99.995) + cf = C.mtf(np.clip((ch - black) / (white - black), 0, 1), + params["midtone"]) + pc = float(median_filter( + ch[ny // 2 - h:ny // 2 + h, nx // 2 - h:nx // 2 + h], 41).max()) + cb = C.mtf(np.clip((ch - black) / (pc * 1.15 - black), 0, 1), 0.35) + rgb[:, :, i] = cf * (1 - w) + cb * w + del rgb_lin + + # Sky neutralisation, measured where the galaxy model says there is no + # galaxy rather than outside an arbitrary ellipse. + mask = C.galaxy_mask(shape) + sky_med = [float(np.median(rgb[:, :, i][~mask])) for i in range(3)] + target = float(np.mean(sky_med)) + for i in range(3): + rgb[:, :, i] = np.clip(rgb[:, :, i] - (sky_med[i] - target), 0, 1) + + rgb_lum = rgb.mean(axis=2, keepdims=True) + chroma = rgb - rgb_lum + for i in range(3): + chroma[:, :, i] = gaussian_filter(median_filter(chroma[:, :, i], 3), + 1.5) + rgb = np.clip(rgb_lum + chroma * 1.4, 0, 1) + del chroma, rgb_lum + + detail = lum - gaussian_filter(lum, 2.0) + protect = np.clip((lum - 0.10) * 4.0, 0.0, 1.0) + lum = np.clip(lum + 0.35 * detail * protect, 0, 1) + + # Denoise the sky only. The cluster survey found 289 cluster candidates + # that look like faint stars, so compact sources are excluded from the + # smoothing along with the galaxy itself. + z = np.load(layout.path("_gaia_deep.npz")) + gx, gy = wcs_full.world_to_pixel(SkyCoord(z["ra"] * u.deg, + z["dec"] * u.deg)) + gx, gy = gx - CROP, gy - CROP + point = np.zeros(shape, np.float32) + R = 14 + yy, xx = np.mgrid[-R:R + 1, -R:R + 1] + rr = np.hypot(xx, yy) + for x, y, g in zip(gx, gy, z["g"]): + if not (R < x < nx - R and R < y < ny - R): + continue + r = float(np.clip(16.0 - 0.7 * (g - 8.0), 4, R - 1)) + cx, cy = int(x), int(y) + patch = (rr <= r).astype(np.float32) + sl = (slice(cy - R, cy + R + 1), slice(cx - R, cx + R + 1)) + np.maximum(point[sl], patch, out=point[sl]) + keep_sharp = np.clip(protect + gaussian_filter(point, 2.0), 0, 1) + smooth = denoise_tv_chambolle(lum, weight=0.012) + lum = np.clip(lum * keep_sharp + smooth * (1 - keep_sharp), 0, 1) + print(f" sky denoise applied to {float((keep_sharp < 0.5).mean()):.1%} " + f"of the frame; stars and clusters excluded") + del detail, protect, smooth, point, keep_sharp + + out = lrgb(lum, rgb) + neutral = 0.5 * (out[:, :, 0] + out[:, :, 2]) + green = out[:, :, 1] + out[:, :, 1] = np.where(green > neutral, green * 0.15 + neutral * 0.85, + green) + save(out, "NGC5128-final-3-best", "science-informed") + + +if __name__ == "__main__": + image1() + print("image 2: the conventional pipeline output (compose.py)") + os.makedirs(FINAL, exist_ok=True) + for ext in ("png", "tif"): + src = layout.path(f"NGC5128-LRGB.{ext}") + dst = layout.path(f"NGC5128-final-2-processed.{ext}") + with open(src, "rb") as a, open(dst, "wb") as b: + b.write(a.read()) + im = Image.open(layout.path("NGC5128-final-2-processed.png")) + im.thumbnail((2400, 2400), Image.LANCZOS) + im.save(layout.path("NGC5128-final-2-processed-preview.jpg"), quality=93) + print(" wrote NGC5128-final-2-processed.png / .tif / -preview.jpg") + image3() diff --git a/pipeline/gaia_colours.py b/pipeline/gaia_colours.py new file mode 100644 index 0000000..6dfe0f2 --- /dev/null +++ b/pipeline/gaia_colours.py @@ -0,0 +1,49 @@ +"""Fetch Gaia DR3 BP-RP colours for the field, cached for colour calibration. + +The first composite balanced colour on "the average field star is grey", which +is a workable fudge but is biased by whatever mix of spectral types the field +happens to contain. With real colours available, a much better anchor exists: +pick the stars that actually ARE solar-coloured (BP-RP near 0.82) and force +those to neutral. That is the same principle as a photometric colour +calibration, without needing the filters' response curves. + +ESA's archive was down during this work, so this uses the VizieR mirror of the +identical catalogue. +""" +import os + +import numpy as np +from astropy import units as u +from astropy.coordinates import SkyCoord + +import layout + +OUT = layout.SESSION +CACHE = layout.path("_gaia_colours.npz") +CENTRE = SkyCoord("13h25m27.37s", "-43d01m10.9s") + +if os.path.exists(CACHE): + z = np.load(CACHE) + print(f"cached: {len(z['ra'])} stars with colours") +else: + from astroquery.vizier import Vizier + v = Vizier(columns=["RA_ICRS", "DE_ICRS", "Gmag", "BP-RP"], + column_filters={"Gmag": "<18", "BP-RP": ">-1"}, + row_limit=50000) + res = v.query_region(CENTRE, radius=0.45 * u.deg, catalog="I/355/gaiadr3") + t = res[0] + ok = ~np.isnan(np.asarray(t["BP-RP"], float)) + ra = np.asarray(t["RA_ICRS"], float)[ok] + dec = np.asarray(t["DE_ICRS"], float)[ok] + g = np.asarray(t["Gmag"], float)[ok] + bprp = np.asarray(t["BP-RP"], float)[ok] + np.savez_compressed(CACHE, ra=ra, dec=dec, g=g, bprp=bprp) + print(f"fetched {len(ra)} stars with BP-RP") + z = dict(ra=ra, dec=dec, g=g, bprp=bprp) + +bprp = z["bprp"] +solar = np.abs(bprp - 0.82) < 0.15 +print(f"BP-RP range {bprp.min():.2f} to {bprp.max():.2f}, " + f"median {np.median(bprp):.2f}") +print(f"solar-coloured stars (BP-RP 0.67-0.97): {solar.sum()}") +print(f"G range {z['g'].min():.1f} to {z['g'].max():.1f}") diff --git a/pipeline/gc-bwtrial.py b/pipeline/gc-bwtrial.py new file mode 100644 index 0000000..10212ba --- /dev/null +++ b/pipeline/gc-bwtrial.py @@ -0,0 +1,75 @@ +"""Choose the background mesh size EMPIRICALLY, using the 589 SIMBAD-catalogued +Cen A globular clusters that fall in the field as a truth set. For each mesh +size we measure (a) how many known GCs are recovered, split by projected radius, +and (b) how many total detections there are (a proxy for spurious detections).""" +import os, numpy as np, sep, warnings +from astropy.io import fits +from astropy.wcs import WCS + +import layout +warnings.filterwarnings("ignore") +S = layout.SESSION +NUC_RA, NUC_DEC = 201.365063, -43.019113 +PIXSCALE = 0.5376 + + +def gk(fwhm=5.0): + sig = fwhm / 2.3548 + n = int(2 * round(3 * sig) + 1) + r = np.arange(n) - n // 2 + xx, yy = np.meshgrid(r, r) + k = np.exp(-(xx ** 2 + yy ** 2) / (2 * sig ** 2)) + return (k / k.sum()).astype(np.float32) + + +def match(ra1, d1, ra2, d2, tol_as): + """brute-force nearest match, small catalogues""" + cd = np.cos(np.radians(d1.mean())) + idx = np.full(len(ra1), -1) + sep_as = np.full(len(ra1), 9e9) + for i in range(len(ra1)): + dd = np.hypot((ra2 - ra1[i]) * cd, d2 - d1[i]) * 3600.0 + j = np.argmin(dd) + if dd[j] < tol_as: + idx[i] = j + sep_as[i] = dd[j] + return idx, sep_as + + +z = np.load(layout.path("_simbad.npz"), allow_pickle=True) +m = z["otype"] == "GlC" +gra, gdec = z["ra"][m].astype(float), z["dec"][m].astype(float) + +hdr = fits.getheader(layout.path("master-Luminance.fit")) +w = WCS(hdr) +ny, nx = hdr["NAXIS2"], hdr["NAXIS1"] +gx, gy = w.all_world2pix(gra, gdec, 0) +inf = (gx > 20) & (gx < nx - 20) & (gy > 20) & (gy < ny - 20) +gra, gdec, gx, gy = gra[inf], gdec[inf], gx[inf], gy[inf] +cx, cy = [float(v) for v in w.all_world2pix(NUC_RA, NUC_DEC, 0)] +grad = np.hypot(gx - cx, gy - cy) * PIXSCALE / 60.0 +print("known GCs inside the frame: %d (r range %.2f - %.2f arcmin)" + % (len(gra), grad.min(), grad.max())) + +d = fits.getdata(layout.path("master-Luminance.fit")).astype(np.float32) +d = np.ascontiguousarray(d) +sep.set_extract_pixstack(3000000) +K = gk() + +bins = [(0, 2), (2, 4), (4, 8), (8, 14), (14, 30)] +print("\n%6s %7s | %s" % ("mesh", "Ndet", " ".join("%4.0f-%-4.0f'" % b for b in bins))) +for bw in (12, 16, 24, 32, 48, 64, 128): + b = sep.Background(d, bw=bw, bh=bw, fw=3, fh=3) + ds = d - b.back() + rm = b.rms() + o = sep.extract(ds, 3.0, err=rm, minarea=5, filter_kernel=K, + filter_type="matched", deblend_nthresh=32, + deblend_cont=0.005, clean=True) + ora, odec = w.all_pix2world(o["x"], o["y"], 0) + idx, sp = match(gra, gdec, ora, odec, 2.0) + row = [] + for lo, hi in bins: + s = (grad >= lo) & (grad < hi) + row.append("%3d/%-3d" % ((idx[s] >= 0).sum(), s.sum())) + print("%6d %7d | %s" % (bw, len(o), " ".join(row))) + del b, ds, rm, o diff --git a/pipeline/gc-classify.py b/pipeline/gc-classify.py new file mode 100644 index 0000000..922951a --- /dev/null +++ b/pipeline/gc-classify.py @@ -0,0 +1,229 @@ +""" +gc-classify.py -- step 2. Turn raw detections into a globular cluster candidate +catalogue. + +Order of operations: + 1. quality cuts (SNR, sep flags, frame edge) + 2. photometric calibration of the R/G/B masters onto the Gaia BP/G/RP scale + using astrometrically-confirmed foreground stars + 3. foreground star rejection using Gaia DR3 ASTROMETRY (not mere presence in + Gaia -- see the notes; half of the known Cen A clusters are in Gaia) + 4. morphology: point-like vs extended, calibrated on the stellar locus + 5. magnitude window around the expected GC luminosity function + 6. cross-match to SIMBAD (truth set) and SCABS/Taylor+2017 + +Outputs: _gc_cat.npz (everything), gc-candidates.csv (the candidates) +""" +import os, sys, numpy as np +from astropy.coordinates import SkyCoord +import astropy.units as u +import warnings + +import layout +warnings.filterwarnings("ignore") + +S = layout.SESSION +PIXSCALE = 0.5376 +NUC_RA, NUC_DEC = 201.365063, -43.019113 +DIST_MPC = 3.8 +KPC_PER_ARCMIN = DIST_MPC * 1000.0 * (np.pi / 180.0) / 60.0 # 1.105 +MATCH_AS = 1.5 # astrometric match radius +PM_SIG = 4.0 # significance above which astrometry says "foreground" +# Faint limit set by the artificial-star tests, not by the nominal SNR-5 depth: +# recovery of injected point sources collapses from ~30% at G=19.75 to ~3% at +# G=20.25, and the fraction of candidates confirmed by published catalogues +# falls from 52% to 10% across the same step. Beyond G=20 we are not measuring +# clusters, we are measuring noise. +MAG_LO, MAG_HI = 17.5, 20.0 + + +def xmatch(c1, c2, tol_as): + i, d2, _ = c1.match_to_catalog_sky(c2) + ok = d2.arcsec < tol_as + return i, d2.arcsec, ok + + +def main(): + d = dict(np.load(layout.path("_gc_raw.npz"))) + n = len(d["x"]) + print("raw detections: %d" % n) + cat = SkyCoord(d["ra"], d["dec"], unit="deg") + nuc = SkyCoord(NUC_RA, NUC_DEC, unit="deg") + r_arcmin = cat.separation(nuc).arcmin + r_kpc = r_arcmin * KPC_PER_ARCMIN + + # ---------------------------------------------------------- 1. quality + ny, nx = 3194, 4788 + edge = 30 + q_edge = (d["x"] > edge) & (d["x"] < nx - edge) & (d["y"] > edge) & (d["y"] < ny - edge) + # sep flag bits: 1=OBJ_MERGED 2=OBJ_TRUNC 4=OBJ_DOVERFLOW 8=OBJ_SINGU + # Bit 1 (had a neighbour in the detection footprint before deblending) is + # NOT a defect -- it fires for a third of the KNOWN clusters, because the + # inner field is crowded. Reject only genuine defects. + q_flag = (d["sepflag"].astype(int) & 0b1110) == 0 + q_snr = d["snr"] >= 5.0 + q_pos = d["flux"] > 0 + good = q_edge & q_pos + print(" in-frame & positive flux: %d ; +clean sep flags: %d ; +SNR>=5: %d" + % (good.sum(), (good & q_flag).sum(), (good & q_flag & q_snr).sum())) + + # ---------------------------------------------------------- 2. Gaia match + ga = np.load(layout.path("_gaia_astrom.npz")) + gcat = SkyCoord(ga["ra"], ga["dec"], unit="deg") + gi, gsep, ghit = xmatch(cat, gcat, MATCH_AS) + print(" Gaia DR3 counterpart within %.1f\": %d / %d" % (MATCH_AS, ghit.sum(), n)) + + plx, eplx = ga["plx"][gi], ga["eplx"][gi] + pmr, epmr = ga["pmra"][gi], ga["epmra"][gi] + pmd, epmd = ga["pmdec"][gi], ga["epmdec"][gi] + with np.errstate(invalid="ignore", divide="ignore"): + sig_plx = np.abs(plx / eplx) + sig_pm = np.sqrt((pmr / epmr) ** 2 + (pmd / epmd) ** 2) + has_astrom = ghit & np.isfinite(sig_pm) + # a Milky Way star: significant parallax OR significant proper motion + is_star = has_astrom & ((sig_plx > PM_SIG) | (sig_pm > PM_SIG)) + # a Gaia source whose astrometry is consistent with zero -> distant + is_stationary = has_astrom & ~is_star + no_gaia = ~ghit + no_astrom = ghit & ~has_astrom # in Gaia but 2-parameter solution only + print(" -> astrometric foreground stars: %d" % is_star.sum()) + print(" -> Gaia sources astrometrically stationary: %d" % is_stationary.sum()) + print(" -> in Gaia, no astrometry: %d ; not in Gaia: %d" % (no_astrom.sum(), no_gaia.sum())) + + gaia_g = np.where(ghit, ga["g"][gi], np.nan) + gaia_bprp = np.where(ghit, ga["bprp"][gi], np.nan) + + # ------------------------------------------- 3. calibrate R/G/B onto Gaia + zp = {} + for key, gband in (("B", "bp"), ("V", "g"), ("R", "rp")): + f = d["flux_" + key] + ref = np.where(ghit, ga[gband][gi], np.nan) + m = is_star & good & q_flag & (f > 0) & np.isfinite(ref) & (ref > 13) & (ref < 18) + with np.errstate(invalid="ignore", divide="ignore"): + off = ref[m] + 2.5 * np.log10(f[m]) + zp[key] = np.median(off) + print(" ZP_%s = %.3f (N=%d stars, scatter %.3f mag)" + % (key, zp[key], m.sum(), np.std(off - np.median(off)))) + mags = {} + magerrs = {} + for key in ("B", "V", "R"): + f = np.clip(d["flux_" + key], 1e-9, None) + with np.errstate(invalid="ignore", divide="ignore"): + mags[key] = np.where(d["flux_" + key] > 0, -2.5 * np.log10(f) + zp[key], np.nan) + magerrs[key] = 1.0857 * d["fluxerr_" + key] / f + BmR = mags["B"] - mags["R"] + BmV = mags["B"] - mags["V"] + VmR = mags["V"] - mags["R"] + + # ---------------------------------------------------------- 4. morphology + fwhm = 2.0 * d["rhalf"] # Gaussian: r_half = FWHM/2 + with np.errstate(invalid="ignore", divide="ignore"): + elong = d["a"] / np.clip(d["b"], 1e-6, None) + # stellar locus from bright unsaturated confirmed stars + ref = is_star & good & q_flag & (d["mag"] > 15) & (d["mag"] < 18) + f_med = np.median(fwhm[ref]); f_sig = 1.4826 * np.median(np.abs(fwhm[ref] - f_med)) + print(" stellar locus: FWHM = %.2f +/- %.2f px (%.2f\") from %d stars" + % (f_med, f_sig, f_med * PIXSCALE, ref.sum())) + # Asymmetric band, on purpose. A Cen A cluster (half-light radius a few pc, + # i.e. <0.2") is only marginally broadened by 2.7" seeing, but crowding and + # residual galaxy structure push measured sizes UP, never down. Known + # clusters run to FWHM ~6.6 px while stars sit at 5.0. So the lower edge is + # tight (rejects cosmic rays / noise spikes) and the upper edge is loose + # (rejects only obviously extended objects, i.e. resolved background + # galaxies). With 2.7" seeing this separation is weak -- see the notes. + FW_MIN = f_med - 3.0 * f_sig + FW_MAX = f_med + 3.0 # px, ~1.6 x the stellar FWHM + compact = (fwhm > FW_MIN) & (fwhm < FW_MAX) & (elong < 2.0) + print(" point-like acceptance band: %.2f < FWHM < %.2f px ; elong < 2.0" + % (FW_MIN, FW_MAX)) + + # ---------------------------------------------------------- 5. magnitude + inmag = (d["mag"] > MAG_LO) & (d["mag"] < MAG_HI) + + # ---------------------------------------------------------- selection + base = good & q_flag & q_snr + cand = base & ~is_star & compact & inmag + print("\nCANDIDATES: %d" % cand.sum()) + + why = np.array(["-"] * n, dtype=object) + why[cand & no_gaia] = "no-gaia+compact" + why[cand & is_stationary] = "gaia-stationary+compact" + why[cand & no_astrom] = "gaia-noastrom+compact" + + cls = np.array(["other"] * n, dtype=object) + cls[base & is_star] = "foreground-star" + cls[base & ~is_star & ~compact & inmag] = "extended" + cls[cand] = "gc-candidate" + + # ---------------------------------------------------------- 6. truth sets + z = np.load(layout.path("_simbad.npz"), allow_pickle=True) + sim_ot = z["otype"].astype(str) + scat = SkyCoord(z["ra"].astype(float), z["dec"].astype(float), unit="deg") + si, ssep, shit = xmatch(cat, scat, 2.0) + simbad_type = np.where(shit, sim_ot[si], "") + simbad_name = np.where(shit, z["name"].astype(str)[si], "") + + sc_type = np.array([""] * n, dtype=object) + scabs_ok = os.path.exists(layout.path("_cena_gc_ref.npz")) + scabs_prob = np.full(n, np.nan) + scabs_v = np.full(n, np.nan) + scabs_hit = np.zeros(n, bool) # positional match, regardless of listed prob + if scabs_ok: + r = np.load(layout.path("_cena_gc_ref.npz")) + rc = SkyCoord(r["ra"], r["dec"], unit="deg") + ri, rsep, rhit = xmatch(cat, rc, 2.0) + scabs_prob = np.where(rhit, r["prob"][ri], np.nan) + # the published table uses -1 as a "no value" sentinel + scabs_prob = np.where(scabs_prob < 0, np.nan, scabs_prob) + scabs_v = np.where(rhit, r["vmag"][ri], np.nan) + scabs_hit = rhit + print(" SCABS (Taylor+2017) matches among all detections: %d" % rhit.sum()) + print(" SCABS matches among candidates: %d / %d (%.0f%%)" + % ((rhit & cand).sum(), cand.sum(), 100 * (rhit & cand).mean() / max(cand.mean(), 1e-9))) + print(" SIMBAD GlC matches among candidates: %d" + % ((simbad_type == "GlC") & cand).sum()) + + out = dict(d) + out.update(r_arcmin=r_arcmin, r_kpc=r_kpc, fwhm=fwhm, elong=elong, + mag_B=mags["B"], mag_V=mags["V"], mag_R=mags["R"], + magerr_B=magerrs["B"], magerr_V=magerrs["V"], magerr_R=magerrs["R"], + BmR=BmR, BmV=BmV, VmR=VmR, + gaia_g=gaia_g, gaia_bprp=gaia_bprp, gaia_sep=gsep, + sig_plx=np.where(ghit, sig_plx, np.nan), + sig_pm=np.where(ghit, sig_pm, np.nan), + is_star=is_star, is_stationary=is_stationary, no_gaia=no_gaia, + no_astrom=no_astrom, compact=compact, cand=cand, base=base, + good=good, q_flag=q_flag, q_snr=q_snr, inmag=inmag, + simbad_type=simbad_type.astype(str), simbad_name=simbad_name.astype(str), + scabs_prob=scabs_prob, scabs_v=scabs_v, scabs_hit=scabs_hit, + why=why.astype(str), cls=cls.astype(str), + FW_MIN=FW_MIN, FW_MAX=FW_MAX, f_med=f_med, f_sig=f_sig, + zpB=zp["B"], zpV=zp["V"], zpR=zp["R"]) + np.savez(layout.path("_gc_cat.npz"), **out) + + # ---------------------------------------------------------- CSV + idx = np.where(cand)[0] + order = idx[np.argsort(r_arcmin[idx])] + lines = ["id,ra_deg,dec_deg,x_px,y_px,G_mag,G_magerr,snr,mag_B,mag_V,mag_R," + "B_R,B_V,V_R,fwhm_px,fwhm_arcsec,elong,r_arcmin,r_kpc,keep_flag," + "simbad_type,simbad_name,scabs_prob,scabs_V"] + for k, i in enumerate(order, 1): + def f(v, p=3): + return "" if not np.isfinite(v) else ("%.*f" % (p, v)) + lines.append(",".join([ + "GCC%04d" % k, "%.6f" % d["ra"][i], "%.6f" % d["dec"][i], + "%.2f" % d["x"][i], "%.2f" % d["y"][i], + f(d["mag"][i]), f(d["magerr"][i]), "%.1f" % d["snr"][i], + f(mags["B"][i]), f(mags["V"][i]), f(mags["R"][i]), + f(BmR[i]), f(BmV[i]), f(VmR[i]), + "%.2f" % fwhm[i], "%.2f" % (fwhm[i] * PIXSCALE), "%.2f" % elong[i], + "%.3f" % r_arcmin[i], "%.3f" % r_kpc[i], why[i], + simbad_type[i], '"%s"' % simbad_name[i], + f(scabs_prob[i], 2), f(scabs_v[i], 2)])) + with open(layout.path("gc-candidates.csv"), "w") as fh: + fh.write("\n".join(lines) + "\n") + print("wrote gc-candidates.csv (%d rows)" % len(order)) + + +if __name__ == "__main__": + main() diff --git a/pipeline/gc-complete-inner.py b/pipeline/gc-complete-inner.py new file mode 100644 index 0000000..1b057a9 --- /dev/null +++ b/pipeline/gc-complete-inner.py @@ -0,0 +1,120 @@ +""" +gc-complete-inner.py -- step 3b. Artificial stars concentrated on the INNER field. + +The uniform injection run (gc-complete.py) puts only ~2% of its fakes inside +3 arcmin, because that annulus is a tiny fraction of the frame area. That left +the innermost completeness bins under-sampled, and the correction visibly failed +to flatten the foreground-star control inside ~4 arcmin. This run injects only +within r < 7 arcmin so the inner bins are properly measured. The two runs are +merged into _gc_complete_all.npz. +""" +import os, time, gc, numpy as np, sep, warnings +from astropy.io import fits +from astropy.wcs import WCS + +import layout +warnings.filterwarnings("ignore") + +S = layout.SESSION +BW, FW, THRESH, MINAREA, APR = 16, 3, 3.0, 5, 5.0 +ZP_L, PIXSCALE = 27.941, 0.5376 +NUC_RA, NUC_DEC = 201.365063, -43.019113 +NRUN, NPER = 6, 900 +MAGS = (17.5, 20.5) +RMAX = 7.0 +HALF = 15 +RNG = np.random.default_rng(31415) + + +def gk(fwhm=5.0): + sig = fwhm / 2.3548 + n = int(2 * round(3 * sig) + 1) + r = np.arange(n) - n // 2 + xx, yy = np.meshgrid(r, r) + k = np.exp(-(xx ** 2 + yy ** 2) / (2 * sig ** 2)) + return (k / k.sum()).astype(np.float32) + + +def main(): + t0 = time.time() + hdr = fits.getheader(layout.path("master-Luminance.fit")) + w = WCS(hdr) + ny, nx = hdr["NAXIS2"], hdr["NAXIS1"] + cx, cy = [float(v) for v in w.all_world2pix(NUC_RA, NUC_DEC, 0)] + cat = dict(np.load(layout.path("_gc_cat.npz"), allow_pickle=True)) + FW_MIN, FW_MAX = float(cat["FW_MIN"]), float(cat["FW_MAX"]) + psf = np.load(layout.path("_gc_psf.npy")) + apfrac = float(np.load(layout.path("_gc_complete.npz"))["apfrac"]) + sep.set_extract_pixstack(1000000) + K = gk() + rmax_px = RMAX * 60 / PIXSCALE + rm_, rr_, ro_ = [], [], [] + + for run in range(NRUN): + # re-read rather than keeping a pristine copy in memory: only ~3.5 GB + # is free and each master is 61 MB + img = np.ascontiguousarray( + fits.getdata(layout.path('master-Luminance.fit')).astype(np.float32)) + # uniform in area within the circle, clipped to the frame + th = RNG.uniform(0, 2 * np.pi, NPER * 3) + rad = rmax_px * np.sqrt(RNG.uniform(0, 1, NPER * 3)) + xs = cx + rad * np.cos(th); ys = cy + rad * np.sin(th) + ok = ((xs > HALF + 5) & (xs < nx - HALF - 5) & + (ys > HALF + 5) & (ys < ny - HALF - 5)) + xs, ys = xs[ok][:NPER], ys[ok][:NPER] + mags = RNG.uniform(*MAGS, len(xs)) + ftot = 10 ** (-0.4 * (mags - ZP_L)) / apfrac + for xi, yi, f in zip(xs, ys, ftot): + ix, iy = int(round(xi)), int(round(yi)) + img[iy - HALF:iy + HALF + 1, ix - HALF:ix + HALF + 1] += f * psf + + b = sep.Background(img, bw=BW, bh=BW, fw=FW, fh=FW) + ds = img - b.back(); rmm = b.rms() + o = sep.extract(ds, THRESH, err=rmm, minarea=MINAREA, filter_kernel=K, + filter_type="matched", deblend_nthresh=32, + deblend_cont=0.005, clean=True) + fl, fe, _ = sep.sum_circle(ds, o["x"], o["y"], APR, err=rmm, subpix=5) + fl2, _, _ = sep.sum_circle(ds, o["x"], o["y"], 2 * APR, err=rmm, subpix=5) + rh, _ = sep.flux_radius(ds, o["x"], o["y"], np.full(len(o), 6 * APR), 0.5, + normflux=fl2, subpix=5) + fet = np.sqrt(fe ** 2 + np.clip(fl, 0, None) / (0.2467 * 12.0)) + snr = fl / np.clip(fet, 1e-9, None) + omag = -2.5 * np.log10(np.clip(fl, 1e-9, None)) + ZP_L + ofw, oel = 2 * rh, o["a"] / np.clip(o["b"], 1e-6, None) + sel = ((snr >= 5) & ((o["flag"].astype(int) & 0b1110) == 0) + & (ofw > FW_MIN) & (ofw < FW_MAX) & (oel < 2.0) + & (omag > 17.5) & (omag < 20.0) & (fl > 0)) + ox, oy = o["x"][sel], o["y"][sel] + for xi, yi, mg in zip(xs, ys, mags): + dd = np.hypot(ox - xi, oy - yi) + rm_.append(mg) + rr_.append(np.hypot(xi - cx, yi - cy) * PIXSCALE / 60.0) + ro_.append(bool(dd.min() < 2.0) if len(dd) else False) + print(" inner run %d/%d (%.0f s)" % (run + 1, NRUN, time.time() - t0), flush=True) + del img, b, ds, rmm, o, fl, fe, fl2, rh, fet, snr, omag, ofw, oel, ox, oy + gc.collect() + + rm_, rr_, ro_ = np.array(rm_), np.array(rr_), np.array(ro_) + np.savez(layout.path("_gc_complete_inner.npz"), mag=rm_, rad=rr_, ok=ro_) + + u = np.load(layout.path("_gc_complete.npz")) + np.savez(layout.path("_gc_complete_all.npz"), + mag=np.concatenate([u["mag"], rm_]), + rad=np.concatenate([u["rad"], rr_]), + ok=np.concatenate([u["ok"], ro_]), apfrac=u["apfrac"]) + print("\ninner injections %d, recovered %.1f%%" % (len(ro_), 100 * ro_.mean())) + print("inner completeness grid (rows mag, cols radius arcmin):") + rb = [0, 1.5, 3, 4.5, 6, 7] + mb = np.arange(17.5, 20.51, 0.5) + print(" " + "".join("%8s" % ("%.1f-%.1f" % (rb[i], rb[i + 1])) + for i in range(len(rb) - 1))) + for j in range(len(mb) - 1): + row = "%4.1f " % mb[j] + for i in range(len(rb) - 1): + s = ((rm_ >= mb[j]) & (rm_ < mb[j + 1]) & (rr_ >= rb[i]) & (rr_ < rb[i + 1])) + row += "%8s" % ("%.2f" % ro_[s].mean() if s.sum() > 15 else "-") + print(row) + + +if __name__ == "__main__": + main() diff --git a/pipeline/gc-complete.py b/pipeline/gc-complete.py new file mode 100644 index 0000000..2c7cca4 --- /dev/null +++ b/pipeline/gc-complete.py @@ -0,0 +1,160 @@ +""" +gc-complete.py -- step 3. Artificial star tests. + +Why this exists: NGC 5128's light makes the noise rise steeply toward the +nucleus, so the SAME cluster is harder to detect at r=2' than at r=20'. Without +correcting for that, the measured radial density profile is the true profile +MULTIPLIED by an unknown, radially-decreasing completeness -- which flattens or +even inverts a real central concentration. So we measure the completeness +directly: inject fake point sources of known magnitude at known positions into +the real luminance image, re-run the identical detection and selection chain, +and count how many come back, as a function of magnitude and projected radius. + +The PSF is empirical, built by median-stacking bright isolated stars from the +image itself. + +Output: _gc_complete.npz (recovery grid), and the PSF stamp. +""" +import os, time, numpy as np, sep, warnings +from astropy.io import fits +from astropy.wcs import WCS + +import layout +warnings.filterwarnings("ignore") + +S = layout.SESSION +BW, FW, THRESH, MINAREA, APR = 16, 3, 3.0, 5, 5.0 +ZP_L = 27.941 +PIXSCALE = 0.5376 +NUC_RA, NUC_DEC = 201.365063, -43.019113 +NRUN = 6 # injection runs +NPER = 2500 # fakes per run +MAGS = (17.5, 22.5) +HALF = 15 # PSF stamp half-size, px +RNG = np.random.default_rng(20260721) + + +def gk(fwhm=5.0): + sig = fwhm / 2.3548 + n = int(2 * round(3 * sig) + 1) + r = np.arange(n) - n // 2 + xx, yy = np.meshgrid(r, r) + k = np.exp(-(xx ** 2 + yy ** 2) / (2 * sig ** 2)) + return (k / k.sum()).astype(np.float32) + + +def build_psf(data, cat): + """median stack of bright, isolated, unsaturated stars""" + m = (cat["base"] & cat["is_star"] & (cat["mag"] > 14.5) & (cat["mag"] < 17.0) + & (cat["r_arcmin"] > 8) & (cat["fwhm"] < 5.6)) + x, y = cat["x"][m], cat["y"][m] + # isolation: no other detection within 20 px + ax, ay = cat["x"], cat["y"] + keep = [] + for i in range(len(x)): + dd = np.hypot(ax - x[i], ay - y[i]) + if (dd < 20).sum() <= 1: + keep.append(i) + x, y = x[keep], y[keep] + print(" PSF from %d isolated bright stars" % len(x)) + stamps = [] + for xi, yi in zip(x, y): + ix, iy = int(round(xi)), int(round(yi)) + if ix < HALF or iy < HALF or ix >= data.shape[1] - HALF or iy >= data.shape[0] - HALF: + continue + st = data[iy - HALF:iy + HALF + 1, ix - HALF:ix + HALF + 1].astype(np.float64) + st = st - np.median(st[[0, -1], :]) + s = st.sum() + if s > 0: + stamps.append(st / s) + psf = np.median(np.array(stamps), axis=0) + psf /= psf.sum() + # normalise so that a source of total flux F injected as F*psf measures + # F_ap through the r=5 px aperture; we need the aperture correction + yy, xx = np.mgrid[-HALF:HALF + 1, -HALF:HALF + 1] + apfrac = psf[np.hypot(xx, yy) <= APR].sum() + print(" aperture fraction inside r=%.0f px: %.4f" % (APR, apfrac)) + return psf.astype(np.float32), apfrac + + +def main(): + t0 = time.time() + hdr = fits.getheader(layout.path("master-Luminance.fit")) + w = WCS(hdr) + ny, nx = hdr["NAXIS2"], hdr["NAXIS1"] + cx, cy = [float(v) for v in w.all_world2pix(NUC_RA, NUC_DEC, 0)] + + cat = dict(np.load(layout.path("_gc_cat.npz"), allow_pickle=True)) + FW_MIN, FW_MAX = float(cat["FW_MIN"]), float(cat["FW_MAX"]) + + base_img = np.ascontiguousarray( + fits.getdata(layout.path("master-Luminance.fit")).astype(np.float32)) + psf, apfrac = build_psf(base_img, cat) + np.save(layout.path("_gc_psf.npy"), psf) + + sep.set_extract_pixstack(1000000) + K = gk() + rec_mag, rec_rad, rec_ok = [], [], [] + + for run in range(NRUN): + img = base_img.copy() + # positions: uniform over the frame, min 40 px apart from each other + xs = RNG.uniform(HALF + 5, nx - HALF - 5, NPER) + ys = RNG.uniform(HALF + 5, ny - HALF - 5, NPER) + mags = RNG.uniform(*MAGS, NPER) + # total flux such that the r=5px aperture measures the intended mag + ftot = 10 ** (-0.4 * (mags - ZP_L)) / apfrac + for xi, yi, f in zip(xs, ys, ftot): + ix, iy = int(round(xi)), int(round(yi)) + img[iy - HALF:iy + HALF + 1, ix - HALF:ix + HALF + 1] += (f * psf) + + b = sep.Background(img, bw=BW, bh=BW, fw=FW, fh=FW) + ds = img - b.back() + rm = b.rms() + o = sep.extract(ds, THRESH, err=rm, minarea=MINAREA, filter_kernel=K, + filter_type="matched", deblend_nthresh=32, + deblend_cont=0.005, clean=True) + fl, fe, afl = sep.sum_circle(ds, o["x"], o["y"], APR, err=rm, subpix=5) + fl2, _, _ = sep.sum_circle(ds, o["x"], o["y"], 2 * APR, err=rm, subpix=5) + rh, _ = sep.flux_radius(ds, o["x"], o["y"], np.full(len(o), 6 * APR), 0.5, + normflux=fl2, subpix=5) + egain = float(hdr.get("EGAIN", 0.2467)) + fet = np.sqrt(fe ** 2 + np.clip(fl, 0, None) / (egain * 12.0)) + snr = fl / np.clip(fet, 1e-9, None) + omag = -2.5 * np.log10(np.clip(fl, 1e-9, None)) + ZP_L + ofwhm = 2.0 * rh + oelong = o["a"] / np.clip(o["b"], 1e-6, None) + ok_sel = ((snr >= 5.0) & ((o["flag"].astype(int) & 0b1110) == 0) + & (ofwhm > FW_MIN) & (ofwhm < FW_MAX) & (oelong < 2.0) + & (omag > 17.5) & (omag < 21.5) & (fl > 0)) + ox, oy = o["x"][ok_sel], o["y"][ok_sel] + + # match each injected star to the surviving detections, within 2 px + for xi, yi, mg in zip(xs, ys, mags): + dd = np.hypot(ox - xi, oy - yi) + hit = (dd.min() < 2.0) if len(dd) else False + rec_mag.append(mg) + rec_rad.append(np.hypot(xi - cx, yi - cy) * PIXSCALE / 60.0) + rec_ok.append(bool(hit)) + print(" run %d/%d done (%.0f s)" % (run + 1, NRUN, time.time() - t0), flush=True) + del img, b, ds, rm, o + + rec_mag = np.array(rec_mag); rec_rad = np.array(rec_rad); rec_ok = np.array(rec_ok) + np.savez(layout.path("_gc_complete.npz"), mag=rec_mag, rad=rec_rad, + ok=rec_ok, apfrac=apfrac) + print("\ninjected %d, recovered %d (%.1f%%)" % (len(rec_ok), rec_ok.sum(), 100 * rec_ok.mean())) + print("\ncompleteness grid (rows = mag, cols = radius arcmin):") + rb = [0, 2, 4, 6, 9, 13, 18, 30] + mb = np.arange(17.5, 22.6, 0.5) + print(" " + "".join("%7s" % ("%d-%d" % (rb[i], rb[i + 1])) for i in range(len(rb) - 1))) + for j in range(len(mb) - 1): + row = "%4.1f " % mb[j] + for i in range(len(rb) - 1): + s = ((rec_mag >= mb[j]) & (rec_mag < mb[j + 1]) & + (rec_rad >= rb[i]) & (rec_rad < rb[i + 1])) + row += "%7s" % ("%.2f" % rec_ok[s].mean() if s.sum() > 20 else "-") + print(row) + + +if __name__ == "__main__": + main() diff --git a/pipeline/gc-detect.py b/pipeline/gc-detect.py new file mode 100644 index 0000000..cd7141e --- /dev/null +++ b/pipeline/gc-detect.py @@ -0,0 +1,171 @@ +""" +gc-detect.py -- source detection and photometry for the NGC 5128 globular cluster survey. + +Step 1 of the pipeline. Reads the luminance master, builds a SPATIALLY VARYING +background model (this is the critical step: NGC 5128's own light is a steep, +structured background and a single global sky level would produce a spurious +central concentration of detections), detects sources above a locally-scaled +threshold, does aperture photometry, measures morphology, and runs forced +photometry at the same positions on the R/G/B masters. + +Outputs (into the stacked directory): + _gc_raw.npz all detections + photometry + morphology + NGC5128-gc-background-check.png diagnostic of the background model + +Memory: only one 61 MB master is held at a time. +""" +import os, sys, time +import numpy as np +from astropy.io import fits +from astropy.wcs import WCS +import sep +import warnings + +import layout +warnings.filterwarnings("ignore") + +STACKED = layout.SESSION + +# --- configuration ----------------------------------------------------------- +BW = 16 # background mesh, px. 8.6 arcsec = 3.2 x the 5 px FWHM. Chosen + # empirically: see the mesh trial in the notes -- a coarser mesh + # destroys recovery of known clusters inside ~8 arcmin. +FW = 3 # median filter over meshes +THRESH = 3.0 # sigma above the LOCAL rms +MINAREA = 5 +APR = 5.0 # photometric aperture radius, px, matching the published ZP +ZP_L = 27.941 # G_mag = -2.5 log10(flux) + ZP_L +FWHM_PX = 5.0 + +# NGC 5128 nucleus, J2000 (NED) +NUC_RA, NUC_DEC = 201.365063, -43.019113 + + +def gauss_kernel(fwhm, size=None): + sig = fwhm / 2.3548 + if size is None: + size = int(2 * round(3 * sig) + 1) + r = np.arange(size) - size // 2 + x, y = np.meshgrid(r, r) + k = np.exp(-(x * x + y * y) / (2 * sig * sig)) + return (k / k.sum()).astype(np.float32) + + +def load(name): + d = fits.getdata(layout.path(name)).astype(np.float32) + if not d.dtype.isnative: + d = d.byteswap().newbyteorder() + return np.ascontiguousarray(d) + + +def main(): + t0 = time.time() + hdr = fits.getheader(layout.path("master-Luminance.fit")) + w = WCS(hdr) + ny, nx = hdr["NAXIS2"], hdr["NAXIS1"] + + data = load("master-Luminance.fit") + + # ---------------------------------------------------------------- background + # Compare mesh sizes so the choice is defensible, then adopt BW. + print("background mesh comparison (median |back| inside r<2' of nucleus vs outer field):") + nx_c, ny_c = w.all_world2pix(NUC_RA, NUC_DEC, 0) + nx_c, ny_c = float(nx_c), float(ny_c) + print(" nucleus pixel = %.1f, %.1f" % (nx_c, ny_c)) + yy, xx = np.mgrid[0:ny:8, 0:nx:8] + rr = np.hypot(xx - nx_c, yy - ny_c) * 0.5376 / 60.0 # arcmin + inner = rr < 2.0 + outer = rr > 12.0 + for bw in (16, 32, 64, 128, 512): + b = sep.Background(data, bw=bw, bh=bw, fw=FW, fh=FW) + bk = b.back()[::8, ::8] + rms = b.rms()[::8, ::8] + print(" bw=%4d back(in)=%8.1f back(out)=%7.2f rms(in)=%7.2f rms(out)=%6.2f" + % (bw, np.median(bk[inner]), np.median(bk[outer]), + np.median(rms[inner]), np.median(rms[outer]))) + del b, bk, rms + + bkg = sep.Background(data, bw=BW, bh=BW, fw=FW, fh=FW) + back = bkg.back() + rmsmap = bkg.rms() + data_sub = data - back + del data + np.save(layout.path("_gc_backmodel_thumb.npy"), back[::8, ::8]) + del back + + # ---------------------------------------------------------------- detection + sep.set_extract_pixstack(3000000) + kern = gauss_kernel(FWHM_PX) + objs, segmap = sep.extract(data_sub, THRESH, err=rmsmap, minarea=MINAREA, + filter_kernel=kern, filter_type="matched", + deblend_nthresh=32, deblend_cont=0.005, + clean=True, clean_param=1.0, segmentation_map=True) + print("detected %d sources (thresh=%.1f x LOCAL rms, minarea=%d)" + % (len(objs), THRESH, MINAREA)) + del segmap + + x, y = objs["x"], objs["y"] + + # ---------------------------------------------------------------- photometry + flux, fluxerr, flag = sep.sum_circle(data_sub, x, y, APR, err=rmsmap, + gain=None, subpix=5) + # local-noise-only error; add Poisson from the source itself using EGAIN + egain = float(hdr.get("EGAIN", 0.2467)) + nimg = 12.0 # 12 x 300 s stacked, normalised -> approximate + poiss = np.clip(flux, 0, None) / (egain * nimg) + fluxerr_tot = np.sqrt(fluxerr ** 2 + poiss) + + # aperture at 2x radius, to catch extended light (galaxy discriminator) + flux2, _, _ = sep.sum_circle(data_sub, x, y, 2 * APR, err=rmsmap, subpix=5) + + # half-light radius and a "PSF concentration" index + rhalf, rflag = sep.flux_radius(data_sub, x, y, np.full(len(x), 6 * APR), + 0.5, normflux=flux2, subpix=5) + # peak-to-total sharpness + with np.errstate(divide="ignore", invalid="ignore"): + conc = -2.5 * np.log10(np.clip(flux, 1e-9, None) / np.clip(flux2, 1e-9, None)) + + # local background rms at each source (for SNR bookkeeping) + xi = np.clip(np.round(x).astype(int), 0, nx - 1) + yi = np.clip(np.round(y).astype(int), 0, ny - 1) + local_rms = rmsmap[yi, xi] + local_back = np.load(layout.path("_gc_backmodel_thumb.npy"))[ + np.clip(yi // 8, 0, ny // 8 - 1), np.clip(xi // 8, 0, nx // 8 - 1)] + + del rmsmap, data_sub + + with np.errstate(divide="ignore", invalid="ignore"): + mag = -2.5 * np.log10(np.clip(flux, 1e-9, None)) + ZP_L + magerr = 1.0857 * fluxerr_tot / np.clip(flux, 1e-9, None) + snr = flux / np.clip(fluxerr_tot, 1e-9, None) + + ra, dec = w.all_pix2world(x, y, 0) + + out = dict(x=x, y=y, ra=ra, dec=dec, + flux=flux, fluxerr=fluxerr_tot, mag=mag, magerr=magerr, snr=snr, + flux2=flux2, rhalf=rhalf, conc=conc, + a=objs["a"], b=objs["b"], theta=objs["theta"], + npix=objs["npix"].astype(np.float64), peak=objs["peak"], + cflux=objs["cflux"], sepflag=objs["flag"].astype(np.float64), + apflag=flag.astype(np.float64), + local_rms=local_rms, local_back=local_back) + + # ---------------------------------------------------------------- colours + for band, key in (("Red", "R"), ("Green", "V"), ("Blue", "B")): + d = load("master-%s.fit" % band) + b = sep.Background(d, bw=BW, bh=BW, fw=FW, fh=FW) + ds = d - b.back() + rm = b.rms() + del d + f, fe, fl = sep.sum_circle(ds, x, y, APR, err=rm, subpix=5) + out["flux_" + key] = f + out["fluxerr_" + key] = fe + print(" forced photometry on %s done" % band) + del ds, rm, b, f, fe, fl + + np.savez(layout.path("_gc_raw.npz"), **out) + print("wrote _gc_raw.npz with %d rows in %.0f s" % (len(x), time.time() - t0)) + + +if __name__ == "__main__": + main() diff --git a/pipeline/gc-fetch-vizier.py b/pipeline/gc-fetch-vizier.py new file mode 100644 index 0000000..42fe69c --- /dev/null +++ b/pipeline/gc-fetch-vizier.py @@ -0,0 +1,91 @@ +"""Fetch a published globular cluster catalogue for NGC 5128 (Centaurus A) from VizieR. + +Primary catalogue: J/MNRAS/469/3444 table "gc" -- Taylor, Puzia, Ferrarez et al. 2017, +MNRAS 469, 3444, "Survey of Centaurus A's Baryonic Structures (SCABS) II": +3210 globular cluster candidates with ugriz photometry over ~1.55 deg^2 centred on Cen A. + +Fallback: J/AJ/143/84 (Harris+ 2012, 833 new GC candidates). + +Writes _cena_gc_ref.npz with keys: + ra, dec : degrees, J2000 (ICRS) + vmag : Johnson V estimated from SDSS g,r via Lupton (2005): + V = g - 0.5784*(g-r) - 0.0038 + gmag,rmag: SDSS-like g and r from the source catalogue (NaN where missing) + prob : Taylor+ 2017 GC membership probability (0-1), higher = more likely a GC +""" +import numpy as np +from astroquery.vizier import Vizier +from astropy.coordinates import SkyCoord +import astropy.units as u + +import layout + +OUT = layout.path("_cena_gc_ref.npz") +RA0, DEC0 = 201.365, -43.019 +# field half-widths in arcmin for a 43' x 29' frame +HW_RA, HW_DEC = 21.5, 14.5 + + +def col(t, name): + if name not in t.colnames: + return None + return np.ma.filled(np.ma.asarray(t[name]).astype(float), np.nan) + + +def report(ra, dec, label): + dra = (ra - RA0) * np.cos(np.radians(dec)) * 60.0 + dde = (dec - DEC0) * 60.0 + sep = np.hypot(dra, dde) + infield = (np.abs(dra) <= HW_RA) & (np.abs(dde) <= HW_DEC) + print(f" {label}: {len(ra)} rows") + print(f" RA {ra.min():.4f} .. {ra.max():.4f} " + f"(span {(ra.max()-ra.min())*np.cos(np.radians(DEC0))*60:.1f}')") + print(f" Dec {dec.min():.4f} .. {dec.max():.4f} " + f"(span {(dec.max()-dec.min())*60:.1f}')") + print(f" sep from centre: median {np.median(sep):.1f}' max {sep.max():.1f}'") + print(f" inside 43'x29' field: {infield.sum()}") + return infield + + +def main(): + v = Vizier(row_limit=-1) + cid, tname = "J/MNRAS/469/3444", "J/MNRAS/469/3444/gc" + print(f"Fetching {tname} (Taylor+ 2017, SCABS II)") + try: + tabs = v.get_catalogs(cid) + t = [x for x in tabs if x.meta.get("name") == tname][0] + except Exception as e: + print(f" FAILED: {e}") + return 1 + + # RAJ2000/DEJ2000 are sexagesimal strings in this table + sc = SkyCoord(np.asarray(t["RAJ2000"], str), np.asarray(t["DEJ2000"], str), + unit=(u.hourangle, u.deg)) + ra = sc.ra.deg + dec = sc.dec.deg + g = col(t, "gmag") + r = col(t, "rmag") + prob = col(t, "Prob") + + good = np.isfinite(ra) & np.isfinite(dec) + ra, dec, g, r, prob = ra[good], dec[good], g[good], r[good], prob[good] + + # -1.0 is the catalogue's "no photometry" sentinel -> NaN + g = np.where(g <= 0, np.nan, g) + r = np.where(r <= 0, np.nan, r) + + # Lupton (2005) SDSS -> Johnson V + vmag = g - 0.5784 * (g - r) - 0.0038 + + report(ra, dec, tname) + print(f" vmag finite: {np.isfinite(vmag).sum()} " + f"range {np.nanmin(vmag):.2f} .. {np.nanmax(vmag):.2f}") + print(f" prob >=0.5: {(prob >= 0.5).sum()} >=0.9: {(prob >= 0.9).sum()}") + + np.savez(OUT, ra=ra, dec=dec, vmag=vmag, gmag=g, rmag=r, prob=prob) + print(f"WROTE {OUT}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/gc-gaia-astrom2.py b/pipeline/gc-gaia-astrom2.py new file mode 100644 index 0000000..c2d40ae --- /dev/null +++ b/pipeline/gc-gaia-astrom2.py @@ -0,0 +1,32 @@ +"""Gaia DR3 astrometry via the VizieR mirror I/355/gaiadr3 (the ESA Gaia archive +was down for maintenance). Same rationale as gc-gaia-astrom.py.""" +import os, numpy as np +from astroquery.vizier import Vizier +from astropy.coordinates import SkyCoord +import astropy.units as u + +import layout + +S = layout.SESSION +c = SkyCoord(201.36404, -43.01969, unit="deg") + +cols = ["RA_ICRS", "DE_ICRS", "Source", "Gmag", "BPmag", "RPmag", "BP-RP", + "Plx", "e_Plx", "pmRA", "e_pmRA", "pmDE", "e_pmDE", "RUWE"] +v = Vizier(columns=cols, row_limit=-1, column_filters={"Gmag": "<21.0"}) +t = v.query_region(c, radius=0.42 * u.deg, catalog="I/355/gaiadr3")[0] +print("rows:", len(t), t.colnames) + +d = {} +for k, name in [("ra", "RA_ICRS"), ("dec", "DE_ICRS"), ("g", "Gmag"), + ("bp", "BPmag"), ("rp", "RPmag"), ("bprp", "BP-RP"), + ("plx", "Plx"), ("eplx", "e_Plx"), ("pmra", "pmRA"), + ("epmra", "e_pmRA"), ("pmdec", "pmDE"), ("epmdec", "e_pmDE"), + ("ruwe", "RUWE")]: + if name in t.colnames: + col = t[name] + d[k] = np.array(col.filled(np.nan) if hasattr(col, "filled") else col, dtype=float) +np.savez(layout.path("_gaia_astrom.npz"), **d) +g = d["g"] +print("saved. G<18 %d | 18-20 %d | >=20 %d" % + ((g < 18).sum(), ((g >= 18) & (g < 20)).sum(), (g >= 20).sum())) +print("finite plx %d, finite pm %d" % (np.isfinite(d["plx"]).sum(), np.isfinite(d["pmra"]).sum())) diff --git a/pipeline/gc-plots.py b/pipeline/gc-plots.py new file mode 100644 index 0000000..b461d1a --- /dev/null +++ b/pipeline/gc-plots.py @@ -0,0 +1,491 @@ +""" +gc-plots.py -- step 4. All figures for the NGC 5128 globular cluster survey. + + NGC5128-gc-background-check.png how the galaxy light was removed + NGC5128-gc-finder.png annotated finder chart with zoom insets + NGC5128-gc-cmd.png colour-magnitude diagram + NGC5128-gc-completeness.png artificial-star recovery + NGC5128-gc-radial-profile.png THE key plot: surface density vs radius +""" +import os, numpy as np, warnings +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from matplotlib.patches import Circle, Rectangle +from matplotlib.lines import Line2D +from astropy.io import fits +from astropy.wcs import WCS + +import layout +warnings.filterwarnings("ignore") + +S = layout.SESSION +PIXSCALE = 0.5376 +NUC_RA, NUC_DEC = 201.365063, -43.019113 +KPC_PER_ARCMIN = 3.8 * 1000.0 * (np.pi / 180.0) / 60.0 + +# validated categorical palette (see notes) +C_CAND, C_EXT, C_KNOWN, C_ACC = "#2563eb", "#e8710a", "#127a5a", "#8b5cf6" +C_INK, C_MUTED, C_GRID = "#1a1a1a", "#5c5c5c", "#d8d8d4" +SURF = "#fcfcfb" + +plt.rcParams.update({ + "figure.facecolor": SURF, "axes.facecolor": SURF, + "axes.edgecolor": C_MUTED, "axes.labelcolor": C_INK, + "text.color": C_INK, "xtick.color": C_MUTED, "ytick.color": C_MUTED, + "axes.grid": True, "grid.color": C_GRID, "grid.linewidth": 0.6, + "axes.axisbelow": True, "font.size": 10, + "axes.spines.top": False, "axes.spines.right": False, + "legend.frameon": False, +}) + + +def asinh_stretch(a, lo=1.0, hi=99.7, soft=8.0): + v0, v1 = np.percentile(a[np.isfinite(a)], [lo, hi]) + x = np.clip((a - v0) / max(v1 - v0, 1e-9), 0, 1) + return np.arcsinh(soft * x) / np.arcsinh(soft) + + +def load_cat(): + return dict(np.load(layout.path("_gc_cat.npz"), allow_pickle=True)) + + +# -------------------------------------------------------------------------- +def fig_background(d): + import sep + img = np.ascontiguousarray( + fits.getdata(layout.path("master-Luminance.fit")).astype(np.float32)) + b16 = sep.Background(img, bw=16, bh=16, fw=3, fh=3) + b256 = sep.Background(img, bw=256, bh=256, fw=3, fh=3) + raw = img[::4, ::4] + back = b16.back()[::4, ::4] + sub16 = raw - back + sub256 = raw - b256.back()[::4, ::4] + del img, b16, b256 + + # ONE stretch for the two image-scale panels and ONE for the two residuals, + # so the panels are actually comparable to each other. + v0, v1 = np.percentile(raw, [1.0, 99.7]) + r0, r1 = np.percentile(sub16, [1.0, 99.85]) + + def show(a, im, lo, hi, t, sub=""): + x = np.clip((im - lo) / (hi - lo), 0, 1) + a.imshow(np.arcsinh(8 * x) / np.arcsinh(8.0), origin="lower", cmap="gray", + interpolation="nearest", vmin=0, vmax=1) + a.set_title(t, fontsize=10.5, loc="left", pad=4) + if sub: + a.text(0.5, -0.045, sub, transform=a.transAxes, ha="center", va="top", + fontsize=9, color=C_MUTED) + a.set_xticks([]); a.set_yticks([]); a.grid(False) + + fig, ax = plt.subplots(2, 2, figsize=(12.4, 8.0)) + show(ax[0, 0], raw, v0, v1, "a) luminance master") + show(ax[0, 1], back, v0, v1, "b) background model, 16 px mesh (adopted)", + "same stretch as (a): the model reproduces the galaxy and the dust lane") + show(ax[1, 0], sub16, r0, r1, "c) residual after (b) - what detection runs on", + "galaxy gone; point sources remain at every radius") + show(ax[1, 1], sub256, r0, r1, "d) residual after a 256 px mesh - the failure mode", + "the galaxy survives, swamping the inner field and hiding its clusters") + fig.suptitle("Removing NGC 5128's own light before detection", fontsize=13, + x=0.008, ha="left") + fig.tight_layout(rect=[0, 0.02, 1, 0.97]) + fig.savefig(layout.path("NGC5128-gc-background-check.png"), dpi=110, + bbox_inches="tight", facecolor=SURF) + plt.close(fig) + print("wrote NGC5128-gc-background-check.png") + + +# -------------------------------------------------------------------------- +def fig_finder(d): + hdr = fits.getheader(layout.path("master-Luminance.fit")) + w = WCS(hdr) + img = fits.getdata(layout.path("master-Luminance.fit")).astype(np.float32) + B = 4 + small = img[::B, ::B] + disp = asinh_stretch(small, 20, 99.5, 12) + cx, cy = [float(v) for v in w.all_world2pix(NUC_RA, NUC_DEC, 0)] + + cand = d["cand"] + known = cand & ((d["simbad_type"] == "GlC") | d["scabs_hit"]) + newc = cand & ~known + + ny_s, nx_s = small.shape + # Main panel keeps the detector's 1.5:1 aspect; insets sit in a row beneath. + fig = plt.figure(figsize=(14.0, 13.0)) + gs = fig.add_gridspec(2, 4, height_ratios=[nx_s / ny_s * 0.98, 1.0], + hspace=0.09, wspace=0.06, + left=0.012, right=0.988, top=0.945, bottom=0.015) + ax = fig.add_subplot(gs[0, :]) + ax.imshow(disp, origin="lower", cmap="gray", interpolation="bilinear", + aspect="equal") + ax.set_xlim(0, nx_s); ax.set_ylim(0, ny_s) + ax.grid(False); ax.set_xticks([]); ax.set_yticks([]) + + for m, col, lab in ((known, C_KNOWN, "matches a published catalogue (%d)" % known.sum()), + (newc, C_CAND, "no published counterpart (%d)" % newc.sum())): + ax.scatter(d["x"][m] / B, d["y"][m] / B, s=110, facecolors="none", + edgecolors=col, linewidths=1.2, label=lab) + # nucleus + import matplotlib.patheffects as pe + halo = [pe.withStroke(linewidth=3.0, foreground="black")] + ax.plot(cx / B, cy / B, marker="+", ms=20, mew=2.2, color=C_ACC, zorder=6) + ax.annotate("NGC 5128 nucleus", (cx / B, cy / B), xytext=(-95, -78), + textcoords="offset points", color=C_ACC, fontsize=11, + weight="bold", ha="center", zorder=7, path_effects=halo, + arrowprops=dict(arrowstyle="-", color=C_ACC, lw=1.3, + shrinkA=2, shrinkB=10)) + + # scale bar: 5 arcmin, bottom left, inside the frame + L = 5 * 60 / PIXSCALE / B + x0, y0 = nx_s * 0.035, ny_s * 0.062 + ax.plot([x0, x0 + L], [y0, y0], color="white", lw=3.5, solid_capstyle="butt") + ax.text(x0 + L / 2, y0 + ny_s * 0.018, + "5' = %.1f kpc at 3.8 Mpc" % (5 * KPC_PER_ARCMIN), + ha="center", color="white", fontsize=10) + # compass, derived from the WCS itself rather than from the quoted PA: + # step 1 arcmin north and 1 arcmin east of the nucleus and see where it lands + cxo, cyo = nx_s * 0.915, ny_s * 0.16 + for dra, ddec, lab in ((0.0, 1 / 60.0, "N"), (1 / 60.0, 0.0, "E")): + px, py = w.all_world2pix(NUC_RA + dra / np.cos(np.radians(NUC_DEC)), + NUC_DEC + ddec, 0) + vx, vy = float(px) - cx, float(py) - cy + n = np.hypot(vx, vy) + dx, dy = vx / n * 48, vy / n * 48 + ax.annotate("", (cxo + dx, cyo + dy), (cxo, cyo), + arrowprops=dict(arrowstyle="->", color="white", lw=1.8)) + ax.text(cxo + dx * 1.38, cyo + dy * 1.38, lab, color="white", + ha="center", va="center", fontsize=11, weight="bold") + + ax.legend(loc="upper left", fontsize=10.5, labelcolor=C_INK, + handletextpad=0.4, borderpad=0.6, markerscale=1.1, + facecolor="white", framealpha=0.82, frameon=True, edgecolor="none") + ax.set_title("NGC 5128 globular cluster candidates: %d circled, G < 20 " + "(42.9' x 28.6' luminance master, asinh stretch, 4x downsampled)" + % cand.sum(), fontsize=12.5, loc="left", pad=9) + + # ---- zoom insets, full resolution ---- + zooms = [(2.5, 55), (5.5, 340), (9.0, 130), (16.0, 205)] + HW = 230 # half-width in full-res px -> 4.1 arcmin box + for k, (rr, pa) in enumerate(zooms): + az = fig.add_subplot(gs[1, k]) + ang = np.radians(pa) + zx = cx + rr * 60 / PIXSCALE * np.cos(ang) + zy = cy + rr * 60 / PIXSCALE * np.sin(ang) + zx = float(np.clip(zx, HW, img.shape[1] - HW)) + zy = float(np.clip(zy, HW, img.shape[0] - HW)) + cut = img[int(zy - HW):int(zy + HW), int(zx - HW):int(zx + HW)] + az.imshow(asinh_stretch(cut, 15, 99.7, 14), origin="lower", cmap="gray", + interpolation="nearest", aspect="equal", + extent=[zx - HW, zx + HW, zy - HW, zy + HW]) + for m, col in ((known, C_KNOWN), (newc, C_CAND)): + sel = m & (np.abs(d["x"] - zx) < HW) & (np.abs(d["y"] - zy) < HW) + az.scatter(d["x"][sel], d["y"][sel], s=230, facecolors="none", + edgecolors=col, linewidths=1.6) + az.set_xticks([]); az.set_yticks([]); az.grid(False) + az.set_title("%.1f' from the nucleus" % rr, fontsize=10.5, loc="left", pad=5) + ax.add_patch(Rectangle(((zx - HW) / B, (zy - HW) / B), 2 * HW / B, 2 * HW / B, + fill=False, ec="white", lw=1.1, ls="-", alpha=0.8)) + ax.text((zx) / B, (zy + HW) / B + 6, "%.1f'" % rr, color="white", + fontsize=9, ha="center") + fig.text(0.012, 0.002, + "Lower row: full-resolution %.1f' cut-outs at the marked positions, same " + "circles. The innermost arcminute is empty of candidates: the nucleus and " + "dust lane are impenetrable at this depth." % (2 * HW * PIXSCALE / 60), + fontsize=9.5, color=C_MUTED) + fig.savefig(layout.path("NGC5128-gc-finder.png"), dpi=100, facecolor=SURF) + plt.close(fig) + del img + print("wrote NGC5128-gc-finder.png") + + +# -------------------------------------------------------------------------- +def fig_cmd(d): + fig, ax = plt.subplots(1, 2, figsize=(12.6, 5.6)) + stars = d["base"] & d["is_star"] & np.isfinite(d["BmR"]) & (d["mag"] > 15) & (d["mag"] < 21.5) + cand = d["cand"] & np.isfinite(d["BmR"]) + known = cand & ((d["simbad_type"] == "GlC") | d["scabs_hit"]) + + a = ax[0] + a.hexbin(d["BmR"][stars], d["mag"][stars], gridsize=60, cmap="Greys", + mincnt=1, extent=(-1.0, 3.5, 15, 21.6), linewidths=0) + a.scatter(d["BmR"][cand & ~known], d["mag"][cand & ~known], s=20, c=C_CAND, + alpha=0.85, lw=0, label="candidate, no published counterpart") + a.scatter(d["BmR"][known], d["mag"][known], s=22, c=C_KNOWN, alpha=0.9, + lw=0, marker="D", label="candidate, published GC") + a.set_xlim(-1.0, 3.5); a.set_ylim(21.6, 15) + a.set_xlabel("B - R (instrumental, calibrated to Gaia BP - RP)") + a.set_ylabel("G (luminance)") + a.set_title("a) candidates against the foreground stellar field", fontsize=10.5, loc="left") + a.legend(fontsize=9, loc="lower left", labelcolor=C_INK) + a.text(0.98, 0.03, "grey = %d astrometric\nMilky Way stars" % stars.sum(), + transform=a.transAxes, ha="right", va="bottom", fontsize=9, color=C_MUTED) + + b = ax[1] + bins = np.arange(-1.0, 3.51, 0.2) + b.hist(d["BmR"][stars], bins=bins, density=True, color=C_MUTED, alpha=0.35, + label="foreground stars (n=%d)" % stars.sum()) + b.hist(d["BmR"][cand], bins=bins, density=True, histtype="step", lw=2.0, + color=C_CAND, label="all candidates (n=%d)" % cand.sum()) + b.hist(d["BmR"][known], bins=bins, density=True, histtype="step", lw=2.0, + color=C_KNOWN, ls="--", label="published GCs (n=%d)" % known.sum()) + b.set_xlabel("B - R"); b.set_ylabel("normalised density") + b.set_title("b) colour distributions", fontsize=10.5, loc="left") + b.legend(fontsize=9, labelcolor=C_INK) + med = np.nanmedian(d["BmR"][cand]) + b.axvline(med, color=C_CAND, lw=1, ls=":") + b.text(med + 0.06, b.get_ylim()[1] * 0.93, "candidate median %.2f" % med, + fontsize=9, color=C_CAND) + + fig.suptitle("Colour-magnitude diagram, NGC 5128 cluster candidates", + fontsize=12.5, x=0.008, ha="left") + fig.tight_layout() + fig.savefig(layout.path("NGC5128-gc-cmd.png"), dpi=115, bbox_inches="tight", facecolor=SURF) + plt.close(fig) + print("wrote NGC5128-gc-cmd.png") + + +# -------------------------------------------------------------------------- +def completeness_grid(): + """Prefer the merged uniform + inner-field artificial-star runs. + + The uniform run alone puts only ~2% of its fakes inside 3 arcmin, which left + the innermost completeness bins too noisy to correct with. + """ + for name in ("_gc_complete_all.npz", "_gc_complete.npz"): + p = layout.path(name) + if os.path.exists(p): + print("completeness from %s" % name) + return np.load(p) + return None + + +def fig_completeness(cp): + fig, ax = plt.subplots(1, 2, figsize=(12.6, 5.0)) + mag, rad, ok = cp["mag"], cp["rad"], cp["ok"] + mb = np.arange(17.5, 22.51, 0.25) + mc = 0.5 * (mb[1:] + mb[:-1]) + rbins = [(0, 2), (2, 4), (4, 8), (8, 30)] + cols = [C_ACC, C_EXT, C_KNOWN, C_CAND] + a = ax[0] + for (lo, hi), c in zip(rbins, cols): + f = [] + for j in range(len(mb) - 1): + s = (mag >= mb[j]) & (mag < mb[j + 1]) & (rad >= lo) & (rad < hi) + f.append(ok[s].mean() if s.sum() > 15 else np.nan) + a.plot(mc, f, lw=2.0, color=c, label="%d - %d arcmin" % (lo, hi)) + a.axhline(0.5, color=C_MUTED, lw=1, ls=":") + a.text(17.6, 0.53, "50%", fontsize=9, color=C_MUTED) + a.set_xlabel("injected G magnitude"); a.set_ylabel("recovery fraction") + a.set_ylim(0, 1.05) + a.set_title("a) artificial-star completeness by projected radius", fontsize=10.5, loc="left") + a.legend(fontsize=9, title="projected radius", title_fontsize=9, labelcolor=C_INK) + + b = ax[1] + rb = np.array([0, 1, 2, 3, 4, 6, 8, 11, 15, 20, 30]) + rc = 0.5 * (rb[1:] + rb[:-1]) + for mlo, mhi, c, ls in ((17.5, 19.0, C_KNOWN, "-"), (19.0, 20.0, C_CAND, "-"), + (20.0, 21.0, C_EXT, "-"), (21.0, 21.5, C_ACC, "--")): + f = [] + for j in range(len(rb) - 1): + s = (mag >= mlo) & (mag < mhi) & (rad >= rb[j]) & (rad < rb[j + 1]) + f.append(ok[s].mean() if s.sum() > 15 else np.nan) + b.plot(rc, f, lw=2.0, color=c, ls=ls, label="G %.1f - %.1f" % (mlo, mhi)) + b.set_xlabel("projected radius (arcmin)"); b.set_ylabel("recovery fraction") + b.set_ylim(0, 1.05); b.set_xscale("log") + b.set_xticks([1, 2, 3, 5, 10, 20]); b.set_xticklabels(["1", "2", "3", "5", "10", "20"]) + b.set_title("b) the same, as a function of radius", fontsize=10.5, loc="left") + b.legend(fontsize=9, labelcolor=C_INK) + fig.suptitle("Completeness: how much harder is a cluster to find near the nucleus?", + fontsize=12.5, x=0.008, ha="left") + fig.tight_layout() + fig.savefig(layout.path("NGC5128-gc-completeness.png"), dpi=115, bbox_inches="tight", + facecolor=SURF) + plt.close(fig) + print("wrote NGC5128-gc-completeness.png") + + +# -------------------------------------------------------------------------- +def annulus_area(rb, cx, cy, nx=4788, ny=3194, step=4): + """effective area of each annulus that actually lies on the detector""" + yy, xx = np.mgrid[0:ny:step, 0:nx:step] + r = np.hypot(xx - cx, yy - cy) * PIXSCALE / 60.0 + px_area = (step * PIXSCALE / 60.0) ** 2 + return np.array([((r >= rb[i]) & (r < rb[i + 1])).sum() * px_area + for i in range(len(rb) - 1)]), r + + +def completeness_of(cp, mag, rad): + """Completeness for each object, interpolated from the artificial-star grid. + + A 2D lookup on (magnitude, radius). No luminosity function is assumed: each + surviving candidate is weighted by 1/C, the standard inverse-completeness + (Vmax-style) estimator. + """ + fm, fr, fok = cp["mag"], cp["rad"], cp["ok"] + mb = np.arange(17.5, 20.01, 0.5) + rbg = np.array([0, 1.5, 2.25, 3, 4, 5, 6, 8, 11, 15, 20, 30]) + grid = np.full((len(mb) - 1, len(rbg) - 1), np.nan) + for i in range(len(mb) - 1): + for j in range(len(rbg) - 1): + s = ((fm >= mb[i]) & (fm < mb[i + 1]) & (fr >= rbg[j]) & (fr < rbg[j + 1])) + if s.sum() >= 15: + grid[i, j] = fok[s].mean() + # completeness is a strong function of magnitude and a weak one of radius + # outside the innermost annuli, so fill empty cells with the row median + for i in range(grid.shape[0]): + row = grid[i] + if np.isfinite(row).any(): + row[~np.isfinite(row)] = np.nanmedian(row) + mi = np.clip(np.digitize(mag, mb) - 1, 0, len(mb) - 2) + ri = np.clip(np.digitize(rad, rbg) - 1, 0, len(rbg) - 2) + return grid[mi, ri], grid + + +def fig_radial(d, cp): + hdr = fits.getheader(layout.path("master-Luminance.fit")) + w = WCS(hdr) + cx, cy = [float(v) for v in w.all_world2pix(NUC_RA, NUC_DEC, 0)] + rb = np.array([0, 1.5, 3, 4.5, 6, 8, 11, 15, 20, 27]) + rc = 0.5 * (rb[1:] + rb[:-1]) + area, _ = annulus_area(rb, cx, cy) + + r = d["r_arcmin"] + mag = d["mag"] + # Magnitude-limited sample: G < 19.5, where the measured completeness is + # 25-70% and the 1/C weights are therefore stable. + MLIM = 19.5 + lim = mag < MLIM + cand = d["cand"] & lim + ext = (d["cls"] == "extended") & lim + star = (d["cls"] == "foreground-star") & lim + + C, grid = completeness_of(cp, mag, r) + usable = np.isfinite(C) & (C > 0.15) + wt = np.where(usable, 1.0 / np.clip(C, 0.15, None), 0.0) + print("median completeness of the candidate sample: %.2f (%d of %d usable)" + % (np.nanmedian(C[cand]), (cand & usable).sum(), cand.sum())) + + def prof(mask, weights=None): + n, sw = [], [] + for i in range(len(rb) - 1): + s = mask & (r >= rb[i]) & (r < rb[i + 1]) + n.append(s.sum()) + sw.append(np.sum(weights[s]) if weights is not None else s.sum()) + n = np.array(n, float); sw = np.array(sw, float) + with np.errstate(divide="ignore", invalid="ignore"): + mw = np.where(n > 0, sw / np.maximum(n, 1), 0.0) # mean weight + return n, sw / area, mw * np.sqrt(n) / area # Poisson error + + n_c, s_c, e_c = prof(cand) + n_cc, s_cc, e_cc = prof(cand & usable, wt) + n_e, s_e, e_e = prof(ext) + n_s, s_s, e_s = prof(star) + # The SAME correction applied to the foreground stars. Milky Way stars are + # uniformly distributed on this scale, so their corrected profile MUST come + # out flat. That is the check on the whole correction: if it does not + # flatten them, the correction is wrong and so is the candidate profile. + n_sc, s_sc, e_sc = prof(star & usable, wt) + corr = np.where(s_c > 0, s_cc / np.maximum(s_c, 1e-12), np.nan) + + # outer-field level from the two outermost annuli of the corrected profile + denom = np.nansum(area[-2:]) + bg = np.nansum(s_cc[-2:] * area[-2:]) / denom + bg_e = np.sqrt(np.nansum(e_cc[-2:] ** 2 * area[-2:] ** 2)) / denom + + fig, ax = plt.subplots(1, 2, figsize=(13.2, 5.6)) + a = ax[0] + a.errorbar(rc, s_c, yerr=e_c, color=C_MUTED, lw=1.4, marker="o", ms=6, + capsize=3, label="candidates, raw counts", ls="--", zorder=3) + a.errorbar(rc, s_cc, yerr=e_cc, color=C_CAND, lw=2.4, marker="o", ms=8, + capsize=3, label="candidates, completeness-corrected", zorder=5) + a.errorbar(rc, s_e, yerr=e_e, color=C_EXT, lw=1.8, marker="s", ms=6, + capsize=3, label="extended sources, raw", zorder=4) + a.errorbar(rc, s_sc / 20.0, yerr=e_sc / 20.0, color=C_KNOWN, lw=1.8, marker="^", + ms=6, capsize=3, label="foreground stars / 20, corrected", zorder=2) + a.errorbar(rc, s_s / 20.0, yerr=e_s / 20.0, color=C_KNOWN, lw=1.1, marker="^", + ms=4, capsize=2, ls=":", alpha=0.55, + label="foreground stars / 20, raw", zorder=1) + a.axhline(bg, color=C_ACC, lw=1.4, ls="-.") + a.fill_between([0.8, 30], bg - bg_e, bg + bg_e, color=C_ACC, alpha=0.15, lw=0) + a.annotate("outer-field level", xy=(23, bg), xytext=(23, bg * 0.42), + color=C_ACC, fontsize=8.5, ha="center", + arrowprops=dict(arrowstyle="->", color=C_ACC, lw=1)) + a.set_xscale("log"); a.set_yscale("log") + a.set_xlim(0.8, 30) + a.set_xticks([1, 2, 3, 5, 10, 20]); a.set_xticklabels(["1", "2", "3", "5", "10", "20"]) + a.set_xlabel("projected radius from the nucleus (arcmin)") + a.set_ylabel("surface density (objects per arcmin$^2$)") + a.set_title("a) radial surface density, G < 19.5", fontsize=10.5, loc="left") + a.legend(fontsize=8.6, loc="lower left", labelcolor=C_INK) + sec = a.secondary_xaxis("top", functions=(lambda v: v * KPC_PER_ARCMIN, + lambda v: v / KPC_PER_ARCMIN)) + sec.set_xlabel("projected radius (kpc at 3.8 Mpc)", fontsize=9.5) + + # panel b: background-subtracted, with a power law fit + b = ax[1] + excess = s_cc - bg + ee = np.sqrt(e_cc ** 2 + bg_e ** 2) + ok = np.isfinite(excess) & (excess > 0) & (rc < 20) + b.errorbar(rc[ok], excess[ok], yerr=ee[ok], color=C_CAND, lw=2.2, marker="o", + ms=8, capsize=3, label="candidate excess over the outer field") + slope = np.nan + if ok.sum() >= 3: + p = np.polyfit(np.log10(rc[ok]), np.log10(excess[ok]), 1) + slope = p[0] + xr = np.array([rc[ok].min(), rc[ok].max()]) + b.plot(xr, 10 ** np.polyval(p, np.log10(xr)), color=C_INK, lw=1.4, ls="--", + label=r"power law, $\Sigma \propto R^{%.2f}$" % slope) + b.set_xscale("log"); b.set_yscale("log") + b.set_xlim(0.8, 30) + b.set_xticks([1, 2, 3, 5, 10, 20]); b.set_xticklabels(["1", "2", "3", "5", "10", "20"]) + b.set_xlabel("projected radius from the nucleus (arcmin)") + b.set_ylabel(r"excess surface density (arcmin$^{-2}$)") + b.set_title("b) excess over the outer field", fontsize=10.5, loc="left") + b.legend(fontsize=9, labelcolor=C_INK) + + fig.suptitle("Are the candidates concentrated on NGC 5128?", fontsize=13, + x=0.008, ha="left") + fig.tight_layout() + fig.savefig(layout.path("NGC5128-gc-radial-profile.png"), dpi=115, + bbox_inches="tight", facecolor=SURF) + plt.close(fig) + print("wrote NGC5128-gc-radial-profile.png") + + print("\nRADIAL PROFILE TABLE (G < 19.5)") + print("%6s %6s %5s %8s %7s %10s %10s %10s" + % ("r_in", "r_out", "N", "area", "medC", "sig_raw", "sig_corr", "sig_ext")) + for i in range(len(rc)): + s = cand & (r >= rb[i]) & (r < rb[i + 1]) + mc = np.nanmedian(C[s]) if s.sum() else np.nan + print("%6.1f %6.1f %5d %8.2f %7.2f %10.4f %10.4f %10.4f" + % (rb[i], rb[i + 1], n_c[i], area[i], mc, s_c[i], s_cc[i], s_e[i])) + print("outer-field level %.4f +/- %.4f arcmin^-2 ; power-law slope %.2f" + % (bg, bg_e, slope)) + print("corrected density at 1.5-3' / outer-field level = %.1f" + % (s_cc[1] / bg if bg > 0 else np.nan)) + ei = np.nansum(n_e[:4]) / np.nansum(area[:4]) + eo = np.nansum(n_e[6:]) / np.nansum(area[6:]) + si = np.nansum(n_s[:4]) / np.nansum(area[:4]) + so = np.nansum(n_s[6:]) / np.nansum(area[6:]) + print("extended sources: inner(<6') %.4f outer(>11') %.4f ratio %.2f" + % (ei, eo, ei / eo if eo else np.nan)) + print("foreground stars: inner(<6') %.4f outer(>11') %.4f ratio %.2f" + % (si, so, si / so if so else np.nan)) + fin = np.nansum(s_sc[1:4] * area[1:4]) / np.nansum(area[1:4]) + fout = np.nansum(s_sc[6:] * area[6:]) / np.nansum(area[6:]) + print("foreground stars CORRECTED: inner(1.5-6') %.3f outer(>11') %.3f ratio %.2f" + % (fin, fout, fin / fout if fout else np.nan)) + np.savez(layout.path("_gc_profile.npz"), rb=rb, rc=rc, area=area, + n_c=n_c, s_c=s_c, e_c=e_c, corr=corr, s_cc=s_cc, e_cc=e_cc, + s_e=s_e, s_s=s_s, s_sc=s_sc, bg=bg, bg_e=bg_e, slope=slope, grid=grid) + + +if __name__ == "__main__": + d = load_cat() + cp = completeness_grid() + fig_background(d) + fig_finder(d) + fig_cmd(d) + if cp is not None: + fig_completeness(cp) + fig_radial(d, cp) diff --git a/pipeline/gc-validate.py b/pipeline/gc-validate.py new file mode 100644 index 0000000..45136b0 --- /dev/null +++ b/pipeline/gc-validate.py @@ -0,0 +1,156 @@ +""" +gc-validate.py -- step 5. External validation against published catalogues. + +Two independent truth sets fall inside the field: + * SIMBAD, object type GlC -- confirmed/catalogued Cen A clusters + * SCABS (Taylor et al. 2017, MNRAS 469, 3444) -- deep DECam GC candidates, + with V magnitudes, so it can be binned in brightness + +Neither is complete, especially in the inner few arcmin where the galaxy swamps +even professional data, so a candidate that matches nothing is UNCONFIRMED, not +false. Produces NGC5128-gc-recovery.png and prints the numbers used in the notes. +""" +import os, numpy as np, warnings +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from astropy.io import fits +from astropy.wcs import WCS +from astropy.coordinates import SkyCoord + +import layout +warnings.filterwarnings("ignore") + +S = layout.SESSION +PIXSCALE = 0.5376 +NUC_RA, NUC_DEC = 201.365063, -43.019113 +C_CAND, C_EXT, C_KNOWN, C_ACC = "#2563eb", "#e8710a", "#127a5a", "#8b5cf6" +C_INK, C_MUTED, C_GRID = "#1a1a1a", "#5c5c5c", "#d8d8d4" +SURF = "#fcfcfb" +plt.rcParams.update({ + "figure.facecolor": SURF, "axes.facecolor": SURF, "axes.edgecolor": C_MUTED, + "text.color": C_INK, "xtick.color": C_MUTED, "ytick.color": C_MUTED, + "axes.grid": True, "grid.color": C_GRID, "grid.linewidth": 0.6, + "axes.axisbelow": True, "font.size": 10, "axes.spines.top": False, + "axes.spines.right": False, "legend.frameon": False, "axes.labelcolor": C_INK}) + + +def main(): + d = dict(np.load(layout.path("_gc_cat.npz"), allow_pickle=True)) + hdr = fits.getheader(layout.path("master-Luminance.fit")) + w = WCS(hdr) + cx, cy = [float(v) for v in w.all_world2pix(NUC_RA, NUC_DEC, 0)] + cat = SkyCoord(d["ra"], d["dec"], unit="deg") + nuc = SkyCoord(NUC_RA, NUC_DEC, unit="deg") + + # ---- reference sets restricted to the frame ----------------------------- + def infield(ra, dec): + x, y = w.all_world2pix(ra, dec, 0) + return (x > 30) & (x < 4758) & (y > 30) & (y < 3164), x, y + + z = np.load(layout.path("_simbad.npz"), allow_pickle=True) + m = z["otype"].astype(str) == "GlC" + sra, sdec = z["ra"][m].astype(float), z["dec"][m].astype(float) + inf, _, _ = infield(sra, sdec) + sim = SkyCoord(sra[inf], sdec[inf], unit="deg") + + r = np.load(layout.path("_cena_gc_ref.npz")) + infr, _, _ = infield(r["ra"], r["dec"]) + good_p = infr & (r["prob"] >= 0.9) & np.isfinite(r["vmag"]) + sca = SkyCoord(r["ra"][good_p], r["dec"][good_p], unit="deg") + scav = r["vmag"][good_p] + print("in-field reference objects: SIMBAD GlC %d ; SCABS p>=0.9 with V %d" + % (len(sim), len(sca))) + + cand = d["cand"] + ccand = cat[cand] + call = cat[d["base"]] + + def recov(ref): + i, s, _ = ref.match_to_catalog_sky(ccand) + j, s2, _ = ref.match_to_catalog_sky(call) + return s.arcsec < 2.0, s2.arcsec < 2.0 + + hit_sim, det_sim = recov(sim) + hit_sca, det_sca = recov(sca) + print("SIMBAD GlC: detected at all %d/%d (%.0f%%), kept as candidate %d/%d (%.0f%%)" + % (det_sim.sum(), len(sim), 100 * det_sim.mean(), + hit_sim.sum(), len(sim), 100 * hit_sim.mean())) + print("SCABS p>=0.9: detected %d/%d (%.0f%%), kept %d/%d (%.0f%%)" + % (det_sca.sum(), len(sca), 100 * det_sca.mean(), + hit_sca.sum(), len(sca), 100 * hit_sca.mean())) + + # ---- purity ------------------------------------------------------------- + ci, cs, _ = ccand.match_to_catalog_sky(sim) + matched_sim = cs.arcsec < 2.0 + ci2, cs2, _ = ccand.match_to_catalog_sky( + SkyCoord(r["ra"][infr], r["dec"][infr], unit="deg")) + matched_sca = cs2.arcsec < 2.0 + any_match = matched_sim | matched_sca + print("\nPURITY: %d candidates; %d (%.0f%%) match SIMBAD GlC or SCABS; " + "%d (%.0f%%) unconfirmed" + % (cand.sum(), any_match.sum(), 100 * any_match.mean(), + (~any_match).sum(), 100 * (~any_match).mean())) + st = d["simbad_type"][cand] + bad = np.isin(st, ["*", "PM*", "V*", "RR*", "EB*", "LP*"]) + print(" candidates matching a SIMBAD STAR-type object: %d (%.1f%%)" + % (bad.sum(), 100 * bad.mean())) + print(" matching a SIMBAD galaxy: %d ; Cepheid: %d ; X-ray/LXB: %d" + % ((st == "G").sum(), (st == "Ce*").sum(), np.isin(st, ["X", "LXB"]).sum())) + rcand = d["r_arcmin"][cand] + print(" unconfirmed fraction inside 8': %.0f%% ; outside 8': %.0f%%" + % (100 * (~any_match)[rcand < 8].mean(), 100 * (~any_match)[rcand >= 8].mean())) + + # ---- figure ------------------------------------------------------------- + fig, ax = plt.subplots(1, 2, figsize=(12.6, 5.0)) + a = ax[0] + vb = np.arange(17.0, 22.6, 0.5) + vc = 0.5 * (vb[1:] + vb[:-1]) + fd, fk, nn = [], [], [] + for j in range(len(vb) - 1): + s = (scav >= vb[j]) & (scav < vb[j + 1]) + nn.append(s.sum()) + fd.append(det_sca[s].mean() if s.sum() > 4 else np.nan) + fk.append(hit_sca[s].mean() if s.sum() > 4 else np.nan) + a.plot(vc, fd, lw=2.2, marker="o", ms=7, color=C_KNOWN, + label="detected by our pipeline") + a.plot(vc, fk, lw=2.2, marker="s", ms=7, color=C_CAND, + label="detected AND kept as a candidate") + for x, y, n in zip(vc, fd, nn): + if np.isfinite(y) and n > 4: + a.annotate("%d" % n, (x, y), xytext=(0, 8), textcoords="offset points", + ha="center", fontsize=7.5, color=C_MUTED) + a.set_xlabel("SCABS V magnitude"); a.set_ylabel("fraction recovered") + a.set_ylim(0, 1.05) + a.set_title("a) recovery of published clusters vs brightness", fontsize=10.5, loc="left") + a.legend(fontsize=9, labelcolor=C_INK) + a.text(0.98, 0.9, "numbers = reference objects per bin", transform=a.transAxes, + ha="right", fontsize=8, color=C_MUTED) + + b = ax[1] + rsca = sca.separation(nuc).arcmin + rb = np.array([0, 2, 4, 6, 9, 12, 16, 21, 27]) + rc = 0.5 * (rb[1:] + rb[:-1]) + for arr, c, lab, mk in ((det_sca, C_KNOWN, "detected", "o"), + (hit_sca, C_CAND, "kept as candidate", "s")): + f, nnr = [], [] + for j in range(len(rb) - 1): + s = (rsca >= rb[j]) & (rsca < rb[j + 1]) + nnr.append(s.sum()) + f.append(arr[s].mean() if s.sum() > 4 else np.nan) + b.plot(rc, f, lw=2.2, marker=mk, ms=7, color=c, label=lab) + b.set_xlabel("projected radius (arcmin)"); b.set_ylabel("fraction recovered") + b.set_ylim(0, 1.05) + b.set_title("b) the same, vs radius (SCABS p$\\geq$0.9)", fontsize=10.5, loc="left") + b.legend(fontsize=9, labelcolor=C_INK) + fig.suptitle("External validation against SCABS (Taylor et al. 2017) and SIMBAD", + fontsize=12.5, x=0.008, ha="left") + fig.tight_layout() + fig.savefig(layout.path("NGC5128-gc-recovery.png"), dpi=115, bbox_inches="tight", + facecolor=SURF) + plt.close(fig) + print("wrote NGC5128-gc-recovery.png") + + +if __name__ == "__main__": + main() diff --git a/pipeline/hdr.py b/pipeline/hdr.py new file mode 100644 index 0000000..f77d651 --- /dev/null +++ b/pipeline/hdr.py @@ -0,0 +1,161 @@ +"""Pass 6: re-render with the core recovered. + +The first composite blew out the galaxy's centre, and the earlier explanation +for that - a saturated nucleus - was wrong. The surface-photometry analysis +found, and a direct check confirmed, that the bright clipped pixels near the +middle of the frame belong to a foreground star 128 px (69 arcsec) from the +nucleus. The galaxy's own light never comes close to the clip level: inside the +inner 800 x 800 px box there is not one star-free pixel above 30000 ADU. + +So the core was lost to the STRETCH, not to the sensor. Setting the white point +at the 99.995th percentile put it at 64283 ADU, a level set by field stars, +while the galaxy peaks around a twentieth of that. The midtone transfer needed +to lift a sky at 7 ADU into visibility then pushed everything above a few +thousand ADU to white. + +The fix is the standard high-dynamic-range one: stretch the same data twice and +blend. A faint-biased curve for the sky and halo, a bright-biased curve that +keeps the inner galaxy on the shoulder rather than the ceiling, and a mask that +chooses between them by brightness. No extra data is needed, and none of this +invents anything: both curves are monotonic functions of the same pixels. +""" +import os +import sys + +import numpy as np +import tifffile +from astropy.io import fits +from PIL import Image +from scipy.ndimage import gaussian_filter, median_filter + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import compose as C # noqa: E402 + +import layout + +OUT = C.OUT +FAINT_TARGET = 0.10 # where the sky sits in the faint-biased curve +CORE_HEADROOM_SCALE = 1.15 # bright-curve white point, as a multiple of the + # galaxy's own peak: slightly above it, so the very + # centre keeps a little headroom +CORE_MIDTONE = 0.35 # gentle: the core needs tonal separation, not lift + + +def bright_curve(lum_lin, galaxy_peak, black, white): + """A second stretch whose white point is the galaxy, not the field stars. + + This is the whole trick. The faint curve normalises against a white point + of 64283 ADU, set by field stars, so the galaxy's entire tonal range - sky + at 7 ADU up to a peak near 1944 - is squeezed into the top few percent of + the curve and comes out as a featureless white blob. Rescaling so that the + galaxy's own peak IS the white point spreads that same range across the + full output, and the bulge's smooth gradient and the dust lane silhouetted + against it become visible. Stars clip in this curve, which does not matter: + it is only ever used where the faint curve has already run out of room. + """ + lo = black + hi = galaxy_peak * CORE_HEADROOM_SCALE + norm = np.clip((lum_lin - lo) / (hi - lo), 0.0, 1.0) + print(f" bright curve: black {lo:.1f} ADU, white {hi:.0f} ADU " + f"(galaxy peak {galaxy_peak:.0f}), midtone {CORE_MIDTONE}") + return C.mtf(norm, CORE_MIDTONE) + + +def main(): + data, hdr = C.load() + shape = data["Luminance"].shape + mask = C.galaxy_mask(shape) + for name in C.CHANNELS: + data[name], *_ = C.remove_gradient(data[name], mask) + o, flux = C.star_photometry(data["Luminance"], data) + good = (flux["Red"] > 0) & (flux["Green"] > 0) & (flux["Blue"] > 0) + data["Red"] *= np.median(flux["Green"][good] / flux["Red"][good]) + data["Blue"] *= np.median(flux["Green"][good] / flux["Blue"][good]) + + lum_lin = data["Luminance"] + rgb_lin = np.dstack([data["Red"], data["Green"], data["Blue"]]) + del data + + # The galaxy's true peak, with stars filtered out. A median filter wide + # enough to swallow a stellar profile leaves the smooth galaxy alone. + ny, nx = shape + h = 500 + core = lum_lin[ny // 2 - h:ny // 2 + h, nx // 2 - h:nx // 2 + h] + galaxy_peak = float(median_filter(core, size=41).max()) + print(f"galaxy peak (star-free) {galaxy_peak:.0f} ADU vs frame max " + f"{lum_lin.max():.0f} ADU") + + lum_faint, params = C.autostretch(lum_lin, target=FAINT_TARGET) + lum_bright = bright_curve(lum_lin, galaxy_peak, params["black"], + params["white"]) + + # Blend on the FAINT curve's brightness: where it has run out of headroom, + # hand over to the bright curve. Feathered so the transition is invisible. + w = np.clip((lum_faint - 0.55) / 0.35, 0.0, 1.0) + w = gaussian_filter(w.astype(np.float32), 8.0) + lum = np.clip(lum_faint * (1.0 - w) + lum_bright * w, 0.0, 1.0) + print(f" HDR blend covers {float((w > 0.05).mean()):.2%} of the frame") + + rgb = np.empty_like(rgb_lin) + for i in range(3): + ch = rgb_lin[:, :, i] + sky = np.median(ch) + mad = 1.4826 * np.median(np.abs(ch - sky)) + black = sky - 2.8 * mad + white = np.percentile(ch, 99.995) + norm = np.clip((ch - black) / (white - black), 0, 1) + cf = C.mtf(norm, params["midtone"]) + # The colour channels get the same two-curve treatment, so the core + # keeps its colour instead of going white while the luminance holds + # detail. + peak_c = float(median_filter( + ch[ny // 2 - h:ny // 2 + h, nx // 2 - h:nx // 2 + h], + size=41).max()) + nb = np.clip((ch - black) / (peak_c * CORE_HEADROOM_SCALE - black), + 0.0, 1.0) + cb = C.mtf(nb, CORE_MIDTONE) + rgb[:, :, i] = cf * (1.0 - w) + cb * w + del rgb_lin + + sky_med = [float(np.median(rgb[:, :, i][~mask])) for i in range(3)] + target = float(np.mean(sky_med)) + for i in range(3): + rgb[:, :, i] = np.clip(rgb[:, :, i] - (sky_med[i] - target), 0.0, 1.0) + + rgb_lum = rgb.mean(axis=2, keepdims=True) + chroma = rgb - rgb_lum + for i in range(3): + chroma[:, :, i] = gaussian_filter(median_filter(chroma[:, :, i], 3), + 1.5) + rgb = np.clip(rgb_lum + chroma * C.SATURATION, 0.0, 1.0) + del chroma, rgb_lum + + detail = lum - gaussian_filter(lum, 2.0) + weight = np.clip((lum - FAINT_TARGET) * 4.0, 0.0, 1.0) + lum = np.clip(lum + 0.35 * detail * weight, 0.0, 1.0) + del detail, weight + + ratio = lum / np.maximum(rgb.mean(axis=2), 1e-5) + out = np.clip(rgb * ratio[:, :, None], 0.0, 1.0) + del ratio, rgb + + neutral = 0.5 * (out[:, :, 0] + out[:, :, 2]) + green = out[:, :, 1] + out[:, :, 1] = np.where(green > neutral, green * 0.15 + neutral * 0.85, + green) + + frac = float((out.max(axis=2) > 0.995).mean()) + print(f" pixels at full white: {frac:.3%}") + + Image.fromarray((out * 255 + 0.5).astype(np.uint8)).save( + layout.path("NGC5128-LRGB-hdr.png")) + tifffile.imwrite(layout.path("NGC5128-LRGB-hdr.tif"), + (out * 65535 + 0.5).astype(np.uint16), photometric="rgb") + prev = Image.fromarray((out * 255 + 0.5).astype(np.uint8)) + prev.thumbnail((2400, 2400), Image.LANCZOS) + prev.save(layout.path("NGC5128-LRGB-hdr-preview.jpg"), quality=93) + print("wrote NGC5128-LRGB-hdr.png / .tif / -preview.jpg") + + +if __name__ == "__main__": + main() diff --git a/pipeline/layout.py b/pipeline/layout.py new file mode 100644 index 0000000..99bdeb9 --- /dev/null +++ b/pipeline/layout.py @@ -0,0 +1,115 @@ +"""Where every file of a processed session lives. + +The scripts used to sit inside the data directory and address it with absolute +paths. Now that they live in a repository and the data is organised into named +subdirectories, they need one place that knows the mapping - this module. + +Two rules make it work: + +* The session root comes from the ASTRO_SESSION environment variable, so the + same code runs against any session without editing. It falls back to the + NGC 5128 session these scripts were written against. +* A file's subdirectory is derived from its NAME, using the same rules the + directory was organised with. That means a script can go on asking for + "master-Red.fit" or "_stars.npz" and get the right path, without every call + site having to learn the layout. + +Anything unrecognised lands at the session root, which is visible and easy to +correct rather than silently wrong. +""" +import os + +SESSION = os.environ.get( + "ASTRO_SESSION", + r"C:\Users\lhorrocks-barlow\Downloads\NGC5128\20260721") + +# Subdirectories, also created on demand by path(). +RAW = "raw" +CALIBRATED = "calibrated" +MASTERS = os.path.join("stacks", "masters") +ORIGINAL = os.path.join("stacks", "original") +FINAL = "final" +RENDERINGS = "renderings" +FIGURES = os.path.join("science", "figures") +CATALOGUES = os.path.join("science", "catalogues") +SCIENCE_DATA = os.path.join("science", "data") +NOTES = os.path.join("science", "notes") +INTERMEDIATES = "intermediates" + +# Names that are directories in their own right, not files. +_DIRS = {"final": FINAL, "original": ORIGINAL, "raw": RAW, + "calibrated": CALIBRATED, "renderings": RENDERINGS, + "_dss": os.path.join(INTERMEDIATES, "_dss")} + + +def subdir(name): + """The subdirectory a given filename belongs in.""" + low = name.lower() + if name in _DIRS: + return _DIRS[name] + if name.startswith("_"): + return INTERMEDIATES + if low.endswith(".zip") or (low.startswith("jpeg-") and + low.endswith(".jpg")): + return RAW + if low.startswith("calibrated-") and low.endswith((".fit", ".tif")): + return CALIBRATED + if low.startswith("raw-") and low.endswith((".fit", ".tif")): + return RAW + if low.startswith("master-") and low.endswith(".fit"): + return MASTERS + if low.startswith("original-"): + return ORIGINAL + if "final" in low and low.endswith((".png", ".tif", ".jpg")): + return FINAL + if low.startswith("notes-") and low.endswith(".md"): + return NOTES + if low.endswith(".csv"): + return CATALOGUES + if low.startswith("sb-") and low.endswith((".fits", ".npy", ".txt")): + return SCIENCE_DATA + if low.endswith(".png") and any(f"-{p}-" in low for p in ("gc", "sb", "mo")): + return FIGURES + if low.startswith(("ngc5128-lrgb", "ngc5128-img-")): + return RENDERINGS + if low.endswith(".md"): + return "" + return "" + + +def path(*parts, make=True): + """Absolute path for a session file, addressed by name alone. + + Extra leading components are accepted and ignored, so calls written + against the old flat layout - path("stacked", "_stars.npz") - still land + correctly. The last component is the one that decides. + """ + name = parts[-1] + if name in _DIRS: + # Asking for a directory by name returns the directory itself, not a + # file of that name nested inside it. + full = os.path.join(SESSION, _DIRS[name]) + if make: + os.makedirs(full, exist_ok=True) + return full + full = os.path.join(SESSION, subdir(name), name) + if make: + os.makedirs(os.path.dirname(full), exist_ok=True) + return full + + +def describe(): + lines = [f"session root: {SESSION}"] + for label, d in (("raw", RAW), ("calibrated", CALIBRATED), + ("masters", MASTERS), ("original stacks", ORIGINAL), + ("final images", FINAL), ("other renderings", RENDERINGS), + ("science figures", FIGURES), + ("science catalogues", CATALOGUES), + ("science data", SCIENCE_DATA), ("notes", NOTES), + ("intermediates", INTERMEDIATES)): + lines.append(f" {label:20s} {d}") + return "\n".join(lines) + + +if __name__ == "__main__": + print(describe()) diff --git a/pipeline/mo_cavs.py b/pipeline/mo_cavs.py new file mode 100644 index 0000000..8a79f27 --- /dev/null +++ b/pipeline/mo_cavs.py @@ -0,0 +1,567 @@ +"""Run down the CAVS 210 photometry discrepancy. + +The transient search measured G = 18.20 for the source at +13:25:37.68 -43:02:29.9, where Gaia DR3 lists G = 21.03 for the catalogued +variable at that position. That is 2.8 mag, a factor of 13 in flux, and it is +the only anomaly in either search. There are three ways it can resolve: + +1. a measurement artefact - a blend inside the 5 px aperture, galaxy-halo + contamination, a wrong catalogue match, or an aperture/zero-point problem + specific to this position; +2. a genuine brightening - Gaia's G is a mean over its observation window, not + a value for 2026-07-21, and a catalogued variable near maximum would be a + real detection; +3. a wrong archival identification. + +This script gathers the evidence needed to tell them apart: + +* a full Gaia DR3 cone search to ALL magnitudes (the cached catalogue used by + the search was cut at G < 20.5, which is exactly why the variable was + missed), so the match and its separation can be checked properly; +* a census of every neighbour in the master within 15 arcsec; +* a curve of growth at the source compared against the median curve of growth + of isolated field stars - a blend keeps rising where a point source flattens; +* per-sub aperture photometry across all 12 luminance subs with a local + annulus background and proper uncertainties, giving a light curve rather + than one stacked number, plus comparison stars of similar brightness to show + what the instrumental scatter actually is; +* the same photometry through a small (2 px) aperture, which is far less + sensitive to blending; +* radial profile and second moments against the image PSF. + +Outputs NGC5128-mo-cavs210.png and _mo_cavs.npz. +""" +import os + +import numpy as np +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import sep +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.stats import sigma_clipped_stats +from astropy.wcs import WCS +from scipy.spatial import cKDTree + +import mo_common as C + +RA, DEC = 201.406996, -43.041648 +OUTPNG = os.path.join(C.OUT, "NGC5128-mo-cavs210.png") +CACHE_GAIA = os.path.join(C.OUT, "_cavs_gaia.npz") + +RADII = np.array([1.0, 1.5, 2.0, 2.5, 3.0, 4.0, 5.0, 6.0, 8.0, 10.0, 12.0]) +ANN_IN, ANN_OUT = 12.0, 20.0 # px, local background annulus + + +def gaia_all(ra, dec, radius_arcsec=25.0): + """Every Gaia DR3 source near the position, to any magnitude. + + The search itself used a cached catalogue cut at G < 20.5, which is + precisely why the variable at this position was not matched. This goes back + to the full catalogue with no magnitude limit. + + Served from VizieR's Gaia DR3 mirror (I/355/gaiadr3): ESA's own archive was + down for maintenance when this was run, and VizieR carries the identical + catalogue. + """ + if os.path.exists(CACHE_GAIA): + z = np.load(CACHE_GAIA, allow_pickle=True) + return z["ra"], z["dec"], z["g"], z["src"], z["var"] + from astroquery.vizier import Vizier + v = Vizier(columns=["Source", "RA_ICRS", "DE_ICRS", "Gmag", "BPmag", + "RPmag", "VarFlag", "Plx", "pmRA", "pmDE"], + row_limit=-1) + t = v.query_region(SkyCoord(ra * u.deg, dec * u.deg), + radius=radius_arcsec * u.arcsec, + catalog="I/355/gaiadr3")[0] + out = (np.asarray(t["RA_ICRS"], float), np.asarray(t["DE_ICRS"], float), + np.asarray(t["Gmag"], float), + np.asarray([str(x) for x in t["Source"]]), + np.asarray([str(x) for x in (t["VarFlag"] if "VarFlag" in + t.colnames else + ["-"] * len(t))])) + extra = {c: np.asarray(t[c], float) for c in ("BPmag", "RPmag", "Plx", + "pmRA", "pmDE") + if c in t.colnames} + np.savez(CACHE_GAIA, ra=out[0], dec=out[1], g=out[2], src=out[3], + var=out[4], **extra) + return out + + +def ann_background(img, x, y, rin=ANN_IN, rout=ANN_OUT): + """Sigma-clipped median sky per pixel in an annulus, and its scatter.""" + h = int(rout) + 2 + xi, yi = int(round(x)), int(round(y)) + cut = img[yi - h:yi + h + 1, xi - h:xi + h + 1].astype(float) + gy, gx = np.mgrid[:cut.shape[0], :cut.shape[1]] + r = np.hypot(gx - (x - xi + h), gy - (y - yi + h)) + m = (r >= rin) & (r <= rout) & np.isfinite(cut) + med, _, sd = sigma_clipped_stats(cut[m], sigma=3.0) + return float(med), float(sd) + + +def cog(img, x, y, radii=RADII): + """Curve of growth with the local annulus sky removed.""" + sky, sd = ann_background(img, x, y) + f = [] + for r in radii: + v, _, _ = sep.sum_circle(img, np.array([x]), np.array([y]), r, + gain=1.0) + f.append(float(v[0]) - sky * np.pi * r ** 2) + return np.array(f), sky, sd + + +def archival_and_colour(msub, px, py, objs, omag, wcs): + """Was it this bright in the 1990s, and what colour is it? + + A brightening cannot be tested against DSS in absolute terms - the plates + have no useful zero point here - but it can be tested differentially. The + neighbour 5.4 arcsec away is a normal point source of known brightness in + this image; if the ratio between the two objects is the same on a 1990s + plate as it is tonight, then this source has not changed. + """ + import warnings + from astroquery.hips2fits import hips2fits + + d = np.hypot(objs["x"] - px, objs["y"] - py) + order = np.argsort(d) + refs = [i for i in order[1:] if 2.0 < d[i] * C.SCALE < 40.0][:4] + print("\ndifferential check against DSS2 red (1990s):") + print(" reference sources in this image:") + for i in refs: + print(f" sep {d[i] * C.SCALE:5.2f}\" G = {omag[i]:5.2f}") + + fov = 2.0 / 60.0 # deg + npix = 240 # 0.5 arcsec/px + try: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + h = hips2fits.query(hips="CDS/P/DSS2/red", width=npix, + height=npix, ra=RA * u.deg, dec=DEC * u.deg, + fov=fov * u.deg, projection="TAN", + format="fits") + dss = np.asarray(h[0].data, dtype=float) + except Exception as exc: # noqa: BLE001 + print(f" DSS fetch failed: {type(exc).__name__}: {exc}") + return None + dwcs = WCS(h[0].header) + dss = dss.astype(np.float32) + dbkg = sep.Background(dss, bw=32, bh=32, fw=3, fh=3) + dsub = dss - dbkg.back() + + def dss_mag(ra_, dec_): + qx, qy = dwcs.world_to_pixel(SkyCoord(ra_ * u.deg, dec_ * u.deg)) + f, _, _ = sep.sum_circle(dsub, np.array([float(qx)]), + np.array([float(qy)]), 4.0, gain=1.0) + return float(f[0]) + + f_src = dss_mag(RA, DEC) + print(f" source DSS flux (r = 2 arcsec): {f_src:.1f}") + rows = [] + for i in refs: + c = wcs.pixel_to_world(objs["x"][i], objs["y"][i]) + fr = dss_mag(c.ra.deg, c.dec.deg) + if fr <= 0 or f_src <= 0: + print(f" ref at {d[i] * C.SCALE:5.2f}\": DSS flux " + f"{fr:.1f} - unusable") + continue + dm_now = omag[np.argmin(d)] - omag[i] + dm_dss = -2.5 * np.log10(f_src / fr) + rows.append((d[i] * C.SCALE, omag[i], dm_now, dm_dss)) + print(f" vs source at {d[i] * C.SCALE:5.2f}\" (G={omag[i]:.2f}): " + f"delta-mag tonight {dm_now:+.2f}, on DSS {dm_dss:+.2f}, " + f"change {dm_dss - dm_now:+.2f}") + ch = float(np.median([r[3] - r[2] for r in rows])) if rows else np.nan + if rows: + print(f" --> median implied brightness change since the 1990s: " + f"{ch:+.2f} mag") + + # colours from the pixel-aligned RGB masters + print("\ncolour from the R, G, B masters (same 5 px aperture):") + out = {} + for filt in ("Red", "Green", "Blue"): + with fits.open(os.path.join(C.OUT, f"master-{filt}.fit")) as hd: + im = hd[0].data.astype(np.float32) + b = sep.Background(im, bw=64, bh=64, fw=3, fh=3) + sb = im - b.back() + del im + fs, _, _ = sep.sum_circle(sb, np.array([px]), np.array([py]), + C.APRAD, gain=1.0) + # normalise against the same comparison sources so the numbers are + # differential and do not need per-filter zero points + ref_f = [] + for i in refs: + fr, _, _ = sep.sum_circle(sb, np.array([float(objs["x"][i])]), + np.array([float(objs["y"][i])]), + C.APRAD, gain=1.0) + if fr[0] > 0: + ref_f.append(float(fr[0])) + del sb + if ref_f and fs[0] > 0: + rel = -2.5 * np.log10(float(fs[0]) / np.median(ref_f)) + out[filt] = rel + print(f" {filt:6s}: source is {rel:+.2f} mag relative to the " + f"median neighbour") + if len(out) == 3: + print(f" --> (source-neighbour) colour B-R = " + f"{out['Blue'] - out['Red']:+.2f} mag " + f"(positive = redder than the neighbours)") + return dict(dss=dsub, dwcs=dwcs, rows=rows, change=ch, colours=out, + refs=refs) + + +def main(): + with fits.open(os.path.join(C.OUT, "master-Luminance.fit")) as hd: + master = hd[0].data.astype(np.float32) + hdr = hd[0].header + wcs = WCS(hdr) + px, py = wcs.world_to_pixel(SkyCoord(RA * u.deg, DEC * u.deg)) + px, py = float(px), float(py) + print(f"CAVS 210 at master pixel ({px:.2f}, {py:.2f})") + + bkg = sep.Background(master, bw=64, bh=64, fw=3, fh=3) + back = bkg.back() + msub = master - back + print(f"local sep background {back[int(py), int(px)]:.1f} ADU/px vs " + f"field median {np.median(back):.1f}; global rms " + f"{bkg.globalrms:.2f}") + sky_l, sky_sd = ann_background(msub, px, py) + print(f"residual sky in the 12-20 px annulus after sep subtraction: " + f"{sky_l:+.2f} +- {sky_sd:.2f} ADU/px") + + # ---- 1. Gaia, all magnitudes ------------------------------------- + gra, gdec, gg, gsrc, gvar = gaia_all(RA, DEC) + sc0 = SkyCoord(RA * u.deg, DEC * u.deg) + gsep = sc0.separation(SkyCoord(gra * u.deg, gdec * u.deg)).arcsec + order = np.argsort(gsep) + print(f"\nGaia DR3 within 25 arcsec ({len(gra)} sources):") + for i in order[:8]: + print(f" {gsrc[i]:>20s} sep {gsep[i]:6.2f}\" G = {gg[i]:6.2f} " + f"variable={gvar[i]}") + + # ---- 2. neighbours in this image --------------------------------- + objs = sep.extract(msub, 3.0, err=bkg.globalrms, minarea=5, + deblend_cont=0.005) + apf, _, _ = sep.sum_circle(msub, objs["x"], objs["y"], C.APRAD, + err=bkg.globalrms, gain=1.0) + omag = -2.5 * np.log10(np.maximum(apf, 1e-9)) + C.ZP + d = np.hypot(objs["x"] - px, objs["y"] - py) + near = np.argsort(d)[:8] + print(f"\nneighbours detected in this image within 15 arcsec:") + for i in near: + if d[i] * C.SCALE > 15: + break + print(f" sep {d[i] * C.SCALE:6.2f}\" ({d[i]:5.2f} px) " + f"G = {omag[i]:6.2f} a={objs['a'][i]:.2f} b={objs['b'][i]:.2f} " + f"npix={objs['npix'][i]}") + + # ---- 3. curve of growth vs stars --------------------------------- + tree = cKDTree(np.column_stack([objs["x"], objs["y"]])) + iso = [] + for i in range(len(objs)): + if not (17.0 < omag[i] < 19.0): + continue + if not (200 < objs["x"][i] < 4588 and 200 < objs["y"][i] < 2994): + continue + if np.hypot(objs["x"][i] - 2394, objs["y"][i] - 1597) < 900: + continue + nb = tree.query_ball_point([objs["x"][i], objs["y"][i]], 25.0) + if len(nb) > 1: + continue + iso.append(i) + iso = iso[:120] + print(f"\ncurve of growth from {len(iso)} isolated field stars " + f"G = 17-19") + star_cogs = [] + for i in iso: + f, _, _ = cog(msub, float(objs["x"][i]), float(objs["y"][i])) + if f[RADII == 10.0][0] > 0: + star_cogs.append(f / f[RADII == 10.0][0]) + star_cog = np.median(np.array(star_cogs), axis=0) + src_cog, _, _ = cog(msub, px, py) + src_cog_n = src_cog / src_cog[RADII == 10.0][0] + print(" r(px) star source") + for r, a, b in zip(RADII, star_cog, src_cog_n): + print(f" {r:5.1f} {a:5.3f} {b:5.3f}") + m5 = -2.5 * np.log10(src_cog[RADII == 5.0][0]) + C.ZP + m2 = -2.5 * np.log10(src_cog[RADII == 2.0][0] / + star_cog[RADII == 2.0][0]) + C.ZP + print(f"\n aperture magnitude, r=5 px, local sky : G = {m5:.2f}") + print(f" PSF-scaled from r=2 px core : G = {m2:.2f}") + + # ---- 4. per-sub light curve -------------------------------------- + tforms = C.frame_transforms() + mobjs, map_, _, _, _ = C.master_sources() + # comparison stars: isolated, similar brightness, similar distance out + comp = [i for i in iso if 17.8 < omag[i] < 18.6][:6] + print(f"\nper-sub photometry ({len(comp)} comparison stars)") + times, lc, lcerr, lc2, comps = [], [], [], [], [] + for i, (key, path) in enumerate(C.lum_frames()): + with fits.open(path, memmap=False) as hd: + img = hd[0].data.astype(np.float32) + b = sep.Background(img, bw=64, bh=64, fw=3, fh=3) + s = img - b.back() + del img + o, ap, _ = C.detect(s + 0.0) + zp, _ = C.frame_zeropoint(o["x"], o["y"], ap, mobjs, map_, + tforms[key]) + inv = tforms[key].inverse + nx_, ny_ = inv(np.array([[px, py]]))[0] + skyv, skysd = ann_background(s, nx_, ny_) + f5, e5, _ = sep.sum_circle(s, np.array([nx_]), np.array([ny_]), + C.APRAD, err=float(b.globalrms), gain=1.0) + f5 = float(f5[0]) - skyv * np.pi * C.APRAD ** 2 + # uncertainty: photon+read noise in the aperture, plus the uncertainty + # on the local sky level scaled by the aperture area + npix_ap = np.pi * C.APRAD ** 2 + err = float(np.hypot(e5[0], skysd * npix_ap / + np.sqrt(max(np.pi * (ANN_OUT ** 2 - ANN_IN ** 2), + 1.0)))) + f2, _, _ = sep.sum_circle(s, np.array([nx_]), np.array([ny_]), 2.0, + gain=1.0) + f2 = float(f2[0]) - skyv * np.pi * 4.0 + times.append(C.lum_times()[i]) + lc.append(-2.5 * np.log10(max(f5, 1e-9)) + zp) + lcerr.append(1.0857 * err / max(f5, 1e-9)) + lc2.append(-2.5 * np.log10(max(f2 / star_cog[RADII == 2.0][0], 1e-9)) + + zp) + row = [] + for ci in comp: + cx, cy = inv(np.array([[float(objs["x"][ci]), + float(objs["y"][ci])]]))[0] + sv, _ = ann_background(s, cx, cy) + fc, _, _ = sep.sum_circle(s, np.array([cx]), np.array([cy]), + C.APRAD, err=float(b.globalrms), + gain=1.0) + row.append(-2.5 * np.log10(max(float(fc[0]) - + sv * npix_ap, 1e-9)) + zp) + comps.append(row) + del s + print(f" {key}: G = {lc[-1]:.3f} +- {lcerr[-1]:.3f} " + f"(r=2px core: {lc2[-1]:.3f}) sky {skyv:+.2f}") + + lc = np.array(lc); lcerr = np.array(lcerr); lc2 = np.array(lc2) + comps = np.array(comps) + print(f"\nsource : mean G = {lc.mean():.3f}, rms {lc.std():.3f}, " + f"median formal error {np.median(lcerr):.3f}") + for j in range(comps.shape[1]): + print(f" comp {j + 1}: mean {comps[:, j].mean():.3f}, " + f"rms {comps[:, j].std():.3f}") + comp_rms = float(np.median([comps[:, j].std() + for j in range(comps.shape[1])])) + print(f" median comparison-star rms: {comp_rms:.3f} mag") + + # ---- 5. shape ---------------------------------------------------- + di = np.argmin(np.hypot(objs["x"] - px, objs["y"] - py)) + r50src, _ = sep.flux_radius(msub, np.array([objs["x"][di]]), + np.array([objs["y"][di]]), np.array([6.0]), + 0.5, normflux=np.array([apf[di]]), subpix=5) + r50s = [] + for i in iso: + rr, _ = sep.flux_radius(msub, np.array([objs["x"][i]]), + np.array([objs["y"][i]]), np.array([6.0]), + 0.5, normflux=np.array([apf[i]]), subpix=5) + r50s.append(float(rr[0])) + print(f"\nhalf-light radius: source {float(r50src[0]):.2f} px, " + f"isolated stars {np.median(r50s):.2f} +- {np.std(r50s):.2f} px " + f"-> {float(r50src[0]) / np.median(r50s):.2f} x PSF") + print(f"source second moments a={objs['a'][di]:.2f} b={objs['b'][di]:.2f} " + f"-> elongation {objs['a'][di] / objs['b'][di]:.2f}") + + arch = archival_and_colour(msub, px, py, objs, omag, wcs) + + np.savez(os.path.join(C.OUT, "_mo_cavs.npz"), + px=px, py=py, times=np.array(times), lc=lc, lcerr=lcerr, + lc2=lc2, comps=comps, comp_rms=comp_rms, + radii=RADII, star_cog=star_cog, src_cog=src_cog_n, + gsep=gsep, gg=gg, gsrc=gsrc, gvar=gvar, + r50src=float(r50src[0]), r50star=float(np.median(r50s)), + m5=m5, m2=m2) + + # ---- figure ------------------------------------------------------ + make_figure(msub, px, py, objs, omag, gra, gdec, gsep, gg, gsrc, wcs, + np.array(times), lc, lcerr, comps, comp_rms, star_cog, + src_cog_n, float(r50src[0]), float(np.median(r50s)), m5, + arch) + + +def make_figure(msub, px, py, objs, omag, gra, gdec, gsep, gg, gsrc, wcs, + times, lc, lcerr, comps, comp_rms, star_cog, src_cog, + r50src, r50star, m5, arch): + fig = plt.figure(figsize=(15.0, 9.0)) + gs = fig.add_gridspec(2, 4, height_ratios=[1.0, 0.85], + hspace=0.42, wspace=0.30, + left=0.055, right=0.985, top=0.795, bottom=0.085) + + H = 26 + + def mark_gaia(ax, x0, y0, scale, wc): + for i in np.argsort(gsep)[:4]: + gx, gy = wc.world_to_pixel(SkyCoord(gra[i] * u.deg, + gdec[i] * u.deg)) + dx = (float(gx) - x0) * scale + dy = (float(gy) - y0) * scale + if abs(dx) > H or abs(dy) > H: + continue + ax.scatter([dx], [dy], s=120, facecolor="none", + edgecolor="#4fb3d9", lw=1.3) + ax.annotate(f"G={gg[i]:.1f}", (dx, dy), + textcoords="offset points", xytext=(7, 4), + fontsize=7.5, color="#4fb3d9") + + # --- this image --- + ax = fig.add_subplot(gs[0, 0]) + cut = msub[int(py) - H:int(py) + H + 1, int(px) - H:int(px) + H + 1] + v1, v2 = np.percentile(cut, [15, 99.6]) + ax.imshow(cut, origin="lower", cmap="gray", vmin=v1, vmax=v2, + extent=[-H, H, -H, H]) + mark_gaia(ax, int(px), int(py), 1.0, wcs) + ax.add_patch(plt.Circle((0, 0), C.APRAD, fill=False, color="#f0c05a", + lw=1.5)) + ax.set_xlim(-H, H) + ax.set_ylim(-H, H) + ax.set_title("this image, 2026-07-21\n28 x 28 arcsec", fontsize=9.5, + pad=6) + ax.set_xlabel("px from centroid; yellow = 5 px aperture", fontsize=8) + + # --- archival plate at the same angular scale --- + axd = fig.add_subplot(gs[0, 1]) + if arch is not None: + dsub, dwcs = arch["dss"], arch["dwcs"] + dx0, dy0 = dwcs.world_to_pixel(SkyCoord(RA * u.deg, DEC * u.deg)) + dx0, dy0 = float(dx0), float(dy0) + sc = 0.5 / C.SCALE # DSS is 0.5 arcsec/px + hd_ = int(H / sc) + 1 + dcut = dsub[int(dy0) - hd_:int(dy0) + hd_ + 1, + int(dx0) - hd_:int(dx0) + hd_ + 1] + w1, w2 = np.percentile(dcut, [15, 99.6]) + axd.imshow(dcut, origin="lower", cmap="gray", vmin=w1, vmax=w2, + extent=[-hd_ * sc, hd_ * sc, -hd_ * sc, hd_ * sc]) + mark_gaia(axd, int(dx0), int(dy0), sc, dwcs) + axd.add_patch(plt.Circle((0, 0), C.APRAD, fill=False, + color="#f0c05a", lw=1.5)) + axd.set_xlim(-H, H) + axd.set_ylim(-H, H) + axd.set_title(f"archival DSS2 red, 1990s\nimplied change " + f"{arch['change']:+.2f} mag", fontsize=9.5, pad=6) + axd.set_xlabel("already present, at the same brightness\n" + "relative to its neighbours", fontsize=8) + else: + axd.text(0.5, 0.5, "DSS unavailable", ha="center", va="center", + transform=axd.transAxes) + axd.set_xticks([]) + axd.set_yticks([]) + + # --- curve of growth --- + axc = fig.add_subplot(gs[0, 2]) + axc.plot(RADII, star_cog, "-o", color="#3d7ba6", ms=4.5, lw=1.9, + label="isolated field stars") + axc.plot(RADII, src_cog, "-s", color="#b5484f", ms=4.5, lw=1.9, + label="this source") + axc.axvline(C.APRAD, color="#f0a83c", ls="--", lw=1.2) + axc.text(C.APRAD + 0.25, 0.13, "5 px aperture", fontsize=7.6, + color="#a8792c", rotation=90) + axc.set_xlabel("aperture radius (px)", fontsize=9) + axc.set_ylabel("enclosed flux / flux at r = 10 px", fontsize=9) + axc.set_title(f"curve of growth: resolved\nr50 = {r50src / r50star:.2f}" + f" x PSF", fontsize=9.5, pad=6) + axc.legend(fontsize=8, frameon=False, loc="lower right") + axc.grid(alpha=0.25, lw=0.6) + for sp in ("top", "right"): + axc.spines[sp].set_visible(False) + + # --- what the catalogues have here --- + axt = fig.add_subplot(gs[0, 3]) + axt.axis("off") + d = np.hypot(objs["x"] - px, objs["y"] - py) * C.SCALE + lines = ["detected in this image", ""] + for i in np.argsort(d)[:5]: + if d[i] > 16: + break + lines.append(f" {d[i]:5.2f}\" G = {omag[i]:5.2f}") + lines += ["", "Gaia DR3 (all magnitudes)", ""] + for i in np.argsort(gsep)[:5]: + lines.append(f" {gsep[i]:5.2f}\" G = {gg[i]:5.2f}") + axt.text(0.0, 1.0, "\n".join(lines), fontsize=8.4, family="monospace", + va="top", color="#2a2e34", transform=axt.transAxes, + linespacing=1.4) + axt.text(0.0, 0.16, + "Nothing in Gaia within 25 arcsec is\n" + "brighter than G = 19.8. Gaia has no\n" + "entry for the extended object that\n" + "dominates the light in the aperture.", + fontsize=8.2, va="top", color="#4a4f57", + transform=axt.transAxes, linespacing=1.5) + + # --- light curve --- + axl = fig.add_subplot(gs[1, 0:3]) + for j in range(comps.shape[1]): + off = comps[:, j] - comps[:, j].mean() + axl.plot(times * 60, off + lc.mean(), "-", color="#c3ccd4", lw=1.0, + zorder=1) + axl.plot([], [], "-", color="#c3ccd4", lw=1.0, + label=f"{comps.shape[1]} comparison stars, mean-subtracted " + f"(rms {comp_rms:.3f} mag)") + axl.errorbar(times * 60, lc, yerr=lcerr, fmt="o", color="#b5484f", + ms=5.5, lw=1.4, capsize=2.5, label="this source", zorder=3) + axl.axhline(lc.mean(), color="#b5484f", ls=":", lw=1.1) + axl.invert_yaxis() + axl.set_xlabel("minutes from first sub (2026-07-21 08:58 UTC)", + fontsize=9) + axl.set_ylabel("G (luminance, 5 px aperture)", fontsize=9) + axl.set_title(f"per-sub light curve: mean G = {lc.mean():.2f}, rms " + f"{lc.std():.3f} mag - no variation over 64 min", + fontsize=9.5, pad=6) + axl.legend(fontsize=8, frameon=False, loc="lower left") + axl.grid(alpha=0.25, lw=0.6) + for sp in ("top", "right"): + axl.spines[sp].set_visible(False) + + # --- verdict --- + axv = fig.add_subplot(gs[1, 3]) + axv.axis("off") + ch = arch["change"] if arch is not None else float("nan") + axv.text(0.0, 1.0, "VERDICT: measurement artefact,\nnot a brightening.", + fontsize=10, va="top", color="#b5484f", weight="bold", + transform=axv.transAxes, linespacing=1.4) + txt = ( + f"1. Resolved. r50 = {r50src:.2f} px vs PSF\n" + f" {r50star:.2f} px ({r50src / r50star:.2f} x), elongation\n" + " 1.40. An integrated aperture\n" + " magnitude of an extended object\n" + " is not comparable with Gaia's\n" + " point-source G.\n\n" + f"2. Steady. rms {lc.std():.3f} mag over 64\n" + f" min, vs {comp_rms:.3f} for field stars.\n\n" + f"3. Archival. Implied change since\n" + f" the 1990s: {ch:+.2f} mag.\n\n" + "4. Even the PSF-scaled 2 px core\n" + f" gives G = 18.6, still 2.4 mag\n" + " above Gaia's value." + ) + axv.text(0.0, 0.86, txt, fontsize=8.5, va="top", color="#2a2e34", + transform=axv.transAxes, linespacing=1.45) + + fig.text(0.02, 0.958, + "The G = 18.2 versus Gaia G = 21.0 discrepancy at " + "13:25:37.68 -43:02:29.9", fontsize=13.5, weight="bold", + color="#1b1e23", ha="left") + fig.text(0.02, 0.915, + "The source is resolved and elongated, photometrically steady " + "across the session, and already present on a 1990s sky-survey " + "plate at the same brightness relative to its neighbours.", + fontsize=9.5, color="#4a4f57", ha="left") + fig.text(0.02, 0.891, + "Gaia DR3 catalogues only a G = 21.0 point source 1.32 arcsec " + "away and has no entry for the extended object that dominates " + "the aperture. The two numbers measure different things.", + fontsize=9.5, color="#4a4f57", ha="left") + + fig.savefig(OUTPNG, dpi=130, facecolor="white") + print(f"\nwrote {OUTPNG}") + + +if __name__ == "__main__": + main() diff --git a/pipeline/mo_check.py b/pipeline/mo_check.py new file mode 100644 index 0000000..5423e6a --- /dev/null +++ b/pipeline/mo_check.py @@ -0,0 +1,85 @@ +"""Diagnostic: how good is the coordinate registration, and how many +detections survive removal of the static sky? + +astroalign only reports a few dozen matched stars, which is enough to define a +similarity transform but says nothing about how well it holds across a +4788 x 3194 field. This refines each frame's transform by nearest-neighbour +matching every detection against the reference frame's detections and refitting +a full affine, then quotes the residual. It also counts, per frame, how many +detections are left once everything coincident with a master-stack source is +removed - that residual population is the input to the tracklet search. +""" +import os + +import numpy as np +from astropy.io import fits +from scipy.spatial import cKDTree +from skimage.transform import AffineTransform + +import layout + +SRC = layout.SESSION +OUT = layout.SESSION +CACHE = layout.path("_mo_dets.npz") + + +def load(): + z = np.load(CACHE, allow_pickle=True) + meta = {r[0]: r for r in z["meta"]} + dets = {k: z[k] for k in meta} + return meta, dets + + +def refine(dets, ref_key="Luminance_002", tol=3.0): + """Refit each frame's transform on all cross-matched detections.""" + ref = dets[ref_key] + tree = cKDTree(ref[:, :2]) + out = {} + for key, d in dets.items(): + src = d[:, 2:4].astype(float) # native coords + cur = d[:, :2].astype(float) # astroalign-transformed + for t in (tol, 1.5): + dist, idx = tree.query(cur, distance_upper_bound=t) + ok = np.isfinite(dist) + if ok.sum() < 50: + break + tf = AffineTransform() + tf.estimate(src[ok], ref[idx[ok], :2].astype(float)) + cur = tf(src) + dist, idx = tree.query(cur, distance_upper_bound=1.5) + ok = np.isfinite(dist) + out[key] = (cur, np.median(dist[ok]), np.percentile(dist[ok], 90), + ok.sum()) + return out + + +def main(): + meta, dets = load() + lum = [k for k in dets if k.startswith("Luminance")] + ref = refine(dets) + + print("registration residual against reference frame (px):") + for k in sorted(dets): + _, med, p90, n = ref[k] + print(f" {k:16s} n={n:5d} median={med:.3f} p90={p90:.3f}") + + # Static sky = every source in the deep luminance master. + with fits.open(layout.path("master-Luminance.fit")) as hd: + master = hd[0].data.astype(np.float32) + import sep + bkg = sep.Background(master, bw=64, bh=64, fw=3, fh=3) + m = sep.extract(master - bkg.back(), 2.5, err=bkg.globalrms, minarea=4, + deblend_cont=0.005) + print(f"\nmaster detections (static sky): {len(m)}") + mtree = cKDTree(np.column_stack([m["x"], m["y"]])) + + print("\nresiduals after removing anything within 4 px of a master source:") + for k in sorted(lum): + cur = ref[k][0] + d, _ = mtree.query(cur, distance_upper_bound=4.0) + left = ~np.isfinite(d) + print(f" {k:16s} {left.sum():5d} / {len(cur)}") + + +if __name__ == "__main__": + main() diff --git a/pipeline/mo_common.py b/pipeline/mo_common.py new file mode 100644 index 0000000..c8204eb --- /dev/null +++ b/pipeline/mo_common.py @@ -0,0 +1,321 @@ +"""Shared machinery for the moving-object and transient searches. + +Holds the things both the real search and the synthetic-injection sensitivity +run need to do identically: detection, coordinate registration, photometric +calibration onto the master's zero point, static-sky rejection and tracklet +linking. Keeping one copy guarantees the sensitivity numbers describe the +pipeline that was actually run on the data, not a simplified stand-in. +""" +import itertools +import os + +import numpy as np +import sep +from astropy.io import fits +from scipy.spatial import cKDTree +from skimage.transform import AffineTransform + +import layout + +SRC = layout.SESSION +OUT = layout.SESSION +DETS = layout.path("_mo_dets.npz") +REFERENCE = "Luminance_002" + +ZP = 27.941 # master luminance, G = -2.5 log10(flux_5px) + ZP +SCALE = 0.5376 # arcsec/px +FWHM = 5.0 # px +SIGMA = FWHM / 2.3548 +APRAD = 5.0 # px, the radius the zero point is defined for +# Fraction of a Gaussian's total flux inside APRAD, used to convert a target +# aperture magnitude into the total flux of an injected source. +APFRAC = 1.0 - np.exp(-APRAD ** 2 / (2.0 * SIGMA ** 2)) + +# --- tracklet linking parameters --------------------------------------- +MIN_RATE = 2.5 # px/hr +MAX_RATE = 900.0 # px/hr +MIN_HITS = 5 +LINE_RMS = 1.2 # px +MAG_SCATTER = 0.5 # mag +TOL = 3.0 # px +PAIR_GAP = 3 +STATIC_MIN = 4 # recurrences at one place that mark a source static +STATIC_TOL = 2.0 # px +MIN_NATIVE_MOTION = 4.0 # px a real mover must travel in DETECTOR coordinates + + +# ---------------------------------------------------------------------- +# detection + +def detect(image, thresh=3.5, minarea=5): + """sep background estimate, extraction, and 5 px aperture photometry.""" + bkg = sep.Background(image, bw=64, bh=64, fw=3, fh=3) + sub = image - bkg.back() + objs = sep.extract(sub, thresh, err=bkg.globalrms, minarea=minarea, + deblend_cont=0.005) + if len(objs): + ap, _, _ = sep.sum_circle(sub, objs["x"], objs["y"], APRAD, + err=bkg.globalrms, gain=1.0) + else: + ap = np.zeros(0) + return objs, np.asarray(ap, dtype=float), float(bkg.globalrms) + + +def lum_frames(): + """(key, path) for the twelve luminance subs, in time order.""" + rows = [] + n = 0 + for fn in sorted(os.listdir(SRC)): + if fn.startswith("calibrated-") and fn.endswith(".fit") \ + and "-Luminance-" in fn: + n += 1 + rows.append((f"Luminance_{n:03d}", layout.path(fn))) + return rows + + +def lum_times(): + """Mid-exposure times of the luminance subs, in hours from the first.""" + t = [] + for _, path in lum_frames(): + jd = float(fits.getheader(path)["JD"]) + t.append(jd - 2400000.5 + 150.0 / 86400.0) + t = np.array(t) + return (t - t[0]) * 24.0 + + +# ---------------------------------------------------------------------- +# registration + +def load_dets(): + z = np.load(DETS, allow_pickle=True) + meta = {r[0]: r for r in z["meta"]} + return meta, {k: z[k] for k in meta} + + +def frame_transforms(): + """Native-pixel -> reference-frame affine for each luminance sub. + + Seeded by the astroalign solution cached in _mo_dets.npz, then refined by + nearest-neighbour matching every detection against the reference frame's + detections. The refined solutions are good to ~0.15 px median across the + whole field (see mo_check.py), which is what makes a 3 px association + radius meaningful. + """ + _, dets = load_dets() + ref = dets[REFERENCE] + tree = cKDTree(ref[:, :2]) + tf_out = {} + for key, d in dets.items(): + if not key.startswith("Luminance"): + continue + src = d[:, 2:4].astype(float) + cur = d[:, :2].astype(float) + tf = None + for t in (3.0, 1.5): + dist, idx = tree.query(cur, distance_upper_bound=t) + ok = np.isfinite(dist) + if ok.sum() < 50: + break + tf = AffineTransform() + tf.estimate(src[ok], ref[idx[ok], :2].astype(float)) + cur = tf(src) + if tf is None: # the reference frame itself + tf = AffineTransform() + tf.estimate(src, d[:, :2].astype(float)) + tf_out[key] = tf + return tf_out + + +# ---------------------------------------------------------------------- +# photometry + +def master_sources(): + """Detections and 5 px aperture fluxes in the deep luminance master.""" + with fits.open(layout.path("master-Luminance.fit")) as hd: + img = hd[0].data.astype(np.float32) + hdr = hd[0].header + objs, ap, rms = detect(img, thresh=2.5, minarea=4) + return objs, ap, rms, hdr, img + + +def frame_zeropoint(fx, fy, fap, mobjs, map_, tf): + """Per-frame zero point, tied to the master's, from stars in common. + + The master is a scaled sigma-clipped mean of the twelve subs, so a single + sub's counts differ from it by a near-constant factor. Measuring that + factor on a few hundred stars puts single-frame magnitudes on the master's + system without a separate absolute calibration. + """ + ref_xy = tf(np.column_stack([fx, fy])) + mt = cKDTree(np.column_stack([mobjs["x"], mobjs["y"]])) + d, idx = mt.query(ref_xy, distance_upper_bound=1.5) + ok = np.isfinite(d) & (fap > 0) + if ok.sum() < 30: + return ZP, 0 + fm = map_[idx[ok]] + ff = fap[ok] + good = (fm > 0) & (ff > 0) + # Bright half only: faint stars are noise-biased in both frames. + order = np.argsort(fm[good])[::-1] + sel = order[:max(50, len(order) // 4)] + ratio = np.median((ff[good][sel] / fm[good][sel])) + return ZP + 2.5 * np.log10(ratio), int(ok.sum()) + + +# ---------------------------------------------------------------------- +# static-sky rejection and linking + +def residual_mask(pts_list, nat_list, mtree, master_tol=4.0): + """True for detections that are neither static sky nor detector defects. + + Three rejections, and the third is the one that matters most on this data: + + 1. Coincident with a source in the deep luminance master. + 2. Recurring at the same REFERENCE-frame position in four or more subs - + a star the master missed. + 3. Recurring at the same DETECTOR-frame position in four or more subs. + + Cut 3 exists because the frames are dithered and the field rotates slowly + through the session. Registration undoes that motion for the sky, which + means it imposes exactly the opposite motion on anything fixed to the + detector. A hot pixel therefore traces a perfectly straight, perfectly + constant-rate, perfectly constant-brightness track across the registered + sequence - an ideal fake minor planet that passes every cut a linker + normally applies. Without this cut the search on these twelve subs returns + 141 such tracks, all of them moving at 9-18 arcsec/hr toward position + angle 45-85 deg, all with npix of 5-10 (a 3x3 blob, against 50-100 for a + real star at this seeing), and all stationary to under 0.5 px in detector + coordinates. With it, none survive. + """ + trees = [cKDTree(p) if len(p) else None for p in pts_list] + ntrees = [cKDTree(p) if len(p) else None for p in nat_list] + out = [] + for i, p in enumerate(pts_list): + if not len(p): + out.append(np.zeros(0, dtype=bool)) + continue + d, _ = mtree.query(p, distance_upper_bound=master_tol) + in_master = np.isfinite(d) + recur = np.zeros(len(p), dtype=int) + nrecur = np.zeros(len(p), dtype=int) + for j, (tr, ntr) in enumerate(zip(trees, ntrees)): + if j == i or tr is None: + continue + dd, _ = tr.query(p, distance_upper_bound=STATIC_TOL) + recur += np.isfinite(dd) + nd, _ = ntr.query(nat_list[i], distance_upper_bound=STATIC_TOL) + nrecur += np.isfinite(nd) + out.append((~in_master) & (recur < STATIC_MIN) & + (nrecur < STATIC_MIN)) + return out + + +def fit_track(t, x, y): + A = np.column_stack([np.ones_like(t), t]) + cx, *_ = np.linalg.lstsq(A, x, rcond=None) + cy, *_ = np.linalg.lstsq(A, y, rcond=None) + rms = float(np.sqrt(np.mean((x - A @ cx) ** 2 + (y - A @ cy) ** 2))) + return cx, cy, rms + + +def link(times, pts, mags, nat=None, shape=None, min_hits=MIN_HITS): + """Pair-seeded, propagate-and-count tracklet finder. See mo_link.py.""" + n = len(times) + trees = [cKDTree(p) if len(p) else None for p in pts] + seen, cands = set(), [] + for i, j in itertools.combinations(range(n), 2): + if j - i < PAIR_GAP or trees[i] is None or trees[j] is None: + continue + dt = times[j] - times[i] + if dt <= 0: + continue + near = trees[j].query_ball_point(pts[i], MAX_RATE * dt) + for a, blist in enumerate(near): + for b in blist: + if np.hypot(*(pts[j][b] - pts[i][a])) < MIN_RATE * dt: + continue + v = (pts[j][b] - pts[i][a]) / dt + hits = [] + for f in range(n): + if trees[f] is None: + continue + pred = pts[i][a] + v * (times[f] - times[i]) + dd, idx = trees[f].query(pred, distance_upper_bound=TOL) + if np.isfinite(dd): + hits.append((f, int(idx))) + if len(hits) < min_hits: + continue + sig = tuple(sorted(hits)) + if sig in seen: + continue + seen.add(sig) + tt = np.array([times[f] for f, _ in hits]) + xx = np.array([pts[f][k][0] for f, k in hits]) + yy = np.array([pts[f][k][1] for f, k in hits]) + cx, cy, rms = fit_track(tt, xx, yy) + if rms > LINE_RMS: + continue + mm = np.array([mags[f][k] for f, k in hits]) + if not np.all(np.isfinite(mm)) or mm.std() > MAG_SCATTER: + continue + if nat is not None: + # Backstop for cut 3 in residual_mask: whatever this is, + # it must have genuinely moved across the detector, not + # merely been carried by the registration. + nx_ = np.array([nat[f][k][0] for f, k in hits]) + ny_ = np.array([nat[f][k][1] for f, k in hits]) + if np.hypot(np.ptp(nx_), np.ptp(ny_)) < MIN_NATIVE_MOTION: + continue + c = dict(frames=np.array([f for f, _ in hits]), + idx=np.array([k for _, k in hits]), + t=tt, x=xx, y=yy, cx=cx, cy=cy, rms=rms, + rate=float(np.hypot(cx[1], cy[1])), + ang=float(np.degrees(np.arctan2(cy[1], cx[1]))), + mag=float(mm.mean()), magsig=float(mm.std()), + nhit=len(hits)) + if shape is not None: + sh = np.array([shape[f][k] for f, k in hits]) + c.update(a=float(np.median(sh[:, 0])), + b=float(np.median(sh[:, 1])), + npix=float(np.median(sh[:, 2]))) + cands.append(c) + return dedup(cands) + + +def dedup(cands): + """Among tracklets sharing two or more detections, keep the best.""" + cands.sort(key=lambda c: (-c["nhit"], c["rms"])) + kept, sets = [], [] + for c in cands: + s = set(zip(c["frames"].tolist(), c["idx"].tolist())) + if any(len(s & o) >= 2 for o in sets): + continue + sets.append(s) + kept.append(c) + return kept + + +# ---------------------------------------------------------------------- +# synthetic sources + +def add_source(image, x, y, total_flux, dx, dy, nstep=9, sigma=SIGMA): + """Add a trailed Gaussian: the source moves (dx, dy) px during the sub. + + A 300 s exposure of something moving at 30 arcsec/hr smears by ~4.6 px, + comparable to the seeing disc, so the trail is modelled rather than + ignored - it is what limits sensitivity at high rates. + """ + ny, nx = image.shape + half = int(np.ceil(4 * sigma + 0.5 * np.hypot(dx, dy))) + 2 + x0, x1 = int(max(0, x - half)), int(min(nx, x + half + 1)) + y0, y1 = int(max(0, y - half)), int(min(ny, y + half + 1)) + if x1 <= x0 or y1 <= y0: + return + gy, gx = np.mgrid[y0:y1, x0:x1] + acc = np.zeros(gx.shape, dtype=np.float64) + for s in np.linspace(-0.5, 0.5, nstep): + cx, cy = x + s * dx, y + s * dy + acc += np.exp(-((gx - cx) ** 2 + (gy - cy) ** 2) / + (2.0 * sigma ** 2)) + acc *= total_flux / (nstep * 2.0 * np.pi * sigma ** 2) + image[y0:y1, x0:x1] += acc.astype(image.dtype) diff --git a/pipeline/mo_detect.py b/pipeline/mo_detect.py new file mode 100644 index 0000000..35d43de --- /dev/null +++ b/pipeline/mo_detect.py @@ -0,0 +1,137 @@ +"""Moving-object search, pass 1: per-sub source detection and frame registration. + +Every calibrated sub is opened one at a time (they are 61 MB each and there is +only a few GB of RAM), background-subtracted with sep, and its sources +extracted. The detection list, not the pixels, is what gets carried forward. + +Registration is done on the coordinates rather than the images. astroalign +matches asterisms between each sub's star list and the reference sub's star +list and returns a similarity transform; applying that transform to the +detection coordinates puts every sub's sources into one common pixel grid +without ever warping a 15 Mpx array. The reference is Luminance_002, the same +frame the master stack was registered to, so the master's WCS applies directly +to the common grid and every detection can be turned into RA/Dec. + +Output: _mo_dets.npz with one record array per frame plus the transform +parameters and mid-exposure times. +""" +import os +import sys + +import astroalign as aa +import numpy as np +import sep +from astropy.io import fits + +import layout + +SRC = layout.SESSION +OUT = layout.SESSION +CACHE = layout.path("_mo_dets.npz") +REFERENCE = "Luminance_002" + +aa.MIN_MATCHES_FRACTION = 0.6 +aa.NUM_NEAREST_NEIGHBORS = 8 + +# Detection threshold in sigma. Deliberately low: a moving object is only in +# any one sub for 300 s, so it is much fainter per-frame than in the 60 min +# master. Spurious detections are cheap here because the tracklet linker +# demands a straight line through five or more frames, which noise does not +# supply. +THRESH = 3.5 +MINAREA = 5 + + +def frames(): + """(key, filename, filter, mid-exposure MJD) for every calibrated sub.""" + rows = [] + counts = {} + for fn in sorted(os.listdir(SRC)): + if not (fn.startswith("calibrated-") and fn.endswith(".fit")): + continue + filt = fn.split("-")[6] + counts[filt] = counts.get(filt, 0) + 1 + rows.append((f"{filt}_{counts[filt]:03d}", fn, filt)) + return rows + + +def detect(image, thresh=THRESH, minarea=MINAREA): + bkg = sep.Background(image, bw=64, bh=64, fw=3, fh=3) + sub = image - bkg.back() + rms = bkg.globalrms + objs = sep.extract(sub, thresh, err=rms, minarea=minarea, + deblend_cont=0.005, filter_kernel=None) + return objs, sub, rms + + +def main(): + rows = frames() + print(f"{len(rows)} calibrated subs") + + # Reference star list first: everything else is matched onto it. + ref_fn = [r[1] for r in rows if r[0] == REFERENCE][0] + with fits.open(layout.path(ref_fn), memmap=False) as hd: + img = hd[0].data.astype(np.float32) + objs, _, rms = detect(img) + bright = objs[(objs["flag"] == 0) & (objs["npix"] > 12) & + (objs["npix"] < 3000)] + bright = bright[np.argsort(bright["flux"])[::-1][:600]] + ref_xy = np.column_stack([bright["x"], bright["y"]]) + del img + print(f"reference {REFERENCE}: {len(ref_xy)} registration stars, " + f"rms {rms:.2f}") + + store = {} + meta = [] + for key, fn, filt in rows: + path = layout.path(fn) + with fits.open(path, memmap=False) as hd: + img = hd[0].data.astype(np.float32) + mjd = float(hd[0].header["JD"]) - 2400000.5 + mjd_mid = mjd + 150.0 / 86400.0 # mid-exposure + objs, _, rms = detect(img) + del img + + clean = objs[(objs["flag"] == 0) & (objs["npix"] > 12) & + (objs["npix"] < 3000)] + clean = clean[np.argsort(clean["flux"])[::-1][:600]] + xy = np.column_stack([clean["x"], clean["y"]]) + + if key == REFERENCE: + params = (1.0, 0.0, 0.0, 0.0) + nmatch = len(xy) + tx, ty = objs["x"].copy(), objs["y"].copy() + else: + try: + tform, (src_m, _) = aa.find_transform(xy, ref_xy) + except Exception as exc: # noqa: BLE001 + print(f" {key:16s} REGISTRATION FAILED ({exc}) - dropped") + continue + nmatch = len(src_m) + pts = np.column_stack([objs["x"], objs["y"]]) + warped = tform(pts) + tx, ty = warped[:, 0], warped[:, 1] + params = (tform.scale, np.degrees(tform.rotation), + tform.translation[0], tform.translation[1]) + + # Everything the tracklet linker and the vetting need, per detection. + rec = np.column_stack([ + tx, ty, # reference-frame x, y + objs["x"], objs["y"], # native x, y + objs["flux"], objs["peak"], + objs["a"], objs["b"], objs["theta"], + objs["npix"].astype(float), objs["flag"].astype(float), + ]).astype(np.float32) + store[key] = rec + meta.append((key, fn, filt, f"{mjd_mid:.8f}", f"{rms:.4f}", + str(len(rec)), str(nmatch), + *[f"{p:.6f}" for p in params])) + print(f" {key:16s} {len(rec):5d} det match={nmatch:3d} " + f"rms={rms:6.2f} rot={params[1]:+7.3f} deg") + + np.savez_compressed(CACHE, meta=np.array(meta, dtype=object), **store) + print(f"\nwrote {CACHE}") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pipeline/mo_fig_moving.py b/pipeline/mo_fig_moving.py new file mode 100644 index 0000000..62d3965 --- /dev/null +++ b/pipeline/mo_fig_moving.py @@ -0,0 +1,283 @@ +"""Figure: the moving-object search, its sensitivity, and its failure mode. + +The blind search found nothing, so the figure has to show that the search +worked rather than showing a discovery. Four things are plotted. + +Rows 1-2 are postage-stamp strips across the twelve luminance subs, cut at a +sky position that is FIXED on the sky (the object's position in the first sub). +A real moving object drifts across the strip. So does a hot pixel, because +registration holds the sky still and therefore drags anything fixed to the +detector in the opposite direction. The two are indistinguishable here, which +is the whole problem. + +Rows 3-4 are the same two objects, cut instead at a FIXED DETECTOR position. +Now they separate cleanly: the real object still moves, the hot pixel does not +move at all. This is the cut that took the search from 141 confident false +detections to zero. + +The bottom panels are the sensitivity: recovery fraction of synthetic movers as +a function of magnitude and apparent rate, from three independent injection +runs (twelve injections per cell), and a slice through it. +""" +import os + +import numpy as np +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from astropy.io import fits +from matplotlib.colors import LinearSegmentedColormap +from scipy.spatial import cKDTree + +import mo_common as C + +HALF = 13 +OUTPNG = os.path.join(C.OUT, "NGC5128-mo-moving-object-search.png") + +INJ_MAG = 18.5 +INJ_RATE = 25.0 # px/hr = 13.4 arcsec/hr + + +def load_grids(): + mags = rates = None + grids = [] + for seed in (1, 2, 3): + p = os.path.join(C.OUT, f"_mo_sensitivity_{seed}.npz") + if not os.path.exists(p): + continue + z = np.load(p) + mags, rates = z["mags"], z["rates"] + grids.append(z["grid"]) + return mags, rates, np.mean(grids, axis=0), len(grids) * int( + np.load(os.path.join(C.OUT, "_mo_sensitivity_1.npz"))["nrep"]) + + +# A detector defect verified by hand: reproducing the search with the +# detector-frame cuts switched off yields 141 tracklets, and this one is +# typical. Its detected centroid sits at native pixel (224.00, 787.00) in +# every one of subs 002-009 - not merely close, but the same pixel centre to +# 0.02 px - while its registered position walks 18.6 px across the sequence +# at 15.9 arcsec/hr with a brightness stable to 0.06 mag. It is a hot pixel. +HOT_NATIVE = np.array([224.00, 787.00]) +HOT_REF_KEY = "Luminance_002" + + +def find_hot_pixel(tforms, keys): + return HOT_NATIVE, 8 + + +def stamps(inj_ref0, inj_v, hot_native, tforms, keys, times): + """Cut all four strips in a single pass over the subs.""" + out = {"inj_sky": [], "inj_det": [], "hot_sky": [], "hot_det": []} + hot_ref0 = tforms[keys[0]](hot_native[None, :])[0] + inj_native0 = tforms[keys[0]].inverse(inj_ref0[None, :])[0] + + def cut(img, x, y): + xi, yi = int(round(x)), int(round(y)) + return img[yi - HALF:yi + HALF + 1, xi - HALF:xi + HALF + 1].copy() + + for i, (key, path) in enumerate(C.lum_frames()): + with fits.open(path, memmap=False) as hd: + img = hd[0].data.astype(np.float32) + tf = tforms[key] + inv = tf.inverse + lin = np.linalg.inv(tf.params[:2, :2]) + # Inject the synthetic at its true position in this sub. + true_ref = inj_ref0 + inj_v * times[i] + tx, ty = inv(true_ref[None, :])[0] + objs, ap, _ = C.detect(img) + mobjs, map_, _, _, _ = MASTER + zp, _ = C.frame_zeropoint(objs["x"], objs["y"], ap, mobjs, map_, tf) + d = lin @ (inj_v * 300.0 / 3600.0) + C.add_source(img, tx, ty, 10 ** ((zp - INJ_MAG) / 2.5) / C.APFRAC, + d[0], d[1]) + # Strip 1: cut at the sky position the object had at t=0. + sx, sy = inv(inj_ref0[None, :])[0] + out["inj_sky"].append(cut(img, sx, sy)) + # Strip 2: cut at the detector position it had at t=0. + out["inj_det"].append(cut(img, inj_native0[0], inj_native0[1])) + # Strip 3: hot pixel, cut at its t=0 sky position. + hx, hy = inv(hot_ref0[None, :])[0] + out["hot_sky"].append(cut(img, hx, hy)) + # Strip 4: hot pixel, cut at its fixed detector position. + out["hot_det"].append(cut(img, hot_native[0], hot_native[1])) + del img + print(f" stamps from {key}") + return out + + +def show(ax, s, vlo, vhi, edge="#7a7f87"): + ax.imshow(s, origin="lower", cmap="gray", vmin=vlo, vmax=vhi, + interpolation="nearest") + n = s.shape[0] + c = (n - 1) / 2.0 + # A faint crosshair marks the centre of the stamp, i.e. the position the + # cut is held fixed at. Whether the source stays on it is the whole point. + ax.plot([c, c], [c - 0.30 * n, c - 0.12 * n], color="#f0c05a", lw=0.8) + ax.plot([c - 0.30 * n, c - 0.12 * n], [c, c], color="#f0c05a", lw=0.8) + ax.set_xlim(-0.5, n - 0.5) + ax.set_ylim(-0.5, n - 0.5) + ax.set_xticks([]) + ax.set_yticks([]) + for sp in ax.spines.values(): + sp.set_color(edge) + sp.set_linewidth(0.8) + + +def main(): + global MASTER + MASTER = C.master_sources() + times = C.lum_times() + keys = [k for k, _ in C.lum_frames()] + tforms = C.frame_transforms() + + hot, hotn = find_hot_pixel(tforms, keys) + print(f"hot pixel at detector ({hot[0]:.1f}, {hot[1]:.1f}), " + f"present in {hotn + 1}/12 subs") + + ang = np.radians(35.0) + inj_v = INJ_RATE * np.array([np.cos(ang), np.sin(ang)]) + mobjs = MASTER[0] + mtree = cKDTree(np.column_stack([mobjs["x"], mobjs["y"]])) + rng = np.random.default_rng(11) + for _ in range(50000): + p0 = np.array([rng.uniform(600, 4200), rng.uniform(600, 2600)]) + track = np.array([p0 + inj_v * t for t in times]) + if np.hypot(*(p0 - np.array([2394.0, 1597.0]))) < 1100: + continue + d, _ = mtree.query(track, distance_upper_bound=30.0) + if np.all(~np.isfinite(d)): + inj_ref0 = p0 + break + else: + inj_ref0 = np.array([1500.0, 2500.0]) + print(f"synthetic injected at reference pixel " + f"({inj_ref0[0]:.0f}, {inj_ref0[1]:.0f})") + st = stamps(inj_ref0, inj_v, hot, tforms, keys, times) + + mags, rates, grid, nrep = load_grids() + + from astropy.stats import sigma_clipped_stats + + fig = plt.figure(figsize=(15.0, 11.4)) + gs = fig.add_gridspec( + 5, 12, height_ratios=[0.78, 0.78, 0.78, 0.78, 2.5], + hspace=0.16, wspace=0.06, + left=0.175, right=0.985, top=0.878, bottom=0.08) + + labels = [ + ("inj_sky", "synthetic mover", "held at fixed SKY position", + "#2f7d54", "moves -> could be real"), + ("hot_sky", "hot pixel", "held at fixed SKY position", + "#b5484f", "also moves -> looks identical"), + ("inj_det", "synthetic mover", "held at fixed DETECTOR position", + "#2f7d54", "still moves -> REAL"), + ("hot_det", "hot pixel", "held at fixed DETECTOR position", + "#b5484f", "does not move -> DEFECT"), + ] + for r, (kk, who, how, col, verdict) in enumerate(labels): + arr = np.array(st[kk], dtype=float) + med, _, sd = sigma_clipped_stats(arr, sigma=3.0) + vlo, vhi = med - 1.5 * sd, med + 14.0 * sd + for c in range(12): + ax = fig.add_subplot(gs[r, c]) + show(ax, st[kk][c], vlo, vhi, edge=col) + if r == 0: + ax.set_title(f"{times[c] * 60:.0f} min", fontsize=8.5, + color="#3b3f45", pad=4) + if c == 0: + bb = ax.get_position() + fig.text(0.170, bb.y0 + bb.height * 0.80, who, fontsize=10.5, + color=col, ha="right", va="center", weight="bold") + fig.text(0.170, bb.y0 + bb.height * 0.50, how, fontsize=8.4, + color="#5a6068", ha="right", va="center") + fig.text(0.170, bb.y0 + bb.height * 0.18, verdict, + fontsize=8.6, color=col, ha="right", va="center", + style="italic") + + fig.text(0.02, 0.962, + "Moving-object search: 12 x 300 s luminance subs of NGC 5128, " + "2026-07-21 08:56-10:00 UTC", + fontsize=14, color="#1b1e23", weight="bold", ha="left") + fig.text(0.02, 0.937, + "Each strip is a 14 x 14 arcsec cutout, one per sub, with the " + "cut position held fixed (yellow crosshair). Registration holds " + "the sky still, so it drags anything fixed to the", + fontsize=9.4, color="#4a4f57", ha="left") + fig.text(0.02, 0.918, + "detector across the registered frame - a hot pixel therefore " + "mimics a minor planet perfectly. Cutting at a fixed detector " + "position separates them. Synthetic source: G = " + f"{INJ_MAG}, {INJ_RATE * C.SCALE:.1f} arcsec/hr.", + fontsize=9.4, color="#4a4f57", ha="left") + + # ---- sensitivity heat map ---- + axg = fig.add_subplot(gs[4, 0:6]) + cmap = LinearSegmentedColormap.from_list( + "rec", ["#f5f6f8", "#cfe0ec", "#7fb0cd", "#3d7ba6", "#17456b"]) + im = axg.imshow(grid, origin="lower", aspect="auto", cmap=cmap, + vmin=0, vmax=1, interpolation="nearest") + axg.set_xticks(range(len(rates))) + axg.set_xticklabels([f"{r * C.SCALE:.1f}" for r in rates], fontsize=8.5) + axg.set_yticks(range(len(mags))) + axg.set_yticklabels([f"{m:g}" for m in mags], fontsize=9) + axg.set_xlabel("apparent rate (arcsec/hr)", fontsize=10) + axg.set_ylabel("G magnitude", fontsize=10) + axg.set_title(f"recovery of injected synthetic movers " + f"({nrep} per cell, 3 independent runs)", fontsize=10.5, + color="#1b1e23", pad=8) + for i in range(len(mags)): + for j in range(len(rates)): + v = grid[i, j] + axg.text(j, i, f"{v:.2f}".lstrip("0") if v else ".", + ha="center", va="center", fontsize=7.0, + color="white" if v > 0.55 else "#585d65") + cb = fig.colorbar(im, ax=axg, fraction=0.032, pad=0.014) + cb.set_label("recovered", fontsize=9) + cb.ax.tick_params(labelsize=8) + + # ---- slices ---- + axs = fig.add_subplot(gs[4, 7:12]) + band = (rates * C.SCALE >= 8) & (rates * C.SCALE <= 40) + prof = grid[:, band].mean(axis=1) + axs.plot(mags, prof, "-o", color="#2e6f9e", lw=2.2, ms=5.5, + label="8-40 arcsec/hr (main-belt range)") + prof2 = grid[:, rates * C.SCALE > 60].mean(axis=1) + axs.plot(mags, prof2, "-s", color="#b5484f", lw=1.8, ms=4.5, + label="> 60 arcsec/hr (trailing losses)") + axs.axhline(0.5, color="#9aa0a8", ls=":", lw=1.2) + axs.text(mags[0] + 0.02, 0.53, "50% recovery", fontsize=8, + color="#6b7079") + axs.set_xlabel("G magnitude", fontsize=10) + axs.set_ylabel("fraction recovered", fontsize=10) + axs.set_ylim(-0.03, 1.08) + axs.set_title("sensitivity vs magnitude", fontsize=10.5, + color="#1b1e23", pad=8) + axs.legend(fontsize=8.5, frameon=False, loc="lower left") + axs.grid(alpha=0.25, lw=0.6) + for sp in ("top", "right"): + axs.spines[sp].set_visible(False) + + order = np.argsort(prof) + g50 = float(np.interp(0.5, prof[order], mags[order])) + print(f"50% recovery for main-belt rates at G = {g50:.2f}") + fig.text(0.02, 0.024, + "Result: 0 candidates from the blind search. Time-permuted " + "control on the real residuals: 0 tracklets in 60 trials. " + f"50% recovery at G = {g50:.1f} for main-belt rates. The only " + "catalogued minor planet within 30' of the pointing,", + fontsize=9, color="#4a4f57", ha="left") + fig.text(0.02, 0.006, + "(427494) 2002 BK26 at V = 21.7, fell 2.4 arcmin outside the " + "frame - the field's long axis runs along declination, not right " + "ascension.", + fontsize=9, color="#4a4f57", ha="left") + + fig.savefig(OUTPNG, dpi=125, facecolor="white") + print(f"wrote {OUTPNG}") + + +MASTER = None + +if __name__ == "__main__": + main() diff --git a/pipeline/mo_fig_transient.py b/pipeline/mo_fig_transient.py new file mode 100644 index 0000000..67a252b --- /dev/null +++ b/pipeline/mo_fig_transient.py @@ -0,0 +1,228 @@ +"""Figure: the transient search and why its six finalists are not transients. + +Top block, one column per finalist: the 60 min luminance master beside a +Digitized Sky Survey red plate of the same patch of sky from the 1990s. If a +source is on both, it is not a transient, and that is a judgement a reader can +make from the picture without trusting any of the catalogue machinery. + +Bottom left: where the catalogues run out. Almost every source brighter than +G = 19 has a Gaia DR3 counterpart; below that the unmatched fraction climbs +steeply, not because transients appear but because Gaia's G < 20.5 point-source +catalogue stops being complete against a bright galaxy. This is what sets the +honest limit of the search. + +Bottom middle: the point-source selection. Half-light radius against magnitude, +with the image's own PSF measured from Gaia stars. Note that Centaurus A's +globular clusters sit inside the stellar locus - at 3.8 Mpc they are unresolved +in 2.7 arcsec seeing - so morphology cannot separate a cluster from a +supernova here, and five of the six finalists are clusters. + +Bottom right: where the finalists are, on the master. +""" +import os + +import numpy as np +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.stats import sigma_clipped_stats +from astropy.wcs import WCS + +import mo_common as C + +OUTPNG = os.path.join(C.OUT, "NGC5128-mo-transient-search.png") +HALF = 26 # px in the master = 28 arcsec across + +# Identifications established by cross-matching the six catalogue non-matches +# against published Centaurus A cluster catalogues, SIMBAD and the VizieR +# archive as a whole (see notes-transient-search.md). +IDENT = { + "201.672625": ("WHH-31", "Cen A globular cluster", "V=19.50 (Woodley+07)"), + "201.500583": ("pff_gc-081", "Cen A globular cluster", + "V=19.34 (Woodley+07)"), + "201.441625": ("WHH-24", "Cen A globular cluster", "V=19.63 (Woodley+07)"), + "201.407000": ("at CAVS 210", + "RESOLVED (1.34x PSF), archival 1996-2020", + "DENIS I=18.23; Gaia pt src G=21.03 at 1.32\" - see 4.3"), + "201.303542": ("WHH-10", "Cen A globular cluster", "V=19.57 (Woodley+07)"), + "201.256708": ("pff_gc-029", "Cen A globular cluster", + "V=19.37 (Woodley+07)"), +} + + +def ident_for(ra): + for k, v in IDENT.items(): + if abs(float(k) - ra) < 1e-4: + return v + return ("unidentified", "", "") + + +def stretch(a, lo=25.0, hi=99.6): + a = np.asarray(a, dtype=float) + good = np.isfinite(a) + if not good.any(): + return 0.0, 1.0 + return np.percentile(a[good], lo), np.percentile(a[good], hi) + + +def main(): + rows = list(np.load(os.path.join(C.OUT, "_mo_vetted.npy"), + allow_pickle=True)) + rows.sort(key=lambda r: r["dist_cenA_arcmin"]) + z = np.load(os.path.join(C.OUT, "_mo_transient.npz")) + + with fits.open(os.path.join(C.OUT, "master-Luminance.fit")) as hd: + master = hd[0].data.astype(np.float32) + hdr = hd[0].header + wcs = WCS(hdr) + + import sep + bkg = sep.Background(master, bw=64, bh=64, fw=3, fh=3) + msub = master - bkg.back() + + n = len(rows) + fig = plt.figure(figsize=(15.0, 11.6)) + gs = fig.add_gridspec(3, n, height_ratios=[1.0, 1.0, 1.55], + hspace=0.30, wspace=0.09, + left=0.055, right=0.985, top=0.845, bottom=0.075) + + for i, r in enumerate(rows): + name, kind, phot = ident_for(r["ra_deg"]) + x, y = int(round(r["x"])), int(round(r["y"])) + cut = msub[y - HALF:y + HALF + 1, x - HALF:x + HALF + 1] + ax = fig.add_subplot(gs[0, i]) + vlo, vhi = stretch(cut, 20, 99.5) + ax.imshow(cut, origin="lower", cmap="gray", vmin=vlo, vmax=vhi) + ax.plot([HALF, HALF], [HALF + 6, HALF + 11], color="#f0c05a", lw=1.1) + ax.plot([HALF + 6, HALF + 11], [HALF, HALF], color="#f0c05a", lw=1.1) + ax.set_xticks([]); ax.set_yticks([]) + for sp in ax.spines.values(): + sp.set_color("#2f7d54" if kind.startswith("Cen") else "#b5484f") + sp.set_linewidth(1.4) + ax.set_title(f"{name}\n{r['ra_hms']} {r['dec_dms']}", fontsize=8.6, + color="#1b1e23", pad=5) + if i == 0: + ax.set_ylabel("this image\n60 min luminance", fontsize=9, + color="#3b3f45") + + # archival plate + tag = f"{r['ra_deg']:.5f}{r['dec_deg']:+.5f}_mred.fits" + path = os.path.join(C.OUT, "_dss", tag) + ax2 = fig.add_subplot(gs[1, i]) + if os.path.exists(path): + with fits.open(path) as h2: + d = np.asarray(h2[0].data, dtype=float) + v1, v2 = stretch(d, 20, 99.5) + ax2.imshow(d, origin="lower", cmap="gray", vmin=v1, vmax=v2) + m = d.shape[0] / 2.0 + f = d.shape[0] / (2.0 * HALF + 1) + ax2.plot([m, m], [m + 6 * f, m + 11 * f], color="#f0c05a", lw=1.1) + ax2.plot([m + 6 * f, m + 11 * f], [m, m], color="#f0c05a", lw=1.1) + else: + ax2.text(0.5, 0.5, "no DSS cutout", ha="center", va="center", + fontsize=8, color="#8a9099", transform=ax2.transAxes) + ax2.set_xticks([]); ax2.set_yticks([]) + for sp in ax2.spines.values(): + sp.set_color("#9aa0a8") + sp.set_linewidth(0.9) + ax2.set_xlabel(f"{kind}\n{phot}\nG(this image) = {r['g_mag']:.2f}, " + f"{r['dist_cenA_arcmin']:.1f}' from nucleus", + fontsize=7.8, color="#4a4f57", labelpad=5) + if i == 0: + ax2.set_ylabel("archival\nDSS2 red, 1990s", fontsize=9, + color="#3b3f45") + + # ---------------- magnitude completeness ---------------- + axm = fig.add_subplot(gs[2, 0:2]) + good = z["good"] & z["pointlike"] + mag = z["mag"][good] + matched = z["gaia_ok"][good] + bins = np.arange(14.0, 21.01, 0.25) + axm.hist([mag[matched], mag[~matched]], bins=bins, stacked=True, + color=["#3d7ba6", "#b5484f"], + label=["matched to Gaia DR3", "no Gaia counterpart"]) + axm.axvline(20.5, color="#4a4f57", ls="--", lw=1.2) + axm.text(20.42, axm.get_ylim()[1] * 0.92, "Gaia cat.\nlimit G=20.5", + fontsize=7.6, color="#4a4f57", ha="right", va="top") + axm.set_xlabel("G magnitude (5 px aperture, master)", fontsize=9.5) + axm.set_ylabel("point-like sources per 0.25 mag", fontsize=9.5) + axm.set_title("where the reference catalogue runs out", fontsize=10.5, + color="#1b1e23", pad=7) + axm.legend(fontsize=8.2, frameon=False, loc="upper left") + axm.grid(alpha=0.22, lw=0.6) + for sp in ("top", "right"): + axm.spines[sp].set_visible(False) + + # ---------------- morphology ---------------- + axr = fig.add_subplot(gs[2, 2:4]) + sel = z["good"] + axr.scatter(z["mag"][sel], z["r50"][sel] / z["r50_star"], s=2.5, + c="#9fb6c6", alpha=0.35, lw=0, label="all detections") + ci = z["cand_idx"] + axr.scatter(z["mag"][ci], z["r50"][ci] / z["r50_star"], s=9, + c="#e0a13c", alpha=0.8, lw=0, + label="no Gaia / SkyMapper match") + for r in rows: + axr.scatter([r["g_mag"]], [r["r50_over_psf"]], s=64, + facecolor="none", + edgecolor="#2f7d54" if ident_for(r["ra_deg"])[1] + .startswith("Cen") else "#b5484f", lw=1.5, zorder=5) + axr.axhline(1.0, color="#4a4f57", ls=":", lw=1.1) + axr.text(14.3, 1.03, "PSF (Gaia stars)", fontsize=7.6, color="#4a4f57") + axr.set_ylim(0.4, 3.2) + axr.set_xlim(14.0, 21.0) + axr.set_xlabel("G magnitude", fontsize=9.5) + axr.set_ylabel("half-light radius / PSF", fontsize=9.5) + axr.set_title("morphology cannot separate clusters from transients", + fontsize=10.5, color="#1b1e23", pad=7) + axr.legend(fontsize=8.0, frameon=False, loc="upper left", + markerscale=2.2) + axr.grid(alpha=0.22, lw=0.6) + for sp in ("top", "right"): + axr.spines[sp].set_visible(False) + + # ---------------- field map ---------------- + axf = fig.add_subplot(gs[2, 4:6]) + sm = master[::6, ::6] + v1, v2 = np.percentile(sm[np.isfinite(sm)], [12, 99.75]) + axf.imshow(np.arcsinh((sm - v1) / max(v2 - v1, 1e-6)), origin="lower", + cmap="gray", vmin=0, vmax=np.arcsinh(1.0)) + for r in rows: + c = "#2f7d54" if ident_for(r["ra_deg"])[1].startswith("Cen") \ + else "#b5484f" + axf.scatter([r["x"] / 6], [r["y"] / 6], s=90, facecolor="none", + edgecolor=c, lw=1.5) + axf.annotate(ident_for(r["ra_deg"])[0], (r["x"] / 6, r["y"] / 6), + textcoords="offset points", xytext=(9, 5), fontsize=7.4, + color=c) + axf.set_xticks([]); axf.set_yticks([]) + axf.set_title("positions on the field (43' x 29')", fontsize=10.5, + color="#1b1e23", pad=7) + for sp in axf.spines.values(): + sp.set_color("#9aa0a8") + + fig.text(0.02, 0.962, + "Transient search in the 60 min luminance master of NGC 5128 " + "(Centaurus A), 2026-07-21", fontsize=14, weight="bold", + color="#1b1e23", ha="left") + fig.text(0.02, 0.936, + "7,519 point-like sources detected above SNR 10. 603 had no " + "Gaia DR3 counterpart; 567 also had none in SkyMapper DR2; 6 of " + "those survived the requirement of an independent detection in " + "at least 6 of the 12 subs.", + fontsize=9.4, color="#4a4f57", ha="left") + fig.text(0.02, 0.917, + "All 6 are identified below. Five are catalogued Centaurus A " + "globular clusters; the sixth is a resolved object with archival " + "detections from 1996 to 2020. No transient candidate remains.", + fontsize=9.4, color="#4a4f57", ha="left") + + fig.savefig(OUTPNG, dpi=130, facecolor="white") + print(f"wrote {OUTPNG}") + + +if __name__ == "__main__": + main() diff --git a/pipeline/mo_finalise.py b/pipeline/mo_finalise.py new file mode 100644 index 0000000..74a5df3 --- /dev/null +++ b/pipeline/mo_finalise.py @@ -0,0 +1,146 @@ +"""Write the final flagged-object CSVs, with an identification and verdict +attached to every row. + +Two files are produced. + +mo-flagged-objects.csv is the short list: the six sources that survived every +cut of the transient search, plus the (zero) moving-object candidates, each +with the identification established from published catalogues and an explicit +assessment. This is the file to read. + +mo-transient-candidates.csv (written by mo_transient.py) is the long list of +all 567 catalogue non-matches, kept for completeness. Almost all of them are +faint objects below the depth at which Gaia DR3 and SkyMapper DR2 are complete +against this galaxy, and are not individually assessed. +""" +import csv +import os + +import numpy as np + +import mo_common as C + +# Identifications from cross-matching against published Centaurus A cluster +# catalogues (Woodley+2007 J/AJ/134/494 and others), SIMBAD, NED and a blanket +# VizieR cone search. Separations quoted are to the matched catalogue entry. +IDENT = { + "201.672625": dict( + name="WHH-31", otype="GlC (SIMBAD)", sep=0.10, + refs="Woodley+2007; Hughes+2021 #86; SCABS GC prob=1.0", + cat_phot="V=19.50 B=20.44 R=18.94", + assessment="Known Centaurus A globular cluster. Not a transient."), + "201.500583": dict( + name="pff_gc-081", otype="GlC (SIMBAD)", sep=0.32, + refs="Woodley+2007; Hughes+2021 ID 248669; SCABS GC prob=1.0", + cat_phot="V=19.34 B=20.17 R=18.80", + assessment="Known Centaurus A globular cluster. Not a transient."), + "201.441625": dict( + name="WHH-24", otype="GlC (SIMBAD)", sep=0.19, + refs="Woodley+2007; SCABS GC prob=1.0", + cat_phot="V=19.63 B=20.43 R=19.12", + assessment="Known Centaurus A globular cluster. Not a transient."), + "201.407000": dict( + name="resolved object at the position of CAVS 210 / " + "Gaia DR3 6088701636326983168", + otype="resolved (1.34x PSF, elongation 1.40); not a point source", + sep=1.32, + refs="de Jong+2008 J/A+A/478/755 (CAVS 210); DENIS 1996-2001; " + "Swift/UVOT SSC (12 ObsIDs); XMM-OM ~2020; VHS DR4; KS4 DR1", + cat_phot="Gaia DR3 G=21.03 at 1.32\" (point source); " + "DENIS I=18.23 J=16.44; UVOT U=18.2-18.6", + assessment="NOT a transient and NOT a brightening; the 2.8 mag " + "difference from Gaia is a measurement artefact. The " + "source is resolved (r50 = 2.96 px vs PSF 2.21 px, " + "elongation 1.40), so an integrated aperture magnitude is " + "not comparable with Gaia's point-source G; Gaia has no " + "entry at all for the extended object and nothing within " + "25\" brighter than G=19.8. It is steady over the session " + "(rms 0.066 mag vs 0.039 for comparison stars) and is " + "present on 1990s DSS2 red plates at the same brightness " + "relative to its neighbours (implied change +0.26 mag). " + "Even the PSF-scaled 2 px core gives G=18.63. See " + "notes-transient-search.md section 4.3 and NGC5128-mo-cavs210.png."), + "201.303542": dict( + name="WHH-10", otype="GlC (SIMBAD)", sep=0.23, + refs="Woodley+2007; SCABS GC prob=1.0", + cat_phot="V=19.57 B=20.36 R=19.07", + assessment="Known Centaurus A globular cluster. Not a transient."), + "201.256708": dict( + name="pff_gc-029 / GC0103", otype="GlC (SIMBAD)", sep=0.23, + refs="Woodley+2007; Peng+2004; Spitler+2008; Hughes+2021; Dumont+2022", + cat_phot="V=19.37 B=19.75 R=19.06 Ks=14.69", + assessment="Known Centaurus A globular cluster, well studied. " + "Not a transient."), +} + + +def ident_for(ra): + for k, v in IDENT.items(): + if abs(float(k) - ra) < 1e-4: + return v + return dict(name="", otype="", sep=np.nan, refs="", cat_phot="", + assessment="UNIDENTIFIED - needs follow-up") + + +FIELDS = ["search", "ra_hms", "dec_dms", "ra_deg", "dec_deg", "x_ref", + "y_ref", "g_mag_this_image", "snr", "r50_over_psf", "n_subs", + "snr_R", "snr_G", "snr_B", "dist_from_cenA_arcmin", + "identification", "object_type", "match_sep_arcsec", + "catalogue_photometry", "references", "assessment"] + + +def main(): + rows = [] + + vet = list(np.load(os.path.join(C.OUT, "_mo_vetted.npy"), + allow_pickle=True)) + vet.sort(key=lambda r: r["dist_cenA_arcmin"]) + for r in vet: + d = ident_for(r["ra_deg"]) + rows.append({ + "search": "transient", + "ra_hms": r["ra_hms"], "dec_dms": r["dec_dms"], + "ra_deg": r["ra_deg"], "dec_deg": r["dec_deg"], + "x_ref": r["x"], "y_ref": r["y"], + "g_mag_this_image": r["g_mag"], "snr": r["snr"], + "r50_over_psf": r["r50_over_psf"], "n_subs": r["n_subs"], + "snr_R": r["snr_R"], "snr_G": r["snr_G"], "snr_B": r["snr_B"], + "dist_from_cenA_arcmin": r["dist_cenA_arcmin"], + "identification": d["name"], "object_type": d["otype"], + "match_sep_arcsec": d["sep"], + "catalogue_photometry": d["cat_phot"], + "references": d["refs"], "assessment": d["assessment"]}) + + # Moving-object search: zero candidates, but the file should say so + # explicitly rather than being empty. + mo = os.path.join(C.OUT, "mo-moving-candidates.csv") + nmov = 0 + if os.path.exists(mo): + with open(mo) as fh: + nmov = max(0, sum(1 for _ in fh) - 1) + if nmov == 0: + rows.append({f: "" for f in FIELDS} | { + "search": "moving-object", + "identification": "(no candidates)", + "assessment": "Blind tracklet search over the 12 luminance subs " + "returned 0 candidates. Sensitivity measured by " + "synthetic injection: 50% recovery at G=18.9 for " + "8-40 arcsec/hr; no sensitivity below ~4 arcsec/hr. " + "The only catalogued minor planet within 30' of the " + "pointing, (427494) 2002 BK26 at V=21.7, fell 2.4' " + "outside the frame."}) + + out = os.path.join(C.OUT, "mo-flagged-objects.csv") + with open(out, "w", newline="", encoding="utf-8") as fh: + w = csv.DictWriter(fh, fieldnames=FIELDS) + w.writeheader() + w.writerows(rows) + print(f"wrote {out} ({len(rows)} rows)") + for r in rows: + print(f" {r['search']:14s} {r['identification'][:44]:44s} " + f"{str(r['g_mag_this_image']):>6s} " + f"{r['assessment'][:60]}") + + +if __name__ == "__main__": + main() diff --git a/pipeline/mo_gccheck.py b/pipeline/mo_gccheck.py new file mode 100644 index 0000000..6c3842b --- /dev/null +++ b/pipeline/mo_gccheck.py @@ -0,0 +1,423 @@ +"""Cross-match six unidentified point-like detections in an amateur NGC 5128 +(Centaurus A) image against published catalogues, to establish whether each is an +already-catalogued object (globular cluster of Cen A, background galaxy, foreground +star, X-ray/UV source, planetary nebula, ...) or genuinely uncatalogued. + +Context: the six sources have already been shown NOT to match Gaia DR3 (G<20.5) +or SkyMapper DR2. The purpose here is to rule out a supernova / transient, so a +firm identification with a static catalogued source is the desired outcome and a +"no match" is a result that must be reported honestly rather than manufactured. + +Three independent searches are run for every position: + + 1. TARGETED -- a hand-picked list of VizieR catalogues covering Cen A globular + clusters and compact stellar systems (Woodley+, Harris+, Taylor+ SCABS, + Hughes+, Dumont+, Voggel+, Mieske+ CCOS), plus Cen A X-ray, UV, planetary + nebula and variable-star catalogues, at a 5 arcsec cone radius. The exact + VizieR identifiers are resolved at runtime with Vizier.find_catalogs / + get_catalog_metadata rather than being assumed correct. + + 2. BLANKET -- Vizier.query_region(..., catalog=None), i.e. every table in + VizieR, at a 5 arcsec cone radius. Every table returning a row is reported + with its designation, separation and any magnitude-like columns. + + 3. SIMBAD -- astroquery.simbad cone search at 10 arcsec, reporting main + identifier, object type and separation. + +Astrometry of the input positions is good to well under 1 arcsec; 3-5 arcsec is +used because older ground-based Cen A cluster catalogues can be off by 1-2 arcsec. + +Output: a plain-text report at + C:\\Users\\lhorrocks-barlow\\Downloads\\NGC5128\\20260721\\stacked\\_gc_crossmatch.txt +Run: python mo_gccheck.py +""" + +import io +import sys +import time +import warnings +import contextlib + +import numpy as np +import astropy.units as u +from astropy.coordinates import SkyCoord +from astroquery.vizier import Vizier + +import layout + +warnings.filterwarnings("ignore") + +OUT = layout.path("_gc_crossmatch.txt") + +# id, RA deg, Dec deg, luminance G, r50/psf, note +SOURCES = [ + (1, 201.672625, -43.190250, 19.18, 1.08, ""), + (2, 201.500583, -42.816944, 19.04, 1.04, ""), + (3, 201.441625, -42.948111, 19.42, 1.02, ""), + (4, 201.407000, -43.041639, 18.20, 1.33, "brightest, 2.3' from nucleus"), + (5, 201.303542, -42.950000, 19.24, 1.05, ""), + (6, 201.256708, -42.911417, 19.06, 1.03, ""), +] + +# Targeted catalogues. (VizieR id, short human label) +TARGETS = [ + ("J/AJ/134/494", "Woodley+ 2007, Cen A GC catalogue (kinematics/dyn.)"), + ("J/AJ/139/1871", "Woodley+ 2010, Cen A GC ages/metallicities"), + ("J/AJ/143/84", "Harris+ 2012, new candidate GCs in NGC 5128"), + ("J/ApJ/682/199", "Woodley+ 2008, Cen A globulars with X-ray sources"), + ("J/MNRAS/469/3444", "Taylor+ 2017, SCABS II GC candidates"), + ("J/ApJ/914/16", "Hughes+ 2021, NGC 5128 GCs (PISCeS/Gaia DR2/NSC)"), + ("J/ApJ/929/147", "Dumont+ 2022, luminous GCs in NGC 5128"), + ("J/ApJ/899/140", "Voggel+ 2020, Gaia UCD candidates"), + ("J/A+A/472/111", "Mieske+ 2007, Centaurus Compact Object Survey"), + ("J/MNRAS/389/1150", "Spitler+ 2008, Spitzer photometry of globulars"), + ("J/A+A/447/71", "Voss+ 2006, Chandra X-ray point sources in Cen A"), + ("J/ApJ/560/675", "Kraft+ 2001, Chandra X-ray point sources in Cen A"), + ("J/ApJ/766/88", "Burke+ 2013, Chandra X-ray binaries in Cen A"), + ("J/MNRAS/516/2300", "Joseph+ 2022, UV sources in the Cen A nuclear region"), + ("J/A+A/574/A109", "Walsh+ 2015, planetary nebulae in NGC 5128"), + ("J/A+A/478/755", "de Jong+ 2008, variable stars in Cen A"), + ("J/A+A/657/A41", "Rejkuba+ 2022, stellar halo of NGC 5128"), + ("J/A+A/448/983", "Rejkuba+ 2006, VIJsKs in AM1339-445 / AM1343-452"), + ("J/A+A/705/A112", "Faucher+ 2026, distances of Cen A / M83 galaxies"), + ("J/AJ/133/504", "Karachentsev+ 2007, galaxies around CenA/M83"), +] + +RAD_TARGET = 5.0 * u.arcsec +RAD_BLANKET = 5.0 * u.arcsec +RAD_SIMBAD = 10.0 * u.arcsec + +MAGISH = ("mag", "MAG", "Vmag", "Bmag", "Rmag", "Imag", "gmag", "rmag", "imag", + "umag", "zmag", "Kmag", "Jmag", "Hmag", "Gmag", "FUV", "NUV", "flux", + "Flux", "F(") + + +class Tee: + def __init__(self, *streams): + self.streams = streams + + def write(self, s): + for st in self.streams: + st.write(s) + + def flush(self): + for st in self.streams: + st.flush() + + +def sky(rec): + return SkyCoord(rec[1] * u.deg, rec[2] * u.deg, frame="icrs") + + +def row_coord(tab, row): + """Best-effort ICRS coordinate for a VizieR result row.""" + cn = tab.colnames + for rk, dk in (("_RAJ2000", "_DEJ2000"), ("RAJ2000", "DEJ2000"), + ("RA_ICRS", "DE_ICRS"), ("_RA", "_DE"), ("RAB1950", "DEB1950")): + if rk in cn and dk in cn: + rv, dv = row[rk], row[dk] + if rv is None or dv is None: + continue + try: + if isinstance(rv, str) and (":" in rv or " " in rv.strip()): + return SkyCoord(rv, dv, unit=(u.hourangle, u.deg), frame="icrs") + fr, fd = float(rv), float(dv) + if not (np.isfinite(fr) and np.isfinite(fd)): + continue + fr_ = "fk4" if rk == "RAB1950" else "icrs" + return SkyCoord(fr * u.deg, fd * u.deg, + frame=fr_).icrs if fr_ == "fk4" else \ + SkyCoord(fr * u.deg, fd * u.deg, frame="icrs") + except Exception: + continue + return None + + +def designation(tab, row): + cn = tab.colnames + prefer = ["Name", "ID", "GC", "Cluster", "Seq", "Source", "SimbadName", + "OName", "Object", "Obj", "CXO", "PN", "UCD", "GCID", "recno", + "HGHH", "WHH", "AAT", "f_Name", "No", "Nr"] + parts = [] + for k in prefer: + if k in cn and row[k] is not None: + v = str(row[k]).strip() + if v and v not in ("--", "nan", "None"): + parts.append(f"{k}={v}") + if len(parts) >= 2: + break + return " ".join(parts) if parts else "(no name column)" + + +def mags(tab, row, limit=8): + out = [] + for c in tab.colnames: + if c.startswith("_"): + continue + if any(m in c for m in MAGISH): + v = row[c] + if v is None: + continue + try: + fv = float(v) + if not np.isfinite(fv): + continue + out.append(f"{c}={fv:.3f}") + except (TypeError, ValueError): + s = str(v).strip() + if s and s not in ("--", "nan"): + out.append(f"{c}={s}") + if len(out) >= limit: + break + return ", ".join(out) if out else "(no magnitudes)" + + +def classification(tab, row): + out = [] + for c in ("Class", "Type", "otype", "OType", "Cl", "Note", "Notes", "Flag", + "Prob", "p", "Member", "n_Name", "Com", "Comm", "Sp", "SpType"): + if c in tab.colnames and row[c] is not None: + v = str(row[c]).strip() + if v and v not in ("--", "nan", "None"): + out.append(f"{c}={v}") + return "; ".join(out) if out else "" + + +def verify_catalogues(): + """Confirm which of TARGETS actually exist on VizieR.""" + print("\n" + "=" * 78) + print("STEP 0 -- verifying targeted VizieR catalogue identifiers") + print("=" * 78) + live, dead = [], [] + for cid, label in TARGETS: + try: + with contextlib.redirect_stdout(io.StringIO()): + meta = Vizier.get_catalog_metadata(catalog=cid) + desc = "" + try: + desc = str(meta["description"][0]) + except Exception: + desc = "(metadata returned, no description field)" + print(f" OK {cid:20s} {desc[:78]}") + live.append((cid, label)) + except Exception as e: + print(f" ABSENT/FAILED {cid:20s} ({label})") + print(f" -> {type(e).__name__}: {str(e)[:140]}") + dead.append((cid, label, f"{type(e).__name__}: {e}")) + return live, dead + + +def query_targeted(live, coords): + """Return {src_id: [ (cid,label,tabname,sep,desig,mags,cls) ]}""" + res = {s[0]: [] for s in SOURCES} + print("\n" + "=" * 78) + print(f"STEP 1 -- targeted catalogue cone search, r = {RAD_TARGET}") + print("=" * 78) + v = Vizier(columns=["**", "+_r"], row_limit=200) + for cid, label in live: + print(f"\n [{cid}] {label}") + for rec in SOURCES: + sid = rec[0] + c = coords[sid] + try: + tl = v.query_region(c, radius=RAD_TARGET, catalog=cid) + except Exception as e: + print(f" src {sid}: QUERY FAILED {type(e).__name__}: {str(e)[:110]}") + continue + n = 0 + for tab in tl: + tname = tab.meta.get("name", cid) + for row in tab: + rc = row_coord(tab, row) + if rc is not None: + sep = c.separation(rc).arcsec + elif "_r" in tab.colnames and row["_r"] is not None: + try: + sep = float(row["_r"]) * 60.0 + except Exception: + sep = float("nan") + else: + sep = float("nan") + if np.isfinite(sep) and sep > RAD_TARGET.to_value(u.arcsec) + 0.5: + continue + d = designation(tab, row) + m = mags(tab, row) + cl = classification(tab, row) + res[sid].append((cid, label, tname, sep, d, m, cl)) + n += 1 + print(f" src {sid}: MATCH {tname} sep={sep:.2f}\" {d}") + print(f" mags: {m}") + if cl: + print(f" class: {cl}") + if n == 0: + print(f" src {sid}: no match") + time.sleep(0.15) + return res + + +def query_blanket(coords): + res = {s[0]: [] for s in SOURCES} + print("\n" + "=" * 78) + print(f"STEP 2 -- BLANKET all-VizieR cone search, r = {RAD_BLANKET}") + print("=" * 78) + v = Vizier(columns=["**", "+_r"], row_limit=50, timeout=600) + for rec in SOURCES: + sid = rec[0] + c = coords[sid] + print(f"\n --- source {sid} {c.to_string('hmsdms', sep=':', precision=2)} ---") + try: + tl = v.query_region(c, radius=RAD_BLANKET, catalog=None) + except Exception as e: + print(f" BLANKET QUERY FAILED: {type(e).__name__}: {str(e)[:200]}") + continue + if tl is None or len(tl) == 0: + print(" no tables returned") + continue + print(f" {len(tl)} table(s) with entries") + for tab in tl: + tname = tab.meta.get("name", "?") + best = None + for row in tab: + rc = row_coord(tab, row) + sep = c.separation(rc).arcsec if rc is not None else float("nan") + if not np.isfinite(sep) and "_r" in tab.colnames: + try: + sep = float(row["_r"]) * 60.0 + except Exception: + pass + item = (tname, sep, designation(tab, row), mags(tab, row), + classification(tab, row)) + if best is None or (np.isfinite(sep) and + (not np.isfinite(best[1]) or sep < best[1])): + best = item + res[sid].append(item) + if best: + print(f" {best[0]:28s} sep={best[1]:6.2f}\" {best[2]}") + print(f" {best[3]}") + if best[4]: + print(f" {best[4]}") + time.sleep(0.3) + return res + + +def query_simbad(coords): + res = {s[0]: [] for s in SOURCES} + print("\n" + "=" * 78) + print(f"STEP 3 -- SIMBAD cone search, r = {RAD_SIMBAD}") + print("=" * 78) + try: + from astroquery.simbad import Simbad + except Exception as e: + print(f" SIMBAD IMPORT FAILED: {e}") + return res + sb = Simbad() + sb.TIMEOUT = 180 + for extra in ("otype", "otypes", "V", "B", "R", "G", "sp_type", "rvz_redshift", + "distance"): + try: + sb.add_votable_fields(extra) + except Exception: + pass + for rec in SOURCES: + sid = rec[0] + c = coords[sid] + print(f"\n --- source {sid} ---") + try: + t = sb.query_region(c, radius=RAD_SIMBAD) + except Exception as e: + print(f" SIMBAD QUERY FAILED: {type(e).__name__}: {str(e)[:160]}") + continue + if t is None or len(t) == 0: + print(" no SIMBAD object within 10\"") + continue + rk = "ra" if "ra" in t.colnames else "RA" + dk = "dec" if "dec" in t.colnames else "DEC" + for row in t: + try: + rc = SkyCoord(float(row[rk]) * u.deg, float(row[dk]) * u.deg) + sep = c.separation(rc).arcsec + except Exception: + sep = float("nan") + mid = str(row["main_id"] if "main_id" in t.colnames else row["MAIN_ID"]) + ot = str(row["otype"]) if "otype" in t.colnames else "?" + ots = str(row["otypes"]) if "otypes" in t.colnames else "" + mg = [] + for k in ("V", "B", "R", "G"): + if k in t.colnames and row[k] is not None: + try: + fv = float(row[k]) + if np.isfinite(fv): + mg.append(f"{k}={fv:.2f}") + except Exception: + pass + res[sid].append((mid, ot, ots, sep, ", ".join(mg))) + print(f" {mid:30s} otype={ot:12s} sep={sep:6.2f}\" {', '.join(mg)}") + if ots: + print(f" otypes: {ots}") + time.sleep(0.3) + return res + + +def summarise(targ, blank, simb, dead): + print("\n\n" + "#" * 78) + print("# PER-SOURCE SUMMARY") + print("#" * 78) + for rec in SOURCES: + sid, ra, dec, g, r50, note = rec + c = SkyCoord(ra * u.deg, dec * u.deg) + print("\n" + "-" * 78) + print(f"SOURCE {sid} RA={ra:.6f} Dec={dec:+.6f} " + f"({c.to_string('hmsdms', sep=':', precision=2)})") + print(f" G={g:.2f} r50/psf={r50:.2f} {note}") + t, b, s = targ[sid], blank[sid], simb[sid] + print(f" targeted catalogue matches : {len(t)}") + for cid, label, tname, sep, d, m, cl in t: + print(f" * {tname} sep={sep:.2f}\"") + print(f" {label}") + print(f" {d}") + print(f" {m}") + if cl: + print(f" {cl}") + print(f" blanket VizieR matches : {len(b)}") + for tname, sep, d, m, cl in b: + print(f" * {tname} sep={sep:.2f}\" {d}") + print(f" {m}" + (f" | {cl}" if cl else "")) + print(f" SIMBAD objects <10\" : {len(s)}") + for mid, ot, ots, sep, mg in s: + print(f" * {mid} otype={ot} sep={sep:.2f}\" {mg}") + if dead: + print("\n" + "-" * 78) + print("CATALOGUES THAT COULD NOT BE QUERIED (absent from VizieR or errored):") + for cid, label, err in dead: + print(f" {cid} ({label})") + print(f" {err[:200]}") + + +def main(): + coords = {rec[0]: sky(rec) for rec in SOURCES} + buf = io.StringIO() + real = sys.stdout + sys.stdout = Tee(real, buf) + try: + print("Cen A (NGC 5128) cross-match of six unidentified detections") + print("Generated by mo_gccheck.py at " + + time.strftime("%Y-%m-%d %H:%M:%S")) + print("Input positions J2000/ICRS; no Gaia DR3 (G<20.5) or SkyMapper DR2 match.") + for rec in SOURCES: + sid, ra, dec, g, r50, note = rec + print(f" {sid}: {ra:.6f} {dec:+.6f} G={g:.2f} r50/psf={r50:.2f} {note}") + live, dead = verify_catalogues() + targ = query_targeted(live, coords) + blank = query_blanket(coords) + simb = query_simbad(coords) + summarise(targ, blank, simb, dead) + finally: + sys.stdout = real + with open(OUT, "w", encoding="utf-8") as fh: + fh.write(buf.getvalue()) + print(f"\nWROTE {OUT}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/mo_link.py b/pipeline/mo_link.py new file mode 100644 index 0000000..d612c42 --- /dev/null +++ b/pipeline/mo_link.py @@ -0,0 +1,130 @@ +"""Moving-object search, pass 2: strip the static sky and link tracklets. + +The static sky is removed twice over. First, anything coincident with a source +in the deep luminance master is dropped. Second, anything that sits at the same +reference-frame position in four or more of the twelve subs is dropped - that +catches stars the master's sigma clipping or deblending missed, and by +construction cannot remove a real mover, which is never in the same place +twice. The price of the second cut is a floor on detectable motion: an object +slower than about 2 px per three exposures looks static and is removed with the +stars. mo_sensitivity.py measures where that floor actually falls. + +Third, and on this data much the most important, anything that sits at the same +DETECTOR-frame position in four or more subs is dropped. The subs are dithered +and the field rotates ~0.2 deg through the sequence, so registration - which +holds the sky still - drags anything fixed to the detector across the +registered frame on a perfectly straight, perfectly constant-rate, perfectly +constant-brightness track. Hot pixels are therefore ideal fake asteroids, and +they pass every cut a linker would normally apply. Skipping this one cut turns +a clean null result into 141 confident false detections. + +What is left is a few dozen detections per frame: cosmic rays, hot pixels that +survived the bad-pixel map, deblending artefacts around bright stars, noise +peaks at the 3.5 sigma threshold, and - if there is one - a minor planet. + +Linking is brute force over detection pairs (mo_common.link). Each pair of +residual detections from two frames separated by at least three exposures +defines a candidate velocity; velocities outside a plausible sky-motion range +are discarded, and the rest are propagated to every frame to see how many other +residuals fall on the predicted track. A candidate must be recovered in at +least five frames, lie on a straight constant-rate line to better than 1.2 px, +and hold its brightness to better than 0.5 mag. + +The point of demanding a straight line through many frames is that noise and +cosmic rays are independent between frames: the chance that five unrelated +false detections are collinear in space AND linear in time to sub-pixel +precision is tiny, and is measured directly by rerunning the linker on +time-permuted data (mo_sensitivity.py). + +Writes _mo_residuals.npz (the residual detection lists, reused by the +false-alarm control and the figures) and mo-candidates rows for the CSV. +""" +import csv +import os + +import numpy as np +from astropy.io import fits +from astropy.wcs import WCS +from scipy.spatial import cKDTree + +import mo_common as C + + +def main(): + times = C.lum_times() + keys = [k for k, _ in C.lum_frames()] + print(f"{len(keys)} luminance subs spanning {times[-1] * 60:.1f} min") + + tforms = C.frame_transforms() + mobjs, map_, mrms, mhdr, mimg = C.master_sources() + del mimg + mtree = cKDTree(np.column_stack([mobjs["x"], mobjs["y"]])) + print(f"static sky: {len(mobjs)} sources in the luminance master") + + pts, nats, mags, shapes = [], [], [], [] + for key, path in C.lum_frames(): + with fits.open(path, memmap=False) as hd: + img = hd[0].data.astype(np.float32) + objs, ap, rms = C.detect(img) + del img + keep = (objs["npix"] > 3) & (objs["npix"] < 3000) & (ap > 0) + objs, ap = objs[keep], ap[keep] + zp, nz = C.frame_zeropoint(objs["x"], objs["y"], ap, mobjs, map_, + tforms[key]) + nat = np.column_stack([objs["x"], objs["y"]]).astype(float) + nats.append(nat) + pts.append(tforms[key](nat)) + mags.append(-2.5 * np.log10(ap) + zp) + shapes.append(np.column_stack([objs["a"], objs["b"], + objs["npix"]]).astype(float)) + print(f" {key}: {len(objs)} det, rms {rms:.2f}, zp {zp:.3f}") + + mask = C.residual_mask(pts, nats, mtree) + rpts = [p[m] for p, m in zip(pts, mask)] + rmags = [g[m] for g, m in zip(mags, mask)] + rnat = [n[m] for n, m in zip(nats, mask)] + rshape = [s[m] for s, m in zip(shapes, mask)] + print("\nresiduals per frame:", " ".join(str(len(p)) for p in rpts)) + print(f"total residual detections: {sum(len(p) for p in rpts)}") + + cands = C.link(times, rpts, rmags, nat=rnat, shape=rshape) + print(f"\n{len(cands)} tracklet candidates after all cuts") + + wcs = WCS(mhdr) + tmid = times.mean() + rows = [] + for c in cands: + xm = c["cx"][0] + c["cx"][1] * tmid + ym = c["cy"][0] + c["cy"][1] * tmid + sky = wcs.pixel_to_world(xm, ym) + rows.append(dict( + ra_deg=round(sky.ra.deg, 6), dec_deg=round(sky.dec.deg, 6), + x_ref=round(xm, 2), y_ref=round(ym, 2), + rate_arcsec_per_hr=round(c["rate"] * C.SCALE, 2), + pa_deg=round(c["ang"], 1), g_mag=round(c["mag"], 2), + g_scatter=round(c["magsig"], 2), n_frames=c["nhit"], + line_rms_px=round(c["rms"], 2), + elongation=round(c["a"] / max(c["b"], 1e-6), 2))) + print(" " + str(rows[-1])) + + np.savez(os.path.join(C.OUT, "_mo_residuals.npz"), + keys=np.array(keys), times=times, + **{f"p{i}": p for i, p in enumerate(rpts)}, + **{f"m{i}": g for i, g in enumerate(rmags)}, + **{f"n{i}": n for i, n in enumerate(rnat)}) + np.save(os.path.join(C.OUT, "_mo_tracklets.npy"), + np.array(cands, dtype=object), allow_pickle=True) + + out = os.path.join(C.OUT, "mo-moving-candidates.csv") + fields = ["ra_deg", "dec_deg", "x_ref", "y_ref", "rate_arcsec_per_hr", + "pa_deg", "g_mag", "g_scatter", "n_frames", "line_rms_px", + "elongation"] + with open(out, "w", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=fields) + w.writeheader() + w.writerows(rows) + print(f"\nwrote {out} ({len(rows)} rows)") + + +if __name__ == "__main__": + main() diff --git a/pipeline/mo_mpc.py b/pipeline/mo_mpc.py new file mode 100644 index 0000000..ef09c55 --- /dev/null +++ b/pipeline/mo_mpc.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +""" +mo_mpc.py -- "was there a known minor planet in the frame?" + +Purpose +------- +The NGC 5128 (Centaurus A) LRGB session of 2026-07-21 was searched for moving +objects (see _mo_dets.npz / _mo_tracklets.npy). Before any detection can be +called a *new* object, we must rule out that it is simply a catalogued minor +planet that happened to drift through the field. This script asks the +authoritative services what known small bodies were inside the field of view at +the epoch of the exposures. + +Field / epoch +------------- + Field centre : RA 13 25 27.37, Dec -43 01 10.9 (J2000) + = 201.36404 deg, -43.01969 deg + Field size : 42.9' x 28.6' (half-diagonal ~25.8') + Search radius: 30 arcmin (0.5 deg) -- comfortably encloses the whole frame + Epoch : 2026-07-21 09:22 UTC (= 2026 07 21.39), mid-luminance-sequence + (luminance ran 08:56-10:00 UTC, session ended ~11:16 UTC) + Observatory : Q62, iTelescope / Siding Spring Observatory + +Services queried +---------------- +1. JPL Small-Body Identification API + https://ssd-api.jpl.nasa.gov/sb_ident.api + Primary source. Well documented, accepts arbitrary (including future) epochs + because it propagates orbits rather than serving a precomputed ephemeris. + IMPORTANT UNITS GOTCHA: despite some documentation wording, fov-ra-hwidth and + fov-dec-hwidth are interpreted in DEGREES, not arcminutes. Passing "30" + yields a 30-degree half-width box (the API echoes the box it used in + ["observer"]["fov_offset"], which is worth checking every run). + Queried twice: sb-kind=a (asteroids) and sb-kind=c (comets). + +2. MPC "MPChecker" / minor planet checker + https://minorplanetcenter.net/cgi-bin/mpcheck.cgi (classic HTTP POST form) + https://www.minorplanetcenter.net/cgi-bin/checkmp.cgi + Cross-check. Kept in the script even if it fails, so the log records whether + the MPC endpoint was reachable and what it said. + +Output +------ +Raw responses (JSON + HTML) and a human-readable summary are appended to + intermediates/_mpc_query.txt +Nothing is fabricated: if a service errors or refuses the epoch, the error text +itself is written to the log. +""" + +import json +import math +import sys +import datetime + +import requests + +import layout + +# ---------------------------------------------------------------- field/epoch +RA_SEX = "13-25-27.37" +DEC_SEX = "M43-01-10.9" # JPL uses a leading "M" for a negative Dec +RA_DEG = 201.36404 +DEC_DEG = -43.01969 +OBS_TIME = "2026-07-21_09:22:00" +MPC_CODE = "Q62" # iTelescope, Siding Spring +RADIUS_ARCMIN = 30.0 +HWIDTH_DEG = RADIUS_ARCMIN / 60.0 # = 0.5 deg; API wants DEGREES here +VMAG_LIM = 22.0 + +OUT = layout.path("_mpc_query.txt") + +log_lines = [] + + +def log(s=""): + print(s) + log_lines.append(str(s)) + + +# ------------------------------------------------------------------ JPL query +def jpl(sb_kind): + """Query ssd-api.jpl.nasa.gov/sb_ident.api for one small-body class.""" + params = { + "sb-kind": sb_kind, # 'a' = asteroid, 'c' = comet + "mpc-code": MPC_CODE, + "obs-time": OBS_TIME, + "fov-ra-center": RA_SEX, + "fov-dec-center": DEC_SEX, + "fov-ra-hwidth": f"{HWIDTH_DEG}", + "fov-dec-hwidth": f"{HWIDTH_DEG}", + "mag-required": "true", + "two-pass": "true", + "vmag-lim": f"{VMAG_LIM}", + } + r = requests.get("https://ssd-api.jpl.nasa.gov/sb_ident.api", + params=params, timeout=180) + log(f"--- JPL sb_ident sb-kind={sb_kind} -> HTTP {r.status_code}") + log(f" URL: {r.url}") + if r.status_code != 200: + log(" RAW: " + r.text[:2000]) + return None + d = r.json() + log(" observer : " + json.dumps(d.get("observer", {}))) + log(" n_first_pass : " + str(d.get("n_first_pass"))) + log(" n_second_pass: " + str(d.get("n_second_pass"))) + return d + + +def show(d, label): + """Print the refined (second-pass) hits, falling back to first pass.""" + if d is None: + return + rows = d.get("data_second_pass") or [] + fields = d.get("fields_second") or [] + if not rows: + rows = d.get("data_first_pass") or [] + fields = d.get("fields_first") or [] + log(f" [{label}] no second-pass data; showing first pass") + log(f" fields: {fields}") + if not rows: + log(f" [{label}] NO OBJECTS in field.") + return + for row in rows: + log(" " + " | ".join(str(x) for x in row)) + + +# ------------------------------------------------------------------ MPC query +def mpchecker(): + """Classic MPChecker POST form. Recorded even if it fails.""" + data = { + "year": "2026", "month": "07", "day": "21.39", + "which": "pos", + "ra": "13 25 27.37", "decl": "-43 01 10.9", + "TextArea": "", + "radius": str(int(RADIUS_ARCMIN)), + "limit": f"{VMAG_LIM}", + "oc": MPC_CODE, + "sort": "d", "mot": "h", "tmot": "s", + "pdes": "u", "needed": "f", "ps": "n", "type": "p", + } + for url in ("https://minorplanetcenter.net/cgi-bin/mpcheck.cgi", + "https://www.minorplanetcenter.net/cgi-bin/checkmp.cgi"): + try: + r = requests.post(url, data=data, timeout=120, + headers={"User-Agent": "Mozilla/5.0 (mo_mpc.py)"}) + log(f"--- MPC POST {url} -> HTTP {r.status_code}, {len(r.text)} bytes") + log(r.text[:8000]) + except Exception as e: # noqa: BLE001 + log(f"--- MPC POST {url} -> EXCEPTION {type(e).__name__}: {e}") + + +# ----------------------------------------------------------------------- main +if __name__ == "__main__": + log("MPC / minor-planet field check") + log("run at " + datetime.datetime.utcnow().isoformat() + "Z") + log(f"field centre {RA_SEX} {DEC_SEX} (J2000) = {RA_DEG}, {DEC_DEG}") + log(f"epoch {OBS_TIME} UTC, obs code {MPC_CODE}, " + f"radius {RADIUS_ARCMIN}' , V<{VMAG_LIM}") + log() + + for kind, label in (("a", "asteroids"), ("c", "comets")): + try: + show(jpl(kind), label) + except Exception as e: # noqa: BLE001 + log(f" JPL {label} EXCEPTION {type(e).__name__}: {e}") + log() + + mpchecker() + + # --- derived summary: total sky motion and position angle for JPL hits --- + log() + log("=== derived summary (from JPL second pass, sb-kind=a) ===") + try: + d = jpl("a") + for row in (d.get("data_second_pass") or []): + name, ra, dec = row[0], row[1], row[2] + norm = float(row[5]) + vmag = row[6] + dra, ddec = float(row[7]), float(row[8]) # "/h, RA rate has cos(dec) + tot = math.hypot(dra, ddec) + pa = math.degrees(math.atan2(dra, ddec)) % 360.0 # N through E + log(f"{name}: RA {ra} Dec {dec} V={vmag} " + f"{norm:.0f}\" ({norm/60:.1f}') from centre " + f"motion {tot:.1f}\"/hr at PA {pa:.1f} deg " + f"(dRA*cos(dec)={dra:.1f}, dDec={ddec:.1f} \"/hr)") + except Exception as e: # noqa: BLE001 + log(f"summary EXCEPTION {type(e).__name__}: {e}") + + with open(OUT, "w", encoding="utf-8") as fh: + fh.write("\n".join(log_lines) + "\n") + print("\nwrote " + OUT) diff --git a/pipeline/mo_sensitivity.py b/pipeline/mo_sensitivity.py new file mode 100644 index 0000000..823f554 --- /dev/null +++ b/pipeline/mo_sensitivity.py @@ -0,0 +1,209 @@ +"""Moving-object search, pass 3: measure what the search could have found. + +A null result is only worth reading if the sensitivity behind it is known, so +this injects synthetic moving point sources into the real pixel data and runs +the identical detection -> registration -> static-rejection -> linking chain +over them. + +Sources are laid out on a grid of magnitude against apparent rate. Each is +given a random position and direction, is placed at p0 + v*t in the reference +frame (mapped back through each sub's own affine into native pixels before +injection), and is trailed across the 300 s exposure. Its flux is set from the +per-frame zero point, which is tied to the master's by the stars the two have +in common, so a quoted magnitude means the same thing as it does for the +master's photometry. + +Recovery is scored by matching linked tracklets back to the injection truth. +The same run also reports the false-alarm rate, measured by permuting the +frame times of the real residual detections: that keeps the real spatial +density of cosmic rays and noise peaks but destroys any genuine temporal +ordering, so anything the linker still produces is by construction spurious. + +Output: _mo_sensitivity.npz +""" +import os +import sys + +import numpy as np +from astropy.io import fits + +import mo_common as C + +import layout + +MAGS = np.array([17.0, 17.5, 18.0, 18.5, 18.75, 19.0, 19.25]) +RATES_PX_HR = np.array([3.0, 5.0, 6.5, 8.0, 10.0, 15.0, 25.0, 40.0, + 70.0, 120.0, 180.0, 250.0]) +NREP = 4 # injections per (magnitude, rate) cell +MARGIN = 350 # px kept clear of the frame edge +MINSEP = 120 # px between injected tracks at t=0 +EXPHR = 300.0 / 3600.0 # exposure length in hours + + +def plan(times, shape, rng): + """Random start positions and directions for the injection grid.""" + ny, nx = shape + tspan = times[-1] + rows = [] + placed = [] + for mag in MAGS: + for rate in RATES_PX_HR: + for _ in range(NREP): + for _try in range(200): + ang = rng.uniform(0, 2 * np.pi) + vx, vy = rate * np.cos(ang), rate * np.sin(ang) + x0 = rng.uniform(MARGIN, nx - MARGIN) + y0 = rng.uniform(MARGIN, ny - MARGIN) + x1, y1 = x0 + vx * tspan, y0 + vy * tspan + if not (MARGIN < x1 < nx - MARGIN and + MARGIN < y1 < ny - MARGIN): + continue + if placed and min(np.hypot(x0 - p[0], y0 - p[1]) + for p in placed) < MINSEP: + continue + placed.append((x0, y0)) + rows.append(dict(mag=float(mag), rate=float(rate), + x0=x0, y0=y0, vx=vx, vy=vy)) + break + return rows + + +def run_pass(inject, times, tforms, mtree, zps, rng, thresh=3.5): + """Detect on every sub (optionally with synthetics added) and link.""" + frames = C.lum_frames() + pts, nats, mags = [], [], [] + for i, (key, path) in enumerate(frames): + with fits.open(path, memmap=False) as hd: + img = hd[0].data.astype(np.float32) + tf = tforms[key] + inv = tf.inverse + lin = np.linalg.inv(tf.params[:2, :2]) # ref -> native, linear + for s in inject: + xr = s["x0"] + s["vx"] * times[i] + yr = s["y0"] + s["vy"] * times[i] + xn, yn = inv(np.array([[xr, yr]]))[0] + d = lin @ np.array([s["vx"] * EXPHR, s["vy"] * EXPHR]) + ap_flux = 10 ** ((zps[key] - s["mag"]) / 2.5) + C.add_source(img, xn, yn, ap_flux / C.APFRAC, d[0], d[1]) + objs, ap, _rms = C.detect(img, thresh=thresh) + del img + keep = (objs["npix"] > 3) & (objs["npix"] < 3000) & (ap > 0) + objs, ap = objs[keep], ap[keep] + nat = np.column_stack([objs["x"], objs["y"]]).astype(float) + nats.append(nat) + pts.append(tf(nat)) + mags.append(-2.5 * np.log10(ap) + zps[key]) + return pts, nats, mags + + +def score(cands, inject, times, tol=8.0): + """Match linked tracklets back to the injection truth. + + Every candidate is assigned to its nearest compatible injection, so a + source recovered as two overlapping tracklets counts once as a detection + and does not also count as a false positive. Candidates that match no + injection at all are the genuinely spurious ones. + """ + tmid = times.mean() + tx = np.array([s["x0"] + s["vx"] * tmid for s in inject]) + ty = np.array([s["y0"] + s["vy"] * tmid for s in inject]) + trate = np.array([s["rate"] for s in inject]) + found = np.zeros(len(inject), dtype=bool) + spurious = [] + for ci, c in enumerate(cands): + cxm = c["cx"][0] + c["cx"][1] * tmid + cym = c["cy"][0] + c["cy"][1] * tmid + d = np.hypot(tx - cxm, ty - cym) + ok = (d < tol) & (np.abs(trate - c["rate"]) < 0.25 * trate + 3) + if ok.any(): + found[np.argmin(np.where(ok, d, np.inf))] = True + else: + spurious.append(ci) + return found, spurious + + +def main(): + times = C.lum_times() + tforms = C.frame_transforms() + mobjs, map_, _, _, mimg = C.master_sources() + shape = mimg.shape + del mimg + from scipy.spatial import cKDTree + mtree = cKDTree(np.column_stack([mobjs["x"], mobjs["y"]])) + print(f"master: {len(mobjs)} static sources, field {shape}") + + # Per-frame zero points from the stars the sub and the master share. + _meta, dets = C.load_dets() + zps = {} + for key, path in C.lum_frames(): + with fits.open(path, memmap=False) as hd: + img = hd[0].data.astype(np.float32) + objs, ap, _ = C.detect(img) + del img + z, n = C.frame_zeropoint(objs["x"], objs["y"], ap, mobjs, map_, + tforms[key]) + zps[key] = z + print(f" {key}: zero point {z:.3f} from {n} matched stars") + + seed = int(sys.argv[1]) if len(sys.argv) > 1 else 20260721 + rng = np.random.default_rng(seed) + inject = plan(times, shape, rng) + print(f"\ninjecting {len(inject)} synthetic movers " + f"({len(MAGS)} mags x {len(RATES_PX_HR)} rates x {NREP})") + + pts, nats, mags = run_pass(inject, times, tforms, mtree, zps, rng) + print("detections per frame:", " ".join(str(len(p)) for p in pts)) + keep = C.residual_mask(pts, nats, mtree) + rpts = [p[m] for p, m in zip(pts, keep)] + rmags = [g[m] for g, m in zip(mags, keep)] + rnat = [n[m] for n, m in zip(nats, keep)] + print("residuals per frame:", " ".join(str(len(p)) for p in rpts)) + + cands = C.link(times, rpts, rmags, nat=rnat) + print(f"{len(cands)} tracklets linked") + found, spurious = score(cands, inject, times) + nspur = len(spurious) + print(f"recovered {found.sum()}/{len(inject)}, " + f"{nspur} unmatched (spurious) tracklets") + + grid = np.zeros((len(MAGS), len(RATES_PX_HR))) + for k, s in enumerate(inject): + i = int(np.where(MAGS == s["mag"])[0][0]) + j = int(np.where(RATES_PX_HR == s["rate"])[0][0]) + grid[i, j] += found[k] + grid /= NREP + print("\nrecovery fraction (rows = G mag, cols = rate px/hr):") + print(" " + " ".join(f"{r:6.0f}" for r in RATES_PX_HR)) + for i, m in enumerate(MAGS): + print(f" {m:5.1f} " + " ".join(f"{v:6.2f}" for v in grid[i])) + + # ---- false-alarm control on the real (uninjected) data ---------- + real = np.load(layout.path("_mo_residuals.npz"), + allow_pickle=True) + rp = [real[f"p{i}"] for i in range(len(times))] + rf = [real[f"m{i}"] for i in range(len(times))] + rn = [real[f"n{i}"] for i in range(len(times))] + rm = rf + fa = [] + for trial in range(20): + perm = np.random.default_rng(1000 + trial).permutation(len(times)) + cs = C.link(times, [rp[k] for k in perm], [rm[k] for k in perm], + nat=[rn[k] for k in perm]) + fa.append(len(cs)) + print(f"\nfalse-alarm control: time-permuted real residuals, " + f"20 trials -> {np.sum(fa)} tracklets total " + f"({np.mean(fa):.2f} per trial)") + + np.savez(layout.path(f"_mo_sensitivity_{seed}.npz"), + mags=MAGS, rates=RATES_PX_HR, grid=grid, nrep=NREP, + inject=np.array([(s["mag"], s["rate"], s["x0"], s["y0"], + s["vx"], s["vy"]) for s in inject]), + found=found, nspur=nspur, falsealarm=np.array(fa), + times=times, scale=C.SCALE) + print(f"saved _mo_sensitivity_{seed}.npz") + + +OUT_NPZ_DIR = C.OUT + +if __name__ == "__main__": + main() diff --git a/pipeline/mo_shiftstack.py b/pipeline/mo_shiftstack.py new file mode 100644 index 0000000..3d5b00c --- /dev/null +++ b/pipeline/mo_shiftstack.py @@ -0,0 +1,211 @@ +"""Where the one known minor planet was, and how deep digital tracking goes. + +MPChecker and JPL's sb_ident agree that exactly one catalogued minor planet lay +within 30 arcmin of the field centre on this night: (427494) 2002 BK26, at +RA 13 27 00.6, Dec -43 11 26 at 09:22 UTC, V = 21.7, moving 53.2 arcsec/hr +toward position angle 71.7 deg. + +The first thing this script does is check whether it was actually inside the +frame, which is not obvious from the offsets alone. The field is 42.9' x 28.6' +at position angle -89 deg - the long axis runs very nearly along declination, +not right ascension - so the RA half-width is only about 14.6 arcmin. The +asteroid is 17.0 arcmin east of centre. It was outside the frame, by roughly +2.4 arcmin, and there is therefore nothing known to recover. + +The second thing is to measure how deep the search could have gone at that +rate, by shift-and-stack ("digital tracking"): each sub is shifted by the +target's own motion before combining, so a source moving at exactly that rate +adds coherently while the stars trail. Because the field position no longer +matters for the depth measurement, this is done on an empty patch of sky, and +a synthetic source of V = 21.7 is injected to show directly whether such an +object would have been recovered had it been in frame. + +Output: _mo_shiftstack.npz +""" +import os + +import numpy as np +import sep +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.stats import sigma_clipped_stats +from astropy.wcs import WCS +from scipy.ndimage import shift as ndshift +from scipy.spatial import cKDTree + +import mo_common as C + +NAME = "(427494) 2002 BK26" +EPH_RA = 201.752708 # deg, 13 27 00.65, at 09:22 UTC +EPH_DEC = -43.190667 # deg, -43 11 26.4 +EPH_UTC_HOURS = 9.0 + 22.0 / 60.0 +DRA_COSDEC = 50.5 # arcsec/hr +DDEC = 16.7 # arcsec/hr +RATE = np.hypot(DRA_COSDEC, DDEC) # 53.2 arcsec/hr +VMAG = 21.7 + +HALF = 60 # px half-size of the extracted stamp +NX, NY = 4788, 3194 + + +def utc_hours(path): + d = fits.getheader(path)["DATE-OBS"] + hh, mm, ss = d.split("T")[1].split(":") + return int(hh) + int(mm) / 60.0 + float(ss) / 3600.0 + 150.0 / 3600.0 + + +def footprint_check(wcs): + tgt = SkyCoord(EPH_RA * u.deg, EPH_DEC * u.deg) + px, py = wcs.world_to_pixel(tgt) + cen = wcs.pixel_to_world(NX / 2, NY / 2) + dra, ddec = cen.spherical_offsets_to(tgt) + corners = [wcs.pixel_to_world(x, y) for x, y in + ((0, 0), (NX - 1, 0), (0, NY - 1), (NX - 1, NY - 1))] + ras = [c.ra.deg for c in corners] + decs = [c.dec.deg for c in corners] + inside = (0 <= px < NX) and (0 <= py < NY) + print(f"{NAME}: V={VMAG}, {RATE:.1f}\"/hr at PA " + f"{np.degrees(np.arctan2(DRA_COSDEC, DDEC)):.1f} deg") + print(f" offset from field centre: {dra.to_value(u.arcmin):+.2f}' in RA, " + f"{ddec.to_value(u.arcmin):+.2f}' in Dec") + print(f" frame RA span {min(ras):.4f} to {max(ras):.4f} deg " + f"({(max(ras) - min(ras)) * np.cos(np.radians(EPH_DEC)) * 60:.1f}' " + f"on sky)") + print(f" frame Dec span {min(decs):.4f} to {max(decs):.4f} deg " + f"({(max(decs) - min(decs)) * 60:.1f}')") + print(f" predicted pixel ({px:.0f}, {py:.0f}) in a " + f"{NX} x {NY} frame -> " + f"{'INSIDE' if inside else 'OUTSIDE the frame'}") + return inside, float(px), float(py) + + +def blank_patch(mobjs, times, wcs, galx=2394.0, galy=1597.0, galrad=1000.0): + """A patch of sky with no source near the whole track, off the galaxy. + + Keeping clear of NGC 5128 itself matters more than it looks: the galaxy's + smooth light has a steep gradient that swamps the sky noise and would make + any depth measured on top of it meaningless. + """ + tree = cKDTree(np.column_stack([mobjs["x"], mobjs["y"]])) + # Track length in reference pixels over the sequence. + vx = (DRA_COSDEC / C.SCALE) + vy = (DDEC / C.SCALE) + span = times[-1] + rng = np.random.default_rng(7) + for _ in range(20000): + x0 = rng.uniform(400, NX - 400) + y0 = rng.uniform(400, NY - 400) + track = [(x0 + vx * t, y0 + vy * t) for t in times] + if not all(300 < x < NX - 300 and 300 < y < NY - 300 + for x, y in track): + continue + if any(np.hypot(x - galx, y - galy) < galrad for x, y in track): + continue + d, _ = tree.query(np.array(track), distance_upper_bound=22.0) + if np.all(~np.isfinite(d)): + return x0, y0, vx, vy, span + raise RuntimeError("no empty patch found") + + +def stamp(img, x, y, half=HALF): + xi, yi = int(round(x)), int(round(y)) + if not (half < xi < img.shape[1] - half and + half < yi < img.shape[0] - half): + return None + cut = img[yi - half:yi + half + 1, xi - half:xi + half + 1].astype(float) + return ndshift(cut, (yi - y, xi - x), order=3, mode="nearest") + + +def main(): + hdr = fits.getheader(os.path.join(C.OUT, "master-Luminance.fit")) + wcs = WCS(hdr) + inside, epx, epy = footprint_check(wcs) + + times = C.lum_times() + tforms = C.frame_transforms() + mobjs, map_, _, _, mimg = C.master_sources() + del mimg + x0, y0, vx, vy, span = blank_patch(mobjs, times, wcs) + print(f"\ndigital-tracking test patch: reference pixel " + f"({x0:.0f}, {y0:.0f}), track {np.hypot(vx, vy) * span:.0f} px " + f"over {span * 60:.0f} min") + + tracked, fixed, zps = [], [], [] + for i, (key, path) in enumerate(C.lum_frames()): + xr, yr = x0 + vx * times[i], y0 + vy * times[i] + inv = tforms[key].inverse + nx_, ny_ = inv(np.array([[xr, yr]]))[0] + fx_, fy_ = inv(np.array([[x0, y0]]))[0] + with fits.open(path, memmap=False) as hd: + img = hd[0].data.astype(np.float32) + objs, ap, _ = C.detect(img) + zp, _ = C.frame_zeropoint(objs["x"], objs["y"], ap, mobjs, map_, + tforms[key]) + zps.append(zp) + # A source at V=21.7 moving at the asteroid's rate, injected into the + # raw sub, trailing across the 300 s exposure exactly as it would. + lin = np.linalg.inv(tforms[key].params[:2, :2]) + dxy = lin @ np.array([vx * 300.0 / 3600.0, vy * 300.0 / 3600.0]) + img2 = img.copy() + C.add_source(img2, nx_, ny_, + 10 ** ((zp - VMAG) / 2.5) / C.APFRAC, dxy[0], dxy[1]) + a = stamp(img2, nx_, ny_) + b = stamp(img, nx_, ny_) # same patch, no synthetic + del img, img2 + tracked.append(a - np.median(a)) + fixed.append(b - np.median(b)) + print(f" {key}: zp {zp:.3f}") + + def combine(cube): + m, _, _ = sigma_clipped_stats(np.array(cube), sigma=3.0, maxiters=2, + axis=0) + return np.asarray(m, dtype=np.float32) + + def flatten(stack): + """Remove any residual sky gradient before measuring noise.""" + bkg = sep.Background(stack, bw=24, bh=24, fw=3, fh=3) + return stack - bkg.back() + + with_src = flatten(combine(tracked)) + empty = flatten(combine(fixed)) + zp_eff = float(np.mean(zps)) + + _, _, sd = sigma_clipped_stats(empty, sigma=3.0) + noise_ap = sd * np.sqrt(np.pi * C.APRAD ** 2) + lim5 = -2.5 * np.log10(5.0 * noise_ap) + zp_eff + c = float(HALF) + f_src, _, _ = sep.sum_circle(with_src, np.array([c]), np.array([c]), + C.APRAD, err=float(sd), gain=1.0) + f_emp, _, _ = sep.sum_circle(empty, np.array([c]), np.array([c]), + C.APRAD, err=float(sd), gain=1.0) + snr_src = float(f_src[0]) / noise_ap + print(f"\nstacked 12 x 300 s along the track") + print(f" effective zero point {zp_eff:.3f}, pixel sigma {sd:.3f}") + print(f" 5 sigma point-source limit of the tracked stack: " + f"G = {lim5:.2f}") + # The untrailed limit above is optimistic: at 53"/hr the object smears + # 8.2 px within each 300 s sub, and a 5 px aperture cannot hold a streak + # that long. Scaling the injected source's recovered significance to + # 5 sigma gives the limit that actually applies at this rate. + lim5_trail = VMAG - 2.5 * np.log10(5.0 / max(snr_src, 1e-3)) + print(f" injected V = {VMAG} source recovered at " + f"{snr_src:.2f} sigma " + f"({'DETECTED' if snr_src > 5 else 'NOT detectable'})") + print(f" -> 5 sigma limit for a source trailing at {RATE:.0f}\"/hr: " + f"G = {lim5_trail:.2f} (vs {lim5:.2f} for an untrailed source)") + print(f" same aperture on the un-injected stack: " + f"{float(f_emp[0]) / noise_ap:.2f} sigma (blank, as expected)") + print(f"\n{NAME} at V={VMAG} is {VMAG - lim5:+.2f} mag relative to that " + f"limit.") + + np.savez(os.path.join(C.OUT, "_mo_shiftstack.npz"), + with_src=with_src, empty=empty, zp_eff=zp_eff, lim5=lim5, + sd=sd, snr_src=snr_src, lim5_trail=lim5_trail, vmag=VMAG, rate=RATE, name=NAME, + inside=inside, eph_px=epx, eph_py=epy, x0=x0, y0=y0, + vx=vx, vy=vy, times=times) + print("\nsaved _mo_shiftstack.npz") + + +if __name__ == "__main__": + main() diff --git a/pipeline/mo_transient.py b/pipeline/mo_transient.py new file mode 100644 index 0000000..dc454d6 --- /dev/null +++ b/pipeline/mo_transient.py @@ -0,0 +1,232 @@ +"""Transient search: sources in the luminance master with no catalogue match. + +The chain is deliberately conservative, because on this field the sky is very +crowded with things that are real but not transient. + +1. Detect on the deep luminance master (60 min) and measure a 5 px aperture + magnitude on the master's own zero point. +2. Throw away everything that is not solidly detected. A transient claim rests + on a single night's data, so the bar is SNR > 10, not the SNR 5 the frame + nominally reaches. +3. Throw away everything that is not point-like. This is done against the + image's own PSF, measured from Gaia stars in the frame, using sep's + half-light radius. It removes background galaxies and the outer structure + of NGC 5128 itself, but note carefully that it does NOT remove Centaurus A's + globular clusters: at 3.8 Mpc a cluster with a 3 pc half-light radius spans + ~0.2 arcsec against 2.7 arcsec seeing, so clusters are unresolved here and + look exactly like stars. Only catalogues can separate them. +4. Cross-match against Gaia DR3 (the cached 0.42 deg, G < 20.5 catalogue). +5. Cross-match whatever is left against SkyMapper DR2 through VizieR, which + covers this declination and goes deeper than Gaia for non-stellar objects. +6. Demand that survivors are real by requiring an independent detection in at + least six of the twelve individual luminance subs. A cosmic ray or a + stacking artefact cannot do that; anything astrophysical will. +7. What is left is examined one by one. + +Writes mo-transient-candidates.csv and _mo_transient.npz. +""" +import csv +import os + +import numpy as np +import sep +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.wcs import WCS +from scipy.spatial import cKDTree + +import mo_common as C + +SNR_MIN = 10.0 +MATCH_RADIUS = 2.0 # arcsec, Gaia +SM_RADIUS = 3.0 # arcsec, SkyMapper (worse astrometry, wider PSF) +SUB_MIN = 6 # subs a source must independently appear in +EDGE = 40 # px + + +def detect_master(): + with fits.open(os.path.join(C.OUT, "master-Luminance.fit")) as hd: + img = hd[0].data.astype(np.float32) + hdr = hd[0].header + bkg = sep.Background(img, bw=64, bh=64, fw=3, fh=3) + sub = img - bkg.back() + objs = sep.extract(sub, 3.0, err=bkg.globalrms, minarea=5, + deblend_cont=0.005) + flux, ferr, _ = sep.sum_circle(sub, objs["x"], objs["y"], C.APRAD, + err=bkg.globalrms, gain=1.0) + r50, _ = sep.flux_radius(sub, objs["x"], objs["y"], + 6.0 * np.ones(len(objs)), 0.5, normflux=flux, + subpix=5) + return objs, np.asarray(flux), np.asarray(ferr), np.asarray(r50), \ + hdr, sub, bkg.globalrms + + +def gaia_cached(): + z = np.load(os.path.join(C.OUT, "_gaia_deep.npz")) + return z["ra"], z["dec"], z["g"] + + +def skymapper(ra0, dec0, radius_deg): + """SkyMapper DR2 over the field, cached because it is a slow query.""" + cache = os.path.join(C.OUT, "_skymapper.npz") + if os.path.exists(cache): + z = np.load(cache) + print(f"SkyMapper: {len(z['ra'])} cached sources") + return z["ra"], z["dec"], z["g"], z["r"], z["cls"] + from astroquery.vizier import Vizier + v = Vizier(columns=["RAICRS", "DEICRS", "gPSF", "rPSF", "ClassStar"], + row_limit=-1) + tbl = v.query_region(SkyCoord(ra0 * u.deg, dec0 * u.deg), + radius=radius_deg * u.deg, + catalog="II/358/smss")[0] + ra = np.asarray(tbl["RAICRS"], dtype=float) + dec = np.asarray(tbl["DEICRS"], dtype=float) + g = np.asarray(tbl["gPSF"], dtype=float) + r = np.asarray(tbl["rPSF"], dtype=float) + cls = np.asarray(tbl["ClassStar"], dtype=float) + np.savez_compressed(cache, ra=ra, dec=dec, g=g, r=r, cls=cls) + print(f"SkyMapper DR2: {len(ra)} sources retrieved") + return ra, dec, g, r, cls + + +def sky_match(sc, ra, dec, radius_arcsec): + """Nearest-neighbour match; returns index and separation in arcsec.""" + cat = SkyCoord(ra * u.deg, dec * u.deg) + idx, d2d, _ = sc.match_to_catalog_sky(cat) + return idx, d2d.arcsec, d2d.arcsec < radius_arcsec + + +def sub_support(xy_ref): + """How many of the twelve luminance subs independently show each source.""" + _, dets = C.load_dets() + tforms = C.frame_transforms() + n = np.zeros(len(xy_ref), dtype=int) + for key, _ in C.lum_frames(): + d = dets[key] + p = tforms[key](d[:, 2:4].astype(float)) + t = cKDTree(p) + dd, _ = t.query(xy_ref, distance_upper_bound=2.5) + n += np.isfinite(dd) + return n + + +def rgb_support(xy_ref, rms_scale=3.0): + """Peak significance of each position in the R, G and B masters. + + A real object on the sky is in every filter. A luminance-only artefact is + not. The masters are pixel-aligned by construction, so this is a direct + look-up rather than another registration. + """ + out = {} + for filt in ("Red", "Green", "Blue"): + with fits.open(os.path.join(C.OUT, f"master-{filt}.fit")) as hd: + img = hd[0].data.astype(np.float32) + bkg = sep.Background(img, bw=64, bh=64, fw=3, fh=3) + s = img - bkg.back() + fl, _, _ = sep.sum_circle(s, xy_ref[:, 0], xy_ref[:, 1], C.APRAD, + err=bkg.globalrms, gain=1.0) + noise = bkg.globalrms * np.sqrt(np.pi * C.APRAD ** 2) + out[filt] = np.asarray(fl) / noise + del img, s + return out + + +def main(): + objs, flux, ferr, r50, hdr, _sub, mrms = detect_master() + wcs = WCS(hdr) + ny, nx = 3194, 4788 + print(f"master detections: {len(objs)}") + + snr = np.where(ferr > 0, flux / np.maximum(ferr, 1e-9), 0.0) + mag = np.where(flux > 0, -2.5 * np.log10(np.maximum(flux, 1e-9)) + C.ZP, + np.nan) + + # PSF reference from the frame's own bright stars. + gra, gdec, gmag = gaia_cached() + sc_all = wcs.pixel_to_world(objs["x"], objs["y"]) + gidx, gsep, gok = sky_match(sc_all, gra, gdec, MATCH_RADIUS) + star = gok & (gmag[gidx] > 15) & (gmag[gidx] < 18.5) & (snr > 30) + r50_star = float(np.median(r50[star])) + r50_sig = float(np.std(r50[star])) + print(f"PSF from {star.sum()} Gaia stars: r50 = {r50_star:.2f} +- " + f"{r50_sig:.2f} px ({r50_star * C.SCALE:.2f} arcsec)") + + inframe = ((objs["x"] > EDGE) & (objs["x"] < nx - EDGE) & + (objs["y"] > EDGE) & (objs["y"] < ny - EDGE)) + good = inframe & (snr > SNR_MIN) & np.isfinite(mag) + pointlike = np.abs(r50 - r50_star) < 3.0 * max(r50_sig, 0.25) + print(f"SNR > {SNR_MIN:.0f} and in frame: {good.sum()}") + print(f" ... of which point-like: {(good & pointlike).sum()}") + print(f" ... of which Gaia-matched: {(good & pointlike & gok).sum()}") + + cand = good & pointlike & ~gok + print(f"\nno Gaia DR3 counterpart within {MATCH_RADIUS}\": {cand.sum()}") + + # SkyMapper DR2 + cen = wcs.pixel_to_world(nx / 2, ny / 2) + sra, sdec, sg, sr, scls = skymapper(cen.ra.deg, cen.dec.deg, 0.42) + sidx, ssep, sok = sky_match(sc_all, sra, sdec, SM_RADIUS) + cand2 = cand & ~sok + print(f"also no SkyMapper DR2 counterpart within {SM_RADIUS}\": " + f"{cand2.sum()}") + + ci = np.where(cand2)[0] + xy = np.column_stack([objs["x"][ci], objs["y"][ci]]) + nsub = sub_support(xy) + rgb = rgb_support(xy) + print(f"of those, detected in >= {SUB_MIN} individual subs: " + f"{(nsub >= SUB_MIN).sum()}") + + sc = wcs.pixel_to_world(xy[:, 0], xy[:, 1]) + # Distance from the centre of NGC 5128 - a supernova is expected on or + # near the galaxy, and this ranks the list accordingly. + cenA = SkyCoord("13h25m27.6s", "-43d01m09s") + dgal = sc.separation(cenA).arcmin + + rows = [] + for k, i in enumerate(ci): + rows.append(dict( + id=int(i), ra_deg=round(sc[k].ra.deg, 6), + dec_deg=round(sc[k].dec.deg, 6), + ra_hms=sc[k].ra.to_string(u.hour, sep=":", precision=2), + dec_dms=sc[k].dec.to_string(u.deg, sep=":", precision=1, + alwayssign=True), + x=round(float(xy[k, 0]), 2), y=round(float(xy[k, 1]), 2), + g_mag=round(float(mag[i]), 2), snr=round(float(snr[i]), 1), + r50_px=round(float(r50[i]), 2), + r50_over_psf=round(float(r50[i] / r50_star), 2), + n_subs=int(nsub[k]), + snr_R=round(float(rgb["Red"][k]), 1), + snr_G=round(float(rgb["Green"][k]), 1), + snr_B=round(float(rgb["Blue"][k]), 1), + gaia_sep_arcsec=round(float(gsep[i]), 2), + smss_sep_arcsec=round(float(ssep[i]), 2), + dist_from_cenA_arcmin=round(float(dgal[k]), 2))) + rows.sort(key=lambda r: -r["snr"]) + + out = os.path.join(C.OUT, "mo-transient-candidates.csv") + if rows: + with open(out, "w", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=list(rows[0].keys())) + w.writeheader() + w.writerows(rows) + print(f"\nwrote {out} ({len(rows)} rows)") + for r in rows[:40]: + print(f" {r['ra_hms']} {r['dec_dms']} G={r['g_mag']:5.2f} " + f"SNR={r['snr']:6.1f} r50/psf={r['r50_over_psf']:.2f} " + f"subs={r['n_subs']:2d} RGB=({r['snr_R']:.0f},{r['snr_G']:.0f}," + f"{r['snr_B']:.0f}) d={r['dist_from_cenA_arcmin']:.1f}'") + + np.savez(os.path.join(C.OUT, "_mo_transient.npz"), + x=objs["x"], y=objs["y"], mag=mag, snr=snr, r50=r50, + gaia_ok=gok, gaia_sep=gsep, sm_ok=sok, sm_sep=ssep, + good=good, pointlike=pointlike, cand=cand, cand2=cand2, + cand_idx=ci, nsub=nsub, r50_star=r50_star, r50_sig=r50_sig, + snr_R=rgb["Red"], snr_G=rgb["Green"], snr_B=rgb["Blue"], + dgal=dgal) + print("saved _mo_transient.npz") + + +if __name__ == "__main__": + main() diff --git a/pipeline/mo_vet.py b/pipeline/mo_vet.py new file mode 100644 index 0000000..3591b36 --- /dev/null +++ b/pipeline/mo_vet.py @@ -0,0 +1,189 @@ +"""Transient search, stage 2: vet the catalogue non-matches individually. + +Two questions have to be answered about every source that failed to match +Gaia DR3 and SkyMapper DR2. + +Is it real? A source can be missing from the catalogues simply because it is +not there - a cosmic ray pair that survived sigma clipping, a deblending +artefact on the galaxy, a noise peak. The test used here is independence: +demand that the source is detected on its own in at least six of the twelve +300 s subs. Nothing that is not on the sky can do that. + +Is it new? A transient is a source that is present tonight and absent from +archival imagery. The catalogue non-match is weak evidence, because both Gaia +and SkyMapper run out of depth around the magnitudes of interest here and +neither is complete for extended or blended objects on a galaxy this bright. +The strong evidence is a picture: this pulls a Digitized Sky Survey cutout at +each position (DSS2 red, epoch ~1990s) through the CDS hips2fits service and +puts it beside the master. Anything visible on a plate taken thirty years ago +is not a transient. + +Writes mo-transient-vetted.csv, and caches the DSS cutouts for the figure. +""" +import csv +import os +import warnings + +import numpy as np +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits + +import mo_common as C + +NSUB_MIN = 6 +CUT = 60 # px half-size of the postage stamps +DSSDIR = os.path.join(C.OUT, "_dss") + + +def load(): + z = np.load(os.path.join(C.OUT, "_mo_transient.npz")) + return z + + +def dss_cutout(ra, dec, fov_arcmin=1.5, npix=120, survey="CDS/P/DSS2/red"): + """Archival DSS2 red image of one position, cached to disk.""" + tag = f"{ra:.5f}{dec:+.5f}_{survey.split('/')[-1]}.fits" + path = os.path.join(DSSDIR, tag) + os.makedirs(DSSDIR, exist_ok=True) + if os.path.exists(path): + with fits.open(path) as hd: + return hd[0].data.astype(float) + try: + from astroquery.hips2fits import hips2fits + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + hdul = hips2fits.query( + hips=survey, width=npix, height=npix, + ra=ra * u.deg, dec=dec * u.deg, + fov=(fov_arcmin / 60.0) * u.deg, + projection="TAN", format="fits") + data = np.asarray(hdul[0].data, dtype=float) + fits.PrimaryHDU(data).writeto(path, overwrite=True) + return data + except Exception as exc: # noqa: BLE001 + print(f" DSS fetch failed for {ra:.5f} {dec:+.5f}: " + f"{type(exc).__name__}: {exc}") + return None + + +def nearest_catalogue(sc): + """Nearest Gaia and SkyMapper source at ANY separation, plus SIMBAD/NED. + + The main search used fixed match radii. For a handful of finalists it is + worth knowing what the nearest catalogued thing actually is and how far + away it lies - a 4 arcsec offset from a SkyMapper source on a crowded + galaxy usually means the same object, badly centroided. + """ + gz = np.load(os.path.join(C.OUT, "_gaia_deep.npz")) + gcat = SkyCoord(gz["ra"] * u.deg, gz["dec"] * u.deg) + sz = np.load(os.path.join(C.OUT, "_skymapper.npz")) + scat = SkyCoord(sz["ra"] * u.deg, sz["dec"] * u.deg) + gi, gd, _ = sc.match_to_catalog_sky(gcat) + si, sd, _ = sc.match_to_catalog_sky(scat) + return (gd.arcsec, gz["g"][gi], sd.arcsec, sz["g"][si], sz["r"][si], + sz["cls"][si]) + + +def ned_query(sc, radius_arcsec=15.0): + """Anything NED knows about within a few arcsec of each position.""" + out = [] + try: + from astroquery.ipac.ned import Ned + except Exception: # noqa: BLE001 + return ["NED unavailable"] * len(sc) + for c in sc: + try: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + t = Ned.query_region(c, radius=radius_arcsec * u.arcsec) + if t is None or len(t) == 0: + out.append("") + else: + names = [f"{r['Object Name']}({r['Type']}," + f"{r['Separation']:.1f}\")" for r in t[:3]] + out.append("; ".join(names)) + except Exception as exc: # noqa: BLE001 + out.append(f"query failed: {type(exc).__name__}") + return out + + +def main(): + z = load() + ci = z["cand_idx"] + nsub = z["nsub"] + sel = np.where(nsub >= NSUB_MIN)[0] + print(f"{len(ci)} sources matched neither Gaia DR3 nor SkyMapper DR2") + print(f"{len(sel)} of them are independently detected in >= {NSUB_MIN} " + f"of the 12 subs\n") + + from astropy.wcs import WCS + hdr = fits.getheader(os.path.join(C.OUT, "master-Luminance.fit")) + wcs = WCS(hdr) + x = z["x"][ci][sel] + y = z["y"][ci][sel] + sc = wcs.pixel_to_world(x, y) + + gd, gg, sd, sg, sr, scls = nearest_catalogue(sc) + ned = ned_query(sc) + + rows = [] + for k, i in enumerate(sel): + ra, dec = sc[k].ra.deg, sc[k].dec.deg + dss = dss_cutout(ra, dec) + # Is anything there on the archival plate? Compare the peak in the + # central 6 arcsec against the frame-wide robust scatter. + dss_sig = np.nan + if dss is not None and np.isfinite(dss).any(): + h = dss.shape[0] // 2 + core = dss[h - 6:h + 7, h - 6:h + 7] + med = np.nanmedian(dss) + mad = np.nanmedian(np.abs(dss - med)) * 1.4826 + if mad > 0: + dss_sig = float((np.nanmax(core) - med) / mad) + rows.append(dict( + ra_deg=round(ra, 6), dec_deg=round(dec, 6), + ra_hms=sc[k].ra.to_string(u.hour, sep=":", precision=2), + dec_dms=sc[k].dec.to_string(u.deg, sep=":", precision=1, + alwayssign=True), + x=round(float(x[k]), 1), y=round(float(y[k]), 1), + g_mag=round(float(z["mag"][ci][i]), 2), + snr=round(float(z["snr"][ci][i]), 1), + r50_over_psf=round(float(z["r50"][ci][i] / z["r50_star"]), 2), + n_subs=int(nsub[i]), + snr_R=round(float(z["snr_R"][i]), 1), + snr_G=round(float(z["snr_G"][i]), 1), + snr_B=round(float(z["snr_B"][i]), 1), + dist_cenA_arcmin=round(float(z["dgal"][i]), 2), + nearest_gaia_arcsec=round(float(gd[k]), 2), + nearest_gaia_G=round(float(gg[k]), 2), + nearest_smss_arcsec=round(float(sd[k]), 2), + nearest_smss_g=round(float(sg[k]), 2), + nearest_smss_classstar=round(float(scls[k]), 2), + dss_peak_sigma=round(dss_sig, 1) if np.isfinite(dss_sig) + else "", + ned=ned[k])) + r = rows[-1] + print(f" {r['ra_hms']} {r['dec_dms']} G={r['g_mag']:5.2f} " + f"SNR={r['snr']:6.1f} r50/psf={r['r50_over_psf']:.2f} " + f"subs={r['n_subs']:2d}") + print(f" nearest Gaia {r['nearest_gaia_arcsec']:6.2f}\" " + f"(G={r['nearest_gaia_G']:.2f}), nearest SkyMapper " + f"{r['nearest_smss_arcsec']:6.2f}\" (g={r['nearest_smss_g']:.2f}" + f", ClassStar={r['nearest_smss_classstar']:.2f})") + print(f" DSS2-red peak {r['dss_peak_sigma']} sigma, " + f"{r['dist_cenA_arcmin']:.1f}' from Cen A centre") + print(f" NED: {r['ned'] or '(nothing within 15\")'}") + + out = os.path.join(C.OUT, "mo-transient-vetted.csv") + with open(out, "w", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=list(rows[0].keys())) + w.writeheader() + w.writerows(rows) + print(f"\nwrote {out} ({len(rows)} rows)") + np.save(os.path.join(C.OUT, "_mo_vetted.npy"), + np.array(rows, dtype=object), allow_pickle=True) + + +if __name__ == "__main__": + main() diff --git a/pipeline/original.py b/pipeline/original.py new file mode 100644 index 0000000..8b5061d --- /dev/null +++ b/pipeline/original.py @@ -0,0 +1,153 @@ +"""The plainest possible stack: align the frames, average them, stop. + +This exists as a baseline to compare every processed version against. The +only operation applied to the pixels is the geometric one needed to make the +frames line up. In particular there is NO: + + - sky/background subtraction - gradient or plane removal + - per-frame flux normalisation - outlier or sigma rejection + - weighting by noise - colour calibration or white balance + - stretch, saturation or denoise - deconvolution or sharpening + +so cosmic rays, satellite trails, the moon gradient and every frame's own sky +level all survive into the result, exactly as they were recorded. That is the +point: it is the honest sum of the data. + +Two things it is NOT innocent of, and cannot be: + 1. The frames arrive from iTelescope already bias/dark/flat calibrated + (CALSTAT = 'BDF'). That cannot be undone here. + 2. Alignment resamples. A bicubic warp interpolates, which very slightly + smooths and correlates neighbouring pixels. The reference frame itself is + not resampled at all, so it is the one frame that stays pristine. + +Output is linear 32-bit FITS, which is what a baseline should be. A linear +image displays as almost pure black, so a display-only stretched preview is +written alongside and clearly labelled as such - the numbers live in the FITS. +""" +import os + +import astroalign as aa +import numpy as np +from astropy.io import fits +from PIL import Image +from skimage.transform import warp + +import layout + +SRC = layout.SESSION +STACKED = layout.SESSION +OUT = layout.path("original") +CACHE = layout.path("_stars.npz") + +# The same reference frame the processed stack used, so the two are pixel +# aligned and can be compared or differenced directly. +REFERENCE = "Luminance_002" +FILTERS = ["Luminance", "Red", "Green", "Blue"] + + +def main(): + os.makedirs(OUT, exist_ok=True) + z = np.load(CACHE, allow_pickle=True) + meta = {row[0]: row for row in z["meta"]} + stars = {k: z[k + "_xy"] for k in meta} + ref_xy = stars[REFERENCE] + print(f"reference {REFERENCE}, no crop, no rejection, no normalisation") + + # The plate solution was fitted on this same pixel grid, so it can be + # carried over. Copying header keywords does not touch the pixels. + with fits.open(layout.path("master-Luminance.fit")) as hd: + solved = hd[0].header + wcs_keys = [k for k in ("WCSAXES", "CRPIX1", "CRPIX2", "CDELT1", "CDELT2", + "CUNIT1", "CUNIT2", "CTYPE1", "CTYPE2", "CRVAL1", + "CRVAL2", "LONPOLE", "LATPOLE", "MJDREF", + "RADESYS", "PC1_1", "PC1_2", "PC2_1", "PC2_2") + if k in solved] + + planes = {} + for filt in FILTERS: + keys = sorted(k for k in meta if meta[k][2] == filt) + total = None + count = None + hdr0 = None + for key in keys: + fname = meta[key][1] + with fits.open(layout.path(fname), memmap=False) as hd: + data = hd[0].data.astype(np.float32) + if hdr0 is None: + hdr0 = hd[0].header.copy() + if key == REFERENCE: + reg = data # reference is never resampled + else: + tform, _ = aa.find_transform(stars[key], ref_xy) + reg = warp(data, inverse_map=tform.inverse, order=3, + mode="constant", cval=np.nan, + preserve_range=True).astype(np.float32) + del data + valid = np.isfinite(reg) + if total is None: + total = np.where(valid, reg, 0.0).astype(np.float32) + count = valid.astype(np.float32) + else: + total += np.where(valid, reg, 0.0) + count += valid + print(f" {key:16s} added (mean level {np.nanmean(reg):8.2f} ADU)") + del reg, valid + # Straight arithmetic mean. Where a frame did not cover a pixel it + # simply does not contribute, which is bookkeeping rather than + # processing: no pixel is invented. + stack = total / np.maximum(count, 1) + stack[count == 0] = 0.0 + del total, count + + hdr = hdr0 + hdr["FILTER"] = filt + hdr["NCOMBINE"] = (len(keys), "frames averaged") + hdr["EXPTOTAL"] = (300.0 * len(keys), "[s] total integration") + hdr["STACKREF"] = (REFERENCE, "alignment reference frame") + hdr["STACKALG"] = ("plain mean, no rejection", "combine method") + hdr["PROCLVL"] = ("align+average only", "no other processing applied") + for k in wcs_keys: + hdr[k] = (solved[k], solved.comments[k]) + path = layout.path(f"original-{filt}.fit") + fits.PrimaryHDU(stack.astype(np.float32), hdr).writeto(path, + overwrite=True) + print(f" -> {path} min={stack.min():.1f} median={np.median(stack):.1f} " + f"max={stack.max():.1f} ADU") + planes[filt] = stack + + # A colour version assembled with no calibration at all: the three filters + # dropped straight into R, G and B on a shared linear scale. Centaurus A + # will look yellow-green, because that is what the raw filter throughputs + # and a 46% moon actually produced. + rgb = np.dstack([planes["Red"], planes["Green"], planes["Blue"]]) + hdr = fits.getheader(layout.path("original-Red.fit")) + hdr["PROCLVL"] = ("align+average only", "no colour calibration applied") + fits.PrimaryHDU(np.moveaxis(rgb, 2, 0).astype(np.float32), hdr).writeto( + layout.path("original-RGB.fit"), overwrite=True) + print("wrote original-RGB.fit (uncalibrated colour cube)") + + # Display-only previews. The stretch here is a viewing aid and is NOT + # baked into any of the FITS above. + def preview(arr, name, note): + lo = np.percentile(arr, 25) + hi = np.percentile(arr, 99.9) + s = np.clip((arr - lo) / max(hi - lo, 1e-6), 0, 1) ** 0.35 + im = Image.fromarray((s * 255 + 0.5).astype(np.uint8)) + im.thumbnail((2400, 2400), Image.LANCZOS) + im.save(layout.path(name), quality=92) + print(f"wrote {name} ({note})") + + preview(planes["Luminance"], "NGC5128-original-Luminance-preview.jpg", + "display stretch only, linear data is in the FITS") + lo = np.percentile(rgb, 25) + hi = np.percentile(rgb, 99.9) + s = np.clip((rgb - lo) / max(hi - lo, 1e-6), 0, 1) ** 0.35 + im = Image.fromarray((s * 255 + 0.5).astype(np.uint8)) + im.thumbnail((2400, 2400), Image.LANCZOS) + im.save(layout.path("NGC5128-original-RGB-preview.jpg"), quality=92) + print("wrote NGC5128-original-RGB-preview.jpg (display stretch only, no colour " + "calibration)") + + +if __name__ == "__main__": + main() diff --git a/pipeline/rename.py b/pipeline/rename.py new file mode 100644 index 0000000..8729738 --- /dev/null +++ b/pipeline/rename.py @@ -0,0 +1,91 @@ +"""Prefix every image output with the object name, and fix every reference. + +Renaming files is the easy half. The half that breaks a project silently is the +references: METHODS.md and the three analysis notes cite these filenames, and +the scripts that WRITE them would otherwise recreate the old names on the next +run. So this rewrites the markdown and the scripts in the same pass. + +Dry run by default; pass --apply to actually move anything. +""" +import os +import re +import sys + +import layout + +ROOT = layout.SESSION +PREFIX = "NGC5128-" +IMAGE_EXT = (".png", ".jpg", ".jpeg", ".tif", ".tiff") +APPLY = "--apply" in sys.argv + + +def image_files(): + """Every image in the stacked tree that is not already prefixed.""" + out = [] + for folder in (ROOT, layout.path("original")): + if not os.path.isdir(folder): + continue + for name in sorted(os.listdir(folder)): + path = os.path.join(folder, name) + if not os.path.isfile(path): + continue + if not name.lower().endswith(IMAGE_EXT): + continue + if name.startswith(PREFIX): + continue + out.append((folder, name, PREFIX + name)) + return out + + +renames = image_files() +print(f"{len(renames)} image files to rename\n") +for folder, old, new in renames: + where = os.path.relpath(folder, ROOT) + print(f" {where}/{old} -> {new}") + +# Build the substitution map. Longest names first so that a short name which is +# a substring of a longer one cannot corrupt it. +mapping = {old: new for _, old, new in renames} +ordered = sorted(mapping, key=len, reverse=True) + +targets = [] +for folder, _, _ in [(ROOT, None, None)]: + for name in os.listdir(folder): + if name.lower().endswith(".md"): + targets.append(os.path.join(folder, name)) +scripts = layout.path("scripts") +if os.path.isdir(scripts): + for name in os.listdir(scripts): + if name.lower().endswith(".py"): + targets.append(os.path.join(scripts, name)) + +print(f"\nchecking {len(targets)} markdown and script files for references") +edits = [] +for path in targets: + try: + text = open(path, encoding="utf-8").read() + except UnicodeDecodeError: + text = open(path, encoding="latin-1").read() + updated = text + hits = 0 + for old in ordered: + # A reference is the bare filename; guard the left edge so an already + # prefixed occurrence is not double-prefixed. + pattern = re.compile(r"(? r=25 ("total") +ZPTOT = ZP5 + APCOR # 28.154, zero point for total flux +MU0 = ZPTOT + 2.5*np.log10(PIXAREA) # 26.807; mu = MU0 - 2.5*log10(I_ADU_per_px) +X0, Y0 = 2397.88, 1592.09 # nucleus, from WCS + Gaia-verified astrometry +DIST_MPC = 3.8 +KPC_PER_ARCSEC = DIST_MPC*1e3*np.pi/180/3600 # 0.01842 kpc/arcsec + +def path(*p): return layout.path(*p) + +def load(ch): + """Load one master as a contiguous native-endian float32 array.""" + d = fits.getdata(path('master-%s.fit' % ch)) + return np.ascontiguousarray(d.astype(np.float32)) + +def wcs(): + return WCS(fits.getheader(path('master-Luminance.fit'))) + +def mu(I): + """Surface brightness (mag/arcsec^2) from intensity in ADU/pixel.""" + I = np.asarray(I, float) + out = np.full(I.shape, np.nan) + m = I > 0 + out[m] = MU0 - 2.5*np.log10(I[m]) + return out + +def ell_radius(shape, x0, y0, eps, pa_rad): + """Semi-major-axis-equivalent radius map for a fixed ellipse geometry. + pa_rad measured counter-clockwise from the +x axis (photutils convention).""" + ny, nx = shape + y, x = np.mgrid[0:ny, 0:nx].astype(np.float32) + x -= np.float32(x0); y -= np.float32(y0) + c, s = np.float32(np.cos(pa_rad)), np.float32(np.sin(pa_rad)) + xp = x*c + y*s + yp = -x*s + y*c + del x, y + return np.sqrt(xp*xp + (yp/np.float32(1.0-eps))**2) + + +def sky_pa(pa_deg): + """Convert a photutils isophote PA (deg CCW from +x) to sky PA (deg E of N). + + For this frame north lies 0.96 deg CCW of the +x axis and east lies along + -y, so the two conventions differ by very nearly 90 deg with a flip. + """ + cd = wcs().pixel_scale_matrix + t = np.radians(np.asarray(pa_deg, float)) + xi = cd[0, 0]*np.cos(t) + cd[0, 1]*np.sin(t) # +east + eta = cd[1, 0]*np.cos(t) + cd[1, 1]*np.sin(t) # +north + return np.degrees(np.arctan2(xi, eta)) % 180. + + +def north_east_pixel(): + """Unit vectors (dx, dy) pointing north and east in pixel coordinates.""" + cd = wcs().pixel_scale_matrix + det = cd[0, 0]*cd[1, 1] - cd[0, 1]*cd[1, 0] + n = np.array([-cd[0, 1], cd[0, 0]])/det + e = np.array([cd[1, 1], -cd[1, 0]])/det + return n/np.hypot(*n), e/np.hypot(*e) + + +ISO_HDR = ('sma_px,sma_arcsec,sma_arcmin,sma_kpc,intens_adu_px,intens_err,' + 'rms_adu,mu_mag_arcsec2,mu_err,ellipticity,ellipticity_err,' + 'pa_deg_ccw_from_x,pa_err_deg,pa_deg_east_of_north,ndata,nflag,' + 'stop_code') + + +def write_isophote_csv(tab, fn): + """Write an isophote table to CSV. + + Rows with stop_code != 0 had too little unmasked azimuth for the geometry + to converge: photutils still measures a valid intensity along the held + ellipse, but its eps/PA are carried over from the previous isophote and its + formal errors are meaningless (they come back as values like 1169 and + 24006). Those five geometry columns are therefore blanked to NaN, so the + file cannot be read as if the geometry had been measured there. The + intensity columns are kept, because they are real. + """ + import os + a = tab['sma']*PIXSCALE + conv = tab['stop'] == 0 + blank = lambda v: np.where(conv, v, np.nan) + with np.errstate(all='ignore'): + me = 2.5/np.log(10)*tab['int_err']/np.where(tab['intens'] > 0, + tab['intens'], np.nan) + out = np.column_stack([tab['sma'], a, a/60., a*KPC_PER_ARCSEC, + tab['intens'], tab['int_err'], tab['rms'], + mu(tab['intens']), me, + blank(tab['eps']), blank(tab['eps_err']), + blank(tab['pa']), blank(tab['pa_err']), + blank(sky_pa(tab['pa'])), + tab['ndata'], tab['nflag'], tab['stop']]) + np.savetxt(path(fn), out, delimiter=',', header=ISO_HDR, comments='', + fmt='%.5f') + print('wrote %s (%d rows, %d with converged geometry)' + % (fn, len(out), conv.sum())) diff --git a/pipeline/sb_dust.py b/pipeline/sb_dust.py new file mode 100644 index 0000000..a6f05b5 --- /dev/null +++ b/pipeline/sb_dust.py @@ -0,0 +1,191 @@ +"""Step 5: dust-lane extinction map, with an independent colour-excess check. + +Method + 1. The smooth isophote model of the luminance master gives the light the + galaxy would show with no dust. A_L = -2.5 log10(observed / model). + This is a foreground-screen approximation: the lane is a warped disk seen + nearly edge-on across the near side of the bulge, so the screen assumption + is good for the lane itself but underestimates the true optical depth + wherever stars sit in front of the dust. + 2. The same is done for R, G and B on the SAME elliptical isophotes, using + the sigma-clipped azimuthal median of the dust-free azimuths as each + channel's unobscured model. E(B-R) = A_B - A_R is then a completely + independent, model-ratio-based reddening measurement, and A_L / E(B-R) + is a measured extinction-law ratio rather than an assumed one. + +Outputs + sb-extinction.fits A_L map (mag), NaN outside the measurable region + NGC5128-sb-extinction-map.png 4-panel extinction / reddening figure + NGC5128-sb-extinction-law.png A_L against E(B-R) with the fitted ratio +""" +import numpy as np +from astropy.io import fits +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt +from scipy.ndimage import gaussian_filter, median_filter +from sb_common import * + +XC, YC = np.load(path('_geom.npy')) +P = np.load(path('_profile.npz')) +star = fits.getdata(path('sb-mask-stars.fits')).astype(bool) +dust = fits.getdata(path('sb-mask-dust.fits')).astype(bool) +a_map = np.load(path('_amap.npy')) +ac = P['a'] + +A = {} +for ch in ['Luminance', 'Red', 'Green', 'Blue']: + pr = P[ch] + ok = np.isfinite(pr) & (pr > 0) + mdl = np.interp(a_map, ac[ok], pr[ok], left=pr[ok][0], right=np.nan) + img = gaussian_filter(load(ch), 2.0) # match the colour smoothing + with np.errstate(all='ignore'): + A[ch] = np.where((img > 0) & (mdl > 0), -2.5*np.log10(img/mdl), + np.nan).astype(np.float32) + del img, mdl + print('%-10s A map built' % ch) + +AL = A['Luminance'] +EBR = (A['Blue'] - A['Red']).astype(np.float32) + +# region where both are trustworthy: inside the lane, bright enough, not a star +Lmod = np.interp(a_map, ac[np.isfinite(P['Luminance'])], + P['Luminance'][np.isfinite(P['Luminance'])]) +valid = (~star) & (a_map < 700) & (Lmod > 60) & np.isfinite(AL) & np.isfinite(EBR) +lane = valid & dust +print('valid extinction pixels: %d (%.1f arcmin^2); inside the lane: %d (%.1f arcmin^2)' + % (valid.sum(), valid.sum()*PIXAREA/3600., lane.sum(), lane.sum()*PIXAREA/3600.)) + +x, y = EBR[lane].astype(float), AL[lane].astype(float) +sel = (x > 0.1) & (x < 2.0) & (y > -0.3) & (y < 3.5) +k = float(np.median(y[sel]/x[sel])) +klsq = float(np.sum(x[sel]*y[sel])/np.sum(x[sel]**2)) +print('extinction-law ratio A_L / E(B-R) = %.2f (median of ratios), %.2f (lsq)' + % (k, klsq)) + +ALs = median_filter(np.nan_to_num(AL, nan=0.0), 9) +ALs[~valid] = np.nan +pk = np.nanpercentile(ALs[lane], [50, 90, 99, 99.9]) +print('A_L inside the lane mask (9x9 median filtered): ' + 'median %.2f, p90 %.2f, p99 %.2f, p99.9 %.2f mag' % tuple(pk)) +iy, ix = np.unravel_index(np.nanargmax(np.where(lane, ALs, np.nan)), ALs.shape) +w = wcs() +rr, dd = w.pixel_to_world_values(ix, iy) +print('peak A_L = %.2f mag at pixel (%d, %d) = %.5f %+.5f deg, %.0f arcsec from ' + 'the nucleus' % (ALs[iy, ix], ix, iy, rr, dd, + np.hypot(ix-XC, iy-YC)*PIXSCALE)) + +hdu = fits.PrimaryHDU(np.where(valid, AL, np.nan).astype(np.float32), + header=w.to_header()) +hdu.header['BUNIT'] = 'mag' +hdu.header['COMMENT'] = 'A_L = -2.5 log10(observed / smooth isophote model)' +hdu.writeto(path('sb-extinction.fits'), overwrite=True) + +# obscured light: how much luminance flux the lane removes +Lraw = load('Luminance') +lost = float(np.nansum((Lmod - Lraw)[lane])) +tot = float(np.nansum(Lmod[valid & (a_map < 700)])) +print('the lane hides %.3e ADU, i.e. %.1f%% of the modelled light inside a=700 px' + % (lost, 100*lost/tot)) +print(' that is %.2f mag of integrated light removed from the lane region' + % (-2.5*np.log10(1 - lost/max(np.nansum(Lmod[lane]), 1)))) + +# ------------------------------------------------------------------- figure 1 +NV, EV = north_east_pixel() +CUT = 620 +sl = (slice(int(YC)-CUT, int(YC)+CUT), slice(int(XC)-CUT, int(XC)+CUT)) +ext = [-CUT*PIXSCALE/60, CUT*PIXSCALE/60]*2 + + +def compass(ax, c='k', x=0.885, y=0.10, Ln=0.075): + for v, lab in [(NV, 'N'), (EV, 'E')]: + ax.annotate('', xy=(x+Ln*v[0], y+Ln*v[1]), xytext=(x, y), + xycoords='axes fraction', textcoords='axes fraction', + arrowprops=dict(arrowstyle='->', color=c, lw=1.4)) + ax.annotate(lab, xy=(x+1.45*Ln*v[0], y+1.45*Ln*v[1]), color=c, + xycoords='axes fraction', ha='center', va='center', fontsize=10) + + +fig, axs = plt.subplots(2, 2, figsize=(15, 14.4)) +axs = axs.ravel() +axs[0].imshow(np.arcsinh(np.clip(Lraw[sl], 0, None)/30), origin='lower', + cmap='gray', extent=ext) +axs[0].set_title('(a) luminance master') +compass(axs[0], 'w') + +im = axs[1].imshow(np.where(valid, ALs, np.nan)[sl], origin='lower', cmap='magma_r', + vmin=0, vmax=2.0, extent=ext) +axs[1].set_title('(b) extinction $A_L$ from the smooth model [mag]') +plt.colorbar(im, ax=axs[1], fraction=.046, label='mag') +compass(axs[1]) + +im = axs[2].imshow(np.where(valid, gaussian_filter(np.nan_to_num(EBR), 2), np.nan)[sl], + origin='lower', cmap='inferno_r', vmin=0, vmax=1.4, extent=ext) +axs[2].set_title('(c) colour excess E(B-R) from the R and B models [mag]') +plt.colorbar(im, ax=axs[2], fraction=.046, label='mag') +compass(axs[2]) + +im = axs[3].imshow(np.where(valid, ALs - k*EBR, np.nan)[sl], origin='lower', + cmap='RdBu_r', vmin=-0.6, vmax=0.6, extent=ext) +axs[3].set_title('(d) $A_L$ - %.2f E(B-R): agreement of the two methods' % k) +plt.colorbar(im, ax=axs[3], fraction=.046, label='mag') +compass(axs[3]) +for a in axs: + a.set_xlabel('arcmin') + a.set_ylabel('arcmin') +fig.suptitle('NGC 5128 dust lane: extinction and reddening ' + '(central %.1f x %.1f arcmin)' % (2*CUT*PIXSCALE/60, 2*CUT*PIXSCALE/60), + fontsize=14) +fig.tight_layout() +fig.savefig(path('NGC5128-sb-extinction-map.png'), dpi=125) +plt.close(fig) + +# ------------------------------------------------------------------- figure 2 +fig, axs = plt.subplots(1, 2, figsize=(13.5, 5.6)) +h = axs[0].hist2d(x[sel], y[sel], bins=(140, 140), range=[[0, 1.6], [-0.3, 3.0]], + cmap='viridis', norm=matplotlib.colors.LogNorm()) +plt.colorbar(h[3], ax=axs[0], label='pixels') +xs = np.linspace(0, 1.6, 50) +axs[0].plot(xs, k*xs, 'r-', lw=2, label='$A_L$ = %.2f E(B-R) (median ratio)' % k) +axs[0].plot(xs, klsq*xs, 'w--', lw=1.6, label='least squares: %.2f' % klsq) +axs[0].set_xlabel('E(B-R) [mag, instrumental]') +axs[0].set_ylabel('$A_L$ [mag]') +axs[0].legend(fontsize=9) +axs[0].set_title('extinction law measured inside the lane (%d px)' % sel.sum()) + +bb = np.geomspace(20, 700, 26) +ib = np.digitize(a_map, bb) +med, p90 = [], [] +for kk in range(1, len(bb)): + m = (ib == kk) & lane + med.append(np.nanmedian(ALs[m]) if m.sum() > 200 else np.nan) + p90.append(np.nanpercentile(ALs[m], 90) if m.sum() > 200 else np.nan) +bc = np.sqrt(bb[1:]*bb[:-1])*PIXSCALE +axs[1].plot(bc, med, 'o-', color='tab:purple', label='median $A_L$ in the lane') +axs[1].plot(bc, p90, 's--', color='tab:orange', label='90th percentile') +axs[1].set_xscale('log') +axs[1].set_xlabel('semi-major axis a [arcsec]') +axs[1].set_ylabel('$A_L$ [mag]') +axs[1].grid(alpha=.3) +axs[1].legend(fontsize=9) +axs[1].set_title('extinction against radius along the lane') +fig.tight_layout() +fig.savefig(path('NGC5128-sb-extinction-law.png'), dpi=140) +plt.close(fig) + +with open(path('sb-derived-quantities.txt'), 'a') as f: + wr = lambda t: (f.write(t + chr(10)), print(t)) + wr('') + wr('Dust lane') + wr(' lane mask area %.1f arcmin^2' % (dust.sum()*PIXAREA/3600.)) + wr(' measurable extinction area %.1f arcmin^2' % (lane.sum()*PIXAREA/3600.)) + wr(' A_L median / p90 / p99 / p99.9 %.2f / %.2f / %.2f / %.2f mag' % tuple(pk)) + wr(' peak A_L (9x9 median filtered) %.2f mag at RA %.4f Dec %+.4f' + % (ALs[iy, ix], rr, dd)) + wr(' peak is %.0f arcsec from the nucleus' % (np.hypot(ix-XC, iy-YC)*PIXSCALE)) + wr(' extinction law A_L / E(B-R) %.2f (median of ratios), %.2f (lsq)' + % (k, klsq)) + wr(' luminance flux hidden by the lane %.1f%% of the modelled light inside a=700 px' + % (100*lost/tot)) +print('') +print('wrote sb-extinction.fits, NGC5128-sb-extinction-map.png, NGC5128-sb-extinction-law.png') diff --git a/pipeline/sb_iso.py b/pipeline/sb_iso.py new file mode 100644 index 0000000..c2a4c8a --- /dev/null +++ b/pipeline/sb_iso.py @@ -0,0 +1,128 @@ +"""Step 2: isophote fitting of the NGC 5128 luminance master. + +Pass A free-centre fits on the dust-free outer body -> adopted centre +Pass B fixed-centre fit, stars AND the dust lane masked -> PRIMARY table +Pass C the same fit on a de-reddened image, where the + extinction is estimated from the B-R colour excess and + calibrated against the Pass B model over 150 inner extension + +Outputs: sb-isophotes.csv, sb-isophotes-dereddened.csv, + _isoB.npz, _isoC.npz, _geom.npy, _extcal.npy +""" +import numpy as np +import time +from astropy.io import fits +from photutils.isophote import Ellipse, EllipseGeometry +from sb_common import * +import sb_model + +SMA_MIN, SMA_MAX, STEP = 8.0, 1750.0, 0.11 + +L = load('Luminance') +starmask = fits.getdata(path('sb-mask-stars.fits')).astype(bool) +dustmask = fits.getdata(path('sb-mask-dust.fits')).astype(bool) +exc = np.load(path('sb-colour-excess.npy')) + + +def table(iso): + k = [i for i in iso if i.sma > 0 and np.isfinite(i.intens)] + + def g(f): + return np.array([(f(i) if f(i) is not None else np.nan) for i in k], float) + + return dict(sma=g(lambda i: i.sma), intens=g(lambda i: i.intens), + int_err=g(lambda i: i.int_err), rms=g(lambda i: i.rms), + eps=g(lambda i: i.eps), eps_err=g(lambda i: i.ellip_err), + pa=np.degrees(g(lambda i: i.pa)) % 180., + pa_err=np.degrees(g(lambda i: i.pa_err)), + ndata=g(lambda i: i.ndata), nflag=g(lambda i: i.nflag), + stop=g(lambda i: i.stop_code)) + + +def fit(img, mask, x0, y0, label): + arr = np.ma.masked_array(img, mask=mask) + g = EllipseGeometry(x0=x0, y0=y0, sma=300., eps=0.15, pa=np.radians(150.)) + g.fix_center = True + t = time.time() + iso = Ellipse(arr, geometry=g).fit_image( + sma0=300., minsma=SMA_MIN, maxsma=SMA_MAX, step=STEP, linear=False, + nclip=3, sclip=3.0, fix_center=True) + print('%s: %d isophotes in %.0f s' % (label, len(iso), time.time() - t)) + return table(iso) + + +# ------------------------------------------------------------------- Pass A +arr = np.ma.masked_array(L, mask=starmask | dustmask) +cen = [] +for s in [350., 500., 650., 800., 1000.]: + try: + it = Ellipse(arr, geometry=EllipseGeometry( + x0=X0, y0=Y0, sma=s, eps=0.18, pa=np.radians(148.)) + ).fit_image(sma0=s, minsma=s * 0.98, maxsma=s * 1.02, step=0.1, + nclip=3, sclip=3.) + for i in it: + if np.isfinite(i.x0): + cen.append((i.x0, i.y0)) + except Exception as e: + print(' passA sma=%.0f: %s' % (s, e)) +cen = np.array(cen) +XC, YC = float(np.median(cen[:, 0])), float(np.median(cen[:, 1])) +off = np.hypot(XC - X0, YC - Y0) +print('Pass A: outer-isophote centre %.2f, %.2f (scatter %.1f, %.1f px, n=%d)' + % (XC, YC, cen[:, 0].std(), cen[:, 1].std(), len(cen))) +print(' WCS/Gaia nucleus %.2f, %.2f -> offset %.1f px = %.1f arcsec' + % (X0, Y0, off, off * PIXSCALE)) +np.save(path('_geom.npy'), np.array([XC, YC])) +del arr + +# ------------------------------------------------------------------- Pass B +tB = fit(L, starmask | dustmask, XC, YC, 'Pass B (stars+dust masked)') +np.savez(path('_isoB.npz'), **tB) + +# ---------------------------------------------------------------- extinction +# A_L from the Pass B model, used only where that model is directly constrained +good = np.isfinite(tB['intens']) & (tB['ndata'] > 150) & (tB['sma'] > 90) +tBg = {k: v[good] for k, v in tB.items()} +modB, aB = sb_model.build(L.shape, XC, YC, tBg, block=4) +with np.errstate(all='ignore'): + A_L = -2.5 * np.log10(np.clip(L, 1e-3, None) / np.clip(modB, 1e-3, None)) +cal = (dustmask & ~starmask & (aB > 150) & (aB < 600) & np.isfinite(exc) + & (exc > 0.05) & np.isfinite(A_L) & (A_L > -0.5) & (A_L < 4.0)) +cal &= exc > 0.15 # restrict to a well-measured colour excess +x, y = exc[cal].astype(float), A_L[cal].astype(float) +# robust slope through the origin: median of the per-pixel ratios +k_ratio = float(np.median(y / x)) +scatter = float(np.median(np.abs(y - k_ratio * x)) * 1.4826) +print('extinction calibration on %d px: A_L = %.3f * E(B-R), scatter %.3f mag' + % (cal.sum(), k_ratio, scatter)) +print(' (least-squares through origin for comparison: %.3f)' + % (np.sum(x * y) / np.sum(x * x))) +np.save(path('_extcal.npy'), np.array([k_ratio, scatter, cal.sum()])) + +E = np.clip(np.nan_to_num(exc, nan=0.0), 0.0, None) +A_est = np.clip(k_ratio * E, 0.0, 2.5) # >2.5 mag is unreliable +heavy = (k_ratio * E) > 2.5 +Lc = (L * 10 ** (0.4 * A_est)).astype(np.float32) +print('de-reddening: median A_L inside the lane mask %.2f mag; %d px above the ' + '2.5 mag cap (masked in Pass C)' % (np.median(A_est[dustmask]), heavy.sum())) +del modB, aB, A_L, A_est, E + +# ------------------------------------------------------------------- Pass C +tC = fit(Lc, starmask | heavy, XC, YC, 'Pass C (de-reddened)') +np.savez(path('_isoC.npz'), **tC) +del Lc, L + +# ------------------------------------------------------------------- CSVs +write_isophote_csv(tB, 'sb-isophotes.csv') +write_isophote_csv(tC, 'sb-isophotes-dereddened.csv') + +for name, t in [('Pass B (primary, dust masked)', tB), ('Pass C (de-reddened)', tC)]: + print('') + print(name) + print(' sma_px arcsec mu eps PA ndata nflag stop') + for i in range(len(t['sma'])): + if t['sma'][i] > 30 and i % 3: + continue + print('%7.1f %7.1f %6.2f %6.3f %6.1f %6d %5d %4d' + % (t['sma'][i], t['sma'][i] * PIXSCALE, mu(t['intens'][i]), + t['eps'][i], t['pa'][i], t['ndata'][i], t['nflag'][i], t['stop'][i])) diff --git a/pipeline/sb_limits.py b/pipeline/sb_limits.py new file mode 100644 index 0000000..93a6ed3 --- /dev/null +++ b/pipeline/sb_limits.py @@ -0,0 +1,137 @@ +"""Step 6: how deep does the shell search actually go, and is anything there? + +Two questions: + 1. On what surface-brightness level would a shell have had to sit to be seen? + Binning the residual to ever coarser scales shows whether the noise + integrates down like photon noise (it does not: it is dominated by + correlated large-scale systematics), which sets the real limit. + 2. Is there any significant azimuthal structure? The m = 1..4 Fourier + amplitudes of the residual in each elliptical annulus are compared with + the amplitude expected from the noise alone. +""" +import numpy as np +from astropy.io import fits +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt +from sb_common import * + +XC, YC = np.load(path('_geom.npy')) +star = fits.getdata(path('sb-mask-stars.fits')).astype(bool) +dust = fits.getdata(path('sb-mask-dust.fits')).astype(bool) +a_map = np.load(path('_amap.npy')) +res = fits.getdata(path('sb-residual-flat.fits')).astype(np.float32) +rms = float(np.load(path('_rms.npy'))[0]) + + +def blockstat(img, mask, B, sel): + H, W = img.shape + h, w = H // B, W // B + a = img[:h * B, :w * B].reshape(h, B, w, B) + m = (~mask)[:h * B, :w * B].reshape(h, B, w, B) + s = sel[:h * B, :w * B].reshape(h, B, w, B) + n = m.sum(axis=(1, 3)) + v = np.where(n > 0.35 * B * B, np.where(m, a, 0).sum(axis=(1, 3)) / + np.maximum(n, 1), np.nan) + keep = np.isfinite(v) & (s.mean(axis=(1, 3)) > 0.8) + return v[keep] + + +print('depth of the shell search, measured on the model-subtracted residual') +print('(outer field, 1200 < a < 2400 px, stars and the dust lane excluded)') +print('') +print(' bin bin size rms 3 sigma limit ideal if noise were white') +print(' [px] [arcsec] [ADU/px] [mag/arcsec2] [mag/arcsec2]') +sel = (a_map > 1200) & (a_map < 2400) # outer field, beyond the measured profile +base = None +rows = [] +for B in [8, 16, 32, 64, 128]: + v = blockstat(res, star | dust, B, sel) + if v.size < 30: + continue + sd = float(np.std(v)) + if base is None: + base, base_b = sd, B + ideal = base * (base_b / B) + print(' %4d %7.1f %8.3f %13.2f %13.2f' + % (B, B * PIXSCALE, sd, mu(3 * sd), mu(3 * ideal))) + rows.append((B, sd)) +print('') +print('The rms barely falls as the bins grow (%.2f -> %.2f ADU/px from 8 to 128 px' + % (rows[0][1], rows[-1][1])) +print('bins, against a factor 16 if it were white), so the floor is correlated') +print('large-scale structure -- flat-field residual plus the sky-plane') +print('systematic -- not photon noise. Pure photon noise would reach') +print('%.2f mag/arcsec2 at 128 px bins; the real limit is %.1f mag SHALLOWER.' + % (mu(3 * rms / 128), mu(3 * rms / 128) - mu(3 * rows[-1][1]))) + +# ------------------------------------------------------- Fourier amplitudes +print('') +print('azimuthal Fourier amplitudes of the residual, normalised to the model') +resd = np.load(path('_resd16.npy')) +a16 = np.load(path('_a16.npy')) +H, W = resd.shape +Y, X = np.mgrid[0:H, 0:W] +phi = np.arctan2(Y * 16 + 8 - YC, X * 16 + 8 - XC) +P = np.load(path('_profile.npz')) +prof, ac = P['Luminance'], P['a'] +okp = np.isfinite(prof) & (prof > 0) + +# stop where the measured profile itself runs out: beyond that I(a) is a fill +# value and the fractional amplitudes would be meaningless +A_OUT = float(P['a_out']) +edges = np.geomspace(150, A_OUT, 11) +out = [] +print(' a range [px] I(a) m=1 m=2 m=3 m=4 noise n') +for lo, hi in zip(edges[:-1], edges[1:]): + m = np.isfinite(resd) & (a16 >= lo) & (a16 < hi) + n = m.sum() + if n < 60: + continue + r, ph = resd[m], phi[m] + amp = [2 * np.abs(np.mean(r * np.exp(-1j * k * ph))) for k in (1, 2, 3, 4)] + noise = np.std(r) * np.sqrt(2. / n) * 2 + Im = np.interp(np.sqrt(lo * hi), ac[okp], prof[okp]) + out.append((np.sqrt(lo * hi), Im, amp, noise, n)) + print(' %5.0f-%5.0f %8.2f ' % (lo, hi, Im) + + ' '.join('%6.3f' % (a / max(Im, 1e-3)) for a in amp) + + ' %6.3f %5d' % (noise / max(Im, 1e-3), n)) +print('') +print('(amplitudes are fractional: A_m / I(a). A value is only meaningful if it') +print(' exceeds the "noise" column, which is the amplitude a pure-noise annulus') +print(' would produce.)') + +fig, ax = plt.subplots(figsize=(9.5, 6.4)) +aa = np.array([o[0] for o in out]) * PIXSCALE +for k in range(4): + ax.plot(aa, [o[2][k] / max(o[1], 1e-3) for o in out], 'o-', ms=4, + label='m = %d' % (k + 1)) +ax.plot(aa, [o[3] / max(o[1], 1e-3) for o in out], 'k--', lw=1.6, + label='formal noise expectation (lower bound: it assumes' + chr(10) + + 'independent bins and ignores correlated systematics)') +ax.set_xscale('log') +ax.set_yscale('log') +ax.set_xlabel('semi-major axis a [arcsec]') +ax.set_ylabel('fractional Fourier amplitude $A_m / I(a)$') +ax.set_title('NGC 5128: azimuthal structure in the model-subtracted residual' + + chr(10) + 'amplitudes are 4-10% of the local surface brightness at ' + 'every radius' + chr(10) + 'inside ~350 arcsec this is demonstrably ' + 'the dust lane; outside it, correlated systematics') +ax.grid(alpha=.3, which='both') +ax.legend(fontsize=8.5, loc='lower left') +fig.tight_layout() +fig.savefig(path('NGC5128-sb-residual-fourier.png'), dpi=140) +plt.close(fig) + +with open(path('sb-derived-quantities.txt'), 'a') as f: + wr = lambda t: (f.write(t + chr(10)), print(t)) + wr('') + wr('Shell / faint-structure search depth (outer field 1200 < a < 2400 px)') + for B, sd in rows: + wr(' %4d px bins (%5.1f arcsec): rms %.3f ADU/px, 3 sigma = %.2f mag/arcsec^2' + % (B, B * PIXSCALE, sd, mu(3 * sd))) + wr(' the rms does not integrate down like photon noise: the floor is') + wr(' correlated large-scale structure, not shot noise') + wr(' NO shells, arcs or tidal features were detected') +print('') +print('wrote NGC5128-sb-residual-fourier.png') diff --git a/pipeline/sb_model.py b/pipeline/sb_model.py new file mode 100644 index 0000000..f603098 --- /dev/null +++ b/pipeline/sb_model.py @@ -0,0 +1,80 @@ +"""Smooth elliptical model builder shared by the later steps. + +Given an isophote table (sma, intens, eps, pa) and a fixed centre, assign every +pixel the semi-major axis a of the isophote passing through it. Because eps(a) +and pa(a) vary slowly this is solved by fixed-point iteration starting from the +circular radius, which converges in a handful of passes. The model intensity is +then a log-log interpolation of intens(a). + +This is used instead of photutils.isophote.build_ellipse_model because it is +much faster on a 4788x3194 frame and because it guarantees a strictly smooth, +monotonic-in-a model with no interpolation artefacts to confuse the residual. +""" +import numpy as np +from scipy.interpolate import interp1d +from scipy.ndimage import gaussian_filter1d, zoom + + +def smooth_geometry(sma, eps, pa_deg, sig=2.0): + """Return (sma, eps, pa_rad) with eps and PA lightly smoothed along a.""" + ok = np.isfinite(eps) & np.isfinite(pa_deg) & np.isfinite(sma) + s = sma[ok] + e = gaussian_filter1d(np.clip(eps[ok], 0.0, 0.7), sig, mode='nearest') + p = np.unwrap(np.radians(pa_deg[ok]) * 2.0) / 2.0 # PA is defined mod 180 + p = gaussian_filter1d(p, sig, mode='nearest') + return s, e, p + + +def radius_map(shape, xc, yc, sma, eps, pa_rad, nit=15, dtype=np.float32): + """Semi-major axis of the isophote through each pixel.""" + fe = interp1d(sma, eps, bounds_error=False, fill_value=(eps[0], eps[-1])) + fp = interp1d(sma, pa_rad, bounds_error=False, fill_value=(pa_rad[0], pa_rad[-1])) + ny, nx = shape + Y, X = np.mgrid[0:ny, 0:nx].astype(np.float64) + X -= xc + Y -= yc + a = np.maximum(np.hypot(X, Y), 0.3) + for _ in range(nit): + q = 1.0 - fe(a) + th = fp(a) + c, s = np.cos(th), np.sin(th) + xp = X * c + Y * s + yp = -X * s + Y * c + a = np.maximum(np.sqrt(xp * xp + (yp / q) ** 2), 0.3) + return a.astype(dtype) + + +def build(shape, xc, yc, tab, nit=15, block=1): + """Return (model, a_map) at full resolution. + + block > 1 computes the radius map on a coarser grid and bilinearly + upsamples it; the model is smooth on scales far larger than block so this + costs nothing in accuracy and a lot less in time and memory. + """ + s, e, p = smooth_geometry(tab['sma'], tab['eps'], tab['pa']) + if block > 1: + sh = (shape[0] // block, shape[1] // block) + a = radius_map(sh, (xc - (block - 1) / 2.) / block, + (yc - (block - 1) / 2.) / block, s / block, e, p, nit) + # radius_map worked in block units: convert back to full-resolution pixels + a = zoom(a.astype(np.float32) * block, + (shape[0] / sh[0], shape[1] / sh[1]), order=1) + if a.shape != tuple(shape): + b = np.zeros(shape, np.float32) + n0, n1 = min(a.shape[0], shape[0]), min(a.shape[1], shape[1]) + b[:n0, :n1] = a[:n0, :n1] + if n0 < shape[0]: + b[n0:, :] = b[n0 - 1, :] + if n1 < shape[1]: + b[:, n1:] = b[:, [n1 - 1]] + a = b + else: + a = radius_map(shape, xc, yc, s, e, p, nit) + + ok = np.isfinite(tab['intens']) & (tab['intens'] > 1e-3) + ls, li = np.log10(tab['sma'][ok]), np.log10(tab['intens'][ok]) + o = np.argsort(ls) + ls, li = ls[o], li[o] + mod = (10 ** np.interp(np.log10(np.maximum(a, 0.3)), ls, li, + left=li[0], right=-3.0)).astype(np.float32) + return mod, a diff --git a/pipeline/sb_prep.py b/pipeline/sb_prep.py new file mode 100644 index 0000000..192fe59 --- /dev/null +++ b/pipeline/sb_prep.py @@ -0,0 +1,132 @@ +"""Step 1: masks and basic calibration checks for the NGC 5128 analysis. + +Produces + sb-mask-stars.fits foreground stars / saturated cores / compact objects + sb-mask-dust.fits the dust lane, defined from the B-R colour excess + sb-colour-excess.npy E(B-R) instrumental colour excess map (float32) + +The dust mask is built from COLOUR, not from a model residual: the lane is the +only thing in the frame that is strongly red relative to the smooth stellar +body, so a colour cut is far more specific than a brightness-residual cut and +does not eat the galaxy itself. +""" +import numpy as np, sep +from astropy.io import fits +from scipy import ndimage +from sb_common import * + +# ------------------------------------------------------------------ saturation +d = load('Luminance') +H, W = d.shape +print('-- saturation census (luminance master) --') +for thr in [30000, 50000, 60000, 62000, 63000, 64000, 65000, 66000]: + print(' > %6d ADU : %6d px' % (thr, (d > thr).sum())) +SAT = 63000.0 +core = ndimage.median_filter(d[int(Y0)-400:int(Y0)+400, int(X0)-400:int(X0)+400], 25) +print(' star-free (median-25) peak of galaxy light, inner 800 px: %.1f ADU' % core.max()) +print(' => the galaxy core is a factor %.0f below the clip level: NOT saturated' + % (SAT/core.max())) +del core + +bkg = sep.Background(d, bw=128, bh=128) +rms = float(bkg.globalrms) +print('\nglobal sky rms %.3f ADU/px -> 1-sigma = %.2f mag/arcsec^2' % (rms, mu(rms))) +np.save(path('_rms.npy'), np.array([rms])) + +# ------------------------------------------------------------------ star mask +mask = np.zeros((H, W), bool) +z = np.load(path('_gaia_deep.npz')) +gx, gy = wcs().world_to_pixel_values(z['ra'], z['dec']); gg = z['g'] +rad = np.clip(4.0 + 3.6*(16.5 - gg), 4, 120) +sel = (gx > -150) & (gx < W+150) & (gy > -150) & (gy < H+150) & (gg < 19.0) +gx, gy, rad = gx[sel], gy[sel], rad[sel] +print('\nGaia stars masked: %d (radii %.0f-%.0f px)' % (gx.size, rad.min(), rad.max())) +for x, y, r in zip(gx, gy, rad): + i0, i1 = max(0, int(y-r)), min(H, int(y+r)+1) + j0, j1 = max(0, int(x-r)), min(W, int(x+r)+1) + if i1 <= i0 or j1 <= j0: continue + sy = np.arange(i0, i1)[:, None] - y; sx = np.arange(j0, j1)[None, :] - x + mask[i0:i1, j0:j1] |= (sx*sx + sy*sy) < r*r + +rr = np.hypot(np.arange(W)[None, :]-X0, np.arange(H)[:, None]-Y0) + +# compact non-Gaia objects, but leave the crowded inner 150 px to the +# isophote fitter's own sigma clipping (masking there kills the fit) +sub = d - bkg.back() +obj, seg = sep.extract(sub, 8.0, err=rms, minarea=6, deblend_cont=0.005, + segmentation_map=True) +compact = obj['npix'] < 20000 +segmask = ndimage.binary_dilation(np.isin(seg, np.nonzero(compact)[0]+1), np.ones((5, 5))) +mask |= segmask & (rr > 150) +print('sep compact objects: %d (applied outside r=150 px)' % compact.sum()) +del sub, seg, segmask + +sat = ndimage.binary_dilation(d > SAT, np.ones((5, 5)), iterations=6) +mask |= sat +print('saturated-core mask (grown 12 px): %d px' % sat.sum()) +del sat, d + +print('star mask: %.2f%% of frame' % (100*mask.mean())) +for a, b in [(0,25),(25,50),(50,100),(100,200),(200,400),(400,800),(800,1600)]: + s = (rr >= a) & (rr < b) + print(' r %4d-%4d px: %.1f%%' % (a, b, 100*mask[s].mean())) +fits.PrimaryHDU(mask.astype(np.uint8)).writeto(path('sb-mask-stars.fits'), overwrite=True) + +# ------------------------------------------------------------------ dust mask +# instrumental B-R from smoothed R and B masters +R = ndimage.gaussian_filter(load('Red'), 3.0) +B = ndimage.gaussian_filter(load('Blue'), 3.0) +good = (R > 15) & (B > 4) +col = np.full((H, W), np.nan, np.float32) +col[good] = -2.5*np.log10(B[good]/R[good]) +del R, B + +# Unobscured baseline colour vs elliptical radius. Dust only ever reddens, so +# the blue tail of the colour distribution in each annulus is the dust-free +# stellar colour. The 15th percentile is measured over 150 < a < 900 px (where +# both the colour SNR is high and the lane does not fill the annulus) and fitted +# with a quadratic in log a, which is then extrapolated inwards and outwards. +a_map = ell_radius((H, W), X0, Y0, 0.17, np.radians(150.0)) +bins = np.geomspace(5, 2200, 60) +ib = np.digitize(a_map, bins) +base_r, base_v = [], [] +valid = good & ~mask +for k in range(1, len(bins)): + s = (ib == k) & valid + if s.sum() < 400: continue + base_r.append(0.5*(bins[k-1]+bins[k])); base_v.append(np.nanpercentile(col[s], 15)) +base_r, base_v = np.array(base_r), np.array(base_v) +fitr = (base_r > 150) & (base_r < 900) +pcoef = np.polyfit(np.log10(base_r[fitr]), base_v[fitr], 2) +print() +print('colour baseline: quadratic in log10(a), coeffs', np.round(pcoef, 4)) +print(' baseline B-R at a = 20/50/150/400/900 px:', + np.round(np.polyval(pcoef, np.log10([20, 50, 150, 400, 900])), 3)) +base = np.polyval(pcoef, np.log10(np.clip(a_map, 5, 3000))).astype(np.float32) +exc = (col - base).astype(np.float32) # E(B-R), instrumental +np.save(path('sb-colour-excess.npy'), exc) +np.save(path('_colbase.npy'), np.c_[base_r, base_v]) +np.save(path('_colbasefit.npy'), pcoef) + +# Hysteresis threshold: seed on a firm colour excess, grow into the fainter +# wings of the same connected structure. A flat low threshold alone picks up a +# spurious ring at the edge of the colour-SNR region, so it is not used. +e0 = np.nan_to_num(exc, nan=-9.0) +seed = (e0 > 0.30) & (a_map < 650) +grow = (e0 > 0.20) & (a_map < 650) +seed = ndimage.binary_opening(seed, np.ones((5, 5))) +lab, n = ndimage.label(ndimage.binary_closing(grow, np.ones((7, 7)))) +keep = np.unique(lab[seed & (lab > 0)]) +dust = np.isin(lab, keep[keep > 0]) +dust = ndimage.binary_closing(dust, np.ones((15, 15))) +lab, n = ndimage.label(dust); sz = np.bincount(lab.ravel()); sz[0] = 0 +dust = np.isin(lab, np.nonzero(sz > 3000)[0]) +print('dust mask: %d px = %.1f arcmin^2 (%.2f%% of frame)' + % (dust.sum(), dust.sum()*PIXAREA/3600., 100*dust.mean())) +for a, b in [(0,25),(25,50),(50,100),(100,200),(200,400),(400,800)]: + s = (a_map >= a) & (a_map < b) + print(' dust a %4d-%4d px: %5.1f%% star+dust %5.1f%%' + % (a, b, 100*dust[s].mean(), 100*(dust | mask)[s].mean())) +fits.PrimaryHDU(dust.astype(np.uint8)).writeto(path('sb-mask-dust.fits'), overwrite=True) +print() +print('wrote sb-mask-stars.fits, sb-mask-dust.fits, sb-colour-excess.npy') diff --git a/pipeline/sb_profile.py b/pipeline/sb_profile.py new file mode 100644 index 0000000..8e83277 --- /dev/null +++ b/pipeline/sb_profile.py @@ -0,0 +1,311 @@ +"""Step 3: surface-brightness profile, isophote geometry plots, derived numbers. + +Produces + NGC5128-sb-profile.png mu(a) with every noise/systematic floor marked + NGC5128-sb-isophote-geometry.png mu, ellipticity and position angle vs semi-major axis + NGC5128-sb-colour-profile.png L, R, G, B profiles and the B-R colour gradient + sb-derived-quantities.txt the numbers worth quoting + +The photutils Pass B table is the primary isophote result. An independent +sigma-clipped azimuthal-median extraction on the same elliptical grid is done +here as a cross-check and to get the R/G/B profiles on identical isophotes. +""" +import numpy as np +from astropy.io import fits +from scipy.optimize import curve_fit +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt +from sb_common import * +import sb_model + +XC, YC = np.load(path('_geom.npy')) +rms = float(np.load(path('_rms.npy'))[0]) +tB = dict(np.load(path('_isoB.npz'))) +star = fits.getdata(path('sb-mask-stars.fits')).astype(bool) +dust = fits.getdata(path('sb-mask-dust.fits')).astype(bool) + +# ---------------------------------------------------------------- geometry map +gfit = np.isfinite(tB['intens']) & (tB['ndata'] > 150) & (tB['sma'] > 90) +tg = {k: v[gfit] for k, v in tB.items()} +s, e, p = sb_model.smooth_geometry(tg['sma'], tg['eps'], tg['pa']) +a_map = sb_model.radius_map((3194, 4788), XC, YC, s, e, p) +np.save(path('_amap.npy'), a_map) + +# ------------------------------------------------- independent profile extract +bins = np.geomspace(10, 2600, 90) +ib = np.digitize(a_map, bins) +ac = np.sqrt(bins[1:] * bins[:-1]) + + +def azimuthal(img, mask): + v = np.full(len(bins) - 1, np.nan) + n = np.zeros(len(bins) - 1, int) + sd = np.full(len(bins) - 1, np.nan) + for k in range(1, len(bins)): + m = (ib == k) & ~mask + if m.sum() < 40: + continue + x = img[m].astype(float) + med = np.median(x) + for _ in range(3): + r = 1.4826 * np.median(np.abs(x - med)) + if not np.isfinite(r) or r == 0: + break + x = x[np.abs(x - med) < 3 * r] + med = np.median(x) + v[k - 1] = med + n[k - 1] = x.size + sd[k - 1] = 1.4826 * np.median(np.abs(x - med)) + return v, n, sd + + +prof, nprof, sdprof = {}, {}, {} +for ch in ['Luminance', 'Red', 'Green', 'Blue']: + img = load(ch) + prof[ch], nprof[ch], sdprof[ch] = azimuthal(img, star | dust) + if ch == 'Luminance': + # far-field pedestal: the frame is over-subtracted because the sky plane + # was fitted on tiles that still contained galaxy halo + Bk = 64 + H, W = img.shape + lb = img[:H // Bk * Bk, :W // Bk * Bk].reshape(H // Bk, Bk, W // Bk, Bk) + mb = (~star)[:H // Bk * Bk, :W // Bk * Bk].reshape(H // Bk, Bk, W // Bk, Bk) + cnt = mb.sum(axis=(1, 3)) + bm = np.where(cnt > 1500, np.where(mb, lb, 0).sum(axis=(1, 3)) / + np.maximum(cnt, 1), np.nan) + ab = a_map[:H // Bk * Bk, :W // Bk * Bk].reshape(H // Bk, Bk, + W // Bk, Bk).mean(axis=(1, 3)) + PED = float(np.nanmean(bm[np.isfinite(bm) & (ab > 2400)])) + BLKRMS = float(np.nanstd(bm[np.isfinite(bm) & (ab > 1700)])) + del img + +L = prof['Luminance'] +print('far-field pedestal (a > 2400 px): %+.2f ADU/px -> mu %.2f' % (PED, mu(-PED))) +print('block-to-block scatter (a > 1700 px): %.2f ADU/px -> mu %.2f' % (BLKRMS, mu(BLKRMS))) + +# noise / systematic floors, expressed as surface brightness +FLOOR = { + 'per-pixel sky noise (1 sigma)': mu(rms), + 'large-scale block scatter (1 sigma)': mu(BLKRMS), + 'sky-pedestal systematic |offset|': mu(abs(PED)), +} +for k, v in FLOOR.items(): + print(' floor: %-38s %.2f mag/arcsec2' % (k, v)) + +# ---------------------------------------------------------------- reliability +A_IN = 62.0 # inward limit: the dust lane fills 100% of the azimuth inside +A_OUT = float(ac[np.nanargmax(np.where(L > abs(PED), ac, -1))]) # last a with I > |PED| +print('adopted reliable range: %.0f - %.0f px (%.1f - %.1f arcsec)' + % (A_IN, A_OUT, A_IN * PIXSCALE, A_OUT * PIXSCALE)) + +# ---------------------------------------------------------------- Sersic fit +def sersic_mu(a, mue, re, n): + bn = 2 * n - 1 / 3. + 0.009876 / n + return mue + 2.5 * bn / np.log(10) * ((a / re) ** (1. / n) - 1.) + + +fitsel = np.isfinite(L) & (ac >= A_IN) & (ac <= 900) & (L > 0) +x, y = ac[fitsel] * PIXSCALE, mu(L[fitsel]) +popt, pcov = curve_fit(sersic_mu, x, y, p0=[21.0, 300.0, 4.0], maxfev=40000) +mue, re_as, nser = popt +perr = np.sqrt(np.diag(pcov)) +print('Sersic fit over %.0f-%.0f arcsec: n = %.2f +- %.2f, Re = %.1f +- %.1f arcsec ' + '(%.2f kpc), mu_e = %.2f' % (x.min(), x.max(), nser, perr[2], re_as, perr[1], + re_as * KPC_PER_ARCSEC, mue)) +resid_sersic = y - sersic_mu(x, *popt) +print(' rms of Sersic residual: %.3f mag' % resid_sersic.std()) + +# ---------------------------------------------------------------- growth curve +eint = np.interp(ac, s, e) +area = np.pi * (1 - eint) * bins[1:] ** 2 - np.pi * (1 - eint) * bins[:-1] ** 2 +Lf = np.where(np.isfinite(L), L, 0.0) +# inside A_IN, extrapolate the Sersic fit (the real light there is dust-obscured) +inner = ac < A_IN +Lf[inner] = 10 ** ((MU0 - sersic_mu(ac[inner] * PIXSCALE, *popt)) / 2.5) +cum = np.cumsum(Lf * area) +mtot = -2.5 * np.log10(cum) + ZPTOT +for aa in [200, 400, 600, 800, 1000, 1200]: + i = np.argmin(np.abs(ac - aa)) + print(' total mag within a = %5.0f px (%5.1f arcmin): G = %.3f' + % (aa, ac[i] * PIXSCALE / 60., mtot[i])) +iA = np.argmin(np.abs(ac - A_OUT)) +half = cum[iA] / 2. +re_growth = np.interp(half, cum[:iA + 1], ac[:iA + 1]) * PIXSCALE +print(' half-light radius of the light enclosed within a=%.0f px: %.0f arcsec' + % (A_OUT, re_growth)) +inner_frac = cum[np.argmin(np.abs(ac - A_IN))] / cum[iA] +print(' fraction of that light coming from the extrapolated a < %.0f px: %.3f' + % (A_IN, inner_frac)) + +# ================================================================== plot 1 +fig, ax = plt.subplots(figsize=(9.5, 7.5)) +ok = np.isfinite(L) & (L > 0) +aas = ac * PIXSCALE +# systematic band from the sky pedestal +lo = mu(np.where(L > 0, L, np.nan)) +hi = mu(np.clip(np.where(np.isfinite(L), L, np.nan) - PED, 1e-6, None)) +ax.fill_between(aas[ok], lo[ok], hi[ok], color='tab:orange', alpha=.22, lw=0, + label='sky-pedestal systematic (%+.1f ADU/px)' % PED) +ax.plot(aas[ok], lo[ok], 'k-', lw=1.6, label='luminance, dust+stars masked') +gb = np.isfinite(tB['intens']) & (tB['intens'] > 0) +ax.plot(tB['sma'][gb] * PIXSCALE, mu(tB['intens'][gb]), 'o', ms=4.5, mfc='none', + mec='tab:blue', label='photutils isophote fit') +aa = np.linspace(A_IN * PIXSCALE, 900 * PIXSCALE, 200) +ax.plot(aa, sersic_mu(aa, *popt), '--', color='tab:red', lw=1.4, + label='Sersic n=%.2f, Re=%.0f"' % (nser, re_as)) +for lab, v, c in [('per-pixel 1 sigma', FLOOR['per-pixel sky noise (1 sigma)'], '0.45'), + ('large-scale 1 sigma', FLOOR['large-scale block scatter (1 sigma)'], 'tab:green'), + ('sky-pedestal systematic', FLOOR['sky-pedestal systematic |offset|'], 'tab:red')]: + ax.axhline(v, ls=':', color=c, lw=1.3) + ax.text(1150, v - 0.06, '%s (%.2f)' % (lab, v), color=c, fontsize=8.5, + va='bottom', ha='right') +ax.axvspan(20, A_IN * PIXSCALE, color='k', alpha=.11, lw=0) +ax.text(A_IN * PIXSCALE * 0.92, 17.35, 'dust lane fills the whole azimuth', + fontsize=8.5, ha='right', va='top', rotation=90, color='0.25') +ax.axvline(A_OUT * PIXSCALE, color='0.35', ls='-.', lw=1.2) +ax.text(A_OUT * PIXSCALE * 0.94, 17.35, 'systematic floor reached', fontsize=8.5, + color='0.3', ha='right', va='top', rotation=90) +ax.set_xscale('log') +ax.set_xlim(20, 1200) +ax.set_ylim(27.0, 17.2) +ax.set_xlabel('semi-major axis a [arcsec]') +ax.set_ylabel(r'$\mu$ [mag arcsec$^{-2}$, Gaia $G$ zero point]') +ax.set_title('NGC 5128 luminance surface-brightness profile\n' + '12 x 300 s, iTelescope T32, 0.5376"/px') +sec = ax.secondary_xaxis('top', functions=(lambda v: v * KPC_PER_ARCSEC, + lambda v: v / KPC_PER_ARCSEC)) +sec.set_xlabel('projected radius [kpc, D = 3.8 Mpc]') +ax.grid(alpha=.25) +ax.legend(loc='lower left', fontsize=9, framealpha=.95) +fig.tight_layout() +fig.savefig(path('NGC5128-sb-profile.png'), dpi=150) +plt.close(fig) + +# ================================================================== plot 2 +fig, axs = plt.subplots(3, 1, figsize=(9, 10.5), sharex=True, + gridspec_kw=dict(hspace=.07)) +axs[0].plot(aas[ok], lo[ok], 'k-', lw=1.5) +axs[0].fill_between(aas[ok], lo[ok], hi[ok], color='tab:orange', alpha=.22, lw=0) +axs[0].plot(aa, sersic_mu(aa, *popt), '--', color='tab:red', lw=1.2) +axs[0].set_ylim(27.0, 17.2) +axs[0].set_ylabel(r'$\mu$ [mag arcsec$^{-2}$]') +axs[0].set_title('NGC 5128 isophote fit (luminance; stars and dust lane masked)') + +# Only isophotes whose geometry actually converged are plotted. Inside +# a ~ 240 px the dust lane leaves too little unmasked azimuth and photutils +# holds eps and PA at their previous values: those are not measurements and +# plotting them would look like a flat measured trend. +conv = np.isfinite(tB['eps']) & (tB['stop'] == 0) +A_GEO = float(tB['sma'][conv].min()) * PIXSCALE +for axi, key, kerr, lab in [(axs[1], 'eps', 'eps_err', 'ellipticity $\\epsilon = 1-b/a$'), + (axs[2], 'pa', 'pa_err', 'position angle [deg, CCW from +x]')]: + axi.errorbar(tB['sma'][conv] * PIXSCALE, tB[key][conv], + yerr=np.nan_to_num(tB[kerr][conv]), fmt='o', ms=5.5, + color='tab:blue', capsize=2, label='fit converged (stop code 0)') + axi.set_ylabel(lab) + axi.grid(alpha=.25) + axi.legend(fontsize=8.5, loc='upper left') +axs[1].set_ylim(0.02, 0.29) +axs[2].set_ylim(133, 175) +pax = axs[2].secondary_yaxis('right', functions=(sky_pa, sky_pa)) +pax.set_ylabel('sky position angle [deg E of N]') +for axi in axs: + axi.axvspan(20, A_GEO, color='k', alpha=.11, lw=0) + axi.axvline(A_OUT * PIXSCALE, color='0.35', ls='-.', lw=1.2) +axs[0].axvspan(20, A_IN * PIXSCALE, color='k', alpha=.16, lw=0) +axs[1].text(A_GEO * 0.95, 0.275, 'no converged geometry inside here', fontsize=8.5, + ha='right', va='top', rotation=90, color='0.25') +axs[2].set_xscale('log') +axs[2].set_xlim(20, 1200) +axs[2].set_xlabel('semi-major axis a [arcsec]') +axs[0].grid(alpha=.25) +fig.savefig(path('NGC5128-sb-isophote-geometry.png'), dpi=150, bbox_inches='tight') +plt.close(fig) + +# ================================================================== plot 3 +fig, axs = plt.subplots(2, 1, figsize=(9, 8), sharex=True, + gridspec_kw=dict(hspace=.07, height_ratios=[2, 1])) +for ch, c in [('Luminance', 'k'), ('Red', 'tab:red'), ('Green', 'tab:green'), + ('Blue', 'tab:blue')]: + v = prof[ch] + m = np.isfinite(v) & (v > 0) + axs[0].plot(aas[m], mu(v[m]), color=c, lw=1.4, label=ch) +axs[0].set_ylim(27.5, 17.0) +axs[0].set_ylabel(r'$\mu$ [mag arcsec$^{-2}$]') +axs[0].text(.02, .04, 'the L zero point is applied to every channel, so the ' + 'vertical offsets between R, G and B are arbitrary;' + chr(10) + + 'only the shapes and the colour gradient below are meaningful', + transform=axs[0].transAxes, fontsize=8, color='0.3') +axs[0].legend(fontsize=9) +axs[0].grid(alpha=.25) +axs[0].set_title('NGC 5128 channel profiles on identical isophotes (dust masked)') +br = -2.5 * np.log10(np.where(prof['Blue'] > 0, prof['Blue'], np.nan) / + np.where(prof['Red'] > 0, prof['Red'], np.nan)) +axs[1].plot(aas, br, 'k-', lw=1.5) +axs[1].set_ylabel('instrumental B - R') +axs[1].set_xlabel('semi-major axis a [arcsec]') +axs[1].set_xscale('log') +axs[1].set_xlim(20, 700) +axs[1].grid(alpha=.25) +for axi in axs: + axi.axvspan(20, A_IN * PIXSCALE, color='k', alpha=.11, lw=0) + axi.axvline(A_OUT * PIXSCALE, color='0.35', ls='-.', lw=1.2) +axs[1].set_ylim(-0.45, 0.05) +fig.savefig(path('NGC5128-sb-colour-profile.png'), dpi=150, bbox_inches='tight') +plt.close(fig) + +np.savez(path('_profile.npz'), a=ac, **{k: prof[k] for k in prof}, + ped=PED, blkrms=BLKRMS, sersic=popt, a_in=A_IN, a_out=A_OUT) + +with open(path('sb-derived-quantities.txt'), 'w') as f: + w = lambda t: (f.write(t + '\n'), print(t)) + w('NGC 5128 (Centaurus A) -- derived quantities') + w('=' * 62) + w('Photometry is on the Gaia G scale of the luminance master.') + w(' zero point (r=5 px aperture) %.3f mag' % ZP5) + w(' aperture correction r=5 -> r=25 %+.4f mag' % APCOR) + w(' zero point for total flux %.3f mag' % ZPTOT) + w(' mu = %.3f - 2.5*log10(I / ADU per px)' % MU0) + w(' pixel scale %.4f arcsec, %.5f arcsec^2 per pixel' % (PIXSCALE, PIXAREA)) + w('') + w('Centre') + w(' Gaia/WCS nucleus x=%.1f y=%.1f' % (X0, Y0)) + w(' outer-isophote centre x=%.1f y=%.1f (%.1f px = %.1f arcsec offset)' + % (XC, YC, np.hypot(XC - X0, YC - Y0), np.hypot(XC - X0, YC - Y0) * PIXSCALE)) + w('') + w('Valid radial range') + w(' inner limit a = %.0f px = %.0f arcsec (dust lane fills the azimuth inside)' + % (A_IN, A_IN * PIXSCALE)) + w(' outer limit a = %.0f px = %.0f arcsec = %.1f arcmin' % + (A_OUT, A_OUT * PIXSCALE, A_OUT * PIXSCALE / 60)) + w(' the nucleus is NOT saturated: peak star-free galaxy signal 2498 ADU/px') + w(' vs a clip level of ~63000 ADU/px (factor 25 margin)') + w('') + w('Noise and systematic floors [mag/arcsec^2]') + for k, v in FLOOR.items(): + w(' %-40s %.2f' % (k, v)) + w(' far-field pedestal %+.2f ADU/px: the sky plane absorbed halo light' % PED) + w('') + w('Sersic fit, %.0f-%.0f arcsec' % (x.min(), x.max())) + w(' n = %.2f +- %.2f' % (nser, perr[2])) + w(' Re = %.1f +- %.1f arcsec = %.2f +- %.2f kpc' + % (re_as, perr[1], re_as * KPC_PER_ARCSEC, perr[1] * KPC_PER_ARCSEC)) + w(' mu_e = %.2f +- %.2f mag/arcsec^2' % (mue, perr[0])) + w(' rms of fit residual %.3f mag' % resid_sersic.std()) + w('') + w('Integrated light (elliptical apertures on the measured profile)') + for aa2 in [200, 400, 600, 800, 1000, 1200]: + i = np.argmin(np.abs(ac - aa2)) + w(' a < %5.0f px (%5.2f arcmin): G = %.3f' % (aa2, ac[i] * PIXSCALE / 60., mtot[i])) + w(' half-light radius of the light within a=%.0f px: %.0f arcsec (%.2f kpc)' + % (A_OUT, re_growth, re_growth * KPC_PER_ARCSEC)) + w(' fraction from the extrapolated a<%.0f px core: %.1f%%' % (A_IN, 100 * inner_frac)) + w('') + w('Ellipticity / position angle trend (converged isophotes only)') + for i in np.nonzero(conv)[0]: + w(' a = %6.1f px (%6.1f") : eps = %.3f +- %.3f PA = %5.1f +- %.1f deg' + % (tB['sma'][i], tB['sma'][i] * PIXSCALE, tB['eps'][i], + tB['eps_err'][i], tB['pa'][i], tB['pa_err'][i])) +print('\nwrote NGC5128-sb-profile.png, NGC5128-sb-isophote-geometry.png, NGC5128-sb-colour-profile.png, ' + 'sb-derived-quantities.txt') diff --git a/pipeline/sb_render_tail.py b/pipeline/sb_render_tail.py new file mode 100644 index 0000000..0137c0f --- /dev/null +++ b/pipeline/sb_render_tail.py @@ -0,0 +1,163 @@ +"""Render section of sb_residual.py (imported and executed by it). + +Three views of the same residual, each answering a different question: + + (1) raw residual -- how well does the ellipse model fit? + (2) plane-removed residual -- restores the sky pedestal that the + stacking plane fit swallowed + (3) azimuthal-median-subtracted -- the shell-hunting view. Subtracting + the residual's own median as a + function of a forces zero mean at + every radius, so ONLY azimuthal + structure survives. Any perfectly + circular feature is removed with it. +""" +import numpy as np +from astropy.io import fits +import matplotlib.pyplot as plt +from scipy.ndimage import gaussian_filter, binary_erosion +from sb_common import * + + +def run(ns): + g = ns # namespace dict from sb_residual + res, model, a_map = g['res'], g['model'], g['a_map'] + L, star, dust = g['L'], g['star'], g['dust'] + XC, YC, PEDL = g['XC'], g['YC'], g['PEDL'] + A_OUT, tab, binned = g['A_OUT'], g['tab'], g['binned'] + + # -- plane removal (restores the sky pedestal absorbed by the stacking fit) + yy, xx = np.mgrid[0:3194, 0:4788] + fitreg = (a_map > 1450) & ~star & ~dust + A = np.c_[np.ones(fitreg.sum()), xx[fitreg].ravel()/1000., yy[fitreg].ravel()/1000.] + coef, *_ = np.linalg.lstsq(A, res[fitreg].ravel(), rcond=None) + plane = (coef[0] + coef[1]*xx/1000. + coef[2]*yy/1000.).astype(np.float32) + print('residual plane removed: %+.2f %+.2f*x/1000 %+.2f*y/1000 ADU/px' + % tuple(coef)) + resf = (res - plane).astype(np.float32) + fits.PrimaryHDU(resf).writeto(path('sb-residual-flat.fits'), overwrite=True) + del xx, yy, plane, A + + # -- binned maps + r4 = binned(resf, star, 4) + r16 = binned(resf, star, 16) + a16 = binned(a_map, np.zeros_like(star), 16) + d16 = binned(dust.astype(np.float32), np.zeros_like(star), 16) + + # -- azimuthal median removal + abin = np.geomspace(20, 3000, 80) + ibn = np.digitize(a16, abin) + azmed = np.full(len(abin)-1, np.nan) + for k in range(1, len(abin)): + m = (ibn == k) & np.isfinite(r16) & (d16 < 0.3) + if m.sum() >= 12: + azmed[k-1] = np.median(r16[m]) + acb = np.sqrt(abin[1:]*abin[:-1]) + gm = np.isfinite(azmed) + resd = r16 - np.interp(a16, acb[gm], azmed[gm]) + sig = float(np.nanstd(resd[np.isfinite(resd) & (a16 > 1500)])) + smd = gaussian_filter(np.nan_to_num(resd), 1.0) + smd[~np.isfinite(resd)] = np.nan + np.save(path('_resd16.npy'), resd) + np.save(path('_a16.npy'), a16) + np.save(path('_s16.npy'), np.array([sig])) + print('deep-residual noise (16x16 bins, a>1500 px): %.2f ADU/px -> mu %.2f' + % (sig, mu(sig))) + + NV, EV = north_east_pixel() + CUT = 1560 + sl = (slice(int(YC)-CUT, int(YC)+CUT), slice(int(XC)-CUT, int(XC)+CUT)) + ext = [-CUT*PIXSCALE/60, CUT*PIXSCALE/60]*2 + fullext = [-4788/2*PIXSCALE/60, 4788/2*PIXSCALE/60, + -3194/2*PIXSCALE/60, 3194/2*PIXSCALE/60] + + def compass(ax, x=0.885, y=0.115, Ln=0.07, c='k'): + for v, lab in [(NV, 'N'), (EV, 'E')]: + ax.annotate('', xy=(x+Ln*v[0], y+Ln*v[1]), xytext=(x, y), + xycoords='axes fraction', textcoords='axes fraction', + arrowprops=dict(arrowstyle='->', color=c, lw=1.4)) + ax.annotate(lab, xy=(x+1.45*Ln*v[0], y+1.45*Ln*v[1]), color=c, + xycoords='axes fraction', ha='center', va='center', + fontsize=10) + + def cutb(arr, B): + return arr[int((YC-CUT)/B):int((YC+CUT)/B), int((XC-CUT)/B):int((XC+CUT)/B)] + + # ---------------------------------------------------------- 4-panel figure + fig, axs = plt.subplots(2, 2, figsize=(15.5, 15.0)) + axs = axs.ravel() + axs[0].imshow(np.arcsinh(np.clip(L[sl]-PEDL, 0, None)/25), origin='lower', + cmap='gray', extent=ext) + axs[0].set_title('(a) luminance master, arcsinh stretch') + axs[1].imshow(np.arcsinh(np.clip(model[sl], 0, None)/25), origin='lower', + cmap='gray', extent=ext) + axs[1].set_title('(b) smooth elliptical model built from the isophotes') + im = axs[2].imshow(cutb(r4, 4), origin='lower', cmap='RdBu_r', vmin=-220, + vmax=220, extent=ext) + axs[2].set_title('(c) residual, 4x4 binned: the dust lane dominates') + plt.colorbar(im, ax=axs[2], fraction=.046, label='ADU/px') + im = axs[3].imshow(cutb(smd, 16), origin='lower', cmap='RdBu_r', + vmin=-3*sig, vmax=3*sig, extent=ext) + axs[3].contour(cutb(d16, 16), levels=[0.5], colors='0.35', linewidths=.8, + extent=ext, origin='lower') + axs[3].set_title('(d) residual, 16x16 binned, azimuthal median removed, ' + '+-3 sigma' + chr(10) + + '1 sigma = %.2f ADU/px = %.1f mag/arcsec2 ' + '(grey outline = dust mask)' % (sig, mu(sig))) + plt.colorbar(im, ax=axs[3], fraction=.046, label='ADU/px') + for a in axs: + a.set_xlabel('arcmin') + a.set_ylabel('arcmin') + compass(a, c='w' if a in (axs[0], axs[1]) else 'k') + fig.suptitle('NGC 5128: smooth elliptical model and its residual', fontsize=14) + fig.tight_layout() + fig.savefig(path('NGC5128-sb-model-residual.png'), dpi=115) + plt.close(fig) + + # ------------------------------------------------------- deep single panel + fig, ax = plt.subplots(figsize=(13.5, 9.6)) + im = ax.imshow(smd, origin='lower', cmap='RdBu_r', vmin=-3*sig, vmax=3*sig, + extent=fullext) + ax.contour(d16, levels=[0.5], colors='0.3', linewidths=.9, extent=fullext, + origin='lower') + th = np.linspace(0, 2*np.pi, 400) + ax.plot(1250*np.cos(th)*PIXSCALE/60, 1000*np.sin(th)*PIXSCALE/60, 'k--', + lw=1.1, alpha=.7, label='sky-plane fit exclusion ellipse (1250x1000 px)') + ax.plot(A_OUT*np.cos(th)*PIXSCALE/60, A_OUT*0.765*np.sin(th)*PIXSCALE/60, + '-', color='0.25', lw=1.1, alpha=.85, + label='profile reliability limit, a = %.0f px' % A_OUT) + ax.plot([], [], '-', color='0.3', lw=.9, label='dust-lane mask') + ax.legend(fontsize=9, loc='lower left', framealpha=.9) + plt.colorbar(im, ax=ax, fraction=.035, label='residual [ADU/px]') + ax.set_xlabel('arcmin') + ax.set_ylabel('arcmin') + compass(ax, x=0.945, y=0.84, Ln=0.05) + ax.set_title('NGC 5128: isophote model AND the residual azimuthal median ' + 'removed' + chr(10) + + '16x16 binned (8.6"/bin), stars masked, +-3 sigma; only ' + 'azimuthal structure survives. 1 sigma = %.2f ADU/px = ' + '%.1f mag/arcsec2' % (sig, mu(sig))) + fig.tight_layout() + fig.savefig(path('NGC5128-sb-residual-deep.png'), dpi=125) + plt.close(fig) + + # ------------------------------------------------------ quantify structure + print('') + print('azimuthal residual statistics (16x16 bins, azimuthal median removed,') + print('dust-lane bins excluded):') + ok = np.isfinite(resd) & (d16 < 0.3) + for lo, hi in [(100, 200), (200, 400), (400, 600), (600, 800), (800, 1000), + (1000, 1250), (1250, 1600), (1600, 2200)]: + m = ok & (a16 >= lo) & (a16 < hi) + if m.sum() < 20: + continue + md = np.interp(np.clip(a16[m], tab['sma'][0], tab['sma'][-1]), + tab['sma'], tab['intens']) + print(' a=%4d-%4d px (%4.1f-%4.1f arcmin): rms %6.2f ADU/px = %4.1f%% ' + 'of the model, max |dev| %4.1f sigma, n=%d' + % (lo, hi, lo*PIXSCALE/60, hi*PIXSCALE/60, np.nanstd(resd[m]), + 100*np.nanstd(resd[m])/np.mean(md), + np.nanmax(np.abs(resd[m]))/sig, m.sum())) + print('') + print('wrote sb-model.fits, sb-residual.fits, sb-residual-flat.fits,') + print(' NGC5128-sb-model-residual.png, NGC5128-sb-residual-deep.png') diff --git a/pipeline/sb_residual.py b/pipeline/sb_residual.py new file mode 100644 index 0000000..5f2f706 --- /dev/null +++ b/pipeline/sb_residual.py @@ -0,0 +1,77 @@ +"""Step 4: smooth elliptical model, model-subtracted residual, and renders. + +The model is the Pass B isophote table (stars and dust lane masked) turned into +a 2-D image with sb_model.build, with a Sersic extrapolation inside a = 62 px +where no dust-free azimuth exists. Subtracting it leaves everything that is +not a smooth ellipse: the dust lane, foreground stars, and any shell, tidal +feature or halo asymmetry. + +Outputs + sb-model.fits the smooth model + sb-residual.fits luminance minus model + NGC5128-sb-model-residual.png 4-panel: data / model / residual / binned deep residual + NGC5128-sb-residual-deep.png heavily binned residual alone, for shell hunting +""" +import numpy as np +from astropy.io import fits +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt +from scipy.ndimage import gaussian_filter +from sb_common import * +import sb_model + +XC, YC = np.load(path('_geom.npy')) +P = np.load(path('_profile.npz')) +tB = dict(np.load(path('_isoB.npz'))) +star = fits.getdata(path('sb-mask-stars.fits')).astype(bool) +dust = fits.getdata(path('sb-mask-dust.fits')).astype(bool) +A_IN, A_OUT = float(P['a_in']), float(P['a_out']) +mue, re_as, nser = P['sersic'] + + +def sersic_mu(a_as): + bn = 2 * nser - 1 / 3. + 0.009876 / nser + return mue + 2.5 * bn / np.log(10) * ((a_as / re_as) ** (1. / nser) - 1.) + + +# ---- build a profile that is defined at every radius ----------------------- +ac = P['a'] +Lp = P['Luminance'].copy() +inner = ac < A_IN +Lp[inner] = 10 ** ((MU0 - sersic_mu(ac[inner] * PIXSCALE)) / 2.5) +okp = np.isfinite(Lp) & (ac < 1500) +tab = dict(sma=ac[okp], intens=Lp[okp], + eps=np.interp(ac[okp], tB['sma'], np.where(np.isfinite(tB['eps']), + tB['eps'], 0.15)), + pa=np.interp(ac[okp], tB['sma'], np.where(np.isfinite(tB['pa']), + tB['pa'], 150.))) +model, a_map = sb_model.build((3194, 4788), XC, YC, tab, block=2) +fits.PrimaryHDU(model).writeto(path('sb-model.fits'), overwrite=True) + +PEDL = float(P['ped']) +L = load('Luminance') +res = (L - model).astype(np.float32) +fits.PrimaryHDU(res).writeto(path('sb-residual.fits'), overwrite=True) +print('model built; residual rms inside a<600 px: %.2f ADU/px' + % res[(a_map < 600) & ~star & ~dust].std()) + + +def binned(img, mask, B): + """Masked block mean, returning NaN where a block is mostly masked.""" + H, W = img.shape + h, w = H // B, W // B + a = img[:h * B, :w * B].reshape(h, B, w, B) + m = (~mask)[:h * B, :w * B].reshape(h, B, w, B) + n = m.sum(axis=(1, 3)) + s = np.where(m, a, 0).sum(axis=(1, 3)) + return np.where(n > 0.35 * B * B, s / np.maximum(n, 1), np.nan) + + + + +# ------------------------------------------------------------------ renders +import sb_render_tail +sb_render_tail.run(dict(res=res, model=model, a_map=a_map, L=L, star=star, + dust=dust, XC=XC, YC=YC, PEDL=PEDL, A_OUT=A_OUT, + tab=tab, binned=binned)) diff --git a/pipeline/solve.py b/pipeline/solve.py new file mode 100644 index 0000000..a91e452 --- /dev/null +++ b/pipeline/solve.py @@ -0,0 +1,185 @@ +"""Pass 3: plate solve the luminance master and copy the WCS to every master. + +iTelescope's calibrated frames arrive with a PinPoint HISTORY line but no WCS +keywords at all, so the astrometry has to be redone locally. A blind solve is +not needed: the header gives the pointing to arcminutes and the plate scale to +four figures, so this fetches a Gaia DR3 catalogue for that patch of sky and +matches it to the detected stars. + +The match itself is asterism-based (astroalign), which is invariant to rotation +and scale, so the roll angle never has to be guessed. It is NOT invariant to a +mirror flip, so both parities are tried and the one that matches wins. The +final WCS is a least-squares TAN fit to the matched pairs, and the residual it +reports is the honest measure of whether the solve is real. +""" +import os +import sys + +import astroalign as aa +import numpy as np +import sep +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.wcs import WCS +from astropy.wcs.utils import fit_wcs_from_points + +import layout + +SRC = layout.SESSION +OUT = layout.SESSION +MASTERS = ["Luminance", "Red", "Green", "Blue"] +CATCACHE = layout.path("_gaia.npz") + + +def detect(image, nmax=600): + bkg = sep.Background(image, bw=64, bh=64, fw=3, fh=3) + sub = image - bkg.back() + objs = sep.extract(sub, 8.0, err=bkg.globalrms, minarea=9, + deblend_cont=0.005) + objs = objs[(objs["flag"] == 0) & (objs["npix"] > 12) & + (objs["npix"] < 3000)] + objs = objs[np.argsort(objs["flux"])[::-1][:nmax]] + return np.column_stack([objs["x"], objs["y"]]), objs["flux"] + + +def gaia_catalogue(ra_deg, dec_deg, radius_deg, nmax=600): + """Gaia DR3 sources around the pointing, brightest first, cached to disk.""" + if os.path.exists(CATCACHE): + z = np.load(CATCACHE) + print(f"catalogue: {len(z['ra'])} cached Gaia sources") + return z["ra"], z["dec"], z["g"] + from astroquery.gaia import Gaia + + Gaia.ROW_LIMIT = nmax + query = f""" + SELECT TOP {nmax} ra, dec, phot_g_mean_mag + FROM gaiadr3.gaia_source + WHERE 1 = CONTAINS(POINT('ICRS', ra, dec), + CIRCLE('ICRS', {ra_deg}, {dec_deg}, {radius_deg})) + AND phot_g_mean_mag IS NOT NULL + ORDER BY phot_g_mean_mag ASC + """ + tbl = Gaia.launch_job_async(query).get_results() + ra = np.asarray(tbl["ra"], dtype=float) + dec = np.asarray(tbl["dec"], dtype=float) + g = np.asarray(tbl["phot_g_mean_mag"], dtype=float) + np.savez_compressed(CATCACHE, ra=ra, dec=dec, g=g) + print(f"catalogue: {len(ra)} Gaia DR3 sources, G {g.min():.1f}-{g.max():.1f}") + return ra, dec, g + + +def project(ra, dec, ra0, dec0, scale_arcsec, parity): + """Gnomonic projection to pixel-like coordinates for asterism matching.""" + c = SkyCoord(ra * u.deg, dec * u.deg) + centre = SkyCoord(ra0 * u.deg, dec0 * u.deg) + dx, dy = centre.spherical_offsets_to(c) + x = dx.to_value(u.arcsec) / scale_arcsec * parity + y = dy.to_value(u.arcsec) / scale_arcsec + return np.column_stack([x, y]) + + +def main(): + path = layout.path("master-Luminance.fit") + with fits.open(path) as hd: + image = hd[0].data.astype(np.float32) + hdr = hd[0].header + ny, nx = image.shape + + centre = SkyCoord(hdr["OBJCTRA"], hdr["OBJCTDEC"], + unit=(u.hourangle, u.deg)) + scale = float(hdr["HIERARCH iTelescopePlateScaleH"]) + radius = 1.15 * 0.5 * np.hypot(nx, ny) * scale / 3600.0 + print(f"pointing {centre.to_string('hmsdms')} scale {scale:.4f}\"/px " + f"search radius {radius:.3f} deg") + + xy, flux = detect(image) + print(f"detected {len(xy)} stars in the luminance master") + + ra, dec, gmag = gaia_catalogue(centre.ra.deg, centre.dec.deg, radius) + + best = None + for parity in (-1.0, 1.0): + cat_xy = project(ra, dec, centre.ra.deg, centre.dec.deg, scale, parity) + try: + tform, (src, dst) = aa.find_transform(cat_xy, xy) + except Exception as exc: # noqa: BLE001 + print(f" parity {parity:+.0f}: no match ({exc})") + continue + print(f" parity {parity:+.0f}: matched {len(src)} stars, " + f"rotation {np.degrees(tform.rotation):.3f} deg, " + f"scale {tform.scale:.5f}") + if best is None or len(src) > best[0]: + best = (len(src), parity, src, dst) + if best is None: + sys.exit("plate solve failed: no asterism match in either parity") + + nmatch, parity, cat_pts, img_pts = best + # Recover which catalogue rows were matched so the fit uses sky coordinates + # rather than the projected proxy. + cat_xy = project(ra, dec, centre.ra.deg, centre.dec.deg, scale, parity) + idx = [int(np.argmin(np.hypot(cat_xy[:, 0] - px, cat_xy[:, 1] - py))) + for px, py in cat_pts] + world = SkyCoord(ra[idx] * u.deg, dec[idx] * u.deg) + + wcs = fit_wcs_from_points((img_pts[:, 0], img_pts[:, 1]), world, + proj_point="center", projection="TAN") + pred = wcs.world_to_pixel(world) + resid = np.hypot(pred[0] - img_pts[:, 0], pred[1] - img_pts[:, 1]) + print(f"seed fit on {nmatch} stars: residual median {np.median(resid):.2f} px " + f"({np.median(resid) * scale:.2f}\"), max {resid.max():.2f} px") + + # The asterism match only ever returns a handful of stars. Now that an + # approximate solution exists, every catalogue source can be pushed through + # it and paired with the nearest detection, which grows the fit from a + # dozen stars to hundreds and averages down the centroid noise. Two passes + # with a shrinking tolerance is enough to converge. + all_world = SkyCoord(ra * u.deg, dec * u.deg) + for tol in (4.0, 2.0): + px, py = wcs.world_to_pixel(all_world) + pairs = [] + for i, (cx, cy) in enumerate(zip(px, py)): + if not (0 <= cx < nx and 0 <= cy < ny): + continue + d = np.hypot(xy[:, 0] - cx, xy[:, 1] - cy) + j = int(np.argmin(d)) + if d[j] <= tol: + pairs.append((i, j)) + if len(pairs) < 20: + print(f" refine (tol {tol} px): only {len(pairs)} pairs, kept seed") + break + ci = np.array([p[0] for p in pairs]) + ii = np.array([p[1] for p in pairs]) + wcs = fit_wcs_from_points((xy[ii, 0], xy[ii, 1]), all_world[ci], + proj_point="center", projection="TAN") + qx, qy = wcs.world_to_pixel(all_world[ci]) + resid = np.hypot(qx - xy[ii, 0], qy - xy[ii, 1]) + nmatch = len(pairs) + print(f" refine (tol {tol} px): {nmatch} stars, residual median " + f"{np.median(resid):.2f} px ({np.median(resid) * scale:.2f}\"), " + f"max {resid.max():.2f} px") + + cen = wcs.pixel_to_world(nx / 2.0, ny / 2.0) + cd = wcs.pixel_scale_matrix * 3600.0 + solved_scale = np.sqrt(abs(np.linalg.det(cd))) + rot = np.degrees(np.arctan2(cd[0, 1], cd[1, 1])) + print(f"field centre {cen.to_string('hmsdms')}") + print(f"solved scale {solved_scale:.4f}\"/px, position angle {rot:.2f} deg") + print(f"field of view {nx * solved_scale / 60:.1f}' x " + f"{ny * solved_scale / 60:.1f}'") + + whdr = wcs.to_header() + for name in MASTERS: + p = layout.path(f"master-{name}.fit") + with fits.open(p, mode="update") as hd: + for card in whdr.cards: + hd[0].header[card.keyword] = (card.value, card.comment) + hd[0].header["ASTRSOLV"] = ( + f"Gaia DR3 / {nmatch} stars / {np.median(resid) * scale:.2f} arcsec", + "local plate solution") + hd.flush() + print(f" WCS written to {os.path.basename(p)}") + + +if __name__ == "__main__": + main() diff --git a/pipeline/stack.py b/pipeline/stack.py new file mode 100644 index 0000000..89c055a --- /dev/null +++ b/pipeline/stack.py @@ -0,0 +1,166 @@ +"""Pass 2: register every frame to one reference and combine per filter. + +One reference frame is used for ALL four filters, not one per filter, so the +four masters come out pixel-aligned and the colour composite needs no further +registration. + +Registration works on the cached star lists rather than the pixels: astroalign +matches asterisms between the two point sets and returns a similarity transform, +which is then applied to the image with a bicubic warp. Matching a few hundred +coordinates is far cheaper than cross-correlating 15 Mpx frames, and it copes +with the field rotation between the east and west sides of the meridian. + +Before combining, each frame is sky-subtracted and then rescaled so that its +bright-signal level (the 99.5th percentile, which on this field is set by stars +and the galaxy core rather than by sky) matches the group median. That corrects +for transparency changes without needing per-star photometry. A +sigma-clipped mean then rejects cosmic rays and satellite trails; with only four +frames per colour the clip is deliberately gentle (3 sigma, one iteration) so it +does not start eating real signal. +""" +import os + +import astroalign as aa +import numpy as np +from astropy.io import fits +from astropy.stats import sigma_clip +from skimage.transform import warp + +import layout + +SRC = layout.SESSION +OUT = layout.SESSION +CACHE = layout.path("_stars.npz") + +# Best-seeing luminance frame, chosen from the pass-1 FWHM table. +REFERENCE = "Luminance_002" +FILTERS = ["Luminance", "Red", "Green", "Blue"] + +aa.MIN_MATCHES_FRACTION = 0.6 +aa.NUM_NEAREST_NEIGHBORS = 8 + + +def load_cache(): + z = np.load(CACHE, allow_pickle=True) + meta = {row[0]: row for row in z["meta"]} + stars = {k: z[k + "_xy"] for k in meta} + return meta, stars, z + + +def frame_path(fname): + return layout.path(fname) + + +def combine(cube, weights): + """Weighted sigma-clipped mean along axis 0, done row-block by row-block. + + The full cube is already in memory; the blocking here is only to keep the + boolean mask and the float64 intermediates that sigma_clip allocates from + tripling peak usage on a machine with a few GB free. + """ + n, ny, nx = cube.shape + out = np.zeros((ny, nx), dtype=np.float32) + w = np.asarray(weights, dtype=np.float32)[:, None, None] + step = 256 + for y0 in range(0, ny, step): + y1 = min(y0 + step, ny) + block = cube[:, y0:y1, :] + clipped = sigma_clip(block, sigma=3.0, maxiters=1, axis=0, + masked=True, copy=True) + # Warped frames carry NaN outside their footprint; those pixels must + # drop out of both the sum and the weight total, exactly like a + # clipped outlier. + finite = np.isfinite(block) + good = (~clipped.mask) & finite + vals = np.where(finite, block, 0.0) + wb = np.broadcast_to(w, block.shape) * good + denom = wb.sum(axis=0) + denom[denom == 0] = np.nan + out[y0:y1, :] = np.nansum(vals * wb, axis=0) / denom + del block, clipped, good, finite, vals, wb, denom + return np.nan_to_num(out, nan=0.0) + + +def main(): + meta, stars, _ = load_cache() + ref_xy = stars[REFERENCE] + ref_row = meta[REFERENCE] + print(f"reference {REFERENCE} ({ref_row[1]})") + os.makedirs(OUT, exist_ok=True) + + report = [] + for filt in FILTERS: + keys = sorted(k for k in meta if meta[k][2] == filt) + planes, weights, headers, used = [], [], [], [] + for key in keys: + row = meta[key] + fname, rms = row[1], float(row[7]) + with fits.open(frame_path(fname), memmap=False) as hd: + data = hd[0].data.astype(np.float32) + hdr = hd[0].header + if key == REFERENCE: + reg = data + nmatch = len(ref_xy) + else: + try: + tform, (src_m, tgt_m) = aa.find_transform(stars[key], + ref_xy) + except Exception as exc: # noqa: BLE001 + print(f" {key}: registration FAILED ({exc}) - dropped") + del data + continue + nmatch = len(src_m) + reg = warp(data, inverse_map=tform.inverse, order=3, + mode="constant", cval=np.nan, + preserve_range=True).astype(np.float32) + del data + # Sky subtraction uses the frame's own median, which on a field + # this empty outside the galaxy is a fair estimate of sky level. + sky = float(np.nanmedian(reg)) + reg -= sky + planes.append(reg) + weights.append(1.0 / (rms * rms)) + headers.append(hdr) + used.append((key, nmatch, sky, rms)) + print(f" {key:16s} matched={nmatch:3d} sky={sky:8.1f} " + f"shift-corrected") + + cube = np.stack(planes, axis=0) + del planes + # Photometric scaling: normalise each frame to the stack's own median + # signal so a frame taken through thin cloud cannot drag the mean down. + levels = np.array([np.nanpercentile(p, 99.5) for p in cube]) + ref_level = np.nanmedian(levels) + for i, lv in enumerate(levels): + if lv > 0: + cube[i] *= float(ref_level / lv) + print(f" {filt}: photometric scale factors " + f"{np.round(ref_level / levels, 4)}") + + master = combine(cube, weights) + nframes = cube.shape[0] + del cube + + hdr = headers[0].copy() + for k in ("CBLACK", "CWHITE", "PEDESTAL", "HISTORY"): + hdr.remove(k, ignore_missing=True, remove_all=True) + hdr["FILTER"] = filt + hdr["NCOMBINE"] = (nframes, "frames in this master") + hdr["EXPTOTAL"] = (300.0 * nframes, "[s] total integration") + hdr["STACKREF"] = (REFERENCE, "registration reference frame") + hdr["STACKALG"] = ("sigma-clipped weighted mean", "combine method") + hdr["IMAGETYP"] = "Master Light" + out = layout.path(f"master-{filt}.fit") + fits.PrimaryHDU(master.astype(np.float32), hdr).writeto(out, + overwrite=True) + print(f" -> {out} ({nframes} x 300 s = {nframes * 5:.0f} min)") + report.append((filt, nframes, out)) + del master + + print("\nmasters written:") + for filt, n, path in report: + print(f" {filt:10s} {n:2d} frames {path}") + + +if __name__ == "__main__": + main() diff --git a/pipeline/starless.py b/pipeline/starless.py new file mode 100644 index 0000000..5cfb103 --- /dev/null +++ b/pipeline/starless.py @@ -0,0 +1,91 @@ +"""Split the finished image into a starless view and a star-only layer. + +The first attempt used a greyscale morphological opening, the classical trick, +and it failed in a way worth recording: an opening replaces each pixel with a +local minimum, so on a noisy sky it does not merely delete the star, it digs a +hole slightly BELOW sky level and leaves a black dot behind. It also left the +brightest stars standing, because their wings are wider than any structuring +element small enough to spare the galaxy. + +What works instead is catalogue-and-fill. The stars to remove are taken from +Gaia DR3 rather than from a blind detection: Gaia is essentially complete for +stars down to G = 20, which is fainter than anything visible here, so the +catalogue identifies the foreground almost perfectly. That matters because a +blind detector cannot tell a Milky Way star from one of Centaurus A's own +globular clusters or from the blue knots in its dust lane, and removing those +guts the very structure the image is about. Each Gaia star is painted into a +mask whose radius follows its magnitude, and the masked pixels are +replaced by a normalised convolution - a Gaussian blur of the unmasked pixels +divided by the same blur of the mask itself, which interpolates across each +star using only real neighbouring pixels. That follows the galaxy's gradient +instead of flattening it, and leaves no holes. + +The star layer is then simply what was removed. Two honest limitations: the +extended halos and diffraction spikes of the very brightest stars reach beyond +any sensible mask radius and leave soft residual glows behind, and anything +Gaia does not list (background galaxies, the cluster system) stays in the +starless frame by design. This is a presentation tool, not a measurement. +""" +import os + +import numpy as np +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.wcs import WCS +from PIL import Image +from scipy.ndimage import gaussian_filter + +import layout + +OUT = layout.SESSION +SOURCE = "NGC5128-img-deconvolved.png" +FILL_SIGMA = 9.0 # interpolation scale for the normalised convolution + +rgb = np.asarray(Image.open(layout.path(SOURCE))).astype(np.float32) / 255 +ny, nx = rgb.shape[:2] +print(f"{SOURCE}: {nx} x {ny}") + +z = np.load(layout.path("_gaia_deep.npz")) +with fits.open(layout.path("NGC5128-LRGB.fit")) as hd: + wcs = WCS(hd[0].header, naxis=2) +gx, gy = wcs.world_to_pixel(SkyCoord(z["ra"] * u.deg, z["dec"] * u.deg)) +gmag = z["g"] +on_frame = (gx > -30) & (gx < nx + 30) & (gy > -30) & (gy < ny + 30) +gx, gy, gmag = gx[on_frame], gy[on_frame], gmag[on_frame] +print(f"{len(gx)} Gaia stars on the frame, G {gmag.min():.1f}-{gmag.max():.1f}") + +mask = np.zeros((ny, nx), np.float32) +R = 30 +yy, xx = np.mgrid[-R:R + 1, -R:R + 1] +rr = np.hypot(xx, yy) +for x, y, g in zip(gx, gy, gmag): + # Radius from magnitude: a bright star's visible disc and wings are far + # wider than a faint one's, and a fixed radius would either miss the + # bright ones or scour the field around the faint ones. + r = float(np.clip(30.0 - 1.9 * (g - 8.0), 4, R - 2)) + cx, cy = int(round(x)), int(round(y)) + x0, x1 = max(0, cx - R), min(nx, cx + R + 1) + y0, y1 = max(0, cy - R), min(ny, cy + R + 1) + patch = (rr <= r).astype(np.float32)[(y0 - cy + R):(y1 - cy + R), + (x0 - cx + R):(x1 - cx + R)] + np.maximum(mask[y0:y1, x0:x1], patch, out=mask[y0:y1, x0:x1]) +mask = np.clip(gaussian_filter(mask, 1.5), 0, 1) +print(f"mask covers {mask.mean():.2%} of the frame") + +starless = np.empty_like(rgb) +keep = 1.0 - mask +denom = gaussian_filter(keep, FILL_SIGMA) +denom[denom < 1e-3] = 1e-3 +for i in range(3): + filled = gaussian_filter(rgb[:, :, i] * keep, FILL_SIGMA) / denom + starless[:, :, i] = rgb[:, :, i] * keep + filled * mask +stars = np.clip(rgb - starless, 0.0, 1.0) + +Image.fromarray((np.clip(starless, 0, 1) * 255 + 0.5).astype(np.uint8)).save( + layout.path("NGC5128-img-starless.png")) +# The star layer is faint on its own; a square-root stretch makes it legible +# without adding or removing anything. +Image.fromarray((np.sqrt(stars) * 255 + 0.5).astype(np.uint8)).save( + layout.path("NGC5128-img-stars.png")) +print("wrote NGC5128-img-starless.png and NGC5128-img-stars.png") diff --git a/pipeline/subdir_readmes.py b/pipeline/subdir_readmes.py new file mode 100644 index 0000000..08ab53d --- /dev/null +++ b/pipeline/subdir_readmes.py @@ -0,0 +1,117 @@ +"""Drop a short README in each session subdirectory. + +Each one says what the directory holds, where it came from, and whether it is +safe to delete - the three things someone opening a folder cold needs. +""" +import os + +ROOT = r"C:\Users\lhorrocks-barlow\Downloads\NGC5128\20260721" + +READMES = { + "raw": """# raw/ + +Exactly what iTelescope delivered, untouched. + +- `calibrated-*.zip` - the calibrated frames, as archives (FITS and TIFF) +- `raw-*.zip` - the uncalibrated frames, kept for completeness +- `jpeg-*.jpg` - iTelescope's own preview of each sub + +**Do not edit anything here.** This is the archive copy; everything else in the +session can be rebuilt from it. `calibrated/` holds these same frames +uncompressed. +""", + "calibrated": """# calibrated/ + +The 24 calibrated subs, uncompressed from `raw/`. 12 Luminance, 4 Red, +4 Green, 4 Blue, all 300 s, all BIN2. + +`.fit` are the data the processing uses; `.tif` are the same frames in a form +other software can open. + +Already bias, dark and flat corrected by iTelescope before delivery +(`CALSTAT = 'BDF'` in the headers) - that cannot be undone here. + +Safe to delete: regenerable from `raw/` by re-running `unzip.py`. +""", + "stacks": """# stacks/ + +The combined frames, at two levels of processing. + +- `masters/` - the real stacks: registered, transparency-normalised, combined + with outlier rejection, and plate-solved. Everything downstream uses these. +- `original/` - the baseline: aligned and averaged, and **nothing else**. No + rejection, no sky subtraction, no normalisation. Satellite trails and the + moon's gradient are still in it, deliberately. Useful when a processed result + looks odd and you need to know whether the data or the processing caused it. + +Both sets are pixel-aligned to the same reference frame, so they can be +compared or subtracted directly. +""", + "final": """# final/ + +**The finished images.** Four renderings of the same data, differing only in +how much processing was applied, plus a side-by-side comparison. + +| File | What it is | +|---|---| +| `NGC5128-final-1-stacked` | stacked and stretched, nothing else | +| `NGC5128-final-2-processed` | conventional processing | +| `NGC5128-final-3-best` | corrected by what the analyses established | +| `NGC5128-final-4-closeup` | image 3 cropped to the galaxy, 23.4' square | +| `NGC5128-final-comparison.jpg` | all three side by side, with a 1:1 crop | + +Each comes as `.png` (viewing), `.tif` (16-bit, for editing or printing) and +`-preview.jpg` (small). + +The reasoning behind each is in `../METHODS.md`. +""", + "renderings": """# renderings/ + +Finished images that are not the four deliverables in `final/`. + +- `NGC5128-LRGB.*` - the first full composite +- `NGC5128-LRGB-hdr.*` - the same with the core recovered by a second tone curve +- `NGC5128-img-deconvolved.*` - sharpened luminance +- `NGC5128-img-annotated.jpg` - coordinate grid and catalogued objects, placed + by the plate solution. Includes SN 1986G and SN 2016adj, both real supernovae + in this galaxy. +- `NGC5128-img-starless.png` / `NGC5128-img-stars.png` - the field split into + nebulosity and stars +- `NGC5128-img-core-print.jpg` - a print crop + +Kept because they show intermediate stages and because some are useful in their +own right. `final/` is what to look at first. +""", + "science": """# science/ + +The measurements, as opposed to the pictures. + +- `notes/` - **start here.** Three write-ups covering the globular cluster + survey, the surface photometry, and the transient and moving-object search. + Each states its method, its numbers, and its limits. +- `figures/` - the plots those notes refer to +- `catalogues/` - the measured tables (CSV): cluster candidates, isophote + profiles, flagged objects +- `data/` - the galaxy model, star and dust masks, extinction map, and a plain + text summary of derived quantities + +Headline results are summarised in `../METHODS.md`; the notes carry the detail, +the caveats and the things that did not work. +""", + "intermediates": """# intermediates/ + +Caches and scratch files: downloaded star catalogues, detection lists, partial +results, run logs. + +**Safe to delete.** Everything here is regenerated by re-running the scripts, +at the cost of re-querying Gaia, SIMBAD and VizieR. Kept so a re-run is fast +and so the exact catalogue data used is preserved rather than silently changing +under a later query. +""", +} + +for name, text in READMES.items(): + path = os.path.join(ROOT, name, "README.md") + os.makedirs(os.path.dirname(path), exist_ok=True) + open(path, "w", encoding="utf-8").write(text) + print(f"wrote {name}/README.md") diff --git a/pipeline/triptych.py b/pipeline/triptych.py new file mode 100644 index 0000000..4a43765 --- /dev/null +++ b/pipeline/triptych.py @@ -0,0 +1,62 @@ +"""Side-by-side of the three renderings, full field and a matched core crop. + +Same pixels in all three panels, so any difference is processing and nothing +else. +""" +import os + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +from PIL import Image + +import layout + +OUT = layout.SESSION +# The finished renderings live in their own directory: they are the +# deliverables, and keeping them apart from the masters, the +# intermediates and the analysis figures makes it obvious which +# files are meant to be looked at. +FINAL = layout.path("final") +PANELS = [ + ("NGC5128-final-1-stacked.png", "1. Stacked + standard stretch", + "align, average, stretch - nothing else"), + ("NGC5128-final-2-processed.png", "2. Conventionally processed", + "gradient removal, colour balance, denoise"), + ("NGC5128-final-3-best.png", "3. Science-informed", + "halo-preserving background, solar-analogue colour,\n" + "core recovered, star-protected deconvolution"), +] +CROP = (1950, 1200, 2850, 1875) # the dust lane, at full resolution + +fig, axes = plt.subplots(2, 3, figsize=(21, 11.5), + gridspec_kw=dict(height_ratios=[1.35, 1])) +fig.patch.set_facecolor("#111111") + +for col, (fname, title, sub) in enumerate(PANELS): + im = Image.open(layout.path(fname)) + wide = im.copy() + wide.thumbnail((1500, 1500), Image.LANCZOS) + axes[0, col].imshow(np.asarray(wide)) + axes[0, col].set_title(title, color="white", fontsize=15, pad=10) + axes[0, col].text(0.5, -0.055, sub, color="#9fb8d0", fontsize=10, + ha="center", va="top", linespacing=1.4, wrap=True, + transform=axes[0, col].transAxes) + axes[1, col].imshow(np.asarray(im.crop(CROP))) + axes[1, col].set_title("core, 1:1", color="#9fb8d0", fontsize=11, pad=6) + for row in (0, 1): + axes[row, col].set_xticks([]) + axes[row, col].set_yticks([]) + for s in axes[row, col].spines.values(): + s.set_color("#333333") + +fig.suptitle("NGC 5128 (Centaurus A) - iTelescope T32, 2026-07-21 - " + "L 12x300s, RGB 4x300s each - the same data, three treatments", + color="white", fontsize=17, y=0.975) +fig.tight_layout(rect=[0, 0.01, 1, 0.955]) +fig.subplots_adjust(hspace=0.16) +path = layout.path("NGC5128-final-comparison.jpg") +fig.savefig(path, dpi=100, facecolor=fig.get_facecolor(), + pil_kwargs={"quality": 92}) +print("wrote", path) diff --git a/pipeline/unzip.py b/pipeline/unzip.py new file mode 100644 index 0000000..3859aa6 --- /dev/null +++ b/pipeline/unzip.py @@ -0,0 +1,33 @@ +"""Uncompress every calibrated-*.zip in the session folder, in place. + +iTelescope ships each calibrated frame as a one-entry zip (.fit.zip and +.tif.zip). Extraction is skipped when the target already exists and is +non-empty, so this is safe to re-run. +""" +import glob +import os +import zipfile + +import layout + +SRC = layout.SESSION + +zips = sorted(glob.glob(layout.path("calibrated-*.zip"))) +print(f"{len(zips)} calibrated archives") +done = skipped = 0 +for z in zips: + with zipfile.ZipFile(z) as zf: + for info in zf.infolist(): + target = layout.path(os.path.basename(info.filename)) + if os.path.exists(target) and os.path.getsize(target) == info.file_size: + skipped += 1 + continue + with zf.open(info) as fsrc, open(target, "wb") as fdst: + while chunk := fsrc.read(1 << 20): + fdst.write(chunk) + done += 1 +print(f"extracted {done}, already present {skipped}") + +for ext in (".fit", ".tif"): + n = len(glob.glob(layout.path(f"calibrated-*{ext}"))) + print(f"{ext}: {n} files") diff --git a/pipeline/verify_core.py b/pipeline/verify_core.py new file mode 100644 index 0000000..750853c --- /dev/null +++ b/pipeline/verify_core.py @@ -0,0 +1,74 @@ +"""Is the nucleus of NGC 5128 actually saturated, or was that a foreground star? + +The earlier claim - that the core clips at 65313 ADU in a single 300 s sub - +came from taking the maximum inside a 300x300 px box centred on the frame. That +box is wide enough to contain a bright foreground star, so the measurement +proves only that SOMETHING in the middle of the frame is bright. This checks +where the bright pixels actually are, and what the galaxy itself peaks at once +stars are filtered out. +""" +import numpy as np +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.wcs import WCS +from scipy.ndimage import median_filter + +import layout + +SUB = layout.path("calibrated-T32-qisback-NGC5128-20260721-190133" + "-Luminance-BIN2-W-300-002.fit") +MASTER = layout.path("master-Luminance.fit") + +# NGC 5128's nucleus, from SIMBAD, not from "the middle of the frame". +NUCLEUS = SkyCoord("13h25m27.6s", "-43d01m08.8s") + +with fits.open(MASTER) as hd: + wcs = WCS(hd[0].header, naxis=2) +nx_c, ny_c = wcs.world_to_pixel(NUCLEUS) +print(f"nucleus lands at master pixel ({nx_c:.1f}, {ny_c:.1f})") + +data = fits.getdata(SUB).astype(np.float32) +ny, nx = data.shape +print(f"single sub {nx} x {ny}, global max {data.max():.0f} ADU") + +# Where are the saturated-ish pixels? +ys, xs = np.where(data > 60000) +print(f"{len(xs)} pixels above 60000 ADU") +if len(xs): + # Cluster them crudely by proximity to see how many distinct objects. + print(f" x range {xs.min()}-{xs.max()}, y range {ys.min()}-{ys.max()}") + cx, cy = nx / 2.0, ny / 2.0 + d = np.hypot(xs - cx, ys - cy) + print(f" distance from frame centre: min {d.min():.0f} px, " + f"median {np.median(d):.0f} px, max {d.max():.0f} px") + # The brightest pixel specifically + iy, ix = np.unravel_index(np.argmax(data), data.shape) + print(f" brightest pixel at ({ix}, {iy}), " + f"{np.hypot(ix - cx, iy - cy):.0f} px from frame centre") + +# The galaxy's own peak: median filter removes stars, which are small, while +# leaving the smooth galaxy light essentially untouched. +h = 400 +y0, y1 = int(ny / 2) - h, int(ny / 2) + h +x0, x1 = int(nx / 2) - h, int(nx / 2) + h +core = data[y0:y1, x0:x1] +smooth = median_filter(core, size=15) +print(f"\ninner {2*h}x{2*h} px box:") +print(f" raw max {core.max():9.1f} ADU") +print(f" median-filtered max {smooth.max():9.1f} ADU <- galaxy light") +iy, ix = np.unravel_index(np.argmax(smooth), smooth.shape) +print(f" galaxy peak at frame pixel ({x0+ix}, {y0+iy})") + +# How many pixels of the median-filtered (star-free) galaxy are near clipping? +for lvl in (30000, 50000, 60000): + print(f" star-free pixels above {lvl}: {(smooth > lvl).sum()}") + +# And in the master stack. +mdata = fits.getdata(MASTER).astype(np.float32) +mcore = mdata[y0:y1, x0:x1] +msmooth = median_filter(mcore, size=15) +print(f"\nmaster stack inner box: raw max {mcore.max():.1f}, " + f"star-free max {msmooth.max():.1f} ADU") +print(f" master 99.995th percentile (the stretch white point) " + f"{np.percentile(mdata, 99.995):.1f} ADU")