From 015bff47fc5c74debc20488007c8807d91e8cdef Mon Sep 17 00:00:00 2001 From: laurence Date: Tue, 21 Jul 2026 17:28:26 +0100 Subject: [PATCH 1/2] Add session discovery and ingest, driven by headers not filenames First stage of the generic pipeline. The existing scripts parse filenames, expecting calibrated-T32-...-Luminance-BIN2-W-300-001.fit, which works for exactly one telescope, one binning and one exposure length. Three of the four archived sessions have no luminance channel at all, so that approach cannot process them. session.py opens each frame and reads FILTER, EXPTIME, XBINNING, OBJECT and DATE-OBS instead. It groups frames by filter, works out which colour scheme the session supports (LRGB, RGB, SHO, HOO or mono), and writes a manifest. Filter aliases are mapped, so a header saying H-alpha, Halpha or HA all resolve to Ha. Three real defects were found by running it against the archive rather than by reasoning about it: Frame counts came out doubled. iTelescope ships every frame twice, as 'calibrated-*' with bias, dark and flat applied and 'raw-*' without, and extracting both put two copies of each frame into the stack. Only the calibrated archives are extracted now, with CALSTAT used as a header level safety net for sessions that were already extracted the old way. NGC 5128 now scans as 12/4/4/4, exactly matching the session processed by hand, which makes it a usable regression test. A re-mirror over the organised tree produced duplicates and mirror damage. An interrupted segmented download leaves DIRECTORIES named after the file being fetched - a directory called something.fit - plus .cyberducksegment parts. Anything that trusts a file extension will try to open a directory as a frame. Those are now skipped explicitly at discovery, ingest and scan, and byte-identical copies that a re-mirror puts back are removed as they are refiled. Two T20 archives are corrupt and cannot be extracted. That is real data loss, not a bug: the session has 14 of its 16 frames and the warning is printed rather than swallowed. Not yet done: NGC 6744 mixes bin1 luminance at 4096x4096 with bin2 colour at 2048x2048, so registration will need to resample onto a common grid rather than assuming every frame shares a shape. --- pipeline/session.py | 353 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 353 insertions(+) create mode 100644 pipeline/session.py diff --git a/pipeline/session.py b/pipeline/session.py new file mode 100644 index 0000000..36a3a00 --- /dev/null +++ b/pipeline/session.py @@ -0,0 +1,353 @@ +"""What a session contains, decided by reading the frames rather than the names. + +The first version of this pipeline parsed filenames: it expected +`calibrated-T32-...-Luminance-BIN2-W-300-001.fit` and broke on anything else. +That works for exactly one telescope, one binning and one exposure length. Every +other session in the archive - narrowband from 2024, an Ha-only run from 2021, +an RGB set with no luminance at all - fails that pattern. + +So this module opens each frame and asks the header. FITS carries FILTER, +EXPTIME, XBINNING, OBJECT and DATE-OBS precisely so that software does not have +to guess from a filename, and iTelescope fills them in properly. + +The unit of work is a *session*: one target, one telescope, one night, in one +directory. Everything else in the pipeline takes a Session and asks it what +filters exist and which frames belong to each. +""" +import glob +import json +import os +import zipfile +from collections import defaultdict + +from astropy.io import fits + +import layout + +# Filters that carry broadband colour, in the order a colour image wants them. +BROADBAND = ("Red", "Green", "Blue") +LUMINANCE = ("Luminance", "Clear", "Lum", "L") +# Narrowband, with the aliases different telescopes write into FILTER. +NARROWBAND = { + "Ha": ("Ha", "H-alpha", "Halpha", "HA", "H_Alpha"), + "OIII": ("OIII", "O3", "Oiii", "O-III"), + "SII": ("SII", "S2", "Sii", "S-II"), +} + + +def canonical_filter(raw): + """Map whatever the header says onto a name the pipeline knows.""" + if raw is None: + return "Unknown" + name = str(raw).strip() + for canon, aliases in NARROWBAND.items(): + if name in aliases or name.lower() == canon.lower(): + return canon + for canon in BROADBAND: + if name.lower() == canon.lower() or name.lower() == canon[0].lower(): + return canon + if name.lower() in [a.lower() for a in LUMINANCE]: + return "Luminance" + return name or "Unknown" + + +class Frame: + """One light frame and the header facts the pipeline needs.""" + + __slots__ = ("path", "filter", "exptime", "binning", "object", "date_obs", + "telescope", "instrument", "shape", "scale", "calibrated") + + def __init__(self, path, header, shape, calibrated=True): + self.calibrated = calibrated + self.path = path + self.filter = canonical_filter(header.get("FILTER")) + self.exptime = float(header.get("EXPTIME", header.get("EXPOSURE", 0))) + self.binning = int(header.get("XBINNING", 1)) + self.object = str(header.get("OBJECT", "")).strip() + self.date_obs = str(header.get("DATE-OBS", "")) + self.telescope = str(header.get("TELESCOP", "")).strip() + self.instrument = str(header.get("INSTRUME", "")).strip() + self.shape = shape + # iTelescope writes the plate scale in; without it, solve.py estimates. + scale = header.get("HIERARCH iTelescopePlateScaleH") + try: + self.scale = float(scale) if scale is not None else None + except (TypeError, ValueError): + self.scale = None + + @property + def name(self): + return os.path.basename(self.path) + + def __repr__(self): + return (f"<{self.filter} {self.exptime:.0f}s bin{self.binning} " + f"{self.name}>") + + +class Session: + """One target, one telescope, one night.""" + + def __init__(self, root): + self.root = os.path.abspath(root) + self.frames = [] + self._by_filter = {} + + # ---------------------------------------------------------------- ingest + @staticmethod + def _is_mirror_junk(path): + """True for artefacts a broken segmented download leaves behind. + + Cyberduck writes a DIRECTORY named after the file it is fetching, and + leaves `.cyberducksegment` parts inside when interrupted. Those + directories are named `something.fit`, so anything that trusts the + extension will try to open a directory as a frame. A re-mirror over an + already-organised tree produces dozens of them. + """ + name = os.path.basename(path).lower() + if ".cyberducksegment" in name or name.endswith((".part", ".partial")): + return True + parent = os.path.basename(os.path.dirname(path)).lower() + return parent.endswith((".fit", ".fits", ".tif", ".tiff", ".zip")) + + def ingest(self, verbose=True): + """Put the session into the standard layout and uncompress the lights. + + Handles the three states a downloaded session can be in: still zipped, + already extracted, or a mixture of both (which is what happens when + someone unzips a few by hand). + """ + raw = os.path.join(self.root, layout.RAW) + cal = os.path.join(self.root, layout.CALIBRATED) + os.makedirs(raw, exist_ok=True) + os.makedirs(cal, exist_ok=True) + + moved = extracted = skipped = duplicates = 0 + # Loose files at the session root get filed first. + for path in sorted(glob.glob(os.path.join(self.root, "*"))): + if os.path.isdir(path): + continue + name = os.path.basename(path) + low = name.lower() + if self._is_mirror_junk(path): + continue + if low.endswith(".zip") or (low.startswith("jpeg-") and + low.endswith(".jpg")): + dest = os.path.join(raw, name) + elif low.endswith((".fit", ".fits", ".tif", ".tiff")): + dest = os.path.join(cal, name) + else: + continue + if not os.path.exists(dest): + os.replace(path, dest) + moved += 1 + elif os.path.getsize(dest) == os.path.getsize(path): + # Already filed, and the mirror has put an identical copy back. + os.remove(path) + duplicates += 1 + + # iTelescope ships each frame twice: 'calibrated-*' with bias, dark + # and flat applied, and 'raw-*' without. Extracting both put two copies + # of every frame into the stack. Only the calibrated set is extracted; + # the raw archives stay zipped in raw/ as the untouched originals. If a + # session has no calibrated archives at all, the raw ones are used and + # the fact is recorded. + archives = sorted(glob.glob(os.path.join(raw, "*.zip"))) + calibrated_archives = [z for z in archives + if os.path.basename(z).lower().startswith( + "calibrated")] + self.used_raw = not calibrated_archives and bool(archives) + if self.used_raw: + print(" NOTE: no calibrated archives in this session - " + "extracting the raw frames instead. They have NOT had bias, " + "dark or flat correction applied.") + else: + archives = calibrated_archives + + for zpath in archives: + try: + with zipfile.ZipFile(zpath) as zf: + for info in zf.infolist(): + base = os.path.basename(info.filename) + if not base.lower().endswith((".fit", ".fits", ".tif", + ".tiff")): + continue + target = os.path.join(cal, base) + if os.path.exists(target) and \ + os.path.getsize(target) == info.file_size: + skipped += 1 + continue + with zf.open(info) as src, open(target, "wb") as dst: + while chunk := src.read(1 << 20): + dst.write(chunk) + extracted += 1 + except zipfile.BadZipFile: + print(f" WARNING: {os.path.basename(zpath)} is not a " + f"readable zip - skipped") + if verbose: + print(f" filed {moved}, extracted {extracted}, " + f"already present {skipped}, " + f"duplicate copies removed {duplicates}") + return dict(moved=moved, extracted=extracted, skipped=skipped, + duplicates=duplicates) + + # ----------------------------------------------------------------- scan + def scan(self, verbose=True): + """Read every calibrated light frame's header.""" + cal = os.path.join(self.root, layout.CALIBRATED) + paths = sorted(p for p in glob.glob(os.path.join(cal, "*.fit")) + + glob.glob(os.path.join(cal, "*.fits")) + if os.path.isfile(p) and not self._is_mirror_junk(p)) + self.frames = [] + for path in paths: + try: + with fits.open(path, memmap=True) as hd: + hdr = hd[0].header + shape = hd[0].shape + except Exception as exc: # noqa: BLE001 + print(f" WARNING: cannot read {os.path.basename(path)}: " + f"{exc}") + continue + # Skip anything that is not a light frame. + if str(hdr.get("IMAGETYP", "Light")).lower().startswith( + ("dark", "bias", "flat")): + continue + self.frames.append(Frame(path, hdr, + shape, calibrated=bool( + str(hdr.get("CALSTAT", "")).strip()))) + + # If both calibrated and uncalibrated frames are present - which + # happens when a session was extracted before this rule existed - keep + # only the calibrated ones rather than stacking each frame twice. + if any(f.calibrated for f in self.frames): + dropped = [f for f in self.frames if not f.calibrated] + if dropped and verbose: + print(f" ignoring {len(dropped)} uncalibrated duplicate " + f"frames (no CALSTAT)") + self.frames = [f for f in self.frames if f.calibrated] + + self._by_filter = defaultdict(list) + for f in self.frames: + self._by_filter[f.filter].append(f) + if verbose: + self.describe() + return self.frames + + # ------------------------------------------------------------ properties + @property + def filters(self): + """Filters present, deepest first - the deepest becomes the reference.""" + return sorted(self._by_filter, + key=lambda f: -sum(x.exptime for x in self._by_filter[f])) + + def by_filter(self, name): + return list(self._by_filter.get(name, [])) + + @property + def target(self): + names = [f.object for f in self.frames if f.object] + return max(set(names), key=names.count) if names else \ + os.path.basename(os.path.dirname(self.root)) + + @property + def telescope(self): + names = [f.telescope for f in self.frames if f.telescope] + return max(set(names), key=names.count) if names else "unknown" + + @property + def scale(self): + vals = [f.scale for f in self.frames if f.scale] + return sum(vals) / len(vals) if vals else None + + @property + def palette(self): + """Which colour scheme this session's filters support.""" + present = set(self._by_filter) + has_lum = "Luminance" in present + broad = [c for c in BROADBAND if c in present] + narrow = [n for n in ("SII", "Ha", "OIII") if n in present] + if len(broad) == 3 and has_lum: + return "LRGB" + if len(broad) == 3: + return "RGB" + if len(narrow) >= 3: + return "SHO" + if "Ha" in narrow and "OIII" in narrow: + return "HOO" + if len(present) == 1: + return "MONO" + if broad or narrow: + return "PARTIAL" + return "UNKNOWN" + + def describe(self): + print(f" target {self.target}") + print(f" telescope {self.telescope}") + if self.scale: + print(f" plate scale {self.scale:.4f} arcsec/px") + print(f" palette {self.palette}") + for name in self.filters: + fs = self._by_filter[name] + exps = sorted({f.exptime for f in fs}) + bins = sorted({f.binning for f in fs}) + total = sum(f.exptime for f in fs) / 60.0 + shapes = sorted({f.shape for f in fs}) + shape_txt = "/".join(f"{s[1]}x{s[0]}" for s in shapes) + print(f" {name:10s} {len(fs):3d} frames " + f"{'/'.join(f'{e:.0f}' for e in exps):>12s} s " + f"bin{'/'.join(str(b) for b in bins)} " + f"{shape_txt:>11s} {total:6.1f} min") + + # ------------------------------------------------------------- manifest + def manifest(self): + return { + "root": self.root, + "target": self.target, + "telescope": self.telescope, + "plate_scale": self.scale, + "palette": self.palette, + "filters": { + name: { + "frames": len(self._by_filter[name]), + "total_exposure_s": sum(f.exptime + for f in self._by_filter[name]), + "exposures": sorted({f.exptime + for f in self._by_filter[name]}), + "binning": sorted({f.binning + for f in self._by_filter[name]}), + } for name in self.filters + }, + } + + def save_manifest(self): + path = os.path.join(self.root, "manifest.json") + with open(path, "w", encoding="utf-8") as fh: + json.dump(self.manifest(), fh, indent=2) + return path + + +def discover(root): + """Find every session under a directory tree. + + A session is any directory that directly contains light frames or their + archives, which matches how iTelescope's downloads are organised + (telescope / object / date). + """ + found = [] + for dirpath, dirnames, filenames in os.walk(root): + base = os.path.basename(dirpath) + if base.lower().endswith((".fit", ".fits", ".tif", ".tiff", ".zip")) or ".cyberducksegment" in base.lower(): + dirnames[:] = [] + continue + if base in (layout.RAW, layout.CALIBRATED, "intermediates", + "final", "renderings", "science", "stacks"): + dirnames[:] = [] + continue + has_data = any(f.lower().endswith((".zip", ".fit", ".fits")) + for f in filenames) + has_layout = any(d in dirnames for d in (layout.RAW, + layout.CALIBRATED)) + if has_data or has_layout: + found.append(dirpath) + dirnames[:] = [d for d in dirnames + if d in (layout.RAW, layout.CALIBRATED)] + return sorted(found) From 3f785136ed5f2a99835464cf67236b3513db47e9 Mon Sep 17 00:00:00 2001 From: laurence Date: Tue, 21 Jul 2026 17:32:00 +0100 Subject: [PATCH 2/2] Bring the state files up to date so a cold session can resume Rule 2 of the workflow is that state lives in the repo rather than the chat, and a long session had put most of the current position only in the chat. This writes it down. TODO.md gains a RESUME HERE block at the top, because the useful question for the next session is not what has been done but what to do next and what is blocking it. It names the three live strands, the three decisions waiting on the user, and the one thing that will break the batch run if it is forgotten: NGC 6744 mixes bin1 luminance at 4096x4096 with bin2 colour at 2048x2048, so registration has to resample onto a common grid. Nothing else in the archive does that, which is exactly why it will be forgotten. It also drops three duplicate "Done (recent)" headings that had accumulated, and tabulates what the five archived sessions actually contain. NOTES.md gains the gotchas, all of which produced plausible wrong answers rather than errors: every frame is delivered twice as calibrated and raw, so extracting both silently doubles the stack; an interrupted segmented download leaves DIRECTORIES named like files, which defeats anything that trusts an extension; a re-mirror duplicates rather than replaces; two T20 archives are genuinely corrupt; extraction triples the disk footprint and filled the drive. Alongside them the analysis lessons from NGC 5128, kept for the same reason - each is a way to be confidently wrong. DECISIONS.md records why the pipeline reads headers rather than filenames, and why imaging is being made push-button while the science stays hand-driven. ARCHITECTURE.md described a documentation-only repository with no code, which stopped being true when the pipeline merged in. It now covers all three parts, the code-and-data separation the pipeline is built around, and the on-disk session layout. Both of this session's own errors are recorded in TODO.md as things not to repeat: measuring a foreground star instead of the galaxy nucleus and booking telescope time to fix it, and telling the user NGC 6744 had not been processed when it had. --- state/ARCHITECTURE.md | 79 ++++++++++++-- state/DECISIONS.md | 27 +++++ state/NOTES.md | 69 +++++++++++++ state/TODO.md | 234 +++++++++++++++++++++++++----------------- 4 files changed, 303 insertions(+), 106 deletions(-) diff --git a/state/ARCHITECTURE.md b/state/ARCHITECTURE.md index 6de98f7..dab87b5 100644 --- a/state/ARCHITECTURE.md +++ b/state/ARCHITECTURE.md @@ -3,14 +3,73 @@ > How the system is built and why. Update this when the structure changes; a change is > not finished until this reflects it. -A documentation-only repo; there is no code. +Three parts, which is why the repository has three top-level content +directories. Until 2026-07-21 the first two lived in separate repositories. -- `TELESCOPES.md`: the deliverable. Per-observatory sections, one entry per telescope - (spec bullets then a review paragraph), a "choosing a telescope" table, and general - observations. Compiled from the sources below, dated in its header. -- `data/itelescope-telescopes.csv`: verbatim CSV export of iTelescope's maintained - specs Google Sheet. Treated as the source of truth for numbers; re-exported to - refresh (see state/NOTES.md). -- `state/`: Default Workflow project memory. -- `CLAUDE.md` + `docs/`: the Default Workflow itself, copied from the - Default-Workflow repo so the project is self-contained. +## itelescope/ - the network and the campaign + +Documentation and data, no code. + +- `TELESCOPES.md`: the deliverable. Per-observatory sections, one entry per + telescope (spec bullets then a review paragraph), a "choosing a telescope" + table, and general observations. Compiled from the sources below. +- `TARGETS.md`: southern targets invisible from the UK, matched to scopes and + to the time of year. +- `CAMPAIGN.md`: the points drain campaign - ledger, bookings, queue, rate + model. Updated after every session from the account's transaction history. +- `data/itelescope-telescopes.csv`: verbatim CSV export of iTelescope's + maintained specs sheet. The source of truth for numbers; re-exported to + refresh (see `state/NOTES.md`). +- `plans/`: ACP observing plans, one per booked run, uploaded to the scope. + +## pipeline/ - the processing code + +Python 3.12. The design rule is that **code and data are separate**: nothing +here knows an absolute path. A session is located at runtime through the +`ASTRO_SESSION` environment variable. + +- `layout.py` maps a **filename** to the subdirectory it belongs in, using the + same rules the session directories are organised with. A script asks for + `master-Red.fit` or `_stars.npz` and gets the right path without any call + site knowing the structure. Unrecognised names resolve to the session root, + which is visible and correctable rather than silently wrong. +- `session.py` is the entry point for anything generic. It discovers sessions, + ingests and uncompresses them, reads every frame's header, groups frames by + filter, and derives the colour scheme (LRGB / RGB / SHO / HOO / MONO) from + what is actually present. **Header-driven, never filename-driven** - see + DECISIONS. +- `restructure.py` reorganises a flat session directory into the standard + layout. Idempotent, dry run by default. +- The remaining scripts are the NGC 5128 session as it was actually run: + `unzip -> analyse -> stack -> solve -> depth -> compose -> hdr/enhance/ + starless/annotate -> final -> closeup -> triptych`, plus the analysis + families `gc-*` (globular clusters), `sb_*` (surface photometry) and `mo_*` + (moving objects and transients). They are kept as provenance for published + results and are being generalised stage by stage. + +### Session layout on disk (not in this repo) + +``` +/ + METHODS.md what was done to this data and what was found + raw/ exactly what the telescope delivered + calibrated/ uncompressed calibrated subs + stacks/masters/ per-filter registered, combined, plate-solved + stacks/original/ alignment-only baseline, no other processing + final/ the deliverable renderings + renderings/ other finished images + science/figures|catalogues|data|notes + intermediates/ caches; deletable, regenerated by a re-run +``` + +## observing/ - in-person plans + +Field plans worked out from real numbers, revised as a date approaches. The +reasoning is the part worth keeping, so plans record why a decision was made, +not only what was decided. + +## The workflow itself + +`CLAUDE.md` + `docs/`: the Default Workflow, copied in so the project is +self-contained. `state/`: project memory - objective, current work, decisions, +notes. diff --git a/state/DECISIONS.md b/state/DECISIONS.md index 25eb138..cc15ca9 100644 --- a/state/DECISIONS.md +++ b/state/DECISIONS.md @@ -3,6 +3,33 @@ > A dated, append only log of decisions and their rationale. Newest at the top. Never > rewrite past entries; if a decision is reversed, add a new entry that says so. +## 2026-07-21: the pipeline reads headers, not filenames + +The first pipeline parsed `calibrated-T32-...-Luminance-BIN2-W-300-001.fit` to +learn a frame's filter, binning and exposure. That works for one telescope, one +binning and one exposure length. Three of the four archived sessions have no +luminance channel at all and one is from 2021 with different naming, so the +approach could not survive contact with the archive. + +FITS headers carry FILTER, EXPTIME, XBINNING, OBJECT and DATE-OBS precisely so +software does not have to guess, and iTelescope fills them in properly. The +pipeline opens each frame and asks. Filter aliases are mapped, so H-alpha, +Halpha and HA all resolve to Ha. + +The colour scheme is then derived from what is present rather than assumed: +LRGB, RGB without luminance, SHO, HOO, or mono. That is what lets one command +handle a galaxy in LRGB and a nebula in narrowband without being told which is +which. + +## 2026-07-21: push-button imaging, hand-driven science + +The imaging half - unzip, measure, register, stack, plate solve, compose - is +mechanical and should run unattended on any session. The analyses are not: a +globular cluster survey suits an elliptical galaxy and is meaningless for an +emission nebula, and much of the value in the NGC 5128 work came from noticing +when a result was wrong rather than from running the code. So the pipeline is +being built to be push-button, and the science stays step by step. + ## 2026-07-21: merge itelescope and astro-pipeline into one repository The two were split by accident of chronology rather than by design. `itelescope` diff --git a/state/NOTES.md b/state/NOTES.md index 3a4323c..7c7d938 100644 --- a/state/NOTES.md +++ b/state/NOTES.md @@ -78,3 +78,72 @@ Paid imaging needs no reservation when a scope is idle: same walk-in ACP flow as the free scopes, points just deduct. Reservations (to guarantee a slot) go through lookup.itelescope.online, which is Cloudflare-protected: Playwright needed. + + +## Processing the downloaded archive (2026-07-21) + +Gotchas found by running code against the real archive rather than reasoning +about it. Each one produced a plausible-looking wrong answer, not an error. + +- **Every frame is delivered twice.** iTelescope ships `calibrated-*` (bias, + dark and flat applied) and `raw-*` (not). Extracting both puts two copies of + every frame into the stack, and the counts look merely generous rather than + wrong: NGC 5128 scanned as 24 luminance frames when it has 12. `CALSTAT` in + the header is the reliable discriminator - calibrated frames carry it, raw + frames do not. Only the calibrated archives are extracted; the raw archives + stay zipped as the untouched originals. + +- **An interrupted segmented download leaves DIRECTORIES named like files.** + Cyberduck creates a directory called `something.fit` while fetching, with + `.cyberducksegment` parts inside. Anything that trusts a file extension will + try to open a directory as a frame. After a re-mirror over an already + organised tree there were 28 of them. Discovery, ingest and scan all skip + them explicitly. + +- **A re-mirror duplicates rather than replaces.** Mirroring on top of an + organised tree restored ~1 GB of files to session roots that had already been + filed into `raw/`. Ingest now removes byte-identical copies as it refiles. + +- **Two T20 archives are genuinely corrupt** and cannot be extracted + (`ic1805-...-Ha-...-015` and `-016`). That is real data loss, not a bug: the + session has 14 of 16 frames. The warning is printed rather than swallowed. + +- **Extraction triples the disk footprint.** The archive went from 6.1 GB of + zips to 14 GB extracted, which filled the drive. The TIFFs are 1.69 GB of + that and the pipeline never reads them - it only uses the FITS. + +- **Channels do not always share a pixel grid.** NGC 6744 was shot with + luminance at bin1 (4096x4096) and colour at bin2 (2048x2048). Registration + has to resample onto a common grid; code that assumes a single frame shape + will fail on this session and only this session. + +## Processing lessons from NGC 5128 (2026-07-21) + +Kept because each is a way to get a confident wrong answer: + +- **Measure a core with the stars filtered out.** Taking the maximum pixel in a + box around the frame centre found a foreground star, not the galaxy, and led + to booking telescope time to fix a saturation problem that did not exist. + +- **Registration makes sensor defects look like perfect asteroids.** Holding + the sky still drags anything detector-fixed across the frame on a dead + straight, constant-rate, constant-brightness track. Hot pixels pass every + standard quality cut. Requiring the motion to be real in DETECTOR coordinates + took 141 confident detections to zero. + +- **Never compare an aperture magnitude to a point-source catalogue without + checking the source is a point.** A resolved object looks exactly like a + 2.8 magnitude outburst. Carry r50/psf through to the cross-match. + +- **Never fit a sky background to a field the target fills.** A plane fitted + around a large galaxy absorbs its halo, measured at -17.9 ADU/px. Fit the + background and a source model together. + +- **"No Gaia counterpart" is not a star-rejection test** at extragalactic + distances: half of Centaurus A's catalogued globular clusters have a Gaia + source, because a cluster at 3.8 Mpc is a point source. Reject on parallax + and proper motion instead. + +- **A coarse background mesh fails in the direction that flatters you.** It + destroys detection near a bright galaxy while leaving the outer field intact, + manufacturing a falsely flat radial profile. diff --git a/state/TODO.md b/state/TODO.md index 5645984..efbcbf9 100644 --- a/state/TODO.md +++ b/state/TODO.md @@ -1,121 +1,163 @@ # TODO -> The current state of play. Keep it honest and current; this is what the next session -> reads to know what to do. Move items between sections as they progress. +> The current state of play. Keep it honest and current; this is what the next +> session reads to know what to do. Move items between sections as they progress. -## Done +--- -- 2026-07-17: repo created on git.discworld.casa, Default Workflow scaffold copied in. -- 2026-07-17: specs sheet snapshotted to data/itelescope-telescopes.csv (24 scopes). -- 2026-07-17: TELESCOPES.md written: per-scope reviews for all 23 active telescopes - plus a choosing guide (T74 noted as in commissioning, no specs published). -- 2026-07-18: TARGETS.md written: southern targets invisible from the UK (dec tiers - from lat 51.5°N), matched to Q62/X07 scopes, seasonal booking calendar, and a - July-August starter campaign (T33 free, T71 narrowband, T73 galaxy, T8 LMC later). +## RESUME HERE - 2026-07-21, end of session -## Done (recent) +Three strands are live. Each has a clear next action. -- 2026-07-20: NGC 6744 on T59 SUCCEEDED (reservation 749227, txn 851148): full - LRGB dataset on the scope server (L 15x300s BIN1, RGB 7x300s each BIN2), - previews verified clean. Billed 381 pts (est was ~240: see the rate - correction in NOTES.md). Balance now 2,257. Ledger updated in CAMPAIGN.md. +### 1. Generic pipeline - MID-BUILD, this is the active work -## Done (recent) +Building a push-button pipeline that processes any session, not just LRGB from +one telescope. Branch `generic-pipeline`. -- 2026-07-21 ~08:55 UTC: free T33 47 Tuc retry booked as reservation 749314 - (grid-verified), third attempt at the first-light test, runs 14:05-14:50 UTC - 21 Jul, costs 0. +**Done:** `pipeline/session.py` - discovery, ingest, header-driven scanning, +filter grouping, palette detection (LRGB / RGB / SHO / HOO / MONO), manifest +writing. Tested against all five archived sessions. -## Done (recent) +**Next:** `measure` -> `register/stack` -> `solve` -> `compose` -> `report`, +then a `run.py` orchestrator with resumable stages. -- 2026-07-21 11:40 UTC: Cen A on T32 verified COMPLETE (billed 160, NGC5128 - images on the scope server), and queue #2 (Corona Australis on T8) booked for - the night of 22 Jul, 21:45-00:45 SSO local, which brackets the target's - culmination instead of chasing it down the western sky. Rate model rebuilt - from two actual bills. Details in the CAMPAIGN.md session note. +**The blocker before any of that:** the disk is full (see below). -## In progress +**The known hard part:** NGC 6744 mixes bin1 luminance at 4096x4096 with bin2 +colour at 2048x2048. Registration must resample onto a common grid rather than +assume every frame shares a shape. Nothing else in the archive does this, so it +is easy to forget and it will fail loudly when it happens. -- T33 first-light test on its THIRD booking (749314), which had NOT yet run as of - 11:40 UTC 21 Jul: it starts 14:05 UTC. Verify after 14:50 UTC (images and logs - on t33.itelescope.net:8033 were still empty/stale at 11:40, as expected). The - two prior free reservations (749228, 749229) expired unused with reschedule=No; - if 749314 also fails to run, stop rebooking and either walk it in (POST - plan.asp per plans/README.md) or drop the free objective. -- The T32 HDR core set (plans/hdrcore-t32.txt) was booked and then CANCELLED on - 2026-07-21 without running: the Cen A nucleus is not saturated, the earlier - claim having measured a foreground star. See the correction in CAMPAIGN.md. - Zero points spent. The plan file remains available on T32 if a genuinely - saturated target turns up. -- The 21 Jul Cen A data has been fully processed off-repo, in the user's - Downloads\NGC5128\20260721\stacked folder: masters, plate solution (0.27 - arcsec on 237 stars), an LRGB composite, a globular cluster survey (289 - candidates, 62% catalogue-confirmed, density falling 10.5x outward), surface - photometry (Sersic n = 4.27, Re = 400 arcsec, total G = 6.56, which matches - the catalogued brightness and so validates the photometric chain), and a - transient/moving-object search (both null, with measured limits). METHODS.md - there documents the lot. Nothing from that analysis needs to come back into - this repo, but two results affect FUTURE sessions and are recorded here: - **(a)** measure saturation properly before buying short subs, and **(b)** the - faint outer halo in that stack is limited by the sky-plane subtraction - (-17.9 ADU/px of real halo light absorbed), not by exposure time, so more - integration alone would not have gone deeper. -- CrA on T8 runs 11:45-14:45 UTC 22 Jul. Verify after 14:45 UTC: images under - /images/qisback/T8/ on t8.itelescope.net:8008, then the actual bill from - account/history.aspx into the ledger. This is the first T8 bill, so it also - fixes T8's effective rate (card says 130/hr; trust nothing until observed). -- DRAIN CAMPAIGN ACTIVE (see CAMPAIGN.md): spend the full balance by 10 Aug, - then cancel before the 12 Aug renewal (user confirmation required for the - cancel itself). 541 of 2,638 spent; 2,097 remain. The x1.6 estimate multiplier - is WITHDRAWN: billing is on plan imaging minutes, and per-scope rates must come - from actual bills (T59 127/hr, T32 80/hr observed; the NOTES rate card is - unreliable). On current numbers the campaign UNDER-spends by ~800: plan extra - or longer runs, do not trim. Reschedule flag never sticks (grid always shows - No): treat every reservation as one-shot and verify the morning after. - Download the NGC 6744 and NGC 5128 data from data.itelescope.net before the - 10 Aug cancellation sweep. Update the CAMPAIGN.md ledger after every session - from account/history.aspx. +### 2. Three decisions waiting on the user + +- **Where should the image archive live?** `C:` has ~2.6 GB free and the + archive is 14 GB after extraction. ~2.4 GB is safely reclaimable (1.69 GB of + extracted TIFFs the pipeline never reads, 0.74 GB of mirror-junk duplicates) + but that only buys headroom, it does not fix it. The pipeline is location + independent via `ASTRO_SESSION`, so moving the tree costs nothing but a copy. + **Do not start batch processing until this is resolved - it will fail + partway through.** +- **NGC 6744 already has outputs** from a different session on 2026-07-20, in + `T59/NGC6744/20260719/stacked/` under a different naming convention + (`NGC6744_LRGB_final.png`, `master_Blue_NGC6744_20260719.fit`). Re-process + into the new structure, or leave it alone? +- **T08 NGC 2070 has one frame per filter.** No stacking, no outlier rejection + possible. Worth an SHO composite anyway, or skip? + +### 3. Drain campaign - runs itself, but needs checking daily + +See `itelescope/CAMPAIGN.md`. Next checkpoints below under "iTelescope". + +--- + +## The archive being processed + +`Downloads\itelescope\` - five sessions, four telescopes, 2021 to 2026. Frame +counts below are AFTER removing the raw/calibrated duplication (see NOTES). + +| Session | Palette | Frames | Note | +|---|---|---|---| +| T32 NGC 5128 | LRGB | 12/4/4/4 | matches the hand-processed run exactly: a working regression test | +| T59 NGC 6744 | LRGB | 15/7/7/7 | **L bin1 4096², RGB bin2 2048²** - mixed grids | +| T20 IC 1805 | Ha only | 14 | mono; **2 frames lost to corrupt archives** | +| T17 NGC 2030 | RGB, no L | 3/3/3 | only 6 min per channel | +| T08 NGC 2070 | SHO | 1/1/1 | one 60 s frame per filter | + +--- + +## Done - 2026-07-21 + +### Repositories +- **Merged `itelescope` + `astro-pipeline` into `astrophotography`**, both + histories preserved (45 commits; `git log --follow` traces files across the + move). Old repos carry redirect notices in README and CLAUDE.md; **not + archived**, because the campaign is live and other sessions work in + `itelescope` most nights. Archive after 12 Aug. + +### NGC 5128 processed end to end (the reference session) +Lives in `Downloads\NGC5128\20260721\`, organised into `raw/ calibrated/ +stacks/ final/ renderings/ science/ intermediates/` with a `METHODS.md` and a +README in each subdirectory. Results: +- Plate solve: 237 stars, **0.27 arcsec residual**, 0.5376 arcsec/px +- Photometry: zero point G = -2.5 log10(flux) + 27.941, scatter 0.164 mag, + limiting **G ~ 20.7** at SNR 5 +- **Four deliverable images** in `final/`, from raw stack to science-informed, + plus a close-up framed by the galaxy's measured extent +- **Globular clusters:** 289 candidates, 179 (62%) catalogue-confirmed, density + falling **10.5x** outward. A real detection. +- **Surface photometry:** Sersic n = 4.27, Re = 400 arcsec, total G = 6.56 + which matches the catalogued value and validates the whole calibration chain. + No shells, and the data could never have shown them (systematics-limited, + 3 mag short). +- **Transients/moving objects:** two clean nulls with measured limits. + +### Corrections made (both were Claude's errors - do not repeat them) +- **The Cen A nucleus was never saturated.** A foreground star 69 arcsec from + the nucleus was measured instead of the galaxy. A T32 booking made to "fix" + it was cancelled before running, costing nothing. Always filter stars out + before measuring a core. +- **NGC 6744 HAD already been processed** by another session on 20 Jul, and the + user was told otherwise. Check for existing outputs before asserting none + exist. + +### Eclipse plan written +`observing/eclipse-2026-menorca/PLAN.md` - full field plan. + +--- ## Pending - pipeline -- Productionise the processing code: a staged CLI - (ingest -> calibrate -> measure -> register -> stack -> solve -> compose -> - analyse), each stage resumable, packaged as an Apptainer image. Slurm array - jobs afterwards and ONLY for the expensive stages: measured on the NGC 5128 - session, per-frame work is 2.9 s and all 24 frames take 1.1 min - single-threaded, so a scheduler buys nothing there. It pays for the Monte - Carlo work (completeness and injection-recovery tests, difference imaging, - shift-and-stack), which is where compute actually bit. +- Finish the stages: measure, register/stack, solve, compose, report, run.py. +- Then productionise: staged CLI, each stage resumable, packaged as an + **Apptainer image**. **Slurm array jobs only for the expensive stages** - + measured on NGC 5128, per-frame work is 2.9 s and all 24 frames take 1.1 min + single-threaded, so a scheduler buys nothing on stacking. It pays for the + Monte Carlo work (completeness and injection-recovery tests, difference + imaging, shift-and-stack), which is where compute actually bit. - Must handle beyond mono LRGB: narrowband palettes, one-shot colour with debayering, other observatories' header conventions, and full calibration from bias/dark/flat for sources that do not pre-calibrate. -- Three findings from the first session are requirements, not options: vet - moving objects in DETECTOR coordinates; carry r50/psf into any catalogue +- **Three findings from the first session are requirements, not options:** + vet moving objects in DETECTOR coordinates; carry r50/psf into any catalogue cross-match; never fit a sky background to a field the target fills. +- Object-prefixed output naming should be derived from the OBJECT header, not + typed in. On a cluster, `final-3-best.png` is ambiguous the moment there is a + second dataset. +- Emit the alignment-only stack as a standard artefact of every run. It costs + almost nothing and it is the reference every later stage can be diffed + against when a processed result looks wrong. + +## Pending - science (deliberately after the pipeline) + +The analyses are per-object and need judgement; they are not push-button. A +globular cluster survey suits Centaurus A and is meaningless for the Heart +Nebula. Approach these one at a time once the imaging half runs unattended. ## Pending - observing -- Total solar eclipse, 12 Aug 2026, Menorca (`observing/eclipse-2026-menorca/`). - Totality 1m12s at Ciutadella with the Sun at 2 degrees, azimuth 288. - **Order the certified solar filter - it is the only item with no substitute - and there are three weeks left.** Then practise on a low sunset to calibrate - exposures for the extreme atmospheric extinction. +- **Total solar eclipse, 12 Aug 2026, Menorca.** Totality 1m12s at Ciutadella, + Sun at **2 degrees**, azimuth 288, sunset 20:42. + `observing/eclipse-2026-menorca/PLAN.md` has the full plan. + - **ORDER THE CERTIFIED SOLAR FILTER.** Only item with no substitute. + - Practise on a low sunset to calibrate exposures for the extreme extinction. + - Outstanding: tailor the exposure table once the A7 body and lenses are known. ## Pending - iTelescope -- Automation: map the ACP web endpoints on a per-scope server (start with free - T33) for plan submission and status; explore lookup.itelescope.online (the new - planner); enumerate more /DataService.svc methods. Recon notes in NOTES.md. -- Campaign impact (status seen 2026-07-18): T71 and T73 are offline - (development/mount testing), as are T2, T17, T18, T20, T25, T30, T72, T74, - T75, T80. T8, T33, T32, T59 at Siding Spring are operational. Re-plan the - July-August campaign around T33 free time and Siding Spring scopes until the - Chile rebuild finishes; re-check status before any booking. - -- Fill in T74 when iTelescope publishes its specs. -- Reconcile the retired scopes (T9, T19, T31, T69): the support article still lists - them but the maintained sheet does not; confirm status and note it. -- Optional: per-telescope detail pages on support.itelescope.net (links in - DECISIONS.md) hold photos and operational history if deeper entries are ever wanted. -- Periodic refresh: re-export the Google Sheet and diff against data/ to catch new or - changed systems. +- **CrA on T8** ran 11:45-14:45 UTC 22 Jul. Verify images under + `/images/qisback/T8/` on t8.itelescope.net:8008, then put the actual bill from + account/history.aspx into the ledger. **First T8 bill, so it fixes T8's + effective rate** (card says 130/hr; trust nothing until observed). +- **T33 first-light test** on its third booking (749314, ran 14:05 UTC 21 Jul). + If it failed again, stop rebooking: either walk it in (POST plan.asp per + `itelescope/plans/README.md`) or drop the free-imaging objective. +- **Campaign:** 541 of 2,638 points spent, 2,097 remain, all to be gone by + 10 Aug with cancellation before the 12 Aug renewal (**user confirmation + required for the cancel itself**). On current numbers the campaign + **under-spends by ~800** - plan extra or longer runs, do not trim. +- Download all data from data.itelescope.net before the 10 Aug sweep. +- Automation: map ACP endpoints per scope; explore lookup.itelescope.online; + enumerate more /DataService.svc methods. +- Fill in T74 when specs are published; reconcile retired scopes T9, T19, T31, + T69; periodically re-export the specs sheet and diff against + `itelescope/data/`.