Merge pull request 'Generic pipeline stage 1, and state files a cold session can resume from' (#1) from generic-pipeline into main

pipeline/session.py

@@ -0,0 +1,353 @@
+"""What a session contains, decided by reading the frames rather than the names.
+
+The first version of this pipeline parsed filenames: it expected
+`calibrated-T32-...-Luminance-BIN2-W-300-001.fit` and broke on anything else.
+That works for exactly one telescope, one binning and one exposure length. Every
+other session in the archive - narrowband from 2024, an Ha-only run from 2021,
+an RGB set with no luminance at all - fails that pattern.
+
+So this module opens each frame and asks the header. FITS carries FILTER,
+EXPTIME, XBINNING, OBJECT and DATE-OBS precisely so that software does not have
+to guess from a filename, and iTelescope fills them in properly.
+
+The unit of work is a *session*: one target, one telescope, one night, in one
+directory. Everything else in the pipeline takes a Session and asks it what
+filters exist and which frames belong to each.
+"""
+import glob
+import json
+import os
+import zipfile
+from collections import defaultdict
+
+from astropy.io import fits
+
+import layout
+
+# Filters that carry broadband colour, in the order a colour image wants them.
+BROADBAND = ("Red", "Green", "Blue")
+LUMINANCE = ("Luminance", "Clear", "Lum", "L")
+# Narrowband, with the aliases different telescopes write into FILTER.
+NARROWBAND = {
+    "Ha": ("Ha", "H-alpha", "Halpha", "HA", "H_Alpha"),
+    "OIII": ("OIII", "O3", "Oiii", "O-III"),
+    "SII": ("SII", "S2", "Sii", "S-II"),
+}
+
+
+def canonical_filter(raw):
+    """Map whatever the header says onto a name the pipeline knows."""
+    if raw is None:
+        return "Unknown"
+    name = str(raw).strip()
+    for canon, aliases in NARROWBAND.items():
+        if name in aliases or name.lower() == canon.lower():
+            return canon
+    for canon in BROADBAND:
+        if name.lower() == canon.lower() or name.lower() == canon[0].lower():
+            return canon
+    if name.lower() in [a.lower() for a in LUMINANCE]:
+        return "Luminance"
+    return name or "Unknown"
+
+
+class Frame:
+    """One light frame and the header facts the pipeline needs."""
+
+    __slots__ = ("path", "filter", "exptime", "binning", "object", "date_obs",
+                 "telescope", "instrument", "shape", "scale", "calibrated")
+
+    def __init__(self, path, header, shape, calibrated=True):
+        self.calibrated = calibrated
+        self.path = path
+        self.filter = canonical_filter(header.get("FILTER"))
+        self.exptime = float(header.get("EXPTIME", header.get("EXPOSURE", 0)))
+        self.binning = int(header.get("XBINNING", 1))
+        self.object = str(header.get("OBJECT", "")).strip()
+        self.date_obs = str(header.get("DATE-OBS", ""))
+        self.telescope = str(header.get("TELESCOP", "")).strip()
+        self.instrument = str(header.get("INSTRUME", "")).strip()
+        self.shape = shape
+        # iTelescope writes the plate scale in; without it, solve.py estimates.
+        scale = header.get("HIERARCH iTelescopePlateScaleH")
+        try:
+            self.scale = float(scale) if scale is not None else None
+        except (TypeError, ValueError):
+            self.scale = None
+
+    @property
+    def name(self):
+        return os.path.basename(self.path)
+
+    def __repr__(self):
+        return (f"<{self.filter} {self.exptime:.0f}s bin{self.binning} "
+                f"{self.name}>")
+
+
+class Session:
+    """One target, one telescope, one night."""
+
+    def __init__(self, root):
+        self.root = os.path.abspath(root)
+        self.frames = []
+        self._by_filter = {}
+
+    # ---------------------------------------------------------------- ingest
+    @staticmethod
+    def _is_mirror_junk(path):
+        """True for artefacts a broken segmented download leaves behind.
+
+        Cyberduck writes a DIRECTORY named after the file it is fetching, and
+        leaves `.cyberducksegment` parts inside when interrupted. Those
+        directories are named `something.fit`, so anything that trusts the
+        extension will try to open a directory as a frame. A re-mirror over an
+        already-organised tree produces dozens of them.
+        """
+        name = os.path.basename(path).lower()
+        if ".cyberducksegment" in name or name.endswith((".part", ".partial")):
+            return True
+        parent = os.path.basename(os.path.dirname(path)).lower()
+        return parent.endswith((".fit", ".fits", ".tif", ".tiff", ".zip"))
+
+    def ingest(self, verbose=True):
+        """Put the session into the standard layout and uncompress the lights.
+
+        Handles the three states a downloaded session can be in: still zipped,
+        already extracted, or a mixture of both (which is what happens when
+        someone unzips a few by hand).
+        """
+        raw = os.path.join(self.root, layout.RAW)
+        cal = os.path.join(self.root, layout.CALIBRATED)
+        os.makedirs(raw, exist_ok=True)
+        os.makedirs(cal, exist_ok=True)
+
+        moved = extracted = skipped = duplicates = 0
+        # Loose files at the session root get filed first.
+        for path in sorted(glob.glob(os.path.join(self.root, "*"))):
+            if os.path.isdir(path):
+                continue
+            name = os.path.basename(path)
+            low = name.lower()
+            if self._is_mirror_junk(path):
+                continue
+            if low.endswith(".zip") or (low.startswith("jpeg-") and
+                                        low.endswith(".jpg")):
+                dest = os.path.join(raw, name)
+            elif low.endswith((".fit", ".fits", ".tif", ".tiff")):
+                dest = os.path.join(cal, name)
+            else:
+                continue
+            if not os.path.exists(dest):
+                os.replace(path, dest)
+                moved += 1
+            elif os.path.getsize(dest) == os.path.getsize(path):
+                # Already filed, and the mirror has put an identical copy back.
+                os.remove(path)
+                duplicates += 1
+
+        # iTelescope ships each frame twice: 'calibrated-*' with bias, dark
+        # and flat applied, and 'raw-*' without. Extracting both put two copies
+        # of every frame into the stack. Only the calibrated set is extracted;
+        # the raw archives stay zipped in raw/ as the untouched originals. If a
+        # session has no calibrated archives at all, the raw ones are used and
+        # the fact is recorded.
+        archives = sorted(glob.glob(os.path.join(raw, "*.zip")))
+        calibrated_archives = [z for z in archives
+                               if os.path.basename(z).lower().startswith(
+                                   "calibrated")]
+        self.used_raw = not calibrated_archives and bool(archives)
+        if self.used_raw:
+            print("    NOTE: no calibrated archives in this session - "
+                  "extracting the raw frames instead. They have NOT had bias, "
+                  "dark or flat correction applied.")
+        else:
+            archives = calibrated_archives
+
+        for zpath in archives:
+            try:
+                with zipfile.ZipFile(zpath) as zf:
+                    for info in zf.infolist():
+                        base = os.path.basename(info.filename)
+                        if not base.lower().endswith((".fit", ".fits", ".tif",
+                                                      ".tiff")):
+                            continue
+                        target = os.path.join(cal, base)
+                        if os.path.exists(target) and \
+                                os.path.getsize(target) == info.file_size:
+                            skipped += 1
+                            continue
+                        with zf.open(info) as src, open(target, "wb") as dst:
+                            while chunk := src.read(1 << 20):
+                                dst.write(chunk)
+                        extracted += 1
+            except zipfile.BadZipFile:
+                print(f"    WARNING: {os.path.basename(zpath)} is not a "
+                      f"readable zip - skipped")
+        if verbose:
+            print(f"    filed {moved}, extracted {extracted}, "
+                  f"already present {skipped}, "
+                  f"duplicate copies removed {duplicates}")
+        return dict(moved=moved, extracted=extracted, skipped=skipped,
+                    duplicates=duplicates)
+
+    # ----------------------------------------------------------------- scan
+    def scan(self, verbose=True):
+        """Read every calibrated light frame's header."""
+        cal = os.path.join(self.root, layout.CALIBRATED)
+        paths = sorted(p for p in glob.glob(os.path.join(cal, "*.fit")) +
+                       glob.glob(os.path.join(cal, "*.fits"))
+                       if os.path.isfile(p) and not self._is_mirror_junk(p))
+        self.frames = []
+        for path in paths:
+            try:
+                with fits.open(path, memmap=True) as hd:
+                    hdr = hd[0].header
+                    shape = hd[0].shape
+            except Exception as exc:                          # noqa: BLE001
+                print(f"    WARNING: cannot read {os.path.basename(path)}: "
+                      f"{exc}")
+                continue
+            # Skip anything that is not a light frame.
+            if str(hdr.get("IMAGETYP", "Light")).lower().startswith(
+                    ("dark", "bias", "flat")):
+                continue
+            self.frames.append(Frame(path, hdr,
+                                     shape, calibrated=bool(
+                                         str(hdr.get("CALSTAT", "")).strip())))
+
+        # If both calibrated and uncalibrated frames are present - which
+        # happens when a session was extracted before this rule existed - keep
+        # only the calibrated ones rather than stacking each frame twice.
+        if any(f.calibrated for f in self.frames):
+            dropped = [f for f in self.frames if not f.calibrated]
+            if dropped and verbose:
+                print(f"    ignoring {len(dropped)} uncalibrated duplicate "
+                      f"frames (no CALSTAT)")
+            self.frames = [f for f in self.frames if f.calibrated]
+
+        self._by_filter = defaultdict(list)
+        for f in self.frames:
+            self._by_filter[f.filter].append(f)
+        if verbose:
+            self.describe()
+        return self.frames
+
+    # ------------------------------------------------------------ properties
+    @property
+    def filters(self):
+        """Filters present, deepest first - the deepest becomes the reference."""
+        return sorted(self._by_filter,
+                      key=lambda f: -sum(x.exptime for x in self._by_filter[f]))
+
+    def by_filter(self, name):
+        return list(self._by_filter.get(name, []))
+
+    @property
+    def target(self):
+        names = [f.object for f in self.frames if f.object]
+        return max(set(names), key=names.count) if names else \
+            os.path.basename(os.path.dirname(self.root))
+
+    @property
+    def telescope(self):
+        names = [f.telescope for f in self.frames if f.telescope]
+        return max(set(names), key=names.count) if names else "unknown"
+
+    @property
+    def scale(self):
+        vals = [f.scale for f in self.frames if f.scale]
+        return sum(vals) / len(vals) if vals else None
+
+    @property
+    def palette(self):
+        """Which colour scheme this session's filters support."""
+        present = set(self._by_filter)
+        has_lum = "Luminance" in present
+        broad = [c for c in BROADBAND if c in present]
+        narrow = [n for n in ("SII", "Ha", "OIII") if n in present]
+        if len(broad) == 3 and has_lum:
+            return "LRGB"
+        if len(broad) == 3:
+            return "RGB"
+        if len(narrow) >= 3:
+            return "SHO"
+        if "Ha" in narrow and "OIII" in narrow:
+            return "HOO"
+        if len(present) == 1:
+            return "MONO"
+        if broad or narrow:
+            return "PARTIAL"
+        return "UNKNOWN"
+
+    def describe(self):
+        print(f"    target      {self.target}")
+        print(f"    telescope   {self.telescope}")
+        if self.scale:
+            print(f"    plate scale {self.scale:.4f} arcsec/px")
+        print(f"    palette     {self.palette}")
+        for name in self.filters:
+            fs = self._by_filter[name]
+            exps = sorted({f.exptime for f in fs})
+            bins = sorted({f.binning for f in fs})
+            total = sum(f.exptime for f in fs) / 60.0
+            shapes = sorted({f.shape for f in fs})
+            shape_txt = "/".join(f"{s[1]}x{s[0]}" for s in shapes)
+            print(f"      {name:10s} {len(fs):3d} frames  "
+                  f"{'/'.join(f'{e:.0f}' for e in exps):>12s} s  "
+                  f"bin{'/'.join(str(b) for b in bins)}  "
+                  f"{shape_txt:>11s}  {total:6.1f} min")
+
+    # ------------------------------------------------------------- manifest
+    def manifest(self):
+        return {
+            "root": self.root,
+            "target": self.target,
+            "telescope": self.telescope,
+            "plate_scale": self.scale,
+            "palette": self.palette,
+            "filters": {
+                name: {
+                    "frames": len(self._by_filter[name]),
+                    "total_exposure_s": sum(f.exptime
+                                            for f in self._by_filter[name]),
+                    "exposures": sorted({f.exptime
+                                         for f in self._by_filter[name]}),
+                    "binning": sorted({f.binning
+                                       for f in self._by_filter[name]}),
+                } for name in self.filters
+            },
+        }
+
+    def save_manifest(self):
+        path = os.path.join(self.root, "manifest.json")
+        with open(path, "w", encoding="utf-8") as fh:
+            json.dump(self.manifest(), fh, indent=2)
+        return path
+
+
+def discover(root):
+    """Find every session under a directory tree.
+
+    A session is any directory that directly contains light frames or their
+    archives, which matches how iTelescope's downloads are organised
+    (telescope / object / date).
+    """
+    found = []
+    for dirpath, dirnames, filenames in os.walk(root):
+        base = os.path.basename(dirpath)
+        if base.lower().endswith((".fit", ".fits", ".tif", ".tiff", ".zip"))                 or ".cyberducksegment" in base.lower():
+            dirnames[:] = []
+            continue
+        if base in (layout.RAW, layout.CALIBRATED, "intermediates",
+                    "final", "renderings", "science", "stacks"):
+            dirnames[:] = []
+            continue
+        has_data = any(f.lower().endswith((".zip", ".fit", ".fits"))
+                       for f in filenames)
+        has_layout = any(d in dirnames for d in (layout.RAW,
+                                                 layout.CALIBRATED))
+        if has_data or has_layout:
+            found.append(dirpath)
+            dirnames[:] = [d for d in dirnames
+                           if d in (layout.RAW, layout.CALIBRATED)]
+    return sorted(found)

state/ARCHITECTURE.md

@@ -3,14 +3,73 @@
 > How the system is built and why. Update this when the structure changes; a change is
 > not finished until this reflects it.
 
-A documentation-only repo; there is no code.
+Three parts, which is why the repository has three top-level content
+directories. Until 2026-07-21 the first two lived in separate repositories.
 
-- `TELESCOPES.md`: the deliverable. Per-observatory sections, one entry per telescope
+## itelescope/ - the network and the campaign
-  (spec bullets then a review paragraph), a "choosing a telescope" table, and general
+
-  observations. Compiled from the sources below, dated in its header.
+Documentation and data, no code.
-- `data/itelescope-telescopes.csv`: verbatim CSV export of iTelescope's maintained
+
-  specs Google Sheet. Treated as the source of truth for numbers; re-exported to
+- `TELESCOPES.md`: the deliverable. Per-observatory sections, one entry per
-  refresh (see state/NOTES.md).
+  telescope (spec bullets then a review paragraph), a "choosing a telescope"
-- `state/`: Default Workflow project memory.
+  table, and general observations. Compiled from the sources below.
-- `CLAUDE.md` + `docs/`: the Default Workflow itself, copied from the
+- `TARGETS.md`: southern targets invisible from the UK, matched to scopes and
-  Default-Workflow repo so the project is self-contained.
+  to the time of year.
+- `CAMPAIGN.md`: the points drain campaign - ledger, bookings, queue, rate
+  model. Updated after every session from the account's transaction history.
+- `data/itelescope-telescopes.csv`: verbatim CSV export of iTelescope's
+  maintained specs sheet. The source of truth for numbers; re-exported to
+  refresh (see `state/NOTES.md`).
+- `plans/`: ACP observing plans, one per booked run, uploaded to the scope.
+
+## pipeline/ - the processing code
+
+Python 3.12. The design rule is that **code and data are separate**: nothing
+here knows an absolute path. A session is located at runtime through the
+`ASTRO_SESSION` environment variable.
+
+- `layout.py` maps a **filename** to the subdirectory it belongs in, using the
+  same rules the session directories are organised with. A script asks for
+  `master-Red.fit` or `_stars.npz` and gets the right path without any call
+  site knowing the structure. Unrecognised names resolve to the session root,
+  which is visible and correctable rather than silently wrong.
+- `session.py` is the entry point for anything generic. It discovers sessions,
+  ingests and uncompresses them, reads every frame's header, groups frames by
+  filter, and derives the colour scheme (LRGB / RGB / SHO / HOO / MONO) from
+  what is actually present. **Header-driven, never filename-driven** - see
+  DECISIONS.
+- `restructure.py` reorganises a flat session directory into the standard
+  layout. Idempotent, dry run by default.
+- The remaining scripts are the NGC 5128 session as it was actually run:
+  `unzip -> analyse -> stack -> solve -> depth -> compose -> hdr/enhance/
+  starless/annotate -> final -> closeup -> triptych`, plus the analysis
+  families `gc-*` (globular clusters), `sb_*` (surface photometry) and `mo_*`
+  (moving objects and transients). They are kept as provenance for published
+  results and are being generalised stage by stage.
+
+### Session layout on disk (not in this repo)
+
+```
+<session>/
+  METHODS.md            what was done to this data and what was found
+  raw/                  exactly what the telescope delivered
+  calibrated/           uncompressed calibrated subs
+  stacks/masters/       per-filter registered, combined, plate-solved
+  stacks/original/      alignment-only baseline, no other processing
+  final/                the deliverable renderings
+  renderings/           other finished images
+  science/figures|catalogues|data|notes
+  intermediates/        caches; deletable, regenerated by a re-run
+```
+
+## observing/ - in-person plans
+
+Field plans worked out from real numbers, revised as a date approaches. The
+reasoning is the part worth keeping, so plans record why a decision was made,
+not only what was decided.
+
+## The workflow itself
+
+`CLAUDE.md` + `docs/`: the Default Workflow, copied in so the project is
+self-contained. `state/`: project memory - objective, current work, decisions,
+notes.

state/DECISIONS.md

@@ -3,6 +3,33 @@
 > A dated, append only log of decisions and their rationale. Newest at the top. Never
 > rewrite past entries; if a decision is reversed, add a new entry that says so.
 
+## 2026-07-21: the pipeline reads headers, not filenames
+
+The first pipeline parsed `calibrated-T32-...-Luminance-BIN2-W-300-001.fit` to
+learn a frame's filter, binning and exposure. That works for one telescope, one
+binning and one exposure length. Three of the four archived sessions have no
+luminance channel at all and one is from 2021 with different naming, so the
+approach could not survive contact with the archive.
+
+FITS headers carry FILTER, EXPTIME, XBINNING, OBJECT and DATE-OBS precisely so
+software does not have to guess, and iTelescope fills them in properly. The
+pipeline opens each frame and asks. Filter aliases are mapped, so H-alpha,
+Halpha and HA all resolve to Ha.
+
+The colour scheme is then derived from what is present rather than assumed:
+LRGB, RGB without luminance, SHO, HOO, or mono. That is what lets one command
+handle a galaxy in LRGB and a nebula in narrowband without being told which is
+which.
+
+## 2026-07-21: push-button imaging, hand-driven science
+
+The imaging half - unzip, measure, register, stack, plate solve, compose - is
+mechanical and should run unattended on any session. The analyses are not: a
+globular cluster survey suits an elliptical galaxy and is meaningless for an
+emission nebula, and much of the value in the NGC 5128 work came from noticing
+when a result was wrong rather than from running the code. So the pipeline is
+being built to be push-button, and the science stays step by step.
+
 ## 2026-07-21: merge itelescope and astro-pipeline into one repository
 
 The two were split by accident of chronology rather than by design. `itelescope`

state/NOTES.md

@@ -78,3 +78,72 @@ Paid imaging needs no reservation when a scope is idle: same walk-in ACP flow
 as the free scopes, points just deduct. Reservations (to guarantee a slot) go
 through lookup.itelescope.online, which is Cloudflare-protected: Playwright
 needed.
+
+
+## Processing the downloaded archive (2026-07-21)
+
+Gotchas found by running code against the real archive rather than reasoning
+about it. Each one produced a plausible-looking wrong answer, not an error.
+
+- **Every frame is delivered twice.** iTelescope ships `calibrated-*` (bias,
+  dark and flat applied) and `raw-*` (not). Extracting both puts two copies of
+  every frame into the stack, and the counts look merely generous rather than
+  wrong: NGC 5128 scanned as 24 luminance frames when it has 12. `CALSTAT` in
+  the header is the reliable discriminator - calibrated frames carry it, raw
+  frames do not. Only the calibrated archives are extracted; the raw archives
+  stay zipped as the untouched originals.
+
+- **An interrupted segmented download leaves DIRECTORIES named like files.**
+  Cyberduck creates a directory called `something.fit` while fetching, with
+  `.cyberducksegment` parts inside. Anything that trusts a file extension will
+  try to open a directory as a frame. After a re-mirror over an already
+  organised tree there were 28 of them. Discovery, ingest and scan all skip
+  them explicitly.
+
+- **A re-mirror duplicates rather than replaces.** Mirroring on top of an
+  organised tree restored ~1 GB of files to session roots that had already been
+  filed into `raw/`. Ingest now removes byte-identical copies as it refiles.
+
+- **Two T20 archives are genuinely corrupt** and cannot be extracted
+  (`ic1805-...-Ha-...-015` and `-016`). That is real data loss, not a bug: the
+  session has 14 of 16 frames. The warning is printed rather than swallowed.
+
+- **Extraction triples the disk footprint.** The archive went from 6.1 GB of
+  zips to 14 GB extracted, which filled the drive. The TIFFs are 1.69 GB of
+  that and the pipeline never reads them - it only uses the FITS.
+
+- **Channels do not always share a pixel grid.** NGC 6744 was shot with
+  luminance at bin1 (4096x4096) and colour at bin2 (2048x2048). Registration
+  has to resample onto a common grid; code that assumes a single frame shape
+  will fail on this session and only this session.
+
+## Processing lessons from NGC 5128 (2026-07-21)
+
+Kept because each is a way to get a confident wrong answer:
+
+- **Measure a core with the stars filtered out.** Taking the maximum pixel in a
+  box around the frame centre found a foreground star, not the galaxy, and led
+  to booking telescope time to fix a saturation problem that did not exist.
+
+- **Registration makes sensor defects look like perfect asteroids.** Holding
+  the sky still drags anything detector-fixed across the frame on a dead
+  straight, constant-rate, constant-brightness track. Hot pixels pass every
+  standard quality cut. Requiring the motion to be real in DETECTOR coordinates
+  took 141 confident detections to zero.
+
+- **Never compare an aperture magnitude to a point-source catalogue without
+  checking the source is a point.** A resolved object looks exactly like a
+  2.8 magnitude outburst. Carry r50/psf through to the cross-match.
+
+- **Never fit a sky background to a field the target fills.** A plane fitted
+  around a large galaxy absorbs its halo, measured at -17.9 ADU/px. Fit the
+  background and a source model together.
+
+- **"No Gaia counterpart" is not a star-rejection test** at extragalactic
+  distances: half of Centaurus A's catalogued globular clusters have a Gaia
+  source, because a cluster at 3.8 Mpc is a point source. Reject on parallax
+  and proper motion instead.
+
+- **A coarse background mesh fails in the direction that flatters you.** It
+  destroys detection near a bright galaxy while leaving the outer field intact,
+  manufacturing a falsely flat radial profile.

state/TODO.md

@@ -1,121 +1,163 @@
 # TODO
 
-> The current state of play. Keep it honest and current; this is what the next session
+> The current state of play. Keep it honest and current; this is what the next
-> reads to know what to do. Move items between sections as they progress.
+> session reads to know what to do. Move items between sections as they progress.
 
-## Done
+---
 
-- 2026-07-17: repo created on git.discworld.casa, Default Workflow scaffold copied in.
+## RESUME HERE - 2026-07-21, end of session
-- 2026-07-17: specs sheet snapshotted to data/itelescope-telescopes.csv (24 scopes).
-- 2026-07-17: TELESCOPES.md written: per-scope reviews for all 23 active telescopes
-  plus a choosing guide (T74 noted as in commissioning, no specs published).
-- 2026-07-18: TARGETS.md written: southern targets invisible from the UK (dec tiers
-  from lat 51.5°N), matched to Q62/X07 scopes, seasonal booking calendar, and a
-  July-August starter campaign (T33 free, T71 narrowband, T73 galaxy, T8 LMC later).
 
-## Done (recent)
+Three strands are live. Each has a clear next action.
 
-- 2026-07-20: NGC 6744 on T59 SUCCEEDED (reservation 749227, txn 851148): full
+### 1. Generic pipeline - MID-BUILD, this is the active work
-  LRGB dataset on the scope server (L 15x300s BIN1, RGB 7x300s each BIN2),
-  previews verified clean. Billed 381 pts (est was ~240: see the rate
-  correction in NOTES.md). Balance now 2,257. Ledger updated in CAMPAIGN.md.
 
-## Done (recent)
+Building a push-button pipeline that processes any session, not just LRGB from
+one telescope. Branch `generic-pipeline`.
 
-- 2026-07-21 ~08:55 UTC: free T33 47 Tuc retry booked as reservation 749314
+**Done:** `pipeline/session.py` - discovery, ingest, header-driven scanning,
-  (grid-verified), third attempt at the first-light test, runs 14:05-14:50 UTC
+filter grouping, palette detection (LRGB / RGB / SHO / HOO / MONO), manifest
-  21 Jul, costs 0.
+writing. Tested against all five archived sessions.
 
-## Done (recent)
+**Next:** `measure` -> `register/stack` -> `solve` -> `compose` -> `report`,
+then a `run.py` orchestrator with resumable stages.
 
-- 2026-07-21 11:40 UTC: Cen A on T32 verified COMPLETE (billed 160, NGC5128
+**The blocker before any of that:** the disk is full (see below).
-  images on the scope server), and queue #2 (Corona Australis on T8) booked for
-  the night of 22 Jul, 21:45-00:45 SSO local, which brackets the target's
-  culmination instead of chasing it down the western sky. Rate model rebuilt
-  from two actual bills. Details in the CAMPAIGN.md session note.
 
-## In progress
+**The known hard part:** NGC 6744 mixes bin1 luminance at 4096x4096 with bin2
+colour at 2048x2048. Registration must resample onto a common grid rather than
+assume every frame shares a shape. Nothing else in the archive does this, so it
+is easy to forget and it will fail loudly when it happens.
 
-- T33 first-light test on its THIRD booking (749314), which had NOT yet run as of
+### 2. Three decisions waiting on the user
-  11:40 UTC 21 Jul: it starts 14:05 UTC. Verify after 14:50 UTC (images and logs
+
-  on t33.itelescope.net:8033 were still empty/stale at 11:40, as expected). The
+- **Where should the image archive live?** `C:` has ~2.6 GB free and the
-  two prior free reservations (749228, 749229) expired unused with reschedule=No;
+  archive is 14 GB after extraction. ~2.4 GB is safely reclaimable (1.69 GB of
-  if 749314 also fails to run, stop rebooking and either walk it in (POST
+  extracted TIFFs the pipeline never reads, 0.74 GB of mirror-junk duplicates)
-  plan.asp per plans/README.md) or drop the free objective.
+  but that only buys headroom, it does not fix it. The pipeline is location
-- The T32 HDR core set (plans/hdrcore-t32.txt) was booked and then CANCELLED on
+  independent via `ASTRO_SESSION`, so moving the tree costs nothing but a copy.
-  2026-07-21 without running: the Cen A nucleus is not saturated, the earlier
+  **Do not start batch processing until this is resolved - it will fail
-  claim having measured a foreground star. See the correction in CAMPAIGN.md.
+  partway through.**
-  Zero points spent. The plan file remains available on T32 if a genuinely
+- **NGC 6744 already has outputs** from a different session on 2026-07-20, in
-  saturated target turns up.
+  `T59/NGC6744/20260719/stacked/` under a different naming convention
-- The 21 Jul Cen A data has been fully processed off-repo, in the user's
+  (`NGC6744_LRGB_final.png`, `master_Blue_NGC6744_20260719.fit`). Re-process
-  Downloads\NGC5128\20260721\stacked folder: masters, plate solution (0.27
+  into the new structure, or leave it alone?
-  arcsec on 237 stars), an LRGB composite, a globular cluster survey (289
+- **T08 NGC 2070 has one frame per filter.** No stacking, no outlier rejection
-  candidates, 62% catalogue-confirmed, density falling 10.5x outward), surface
+  possible. Worth an SHO composite anyway, or skip?
-  photometry (Sersic n = 4.27, Re = 400 arcsec, total G = 6.56, which matches
+
-  the catalogued brightness and so validates the photometric chain), and a
+### 3. Drain campaign - runs itself, but needs checking daily
-  transient/moving-object search (both null, with measured limits). METHODS.md
+
-  there documents the lot. Nothing from that analysis needs to come back into
+See `itelescope/CAMPAIGN.md`. Next checkpoints below under "iTelescope".
-  this repo, but two results affect FUTURE sessions and are recorded here:
+
-  **(a)** measure saturation properly before buying short subs, and **(b)** the
+---
-  faint outer halo in that stack is limited by the sky-plane subtraction
+
-  (-17.9 ADU/px of real halo light absorbed), not by exposure time, so more
+## The archive being processed
-  integration alone would not have gone deeper.
+
-- CrA on T8 runs 11:45-14:45 UTC 22 Jul. Verify after 14:45 UTC: images under
+`Downloads\itelescope\` - five sessions, four telescopes, 2021 to 2026. Frame
-  /images/qisback/T8/ on t8.itelescope.net:8008, then the actual bill from
+counts below are AFTER removing the raw/calibrated duplication (see NOTES).
-  account/history.aspx into the ledger. This is the first T8 bill, so it also
+
-  fixes T8's effective rate (card says 130/hr; trust nothing until observed).
+| Session | Palette | Frames | Note |
-- DRAIN CAMPAIGN ACTIVE (see CAMPAIGN.md): spend the full balance by 10 Aug,
+|---|---|---|---|
-  then cancel before the 12 Aug renewal (user confirmation required for the
+| T32 NGC 5128 | LRGB | 12/4/4/4 | matches the hand-processed run exactly: a working regression test |
-  cancel itself). 541 of 2,638 spent; 2,097 remain. The x1.6 estimate multiplier
+| T59 NGC 6744 | LRGB | 15/7/7/7 | **L bin1 4096², RGB bin2 2048²** - mixed grids |
-  is WITHDRAWN: billing is on plan imaging minutes, and per-scope rates must come
+| T20 IC 1805 | Ha only | 14 | mono; **2 frames lost to corrupt archives** |
-  from actual bills (T59 127/hr, T32 80/hr observed; the NOTES rate card is
+| T17 NGC 2030 | RGB, no L | 3/3/3 | only 6 min per channel |
-  unreliable). On current numbers the campaign UNDER-spends by ~800: plan extra
+| T08 NGC 2070 | SHO | 1/1/1 | one 60 s frame per filter |
-  or longer runs, do not trim. Reschedule flag never sticks (grid always shows
+
-  No): treat every reservation as one-shot and verify the morning after.
+---
-  Download the NGC 6744 and NGC 5128 data from data.itelescope.net before the
+
-  10 Aug cancellation sweep. Update the CAMPAIGN.md ledger after every session
+## Done - 2026-07-21
-  from account/history.aspx.
+
+### Repositories
+- **Merged `itelescope` + `astro-pipeline` into `astrophotography`**, both
+  histories preserved (45 commits; `git log --follow` traces files across the
+  move). Old repos carry redirect notices in README and CLAUDE.md; **not
+  archived**, because the campaign is live and other sessions work in
+  `itelescope` most nights. Archive after 12 Aug.
+
+### NGC 5128 processed end to end (the reference session)
+Lives in `Downloads\NGC5128\20260721\`, organised into `raw/ calibrated/
+stacks/ final/ renderings/ science/ intermediates/` with a `METHODS.md` and a
+README in each subdirectory. Results:
+- Plate solve: 237 stars, **0.27 arcsec residual**, 0.5376 arcsec/px
+- Photometry: zero point G = -2.5 log10(flux) + 27.941, scatter 0.164 mag,
+  limiting **G ~ 20.7** at SNR 5
+- **Four deliverable images** in `final/`, from raw stack to science-informed,
+  plus a close-up framed by the galaxy's measured extent
+- **Globular clusters:** 289 candidates, 179 (62%) catalogue-confirmed, density
+  falling **10.5x** outward. A real detection.
+- **Surface photometry:** Sersic n = 4.27, Re = 400 arcsec, total G = 6.56
+  which matches the catalogued value and validates the whole calibration chain.
+  No shells, and the data could never have shown them (systematics-limited,
+  3 mag short).
+- **Transients/moving objects:** two clean nulls with measured limits.
+
+### Corrections made (both were Claude's errors - do not repeat them)
+- **The Cen A nucleus was never saturated.** A foreground star 69 arcsec from
+  the nucleus was measured instead of the galaxy. A T32 booking made to "fix"
+  it was cancelled before running, costing nothing. Always filter stars out
+  before measuring a core.
+- **NGC 6744 HAD already been processed** by another session on 20 Jul, and the
+  user was told otherwise. Check for existing outputs before asserting none
+  exist.
+
+### Eclipse plan written
+`observing/eclipse-2026-menorca/PLAN.md` - full field plan.
+
+---
 
 ## Pending - pipeline
 
-- Productionise the processing code: a staged CLI
+- Finish the stages: measure, register/stack, solve, compose, report, run.py.
-  (ingest -> calibrate -> measure -> register -> stack -> solve -> compose ->
+- Then productionise: staged CLI, each stage resumable, packaged as an
-  analyse), each stage resumable, packaged as an Apptainer image. Slurm array
+  **Apptainer image**. **Slurm array jobs only for the expensive stages** -
-  jobs afterwards and ONLY for the expensive stages: measured on the NGC 5128
+  measured on NGC 5128, per-frame work is 2.9 s and all 24 frames take 1.1 min
-  session, per-frame work is 2.9 s and all 24 frames take 1.1 min
+  single-threaded, so a scheduler buys nothing on stacking. It pays for the
-  single-threaded, so a scheduler buys nothing there. It pays for the Monte
+  Monte Carlo work (completeness and injection-recovery tests, difference
-  Carlo work (completeness and injection-recovery tests, difference imaging,
+  imaging, shift-and-stack), which is where compute actually bit.
-  shift-and-stack), which is where compute actually bit.
 - Must handle beyond mono LRGB: narrowband palettes, one-shot colour with
   debayering, other observatories' header conventions, and full calibration
   from bias/dark/flat for sources that do not pre-calibrate.
-- Three findings from the first session are requirements, not options: vet
+- **Three findings from the first session are requirements, not options:**
-  moving objects in DETECTOR coordinates; carry r50/psf into any catalogue
+  vet moving objects in DETECTOR coordinates; carry r50/psf into any catalogue
   cross-match; never fit a sky background to a field the target fills.
+- Object-prefixed output naming should be derived from the OBJECT header, not
+  typed in. On a cluster, `final-3-best.png` is ambiguous the moment there is a
+  second dataset.
+- Emit the alignment-only stack as a standard artefact of every run. It costs
+  almost nothing and it is the reference every later stage can be diffed
+  against when a processed result looks wrong.
+
+## Pending - science (deliberately after the pipeline)
+
+The analyses are per-object and need judgement; they are not push-button. A
+globular cluster survey suits Centaurus A and is meaningless for the Heart
+Nebula. Approach these one at a time once the imaging half runs unattended.
 
 ## Pending - observing
 
-- Total solar eclipse, 12 Aug 2026, Menorca (`observing/eclipse-2026-menorca/`).
+- **Total solar eclipse, 12 Aug 2026, Menorca.** Totality 1m12s at Ciutadella,
-  Totality 1m12s at Ciutadella with the Sun at 2 degrees, azimuth 288.
+  Sun at **2 degrees**, azimuth 288, sunset 20:42.
-  **Order the certified solar filter - it is the only item with no substitute
+  `observing/eclipse-2026-menorca/PLAN.md` has the full plan.
-  and there are three weeks left.** Then practise on a low sunset to calibrate
+  - **ORDER THE CERTIFIED SOLAR FILTER.** Only item with no substitute.
-  exposures for the extreme atmospheric extinction.
+  - Practise on a low sunset to calibrate exposures for the extreme extinction.
+  - Outstanding: tailor the exposure table once the A7 body and lenses are known.
 
 ## Pending - iTelescope
 
-- Automation: map the ACP web endpoints on a per-scope server (start with free
+- **CrA on T8** ran 11:45-14:45 UTC 22 Jul. Verify images under
-  T33) for plan submission and status; explore lookup.itelescope.online (the new
+  `/images/qisback/T8/` on t8.itelescope.net:8008, then put the actual bill from
-  planner); enumerate more /DataService.svc methods. Recon notes in NOTES.md.
+  account/history.aspx into the ledger. **First T8 bill, so it fixes T8's
-- Campaign impact (status seen 2026-07-18): T71 and T73 are offline
+  effective rate** (card says 130/hr; trust nothing until observed).
-  (development/mount testing), as are T2, T17, T18, T20, T25, T30, T72, T74,
+- **T33 first-light test** on its third booking (749314, ran 14:05 UTC 21 Jul).
-  T75, T80. T8, T33, T32, T59 at Siding Spring are operational. Re-plan the
+  If it failed again, stop rebooking: either walk it in (POST plan.asp per
-  July-August campaign around T33 free time and Siding Spring scopes until the
+  `itelescope/plans/README.md`) or drop the free-imaging objective.
-  Chile rebuild finishes; re-check status before any booking.
+- **Campaign:** 541 of 2,638 points spent, 2,097 remain, all to be gone by
-
+  10 Aug with cancellation before the 12 Aug renewal (**user confirmation
-- Fill in T74 when iTelescope publishes its specs.
+  required for the cancel itself**). On current numbers the campaign
-- Reconcile the retired scopes (T9, T19, T31, T69): the support article still lists
+  **under-spends by ~800** - plan extra or longer runs, do not trim.
-  them but the maintained sheet does not; confirm status and note it.
+- Download all data from data.itelescope.net before the 10 Aug sweep.
-- Optional: per-telescope detail pages on support.itelescope.net (links in
+- Automation: map ACP endpoints per scope; explore lookup.itelescope.online;
-  DECISIONS.md) hold photos and operational history if deeper entries are ever wanted.
+  enumerate more /DataService.svc methods.
-- Periodic refresh: re-export the Google Sheet and diff against data/ to catch new or
+- Fill in T74 when specs are published; reconcile retired scopes T9, T19, T31,
-  changed systems.
+  T69; periodically re-export the specs sheet and diff against
+  `itelescope/data/`.