diff --git a/pipeline/session.py b/pipeline/session.py new file mode 100644 index 0000000..36a3a00 --- /dev/null +++ b/pipeline/session.py @@ -0,0 +1,353 @@ +"""What a session contains, decided by reading the frames rather than the names. + +The first version of this pipeline parsed filenames: it expected +`calibrated-T32-...-Luminance-BIN2-W-300-001.fit` and broke on anything else. +That works for exactly one telescope, one binning and one exposure length. Every +other session in the archive - narrowband from 2024, an Ha-only run from 2021, +an RGB set with no luminance at all - fails that pattern. + +So this module opens each frame and asks the header. FITS carries FILTER, +EXPTIME, XBINNING, OBJECT and DATE-OBS precisely so that software does not have +to guess from a filename, and iTelescope fills them in properly. + +The unit of work is a *session*: one target, one telescope, one night, in one +directory. Everything else in the pipeline takes a Session and asks it what +filters exist and which frames belong to each. +""" +import glob +import json +import os +import zipfile +from collections import defaultdict + +from astropy.io import fits + +import layout + +# Filters that carry broadband colour, in the order a colour image wants them. +BROADBAND = ("Red", "Green", "Blue") +LUMINANCE = ("Luminance", "Clear", "Lum", "L") +# Narrowband, with the aliases different telescopes write into FILTER. +NARROWBAND = { + "Ha": ("Ha", "H-alpha", "Halpha", "HA", "H_Alpha"), + "OIII": ("OIII", "O3", "Oiii", "O-III"), + "SII": ("SII", "S2", "Sii", "S-II"), +} + + +def canonical_filter(raw): + """Map whatever the header says onto a name the pipeline knows.""" + if raw is None: + return "Unknown" + name = str(raw).strip() + for canon, aliases in NARROWBAND.items(): + if name in aliases or name.lower() == canon.lower(): + return canon + for canon in BROADBAND: + if name.lower() == canon.lower() or name.lower() == canon[0].lower(): + return canon + if name.lower() in [a.lower() for a in LUMINANCE]: + return "Luminance" + return name or "Unknown" + + +class Frame: + """One light frame and the header facts the pipeline needs.""" + + __slots__ = ("path", "filter", "exptime", "binning", "object", "date_obs", + "telescope", "instrument", "shape", "scale", "calibrated") + + def __init__(self, path, header, shape, calibrated=True): + self.calibrated = calibrated + self.path = path + self.filter = canonical_filter(header.get("FILTER")) + self.exptime = float(header.get("EXPTIME", header.get("EXPOSURE", 0))) + self.binning = int(header.get("XBINNING", 1)) + self.object = str(header.get("OBJECT", "")).strip() + self.date_obs = str(header.get("DATE-OBS", "")) + self.telescope = str(header.get("TELESCOP", "")).strip() + self.instrument = str(header.get("INSTRUME", "")).strip() + self.shape = shape + # iTelescope writes the plate scale in; without it, solve.py estimates. + scale = header.get("HIERARCH iTelescopePlateScaleH") + try: + self.scale = float(scale) if scale is not None else None + except (TypeError, ValueError): + self.scale = None + + @property + def name(self): + return os.path.basename(self.path) + + def __repr__(self): + return (f"<{self.filter} {self.exptime:.0f}s bin{self.binning} " + f"{self.name}>") + + +class Session: + """One target, one telescope, one night.""" + + def __init__(self, root): + self.root = os.path.abspath(root) + self.frames = [] + self._by_filter = {} + + # ---------------------------------------------------------------- ingest + @staticmethod + def _is_mirror_junk(path): + """True for artefacts a broken segmented download leaves behind. + + Cyberduck writes a DIRECTORY named after the file it is fetching, and + leaves `.cyberducksegment` parts inside when interrupted. Those + directories are named `something.fit`, so anything that trusts the + extension will try to open a directory as a frame. A re-mirror over an + already-organised tree produces dozens of them. + """ + name = os.path.basename(path).lower() + if ".cyberducksegment" in name or name.endswith((".part", ".partial")): + return True + parent = os.path.basename(os.path.dirname(path)).lower() + return parent.endswith((".fit", ".fits", ".tif", ".tiff", ".zip")) + + def ingest(self, verbose=True): + """Put the session into the standard layout and uncompress the lights. + + Handles the three states a downloaded session can be in: still zipped, + already extracted, or a mixture of both (which is what happens when + someone unzips a few by hand). + """ + raw = os.path.join(self.root, layout.RAW) + cal = os.path.join(self.root, layout.CALIBRATED) + os.makedirs(raw, exist_ok=True) + os.makedirs(cal, exist_ok=True) + + moved = extracted = skipped = duplicates = 0 + # Loose files at the session root get filed first. + for path in sorted(glob.glob(os.path.join(self.root, "*"))): + if os.path.isdir(path): + continue + name = os.path.basename(path) + low = name.lower() + if self._is_mirror_junk(path): + continue + if low.endswith(".zip") or (low.startswith("jpeg-") and + low.endswith(".jpg")): + dest = os.path.join(raw, name) + elif low.endswith((".fit", ".fits", ".tif", ".tiff")): + dest = os.path.join(cal, name) + else: + continue + if not os.path.exists(dest): + os.replace(path, dest) + moved += 1 + elif os.path.getsize(dest) == os.path.getsize(path): + # Already filed, and the mirror has put an identical copy back. + os.remove(path) + duplicates += 1 + + # iTelescope ships each frame twice: 'calibrated-*' with bias, dark + # and flat applied, and 'raw-*' without. Extracting both put two copies + # of every frame into the stack. Only the calibrated set is extracted; + # the raw archives stay zipped in raw/ as the untouched originals. If a + # session has no calibrated archives at all, the raw ones are used and + # the fact is recorded. + archives = sorted(glob.glob(os.path.join(raw, "*.zip"))) + calibrated_archives = [z for z in archives + if os.path.basename(z).lower().startswith( + "calibrated")] + self.used_raw = not calibrated_archives and bool(archives) + if self.used_raw: + print(" NOTE: no calibrated archives in this session - " + "extracting the raw frames instead. They have NOT had bias, " + "dark or flat correction applied.") + else: + archives = calibrated_archives + + for zpath in archives: + try: + with zipfile.ZipFile(zpath) as zf: + for info in zf.infolist(): + base = os.path.basename(info.filename) + if not base.lower().endswith((".fit", ".fits", ".tif", + ".tiff")): + continue + target = os.path.join(cal, base) + if os.path.exists(target) and \ + os.path.getsize(target) == info.file_size: + skipped += 1 + continue + with zf.open(info) as src, open(target, "wb") as dst: + while chunk := src.read(1 << 20): + dst.write(chunk) + extracted += 1 + except zipfile.BadZipFile: + print(f" WARNING: {os.path.basename(zpath)} is not a " + f"readable zip - skipped") + if verbose: + print(f" filed {moved}, extracted {extracted}, " + f"already present {skipped}, " + f"duplicate copies removed {duplicates}") + return dict(moved=moved, extracted=extracted, skipped=skipped, + duplicates=duplicates) + + # ----------------------------------------------------------------- scan + def scan(self, verbose=True): + """Read every calibrated light frame's header.""" + cal = os.path.join(self.root, layout.CALIBRATED) + paths = sorted(p for p in glob.glob(os.path.join(cal, "*.fit")) + + glob.glob(os.path.join(cal, "*.fits")) + if os.path.isfile(p) and not self._is_mirror_junk(p)) + self.frames = [] + for path in paths: + try: + with fits.open(path, memmap=True) as hd: + hdr = hd[0].header + shape = hd[0].shape + except Exception as exc: # noqa: BLE001 + print(f" WARNING: cannot read {os.path.basename(path)}: " + f"{exc}") + continue + # Skip anything that is not a light frame. + if str(hdr.get("IMAGETYP", "Light")).lower().startswith( + ("dark", "bias", "flat")): + continue + self.frames.append(Frame(path, hdr, + shape, calibrated=bool( + str(hdr.get("CALSTAT", "")).strip()))) + + # If both calibrated and uncalibrated frames are present - which + # happens when a session was extracted before this rule existed - keep + # only the calibrated ones rather than stacking each frame twice. + if any(f.calibrated for f in self.frames): + dropped = [f for f in self.frames if not f.calibrated] + if dropped and verbose: + print(f" ignoring {len(dropped)} uncalibrated duplicate " + f"frames (no CALSTAT)") + self.frames = [f for f in self.frames if f.calibrated] + + self._by_filter = defaultdict(list) + for f in self.frames: + self._by_filter[f.filter].append(f) + if verbose: + self.describe() + return self.frames + + # ------------------------------------------------------------ properties + @property + def filters(self): + """Filters present, deepest first - the deepest becomes the reference.""" + return sorted(self._by_filter, + key=lambda f: -sum(x.exptime for x in self._by_filter[f])) + + def by_filter(self, name): + return list(self._by_filter.get(name, [])) + + @property + def target(self): + names = [f.object for f in self.frames if f.object] + return max(set(names), key=names.count) if names else \ + os.path.basename(os.path.dirname(self.root)) + + @property + def telescope(self): + names = [f.telescope for f in self.frames if f.telescope] + return max(set(names), key=names.count) if names else "unknown" + + @property + def scale(self): + vals = [f.scale for f in self.frames if f.scale] + return sum(vals) / len(vals) if vals else None + + @property + def palette(self): + """Which colour scheme this session's filters support.""" + present = set(self._by_filter) + has_lum = "Luminance" in present + broad = [c for c in BROADBAND if c in present] + narrow = [n for n in ("SII", "Ha", "OIII") if n in present] + if len(broad) == 3 and has_lum: + return "LRGB" + if len(broad) == 3: + return "RGB" + if len(narrow) >= 3: + return "SHO" + if "Ha" in narrow and "OIII" in narrow: + return "HOO" + if len(present) == 1: + return "MONO" + if broad or narrow: + return "PARTIAL" + return "UNKNOWN" + + def describe(self): + print(f" target {self.target}") + print(f" telescope {self.telescope}") + if self.scale: + print(f" plate scale {self.scale:.4f} arcsec/px") + print(f" palette {self.palette}") + for name in self.filters: + fs = self._by_filter[name] + exps = sorted({f.exptime for f in fs}) + bins = sorted({f.binning for f in fs}) + total = sum(f.exptime for f in fs) / 60.0 + shapes = sorted({f.shape for f in fs}) + shape_txt = "/".join(f"{s[1]}x{s[0]}" for s in shapes) + print(f" {name:10s} {len(fs):3d} frames " + f"{'/'.join(f'{e:.0f}' for e in exps):>12s} s " + f"bin{'/'.join(str(b) for b in bins)} " + f"{shape_txt:>11s} {total:6.1f} min") + + # ------------------------------------------------------------- manifest + def manifest(self): + return { + "root": self.root, + "target": self.target, + "telescope": self.telescope, + "plate_scale": self.scale, + "palette": self.palette, + "filters": { + name: { + "frames": len(self._by_filter[name]), + "total_exposure_s": sum(f.exptime + for f in self._by_filter[name]), + "exposures": sorted({f.exptime + for f in self._by_filter[name]}), + "binning": sorted({f.binning + for f in self._by_filter[name]}), + } for name in self.filters + }, + } + + def save_manifest(self): + path = os.path.join(self.root, "manifest.json") + with open(path, "w", encoding="utf-8") as fh: + json.dump(self.manifest(), fh, indent=2) + return path + + +def discover(root): + """Find every session under a directory tree. + + A session is any directory that directly contains light frames or their + archives, which matches how iTelescope's downloads are organised + (telescope / object / date). + """ + found = [] + for dirpath, dirnames, filenames in os.walk(root): + base = os.path.basename(dirpath) + if base.lower().endswith((".fit", ".fits", ".tif", ".tiff", ".zip")) or ".cyberducksegment" in base.lower(): + dirnames[:] = [] + continue + if base in (layout.RAW, layout.CALIBRATED, "intermediates", + "final", "renderings", "science", "stacks"): + dirnames[:] = [] + continue + has_data = any(f.lower().endswith((".zip", ".fit", ".fits")) + for f in filenames) + has_layout = any(d in dirnames for d in (layout.RAW, + layout.CALIBRATED)) + if has_data or has_layout: + found.append(dirpath) + dirnames[:] = [d for d in dirnames + if d in (layout.RAW, layout.CALIBRATED)] + return sorted(found) diff --git a/state/ARCHITECTURE.md b/state/ARCHITECTURE.md index 6de98f7..dab87b5 100644 --- a/state/ARCHITECTURE.md +++ b/state/ARCHITECTURE.md @@ -3,14 +3,73 @@ > How the system is built and why. Update this when the structure changes; a change is > not finished until this reflects it. -A documentation-only repo; there is no code. +Three parts, which is why the repository has three top-level content +directories. Until 2026-07-21 the first two lived in separate repositories. -- `TELESCOPES.md`: the deliverable. Per-observatory sections, one entry per telescope - (spec bullets then a review paragraph), a "choosing a telescope" table, and general - observations. Compiled from the sources below, dated in its header. -- `data/itelescope-telescopes.csv`: verbatim CSV export of iTelescope's maintained - specs Google Sheet. Treated as the source of truth for numbers; re-exported to - refresh (see state/NOTES.md). -- `state/`: Default Workflow project memory. -- `CLAUDE.md` + `docs/`: the Default Workflow itself, copied from the - Default-Workflow repo so the project is self-contained. +## itelescope/ - the network and the campaign + +Documentation and data, no code. + +- `TELESCOPES.md`: the deliverable. Per-observatory sections, one entry per + telescope (spec bullets then a review paragraph), a "choosing a telescope" + table, and general observations. Compiled from the sources below. +- `TARGETS.md`: southern targets invisible from the UK, matched to scopes and + to the time of year. +- `CAMPAIGN.md`: the points drain campaign - ledger, bookings, queue, rate + model. Updated after every session from the account's transaction history. +- `data/itelescope-telescopes.csv`: verbatim CSV export of iTelescope's + maintained specs sheet. The source of truth for numbers; re-exported to + refresh (see `state/NOTES.md`). +- `plans/`: ACP observing plans, one per booked run, uploaded to the scope. + +## pipeline/ - the processing code + +Python 3.12. The design rule is that **code and data are separate**: nothing +here knows an absolute path. A session is located at runtime through the +`ASTRO_SESSION` environment variable. + +- `layout.py` maps a **filename** to the subdirectory it belongs in, using the + same rules the session directories are organised with. A script asks for + `master-Red.fit` or `_stars.npz` and gets the right path without any call + site knowing the structure. Unrecognised names resolve to the session root, + which is visible and correctable rather than silently wrong. +- `session.py` is the entry point for anything generic. It discovers sessions, + ingests and uncompresses them, reads every frame's header, groups frames by + filter, and derives the colour scheme (LRGB / RGB / SHO / HOO / MONO) from + what is actually present. **Header-driven, never filename-driven** - see + DECISIONS. +- `restructure.py` reorganises a flat session directory into the standard + layout. Idempotent, dry run by default. +- The remaining scripts are the NGC 5128 session as it was actually run: + `unzip -> analyse -> stack -> solve -> depth -> compose -> hdr/enhance/ + starless/annotate -> final -> closeup -> triptych`, plus the analysis + families `gc-*` (globular clusters), `sb_*` (surface photometry) and `mo_*` + (moving objects and transients). They are kept as provenance for published + results and are being generalised stage by stage. + +### Session layout on disk (not in this repo) + +``` +/ + METHODS.md what was done to this data and what was found + raw/ exactly what the telescope delivered + calibrated/ uncompressed calibrated subs + stacks/masters/ per-filter registered, combined, plate-solved + stacks/original/ alignment-only baseline, no other processing + final/ the deliverable renderings + renderings/ other finished images + science/figures|catalogues|data|notes + intermediates/ caches; deletable, regenerated by a re-run +``` + +## observing/ - in-person plans + +Field plans worked out from real numbers, revised as a date approaches. The +reasoning is the part worth keeping, so plans record why a decision was made, +not only what was decided. + +## The workflow itself + +`CLAUDE.md` + `docs/`: the Default Workflow, copied in so the project is +self-contained. `state/`: project memory - objective, current work, decisions, +notes. diff --git a/state/DECISIONS.md b/state/DECISIONS.md index 25eb138..cc15ca9 100644 --- a/state/DECISIONS.md +++ b/state/DECISIONS.md @@ -3,6 +3,33 @@ > A dated, append only log of decisions and their rationale. Newest at the top. Never > rewrite past entries; if a decision is reversed, add a new entry that says so. +## 2026-07-21: the pipeline reads headers, not filenames + +The first pipeline parsed `calibrated-T32-...-Luminance-BIN2-W-300-001.fit` to +learn a frame's filter, binning and exposure. That works for one telescope, one +binning and one exposure length. Three of the four archived sessions have no +luminance channel at all and one is from 2021 with different naming, so the +approach could not survive contact with the archive. + +FITS headers carry FILTER, EXPTIME, XBINNING, OBJECT and DATE-OBS precisely so +software does not have to guess, and iTelescope fills them in properly. The +pipeline opens each frame and asks. Filter aliases are mapped, so H-alpha, +Halpha and HA all resolve to Ha. + +The colour scheme is then derived from what is present rather than assumed: +LRGB, RGB without luminance, SHO, HOO, or mono. That is what lets one command +handle a galaxy in LRGB and a nebula in narrowband without being told which is +which. + +## 2026-07-21: push-button imaging, hand-driven science + +The imaging half - unzip, measure, register, stack, plate solve, compose - is +mechanical and should run unattended on any session. The analyses are not: a +globular cluster survey suits an elliptical galaxy and is meaningless for an +emission nebula, and much of the value in the NGC 5128 work came from noticing +when a result was wrong rather than from running the code. So the pipeline is +being built to be push-button, and the science stays step by step. + ## 2026-07-21: merge itelescope and astro-pipeline into one repository The two were split by accident of chronology rather than by design. `itelescope` diff --git a/state/NOTES.md b/state/NOTES.md index 3a4323c..7c7d938 100644 --- a/state/NOTES.md +++ b/state/NOTES.md @@ -78,3 +78,72 @@ Paid imaging needs no reservation when a scope is idle: same walk-in ACP flow as the free scopes, points just deduct. Reservations (to guarantee a slot) go through lookup.itelescope.online, which is Cloudflare-protected: Playwright needed. + + +## Processing the downloaded archive (2026-07-21) + +Gotchas found by running code against the real archive rather than reasoning +about it. Each one produced a plausible-looking wrong answer, not an error. + +- **Every frame is delivered twice.** iTelescope ships `calibrated-*` (bias, + dark and flat applied) and `raw-*` (not). Extracting both puts two copies of + every frame into the stack, and the counts look merely generous rather than + wrong: NGC 5128 scanned as 24 luminance frames when it has 12. `CALSTAT` in + the header is the reliable discriminator - calibrated frames carry it, raw + frames do not. Only the calibrated archives are extracted; the raw archives + stay zipped as the untouched originals. + +- **An interrupted segmented download leaves DIRECTORIES named like files.** + Cyberduck creates a directory called `something.fit` while fetching, with + `.cyberducksegment` parts inside. Anything that trusts a file extension will + try to open a directory as a frame. After a re-mirror over an already + organised tree there were 28 of them. Discovery, ingest and scan all skip + them explicitly. + +- **A re-mirror duplicates rather than replaces.** Mirroring on top of an + organised tree restored ~1 GB of files to session roots that had already been + filed into `raw/`. Ingest now removes byte-identical copies as it refiles. + +- **Two T20 archives are genuinely corrupt** and cannot be extracted + (`ic1805-...-Ha-...-015` and `-016`). That is real data loss, not a bug: the + session has 14 of 16 frames. The warning is printed rather than swallowed. + +- **Extraction triples the disk footprint.** The archive went from 6.1 GB of + zips to 14 GB extracted, which filled the drive. The TIFFs are 1.69 GB of + that and the pipeline never reads them - it only uses the FITS. + +- **Channels do not always share a pixel grid.** NGC 6744 was shot with + luminance at bin1 (4096x4096) and colour at bin2 (2048x2048). Registration + has to resample onto a common grid; code that assumes a single frame shape + will fail on this session and only this session. + +## Processing lessons from NGC 5128 (2026-07-21) + +Kept because each is a way to get a confident wrong answer: + +- **Measure a core with the stars filtered out.** Taking the maximum pixel in a + box around the frame centre found a foreground star, not the galaxy, and led + to booking telescope time to fix a saturation problem that did not exist. + +- **Registration makes sensor defects look like perfect asteroids.** Holding + the sky still drags anything detector-fixed across the frame on a dead + straight, constant-rate, constant-brightness track. Hot pixels pass every + standard quality cut. Requiring the motion to be real in DETECTOR coordinates + took 141 confident detections to zero. + +- **Never compare an aperture magnitude to a point-source catalogue without + checking the source is a point.** A resolved object looks exactly like a + 2.8 magnitude outburst. Carry r50/psf through to the cross-match. + +- **Never fit a sky background to a field the target fills.** A plane fitted + around a large galaxy absorbs its halo, measured at -17.9 ADU/px. Fit the + background and a source model together. + +- **"No Gaia counterpart" is not a star-rejection test** at extragalactic + distances: half of Centaurus A's catalogued globular clusters have a Gaia + source, because a cluster at 3.8 Mpc is a point source. Reject on parallax + and proper motion instead. + +- **A coarse background mesh fails in the direction that flatters you.** It + destroys detection near a bright galaxy while leaving the outer field intact, + manufacturing a falsely flat radial profile. diff --git a/state/TODO.md b/state/TODO.md index 5645984..efbcbf9 100644 --- a/state/TODO.md +++ b/state/TODO.md @@ -1,121 +1,163 @@ # TODO -> The current state of play. Keep it honest and current; this is what the next session -> reads to know what to do. Move items between sections as they progress. +> The current state of play. Keep it honest and current; this is what the next +> session reads to know what to do. Move items between sections as they progress. -## Done +--- -- 2026-07-17: repo created on git.discworld.casa, Default Workflow scaffold copied in. -- 2026-07-17: specs sheet snapshotted to data/itelescope-telescopes.csv (24 scopes). -- 2026-07-17: TELESCOPES.md written: per-scope reviews for all 23 active telescopes - plus a choosing guide (T74 noted as in commissioning, no specs published). -- 2026-07-18: TARGETS.md written: southern targets invisible from the UK (dec tiers - from lat 51.5°N), matched to Q62/X07 scopes, seasonal booking calendar, and a - July-August starter campaign (T33 free, T71 narrowband, T73 galaxy, T8 LMC later). +## RESUME HERE - 2026-07-21, end of session -## Done (recent) +Three strands are live. Each has a clear next action. -- 2026-07-20: NGC 6744 on T59 SUCCEEDED (reservation 749227, txn 851148): full - LRGB dataset on the scope server (L 15x300s BIN1, RGB 7x300s each BIN2), - previews verified clean. Billed 381 pts (est was ~240: see the rate - correction in NOTES.md). Balance now 2,257. Ledger updated in CAMPAIGN.md. +### 1. Generic pipeline - MID-BUILD, this is the active work -## Done (recent) +Building a push-button pipeline that processes any session, not just LRGB from +one telescope. Branch `generic-pipeline`. -- 2026-07-21 ~08:55 UTC: free T33 47 Tuc retry booked as reservation 749314 - (grid-verified), third attempt at the first-light test, runs 14:05-14:50 UTC - 21 Jul, costs 0. +**Done:** `pipeline/session.py` - discovery, ingest, header-driven scanning, +filter grouping, palette detection (LRGB / RGB / SHO / HOO / MONO), manifest +writing. Tested against all five archived sessions. -## Done (recent) +**Next:** `measure` -> `register/stack` -> `solve` -> `compose` -> `report`, +then a `run.py` orchestrator with resumable stages. -- 2026-07-21 11:40 UTC: Cen A on T32 verified COMPLETE (billed 160, NGC5128 - images on the scope server), and queue #2 (Corona Australis on T8) booked for - the night of 22 Jul, 21:45-00:45 SSO local, which brackets the target's - culmination instead of chasing it down the western sky. Rate model rebuilt - from two actual bills. Details in the CAMPAIGN.md session note. +**The blocker before any of that:** the disk is full (see below). -## In progress +**The known hard part:** NGC 6744 mixes bin1 luminance at 4096x4096 with bin2 +colour at 2048x2048. Registration must resample onto a common grid rather than +assume every frame shares a shape. Nothing else in the archive does this, so it +is easy to forget and it will fail loudly when it happens. -- T33 first-light test on its THIRD booking (749314), which had NOT yet run as of - 11:40 UTC 21 Jul: it starts 14:05 UTC. Verify after 14:50 UTC (images and logs - on t33.itelescope.net:8033 were still empty/stale at 11:40, as expected). The - two prior free reservations (749228, 749229) expired unused with reschedule=No; - if 749314 also fails to run, stop rebooking and either walk it in (POST - plan.asp per plans/README.md) or drop the free objective. -- The T32 HDR core set (plans/hdrcore-t32.txt) was booked and then CANCELLED on - 2026-07-21 without running: the Cen A nucleus is not saturated, the earlier - claim having measured a foreground star. See the correction in CAMPAIGN.md. - Zero points spent. The plan file remains available on T32 if a genuinely - saturated target turns up. -- The 21 Jul Cen A data has been fully processed off-repo, in the user's - Downloads\NGC5128\20260721\stacked folder: masters, plate solution (0.27 - arcsec on 237 stars), an LRGB composite, a globular cluster survey (289 - candidates, 62% catalogue-confirmed, density falling 10.5x outward), surface - photometry (Sersic n = 4.27, Re = 400 arcsec, total G = 6.56, which matches - the catalogued brightness and so validates the photometric chain), and a - transient/moving-object search (both null, with measured limits). METHODS.md - there documents the lot. Nothing from that analysis needs to come back into - this repo, but two results affect FUTURE sessions and are recorded here: - **(a)** measure saturation properly before buying short subs, and **(b)** the - faint outer halo in that stack is limited by the sky-plane subtraction - (-17.9 ADU/px of real halo light absorbed), not by exposure time, so more - integration alone would not have gone deeper. -- CrA on T8 runs 11:45-14:45 UTC 22 Jul. Verify after 14:45 UTC: images under - /images/qisback/T8/ on t8.itelescope.net:8008, then the actual bill from - account/history.aspx into the ledger. This is the first T8 bill, so it also - fixes T8's effective rate (card says 130/hr; trust nothing until observed). -- DRAIN CAMPAIGN ACTIVE (see CAMPAIGN.md): spend the full balance by 10 Aug, - then cancel before the 12 Aug renewal (user confirmation required for the - cancel itself). 541 of 2,638 spent; 2,097 remain. The x1.6 estimate multiplier - is WITHDRAWN: billing is on plan imaging minutes, and per-scope rates must come - from actual bills (T59 127/hr, T32 80/hr observed; the NOTES rate card is - unreliable). On current numbers the campaign UNDER-spends by ~800: plan extra - or longer runs, do not trim. Reschedule flag never sticks (grid always shows - No): treat every reservation as one-shot and verify the morning after. - Download the NGC 6744 and NGC 5128 data from data.itelescope.net before the - 10 Aug cancellation sweep. Update the CAMPAIGN.md ledger after every session - from account/history.aspx. +### 2. Three decisions waiting on the user + +- **Where should the image archive live?** `C:` has ~2.6 GB free and the + archive is 14 GB after extraction. ~2.4 GB is safely reclaimable (1.69 GB of + extracted TIFFs the pipeline never reads, 0.74 GB of mirror-junk duplicates) + but that only buys headroom, it does not fix it. The pipeline is location + independent via `ASTRO_SESSION`, so moving the tree costs nothing but a copy. + **Do not start batch processing until this is resolved - it will fail + partway through.** +- **NGC 6744 already has outputs** from a different session on 2026-07-20, in + `T59/NGC6744/20260719/stacked/` under a different naming convention + (`NGC6744_LRGB_final.png`, `master_Blue_NGC6744_20260719.fit`). Re-process + into the new structure, or leave it alone? +- **T08 NGC 2070 has one frame per filter.** No stacking, no outlier rejection + possible. Worth an SHO composite anyway, or skip? + +### 3. Drain campaign - runs itself, but needs checking daily + +See `itelescope/CAMPAIGN.md`. Next checkpoints below under "iTelescope". + +--- + +## The archive being processed + +`Downloads\itelescope\` - five sessions, four telescopes, 2021 to 2026. Frame +counts below are AFTER removing the raw/calibrated duplication (see NOTES). + +| Session | Palette | Frames | Note | +|---|---|---|---| +| T32 NGC 5128 | LRGB | 12/4/4/4 | matches the hand-processed run exactly: a working regression test | +| T59 NGC 6744 | LRGB | 15/7/7/7 | **L bin1 4096², RGB bin2 2048²** - mixed grids | +| T20 IC 1805 | Ha only | 14 | mono; **2 frames lost to corrupt archives** | +| T17 NGC 2030 | RGB, no L | 3/3/3 | only 6 min per channel | +| T08 NGC 2070 | SHO | 1/1/1 | one 60 s frame per filter | + +--- + +## Done - 2026-07-21 + +### Repositories +- **Merged `itelescope` + `astro-pipeline` into `astrophotography`**, both + histories preserved (45 commits; `git log --follow` traces files across the + move). Old repos carry redirect notices in README and CLAUDE.md; **not + archived**, because the campaign is live and other sessions work in + `itelescope` most nights. Archive after 12 Aug. + +### NGC 5128 processed end to end (the reference session) +Lives in `Downloads\NGC5128\20260721\`, organised into `raw/ calibrated/ +stacks/ final/ renderings/ science/ intermediates/` with a `METHODS.md` and a +README in each subdirectory. Results: +- Plate solve: 237 stars, **0.27 arcsec residual**, 0.5376 arcsec/px +- Photometry: zero point G = -2.5 log10(flux) + 27.941, scatter 0.164 mag, + limiting **G ~ 20.7** at SNR 5 +- **Four deliverable images** in `final/`, from raw stack to science-informed, + plus a close-up framed by the galaxy's measured extent +- **Globular clusters:** 289 candidates, 179 (62%) catalogue-confirmed, density + falling **10.5x** outward. A real detection. +- **Surface photometry:** Sersic n = 4.27, Re = 400 arcsec, total G = 6.56 + which matches the catalogued value and validates the whole calibration chain. + No shells, and the data could never have shown them (systematics-limited, + 3 mag short). +- **Transients/moving objects:** two clean nulls with measured limits. + +### Corrections made (both were Claude's errors - do not repeat them) +- **The Cen A nucleus was never saturated.** A foreground star 69 arcsec from + the nucleus was measured instead of the galaxy. A T32 booking made to "fix" + it was cancelled before running, costing nothing. Always filter stars out + before measuring a core. +- **NGC 6744 HAD already been processed** by another session on 20 Jul, and the + user was told otherwise. Check for existing outputs before asserting none + exist. + +### Eclipse plan written +`observing/eclipse-2026-menorca/PLAN.md` - full field plan. + +--- ## Pending - pipeline -- Productionise the processing code: a staged CLI - (ingest -> calibrate -> measure -> register -> stack -> solve -> compose -> - analyse), each stage resumable, packaged as an Apptainer image. Slurm array - jobs afterwards and ONLY for the expensive stages: measured on the NGC 5128 - session, per-frame work is 2.9 s and all 24 frames take 1.1 min - single-threaded, so a scheduler buys nothing there. It pays for the Monte - Carlo work (completeness and injection-recovery tests, difference imaging, - shift-and-stack), which is where compute actually bit. +- Finish the stages: measure, register/stack, solve, compose, report, run.py. +- Then productionise: staged CLI, each stage resumable, packaged as an + **Apptainer image**. **Slurm array jobs only for the expensive stages** - + measured on NGC 5128, per-frame work is 2.9 s and all 24 frames take 1.1 min + single-threaded, so a scheduler buys nothing on stacking. It pays for the + Monte Carlo work (completeness and injection-recovery tests, difference + imaging, shift-and-stack), which is where compute actually bit. - Must handle beyond mono LRGB: narrowband palettes, one-shot colour with debayering, other observatories' header conventions, and full calibration from bias/dark/flat for sources that do not pre-calibrate. -- Three findings from the first session are requirements, not options: vet - moving objects in DETECTOR coordinates; carry r50/psf into any catalogue +- **Three findings from the first session are requirements, not options:** + vet moving objects in DETECTOR coordinates; carry r50/psf into any catalogue cross-match; never fit a sky background to a field the target fills. +- Object-prefixed output naming should be derived from the OBJECT header, not + typed in. On a cluster, `final-3-best.png` is ambiguous the moment there is a + second dataset. +- Emit the alignment-only stack as a standard artefact of every run. It costs + almost nothing and it is the reference every later stage can be diffed + against when a processed result looks wrong. + +## Pending - science (deliberately after the pipeline) + +The analyses are per-object and need judgement; they are not push-button. A +globular cluster survey suits Centaurus A and is meaningless for the Heart +Nebula. Approach these one at a time once the imaging half runs unattended. ## Pending - observing -- Total solar eclipse, 12 Aug 2026, Menorca (`observing/eclipse-2026-menorca/`). - Totality 1m12s at Ciutadella with the Sun at 2 degrees, azimuth 288. - **Order the certified solar filter - it is the only item with no substitute - and there are three weeks left.** Then practise on a low sunset to calibrate - exposures for the extreme atmospheric extinction. +- **Total solar eclipse, 12 Aug 2026, Menorca.** Totality 1m12s at Ciutadella, + Sun at **2 degrees**, azimuth 288, sunset 20:42. + `observing/eclipse-2026-menorca/PLAN.md` has the full plan. + - **ORDER THE CERTIFIED SOLAR FILTER.** Only item with no substitute. + - Practise on a low sunset to calibrate exposures for the extreme extinction. + - Outstanding: tailor the exposure table once the A7 body and lenses are known. ## Pending - iTelescope -- Automation: map the ACP web endpoints on a per-scope server (start with free - T33) for plan submission and status; explore lookup.itelescope.online (the new - planner); enumerate more /DataService.svc methods. Recon notes in NOTES.md. -- Campaign impact (status seen 2026-07-18): T71 and T73 are offline - (development/mount testing), as are T2, T17, T18, T20, T25, T30, T72, T74, - T75, T80. T8, T33, T32, T59 at Siding Spring are operational. Re-plan the - July-August campaign around T33 free time and Siding Spring scopes until the - Chile rebuild finishes; re-check status before any booking. - -- Fill in T74 when iTelescope publishes its specs. -- Reconcile the retired scopes (T9, T19, T31, T69): the support article still lists - them but the maintained sheet does not; confirm status and note it. -- Optional: per-telescope detail pages on support.itelescope.net (links in - DECISIONS.md) hold photos and operational history if deeper entries are ever wanted. -- Periodic refresh: re-export the Google Sheet and diff against data/ to catch new or - changed systems. +- **CrA on T8** ran 11:45-14:45 UTC 22 Jul. Verify images under + `/images/qisback/T8/` on t8.itelescope.net:8008, then put the actual bill from + account/history.aspx into the ledger. **First T8 bill, so it fixes T8's + effective rate** (card says 130/hr; trust nothing until observed). +- **T33 first-light test** on its third booking (749314, ran 14:05 UTC 21 Jul). + If it failed again, stop rebooking: either walk it in (POST plan.asp per + `itelescope/plans/README.md`) or drop the free-imaging objective. +- **Campaign:** 541 of 2,638 points spent, 2,097 remain, all to be gone by + 10 Aug with cancellation before the 12 Aug renewal (**user confirmation + required for the cancel itself**). On current numbers the campaign + **under-spends by ~800** - plan extra or longer runs, do not trim. +- Download all data from data.itelescope.net before the 10 Aug sweep. +- Automation: map ACP endpoints per scope; explore lookup.itelescope.online; + enumerate more /DataService.svc methods. +- Fill in T74 when specs are published; reconcile retired scopes T9, T19, T31, + T69; periodically re-export the specs sheet and diff against + `itelescope/data/`.