diff --git a/pipeline/astrometry.py b/pipeline/astrometry.py index 4035d8a..5b29013 100644 --- a/pipeline/astrometry.py +++ b/pipeline/astrometry.py @@ -245,8 +245,10 @@ def run(session, verbose=True): centre = pointing(header) scale = plate_scale(header, session) if centre is None or not scale: + # Not fatal, and no longer the end of the road: returning None sends + # run.py to blind.py, which needs neither the pointing nor the scale. print(" no pointing or plate scale in the header - " - "cannot solve without a blind solver") + "handing off to the blind solver") return None # Match over the frame's INSCRIBED circle, not its circumscribed one. A # cone big enough to reach the corners also reaches well outside the diff --git a/pipeline/astrometry_net.py b/pipeline/astrometry_net.py new file mode 100644 index 0000000..1e14c0e --- /dev/null +++ b/pipeline/astrometry_net.py @@ -0,0 +1,142 @@ +"""Client for the estate's own Astrometry.net service. + + https://astrometry.ankh-morpork.discworld.network + +This is solve-field running on docker1/2/3 against ~5 GB of local index files. +It exists so that blind solving costs seconds instead of minutes and, more to +the point, so that no image has to leave the house to get solved. + +It is used by blind.py, which prefers this service and falls back to the hosted +nova.astrometry.net when it is unreachable. Everything a solution has to earn - +verification against Gaia, the star-count and residual gate, writing the WCS +into the masters - stays in blind.py, shared by both sources. This module only +gets a WCS out of the estate service; it does not decide whether to believe it. + +Only the stdlib is used for the request. A solve is one POST; pulling in +requests to do it would add a dependency for no gain. +""" +import base64 +import io +import os +import ssl +import time +import urllib.error +import urllib.request + +from astropy.io import fits +from astropy.wcs import WCS + +SERVICE = os.environ.get("ASTROMETRY_SERVICE", + "https://astrometry.ankh-morpork.discworld.network") +TIMEOUT = int(os.environ.get("ASTROMETRY_TIMEOUT", "180")) + + +def _context(): + """TLS context for the estate service. + + The service presents a step-ca certificate from the estate's internal CA. + On a machine that trusts the Discworld root CA this verifies normally. On + one that does not, verification fails - and the right fix is to install the + CA, not to turn checking off, so that is what the error message says. + ASTROMETRY_CA_BUNDLE points at the root CA if it is not in the system store. + ASTROMETRY_INSECURE=1 exists as an explicit, visible escape hatch; it is + never the default and it prints when used. + """ + bundle = os.environ.get("ASTROMETRY_CA_BUNDLE") + if os.environ.get("ASTROMETRY_INSECURE") == "1": + print(" WARNING: TLS verification disabled " + "(ASTROMETRY_INSECURE=1)") + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + return ctx + return ssl.create_default_context(cafile=bundle) if bundle else \ + ssl.create_default_context() + + +def available(timeout=5): + """Is the service up and does it have indexes? Cheap enough to ask first.""" + try: + with urllib.request.urlopen(SERVICE + "/healthz", timeout=timeout, + context=_context()) as resp: + return resp.status == 200 + except Exception: # noqa: BLE001 + return False + + +def solve(path, centre=None, scale=None, radius_deg=2.0, timeout=TIMEOUT, + verbose=True): + """Solve one FITS file. Returns (WCS, info dict) or (None, reason). + + `centre` and `scale` are hints, not requirements - pass whatever the header + offered. A hinted solve turns a whole-sky search into a cone and is an order + of magnitude faster, but the whole point of this fallback is that it still + works when there is nothing to hint with. + """ + params = [] + if centre is not None: + params += [f"ra={centre.ra.deg:.6f}", f"dec={centre.dec.deg:.6f}", + f"radius={radius_deg:.3f}"] + if scale: + # +/-20% around the nominal scale: wide enough to absorb a focal + # reducer or a binning the header did not mention, tight enough to + # still rule out most index scales. + params += [f"scale_low={scale * 0.8:.4f}", + f"scale_high={scale * 1.2:.4f}"] + params.append(f"timeout={timeout}") + url = f"{SERVICE}/solve?" + "&".join(params) + + with open(path, "rb") as handle: + body = handle.read() + + started = time.time() + if verbose: + hint = "hinted" if (centre is not None or scale) else "blind" + print(f" astrometry.net ({hint}): " + f"{len(body) / 1e6:.1f} MB to {SERVICE}") + request = urllib.request.Request( + url, data=body, method="POST", + headers={"Content-Type": "application/octet-stream"}) + try: + with urllib.request.urlopen(request, timeout=timeout + 30, + context=_context()) as resp: + import json + payload = json.load(resp) + except urllib.error.HTTPError as exc: + # 422 is the service saying "no solution", which is an answer, not a + # fault. Anything else is the service or the network being broken, and + # the two should not read the same in the log. + import json + try: + detail = json.load(exc) + except Exception: # noqa: BLE001 + detail = {} + if exc.code == 422: + return None, "no solution found" + return None, f"service error HTTP {exc.code}: {detail.get('error', '')}" + except ssl.SSLCertVerificationError: + return None, ("TLS verification failed - install the Discworld root CA " + "or set ASTROMETRY_CA_BUNDLE to it") + except Exception as exc: # noqa: BLE001 + return None, f"{type(exc).__name__}: {exc}" + + if not payload.get("solved"): + return None, "no solution found" + + # The service hands back the .wcs file - a bare FITS header - base64 + # encoded. Parsing it here rather than trusting the summary numbers means + # the full distortion-free TAN solution is what gets written to the + # masters, not a re-derivation from ra/dec/scale. + with fits.open(io.BytesIO(base64.b64decode(payload["wcs_b64"]))) as hdul: + wcs = WCS(hdul[0].header) + + info = {k: payload.get(k) for k in + ("ra", "dec", "pixscale", "orientation", "parity", + "fieldw", "fieldh", "fieldunits")} + info["seconds"] = payload.get("seconds", round(time.time() - started, 1)) + if verbose: + print(f" astrometry.net solved in {info['seconds']}s: " + f"centre {info['ra']:.5f} {info['dec']:+.5f}, " + f"{info['pixscale']:.3f} arcsec/px, " + f"{info['fieldw']:.1f}' x {info['fieldh']:.1f}'") + return wcs, info diff --git a/pipeline/blind.py b/pipeline/blind.py index 26812a2..f397347 100644 --- a/pipeline/blind.py +++ b/pipeline/blind.py @@ -10,22 +10,33 @@ and no amount of window-sliding fixes a population mismatch. A blind solver does not care. It builds geometric hashes from the image's own stars and looks them up in pre-built index files, so it needs no pointing, no scale and no orientation - only the pixels. astrometry.net is the standard -implementation and this uses the hosted nova.astrometry.net service through -astroquery. +implementation, and there are now two of it available: -Two things to know: +1. **The estate's own** (`astrometry_net.py`) - solve-field on docker1/2/3 + against local index files. Tried first: the image never leaves the house, it + needs no API key, and it answers in seconds rather than queueing. Its index + set covers 5.6 arcmin to 33 degree quads, which is the whole iTelescope + fleet's range, but NOT arbitrarily narrow fields. +2. **The hosted nova.astrometry.net**, through astroquery. The fallback, for + when the estate service is unreachable or its index set cannot reach the + field. Two things to know about it: -* **It needs an API key.** Free, from an account at https://nova.astrometry.net - (Profile -> API Key). Provide it as the ASTROMETRY_API_KEY environment - variable, or in a file named `astrometry.key` beside the session or in - c:\\temp\\claudetemp. Without one this module reports that clearly and the - pipeline carries on unsolved rather than failing. -* **It uploads the image.** A stacked master goes to a third-party service. - That is fine for these targets and worth knowing before pointing it at - anything you would not publish. + * **It needs an API key.** Free, from an account at + https://nova.astrometry.net (Profile -> API Key). Provide it as the + ASTROMETRY_API_KEY environment variable, or in a file named + `astrometry.key` beside the session or in c:\\temp\\claudetemp. Without one + this module says so and the pipeline carries on unsolved rather than + failing. + * **It uploads the image.** A stacked master goes to a third-party service. + Fine for these targets, and worth knowing before pointing it at anything + you would not publish. This is the main reason the estate solver is tried + first. -Results are still put through the same quality gate as the seeded solver: a -solution has to be good, not merely returned. +Whichever source answers, the solution is put through the same quality gate as +the seeded solver - verified against Gaia, with a star count and residual it +has to meet. A solution has to be good, not merely returned. The source is +recorded in the ASTRSOLV card and in the returned `method`, so it is always +possible to tell afterwards whether a frame was solved in-house or uploaded. """ import os @@ -58,31 +69,50 @@ def find_key(session=None): return None -def run(session, scale_hint=None, verbose=True): - """Solve the deepest master blind. Returns the same dict as the seeded - solver, or None.""" +def _solve_via_estate(path, scale_hint=None, verbose=True): + """Try the estate's own solver first. Returns a WCS header, or None. + + Preferred over nova for three reasons, in order of how much they matter: + the image never leaves the house, it needs no API key, and it answers in + seconds rather than sitting in a public queue. If the service is down or + unreachable this returns None and the caller goes to nova, so the estate + being offline costs speed and privacy, not the ability to solve. + """ + try: + import astrometry_net + except ImportError: + return None + if not astrometry_net.available(): + if verbose: + print(f" estate solver unreachable " + f"({astrometry_net.SERVICE}) - falling back to nova") + return None + + wcs, info = astrometry_net.solve(path, scale=scale_hint, verbose=verbose) + if wcs is None: + # `info` is the reason string here. A genuine "no solution" is worth + # reporting but is not a reason to then try nova with the same pixels + # and the same index scales - nova's indexes are wider, so it is worth + # exactly one more attempt, and the caller makes it. + if verbose: + print(f" estate solver: {info}") + return None + return wcs.to_header() + + +def _solve_via_nova(path, session, scale_hint=None, verbose=True): + """The hosted service. Needs an API key and uploads the image.""" key = find_key(session) if not key: if verbose: - print(" blind solve unavailable: no astrometry.net API key." - "\n Get one free at https://nova.astrometry.net " - "(Profile -> API Key), then set ASTROMETRY_API_KEY or write " - "it to c:\\temp\\claudetemp\\astrometry.key") + print(" nova blind solve unavailable: no astrometry.net API" + " key.\n Get one free at https://nova.astrometry.net" + " (Profile -> API Key), then set ASTROMETRY_API_KEY or write" + " it to c:\\temp\\claudetemp\\astrometry.key") return None from astroquery.astrometry_net import AstrometryNet - masters = os.path.join(session.root, layout.MASTERS) - deepest = session.filters[0] - path = os.path.join(masters, f"master-{deepest}.fit") - if not os.path.exists(path): - return None - - with fits.open(path) as hd: - image = hd[0].data.astype(np.float32) - header = hd[0].header - ny, nx = image.shape - ast = AstrometryNet() ast.api_key = key ast.TIMEOUT = 600 @@ -104,10 +134,40 @@ def run(session, scale_hint=None, verbose=True): wcs_header = ast.solve_from_image(path, force_image_upload=True, **kwargs) except Exception as exc: # noqa: BLE001 - print(f" blind solve failed: {type(exc).__name__}: {exc}") + print(f" nova blind solve failed: {type(exc).__name__}: {exc}") return None if not wcs_header: - print(" blind solve returned no solution") + print(" nova blind solve returned no solution") + return None + return wcs_header + + +def run(session, scale_hint=None, verbose=True): + """Solve the deepest master blind. Returns the same dict as the seeded + solver, or None. + + Two possible sources, tried in that order: the estate's own solver, then + nova. Whichever answers, the solution then earns its place the same way - + verified against Gaia and put through the same gate as the seeded solver. + A blind solve that is merely *returned* is not a blind solve that is right. + """ + masters = os.path.join(session.root, layout.MASTERS) + deepest = session.filters[0] + path = os.path.join(masters, f"master-{deepest}.fit") + if not os.path.exists(path): + return None + + with fits.open(path) as hd: + image = hd[0].data.astype(np.float32) + ny, nx = image.shape + + source = "estate" + wcs_header = _solve_via_estate(path, scale_hint=scale_hint, verbose=verbose) + if wcs_header is None: + source = "nova" + wcs_header = _solve_via_nova(path, session, scale_hint=scale_hint, + verbose=verbose) + if wcs_header is None: return None wcs = WCS(wcs_header) @@ -150,13 +210,13 @@ def run(session, scale_hint=None, verbose=True): med = float(np.median(resid)) if resid is not None else float("nan") if nmatch < MIN_STARS or not np.isfinite(med) or med > MAX_RESID_PX: - print(f" blind solution REJECTED on verification: {nmatch} " - f"Gaia stars at {med:.2f} px " + print(f" {source} blind solution REJECTED on verification: " + f"{nmatch} Gaia stars at {med:.2f} px " f"(need >= {MIN_STARS} and <= {MAX_RESID_PX})") return None if verbose: - print(f" blind solve verified: {nmatch} Gaia stars, " + print(f" {source} blind solve verified: {nmatch} Gaia stars, " f"{med:.2f} px ({med * solved_scale:.2f} arcsec)") print(f" centre {centre.to_string('hmsdms')}, " f"{solved_scale:.4f} arcsec/px, PA {rot:.2f} deg") @@ -168,8 +228,11 @@ def run(session, scale_hint=None, verbose=True): with fits.open(p, mode="update") as hd: for card in wcs.to_header().cards: hd[0].header[card.keyword] = (card.value, card.comment) + # Name the SOURCE, not just "astrometry.net": a year from now the + # only way to tell whether a frame was solved in-house or uploaded + # to a public service is this card. hd[0].header["ASTRSOLV"] = ( - f"astrometry.net blind / {nmatch} Gaia stars / " + f"astrometry.net blind ({source}) / {nmatch} Gaia stars / " f"{med * solved_scale:.2f} arcsec", "blind plate solve") hd.flush() @@ -177,4 +240,4 @@ def run(session, scale_hint=None, verbose=True): residual_arcsec=med * solved_scale, scale=solved_scale, position_angle=rot, centre=centre.to_string("hmsdms"), fov_arcmin=(nx * solved_scale / 60, ny * solved_scale / 60), - method="astrometry.net blind") + method=f"astrometry.net blind ({source})") diff --git a/state/DECISIONS.md b/state/DECISIONS.md index cc15ca9..404eb1c 100644 --- a/state/DECISIONS.md +++ b/state/DECISIONS.md @@ -3,6 +3,29 @@ > A dated, append only log of decisions and their rationale. Newest at the top. Never > rewrite past entries; if a decision is reversed, add a new entry that says so. +## 2026-07-21: blind solving goes to the estate's own solver first, nova second + +The estate now runs Astrometry.net itself +(`https://astrometry.ankh-morpork.discworld.network`, config in +ankh-morpork-infra `astrometry/`): solve-field against ~5 GB of local index +files, sized from this repo's own TELESCOPES.md so it covers the whole +iTelescope fleet's fields of view (21.5 arcmin to 10.4 degrees). + +`blind.py` now tries it before nova.astrometry.net. The order is not about +speed, though it is far faster - a blind solve of a real DSS field came back in +0.7s against nova's minutes-in-a-queue. It is that **nova requires uploading +the image to a third party and an API key**, and neither is now necessary for +the ordinary case. nova stays as the fallback for when the estate service is +unreachable or the field is narrower than the local index set reaches, so +losing the estate costs speed and privacy, not the ability to solve at all. + +What deliberately did NOT change: the seeded solver in `astrometry.py` remains +the primary path (a good header beats a blind search), and every blind solution +- from either source - still has to pass the same Gaia verification and +star-count/residual gate. A wrong WCS is worse than no WCS, and a second solver +does not change that. The source is recorded in the ASTRSOLV card so it is +always possible to tell later which one produced a frame's astrometry. + ## 2026-07-21: the pipeline reads headers, not filenames The first pipeline parsed `calibrated-T32-...-Luminance-BIN2-W-300-001.fit` to