laurence
5286a2e81b
The scripts that processed the NGC 5128 session of 2026-07-21 previously lived inside the data directory and addressed it with absolute paths. Code and data are now separated: the code lives here, and a session is located at runtime through the ASTRO_SESSION environment variable. layout.py is what makes that work. It maps a FILENAME to the subdirectory that file belongs in, using the same rules the session directories are organised with, so a script can go on asking for 'master-Red.fit' or '_stars.npz' without any call site knowing the directory structure. Anything unrecognised resolves to the session root, which is visible and correctable rather than silently wrong. restructure.py reorganises a flat session directory into that layout. It is idempotent and dry-run by default. The 50 session scripts are kept as they were run rather than tidied into a library. They were written in sequence as the work went along, several of them by parallel agents, and they show it - but they are the honest provenance of a published set of results, and the productionised pipeline should be able to reproduce those results exactly. Verified before committing: all 51 files compile without warnings, and verify_core.py, closeup.py and triptych.py were run end to end against the reorganised session, correctly finding inputs across calibrated/, stacks/masters/ and final/ and writing outputs back to the right places.
124 lines
4.6 KiB
Python
124 lines
4.6 KiB
Python
"""Reorganise the session directory so a file's kind is obvious from where it sits.
|
|
|
|
Placement is decided by filename rules rather than by hand, so the same layout
|
|
can be reproduced for any other session. Dry run by default; --apply to move.
|
|
|
|
raw/ exactly what iTelescope delivered: the zips and jpegs
|
|
calibrated/ the uncompressed calibrated subs
|
|
stacks/masters/ per-filter registered, plate-solved masters
|
|
stacks/original/ the alignment-only baseline stacks
|
|
final/ the four deliverable renderings + the comparison
|
|
renderings/ other finished images (LRGB, HDR, annotated, starless)
|
|
science/figures/ analysis plots
|
|
science/catalogues/ measured tables (CSV)
|
|
science/data/ models, masks and derived quantities
|
|
science/notes/ the analysis write-ups
|
|
intermediates/ caches and scratch that a re-run can regenerate
|
|
"""
|
|
import os
|
|
import shutil
|
|
import sys
|
|
|
|
ROOT = r"C:\Users\lhorrocks-barlow\Downloads\NGC5128\20260721"
|
|
STACKED = os.path.join(ROOT, "stacked")
|
|
APPLY = "--apply" in sys.argv
|
|
|
|
DIRS = ["raw", "calibrated", "stacks/masters", "stacks/original", "final",
|
|
"renderings", "science/figures", "science/catalogues", "science/data",
|
|
"science/notes", "intermediates"]
|
|
|
|
|
|
def destination(name, from_stacked):
|
|
"""Where a file belongs, decided by its name alone."""
|
|
low = name.lower()
|
|
if not from_stacked:
|
|
if low.endswith(".zip"):
|
|
return "raw"
|
|
if low.startswith("jpeg-") and low.endswith(".jpg"):
|
|
return "raw"
|
|
if low.startswith("calibrated-") and low.endswith((".fit", ".tif")):
|
|
return "calibrated"
|
|
return None
|
|
|
|
if name.startswith("_"):
|
|
return "intermediates"
|
|
if name.startswith("master-") and low.endswith(".fit"):
|
|
return "stacks/masters"
|
|
if name.startswith("notes-") and low.endswith(".md"):
|
|
return "science/notes"
|
|
if low.endswith(".csv"):
|
|
return "science/catalogues"
|
|
if name.startswith("sb-") and low.endswith((".fits", ".npy", ".txt")):
|
|
return "science/data"
|
|
if low.endswith(".png") and any(f"-{p}-" in name for p in ("gc", "sb", "mo")):
|
|
return "science/figures"
|
|
if name.startswith(("NGC5128-LRGB", "NGC5128-img-")):
|
|
return "renderings"
|
|
if name == "METHODS.md":
|
|
return "."
|
|
return None
|
|
|
|
|
|
def plan():
|
|
moves = []
|
|
for name in sorted(os.listdir(ROOT)):
|
|
path = os.path.join(ROOT, name)
|
|
if os.path.isfile(path):
|
|
d = destination(name, from_stacked=False)
|
|
if d:
|
|
moves.append((path, os.path.join(ROOT, d, name)))
|
|
for name in sorted(os.listdir(STACKED)):
|
|
path = os.path.join(STACKED, name)
|
|
if name in ("final", "original", "scripts"):
|
|
continue
|
|
if os.path.isdir(path):
|
|
if name.startswith("_"):
|
|
moves.append((path, os.path.join(ROOT, "intermediates", name)))
|
|
continue
|
|
d = destination(name, from_stacked=True)
|
|
if d:
|
|
target = ROOT if d == "." else os.path.join(ROOT, d)
|
|
moves.append((path, os.path.join(target, name)))
|
|
moves.append((os.path.join(STACKED, "final"), os.path.join(ROOT, "final")))
|
|
moves.append((os.path.join(STACKED, "original"),
|
|
os.path.join(ROOT, "stacks", "original")))
|
|
return moves
|
|
|
|
|
|
if __name__ == "__main__":
|
|
moves = plan()
|
|
counts = {}
|
|
for src, dst in moves:
|
|
rel = os.path.relpath(os.path.dirname(dst), ROOT)
|
|
counts[rel] = counts.get(rel, 0) + 1
|
|
print(f"{len(moves)} moves\n")
|
|
for d in sorted(counts):
|
|
print(f" {d + '/':24s} {counts[d]:3d}")
|
|
|
|
unplaced = []
|
|
for name in sorted(os.listdir(ROOT)):
|
|
if os.path.isfile(os.path.join(ROOT, name)) and \
|
|
not destination(name, False):
|
|
unplaced.append(name)
|
|
for name in sorted(os.listdir(STACKED)):
|
|
p = os.path.join(STACKED, name)
|
|
if os.path.isfile(p) and not destination(name, True):
|
|
unplaced.append("stacked/" + name)
|
|
if unplaced:
|
|
print(f"\nNOT PLACED by any rule ({len(unplaced)}):")
|
|
for n in unplaced:
|
|
print(f" {n}")
|
|
|
|
if not APPLY:
|
|
print("\nDRY RUN - nothing moved. Re-run with --apply")
|
|
sys.exit(0)
|
|
|
|
for d in DIRS:
|
|
os.makedirs(os.path.join(ROOT, d), exist_ok=True)
|
|
for src, dst in moves:
|
|
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
|
if os.path.exists(dst):
|
|
print(f" SKIP (exists): {dst}")
|
|
continue
|
|
shutil.move(src, dst)
|
|
print(f"moved {len(moves)} items")
|