Merge pull request 'M2: On-device WiFi + REST API' (#2) from feature/fw-rest-api into main
Some checks failed
ci / firmware (push) Failing after 28s
ci / server (push) Failing after 31s

This commit is contained in:
laurence 2026-07-03 12:09:54 +01:00
commit 72633f02c3
13 changed files with 510 additions and 20 deletions

View file

@ -17,7 +17,9 @@ build_flags =
; PSRAM is required for audio ring buffers
-DBOARD_HAS_PSRAM
lib_deps =
; Pinned in later milestones as modules land (I2S mic, SD, BLE, HTTP server).
; JSON for the on-device REST API (M2). I2S/SD/WiFi/WebServer/mDNS/Preferences are all
; bundled with the Arduino-ESP32 core, so no other deps are needed yet.
bblanchon/ArduinoJson@^7.0
; --- Build B: ESP32-S3 dev board with PSRAM (N16R8), external I2S mic + SD ---
[env:esp32s3_dev]

211
firmware/src/api_http.cpp Normal file
View file

@ -0,0 +1,211 @@
// SPDX-License-Identifier: GPL-3.0-only
#include "api_http.h"
#include <ArduinoJson.h>
#include <SD.h>
#include <WebServer.h>
#include "config.h"
#include "net_wifi.h"
#include "recorder.h"
#include "storage.h"
namespace {
WebServer server(80);
const char *FW_VERSION = "0.2.0";
// Format a uint64 (SD sizes exceed 32 bits) into a decimal String.
String u64(uint64_t v) {
char b[21];
snprintf(b, sizeof(b), "%llu", (unsigned long long)v);
return String(b);
}
void sendJson(int code, const String &body) {
server.sendHeader("Access-Control-Allow-Origin", "*");
server.send(code, "application/json", body);
}
void sendError(int code, const String &msg) {
sendJson(code, String("{\"error\":\"") + msg + "\"}");
}
// Bearer-token check for mutating endpoints. Empty stored token = auth disabled (open on a
// trusted LAN); tightening this to require a token is a security follow-up (see TODO).
bool authed() {
String tok = config::apiToken();
if (tok.isEmpty()) return true;
return server.header("Authorization") == (String("Bearer ") + tok);
}
const char *recStateStr() {
return recorder::state() == recorder::State::Recording ? "recording" : "idle";
}
// GET /api/v1/device
void handleDevice() {
JsonDocument doc;
doc["device_id"] = config::deviceId();
doc["firmware_version"] = FW_VERSION;
doc["state"] = recStateStr();
doc["power"] = "unknown"; // battery/charge detection lands in M3
doc["battery_pct"] = nullptr;
doc["storage_free_bytes"] = storage::freeBytes();
doc["storage_total_bytes"] = storage::totalBytes();
doc["recordings_count"] = storage::countRecordings();
doc["wifi_connected"] = net_wifi::connected();
doc["ip"] = net_wifi::ip();
String out;
serializeJson(doc, out);
sendJson(200, out);
}
// GET /api/v1/recordings
void handleList() { sendJson(200, storage::listRecordingsJson()); }
// GET /api/v1/recordings/{id}
void handleGetMeta() {
const String id = server.pathArg(0);
const String meta = storage::readFile(storage::recMetaPath(id));
if (meta.isEmpty()) return sendError(404, "No such recording");
sendJson(200, meta);
}
// DELETE /api/v1/recordings/{id}
void handleDelete() {
if (!authed()) return sendError(401, "Unauthorized");
const String id = server.pathArg(0);
if (!storage::removeRecording(id)) return sendError(404, "No such recording");
server.send(204, "application/json", "");
}
// GET /api/v1/recordings/{id}/audio (supports HTTP Range)
void handleAudio() {
const String id = server.pathArg(0);
File f = SD.open(storage::recWavPath(id), FILE_READ);
if (!f) return sendError(404, "No such recording");
const size_t total = f.size();
server.sendHeader("Accept-Ranges", "bytes");
size_t start = 0, end = (total ? total - 1 : 0);
bool partial = false;
String range = server.header("Range");
if (range.startsWith("bytes=")) {
partial = true;
range = range.substring(6);
int dash = range.indexOf('-');
start = range.substring(0, dash).toInt();
String es = range.substring(dash + 1);
if (es.length()) end = es.toInt();
}
if (start >= total) {
f.close();
return sendError(416, "Range not satisfiable");
}
if (end >= total) end = total - 1;
const size_t len = end - start + 1;
f.seek(start);
if (partial) {
server.sendHeader("Content-Range",
"bytes " + String(start) + "-" + String(end) + "/" + String(total));
server.setContentLength(len);
server.send(206, "audio/wav", "");
} else {
server.setContentLength(total);
server.send(200, "audio/wav", "");
}
uint8_t buf[1024];
size_t remaining = len;
while (remaining > 0) {
size_t want = remaining < sizeof(buf) ? remaining : sizeof(buf);
size_t n = f.read(buf, want);
if (n == 0) break;
server.sendContent(reinterpret_cast<char *>(buf), n);
remaining -= n;
}
f.close();
}
// POST /api/v1/record/start
void handleRecordStart() {
if (!authed()) return sendError(401, "Unauthorized");
if (!recorder::startRecording()) return sendError(409, "Already recording");
sendJson(200, String("{\"id\":\"") + recorder::currentId() +
"\",\"state\":\"recording\"}");
}
// POST /api/v1/record/stop
void handleRecordStop() {
if (!authed()) return sendError(401, "Unauthorized");
const String id = recorder::currentId();
if (!recorder::stopRecording()) return sendError(409, "Not recording");
const String meta = storage::readFile(storage::recMetaPath(id));
sendJson(200, meta.isEmpty() ? String("{\"id\":\"") + id + "\"}" : meta);
}
// GET /api/v1/device/config (secrets never returned)
void handleGetConfig() {
JsonDocument doc;
doc["device_id"] = config::deviceId();
doc["wifi_ssid"] = config::wifiSsid();
doc["upload_target"] = config::uploadTarget();
doc["codec"] = "wav_pcm_s16le";
doc["sample_rate"] = 16000;
String out;
serializeJson(doc, out);
sendJson(200, out);
}
// PUT /api/v1/device/config
void handlePutConfig() {
if (!authed()) return sendError(401, "Unauthorized");
JsonDocument doc;
if (deserializeJson(doc, server.arg("plain"))) {
return sendError(400, "Invalid JSON");
}
if (doc["wifi_ssid"].is<const char *>()) {
config::setWifi(doc["wifi_ssid"].as<String>(),
doc["wifi_psk"] | config::wifiPsk());
}
if (doc["api_token"].is<const char *>()) {
config::setApiToken(doc["api_token"].as<String>());
}
if (doc["upload_target"].is<const char *>()) {
config::setUploadTarget(doc["upload_target"].as<String>());
}
// WiFi changes take effect on next reboot/reconnect.
sendJson(200, "{\"status\":\"ok\",\"note\":\"reboot to apply WiFi changes\"}");
}
} // namespace
namespace api_http {
void begin() {
// WebServer only retains headers we ask for.
const char *headers[] = {"Authorization", "Range"};
server.collectHeaders(headers, 2);
server.on("/api/v1/device", HTTP_GET, handleDevice);
server.on("/api/v1/device/config", HTTP_GET, handleGetConfig);
server.on("/api/v1/device/config", HTTP_PUT, handlePutConfig);
server.on("/api/v1/recordings", HTTP_GET, handleList);
// Register the more specific /audio route before the generic {id} route.
server.on("/api/v1/recordings/{}/audio", HTTP_GET, handleAudio);
server.on("/api/v1/recordings/{}", HTTP_GET, handleGetMeta);
server.on("/api/v1/recordings/{}", HTTP_DELETE, handleDelete);
server.on("/api/v1/record/start", HTTP_POST, handleRecordStart);
server.on("/api/v1/record/stop", HTTP_POST, handleRecordStop);
server.onNotFound([]() { sendError(404, "Not found"); });
server.begin();
}
void loop() { server.handleClient(); }
} // namespace api_http

12
firmware/src/api_http.h Normal file
View file

@ -0,0 +1,12 @@
// SPDX-License-Identifier: GPL-3.0-only
// On-device REST API. Implements the "device" paths of api/openapi.yaml on the LAN:
// device status, recording list/metadata/audio(Range)/delete, record start/stop, and
// device config get/put. Served with the core-bundled synchronous WebServer.
#pragma once
namespace api_http {
void begin(); // register routes and start the HTTP server on port 80
void loop(); // service pending requests (call from loop())
} // namespace api_http

43
firmware/src/config.cpp Normal file
View file

@ -0,0 +1,43 @@
// SPDX-License-Identifier: GPL-3.0-only
#include "config.h"
#include <Preferences.h>
namespace {
Preferences prefs;
String deviceId_;
} // namespace
namespace config {
bool begin() {
if (!prefs.begin("openscribe", false)) return false;
deviceId_ = prefs.getString("device_id", "");
if (deviceId_.isEmpty()) {
// Derive a stable id from the eFuse MAC (low 3 bytes) on first boot.
uint64_t mac = ESP.getEfuseMac();
char b[24];
snprintf(b, sizeof(b), "openscribe-%02x%02x%02x", (uint8_t)(mac >> 0),
(uint8_t)(mac >> 8), (uint8_t)(mac >> 16));
deviceId_ = String(b);
prefs.putString("device_id", deviceId_);
}
return true;
}
String deviceId() { return deviceId_; }
String wifiSsid() { return prefs.getString("wifi_ssid", ""); }
String wifiPsk() { return prefs.getString("wifi_psk", ""); }
void setWifi(const String &ssid, const String &psk) {
prefs.putString("wifi_ssid", ssid);
prefs.putString("wifi_psk", psk);
}
String apiToken() { return prefs.getString("api_token", ""); }
void setApiToken(const String &token) { prefs.putString("api_token", token); }
String uploadTarget() { return prefs.getString("upload_target", "none"); }
void setUploadTarget(const String &target) { prefs.putString("upload_target", target); }
} // namespace config

28
firmware/src/config.h Normal file
View file

@ -0,0 +1,28 @@
// SPDX-License-Identifier: GPL-3.0-only
// Persistent device configuration in NVS (Preferences). Holds the device id, WiFi
// credentials, the API bearer token, and upload settings (upload is used from M3).
// Secrets live here, never on the SD card in clear.
#pragma once
#include <Arduino.h>
namespace config {
// Open NVS and ensure a stable device id exists (derived from the MAC on first boot).
bool begin();
String deviceId();
String wifiSsid();
String wifiPsk();
void setWifi(const String &ssid, const String &psk);
// Bearer token for mutating API calls. Empty = auth disabled (open on a trusted LAN).
String apiToken();
void setApiToken(const String &token);
// Upload settings (consumed in M3). Getters return stored values or sensible defaults.
String uploadTarget(); // "none" | "s3" | "webdav"
void setUploadTarget(const String &target);
} // namespace config

View file

@ -2,26 +2,27 @@
//
// OpenScribe firmware entry point.
//
// M1 (recording core): boots audio + storage, then records to a WAV file on the microSD
// card, toggled by the button, with the status LED showing idle/recording/error. Each
// recording gets a sidecar JSON metadata file matching the Recording schema in
// api/openapi.yaml.
// M1 (recording core): records to a WAV file on the microSD card, toggled by the button,
// with the status LED showing idle/recording/error and sidecar JSON metadata per file.
// M2 (WiFi + REST API): joins WiFi (or opens a provisioning SoftAP), advertises over mDNS
// as openscribe.local, and serves the on-device REST API (api/openapi.yaml device paths)
// so the app can list, download (with Range) and delete recordings and control the device.
//
// Later milestones add the remaining module seams:
// power - battery ADC, charge/VBUS detect -> mode switch (M3)
// config - NVS settings (WiFi, upload target, codec, token) (M2/M3)
// net_wifi - WiFi join/reconnect + mDNS (M2)
// api_http - on-device REST server (api/openapi.yaml) (M2)
// uploader - S3/WebDAV upload when powered (M3)
// ble - GATT control + WiFi provisioning (M4)
// ota - firmware update over HTTP (M9)
#include <Arduino.h>
#include "api_http.h"
#include "config.h"
#include "net_wifi.h"
#include "recorder.h"
#include "ux.h"
static const char *FIRMWARE_VERSION = "0.1.0";
static const char *FIRMWARE_VERSION = "0.2.0";
void setup() {
Serial.begin(115200);
@ -39,6 +40,7 @@ void setup() {
}
ux::begin();
config::begin();
if (!recorder::begin()) {
Serial.println("ERROR: audio or SD init failed - check the mic wiring and SD card");
@ -46,6 +48,18 @@ void setup() {
return; // stay in error state; loop() still services the LED
}
// Bring up networking + the REST API. Non-fatal: recording still works offline.
net_wifi::begin();
api_http::begin();
if (net_wifi::isAccessPoint()) {
Serial.printf("WiFi provisioning AP up. API at http://%s/api/v1/device\n",
net_wifi::ip().c_str());
} else {
Serial.printf("WiFi %s. API at http://%s/api/v1/device (or http://openscribe.local)\n",
net_wifi::connected() ? "connected" : "not connected",
net_wifi::ip().c_str());
}
Serial.println("Ready. Short-press the button to start/stop recording.");
ux::setLed(ux::Led::Idle);
}
@ -71,4 +85,8 @@ void loop() {
// While recording, drain the mic into the file.
recorder::update();
// Service networking + the REST API.
net_wifi::loop();
api_http::loop();
}

63
firmware/src/net_wifi.cpp Normal file
View file

@ -0,0 +1,63 @@
// SPDX-License-Identifier: GPL-3.0-only
#include "net_wifi.h"
#include <ESPmDNS.h>
#include <WiFi.h>
#include "config.h"
namespace {
bool apMode = false;
uint32_t lastReconnect = 0;
void startMdns() {
// openscribe.local -> device; advertise the HTTP API service.
if (MDNS.begin("openscribe")) {
MDNS.addService("http", "tcp", 80);
}
}
} // namespace
namespace net_wifi {
bool begin() {
const String ssid = config::wifiSsid();
if (!ssid.isEmpty()) {
WiFi.mode(WIFI_STA);
WiFi.setSleep(false);
WiFi.begin(ssid.c_str(), config::wifiPsk().c_str());
const uint32_t t0 = millis();
while (WiFi.status() != WL_CONNECTED && (millis() - t0) < 15000) {
delay(200);
}
}
if (WiFi.status() != WL_CONNECTED) {
// Provisioning fallback: open a SoftAP so a client can reach the API and set WiFi.
apMode = true;
WiFi.mode(WIFI_AP);
String ap = "OpenScribe-" + config::deviceId().substring(11); // MAC suffix
// WPA2 needs an 8+ char passphrase; note in TODO to make this user-set (M4).
WiFi.softAP(ap.c_str(), "openscribe");
}
startMdns();
return true;
}
void loop() {
if (apMode) return;
if (WiFi.status() != WL_CONNECTED && (millis() - lastReconnect) > 10000) {
lastReconnect = millis();
WiFi.reconnect();
}
}
bool connected() { return WiFi.status() == WL_CONNECTED; }
bool isAccessPoint() { return apMode; }
String ip() {
return apMode ? WiFi.softAPIP().toString() : WiFi.localIP().toString();
}
} // namespace net_wifi

18
firmware/src/net_wifi.h Normal file
View file

@ -0,0 +1,18 @@
// SPDX-License-Identifier: GPL-3.0-only
// WiFi bring-up. Joins the configured network in station mode; if no credentials are
// stored or the join fails, falls back to a SoftAP so the REST API is still reachable to
// provision WiFi. Advertises the device over mDNS as openscribe.local.
#pragma once
#include <Arduino.h>
namespace net_wifi {
bool begin();
void loop(); // reconnect logic in station mode
bool connected(); // station connected to the configured AP
bool isAccessPoint(); // running the provisioning SoftAP
String ip(); // current reachable IP (station or AP)
} // namespace net_wifi

View file

@ -94,4 +94,57 @@ bool writeMetadata(const String &path, const String &json) {
return true;
}
String recWavPath(const String &id) { return String(OSC_REC_DIR) + "/" + id + ".wav"; }
String recMetaPath(const String &id) { return String(OSC_REC_DIR) + "/" + id + ".json"; }
String readFile(const String &path) {
File f = SD.open(path, FILE_READ);
if (!f) return String();
String s;
s.reserve(f.size() + 1);
while (f.available()) s += static_cast<char>(f.read());
f.close();
return s;
}
bool removeRecording(const String &id) {
bool a = SD.remove(recWavPath(id));
bool b = SD.remove(recMetaPath(id));
return a || b;
}
uint32_t countRecordings() {
uint32_t n = 0;
File dir = SD.open(OSC_REC_DIR);
if (!dir) return 0;
for (File f = dir.openNextFile(); f; f = dir.openNextFile()) {
if (String(f.name()).endsWith(".wav")) n++;
f.close();
}
dir.close();
return n;
}
String listRecordingsJson() {
String out = "{\"items\":[";
bool first = true;
File dir = SD.open(OSC_REC_DIR);
if (dir) {
for (File f = dir.openNextFile(); f; f = dir.openNextFile()) {
if (String(f.name()).endsWith(".json")) {
String content;
content.reserve(f.size() + 1);
while (f.available()) content += static_cast<char>(f.read());
if (!first) out += ",";
out += content;
first = false;
}
f.close();
}
dir.close();
}
out += "],\"next_cursor\":null}";
return out;
}
} // namespace storage

View file

@ -36,4 +36,21 @@ class WavWriter {
// Write a sidecar JSON metadata file next to a recording.
bool writeMetadata(const String &path, const String &json);
// Path helpers for a recording id.
String recWavPath(const String &id);
String recMetaPath(const String &id);
// Read a whole (small) file into a String; empty String if it does not exist.
String readFile(const String &path);
// Delete a recording's audio + metadata. Returns true if anything was removed.
bool removeRecording(const String &id);
// Count recordings (by .wav files) currently on the card.
uint32_t countRecordings();
// Build the RecordingPage JSON by concatenating every sidecar metadata object. This is
// the body for GET /recordings and matches api/openapi.yaml.
String listRecordingsJson();
} // namespace storage

View file

@ -39,9 +39,9 @@ Three sync paths, exactly as specified:
- `recorder` - session state machine (idle/recording), file naming, metadata. [M1]
- `ux` - button (short-press start/stop) + status LED (haptic later). [M1]
- `power` - battery read (ADC), charge/VBUS detect -> mode switch.
- `config` - NVS-stored settings (WiFi creds, upload target + keys, codec).
- `net_wifi` - WiFi manager (join, reconnect), mDNS.
- `api_http` - on-device REST server (see `api/openapi.yaml`).
- `config` - NVS-stored settings (device id, WiFi creds, api token, upload). [M2]
- `net_wifi` - WiFi station join/reconnect + SoftAP provisioning fallback + mDNS. [M2]
- `api_http` - on-device REST server implementing the device paths. [M2]
- `uploader` - S3-compatible / WebDAV client; pushes audio + metadata when powered.
- `ble` - GATT: device info, battery, record control, WiFi provisioning, status.
- `ota` - firmware update over HTTP.

View file

@ -52,6 +52,27 @@ Per component (see each component's README for detail):
then. See the follow-ups in TODO.md.
- Not yet compiled locally (no PlatformIO on the dev host) - CI builds both profiles.
## Firmware specifics (M2)
- REST API served by the core-bundled synchronous `WebServer` on port 80. Path params via
`server.on("/.../{}" , ...)` + `server.pathArg(0)`. `collectHeaders` is required to read
`Authorization` and `Range`. Register `/recordings/{}/audio` before `/recordings/{}`.
- Audio download implements HTTP Range (206 + Content-Range) by seeking the file and
streaming with `sendContent`; full GET sends `Accept-Ranges: bytes`.
- Only new dep is `bblanchon/ArduinoJson@^7` (request parse + JSON responses). WiFi,
WebServer, ESPmDNS, Preferences, SD are all core-bundled.
- WiFi: station mode with stored creds; on no-creds/failure it opens a SoftAP
`OpenScribe-<macsuffix>` (pass `openscribe`) so the API is reachable to provision WiFi.
mDNS name is `openscribe.local`.
- Config/secrets live in NVS (`Preferences` namespace `openscribe`), never on the SD card.
## Security follow-ups (before any real deployment)
- API auth is open when no token is set (convenient on a trusted LAN). Set an `api_token`
via `PUT /device/config` to require `Authorization: Bearer` on mutating calls; consider
requiring it for reads too. Tracked in TODO.
- SoftAP uses a fixed default passphrase; make it user-set during provisioning (M4/BLE).
## Dead ends
- (none yet)

View file

@ -5,15 +5,14 @@
## In progress
- [ ] (nothing active - M2 is next)
- [ ] M2 On-device WiFi + REST API - code complete on `feature/fw-rest-api`, in PR.
Not compiled locally / hardware-verified; CI builds both profiles.
## Pending (roughly in build order)
- [ ] M2 On-device WiFi + REST API: WiFi manager, mDNS, list/get/download(range)/delete
recordings, device status. Implements `api/openapi.yaml` device paths.
(branch `feature/fw-rest-api`)
- [ ] M3 Independent uploader: NVS config store, charge/VBUS detect -> powered mode,
S3-compatible + WebDAV upload of audio + metadata. (branch `feature/fw-uploader`)
- [ ] M3 Independent uploader: charge/VBUS detect -> powered mode, S3-compatible +
WebDAV upload of audio + metadata (config store already landed in M2).
(branch `feature/fw-uploader`)
- [ ] M4 BLE GATT: device info, battery, record control, WiFi provisioning, status
notifications; hand-off to WiFi for transfer. (branch `feature/fw-ble`)
- [ ] M5 Server ingest + transcription: FastAPI ingest from object store/upload, store,
@ -31,6 +30,9 @@
## Done
- [x] M2 On-device WiFi + REST API - WiFi/SoftAP + mDNS + config(NVS) + device REST API
(status, list, metadata, audio-with-Range, delete, record start/stop, config).
(PR #2 `feature/fw-rest-api`, 2026-07-03). Not yet hardware-verified.
- [x] M1 Firmware recording core - mic capture to WAV on SD + button/LED + metadata
(merged: PR #1 `feature/fw-recorder-core`, 2026-07-03). Not yet hardware-verified.
- [x] M0 Scaffold - repo, licences, state docs, BOM, OpenAPI, skeletons, CI
@ -44,8 +46,10 @@
- [ ] Audio calibration: INMP441 emits 24-bit samples in a 32-bit slot. M1 reads in
16-bit mode (usable for speech, may be quiet). If levels are low on the dev build,
capture 32-bit and right-shift into int16 in `audio.cpp`.
- [ ] Real timestamps: M1 ids/`started_at` are uptime-based (no RTC/NTP yet). M2 adds NTP
over WiFi; switch ids and `started_at` to real UTC then.
- [ ] Real timestamps: ids/`started_at` are uptime-based (no RTC/NTP yet). Add NTP now
that WiFi exists (M2), and switch ids and `started_at` to real UTC.
- [ ] API auth is open when no token set; SoftAP has a fixed default passphrase. Harden
both before any deployment (see NOTES "Security follow-ups").
## Blocked / waiting