Harden API contracts with request IDs and audit telemetry

README.md

@@ -25,6 +25,7 @@ Open:
 
 Version metadata:
 - REST and MCP tool responses include `api_version` and `schema_version`.
+- REST responses also include `request_id` and `x-request-id` for request correlation.
 - Current values: `api_version=0.3.0`, `schema_version=1.0.0`.
 - Compatibility policy (current): additive, backward-compatible fields may be introduced in the same API minor version.
 
@@ -39,6 +40,7 @@ Docs:
 - `docs/operations-runbook.md`
 - `docs/quality-gates.md`
 - `docs/phase4-execution-plan.md`
+- `docs/api-mcp-contracts.md`
 
 CI:
 - `.gitea/workflows/ci.yml` runs syntax checks + full test suite on push/PR.

docs/api-mcp-contracts.md

@@ -0,0 +1,68 @@
+# API + MCP Contract Policy
+
+This document defines compatibility expectations for Schemeta API and MCP tools.
+
+## Version Fields
+
+All successful API/MCP structured responses include:
+
+- `api_version`
+- `schema_version`
+
+HTTP API responses also include:
+
+- `request_id` (response field + `x-request-id` header)
+
+## Compatibility Policy
+
+- Additive changes are allowed within the same `api_version` minor release.
+- Breaking shape changes require a new API major version.
+- `schema_version` increments when SJM schema semantics or required fields change.
+
+## HTTP API Contracts
+
+Stable endpoints:
+
+- `POST /compile`
+- `POST /analyze`
+- `POST /layout/auto`
+- `POST /layout/tidy`
+- `GET /health`
+- `GET /mcp/ui-bundle`
+
+Error envelope shape:
+
+```json
+{
+  "ok": false,
+  "request_id": "uuid-or-forwarded-id",
+  "error": {
+    "code": "stable_machine_code",
+    "message": "human-readable description"
+  }
+}
+```
+
+## MCP Tool Contracts
+
+Stable tool names:
+
+- `schemeta_compile`
+- `schemeta_analyze`
+- `schemeta_ui_bundle`
+
+Contract rules:
+
+- Tool names are stable across minor versions.
+- Input schemas may gain optional fields additively.
+- Structured response fields are additive-only within a minor.
+
+## Persistence and Restore
+
+UI workspace persistence is expected to support:
+
+- deterministic JSON export/import roundtrip
+- local snapshot restore path (`schemeta:snapshots:v2`)
+- reset-to-sample baseline recovery for QA loops
+
+These are validated in release checklist + browser regression flow.

docs/operations-runbook.md

@@ -35,6 +35,18 @@ This runbook covers baseline production operation for Schemeta API + UI.
 - `GET /mcp/ui-bundle`
   - Metadata for MCP UI embedding.
 
+## Request Correlation and Audit Logs
+
+- Every response includes `x-request-id`.
+- API envelopes include `request_id` for correlation in clients and logs.
+- Server emits one JSON audit log entry per request on response finish with:
+  - `request_id`
+  - `method`
+  - `path`
+  - `status`
+  - `duration_ms`
+  - `client`
+
 ## Production Checks
 
 1. Verify process liveness:
@@ -78,6 +90,6 @@ This runbook covers baseline production operation for Schemeta API + UI.
 
 ## Observability Recommendations
 
-- Add structured request logs at reverse proxy layer.
+- Structured request logs are emitted by the app; keep proxy logs for edge-level traces.
 - Track latency percentiles for `/compile` and `/analyze`.
 - Track per-endpoint status code rates and top warning/error IDs.

src/server.js

@@ -1,4 +1,5 @@
 import { createServer } from "node:http";
+import { randomUUID } from "node:crypto";
 import { readFile } from "node:fs/promises";
 import { extname, join, normalize } from "node:path";
 import { pathToFileURL } from "node:url";
@@ -38,14 +39,18 @@ function json(res, status, payload) {
   res.end(JSON.stringify(payload, null, 2));
 }
 
-export function errorEnvelope(code, message) {
-  return {
+export function errorEnvelope(code, message, details = {}) {
+  const out = {
     ok: false,
     error: {
       code,
       message
     }
   };
+  if (details && typeof details === "object") {
+    Object.assign(out, details);
+  }
+  return out;
 }
 
 export function isAuthorizedRequest(req) {
@@ -149,11 +154,32 @@ export function parsePayloadOptions(body) {
 }
 
 export function withEnvelopeMeta(payload) {
-  return {
+  const out = {
     api_version: API_VERSION,
     schema_version: SCHEMA_VERSION,
     ...payload
   };
+  if (!out.request_id && payload?.request_id) {
+    out.request_id = payload.request_id;
+  }
+  return out;
+}
+
+function requestIdFrom(req) {
+  const headerId = req.headers?.["x-request-id"];
+  if (typeof headerId === "string" && headerId.trim()) {
+    return headerId.trim().slice(0, 128);
+  }
+  return randomUUID();
+}
+
+function auditLog(entry) {
+  const line = {
+    ts: new Date().toISOString(),
+    service: "schemeta",
+    ...entry
+  };
+  console.log(JSON.stringify(line));
 }
 
 function clientKey(req) {
@@ -194,8 +220,22 @@ function withinRateLimit(req) {
 
 export function createRequestHandler() {
   return async (req, res) => {
+  const requestId = requestIdFrom(req);
+  const startedAt = Date.now();
+  res.setHeader("x-request-id", requestId);
+  res.on("finish", () => {
+    auditLog({
+      request_id: requestId,
+      method: req.method,
+      path: req.url,
+      status: res.statusCode,
+      duration_ms: Date.now() - startedAt,
+      client: clientKey(req)
+    });
+  });
+
   if (!req.url || !req.method) {
-    return json(res, 400, errorEnvelope("invalid_request", "Invalid request."));
+    return json(res, 400, errorEnvelope("invalid_request", "Invalid request.", { request_id: requestId }));
   }
 
   const pathname = new URL(req.url, "http://localhost").pathname;
@@ -211,6 +251,7 @@ export function createRequestHandler() {
     return json(res, 200, {
       ok: true,
       service: "schemeta",
+      request_id: requestId,
       api_version: API_VERSION,
       schema_version: SCHEMA_VERSION,
       status: "ok",
@@ -221,6 +262,7 @@ export function createRequestHandler() {
   if (req.method === "GET" && pathname === "/mcp/ui-bundle") {
     return json(res, 200, {
       ok: true,
+      request_id: requestId,
       name: "schemeta-workspace",
       version: "0.2.0",
       api_version: API_VERSION,
@@ -234,56 +276,56 @@ export function createRequestHandler() {
   if (req.method === "POST" && pathname === "/analyze") {
     if (!isAuthorizedRequest(req)) {
       res.setHeader("WWW-Authenticate", "Bearer");
-      return json(res, 401, errorEnvelope("unauthorized", "Missing or invalid API token."));
+      return json(res, 401, errorEnvelope("unauthorized", "Missing or invalid API token.", { request_id: requestId }));
     }
     if (!withinRateLimit(req)) {
-      return json(res, 429, errorEnvelope("rate_limited", "Rate limit exceeded. Try again shortly."));
+      return json(res, 429, errorEnvelope("rate_limited", "Rate limit exceeded. Try again shortly.", { request_id: requestId }));
     }
     try {
       const body = await readBody(req);
       const parsed = parsePayloadOptions(body);
-      return json(res, 200, withEnvelopeMeta(analyze(parsed.payload, parsed.options)));
+      return json(res, 200, withEnvelopeMeta({ request_id: requestId, ...analyze(parsed.payload, parsed.options) }));
     } catch (err) {
       if (err?.code === "PAYLOAD_TOO_LARGE") {
-        return json(res, 413, errorEnvelope("payload_too_large", `Request body exceeds ${MAX_BODY_BYTES} bytes.`));
+        return json(res, 413, errorEnvelope("payload_too_large", `Request body exceeds ${MAX_BODY_BYTES} bytes.`, { request_id: requestId }));
       }
       if (err?.code === "INVALID_JSON") {
-        return json(res, 400, errorEnvelope("invalid_json", "Invalid JSON payload."));
+        return json(res, 400, errorEnvelope("invalid_json", "Invalid JSON payload.", { request_id: requestId }));
       }
-      return json(res, 500, errorEnvelope("internal_error", "Request failed."));
+      return json(res, 500, errorEnvelope("internal_error", "Request failed.", { request_id: requestId }));
     }
   }
 
   if (req.method === "POST" && pathname === "/compile") {
     if (!isAuthorizedRequest(req)) {
       res.setHeader("WWW-Authenticate", "Bearer");
-      return json(res, 401, errorEnvelope("unauthorized", "Missing or invalid API token."));
+      return json(res, 401, errorEnvelope("unauthorized", "Missing or invalid API token.", { request_id: requestId }));
     }
     if (!withinRateLimit(req)) {
-      return json(res, 429, errorEnvelope("rate_limited", "Rate limit exceeded. Try again shortly."));
+      return json(res, 429, errorEnvelope("rate_limited", "Rate limit exceeded. Try again shortly.", { request_id: requestId }));
     }
     try {
       const body = await readBody(req);
       const parsed = parsePayloadOptions(body);
-      return json(res, 200, withEnvelopeMeta(compile(parsed.payload, parsed.options)));
+      return json(res, 200, withEnvelopeMeta({ request_id: requestId, ...compile(parsed.payload, parsed.options) }));
     } catch (err) {
       if (err?.code === "PAYLOAD_TOO_LARGE") {
-        return json(res, 413, errorEnvelope("payload_too_large", `Request body exceeds ${MAX_BODY_BYTES} bytes.`));
+        return json(res, 413, errorEnvelope("payload_too_large", `Request body exceeds ${MAX_BODY_BYTES} bytes.`, { request_id: requestId }));
       }
       if (err?.code === "INVALID_JSON") {
-        return json(res, 400, errorEnvelope("invalid_json", "Invalid JSON payload."));
+        return json(res, 400, errorEnvelope("invalid_json", "Invalid JSON payload.", { request_id: requestId }));
       }
-      return json(res, 500, errorEnvelope("internal_error", "Request failed."));
+      return json(res, 500, errorEnvelope("internal_error", "Request failed.", { request_id: requestId }));
     }
   }
 
   if (req.method === "POST" && pathname === "/layout/auto") {
     if (!isAuthorizedRequest(req)) {
       res.setHeader("WWW-Authenticate", "Bearer");
-      return json(res, 401, errorEnvelope("unauthorized", "Missing or invalid API token."));
+      return json(res, 401, errorEnvelope("unauthorized", "Missing or invalid API token.", { request_id: requestId }));
     }
     if (!withinRateLimit(req)) {
-      return json(res, 429, errorEnvelope("rate_limited", "Rate limit exceeded. Try again shortly."));
+      return json(res, 429, errorEnvelope("rate_limited", "Rate limit exceeded. Try again shortly.", { request_id: requestId }));
     }
     try {
       const body = await readBody(req);
@@ -293,6 +335,7 @@ export function createRequestHandler() {
       const laidOut = applyLayoutToModel(model, { respectLocks: false });
       return json(res, 200, {
         ok: true,
+        request_id: requestId,
         api_version: API_VERSION,
         schema_version: SCHEMA_VERSION,
         model: laidOut,
@@ -300,22 +343,22 @@ export function createRequestHandler() {
       });
     } catch (err) {
       if (err?.code === "PAYLOAD_TOO_LARGE") {
-        return json(res, 413, errorEnvelope("payload_too_large", `Request body exceeds ${MAX_BODY_BYTES} bytes.`));
+        return json(res, 413, errorEnvelope("payload_too_large", `Request body exceeds ${MAX_BODY_BYTES} bytes.`, { request_id: requestId }));
       }
       if (err?.code === "INVALID_JSON") {
-        return json(res, 400, errorEnvelope("invalid_json", "Invalid JSON payload."));
+        return json(res, 400, errorEnvelope("invalid_json", "Invalid JSON payload.", { request_id: requestId }));
       }
-      return json(res, 500, errorEnvelope("internal_error", "Layout auto failed."));
+      return json(res, 500, errorEnvelope("internal_error", "Layout auto failed.", { request_id: requestId }));
     }
   }
 
   if (req.method === "POST" && pathname === "/layout/tidy") {
     if (!isAuthorizedRequest(req)) {
       res.setHeader("WWW-Authenticate", "Bearer");
-      return json(res, 401, errorEnvelope("unauthorized", "Missing or invalid API token."));
+      return json(res, 401, errorEnvelope("unauthorized", "Missing or invalid API token.", { request_id: requestId }));
     }
     if (!withinRateLimit(req)) {
-      return json(res, 429, errorEnvelope("rate_limited", "Rate limit exceeded. Try again shortly."));
+      return json(res, 429, errorEnvelope("rate_limited", "Rate limit exceeded. Try again shortly.", { request_id: requestId }));
     }
     try {
       const body = await readBody(req);
@@ -325,6 +368,7 @@ export function createRequestHandler() {
       const laidOut = applyLayoutToModel(model, { respectLocks: true });
       return json(res, 200, {
         ok: true,
+        request_id: requestId,
         api_version: API_VERSION,
         schema_version: SCHEMA_VERSION,
         model: laidOut,
@@ -332,12 +376,12 @@ export function createRequestHandler() {
       });
     } catch (err) {
       if (err?.code === "PAYLOAD_TOO_LARGE") {
-        return json(res, 413, errorEnvelope("payload_too_large", `Request body exceeds ${MAX_BODY_BYTES} bytes.`));
+        return json(res, 413, errorEnvelope("payload_too_large", `Request body exceeds ${MAX_BODY_BYTES} bytes.`, { request_id: requestId }));
       }
       if (err?.code === "INVALID_JSON") {
-        return json(res, 400, errorEnvelope("invalid_json", "Invalid JSON payload."));
+        return json(res, 400, errorEnvelope("invalid_json", "Invalid JSON payload.", { request_id: requestId }));
       }
-      return json(res, 500, errorEnvelope("internal_error", "Layout tidy failed."));
+      return json(res, 500, errorEnvelope("internal_error", "Layout tidy failed.", { request_id: requestId }));
     }
   }
 
@@ -348,7 +392,7 @@ export function createRequestHandler() {
     }
   }
 
-  return json(res, 404, errorEnvelope("not_found", "Not found."));
+  return json(res, 404, errorEnvelope("not_found", "Not found.", { request_id: requestId }));
   };
 }
 

tests/api-contract.test.js

@@ -17,11 +17,12 @@ import {
 
 test("REST compile contract shape is stable with version metadata", () => {
   const parsed = parsePayloadOptions({ payload: fixture, options: { render_mode: "schematic_stub" } });
-  const body = withRestEnvelopeMeta(compile(parsed.payload, parsed.options));
+  const body = withRestEnvelopeMeta({ request_id: "req-test-1", ...compile(parsed.payload, parsed.options) });
 
   assert.equal(body.ok, true);
   assert.equal(body.api_version, REST_API_VERSION);
   assert.equal(body.schema_version, REST_SCHEMA_VERSION);
+  assert.equal(body.request_id, "req-test-1");
   assert.ok(Array.isArray(body.errors));
   assert.ok(Array.isArray(body.warnings));
   assert.ok(Array.isArray(body.bus_groups));
@@ -42,10 +43,11 @@ test("REST analyze contract shape is stable with version metadata", () => {
 });
 
 test("REST error envelope exposes stable code/message fields", () => {
-  const body = errorEnvelope("rate_limited", "Rate limit exceeded. Try again shortly.");
+  const body = errorEnvelope("rate_limited", "Rate limit exceeded. Try again shortly.", { request_id: "req-test-2" });
   assert.equal(body.ok, false);
   assert.equal(body.error.code, "rate_limited");
   assert.equal(typeof body.error.message, "string");
+  assert.equal(body.request_id, "req-test-2");
 });
 
 test("MCP schemeta_compile returns structured content with version metadata", () => {