From e18cf0580e4fccd69023fe9b631548ea3593500e Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Tue, 15 Sep 2026 16:36:08 -0400 Subject: [PATCH 01/45] feat: add sandbox upload endpoint; refs #147 --- .../src/controllers/autobidsify.controller.js | 52 +++++++++++++++++++ backend/src/routes/autobidsify.routes.js | 9 ++++ backend/src/server.js | 2 + 3 files changed, 63 insertions(+) create mode 100644 backend/src/controllers/autobidsify.controller.js create mode 100644 backend/src/routes/autobidsify.routes.js diff --git a/backend/src/controllers/autobidsify.controller.js b/backend/src/controllers/autobidsify.controller.js new file mode 100644 index 0000000..a70eb06 --- /dev/null +++ b/backend/src/controllers/autobidsify.controller.js @@ -0,0 +1,52 @@ +const axios = require("axios"); + +// Upload a JSON document to the sandbox CouchDB database. CouchDB credentials +// come from .env (server-only) and are never exposed to the client. Minimal +// version: POST to the database so CouchDB assigns the document _id. +// (Auth token, rate limiting, and duplicate handling are intentionally not +// included yet.) +const COUCHDB_URL = process.env.COUCHDB_URL; +const UPLOAD_DB = process.env.COUCHDB_UPLOAD_DB; + +const authHeader = () => + "Basic " + + Buffer.from( + `${process.env.COUCHDB_UPLOAD_USER}:${process.env.COUCHDB_UPLOAD_PASSWORD}` + ).toString("base64"); + +const uploadToSandbox = async (req, res) => { + try { + const doc = req.body; + if (!doc || typeof doc !== "object" || Array.isArray(doc)) { + return res + .status(400) + .json({ error: "Request body must be a JSON object" }); + } + + // POST to the database → CouchDB generates the _id. + const response = await axios.post(`${COUCHDB_URL}/${UPLOAD_DB}`, doc, { + headers: { + "Content-Type": "application/json", + Authorization: authHeader(), + }, + }); + + res.status(201).json({ + ok: true, + id: response.data.id, + rev: response.data.rev, + }); + } catch (err) { + console.error( + "Sandbox upload failed:", + err.response?.status, + err.message + ); + res.status(err.response?.status || 500).json({ + error: "Upload failed", + detail: err.response?.data || err.message, + }); + } +}; + +module.exports = { uploadToSandbox }; diff --git a/backend/src/routes/autobidsify.routes.js b/backend/src/routes/autobidsify.routes.js new file mode 100644 index 0000000..b7cde26 --- /dev/null +++ b/backend/src/routes/autobidsify.routes.js @@ -0,0 +1,9 @@ +const express = require("express"); +const { uploadToSandbox } = require("../controllers/autobidsify.controller"); + +const router = express.Router(); + +// POST /api/v1/autobidsify/upload — upload a JSON document to the sandbox db. +router.post("/upload", uploadToSandbox); + +module.exports = router; diff --git a/backend/src/server.js b/backend/src/server.js index d0d043d..893823a 100644 --- a/backend/src/server.js +++ b/backend/src/server.js @@ -15,6 +15,7 @@ const collectionRoutes = require("./routes/collection.route"); const projectRoutes = require("./routes/projects.routes"); const ollamaRoutes = require("./routes/ollama.routes"); const ollamaPublicRoutes = require("./routes/ollama.public.routes"); +const autobidsifyRoutes = require("./routes/autobidsify.routes"); const app = express(); const PORT = process.env.PORT || 5000; @@ -53,6 +54,7 @@ app.use("/api/v1/collections", collectionRoutes); app.use("/api/v1/projects", projectRoutes); app.use("/api/v1/ollama", ollamaRoutes); app.use("/api/v1/ollama-public", ollamaPublicRoutes); +app.use("/api/v1/autobidsify", autobidsifyRoutes); // health check endpoint app.get("/api/health", async (req, res) => { From e878fd54a190b2c32cf965851ee9a150cd3b6a18 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Tue, 15 Sep 2026 17:06:52 -0400 Subject: [PATCH 02/45] feat: upload to zodiac sandbox via timestamp update handler --- .../src/controllers/autobidsify.controller.js | 57 +++++++++++-------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/backend/src/controllers/autobidsify.controller.js b/backend/src/controllers/autobidsify.controller.js index a70eb06..e22addb 100644 --- a/backend/src/controllers/autobidsify.controller.js +++ b/backend/src/controllers/autobidsify.controller.js @@ -1,18 +1,28 @@ const axios = require("axios"); +const crypto = require("crypto"); -// Upload a JSON document to the sandbox CouchDB database. CouchDB credentials -// come from .env (server-only) and are never exposed to the client. Minimal -// version: POST to the database so CouchDB assigns the document _id. -// (Auth token, rate limiting, and duplicate handling are intentionally not -// included yet.) -const COUCHDB_URL = process.env.COUCHDB_URL; +// Upload a JSON document to the sandbox CouchDB database. The upload target +// (URL + db) and any credentials come from .env (server-only) and are never +// exposed to the client. Minimal version: POST to the database so CouchDB +// assigns the document _id. (Auth token, rate limiting, and duplicate handling +// are intentionally not included yet.) +// +// COUCHDB_UPLOAD_URL lets uploads target a separate CouchDB (e.g. the zodiac +// sandbox) while the read endpoints keep using COUCHDB_URL. Basic auth is only +// attached when upload creds are set — the sandbox may allow anonymous upload. +const UPLOAD_URL = process.env.COUCHDB_UPLOAD_URL || process.env.COUCHDB_URL; const UPLOAD_DB = process.env.COUCHDB_UPLOAD_DB; -const authHeader = () => - "Basic " + - Buffer.from( - `${process.env.COUCHDB_UPLOAD_USER}:${process.env.COUCHDB_UPLOAD_PASSWORD}` - ).toString("base64"); +const buildHeaders = () => { + const headers = { "Content-Type": "application/json" }; + const user = process.env.COUCHDB_UPLOAD_USER; + const pass = process.env.COUCHDB_UPLOAD_PASSWORD; + if (user && pass) { + headers.Authorization = + "Basic " + Buffer.from(`${user}:${pass}`).toString("base64"); + } + return headers; +}; const uploadToSandbox = async (req, res) => { try { @@ -23,19 +33,20 @@ const uploadToSandbox = async (req, res) => { .json({ error: "Request body must be a JSON object" }); } - // POST to the database → CouchDB generates the _id. - const response = await axios.post(`${COUCHDB_URL}/${UPLOAD_DB}`, doc, { - headers: { - "Content-Type": "application/json", - Authorization: authHeader(), - }, - }); + // Write via the CouchDB `timestamp` update handler (not a plain POST/PUT) + // so the doc gets `.datainfo` CreateTime/UpdateTime, matching how datasets + // are stamped elsewhere. We mint the id ourselves so we can return it; PUT + // to _update/timestamp/ creates the doc and merges the body. + const id = crypto.randomUUID(); + const url = `${UPLOAD_URL}/${UPLOAD_DB}/_design/qq/_update/timestamp/${encodeURIComponent( + id + )}`; + const response = await axios.put(url, doc, { headers: buildHeaders() }); + // Update handlers return the new revision in this header (the body is a + // plain success message, not JSON). + const rev = response.headers["x-couch-update-newrev"]; - res.status(201).json({ - ok: true, - id: response.data.id, - rev: response.data.rev, - }); + res.status(201).json({ ok: true, id, rev }); } catch (err) { console.error( "Sandbox upload failed:", From becb5354a556310f762e068f2584b08fc0674a9e Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Wed, 16 Sep 2026 11:02:26 -0400 Subject: [PATCH 03/45] feat(uploads): add authenticated POST /api/v1/uploads for JSON submissions; refs #147 --- .../src/controllers/autobidsify.controller.js | 63 ---------------- backend/src/controllers/uploads.controller.js | 72 +++++++++++++++++++ backend/src/routes/autobidsify.routes.js | 9 --- backend/src/routes/uploads.routes.js | 25 +++++++ backend/src/server.js | 4 +- 5 files changed, 99 insertions(+), 74 deletions(-) delete mode 100644 backend/src/controllers/autobidsify.controller.js create mode 100644 backend/src/controllers/uploads.controller.js delete mode 100644 backend/src/routes/autobidsify.routes.js create mode 100644 backend/src/routes/uploads.routes.js diff --git a/backend/src/controllers/autobidsify.controller.js b/backend/src/controllers/autobidsify.controller.js deleted file mode 100644 index e22addb..0000000 --- a/backend/src/controllers/autobidsify.controller.js +++ /dev/null @@ -1,63 +0,0 @@ -const axios = require("axios"); -const crypto = require("crypto"); - -// Upload a JSON document to the sandbox CouchDB database. The upload target -// (URL + db) and any credentials come from .env (server-only) and are never -// exposed to the client. Minimal version: POST to the database so CouchDB -// assigns the document _id. (Auth token, rate limiting, and duplicate handling -// are intentionally not included yet.) -// -// COUCHDB_UPLOAD_URL lets uploads target a separate CouchDB (e.g. the zodiac -// sandbox) while the read endpoints keep using COUCHDB_URL. Basic auth is only -// attached when upload creds are set — the sandbox may allow anonymous upload. -const UPLOAD_URL = process.env.COUCHDB_UPLOAD_URL || process.env.COUCHDB_URL; -const UPLOAD_DB = process.env.COUCHDB_UPLOAD_DB; - -const buildHeaders = () => { - const headers = { "Content-Type": "application/json" }; - const user = process.env.COUCHDB_UPLOAD_USER; - const pass = process.env.COUCHDB_UPLOAD_PASSWORD; - if (user && pass) { - headers.Authorization = - "Basic " + Buffer.from(`${user}:${pass}`).toString("base64"); - } - return headers; -}; - -const uploadToSandbox = async (req, res) => { - try { - const doc = req.body; - if (!doc || typeof doc !== "object" || Array.isArray(doc)) { - return res - .status(400) - .json({ error: "Request body must be a JSON object" }); - } - - // Write via the CouchDB `timestamp` update handler (not a plain POST/PUT) - // so the doc gets `.datainfo` CreateTime/UpdateTime, matching how datasets - // are stamped elsewhere. We mint the id ourselves so we can return it; PUT - // to _update/timestamp/ creates the doc and merges the body. - const id = crypto.randomUUID(); - const url = `${UPLOAD_URL}/${UPLOAD_DB}/_design/qq/_update/timestamp/${encodeURIComponent( - id - )}`; - const response = await axios.put(url, doc, { headers: buildHeaders() }); - // Update handlers return the new revision in this header (the body is a - // plain success message, not JSON). - const rev = response.headers["x-couch-update-newrev"]; - - res.status(201).json({ ok: true, id, rev }); - } catch (err) { - console.error( - "Sandbox upload failed:", - err.response?.status, - err.message - ); - res.status(err.response?.status || 500).json({ - error: "Upload failed", - detail: err.response?.data || err.message, - }); - } -}; - -module.exports = { uploadToSandbox }; diff --git a/backend/src/controllers/uploads.controller.js b/backend/src/controllers/uploads.controller.js new file mode 100644 index 0000000..727526d --- /dev/null +++ b/backend/src/controllers/uploads.controller.js @@ -0,0 +1,72 @@ +const axios = require("axios"); +const crypto = require("crypto"); + +// Upload a JSON document to the sandbox CouchDB database for review. This is a +// general, login-gated endpoint (any authenticated user can submit a JSON file +// — the AutoBIDSify app just links here). The upload target (URL + db) and any +// credentials come from .env (server-only) and are never exposed to the client. +// +// COUCHDB_UPLOAD_URL lets uploads target a separate CouchDB (the zodiac +// sandbox) while the read endpoints keep using COUCHDB_URL. Basic auth is only +// attached when upload creds are set — the sandbox allows anonymous writes and +// access is gated by the website login instead. +const UPLOAD_URL = process.env.COUCHDB_UPLOAD_URL || process.env.COUCHDB_URL; +const UPLOAD_DB = process.env.COUCHDB_UPLOAD_DB; + +const buildHeaders = () => { + const headers = { "Content-Type": "application/json" }; + const user = process.env.COUCHDB_UPLOAD_USER; + const pass = process.env.COUCHDB_UPLOAD_PASSWORD; + if (user && pass) { + headers.Authorization = + "Basic " + Buffer.from(`${user}:${pass}`).toString("base64"); + } + return headers; +}; + +const createUpload = async (req, res) => { + try { + const doc = req.body; + if (!doc || typeof doc !== "object" || Array.isArray(doc)) { + return res + .status(400) + .json({ error: "Request body must be a JSON object" }); + } + + // Stamp who submitted + a pending status into .datainfo (merged alongside + // the CreateTime/UpdateTime the update handler adds) so the review queue + // shows the submitter. requireAuth guarantees req.user is set. + const meta = { + status: "pending", + submittedBy: { + id: req.user.id, + username: req.user.username, + email: req.user.email, + }, + }; + const payload = { + ...doc, + ".datainfo": { ...(doc[".datainfo"] || {}), ...meta }, + }; + + // Write via the CouchDB `timestamp` update handler (not a plain POST/PUT) + // so the doc gets .datainfo CreateTime/UpdateTime. Mint the id so we can + // return it; PUT to _update/timestamp/ creates the doc and merges body. + const id = crypto.randomUUID(); + const url = `${UPLOAD_URL}/${UPLOAD_DB}/_design/qq/_update/timestamp/${encodeURIComponent( + id + )}`; + const response = await axios.put(url, payload, { headers: buildHeaders() }); + const rev = response.headers["x-couch-update-newrev"]; + + res.status(201).json({ ok: true, id, rev }); + } catch (err) { + console.error("Upload failed:", err.response?.status, err.message); + res.status(err.response?.status || 500).json({ + error: "Upload failed", + detail: err.response?.data || err.message, + }); + } +}; + +module.exports = { createUpload }; diff --git a/backend/src/routes/autobidsify.routes.js b/backend/src/routes/autobidsify.routes.js deleted file mode 100644 index b7cde26..0000000 --- a/backend/src/routes/autobidsify.routes.js +++ /dev/null @@ -1,9 +0,0 @@ -const express = require("express"); -const { uploadToSandbox } = require("../controllers/autobidsify.controller"); - -const router = express.Router(); - -// POST /api/v1/autobidsify/upload — upload a JSON document to the sandbox db. -router.post("/upload", uploadToSandbox); - -module.exports = router; diff --git a/backend/src/routes/uploads.routes.js b/backend/src/routes/uploads.routes.js new file mode 100644 index 0000000..567b3c9 --- /dev/null +++ b/backend/src/routes/uploads.routes.js @@ -0,0 +1,25 @@ +const express = require("express"); +const rateLimit = require("express-rate-limit"); +const { requireAuth } = require("../middleware/auth.middleware"); +const { createUpload } = require("../controllers/uploads.controller"); + +const router = express.Router(); + +// Per-USER daily cap so one account can't flood the server. requireAuth runs +// first, so req.user is set and we key by user id (falls back to IP). The limit +// is configurable via UPLOAD_RATE_LIMIT_MAX (default 20) — no code change to tune. +const uploadRateLimit = rateLimit({ + windowMs: 24 * 60 * 60 * 1000, // 24 hours + max: parseInt(process.env.UPLOAD_RATE_LIMIT_MAX || "20", 10), + standardHeaders: true, + legacyHeaders: false, + keyGenerator: (req) => (req.user && req.user.id ? String(req.user.id) : req.ip), + message: { + error: "Daily upload limit reached. Please try again tomorrow.", + }, +}); + +// POST /api/v1/uploads — upload a JSON document (logged-in users only). +router.post("/", requireAuth, uploadRateLimit, createUpload); + +module.exports = router; diff --git a/backend/src/server.js b/backend/src/server.js index 893823a..f7dcb5f 100644 --- a/backend/src/server.js +++ b/backend/src/server.js @@ -15,7 +15,7 @@ const collectionRoutes = require("./routes/collection.route"); const projectRoutes = require("./routes/projects.routes"); const ollamaRoutes = require("./routes/ollama.routes"); const ollamaPublicRoutes = require("./routes/ollama.public.routes"); -const autobidsifyRoutes = require("./routes/autobidsify.routes"); +const uploadsRoutes = require("./routes/uploads.routes"); const app = express(); const PORT = process.env.PORT || 5000; @@ -54,7 +54,7 @@ app.use("/api/v1/collections", collectionRoutes); app.use("/api/v1/projects", projectRoutes); app.use("/api/v1/ollama", ollamaRoutes); app.use("/api/v1/ollama-public", ollamaPublicRoutes); -app.use("/api/v1/autobidsify", autobidsifyRoutes); +app.use("/api/v1/uploads", uploadsRoutes); // health check endpoint app.get("/api/health", async (req, res) => { From 6550035f294cde5c3d4f2b039728ec02dd77a128 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Wed, 16 Sep 2026 11:13:52 -0400 Subject: [PATCH 04/45] feat(uploads): add login-gated /upload page for JSON submissions; refs #147 --- src/components/Routes.tsx | 2 + src/pages/UploadPage.tsx | 175 +++++++++++++++++++++++++++++++++ src/services/upload.service.ts | 26 +++++ src/types/routes.enum.ts | 1 + 4 files changed, 204 insertions(+) create mode 100644 src/pages/UploadPage.tsx create mode 100644 src/services/upload.service.ts diff --git a/src/components/Routes.tsx b/src/components/Routes.tsx index 10b15e9..addfae1 100644 --- a/src/components/Routes.tsx +++ b/src/components/Routes.tsx @@ -13,6 +13,7 @@ import Home from "pages/Home"; import ResendVerification from "pages/ResendVerification"; import SearchPage from "pages/SearchPage"; import UpdatedDatasetDetailPage from "pages/UpdatedDatasetDetailPage"; +import UploadPage from "pages/UploadPage"; import NewDatasetPage from "pages/UpdatedDatasetPage"; import VerifyEmail from "pages/VerifyEmail"; import React from "react"; @@ -70,6 +71,7 @@ const Routes = () => ( /> } /> } /> + } /> diff --git a/src/pages/UploadPage.tsx b/src/pages/UploadPage.tsx new file mode 100644 index 0000000..471993a --- /dev/null +++ b/src/pages/UploadPage.tsx @@ -0,0 +1,175 @@ +import CloudUploadIcon from "@mui/icons-material/CloudUpload"; +import { + Alert, + Box, + Button, + CircularProgress, + Container, + Link as MuiLink, + Paper, + Typography, +} from "@mui/material"; +import { Colors } from "design/theme"; +import { useAppSelector } from "hooks/useAppSelector"; +import React, { useRef, useState } from "react"; +import { Link } from "react-router-dom"; +import { AuthSelector } from "redux/auth/auth.selector"; +import { UploadService } from "services/upload.service"; +import RoutesEnum from "types/routes.enum"; + +const UploadPage: React.FC = () => { + const { isLoggedIn } = useAppSelector(AuthSelector); + const fileInputRef = useRef(null); + + const [file, setFile] = useState(null); + const [parsed, setParsed] = useState | null>(null); + const [parseError, setParseError] = useState(null); + const [submitting, setSubmitting] = useState(false); + const [result, setResult] = useState<{ id: string } | null>(null); + const [submitError, setSubmitError] = useState(null); + + const resetOutcome = () => { + setResult(null); + setSubmitError(null); + }; + + const handleFile = async (f: File | undefined) => { + resetOutcome(); + setParsed(null); + setParseError(null); + if (!f) return; + setFile(f); + try { + const text = await f.text(); + const json = JSON.parse(text); + if (!json || typeof json !== "object" || Array.isArray(json)) { + setParseError("The file must contain a single JSON object."); + return; + } + setParsed(json); + } catch (e: any) { + setParseError(`Not valid JSON: ${e.message}`); + } + }; + + const handleSubmit = async () => { + if (!parsed) return; + setSubmitting(true); + resetOutcome(); + try { + const res = await UploadService.uploadJson(parsed); + setResult({ id: res.id }); + } catch (e: any) { + setSubmitError(e.message || "Upload failed."); + } finally { + setSubmitting(false); + } + }; + + // Login guard — the endpoint requires auth; prompt to log in. + if (!isLoggedIn) { + return ( + + + + Please log in to upload + + + Uploading a dataset for review requires a NeuroJSON account. + + + + + ); + } + + return ( + + + Upload a dataset + + + Select a JSON file (e.g. the file produced by AutoBIDSify). It will be + submitted to our sandbox for the NeuroJSON team to review before it is + added to a database. + + + + handleFile(e.target.files?.[0])} + /> + + {file && ( + + Selected: {file.name}{" "} + {parsed && ( + + ✓ valid JSON + + )} + + )} + + {parseError && ( + + {parseError} + + )} + + + + + + {result && ( + + Submitted for review. Reference id: {result.id} + + )} + {submitError && ( + + {submitError} + + )} + + + + Coming from AutoBIDSify? Convert your dataset in the{" "} + + converter + + , then upload the generated JSON here. + + + ); +}; + +export default UploadPage; diff --git a/src/services/upload.service.ts b/src/services/upload.service.ts new file mode 100644 index 0000000..1f41722 --- /dev/null +++ b/src/services/upload.service.ts @@ -0,0 +1,26 @@ +import { baseURL } from "services/instance"; + +// POST /api/v1/uploads — submit a JSON document for review. Uses fetch with +// credentials:"include" (like the other authenticated calls) so the httpOnly +// auth cookie is sent; the shared axios instance has withCredentials:false. +export const UploadService = { + uploadJson: async (doc: unknown): Promise<{ id: string; rev: string }> => { + const response = await fetch(`${baseURL}/uploads`, { + method: "POST", + credentials: "include", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(doc), + }); + + const data = await response.json().catch(() => ({})); + + if (!response.ok) { + throw new Error( + (data as any).error || + (data as any).message || + `Upload failed (${response.status})` + ); + } + return data as { id: string; rev: string }; + }, +}; diff --git a/src/types/routes.enum.ts b/src/types/routes.enum.ts index e8c2658..d7aa2a3 100644 --- a/src/types/routes.enum.ts +++ b/src/types/routes.enum.ts @@ -5,5 +5,6 @@ enum RoutesEnum { ABOUT = "/about", // New route for the about page DASHBOARD = "/dashboard", BIDS_CONVERTER = "/autobidsify", + UPLOAD = "/upload", // logged-in users submit a JSON dataset for review } export default RoutesEnum; From 0e22f1f7fc3b096d4460d49c60f3640477161016 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Wed, 16 Sep 2026 11:40:02 -0400 Subject: [PATCH 05/45] style(uploads): improve UploadPage readability on the blue background; refs #147 --- src/pages/UploadPage.tsx | 53 +++++++++++++++++++++++++++++++--------- 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/src/pages/UploadPage.tsx b/src/pages/UploadPage.tsx index 471993a..7df8c8e 100644 --- a/src/pages/UploadPage.tsx +++ b/src/pages/UploadPage.tsx @@ -71,17 +71,21 @@ const UploadPage: React.FC = () => { return ( - + Please log in to upload - + Uploading a dataset for review requires a NeuroJSON account. @@ -92,10 +96,15 @@ const UploadPage: React.FC = () => { return ( - + Upload a dataset - + Select a JSON file (e.g. the file produced by AutoBIDSify). It will be submitted to our sandbox for the NeuroJSON team to review before it is added to a database. @@ -111,14 +120,23 @@ const UploadPage: React.FC = () => { /> {file && ( - + Selected: {file.name}{" "} {parsed && ( @@ -137,9 +155,14 @@ const UploadPage: React.FC = () => { + + + ); }; diff --git a/src/services/upload.service.ts b/src/services/upload.service.ts index 1f41722..6874a46 100644 --- a/src/services/upload.service.ts +++ b/src/services/upload.service.ts @@ -1,26 +1,31 @@ import { baseURL } from "services/instance"; -// POST /api/v1/uploads — submit a JSON document for review. Uses fetch with -// credentials:"include" (like the other authenticated calls) so the httpOnly -// auth cookie is sent; the shared axios instance has withCredentials:false. +export interface UploadResponse { + status: number; // HTTP status + data: any; // parsed JSON body +} + +// POST /api/v1/uploads — submit a JSON dataset for review. fetch with +// credentials:"include" sends the httpOnly auth cookie (the shared axios +// instance has withCredentials:false). Returns status + body so the caller +// branches on 201 (ok) / 409 (confirmation or blocked) / 400 / 403 itself. export const UploadService = { - uploadJson: async (doc: unknown): Promise<{ id: string; rev: string }> => { - const response = await fetch(`${baseURL}/uploads`, { + uploadJson: async ( + doc: unknown, + opts?: { datasetId?: string; confirm?: boolean } + ): Promise => { + const params = new URLSearchParams(); + if (opts?.datasetId) params.set("datasetId", opts.datasetId); + if (opts?.confirm) params.set("confirm", "true"); + const qs = params.toString() ? `?${params.toString()}` : ""; + + const response = await fetch(`${baseURL}/uploads${qs}`, { method: "POST", credentials: "include", headers: { "Content-Type": "application/json" }, body: JSON.stringify(doc), }); - const data = await response.json().catch(() => ({})); - - if (!response.ok) { - throw new Error( - (data as any).error || - (data as any).message || - `Upload failed (${response.status})` - ); - } - return data as { id: string; rev: string }; + return { status: response.status, data }; }, }; From 73e90a7a628f5bbadd2259a1347ba870ba662f0c Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Mon, 21 Sep 2026 16:00:06 -0400 Subject: [PATCH 12/45] feat(uploads): add dashboard Uploads tab with status list + update action; refs #147 --- backend/src/controllers/uploads.controller.js | 20 +- backend/src/routes/uploads.routes.js | 8 +- src/components/User/Dashboard/UploadsTab.tsx | 196 ++++++++++++++++++ src/components/User/UserDashboard.tsx | 12 ++ src/pages/UploadPage.tsx | 7 +- src/services/upload.service.ts | 17 ++ 6 files changed, 256 insertions(+), 4 deletions(-) create mode 100644 src/components/User/Dashboard/UploadsTab.tsx diff --git a/backend/src/controllers/uploads.controller.js b/backend/src/controllers/uploads.controller.js index 148008b..54ea449 100644 --- a/backend/src/controllers/uploads.controller.js +++ b/backend/src/controllers/uploads.controller.js @@ -230,4 +230,22 @@ const createUpload = async (req, res) => { } }; -module.exports = { createUpload }; +// List the caller's own submissions for the dashboard "Uploads" tab. +const listMyUploads = async (req, res) => { + try { + const rows = await sequelize.query( + `SELECT dataset_id, dataset_name, status, created_at, updated_at, + reviewed_at, promoted_db, promoted_at + FROM submissions + WHERE user_id = :uid + ORDER BY created_at DESC`, + { replacements: { uid: req.user.id }, type: sequelize.QueryTypes.SELECT } + ); + res.json(rows); + } catch (err) { + console.error("List uploads failed:", err.message); + res.status(500).json({ error: "Failed to list uploads" }); + } +}; + +module.exports = { createUpload, listMyUploads }; diff --git a/backend/src/routes/uploads.routes.js b/backend/src/routes/uploads.routes.js index cde2f6c..6c93b6a 100644 --- a/backend/src/routes/uploads.routes.js +++ b/backend/src/routes/uploads.routes.js @@ -1,7 +1,10 @@ const express = require("express"); const rateLimit = require("express-rate-limit"); const { requireAuth } = require("../middleware/auth.middleware"); -const { createUpload } = require("../controllers/uploads.controller"); +const { + createUpload, + listMyUploads, +} = require("../controllers/uploads.controller"); const router = express.Router(); @@ -25,4 +28,7 @@ const uploadRateLimit = rateLimit({ // POST /api/v1/uploads — upload a JSON document (logged-in users only). router.post("/", requireAuth, uploadRateLimit, createUpload); +// GET /api/v1/uploads — the caller's own submissions (dashboard Uploads tab). +router.get("/", requireAuth, listMyUploads); + module.exports = router; diff --git a/src/components/User/Dashboard/UploadsTab.tsx b/src/components/User/Dashboard/UploadsTab.tsx new file mode 100644 index 0000000..7dc52f6 --- /dev/null +++ b/src/components/User/Dashboard/UploadsTab.tsx @@ -0,0 +1,196 @@ +import { CloudUpload, Edit, Visibility } from "@mui/icons-material"; +import { + Alert, + Box, + Button, + Chip, + CircularProgress, + Divider, + List, + ListItem, + ListItemText, + Paper, + Typography, +} from "@mui/material"; +import { Colors } from "design/theme"; +import React, { useEffect, useState } from "react"; +import { useNavigate } from "react-router-dom"; +import { UploadRecord, UploadService } from "services/upload.service"; + +const STATUS_CHIP: Record< + UploadRecord["status"], + { label: string; color: "default" | "info" | "warning" | "success" | "error" } +> = { + pending: { label: "Pending review", color: "info" }, + approved: { label: "Approved — awaiting promotion", color: "warning" }, + promoted: { label: "Promoted", color: "success" }, + rejected: { label: "Rejected", color: "error" }, +}; + +const UploadsTab: React.FC = () => { + const navigate = useNavigate(); + const [rows, setRows] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + useEffect(() => { + UploadService.listMine() + .then(setRows) + .catch((e) => setError(e.message)) + .finally(() => setLoading(false)); + }, []); + + const formatDate = (d: string) => + new Date(d).toLocaleDateString("en-US", { + year: "numeric", + month: "short", + day: "numeric", + }); + + if (loading) { + return ( + + + + ); + } + if (error) { + return ( + + {error} + + ); + } + + return ( + + + + + My Uploads + + + Datasets you have submitted for review. + + + + + + {rows.length === 0 ? ( + + + + No uploads yet + + + Datasets you submit for review will appear here. + + + ) : ( + + + {rows.map((r, index) => { + const chip = STATUS_CHIP[r.status]; + return ( + + {index > 0 && } + + + + + {r.dataset_name || r.dataset_id} + + + + + } + secondary={`Submitted ${formatDate(r.created_at)}${ + r.updated_at && r.updated_at !== r.created_at + ? ` · updated ${formatDate(r.updated_at)}` + : "" + }`} + /> + + + {r.status === "promoted" && r.promoted_db && ( + + )} + + + + + ); + })} + + + )} + + ); +}; + +export default UploadsTab; diff --git a/src/components/User/UserDashboard.tsx b/src/components/User/UserDashboard.tsx index 42eab48..f8ddeed 100644 --- a/src/components/User/UserDashboard.tsx +++ b/src/components/User/UserDashboard.tsx @@ -3,6 +3,7 @@ import ProfileTab from "./Dashboard/ProfileTab"; import ProjectsTab from "./Dashboard/ProjectsTab"; import SavedDatasetsTab from "./Dashboard/SavedDatasetsTab"; import SecurityTab from "./Dashboard/SecurityTab"; +import UploadsTab from "./Dashboard/UploadsTab"; import LikedDatasetsTab from "./Dashboard/likedDatasetsTab"; import { AccountCircle, @@ -11,6 +12,7 @@ import { Bookmark, Favorite, FolderOpen, + CloudUpload, } from "@mui/icons-material"; import { Box, @@ -56,6 +58,7 @@ const TAB_INDEX: Record = { liked: 3, projects: 4, settings: 5, + uploads: 6, }; const TAB_NAME = Object.fromEntries( @@ -177,6 +180,12 @@ const UserDashboard: React.FC = () => { id="dashboard-tab-2" aria-controls="dashboard-tabpanel-2" /> + } + label="Uploads" + id="dashboard-tab-6" + aria-controls="dashboard-tabpanel-6" + /> @@ -194,6 +203,9 @@ const UserDashboard: React.FC = () => { + + + ); diff --git a/src/pages/UploadPage.tsx b/src/pages/UploadPage.tsx index c7e06cd..0124bca 100644 --- a/src/pages/UploadPage.tsx +++ b/src/pages/UploadPage.tsx @@ -18,7 +18,7 @@ import { import { Colors } from "design/theme"; import { useAppSelector } from "hooks/useAppSelector"; import React, { useRef, useState } from "react"; -import { Link } from "react-router-dom"; +import { Link, useLocation } from "react-router-dom"; import { AuthSelector } from "redux/auth/auth.selector"; import { UploadService } from "services/upload.service"; import RoutesEnum from "types/routes.enum"; @@ -26,11 +26,14 @@ import RoutesEnum from "types/routes.enum"; const UploadPage: React.FC = () => { const { isLoggedIn } = useAppSelector(AuthSelector); const fileInputRef = useRef(null); + const location = useLocation(); + const initialDatasetId = + new URLSearchParams(location.search).get("datasetId") ?? ""; const [file, setFile] = useState(null); const [parsed, setParsed] = useState | null>(null); const [parseError, setParseError] = useState(null); - const [datasetId, setDatasetId] = useState(""); + const [datasetId, setDatasetId] = useState(initialDatasetId); const [submitting, setSubmitting] = useState(false); const [result, setResult] = useState<{ datasetId: string; diff --git a/src/services/upload.service.ts b/src/services/upload.service.ts index 6874a46..a1da21b 100644 --- a/src/services/upload.service.ts +++ b/src/services/upload.service.ts @@ -5,6 +5,17 @@ export interface UploadResponse { data: any; // parsed JSON body } +export interface UploadRecord { + dataset_id: string; + dataset_name: string | null; + status: "pending" | "approved" | "rejected" | "promoted"; + created_at: string; + updated_at: string; + reviewed_at: string | null; + promoted_db: string | null; + promoted_at: string | null; +} + // POST /api/v1/uploads — submit a JSON dataset for review. fetch with // credentials:"include" sends the httpOnly auth cookie (the shared axios // instance has withCredentials:false). Returns status + body so the caller @@ -28,4 +39,10 @@ export const UploadService = { const data = await response.json().catch(() => ({})); return { status: response.status, data }; }, + + listMine: async (): Promise => { + const res = await fetch(`${baseURL}/uploads`, { credentials: "include" }); + if (!res.ok) throw new Error(`Failed to load uploads (${res.status})`); + return res.json(); + }, }; From 18631aeb62c213d05f94f9fe389fc7135a1ba6bf Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Mon, 21 Sep 2026 16:43:24 -0400 Subject: [PATCH 13/45] feat(uploads): add dataset name input on /upload page; refs #147 --- src/components/User/Dashboard/UploadsTab.tsx | 14 ++++++++--- src/pages/UploadPage.tsx | 26 +++++++++++++++++++- src/services/upload.service.ts | 3 ++- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/src/components/User/Dashboard/UploadsTab.tsx b/src/components/User/Dashboard/UploadsTab.tsx index 7dc52f6..55d5e3f 100644 --- a/src/components/User/Dashboard/UploadsTab.tsx +++ b/src/components/User/Dashboard/UploadsTab.tsx @@ -19,9 +19,17 @@ import { UploadRecord, UploadService } from "services/upload.service"; const STATUS_CHIP: Record< UploadRecord["status"], - { label: string; color: "default" | "info" | "warning" | "success" | "error" } + { + label: string; + color?: "default" | "info" | "warning" | "success" | "error"; + sx?: object; + } > = { - pending: { label: "Pending review", color: "info" }, + // Pending uses the theme purple (not MUI's default blue). + pending: { + label: "Pending review", + sx: { backgroundColor: Colors.purple, color: Colors.white }, + }, approved: { label: "Approved — awaiting promotion", color: "warning" }, promoted: { label: "Promoted", color: "success" }, rejected: { label: "Rejected", color: "error" }, @@ -135,7 +143,7 @@ const UploadsTab: React.FC = () => { label={chip.label} color={chip.color} size="small" - sx={{ height: 20 }} + sx={{ height: 20, ...(chip.sx || {}) }} /> } diff --git a/src/pages/UploadPage.tsx b/src/pages/UploadPage.tsx index 0124bca..d5b8a76 100644 --- a/src/pages/UploadPage.tsx +++ b/src/pages/UploadPage.tsx @@ -34,6 +34,7 @@ const UploadPage: React.FC = () => { const [parsed, setParsed] = useState | null>(null); const [parseError, setParseError] = useState(null); const [datasetId, setDatasetId] = useState(initialDatasetId); + const [datasetName, setDatasetName] = useState(""); const [submitting, setSubmitting] = useState(false); const [result, setResult] = useState<{ datasetId: string; @@ -80,6 +81,7 @@ const UploadPage: React.FC = () => { try { const { status, data } = await UploadService.uploadJson(parsed, { datasetId: datasetId.trim() || undefined, + datasetName: datasetName.trim() || undefined, confirm, }); if (status === 201) { @@ -132,6 +134,11 @@ const UploadPage: React.FC = () => { ); } + // If the file already carries a name, use it and disable the name field. + const dd = parsed?.["dataset_description.json"] as any; + const nameFromFile = + dd && typeof dd === "object" && !Array.isArray(dd) ? dd.Name : undefined; + return ( { )} + {/* Dataset name — used when the file has no dataset_description.json Name */} + setDatasetName(e.target.value)} + disabled={!!nameFromFile} + size="small" + fullWidth + sx={{ mt: 2.5 }} + helperText={ + nameFromFile + ? 'Name taken from your file ("dataset_description.json" → Name).' + : 'No name found in your file ("dataset_description.json" → Name) — enter one here.' + } + /> + {/* Optional: updating an existing dataset */} setDatasetId(e.target.value)} diff --git a/src/services/upload.service.ts b/src/services/upload.service.ts index a1da21b..368f1e8 100644 --- a/src/services/upload.service.ts +++ b/src/services/upload.service.ts @@ -23,10 +23,11 @@ export interface UploadRecord { export const UploadService = { uploadJson: async ( doc: unknown, - opts?: { datasetId?: string; confirm?: boolean } + opts?: { datasetId?: string; datasetName?: string; confirm?: boolean } ): Promise => { const params = new URLSearchParams(); if (opts?.datasetId) params.set("datasetId", opts.datasetId); + if (opts?.datasetName) params.set("datasetName", opts.datasetName); if (opts?.confirm) params.set("confirm", "true"); const qs = params.toString() ? `?${params.toString()}` : ""; From d88eceb85088b4c16f13436172b734a223e5be59 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Mon, 21 Sep 2026 16:53:25 -0400 Subject: [PATCH 14/45] fix(uploads): simplify dataset-name-required error message; refs #147 --- backend/src/controllers/uploads.controller.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/src/controllers/uploads.controller.js b/backend/src/controllers/uploads.controller.js index 54ea449..884ae87 100644 --- a/backend/src/controllers/uploads.controller.js +++ b/backend/src/controllers/uploads.controller.js @@ -63,8 +63,7 @@ const createUpload = async (req, res) => { const datasetName = nameFromDoc || req.query.datasetName; if (!datasetName) { return res.status(400).json({ - error: - "Dataset name required: provide dataset_description.json.Name in the file or a datasetName parameter.", + error: "Please enter a dataset name.", }); } // Normalize: ensure the stored doc carries the name. From 280398f6ecc4deb31dabc50bf3a49042c3a632ab Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Tue, 22 Sep 2026 11:40:55 -0400 Subject: [PATCH 15/45] feat(uploads): clarify + restyle the resubmit confirm dialog; refs#147 --- src/pages/UploadPage.tsx | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/pages/UploadPage.tsx b/src/pages/UploadPage.tsx index d5b8a76..d232f65 100644 --- a/src/pages/UploadPage.tsx +++ b/src/pages/UploadPage.tsx @@ -300,9 +300,21 @@ const UploadPage: React.FC = () => { Submit a new version? {confirmPrompt?.message} + {confirmPrompt && ( + + {confirmPrompt.code === "DATASET_ALREADY_PROMOTED" + ? "This replaces your working copy in the sandbox for review. The version already published to the public database stays unchanged — it is only replaced if this new version passes review." + : "This replaces your working copy in the sandbox and starts a new review."} + + )} - + + + + ); + } + + if (loading) { + return ( + + + + + + ); + } + + if (error || !detail) { + return ( + + {error || "Dataset not found."} + + + ); + } + + const chip = detail.status ? STATUS_CHIP[detail.status] : null; + + return ( + + + + {/* Header / metadata */} + + + + {detail.dataset_name || "(unnamed dataset)"} + + {chip && ( + + )} + + + + + Public dataset ID:{" "} + {detail.dataset_id ? ( + {detail.dataset_id} + ) : ( + Not yet assigned + )} + + {detail.requested_dataset_id && ( + + Preferred ID: {detail.requested_dataset_id} + + )} + + Internal reference: {detail.internal_id} + + + Submitted {formatDate(detail.created_at)} + {detail.updated_at && detail.updated_at !== detail.created_at + ? ` · updated ${formatDate(detail.updated_at)}` + : ""} + + + + + + {detail.status === "promoted" && + detail.promoted_db && + detail.dataset_id && ( + + )} + + + + {/* Conversation */} + + + Review discussion + + + {comments.length === 0 ? ( + + No messages yet. Use the box below to reply to the review team. + + ) : ( + + {comments.map((c) => ( + + + + {c.is_owner ? "You" : "Reviewer"} ·{" "} + {new Date(c.created_at).toLocaleString()} + + + {c.message} + + + + ))} + + )} + + + + setMessage(e.target.value)} + multiline + minRows={2} + fullWidth + /> + {postError && ( + + {postError} + + )} + + + + + + ); +}; + +export default UploadDetailPage; diff --git a/src/services/upload.service.ts b/src/services/upload.service.ts index 9fda899..a8630a9 100644 --- a/src/services/upload.service.ts +++ b/src/services/upload.service.ts @@ -24,6 +24,27 @@ export interface UploadRecord { promoted_at: string | null; } +export interface UploadDetail { + internal_id: string; + dataset_id: string | null; + requested_dataset_id: string | null; + dataset_name: string | null; + submission_id: string | null; + status: UploadRecord["status"]; + created_at: string | null; + updated_at: string | null; + promoted_db: string | null; + promoted_at: string | null; +} + +export interface Comment { + id: number; + username: string; // owner's real name, or "Reviewer" (identity masked) + message: string; + created_at: string; + is_owner: boolean; +} + // POST /api/v1/uploads — submit a JSON dataset for review. fetch with // credentials:"include" sends the httpOnly auth cookie (the shared axios // instance has withCredentials:false). Returns status + body so the caller @@ -61,4 +82,37 @@ export const UploadService = { if (!res.ok) throw new Error(`Failed to load uploads (${res.status})`); return res.json(); }, + + getUpload: async (internalId: string): Promise => { + const res = await fetch(`${baseURL}/uploads/${internalId}`, { + credentials: "include", + }); + if (!res.ok) { + const body = await res.json().catch(() => ({})); + throw new Error(body.error || `Failed to load dataset (${res.status})`); + } + return res.json(); + }, + + listComments: async (internalId: string): Promise => { + const res = await fetch(`${baseURL}/uploads/${internalId}/comments`, { + credentials: "include", + }); + if (!res.ok) throw new Error(`Failed to load comments (${res.status})`); + return res.json(); + }, + + postComment: async (internalId: string, message: string): Promise => { + const res = await fetch(`${baseURL}/uploads/${internalId}/comments`, { + method: "POST", + credentials: "include", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ message }), + }); + if (!res.ok) { + const body = await res.json().catch(() => ({})); + throw new Error(body.error || `Failed to post comment (${res.status})`); + } + return res.json(); + }, }; From 08653cf66550aeea08bb195c84eff4f123e7b42d Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Wed, 23 Sep 2026 17:48:13 -0400 Subject: [PATCH 20/45] feat(uploads): enforce unique preferred dataset ID; refs #147 --- ...3214452-add-unique-requested-dataset-id.js | 19 ++++++++ backend/src/controllers/uploads.controller.js | 44 ++++++++++++------- 2 files changed, 47 insertions(+), 16 deletions(-) create mode 100644 backend/migrations/20260923214452-add-unique-requested-dataset-id.js diff --git a/backend/migrations/20260923214452-add-unique-requested-dataset-id.js b/backend/migrations/20260923214452-add-unique-requested-dataset-id.js new file mode 100644 index 0000000..de2ad8d --- /dev/null +++ b/backend/migrations/20260923214452-add-unique-requested-dataset-id.js @@ -0,0 +1,19 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + // At most one dataset may reserve a given preferred public id (first-come). + // Partial so the many NULL (no preference) rows don't collide. Backs up the + // controller's assertRequestedIdFree check against races. + async up(queryInterface) { + await queryInterface.sequelize.query( + "CREATE UNIQUE INDEX uq_requested_dataset_id ON dataset_registry (requested_dataset_id) WHERE requested_dataset_id IS NOT NULL;" + ); + }, + + async down(queryInterface) { + await queryInterface.sequelize.query( + "DROP INDEX IF EXISTS uq_requested_dataset_id;" + ); + } +}; diff --git a/backend/src/controllers/uploads.controller.js b/backend/src/controllers/uploads.controller.js index 304e2b5..eceb32f 100644 --- a/backend/src/controllers/uploads.controller.js +++ b/backend/src/controllers/uploads.controller.js @@ -54,6 +54,30 @@ const validateRequestedId = (id) => { } }; +// Preliminary availability: reject an id already used as a final public +// dataset_id OR another row's requested_dataset_id. `excludeInternalId` skips +// the caller's own row on update. (The authoritative check still runs at +// promotion; a DB partial-unique index also backs this up.) +const assertRequestedIdFree = async (requestedId, excludeInternalId, t) => { + const rows = await sequelize.query( + `SELECT 1 FROM dataset_registry + WHERE (dataset_id = :id OR requested_dataset_id = :id) + AND (:exclude::uuid IS NULL OR internal_id <> :exclude::uuid) + LIMIT 1`, + { + replacements: { id: requestedId, exclude: excludeInternalId || null }, + type: sequelize.QueryTypes.SELECT, + transaction: t, + } + ); + if (rows.length) { + throw new HttpError(409, { + code: "REQUESTED_ID_TAKEN", + error: `The preferred ID "${requestedId}" is already in use.`, + }); + } +}; + const createUpload = async (req, res) => { try { const body = req.body; @@ -103,22 +127,7 @@ const createUpload = async (req, res) => { if (!internalId) { // NEW logical dataset. - if (requestedId) { - const dup = await sequelize.query( - "SELECT 1 FROM dataset_registry WHERE dataset_id = :id LIMIT 1", - { - replacements: { id: requestedId }, - type: sequelize.QueryTypes.SELECT, - transaction: t, - } - ); - if (dup.length) { - throw new HttpError(409, { - code: "REQUESTED_ID_TAKEN", - error: `The preferred ID "${requestedId}" is already in use.`, - }); - } - } + if (requestedId) await assertRequestedIdFree(requestedId, null, t); registryId = crypto.randomUUID(); await sequelize.query( `INSERT INTO dataset_registry @@ -165,6 +174,9 @@ const createUpload = async (req, res) => { }); } + // If changing the preferred id, it must still be free (excluding self). + if (requestedId) await assertRequestedIdFree(requestedId, registryId, t); + // Registry holds the canonical name (+ optional preferred id). await sequelize.query( `UPDATE dataset_registry From f63bb56717103d867bf9d99d7d289de4cca2399e Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Fri, 25 Sep 2026 10:32:25 -0400 Subject: [PATCH 21/45] fix(uploads): show the real error for a taken preferred ID; refs #147 --- src/pages/UploadPage.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pages/UploadPage.tsx b/src/pages/UploadPage.tsx index c195170..1d03082 100644 --- a/src/pages/UploadPage.tsx +++ b/src/pages/UploadPage.tsx @@ -94,11 +94,11 @@ const UploadPage: React.FC = () => { }); } else if (status === 409 && data.requiresConfirmation) { setConfirmPrompt({ code: data.code, message: data.message }); - } else if (status === 409) { - setInfoMessage( - data.message || "This dataset cannot be updated right now." - ); + } else if (status === 409 && data.message) { + // Informational block (e.g. approved, awaiting promotion). + setInfoMessage(data.message); } else { + // Real errors, incl. REQUESTED_ID_TAKEN (which carries `error`). setSubmitError(data.error || data.message || `Upload failed (${status})`); } } catch (e: any) { From 051c09a41c44d17422747f2fb82afd3b9537a418 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Fri, 25 Sep 2026 12:29:38 -0400 Subject: [PATCH 22/45] feat(uploads): let users choose a target database (default public); refs #147 --- ...36-add-requested-db-to-dataset-registry.js | 19 +++++++ backend/src/controllers/uploads.controller.js | 31 +++++++++-- src/pages/UploadDetailPage.tsx | 3 ++ src/pages/UploadPage.tsx | 53 ++++++++++++++++++- src/services/upload.service.ts | 3 ++ 5 files changed, 104 insertions(+), 5 deletions(-) create mode 100644 backend/migrations/20260925162336-add-requested-db-to-dataset-registry.js diff --git a/backend/migrations/20260925162336-add-requested-db-to-dataset-registry.js b/backend/migrations/20260925162336-add-requested-db-to-dataset-registry.js new file mode 100644 index 0000000..c760d73 --- /dev/null +++ b/backend/migrations/20260925162336-add-requested-db-to-dataset-registry.js @@ -0,0 +1,19 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + // The public database a dataset should be published to at promotion. + // A preference/reservation; defaults to "public" when the user picks nothing. + // Actual db is resolved/created at promotion (a new name is review-gated). + async up(queryInterface, Sequelize) { + await queryInterface.addColumn("dataset_registry", "requested_db", { + type: Sequelize.STRING(63), + allowNull: false, + defaultValue: "public", + }); + }, + + async down(queryInterface) { + await queryInterface.removeColumn("dataset_registry", "requested_db"); + } +}; diff --git a/backend/src/controllers/uploads.controller.js b/backend/src/controllers/uploads.controller.js index eceb32f..d46b078 100644 --- a/backend/src/controllers/uploads.controller.js +++ b/backend/src/controllers/uploads.controller.js @@ -54,6 +54,17 @@ const validateRequestedId = (id) => { } }; +// Target public database name (same slug rule; no njds restriction — that is +// only for dataset ids). Blank defaults to "public" upstream. +const validateRequestedDb = (db) => { + if (!/^[a-z0-9][a-z0-9_-]{2,62}$/.test(db)) { + throw new HttpError(400, { + error: + "Database name must be 3–63 characters: lowercase letters, numbers, - or _.", + }); + } +}; + // Preliminary availability: reject an id already used as a final public // dataset_id OR another row's requested_dataset_id. `excludeInternalId` skips // the caller's own row on update. (The authoritative check still runs at @@ -114,10 +125,14 @@ const createUpload = async (req, res) => { const requestedId = req.query.requestedDatasetId ? String(req.query.requestedDatasetId).trim() : null; + const requestedDb = req.query.requestedDb + ? String(req.query.requestedDb).trim() + : null; const confirm = req.query.confirm === "true"; const userId = req.user.id; if (requestedId) validateRequestedId(requestedId); // throws 400 + if (requestedDb) validateRequestedDb(requestedDb); // throws 400 // Keep the PostgreSQL transaction open across the CouchDB write: commit on // success, roll back if the upload fails. Acceptable at low upload volume. @@ -131,12 +146,13 @@ const createUpload = async (req, res) => { registryId = crypto.randomUUID(); await sequelize.query( `INSERT INTO dataset_registry - (internal_id, dataset_id, requested_dataset_id, dataset_name, owner_user_id, created_at, updated_at) - VALUES (:iid, NULL, :req, :name, :uid, NOW(), NOW())`, + (internal_id, dataset_id, requested_dataset_id, requested_db, dataset_name, owner_user_id, created_at, updated_at) + VALUES (:iid, NULL, :req, :db, :name, :uid, NOW(), NOW())`, { replacements: { iid: registryId, req: requestedId, + db: requestedDb || "public", name: datasetName, uid: userId, }, @@ -182,10 +198,16 @@ const createUpload = async (req, res) => { `UPDATE dataset_registry SET dataset_name = :name, requested_dataset_id = COALESCE(:req, requested_dataset_id), + requested_db = COALESCE(:db, requested_db), updated_at = NOW() WHERE internal_id = :iid`, { - replacements: { name: datasetName, req: requestedId, iid: registryId }, + replacements: { + name: datasetName, + req: requestedId, + db: requestedDb, + iid: registryId, + }, transaction: t, } ); @@ -323,7 +345,7 @@ const loadOwnedRegistry = async (internalId, userId) => { throw new HttpError(404, { error: "Dataset not found" }); } const rows = await sequelize.query( - `SELECT internal_id, dataset_id, requested_dataset_id, dataset_name, owner_user_id + `SELECT internal_id, dataset_id, requested_dataset_id, requested_db, dataset_name, owner_user_id FROM dataset_registry WHERE internal_id = :iid`, { replacements: { iid: internalId }, type: sequelize.QueryTypes.SELECT } ); @@ -358,6 +380,7 @@ const getUpload = async (req, res) => { internal_id: reg.internal_id, dataset_id: reg.dataset_id, requested_dataset_id: reg.requested_dataset_id, + requested_db: reg.requested_db, dataset_name: reg.dataset_name, submission_id: sub.submission_id || null, status: sub.status || null, diff --git a/src/pages/UploadDetailPage.tsx b/src/pages/UploadDetailPage.tsx index 8b26973..6db54d7 100644 --- a/src/pages/UploadDetailPage.tsx +++ b/src/pages/UploadDetailPage.tsx @@ -199,6 +199,9 @@ const UploadDetailPage: React.FC = () => { Preferred ID: {detail.requested_dataset_id} )} + + Target database: {detail.requested_db || "public"} + Internal reference: {detail.internal_id} diff --git a/src/pages/UploadPage.tsx b/src/pages/UploadPage.tsx index 1d03082..8217d8e 100644 --- a/src/pages/UploadPage.tsx +++ b/src/pages/UploadPage.tsx @@ -36,6 +36,7 @@ const UploadPage: React.FC = () => { const [internalId, setInternalId] = useState(initialInternalId); const [datasetName, setDatasetName] = useState(""); const [requestedDatasetId, setRequestedDatasetId] = useState(""); + const [requestedDb, setRequestedDb] = useState(""); const [submitting, setSubmitting] = useState(false); const [result, setResult] = useState<{ internalId: string; @@ -84,6 +85,7 @@ const UploadPage: React.FC = () => { internalId: internalId.trim() || undefined, datasetName: datasetName.trim() || undefined, requestedDatasetId: requestedDatasetId.trim() || undefined, + requestedDb: requestedDb.trim() || undefined, confirm, }); if (status === 201) { @@ -158,6 +160,19 @@ const UploadPage: React.FC = () => { ? "Too long — use 63 characters or fewer." : ""; + // Same slug rule for the target database name (no njds restriction). + const dbTrimmed = requestedDb.trim(); + const requestedDbError = + dbTrimmed === "" + ? "" + : !CHAR_RE.test(dbTrimmed) + ? "Use only lowercase letters and numbers; join words with - or _ (e.g. my-lab)." + : dbTrimmed.length < 3 + ? "Too short — use at least 3 characters." + : dbTrimmed.length > 63 + ? "Too long — use 63 characters or fewer." + : ""; + return ( { + {/* Target database — where the dataset is published after review */} + setRequestedDb(e.target.value)} + size="small" + fullWidth + error={!!requestedDbError} + helperText={requestedDbError || undefined} + sx={{ mt: 2.5 }} + /> + +
  • + Where your dataset is published after review (part of the URL, e.g.{" "} + /db/my-lab/…). +
    • +
  • + Leave blank for the default public database. +
    • +
  • A new database name is created only after review.
    • +     + {/* Existing-dataset internal ID — normally auto-filled from the dashboard */} { + {/* Opens the stored doc via our owner-checked backend proxy (the + sandbox CouchDB URL is never exposed to the browser). */} + {detail.status === "promoted" && detail.promoted_db && detail.dataset_id && ( From 5fcd166624ddeabca68846595f3a278a27301c18 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Mon, 28 Sep 2026 17:07:21 -0400 Subject: [PATCH 26/45] feat(uploads): add draft workflow columns to submissions; refs #147 --- ...10352-add-draft-workflow-to-submissions.js | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 backend/migrations/20260928210352-add-draft-workflow-to-submissions.js diff --git a/backend/migrations/20260928210352-add-draft-workflow-to-submissions.js b/backend/migrations/20260928210352-add-draft-workflow-to-submissions.js new file mode 100644 index 0000000..70f1725 --- /dev/null +++ b/backend/migrations/20260928210352-add-draft-workflow-to-submissions.js @@ -0,0 +1,75 @@ +"use strict"; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + // Draft workflow: review status starts at 'draft'; JSON upload state is + // tracked separately from review state; raw ZIP is optional per submission. + async up(queryInterface, Sequelize) { + // New cycles start as draft (not pending). + await queryInterface.changeColumn("submissions", "status", { + type: Sequelize.STRING(50), + allowNull: false, + defaultValue: "draft", // draft|pending|changes_requested|approved|rejected|promoted + }); + + // JSON component state (separate from review status). + await queryInterface.addColumn("submissions", "json_status", { + type: Sequelize.STRING(20), + allowNull: false, + defaultValue: "uploaded", // uploaded | failed + }); + await queryInterface.addColumn("submissions", "json_uploaded_at", { + type: Sequelize.DATE, + allowNull: true, + }); + await queryInterface.addColumn("submissions", "json_error", { + type: Sequelize.TEXT, + allowNull: true, + }); + + // Last time the user sent it to review (draft/changes_requested → pending). + await queryInterface.addColumn("submissions", "submitted_at", { + type: Sequelize.DATE, + allowNull: true, + }); + + // Raw ZIP is optional; required for submit only when this is true. + await queryInterface.addColumn("submissions", "raw_zip_expected", { + type: Sequelize.BOOLEAN, + allowNull: false, + defaultValue: false, + }); + + // Existing rows already had their JSON uploaded. + await queryInterface.sequelize.query( + "UPDATE submissions SET json_uploaded_at = updated_at WHERE json_uploaded_at IS NULL;" + ); + + // One active cycle per dataset — now including draft. + await queryInterface.sequelize.query( + "DROP INDEX IF EXISTS uq_one_open_per_dataset;" + ); + await queryInterface.sequelize.query( + "CREATE UNIQUE INDEX uq_one_open_per_dataset ON submissions (dataset_registry_id) WHERE status IN ('draft','pending','changes_requested','approved');" + ); + }, + + async down(queryInterface, Sequelize) { + await queryInterface.sequelize.query( + "DROP INDEX IF EXISTS uq_one_open_per_dataset;" + ); + await queryInterface.sequelize.query( + "CREATE UNIQUE INDEX uq_one_open_per_dataset ON submissions (dataset_registry_id) WHERE status IN ('pending','changes_requested','approved');" + ); + await queryInterface.removeColumn("submissions", "raw_zip_expected"); + await queryInterface.removeColumn("submissions", "submitted_at"); + await queryInterface.removeColumn("submissions", "json_error"); + await queryInterface.removeColumn("submissions", "json_uploaded_at"); + await queryInterface.removeColumn("submissions", "json_status"); + await queryInterface.changeColumn("submissions", "status", { + type: Sequelize.STRING(50), + allowNull: false, + defaultValue: "pending", + }); + }, +}; From 9c9ffbf09e53dfca81d016555f3a23f6d209c0a7 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Mon, 28 Sep 2026 17:39:57 -0400 Subject: [PATCH 27/45] feat(uploads): add draft workflow with submit and withdraw; refs #147 --- backend/src/controllers/uploads.controller.js | 185 ++++++++++++++++-- backend/src/routes/uploads.routes.js | 6 + src/components/User/Dashboard/UploadsTab.tsx | 42 ++-- src/pages/UploadDetailPage.tsx | 137 ++++++++++++- src/pages/UploadPage.tsx | 24 ++- src/services/upload.service.ts | 33 ++++ 6 files changed, 373 insertions(+), 54 deletions(-) diff --git a/backend/src/controllers/uploads.controller.js b/backend/src/controllers/uploads.controller.js index d8da617..6eb9f5d 100644 --- a/backend/src/controllers/uploads.controller.js +++ b/backend/src/controllers/uploads.controller.js @@ -90,6 +90,9 @@ const assertRequestedIdFree = async (requestedId, excludeInternalId, t) => { }; const createUpload = async (req, res) => { + // Existing editable cycle whose JSON update is being attempted; used to + // record a failure after the transaction rolls back. + let jsonTargetId = null; try { const body = req.body; if (!body || typeof body !== "object" || Array.isArray(body)) { @@ -134,6 +137,9 @@ const createUpload = async (req, res) => { if (requestedId) validateRequestedId(requestedId); // throws 400 if (requestedDb) validateRequestedDb(requestedDb); // throws 400 + // Review status of the cycle this upload lands in (returned to the client). + let reviewStatus = "draft"; + // Keep the PostgreSQL transaction open across the CouchDB write: commit on // success, roll back if the upload fails. Acceptable at low upload volume. const result = await sequelize.transaction(async (t) => { @@ -162,8 +168,8 @@ const createUpload = async (req, res) => { submissionId = crypto.randomUUID(); await sequelize.query( `INSERT INTO submissions - (submission_id, dataset_registry_id, status, created_at, updated_at) - VALUES (:sid, :rid, 'pending', NOW(), NOW())`, + (submission_id, dataset_registry_id, status, json_status, json_uploaded_at, created_at, updated_at) + VALUES (:sid, :rid, 'draft', 'uploaded', NOW(), NOW(), NOW())`, { replacements: { sid: submissionId, rid: registryId }, transaction: t, @@ -223,22 +229,29 @@ const createUpload = async (req, res) => { } ); - // Open = pending / changes_requested → reuse, reset to pending. - const open = subs.find( - (s) => s.status === "pending" || s.status === "changes_requested" + // Editable = draft / changes_requested → reuse this cycle. The review + // status is NOT changed; the user must click Submit/Resubmit to send it + // to review. + const editable = subs.find( + (s) => s.status === "draft" || s.status === "changes_requested" ); - if (open) { - submissionId = open.submission_id; + if (editable) { + submissionId = editable.submission_id; + reviewStatus = editable.status; + jsonTargetId = editable.id; await sequelize.query( - `UPDATE submissions SET status='pending', updated_at=NOW() WHERE id=:id`, - { replacements: { id: open.id }, transaction: t } + `UPDATE submissions + SET json_status='uploaded', json_uploaded_at=NOW(), json_error=NULL, updated_at=NOW() + WHERE id=:id`, + { replacements: { id: editable.id }, transaction: t } ); } else if (subs.length === 0) { // Defensive: registry with no cycle → start one. submissionId = crypto.randomUUID(); await sequelize.query( - `INSERT INTO submissions (submission_id, dataset_registry_id, status, created_at, updated_at) - VALUES (:sid, :rid, 'pending', NOW(), NOW())`, + `INSERT INTO submissions + (submission_id, dataset_registry_id, status, json_status, json_uploaded_at, created_at, updated_at) + VALUES (:sid, :rid, 'draft', 'uploaded', NOW(), NOW(), NOW())`, { replacements: { sid: submissionId, rid: registryId }, transaction: t, @@ -246,6 +259,17 @@ const createUpload = async (req, res) => { ); } else { const latest = subs[0]; + if (latest.status === "pending") { + // Locked while waiting for review; the user can withdraw it to + // draft first. + throw new HttpError(409, { + code: "SUBMISSION_UNDER_REVIEW", + status: "pending", + requiresConfirmation: false, + message: + "This submission is waiting for review. Withdraw it back to draft to make changes.", + }); + } if (latest.status === "approved") { // Still an ACTIVE workflow (awaiting promotion) — block. throw new HttpError(409, { @@ -266,14 +290,15 @@ const createUpload = async (req, res) => { status: latest.status, requiresConfirmation: true, message: promoted - ? "This dataset is already published. Confirm to submit an updated version for a new review." - : "The previous submission was rejected. Confirm to resubmit for review.", + ? "This dataset is already published. Confirm to start a new version (saved as a draft)." + : "The previous submission was rejected. Confirm to start a new version (saved as a draft).", }); } submissionId = crypto.randomUUID(); await sequelize.query( - `INSERT INTO submissions (submission_id, dataset_registry_id, status, created_at, updated_at) - VALUES (:sid, :rid, 'pending', NOW(), NOW())`, + `INSERT INTO submissions + (submission_id, dataset_registry_id, status, json_status, json_uploaded_at, created_at, updated_at) + VALUES (:sid, :rid, 'draft', 'uploaded', NOW(), NOW(), NOW())`, { replacements: { sid: submissionId, rid: registryId }, transaction: t, @@ -292,11 +317,32 @@ const createUpload = async (req, res) => { return { internal_id: registryId, submission_id: submissionId }; }); - res.status(201).json({ ok: true, status: "pending", ...result }); + res.status(201).json({ ok: true, status: reviewStatus, ...result }); } catch (err) { if (err instanceof HttpError) { return res.status(err.status).json(err.body); } + // The transaction rolled back, so the sandbox still has the previous good + // copy. Mark the last JSON update as failed — only for an existing + // draft/changes_requested cycle (a brand-new dataset has no row to mark). + // Stored message is a fixed, safe sentence (never the raw error). + if (jsonTargetId) { + await sequelize + .query( + `UPDATE submissions + SET json_status='failed', json_error=:msg, updated_at=NOW() + WHERE id=:id`, + { + replacements: { + id: jsonTargetId, + msg: "The last JSON update could not be saved.", + }, + } + ) + .catch((e) => + console.error("Could not record JSON failure:", e.message) + ); + } // Full details stay in the server log only. Never send err.message or the // CouchDB response to the browser — a network error message contains the // sandbox host/port (e.g. "connect ECONNREFUSED :7777"). @@ -394,12 +440,107 @@ const latestSubmissionId = async (internalId) => { return s[0] ? s[0].submission_id : null; }; +// What must be true before draft/changes_requested → pending. +// (Raw ZIP checks are added in step 2, once the raw tables exist; until then +// raw_zip_expected is always false.) +const getReadiness = (sub) => { + const problems = []; + if (sub.json_status !== "uploaded") { + problems.push("The last JSON update failed. Upload the JSON again."); + } + return { canSubmit: problems.length === 0, problems }; +}; + +// POST /api/v1/uploads/:internalId/submit — send (or resend) to review. +const submitForReview = async (req, res) => { + try { + const reg = await loadOwnedRegistry( + String(req.params.internalId), + req.user.id + ); + const out = await sequelize.transaction(async (t) => { + const rows = await sequelize.query( + `SELECT id, status, json_status, raw_zip_expected FROM submissions + WHERE dataset_registry_id = :iid + ORDER BY created_at DESC LIMIT 1 FOR UPDATE`, + { + replacements: { iid: reg.internal_id }, + type: sequelize.QueryTypes.SELECT, + transaction: t, + } + ); + const sub = rows[0]; + if (!sub || !["draft", "changes_requested"].includes(sub.status)) { + throw new HttpError(409, { + code: "NOT_SUBMITTABLE", + error: + "Only a draft or a submission with requested changes can be sent for review.", + }); + } + const { canSubmit, problems } = getReadiness(sub); + if (!canSubmit) { + throw new HttpError(409, { + code: "NOT_READY", + error: problems[0], + problems, + }); + } + await sequelize.query( + `UPDATE submissions + SET status='pending', submitted_at=NOW(), updated_at=NOW() + WHERE id=:id`, + { replacements: { id: sub.id }, transaction: t } + ); + return { status: "pending" }; + }); + res.json(out); + } catch (err) { + if (err instanceof HttpError) return res.status(err.status).json(err.body); + console.error("Submit failed:", err.message); + res.status(500).json({ error: "Failed to submit for review." }); + } +}; + +// POST /api/v1/uploads/:internalId/withdraw — pending → draft, only until the +// reviewer changes the status. A single conditional UPDATE, so it can't race a +// reviewer action: whichever lands first wins. +const withdrawSubmission = async (req, res) => { + try { + const reg = await loadOwnedRegistry( + String(req.params.internalId), + req.user.id + ); + const sid = await latestSubmissionId(reg.internal_id); + const rows = sid + ? await sequelize.query( + `UPDATE submissions SET status='draft', updated_at=NOW() + WHERE submission_id = :sid AND status = 'pending' + RETURNING id`, + { replacements: { sid }, type: sequelize.QueryTypes.SELECT } + ) + : []; + if (rows.length === 0) { + return res.status(409).json({ + code: "CANNOT_WITHDRAW", + error: + "This submission can no longer be withdrawn — the reviewer has already acted on it.", + }); + } + res.json({ status: "draft" }); + } catch (err) { + if (err instanceof HttpError) return res.status(err.status).json(err.body); + console.error("Withdraw failed:", err.message); + res.status(500).json({ error: "Failed to withdraw." }); + } +}; + // GET /api/v1/uploads/:internalId — one dataset's detail (owner only). const getUpload = async (req, res) => { try { const reg = await loadOwnedRegistry(String(req.params.internalId), req.user.id); const s = await sequelize.query( - `SELECT submission_id, status, created_at, updated_at, promoted_db, promoted_at + `SELECT submission_id, status, created_at, updated_at, promoted_db, promoted_at, + json_status, json_uploaded_at, json_error, submitted_at, raw_zip_expected FROM submissions WHERE dataset_registry_id = :iid ORDER BY created_at DESC LIMIT 1`, { replacements: { iid: reg.internal_id }, type: sequelize.QueryTypes.SELECT } @@ -417,6 +558,14 @@ const getUpload = async (req, res) => { updated_at: sub.updated_at || null, promoted_db: sub.promoted_db || null, promoted_at: sub.promoted_at || null, + json_status: sub.json_status || null, + json_uploaded_at: sub.json_uploaded_at || null, + json_error: sub.json_error || null, + submitted_at: sub.submitted_at || null, + raw_zip_expected: !!sub.raw_zip_expected, + readiness: sub.submission_id + ? getReadiness(sub) + : { canSubmit: false, problems: ["No submission found."] }, }); } catch (err) { if (err instanceof HttpError) return res.status(err.status).json(err.body); @@ -496,6 +645,8 @@ module.exports = { listMyUploads, getUpload, getUploadDocument, + submitForReview, + withdrawSubmission, listComments, postComment, }; diff --git a/backend/src/routes/uploads.routes.js b/backend/src/routes/uploads.routes.js index cbfaaf1..aa1dc5e 100644 --- a/backend/src/routes/uploads.routes.js +++ b/backend/src/routes/uploads.routes.js @@ -6,6 +6,8 @@ const { listMyUploads, getUpload, getUploadDocument, + submitForReview, + withdrawSubmission, listComments, postComment, } = require("../controllers/uploads.controller"); @@ -38,6 +40,10 @@ router.get("/", requireAuth, listMyUploads); // One dataset's detail + its review conversation (owner only). router.get("/:internalId", requireAuth, getUpload); router.get("/:internalId/document", requireAuth, getUploadDocument); + +// Review workflow actions (owner only). +router.post("/:internalId/submit", requireAuth, submitForReview); +router.post("/:internalId/withdraw", requireAuth, withdrawSubmission); router.get("/:internalId/comments", requireAuth, listComments); router.post("/:internalId/comments", requireAuth, postComment); diff --git a/src/components/User/Dashboard/UploadsTab.tsx b/src/components/User/Dashboard/UploadsTab.tsx index dffc6ea..56a2fce 100644 --- a/src/components/User/Dashboard/UploadsTab.tsx +++ b/src/components/User/Dashboard/UploadsTab.tsx @@ -27,6 +27,7 @@ const STATUS_CHIP: Record< sx?: object; } > = { + draft: { label: "Draft", color: "default" }, // Pending uses the theme purple (not MUI's default blue). pending: { label: "Pending review", @@ -189,24 +190,29 @@ const UploadsTab: React.FC = () => { View )} - + {/* Same rule as the detail page: locked while + pending/approved (withdraw on the detail page to + edit a pending one). */} + {r.status !== "pending" && r.status !== "approved" && ( + + )} diff --git a/src/pages/UploadDetailPage.tsx b/src/pages/UploadDetailPage.tsx index 98fd135..13d474a 100644 --- a/src/pages/UploadDetailPage.tsx +++ b/src/pages/UploadDetailPage.tsx @@ -40,6 +40,7 @@ const STATUS_CHIP: Record< sx?: object; } > = { + draft: { label: "Draft", color: "default" }, pending: { label: "Pending review", sx: { backgroundColor: Colors.purple, color: Colors.white }, @@ -73,6 +74,24 @@ const UploadDetailPage: React.FC = () => { const [posting, setPosting] = useState(false); const [postError, setPostError] = useState(null); + // Review actions (submit / resubmit / withdraw). + const [acting, setActing] = useState(false); + const [actionError, setActionError] = useState(null); + + const runAction = async (fn: (id: string) => Promise) => { + if (!internalId) return; + setActing(true); + setActionError(null); + try { + await fn(internalId); + setDetail(await UploadService.getUpload(internalId)); // refresh status + } catch (e: any) { + setActionError(e.message || "Action failed."); + } finally { + setActing(false); + } + }; + useEffect(() => { if (!internalId || !isLoggedIn) return; setLoading(true); @@ -217,16 +236,22 @@ const UploadDetailPage: React.FC = () => {     - - + + {/* Locked while pending/approved; rejected/promoted use Update to + start a new version. */} + {detail.status !== "pending" && detail.status !== "approved" && ( + + )} {/* Opens the stored doc via our owner-checked backend proxy (the sandbox CouchDB URL is never exposed to the browser). */} + + )} + + {detail.status === "pending" && ( + + + Waiting for review + {detail.submitted_at + ? ` · submitted ${new Date( + detail.submitted_at + ).toLocaleString()}` + : ""} + . Withdraw it to make changes. + + + + )} + + {detail.status === "approved" && ( + + Approved and waiting to be published. It can't be changed now. + + )} + + {actionError && ( + + {actionError} + + )} + + {/* Conversation */} diff --git a/src/pages/UploadPage.tsx b/src/pages/UploadPage.tsx index 3387f7a..e5d24ac 100644 --- a/src/pages/UploadPage.tsx +++ b/src/pages/UploadPage.tsx @@ -209,14 +209,18 @@ const UploadPage: React.FC = () => { lineHeight: 1.6, }} > - Select a JSON file. It will be submitted to our sandbox for the - NeuroJSON team to review before it is added to a database. + Select a JSON file. It is saved as a draft in our sandbox. When + everything is ready, submit it for review from its upload detail page + (Dashboard → Uploads). - Your file is reviewed by the NeuroJSON team before it is added to a - database. A new reference id is assigned on submission — any existing{" "} + Uploading does not send your dataset for review. Review starts only + after you click Submit for review on the upload + detail page; the NeuroJSON team reviews it before it is added to a + database. + A new reference id is assigned on upload — any existing{" "} _id/_rev in your file is ignored. @@ -393,14 +397,16 @@ const UploadPage: React.FC = () => { {submitting ? ( ) : ( - "Submit for review" + "Upload JSON" )} {result && ( - Submitted for review. + {result.status === "changes_requested" + ? "Saved. Resubmit it for review when it's ready." + : "Saved as draft. Submit it for review when it's ready."}
    Internal reference: {result.internalId} @@ -412,7 +418,7 @@ const UploadPage: React.FC = () => { to={`/uploads/${result.internalId}`} sx={{ color: Colors.purple, fontWeight: 600 }} > - View status → + Open upload details →     )} @@ -455,8 +461,8 @@ const UploadPage: React.FC = () => { {confirmPrompt && ( {confirmPrompt.code === "DATASET_ALREADY_PROMOTED" - ? "This replaces your working copy in the sandbox for review. The version already published to the public database stays unchanged — it is only replaced if this new version passes review." - : "This replaces your working copy in the sandbox and starts a new review."} + ? "This starts a new draft version and replaces your working copy in the sandbox. The version already published to the public database stays unchanged — it is only replaced if the new version passes review." + : "This starts a new draft version and replaces your working copy in the sandbox. Submit it for review when it's ready."} )} diff --git a/src/services/upload.service.ts b/src/services/upload.service.ts index 0dc04fe..af5ba06 100644 --- a/src/services/upload.service.ts +++ b/src/services/upload.service.ts @@ -12,6 +12,7 @@ export interface UploadRecord { dataset_name: string | null; submission_id: string | null; status: + | "draft" | "pending" | "changes_requested" | "approved" @@ -36,6 +37,12 @@ export interface UploadDetail { updated_at: string | null; promoted_db: string | null; promoted_at: string | null; + json_status: "uploaded" | "failed" | null; + json_uploaded_at: string | null; + json_error: string | null; + submitted_at: string | null; + raw_zip_expected: boolean; + readiness: { canSubmit: boolean; problems: string[] }; } export interface Comment { @@ -118,4 +125,30 @@ export const UploadService = { } return res.json(); }, + + // draft / changes_requested → pending + submitForReview: async (internalId: string): Promise<{ status: string }> => { + const res = await fetch(`${baseURL}/uploads/${internalId}/submit`, { + method: "POST", + credentials: "include", + }); + if (!res.ok) { + const body = await res.json().catch(() => ({})); + throw new Error(body.error || `Failed to submit (${res.status})`); + } + return res.json(); + }, + + // pending → draft (only before the reviewer acts) + withdraw: async (internalId: string): Promise<{ status: string }> => { + const res = await fetch(`${baseURL}/uploads/${internalId}/withdraw`, { + method: "POST", + credentials: "include", + }); + if (!res.ok) { + const body = await res.json().catch(() => ({})); + throw new Error(body.error || `Failed to withdraw (${res.status})`); + } + return res.json(); + }, }; From 02ea42d13bf92bf5f1a82b7566737228afc2609d Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Mon, 28 Sep 2026 17:59:42 -0400 Subject: [PATCH 28/45] feat(uploads): edit publishing settings on the upload detail page; refs #147 --- backend/src/controllers/uploads.controller.js | 115 +++++++-- backend/src/routes/uploads.routes.js | 4 + .../Upload/PublishSettingsDialog.tsx | 188 +++++++++++++++ src/pages/UploadDetailPage.tsx | 73 +++++- src/pages/UploadPage.tsx | 228 +++++++++--------- src/services/upload.service.ts | 19 ++ src/utils/uploadValidation.ts | 67 +++++ 7 files changed, 554 insertions(+), 140 deletions(-) create mode 100644 src/components/Upload/PublishSettingsDialog.tsx create mode 100644 src/utils/uploadValidation.ts diff --git a/backend/src/controllers/uploads.controller.js b/backend/src/controllers/uploads.controller.js index 6eb9f5d..078689d 100644 --- a/backend/src/controllers/uploads.controller.js +++ b/backend/src/controllers/uploads.controller.js @@ -196,24 +196,15 @@ const createUpload = async (req, res) => { }); } - // If changing the preferred id, it must still be free (excluding self). - if (requestedId) await assertRequestedIdFree(requestedId, registryId, t); - - // Registry holds the canonical name (+ optional preferred id). + // Registry holds the canonical name. Publishing settings (preferred id, + // target db) are NOT changed by an upload — they are edited only via + // PATCH /:internalId/settings, so they're ignored here on update. await sequelize.query( `UPDATE dataset_registry - SET dataset_name = :name, - requested_dataset_id = COALESCE(:req, requested_dataset_id), - requested_db = COALESCE(:db, requested_db), - updated_at = NOW() + SET dataset_name = :name, updated_at = NOW() WHERE internal_id = :iid`, { - replacements: { - name: datasetName, - req: requestedId, - db: requestedDb, - iid: registryId, - }, + replacements: { name: datasetName, iid: registryId }, transaction: t, } ); @@ -534,6 +525,101 @@ const withdrawSubmission = async (req, res) => { } }; +// PATCH /api/v1/uploads/:internalId/settings — change publishing settings +// without re-uploading. A field left out = unchanged; null or "" = reset +// (requestedDatasetId → let NeuroJSON assign; requestedDb → "public"). +const updateSettings = async (req, res) => { + try { + const body = req.body || {}; + const has = (k) => Object.prototype.hasOwnProperty.call(body, k); + const clean = (v) => + v === null || v === undefined ? null : String(v).trim() || null; + + const changeId = has("requestedDatasetId"); + const changeDb = has("requestedDb"); + if (!changeId && !changeDb) { + return res.status(400).json({ error: "Nothing to update." }); + } + const newId = changeId ? clean(body.requestedDatasetId) : null; + const newDb = changeDb ? clean(body.requestedDb) || "public" : null; + if (newId) validateRequestedId(newId); // 400 on bad format / njds + if (changeDb) validateRequestedDb(newDb); // "public" passes + + const reg = await loadOwnedRegistry( + String(req.params.internalId), + req.user.id + ); + + // Once a dataset has a public ID, its URL (/db//) is fixed so + // shared/cited links keep working; later versions publish to the same place. + if (reg.dataset_id) { + throw new HttpError(409, { + code: "PUBLIC_ID_ASSIGNED", + error: + "This dataset is already published, so its database and ID can't be changed.", + }); + } + + const out = await sequelize.transaction(async (t) => { + // Same rule as uploads: only while the submission is editable. + const rows = await sequelize.query( + `SELECT status FROM submissions WHERE dataset_registry_id = :iid + ORDER BY created_at DESC LIMIT 1 FOR UPDATE`, + { + replacements: { iid: reg.internal_id }, + type: sequelize.QueryTypes.SELECT, + transaction: t, + } + ); + if ( + !rows[0] || + !["draft", "changes_requested"].includes(rows[0].status) + ) { + throw new HttpError(409, { + code: "NOT_EDITABLE", + error: + "Settings can only be changed while the submission is a draft or has requested changes.", + }); + } + if (newId) await assertRequestedIdFree(newId, reg.internal_id, t); + + const sets = []; + const repl = { iid: reg.internal_id }; + if (changeId) { + sets.push("requested_dataset_id = :rid"); + repl.rid = newId; + } + if (changeDb) { + sets.push("requested_db = :rdb"); + repl.rdb = newDb; + } + const updated = await sequelize.query( + `UPDATE dataset_registry SET ${sets.join(", ")}, updated_at = NOW() + WHERE internal_id = :iid + RETURNING requested_dataset_id, requested_db`, + { + replacements: repl, + type: sequelize.QueryTypes.SELECT, + transaction: t, + } + ); + return updated[0]; + }); + res.json(out); + } catch (err) { + if (err instanceof HttpError) return res.status(err.status).json(err.body); + // Unique index backstop (two users grabbing the same ID at once). + if (err.original?.code === "23505") { + return res.status(409).json({ + code: "REQUESTED_ID_TAKEN", + error: "That preferred ID is already in use.", + }); + } + console.error("Update settings failed:", err.message); + res.status(500).json({ error: "Failed to update settings." }); + } +}; + // GET /api/v1/uploads/:internalId — one dataset's detail (owner only). const getUpload = async (req, res) => { try { @@ -647,6 +733,7 @@ module.exports = { getUploadDocument, submitForReview, withdrawSubmission, + updateSettings, listComments, postComment, }; diff --git a/backend/src/routes/uploads.routes.js b/backend/src/routes/uploads.routes.js index aa1dc5e..f15f77f 100644 --- a/backend/src/routes/uploads.routes.js +++ b/backend/src/routes/uploads.routes.js @@ -8,6 +8,7 @@ const { getUploadDocument, submitForReview, withdrawSubmission, + updateSettings, listComments, postComment, } = require("../controllers/uploads.controller"); @@ -44,6 +45,9 @@ router.get("/:internalId/document", requireAuth, getUploadDocument); // Review workflow actions (owner only). router.post("/:internalId/submit", requireAuth, submitForReview); router.post("/:internalId/withdraw", requireAuth, withdrawSubmission); + +// Publishing settings (target db + preferred id), editable without re-uploading. +router.patch("/:internalId/settings", requireAuth, updateSettings); router.get("/:internalId/comments", requireAuth, listComments); router.post("/:internalId/comments", requireAuth, postComment); diff --git a/src/components/Upload/PublishSettingsDialog.tsx b/src/components/Upload/PublishSettingsDialog.tsx new file mode 100644 index 0000000..feeb02f --- /dev/null +++ b/src/components/Upload/PublishSettingsDialog.tsx @@ -0,0 +1,188 @@ +import { + Alert, + Box, + Button, + CircularProgress, + Dialog, + DialogActions, + DialogContent, + DialogTitle, + FormControlLabel, + Radio, + RadioGroup, + TextField, + Typography, +} from "@mui/material"; +import { Colors } from "design/theme"; +import React, { useEffect, useState } from "react"; +import { UploadService } from "services/upload.service"; +import { + getRequestedDbError, + getRequestedIdError, + useExistingDbNames, +} from "utils/uploadValidation"; + +interface PublishSettingsDialogProps { + open: boolean; + internalId: string; + currentDb: string | null; // "public" when unset + currentId: string | null; // null = NeuroJSON assigns + onClose: () => void; + onSaved: () => void; +} + +const radioSx = { "&.Mui-checked": { color: Colors.purple } }; + +// Edit the target database + preferred dataset id without re-uploading. +const PublishSettingsDialog: React.FC = ({ + open, + internalId, + currentDb, + currentId, + onClose, + onSaved, +}) => { + const existingDbs = useExistingDbNames(); + const [db, setDb] = useState(""); + const [mode, setMode] = useState<"auto" | "custom">("auto"); + const [customId, setCustomId] = useState(""); + const [saving, setSaving] = useState(false); + const [error, setError] = useState(null); + + // Reset the form to the saved values each time the dialog opens. + useEffect(() => { + if (!open) return; + setDb(currentDb && currentDb !== "public" ? currentDb : ""); + setMode(currentId ? "custom" : "auto"); + setCustomId(currentId || ""); + setError(null); + }, [open, currentDb, currentId]); + + const dbError = getRequestedDbError(db, existingDbs); + const idError = + mode === "custom" + ? customId.trim() + ? getRequestedIdError(customId) + : "Enter a preferred ID, or choose “Let NeuroJSON assign one”." + : ""; + + // Warn that the old preferred id becomes available to others. + const releasesId = + !!currentId && (mode === "auto" || customId.trim() !== currentId); + + const save = async () => { + setSaving(true); + setError(null); + try { + await UploadService.updateSettings(internalId, { + requestedDb: db.trim() || null, // null → "public" + requestedDatasetId: mode === "auto" ? null : customId.trim(), + }); + onSaved(); + } catch (e: any) { + setError(e.message || "Failed to save settings."); + } finally { + setSaving(false); + } + }; + + return ( + + Publishing settings + + setDb(e.target.value)} + size="small" + fullWidth + error={!!dbError} + helperText={ + dbError || + "Becomes part of the public URL after review (e.g. /db/smith-lab/…). Leave blank for public." + } + sx={{ mt: 1 }} + /> + + + Dataset ID + + setMode(e.target.value as "auto" | "custom")} + > + } + label={ + <> + Let NeuroJSON assign one (njds######) + + } + /> + } + label="Use my preferred ID" + /> + + {mode === "custom" && ( + setCustomId(e.target.value)} + size="small" + fullWidth + error={!!idError} + helperText={ + idError || + "Use only lowercase letters and numbers; join words with - or _." + } + sx={{ mt: 0.5, ml: 4, width: "calc(100% - 32px)" }} + /> + )} + + {releasesId && ( + + Your preferred ID {currentId} will be released, and + other users can then use it. + + )} + {error && ( + + {error} + + )} + + + + + + + ); +}; + +export default PublishSettingsDialog; diff --git a/src/pages/UploadDetailPage.tsx b/src/pages/UploadDetailPage.tsx index 13d474a..b591269 100644 --- a/src/pages/UploadDetailPage.tsx +++ b/src/pages/UploadDetailPage.tsx @@ -16,6 +16,7 @@ import { Typography, } from "@mui/material"; import CopyButton from "components/CopyButton"; +import PublishSettingsDialog from "components/Upload/PublishSettingsDialog"; import { Colors } from "design/theme"; import { useAppSelector } from "hooks/useAppSelector"; import React, { useEffect, useState } from "react"; @@ -74,6 +75,8 @@ const UploadDetailPage: React.FC = () => { const [posting, setPosting] = useState(false); const [postError, setPostError] = useState(null); + const [settingsOpen, setSettingsOpen] = useState(false); + // Review actions (submit / resubmit / withdraw). const [acting, setActing] = useState(false); const [actionError, setActionError] = useState(null); @@ -172,6 +175,9 @@ const UploadDetailPage: React.FC = () => { } const chip = detail.status ? STATUS_CHIP[detail.status] : null; + const canEditSettings = + (detail.status === "draft" || detail.status === "changes_requested") && + !detail.dataset_id; return ( @@ -216,14 +222,6 @@ const UploadDetailPage: React.FC = () => { Not yet assigned )} - {detail.requested_dataset_id && ( - - Preferred ID: {detail.requested_dataset_id} - - )} - - Target database: {detail.requested_db || "public"} - Internal reference: {detail.internal_id} @@ -283,6 +281,65 @@ const UploadDetailPage: React.FC = () => {     + {/* Publishing settings: editable while draft/changes_requested and not + yet published (the public URL is fixed once a public ID exists). */} + + + Publishing settings + {canEditSettings && ( + + )} + + + + Database: {detail.requested_db || "public"} + {(detail.requested_db || "public") === "public" ? " (default)" : ""} + + + Dataset ID:{" "} + {detail.requested_dataset_id ? ( + <> + {detail.requested_dataset_id} (preferred) + + ) : ( + Assigned by NeuroJSON after approval + )} + + {detail.dataset_id && ( + + Locked — this dataset is published, so its address can't change. + + )} + + + + setSettingsOpen(false)} + onSaved={async () => { + setSettingsOpen(false); + if (internalId) setDetail(await UploadService.getUpload(internalId)); + }} + /> + {/* Review: readiness checklist + submit / resubmit / withdraw */} diff --git a/src/pages/UploadPage.tsx b/src/pages/UploadPage.tsx index e5d24ac..df03317 100644 --- a/src/pages/UploadPage.tsx +++ b/src/pages/UploadPage.tsx @@ -18,12 +18,16 @@ import { import CopyButton from "components/CopyButton"; import { Colors } from "design/theme"; import { useAppSelector } from "hooks/useAppSelector"; -import React, { useEffect, useRef, useState } from "react"; +import React, { useRef, useState } from "react"; import { Link, useLocation } from "react-router-dom"; import { AuthSelector } from "redux/auth/auth.selector"; -import { NeurojsonService } from "services/neurojson.service"; import { UploadService } from "services/upload.service"; import RoutesEnum from "types/routes.enum"; +import { + getRequestedDbError, + getRequestedIdError, + useExistingDbNames, +} from "utils/uploadValidation"; const UploadPage: React.FC = () => { const { isLoggedIn } = useAppSelector(AuthSelector); @@ -39,10 +43,8 @@ const UploadPage: React.FC = () => { const [datasetName, setDatasetName] = useState(""); const [requestedDatasetId, setRequestedDatasetId] = useState(""); const [requestedDb, setRequestedDb] = useState(""); - // Existing public db names (lowercased) — a user may not publish into another - // collection's database, so typing one of these is blocked. Fetched for the - // check only (not offered as a picker). Non-fatal if it fails to load. - const [existingDbs, setExistingDbs] = useState>(new Set()); + // Existing public db names — typing one of these is blocked (shared helper). + const existingDbs = useExistingDbNames(); const [submitting, setSubmitting] = useState(false); const [result, setResult] = useState<{ internalId: string; @@ -56,17 +58,6 @@ const UploadPage: React.FC = () => { message: string; } | null>(null); - useEffect(() => { - NeurojsonService.getRegistry() - .then((data: any) => { - const names = (data?.database || []).map((d: any) => - String(d.name || "").toLowerCase() - ); - setExistingDbs(new Set(names)); - }) - .catch(() => {}); // non-fatal — skip the collision check if it fails - }, []); - const resetOutcome = () => { setResult(null); setSubmitError(null); @@ -101,8 +92,13 @@ const UploadPage: React.FC = () => { const { status, data } = await UploadService.uploadJson(parsed, { internalId: internalId.trim() || undefined, datasetName: datasetName.trim() || undefined, - requestedDatasetId: requestedDatasetId.trim() || undefined, - requestedDb: requestedDb.trim() || undefined, + // Only sent for a new dataset (the backend ignores them on update). + requestedDatasetId: internalId.trim() + ? undefined + : requestedDatasetId.trim() || undefined, + requestedDb: internalId.trim() + ? undefined + : requestedDb.trim() || undefined, confirm, }); if (status === 201) { @@ -118,7 +114,9 @@ const UploadPage: React.FC = () => { setInfoMessage(data.message); } else { // Real errors, incl. REQUESTED_ID_TAKEN (which carries `error`). - setSubmitError(data.error || data.message || `Upload failed (${status})`); + setSubmitError( + data.error || data.message || `Upload failed (${status})` + ); } } catch (e: any) { setSubmitError(e.message || "Upload failed."); @@ -160,38 +158,17 @@ const UploadPage: React.FC = () => { const nameFromFile = dd && typeof dd === "object" && !Array.isArray(dd) ? dd.Name : undefined; - // Live validation of the preferred public id (mirrors the backend rule: - // ^[a-z0-9][a-z0-9_-]{2,62}$), but with a specific message per problem. - const CHAR_RE = /^[a-z0-9][a-z0-9_-]*$/; // allowed chars + first-char rule - const requestedTrimmed = requestedDatasetId.trim(); - const requestedIdError = - requestedTrimmed === "" - ? "" - : requestedTrimmed.startsWith("njds") - ? 'Cannot start with "njds" (reserved for NeuroJSON-assigned IDs).' - : !CHAR_RE.test(requestedTrimmed) - ? "Use only lowercase letters and numbers; join words with - or _ (e.g. my-fmri-study). Cannot start with - or _." - : requestedTrimmed.length < 3 - ? "Too short — use at least 3 characters." - : requestedTrimmed.length > 63 - ? "Too long — use 63 characters or fewer." - : ""; + // Publishing settings are set here only for a NEW dataset; for an update + // they're edited on the upload detail page (PATCH /settings). + const isNewDataset = !internalId.trim(); - // Same slug rule for the target database name (no njds restriction). - const dbTrimmed = requestedDb.trim(); - const requestedDbError = - dbTrimmed === "" - ? "" - : !CHAR_RE.test(dbTrimmed) - ? "Use only lowercase letters and numbers; join words with - or _ (e.g. my-lab)." - : dbTrimmed.length < 3 - ? "Too short — use at least 3 characters." - : dbTrimmed.length > 63 - ? "Too long — use 63 characters or fewer." - : dbTrimmed.toLowerCase() !== "public" && - existingDbs.has(dbTrimmed.toLowerCase()) - ? `A database named "${dbTrimmed}" already exists. Choose a different name or leave blank for public.` - : ""; + // Live validation (shared with the detail-page settings dialog). + const requestedIdError = isNewDataset + ? getRequestedIdError(requestedDatasetId) + : ""; + const requestedDbError = isNewDataset + ? getRequestedDbError(requestedDb, existingDbs) + : ""; return ( @@ -219,8 +196,7 @@ const UploadPage: React.FC = () => { Uploading does not send your dataset for review. Review starts only after you click Submit for review on the upload detail page; the NeuroJSON team reviews it before it is added to a - database. - A new reference id is assigned on upload — any existing{" "} + database. A new reference id is assigned on upload — any existing{" "} _id/_rev in your file is ignored. @@ -288,72 +264,88 @@ const UploadPage: React.FC = () => { } /> - {/* Target database — where the dataset is published after review */} - setRequestedDb(e.target.value)} - size="small" - fullWidth - error={!!requestedDbError} - helperText={requestedDbError || undefined} - sx={{ mt: 2.5 }} - /> - -
  • - Becomes part of your dataset’s public URL after review (e.g.{" "} - /db/smith-lab/…). -
    • -
  • - Leave blank for the default public database. -
    • -     + {/* Publishing settings: only for a NEW dataset. For an update they're + edited on the upload detail page. */} + {!isNewDataset && ( + + Publishing settings (database and dataset ID) are changed on the + upload detail page. + + )} - {/* Optional preferred public ID (a preference; final id is set at review) */} - setRequestedDatasetId(e.target.value)} - size="small" - fullWidth - error={!!requestedIdError} - helperText={requestedIdError || undefined} - sx={{ mt: 2.5 }} - /> - -
  • - Becomes part of your dataset’s public URL after review (e.g.{" "} - /db/…/my-fmri-study). -
    • -
  • - Use only lowercase letters and numbers; join words with - or _ - (e.g. my-fmri-study). -
    • -     + {isNewDataset && ( + <> + {/* Target database — where the dataset is published after review */} + setRequestedDb(e.target.value)} + size="small" + fullWidth + error={!!requestedDbError} + helperText={requestedDbError || undefined} + sx={{ mt: 2.5 }} + /> + +
  • + Becomes part of your dataset’s public URL after review (e.g.{" "} + /db/smith-lab/…). +
    • +
  • + Leave blank for the default public database. +
    • +     + + {/* Optional preferred public ID (a preference; final id is set at review) */} + setRequestedDatasetId(e.target.value)} + size="small" + fullWidth + error={!!requestedIdError} + helperText={requestedIdError || undefined} + sx={{ mt: 2.5 }} + /> + +
  • + Becomes part of your dataset’s public URL after review (e.g.{" "} + /db/…/my-fmri-study). +
    • +
  • + Use only lowercase letters and numbers; join words with - or _ + (e.g. my-fmri-study). +
    • +     + + )} {/* Existing-dataset internal ID — normally auto-filled from the dashboard */} => { + const res = await fetch(`${baseURL}/uploads/${internalId}/settings`, { + method: "PATCH", + credentials: "include", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(settings), + }); + if (!res.ok) { + const body = await res.json().catch(() => ({})); + throw new Error(body.error || `Failed to update settings (${res.status})`); + } + return res.json(); + }, + // pending → draft (only before the reviewer acts) withdraw: async (internalId: string): Promise<{ status: string }> => { const res = await fetch(`${baseURL}/uploads/${internalId}/withdraw`, { diff --git a/src/utils/uploadValidation.ts b/src/utils/uploadValidation.ts new file mode 100644 index 0000000..cc9fab5 --- /dev/null +++ b/src/utils/uploadValidation.ts @@ -0,0 +1,67 @@ +import { useEffect, useState } from "react"; +import { NeurojsonService } from "services/neurojson.service"; + +// Live checks for the upload publishing settings. They mirror the backend +// rules (^[a-z0-9][a-z0-9_-]{2,62}$; no "njds" prefix for dataset ids) but +// give one specific message per problem. The backend stays authoritative. + +const CHAR_RE = /^[a-z0-9][a-z0-9_-]*$/; // allowed chars + first-char rule + +const lengthError = (v: string) => + v.length < 3 + ? "Too short — use at least 3 characters." + : v.length > 63 + ? "Too long — use 63 characters or fewer." + : ""; + +// Preferred public dataset id. Empty = no preference (valid). +export const getRequestedIdError = (raw: string): string => { + const v = raw.trim(); + if (!v) return ""; + if (v.startsWith("njds")) { + return 'Cannot start with "njds" (reserved for NeuroJSON-assigned IDs).'; + } + if (!CHAR_RE.test(v)) { + return "Use only lowercase letters and numbers; join words with - or _ (e.g. my-fmri-study). Cannot start with - or _."; + } + return lengthError(v); +}; + +// Target database. Empty = default "public" (valid). Names of existing +// databases belong to other collections, so they're blocked (except public). +export const getRequestedDbError = ( + raw: string, + existingDbs: Set +): string => { + const v = raw.trim(); + if (!v) return ""; + if (!CHAR_RE.test(v)) { + return "Use only lowercase letters and numbers; join words with - or _ (e.g. smith-lab)."; + } + const len = lengthError(v); + if (len) return len; + if (v.toLowerCase() !== "public" && existingDbs.has(v.toLowerCase())) { + return `A database named "${v}" already exists. Choose a different name or leave blank for public.`; + } + return ""; +}; + +// Existing public db names (lowercased), fetched for the collision check only — +// never offered as a picker. Non-fatal if the registry can't be loaded. +export const useExistingDbNames = (): Set => { + const [names, setNames] = useState>(new Set()); + useEffect(() => { + NeurojsonService.getRegistry() + .then((data: any) => + setNames( + new Set( + (data?.database || []).map((d: any) => + String(d.name || "").toLowerCase() + ) + ) + ) + ) + .catch(() => {}); + }, []); + return names; +}; From 0d207923737a8b97ae0f5e83c14a6e235eb29f34 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Tue, 29 Sep 2026 15:55:45 -0400 Subject: [PATCH 29/45] fix(uploads): remove the internal ID field from the upload form; refs #147 --- src/pages/UploadPage.tsx | 57 ++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/src/pages/UploadPage.tsx b/src/pages/UploadPage.tsx index df03317..8f6e508 100644 --- a/src/pages/UploadPage.tsx +++ b/src/pages/UploadPage.tsx @@ -19,7 +19,7 @@ import CopyButton from "components/CopyButton"; import { Colors } from "design/theme"; import { useAppSelector } from "hooks/useAppSelector"; import React, { useRef, useState } from "react"; -import { Link, useLocation } from "react-router-dom"; +import { Link, useLocation, useNavigate } from "react-router-dom"; import { AuthSelector } from "redux/auth/auth.selector"; import { UploadService } from "services/upload.service"; import RoutesEnum from "types/routes.enum"; @@ -33,6 +33,7 @@ const UploadPage: React.FC = () => { const { isLoggedIn } = useAppSelector(AuthSelector); const fileInputRef = useRef(null); const location = useLocation(); + const navigate = useNavigate(); const initialInternalId = new URLSearchParams(location.search).get("internalId") ?? ""; @@ -200,6 +201,35 @@ const UploadPage: React.FC = () => { _id/_rev in your file is ignored. + {/* Updating an existing dataset: the id comes only from the URL + (?internalId=…, set by the Update button) — never typed by hand. */} + {!isNewDataset && ( + + You're updating an existing dataset (reference{" "} + {internalId}).{" "} + { + setInternalId(""); + resetOutcome(); + navigate("/upload", { replace: true }); + }} + sx={{ + color: Colors.purple, + fontWeight: 600, + verticalAlign: "baseline", + }} + > + Start a new dataset instead + + + )} + { )} - {/* Existing-dataset internal ID — normally auto-filled from the dashboard */} - setInternalId(e.target.value)} - size="small" - fullWidth - sx={{ mt: 2.5 }} - /> - -
  • Leave blank for a new dataset.
    • -     - )} - {/* Same rule as the detail page: locked while - pending/approved (withdraw on the detail page to - edit a pending one). */} - {r.status !== "pending" && r.status !== "approved" && ( - - )} + {/* All actions (Update, Submit, settings) live on the + upload detail page, which also shows review state. */} + From 893f58be59b32c4ade8f468fbd2f8e7645e67bd5 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Tue, 29 Sep 2026 16:43:14 -0400 Subject: [PATCH 31/45] feat(uploads): resubmit reminder and stricter name validation; refs #147 --- backend/src/controllers/uploads.controller.js | 35 +++++++++++++--- src/components/User/Dashboard/UploadsTab.tsx | 12 ++++++ src/pages/UploadDetailPage.tsx | 29 +++++++++++++ src/pages/UploadPage.tsx | 42 ++++++++++++++----- src/services/upload.service.ts | 1 + src/utils/uploadValidation.ts | 10 +++-- 6 files changed, 111 insertions(+), 18 deletions(-) diff --git a/backend/src/controllers/uploads.controller.js b/backend/src/controllers/uploads.controller.js index 078689d..db3fefe 100644 --- a/backend/src/controllers/uploads.controller.js +++ b/backend/src/controllers/uploads.controller.js @@ -56,11 +56,13 @@ const validateRequestedId = (id) => { // Target public database name (same slug rule; no njds restriction — that is // only for dataset ids). Blank defaults to "public" upstream. +// CouchDB requires db names to start with a lowercase letter, so this is +// stricter than the dataset-id rule. const validateRequestedDb = (db) => { - if (!/^[a-z0-9][a-z0-9_-]{2,62}$/.test(db)) { + if (!/^[a-z][a-z0-9_-]{2,62}$/.test(db)) { throw new HttpError(400, { error: - "Database name must be 3–63 characters: lowercase letters, numbers, - or _.", + "Database name must be 3–63 characters, start with a lowercase letter, and use only lowercase letters, numbers, - or _.", }); } }; @@ -109,12 +111,33 @@ const createUpload = async (req, res) => { // dataset name: file's dataset_description.json.Name → ?datasetName → 400. const dd = clean["dataset_description.json"]; - const nameFromDoc = + const rawFromDoc = dd && typeof dd === "object" && !Array.isArray(dd) ? dd.Name : undefined; - const datasetName = nameFromDoc || req.query.datasetName; + if ( + rawFromDoc !== undefined && + rawFromDoc !== null && + typeof rawFromDoc !== "string" + ) { + return res.status(400).json({ + error: 'The "Name" in dataset_description.json must be text.', + }); + } + // Spaces-only counts as no name; a repeated ?datasetName (array) is ignored. + const nameFromDoc = typeof rawFromDoc === "string" ? rawFromDoc.trim() : ""; + const nameFromQuery = + typeof req.query.datasetName === "string" + ? req.query.datasetName.trim() + : ""; + const datasetName = nameFromDoc || nameFromQuery; if (!datasetName) { return res.status(400).json({ error: "Please enter a dataset name." }); } + // Matches the dataset_name VARCHAR(255) column (clear 400, not a DB error). + if (datasetName.length > 255) { + return res.status(400).json({ + error: "Dataset name is too long (255 characters max).", + }); + } if (!nameFromDoc) { if (!dd || typeof dd !== "object" || Array.isArray(dd)) { clean["dataset_description.json"] = {}; @@ -626,7 +649,8 @@ const getUpload = async (req, res) => { const reg = await loadOwnedRegistry(String(req.params.internalId), req.user.id); const s = await sequelize.query( `SELECT submission_id, status, created_at, updated_at, promoted_db, promoted_at, - json_status, json_uploaded_at, json_error, submitted_at, raw_zip_expected + json_status, json_uploaded_at, json_error, submitted_at, raw_zip_expected, + reviewed_at FROM submissions WHERE dataset_registry_id = :iid ORDER BY created_at DESC LIMIT 1`, { replacements: { iid: reg.internal_id }, type: sequelize.QueryTypes.SELECT } @@ -648,6 +672,7 @@ const getUpload = async (req, res) => { json_uploaded_at: sub.json_uploaded_at || null, json_error: sub.json_error || null, submitted_at: sub.submitted_at || null, + reviewed_at: sub.reviewed_at || null, raw_zip_expected: !!sub.raw_zip_expected, readiness: sub.submission_id ? getReadiness(sub) diff --git a/src/components/User/Dashboard/UploadsTab.tsx b/src/components/User/Dashboard/UploadsTab.tsx index 0a55c2c..7484c33 100644 --- a/src/components/User/Dashboard/UploadsTab.tsx +++ b/src/components/User/Dashboard/UploadsTab.tsx @@ -172,6 +172,18 @@ const UploadsTab: React.FC = () => { : "" }`} /> + {r.status === "changes_requested" && ( + + Action needed: update and resubmit for review + + )} {r.status === "promoted" && r.promoted_db && r.dataset_id && ( diff --git a/src/pages/UploadDetailPage.tsx b/src/pages/UploadDetailPage.tsx index b591269..d436d55 100644 --- a/src/pages/UploadDetailPage.tsx +++ b/src/pages/UploadDetailPage.tsx @@ -175,6 +175,12 @@ const UploadDetailPage: React.FC = () => { } const chip = detail.status ? STATUS_CHIP[detail.status] : null; + // JSON re-uploaded after the reviewer asked for changes? + const updatedSinceReview = + detail.status === "changes_requested" && + !!detail.json_uploaded_at && + !!detail.reviewed_at && + new Date(detail.json_uploaded_at) > new Date(detail.reviewed_at); const canEditSettings = (detail.status === "draft" || detail.status === "changes_requested") && !detail.dataset_id; @@ -368,6 +374,29 @@ const UploadDetailPage: React.FC = () => { {(detail.status === "draft" || detail.status === "changes_requested") && ( <> + {/* Changes requested: remind the user that updating is not enough — + they must click Resubmit. Wording depends on whether the JSON + was updated after the reviewer's decision. */} + {detail.status === "changes_requested" && ( + + {updatedSinceReview ? ( + <> + You've updated your dataset since the reviewer's request. + Click Resubmit for review so the reviewer + can see your changes. + + ) : ( + <> + The reviewer requested changes. Read their message below, + update your files, then click{" "} + Resubmit for review. + + )} + + )} {!detail.readiness.canSubmit && ( {detail.readiness.problems.join(" ")} diff --git a/src/pages/UploadPage.tsx b/src/pages/UploadPage.tsx index 8f6e508..ae25ba8 100644 --- a/src/pages/UploadPage.tsx +++ b/src/pages/UploadPage.tsx @@ -155,9 +155,22 @@ const UploadPage: React.FC = () => { } // If the file already carries a name, use it and disable the name field. + // Mirrors the backend: Name must be text; spaces-only counts as no name. const dd = parsed?.["dataset_description.json"] as any; - const nameFromFile = + const rawName = dd && typeof dd === "object" && !Array.isArray(dd) ? dd.Name : undefined; + const fileNameTypeError = + rawName !== undefined && rawName !== null && typeof rawName !== "string" + ? 'The "Name" in dataset_description.json must be text.' + : ""; + const nameFromFile = typeof rawName === "string" ? rawName.trim() : ""; + const typedName = datasetName.trim(); + const datasetNameError = fileNameTypeError + ? fileNameTypeError + : (nameFromFile || typedName).length > 255 + ? "Too long — 255 characters max." + : ""; + const hasName = !!nameFromFile || !!typedName; // Publishing settings are set here only for a NEW dataset; for an update // they're edited on the upload detail page (PATCH /settings). @@ -275,22 +288,26 @@ const UploadPage: React.FC = () => { setDatasetName(e.target.value)} - disabled={!!nameFromFile} + disabled={!!nameFromFile || !!fileNameTypeError} size="small" fullWidth + error={!!datasetNameError} sx={{ mt: 2.5, - // Highlight the "no name found" prompt so it's easy to notice. - "& .MuiFormHelperText-root": nameFromFile - ? {} - : { color: Colors.purple, fontWeight: 600 }, + // Highlight the "no name found" prompt so it's easy to notice + // (red error styling wins when there's a problem). + "& .MuiFormHelperText-root": + nameFromFile || datasetNameError + ? {} + : { color: Colors.purple, fontWeight: 600 }, }} helperText={ - nameFromFile + datasetNameError || + (nameFromFile ? 'Name taken from your file ("dataset_description.json" → Name).' - : 'No name found in your file ("dataset_description.json" → Name) — enter one here.' + : 'No name found in your file ("dataset_description.json" → Name) — enter one here.') } /> @@ -382,7 +399,12 @@ const UploadPage: React.FC = () => { variant="contained" size="large" disabled={ - !parsed || submitting || !!requestedIdError || !!requestedDbError + !parsed || + submitting || + !hasName || + !!datasetNameError || + !!requestedIdError || + !!requestedDbError } onClick={() => submit(false)} sx={{ diff --git a/src/services/upload.service.ts b/src/services/upload.service.ts index 69116a4..7bb0a35 100644 --- a/src/services/upload.service.ts +++ b/src/services/upload.service.ts @@ -41,6 +41,7 @@ export interface UploadDetail { json_uploaded_at: string | null; json_error: string | null; submitted_at: string | null; + reviewed_at: string | null; raw_zip_expected: boolean; readiness: { canSubmit: boolean; problems: string[] }; } diff --git a/src/utils/uploadValidation.ts b/src/utils/uploadValidation.ts index cc9fab5..5550fab 100644 --- a/src/utils/uploadValidation.ts +++ b/src/utils/uploadValidation.ts @@ -5,7 +5,8 @@ import { NeurojsonService } from "services/neurojson.service"; // rules (^[a-z0-9][a-z0-9_-]{2,62}$; no "njds" prefix for dataset ids) but // give one specific message per problem. The backend stays authoritative. -const CHAR_RE = /^[a-z0-9][a-z0-9_-]*$/; // allowed chars + first-char rule +const CHAR_RE = /^[a-z0-9][a-z0-9_-]*$/; // dataset id: allowed chars + first-char rule +const DB_CHAR_RE = /^[a-z][a-z0-9_-]*$/; // CouchDB: db names start with a letter const lengthError = (v: string) => v.length < 3 @@ -35,12 +36,15 @@ export const getRequestedDbError = ( ): string => { const v = raw.trim(); if (!v) return ""; - if (!CHAR_RE.test(v)) { + if (!/^[a-z]/.test(v)) { + return "Must start with a lowercase letter (e.g. smith-lab)."; + } + if (!DB_CHAR_RE.test(v)) { return "Use only lowercase letters and numbers; join words with - or _ (e.g. smith-lab)."; } const len = lengthError(v); if (len) return len; - if (v.toLowerCase() !== "public" && existingDbs.has(v.toLowerCase())) { + if (v !== "public" && existingDbs.has(v)) { return `A database named "${v}" already exists. Choose a different name or leave blank for public.`; } return ""; From 50d6c69827ae0cad5317b45890c119c255a7dce6 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Wed, 30 Sep 2026 15:45:40 -0400 Subject: [PATCH 32/45] feat(uploads): add raw data tables (raw_objects, submission_raw_files, raw_upload_attempts); refs #147 --- .../20260930193225-create-raw-objects.js | 103 ++++++++++++++++++ ...60930193235-create-submission-raw-files.js | 53 +++++++++ ...260930193244-create-raw-upload-attempts.js | 92 ++++++++++++++++ 3 files changed, 248 insertions(+) create mode 100644 backend/migrations/20260930193225-create-raw-objects.js create mode 100644 backend/migrations/20260930193235-create-submission-raw-files.js create mode 100644 backend/migrations/20260930193244-create-raw-upload-attempts.js diff --git a/backend/migrations/20260930193225-create-raw-objects.js b/backend/migrations/20260930193225-create-raw-objects.js new file mode 100644 index 0000000..78dc49e --- /dev/null +++ b/backend/migrations/20260930193225-create-raw-objects.js @@ -0,0 +1,103 @@ +"use strict"; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + // One stored raw dataset: the uploaded ZIP (transport artifact) and, after a + // reviewer triggers extraction on Zodiac, its extracted file tree. Shared by + // any number of dataset versions via submission_raw_files. All keys are + // RELATIVE to the Zodiac UPLOAD_ROOT (never absolute paths): + // /raw//source.zip + // /raw//files + // /raw//manifest.json + // zip_storage_key is permanent provenance; zip_deleted_at says whether the + // ZIP still physically exists (a future retention rule may delete it). + async up(queryInterface, Sequelize) { + await queryInterface.createTable("raw_objects", { + raw_id: { + type: Sequelize.UUID, + primaryKey: true, + allowNull: false, // generated by REN when it accepts a verified upload + }, + dataset_registry_id: { + type: Sequelize.UUID, + allowNull: false, + references: { model: "dataset_registry", key: "internal_id" }, + onDelete: "RESTRICT", + onUpdate: "CASCADE", + }, + zip_storage_key: { + type: Sequelize.STRING(512), + allowNull: false, + unique: true, + }, + original_filename: { type: Sequelize.STRING(255), allowNull: true }, + size_bytes: { type: Sequelize.BIGINT, allowNull: false }, // of the ZIP + sha256: { type: Sequelize.CHAR(64), allowNull: false }, // of the ZIP; not unique + zip_deleted_at: { type: Sequelize.DATE, allowNull: true }, + + extraction_status: { + type: Sequelize.STRING(20), + allowNull: false, + defaultValue: "not_extracted", // not_extracted|extracting|extracted|failed + }, + extraction_requested_by: { + type: Sequelize.INTEGER, + allowNull: true, + references: { model: "users", key: "id" }, + onDelete: "SET NULL", + onUpdate: "CASCADE", + }, + extraction_started_at: { type: Sequelize.DATE, allowNull: true }, + extracted_storage_key: { + type: Sequelize.STRING(512), + allowNull: true, + unique: true, + }, + manifest_storage_key: { + type: Sequelize.STRING(512), + allowNull: true, + unique: true, + }, + extracted_file_count: { type: Sequelize.INTEGER, allowNull: true }, + extracted_size_bytes: { type: Sequelize.BIGINT, allowNull: true }, + extracted_at: { type: Sequelize.DATE, allowNull: true }, + extraction_error: { type: Sequelize.TEXT, allowNull: true }, // safe message only + + created_at: { + type: Sequelize.DATE, + allowNull: false, + defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"), + }, + updated_at: { + type: Sequelize.DATE, + allowNull: false, + defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"), + }, + }); + + await queryInterface.sequelize.query(` + ALTER TABLE raw_objects + ADD CONSTRAINT chk_raw_objects_extraction_status + CHECK (extraction_status IN ('not_extracted','extracting','extracted','failed')), + ADD CONSTRAINT chk_raw_objects_extracted_complete + CHECK (extraction_status <> 'extracted' + OR (extracted_storage_key IS NOT NULL + AND manifest_storage_key IS NOT NULL + AND extracted_at IS NOT NULL)); + `); + + await queryInterface.addIndex("raw_objects", ["dataset_registry_id"], { + name: "idx_raw_objects_registry", + }); + // Finds extractions stuck in 'extracting' (reconciliation). + await queryInterface.addIndex( + "raw_objects", + ["extraction_status", "updated_at"], + { name: "idx_raw_objects_extraction_status" } + ); + }, + + async down(queryInterface) { + await queryInterface.dropTable("raw_objects"); + }, +}; diff --git a/backend/migrations/20260930193235-create-submission-raw-files.js b/backend/migrations/20260930193235-create-submission-raw-files.js new file mode 100644 index 0000000..cd631ca --- /dev/null +++ b/backend/migrations/20260930193235-create-submission-raw-files.js @@ -0,0 +1,53 @@ +"use strict"; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + // Which raw object(s) a dataset version (submission) uses. Composite PK + // (submission_id, raw_id) so a version could reference several objects in + // the future. For now a version has at most ONE object, enforced by + // uq_submission_raw_files_one_per_submission — drop that index to allow + // several (a path→object rule, e.g. a mount_path column, is then needed). + // Several versions may point to the same object (carry-forward = a new + // mapping row, no file copy/link). Deleting a version removes only its + // mapping; a mapped object can't be deleted (RESTRICT). Code must ensure + // the object and the submission belong to the same dataset. + async up(queryInterface, Sequelize) { + await queryInterface.createTable("submission_raw_files", { + submission_id: { + type: Sequelize.UUID, + primaryKey: true, // composite PK part 1 + allowNull: false, + references: { model: "submissions", key: "submission_id" }, + onDelete: "CASCADE", + onUpdate: "CASCADE", + }, + raw_id: { + type: Sequelize.UUID, + primaryKey: true, // composite PK part 2 + allowNull: false, + references: { model: "raw_objects", key: "raw_id" }, + onDelete: "RESTRICT", + onUpdate: "CASCADE", + }, + created_at: { + type: Sequelize.DATE, + allowNull: false, + defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"), + }, + }); + + // Current rule: one raw object per version (drop to allow several). + await queryInterface.sequelize.query( + "CREATE UNIQUE INDEX uq_submission_raw_files_one_per_submission ON submission_raw_files (submission_id);" + ); + + // "Which versions use this object?" — for cleanup of unmapped objects. + await queryInterface.addIndex("submission_raw_files", ["raw_id"], { + name: "idx_submission_raw_files_raw", + }); + }, + + async down(queryInterface) { + await queryInterface.dropTable("submission_raw_files"); + }, +}; diff --git a/backend/migrations/20260930193244-create-raw-upload-attempts.js b/backend/migrations/20260930193244-create-raw-upload-attempts.js new file mode 100644 index 0000000..0d3a281 --- /dev/null +++ b/backend/migrations/20260930193244-create-raw-upload-attempts.js @@ -0,0 +1,92 @@ +"use strict"; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + // One tus upload of a raw ZIP for a dataset version. Tracks the upload + // process only — the final size/sha256 live on raw_objects. raw_id is set + // when REN accepts the verified upload (same transaction that creates the + // raw_objects row and the submission_raw_files mapping); a repeated + // completion message sees raw_id already set and does nothing. + async up(queryInterface, Sequelize) { + await queryInterface.createTable("raw_upload_attempts", { + upload_id: { + type: Sequelize.UUID, + primaryKey: true, + allowNull: false, // = the tus upload id carried in the upload token + }, + submission_id: { + type: Sequelize.UUID, + allowNull: false, + references: { model: "submissions", key: "submission_id" }, + onDelete: "CASCADE", + onUpdate: "CASCADE", + }, + status: { + type: Sequelize.STRING(20), + allowNull: false, + defaultValue: "initiated", + // initiated|uploading|verifying|complete|failed|cancelled|expired + }, + original_filename: { type: Sequelize.STRING(255), allowNull: true }, + // tus Upload-Length; also used for the size cap, disk check, final check. + expected_size: { type: Sequelize.BIGINT, allowNull: false }, + error: { type: Sequelize.TEXT, allowNull: true }, + uploaded_by: { + type: Sequelize.INTEGER, + allowNull: false, + references: { model: "users", key: "id" }, + onDelete: "RESTRICT", + onUpdate: "CASCADE", + }, + raw_id: { + type: Sequelize.UUID, + allowNull: true, + unique: true, // one upload can never produce two objects + references: { model: "raw_objects", key: "raw_id" }, + onDelete: "SET NULL", + onUpdate: "CASCADE", + }, + last_checked_at: { type: Sequelize.DATE, allowNull: true }, + created_at: { + type: Sequelize.DATE, + allowNull: false, + defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"), + }, + updated_at: { + type: Sequelize.DATE, + allowNull: false, + defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"), + }, + completed_at: { type: Sequelize.DATE, allowNull: true }, + }); + + // Only a completed attempt may point to an object. One direction only: + // a completed attempt may later have raw_id = NULL if cleanup deletes its + // (unmapped) object — ON DELETE SET NULL must not violate this check. + await queryInterface.sequelize.query(` + ALTER TABLE raw_upload_attempts + ADD CONSTRAINT chk_raw_upload_attempts_status + CHECK (status IN ('initiated','uploading','verifying','complete','failed','cancelled','expired')), + ADD CONSTRAINT chk_raw_upload_attempts_raw_only_when_complete + CHECK (raw_id IS NULL OR status = 'complete'); + `); + + await queryInterface.addIndex("raw_upload_attempts", ["submission_id"], { + name: "idx_raw_upload_attempts_submission", + }); + // Reconciliation scans stale in-progress attempts. + await queryInterface.addIndex( + "raw_upload_attempts", + ["status", "updated_at"], + { name: "idx_raw_upload_attempts_status" } + ); + // At most one active upload per version. + await queryInterface.sequelize.query( + "CREATE UNIQUE INDEX uq_one_active_raw_upload ON raw_upload_attempts (submission_id) WHERE status IN ('initiated','uploading','verifying');" + ); + }, + + async down(queryInterface) { + await queryInterface.dropTable("raw_upload_attempts"); + }, +}; From 8b606d4099c247faaa8a9ff5392ca646e7992cc5 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Wed, 30 Sep 2026 16:22:31 -0400 Subject: [PATCH 33/45] feat(uploads): Ed25519 upload tokens and HMAC service signatures; refs #147 --- backend/scripts/generate-upload-keys.js | 37 ++++++++ backend/scripts/test-upload-signing.js | 89 +++++++++++++++++ backend/src/lib/storageSignature.js | 70 ++++++++++++++ backend/src/lib/uploadTokens.js | 121 ++++++++++++++++++++++++ 4 files changed, 317 insertions(+) create mode 100644 backend/scripts/generate-upload-keys.js create mode 100644 backend/scripts/test-upload-signing.js create mode 100644 backend/src/lib/storageSignature.js create mode 100644 backend/src/lib/uploadTokens.js diff --git a/backend/scripts/generate-upload-keys.js b/backend/scripts/generate-upload-keys.js new file mode 100644 index 0000000..a0f3e11 --- /dev/null +++ b/backend/scripts/generate-upload-keys.js @@ -0,0 +1,37 @@ +// Generate a new Ed25519 key pair (upload/file-read tokens) and an HMAC secret +// (REN <-> Zodiac service calls). Prints .env lines; writes nothing to disk. +// Usage: node scripts/generate-upload-keys.js [kid] (default kid: dev-YYYY-MM) +const crypto = require("crypto"); + +const now = new Date(); +const kid = + process.argv[2] || + `dev-${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, "0")}`; +if (!/^[a-z0-9][a-z0-9_-]{2,40}$/.test(kid)) { + console.error("kid must be 3-41 chars: lowercase letters, digits, - or _"); + process.exit(1); +} + +const { privateKey, publicKey } = crypto.generateKeyPairSync("ed25519"); +const privPem = privateKey.export({ type: "pkcs8", format: "pem" }); +const pubPem = publicKey.export({ type: "spki", format: "pem" }); +const hmacSecret = crypto.randomBytes(32).toString("base64url"); + +// dotenv turns \n inside double quotes into real newlines. +const oneLine = (pem) => pem.trim().replace(/\n/g, "\\n"); + +console.log("# ---- REN backend/.env (keep private) ----"); +console.log(`UPLOAD_TOKEN_KID=${kid}`); +console.log(`UPLOAD_TOKEN_PRIVATE_KEY="${oneLine(privPem)}"`); +console.log(`STORAGE_HMAC_ACTIVE_KID=${kid}`); +console.log(`STORAGE_HMAC_KEYS='${JSON.stringify({ [kid]: hmacSecret })}'`); +console.log(""); +console.log("# ---- Zodiac storage-api/.env ----"); +console.log( + `UPLOAD_TOKEN_PUBLIC_KEYS='${JSON.stringify({ [kid]: pubPem.trim() })}'` +); +console.log(`STORAGE_HMAC_ACTIVE_KID=${kid}`); +console.log(`STORAGE_HMAC_KEYS='${JSON.stringify({ [kid]: hmacSecret })}'`); +console.log(""); +console.log("# Rotation: add the new kid to the Zodiac maps first, then switch REN"); +console.log("# to the new kid, then remove the old kid after 15+ minutes."); diff --git a/backend/scripts/test-upload-signing.js b/backend/scripts/test-upload-signing.js new file mode 100644 index 0000000..634c636 --- /dev/null +++ b/backend/scripts/test-upload-signing.js @@ -0,0 +1,89 @@ +// Self-test for the upload token + service-signature helpers (prints no secrets). +// Usage: node scripts/test-upload-signing.js (reads backend/.env) +require("dotenv").config(); +const crypto = require("crypto"); +const { + signUploadToken, + verifyToken, + signToken, +} = require("../src/lib/uploadTokens"); +const { signRequest, verifyRequest } = require("../src/lib/storageSignature"); + +let failed = 0; +const check = (name, fn) => { + try { + fn(); + console.log("PASS", name); + } catch (e) { + failed++; + console.log("FAIL", name, "-", e.message); + } +}; +const expectThrow = (fn, msg) => { + try { + fn(); + } catch { + return; + } + throw new Error(`expected failure: ${msg}`); +}; + +const kid = process.env.UPLOAD_TOKEN_KID; +const pub = { + [kid]: crypto.createPublicKey(process.env.UPLOAD_TOKEN_PRIVATE_KEY), +}; +const { token } = signUploadToken({ + userId: 1, + uploadId: crypto.randomUUID(), + internalId: crypto.randomUUID(), + submissionId: crypto.randomUUID(), + expectedSize: 1234, +}); + +check("valid upload token verifies", () => { + const c = verifyToken(token, pub, "tus-upload"); + if (c.len !== 1234) throw new Error("len claim wrong"); +}); +check("tampered payload is rejected", () => { + const [h, p, s] = token.split("."); + const bad = JSON.parse(Buffer.from(p, "base64url").toString()); + bad.len = 999999; + const forged = `${h}.${Buffer.from(JSON.stringify(bad)).toString( + "base64url" + )}.${s}`; + expectThrow(() => verifyToken(forged, pub, "tus-upload"), "tamper"); +}); +check("wrong token type is rejected", () => + expectThrow(() => verifyToken(token, pub, "file-read"), "typ") +); +check("unknown kid is rejected", () => + expectThrow(() => verifyToken(token, {}, "tus-upload"), "kid") +); +check("expired token is rejected", () => { + const { token: old } = signToken("tus-upload", {}, -120); + expectThrow(() => verifyToken(old, pub, "tus-upload"), "exp"); +}); + +const body = JSON.stringify({ event: "verified" }); +const path = "/api/v1/internal/storage/uploads/x/events"; +const headers = signRequest({ method: "POST", path, body }); +check("valid HMAC request verifies", () => { + const r = verifyRequest({ method: "POST", path, headers, rawBody: body }); + if (!r.ok) throw new Error(r.reason); +}); +check("HMAC rejects changed body", () => { + const r = verifyRequest({ method: "POST", path, headers, rawBody: body + " " }); + if (r.ok) throw new Error("accepted"); +}); +check("HMAC rejects changed path", () => { + const r = verifyRequest({ + method: "POST", + path: path + "?x=1", + headers, + rawBody: body, + }); + if (r.ok) throw new Error("accepted"); +}); + +console.log(failed ? `\n${failed} check(s) FAILED` : "\nAll checks passed"); +process.exit(failed ? 1 : 0); diff --git a/backend/src/lib/storageSignature.js b/backend/src/lib/storageSignature.js new file mode 100644 index 0000000..9d4ba75 --- /dev/null +++ b/backend/src/lib/storageSignature.js @@ -0,0 +1,70 @@ +// HMAC-SHA256 signatures for server-to-server calls between REN and the +// Zodiac storage API (both directions). Signs timestamp + method + path(+query) +// + SHA-256 of the raw body. Secrets are a kid -> secret map so they can rotate. +const crypto = require("crypto"); + +const MAX_SKEW_SECONDS = 300; +const H_KID = "x-nj-key-id"; +const H_TS = "x-nj-timestamp"; +const H_SIG = "x-nj-signature"; + +const loadKeys = () => { + try { + return JSON.parse(process.env.STORAGE_HMAC_KEYS || "{}"); + } catch { + throw new Error("STORAGE_HMAC_KEYS is not valid JSON."); + } +}; + +const sha256hex = (buf) => + crypto.createHash("sha256").update(buf || "").digest("hex"); +const canonical = (ts, method, path, body) => + [ts, String(method).toUpperCase(), path, sha256hex(body)].join("\n"); + +// Headers to attach to an outgoing request. `body` = the exact bytes/string sent. +const signRequest = ({ method, path, body = "" }) => { + const kid = process.env.STORAGE_HMAC_ACTIVE_KID; + const secret = loadKeys()[kid]; + if (!kid || !secret) throw new Error("Storage HMAC key not configured."); + const ts = String(Math.floor(Date.now() / 1000)); + const sig = crypto + .createHmac("sha256", secret) + .update(canonical(ts, method, path, body)) + .digest("base64url"); + return { [H_KID]: kid, [H_TS]: ts, [H_SIG]: sig }; +}; + +// Check an incoming request. `path` must include the query string exactly as +// sent; `rawBody` must be the unparsed bytes. Returns { ok, reason }. +const verifyRequest = ({ method, path, headers, rawBody }) => { + const kid = headers[H_KID]; + const ts = headers[H_TS]; + const sig = headers[H_SIG]; + if (!kid || !ts || !sig) { + return { ok: false, reason: "missing signature headers" }; + } + const secret = loadKeys()[kid]; + if (!secret) return { ok: false, reason: "unknown key id" }; + const now = Math.floor(Date.now() / 1000); + if (!/^\d+$/.test(ts) || Math.abs(now - Number(ts)) > MAX_SKEW_SECONDS) { + return { ok: false, reason: "timestamp out of range" }; + } + const expected = crypto + .createHmac("sha256", secret) + .update(canonical(ts, method, path, rawBody)) + .digest(); + const given = Buffer.from(String(sig), "base64url"); + if ( + given.length !== expected.length || + !crypto.timingSafeEqual(given, expected) + ) { + return { ok: false, reason: "bad signature" }; + } + return { ok: true, kid }; +}; + +module.exports = { + signRequest, + verifyRequest, + HEADER_NAMES: { H_KID, H_TS, H_SIG }, +}; diff --git a/backend/src/lib/uploadTokens.js b/backend/src/lib/uploadTokens.js new file mode 100644 index 0000000..55227df --- /dev/null +++ b/backend/src/lib/uploadTokens.js @@ -0,0 +1,121 @@ +// Short-lived EdDSA (Ed25519) JWTs that let the browser talk to the Zodiac +// storage API. REN signs with the private key; Zodiac verifies with a +// kid -> public-key map (so keys can rotate). Built on Node's crypto (no jose: +// jose 6 is ESM-only and this backend is CommonJS). +const crypto = require("crypto"); + +const ISSUER = "neurojson-api"; +const AUDIENCE = "zodiac-storage"; +const DEFAULT_TTL_SECONDS = 15 * 60; +const CLOCK_LEEWAY_SECONDS = 30; + +const b64url = (v) => + Buffer.from(typeof v === "string" ? v : JSON.stringify(v)).toString( + "base64url" + ); + +let signer; // cached { key, kid } +const getSigner = () => { + if (signer) return signer; + const pem = process.env.UPLOAD_TOKEN_PRIVATE_KEY; + const kid = process.env.UPLOAD_TOKEN_KID; + if (!pem || !kid) { + throw new Error( + "Upload token key not configured (UPLOAD_TOKEN_PRIVATE_KEY / UPLOAD_TOKEN_KID)." + ); + } + const key = crypto.createPrivateKey(pem); + if (key.asymmetricKeyType !== "ed25519") { + throw new Error("UPLOAD_TOKEN_PRIVATE_KEY must be an Ed25519 key."); + } + signer = { key, kid }; + return signer; +}; + +// Sign a token of a given purpose (typ) with extra claims. +const signToken = (typ, claims, ttlSeconds = DEFAULT_TTL_SECONDS) => { + const { key, kid } = getSigner(); + const iat = Math.floor(Date.now() / 1000); + const header = { alg: "EdDSA", typ: "JWT", kid }; + const payload = { + iss: ISSUER, + aud: AUDIENCE, + typ, + iat, + exp: iat + ttlSeconds, + jti: crypto.randomUUID(), + ...claims, + }; + const input = `${b64url(header)}.${b64url(payload)}`; + const sig = crypto.sign(null, Buffer.from(input), key).toString("base64url"); + return { + token: `${input}.${sig}`, + expiresAt: new Date(payload.exp * 1000).toISOString(), + }; +}; + +// Token for all tus requests of ONE upload. `len` = exact Upload-Length. +const signUploadToken = ({ + userId, + uploadId, + internalId, + submissionId, + expectedSize, +}) => + signToken("tus-upload", { + sub: String(userId), + uid: uploadId, + iid: internalId, + sid: submissionId, + len: expectedSize, + }); + +// Strict verification (used by Zodiac; also by REN's self-test). +// publicKeys: { [kid]: pem | KeyObject }. Throws on any problem. +const verifyToken = (token, publicKeys, expectedTyp) => { + const parts = String(token || "").split("."); + if (parts.length !== 3) throw new Error("malformed token"); + const [h, p, s] = parts; + const header = JSON.parse(Buffer.from(h, "base64url").toString()); + if (header.alg !== "EdDSA") throw new Error("bad alg"); // never trust other algs + const keyMaterial = publicKeys[header.kid]; + if (!keyMaterial) throw new Error("unknown kid"); + const key = + typeof keyMaterial === "string" + ? crypto.createPublicKey(keyMaterial) + : keyMaterial; + if (key.asymmetricKeyType !== "ed25519") throw new Error("bad key type"); + const ok = crypto.verify( + null, + Buffer.from(`${h}.${p}`), + key, + Buffer.from(s, "base64url") + ); + if (!ok) throw new Error("bad signature"); + const claims = JSON.parse(Buffer.from(p, "base64url").toString()); + const now = Math.floor(Date.now() / 1000); + if (claims.iss !== ISSUER || claims.aud !== AUDIENCE) { + throw new Error("bad issuer/audience"); + } + if (claims.typ !== expectedTyp) throw new Error("wrong token type"); + if ( + typeof claims.exp !== "number" || + claims.exp + CLOCK_LEEWAY_SECONDS < now + ) { + throw new Error("expired"); + } + if ( + typeof claims.iat !== "number" || + claims.iat - CLOCK_LEEWAY_SECONDS > now + ) { + throw new Error("issued in the future"); + } + return claims; +}; + +module.exports = { + signToken, + signUploadToken, + verifyToken, + DEFAULT_TTL_SECONDS, +}; From da6a6494217f59ee89e21dff42a7c0e0e0933774 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Wed, 30 Sep 2026 17:50:36 -0400 Subject: [PATCH 34/45] feat(uploads): add raw ZIP upload session routes; refs #147 --- .../src/controllers/rawUploads.controller.js | 425 ++++++++++++++++++ backend/src/controllers/uploads.controller.js | 61 ++- backend/src/lib/uploadAccess.js | 34 ++ backend/src/routes/uploads.routes.js | 20 + 4 files changed, 505 insertions(+), 35 deletions(-) create mode 100644 backend/src/controllers/rawUploads.controller.js create mode 100644 backend/src/lib/uploadAccess.js diff --git a/backend/src/controllers/rawUploads.controller.js b/backend/src/controllers/rawUploads.controller.js new file mode 100644 index 0000000..bf56859 --- /dev/null +++ b/backend/src/controllers/rawUploads.controller.js @@ -0,0 +1,425 @@ +// Raw ZIP upload sessions (owner side). REN only authorizes and tracks state; +// the bytes go browser → Zodiac storage API via tus with a short-lived token. +// Completion (Zodiac events + raw_objects creation) is handled separately. +const crypto = require("crypto"); +const { sequelize } = require("../config/database"); +const { + HttpError, + UUID_RE, + loadOwnedRegistry, +} = require("../lib/uploadAccess"); +const { signUploadToken } = require("../lib/uploadTokens"); + +const MAX_UPLOAD_BYTES = Number( + process.env.RAW_MAX_UPLOAD_BYTES || 536870912000 +); // 500 GB +const CHUNK_BYTES = Number(process.env.RAW_CHUNK_BYTES || 67108864); // 64 MB +const ACTIVE = ["initiated", "uploading", "verifying"]; +const EDITABLE = ["draft", "changes_requested"]; + +const sendError = (res, err, label) => { + if (err instanceof HttpError) return res.status(err.status).json(err.body); + console.error(`${label} failed:`, err.message); + res.status(500).json({ error: `${label} failed.` }); +}; + +// Storage endpoint the browser uploads to (env only; differs dev vs prod). +const storageEndpoint = () => { + const base = process.env.STORAGE_PUBLIC_URL; + if (!base) { + throw new HttpError(503, { + error: "Raw data upload is not configured on this server.", + }); + } + return `${base.replace(/\/+$/, "")}/files`; +}; + +// Display filename only — never used for paths. Basename, trimmed, .zip. +const cleanFilename = (raw) => { + const name = String(raw || "") + .split(/[\\/]/) + .pop() + .trim(); + if (!name || name.length > 255) { + throw new HttpError(400, { error: "Invalid file name." }); + } + if (!/\.zip$/i.test(name)) { + throw new HttpError(400, { error: "Raw data must be a .zip file." }); + } + return name; +}; + +const cleanSize = (raw) => { + const size = Number(raw); + if (!Number.isSafeInteger(size) || size <= 0) { + throw new HttpError(400, { error: "Invalid file size." }); + } + if (size > MAX_UPLOAD_BYTES) { + throw new HttpError(413, { + code: "FILE_TOO_LARGE", + error: `The file is too large (max ${Math.floor( + MAX_UPLOAD_BYTES / 1e9 + )} GB).`, + }); + } + return size; +}; + +// Latest submission (cycle) of the dataset, locked for this transaction. +const lockLatestSubmission = async (internalId, t) => { + const rows = await sequelize.query( + `SELECT id, submission_id, status FROM submissions + WHERE dataset_registry_id = :iid ORDER BY created_at DESC LIMIT 1 FOR UPDATE`, + { + replacements: { iid: internalId }, + type: sequelize.QueryTypes.SELECT, + transaction: t, + } + ); + if (!rows[0]) throw new HttpError(404, { error: "No submission found." }); + return rows[0]; +}; + +const assertEditable = (sub) => { + if (!EDITABLE.includes(sub.status)) { + throw new HttpError(409, { + code: "NOT_EDITABLE", + error: + "Raw data can only be changed while the submission is a draft or has requested changes.", + }); + } +}; + +const tokenFor = (req, reg, sub, attempt) => + signUploadToken({ + userId: req.user.id, + uploadId: attempt.upload_id, + internalId: reg.internal_id, + submissionId: sub.submission_id, + expectedSize: Number(attempt.expected_size), // BIGINT comes back as a string + }); + +// POST /api/v1/uploads/:internalId/raw {filename, size} +// Start a raw ZIP upload, or resume the active one (same name + size). +const startRawUpload = async (req, res) => { + try { + const filename = cleanFilename(req.body?.filename); + const size = cleanSize(req.body?.size); + const endpoint = storageEndpoint(); + const reg = await loadOwnedRegistry( + String(req.params.internalId), + req.user.id + ); + + const out = await sequelize.transaction(async (t) => { + const sub = await lockLatestSubmission(reg.internal_id, t); + assertEditable(sub); + + const active = await sequelize.query( + `SELECT upload_id, original_filename, expected_size FROM raw_upload_attempts + WHERE submission_id = :sid AND status IN (:active)`, + { + replacements: { sid: sub.submission_id, active: ACTIVE }, + type: sequelize.QueryTypes.SELECT, + transaction: t, + } + ); + if (active[0]) { + const a = active[0]; + if ( + a.original_filename === filename && + Number(a.expected_size) === size + ) { + return { sub, attempt: a, resumed: true }; // same file → resume + } + throw new HttpError(409, { + code: "ACTIVE_UPLOAD_EXISTS", + error: + "Another raw data upload is in progress. Resume it with the same file, or cancel it first.", + upload: { + uploadId: a.upload_id, + filename: a.original_filename, + size: Number(a.expected_size), + }, + }); + } + + const attempt = { + upload_id: crypto.randomUUID(), + original_filename: filename, + expected_size: size, + }; + await sequelize.query( + `INSERT INTO raw_upload_attempts + (upload_id, submission_id, status, original_filename, expected_size, uploaded_by, created_at, updated_at) + VALUES (:uid, :sid, 'initiated', :name, :size, :by, NOW(), NOW())`, + { + replacements: { + uid: attempt.upload_id, + sid: sub.submission_id, + name: filename, + size, + by: req.user.id, + }, + transaction: t, + } + ); + // Starting a raw upload means this version includes raw data. + await sequelize.query( + `UPDATE submissions SET raw_zip_expected = true, updated_at = NOW() WHERE id = :id`, + { replacements: { id: sub.id }, transaction: t } + ); + return { sub, attempt, resumed: false }; + }); + + const { token, expiresAt } = tokenFor(req, reg, out.sub, out.attempt); + res.status(out.resumed ? 200 : 201).json({ + uploadId: out.attempt.upload_id, + endpoint, + token, + expiresAt, + chunkSize: CHUNK_BYTES, + resumed: out.resumed, + }); + } catch (err) { + // Unique index backstop: two starts at the same moment. + if (err.original?.code === "23505") { + return res.status(409).json({ + code: "ACTIVE_UPLOAD_EXISTS", + error: "Another raw data upload is in progress.", + }); + } + sendError(res, err, "Start raw upload"); + } +}; + +// Load an attempt that belongs to this dataset (owner already checked). +const loadAttempt = async (reg, uploadId, t) => { + if (!UUID_RE.test(uploadId)) { + throw new HttpError(404, { error: "Upload not found." }); + } + const rows = await sequelize.query( + `SELECT a.upload_id, a.status, a.expected_size, a.original_filename, + s.submission_id, s.status AS submission_status + FROM raw_upload_attempts a + JOIN submissions s ON s.submission_id = a.submission_id + WHERE a.upload_id = :uid AND s.dataset_registry_id = :iid`, + { + replacements: { uid: uploadId, iid: reg.internal_id }, + type: sequelize.QueryTypes.SELECT, + transaction: t, + } + ); + if (!rows[0]) throw new HttpError(404, { error: "Upload not found." }); + return rows[0]; +}; + +// POST /api/v1/uploads/:internalId/raw/:uploadId/token — fresh token for a +// long-running upload. +const refreshRawUploadToken = async (req, res) => { + try { + const reg = await loadOwnedRegistry( + String(req.params.internalId), + req.user.id + ); + const a = await loadAttempt(reg, String(req.params.uploadId)); + if (!ACTIVE.includes(a.status)) { + throw new HttpError(409, { + code: "UPLOAD_NOT_ACTIVE", + error: "This upload is no longer active.", + }); + } + if (!EDITABLE.includes(a.submission_status)) { + throw new HttpError(409, { + code: "NOT_EDITABLE", + error: "The submission can no longer be changed.", + }); + } + const { token, expiresAt } = tokenFor( + req, + reg, + { submission_id: a.submission_id }, + a + ); + res.json({ token, expiresAt }); + } catch (err) { + sendError(res, err, "Refresh upload token"); + } +}; + +// GET /api/v1/uploads/:internalId/raw — current raw object + active/last attempt. +const getRawStatus = async (req, res) => { + try { + const reg = await loadOwnedRegistry( + String(req.params.internalId), + req.user.id + ); + const subs = await sequelize.query( + `SELECT submission_id, status, raw_zip_expected FROM submissions + WHERE dataset_registry_id = :iid ORDER BY created_at DESC LIMIT 1`, + { replacements: { iid: reg.internal_id }, type: sequelize.QueryTypes.SELECT } + ); + const sub = subs[0]; + if (!sub) { + return res.json({ + rawZipExpected: false, + editable: false, + current: null, + activeUpload: null, + lastUpload: null, + }); + } + + const current = await sequelize.query( + `SELECT o.raw_id, o.original_filename, o.size_bytes, o.sha256, + o.extraction_status, o.created_at, + -- carried forward = the upload that made this object belongs + -- to a different version (exact; not a timestamp guess) + (a.submission_id IS NOT NULL AND a.submission_id <> f.submission_id) + AS carried_forward + FROM submission_raw_files f + JOIN raw_objects o ON o.raw_id = f.raw_id + LEFT JOIN raw_upload_attempts a ON a.raw_id = o.raw_id + WHERE f.submission_id = :sid`, + { + replacements: { sid: sub.submission_id }, + type: sequelize.QueryTypes.SELECT, + } + ); + const attempts = await sequelize.query( + `SELECT upload_id, status, original_filename, expected_size, error, + created_at, updated_at, completed_at + FROM raw_upload_attempts WHERE submission_id = :sid + ORDER BY created_at DESC LIMIT 1`, + { + replacements: { sid: sub.submission_id }, + type: sequelize.QueryTypes.SELECT, + } + ); + const last = attempts[0] || null; + const shape = (a) => + a && { + uploadId: a.upload_id, + status: a.status, + filename: a.original_filename, + size: Number(a.expected_size), + error: a.error || null, + createdAt: a.created_at, + updatedAt: a.updated_at, + completedAt: a.completed_at, + }; + const c = current[0]; + res.json({ + rawZipExpected: !!sub.raw_zip_expected, + editable: EDITABLE.includes(sub.status), + // Storage keys stay server-side; only display facts are returned. + current: c + ? { + rawId: c.raw_id, + filename: c.original_filename, + size: Number(c.size_bytes), + sha256: c.sha256, + extractionStatus: c.extraction_status, + carriedForward: !!c.carried_forward, + uploadedAt: c.created_at, + } + : null, + activeUpload: last && ACTIVE.includes(last.status) ? shape(last) : null, + // "Last upload failed/cancelled/expired" (the current ZIP is untouched). + lastUpload: + last && ["failed", "cancelled", "expired"].includes(last.status) + ? shape(last) + : null, + }); + } catch (err) { + sendError(res, err, "Load raw data status"); + } +}; + +// DELETE /api/v1/uploads/:internalId/raw/:uploadId — cancel an active upload. +// One conditional UPDATE, so it can't race the completion step: if REN has +// already committed it, the row is 'complete' and this returns 409. +const cancelRawUpload = async (req, res) => { + try { + const reg = await loadOwnedRegistry( + String(req.params.internalId), + req.user.id + ); + const a = await loadAttempt(reg, String(req.params.uploadId)); + const rows = await sequelize.query( + `UPDATE raw_upload_attempts + SET status = 'cancelled', completed_at = NOW(), updated_at = NOW() + WHERE upload_id = :uid AND status IN (:active) + RETURNING upload_id`, + { + replacements: { uid: a.upload_id, active: ACTIVE }, + type: sequelize.QueryTypes.SELECT, + } + ); + if (!rows.length) { + throw new HttpError(409, { + code: "UPLOAD_NOT_ACTIVE", + error: "This upload has already finished.", + }); + } + // Zodiac's partial data is removed by the reconciliation/cleanup step; + // the browser also sends a tus DELETE when the user cancels. + res.json({ status: "cancelled" }); + } catch (err) { + sendError(res, err, "Cancel raw upload"); + } +}; + +// PATCH /api/v1/uploads/:internalId/raw/expected {expected: boolean} +// "This dataset includes raw data". Can only be turned off when there is no +// raw ZIP and no active upload. +const setRawExpected = async (req, res) => { + try { + if (typeof req.body?.expected !== "boolean") { + throw new HttpError(400, { error: "`expected` must be true or false." }); + } + const expected = req.body.expected; + const reg = await loadOwnedRegistry( + String(req.params.internalId), + req.user.id + ); + await sequelize.transaction(async (t) => { + const sub = await lockLatestSubmission(reg.internal_id, t); + assertEditable(sub); + if (!expected) { + const [{ busy }] = await sequelize.query( + `SELECT (EXISTS (SELECT 1 FROM submission_raw_files WHERE submission_id = :sid) + OR EXISTS (SELECT 1 FROM raw_upload_attempts + WHERE submission_id = :sid AND status IN (:active))) AS busy`, + { + replacements: { sid: sub.submission_id, active: ACTIVE }, + type: sequelize.QueryTypes.SELECT, + transaction: t, + } + ); + if (busy) { + throw new HttpError(409, { + code: "RAW_DATA_PRESENT", + error: + "This version already has raw data (or an upload in progress), so it can't be unmarked.", + }); + } + } + await sequelize.query( + `UPDATE submissions SET raw_zip_expected = :e, updated_at = NOW() WHERE id = :id`, + { replacements: { e: expected, id: sub.id }, transaction: t } + ); + }); + res.json({ rawZipExpected: expected }); + } catch (err) { + sendError(res, err, "Update raw data setting"); + } +}; + +module.exports = { + startRawUpload, + refreshRawUploadToken, + getRawStatus, + cancelRawUpload, + setRawExpected, +}; diff --git a/backend/src/controllers/uploads.controller.js b/backend/src/controllers/uploads.controller.js index db3fefe..48fe949 100644 --- a/backend/src/controllers/uploads.controller.js +++ b/backend/src/controllers/uploads.controller.js @@ -1,6 +1,7 @@ const axios = require("axios"); const crypto = require("crypto"); const { sequelize } = require("../config/database"); +const { HttpError, loadOwnedRegistry } = require("../lib/uploadAccess"); // Upload a JSON dataset into the sandbox CouchDB db for review, tracked in // PostgreSQL. See migrations create-dataset-registry / create-submissions / @@ -28,15 +29,6 @@ const buildHeaders = () => { return headers; }; -// Errors thrown inside the transaction to signal a specific HTTP response. -class HttpError extends Error { - constructor(status, body) { - super(typeof body === "string" ? body : body.error || "error"); - this.status = status; - this.body = typeof body === "string" ? { error: body } : body; - } -} - // Preferred public id rules (final availability is re-checked at promotion). const REQUESTED_ID_RE = /^[a-z0-9][a-z0-9_-]{2,62}$/; const validateRequestedId = (id) => { @@ -424,27 +416,6 @@ const listMyUploads = async (req, res) => { } }; -const UUID_RE = - /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; - -// Load a registry row the caller owns, or throw (404 unknown, 403 not owner). -// Guards the UUID format first so a bad param can't crash the SQL uuid cast. -const loadOwnedRegistry = async (internalId, userId) => { - if (!UUID_RE.test(internalId)) { - throw new HttpError(404, { error: "Dataset not found" }); - } - const rows = await sequelize.query( - `SELECT internal_id, dataset_id, requested_dataset_id, requested_db, dataset_name, owner_user_id - FROM dataset_registry WHERE internal_id = :iid`, - { replacements: { iid: internalId }, type: sequelize.QueryTypes.SELECT } - ); - if (rows.length === 0) throw new HttpError(404, { error: "Dataset not found" }); - if (rows[0].owner_user_id !== userId) { - throw new HttpError(403, { error: "This dataset belongs to another user" }); - } - return rows[0]; -}; - const latestSubmissionId = async (internalId) => { const s = await sequelize.query( `SELECT submission_id FROM submissions @@ -454,17 +425,36 @@ const latestSubmissionId = async (internalId) => { return s[0] ? s[0].submission_id : null; }; -// What must be true before draft/changes_requested → pending. -// (Raw ZIP checks are added in step 2, once the raw tables exist; until then -// raw_zip_expected is always false.) +// What must be true before draft/changes_requested → pending: the JSON is +// uploaded, no raw upload is in progress, and — if this version is marked as +// including raw data — a raw object is attached. `sub` needs the +// RAW_READINESS_COLUMNS below (has_raw, raw_upload_active). const getReadiness = (sub) => { const problems = []; if (sub.json_status !== "uploaded") { problems.push("The last JSON update failed. Upload the JSON again."); } + if (sub.raw_upload_active) { + problems.push( + "A raw data upload is still in progress. Wait for it to finish, or cancel it." + ); + } + if (sub.raw_zip_expected && !sub.has_raw) { + problems.push( + "This version is marked as including raw data, but no raw ZIP has been uploaded yet." + ); + } return { canSubmit: problems.length === 0, problems }; }; +// Extra readiness columns for a `FROM submissions` query (raw ZIP state). +const RAW_READINESS_COLUMNS = ` + EXISTS (SELECT 1 FROM submission_raw_files f + WHERE f.submission_id = submissions.submission_id) AS has_raw, + EXISTS (SELECT 1 FROM raw_upload_attempts a + WHERE a.submission_id = submissions.submission_id + AND a.status IN ('initiated','uploading','verifying')) AS raw_upload_active`; + // POST /api/v1/uploads/:internalId/submit — send (or resend) to review. const submitForReview = async (req, res) => { try { @@ -474,7 +464,8 @@ const submitForReview = async (req, res) => { ); const out = await sequelize.transaction(async (t) => { const rows = await sequelize.query( - `SELECT id, status, json_status, raw_zip_expected FROM submissions + `SELECT id, status, json_status, raw_zip_expected, ${RAW_READINESS_COLUMNS} + FROM submissions WHERE dataset_registry_id = :iid ORDER BY created_at DESC LIMIT 1 FOR UPDATE`, { @@ -650,7 +641,7 @@ const getUpload = async (req, res) => { const s = await sequelize.query( `SELECT submission_id, status, created_at, updated_at, promoted_db, promoted_at, json_status, json_uploaded_at, json_error, submitted_at, raw_zip_expected, - reviewed_at + reviewed_at, ${RAW_READINESS_COLUMNS} FROM submissions WHERE dataset_registry_id = :iid ORDER BY created_at DESC LIMIT 1`, { replacements: { iid: reg.internal_id }, type: sequelize.QueryTypes.SELECT } diff --git a/backend/src/lib/uploadAccess.js b/backend/src/lib/uploadAccess.js new file mode 100644 index 0000000..6ce8c46 --- /dev/null +++ b/backend/src/lib/uploadAccess.js @@ -0,0 +1,34 @@ +// Shared by uploads.controller.js and rawUploads.controller.js. +const { sequelize } = require("../config/database"); + +// Errors thrown inside handlers/transactions to signal a specific HTTP response. +class HttpError extends Error { + constructor(status, body) { + super(typeof body === "string" ? body : body.error || "error"); + this.status = status; + this.body = typeof body === "string" ? { error: body } : body; + } +} + +const UUID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +// Load a registry row the caller owns, or throw (404 unknown, 403 not owner). +// Guards the UUID format first so a bad param can't crash the SQL uuid cast. +const loadOwnedRegistry = async (internalId, userId) => { + if (!UUID_RE.test(internalId)) { + throw new HttpError(404, { error: "Dataset not found" }); + } + const rows = await sequelize.query( + `SELECT internal_id, dataset_id, requested_dataset_id, requested_db, dataset_name, owner_user_id + FROM dataset_registry WHERE internal_id = :iid`, + { replacements: { iid: internalId }, type: sequelize.QueryTypes.SELECT } + ); + if (rows.length === 0) throw new HttpError(404, { error: "Dataset not found" }); + if (rows[0].owner_user_id !== userId) { + throw new HttpError(403, { error: "This dataset belongs to another user" }); + } + return rows[0]; +}; + +module.exports = { HttpError, UUID_RE, loadOwnedRegistry }; diff --git a/backend/src/routes/uploads.routes.js b/backend/src/routes/uploads.routes.js index f15f77f..a59ab39 100644 --- a/backend/src/routes/uploads.routes.js +++ b/backend/src/routes/uploads.routes.js @@ -12,6 +12,13 @@ const { listComments, postComment, } = require("../controllers/uploads.controller"); +const { + startRawUpload, + refreshRawUploadToken, + getRawStatus, + cancelRawUpload, + setRawExpected, +} = require("../controllers/rawUploads.controller"); const router = express.Router(); @@ -51,4 +58,17 @@ router.patch("/:internalId/settings", requireAuth, updateSettings); router.get("/:internalId/comments", requireAuth, listComments); router.post("/:internalId/comments", requireAuth, postComment); +// Raw ZIP upload sessions (owner only). The file itself goes straight to +// Zodiac; REN only authorizes and tracks state. Starting shares the daily +// upload limit; token refreshes aren't limited (long uploads need several). +router.post("/:internalId/raw", requireAuth, uploadRateLimit, startRawUpload); +router.get("/:internalId/raw", requireAuth, getRawStatus); +router.patch("/:internalId/raw/expected", requireAuth, setRawExpected); +router.post( + "/:internalId/raw/:uploadId/token", + requireAuth, + refreshRawUploadToken +); +router.delete("/:internalId/raw/:uploadId", requireAuth, cancelRawUpload); + module.exports = router; From 508732bdff079c639d919ebb0bb53457009b0908 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Thu, 1 Oct 2026 11:24:41 -0400 Subject: [PATCH 35/45] feat(uploads): Zodiac storage events and raw upload completion; refs #147 --- backend/scripts/send-storage-event.js | 32 +++ .../controllers/storageEvents.controller.js | 225 ++++++++++++++++++ .../src/middleware/verifyStorageSignature.js | 25 ++ backend/src/routes/internal.routes.js | 26 ++ backend/src/server.js | 5 + 5 files changed, 313 insertions(+) create mode 100644 backend/scripts/send-storage-event.js create mode 100644 backend/src/controllers/storageEvents.controller.js create mode 100644 backend/src/middleware/verifyStorageSignature.js create mode 100644 backend/src/routes/internal.routes.js diff --git a/backend/scripts/send-storage-event.js b/backend/scripts/send-storage-event.js new file mode 100644 index 0000000..e6e51b3 --- /dev/null +++ b/backend/scripts/send-storage-event.js @@ -0,0 +1,32 @@ +// Send a signed storage event to the local backend, as Zodiac would (testing). +// Usage: node scripts/send-storage-event.js [size] [sha256|reason] +require("dotenv").config({ quiet: true }); +const { signRequest } = require("../src/lib/storageSignature"); + +const [uploadId, event, size, extra] = process.argv.slice(2); +if (!uploadId || !event) { + console.log( + "usage: node scripts/send-storage-event.js [size] [sha256|reason]" + ); + process.exit(1); +} +const payload = { event }; +if (event === "verified" || event === "committed") { + Object.assign(payload, { size: Number(size), sha256: extra }); +} +if (event === "failed") payload.reason = size || "storage_error"; + +const base = process.env.MAIN_API_LOCAL_URL || "http://localhost:5000"; +const path = `/api/v1/internal/storage/uploads/${uploadId}/events`; +const body = JSON.stringify(payload); + +fetch(base + path, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...signRequest({ method: "POST", path, body }), + }, + body, +}) + .then(async (r) => console.log(r.status, await r.text())) + .catch((e) => console.log("request failed:", e.message)); diff --git a/backend/src/controllers/storageEvents.controller.js b/backend/src/controllers/storageEvents.controller.js new file mode 100644 index 0000000..e5a15fb --- /dev/null +++ b/backend/src/controllers/storageEvents.controller.js @@ -0,0 +1,225 @@ +// Events from the Zodiac storage API about one raw ZIP upload. Idempotent and +// forward-only: Zodiac may resend any event (outbox retries, reconciliation). +// started → initiated → uploading +// verified → REN decides commit | discard +// committed → one transaction: raw_objects + attempt complete + mapping +// failed → attempt failed (safe message) +// raw_id = upload_id (deterministic, so a repeated "verified" gets the same +// answer and Zodiac always moves the file to the same path). +const { sequelize } = require("../config/database"); +const { HttpError, UUID_RE } = require("../lib/uploadAccess"); + +const ACTIVE = ["initiated", "uploading", "verifying"]; +const EDITABLE = ["draft", "changes_requested"]; +const SHA_RE = /^[0-9a-f]{64}$/; + +// Zodiac sends a reason code; users only ever see these fixed sentences. +const FAILURE_MESSAGES = { + not_zip: "The file is not a ZIP archive.", + size_mismatch: "The uploaded file size did not match the expected size.", + storage_error: "The storage server could not save the file.", +}; +const failureMessage = (code) => + FAILURE_MESSAGES[code] || "The upload could not be completed."; + +const zipKey = (internalId, rawId) => `${internalId}/raw/${rawId}/source.zip`; + +// Lock the attempt row and read its submission + dataset. +const lockAttempt = async (uploadId, t) => { + const rows = await sequelize.query( + `SELECT a.upload_id, a.status, a.expected_size, a.original_filename, a.raw_id, + s.id AS submission_pk, s.submission_id, s.status AS submission_status, + s.dataset_registry_id + FROM raw_upload_attempts a + JOIN submissions s ON s.submission_id = a.submission_id + WHERE a.upload_id = :uid + FOR UPDATE OF a`, + { + replacements: { uid: uploadId }, + type: sequelize.QueryTypes.SELECT, + transaction: t, + } + ); + if (!rows[0]) throw new HttpError(404, { error: "Unknown upload." }); + return rows[0]; +}; + +const markFailed = (uploadId, message, t) => + sequelize.query( + `UPDATE raw_upload_attempts + SET status = 'failed', error = :msg, completed_at = NOW(), updated_at = NOW() + WHERE upload_id = :uid AND status IN (:active)`, + { + replacements: { uid: uploadId, msg: message, active: ACTIVE }, + transaction: t, + } + ); + +// size + sha256 sent by Zodiac must match what this upload expected. +const checkFile = (a, body) => { + const size = Number(body.size); + if (!Number.isSafeInteger(size) || !SHA_RE.test(String(body.sha256 || ""))) { + throw new HttpError(400, { error: "size and sha256 are required." }); + } + return { + size, + sha256: body.sha256, + sizeOk: size === Number(a.expected_size), + }; +}; + +const onStarted = async (a, body, t) => { + await sequelize.query( + `UPDATE raw_upload_attempts SET status = 'uploading', updated_at = NOW() + WHERE upload_id = :uid AND status = 'initiated'`, + { replacements: { uid: a.upload_id }, transaction: t } + ); + return { ok: true }; +}; + +const onVerified = async (a, body, t) => { + if (a.status === "complete") { + // Already committed earlier — repeat the same answer. + return { + decision: "commit", + rawId: a.raw_id, + storageKey: zipKey(a.dataset_registry_id, a.raw_id), + }; + } + if (!ACTIVE.includes(a.status)) { + // cancelled / failed / expired + return { decision: "discard", reason: `upload is ${a.status}` }; + } + const { sizeOk } = checkFile(a, body); + if (!sizeOk) { + await markFailed(a.upload_id, failureMessage("size_mismatch"), t); + return { decision: "discard", reason: "size_mismatch" }; + } + if (!EDITABLE.includes(a.submission_status)) { + await markFailed( + a.upload_id, + "The submission was locked before the upload finished.", + t + ); + return { decision: "discard", reason: "submission_locked" }; + } + await sequelize.query( + `UPDATE raw_upload_attempts SET status = 'verifying', updated_at = NOW() + WHERE upload_id = :uid AND status IN ('initiated','uploading')`, + { replacements: { uid: a.upload_id }, transaction: t } + ); + const rawId = a.upload_id; // deterministic: same answer every time + return { + decision: "commit", + rawId, + storageKey: zipKey(a.dataset_registry_id, rawId), + }; +}; + +const onCommitted = async (a, body, t) => { + const { size, sha256, sizeOk } = checkFile(a, body); + const rawId = a.upload_id; + + if (a.status === "complete") { + // Repeat: confirm it's the same file, change nothing. + const [obj] = await sequelize.query( + `SELECT sha256 FROM raw_objects WHERE raw_id = :rid`, + { + replacements: { rid: a.raw_id }, + type: sequelize.QueryTypes.SELECT, + transaction: t, + } + ); + if (obj && obj.sha256 !== sha256) { + throw new HttpError(409, { + error: "Upload already completed with a different file.", + }); + } + return { action: "keep" }; + } + if (!ACTIVE.includes(a.status)) { + // e.g. the user cancelled after REN said "commit" — Zodiac deletes the file. + return { action: "discard", reason: `upload is ${a.status}` }; + } + if (!sizeOk) { + await markFailed(a.upload_id, failureMessage("size_mismatch"), t); + return { action: "discard", reason: "size_mismatch" }; + } + + // ONE transaction: the object, the finished attempt, and the version mapping. + await sequelize.query( + `INSERT INTO raw_objects + (raw_id, dataset_registry_id, zip_storage_key, original_filename, + size_bytes, sha256, created_at, updated_at) + VALUES (:rid, :iid, :key, :name, :size, :sha, NOW(), NOW())`, + { + replacements: { + rid: rawId, + iid: a.dataset_registry_id, + key: zipKey(a.dataset_registry_id, rawId), + name: a.original_filename, + size, + sha: sha256, + }, + transaction: t, + } + ); + await sequelize.query( + `UPDATE raw_upload_attempts + SET status = 'complete', raw_id = :rid, error = NULL, + completed_at = NOW(), updated_at = NOW() + WHERE upload_id = :uid`, + { replacements: { rid: rawId, uid: a.upload_id }, transaction: t } + ); + // Replace (or create) this version's raw ZIP. The previous object, if any, + // becomes unmapped and is left for the cleanup job. + await sequelize.query( + `INSERT INTO submission_raw_files (submission_id, raw_id, created_at) + VALUES (:sid, :rid, NOW()) + ON CONFLICT (submission_id) DO UPDATE SET raw_id = EXCLUDED.raw_id, created_at = NOW()`, + { replacements: { sid: a.submission_id, rid: rawId }, transaction: t } + ); + await sequelize.query( + `UPDATE submissions SET raw_zip_expected = true, updated_at = NOW() WHERE id = :id`, + { replacements: { id: a.submission_pk }, transaction: t } + ); + return { action: "keep" }; +}; + +const onFailed = async (a, body, t) => { + await markFailed(a.upload_id, failureMessage(String(body.reason || "")), t); + return { ok: true }; +}; + +const HANDLERS = { + started: onStarted, + verified: onVerified, + committed: onCommitted, + failed: onFailed, +}; + +// POST /api/v1/internal/storage/uploads/:uploadId/events {event, ...} +const handleUploadEvent = async (req, res) => { + try { + const uploadId = String(req.params.uploadId); + if (!UUID_RE.test(uploadId)) { + throw new HttpError(404, { error: "Unknown upload." }); + } + const body = req.body || {}; + const handler = HANDLERS[body.event]; + if (!handler) throw new HttpError(400, { error: "Unknown event." }); + + const out = await sequelize.transaction(async (t) => { + const a = await lockAttempt(uploadId, t); + return handler(a, body, t); + }); + res.json(out); + } catch (err) { + if (err instanceof HttpError) return res.status(err.status).json(err.body); + console.error("Storage event failed:", err.message); + // 5xx → Zodiac's outbox retries later. + res.status(500).json({ error: "Storage event failed." }); + } +}; + +module.exports = { handleUploadEvent }; diff --git a/backend/src/middleware/verifyStorageSignature.js b/backend/src/middleware/verifyStorageSignature.js new file mode 100644 index 0000000..28e4d1e --- /dev/null +++ b/backend/src/middleware/verifyStorageSignature.js @@ -0,0 +1,25 @@ +// Only the Zodiac storage API may call /api/v1/internal/*: every request must +// carry a valid HMAC signature over method + path + raw body (see +// src/lib/storageSignature.js). Needs req.rawBody (set by the internal router). +const { verifyRequest } = require("../lib/storageSignature"); + +const verifyStorageSignature = (req, res, next) => { + const result = verifyRequest({ + method: req.method, + path: req.originalUrl, // includes the query string exactly as sent + headers: req.headers, + rawBody: req.rawBody || "", + }); + if (!result.ok) { + console.warn( + "Rejected internal storage call:", + result.reason, + req.method, + req.originalUrl + ); + return res.status(401).json({ error: "Invalid service signature." }); + } + next(); +}; + +module.exports = { verifyStorageSignature }; diff --git a/backend/src/routes/internal.routes.js b/backend/src/routes/internal.routes.js new file mode 100644 index 0000000..371ee67 --- /dev/null +++ b/backend/src/routes/internal.routes.js @@ -0,0 +1,26 @@ +// Server-to-server routes called by the Zodiac storage API (never the browser). +// Mounted in server.js BEFORE the global express.json(), so this router can +// keep the raw body for the HMAC check. +const express = require("express"); +const { + verifyStorageSignature, +} = require("../middleware/verifyStorageSignature"); +const { + handleUploadEvent, +} = require("../controllers/storageEvents.controller"); + +const router = express.Router(); + +router.use( + express.json({ + limit: "1mb", + verify: (req, res, buf) => { + req.rawBody = buf; + }, + }) +); +router.use(verifyStorageSignature); + +router.post("/storage/uploads/:uploadId/events", handleUploadEvent); + +module.exports = router; diff --git a/backend/src/server.js b/backend/src/server.js index f7dcb5f..df6f340 100644 --- a/backend/src/server.js +++ b/backend/src/server.js @@ -16,6 +16,7 @@ const projectRoutes = require("./routes/projects.routes"); const ollamaRoutes = require("./routes/ollama.routes"); const ollamaPublicRoutes = require("./routes/ollama.public.routes"); const uploadsRoutes = require("./routes/uploads.routes"); +const internalRoutes = require("./routes/internal.routes"); const app = express(); const PORT = process.env.PORT || 5000; @@ -36,6 +37,10 @@ app.use( }) ); +// Zodiac → REN service calls (HMAC-signed). Mounted BEFORE express.json() so +// the internal router can read the raw body; no cookies/restoreUser needed. +app.use("/api/v1/internal", internalRoutes); + app.use(express.json({ limit: "50mb" })); app.use(express.urlencoded({ limit: "50mb", extended: true })); app.use(cookieParser()); // parse cookies From 5ec8a4752bbcbd202d73051c7c9fb28858a1b309 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Fri, 2 Oct 2026 10:01:13 -0400 Subject: [PATCH 36/45] feat(storage-api): skeleton for the Zodiac storage service; refs #147 --- .gitignore | 6 +- storage-api/.env.example | 22 + storage-api/package-lock.json | 1091 +++++++++++++++++++++++ storage-api/package.json | 21 + storage-api/scripts/self-test.js | 113 +++ storage-api/src/config.js | 101 +++ storage-api/src/lib/storageSignature.js | 59 ++ storage-api/src/lib/uploadTokens.js | 50 ++ storage-api/src/server.js | 36 + 9 files changed, 1498 insertions(+), 1 deletion(-) create mode 100644 storage-api/.env.example create mode 100644 storage-api/package-lock.json create mode 100644 storage-api/package.json create mode 100644 storage-api/scripts/self-test.js create mode 100644 storage-api/src/config.js create mode 100644 storage-api/src/lib/storageSignature.js create mode 100644 storage-api/src/lib/uploadTokens.js create mode 100644 storage-api/src/server.js diff --git a/.gitignore b/.gitignore index 3e3dfe2..32a3aae 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,8 @@ backend/.env.local backend/.env.*.local backend/*.sqlite backend/*.db -!backend/package-lock.json \ No newline at end of file +!backend/package-lock.json + +#storage-api +storage-api/storage-dev/ +!storage-api/package-lock.json \ No newline at end of file diff --git a/storage-api/.env.example b/storage-api/.env.example new file mode 100644 index 0000000..ed300cd --- /dev/null +++ b/storage-api/.env.example @@ -0,0 +1,22 @@ +# Where Node listens (Apache proxies to it in prod) +HOST=127.0.0.1 +PORT=8090 +BASE_PATH=/neurojson-storage + +# Storage root. Dev on the Mac: ./storage-dev ; Zodiac: /local_mount/space/zodiac/1/users/neurojson/uploads +UPLOAD_ROOT=./storage-dev + +# Browser origins allowed to upload (comma separated) +ALLOWED_ORIGINS=http://localhost:3000 + +# REN main API for pushes. EMPTY = pushes off (dev: REN pulls instead). +MAIN_API_URL= + +# From the "Zodiac" block of backend/scripts/generate-upload-keys.js (same run as REN's keys) +UPLOAD_TOKEN_PUBLIC_KEYS='{"dev-2026-09":"-----BEGIN PUBLIC KEY-----\n...\n-----END PUBLIC KEY-----"}' +STORAGE_HMAC_ACTIVE_KID=dev-2026-09 +STORAGE_HMAC_KEYS='{"dev-2026-09":"..."}' + +# Limits +MAX_UPLOAD_BYTES=536870912000 +DISK_RESERVE_PERCENT=5 diff --git a/storage-api/package-lock.json b/storage-api/package-lock.json new file mode 100644 index 0000000..361874d --- /dev/null +++ b/storage-api/package-lock.json @@ -0,0 +1,1091 @@ +{ + "name": "neurojson-storage-api", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "neurojson-storage-api", + "version": "0.1.0", + "dependencies": { + "@tus/file-store": "^2.1.1", + "@tus/server": "^2.4.5", + "dotenv": "^17.2.3", + "express": "^5.1.0" + }, + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/@ioredis/commands": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.10.0.tgz", + "integrity": "sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q==", + "license": "MIT", + "optional": true + }, + "node_modules/@redis/client": { + "version": "5.12.1", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-5.12.1.tgz", + "integrity": "sha512-7aPGWeqA3uFm43o19umzdl16CEjK/JQGtSXVPevplTaOU3VJA/rseBC1QvYUz9lLDIMBimc4SW/zrW4S89BaCA==", + "license": "MIT", + "optional": true, + "dependencies": { + "cluster-key-slot": "1.1.2" + }, + "engines": { + "node": ">= 18.19.0" + }, + "peerDependencies": { + "@node-rs/xxhash": "^1.1.0", + "@opentelemetry/api": ">=1 <2" + }, + "peerDependenciesMeta": { + "@node-rs/xxhash": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + } + } + }, + "node_modules/@tus/file-store": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@tus/file-store/-/file-store-2.1.1.tgz", + "integrity": "sha512-/uXV3Ibb4Upirrsj7lSrX+o08H7IKYEkPHV8TKAPPK6CYUcOMTilluOkwPrb9JZydLSYlIBQkhZZVS9sPEfDwQ==", + "license": "MIT", + "dependencies": { + "@tus/utils": "^0.7.1", + "debug": "^4.3.4" + }, + "engines": { + "node": ">=20.19.0" + }, + "optionalDependencies": { + "@redis/client": "^5.0.0" + } + }, + "node_modules/@tus/server": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/@tus/server/-/server-2.4.5.tgz", + "integrity": "sha512-DaQY5F1/JC1E+eDNY+Q5L08wgQ8NHFDNW/E6NclSx10BOHJk85BT4mSId49Ep890Utsk+6hEEuhNFb/LH+Og0w==", + "license": "MIT", + "dependencies": { + "@tus/utils": "^0.7.1", + "debug": "^4.3.4", + "lodash.throttle": "^4.1.1", + "set-cookie-parser": "^2.7.1", + "srvx": "~0.11.15" + }, + "engines": { + "node": ">=20.19.0" + }, + "optionalDependencies": { + "@redis/client": "^5.0.0", + "ioredis": "^5.4.1" + } + }, + "node_modules/@tus/utils": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/@tus/utils/-/utils-0.7.1.tgz", + "integrity": "sha512-KXKK6mhsRto7cApRtJFoIovd6rAXJBUd1qR6S1ynVXApdd+jT1eBHprKP0D9oZtJwwD/B8+n3DLAm68DYPqfuQ==", + "license": "MIT", + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/body-parser": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/dotenv": { + "version": "17.4.2", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.2.tgz", + "integrity": "sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ioredis": { + "version": "5.11.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.11.1.tgz", + "integrity": "sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A==", + "license": "MIT", + "optional": true, + "dependencies": { + "@ioredis/commands": "1.10.0", + "cluster-key-slot": "1.1.1", + "debug": "4.4.3", + "denque": "2.1.0", + "redis-errors": "1.2.0", + "redis-parser": "3.0.0", + "standard-as-callback": "2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/ioredis/node_modules/cluster-key-slot": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.1.tgz", + "integrity": "sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/lodash.throttle": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.throttle/-/lodash.throttle-4.1.1.tgz", + "integrity": "sha512-wIkUCfVKpVsWo3JSZlc+8MB5it+2AN5W8J7YVMST30UrvcQNZ1Okbj+rbVniijTWE6FGYy4XJq/rHkas8qJMLQ==", + "license": "MIT" + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz", + "integrity": "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.1.0.tgz", + "integrity": "sha512-NMPBRMJgiQHjbd8phG3Vebdx4kZ1H121rbl5IkMqeOsahptB9BKo/d7oJ3zTXqTgagn2bWlNSXkh0QUGM31RYg==", + "license": "MIT", + "dependencies": { + "content-type": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/negotiator/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.8.tgz", + "integrity": "sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/qs": { + "version": "6.16.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz", + "integrity": "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==", + "license": "BSD-3-Clause", + "dependencies": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", + "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "optional": true, + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/set-cookie-parser": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.2.tgz", + "integrity": "sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==", + "license": "MIT" + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/srvx": { + "version": "0.11.22", + "resolved": "https://registry.npmjs.org/srvx/-/srvx-0.11.22.tgz", + "integrity": "sha512-LqZxxBDMKuMAZzFzJnDCkFOrs9MZQZr0LvHiO/SuSZVdQaXD7xQ5UWTUxheJrQPve1qk9MG2B/yttUvJxw8egQ==", + "license": "MIT", + "bin": { + "srvx": "bin/srvx.mjs" + }, + "engines": { + "node": ">=20.16.0" + } + }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT", + "optional": true + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + } + } +} diff --git a/storage-api/package.json b/storage-api/package.json new file mode 100644 index 0000000..8f0a5ec --- /dev/null +++ b/storage-api/package.json @@ -0,0 +1,21 @@ +{ + "name": "neurojson-storage-api", + "version": "0.1.0", + "private": true, + "description": "Zodiac storage service for NeuroJSON raw uploads (tus).", + "type": "module", + "engines": { + "node": ">=20.19.0" + }, + "scripts": { + "start": "node src/server.js", + "dev": "node --watch src/server.js", + "self-test": "node scripts/self-test.js" + }, + "dependencies": { + "@tus/file-store": "^2.1.1", + "@tus/server": "^2.4.5", + "dotenv": "^17.2.3", + "express": "^5.1.0" + } +} diff --git a/storage-api/scripts/self-test.js b/storage-api/scripts/self-test.js new file mode 100644 index 0000000..0b5afee --- /dev/null +++ b/storage-api/scripts/self-test.js @@ -0,0 +1,113 @@ +// Self-test for the storage API's token + HMAC helpers and config (prints no +// secrets). Usage: npm run self-test (reads storage-api/.env) +import crypto from "node:crypto"; +import { config } from "../src/config.js"; // also validates .env (exits if wrong) +import { verifyToken } from "../src/lib/uploadTokens.js"; +import { signRequest, verifyRequest } from "../src/lib/storageSignature.js"; + +let failed = 0; +const check = (name, fn) => { + try { + fn(); + console.log("PASS", name); + } catch (e) { + failed++; + console.log("FAIL", name, "-", e.message); + } +}; +const expectThrow = (fn, msg) => { + try { + fn(); + } catch { + return; + } + throw new Error(`expected failure: ${msg}`); +}; + +// --- Tokens: sign with a THROWAWAY key in the same format REN uses --------- +const { privateKey, publicKey } = crypto.generateKeyPairSync("ed25519"); +const pub = { "test-kid": publicKey }; +const b64 = (v) => Buffer.from(JSON.stringify(v)).toString("base64url"); +const makeToken = (claims, ttl = 900) => { + const iat = Math.floor(Date.now() / 1000); + const header = { alg: "EdDSA", typ: "JWT", kid: "test-kid" }; + const payload = { + iss: "neurojson-api", + aud: "zodiac-storage", + typ: "tus-upload", + iat, + exp: iat + ttl, + ...claims, + }; + const input = `${b64(header)}.${b64(payload)}`; + const sig = crypto.sign(null, Buffer.from(input), privateKey); + return `${input}.${sig.toString("base64url")}`; +}; +const token = makeToken({ uid: crypto.randomUUID(), len: 1234 }); + +check("valid token verifies", () => { + if (verifyToken(token, pub, "tus-upload").len !== 1234) { + throw new Error("len claim wrong"); + } +}); +check("tampered payload is rejected", () => { + const [h, p, s] = token.split("."); + const bad = JSON.parse(Buffer.from(p, "base64url").toString()); + bad.len = 999999; + expectThrow( + () => verifyToken(`${h}.${b64(bad)}.${s}`, pub, "tus-upload"), + "tamper" + ); +}); +check("wrong token type is rejected", () => + expectThrow(() => verifyToken(token, pub, "file-read"), "typ") +); +check("unknown kid is rejected", () => + expectThrow(() => verifyToken(token, {}, "tus-upload"), "kid") +); +check("expired token is rejected", () => + expectThrow( + () => verifyToken(makeToken({}, -120), pub, "tus-upload"), + "exp" + ) +); + +// --- HMAC: uses the CONFIGURED Zodiac secret -------------------------------- +const body = JSON.stringify({ event: "verified" }); +const path = "/neurojson-storage/internal/uploads/x"; +const headers = signRequest( + { method: "POST", path, body }, + config.hmacKeys, + config.hmacActiveKid +); +check("valid HMAC request verifies", () => { + const r = verifyRequest( + { method: "POST", path, headers, rawBody: body }, + config.hmacKeys + ); + if (!r.ok) throw new Error(r.reason); +}); +check("HMAC rejects changed body", () => { + const r = verifyRequest( + { method: "POST", path, headers, rawBody: body + " " }, + config.hmacKeys + ); + if (r.ok) throw new Error("accepted"); +}); +check("HMAC rejects changed path", () => { + const r = verifyRequest( + { method: "POST", path: path + "?x=1", headers, rawBody: body }, + config.hmacKeys + ); + if (r.ok) throw new Error("accepted"); +}); + +// --- Config ---------------------------------------------------------------- +check("configured public keys load", () => { + const kids = Object.keys(config.tokenPublicKeys); + if (!kids.length) throw new Error("none"); + console.log(" kids:", kids.join(", ")); +}); + +console.log(failed ? `\n${failed} check(s) FAILED` : "\nAll checks passed"); +process.exit(failed ? 1 : 0); diff --git a/storage-api/src/config.js b/storage-api/src/config.js new file mode 100644 index 0000000..1db3b5d --- /dev/null +++ b/storage-api/src/config.js @@ -0,0 +1,101 @@ +// Read and check all settings at startup. If anything is wrong, list every +// problem and refuse to start (better than failing on the first upload). +import "dotenv/config"; +import crypto from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +const problems = []; +const env = process.env; + +const parseJson = (name) => { + try { + return JSON.parse(env[name] || ""); + } catch { + problems.push(`${name} must be valid JSON`); + return {}; + } +}; + +const basePath = (env.BASE_PATH || "/neurojson-storage").replace(/\/+$/, ""); +if (!basePath.startsWith("/")) problems.push("BASE_PATH must start with /"); + +if (!env.UPLOAD_ROOT) problems.push("UPLOAD_ROOT is required"); +const uploadRoot = path.resolve(env.UPLOAD_ROOT || "."); + +const allowedOrigins = (env.ALLOWED_ORIGINS || "") + .split(",") + .map((s) => s.trim()) + .filter(Boolean); +if (!allowedOrigins.length) problems.push("ALLOWED_ORIGINS is required"); + +// Public keys by kid; each must be a real Ed25519 public key. +const tokenPublicKeys = {}; +for (const [kid, pem] of Object.entries( + parseJson("UPLOAD_TOKEN_PUBLIC_KEYS") +)) { + try { + const key = crypto.createPublicKey(pem); + if (key.asymmetricKeyType !== "ed25519") throw new Error(); + tokenPublicKeys[kid] = key; + } catch { + problems.push( + `UPLOAD_TOKEN_PUBLIC_KEYS["${kid}"] is not an Ed25519 public key` + ); + } +} +if (!Object.keys(tokenPublicKeys).length) { + problems.push("UPLOAD_TOKEN_PUBLIC_KEYS has no keys"); +} + +const hmacKeys = parseJson("STORAGE_HMAC_KEYS"); +const hmacActiveKid = env.STORAGE_HMAC_ACTIVE_KID; +if (!hmacActiveKid || !hmacKeys[hmacActiveKid]) { + problems.push( + "STORAGE_HMAC_ACTIVE_KID must name a key in STORAGE_HMAC_KEYS" + ); +} + +const num = (name, def) => { + const v = Number(env[name] ?? def); + if (!Number.isFinite(v) || v < 0) { + problems.push(`${name} must be a positive number`); + } + return v; +}; + +export const config = { + host: env.HOST || "127.0.0.1", + port: num("PORT", 8090), + basePath, + uploadRoot, + incomingDir: path.join(uploadRoot, ".incoming"), + outboxDir: path.join(uploadRoot, ".outbox"), + allowedOrigins, + mainApiUrl: (env.MAIN_API_URL || "").replace(/\/+$/, ""), // "" = pushes off + tokenPublicKeys, + hmacKeys, + hmacActiveKid, + maxUploadBytes: num("MAX_UPLOAD_BYTES", 536870912000), + diskReservePercent: num("DISK_RESERVE_PERCENT", 5), +}; + +if (problems.length) { + console.error( + "storage-api: invalid configuration:\n - " + problems.join("\n - ") + ); + process.exit(1); +} + +// Working folders (the root itself must already exist and be writable). +try { + fs.accessSync(uploadRoot, fs.constants.W_OK); + for (const dir of [config.incomingDir, config.outboxDir]) { + fs.mkdirSync(dir, { recursive: true }); + } +} catch (e) { + console.error( + `storage-api: UPLOAD_ROOT is not writable: ${uploadRoot} (${e.code})` + ); + process.exit(1); +} diff --git a/storage-api/src/lib/storageSignature.js b/storage-api/src/lib/storageSignature.js new file mode 100644 index 0000000..74f7e43 --- /dev/null +++ b/storage-api/src/lib/storageSignature.js @@ -0,0 +1,59 @@ +// HMAC-SHA256 signatures for server-to-server calls between REN and Zodiac +// (both directions). Same scheme as backend/src/lib/storageSignature.js — +// keep the two in sync. Signs timestamp + method + path(+query) + SHA-256 of +// the raw body; secrets are a kid -> secret map so they can rotate. +import crypto from "node:crypto"; + +const MAX_SKEW_SECONDS = 300; +export const HEADER_NAMES = { + H_KID: "x-nj-key-id", + H_TS: "x-nj-timestamp", + H_SIG: "x-nj-signature", +}; +const { H_KID, H_TS, H_SIG } = HEADER_NAMES; + +const sha256hex = (buf) => + crypto.createHash("sha256").update(buf || "").digest("hex"); +const canonical = (ts, method, path, body) => + [ts, String(method).toUpperCase(), path, sha256hex(body)].join("\n"); + +// Headers to attach to an outgoing request. `body` = the exact bytes/string sent. +export const signRequest = ({ method, path, body = "" }, keys, activeKid) => { + const secret = keys[activeKid]; + if (!activeKid || !secret) throw new Error("Storage HMAC key not configured."); + const ts = String(Math.floor(Date.now() / 1000)); + const sig = crypto + .createHmac("sha256", secret) + .update(canonical(ts, method, path, body)) + .digest("base64url"); + return { [H_KID]: activeKid, [H_TS]: ts, [H_SIG]: sig }; +}; + +// Check an incoming request. `path` must include the query string exactly as +// sent; `rawBody` must be the unparsed bytes. Returns { ok, reason }. +export const verifyRequest = ({ method, path, headers, rawBody }, keys) => { + const kid = headers[H_KID]; + const ts = headers[H_TS]; + const sig = headers[H_SIG]; + if (!kid || !ts || !sig) { + return { ok: false, reason: "missing signature headers" }; + } + const secret = keys[kid]; + if (!secret) return { ok: false, reason: "unknown key id" }; + const now = Math.floor(Date.now() / 1000); + if (!/^\d+$/.test(ts) || Math.abs(now - Number(ts)) > MAX_SKEW_SECONDS) { + return { ok: false, reason: "timestamp out of range" }; + } + const expected = crypto + .createHmac("sha256", secret) + .update(canonical(ts, method, path, rawBody)) + .digest(); + const given = Buffer.from(String(sig), "base64url"); + if ( + given.length !== expected.length || + !crypto.timingSafeEqual(given, expected) + ) { + return { ok: false, reason: "bad signature" }; + } + return { ok: true, kid }; +}; diff --git a/storage-api/src/lib/uploadTokens.js b/storage-api/src/lib/uploadTokens.js new file mode 100644 index 0000000..f755e52 --- /dev/null +++ b/storage-api/src/lib/uploadTokens.js @@ -0,0 +1,50 @@ +// Verify the short-lived EdDSA (Ed25519) tokens REN issues for uploads. +// Same strict checks as backend/src/lib/uploadTokens.js verifyToken — keep the +// two in sync (scripts/self-test.js exercises this copy). Zodiac never signs. +import crypto from "node:crypto"; + +const ISSUER = "neurojson-api"; +const AUDIENCE = "zodiac-storage"; +const CLOCK_LEEWAY_SECONDS = 30; + +// publicKeys: { [kid]: KeyObject | pem }. Throws on any problem; returns claims. +export const verifyToken = (token, publicKeys, expectedTyp) => { + const parts = String(token || "").split("."); + if (parts.length !== 3) throw new Error("malformed token"); + const [h, p, s] = parts; + const header = JSON.parse(Buffer.from(h, "base64url").toString()); + if (header.alg !== "EdDSA") throw new Error("bad alg"); // never trust other algs + const keyMaterial = publicKeys[header.kid]; + if (!keyMaterial) throw new Error("unknown kid"); + const key = + typeof keyMaterial === "string" + ? crypto.createPublicKey(keyMaterial) + : keyMaterial; + if (key.asymmetricKeyType !== "ed25519") throw new Error("bad key type"); + const ok = crypto.verify( + null, + Buffer.from(`${h}.${p}`), + key, + Buffer.from(s, "base64url") + ); + if (!ok) throw new Error("bad signature"); + const claims = JSON.parse(Buffer.from(p, "base64url").toString()); + const now = Math.floor(Date.now() / 1000); + if (claims.iss !== ISSUER || claims.aud !== AUDIENCE) { + throw new Error("bad issuer/audience"); + } + if (claims.typ !== expectedTyp) throw new Error("wrong token type"); + if ( + typeof claims.exp !== "number" || + claims.exp + CLOCK_LEEWAY_SECONDS < now + ) { + throw new Error("expired"); + } + if ( + typeof claims.iat !== "number" || + claims.iat - CLOCK_LEEWAY_SECONDS > now + ) { + throw new Error("issued in the future"); + } + return claims; +}; diff --git a/storage-api/src/server.js b/storage-api/src/server.js new file mode 100644 index 0000000..7042b4f --- /dev/null +++ b/storage-api/src/server.js @@ -0,0 +1,36 @@ +// NeuroJSON storage API (Zodiac). Listens on 127.0.0.1; Apache adds TLS and +// proxies /neurojson-storage/* here. tus uploads and /internal routes are +// added in later steps. +import fs from "node:fs/promises"; +import express from "express"; +import { config } from "./config.js"; + +const app = express(); +app.disable("x-powered-by"); + +// Public health check: basic status only (no disk details — those are internal). +app.get(`${config.basePath}/health`, async (req, res) => { + try { + await fs.access(config.uploadRoot, fs.constants.W_OK); + res.json({ status: "ok" }); + } catch { + res.status(503).json({ status: "error" }); + } +}); + +app.use((req, res) => res.status(404).json({ error: "Not found." })); + +const server = app.listen(config.port, config.host, () => { + console.log( + `storage-api listening on http://${config.host}:${config.port}${config.basePath}` + + ` | root ${config.uploadRoot}` + + ` | pushes ${ + config.mainApiUrl ? "ON → " + config.mainApiUrl : "OFF (pull mode)" + }` + ); +}); + +// Let pm2 stop us cleanly (finish in-flight requests). +for (const sig of ["SIGTERM", "SIGINT"]) { + process.on(sig, () => server.close(() => process.exit(0))); +} From 0bd5225c84cd914369cfd5c23ad0cc15e7fe28bf Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Fri, 2 Oct 2026 10:55:21 -0400 Subject: [PATCH 37/45] feat(storage-api): tus resumable uploads with token checks; refs #147 --- backend/scripts/mint-test-upload-token.js | 19 +++ storage-api/package-lock.json | 190 ++++++++++++++++++++++ storage-api/package.json | 3 + storage-api/scripts/test-upload.js | 72 ++++++++ storage-api/src/lib/diskSpace.js | 34 ++++ storage-api/src/lib/paths.js | 20 +++ storage-api/src/lib/tusAuth.js | 40 +++++ storage-api/src/server.js | 10 +- storage-api/src/tus/tusServer.js | 90 ++++++++++ 9 files changed, 476 insertions(+), 2 deletions(-) create mode 100644 backend/scripts/mint-test-upload-token.js create mode 100644 storage-api/scripts/test-upload.js create mode 100644 storage-api/src/lib/diskSpace.js create mode 100644 storage-api/src/lib/paths.js create mode 100644 storage-api/src/lib/tusAuth.js create mode 100644 storage-api/src/tus/tusServer.js diff --git a/backend/scripts/mint-test-upload-token.js b/backend/scripts/mint-test-upload-token.js new file mode 100644 index 0000000..6592b36 --- /dev/null +++ b/backend/scripts/mint-test-upload-token.js @@ -0,0 +1,19 @@ +// DEV ONLY: print a signed tus-upload token for a test upload of bytes, +// with made-up ids (the storage API doesn't ask REN about uploads yet). +// Usage: node scripts/mint-test-upload-token.js +require("dotenv").config({ quiet: true }); +const crypto = require("crypto"); +const { signUploadToken } = require("../src/lib/uploadTokens"); + +const size = Number(process.argv[2]); +if (!Number.isSafeInteger(size) || size <= 0) { + console.error("usage: node scripts/mint-test-upload-token.js "); + process.exit(1); +} +const ids = { + uploadId: crypto.randomUUID(), + internalId: crypto.randomUUID(), + submissionId: crypto.randomUUID(), +}; +const { token } = signUploadToken({ userId: 0, ...ids, expectedSize: size }); +console.log(JSON.stringify({ ...ids, size, token })); diff --git a/storage-api/package-lock.json b/storage-api/package-lock.json index 361874d..02d6d82 100644 --- a/storage-api/package-lock.json +++ b/storage-api/package-lock.json @@ -13,6 +13,9 @@ "dotenv": "^17.2.3", "express": "^5.1.0" }, + "devDependencies": { + "tus-js-client": "^4.3.1" + }, "engines": { "node": ">=20.19.0" } @@ -144,6 +147,13 @@ "url": "https://opencollective.com/express" } }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true, + "license": "MIT" + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -192,6 +202,16 @@ "node": ">=0.10.0" } }, + "node_modules/combine-errors": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/combine-errors/-/combine-errors-3.0.3.tgz", + "integrity": "sha512-C8ikRNRMygCwaTx+Ek3Yr+OuZzgZjduCOfSQBjbM8V3MfgcjSTeto/GXP6PAwKvJz/v15b7GHZvx5rOlczFw/Q==", + "dev": true, + "dependencies": { + "custom-error-instance": "2.1.1", + "lodash.uniqby": "4.5.0" + } + }, "node_modules/content-disposition": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", @@ -232,6 +252,13 @@ "node": ">=6.6.0" } }, + "node_modules/custom-error-instance": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/custom-error-instance/-/custom-error-instance-2.1.1.tgz", + "integrity": "sha512-p6JFxJc3M4OTD2li2qaHkDCw9SfMw82Ldr6OC9Je1aXiGfhx2W8p3GaoeaGrPJTUN9NirTM/KTxHWMUdR1rsUg==", + "dev": true, + "license": "ISC" + }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", @@ -494,6 +521,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, "node_modules/has-symbols": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", @@ -608,12 +642,95 @@ "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", "license": "MIT" }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/js-base64": { + "version": "3.9.4", + "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.9.4.tgz", + "integrity": "sha512-PtOMXpEGuP0RRiRXsjzHzl44dMHxSu2CPvAhinupR1tBa88me+1DPqsobl7eupn5UukjLkln1ZnAOAOXOrYG0Q==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/lodash._baseiteratee": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/lodash._baseiteratee/-/lodash._baseiteratee-4.7.0.tgz", + "integrity": "sha512-nqB9M+wITz0BX/Q2xg6fQ8mLkyfF7MU7eE+MNBNjTHFKeKaZAPEzEg+E8LWxKWf1DQVflNEn9N49yAuqKh2mWQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "lodash._stringtopath": "~4.8.0" + } + }, + "node_modules/lodash._basetostring": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/lodash._basetostring/-/lodash._basetostring-4.12.0.tgz", + "integrity": "sha512-SwcRIbyxnN6CFEEK4K1y+zuApvWdpQdBHM/swxP962s8HIxPO3alBH5t3m/dl+f4CMUug6sJb7Pww8d13/9WSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/lodash._baseuniq": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash._baseuniq/-/lodash._baseuniq-4.6.0.tgz", + "integrity": "sha512-Ja1YevpHZctlI5beLA7oc5KNDhGcPixFhcqSiORHNsp/1QTv7amAXzw+gu4YOvErqVlMVyIJGgtzeepCnnur0A==", + "dev": true, + "license": "MIT", + "dependencies": { + "lodash._createset": "~4.0.0", + "lodash._root": "~3.0.0" + } + }, + "node_modules/lodash._createset": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/lodash._createset/-/lodash._createset-4.0.3.tgz", + "integrity": "sha512-GTkC6YMprrJZCYU3zcqZj+jkXkrXzq3IPBcF/fIPpNEAB4hZEtXU8zp/RwKOvZl43NUmwDbyRk3+ZTbeRdEBXA==", + "dev": true, + "license": "MIT" + }, + "node_modules/lodash._root": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash._root/-/lodash._root-3.0.1.tgz", + "integrity": "sha512-O0pWuFSK6x4EXhM1dhZ8gchNtG7JMqBtrHdoUFUWXD7dJnNSUze1GuyQr5sOs0aCvgGeI3o/OJW8f4ca7FDxmQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/lodash._stringtopath": { + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/lodash._stringtopath/-/lodash._stringtopath-4.8.0.tgz", + "integrity": "sha512-SXL66C731p0xPDC5LZg4wI5H+dJo/EO4KTqOMwLYCH3+FmmfAKJEZCm6ohGpI+T1xwsDsJCfL4OnhorllvlTPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "lodash._basetostring": "~4.12.0" + } + }, "node_modules/lodash.throttle": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/lodash.throttle/-/lodash.throttle-4.1.1.tgz", "integrity": "sha512-wIkUCfVKpVsWo3JSZlc+8MB5it+2AN5W8J7YVMST30UrvcQNZ1Okbj+rbVniijTWE6FGYy4XJq/rHkas8qJMLQ==", "license": "MIT" }, + "node_modules/lodash.uniqby": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniqby/-/lodash.uniqby-4.5.0.tgz", + "integrity": "sha512-IRt7cfTtHy6f1aRVA5n7kT8rgN3N1nH6MOWLcHfpWG2SH19E3JksLK38MktLxZDhlAjCP9jpIXkOnRXlu6oByQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "lodash._baseiteratee": "~4.7.0", + "lodash._baseuniq": "~4.6.0" + } + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -760,6 +877,18 @@ "url": "https://opencollective.com/express" } }, + "node_modules/proper-lockfile": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, "node_modules/proxy-addr": { "version": "2.0.8", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.8.tgz", @@ -793,6 +922,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", + "dev": true, + "license": "MIT" + }, "node_modules/range-parser": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", @@ -844,6 +980,23 @@ "node": ">=4" } }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, "node_modules/router": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", @@ -995,6 +1148,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true, + "license": "ISC" + }, "node_modules/srvx": { "version": "0.11.22", "resolved": "https://registry.npmjs.org/srvx/-/srvx-0.11.22.tgz", @@ -1032,6 +1192,25 @@ "node": ">=0.6" } }, + "node_modules/tus-js-client": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/tus-js-client/-/tus-js-client-4.3.1.tgz", + "integrity": "sha512-ZLeYmjrkaU1fUsKbIi8JML52uAocjEZtBx4DKjRrqzrZa0O4MYwT6db+oqePlspV+FxXJAyFBc/L5gwUi2OFsg==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer-from": "^1.1.2", + "combine-errors": "^3.0.3", + "is-stream": "^2.0.0", + "js-base64": "^3.7.2", + "lodash.throttle": "^4.1.1", + "proper-lockfile": "^4.1.2", + "url-parse": "^1.5.7" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/type-is": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", @@ -1072,6 +1251,17 @@ "node": ">= 0.8" } }, + "node_modules/url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", diff --git a/storage-api/package.json b/storage-api/package.json index 8f0a5ec..a6a4599 100644 --- a/storage-api/package.json +++ b/storage-api/package.json @@ -17,5 +17,8 @@ "@tus/server": "^2.4.5", "dotenv": "^17.2.3", "express": "^5.1.0" + }, + "devDependencies": { + "tus-js-client": "^4.3.1" } } diff --git a/storage-api/scripts/test-upload.js b/storage-api/scripts/test-upload.js new file mode 100644 index 0000000..a8f2c34 --- /dev/null +++ b/storage-api/scripts/test-upload.js @@ -0,0 +1,72 @@ +// Upload a test file to the storage API with tus-js-client (Node), like the +// browser will. The test file starts with "PK\x03\x04" so the later ZIP check +// passes; it's saved under storage-dev/test-files/ so a resumed run sends the +// exact same bytes. +// +// Usage: +// node scripts/test-upload.js '' \ +// [--stop-after-mb N] [--resume] [--chunk-mb 5] [--endpoint URL] +// --stop-after-mb N abort once N MB are uploaded (to test resuming) +// --resume continue an existing upload (HEAD → PATCH from its offset) +import crypto from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import * as tus from "tus-js-client"; + +const args = process.argv.slice(2); +const opt = (name, def) => { + const i = args.indexOf(name); + return i === -1 ? def : args[i + 1]; +}; +let info; +try { + info = JSON.parse(args[0]); +} catch { + console.log("first argument must be the JSON printed by mint-test-upload-token.js"); + process.exit(1); +} +const endpoint = opt( + "--endpoint", + "http://127.0.0.1:8090/neurojson-storage/files" +); +const chunkSize = Number(opt("--chunk-mb", 5)) * 1024 * 1024; +const stopAfter = opt("--stop-after-mb") && Number(opt("--stop-after-mb")) * 1024 * 1024; +const resume = args.includes("--resume"); + +// Same bytes every run for the same upload id. +const dir = path.resolve("storage-dev/test-files"); +fs.mkdirSync(dir, { recursive: true }); +const file = path.join(dir, `${info.uploadId}.zip`); +if (!fs.existsSync(file)) { + const buf = crypto.randomBytes(info.size); + Buffer.from([0x50, 0x4b, 0x03, 0x04]).copy(buf, 0); // "PK\x03\x04" + fs.writeFileSync(file, buf); +} +const data = fs.readFileSync(file); + +let stopped = false; +const upload = new tus.Upload(data, { + endpoint, + uploadUrl: resume ? `${endpoint}/${info.uploadId}` : undefined, + uploadSize: data.length, + chunkSize, + headers: { Authorization: `Bearer ${info.token}` }, + metadata: { filename: `${info.uploadId}.zip` }, + retryDelays: [0, 1000], + onProgress: (sent, total) => { + process.stdout.write(`\r${(sent / 1048576).toFixed(1)} / ${(total / 1048576).toFixed(1)} MB`); + if (stopAfter && sent >= stopAfter && !stopped) { + stopped = true; + upload.abort().then(() => + console.log(`\nstopped at ${(sent / 1048576).toFixed(1)} MB (run again with --resume)`) + ); + } + }, + onError: (err) => { + const r = err.originalResponse; + console.log(`\nFAILED: ${r ? `HTTP ${r.getStatus()} ${r.getBody().trim()}` : err.message}`); + process.exit(1); + }, + onSuccess: () => console.log(`\nDONE: ${upload.url}`), +}); +upload.start(); diff --git a/storage-api/src/lib/diskSpace.js b/storage-api/src/lib/diskSpace.js new file mode 100644 index 0000000..12424c5 --- /dev/null +++ b/storage-api/src/lib/diskSpace.js @@ -0,0 +1,34 @@ +// Free space minus what in-progress uploads still need, so two big uploads +// can't each pass the check and then fill the disk together. Computed fresh +// from .incoming/ (survives restarts; creates are rare, so a scan is cheap). +import fs from "node:fs/promises"; +import path from "node:path"; +import { config } from "../config.js"; + +// Bytes other active uploads still have to write. +const reservedBytes = async (exceptId) => { + let total = 0; + for (const name of await fs.readdir(config.incomingDir)) { + if (!name.endsWith(".json")) continue; + const id = name.slice(0, -5); + if (id === exceptId) continue; + try { + const meta = JSON.parse( + await fs.readFile(path.join(config.incomingDir, name), "utf8") + ); + const written = (await fs.stat(path.join(config.incomingDir, id))).size; + if (Number.isFinite(meta.size)) total += Math.max(0, meta.size - written); + } catch { + // half-written or removed meanwhile — ignore + } + } + return total; +}; + +// true if `size` more bytes fit, keeping DISK_RESERVE_PERCENT of the disk free. +export const hasRoomFor = async (size, uploadId) => { + const s = await fs.statfs(config.uploadRoot); + const free = s.bavail * s.bsize; + const margin = (s.blocks * s.bsize * config.diskReservePercent) / 100; + return free - (await reservedBytes(uploadId)) - margin >= size; +}; diff --git a/storage-api/src/lib/paths.js b/storage-api/src/lib/paths.js new file mode 100644 index 0000000..1a45137 --- /dev/null +++ b/storage-api/src/lib/paths.js @@ -0,0 +1,20 @@ +// All storage paths are built ONLY from token claims (UUIDs), never from +// anything else the client sends, and must stay inside UPLOAD_ROOT. +import path from "node:path"; +import { config } from "../config.js"; + +const inside = (p) => { + const root = config.uploadRoot + path.sep; + if (!p.startsWith(root)) throw new Error("path escapes UPLOAD_ROOT"); + return p; +}; + +export const incomingFile = (uploadId) => + inside(path.join(config.incomingDir, uploadId)); + +export const rawObjectDir = (internalId, rawId) => + inside(path.join(config.uploadRoot, internalId, "raw", rawId)); + +// Relative key stored in Postgres (never an absolute path). +export const rawZipKey = (internalId, rawId) => + `${internalId}/raw/${rawId}/source.zip`; diff --git a/storage-api/src/lib/tusAuth.js b/storage-api/src/lib/tusAuth.js new file mode 100644 index 0000000..a8b2bfb --- /dev/null +++ b/storage-api/src/lib/tusAuth.js @@ -0,0 +1,40 @@ +// Upload-token checks for tus requests. Every tus request (create, chunk, +// resume check, cancel) must carry `Authorization: Bearer `. Verified +// once per request and cached (tus calls several hooks with the same Request). +import { config } from "../config.js"; +import { verifyToken } from "./uploadTokens.js"; + +const UUID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const cache = new WeakMap(); + +// tus sends { status_code, body } to the client; body must not leak internals. +export const tusError = (status_code, message) => ({ + status_code, + body: `${message}\n`, +}); + +export const getClaims = (req) => { + if (cache.has(req)) return cache.get(req); + const auth = req.headers.get("authorization") || ""; + const token = auth.startsWith("Bearer ") ? auth.slice(7) : ""; + let claims; + try { + claims = verifyToken(token, config.tokenPublicKeys, "tus-upload"); + } catch { + throw tusError(401, "Invalid or expired upload token."); + } + // Paths are built from these claims, so check their shape strictly. + if ( + !UUID_RE.test(String(claims.uid)) || + !UUID_RE.test(String(claims.iid)) || + !UUID_RE.test(String(claims.sid)) || + !Number.isSafeInteger(claims.len) || + claims.len <= 0 || + claims.len > config.maxUploadBytes + ) { + throw tusError(401, "Invalid upload token."); + } + cache.set(req, claims); + return claims; +}; diff --git a/storage-api/src/server.js b/storage-api/src/server.js index 7042b4f..0dda9e8 100644 --- a/storage-api/src/server.js +++ b/storage-api/src/server.js @@ -1,9 +1,9 @@ // NeuroJSON storage API (Zodiac). Listens on 127.0.0.1; Apache adds TLS and -// proxies /neurojson-storage/* here. tus uploads and /internal routes are -// added in later steps. +// proxies /neurojson-storage/* here. /internal routes are added in a later step. import fs from "node:fs/promises"; import express from "express"; import { config } from "./config.js"; +import { tusServer } from "./tus/tusServer.js"; const app = express(); app.disable("x-powered-by"); @@ -18,6 +18,12 @@ app.get(`${config.basePath}/health`, async (req, res) => { } }); +// tus owns everything under /files (POST create, HEAD/PATCH/DELETE +// /files/, and the CORS preflight). No body parser: tus streams the body. +const tusHandler = (req, res) => tusServer.handle(req, res); +app.all(`${config.basePath}/files`, tusHandler); +app.all(`${config.basePath}/files/*splat`, tusHandler); + app.use((req, res) => res.status(404).json({ error: "Not found." })); const server = app.listen(config.port, config.host, () => { diff --git a/storage-api/src/tus/tusServer.js b/storage-api/src/tus/tusServer.js new file mode 100644 index 0000000..96019f8 --- /dev/null +++ b/storage-api/src/tus/tusServer.js @@ -0,0 +1,90 @@ +// tus resumable uploads into UPLOAD_ROOT/.incoming/. +// The upload id = the token's `uid` (assigned by REN), so URLs are +// /files/. Finishing (verify → ask REN → move) comes next. +import fs from "node:fs/promises"; +import { Server } from "@tus/server"; +import { FileStore } from "@tus/file-store"; +import { config } from "../config.js"; +import { getClaims, tusError } from "../lib/tusAuth.js"; +import { hasRoomFor } from "../lib/diskSpace.js"; +import { incomingFile, rawObjectDir } from "../lib/paths.js"; + +const exists = (p) => + fs.access(p).then( + () => true, + () => false + ); + +export const tusServer = new Server({ + path: `${config.basePath}/files`, + datastore: new FileStore({ + directory: config.incomingDir, + expirationPeriodInMilliseconds: 7 * 24 * 3600 * 1000, // idle uploads expire after 7 days + }), + respectForwardedHeaders: true, // Apache sets X-Forwarded-Proto/Host → https Location URLs + allowedOrigins: config.allowedOrigins, + allowedHeaders: ["Authorization"], + allowedCredentials: false, // Bearer token, no cookies + disableTerminationForFinishedUploads: true, + + // Runs FIRST on create: the token decides the upload id. + namingFunction: async (req) => getClaims(req).uid, + + // The upload can never be bigger than the size REN signed into the token. + maxSize: async (req, id) => + id ? getClaims(req).len : config.maxUploadBytes, + + // Every request: valid token, and the URL's upload id must be the token's. + onIncomingRequest: async (req, uploadId) => { + if (getClaims(req).uid !== uploadId) { + throw tusError(403, "This token is not for this upload."); + } + }, + + onUploadCreate: async (req, upload) => { + const c = getClaims(req); + if (upload.size === undefined) { + throw tusError(400, "The file size must be known when the upload starts."); + } + if (upload.size !== c.len) { + throw tusError( + 400, + "Upload-Length must equal the size declared to NeuroJSON." + ); + } + // Never restart an upload that already has data (that would wipe + // progress), and never re-create one that already finished. + if ( + (await exists(incomingFile(c.uid))) || + (await exists(rawObjectDir(c.iid, c.uid))) + ) { + throw tusError(409, "This upload already exists. Resume it instead."); + } + if (!(await hasRoomFor(upload.size, c.uid))) { + throw tusError(507, "Not enough storage space for this file right now."); + } + // Server-side facts for the finish step; these win over client metadata. + return { + metadata: { + ...upload.metadata, + nj_internal_id: c.iid, + nj_submission_id: c.sid, + nj_upload_id: c.uid, + }, + }; + }, + + onUploadFinish: async (req, upload) => { + // Next step replaces this: write outbox record → verify → ask REN → move. + console.log(`upload complete: ${upload.id} (${upload.size} bytes)`); + return {}; + }, + + // Our own errors ({status_code, body}) pass through; anything unexpected + // becomes a generic 500 so file paths / stack details never reach the client. + onResponseError: async (req, err) => { + if (err && typeof err.status_code === "number") return undefined; + console.error("tus error:", err?.message || err); + return { status_code: 500, body: "Internal server error.\n" }; + }, +}); From fb8986d7a5bb5eb659ca296d69cb0fb108573708 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Fri, 2 Oct 2026 11:54:43 -0400 Subject: [PATCH 38/45] feat(storage-api): finish uploads via a crash-safe outbox; refs #147 --- storage-api/src/lib/fileChecks.js | 38 ++++ storage-api/src/lib/mainApiClient.js | 50 ++++++ storage-api/src/outbox/outbox.js | 85 +++++++++ storage-api/src/outbox/worker.js | 258 +++++++++++++++++++++++++++ storage-api/src/server.js | 10 +- storage-api/src/tus/tusServer.js | 18 +- 6 files changed, 454 insertions(+), 5 deletions(-) create mode 100644 storage-api/src/lib/fileChecks.js create mode 100644 storage-api/src/lib/mainApiClient.js create mode 100644 storage-api/src/outbox/outbox.js create mode 100644 storage-api/src/outbox/worker.js diff --git a/storage-api/src/lib/fileChecks.js b/storage-api/src/lib/fileChecks.js new file mode 100644 index 0000000..311a78f --- /dev/null +++ b/storage-api/src/lib/fileChecks.js @@ -0,0 +1,38 @@ +// File checks for a finished upload. The ZIP check is SCREENING ONLY (first +// bytes) — the archive is never opened or extracted here. +import crypto from "node:crypto"; +import fs from "node:fs"; +import fsp from "node:fs/promises"; + +const ZIP_MAGIC = Buffer.from([0x50, 0x4b, 0x03, 0x04]); // "PK\x03\x04" + +export const looksLikeZip = async (file) => { + const fh = await fsp.open(file, "r"); + try { + const buf = Buffer.alloc(4); + const { bytesRead } = await fh.read(buf, 0, 4, 0); + return bytesRead === 4 && buf.equals(ZIP_MAGIC); + } finally { + await fh.close(); + } +}; + +// Streamed (constant memory), so it works for very large files. +export const sha256File = (file) => + new Promise((resolve, reject) => { + const hash = crypto.createHash("sha256"); + fs.createReadStream(file, { highWaterMark: 4 * 1024 * 1024 }) + .on("data", (chunk) => hash.update(chunk)) + .on("error", reject) + .on("end", () => resolve(hash.digest("hex"))); + }); + +// Make a rename durable: fsync the directory that now holds the file. +export const fsyncDir = async (dir) => { + const fh = await fsp.open(dir, "r"); + try { + await fh.sync(); + } finally { + await fh.close(); + } +}; diff --git a/storage-api/src/lib/mainApiClient.js b/storage-api/src/lib/mainApiClient.js new file mode 100644 index 0000000..fa26340 --- /dev/null +++ b/storage-api/src/lib/mainApiClient.js @@ -0,0 +1,50 @@ +// Signed calls from Zodiac to REN. Classifies the answer for the outbox: +// ok → 2xx with a JSON body +// retry → network error / timeout / 5xx (try again later) +// permanent → 4xx (retrying won't help: wrong keys, unknown upload, …) +import { config } from "../config.js"; +import { signRequest } from "./storageSignature.js"; + +export const pushEnabled = () => Boolean(config.mainApiUrl); + +export const sendUploadEvent = async (uploadId, payload) => { + const url = new URL( + `${config.mainApiUrl}/internal/storage/uploads/${uploadId}/events` + ); + const body = JSON.stringify(payload); + const headers = { + "Content-Type": "application/json", + // REN verifies against its req.originalUrl, i.e. this exact path + query. + ...signRequest( + { method: "POST", path: url.pathname + url.search, body }, + config.hmacKeys, + config.hmacActiveKid + ), + }; + let res; + try { + res = await fetch(url, { + method: "POST", + headers, + body, + signal: AbortSignal.timeout(30_000), + }); + } catch (e) { + return { kind: "retry", error: `network: ${e.message}` }; + } + const text = await res.text(); + if (res.ok) { + try { + return { kind: "ok", body: JSON.parse(text) }; + } catch { + return { kind: "retry", error: "REN sent a non-JSON reply" }; + } + } + if (res.status >= 500) { + return { kind: "retry", error: `REN HTTP ${res.status}` }; + } + return { + kind: "permanent", + error: `REN HTTP ${res.status}: ${text.slice(0, 200)}`, + }; +}; diff --git a/storage-api/src/outbox/outbox.js b/storage-api/src/outbox/outbox.js new file mode 100644 index 0000000..6eb0343 --- /dev/null +++ b/storage-api/src/outbox/outbox.js @@ -0,0 +1,85 @@ +// Crash-safe job records on disk, one per finished upload: +// .outbox/.json pending +// .outbox/done/.json finished (kept for status lookups; cleaned up later) +// .outbox/failed/.json parked for an admin (REN said 4xx, or files missing) +import fs from "node:fs/promises"; +import path from "node:path"; +import { config } from "../config.js"; + +const DIR = config.outboxDir; +const DONE = path.join(DIR, "done"); +const FAILED = path.join(DIR, "failed"); +await fs.mkdir(DONE, { recursive: true }); +await fs.mkdir(FAILED, { recursive: true }); + +const fileIn = (dir, id) => path.join(dir, `${id}.json`); + +// temp file → fsync → rename, so a crash never leaves a half-written record. +const writeAtomic = async (file, obj) => { + const tmp = `${file}.tmp`; + const fh = await fs.open(tmp, "w"); + try { + await fh.writeFile(JSON.stringify(obj, null, 2)); + await fh.sync(); + } finally { + await fh.close(); + } + await fs.rename(tmp, file); +}; + +// Look in pending, done and failed (also used by the status endpoint). +export const findRecord = async (id) => { + for (const [where, dir] of [ + ["pending", DIR], + ["done", DONE], + ["failed", FAILED], + ]) { + try { + return { + where, + rec: JSON.parse(await fs.readFile(fileIn(dir, id), "utf8")), + }; + } catch { + // not here + } + } + return null; +}; + +// Returns false if a record for this upload already exists (never overwrite). +export const createRecord = async (rec) => { + if (await findRecord(rec.uploadId)) return false; + const now = new Date().toISOString(); + await writeAtomic(fileIn(DIR, rec.uploadId), { + ...rec, + attempts: 0, + nextAttemptAt: now, + lastError: null, + createdAt: now, + updatedAt: now, + }); + return true; +}; + +export const saveRecord = (rec) => + writeAtomic(fileIn(DIR, rec.uploadId), { + ...rec, + updatedAt: new Date().toISOString(), + }); + +export const listPendingIds = async () => + (await fs.readdir(DIR)) + .filter((n) => n.endsWith(".json")) + .map((n) => n.slice(0, -5)); + +export const readPending = async (id) => + JSON.parse(await fs.readFile(fileIn(DIR, id), "utf8")); + +// Final state: save, then move out of the pending folder. +export const closeRecord = async (rec, outcome) => { + await saveRecord(rec); + await fs.rename( + fileIn(DIR, rec.uploadId), + fileIn(outcome === "failed" ? FAILED : DONE, rec.uploadId) + ); +}; diff --git a/storage-api/src/outbox/worker.js b/storage-api/src/outbox/worker.js new file mode 100644 index 0000000..4d6cb58 --- /dev/null +++ b/storage-api/src/outbox/worker.js @@ -0,0 +1,258 @@ +// Drives each finished upload to completion, one record at a time (hashing +// large files is disk-heavy). Every state change is saved before the next +// step, so after a crash or restart the worker continues where it stopped. +// +// verifying ─► awaiting_decision ─[verified]─► committing ─► committed ─[committed]─► done +// │ └─► discarding ─► done (discard on committed → discarding_committed) +// └─► rejecting ─[failed]─► done +// +// Pull mode (MAIN_API_URL empty): the file work still runs, but the worker +// stops at the states that need REN's answer; REN drives those via /internal. +import fs from "node:fs/promises"; +import path from "node:path"; +import { + listPendingIds, + readPending, + saveRecord, + closeRecord, +} from "./outbox.js"; +import { incomingFile, rawObjectDir, rawZipKey } from "../lib/paths.js"; +import { looksLikeZip, sha256File, fsyncDir } from "../lib/fileChecks.js"; +import { pushEnabled, sendUploadEvent } from "../lib/mainApiClient.js"; + +const TICK_MS = 5000; +const BACKOFF_SECONDS = [5, 30, 120, 600, 1800, 3600]; +const UUID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +// States where the next move needs REN's answer (pull mode waits here). +export const WAITS_FOR_REN = new Set([ + "awaiting_decision", + "rejecting", + "committed", +]); + +const exists = (p) => + fs.access(p).then( + () => true, + () => false + ); +const removeIncoming = async (id) => { + await fs.rm(incomingFile(id), { force: true }); + await fs.rm(`${incomingFile(id)}.json`, { force: true }); +}; + +class Permanent extends Error {} + +const ready = (rec) => { + rec.attempts = 0; + rec.lastError = null; + rec.nextAttemptAt = new Date().toISOString(); +}; + +const retryLater = async (rec, error) => { + const delay = + BACKOFF_SECONDS[Math.min(rec.attempts, BACKOFF_SECONDS.length - 1)]; + rec.attempts += 1; + rec.lastError = error; + rec.nextAttemptAt = new Date(Date.now() + delay * 1000).toISOString(); + await saveRecord(rec); + console.warn( + `outbox ${rec.uploadId}: ${rec.state} retry #${rec.attempts} in ${delay}s (${error})` + ); +}; + +const park = async (rec, error) => { + rec.lastError = error; + rec.parkedFrom = rec.state; + rec.state = "failed"; + await closeRecord(rec, "failed"); + console.error( + `outbox ${rec.uploadId}: PARKED in .outbox/failed/ (${error})` + ); +}; + +const finish = async (rec) => { + rec.state = "done"; + await closeRecord(rec, "done"); + console.log(`outbox ${rec.uploadId}: done`); +}; + +// Ask REN; returns the reply body, or throws Permanent / a retryable Error. +const askRen = async (rec, payload) => { + const r = await sendUploadEvent(rec.uploadId, payload); + if (r.kind === "ok") return r.body; + if (r.kind === "permanent") throw new Permanent(r.error); + throw new Error(r.error); +}; + +const setState = async (rec, state, extra = {}) => { + Object.assign(rec, extra, { state }); + ready(rec); + await saveRecord(rec); + console.log(`outbox ${rec.uploadId}: → ${state}`); +}; + +const STEPS = { + // Size, ZIP signature, SHA-256 of the file sitting in .incoming/. + async verifying(rec) { + const file = incomingFile(rec.uploadId); + let size; + try { + size = (await fs.stat(file)).size; + } catch { + throw new Permanent("incoming file is missing"); + } + let reason = null; + if (size !== rec.expectedSize) reason = "size_mismatch"; + else if (!(await looksLikeZip(file))) reason = "not_zip"; + if (reason) return setState(rec, "rejecting", { reason }); + const sha256 = await sha256File(file); + await setState(rec, "awaiting_decision", { size, sha256 }); + }, + + // Tell REN the file failed the checks, then delete it. + async rejecting(rec) { + await askRen(rec, { event: "failed", reason: rec.reason }); + await removeIncoming(rec.uploadId); + await finish(rec); + }, + + // "verified" → REN answers commit (with rawId) or discard. + async awaiting_decision(rec) { + const reply = await askRen(rec, { + event: "verified", + size: rec.size, + sha256: rec.sha256, + }); + await applyDecision(rec, reply); + }, + + // Move .incoming/ → /raw//source.zip (atomic rename). + async committing(rec) { + const dir = rawObjectDir(rec.internalId, rec.rawId); + const src = incomingFile(rec.uploadId); + const dest = path.join(dir, "source.zip"); + await fs.mkdir(dir, { recursive: true }); + const haveSrc = await exists(src); + const haveDest = await exists(dest); + if (haveSrc && haveDest) { + throw new Permanent("both incoming and final file exist"); + } + if (!haveSrc && !haveDest) { + throw new Permanent("file is missing (neither incoming nor final)"); + } + if (haveSrc) await fs.rename(src, dest); // else: moved before a crash — continue + await fsyncDir(dir); + if ((await fs.stat(dest)).size !== rec.size) { + throw new Permanent("final file size changed"); + } + await fs.rm(`${src}.json`, { force: true }); // tus metadata no longer needed + await setState(rec, "committed", { + storageKey: rawZipKey(rec.internalId, rec.rawId), + }); + }, + + // "committed" → REN records the object; keep, or discard if cancelled meanwhile. + async committed(rec) { + const reply = await askRen(rec, { + event: "committed", + size: rec.size, + sha256: rec.sha256, + }); + if (reply.action === "discard") { + return setState(rec, "discarding_committed", { + reason: reply.reason || null, + }); + } + await finish(rec); + }, + + async discarding(rec) { + await removeIncoming(rec.uploadId); + await finish(rec); + }, + + async discarding_committed(rec) { + await fs.rm(rawObjectDir(rec.internalId, rec.rawId), { + recursive: true, + force: true, + }); + await finish(rec); + }, +}; + +// Apply REN's commit/discard answer (used by push mode here and by the pull +// endpoint later). Validates the rawId so a bad reply can't pick a path. +export const applyDecision = async (rec, reply) => { + if (reply.decision === "commit") { + if (!UUID_RE.test(String(reply.rawId))) { + throw new Permanent("REN sent an invalid rawId"); + } + const key = rawZipKey(rec.internalId, reply.rawId); + if (reply.storageKey && reply.storageKey !== key) { + throw new Permanent("REN storageKey does not match"); + } + return setState(rec, "committing", { rawId: reply.rawId }); + } + if (reply.decision === "discard") { + return setState(rec, "discarding", { reason: reply.reason || null }); + } + throw new Permanent("REN sent an unknown decision"); +}; + +// Run one record forward until it waits, retries, or finishes. +const advance = async (rec) => { + for (let i = 0; i < 10; i++) { + const step = STEPS[rec.state]; + if (!step) return; + if (!pushEnabled() && WAITS_FOR_REN.has(rec.state)) return; // pull mode + const before = rec.state; + try { + await step(rec); + } catch (e) { + if (e instanceof Permanent) await park(rec, e.message); + else await retryLater(rec, e.message); + return; + } + if (rec.state === before || rec.state === "done") return; + } +}; + +let running = false; +let again = false; +let timer = null; + +const tick = async () => { + if (running) { + again = true; + return; + } + running = true; + try { + for (const id of await listPendingIds()) { + let rec; + try { + rec = await readPending(id); + } catch { + continue; // removed meanwhile + } + if (new Date(rec.nextAttemptAt) > new Date()) continue; + await advance(rec); + } + } catch (e) { + console.error("outbox worker:", e.message); + } finally { + running = false; + if (again) { + again = false; + setImmediate(tick); + } + } +}; + +export const wakeWorker = () => setImmediate(tick); +export const startWorker = () => { + timer = setInterval(tick, TICK_MS); + wakeWorker(); // pick up records left from before a restart +}; +export const stopWorker = () => clearInterval(timer); diff --git a/storage-api/src/server.js b/storage-api/src/server.js index 0dda9e8..0d141ca 100644 --- a/storage-api/src/server.js +++ b/storage-api/src/server.js @@ -4,6 +4,7 @@ import fs from "node:fs/promises"; import express from "express"; import { config } from "./config.js"; import { tusServer } from "./tus/tusServer.js"; +import { startWorker, stopWorker } from "./outbox/worker.js"; const app = express(); app.disable("x-powered-by"); @@ -34,9 +35,14 @@ const server = app.listen(config.port, config.host, () => { config.mainApiUrl ? "ON → " + config.mainApiUrl : "OFF (pull mode)" }` ); + startWorker(); }); -// Let pm2 stop us cleanly (finish in-flight requests). +// Let pm2 stop us cleanly (finish in-flight requests). Outbox records are on +// disk, so an interrupted step simply resumes on the next start. for (const sig of ["SIGTERM", "SIGINT"]) { - process.on(sig, () => server.close(() => process.exit(0))); + process.on(sig, () => { + stopWorker(); + server.close(() => process.exit(0)); + }); } diff --git a/storage-api/src/tus/tusServer.js b/storage-api/src/tus/tusServer.js index 96019f8..a4e2565 100644 --- a/storage-api/src/tus/tusServer.js +++ b/storage-api/src/tus/tusServer.js @@ -1,6 +1,7 @@ // tus resumable uploads into UPLOAD_ROOT/.incoming/. // The upload id = the token's `uid` (assigned by REN), so URLs are -// /files/. Finishing (verify → ask REN → move) comes next. +// /files/. When an upload finishes, an outbox record is +// written and the worker verifies it, asks REN, and moves it into place. import fs from "node:fs/promises"; import { Server } from "@tus/server"; import { FileStore } from "@tus/file-store"; @@ -8,6 +9,8 @@ import { config } from "../config.js"; import { getClaims, tusError } from "../lib/tusAuth.js"; import { hasRoomFor } from "../lib/diskSpace.js"; import { incomingFile, rawObjectDir } from "../lib/paths.js"; +import { createRecord } from "../outbox/outbox.js"; +import { wakeWorker } from "../outbox/worker.js"; const exists = (p) => fs.access(p).then( @@ -75,9 +78,18 @@ export const tusServer = new Server({ }, onUploadFinish: async (req, upload) => { - // Next step replaces this: write outbox record → verify → ask REN → move. + // Values set by the server in onUploadCreate (never client-supplied). + const m = upload.metadata || {}; + await createRecord({ + uploadId: upload.id, + internalId: m.nj_internal_id, + submissionId: m.nj_submission_id, + expectedSize: upload.size, + state: "verifying", + }); console.log(`upload complete: ${upload.id} (${upload.size} bytes)`); - return {}; + wakeWorker(); + return {}; // reply to the browser right away; checks run in the background }, // Our own errors ({status_code, body}) pass through; anything unexpected From 790c9bd7895525196d3c7217cc96fda21f0eceb1 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Fri, 2 Oct 2026 13:24:07 -0400 Subject: [PATCH 39/45] feat(storage-api): internal endpoints for REN (pull mode + monitoring); refs #147 --- storage-api/scripts/call-internal.js | 22 +++ storage-api/src/outbox/outbox.js | 20 +++ storage-api/src/outbox/worker.js | 54 ++++-- storage-api/src/routes/internal.js | 250 +++++++++++++++++++++++++++ storage-api/src/server.js | 7 +- 5 files changed, 335 insertions(+), 18 deletions(-) create mode 100644 storage-api/scripts/call-internal.js create mode 100644 storage-api/src/routes/internal.js diff --git a/storage-api/scripts/call-internal.js b/storage-api/scripts/call-internal.js new file mode 100644 index 0000000..d8434ce --- /dev/null +++ b/storage-api/scripts/call-internal.js @@ -0,0 +1,22 @@ +// Send a signed request to the storage API's /internal endpoints, as REN would. +// Usage: node scripts/call-internal.js [''] +// e.g. node scripts/call-internal.js GET /status +// node scripts/call-internal.js GET /uploads/ +// node scripts/call-internal.js POST /uploads//decision '{"decision":"commit","rawId":""}' +// node scripts/call-internal.js POST /uploads//ack '{"action":"keep"}' +import { config } from "../src/config.js"; +import { signRequest } from "../src/lib/storageSignature.js"; + +const [method = "GET", sub = "/status", json = ""] = process.argv.slice(2); +const path = `${config.basePath}/internal${sub}`; +const url = `http://${config.host}:${process.env.CALL_PORT || config.port}${path}`; +const headers = { + ...(json ? { "Content-Type": "application/json" } : {}), + ...signRequest( + { method, path, body: json }, + config.hmacKeys, + config.hmacActiveKid + ), +}; +const res = await fetch(url, { method, headers, body: json || undefined }); +console.log(res.status, await res.text()); diff --git a/storage-api/src/outbox/outbox.js b/storage-api/src/outbox/outbox.js index 6eb0343..0b4cae6 100644 --- a/storage-api/src/outbox/outbox.js +++ b/storage-api/src/outbox/outbox.js @@ -14,6 +14,26 @@ await fs.mkdir(FAILED, { recursive: true }); const fileIn = (dir, id) => path.join(dir, `${id}.json`); +// One change at a time per upload (worker vs. REN's /internal calls). +// In-memory is enough: the service runs as a single process. +const locks = new Map(); +export const withLock = async (id, fn) => { + while (locks.has(id)) await locks.get(id); + let release; + locks.set( + id, + new Promise((r) => { + release = r; + }) + ); + try { + return await fn(); + } finally { + locks.delete(id); + release(); + } +}; + // temp file → fsync → rename, so a crash never leaves a half-written record. const writeAtomic = async (file, obj) => { const tmp = `${file}.tmp`; diff --git a/storage-api/src/outbox/worker.js b/storage-api/src/outbox/worker.js index 4d6cb58..8cade43 100644 --- a/storage-api/src/outbox/worker.js +++ b/storage-api/src/outbox/worker.js @@ -15,6 +15,7 @@ import { readPending, saveRecord, closeRecord, + withLock, } from "./outbox.js"; import { incomingFile, rawObjectDir, rawZipKey } from "../lib/paths.js"; import { looksLikeZip, sha256File, fsyncDir } from "../lib/fileChecks.js"; @@ -41,7 +42,8 @@ const removeIncoming = async (id) => { await fs.rm(`${incomingFile(id)}.json`, { force: true }); }; -class Permanent extends Error {} +// A problem retrying won't fix (bad reply, files missing) → park the record. +export class Permanent extends Error {} const ready = (rec) => { rec.attempts = 0; @@ -113,8 +115,7 @@ const STEPS = { // Tell REN the file failed the checks, then delete it. async rejecting(rec) { await askRen(rec, { event: "failed", reason: rec.reason }); - await removeIncoming(rec.uploadId); - await finish(rec); + await applyAck(rec); }, // "verified" → REN answers commit (with rawId) or discard. @@ -159,12 +160,7 @@ const STEPS = { size: rec.size, sha256: rec.sha256, }); - if (reply.action === "discard") { - return setState(rec, "discarding_committed", { - reason: reply.reason || null, - }); - } - await finish(rec); + await applyAck(rec, reply); }, async discarding(rec) { @@ -200,6 +196,26 @@ export const applyDecision = async (rec, reply) => { throw new Permanent("REN sent an unknown decision"); }; +// REN's acknowledgement of a final result (push reply or pull /ack). +export const applyAck = async (rec, reply = {}) => { + if (rec.state === "committed") { + if (reply.action === "discard") { + return setState(rec, "discarding_committed", { + reason: reply.reason || null, + }); + } + if (reply.action === "keep") return finish(rec); + throw new Permanent( + "acknowledging a committed upload needs action keep or discard" + ); + } + if (rec.state === "rejecting") { + await removeIncoming(rec.uploadId); + return finish(rec); + } + throw new Permanent(`nothing to acknowledge in state ${rec.state}`); +}; + // Run one record forward until it waits, retries, or finishes. const advance = async (rec) => { for (let i = 0; i < 10; i++) { @@ -230,14 +246,18 @@ const tick = async () => { running = true; try { for (const id of await listPendingIds()) { - let rec; - try { - rec = await readPending(id); - } catch { - continue; // removed meanwhile - } - if (new Date(rec.nextAttemptAt) > new Date()) continue; - await advance(rec); + // Read fresh inside the lock, so REN's /internal calls and the worker + // never work on the same record at once (or on a stale copy). + await withLock(id, async () => { + let rec; + try { + rec = await readPending(id); + } catch { + return; // finished or removed meanwhile + } + if (new Date(rec.nextAttemptAt) > new Date()) return; + await advance(rec); + }); } } catch (e) { console.error("outbox worker:", e.message); diff --git a/storage-api/src/routes/internal.js b/storage-api/src/routes/internal.js new file mode 100644 index 0000000..c88ee83 --- /dev/null +++ b/storage-api/src/routes/internal.js @@ -0,0 +1,250 @@ +// Server-to-server endpoints for REN (never the browser). Every request must +// be HMAC-signed with the shared secret; REN signs the full path it calls +// (BASE_PATH included, which Apache keeps when proxying). Used by REN's pull +// reconciliation (dev, and as a safety net in prod) and for monitoring. +import fs from "node:fs/promises"; +import express from "express"; +import { config } from "../config.js"; +import { verifyRequest } from "../lib/storageSignature.js"; +import { incomingFile } from "../lib/paths.js"; +import { + findRecord, + readPending, + listPendingIds, + withLock, + closeRecord, +} from "../outbox/outbox.js"; +import { + applyDecision, + applyAck, + Permanent, + wakeWorker, +} from "../outbox/worker.js"; + +const UUID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const router = express.Router(); + +router.use( + express.json({ + limit: "64kb", + verify: (req, res, buf) => { + req.rawBody = buf; + }, + }) +); +router.use((req, res, next) => { + const r = verifyRequest( + { + method: req.method, + path: req.originalUrl, + headers: req.headers, + rawBody: req.rawBody || "", + }, + config.hmacKeys + ); + if (!r.ok) { + console.warn("Rejected internal call:", r.reason, req.method, req.originalUrl); + return res.status(401).json({ error: "Invalid service signature." }); + } + next(); +}); + +const httpError = (status, message) => + Object.assign(new Error(message), { status }); + +const uploadId = (req) => { + const id = String(req.params.uploadId); + if (!UUID_RE.test(id)) throw httpError(404, "Unknown upload."); + return id; +}; + +const sendError = (res, err) => { + if (err instanceof Permanent) return res.status(409).json({ error: err.message }); + if (err.status) return res.status(err.status).json({ error: err.message }); + console.error("internal endpoint:", err.message); + res.status(500).json({ error: "Internal error." }); +}; + +const removeIncoming = async (id) => { + await fs.rm(incomingFile(id), { force: true }); + await fs.rm(`${incomingFile(id)}.json`, { force: true }); +}; + +// What the disk says about one upload. +const describe = async (id) => { + const found = await findRecord(id); + if (found) { + const r = found.rec; + return { + uploadId: id, + phase: r.state, + where: found.where, + size: r.size ?? r.expectedSize, + sha256: r.sha256 || null, + rawId: r.rawId || null, + storageKey: r.storageKey || null, + reason: r.reason || null, + lastError: r.lastError || null, + updatedAt: r.updatedAt, + }; + } + try { + const st = await fs.stat(incomingFile(id)); + let size = null; + try { + size = + JSON.parse(await fs.readFile(`${incomingFile(id)}.json`, "utf8")).size ?? + null; + } catch { + // no tus metadata + } + return { + uploadId: id, + phase: "incoming", + offset: st.size, + size, + lastActivityAt: st.mtime.toISOString(), + }; + } catch { + return { uploadId: id, phase: "none" }; + } +}; + +// GET /internal/uploads/:uploadId +router.get("/uploads/:uploadId", async (req, res) => { + try { + res.json(await describe(uploadId(req))); + } catch (err) { + sendError(res, err); + } +}); + +// POST /internal/uploads/:uploadId/decision +// {decision: "commit"|"discard", rawId?, storageKey?, reason?} +router.post("/uploads/:uploadId/decision", async (req, res) => { + try { + const id = uploadId(req); + const body = req.body || {}; + const state = await withLock(id, async () => { + const found = await findRecord(id); + if (!found) throw httpError(409, "Upload has not finished uploading."); + const rec = found.rec; + if (rec.state === "awaiting_decision" && found.where === "pending") { + await applyDecision(rec, body); + return rec.state; + } + // Repeat of a decision that already took effect → same answer, no change. + const committedAlready = [ + "committing", + "committed", + "discarding_committed", + "done", + ].includes(rec.state); + if (body.decision === "commit" && committedAlready && rec.rawId === body.rawId) { + return rec.state; + } + if ( + body.decision === "discard" && + ["discarding", "done"].includes(rec.state) && + !rec.rawId + ) { + return rec.state; + } + throw new Permanent(`cannot apply that decision in state ${rec.state}`); + }); + wakeWorker(); // committing / discarding continue right away + res.json({ ok: true, state }); + } catch (err) { + sendError(res, err); + } +}); + +// POST /internal/uploads/:uploadId/ack {action?: "keep"|"discard", reason?} +router.post("/uploads/:uploadId/ack", async (req, res) => { + try { + const id = uploadId(req); + const state = await withLock(id, async () => { + const found = await findRecord(id); + if (!found) throw httpError(404, "Unknown upload."); + if (found.where !== "pending") return found.rec.state; // finished → idempotent + await applyAck(found.rec, req.body || {}); + return found.rec.state; + }); + wakeWorker(); + res.json({ ok: true, state }); + } catch (err) { + sendError(res, err); + } +}); + +// DELETE /internal/uploads/:uploadId — leftovers of a cancelled/expired upload. +// Never removes a committed object (that's the cleanup job's decision). +router.delete("/uploads/:uploadId", async (req, res) => { + try { + const id = uploadId(req); + const result = await withLock(id, async () => { + const found = await findRecord(id); + if (found && found.where === "pending") { + if ( + ["committing", "committed", "discarding_committed"].includes( + found.rec.state + ) + ) { + throw new Permanent( + "upload is already committed; objects are removed by cleanup, not here" + ); + } + await removeIncoming(id); + found.rec.reason = "deleted by REN"; + found.rec.state = "done"; + await closeRecord(found.rec, "done"); + return "removed"; + } + if (found) return "nothing to remove"; // already done/failed + await removeIncoming(id); + return "removed"; + }); + res.json({ ok: true, result }); + } catch (err) { + sendError(res, err); + } +}); + +// GET /internal/status — for REN's monitoring (never public). +router.get("/status", async (req, res) => { + try { + const s = await fs.statfs(config.uploadRoot); + const byState = {}; + let oldest = null; + for (const id of await listPendingIds()) { + try { + const r = await readPending(id); + byState[r.state] = (byState[r.state] || 0) + 1; + if (!oldest || r.updatedAt < oldest.since) { + oldest = { uploadId: id, state: r.state, since: r.updatedAt }; + } + } catch { + // moved meanwhile + } + } + const count = async (sub) => + (await fs.readdir(`${config.outboxDir}/${sub}`)).filter((n) => + n.endsWith(".json") + ).length; + res.json({ + disk: { freeBytes: s.bavail * s.bsize, totalBytes: s.blocks * s.bsize }, + outbox: { + pending: byState, + done: await count("done"), + failed: await count("failed"), + }, + oldestPending: oldest, + pushes: config.mainApiUrl ? "on" : "off", + }); + } catch (err) { + sendError(res, err); + } +}); + +export default router; diff --git a/storage-api/src/server.js b/storage-api/src/server.js index 0d141ca..885e7a3 100644 --- a/storage-api/src/server.js +++ b/storage-api/src/server.js @@ -1,10 +1,12 @@ // NeuroJSON storage API (Zodiac). Listens on 127.0.0.1; Apache adds TLS and -// proxies /neurojson-storage/* here. /internal routes are added in a later step. +// proxies /neurojson-storage/* here: /health (public), /files (tus uploads, +// token-checked), /internal (REN only, HMAC-signed). import fs from "node:fs/promises"; import express from "express"; import { config } from "./config.js"; import { tusServer } from "./tus/tusServer.js"; import { startWorker, stopWorker } from "./outbox/worker.js"; +import internalRoutes from "./routes/internal.js"; const app = express(); app.disable("x-powered-by"); @@ -25,6 +27,9 @@ const tusHandler = (req, res) => tusServer.handle(req, res); app.all(`${config.basePath}/files`, tusHandler); app.all(`${config.basePath}/files/*splat`, tusHandler); +// REN → Zodiac service calls (HMAC-signed): status, decisions, acks. +app.use(`${config.basePath}/internal`, internalRoutes); + app.use((req, res) => res.status(404).json({ error: "Not found." })); const server = app.listen(config.port, config.host, () => { From 86a59ae05766dec0e2fec56c857fb0bbd064e659 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Fri, 2 Oct 2026 13:38:47 -0400 Subject: [PATCH 40/45] feat(uploads): reconciliation job that finishes raw uploads by pulling; refs #147 --- backend/scripts/reconcile-raw-once.js | 10 + .../controllers/storageEvents.controller.js | 32 ++-- backend/src/jobs/reconcileRawUploads.js | 180 ++++++++++++++++++ backend/src/lib/storageClient.js | 55 ++++++ backend/src/server.js | 3 + 5 files changed, 266 insertions(+), 14 deletions(-) create mode 100644 backend/scripts/reconcile-raw-once.js create mode 100644 backend/src/jobs/reconcileRawUploads.js create mode 100644 backend/src/lib/storageClient.js diff --git a/backend/scripts/reconcile-raw-once.js b/backend/scripts/reconcile-raw-once.js new file mode 100644 index 0000000..740aeea --- /dev/null +++ b/backend/scripts/reconcile-raw-once.js @@ -0,0 +1,10 @@ +// Run ONE raw-upload reconciliation pass now and print what happened. +// Usage: node scripts/reconcile-raw-once.js +require("dotenv").config({ quiet: true }); +const { reconcileOnce } = require("../src/jobs/reconcileRawUploads"); +const { sequelize } = require("../src/config/database"); + +reconcileOnce() + .then((s) => console.log("summary:", s)) + .catch((e) => console.error("failed:", e.message)) + .finally(() => sequelize.close()); diff --git a/backend/src/controllers/storageEvents.controller.js b/backend/src/controllers/storageEvents.controller.js index e5a15fb..533e148 100644 --- a/backend/src/controllers/storageEvents.controller.js +++ b/backend/src/controllers/storageEvents.controller.js @@ -198,22 +198,26 @@ const HANDLERS = { failed: onFailed, }; +// Run one storage event for an upload (shared by the HTTP endpoint for +// pushes and by the reconciliation job for pulls). Returns REN's answer. +const processStorageEvent = async (uploadId, body = {}) => { + if (!UUID_RE.test(uploadId)) { + throw new HttpError(404, { error: "Unknown upload." }); + } + const handler = HANDLERS[body.event]; + if (!handler) throw new HttpError(400, { error: "Unknown event." }); + return sequelize.transaction(async (t) => { + const a = await lockAttempt(uploadId, t); + return handler(a, body, t); + }); +}; + // POST /api/v1/internal/storage/uploads/:uploadId/events {event, ...} const handleUploadEvent = async (req, res) => { try { - const uploadId = String(req.params.uploadId); - if (!UUID_RE.test(uploadId)) { - throw new HttpError(404, { error: "Unknown upload." }); - } - const body = req.body || {}; - const handler = HANDLERS[body.event]; - if (!handler) throw new HttpError(400, { error: "Unknown event." }); - - const out = await sequelize.transaction(async (t) => { - const a = await lockAttempt(uploadId, t); - return handler(a, body, t); - }); - res.json(out); + res.json( + await processStorageEvent(String(req.params.uploadId), req.body || {}) + ); } catch (err) { if (err instanceof HttpError) return res.status(err.status).json(err.body); console.error("Storage event failed:", err.message); @@ -222,4 +226,4 @@ const handleUploadEvent = async (req, res) => { } }; -module.exports = { handleUploadEvent }; +module.exports = { handleUploadEvent, processStorageEvent }; diff --git a/backend/src/jobs/reconcileRawUploads.js b/backend/src/jobs/reconcileRawUploads.js new file mode 100644 index 0000000..db0f425 --- /dev/null +++ b/backend/src/jobs/reconcileRawUploads.js @@ -0,0 +1,180 @@ +// Reconciliation: REN asks the Zodiac storage API about raw uploads that +// aren't finished and finishes them. Required in dev (Zodiac can't reach the +// developer's machine, so pushes are off); a safety net in prod. Decisions +// reuse processStorageEvent, so pull and push can never decide differently. +// Assumes ONE backend process (add a Postgres advisory lock if that changes). +const { sequelize } = require("../config/database"); +const { + processStorageEvent, +} = require("../controllers/storageEvents.controller"); +const storage = require("../lib/storageClient"); + +const INTERVAL_MS = Number(process.env.RAW_RECONCILE_INTERVAL_MS ?? 600000); // 0 = off +const MIN_AGE_MS = Number(process.env.RAW_RECONCILE_MIN_AGE_MS ?? 1800000); // let pushes happen first +const EXPIRE_MS = Number(process.env.RAW_INACTIVE_EXPIRE_DAYS ?? 7) * 86400000; +const BATCH = 50; +const ACTIVE = ["initiated", "uploading", "verifying"]; + +const q = (sql, replacements) => + sequelize.query(sql, { replacements, type: sequelize.QueryTypes.SELECT }); +const run = (sql, replacements) => sequelize.query(sql, { replacements }); + +const touch = (id) => + run( + `UPDATE raw_upload_attempts SET last_checked_at = NOW() WHERE upload_id = :id`, + { id } + ); +const expire = (id, msg) => + run( + `UPDATE raw_upload_attempts + SET status = 'expired', error = :msg, completed_at = NOW(), updated_at = NOW() + WHERE upload_id = :id AND status IN (:active)`, + { id, msg, active: ACTIVE } + ); +const idleTooLong = (since) => + Date.now() - new Date(since).getTime() > EXPIRE_MS; + +// One unfinished attempt: look at Zodiac, then take the matching step. +const reconcileActive = async (a) => { + const id = a.upload_id; + const st = await storage.getUpload(id); + switch (st.phase) { + case "none": + if (idleTooLong(a.updated_at)) { + await expire(id, "The upload was never started."); + } + break; + case "incoming": + if (idleTooLong(st.lastActivityAt)) { + await expire(id, "The upload stopped and expired."); + await storage.deleteUpload(id); + } else { + await run( + `UPDATE raw_upload_attempts SET status = 'uploading', updated_at = NOW() + WHERE upload_id = :id AND status = 'initiated'`, + { id } + ); + } + break; + case "awaiting_decision": { + const reply = await processStorageEvent(id, { + event: "verified", + size: st.size, + sha256: st.sha256, + }); + await storage.sendDecision(id, reply); + break; + } + case "committed": { + const reply = await processStorageEvent(id, { + event: "committed", + size: st.size, + sha256: st.sha256, + }); + await storage.sendAck(id, reply); + break; + } + case "rejecting": + await processStorageEvent(id, { event: "failed", reason: st.reason }); + await storage.sendAck(id); + break; + case "failed": // parked on Zodiac for an admin + await processStorageEvent(id, { + event: "failed", + reason: "storage_error", + }); + break; + default: + // verifying / committing / discarding / done: Zodiac is busy (or + // finished a discard) — check again next run. + break; + } + await touch(id); + return st.phase; +}; + +// A finished-in-REN attempt (cancelled/failed/expired): make sure Zodiac +// drops whatever it still holds. Never touches committed objects REN kept. +const cleanUpFinished = async (a) => { + const id = a.upload_id; + const st = await storage.getUpload(id); + if (st.phase === "incoming") { + await storage.deleteUpload(id); + } else if (st.phase === "awaiting_decision") { + await storage.sendDecision(id, { decision: "discard", reason: a.status }); + } else if (st.phase === "committed") { + await storage.sendAck(id, { action: "discard", reason: a.status }); + } else if (st.phase === "rejecting") { + await storage.sendAck(id); + } + await touch(id); // cleaned up → skipped next time + return st.phase; +}; + +let running = false; + +// One full pass. Returns a short summary (used by the one-shot script too). +const reconcileOnce = async () => { + if (running) return { skipped: true }; + running = true; + const summary = { active: 0, cleaned: 0, errors: 0 }; + try { + const active = await q( + `SELECT upload_id, status, updated_at FROM raw_upload_attempts + WHERE status IN (:active) + AND updated_at < NOW() - (:minAge * INTERVAL '1 millisecond') + ORDER BY updated_at LIMIT ${BATCH}`, + { active: ACTIVE, minAge: MIN_AGE_MS } + ); + for (const a of active) { + try { + const phase = await reconcileActive(a); + summary.active++; + console.log(`reconcile ${a.upload_id}: Zodiac says ${phase}`); + } catch (e) { + summary.errors++; + console.warn(`reconcile ${a.upload_id} failed (will retry):`, e.message); + } + } + const finished = await q( + `SELECT upload_id, status FROM raw_upload_attempts + WHERE status IN ('cancelled','failed','expired') + AND (last_checked_at IS NULL OR last_checked_at < completed_at) + ORDER BY completed_at LIMIT ${BATCH}` + ); + for (const a of finished) { + try { + const phase = await cleanUpFinished(a); + summary.cleaned++; + console.log(`cleanup ${a.upload_id} (${a.status}): Zodiac had ${phase}`); + } catch (e) { + summary.errors++; + console.warn(`cleanup ${a.upload_id} failed (will retry):`, e.message); + } + } + } finally { + running = false; + } + return summary; +}; + +const startRawReconciler = () => { + if (!INTERVAL_MS) { + return console.log("raw reconciler: off (RAW_RECONCILE_INTERVAL_MS=0)"); + } + if (!storage.configured()) { + return console.log( + "raw reconciler: off (no STORAGE_INTERNAL_URL / STORAGE_PUBLIC_URL)" + ); + } + console.log( + `raw reconciler: every ${INTERVAL_MS / 1000}s (min age ${MIN_AGE_MS / 1000}s)` + ); + setInterval( + () => + reconcileOnce().catch((e) => console.error("raw reconciler:", e.message)), + INTERVAL_MS + ); +}; + +module.exports = { reconcileOnce, startRawReconciler }; diff --git a/backend/src/lib/storageClient.js b/backend/src/lib/storageClient.js new file mode 100644 index 0000000..0c8cc05 --- /dev/null +++ b/backend/src/lib/storageClient.js @@ -0,0 +1,55 @@ +// Signed calls from REN to the Zodiac storage API's /internal endpoints +// (HMAC, same scheme as Zodiac → REN). Throws on any non-2xx so callers can +// simply try again on the next run. +const { signRequest } = require("./storageSignature"); + +const base = () => + ( + process.env.STORAGE_INTERNAL_URL || + process.env.STORAGE_PUBLIC_URL || + "" + ).replace(/\/+$/, ""); + +const call = async (method, sub, body) => { + const url = new URL(`${base()}/internal${sub}`); + const json = body === undefined ? "" : JSON.stringify(body); + const headers = { + ...(json ? { "Content-Type": "application/json" } : {}), + // Zodiac verifies against its req.originalUrl = this exact path + query. + ...signRequest({ method, path: url.pathname + url.search, body: json }), + }; + const res = await fetch(url, { + method, + headers, + body: json || undefined, + signal: AbortSignal.timeout(30_000), + }); + const text = await res.text(); + if (!res.ok) { + throw new Error( + `storage ${method} ${sub} → HTTP ${res.status}: ${text.slice(0, 200)}` + ); + } + return JSON.parse(text); +}; + +module.exports = { + configured: () => Boolean(base()), + getUpload: (id) => call("GET", `/uploads/${id}`), + // Forward REN's answer exactly as it would be returned to a push. + sendDecision: (id, reply) => + call("POST", `/uploads/${id}/decision`, { + decision: reply.decision, + rawId: reply.rawId, + storageKey: reply.storageKey, + reason: reply.reason, + }), + sendAck: (id, reply = {}) => + call( + "POST", + `/uploads/${id}/ack`, + reply.action ? { action: reply.action, reason: reply.reason } : {} + ), + deleteUpload: (id) => call("DELETE", `/uploads/${id}`), + status: () => call("GET", "/status"), +}; diff --git a/backend/src/server.js b/backend/src/server.js index df6f340..63c376b 100644 --- a/backend/src/server.js +++ b/backend/src/server.js @@ -17,6 +17,7 @@ const ollamaRoutes = require("./routes/ollama.routes"); const ollamaPublicRoutes = require("./routes/ollama.public.routes"); const uploadsRoutes = require("./routes/uploads.routes"); const internalRoutes = require("./routes/internal.routes"); +const { startRawReconciler } = require("./jobs/reconcileRawUploads"); const app = express(); const PORT = process.env.PORT || 5000; @@ -106,6 +107,8 @@ const startServer = async () => { console.log(`🚀 Server is running on port ${PORT}`); console.log(`📡 API available at http://localhost:${PORT}/api`); console.log(`🌍 Environment: ${process.env.NODE_ENV || "development"}`); + // Finish raw uploads by asking the storage API (needed in pull mode). + startRawReconciler(); }); } catch (error) { console.error("❌ Failed to start server:", error); From 34e321c5c29a10bb7a828eed5e4dca25745e80e1 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Fri, 2 Oct 2026 13:54:33 -0400 Subject: [PATCH 41/45] feat(uploads): frontend service for raw ZIP uploads; refs #147 --- .../src/controllers/rawUploads.controller.js | 4 +- src/services/rawUpload.service.ts | 135 ++++++++++++++++++ src/utils/formatBytes.ts | 14 ++ storage-api/.env.example | 2 +- storage-api/src/config.js | 2 +- 5 files changed, 153 insertions(+), 4 deletions(-) create mode 100644 src/services/rawUpload.service.ts create mode 100644 src/utils/formatBytes.ts diff --git a/backend/src/controllers/rawUploads.controller.js b/backend/src/controllers/rawUploads.controller.js index bf56859..111fedd 100644 --- a/backend/src/controllers/rawUploads.controller.js +++ b/backend/src/controllers/rawUploads.controller.js @@ -11,8 +11,8 @@ const { const { signUploadToken } = require("../lib/uploadTokens"); const MAX_UPLOAD_BYTES = Number( - process.env.RAW_MAX_UPLOAD_BYTES || 536870912000 -); // 500 GB + process.env.RAW_MAX_UPLOAD_BYTES || 500000000000 +); // 500 GB (decimal, matches the size shown in the UI) const CHUNK_BYTES = Number(process.env.RAW_CHUNK_BYTES || 67108864); // 64 MB const ACTIVE = ["initiated", "uploading", "verifying"]; const EDITABLE = ["draft", "changes_requested"]; diff --git a/src/services/rawUpload.service.ts b/src/services/rawUpload.service.ts new file mode 100644 index 0000000..d65781d --- /dev/null +++ b/src/services/rawUpload.service.ts @@ -0,0 +1,135 @@ +import { baseURL } from "services/instance"; + +// Calls to REN's raw ZIP routes (/api/v1/uploads/:internalId/raw…). The file +// itself never goes through REN — the browser sends it to the storage API +// with tus (see utils/rawUploader.ts), using the token these routes return. + +export type RawAttemptStatus = + | "initiated" + | "uploading" + | "verifying" + | "complete" + | "failed" + | "cancelled" + | "expired"; + +export type RawExtractionStatus = + | "not_extracted" + | "extracting" + | "extracted" + | "failed"; + +// One upload attempt (the active one, or the last failed/cancelled/expired one). +export interface RawUploadSummary { + uploadId: string; + status: RawAttemptStatus; + filename: string | null; + size: number; + error: string | null; + createdAt: string; + updatedAt: string; + completedAt: string | null; +} + +// The ZIP this version currently uses. +export interface RawCurrentZip { + rawId: string; + filename: string | null; + size: number; + sha256: string; + extractionStatus: RawExtractionStatus; + carriedForward: boolean; + uploadedAt: string; +} + +export interface RawStatus { + rawZipExpected: boolean; + editable: boolean; + current: RawCurrentZip | null; + activeUpload: RawUploadSummary | null; + lastUpload: RawUploadSummary | null; +} + +// What REN returns when an upload starts (or resumes). +export interface RawUploadSession { + uploadId: string; + endpoint: string; // storage API tus endpoint, e.g. https://…/neurojson-storage/files + token: string; // short-lived upload token (memory only — never stored) + expiresAt: string; + chunkSize: number; + resumed: boolean; // true = continue the existing upload for this file +} + +export interface RawUploadToken { + token: string; + expiresAt: string; +} + +// Error from a raw route that keeps REN's code (e.g. ACTIVE_UPLOAD_EXISTS, +// FILE_TOO_LARGE, NOT_EDITABLE) and extra fields (e.g. the other upload), so +// the UI can react to each case instead of only showing a message. +export class RawUploadError extends Error { + status: number; + code?: string; + details: any; + + constructor(message: string, status: number, code?: string, details?: any) { + super(message); + this.name = "RawUploadError"; + this.status = status; + this.code = code; + this.details = details; + } +} + +const request = async ( + method: string, + path: string, + body?: unknown +): Promise => { + const res = await fetch(`${baseURL}/uploads${path}`, { + method, + credentials: "include", + headers: + body !== undefined ? { "Content-Type": "application/json" } : undefined, + body: body !== undefined ? JSON.stringify(body) : undefined, + }); + const data = await res.json().catch(() => ({})); + if (!res.ok) { + throw new RawUploadError( + data.error || `Request failed (${res.status})`, + res.status, + data.code, + data + ); + } + return data as T; +}; + +export const RawUploadService = { + // Current ZIP + active/last upload for the dataset's latest version. + getStatus: (internalId: string) => + request("GET", `/${internalId}/raw`), + + // Start an upload, or get the active one back (same name + size → resumed). + start: (internalId: string, file: { name: string; size: number }) => + request("POST", `/${internalId}/raw`, { + filename: file.name, + size: file.size, + }), + + // Fresh token for a long upload (tokens last 15 minutes). + refreshToken: (internalId: string, uploadId: string) => + request("POST", `/${internalId}/raw/${uploadId}/token`), + + cancel: (internalId: string, uploadId: string) => + request<{ status: string }>("DELETE", `/${internalId}/raw/${uploadId}`), + + // "This dataset includes raw data" checkbox. + setExpected: (internalId: string, expected: boolean) => + request<{ rawZipExpected: boolean }>( + "PATCH", + `/${internalId}/raw/expected`, + { expected } + ), +}; diff --git a/src/utils/formatBytes.ts b/src/utils/formatBytes.ts new file mode 100644 index 0000000..58cd1e1 --- /dev/null +++ b/src/utils/formatBytes.ts @@ -0,0 +1,14 @@ +// Human-readable size in DECIMAL units (1 GB = 1,000,000,000 bytes), matching +// the "500 GB max" limit message. +export const formatBytes = (bytes: number): string => { + if (!Number.isFinite(bytes) || bytes < 0) return "—"; + if (bytes < 1000) return `${bytes} B`; + const units = ["KB", "MB", "GB", "TB"]; + let value = bytes / 1000; + let i = 0; + while (value >= 1000 && i < units.length - 1) { + value /= 1000; + i++; + } + return `${value.toFixed(value < 10 ? 1 : 0)} ${units[i]}`; +}; diff --git a/storage-api/.env.example b/storage-api/.env.example index ed300cd..20cd97d 100644 --- a/storage-api/.env.example +++ b/storage-api/.env.example @@ -18,5 +18,5 @@ STORAGE_HMAC_ACTIVE_KID=dev-2026-09 STORAGE_HMAC_KEYS='{"dev-2026-09":"..."}' # Limits -MAX_UPLOAD_BYTES=536870912000 +MAX_UPLOAD_BYTES=500000000000 DISK_RESERVE_PERCENT=5 diff --git a/storage-api/src/config.js b/storage-api/src/config.js index 1db3b5d..b4526c3 100644 --- a/storage-api/src/config.js +++ b/storage-api/src/config.js @@ -76,7 +76,7 @@ export const config = { tokenPublicKeys, hmacKeys, hmacActiveKid, - maxUploadBytes: num("MAX_UPLOAD_BYTES", 536870912000), + maxUploadBytes: num("MAX_UPLOAD_BYTES", 500000000000), // 500 GB (decimal) diskReservePercent: num("DISK_RESERVE_PERCENT", 5), }; From 0d5fc0a63f999d854aba4d580057fe8b04182f0e Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Fri, 2 Oct 2026 14:00:06 -0400 Subject: [PATCH 42/45] feat(uploads): browser raw ZIP uploader on tus-js-client; refs #147 --- package.json | 1 + src/utils/rawUploader.ts | 173 +++++++++++++++++++++++++++++++++++++++ yarn.lock | 99 +++++++++++++++++++++- 3 files changed, 271 insertions(+), 2 deletions(-) create mode 100644 src/utils/rawUploader.ts diff --git a/package.json b/package.json index 937f15d..8c837f6 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "stats-js": "^1.0.1", "stats.js": "0.17.0", "three": "0.145.0", + "tus-js-client": "^4.3.1", "typescript": "^5.1.6", "uplot": "1.6.32", "web-vitals": "^2.1.0", diff --git a/src/utils/rawUploader.ts b/src/utils/rawUploader.ts new file mode 100644 index 0000000..677d3cf --- /dev/null +++ b/src/utils/rawUploader.ts @@ -0,0 +1,173 @@ +import { + RawUploadError, + RawUploadService, + RawUploadSession, +} from "services/rawUpload.service"; +import * as tus from "tus-js-client"; + +// Sends one raw ZIP from the browser straight to the storage API with tus, +// resumably, in chunks. REN only hands out the session and short-lived +// tokens; the token is kept in memory (never in localStorage). + +export interface RawUploaderCallbacks { + onProgress?: (sent: number, total: number) => void; + onFinished?: () => void; // every byte stored — the server now verifies it + onError?: (message: string, canResume: boolean) => void; +} + +const REFRESH_MARGIN_MS = 3 * 60 * 1000; // renew tokens with < 3 min left +// Statuses where retrying the same request can't help. +const STOP_STATUSES = new Set([400, 403, 404, 409, 410, 413, 507]); + +export const rawUploadSupported = tus.isSupported; + +// Turn any upload error into a short message + whether Resume makes sense. +const describeError = (err: any): { message: string; canResume: boolean } => { + const cause = err?.causingError ?? err; + if (cause instanceof RawUploadError) { + // A token refresh with REN failed. + if (cause.status === 401) { + return { + message: + "Your login session ended. Log in again, then choose the same file to continue.", + canResume: false, + }; + } + return { message: cause.message, canResume: false }; + } + const status: number = err?.originalResponse?.getStatus?.() ?? 0; + const body: string = (err?.originalResponse?.getBody?.() || "").trim(); + if (STOP_STATUSES.has(status)) { + return { + message: body || `The upload was refused (HTTP ${status}).`, + canResume: false, + }; + } + if (status === 401) { + return { + message: "The upload permission expired. Click Resume to continue.", + canResume: true, + }; + } + return { + message: "The connection was interrupted. Click Resume to continue.", + canResume: true, + }; +}; + +export class RawUploader { + readonly internalId: string; + readonly file: File; + private session: RawUploadSession; + private callbacks: RawUploaderCallbacks; + private upload: tus.Upload | null = null; + private token: string; + private tokenExpiresAt: number; + + private constructor( + internalId: string, + file: File, + session: RawUploadSession, + callbacks: RawUploaderCallbacks + ) { + this.internalId = internalId; + this.file = file; + this.session = session; + this.callbacks = callbacks; + this.token = session.token; + this.tokenExpiresAt = Date.parse(session.expiresAt); + } + + // Ask REN for a session (new, or the existing one for this same file) and + // start sending. Throws RawUploadError, e.g. ACTIVE_UPLOAD_EXISTS when a + // DIFFERENT file is already being uploaded for this version. + static async begin( + internalId: string, + file: File, + callbacks: RawUploaderCallbacks + ): Promise { + const session = await RawUploadService.start(internalId, { + name: file.name, + size: file.size, + }); + const uploader = new RawUploader(internalId, file, session, callbacks); + uploader.start(); + return uploader; + } + + get uploadId() { + return this.session.uploadId; + } + + get resumed() { + return this.session.resumed; + } + + private async currentToken(): Promise { + if (Date.now() < this.tokenExpiresAt - REFRESH_MARGIN_MS) return this.token; + const fresh = await RawUploadService.refreshToken( + this.internalId, + this.session.uploadId + ); + this.token = fresh.token; + this.tokenExpiresAt = Date.parse(fresh.expiresAt); + return this.token; + } + + private start() { + const { endpoint, uploadId, chunkSize, resumed } = this.session; + this.upload = new tus.Upload(this.file, { + endpoint, + // The upload id is known, so resume exactly that upload: tus asks the + // server for its offset and continues. (If the server has nothing yet, + // tus creates it — same id, since the server names uploads by token.) + uploadUrl: resumed ? `${endpoint}/${uploadId}` : null, + chunkSize, + metadata: { + filename: this.file.name, // display only; never used for paths + filetype: this.file.type || "application/zip", + }, + storeFingerprintForResuming: false, // we resume by upload id instead + retryDelays: [0, 1000, 3000, 5000, 10000, 20000, 30000, 60000], + onBeforeRequest: async (req) => { + req.setHeader("Authorization", `Bearer ${await this.currentToken()}`); + }, + onShouldRetry: (err) => { + const status = err.originalResponse?.getStatus() ?? 0; + if (status === 401) { + this.tokenExpiresAt = 0; // next attempt fetches a fresh token + return true; + } + return !STOP_STATUSES.has(status); // network errors, 5xx, 423 → retry + }, + onProgress: (sent, total) => this.callbacks.onProgress?.(sent, total), + onSuccess: () => this.callbacks.onFinished?.(), + onError: (err) => { + const { message, canResume } = describeError(err); + this.callbacks.onError?.(message, canResume); + }, + }); + this.upload.start(); + } + + // Stop sending; the server keeps what it has. + async pause() { + await this.upload?.abort(false); + } + + // Continue from the server's offset (tus re-checks it first). + resume() { + this.upload?.start(); + } + + // Stop, ask the storage API to drop the partial data, and tell REN. If the + // storage call fails, REN's cleanup job removes the leftovers anyway. + async cancel() { + try { + await this.upload?.abort(true); + } catch { + // already finished or not created yet — REN's cancel still applies + } + await RawUploadService.cancel(this.internalId, this.session.uploadId); + } +} diff --git a/yarn.lock b/yarn.lock index b91968c..b993abe 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4293,7 +4293,7 @@ bser@2.1.1: dependencies: node-int64 "^0.4.0" -buffer-from@^1.0.0: +buffer-from@^1.0.0, buffer-from@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== @@ -4664,6 +4664,14 @@ colorette@^2.0.10: resolved "https://registry.yarnpkg.com/colorette/-/colorette-2.0.20.tgz#9eb793e6833067f7235902fcd3b09917a000a95a" integrity sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w== +combine-errors@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/combine-errors/-/combine-errors-3.0.3.tgz#f4df6740083e5703a3181110c2b10551f003da86" + integrity sha512-C8ikRNRMygCwaTx+Ek3Yr+OuZzgZjduCOfSQBjbM8V3MfgcjSTeto/GXP6PAwKvJz/v15b7GHZvx5rOlczFw/Q== + dependencies: + custom-error-instance "2.1.1" + lodash.uniqby "4.5.0" + combined-stream@^1.0.8: version "1.0.8" resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" @@ -5230,6 +5238,11 @@ csstype@^3.0.2, csstype@^3.1.3: resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.1.3.tgz#d80ff294d114fb0e6ac500fbf85b60137d7eff81" integrity sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw== +custom-error-instance@2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/custom-error-instance/-/custom-error-instance-2.1.1.tgz#3cf6391487a6629a6247eb0ca0ce00081b7e361a" + integrity sha512-p6JFxJc3M4OTD2li2qaHkDCw9SfMw82Ldr6OC9Je1aXiGfhx2W8p3GaoeaGrPJTUN9NirTM/KTxHWMUdR1rsUg== + cwise-compiler@^1.0.0, cwise-compiler@^1.1.1: version "1.1.3" resolved "https://registry.yarnpkg.com/cwise-compiler/-/cwise-compiler-1.1.3.tgz#f4d667410e850d3a313a7d2db7b1e505bb034cc5" @@ -8506,6 +8519,11 @@ jquery@^3.7.1: resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.7.1.tgz#083ef98927c9a6a74d05a6af02806566d16274de" integrity sha512-m4avr8yL8kmFN8psrbFFFmB/If14iN5o9nw/NgnnM+kybDJpRsAynV2BsfpTYrTRysYUdADVD7CkUUizgkpLfg== +js-base64@^3.7.2: + version "3.9.4" + resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-3.9.4.tgz#bc4d811fd52105848df55bc80ca401d963020e6d" + integrity sha512-PtOMXpEGuP0RRiRXsjzHzl44dMHxSu2CPvAhinupR1tBa88me+1DPqsobl7eupn5UukjLkln1ZnAOAOXOrYG0Q== + "js-tokens@^3.0.0 || ^4.0.0", js-tokens@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" @@ -8849,6 +8867,43 @@ lodash-es@4, lodash-es@^4.17.21: resolved "https://registry.yarnpkg.com/lodash-es/-/lodash-es-4.17.21.tgz#43e626c46e6591b7750beb2b50117390c609e3ee" integrity sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw== +lodash._baseiteratee@~4.7.0: + version "4.7.0" + resolved "https://registry.yarnpkg.com/lodash._baseiteratee/-/lodash._baseiteratee-4.7.0.tgz#34a9b5543572727c3db2e78edae3c0e9e66bd102" + integrity sha512-nqB9M+wITz0BX/Q2xg6fQ8mLkyfF7MU7eE+MNBNjTHFKeKaZAPEzEg+E8LWxKWf1DQVflNEn9N49yAuqKh2mWQ== + dependencies: + lodash._stringtopath "~4.8.0" + +lodash._basetostring@~4.12.0: + version "4.12.0" + resolved "https://registry.yarnpkg.com/lodash._basetostring/-/lodash._basetostring-4.12.0.tgz#9327c9dc5158866b7fa4b9d42f4638e5766dd9df" + integrity sha512-SwcRIbyxnN6CFEEK4K1y+zuApvWdpQdBHM/swxP962s8HIxPO3alBH5t3m/dl+f4CMUug6sJb7Pww8d13/9WSw== + +lodash._baseuniq@~4.6.0: + version "4.6.0" + resolved "https://registry.yarnpkg.com/lodash._baseuniq/-/lodash._baseuniq-4.6.0.tgz#0ebb44e456814af7905c6212fa2c9b2d51b841e8" + integrity sha512-Ja1YevpHZctlI5beLA7oc5KNDhGcPixFhcqSiORHNsp/1QTv7amAXzw+gu4YOvErqVlMVyIJGgtzeepCnnur0A== + dependencies: + lodash._createset "~4.0.0" + lodash._root "~3.0.0" + +lodash._createset@~4.0.0: + version "4.0.3" + resolved "https://registry.yarnpkg.com/lodash._createset/-/lodash._createset-4.0.3.tgz#0f4659fbb09d75194fa9e2b88a6644d363c9fe26" + integrity sha512-GTkC6YMprrJZCYU3zcqZj+jkXkrXzq3IPBcF/fIPpNEAB4hZEtXU8zp/RwKOvZl43NUmwDbyRk3+ZTbeRdEBXA== + +lodash._root@~3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/lodash._root/-/lodash._root-3.0.1.tgz#fba1c4524c19ee9a5f8136b4609f017cf4ded692" + integrity sha512-O0pWuFSK6x4EXhM1dhZ8gchNtG7JMqBtrHdoUFUWXD7dJnNSUze1GuyQr5sOs0aCvgGeI3o/OJW8f4ca7FDxmQ== + +lodash._stringtopath@~4.8.0: + version "4.8.0" + resolved "https://registry.yarnpkg.com/lodash._stringtopath/-/lodash._stringtopath-4.8.0.tgz#941bcf0e64266e5fc1d66fed0a6959544c576824" + integrity sha512-SXL66C731p0xPDC5LZg4wI5H+dJo/EO4KTqOMwLYCH3+FmmfAKJEZCm6ohGpI+T1xwsDsJCfL4OnhorllvlTPQ== + dependencies: + lodash._basetostring "~4.12.0" + lodash.curry@^4.0.1: version "4.1.1" resolved "https://registry.yarnpkg.com/lodash.curry/-/lodash.curry-4.1.1.tgz#248e36072ede906501d75966200a86dab8b23170" @@ -8879,11 +8934,24 @@ lodash.sortby@^4.7.0: resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438" integrity sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA== +lodash.throttle@^4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/lodash.throttle/-/lodash.throttle-4.1.1.tgz#c23e91b710242ac70c37f1e1cda9274cc39bf2f4" + integrity sha512-wIkUCfVKpVsWo3JSZlc+8MB5it+2AN5W8J7YVMST30UrvcQNZ1Okbj+rbVniijTWE6FGYy4XJq/rHkas8qJMLQ== + lodash.uniq@^4.5.0: version "4.5.0" resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773" integrity sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ== +lodash.uniqby@4.5.0: + version "4.5.0" + resolved "https://registry.yarnpkg.com/lodash.uniqby/-/lodash.uniqby-4.5.0.tgz#a3a17bbf62eeb6240f491846e97c1c4e2a5e1e21" + integrity sha512-IRt7cfTtHy6f1aRVA5n7kT8rgN3N1nH6MOWLcHfpWG2SH19E3JksLK38MktLxZDhlAjCP9jpIXkOnRXlu6oByQ== + dependencies: + lodash._baseiteratee "~4.7.0" + lodash._baseuniq "~4.6.0" + lodash@^4.17.15, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.4, lodash@^4.7.0: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" @@ -10510,6 +10578,15 @@ prop-types@^15.6.2, prop-types@^15.8.1: object-assign "^4.1.1" react-is "^16.13.1" +proper-lockfile@^4.1.2: + version "4.1.2" + resolved "https://registry.yarnpkg.com/proper-lockfile/-/proper-lockfile-4.1.2.tgz#c8b9de2af6b2f1601067f98e01ac66baa223141f" + integrity sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA== + dependencies: + graceful-fs "^4.2.4" + retry "^0.12.0" + signal-exit "^3.0.2" + property-information@^7.0.0: version "7.1.0" resolved "https://registry.yarnpkg.com/property-information/-/property-information-7.1.0.tgz#b622e8646e02b580205415586b40804d3e8bfd5d" @@ -11118,6 +11195,11 @@ resolve@^2.0.0-next.5: path-parse "^1.0.7" supports-preserve-symlinks-flag "^1.0.0" +retry@^0.12.0: + version "0.12.0" + resolved "https://registry.yarnpkg.com/retry/-/retry-0.12.0.tgz#1b42a6266a21f07421d1b0b54b7dc167b01c013b" + integrity sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow== + retry@^0.13.1: version "0.13.1" resolved "https://registry.yarnpkg.com/retry/-/retry-0.13.1.tgz#185b1587acf67919d63b357349e03537b2484658" @@ -12415,6 +12497,19 @@ tunnel-agent@^0.6.0: dependencies: safe-buffer "^5.0.1" +tus-js-client@^4.3.1: + version "4.3.1" + resolved "https://registry.yarnpkg.com/tus-js-client/-/tus-js-client-4.3.1.tgz#2f9a47fba006206fb0d08c649fa01254944d5d87" + integrity sha512-ZLeYmjrkaU1fUsKbIi8JML52uAocjEZtBx4DKjRrqzrZa0O4MYwT6db+oqePlspV+FxXJAyFBc/L5gwUi2OFsg== + dependencies: + buffer-from "^1.1.2" + combine-errors "^3.0.3" + is-stream "^2.0.0" + js-base64 "^3.7.2" + lodash.throttle "^4.1.1" + proper-lockfile "^4.1.2" + url-parse "^1.5.7" + type-check@^0.4.0, type-check@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1" @@ -12669,7 +12764,7 @@ uri-js@^4.2.2: dependencies: punycode "^2.1.0" -url-parse@^1.5.3: +url-parse@^1.5.3, url-parse@^1.5.7: version "1.5.10" resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.5.10.tgz#9d3c2f736c1d75dd3bd2be507dcc111f1e2ea9c1" integrity sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ== From a5851b986d8fb67587ce6286c2de5a8d72ba807b Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Fri, 2 Oct 2026 15:38:05 -0400 Subject: [PATCH 43/45] feat(uploads): raw ZIP section on the upload detail page; refs #147 --- src/components/Upload/RawDataUploader.tsx | 605 ++++++++++++++++++++++ src/pages/UploadDetailPage.tsx | 15 +- src/pages/UploadPage.tsx | 3 +- 3 files changed, 621 insertions(+), 2 deletions(-) create mode 100644 src/components/Upload/RawDataUploader.tsx diff --git a/src/components/Upload/RawDataUploader.tsx b/src/components/Upload/RawDataUploader.tsx new file mode 100644 index 0000000..4c93fd2 --- /dev/null +++ b/src/components/Upload/RawDataUploader.tsx @@ -0,0 +1,605 @@ +import { + Alert, + Box, + Button, + Checkbox, + CircularProgress, + Dialog, + DialogActions, + DialogContent, + DialogContentText, + DialogTitle, + FormControlLabel, + LinearProgress, + Paper, + Typography, +} from "@mui/material"; +import CopyButton from "components/CopyButton"; +import { Colors } from "design/theme"; +import React, { useCallback, useEffect, useRef, useState } from "react"; +import { + RawExtractionStatus, + RawStatus, + RawUploadError, + RawUploadService, +} from "services/rawUpload.service"; +import { formatBytes } from "utils/formatBytes"; +import { RawUploader, rawUploadSupported } from "utils/rawUploader"; + +// "Raw data (ZIP)" section of the upload detail page: shows the version's +// current ZIP and uploads a new one straight to the storage API (tus). + +const MAX_BYTES = 500_000_000_000; // 500 GB — same limit as the server +const POLL_MS = 3000; + +type Phase = "idle" | "starting" | "uploading" | "paused" | "verifying"; + +const EXTRACTION_LABEL: Record = { + not_extracted: "Not extracted yet", + extracting: "Extracting…", + extracted: "Extracted", + failed: "Extraction failed", +}; + +const formatEta = (seconds: number) => { + if (!Number.isFinite(seconds) || seconds <= 0) return ""; + if (seconds < 60) return `about ${Math.ceil(seconds)} s left`; + if (seconds < 3600) return `about ${Math.ceil(seconds / 60)} min left`; + const h = Math.floor(seconds / 3600); + return `about ${h} h ${Math.ceil((seconds - h * 3600) / 60)} min left`; +}; + +const purpleButton = { + backgroundColor: Colors.purple, + "&:hover": { backgroundColor: Colors.secondaryPurple }, +}; +const outlinedPurple = { color: Colors.purple, borderColor: Colors.purple }; + +interface RawDataUploaderProps { + internalId: string; + status?: string | null; // submission status from the page; reload when it changes + onChanged?: () => void; // parent reloads the dataset (readiness etc.) +} + +const RawDataUploader: React.FC = ({ + internalId, + status: submissionStatus, + onChanged, +}) => { + const [status, setStatus] = useState(null); + const [loadError, setLoadError] = useState(null); + const [phase, setPhase] = useState("idle"); + const [progress, setProgress] = useState<{ + sent: number; + total: number; + speed: number; + } | null>(null); + const [error, setError] = useState<{ + message: string; + canResume: boolean; + } | null>(null); + const [conflict, setConflict] = useState<{ + uploadId: string; + filename: string; + size: number; + } | null>(null); + const [confirmCancel, setConfirmCancel] = useState(false); + const [busy, setBusy] = useState(false); + const [success, setSuccess] = useState(null); + const uploaderRef = useRef(null); + const verifyingIdRef = useRef(null); + const speedRef = useRef<{ t: number; bytes: number; speed: number } | null>( + null + ); + const fileInput = useRef(null); + + const load = useCallback(async () => { + try { + const s = await RawUploadService.getStatus(internalId); + setStatus(s); + setLoadError(null); + return s; + } catch (e: any) { + setLoadError(e.message || "Could not load raw data status."); + return null; + } + }, [internalId]); + + useEffect(() => { + load(); + }, [load, submissionStatus]); // e.g. Submit / Withdraw changes "editable" + + // Reloaded while the server was already verifying → keep waiting for it. + useEffect(() => { + if (phase === "idle" && status?.activeUpload?.status === "verifying") { + verifyingIdRef.current = status.activeUpload.uploadId; + setPhase("verifying"); + } + }, [status, phase]); + + // Verifying: poll until the new ZIP is attached, or the upload ended + // without it (failed / cancelled / expired). raw_id = upload_id on the + // server, so "attached" means current.rawId === the upload id. + useEffect(() => { + if (phase !== "verifying") return; + const id = verifyingIdRef.current; + const timer = setInterval(async () => { + const s = await load(); + if (!s || !id) return; + const attached = s.current?.rawId === id; + const stillActive = s.activeUpload?.uploadId === id; + if (!attached && stillActive) return; + clearInterval(timer); + uploaderRef.current = null; + setProgress(null); + setPhase("idle"); + if (attached && s.current) { + setSuccess( + `${s.current.filename || "The ZIP"} (${formatBytes( + s.current.size + )}) is verified and attached to this version.` + ); + } else if (s.lastUpload?.uploadId === id) { + setError({ + message: s.lastUpload.error || "The upload could not be completed.", + canResume: false, + }); + } + onChanged?.(); + }, POLL_MS); + return () => clearInterval(timer); + }, [phase, load, onChanged]); + + // Closing the tab mid-upload: ask first (the browser shows its own text). + useEffect(() => { + if (phase !== "uploading" && phase !== "starting") return; + const warn = (e: BeforeUnloadEvent) => { + e.preventDefault(); + e.returnValue = ""; + }; + window.addEventListener("beforeunload", warn); + return () => window.removeEventListener("beforeunload", warn); + }, [phase]); + + // Leaving this page inside the app: stop sending (resumable later). + useEffect( + () => () => { + uploaderRef.current?.pause(); + }, + [] + ); + + const trackSpeed = (sent: number) => { + const now = performance.now(); + const prev = speedRef.current; + if (!prev) { + speedRef.current = { t: now, bytes: sent, speed: 0 }; + return 0; + } + if (now - prev.t < 500) return prev.speed; + const instant = (sent - prev.bytes) / ((now - prev.t) / 1000); + const speed = prev.speed ? 0.3 * instant + 0.7 * prev.speed : instant; + speedRef.current = { t: now, bytes: sent, speed }; + return speed; + }; + + const startUpload = async (file: File) => { + setSuccess(null); + setError(null); + setConflict(null); + if (!/\.zip$/i.test(file.name)) { + return setError({ + message: "Please choose a .zip file.", + canResume: false, + }); + } + if (file.size <= 0) { + return setError({ message: "The file is empty.", canResume: false }); + } + if (file.size > MAX_BYTES) { + return setError({ + message: `The file is too large (max ${formatBytes(MAX_BYTES)}).`, + canResume: false, + }); + } + speedRef.current = null; + setProgress({ sent: 0, total: file.size, speed: 0 }); + setPhase("starting"); + try { + uploaderRef.current = await RawUploader.begin(internalId, file, { + onProgress: (sent, total) => { + setProgress({ sent, total, speed: trackSpeed(sent) }); + setPhase((p) => (p === "starting" ? "uploading" : p)); + }, + onFinished: () => { + verifyingIdRef.current = uploaderRef.current?.uploadId ?? null; + setPhase("verifying"); + }, + onError: (message, canResume) => { + setError({ message, canResume }); + if (canResume) { + setPhase("paused"); + } else { + uploaderRef.current = null; + setProgress(null); + setPhase("idle"); + load(); + } + }, + }); + setPhase((p) => (p === "starting" ? "uploading" : p)); + await load(); + onChanged?.(); // readiness now says "upload in progress" + } catch (e: any) { + setProgress(null); + setPhase("idle"); + if (e instanceof RawUploadError && e.code === "ACTIVE_UPLOAD_EXISTS") { + setConflict(e.details?.upload ?? null); + } else { + setError({ + message: e.message || "Could not start the upload.", + canResume: false, + }); + } + } + }; + + const pause = async () => { + await uploaderRef.current?.pause(); + setPhase("paused"); + }; + + const resume = () => { + setError(null); + speedRef.current = null; + setPhase("uploading"); + uploaderRef.current?.resume(); + }; + + // The upload to cancel: the one running here, an interrupted one (after a + // reload), or the one blocking a different file. + const cancelTargetId = + uploaderRef.current?.uploadId || + status?.activeUpload?.uploadId || + conflict?.uploadId || + null; + + const cancel = async () => { + setSuccess(null); + setConfirmCancel(false); + setBusy(true); + try { + if (uploaderRef.current) await uploaderRef.current.cancel(); + else if (cancelTargetId) { + await RawUploadService.cancel(internalId, cancelTargetId); + } + setError(null); + } catch (e: any) { + setError({ + message: e.message || "Could not cancel the upload.", + canResume: false, + }); + } finally { + uploaderRef.current = null; + setConflict(null); + setProgress(null); + setPhase("idle"); + setBusy(false); + await load(); + onChanged?.(); + } + }; + + const toggleExpected = async (expected: boolean) => { + setBusy(true); + try { + await RawUploadService.setExpected(internalId, expected); + await load(); + onChanged?.(); + } catch (e: any) { + setError({ message: e.message, canResume: false }); + } finally { + setBusy(false); + } + }; + + if (!status) { + return ( + + + Raw data (ZIP) + + {loadError ? ( + {loadError} + ) : ( + + )} + + ); + } + + const { current, activeUpload, lastUpload, editable } = status; + const sending = + phase === "starting" || phase === "uploading" || phase === "paused"; + const interrupted = + !!activeUpload && + !uploaderRef.current && + phase === "idle" && + activeUpload.status !== "verifying"; + const canPick = + editable && rawUploadSupported && phase === "idle" && !activeUpload; + const pct = + progress && progress.total + ? Math.floor((progress.sent / progress.total) * 100) + : 0; + const showLastFailure = + phase === "idle" && + !activeUpload && + !error && + lastUpload && + ["failed", "expired"].includes(lastUpload.status); + + return ( + + + Raw data (ZIP) + + + {success && ( + setSuccess(null)} + sx={{ mb: 2 }} + > + Raw data uploaded: {success} + + )} + + toggleExpected(e.target.checked)} + sx={{ "&.Mui-checked": { color: Colors.purple } }} + /> + } + label="This dataset includes raw data" + /> + + {/* Current ZIP */} + {current ? ( + + + Current ZIP: {current.filename || "(unnamed)"} ·{" "} + {formatBytes(current.size)} · uploaded{" "} + {new Date(current.uploadedAt).toLocaleDateString()} + {current.carriedForward + ? " · carried over from the previous version" + : ""} + + + SHA-256{" "} + + {current.sha256.slice(0, 8)}…{current.sha256.slice(-8)} + + ·{" "} + {EXTRACTION_LABEL[current.extractionStatus]} + + + ) : ( + status.rawZipExpected && + !activeUpload && + phase === "idle" && ( + + No raw ZIP uploaded yet. + + ) + )} + + {!rawUploadSupported && ( + + This browser can't upload large files. Please use a recent Chrome, + Firefox, Safari or Edge. + + )} + + {/* Upload interrupted earlier (e.g. the page was reloaded). */} + {interrupted && ( + + An upload of {activeUpload!.filename} ( + {formatBytes(activeUpload!.size)}) was interrupted. Choose the same + file to continue, or cancel it. + {editable && ( + + + + + )} + + )} + + {/* A different file was picked while another upload exists. */} + {conflict && ( + + Another upload ({conflict.filename},{" "} + {formatBytes(conflict.size)}) is in progress for this version. Choose + that same file to continue it, or cancel it to upload a different + file. + + + + + )} + + {/* Progress while sending. */} + {sending && progress && ( + + + {phase === "paused" + ? "Paused" + : phase === "starting" + ? "Starting" + : "Uploading"}{" "} + {uploaderRef.current?.file.name} + + + + {pct}% · {formatBytes(progress.sent)} /{" "} + {formatBytes(progress.total)} + {phase === "uploading" && + progress.speed > 0 && + ` · ${formatBytes(progress.speed)}/s · ${formatEta( + (progress.total - progress.sent) / progress.speed + )}`} + + + {phase === "uploading" && ( + + )} + {phase === "paused" && ( + + )} + + + + )} + + {phase === "verifying" && ( + + + + Verifying the file — checking it and computing its checksum. This + can take a while for very large files. + + + )} + + {error && ( + + {error.message} + + )} + + {showLastFailure && ( + + The last upload ({lastUpload!.filename}) didn't finish:{" "} + {lastUpload!.error || "it expired."} + {current ? " Your current ZIP is unchanged." : ""} + + )} + + {canPick && ( + + )} + + {!editable && ( + + Raw data can only be changed while the submission is a draft or has + requested changes. + + )} + + { + const f = e.target.files?.[0]; + e.target.value = ""; // allow picking the same file again later + if (f) startUpload(f); + }} + /> + + setConfirmCancel(false)}> + Cancel this upload? + + + The part already uploaded will be deleted. You can start a new + upload afterwards. + + + + + + + + + ); +}; + +export default RawDataUploader; diff --git a/src/pages/UploadDetailPage.tsx b/src/pages/UploadDetailPage.tsx index d436d55..da54469 100644 --- a/src/pages/UploadDetailPage.tsx +++ b/src/pages/UploadDetailPage.tsx @@ -17,9 +17,10 @@ import { } from "@mui/material"; import CopyButton from "components/CopyButton"; import PublishSettingsDialog from "components/Upload/PublishSettingsDialog"; +import RawDataUploader from "components/Upload/RawDataUploader"; import { Colors } from "design/theme"; import { useAppSelector } from "hooks/useAppSelector"; -import React, { useEffect, useState } from "react"; +import React, { useCallback, useEffect, useState } from "react"; import { Link, useNavigate, useParams } from "react-router-dom"; import { AuthSelector } from "redux/auth/auth.selector"; import { baseURL } from "services/instance"; @@ -95,6 +96,11 @@ const UploadDetailPage: React.FC = () => { } }; + // Raw ZIP changes affect readiness, so reload the dataset after them. + const reloadDetail = useCallback(async () => { + if (internalId) setDetail(await UploadService.getUpload(internalId)); + }, [internalId]); + useEffect(() => { if (!internalId || !isLoggedIn) return; setLoading(true); @@ -346,6 +352,13 @@ const UploadDetailPage: React.FC = () => { }} /> + {/* Raw data (ZIP): optional; uploads go straight to the storage API */} + + {/* Review: readiness checklist + submit / resubmit / withdraw */} diff --git a/src/pages/UploadPage.tsx b/src/pages/UploadPage.tsx index ae25ba8..e3f5213 100644 --- a/src/pages/UploadPage.tsx +++ b/src/pages/UploadPage.tsx @@ -211,7 +211,8 @@ const UploadPage: React.FC = () => { after you click Submit for review on the upload detail page; the NeuroJSON team reviews it before it is added to a database. A new reference id is assigned on upload — any existing{" "} - _id/_rev in your file is ignored. + _id/_rev in your file is ignored. Raw data + (a ZIP) is added afterwards, on the upload detail page. {/* Updating an existing dataset: the id comes only from the URL From 91a65ed2c09ffc873d2826d7115566504c6d667d Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Fri, 2 Oct 2026 15:45:51 -0400 Subject: [PATCH 44/45] feat(uploads): raw ZIP chip on the Uploads tab; refs #147 --- backend/src/controllers/uploads.controller.js | 10 ++++- src/components/User/Dashboard/UploadsTab.tsx | 37 +++++++++++++++++++ src/services/upload.service.ts | 3 ++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/backend/src/controllers/uploads.controller.js b/backend/src/controllers/uploads.controller.js index 48fe949..96596b4 100644 --- a/backend/src/controllers/uploads.controller.js +++ b/backend/src/controllers/uploads.controller.js @@ -398,7 +398,15 @@ const listMyUploads = async (req, res) => { const rows = await sequelize.query( `SELECT r.internal_id, r.dataset_id, r.requested_dataset_id, r.dataset_name, s.submission_id, s.status, - s.created_at, s.updated_at, s.promoted_db, s.promoted_at + s.created_at, s.updated_at, s.promoted_db, s.promoted_at, + s.raw_zip_expected, + (SELECT o.size_bytes + FROM submission_raw_files f + JOIN raw_objects o ON o.raw_id = f.raw_id + WHERE f.submission_id = s.submission_id) AS raw_zip_size, + EXISTS (SELECT 1 FROM raw_upload_attempts a + WHERE a.submission_id = s.submission_id + AND a.status IN ('initiated','uploading','verifying')) AS raw_upload_active FROM dataset_registry r LEFT JOIN LATERAL ( SELECT * FROM submissions s2 diff --git a/src/components/User/Dashboard/UploadsTab.tsx b/src/components/User/Dashboard/UploadsTab.tsx index 7484c33..af8d61d 100644 --- a/src/components/User/Dashboard/UploadsTab.tsx +++ b/src/components/User/Dashboard/UploadsTab.tsx @@ -16,6 +16,7 @@ import { Colors } from "design/theme"; import React, { useEffect, useState } from "react"; import { useNavigate } from "react-router-dom"; import { UploadRecord, UploadService } from "services/upload.service"; +import { formatBytes } from "utils/formatBytes"; type SubStatus = Exclude; @@ -39,6 +40,28 @@ const STATUS_CHIP: Record< rejected: { label: "Rejected", color: "error" }, }; +// Raw ZIP state of the latest version (null = no raw data → no chip). +const rawChip = (r: UploadRecord) => { + if (r.raw_upload_active) { + return { + label: "ZIP uploading", + sx: { color: Colors.purple, borderColor: Colors.purple }, + variant: "outlined" as const, + }; + } + if (r.raw_zip_size != null) { + return { label: `ZIP · ${formatBytes(Number(r.raw_zip_size))}` }; + } + if (r.raw_zip_expected) { + return { + label: "ZIP missing", + color: "warning" as const, + variant: "outlined" as const, + }; + } + return null; +}; + const UploadsTab: React.FC = () => { const navigate = useNavigate(); const [rows, setRows] = useState([]); @@ -164,6 +187,20 @@ const UploadsTab: React.FC = () => { sx={{ height: 20, ...(chip.sx || {}) }} /> )} + {(() => { + const raw = rawChip(r); + return ( + raw && ( + + ) + ); + })()} } secondary={`Submitted ${formatDate(r.created_at)}${ diff --git a/src/services/upload.service.ts b/src/services/upload.service.ts index 7bb0a35..a59ff61 100644 --- a/src/services/upload.service.ts +++ b/src/services/upload.service.ts @@ -23,6 +23,9 @@ export interface UploadRecord { updated_at: string | null; promoted_db: string | null; promoted_at: string | null; + raw_zip_expected: boolean | null; + raw_zip_size: string | number | null; // BIGINT arrives as a string + raw_upload_active: boolean | null; } export interface UploadDetail { From 888e01ccbd9e6e18f97aea55309302d02a6a51a1 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Fri, 2 Oct 2026 16:00:49 -0400 Subject: [PATCH 45/45] feat(uploads): 50 GB raw ZIP limit, sent to the page; refs #147 --- backend/src/controllers/rawUploads.controller.js | 6 ++++-- src/components/Upload/RawDataUploader.tsx | 15 ++++++++++++--- src/services/rawUpload.service.ts | 1 + src/utils/formatBytes.ts | 2 +- storage-api/.env.example | 2 +- storage-api/src/config.js | 2 +- 6 files changed, 20 insertions(+), 8 deletions(-) diff --git a/backend/src/controllers/rawUploads.controller.js b/backend/src/controllers/rawUploads.controller.js index 111fedd..f68df74 100644 --- a/backend/src/controllers/rawUploads.controller.js +++ b/backend/src/controllers/rawUploads.controller.js @@ -11,8 +11,8 @@ const { const { signUploadToken } = require("../lib/uploadTokens"); const MAX_UPLOAD_BYTES = Number( - process.env.RAW_MAX_UPLOAD_BYTES || 500000000000 -); // 500 GB (decimal, matches the size shown in the UI) + process.env.RAW_MAX_UPLOAD_BYTES || 50000000000 +); // 50 GB (decimal, matches the size shown in the UI); sent to the page const CHUNK_BYTES = Number(process.env.RAW_CHUNK_BYTES || 67108864); // 64 MB const ACTIVE = ["initiated", "uploading", "verifying"]; const EDITABLE = ["draft", "changes_requested"]; @@ -264,6 +264,7 @@ const getRawStatus = async (req, res) => { return res.json({ rawZipExpected: false, editable: false, + maxUploadBytes: MAX_UPLOAD_BYTES, current: null, activeUpload: null, lastUpload: null, @@ -312,6 +313,7 @@ const getRawStatus = async (req, res) => { res.json({ rawZipExpected: !!sub.raw_zip_expected, editable: EDITABLE.includes(sub.status), + maxUploadBytes: MAX_UPLOAD_BYTES, // Storage keys stay server-side; only display facts are returned. current: c ? { diff --git a/src/components/Upload/RawDataUploader.tsx b/src/components/Upload/RawDataUploader.tsx index 4c93fd2..50f1bd6 100644 --- a/src/components/Upload/RawDataUploader.tsx +++ b/src/components/Upload/RawDataUploader.tsx @@ -29,7 +29,6 @@ import { RawUploader, rawUploadSupported } from "utils/rawUploader"; // "Raw data (ZIP)" section of the upload detail page: shows the version's // current ZIP and uploads a new one straight to the storage API (tus). -const MAX_BYTES = 500_000_000_000; // 500 GB — same limit as the server const POLL_MS = 3000; type Phase = "idle" | "starting" | "uploading" | "paused" | "verifying"; @@ -196,9 +195,10 @@ const RawDataUploader: React.FC = ({ if (file.size <= 0) { return setError({ message: "The file is empty.", canResume: false }); } - if (file.size > MAX_BYTES) { + const max = status?.maxUploadBytes ?? 0; + if (max && file.size > max) { return setError({ - message: `The file is too large (max ${formatBytes(MAX_BYTES)}).`, + message: `The file is too large (max ${formatBytes(max)}).`, canResume: false, }); } @@ -558,6 +558,15 @@ const RawDataUploader: React.FC = ({ {current ? "Replace ZIP" : "Choose ZIP file"} )} + {canPick && ( + + ZIP files up to {formatBytes(status.maxUploadBytes)}. + + )} {!editable && ( diff --git a/src/services/rawUpload.service.ts b/src/services/rawUpload.service.ts index d65781d..ef3dd50 100644 --- a/src/services/rawUpload.service.ts +++ b/src/services/rawUpload.service.ts @@ -45,6 +45,7 @@ export interface RawCurrentZip { export interface RawStatus { rawZipExpected: boolean; editable: boolean; + maxUploadBytes: number; // server's per-file limit (bytes) current: RawCurrentZip | null; activeUpload: RawUploadSummary | null; lastUpload: RawUploadSummary | null; diff --git a/src/utils/formatBytes.ts b/src/utils/formatBytes.ts index 58cd1e1..6be73d1 100644 --- a/src/utils/formatBytes.ts +++ b/src/utils/formatBytes.ts @@ -1,5 +1,5 @@ // Human-readable size in DECIMAL units (1 GB = 1,000,000,000 bytes), matching -// the "500 GB max" limit message. +// the "50 GB max" limit message. export const formatBytes = (bytes: number): string => { if (!Number.isFinite(bytes) || bytes < 0) return "—"; if (bytes < 1000) return `${bytes} B`; diff --git a/storage-api/.env.example b/storage-api/.env.example index 20cd97d..95c5e6b 100644 --- a/storage-api/.env.example +++ b/storage-api/.env.example @@ -18,5 +18,5 @@ STORAGE_HMAC_ACTIVE_KID=dev-2026-09 STORAGE_HMAC_KEYS='{"dev-2026-09":"..."}' # Limits -MAX_UPLOAD_BYTES=500000000000 +MAX_UPLOAD_BYTES=50000000000 DISK_RESERVE_PERCENT=5 diff --git a/storage-api/src/config.js b/storage-api/src/config.js index b4526c3..c028479 100644 --- a/storage-api/src/config.js +++ b/storage-api/src/config.js @@ -76,7 +76,7 @@ export const config = { tokenPublicKeys, hmacKeys, hmacActiveKid, - maxUploadBytes: num("MAX_UPLOAD_BYTES", 500000000000), // 500 GB (decimal) + maxUploadBytes: num("MAX_UPLOAD_BYTES", 50000000000), // 50 GB (decimal) diskReservePercent: num("DISK_RESERVE_PERCENT", 5), };