Release: Postgres landing stats + latest-update board, search fixes, and dataset upload with raw ZIP support

.gitignore

@@ -16,4 +16,8 @@ backend/.env.local
 backend/.env.*.local
 backend/*.sqlite
 backend/*.db
-!backend/package-lock.json
+!backend/package-lock.json
+
+#storage-api
+storage-api/storage-dev/
+!storage-api/package-lock.json

backend/migrations/20260903145021-add-iolinks-dbname-dsname-view-index.js

@@ -0,0 +1,26 @@
+'use strict';
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  // Composite index for the file-type filter's EXISTS subquery in
+  // searchAllDatabases:
+  //   EXISTS (SELECT 1 FROM iolinks l
+  //           WHERE l.dbname = ioviews.dbname AND l.dsname = ioviews.dsname
+  //             AND l.view IN (:fileTypes))
+  // Without it the planner uses idx_iolinks_dbname alone and scans every row
+  // of a database (openneuro ~1.4M), filtering dsname/view in memory — the
+  // dominant cost of a subject search combined with a file_type filter. This
+  // turns that into a direct lookup on (dbname, dsname, view).
+  async up (queryInterface, Sequelize) {
+    await queryInterface.addIndex("iolinks", ["dbname", "dsname", "view"], {
+      name: "idx_iolinks_dbname_dsname_view",
+    });
+  },
+
+  async down (queryInterface, Sequelize) {
+    await queryInterface.removeIndex(
+      "iolinks",
+      "idx_iolinks_dbname_dsname_view"
+    );
+  }
+};

backend/migrations/20260903180911-create-stats-history.js

@@ -0,0 +1,61 @@
+'use strict';
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  // One row per scheduled sync run. Doubles as (1) persistent landing-page
+  // stats and (2) sync execution history. Lifecycle: inserted 'running' at
+  // sync start; finalized 'success' (with totals) or 'failed' (with error)
+  // after the run completes. The latest 'success' row = current NeuroJSON state.
+  async up (queryInterface, Sequelize) {
+    await queryInterface.createTable("stats_history", {
+      id: {
+        type: Sequelize.INTEGER,
+        autoIncrement: true,
+        primaryKey: true,
+        allowNull: false,
+      },
+      started_at: {
+        type: Sequelize.DATE, // TIMESTAMPTZ
+        allowNull: false,
+      },
+      completed_at: {
+        type: Sequelize.DATE,
+        allowNull: true,
+      },
+      status: {
+        type: Sequelize.TEXT, // 'running' | 'success' | 'failed'
+        allowNull: false,
+        defaultValue: "running",
+      },
+      total_datasets: {
+        type: Sequelize.BIGINT,
+        allowNull: true,
+      },
+      total_subjects: {
+        type: Sequelize.BIGINT,
+        allowNull: true,
+      },
+      total_files: {
+        type: Sequelize.BIGINT,
+        allowNull: true,
+      },
+      total_size_bytes: {
+        type: Sequelize.BIGINT,
+        allowNull: true,
+      },
+      error: {
+        type: Sequelize.TEXT, // failure reason when status = 'failed'
+        allowNull: true,
+      },
+      created_at: {
+        type: Sequelize.DATE,
+        allowNull: false,
+        defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"),
+      },
+    });
+  },
+
+  async down (queryInterface, Sequelize) {
+    await queryInterface.dropTable("stats_history");
+  }
+};

backend/migrations/20260910160052-create-dataset-changes.js

@@ -0,0 +1,59 @@
+'use strict';
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  // One row per dataset-level change detected during an incremental sync.
+  // Only the incremental path (processDatasetUpdate / delete branch) writes
+  // here — firstSync never logs, so a full/first sync does not mass-record
+  // every dataset as "added". Powers the "latest update" board on the landing
+  // page. history_id → stats_history.id (the sync run that produced the change).
+  async up (queryInterface, Sequelize) {
+    await queryInterface.createTable("dataset_changes", {
+      id: {
+        type: Sequelize.INTEGER,
+        autoIncrement: true,
+        primaryKey: true,
+        allowNull: false,
+      },
+      history_id: {
+        type: Sequelize.INTEGER, // FK → stats_history.id (same INTEGER type)
+        allowNull: false,
+        references: { model: "stats_history", key: "id" },
+        onDelete: "CASCADE",
+      },
+      dbname: {
+        type: Sequelize.TEXT,
+        allowNull: false,
+      },
+      dsname: {
+        type: Sequelize.TEXT,
+        allowNull: false,
+      },
+      change_type: {
+        type: Sequelize.TEXT, // 'added' | 'updated' | 'deleted'
+        allowNull: false,
+      },
+      created_at: {
+        type: Sequelize.DATE,
+        allowNull: false,
+        defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"),
+      },
+    });
+
+    // Dedupe: one logical change per dataset per sync run (a dataset touched
+    // multiple times in one sync → ON CONFLICT DO NOTHING on this key).
+    await queryInterface.addIndex(
+      "dataset_changes",
+      ["history_id", "dbname", "dsname"],
+      { name: "uq_dataset_changes_run_dataset", unique: true }
+    );
+    // "latest update" lookup filters/groups by run.
+    await queryInterface.addIndex("dataset_changes", ["history_id"], {
+      name: "idx_dataset_changes_history_id",
+    });
+  },
+
+  async down (queryInterface, Sequelize) {
+    await queryInterface.dropTable("dataset_changes");
+  }
+};

backend/migrations/20260923192128-create-dataset-registry.js

@@ -0,0 +1,64 @@
+"use strict";
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  // Permanent record for one logical dataset. `internal_id` is the stable
+  // identity for the dataset's whole lifetime and is also the sandbox CouchDB
+  // `_id`. `dataset_id` (final public id) is assigned only at promotion.
+  async up(queryInterface, Sequelize) {
+    // Sequence backing NeuroJSON-generated public ids (njds######). Consumed
+    // only at promotion when the user did not request a valid custom id.
+    await queryInterface.sequelize.query(
+      "CREATE SEQUENCE IF NOT EXISTS neurojson_dataset_seq START 1;"
+    );
+
+    await queryInterface.createTable("dataset_registry", {
+      internal_id: {
+        type: Sequelize.UUID,
+        primaryKey: true,
+        allowNull: false, // app-generated (crypto.randomUUID); also the sandbox _id
+      },
+      dataset_id: {
+        type: Sequelize.STRING(255),
+        allowNull: true,
+        unique: true, // final public id, set at promotion (njds###### or custom)
+      },
+      requested_dataset_id: {
+        type: Sequelize.STRING(63),
+        allowNull: true, // user's preferred public id — a preference until promotion
+      },
+      dataset_name: {
+        type: Sequelize.STRING(255),
+        allowNull: true, // canonical logical-dataset name
+      },
+      owner_user_id: {
+        type: Sequelize.INTEGER,
+        allowNull: false,
+        references: { model: "users", key: "id" },
+        onDelete: "RESTRICT", // don't orphan a dataset's owner
+        onUpdate: "CASCADE",
+      },
+      created_at: {
+        type: Sequelize.DATE,
+        allowNull: false,
+        defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"),
+      },
+      updated_at: {
+        type: Sequelize.DATE,
+        allowNull: false,
+        defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"),
+      },
+    });
+
+    await queryInterface.addIndex("dataset_registry", ["owner_user_id"], {
+      name: "idx_registry_owner",
+    });
+  },
+
+  async down(queryInterface) {
+    await queryInterface.dropTable("dataset_registry");
+    await queryInterface.sequelize.query(
+      "DROP SEQUENCE IF EXISTS neurojson_dataset_seq;"
+    );
+  },
+};

backend/migrations/20260923192138-create-submissions.js

@@ -0,0 +1,75 @@
+"use strict";
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  // One review workflow (cycle) for a logical dataset. Holds only the current
+  // status + review/promotion metadata; the reviewer<->submitter discussion
+  // lives in submission_comments. A dataset can have many cycles over time,
+  // but only ONE open one (see the partial unique index below).
+  async up(queryInterface, Sequelize) {
+    await queryInterface.createTable("submissions", {
+      id: {
+        type: Sequelize.BIGINT,
+        autoIncrement: true,
+        primaryKey: true,
+        allowNull: false,
+      },
+      submission_id: {
+        type: Sequelize.UUID,
+        allowNull: false,
+        unique: true, // one review cycle; referenced by submission_comments
+      },
+      dataset_registry_id: {
+        type: Sequelize.UUID,
+        allowNull: false,
+        references: { model: "dataset_registry", key: "internal_id" },
+        onDelete: "CASCADE", // removing the logical dataset removes its cycles
+        onUpdate: "CASCADE",
+      },
+      status: {
+        type: Sequelize.STRING(50),
+        allowNull: false,
+        defaultValue: "pending", // pending|changes_requested|approved|rejected|promoted
+      },
+      reviewed_by: {
+        type: Sequelize.INTEGER,
+        allowNull: true,
+        references: { model: "users", key: "id" },
+        onDelete: "SET NULL", // keep the cycle if a reviewer is removed
+        onUpdate: "CASCADE",
+      },
+      reviewed_at: { type: Sequelize.DATE, allowNull: true },
+      promoted_db: { type: Sequelize.STRING(255), allowNull: true },
+      promoted_at: { type: Sequelize.DATE, allowNull: true },
+      created_at: {
+        type: Sequelize.DATE,
+        allowNull: false,
+        defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"),
+      },
+      updated_at: {
+        type: Sequelize.DATE,
+        allowNull: false,
+        defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"),
+      },
+    });
+
+    await queryInterface.addIndex("submissions", ["dataset_registry_id"], {
+      name: "idx_submissions_registry",
+    });
+    await queryInterface.addIndex("submissions", ["status"], {
+      name: "idx_submissions_status",
+    });
+
+    // At most one OPEN workflow per dataset. "Open" = still active:
+    // pending / changes_requested / approved (approved is awaiting promotion,
+    // so it must also block a new cycle). rejected/promoted are terminal, so a
+    // confirmed resubmit can insert a fresh pending row.
+    await queryInterface.sequelize.query(
+      "CREATE UNIQUE INDEX uq_one_open_per_dataset ON submissions (dataset_registry_id) WHERE status IN ('pending','changes_requested','approved');"
+    );
+  },
+
+  async down(queryInterface) {
+    await queryInterface.dropTable("submissions");
+  },
+};

backend/migrations/20260923192147-create-submission-comments.js

@@ -0,0 +1,50 @@
+"use strict";
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  // Chronological reviewer <-> submitter discussion for a submission cycle,
+  // one row per comment. This is the single source of review discussion
+  // (there is no review_note column). Author role is derived at read time
+  // (user_id == dataset_registry.owner_user_id => submitter, else reviewer).
+  async up(queryInterface, Sequelize) {
+    await queryInterface.createTable("submission_comments", {
+      id: {
+        type: Sequelize.BIGINT,
+        autoIncrement: true,
+        primaryKey: true,
+        allowNull: false,
+      },
+      submission_id: {
+        type: Sequelize.UUID,
+        allowNull: false,
+        references: { model: "submissions", key: "submission_id" },
+        onDelete: "CASCADE", // comments die with their cycle
+        onUpdate: "CASCADE",
+      },
+      user_id: {
+        type: Sequelize.INTEGER,
+        allowNull: false,
+        references: { model: "users", key: "id" },
+        onDelete: "RESTRICT", // a comment must keep a real author
+        onUpdate: "CASCADE",
+      },
+      message: { type: Sequelize.TEXT, allowNull: false },
+      created_at: {
+        type: Sequelize.DATE,
+        allowNull: false,
+        defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"),
+      },
+    });
+
+    // Serves the read query: WHERE submission_id = ? ORDER BY created_at ASC.
+    await queryInterface.addIndex(
+      "submission_comments",
+      ["submission_id", "created_at"],
+      { name: "idx_comments_submission_created" }
+    );
+  },
+
+  async down(queryInterface) {
+    await queryInterface.dropTable("submission_comments");
+  },
+};

backend/migrations/20260923214452-add-unique-requested-dataset-id.js

@@ -0,0 +1,19 @@
+'use strict';
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  // At most one dataset may reserve a given preferred public id (first-come).
+  // Partial so the many NULL (no preference) rows don't collide. Backs up the
+  // controller's assertRequestedIdFree check against races.
+  async up(queryInterface) {
+    await queryInterface.sequelize.query(
+      "CREATE UNIQUE INDEX uq_requested_dataset_id ON dataset_registry (requested_dataset_id) WHERE requested_dataset_id IS NOT NULL;"
+    );
+  },
+
+  async down(queryInterface) {
+    await queryInterface.sequelize.query(
+      "DROP INDEX IF EXISTS uq_requested_dataset_id;"
+    );
+  }
+};

backend/migrations/20260925162336-add-requested-db-to-dataset-registry.js

@@ -0,0 +1,19 @@
+'use strict';
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  // The public database a dataset should be published to at promotion.
+  // A preference/reservation; defaults to "public" when the user picks nothing.
+  // Actual db is resolved/created at promotion (a new name is review-gated).
+  async up(queryInterface, Sequelize) {
+    await queryInterface.addColumn("dataset_registry", "requested_db", {
+      type: Sequelize.STRING(63),
+      allowNull: false,
+      defaultValue: "public",
+    });
+  },
+
+  async down(queryInterface) {
+    await queryInterface.removeColumn("dataset_registry", "requested_db");
+  }
+};