From 0cebde9bd7337e7cf26b9b6f0dbdf0b5139310ad Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Fri, 7 Aug 2026 13:00:41 -0500 Subject: [PATCH 1/4] A more generic back end expand via a new /v1/id/_:id/expanded/ endpoint --- controllers/crud.js | 167 ++++++++++++++++- controllers/gog.js | 69 +------ controllers/utils.js | 64 +++++++ database/__mocks__/index.js | 1 + db-controller.js | 3 +- openapi/contracts/core-provider.openapi.yaml | 93 ++++++++++ public/API.html | 186 +++++++++++++++++++ routes/id.js | 9 + 8 files changed, 528 insertions(+), 64 deletions(-) diff --git a/controllers/crud.js b/controllers/crud.js index 758a048a..bcadc537 100644 --- a/controllers/crud.js +++ b/controllers/crud.js @@ -6,7 +6,7 @@ */ import { newID, isValidID, db } from '../database/index.js' import utils from '../utils.js' -import { _contextid, idNegotiation, getPagination, generateSlugId, ObjectID, getAgentClaim, parseDocumentID } from './utils.js' +import { _contextid, idNegotiation, getPagination, generateSlugId, ObjectID, getAgentClaim, parseDocumentID, findLeafAnnotationsFor } from './utils.js' /** * Create a new Linked Open Data object in RERUM v1. @@ -127,8 +127,171 @@ const id = async function (req, res, next) { } } +/** + * The expand job always constrains the Annotations it gathers to the leaf versions, to the + * Annotation types, and to the entity in the request URI. A client cannot influence those, so + * these keys are dropped from a supplied filter body by exact name or dotted prefix. + * The dot matters -- 'targetCollection' is a real property on Gallery of Glosses data and must + * still be usable as a filter. + */ +const RESERVED_FILTER_KEYS = ["target", "type", "@type", "__rerum.history"] + +/** + * Identity and system properties an Annotation body must never overwrite. '@id' and '@context' + * are read by idNegotiation() and res.location() right after the merge, so clobbering them would + * break the response itself. '__proto__' is not data -- assigning it would re-point the response + * object's prototype instead of adding a property, and emitting it would hand a prototype + * pollution vector to every client that parses the response. + */ +const PROTECTED_EXPANSION_KEYS = new Set(["@id", "id", "_id", "__rerum", "__deleted", "@context", "__proto__"]) + +/** + * Reduce a supplied POST body to the literal MongoDB filter keys the expand job will honor. + * @param supplied The parsed JSON request body. + * @return An object of filter keys, minus the ones this endpoint owns. + */ +function sanitizeExpansionFilters(supplied) { + const filters = {} + for (const [key, value] of Object.entries(supplied)) { + if (RESERVED_FILTER_KEYS.some(reserved => key === reserved || key.startsWith(`${reserved}.`))) continue + filters[key] = value + } + return filters +} + +/** + * The [key, value] assertions an Annotation makes about the entity it targets. + * Only 'body' and 'bodyValue' are read -- an Annotation carrying neither is ignored, and no other + * property of the Annotation can leak onto the entity. + * + * Anticipates the likely Annotation body formats + * - bodyValue: 'text' the W3C shorthand, which has no key of its own + * - body: {'key': 'value'} a single assertion + * - body: {'key': {...}} a single assertion, value kept as-is + * - body: {'type':'TextualBody', 'value': 'text', ...} kept whole so 'format' and 'language' survive + * + * @param anno An Annotation document. + * @return An Array of [key, value] pairs to merge onto the entity. + */ +function assertionsFrom(anno) { + const assertions = [] + if (typeof anno.bodyValue === "string") assertions.push(["bodyValue", anno.bodyValue]) + const body = anno.body + // Skip Annotations carrying multiple bodies, and string bodies that are an IRI referencing an + // external resource with no embedded value to expand with. + if (!body || typeof body !== "object" || Array.isArray(body)) return assertions + if ((body.type ?? body["@type"]) === "TextualBody") { + assertions.push(["bodyValue", body]) + return assertions + } + const keys = Object.keys(body) + // Any other multi-key body is structural rather than assertional and cannot be attributed to a + // single entity property. This is what skips the Choice, Composite, and List multiplicity + // constructs, which are all shaped {type, items}. + if (keys.length !== 1) return assertions + assertions.push([keys[0], body[keys[0]]]) + return assertions +} + +/** + * Merge the assertions of the gathered Annotations onto a copy of the entity, as raw values. + * Unlike the Gallery of Glosses expand(), values are not wrapped and not unwrapped -- what the + * Annotation says is what the entity gets. When more than one current Annotation asserts the same + * key, or the entity already carries it, the values collect into an Array. + * @param primitiveEntity The unexpanded entity. + * @param annos The Annotations targeting it. + * @return A new, expanded entity object. + */ +function applyRawExpansion(primitiveEntity, annos) { + const expandedEntity = structuredClone(primitiveEntity) + // Hold __rerum aside so it can be re-appended after the merged properties. It is the + // last property on a stored object and should stay last on an expanded one. + const rerumProp = expandedEntity.__rerum + delete expandedEntity.__rerum + for (const anno of annos) { + for (const [key, value] of assertionsFrom(anno)) { + if (PROTECTED_EXPANSION_KEYS.has(key)) continue + if (!Object.hasOwn(expandedEntity, key)) { + expandedEntity[key] = value + continue + } + const existing = Array.isArray(expandedEntity[key]) ? expandedEntity[key] : [expandedEntity[key]] + expandedEntity[key] = Array.isArray(value) ? [...existing, ...value] : [...existing, value] + } + } + if (rerumProp !== undefined) expandedEntity.__rerum = rerumProp + return expandedEntity +} + +/** + * Query the MongoDB for the object with the _id or __rerum.slug provided in the request URL, then + * merge in the assertions of all the current Annotations targeting it. + * + * GET recognizes the '?generator=' and '?creator=' convenience parameters only. + * POST reads literal MongoDB filter keys from the JSON body and ignores URL parameters as filters. + * Both methods page the Annotation search with the usual '?limit=' and '?skip=' parameters. + * */ +const idExpanded = async function (req, res, next) { + res.set("Content-Type", "application/json; charset=utf-8") + const id = req.params["_id"] + const isPost = req.method === "POST" + //Paging is transport rather than a filter, so it comes off the URL for both methods. + //The default is generous because an expansion wants every Annotation it can get, and an + //entity with more than 200 targeting it is not expected. + const pagination = getPagination(req.query, 200) + let filters = {} + if (isPost) { + //Express leaves the body undefined when a POST supplies none. That is an unfiltered expand. + const supplied = req.body ?? {} + if (typeof supplied !== "object" || Array.isArray(supplied)) { + const err = { + "message": "The /expanded request body must be a JSON object of filter properties.", + "status": 400 + } + return next(utils.createExpressError(err)) + } + filters = sanitizeExpansionFilters(supplied) + } + else { + //Repeated query parameters arrive as an Array, which is not a filter value we support. + if (typeof req.query.generator === "string" && req.query.generator) filters["__rerum.generatedBy"] = req.query.generator + if (typeof req.query.creator === "string" && req.query.creator) filters.creator = req.query.creator + } + try { + const match = await db.findOne({"$or": [{"_id": id}, {"__rerum.slug": id}]}) + if (!match) { + const err = { + "message": `No RERUM object with id '${id}'`, + "status": 404 + } + return next(utils.createExpressError(err)) + } + res.set(utils.configureWebAnnoHeadersFor(match)) + //Support built in browser caching. A POST response is not cacheable. + if (!isPost) res.set("Cache-Control", "max-age=86400, must-revalidate") + // No Last-Modified here, unlike GET /v1/id/:_id. It would compare against the root entity + // before the targeting Annotations are merged in. + // Include current version for optimistic locking + res.set('Current-Overwritten-Version', match.__rerum?.isOverwritten ?? "") + // Annotations target the stored URI, so this must come off the raw match. idNegotiation() + // below rebuilds 'id' from RERUM_ID_PREFIX, which is not necessarily the stored host. + const targetId = match["@id"] ?? match.id + const annos = targetId ? await findLeafAnnotationsFor(targetId, filters, pagination) : [] + // Let clients detect a full page. When this equals the limit there may be more to gather, + // and the entity in hand is expanded from only part of its Annotations. + res.set('Annotations-Merged', String(annos.length)) + let expanded = applyRawExpansion(match, annos) + expanded = idNegotiation(expanded) + res.location(_contextid(expanded["@context"]) ? expanded.id : expanded["@id"]) + res.json(expanded) + } catch (error) { + return next(utils.createExpressError(error)) + } +} + export { create, query, - id + id, + idExpanded } diff --git a/controllers/gog.js b/controllers/gog.js index db69a3e5..bfbe1b7c 100644 --- a/controllers/gog.js +++ b/controllers/gog.js @@ -8,7 +8,7 @@ import { newID, isValidID, db } from '../database/index.js' import utils from '../utils.js' -import { _contextid, ObjectID, getAgentClaim, getPagination, parseDocumentID, idNegotiation } from './utils.js' +import { _contextid, ObjectID, getAgentClaim, getPagination, parseDocumentID, idNegotiation, findLeafAnnotationsFor } from './utils.js' // The Gallery of Glosses agents, by RERUM ObjectId. Prod (store) and dev (devstore) mint different // agents; only the trailing id is compared, so either host spelling matches. @@ -336,66 +336,13 @@ const expand = async function(primitiveEntity, GENERATOR=undefined, CREATOR=unde // An entity is expandable if it carries a URI under either '@id' or 'id'. if(!primitiveEntity?.["@id"] && !primitiveEntity?.id) return primitiveEntity const targetId = primitiveEntity["@id"] ?? primitiveEntity.id ?? "unknown" - // '$and' is always present so the GENERATOR and CREATOR blocks below can push into it from - // either branch. 'annoTypeConditions' is always pushed, so it is never the empty Array Mongo rejects. - let queryObj = { - "__rerum.history.next": { $exists: true, $size: 0 }, - "$and": [] - } - let targetPatterns = ["target", "target.@id", "target.id"] - let targetConditions = [] - let annoTypeConditions = [{"type": "Annotation"}, {"@type":"Annotation"}, {"@type":"oa:Annotation"}] - - if (targetId.startsWith("http")) { - for(const targetKey of targetPatterns){ - targetConditions.push({ [targetKey]: targetId.replace(/^https?/, "http") }) - targetConditions.push({ [targetKey]: targetId.replace(/^https?/, "https") }) - } - queryObj["$and"].push({"$or": targetConditions}, {"$or": annoTypeConditions}) - } - else{ - queryObj["$and"].push({"$or": annoTypeConditions}) - queryObj.target = targetId - } - - // Only expand with data from a specific app - if(GENERATOR) { - // Need to check http:// and https:// - const generatorConditions = [ - {"__rerum.generatedBy": GENERATOR.replace(/^https?/, "http")}, - {"__rerum.generatedBy": GENERATOR.replace(/^https?/, "https")} - ] - if (GENERATOR.startsWith("http")) { - queryObj["$and"].push({"$or": generatorConditions }) - } - else{ - // It should be a URI, but this can be a fallback. - queryObj["__rerum.generatedBy"] = GENERATOR - } - } - // Only expand with data from a specific creator - if(CREATOR) { - // Need to check http:// and https:// - const creatorConditions = [ - {"creator": CREATOR.replace(/^https?/, "http")}, - {"creator": CREATOR.replace(/^https?/, "https")} - ] - if (CREATOR.startsWith("http")) { - queryObj["$and"].push({"$or": creatorConditions }) - } - else{ - // It should be a URI, but this can be a fallback. - queryObj["creator"] = CREATOR - } - } - - // Get the Annotations targeting this Entity from the db. Remove _id property. - // Assuming we do not need paged query here - let matches = await db.find(queryObj).toArray() - matches = matches.map(o => { - delete o._id - return o - }) + // Only expand with data from a specific app and/or a specific creator. The shared helper + // applies the leaf, target, and Annotation type constraints and doubles these two URIs + // across the http/https spellings. + const filters = {} + if(GENERATOR) filters["__rerum.generatedBy"] = GENERATOR + if(CREATOR) filters.creator = CREATOR + const matches = await findLeafAnnotationsFor(targetId, filters) // Combine the Annotation bodies with the primitive object. // Mirror DEER's client-side expand() (deer-utils.js buildValueObject) diff --git a/controllers/utils.js b/controllers/utils.js index dd455d05..72e2614a 100644 --- a/controllers/utils.js +++ b/controllers/utils.js @@ -109,6 +109,69 @@ const generateSlugId = async function(slug_id="", next){ return slug_return } +// RERUM has minted these two under both 'http' and 'https' over the years, so a filter on either +// must match both spellings. Every other supplied filter key is applied exactly as given. +const URI_DOUBLED_FILTER_KEYS = new Set(["__rerum.generatedBy", "creator"]) + +/** + * Find the current (leaf) Annotations targeting an entity, for expansion. + * + * Anticipates the likely Annotation target formats + * - target: 'uri' + * - target: {'id':'uri'} + * - target: {'@id':'uri'} + * and the likely Annotation type formats + * - {"type": "Annotation"}, {"@type": "Annotation"}, {"@type": "oa:Annotation"} + * + * @param targetId The '@id' or 'id' URI of the entity being expanded. + * @param filters Literal MongoDB filter keys to AND into the query. Already sanitized by the + * caller -- the leaf, type, and target constraints here cannot be overruled. + * @param pagination A {limit, skip} pair from getPagination(). When supplied, the query is sorted + * by '_id' first -- Mongo's natural order is not stable across paged calls, so + * without a sort a client walking pages could miss or repeat Annotations. + * Omit it to fetch every match, which is the long standing expand() behavior. + * @return An Array of matching Annotation documents, with '_id' removed. + */ +const findLeafAnnotationsFor = async function (targetId, filters = {}, pagination = null) { + // '$and' is always present so the filter conditions below can push into it from either branch. + // 'annoTypeConditions' is always pushed, so it is never the empty Array Mongo rejects. + const queryObj = { + "__rerum.history.next": { $exists: true, $size: 0 }, + "$and": [] + } + const annoTypeConditions = [{"type": "Annotation"}, {"@type": "Annotation"}, {"@type": "oa:Annotation"}] + if (targetId.startsWith("http")) { + const targetConditions = [] + for (const targetKey of ["target", "target.@id", "target.id"]) { + targetConditions.push({ [targetKey]: targetId.replace(/^https?/, "http") }) + targetConditions.push({ [targetKey]: targetId.replace(/^https?/, "https") }) + } + queryObj["$and"].push({"$or": targetConditions}, {"$or": annoTypeConditions}) + } + else { + queryObj["$and"].push({"$or": annoTypeConditions}) + queryObj.target = targetId + } + for (const [key, value] of Object.entries(filters)) { + if (URI_DOUBLED_FILTER_KEYS.has(key) && typeof value === "string" && /^https?:\/\//.test(value)) { + queryObj["$and"].push({"$or": [ + { [key]: value.replace(/^https?/, "http") }, + { [key]: value.replace(/^https?/, "https") } + ]}) + continue + } + queryObj["$and"].push({ [key]: value }) + } + // Get the Annotations targeting this Entity from the db. Remove _id property. + let cursor = db.find(queryObj) + if (pagination) cursor = cursor.sort({ "_id": 1 }).limit(pagination.limit).skip(pagination.skip) + const matches = await cursor.toArray() + return matches.map(o => { + delete o._id + return o + }) +} + // Handle index actions const index = function (req, res, next) { res.json({ @@ -465,6 +528,7 @@ async function healReleasesTree(releasing) { export { _contextid, idNegotiation, + findLeafAnnotationsFor, getPagination, generateSlugId, index, diff --git a/database/__mocks__/index.js b/database/__mocks__/index.js index 51ec6a2c..b155179a 100644 --- a/database/__mocks__/index.js +++ b/database/__mocks__/index.js @@ -41,6 +41,7 @@ function createMockFunction(implementation = () => undefined) { function createCursor() { return { + sort: createMockFunction(function () { return this }), limit: createMockFunction(function () { return this }), skip: createMockFunction(function () { return this }), toArray: createMockFunction(() => Promise.resolve([])) diff --git a/db-controller.js b/db-controller.js index 7f161667..99f5c163 100644 --- a/db-controller.js +++ b/db-controller.js @@ -8,7 +8,7 @@ // Import controller modules import { index, idNegotiation, generateSlugId, remove } from './controllers/utils.js' -import { create, query, id } from './controllers/crud.js' +import { create, query, id, idExpanded } from './controllers/crud.js' import { searchAsWords, searchAsPhrase } from './controllers/search.js' import { deleteObj } from './controllers/delete.js' import { putUpdate, patchUpdate, patchSet, patchUnset, overwrite } from './controllers/update.js' @@ -32,6 +32,7 @@ export default { searchAsWords, searchAsPhrase, id, + idExpanded, bulkCreate, bulkUpdate, queryHeadRequest, diff --git a/openapi/contracts/core-provider.openapi.yaml b/openapi/contracts/core-provider.openapi.yaml index 7174b34a..fa603009 100644 --- a/openapi/contracts/core-provider.openapi.yaml +++ b/openapi/contracts/core-provider.openapi.yaml @@ -44,6 +44,99 @@ paths: description: Object headers '404': $ref: '#/components/responses/NotFound' + /id/{id}/expanded: + get: + summary: Read object by id with its current Annotations merged in + operationId: getExpandedObjectById + parameters: + - $ref: '#/components/parameters/ObjectId' + - in: query + name: generator + required: false + description: Only expand with Annotations generated by this registered app agent. + schema: + type: string + - in: query + name: creator + required: false + description: Only expand with Annotations attributed to this creator. + schema: + type: string + - in: query + name: limit + required: false + description: Maximum Annotations to gather. Defaults to 200. + schema: + type: integer + - in: query + name: skip + required: false + description: Annotations to skip before gathering. Defaults to 0. + schema: + type: integer + responses: + '200': + description: Expanded object payload + content: + application/json: + schema: + $ref: '#/components/schemas/GenericObject' + '404': + $ref: '#/components/responses/NotFound' + head: + summary: Read expanded object headers by id + operationId: headExpandedObjectById + parameters: + - $ref: '#/components/parameters/ObjectId' + responses: + '200': + description: Expanded object headers + '404': + $ref: '#/components/responses/NotFound' + post: + summary: Read object by id with its current Annotations merged in, filtered by the request body + operationId: postExpandedObjectById + description: >- + The request body is an object of literal MongoDB filter properties ANDed into the search for + Annotations targeting the entity. URL query parameters supply no filters, though 'limit' and + 'skip' still page the search. The leaf version, the Annotation type, and the target + constraints are applied automatically and cannot be overruled, so 'target', 'type', '@type', + and '__rerum.history' keys are ignored. + parameters: + - $ref: '#/components/parameters/ObjectId' + - in: query + name: limit + required: false + description: Maximum Annotations to gather. Defaults to 200. + schema: + type: integer + - in: query + name: skip + required: false + description: Annotations to skip before gathering. Defaults to 0. + schema: + type: integer + requestBody: + required: false + content: + application/json: + schema: + $ref: '#/components/schemas/GenericObject' + responses: + '200': + description: Expanded object payload + content: + application/json: + schema: + $ref: '#/components/schemas/GenericObject' + '400': + $ref: '#/components/responses/BadRequest' + '404': + $ref: '#/components/responses/NotFound' + '413': + $ref: '#/components/responses/PayloadTooLarge' + '415': + $ref: '#/components/responses/UnsupportedMediaType' /since/{id}: get: summary: Read updates since id diff --git a/public/API.html b/public/API.html index 38f2df93..a105fc3c 100644 --- a/public/API.html +++ b/public/API.html @@ -48,6 +48,7 @@

API (1.1.0)

  • GET @@ -61,6 +62,7 @@

    API (1.1.0)

  • Custom Query
  • Text Search
  • Phrase Search
    • +
  • Expanded record with filters
  • HTTP POST Method Override
    • @@ -149,6 +151,99 @@

    Single record by id

    This can be used directly in the browser. Try it to see what the response resp looks like. https://devstore.rerum.io/v1/id/11111

    +

    Expanded record by id

    + + + + + + + + + + + + + + + +
    PatternPayloadResponse
    /id/_id/expandedempty200 {JSON}
    +

    + Gather every current Annotation targeting the record and merge what those Annotations assert onto it. + This does the entity assembly that client applications otherwise perform with many + /query requests. +

    + +

    + Only the current (leaf) versions of Annotations are gathered. No token is required. + The response is a raw assembly—if your application needs the data in a particular shape, format the + response for your own internal needs. +

    +
    What gets merged
    + +

    +

    Javascript Example
    + 
    
+                const expanded = await fetch("https://devstore.rerum.io/v1/id/11111/expanded").then(resp => resp.json()).catch(err => {throw err})
+
    +

    +

    + This can be used directly in the browser. Try it to see what the response resp looks like. + https://devstore.rerum.io/v1/id/11111/expanded +

    History tree before this version

    @@ -748,6 +843,97 @@ Results are returned sorted by relevance score in descending order. The __rerum.score property indicates match quality.

    +

    Expanded record with filters

    +
    + + + + + + + + + + + + + + +
    PatternPayloadResponse
    /id/_id/expanded{JSON}200 {JSON}
    +

    + The same expansion as + GET /id/_id/expanded, + with the search for Annotations narrowed by the request body. Use this when the convenience parameters on + the GET are not enough. Unlike the GET, this response is not browser cached. +

    + +

    + Three constraints belong to the endpoint and cannot be overruled. Supplying them is not an error—they are + ignored, by exact name or as a dotted prefix. +

    + + + + + + + + + + + + + + + + + + + + + +
    IgnoredAlways applied instead
    target, + target.@id, + target.idAnnotations targeting the record at + _id
    type, + @typeAnnotations only
    __rerum.history.next, + __rerum.history.previous, + __rerum.history.primeCurrent (leaf) versions only
    +

    +

    Javascript Example
    + 
    
+                const expanded = await fetch("https://devstore.rerum.io/v1/id/11111/expanded", {
+                    method: "POST",
+                    headers: { "Content-Type": "application/json" },
+                    body: JSON.stringify({
+                        "__rerum.generatedBy": "https://devstore.rerum.io/v1/id/agent7",
+                        "motivation": "describing"
+                    })
+                }).then(resp => resp.json()).catch(err => {throw err})
+
    +

    HTTP POST Method Override

    This section is non-normative.

    diff --git a/routes/id.js b/routes/id.js index fdfca44f..3e2069a1 100644 --- a/routes/id.js +++ b/routes/id.js @@ -2,6 +2,15 @@ import express from 'express' const router = express.Router() //This controller will handle all MongoDB interactions. import controller from '../db-controller.js' +import rest from '../rest.js' + +router.route('/:_id/expanded') + .get(controller.idExpanded) + .post(rest.verifyJsonContentType, controller.idExpanded) + .all((req, res, next) => { + res.statusMessage = 'Improper request method, please use GET or POST.' + res.status(405).end() + }) router.route('/:_id') .get(controller.id) From c060e50abcdf433e8a9d72a3a1d47b6434ad5fb7 Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Fri, 7 Aug 2026 13:40:52 -0500 Subject: [PATCH 2/4] Catch the W3C SpecificResource form target variants as well --- controllers/utils.js | 6 +++++- public/API.html | 14 +++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/controllers/utils.js b/controllers/utils.js index 72e2614a..8aeb499a 100644 --- a/controllers/utils.js +++ b/controllers/utils.js @@ -120,6 +120,8 @@ const URI_DOUBLED_FILTER_KEYS = new Set(["__rerum.generatedBy", "creator"]) * - target: 'uri' * - target: {'id':'uri'} * - target: {'@id':'uri'} + * - target: {'source':'uri', 'type':'SpecificResource'} the W3C SpecificResource + * - target: {'source':{'id':'uri'}} a SpecificResource with an embedded source * and the likely Annotation type formats * - {"type": "Annotation"}, {"@type": "Annotation"}, {"@type": "oa:Annotation"} * @@ -142,7 +144,9 @@ const findLeafAnnotationsFor = async function (targetId, filters = {}, paginatio const annoTypeConditions = [{"type": "Annotation"}, {"@type": "Annotation"}, {"@type": "oa:Annotation"}] if (targetId.startsWith("http")) { const targetConditions = [] - for (const targetKey of ["target", "target.@id", "target.id"]) { + // 'target.source' is the W3C SpecificResource, which is how an Annotation targets a + // fragment or a selected region of a resource rather than the whole of it. + for (const targetKey of ["target", "target.@id", "target.id", "target.source", "target.source.@id", "target.source.id"]) { targetConditions.push({ [targetKey]: targetId.replace(/^https?/, "http") }) targetConditions.push({ [targetKey]: targetId.replace(/^https?/, "https") }) } diff --git a/public/API.html b/public/API.html index a105fc3c..30c56868 100644 --- a/public/API.html +++ b/public/API.html @@ -225,7 +225,10 @@

    Expanded record by id

    _id, __rerum, __deleted, and - @context. + @context. An assertion naming + __proto__ is dropped for the same reason—it + is not data, and emitting it would hand a prototype pollution vector to every client that parses the + response.
  • Skipped for now, as future work: a body with more than one property, an Annotation with multiple bodies, the Choice, @@ -904,9 +907,14 @@

    Expanded record with filters

    target, target.@id, - target.id + target.id, + target.source, + target.source.@id, + target.source.id Annotations targeting the record at - _id + _id, including those + targeting it through a + SpecificResource type, From 1cb50865b180c21dbf47656bcb783921c897fa6b Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Tue, 11 Aug 2026 10:44:32 -0500 Subject: [PATCH 3/4] changes during review --- controllers/crud.js | 19 +++++++++++++++++-- controllers/gog.js | 13 ++++++++++--- controllers/utils.js | 24 ++++++++++++++++++++++-- public/API.html | 4 +++- 4 files changed, 52 insertions(+), 8 deletions(-) diff --git a/controllers/crud.js b/controllers/crud.js index bcadc537..5c7946f3 100644 --- a/controllers/crud.js +++ b/controllers/crud.js @@ -159,6 +159,14 @@ function sanitizeExpansionFilters(supplied) { return filters } +/** + * The Annotation body types whose value is kept whole rather than read as a single assertion. + * The OA prefixed spelling is honored for the Annotation type in findLeafAnnotationsFor(), so it + * is honored here too. Without it an 'oa:TextualBody' falls through to the single key check and + * is dropped, since a TextualBody always carries at least a type and a value. + */ +const TEXTUAL_BODY_TYPES = new Set(["TextualBody", "oa:TextualBody"]) + /** * The [key, value] assertions an Annotation makes about the entity it targets. * Only 'body' and 'bodyValue' are read -- an Annotation carrying neither is ignored, and no other @@ -169,6 +177,7 @@ function sanitizeExpansionFilters(supplied) { * - body: {'key': 'value'} a single assertion * - body: {'key': {...}} a single assertion, value kept as-is * - body: {'type':'TextualBody', 'value': 'text', ...} kept whole so 'format' and 'language' survive + * - body: {'@type':'oa:TextualBody', ...} the OA prefixed spelling of the same * * @param anno An Annotation document. * @return An Array of [key, value] pairs to merge onto the entity. @@ -180,7 +189,7 @@ function assertionsFrom(anno) { // Skip Annotations carrying multiple bodies, and string bodies that are an IRI referencing an // external resource with no embedded value to expand with. if (!body || typeof body !== "object" || Array.isArray(body)) return assertions - if ((body.type ?? body["@type"]) === "TextualBody") { + if (TEXTUAL_BODY_TYPES.has(body.type ?? body["@type"])) { assertions.push(["bodyValue", body]) return assertions } @@ -280,9 +289,15 @@ const idExpanded = async function (req, res, next) { // Let clients detect a full page. When this equals the limit there may be more to gather, // and the entity in hand is expanded from only part of its Annotations. res.set('Annotations-Merged', String(annos.length)) + // This deployment's '/expanded' URI, not the entity URI. The entity URI would hand back + // the unexpanded record, and it cannot be the base for this one either -- an entity minted + // by another RERUM carries that host in its stored '@id', and there is no guarantee the + // other host serves '/expanded' at all. RERUM_ID_PREFIX is how idNegotiation() mints ids, + // so this stays on the host actually answering the request. + const expandedLocation = `${process.env.RERUM_ID_PREFIX}${match._id}/expanded` let expanded = applyRawExpansion(match, annos) expanded = idNegotiation(expanded) - res.location(_contextid(expanded["@context"]) ? expanded.id : expanded["@id"]) + res.location(expandedLocation) res.json(expanded) } catch (error) { return next(utils.createExpressError(error)) diff --git a/controllers/gog.js b/controllers/gog.js index bfbe1b7c..e575910d 100644 --- a/controllers/gog.js +++ b/controllers/gog.js @@ -8,7 +8,7 @@ import { newID, isValidID, db } from '../database/index.js' import utils from '../utils.js' -import { _contextid, ObjectID, getAgentClaim, getPagination, parseDocumentID, idNegotiation, findLeafAnnotationsFor } from './utils.js' +import { ObjectID, getAgentClaim, getPagination, parseDocumentID, idNegotiation, findLeafAnnotationsFor } from './utils.js' // The Gallery of Glosses agents, by RERUM ObjectId. Prod (store) and dev (devstore) mint different // agents; only the trailing id is compared, so either host spelling matches. @@ -364,7 +364,9 @@ const expand = async function(primitiveEntity, GENERATOR=undefined, CREATOR=unde }, evidence: assertion?.evidence ?? anno.evidence ?? "" } - if(expandedEntity.hasOwnProperty(key)){ + // Object.hasOwn() rather than the method on the entity. A merged assertion named + // 'hasOwnProperty' would shadow the method and throw a TypeError on the next iteration. + if(Object.hasOwn(expandedEntity, key)){ expandedEntity[key] = Array.isArray(expandedEntity[key]) ? [...expandedEntity[key], valueObject] : [expandedEntity[key], valueObject] @@ -401,6 +403,11 @@ const expandedId = async function (req, res, next) { }) return next(utils.createExpressError(err)) } + // This '/gog/id' URI, not the entity URI. This response is the expanded representation, + // and the entity URI would hand back the unexpanded record instead. Built off + // RERUM_ID_PREFIX so the origin follows the deployment, and captured before expand() in + // case idNegotiation() reaches the match itself and drops '_id'. + const expandedLocation = new URL(`/gog/id/${match._id}`, process.env.RERUM_ID_PREFIX).href // Same browser-caching policy as GET /v1/id/:_id so this stable URI is cached (24h). res.set(utils.configureWebAnnoHeadersFor(match)) res.set("Cache-Control", "max-age=86400, must-revalidate") @@ -409,7 +416,7 @@ const expandedId = async function (req, res, next) { res.set("Current-Overwritten-Version", match.__rerum?.isOverwritten ?? "") let expanded = await expand(match, generator) expanded = idNegotiation(expanded) - res.location(_contextid(expanded["@context"]) ? expanded.id : expanded["@id"]) + res.location(expandedLocation) res.json(expanded) } catch (error) { return next(utils.createExpressError(error)) diff --git a/controllers/utils.js b/controllers/utils.js index 8aeb499a..4fbf0f2a 100644 --- a/controllers/utils.js +++ b/controllers/utils.js @@ -113,6 +113,19 @@ const generateSlugId = async function(slug_id="", next){ // must match both spellings. Every other supplied filter key is applied exactly as given. const URI_DOUBLED_FILTER_KEYS = new Set(["__rerum.generatedBy", "creator"]) +// The properties an Annotation can carry the URI of its target under. +const TARGET_KEYS = ["target", "target.@id", "target.id", "target.source", "target.source.@id", "target.source.id"] + +/** + * Escape the RegExp metacharacters in a literal so it can be embedded in a pattern and match only + * itself. A RERUM URI has at least the dots of its host to escape. + * @param literal A string to be matched literally. + * @return The same string, safe to concatenate into a RegExp source. + */ +function escapeRegex(literal) { + return literal.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") +} + /** * Find the current (leaf) Annotations targeting an entity, for expansion. * @@ -122,6 +135,7 @@ const URI_DOUBLED_FILTER_KEYS = new Set(["__rerum.generatedBy", "creator"]) * - target: {'@id':'uri'} * - target: {'source':'uri', 'type':'SpecificResource'} the W3C SpecificResource * - target: {'source':{'id':'uri'}} a SpecificResource with an embedded source + * - target: 'uri#xywh=0,0,100,100' a fragment of the resource * and the likely Annotation type formats * - {"type": "Annotation"}, {"@type": "Annotation"}, {"@type": "oa:Annotation"} * @@ -144,11 +158,17 @@ const findLeafAnnotationsFor = async function (targetId, filters = {}, paginatio const annoTypeConditions = [{"type": "Annotation"}, {"@type": "Annotation"}, {"@type": "oa:Annotation"}] if (targetId.startsWith("http")) { const targetConditions = [] + // Hanging a fragment off the URI is the other W3C way to target part of a resource rather + // than the whole of it, and an exact match will not catch one. Anchored at the front so + // the pattern can still use an index, and terminated by the '#' so it cannot spill onto a + // longer id. One pattern covers both spellings, since only the scheme is left unescaped. + const fragmentPattern = new RegExp(`^https?${escapeRegex(targetId.replace(/^https?/, ""))}#`) // 'target.source' is the W3C SpecificResource, which is how an Annotation targets a - // fragment or a selected region of a resource rather than the whole of it. - for (const targetKey of ["target", "target.@id", "target.id", "target.source", "target.source.@id", "target.source.id"]) { + // selected region of a resource rather than the whole of it. + for (const targetKey of TARGET_KEYS) { targetConditions.push({ [targetKey]: targetId.replace(/^https?/, "http") }) targetConditions.push({ [targetKey]: targetId.replace(/^https?/, "https") }) + targetConditions.push({ [targetKey]: fragmentPattern }) } queryObj["$and"].push({"$or": targetConditions}, {"$or": annoTypeConditions}) } diff --git a/public/API.html b/public/API.html index 30c56868..45838f66 100644 --- a/public/API.html +++ b/public/API.html @@ -914,7 +914,9 @@

    Expanded record with filters

    Annotations targeting the record at _id, including those targeting it through a - SpecificResource + SpecificResource or through a + fragment of its URI such as + #xywh=0,0,100,100 type, From e39ff2e5d684bb2b7d6957330deb4e38c4c4e3f2 Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Tue, 11 Aug 2026 11:21:55 -0500 Subject: [PATCH 4/4] changes during review --- controllers/gog.js | 7 ++++++- controllers/utils.js | 21 ++++++++++++++------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/controllers/gog.js b/controllers/gog.js index e575910d..166c02f4 100644 --- a/controllers/gog.js +++ b/controllers/gog.js @@ -323,6 +323,7 @@ const _gog_glosses_from_manuscript = async function (req, res, next) { * * Anticipate likely Annotation type formats * - {"type": "Annotation"} +* - {"type": "oa:Annotation"} * - {"@type": "Annotation"} * - {"@type": "oa:Annotation"} * @@ -350,7 +351,11 @@ const expand = async function(primitiveEntity, GENERATOR=undefined, CREATOR=unde let expandedEntity = structuredClone(primitiveEntity) for(const anno of matches){ const body = anno.body - if(!body || typeof body !== "object") continue + // Array.isArray() as well as the typeof check. An Array is a typeof 'object', and + // Object.keys() on a one element Array is ["0"] -- a length of 1 that would pass the + // single assertion check below and merge the body onto the entity under the key "0". + // Annotations carrying multiple bodies are not expanded with. + if(!body || typeof body !== "object" || Array.isArray(body)) continue const keys = Object.keys(body) if(keys.length !== 1) continue const key = keys[0] diff --git a/controllers/utils.js b/controllers/utils.js index 4fbf0f2a..a4e7a793 100644 --- a/controllers/utils.js +++ b/controllers/utils.js @@ -137,7 +137,8 @@ function escapeRegex(literal) { * - target: {'source':{'id':'uri'}} a SpecificResource with an embedded source * - target: 'uri#xywh=0,0,100,100' a fragment of the resource * and the likely Annotation type formats - * - {"type": "Annotation"}, {"@type": "Annotation"}, {"@type": "oa:Annotation"} + * - {"type": "Annotation"}, {"type": "oa:Annotation"} + * - {"@type": "Annotation"}, {"@type": "oa:Annotation"} * * @param targetId The '@id' or 'id' URI of the entity being expanded. * @param filters Literal MongoDB filter keys to AND into the query. Already sanitized by the @@ -155,20 +156,26 @@ const findLeafAnnotationsFor = async function (targetId, filters = {}, paginatio "__rerum.history.next": { $exists: true, $size: 0 }, "$and": [] } - const annoTypeConditions = [{"type": "Annotation"}, {"@type": "Annotation"}, {"@type": "oa:Annotation"}] + const annoTypeConditions = [ + {"type": "Annotation"}, {"type": "oa:Annotation"}, + {"@type": "Annotation"}, {"@type": "oa:Annotation"} + ] if (targetId.startsWith("http")) { const targetConditions = [] // Hanging a fragment off the URI is the other W3C way to target part of a resource rather - // than the whole of it, and an exact match will not catch one. Anchored at the front so - // the pattern can still use an index, and terminated by the '#' so it cannot spill onto a - // longer id. One pattern covers both spellings, since only the scheme is left unescaped. - const fragmentPattern = new RegExp(`^https?${escapeRegex(targetId.replace(/^https?/, ""))}#`) + // than the whole of it, and an exact match will not catch one. Anchored at the front and + // terminated by the '#' so the pattern cannot spill onto a longer id. One pattern per + // scheme rather than a single '^https?' -- Mongo bounds an index scan by the pattern's + // literal prefix, and '^https?' leaves it only 'http', which is every target URI stored. + const fragmentPatterns = ["http", "https"].map(scheme => + new RegExp(`^${escapeRegex(targetId.replace(/^https?/, scheme))}#`) + ) // 'target.source' is the W3C SpecificResource, which is how an Annotation targets a // selected region of a resource rather than the whole of it. for (const targetKey of TARGET_KEYS) { targetConditions.push({ [targetKey]: targetId.replace(/^https?/, "http") }) targetConditions.push({ [targetKey]: targetId.replace(/^https?/, "https") }) - targetConditions.push({ [targetKey]: fragmentPattern }) + for (const fragmentPattern of fragmentPatterns) targetConditions.push({ [targetKey]: fragmentPattern }) } queryObj["$and"].push({"$or": targetConditions}, {"$or": annoTypeConditions}) }