Skip to content

/v1/id/_:id/expanded endpoint #286

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
thehabes wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

/v1/id/_:id/expanded endpoint #286

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0cebde9
A more generic back end expand via a new /v1/id/_:id/expanded/ endpoint
thehabes Aug 7, 2026
c060e50
Catch the W3C SpecificResource form target variants as well
thehabes Aug 7, 2026
1cb5086
changes during review
thehabes Aug 11, 2026
e39ff2e
changes during review
thehabes Aug 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
182 changes: 180 additions & 2 deletions controllers/crud.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
*/
import { newID, isValidID, db } from '../database/index.js'
import utils from '../utils.js'
import { _contextid, idNegotiation, getPagination, generateSlugId, ObjectID, getAgentClaim, parseDocumentID } from './utils.js'
import { _contextid, idNegotiation, getPagination, generateSlugId, ObjectID, getAgentClaim, parseDocumentID, findLeafAnnotationsFor } from './utils.js'

/**
* Create a new Linked Open Data object in RERUM v1.
Expand Down Expand Up @@ -127,8 +127,186 @@ const id = async function (req, res, next) {
}
}

/**
* The expand job always constrains the Annotations it gathers to the leaf versions, to the
* Annotation types, and to the entity in the request URI. A client cannot influence those, so
* these keys are dropped from a supplied filter body by exact name or dotted prefix.
* The dot matters -- 'targetCollection' is a real property on Gallery of Glosses data and must
* still be usable as a filter.
*/
const RESERVED_FILTER_KEYS = ["target", "type", "@type", "__rerum.history"]

/**
* Identity and system properties an Annotation body must never overwrite. '@id' and '@context'
* are read by idNegotiation() and res.location() right after the merge, so clobbering them would
* break the response itself. '__proto__' is not data -- assigning it would re-point the response
* object's prototype instead of adding a property, and emitting it would hand a prototype
* pollution vector to every client that parses the response.
*/
const PROTECTED_EXPANSION_KEYS = new Set(["@id", "id", "_id", "__rerum", "__deleted", "@context", "__proto__"])

/**
* Reduce a supplied POST body to the literal MongoDB filter keys the expand job will honor.
* @param supplied The parsed JSON request body.
* @return An object of filter keys, minus the ones this endpoint owns.
*/
function sanitizeExpansionFilters(supplied) {
const filters = {}
for (const [key, value] of Object.entries(supplied)) {
if (RESERVED_FILTER_KEYS.some(reserved => key === reserved || key.startsWith(`${reserved}.`))) continue
filters[key] = value
}
return filters
}

/**
* The Annotation body types whose value is kept whole rather than read as a single assertion.
* The OA prefixed spelling is honored for the Annotation type in findLeafAnnotationsFor(), so it
* is honored here too. Without it an 'oa:TextualBody' falls through to the single key check and
* is dropped, since a TextualBody always carries at least a type and a value.
*/
const TEXTUAL_BODY_TYPES = new Set(["TextualBody", "oa:TextualBody"])

/**
* The [key, value] assertions an Annotation makes about the entity it targets.
* Only 'body' and 'bodyValue' are read -- an Annotation carrying neither is ignored, and no other
* property of the Annotation can leak onto the entity.
*
* Anticipates the likely Annotation body formats
* - bodyValue: 'text' the W3C shorthand, which has no key of its own
* - body: {'key': 'value'} a single assertion
* - body: {'key': {...}} a single assertion, value kept as-is
* - body: {'type':'TextualBody', 'value': 'text', ...} kept whole so 'format' and 'language' survive
* - body: {'@type':'oa:TextualBody', ...} the OA prefixed spelling of the same
*
* @param anno An Annotation document.
* @return An Array of [key, value] pairs to merge onto the entity.
*/
function assertionsFrom(anno) {
const assertions = []
if (typeof anno.bodyValue === "string") assertions.push(["bodyValue", anno.bodyValue])
const body = anno.body
// Skip Annotations carrying multiple bodies, and string bodies that are an IRI referencing an
// external resource with no embedded value to expand with.
if (!body || typeof body !== "object" || Array.isArray(body)) return assertions
if (TEXTUAL_BODY_TYPES.has(body.type ?? body["@type"])) {
assertions.push(["bodyValue", body])
return assertions
}
const keys = Object.keys(body)
// Any other multi-key body is structural rather than assertional and cannot be attributed to a
// single entity property. This is what skips the Choice, Composite, and List multiplicity
// constructs, which are all shaped {type, items}.
if (keys.length !== 1) return assertions
assertions.push([keys[0], body[keys[0]]])
return assertions
}

/**
* Merge the assertions of the gathered Annotations onto a copy of the entity, as raw values.
* Unlike the Gallery of Glosses expand(), values are not wrapped and not unwrapped -- what the
* Annotation says is what the entity gets. When more than one current Annotation asserts the same
* key, or the entity already carries it, the values collect into an Array.
* @param primitiveEntity The unexpanded entity.
* @param annos The Annotations targeting it.
* @return A new, expanded entity object.
*/
function applyRawExpansion(primitiveEntity, annos) {
const expandedEntity = structuredClone(primitiveEntity)
// Hold __rerum aside so it can be re-appended after the merged properties. It is the
// last property on a stored object and should stay last on an expanded one.
const rerumProp = expandedEntity.__rerum
delete expandedEntity.__rerum
for (const anno of annos) {
for (const [key, value] of assertionsFrom(anno)) {
if (PROTECTED_EXPANSION_KEYS.has(key)) continue
if (!Object.hasOwn(expandedEntity, key)) {
expandedEntity[key] = value
continue
}
const existing = Array.isArray(expandedEntity[key]) ? expandedEntity[key] : [expandedEntity[key]]
expandedEntity[key] = Array.isArray(value) ? [...existing, ...value] : [...existing, value]
}
}
if (rerumProp !== undefined) expandedEntity.__rerum = rerumProp
return expandedEntity
}

/**
* Query the MongoDB for the object with the _id or __rerum.slug provided in the request URL, then
* merge in the assertions of all the current Annotations targeting it.
*
* GET recognizes the '?generator=' and '?creator=' convenience parameters only.
* POST reads literal MongoDB filter keys from the JSON body and ignores URL parameters as filters.
* Both methods page the Annotation search with the usual '?limit=' and '?skip=' parameters.
* */
const idExpanded = async function (req, res, next) {
res.set("Content-Type", "application/json; charset=utf-8")
const id = req.params["_id"]
const isPost = req.method === "POST"
//Paging is transport rather than a filter, so it comes off the URL for both methods.
//The default is generous because an expansion wants every Annotation it can get, and an
//entity with more than 200 targeting it is not expected.
const pagination = getPagination(req.query, 200)
let filters = {}
if (isPost) {
//Express leaves the body undefined when a POST supplies none. That is an unfiltered expand.
const supplied = req.body ?? {}
if (typeof supplied !== "object" || Array.isArray(supplied)) {
const err = {
"message": "The /expanded request body must be a JSON object of filter properties.",
"status": 400
}
return next(utils.createExpressError(err))
}
filters = sanitizeExpansionFilters(supplied)
}
else {
//Repeated query parameters arrive as an Array, which is not a filter value we support.
if (typeof req.query.generator === "string" && req.query.generator) filters["__rerum.generatedBy"] = req.query.generator
if (typeof req.query.creator === "string" && req.query.creator) filters.creator = req.query.creator
}
try {
const match = await db.findOne({"$or": [{"_id": id}, {"__rerum.slug": id}]})
if (!match) {
const err = {
"message": `No RERUM object with id '${id}'`,
"status": 404
}
return next(utils.createExpressError(err))
}
res.set(utils.configureWebAnnoHeadersFor(match))
//Support built in browser caching. A POST response is not cacheable.
if (!isPost) res.set("Cache-Control", "max-age=86400, must-revalidate")
// No Last-Modified here, unlike GET /v1/id/:_id. It would compare against the root entity
// before the targeting Annotations are merged in.
// Include current version for optimistic locking
res.set('Current-Overwritten-Version', match.__rerum?.isOverwritten ?? "")
// Annotations target the stored URI, so this must come off the raw match. idNegotiation()
// below rebuilds 'id' from RERUM_ID_PREFIX, which is not necessarily the stored host.
const targetId = match["@id"] ?? match.id
const annos = targetId ? await findLeafAnnotationsFor(targetId, filters, pagination) : []
// Let clients detect a full page. When this equals the limit there may be more to gather,
// and the entity in hand is expanded from only part of its Annotations.
res.set('Annotations-Merged', String(annos.length))
// This deployment's '/expanded' URI, not the entity URI. The entity URI would hand back
// the unexpanded record, and it cannot be the base for this one either -- an entity minted
// by another RERUM carries that host in its stored '@id', and there is no guarantee the
// other host serves '/expanded' at all. RERUM_ID_PREFIX is how idNegotiation() mints ids,
// so this stays on the host actually answering the request.
const expandedLocation = `${process.env.RERUM_ID_PREFIX}${match._id}/expanded`
let expanded = applyRawExpansion(match, annos)
expanded = idNegotiation(expanded)
res.location(expandedLocation)
res.json(expanded)
} catch (error) {
return next(utils.createExpressError(error))
}
}

export {
create,
query,
id
id,
idExpanded
}
87 changes: 23 additions & 64 deletions controllers/gog.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

import { newID, isValidID, db } from '../database/index.js'
import utils from '../utils.js'
import { _contextid, ObjectID, getAgentClaim, getPagination, parseDocumentID, idNegotiation } from './utils.js'
import { ObjectID, getAgentClaim, getPagination, parseDocumentID, idNegotiation, findLeafAnnotationsFor } from './utils.js'

// The Gallery of Glosses agents, by RERUM ObjectId. Prod (store) and dev (devstore) mint different
// agents; only the trailing id is compared, so either host spelling matches.
Expand Down Expand Up @@ -323,6 +323,7 @@ const _gog_glosses_from_manuscript = async function (req, res, next) {
*
* Anticipate likely Annotation type formats
* - {"type": "Annotation"}
* - {"type": "oa:Annotation"}
* - {"@type": "Annotation"}
* - {"@type": "oa:Annotation"}
*
Expand All @@ -336,74 +337,25 @@ const expand = async function(primitiveEntity, GENERATOR=undefined, CREATOR=unde
// An entity is expandable if it carries a URI under either '@id' or 'id'.
if(!primitiveEntity?.["@id"] && !primitiveEntity?.id) return primitiveEntity
const targetId = primitiveEntity["@id"] ?? primitiveEntity.id ?? "unknown"
// '$and' is always present so the GENERATOR and CREATOR blocks below can push into it from
// either branch. 'annoTypeConditions' is always pushed, so it is never the empty Array Mongo rejects.
let queryObj = {
"__rerum.history.next": { $exists: true, $size: 0 },
"$and": []
}
let targetPatterns = ["target", "target.@id", "target.id"]
let targetConditions = []
let annoTypeConditions = [{"type": "Annotation"}, {"@type":"Annotation"}, {"@type":"oa:Annotation"}]

if (targetId.startsWith("http")) {
for(const targetKey of targetPatterns){
targetConditions.push({ [targetKey]: targetId.replace(/^https?/, "http") })
targetConditions.push({ [targetKey]: targetId.replace(/^https?/, "https") })
}
queryObj["$and"].push({"$or": targetConditions}, {"$or": annoTypeConditions})
}
else{
queryObj["$and"].push({"$or": annoTypeConditions})
queryObj.target = targetId
}

// Only expand with data from a specific app
if(GENERATOR) {
// Need to check http:// and https://
const generatorConditions = [
{"__rerum.generatedBy": GENERATOR.replace(/^https?/, "http")},
{"__rerum.generatedBy": GENERATOR.replace(/^https?/, "https")}
]
if (GENERATOR.startsWith("http")) {
queryObj["$and"].push({"$or": generatorConditions })
}
else{
// It should be a URI, but this can be a fallback.
queryObj["__rerum.generatedBy"] = GENERATOR
}
}
// Only expand with data from a specific creator
if(CREATOR) {
// Need to check http:// and https://
const creatorConditions = [
{"creator": CREATOR.replace(/^https?/, "http")},
{"creator": CREATOR.replace(/^https?/, "https")}
]
if (CREATOR.startsWith("http")) {
queryObj["$and"].push({"$or": creatorConditions })
}
else{
// It should be a URI, but this can be a fallback.
queryObj["creator"] = CREATOR
}
}

// Get the Annotations targeting this Entity from the db. Remove _id property.
// Assuming we do not need paged query here
let matches = await db.find(queryObj).toArray()
matches = matches.map(o => {
delete o._id
return o
})
// Only expand with data from a specific app and/or a specific creator. The shared helper
// applies the leaf, target, and Annotation type constraints and doubles these two URIs
// across the http/https spellings.
const filters = {}
if(GENERATOR) filters["__rerum.generatedBy"] = GENERATOR
if(CREATOR) filters.creator = CREATOR
const matches = await findLeafAnnotationsFor(targetId, filters)

// Combine the Annotation bodies with the primitive object.
// Mirror DEER's client-side expand() (deer-utils.js buildValueObject)
// When more than one current Annotation asserts the same key, collect the values into an Array.
let expandedEntity = structuredClone(primitiveEntity)
for(const anno of matches){
const body = anno.body
if(!body || typeof body !== "object") continue
// Array.isArray() as well as the typeof check. An Array is a typeof 'object', and
// Object.keys() on a one element Array is ["0"] -- a length of 1 that would pass the
// single assertion check below and merge the body onto the entity under the key "0".
// Annotations carrying multiple bodies are not expanded with.
if(!body || typeof body !== "object" || Array.isArray(body)) continue
const keys = Object.keys(body)
if(keys.length !== 1) continue
const key = keys[0]
Expand All @@ -417,7 +369,9 @@ const expand = async function(primitiveEntity, GENERATOR=undefined, CREATOR=unde
},
evidence: assertion?.evidence ?? anno.evidence ?? ""
}
if(expandedEntity.hasOwnProperty(key)){
// Object.hasOwn() rather than the method on the entity. A merged assertion named
// 'hasOwnProperty' would shadow the method and throw a TypeError on the next iteration.
if(Object.hasOwn(expandedEntity, key)){
expandedEntity[key] = Array.isArray(expandedEntity[key])
? [...expandedEntity[key], valueObject]
: [expandedEntity[key], valueObject]
Expand Down Expand Up @@ -454,6 +408,11 @@ const expandedId = async function (req, res, next) {
})
return next(utils.createExpressError(err))
}
// This '/gog/id' URI, not the entity URI. This response is the expanded representation,
// and the entity URI would hand back the unexpanded record instead. Built off
// RERUM_ID_PREFIX so the origin follows the deployment, and captured before expand() in
// case idNegotiation() reaches the match itself and drops '_id'.
const expandedLocation = new URL(`/gog/id/${match._id}`, process.env.RERUM_ID_PREFIX).href
// Same browser-caching policy as GET /v1/id/:_id so this stable URI is cached (24h).
res.set(utils.configureWebAnnoHeadersFor(match))
res.set("Cache-Control", "max-age=86400, must-revalidate")
Expand All @@ -462,7 +421,7 @@ const expandedId = async function (req, res, next) {
res.set("Current-Overwritten-Version", match.__rerum?.isOverwritten ?? "")
let expanded = await expand(match, generator)
expanded = idNegotiation(expanded)
res.location(_contextid(expanded["@context"]) ? expanded.id : expanded["@id"])
res.location(expandedLocation)
res.json(expanded)
} catch (error) {
return next(utils.createExpressError(error))
Expand Down
Loading
Loading