diff --git a/CLAUDE.md b/CLAUDE.md index 03567b6..26b4685 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -35,13 +35,13 @@ make install # prompts for base URL, keystore path, keystore password and opt Each `install.sh` sources `lib/ldh-app.sh` and is the app's own sequence of steps: -- `ldh admin make-public` — an authorization that makes the app readable by anyone +- `ldh admin make-public` — grants that make the app readable by anyone, the root and everything below it - `ldh_app_import_ns admin/model` — `ldh admin import ontology --file admin/model/ns.ttl`: installs `ns.ttl` as the namespace ontology, replacing the document and clearing the ontology cache - `ldh push --dir . "$LDH_BASE"` — PUTs every RDF file to the document its path spells and uploads every other file into its folder's document - `ldh_app_import_csv . imports.csv` — one `ldh import csv --wait` per manifest row (`query_filename,csv_filename,target,title`) - app-specific steps: `ldh admin create authorization`, `ldh packages add`, `ldh import rdf` -**Re-runs converge but are not clean.** `ldh push` PUTs and the ontology is reset before re-import, so documents and the model end up the same. `make-public`, `create authorization` and every import are POSTs, so each run adds another authorization and another import record. Nothing probes for an earlier install. +**Re-runs converge but are not clean.** `ldh push` PUTs and the ontology is reset before re-import, so documents and the model end up the same. `make-public` writes the same grants again. `create authorization` and every import are POSTs, so each run adds another authorization and another import record. Nothing probes for an earlier install. ### App layout diff --git a/README.md b/README.md index c3f8d99..e01819e 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ cd ../LinkedDataHub/cli && mvn package && export PATH="$PWD/bin:$PATH" `make install` prompts for the base URL, the owner's keystore, its password and an optional proxy URL, exports them as the `LDH_*` variables the CLI reads, and runs the app's `install.sh` — every installer sources the shared [`lib/ldh-app.sh`](lib/ldh-app.sh). To install unattended, export the variables and run `./install.sh`. -__Re-running an install converges but is not clean: `ldh push` replaces each document and the namespace ontology is reset before re-import, but `make-public`, `create authorization` and every import are POSTs, so each run adds another authorization and another import record.__ +__Re-running an install converges but is not clean: `ldh push` replaces each document and the namespace ontology is reset before re-import, and `make-public` writes the same grants again, but `create authorization` and every import are POSTs, so each run adds another authorization and another import record.__ ## Apps diff --git a/demo/copenhagen/install.sh b/demo/copenhagen/install.sh index b9d318b..5ed97c5 100755 --- a/demo/copenhagen/install.sh +++ b/demo/copenhagen/install.sh @@ -4,8 +4,8 @@ # # Reads LDH_BASE, LDH_CERT_FILE, LDH_CERT_PASSWORD and optionally LDH_PROXY; `make install` prompts # for them. Re-running converges on the same documents — PUT replaces each one and the ontology is -# reset before re-import — but make-public and every CSV import are POSTs, so each run adds another -# authorization and another import record. +# reset before re-import, and make-public writes the same grants again — but every CSV import is a +# POST, so each run adds another import record. set -euo pipefail app_dir="$(cd "$(dirname "$0")" && pwd)" diff --git a/demo/northwind-traders/install.sh b/demo/northwind-traders/install.sh index b9d318b..5ed97c5 100755 --- a/demo/northwind-traders/install.sh +++ b/demo/northwind-traders/install.sh @@ -4,8 +4,8 @@ # # Reads LDH_BASE, LDH_CERT_FILE, LDH_CERT_PASSWORD and optionally LDH_PROXY; `make install` prompts # for them. Re-running converges on the same documents — PUT replaces each one and the ontology is -# reset before re-import — but make-public and every CSV import are POSTs, so each run adds another -# authorization and another import record. +# reset before re-import, and make-public writes the same grants again — but every CSV import is a +# POST, so each run adds another import record. set -euo pipefail app_dir="$(cd "$(dirname "$0")" && pwd)" diff --git a/demo/unesco-thesaurus/install.sh b/demo/unesco-thesaurus/install.sh index 0e63776..e0d82ae 100755 --- a/demo/unesco-thesaurus/install.sh +++ b/demo/unesco-thesaurus/install.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash -# Installs the app onto a LinkedDataHub instance with the ldh CLI: makes it public, grants -# authenticated agents read access to its items, pushes the document tree with its files, imports the -# taxonomy editor package and imports the UNESCO Thesaurus SKOS data. The ontology, constructors, +# Installs the app onto a LinkedDataHub instance with the ldh CLI: makes it public, pushes the +# document tree with its files, imports the taxonomy editor package and imports the UNESCO Thesaurus +# SKOS data. The ontology, constructors, # constraints and stylesheet all come from the package. # # Reads LDH_BASE, LDH_CERT_FILE, LDH_CERT_PASSWORD and optionally LDH_PROXY; `make install` prompts -# for them. Re-running converges on the same documents — PUT replaces each one — but make-public, -# the authorization and the RDF import are POSTs, so each run adds another of each. +# for them. Re-running converges on the same documents — PUT replaces each one and make-public +# writes the same grants again — but the RDF import is a POST, so each run adds another. set -euo pipefail app_dir="$(cd "$(dirname "$0")" && pwd)" @@ -16,13 +16,6 @@ ldh_app_require_env ldh_app_step "Creating authorization to make the app public" ldh admin make-public -ldh_app_step "Creating authorization for authenticated agents to read items" -ldh admin create authorization \ - --label "Read access to graph items" \ - --agent-class "http://www.w3.org/ns/auth/acl#AuthenticatedAgent" \ - --to-all-in "https://w3id.org/atomgraph/linkeddatahub/document-hierarchy#Item" \ - --read - ldh_app_step "Pushing documents and files" ldh push --dir "$app_dir" "$LDH_BASE" diff --git a/docs/get-started/setup-1.mp4 b/docs/get-started/setup-1.mp4 index d017512..658e55d 100644 Binary files a/docs/get-started/setup-1.mp4 and b/docs/get-started/setup-1.mp4 differ diff --git a/docs/get-started/setup.ttl b/docs/get-started/setup.ttl index 0289504..2964f17 100644 --- a/docs/get-started/setup.ttl +++ b/docs/get-started/setup.ttl @@ -11,7 +11,7 @@ <#content> a ldh:XHTML ; rdf:value """

Set up LinkedDataHub on your local machine with Docker

- +

Start LinkedDataHub

This section assumes you will be running on your local machine, i.e. localhost. If you intend to run it on a different host, change the system base URI.

diff --git a/docs/install.sh b/docs/install.sh index 15b2432..cd7b3b9 100755 --- a/docs/install.sh +++ b/docs/install.sh @@ -4,8 +4,8 @@ # the sources reference. # # Reads LDH_BASE, LDH_CERT_FILE, LDH_CERT_PASSWORD and optionally LDH_PROXY; `make install` prompts -# for them. Re-running converges: PUT replaces each document. make-public is a POST and adds another -# authorization per run. +# for them. Re-running converges: PUT replaces each document and make-public writes the same grants +# again. set -euo pipefail app_dir="$(cd "$(dirname "$0")" && pwd)" diff --git a/docs/reference/administration/acl.ttl b/docs/reference/administration/acl.ttl index 2ef119b..d2bac98 100644 --- a/docs/reference/administration/acl.ttl +++ b/docs/reference/administration/acl.ttl @@ -14,7 +14,7 @@

LinkedDataHub access control is based on the W3C ACL ontology.

Access modes

-

There are four access modes (classes of operation) that map to HTTP methods:

+

There are three access modes (classes of operation) that map to HTTP methods:

@@ -40,13 +40,10 @@ - - - - -
Access modes and the HTTP methods they permit
Add information without removing or replacing any. POST
ControlChange the authorizations that apply to the resource.— (no corresponding HTTP method)
+

acl:Control grants nothing: authorizations are documents in the administration dataspace, + so changing them takes acl:Write there.

Agents

@@ -68,8 +65,8 @@

By default, only members of the owners group have access to the administration dataspace: its built-in - Full control authorization grants the Control mode (along with read and write) - to that group. See Authorizations.

+ Full control authorization grants them read, append and write access to the administration root and everything + below it. See Authorizations.

Note that an agent being a member of one of the above groups does not automatically provide it with an authorization. A valid authorization for the whole group has to be present.

@@ -78,11 +75,13 @@

Authorizations

An authorization explicitly grants access for an agent or a group - of agents to access a specific end-user dataspace document or a class - of its documents.

-

An agent has to be authorized using the Control mode to be - able to log in to the administration dataspace. By default such an - authorization is granted to the owners group.

+ of agents to a specific document (acl:accessTo), to every document below a container (acl:default), + or to the resources of a class (acl:accessToClass). Grants only add access: + no authorization overrides another.

+

An authorization counts only in its own document, the one whose foaf:primaryTopic it is, and a group's members count + only in the group's own document.

+

To use the administration dataspace, an agent needs access to its + documents like to any others. By default the owners group has it.

The default authorizations for groups and their respective access modes:

@@ -91,7 +90,7 @@ - + @@ -99,7 +98,7 @@ - + @@ -145,19 +144,47 @@ --read --append --write -

An authorization can also target a whole class of documents with acl:accessToClass. Granting the readers group read access to every container:

- 
@prefix acl:  <http://www.w3.org/ns/auth/acl#> .
+        
An authorization can also reach every document below a container with acl:default. Granting the readers group read access to
+            the orders container and every order in it:

+        

+            

+                
+                
+            

+            

+                
@prefix acl:  <http://www.w3.org/ns/auth/acl#> .
 @prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
-@prefix dh:   <https://w3id.org/atomgraph/linkeddatahub/document-hierarchy#> .
 
 <#this> a acl:Authorization ;
-    rdfs:label "Container read access" ;
+    rdfs:label "Order read access" ;
     acl:agentGroup <https://admin.localhost:4443/acl/groups/readers/#this> ;
     acl:mode acl:Read ;
-    acl:accessToClass dh:Container .

+    acl:accessTo <https://localhost:4443/orders/> ;
+    acl:default <https://localhost:4443/orders/> .
+ + + +

A document is below a container when its own sioc:has_parent or sioc:has_container leads there. acl:default does + not cover the container itself, hence the acl:accessTo alongside it.

+

acl:accessToClass grants access to the resources of a class, such as nfo:FileDataObject for file uploads. It refuses the + document classes (def:Root, dh:Container, dh:Item) and the classes above them: documents are granted through the + hierarchy.

A public-access authorization allows access for non-authenticated agents — the Publish stage of the tutorial shows the one ldh admin make-public creates, granting acl:Read to the foaf:Agent class - on every document class.

+ on the root and every document below it.

+
+

Document owners

+

The agent that creates a document is its owner (acl:owner), and stays so for the life of the document. The built-in + Owner access authorization grants read, append and write access to the lacl:OwnerAgent class, which matches + the owners of the requested document, on the end-user root and every document below it.

+
diff --git a/docs/reference/command-line-interface.ttl b/docs/reference/command-line-interface.ttl index 24cf4db..a493a1a 100644 --- a/docs/reference/command-line-interface.ttl +++ b/docs/reference/command-line-interface.ttl @@ -143,6 +143,7 @@ ldh create container --parent "$LDH_BASE" --title "Categories" --slug categories 
item=$(ldh create item --container "$LDH_BASE" --title "Example" --slug example)

ldh push writes many documents in one run and prints one line per written document URL or upload URI, in write order; progress and skipped entries go to standard error, and the first failed request stops the run.

Exit codes are 0 on success, 1 on an HTTP error status or runtime failure (the message goes to standard error; add --verbose for a stack trace), and 2 on a usage error.

+

Redirects are not followed and count as failures, with their Location on standard error: a PUT to a URI without a trailing slash is answered 308 Permanent Redirect and writes nothing. A write answered 303 See Other succeeds; ldh get --head reports any status, and ldh get --timegate takes its 302 Found.

Usage

@@ -398,6 +399,7 @@ ldh add file \\ "$doc"

add file prints the content-addressed URI of the upload ({base}uploads/{sha1}) rather than the document URI — the Data stage of the tutorial feeds exactly that URI into a CSV import.

+

It appends the file's description to the document (a multipart POST); with --replace it replaces the document's description with it instead (a multipart PUT).

See also the data import user guides.

Request bodies

ldh put and ldh post read RDF from an optional FILE argument, recognizing the syntax from its extension (ldh put "$LDH_BASE" root.ttl), or from standard input, where -t/--content-type is required since a stream carries no name. Either way relative URIs resolve against the target URI. ldh patch reads a SPARQL 1.1 update from standard input, validates it and sends it verbatim.

diff --git a/docs/reference/configuration.ttl b/docs/reference/configuration.ttl index 7b6b76d..ec03545 100644 --- a/docs/reference/configuration.ttl +++ b/docs/reference/configuration.ttl @@ -195,7 +195,9 @@ HOST=ec2-54-235-229-141.compute-1.amazonaws.com
Set to false if you are not using the self-signed server certificate (e.g. using a LetsEncrypt certificate instead). Not to be confused with the WebID client certificate. Defaults to true.
MAX_CONTENT_LENGTH
-
Maximum allowed request body size (nginx has a separate setting for this). Defaults to 2097152.
+
Maximum allowed request body size, in bytes; a larger body is refused with 413 Payload Too Large. Defaults to 2097152.
+
MAX_UPLOAD_LENGTH
+
Maximum size, in bytes, of a file upload (a multipart/form-data request body); a larger one is refused with 413 Payload Too Large. Defaults to MAX_CONTENT_LENGTH. nginx limits request bodies to this value when it is set, and to MAX_CONTENT_LENGTH otherwise.
MAX_QUERY_RESPONSE_LENGTH
The longest query result, in bytes, that the /sparql endpoint answers a query with; a longer one is refused with 502 Bad Gateway. Counted on the triplestore's response to the platform, not on what the caller receives. Defaults to 5242880 (5 MiB).
HTTP_MAX_THREADS
diff --git a/docs/reference/http-api.ttl b/docs/reference/http-api.ttl index d981b39..5b429be 100644 --- a/docs/reference/http-api.ttl +++ b/docs/reference/http-api.ttl @@ -150,8 +150,8 @@ ETag: "1e635b4-ntriples"
- + diff --git a/docs/reference/http-api/documents.ttl b/docs/reference/http-api/documents.ttl index e04467d..089d439 100644 --- a/docs/reference/http-api/documents.ttl +++ b/docs/reference/http-api/documents.ttl @@ -21,7 +21,8 @@

Methods

-

The API also supports the PATCH HTTP method which is optional in GSP. It accepts graph-scoped SPARQL updates that will modify the requested document. Only the INSERT/WHERE and DELETE WHERE forms are supported; GRAPH patterns are not allowed.

+

The API also supports the PATCH HTTP method which is optional in GSP. It accepts graph-scoped SPARQL updates that will modify the requested document. Only a single DELETE/INSERT … WHERE or DELETE WHERE operation is supported; GRAPH, WITH, USING and USING NAMED are not allowed (see graph-scoped updates).

+

A multipart/form-data body uploads files: a POST appends their descriptions to the document and a PUT replaces the document with them, as the same writes with an RDF body do. The upload limit applies to such a body instead of the request body limit (see configuration).

Trailing slashes in document URIs are enforced using 308 Permanent Redirect responses.

Default access for each group
Group Read access Write/append accessFull controlAdministration dataspace
Owners Read WriteControlRead, Append, Write
Append
http://www.w3.org/ns/auth/acl#modeOne link per access mode the agent holds on the requested document (acl:Read, acl:Append, acl:Write, - acl:Control); historical versions advertise acl:Read at mostOne link per access mode the agent holds on the requested document (acl:Read, acl:Append, + acl:Write); historical versions advertise acl:Read at most
timemap, timegate, original, self
@@ -47,9 +48,9 @@ - - - + + + @@ -66,19 +67,29 @@ - + - - - + + + + + + + + + + + + + @@ -88,13 +99,19 @@ - + - - - + + + + + + + + + @@ -103,28 +120,44 @@ - - - - - + + + + + + + + + + + + + + +
HTTP methods and the responses they return
Media type not supported
POSTAppends data to a document204 No ContentPOSTAppends data to a document204 No Content 400 Bad Request RDF syntax error
Media type not supported
422 Unprocessable Entity422 Unprocessable Entity Constraint violation in the resulting document
PUTUpserts a document200 OK

201 Created

308 Permanent Redirect		Body states the sioc:has_parent/sioc:has_container of a resource other than the document
Malformed IRI in the body (see below)
PUTUpserts a document204 No Content (replaced)

201 Created (created)

308 Permanent Redirect		 400 Bad Request RDF syntax error
Malformed document URI
409 ConflictThe parent of a document being created does not exist
413 Payload Too Large Request body too largeMedia type not supported
422 Unprocessable Entity422 Unprocessable Entity Constraint violation in the resulting document
DELETERemoves the requested document204 No ContentBody states the sioc:has_parent/sioc:has_container of a resource other than the document
Malformed IRI in the body (see below)
DELETERemoves the requested document204 No Content 404 Not Found Document with request URI not found
Deleting root, owner, or secretary documents is not allowed
PATCHModifies a document using SPARQL Update204 No Content422 Unprocessable EntitySPARQL update is not a single graph-scoped INSERT/DELETE operation, or removes an rdf:type409 ConflictThe container still has documents: delete them first
PATCHModifies a document using SPARQL Update204 No Content422 Unprocessable EntitySPARQL update is not a single DELETE/INSERT … WHERE or DELETE WHERE operation, uses GRAPH, WITH, USING or USING NAMED, or removes an rdf:type
Constraint violation in the updated document
Update changes the document's sioc:has_parent/sioc:has_container, or adds one of another resource
Malformed IRI in the body (see below)

Any method may additionally answer 403 Forbidden — see Access control. Any method that writes may answer 428 Precondition Required when the request carries no precondition, or 412 Precondition Failed when the one it carries is out of date — see Conditional writes.

+

A POST or PATCH answers with the ETag of the document it leaves, so the next write can quote it. A PUT answers with no ETag or Last-Modified: the document it stores is not the payload — the server adds its metadata and skolemizes blank nodes — and RFC 9110 allows a PUT response a validator only for a representation saved unchanged. A client that writes again after a PUT reads the tag with HEAD.

+

An IRI the IRI syntax does not allow — such as <http://[::1> or one with a space in it — is refused with 422 Unprocessable Entity, whether it comes in an RDF body, an RDF/POST form or a SPARQL update. The response names each value as an ldh:URISyntaxViolation, with the value as typed in ldh:violationValue; an RDF body names the first one only, as its parser stops there. A Turtle or N-Triples body with a space or < in an IRI is answered 400 Bad Request instead, as its tokenizer does not report the value.

Document metadata

Since LinkedDataHub 5, the document hierarchy is managed automatically.

By default, LinkedDataHub treats an RDF document as an item by giving it the dh:Item type and attaching it to the parent container using sioc:has_container. If the client wants to create a container instead, it has to explicitly add the dh:Container type on the document resource; the new container will be attached to its parent using sioc:has_parent. In either case, the URI of the new document will be relative to its parent's.

+

Containment is server-managed: a POST drops the document's sioc:has_parent/sioc:has_container from its body, + a PATCH cannot change them, and no write may state the containment of another resource. acl:default + grants rely on it.

LinkedDataHub will also manage additional document metadata, such as its owner and creation/modification timestamps.

For example, this HTTP request creating the Northwind regions container (Turtle syntax):

PUT /regions/ HTTP/1.1
 Host: localhost:4443
 Content-Type: text/turtle
+If-None-Match: *
 
 @prefix dh:     <https://w3id.org/atomgraph/linkeddatahub/document-hierarchy#> .
 @prefix dct:    <http://purl.org/dc/terms/> .
@@ -148,7 +181,8 @@ Content-Type: text/turtle
         
The HTTP request to produce a new item can be empty:

         
PUT /regions/1/ HTTP/1.1
 Host: localhost:4443
-Content-Type: text/turtle

+Content-Type: text/turtle
+If-None-Match: *

It will create an item document with the following triples:

@prefix dh:     <https://w3id.org/atomgraph/linkeddatahub/document-hierarchy#> .
 @prefix dct:    <http://purl.org/dc/terms/> .
@@ -165,10 +199,12 @@ Content-Type: text/turtle

Graph-scoped updates

-

A PATCH applies a SPARQL update scoped to the requested document's graph. The request body must contain a single update operation, in one of two forms: DELETE/INSERT with a WHERE clause (either template can be omitted, giving plain INSERT { } WHERE { } or DELETE { } WHERE { }), or the DELETE WHERE shorthand. Every other form is rejected with 422 Unprocessable Entity, as is a body with more than one operation. Appending a description to the Chai product:

+

A PATCH applies a SPARQL update scoped to the requested document's graph. The request body must contain a single update operation, in one of two forms: DELETE/INSERT with a WHERE clause (either template can be omitted, giving plain INSERT { } WHERE { } or DELETE { } WHERE { }), or the DELETE WHERE shorthand. Every other form is rejected with 422 Unprocessable Entity, as is a body with more than one operation. Like any write to an existing document, it quotes the document's entity tag (see Conditional writes). Appending a description to the Chai product:

PATCH /products/1/ HTTP/1.1
 Host: localhost:4443
+Accept: application/n-triples
 Content-Type: application/sparql-update
+If-Match: "1e635b4-ntriples"
 
 PREFIX schema: <https://schema.org/>
 
@@ -177,11 +213,13 @@ INSERT
     <https://localhost:4443/products/1/#this> schema:description "A delicate black tea blend" .
 }
 WHERE {}
-

The update executes against that one graph, so GRAPH patterns are not allowed — this request is rejected with +

The update executes against that one graph, so it cannot name another: GRAPH is not allowed in a template, in the WHERE pattern (including EXISTS and sub-queries) or in a DELETE WHERE, and neither are WITH, USING and USING NAMED. This request is rejected with 422 Unprocessable Entity:

PATCH /products/1/ HTTP/1.1
 Host: localhost:4443
+Accept: application/n-triples
 Content-Type: application/sparql-update
+If-Match: "1e635b4-ntriples"
 
 PREFIX schema: <https://schema.org/>
 
@@ -213,10 +251,10 @@ WHERE {}

A write to a document that already exists must say which state it was written against. Every write reads the document's graph, changes it in memory and writes the whole graph back, so two writers that name no precondition overwrite each other with nothing to show that anything was lost.

Such a write is answered 428 Precondition Required. To make it conditional, send either:

    -
  • If-Match with the document's current ETag, to modify what you read
    • +
  • If-Match with the document's current ETag, to modify what you read. If-Match: * asserts only that some state exists and does not count
  • If-None-Match: *, to create a document only if it does not exist yet
-

Creating a document needs no precondition — there is no prior state to have been written against — so a PUT to a URL that does not exist is accepted as it always was.

+

Creating a document needs no precondition — there is no prior state to have been written against — so a PUT to a URL that does not exist is accepted without one. A PUT that quotes an If-Match to a URL that does not exist is refused 412 Precondition Failed: the state it was written against is gone, and a document deleted in the meantime is not recreated.

If the document changed since you read it, the write is refused 412 Precondition Failed, and that response carries the document's current ETag so you can re-read, reapply your change and retry without an extra request. If-None-Match: * against a document that already exists is likewise refused 412, which is how a client can create-or-append in two steps.

@@ -235,7 +273,8 @@ Accept: application/n-triples Content-Type: application/sparql-update If-Match: "f731c06f5ac4fec6490dff1399bd5440" -HTTP/1.1 204 No Content +HTTP/1.1 204 No Content +ETag: "9b0e4d1c27a3f58e6c41d2a7b8f05e93"

HEAD is answered for any access mode the agent holds, not acl:Read alone, so an agent granted acl:Write or acl:Append can read the validator its writes have to quote without being able to read the document. GET still requires acl:Read.

The command line interface does this for you: every ldh command that writes reads the document's validator first and sends it.

diff --git a/docs/reference/http-api/system-endpoints.ttl b/docs/reference/http-api/system-endpoints.ttl index dbc6d7d..143e9c3 100644 --- a/docs/reference/http-api/system-endpoints.ttl +++ b/docs/reference/http-api/system-endpoints.ttl @@ -46,7 +46,7 @@ settings - The dataspace's own description in the dataspace configuration. PATCH with a SPARQL update edits it — this is how packages are installed and uninstalled. Editing is restricted to end-user dataspaces: a PATCH on an admin dataspace's settings answers 422 Unprocessable Entity + The dataspace's own description in the dataspace configuration. PATCH with a SPARQL update edits it — this is how packages are installed and uninstalled. The update takes the same forms as a document PATCH, and is held to the same conditional write terms: it quotes the settings' ETag in If-Match or is answered 428 Precondition Required, a stale tag is answered 412 Precondition Failed, and a successful update answers 204 No Content with the new ETag. Editing is restricted to end-user dataspaces: a PATCH on an admin dataspace's settings answers 422 Unprocessable Entity diff --git a/docs/reference/migrating-from-5x.ttl b/docs/reference/migrating-from-5x.ttl index 85d3bd0..5613ad7 100644 --- a/docs/reference/migrating-from-5x.ttl +++ b/docs/reference/migrating-from-5x.ttl @@ -160,6 +160,23 @@

A deployment with its own compose file adds the ./packages bind mount at /var/www/linkeddatahub/packages (PACKAGE_ROOT), where copies of imported package stylesheets survive a redeploy.

+
+

Access control

+

Authorizations follow plain WAC, the same in both dataspaces:

+
    +
  • acl:Control grants nothing. The owners' admin role is read, append and write on the administration root and everything below it.
    • +
  • Documents are granted through the hierarchy, with acl:default on a container. acl:accessToClass refuses the document + classes and the classes above them.
    • +
  • Document owners are matched by the lacl:OwnerAgent agent class, granted by a seeded Owner access authorization.
    • +
  • An authorization counts only in the document whose foaf:primaryTopic it is, and a group's members only in the group's own document.
    • +
  • ldh admin create authorization gains --default and loses --control. ldh admin make-public grants + the root and everything below it, and puts its SPARQL POST grant in acl/authorizations/public-sparql/.
    • +
+

An existing deployment migrates its stores with the two SPARQL updates in the platform's migrations/ directory, which the entrypoint does not + run. 6.0-hierarchy.ru.template completes the seeded hierarchy, once per store of each dataspace. 6.0-acl.ru.template rewrites + the authorizations, once per admin store. Both are idempotent, and their headers show how to run them. A grant on dh:Container or + dh:Item alone has no 6.0 equivalent and is dropped: recreate it with acl:default on the container it was meant for.

+

OAuth grants

The oauth2-login and oauth2-authorize grants move from the administration store to the end-user application's @@ -180,7 +197,7 @@

A write to a document that already exists must be conditional: If-Match with the document's current - ETag, or If-None-Match: * to create. An unconditional write is answered 428 Precondition Required.

+ ETag, or If-None-Match: * to create. An unconditional write, or one with If-Match: *, is answered 428 Precondition Required.

A client that wrote unconditionally now reads the tag first, as the ldh CLI does for you. An entity tag @@ -190,8 +207,11 @@ PATCH already was in 5.10. A POST used to be checked on its body alone and is now checked merged with the document, so appending to a document that violates a constraint is refused 422 until the violation is repaired. A PUT whose body does not type the document is checked as the dh:Item the server makes it, so it needs a dct:title.

-

The access control query fails closed on a typeless resource, so a request for a URL that does not exist is answered 403 rather than - 404 unless the agent is the owner.

+

Access to a URL that does not exist matches no authorization, so the request is answered 403 rather than 404, in both + dataspaces and for owners too.

+

Containment is server-managed. DELETE of a container that still has documents is refused 409 Conflict: delete its children + first. A POST drops sioc:has_parent/sioc:has_container of the document from its body, a PATCH that + changes them is refused 422, and so is any write that states the containment of another resource.

ldh:container leaves the ontology, the ldh:ViewConstructor form and the view query. A view's Create button determines the container from the view's own solutions.

diff --git a/docs/tutorial/app-as-repository.ttl b/docs/tutorial/app-as-repository.ttl index e8d595c..e39ac47 100644 --- a/docs/tutorial/app-as-repository.ttl +++ b/docs/tutorial/app-as-repository.ttl @@ -53,8 +53,8 @@ ldh push "$LDH_BASE"

Idempotency

Re-running the install converges on the same documents instead of duplicating data. PUT replaces each document's description outright, and so does - ldh admin import ontology --file for the namespace ontology, keeping only the ontology document's own description. (The steps that create rather than replace — - make-public and each CSV import — are POSTs, so a re-run records another authorization and another import; the documents the imports write are the same.)

+ ldh admin import ontology --file for the namespace ontology, keeping only the ontology document's own description. make-public writes the same grants again. (The CSV imports create rather than replace — + they are POSTs, so a re-run records another import; the documents the imports write are the same.)

Installing

diff --git a/docs/tutorial/publish.ttl b/docs/tutorial/publish.ttl index 02fbc24..c7e0002 100644 --- a/docs/tutorial/publish.ttl +++ b/docs/tutorial/publish.ttl @@ -24,12 +24,11 @@
diff --git a/docs/user-guide/upload-file.ttl b/docs/user-guide/upload-file.ttl index 4b28d62..29b8961 100644 --- a/docs/user-guide/upload-file.ttl +++ b/docs/user-guide/upload-file.ttl @@ -33,7 +33,7 @@
  • Drop it. The document reloads in Read mode with the file among its resources; an image is shown inline
    • -

    A file larger than the instance's request limit is refused before it reaches the platform, and the page says so without navigating away.

    +

    A file larger than the instance's upload limit is refused before it reaches the platform, and the page says so without navigating away.

    Alternatively, upload through a form:

    1. Click the Create dropdown at the bottom of the page
      2. diff --git a/packages/install.sh b/packages/install.sh index 817e2bd..77dbb00 100755 --- a/packages/install.sh +++ b/packages/install.sh @@ -10,7 +10,7 @@ # # Reads LDH_BASE, LDH_CERT_FILE, LDH_CERT_PASSWORD and optionally LDH_PROXY; `make install` prompts # for them. Re-running converges: PUT replaces each document and the stylesheet reference is added back -# to it. make-public is a POST and adds another authorization per run. +# to it. make-public writes the same grants again. set -euo pipefail app_dir="$(cd "$(dirname "$0")" && pwd)" diff --git a/screencast/lib/fixture.mjs b/screencast/lib/fixture.mjs index aa46dc9..287a59f 100644 --- a/screencast/lib/fixture.mjs +++ b/screencast/lib/fixture.mjs @@ -129,6 +129,6 @@ WHERE { <${url}> ?slot ?block . ?block a ?type . FILTER(?type IN (ldh:XHTML, ldh export async function patchDocument({ ldh, certFile, certPassword, certPasswordFile, url, update }) { const password = certPassword ?? (certPasswordFile ? (await fs.readFile(certPasswordFile, 'utf8')).trim() : null); if (!password) throw new Error('patchDocument needs --cert-password or --cert-password-file'); - const r = await run(ldh, ['patch', url, '-f', certFile, '-p', password], update); + const r = await run(ldh, ['patch', url, '-c', certFile, '-p', password], update); return { ok: r.code === 0, out: String(r.err || r.out).split(password).join('••••').slice(0, 300) }; } diff --git a/screencast/overview/voice.mjs b/screencast/overview/voice.mjs index c77fe85..dff7e48 100644 --- a/screencast/overview/voice.mjs +++ b/screencast/overview/voice.mjs @@ -11,9 +11,10 @@ // reproducible. The with-timestamps endpoint is used, so each take also keeps its // character alignment for the subtitles. // -// node overview/voice.mjs [--api-key-file ~/.config/elevenlabs/api-key] [--dry-run] +// node overview/voice.mjs [web-algebra/SCRIPT.md] [--api-key-file ~/.config/elevenlabs/api-key] [--dry-run] // -// Writes tracks/overview/voice.json — id, text, spoken, file, seconds per line — and +// Another film's script can be named instead; its takes then go under tracks//. +// Writes tracks//voice.json — id, text, spoken, file, seconds per line — and // prints the lengths. import fs from 'node:fs/promises'; @@ -23,15 +24,15 @@ import { createHash } from 'node:crypto'; import { TRACKS } from '../lib/harness.mjs'; const HERE = path.dirname(new URL(import.meta.url).pathname); -const SCRIPT = path.join(HERE, 'SCRIPT.md'); -const OUT = path.join(TRACKS, 'overview'); +const args = process.argv.slice(2); +const SCRIPT = path.resolve(args.find((a, i) => !a.startsWith('--') && args[i - 1] !== '--api-key-file') ?? path.join(HERE, 'SCRIPT.md')); +const OUT = path.join(TRACKS, path.basename(path.dirname(SCRIPT))); const CACHE = path.join(OUT, 'voice'); const VOICE = 'onwK4e9ZLuTAKqWW03F9'; // Daniel const MODEL = 'eleven_multilingual_v2'; const FORMAT = 'mp3_44100_128'; -const args = process.argv.slice(2); const flag = (name, fallback) => { const i = args.indexOf(name); return i === -1 ? fallback : args[i + 1]; diff --git a/screencast/render/overview.mjs b/screencast/render/overview.mjs index 619f6b2..083de71 100644 --- a/screencast/render/overview.mjs +++ b/screencast/render/overview.mjs @@ -29,13 +29,14 @@ const cfg = JSON.parse(await fs.readFile(cfgPath, 'utf8')); const onlyArg = argv.indexOf('--only'); const only = onlyArg === -1 ? null : new Set(argv[onlyArg + 1].split(',')); const noAudio = argv.includes('--no-audio'); -// --reuse keeps the clips already cut (tracks/.overview/.mp4) and re-cuts only the ids +// --reuse keeps the clips already cut (tracks/./.mp4) and re-cuts only the ids // named after it, so a change to one shot does not re-encode the film's thirty. const reuseArg = argv.indexOf('--reuse'); const reuse = reuseArg === -1 ? null : new Set((argv[reuseArg + 1] ?? '').startsWith('--') || !argv[reuseArg + 1] ? [] : argv[reuseArg + 1].split(',')); const at = (p) => path.resolve(ROOT, p); const TRACKS = at('tracks'); -const TMP = path.join(TRACKS, '.overview'); +// each cut keeps its clips in its own folder, named after its output, so two cuts never share them +const TMP = path.join(TRACKS, `.${path.basename(cfg.output, '.mp4')}`); await fs.mkdir(TMP, { recursive: true }); function run(cmd, args) { @@ -137,7 +138,7 @@ async function cardHtml({ eyebrow, title, lede, links, note }, W, H) { .pill.is-primary { background: #98ccff linear-gradient(180deg, rgba(255,255,255,0.22), rgba(0,0,0,0.10)); color: #003259; border-color: transparent; } .note { font-family: 'Geist Mono', monospace; font-size: ${px(14)}; letter-spacing: 0.04em; text-transform: uppercase; color: #87929e; margin: ${px(36)} 0 0; }
      -
      LinkedDataHub
      +
      ${esc(cfg.wordmark ?? 'LinkedDataHub')}
      ${eyebrow ? `

      ${esc(eyebrow)}

      ` : ''}

      ${esc(title)}

      ${lede ? `

      ${esc(lede)}

      ` : ''} diff --git a/screencast/scenes/probe-shacl-adm.mjs b/screencast/scenes/probe-shacl-adm.mjs new file mode 100644 index 0000000..4ff5ac2 --- /dev/null +++ b/screencast/scenes/probe-shacl-adm.mjs @@ -0,0 +1,33 @@ +// Not a scene. The admin Node shape form's target class lookup: which endpoint, which query, what comes back. +import { runScene, resolve, geometryFrom, sleep } from '../lib/harness.mjs'; +const opts = await resolve('/'); +const { identity, base } = opts; +const ADMIN = base.replace('://', '://admin.'); +const identities = identity ? [...identity, { ...identity[0], origin: ADMIN }] : null; +const NS_DOC = `${ADMIN}/ontologies/namespace/`; +const sugg = (page) => page.locator('.ac-cb-panel[role=listbox]:visible li.ac-cb-item').evaluateAll((ls) => ls.map((l) => [l.textContent.replace(/\s+/g, ' ').trim(), l.title])); +await runScene({ + id: 'probe-shacl-adm', target: NS_DOC, identity: identities, + geometry: geometryFrom(opts, { width: 1440, height: 900, deviceScaleFactor: 1 }), + async body({ page }) { + page.on('request', (r) => { const q = decodeURIComponent(r.url()) + (r.postData() ?? ''); if (/SELECT|DESCRIBE/.test(q) && /label|Label/.test(q)) console.log(' → ', r.method(), r.url().slice(0, 90), '|', q.replace(/\s+/g, ' ').slice(q.indexOf('SELECT'), q.indexOf('SELECT') + 400)); }); + page.on('response', async (r) => { if (/sparql|\/ns\?/.test(r.url()) && r.request().method() !== 'OPTIONS') { try { const t = await r.text(); if (/label/i.test(decodeURIComponent(r.url()))) console.log(' ← ', r.status(), t.replace(/\s+/g, ' ').slice(0, 300)); } catch {} } }); + await page.context().addCookies([{ name: 'LinkedDataHub.first-time-message', value: 'true', domain: new URL(ADMIN).hostname, path: '/' }]); + await page.goto(NS_DOC, { waitUntil: 'load' }); + await page.waitForSelector('.ldh-pane.is-active .ldh-block', { timeout: 30_000 }); + await sleep(2500); + await page.locator('.ldh-create-dock button.drop-toggle').first().click(); await sleep(1200); + await page.locator('.add-constructor:visible').filter({ hasText: /Node shape$/ }).first().click(); await sleep(3500); + const g = page.locator('form:visible .ldh-prop-group').filter({ hasText: /target class/i }).first(); + const input = g.locator('input.resource-combobox:visible').first(); + console.log(' input attrs:', await input.evaluate((i) => JSON.stringify({ name: i.name, cls: i.className, ds: { ...i.dataset }, list: i.getAttribute('list') }))); + console.log(' group html:', (await g.evaluate((e) => e.outerHTML)).slice(0, 1500)); + await input.click(); await input.pressSequentially('Person', { delay: 60 }); await sleep(8000); + console.log(' targetClass suggestions (Person):', JSON.stringify(await sugg(page))); + await input.fill(''); await input.pressSequentially('Namespace', { delay: 60 }); await sleep(8000); + console.log(' targetClass suggestions (Namespace):', JSON.stringify(await sugg(page))); + // a typed URI? + await input.fill(''); await input.pressSequentially('http://xmlns.com/foaf/0.1/Person', { delay: 20 }); await sleep(5000); + console.log(' after typing URI: suggestions', JSON.stringify(await sugg(page)), 'hidden ou:', await g.locator('input[name=ou]').evaluateAll((is) => is.map((i) => i.value))); + }, +}); diff --git a/screencast/scenes/probe-shacl-flow.mjs b/screencast/scenes/probe-shacl-flow.mjs new file mode 100644 index 0000000..063929c --- /dev/null +++ b/screencast/scenes/probe-shacl-flow.mjs @@ -0,0 +1,106 @@ +// Not a scene. Dry run of the SHACL flow: before (no Person type), admin creates a property +// shape and a node shape for foaf:Person, clears the ontology, after (Person offered, form +// from the shape, empty save → violation). +import { runScene, resolve, geometryFrom, sleep } from '../lib/harness.mjs'; +import { switchDocumentMode } from '../lib/blocks.mjs'; +import { patchDocument } from '../lib/fixture.mjs'; +const opts = await resolve('/'); +const { identity, base } = opts; +const ADMIN = base.replace('://', '://admin.'); +const identities = identity ? [...identity, { ...identity[0], origin: ADMIN }] : null; +const NS_DOC = `${ADMIN}/ontologies/namespace/`; +const plain = { click: async (l) => l.click() }; +const sugg = (page) => page.locator('.ac-cb-panel[role=listbox]:visible li.ac-cb-item').evaluateAll((ls) => ls.map((l) => [l.textContent.replace(/\s+/g, ' ').trim(), l.title])); +const formOf = (page) => page.locator('form:visible').filter({ has: page.locator('.ldh-prop-group') }).last(); +const dumpForm = async (page, what) => { + const groups = await formOf(page).locator('.ldh-prop-group').evaluateAll((gs) => gs.map((g) => ({ + label: (g.querySelector('.pred')?.textContent ?? '').replace(/\s+/g, ' ').trim().slice(0, 30), + inputs: [...g.querySelectorAll('input:not([type=hidden]), textarea, select')].map((i) => `${i.tagName.toLowerCase()}.${i.className.split(' ')[0]}#${i.name}=${(i.value ?? '').slice(0, 30)}`), + violation: g.className.includes('violation'), text: g.className.includes('violation') ? g.textContent.replace(/\s+/g, ' ').trim().slice(0, 160) : undefined, + }))); + console.log(` ${what}:`, JSON.stringify(groups)); +}; +const group = (page, re) => formOf(page).locator('.ldh-prop-group').filter({ has: page.locator('.pred', { hasText: re }) }).first(); +const typeIn = async (page, re, text) => { const i = group(page, re).locator('input:not([type=hidden]):visible, textarea:visible').first(); await i.click(); await i.fill(''); await i.pressSequentially(text, { delay: 15 }); }; +const removeRow = async (page, re) => { const b = group(page, re).locator('button.btn-remove-property').first(); if (await b.count()) await b.click(); }; +const save = async (page) => { await formOf(page).locator('button.btn-save').first().click(); await sleep(4000); }; +const dockCreate = async (page, label) => { + await page.locator('.ldh-create-dock button.drop-toggle').first().click(); await sleep(1200); + await page.locator('.add-constructor:visible').filter({ hasText: new RegExp(`${label}$`) }).first().click(); await sleep(3500); +}; +const RESET = `PREFIX sh: +DELETE { ?s ?p ?o } WHERE { ?s a ?t . FILTER(?t IN (sh:NodeShape, sh:PropertyShape)) ?s ?p ?o }`; +const instanceTypeInput = async (page) => { const g = formOf(page).locator('.ldh-prop-group').filter({ has: page.locator('input[name=pu][value$="#type"]') }).first(); const edit = g.locator('button.add-type-combobox').first(); if (await edit.count()) { await edit.click(); await sleep(800); } return g.locator('input:not([type=hidden]):visible').first(); }; + +const reset = await patchDocument({ ldh: opts.ldh, certFile: opts.certFile, certPassword: opts.certPassword, certPasswordFile: opts.certPasswordFile, url: NS_DOC, update: RESET }); +console.log(' reset:', JSON.stringify(reset)); +await runScene({ + id: 'probe-shacl-flow', target: base + '/', identity: identities, + geometry: geometryFrom(opts, { width: 1440, height: 900, deviceScaleFactor: 1 }), + async body({ page }) { + page.on('response', async (r) => { if ([422, 400, 500].includes(r.status())) { try { console.log(' ←', r.status(), r.request().method(), r.url().slice(0, 80), (await r.text()).replace(/\s+/g, ' ').slice(0, 600)); } catch {} } }); + await page.context().addCookies([{ name: 'LinkedDataHub.first-time-message', value: 'true', domain: new URL(ADMIN).hostname, path: '/' }]); + + // ── before ── + await page.goto(base + '/', { waitUntil: 'load' }); + await page.waitForSelector('.ldh-pane.is-active .ldh-block', { timeout: 30_000 }); await sleep(2000); + await switchDocumentMode(page, plain, 'read-mode'); await sleep(2500); + await dockCreate(page, 'Instance'); + console.log(' Instance form html:', (await formOf(page).evaluate((f) => f.outerHTML)).slice(0, 2500)); + const tin = await instanceTypeInput(page); + console.log(' type input:', await tin.count(), await tin.evaluate((i) => `${i.tagName}.${i.className}#${i.name}`).catch(() => null)); + if (await tin.count()) { await tin.click(); await tin.pressSequentially('Person', { delay: 40 }); await sleep(6000); console.log(' BEFORE type suggestions (Person):', JSON.stringify(await sugg(page))); } + + // ── admin: property shape ── + await page.goto(NS_DOC, { waitUntil: 'load' }); + await page.waitForSelector('.ldh-pane.is-active .ldh-block', { timeout: 30_000 }); await sleep(2500); + await dockCreate(page, 'Property shape'); + await dumpForm(page, 'Property shape form (fresh)'); + await typeIn(page, /^Name$/, 'Name'); + await typeIn(page, /^Path$/, 'http://xmlns.com/foaf/0.1/name'); await sleep(1500); await page.keyboard.press('Escape'); + await typeIn(page, /^Min count$/, '1'); + await typeIn(page, /^Datatype$/, 'http://www.w3.org/2001/XMLSchema#string'); await sleep(1500); await page.keyboard.press('Escape'); + await removeRow(page, /^Max count$/); + await dumpForm(page, 'Property shape form (filled)'); + await save(page); + console.log(' after PS save: forms', await page.locator('form:visible').filter({ has: page.locator('.ldh-prop-group') }).count(), 'blocks', await page.locator('.ldh-pane.is-active .ldh-block').count()); + if (await page.locator('form:visible').filter({ has: page.locator('.ldh-prop-group') }).count()) await dumpForm(page, 'PS form still open'); + + // ── admin: node shape ── + await dockCreate(page, 'Node shape'); + await typeIn(page, /^Label$/, 'Person shape'); + await typeIn(page, /^Target class$/, 'http://xmlns.com/foaf/0.1/Person'); await sleep(1500); await page.keyboard.press('Escape'); + const pin = group(page, /^Property$/).locator('input:not([type=hidden]):visible').first(); + await pin.click(); await pin.pressSequentially('Name', { delay: 40 }); await sleep(6000); + const s = await sugg(page); console.log(' Property suggestions (Name):', JSON.stringify(s)); + const hit = page.locator('.ac-cb-panel[role=listbox]:visible li.ac-cb-item').filter({ hasText: /Name/ }).first(); + if (await hit.count()) await hit.click(); else console.log(' !! no property shape suggestion'); + await sleep(800); + await dumpForm(page, 'Node shape form (filled)'); + await save(page); + console.log(' after NS save: forms', await page.locator('form:visible').filter({ has: page.locator('.ldh-prop-group') }).count(), 'blocks', await page.locator('.ldh-pane.is-active .ldh-block').count()); + if (await page.locator('form:visible').filter({ has: page.locator('.ldh-prop-group') }).count()) await dumpForm(page, 'NS form still open'); + console.log(' blocks:', JSON.stringify(await page.locator('.ldh-pane.is-active .ldh-block').evaluateAll((bs) => bs.map((b) => b.textContent.replace(/\s+/g, ' ').trim().slice(0, 120))))); + + // ── clear ── + const clear = page.locator('.ldh-pane.is-active form[action$="/clear"] button').first(); + console.log(' clear button:', await clear.count()); + if (await clear.count()) { await clear.click(); await page.waitForLoadState('load'); await sleep(3000); console.log(' after clear url:', page.url()); } + + // ── after ── + await page.goto(base + '/', { waitUntil: 'load' }); + await page.waitForSelector('.ldh-pane.is-active .ldh-block', { timeout: 30_000 }); await sleep(2000); + await switchDocumentMode(page, plain, 'read-mode'); await sleep(2500); + await dockCreate(page, 'Instance'); + const tin2 = await instanceTypeInput(page); + await tin2.click(); await tin2.pressSequentially('Person', { delay: 40 }); await sleep(6000); + const s2 = await sugg(page); console.log(' AFTER type suggestions (Person):', JSON.stringify(s2)); + const person = page.locator('.ac-cb-panel[role=listbox]:visible li.ac-cb-item[title="http://xmlns.com/foaf/0.1/Person"]').first(); + if (await person.count()) { + await person.click(); await sleep(5000); + await dumpForm(page, 'Person form'); + await save(page); + await dumpForm(page, 'after empty save'); + } else console.log(' !! Person not offered'); + }, +}); diff --git a/screencast/scenes/shacl-person.mjs b/screencast/scenes/shacl-person.mjs new file mode 100644 index 0000000..e19a89b --- /dev/null +++ b/screencast/scenes/shacl-person.mjs @@ -0,0 +1,240 @@ +// The model builds the interface, SHACL edition: a dataspace that cannot create a Person +// (S0); in the admin dataspace a property shape — foaf:name, string, at least one — and a +// node shape targeting foaf:Person are created in the namespace ontology, with the ontology's +// own Clear pressed afterwards (A1–A3); back in the end-user dataspace Person is now a type +// on offer and its form carries the Name field the shape asked for (B1); saved without a +// name the server answers 422 with a sh:ValidationResult and the field is marked (B2); named, +// the record is saved and shown (B3). +// +// Both halves are filmed in one tab: the admin origin is the end-user origin with an +// `admin.` prefix, so the owner identity is bound to both (as overview-g-packages does). +// +// The class and property lookups on the shape forms query the admin store, which holds no +// vocabulary, so foaf:Person, foaf:name and xsd:string are typed as URIs — a vocabulary term +// the ontology's author knows, like the URI the user guide has them type for a constructor. +// Everything else is picked by label on camera: the property shape by its sh:name, Person by +// its class label. +// +// Off camera first: the shapes and any Person made by an earlier take are removed and the +// ontology cache cleared, so the "before" is real. +import { spawn } from 'node:child_process'; +import fs from 'node:fs/promises'; +import { runScene, resolve, geometryFrom, sleep, until } from '../lib/harness.mjs'; +import { patchDocument } from '../lib/fixture.mjs'; +import { ui } from '../lib/dom.mjs'; +import { switchDocumentMode } from '../lib/blocks.mjs'; +import { GEOMETRY_2X, zoom2x, focus, centre, load, easeScrollTo } from '../lib/supercut.mjs'; + +const opts = await resolve('/'); +const { base, identity, ldh, certFile, certPasswordFile, certPassword } = opts; +const password = certPassword ?? (await fs.readFile(certPasswordFile, 'utf8')).trim(); +const ADMIN = base.replace('://', '://admin.'); +const NS_DOC = `${ADMIN}/ontologies/namespace/`; +const NS = `${base}/ns#`; +const ROOT = `${base}/`; +const PERSON = 'Ada Lovelace'; +const FOAF = 'http://xmlns.com/foaf/0.1/'; +const XSD_STRING = 'http://www.w3.org/2001/XMLSchema#string'; + +// ── reset ──────────────────────────────────────────────────────────────────── +const fx = { ldh, certFile, certPassword, certPasswordFile }; +const r1 = await patchDocument({ ...fx, url: NS_DOC, update: `PREFIX sh: +DELETE { ?s ?p ?o } WHERE { ?s a ?t . FILTER(?t IN (sh:NodeShape, sh:PropertyShape)) ?s ?p ?o }` }); +if (!r1.ok) throw new Error(`reset: could not remove the shapes: ${r1.out}`); +const r2 = await patchDocument({ ...fx, url: ROOT, update: `PREFIX foaf: <${FOAF}> +DELETE { ?s ?p ?o } WHERE { ?s a foaf:Person ; ?p ?o }` }); +if (!r2.ok) throw new Error(`reset: could not remove earlier people: ${r2.out}`); +const run = (args) => new Promise((res) => { const p = spawn(ldh, args); let out = '', err = ''; p.stdout.on('data', (d) => (out += d)); p.stderr.on('data', (d) => (err += d)); p.on('close', (code) => res({ code, out: out.trim(), err: err.trim().split(password).join('••••') })); }); +const cleared = await run(['admin', 'clear', 'ontology', ROOT, '-c', certFile, '-p', password, '--ontology', NS]); +if (cleared.code !== 0) throw new Error(`reset: could not clear the ontology: ${cleared.err.slice(0, 200)}`); +console.log(' reset: shapes and people removed, ontology cleared'); + +const identities = identity ? [...identity, { ...identity[0], origin: ADMIN }] : null; + +await runScene({ + id: 'shacl-person', target: ROOT, warm: ROOT, identity: identities, + geometry: geometryFrom(opts, GEOMETRY_2X), + async body({ page, cursor, type, marks }) { + await zoom2x(page); + await page.emulateMedia({ colorScheme: 'dark' }); + await page.context().addCookies([{ name: 'LinkedDataHub.first-time-message', value: 'true', domain: new URL(ADMIN).hostname, path: '/' }]); + + const form = () => ui(page).locator('form.ldh-edit-form:visible').last(); + const group = (re) => form().locator('.ldh-prop-group').filter({ has: page.locator('.pred', { hasText: re }) }).first(); + const sugg = () => page.locator('.ac-cb-panel[role=listbox]:visible li.ac-cb-item'); + const typeInto = async (re, text, { uri = false } = {}) => { + const input = group(re).locator('input:not([type=hidden]):visible, textarea:visible').first(); + await input.scrollIntoViewIfNeeded(); + await cursor.click(input); + await type(input, text, uri ? { base: 18, spread: 6 } : { base: 30, spread: 12 }); + // a typed URI opens the lookup panel on nothing; it is closed, the URI stays the value + if (uri) { await sleep(900); await page.keyboard.press('Escape'); await sleep(300); } + }; + const removeRow = async (re) => { + const btn = group(re).locator('button.btn-remove-property').first(); + await btn.scrollIntoViewIfNeeded(); await cursor.click(btn); await sleep(500); + }; + const saveBtn = () => form().locator('button.btn-save').first(); + const dockCreate = async (label) => { + const toggle = page.locator('.ldh-create-dock button.drop-toggle').first(); + await until(toggle.waitFor({ state: 'visible', timeout: 20_000 }), 'the Create dock'); + await cursor.moveTo(...(await centre(toggle)), { duration: 600 }); + await cursor.click(toggle); await sleep(1100); + const item = page.locator('.add-constructor:visible').filter({ hasText: new RegExp(`${label}$`) }).first(); + await until(item.waitFor({ state: 'visible', timeout: 10_000 }), `${label} in the Create menu`); + await cursor.click(item); + await until(form().waitFor({ state: 'visible', timeout: 15_000 }), `the ${label} form`); + await sleep(1500); + }; + // the Instance form's Type is a committed chip; its edit opens the class lookup + const openTypeLookup = async () => { + const g = form().locator('.ldh-prop-group').filter({ has: page.locator('input[name=pu][value$="#type"]') }).first(); + const edit = g.locator('button.add-type-combobox').first(); + await cursor.click(edit); await sleep(800); + const input = g.locator('input:not([type=hidden]):visible').first(); + await cursor.click(input); + await type(input, 'Person', { base: 60, spread: 20 }); + return input; + }; + const blocks = () => ui(page).locator('.ldh-block'); + + // ── S0 · before: no Person to create ─────────────────────────────────────── + // The root opens in Content mode on its children view; the Create dock offers the model's + // classes in Properties, so the switch is on camera (the Properties page of an empty root + // has no blocks, only the dock). + const toProperties = async () => { + await switchDocumentMode(page, cursor, 'read-mode'); + await until(page.locator('.ldh-create-dock button.drop-toggle').first().waitFor({ state: 'visible', timeout: 20_000 }), 'the Create dock in Properties'); + await sleep(1200); + }; + await load(page, ROOT, '.ldh-pane.is-active .ldh-block', 3000); + await marks.beat('S0-start', 'the dataspace, empty but for its root', await focus(blocks().first())); + await sleep(1500); + await toProperties(); + await marks.beat('S0-properties', 'Properties; Create at the foot of the page'); + await dockCreate('Instance'); + await marks.beat('S0-form', 'a new Instance; its Type, a lookup over the model\'s classes', await focus(form())); + await openTypeLookup(); + await sleep(5000); + const before = (await sugg().allInnerTexts()).map((t) => t.replace(/\s+/g, ' ').trim()); + if (before.some((t) => /^Person\b/.test(t))) throw new Error(`S0: Person is already on offer: ${JSON.stringify(before)}`); + await marks.beat('S0-end', `"Person" finds no class: the model has nothing to build a Person from (offered: ${JSON.stringify(before)})`, await focus(form())); + await sleep(1500); + await page.keyboard.press('Escape'); + + // ── A1 · admin: the property shape ───────────────────────────────────────── + await marks.beat('A1-start', 'to the admin dataspace: the namespace ontology'); + await load(page, NS_DOC, '.ldh-pane.is-active .ldh-block', 3000); + await marks.beat('A1-doc', 'the Namespace ontology, one block', await focus(blocks().first())); + await sleep(1500); + await dockCreate('Property shape'); + await marks.beat('A1-form', 'the Property shape form: name, path, counts, datatype', await focus(form())); + await typeInto(/^Name$/, 'Name'); + await typeInto(/^Path$/, `${FOAF}name`, { uri: true }); + await typeInto(/^Min count$/, '1'); + await typeInto(/^Datatype$/, XSD_STRING, { uri: true }); + await removeRow(/^Max count$/); + await marks.beat('A1-filled', 'foaf:name, a string, at least one; max count dropped', await focus(form())); + await saveBtn().scrollIntoViewIfNeeded(); + await cursor.moveTo(...(await centre(saveBtn())), { duration: 500 }); + await cursor.click(saveBtn()); + await until(page.waitForFunction(() => [...document.querySelectorAll('.ldh-pane.is-active .ldh-block')].some((b) => /Property shape/.test(b.textContent) && !b.querySelector('form')), null, { timeout: 20_000 }), 'the property shape as a block'); + await sleep(2000); + const psBlock = blocks().filter({ hasText: 'Property shape' }).first(); + await easeScrollTo(psBlock, { ms: 900, block: 'center' }); await sleep(600); + await marks.beat('A1-end', 'saved: the property shape, a resource of the ontology', await focus(psBlock)); + await sleep(1200); + + // ── A2 · admin: the node shape ───────────────────────────────────────────── + await dockCreate('Node shape'); + await marks.beat('A2-form', 'the Node shape form: label, property, target class', await focus(form())); + await typeInto(/^Label$/, 'Person shape'); + await typeInto(/^Target class$/, `${FOAF}Person`, { uri: true }); + const pin = group(/^Property$/).locator('input:not([type=hidden]):visible').first(); + await pin.scrollIntoViewIfNeeded(); + await cursor.click(pin); + await type(pin, 'Name', { base: 60, spread: 20 }); + const hit = sugg().filter({ hasText: /Name/ }).filter({ hasText: /Property shape/ }).first(); + await until(hit.waitFor({ state: 'visible', timeout: 15_000 }), 'the property shape in the lookup'); + await sleep(1200); + await marks.beat('A2-lookup', 'Property: the shape just made, found by its name', await focus(form())); + await cursor.click(hit); await sleep(800); + await marks.beat('A2-filled', 'targets foaf:Person, carries the Name property shape', await focus(form())); + await saveBtn().scrollIntoViewIfNeeded(); + await cursor.moveTo(...(await centre(saveBtn())), { duration: 500 }); + await cursor.click(saveBtn()); + await until(page.waitForFunction(() => [...document.querySelectorAll('.ldh-pane.is-active .ldh-block')].some((b) => /Node shape/.test(b.textContent) && !b.querySelector('form')), null, { timeout: 20_000 }), 'the node shape as a block'); + await sleep(2000); + const nsBlock = blocks().filter({ hasText: 'Node shape' }).first(); + await easeScrollTo(nsBlock, { ms: 900, block: 'center' }); await sleep(600); + await marks.beat('A2-end', 'saved: the node shape', await focus(nsBlock)); + await sleep(1200); + + // ── A3 · the ontology's Clear: the cached closure is rebuilt ─────────────── + const clear = ui(page).locator('form[action$="/clear"] button').first(); + await until(clear.waitFor({ state: 'visible', timeout: 10_000 }), 'the ontology\'s Clear button'); + await easeScrollTo(clear, { ms: 900, block: 'center' }); await sleep(500); + await cursor.moveTo(...(await centre(clear)), { duration: 600 }); + await marks.beat('A3-start', 'pointer on Clear: the dataspace re-reads its model', await focus(blocks().first())); + await cursor.click(clear); + await page.waitForLoadState('load').catch(() => {}); + await until(page.waitForSelector('.ldh-pane.is-active .ldh-block', { timeout: 30_000 }), 'the ontology, reloaded'); + await sleep(2000); + await marks.beat('A3-end', 'cleared and reloaded'); + + // ── B1 · after: Person on offer, its form from the shape ─────────────────── + await marks.beat('B1-start', 'back to the dataspace'); + await load(page, ROOT, '.ldh-pane.is-active .ldh-block', 3000); + await toProperties(); + await dockCreate('Instance'); + await openTypeLookup(); + const person = sugg().filter({ has: page.locator('[title="http://xmlns.com/foaf/0.1/Person"]') }).first(); + const personAlt = page.locator('.ac-cb-panel[role=listbox]:visible li.ac-cb-item[title="http://xmlns.com/foaf/0.1/Person"]').first(); + const found = await Promise.race([ + person.waitFor({ state: 'visible', timeout: 15_000 }).then(() => person, () => null), + personAlt.waitFor({ state: 'visible', timeout: 15_000 }).then(() => personAlt, () => null), + ]); + if (!found) throw new Error(`B1: Person not offered after the shape (offered: ${JSON.stringify(await sugg().allInnerTexts())})`); + await sleep(1200); + await marks.beat('B1-offered', '"Person" now finds a class: the shape made it one the dataspace can build', await focus(form())); + await cursor.click(found); + const nameField = () => group(/^Name$/); + const failure = form().locator('.ldh-failure'); + const outcome = await Promise.race([ + nameField().waitFor({ state: 'visible', timeout: 20_000 }).then(() => 'field', () => null), + failure.waitFor({ state: 'visible', timeout: 20_000 }).then(() => 'failure', () => null), + ]); + if (outcome === 'failure') throw new Error(`B1: the Person form failed to render — ${(await failure.innerText()).replace(/\s+/g, ' ').slice(0, 300)}`); + if (outcome !== 'field') throw new Error(`B1: no Name field on the Person form (groups: ${JSON.stringify(await form().locator('.pred').allInnerTexts())})`); + await sleep(2000); + await marks.beat('B1-end', 'the Person form: a Name field, from the shape — no constructor was written', await focus(form())); + await sleep(1500); + + // ── B2 · saved without a name: 422, the field marked ─────────────────────── + await saveBtn().scrollIntoViewIfNeeded(); + await cursor.moveTo(...(await centre(saveBtn())), { duration: 600 }); + await marks.beat('B2-save', 'Name empty; pointer on Save', await focus(form())); + await cursor.click(saveBtn()); + const violated = () => form().locator('.ldh-prop-group.is-violation'); + await until(violated().first().waitFor({ state: 'visible', timeout: 20_000 }), 'a violation on a field'); + await sleep(1500); + const which = (await violated().allTextContents()).map((t) => t.replace(/\s+/g, ' ').trim().slice(0, 60)); + if (!which.some((w) => /name/i.test(w))) throw new Error(`B2: the violation landed on ${JSON.stringify(which)}, not on Name`); + await marks.beat('B2-end', `refused by the shape, 422: ${JSON.stringify(which)}`, await focus(form())); + await sleep(1500); + + // ── B3 · named, saved, shown ─────────────────────────────────────────────── + await typeInto(/^Name$/, PERSON); + await saveBtn().scrollIntoViewIfNeeded(); + await cursor.moveTo(...(await centre(saveBtn())), { duration: 500 }); + await marks.beat('B3-filled', 'the name in; pointer on Save', await focus(form())); + await cursor.click(saveBtn()); + await until(page.waitForFunction((t) => (document.querySelector('.ldh-pane.is-active')?.innerText ?? '').includes(t) && !document.querySelector('.ldh-pane.is-active form.ldh-edit-form'), PERSON, { timeout: 30_000 }), 'the person on the page'); + await sleep(1500); + const personBlock = blocks().filter({ hasText: PERSON }).first(); + await easeScrollTo(personBlock, { ms: 900, block: 'center' }); await sleep(800); + await marks.beat('B3-end', `saved: ${PERSON}, a Person, on the page`, await focus(personBlock)); + await sleep(1500); + await marks.beat('end'); + }, +}); diff --git a/screencast/tapes/get-started-setup.tape b/screencast/tapes/get-started-setup.tape index 82eab5f..dc4868e 100644 --- a/screencast/tapes/get-started-setup.tape +++ b/screencast/tapes/get-started-setup.tape @@ -25,14 +25,8 @@ # # The page's advice to pick different ports when 81/4443 are taken does not work today. # -# The clone takes --branch develop, and the .env block is the page's: both are 6.0. -# ABS_PATH is gone from that block because a dataspace serves from the root of its origin -# and there was never a second working value. master is still 5.10.0, which cannot run -# without it — its docker-compose.yml passes ABS_PATH=${ABS_PATH}, which resolves to the -# empty string and overrides the Dockerfile's ENV ABS_PATH=/, so its entrypoint bails on -# [ -z "$ABS_PATH" ]. The page's step 1 is prose — fork the repository and clone the fork -# — so it prints no command the branch flag contradicts. Drop the flag once 6.0 is what a -# clone gets. +# The clone takes the default branch, master, which is the latest release - what a +# visitor gets, and what the frontpage's snippet runs. The .env block is the page's. # # Geometry, framerate and typing speed come from lib/terminal.mjs, which prepends them. # The working directory arrives as $LDH_CLIP_DIR. @@ -49,7 +43,7 @@ Sleep 2s Show Sleep 1s -Type "git clone --branch develop https://github.com/AtomGraph/LinkedDataHub.git" Enter +Type "git clone https://github.com/AtomGraph/LinkedDataHub.git" Enter Sleep 25s Type "cd LinkedDataHub" Enter diff --git a/screencast/web-algebra/SCRIPT.md b/screencast/web-algebra/SCRIPT.md new file mode 100644 index 0000000..11c7242 --- /dev/null +++ b/screencast/web-algebra/SCRIPT.md @@ -0,0 +1,50 @@ +# Web-Algebra in action — narration script (draft 1) + +A short film for the Web-Algebra README's Demo section, replacing "Agentic Content Management +with Web-Algebra MCP" on YouTube. About two minutes, cut from the overview's Flow H takes: the +MCP half (an agent in Claude Code) leads, because the README is the MCP server's, and the +LinkedDataHub assistant follows as the same operations in a second client. + +It opens on the page the agent wrote, then shows how it was written. Voice, synthesis, +pacing and mix are the overview's (`overview/SCRIPT.md`, Production notes): + +```bash +node overview/voice.mjs web-algebra/SCRIPT.md +node render/overview.mjs web-algebra/cut.json +``` + +## The agent + +| # | On screen | Narration | +|---|---|---| +| W1 | The offers container on Northwind: the chart of offer prices | An AI agent built this page: a container of new offers, three discounted products, and a chart of their prices. It was asked in one sentence. | +| W2 | Terminal: `claude mcp add web-algebra …`, then `claude`, `/mcp` listing it connected | Web-Algebra is an open-source MCP server of composable operations for Linked Data: loading, querying and writing RDF. One command registers it with Claude Code. | +| W3 | The prompt typed, aimed at the Northwind dataspace | The request is one sentence, aimed at the Northwind dataspace: a container for new offers, a twenty percent discount on three Beverages products, and a chart of the offer prices. | +| W4 | The tool calls going by; "Items use sioc:has_container, not has_parent. Fixing the query." | The agent reads the data first: a product, then the shape of an existing offer. When its query misses, it checks the model and fixes it. | +| W5 | Claude Code's summary: the three products, list and offer prices | It reports what it wrote: three offers at twenty percent off, and the query behind the chart. | +| W6 | The browser: the offers container, its items, the chart | In LinkedDataHub they are ordinary documents. You can browse them, query them, and keep editing them by hand. | + +## Inside LinkedDataHub + +| # | On screen | Narration | +|---|---|---| +| W7 | A Northwind page; the create bar's Assistant; "Chart the five customers who ordered the most"; the steps report as the plan runs | The same operations run inside LinkedDataHub. Its assistant writes a question as a Web-Algebra plan, and a plan that only reads runs right away. | +| W8 | The answer in words; the rows as a bar chart; the trace, its three steps | The answer comes back in words, the rows drawn as a chart, and every step of the plan in its trace. | +| W9 | "Add the chart to this page"; the plan waits with its operations listed; Execute; the chart becomes a block of the page | A plan that writes waits for you to press Execute. This one adds the chart to the page. | +| Z1 | The page with the chart block; the closing card fades in | Web-Algebra: composable Linked Data operations, for agents and for people. Open source, on GitHub. | + +## Takes + +| Take | Shots | Notes | +|---|---|---| +| `overview-h-agent` | W1, W6 | the browser half of `scenes/overview-h-agent.mjs`, 2026-10-15 | +| `overview-h-agent-terminal` | W2–W5 | `tapes/overview-h-agent.tape`, Web-Algebra 2.0.0 built from the local checkout | +| `overview-h-assistant-take2` | W7–W9, Z1 | `scenes/overview-h-assistant.mjs` | + +## Pronunciation + +| Written | Spoken | +|---|---| +| LinkedDataHub | Linked Data Hub | +| MCP | M C P | +| RDF | R D F | diff --git a/screencast/web-algebra/cut.json b/screencast/web-algebra/cut.json new file mode 100644 index 0000000..7cf1658 --- /dev/null +++ b/screencast/web-algebra/cut.json @@ -0,0 +1,220 @@ +{ + "output": "tracks/web-algebra.mp4", + "width": 1920, + "height": 1080, + "fps": 25, + "crf": 20, + "source": { + "width": 2880, + "height": 1800 + }, + "viewport": { + "width": 2880, + "height": 1620 + }, + "terminal": { + "width": 1440, + "height": 810 + }, + "maxZoom": 1.7, + "minZoom": 1.15, + "leadSeconds": 0.3, + "trailSeconds": 0.1, + "lineOffset": 0.5, + "linePad": 1.0, + "card": { + "seconds": 3.4, + "fade": 0.5 + }, + "wordmark": "Web-Algebra", + "voice": "tracks/web-algebra/voice.json", + "music": { + "file": "tracks/overview/music.mp3", + "gain": -22, + "duck": -14, + "fadeOut": 2.0 + }, + "slide": { + "eyebrow": "Open-source MCP server", + "title": "Linked Data operations, composed.", + "lede": "Load, query and write RDF with operations an agent can call one by one, or compose into a plan that runs in one go.", + "links": [ + { + "label": "github.com/AtomGraph/Web-Algebra", + "primary": true + }, + { + "label": "linkeddatahub.com" + } + ], + "note": "Open source under Apache 2.0", + "fade": 1.2, + "seconds": 6, + "at": 1.0 + }, + "shots": [ + { + "id": "W1", + "line": "W1", + "track": "overview-h-agent", + "from": "H7-chart", + "to": "H7-end", + "move": "zoom", + "minZoom": 1.0 + }, + { + "id": "W2", + "line": "W2", + "track": "overview-h-agent-terminal", + "from": 1, + "to": 31, + "kind": "terminal", + "speed": 1.7 + }, + { + "id": "W3", + "line": "W3", + "track": "overview-h-agent-terminal", + "from": 31, + "to": 45, + "kind": "terminal", + "speed": 1.14 + }, + { + "id": "W3w", + "track": "overview-h-agent-terminal", + "from": 45, + "to": 98, + "kind": "terminal", + "speed": 6 + }, + { + "id": "W4", + "line": "W4", + "track": "overview-h-agent-terminal", + "from": 98, + "to": 110, + "kind": "terminal" + }, + { + "id": "W4w", + "track": "overview-h-agent-terminal", + "from": 110, + "to": 127.5, + "kind": "terminal", + "speed": 6 + }, + { + "id": "W5", + "line": "W5", + "track": "overview-h-agent-terminal", + "from": 127.5, + "to": 134, + "kind": "terminal" + }, + { + "id": "W6", + "line": "W6", + "track": "overview-h-agent", + "from": "H7-start", + "to": "H7-end", + "move": "zoom", + "via": [ + "H7-chart" + ], + "minZoom": 1.0 + }, + { + "card": "Inside LinkedDataHub", + "eyebrow": "The same operations" + }, + { + "id": "W7a", + "track": "overview-h-assistant-take2", + "from": 8.6, + "to": "H1-asked", + "move": "zoom", + "minZoom": 1.0, + "speed": 1.6 + }, + { + "id": "W7w", + "track": "overview-h-assistant-take2", + "from": "H1-asked", + "to": "H1-running", + "move": "zoom", + "minZoom": 1.0, + "speed": 6 + }, + { + "id": "W7", + "line": "W7", + "track": "overview-h-assistant-take2", + "from": "H1-running", + "to": "H1-end", + "move": "zoom", + "minZoom": 1.0, + "speed": 0.71 + }, + { + "id": "W8", + "line": "W8", + "track": "overview-h-assistant-take2", + "from": "H2-start", + "to": "H2-end", + "move": "zoom", + "via": [ + "H2-chart" + ], + "minZoom": 1.0 + }, + { + "id": "W9a", + "track": "overview-h-assistant-take2", + "from": "H3-start", + "to": "H3-asked", + "move": "zoom", + "minZoom": 1.0, + "speed": 1.3 + }, + { + "id": "W9w", + "track": "overview-h-assistant-take2", + "from": "H3-asked", + "to": "H3-plan", + "move": "zoom", + "minZoom": 1.0, + "speed": 7 + }, + { + "id": "W9", + "line": "W9", + "track": "overview-h-assistant-take2", + "from": "H3-plan", + "to": "H3-end", + "move": "zoom", + "via": [ + "H3-execute", + "H3-written" + ], + "minZoom": 1.0 + }, + { + "id": "Z1", + "line": "Z1", + "track": "overview-h-assistant-take2", + "from": "H3-end", + "to": "H3-end", + "hold": 11.2, + "move": "static", + "focus": false, + "scale": 1, + "kind": "close", + "lineOffset": 2.6 + } + ], + "tone": { + "file": "tracks/overview/tone.mp3", + "gain": -8 + } +}