diff --git a/_config.yml b/_config.yml index 49dfa31c3..930355c33 100644 --- a/_config.yml +++ b/_config.yml @@ -88,6 +88,9 @@ collections: openvoxdb_8x: output: true permalink: '/openvoxdb/8.x/:path:output_ext' + openvoxdb_9x: + output: true + permalink: '/openvoxdb/9.x/:path:output_ext' openbolt_latest: output: true @@ -165,6 +168,11 @@ defaults: type: openvoxdb_8x values: nav: openvoxdb_8x + - scope: + path: '' + type: openvoxdb_9x + values: + nav: openvoxdb_9x - scope: path: '' diff --git a/_data/nav/openvoxdb_9x.yml b/_data/nav/openvoxdb_9x.yml new file mode 100644 index 000000000..efc1c0357 --- /dev/null +++ b/_data/nav/openvoxdb_9x.yml @@ -0,0 +1,173 @@ +--- +- text: General information + items: + - text: Release notes + link: release_notes.html + - text: Overview and requirements + link: index.html + - text: Frequently asked questions + link: puppetdb-faq.html + - text: Versioning policy + link: versioning_policy.html + - text: Known issues + link: known_issues.html + - text: Community add-ons + link: community_add_ons.html +- text: Installation + items: + - text: Installing via Puppet module + link: install_via_module.html + - text: Installing from packages + link: install_from_packages.html + - text: Installing from source + link: install_from_source.html + - text: Upgrading OpenVoxDB + link: upgrade.html + - text: Connecting Puppet Servers + link: connect_puppet_server.html + - text: Connecting standalone Puppet nodes + link: connect_puppet_apply.html +- text: Configuration + items: + - text: Configuring OpenVoxDB + link: configure.html + - text: 'puppetdb.conf: Configuring a Puppet/OpenVoxDB connection' + link: puppetdb_connection.html + - text: Configuring Postgres + link: configure_postgres.html + - text: Setting up SSL for PostgreSQL + link: postgres_ssl.html + - text: Coordinating database migrations + link: migration_coordination.html +- text: Usage/admin + items: + - text: Using OpenVoxDB + link: using.html + - text: Maintaining and tuning + link: maintain_and_tune.html + - text: OpenVoxDB CLI + link: pdb_client_tools.html + - text: Exporting and anonymizing data + link: anonymization.html + - text: Scaling recommendations + link: scaling_recommendations.html + - text: Logging configuration + link: logging.html + - text: Debugging with remote REPL + link: repl.html + - text: Load testing + link: load_testing_tool.html +- text: Troubleshooting + items: + - text: General Support Guide + link: pdb_support_guide.html + - text: Session logging + link: trouble_session_logging.html +- text: PQL - Puppet Query Language + items: + - text: Tutorial + link: api/query/tutorial-pql.html + - text: Reference guide + link: api/query/v4/pql.html + - text: Examples + link: api/query/examples-pql.html +- text: API + items: + - text: Overview + link: api/overview.html + - text: Query tutorial + link: api/query/tutorial.html + - text: Curl tips + link: api/query/curl.html +- text: Query API version 4 + items: + - text: Upgrading from version 3 + link: api/query/v4/upgrading-from-v3.html + - text: Query structure + link: api/query/v4/query.html + - text: Entities + link: api/query/v4/entities.html + - text: AST query language + link: api/query/v4/ast.html + - text: Query paging + link: api/query/v4/paging.html + - text: Root endpoint + link: api/query/v4/overview.html + - text: Nodes endpoint + link: api/query/v4/nodes.html + - text: Environments endpoint + link: api/query/v4/environments.html + - text: Producers endpoint + link: api/query/v4/producers.html + - text: Factsets endpoint + link: api/query/v4/factsets.html + - text: Facts endpoint + link: api/query/v4/facts.html + - text: Fact-names endpoint + link: api/query/v4/fact-names.html + - text: Fact-paths endpoint + link: api/query/v4/fact-paths.html + - text: Fact-contents endpoint + link: api/query/v4/fact-contents.html + - text: Inventory endpoint + link: api/query/v4/inventory.html + - text: Catalogs endpoint + link: api/query/v4/catalogs.html + - text: Catalog inputs endpoint + link: api/query/v4/catalog_inputs.html + - text: Catalog input contents endpoint + link: api/query/v4/catalog_input_contents.html + - text: Resources endpoint + link: api/query/v4/resources.html + - text: Edges endpoint + link: api/query/v4/edges.html + - text: Reports endpoint + link: api/query/v4/reports.html + - text: Events endpoint + link: api/query/v4/events.html + - text: Event counts endpoint + link: api/query/v4/event-counts.html + - text: Aggregate event counts endpoint + link: api/query/v4/aggregate-event-counts.html + - text: Package endpoints + link: api/query/v4/packages.html +- text: Admin API version 1 + items: + - text: Archive endpoint + link: api/admin/v1/archive.html + - text: Command (cmd) endpoint + link: api/admin/v1/cmd.html + - text: Summary stats endpoint + link: api/admin/v1/summary-stats.html +- text: Command API version 1 + items: + - text: Commands endpoint + link: api/command/v1/commands.html +- text: Status API version 1 + items: + - text: Status endpoint + link: api/status/v1/status.html +- text: Metadata API version 1 + items: + - text: Version endpoint + link: api/meta/v1/version.html + - text: Server time endpoint + link: api/meta/v1/server-time.html +- text: Metrics API + items: + - text: Metrics API v2 + link: api/metrics/v2/jolokia.html +- text: Wire formats + items: + - text: Catalog wire format - v9 + link: api/wire_format/catalog_format_v9.html + - text: Facts wire format - v5 + link: api/wire_format/facts_format_v5.html + - text: Report wire format - v8 + link: api/wire_format/report_format_v8.html + - text: Deactivate node wire format - v3 + link: api/wire_format/deactivate_node_format_v3.html + - text: Replace catalog inputs wire format - v1 (experimental) + link: api/wire_format/catalog_inputs_format_v1.html + - text: Configure expiration wire format - v1 (experimental) + link: api/wire_format/configure_expiration_format_v1.html diff --git a/_data/nav_map.yml b/_data/nav_map.yml index e45cde4b8..ab4ceb733 100644 --- a/_data/nav_map.yml +++ b/_data/nav_map.yml @@ -26,6 +26,10 @@ collections: openvoxdb_latest|openvoxdb_8x base: /openvoxdb/latest/ +- nav_key: openvoxdb_9x + collections: openvoxdb_9x + base: /openvoxdb/9.x/ + - nav_key: openbolt_5x collections: openbolt_latest|openbolt_5x base: /openbolt/latest/ diff --git a/_data/navigation.yml b/_data/navigation.yml index 6876fa529..6bed42ff7 100644 --- a/_data/navigation.yml +++ b/_data/navigation.yml @@ -9,7 +9,7 @@ collections: [openvox-server_latest, openvox-server_8x, openvox-server_9x] - title: OpenVoxDB url: /openvoxdb/latest/ - collections: [openvoxdb_latest, openvoxdb_8x] + collections: [openvoxdb_latest, openvoxdb_8x, openvoxdb_9x] - title: OpenBolt url: /openbolt/latest/ collections: [openbolt_latest, openbolt_5x] diff --git a/_data/products.yml b/_data/products.yml index eeaac79ff..c406f755c 100644 --- a/_data/products.yml +++ b/_data/products.yml @@ -86,6 +86,10 @@ openvoxdb: label: OpenVoxDB latest: 8x versions: + - id: 9x + label: "9.x" + collection: _openvoxdb_9x + base: /openvoxdb/9.x/ - id: 8x label: "8.x" collection: _openvoxdb_8x diff --git a/docs/_openvoxdb_9x/README.md b/docs/_openvoxdb_9x/README.md new file mode 100644 index 000000000..c20f65835 --- /dev/null +++ b/docs/_openvoxdb_9x/README.md @@ -0,0 +1,8 @@ +# OpenVoxDB documentation + +If you were redirected while trying to reach the OpenVoxDB docs, the version you were trying to reach is no longer maintained. + +For the most recent OpenVoxDB docs, see [docs.openvoxproject.org/openvoxdb/latest](https://docs.openvoxproject.org/openvoxdb/latest). + +If you need docs for an older version of OpenVoxDB, you may be able to find it by switching to the relevant branch of this repository. + diff --git a/docs/_openvoxdb_9x/anonymization.markdown b/docs/_openvoxdb_9x/anonymization.markdown new file mode 100644 index 000000000..5aa76f8c4 --- /dev/null +++ b/docs/_openvoxdb_9x/anonymization.markdown @@ -0,0 +1,125 @@ +--- +title: "Export, import and anonymization" +layout: default +--- +[pdb_client_tools]: ./pdb_client_tools.html + +# Exporting and anonymizing data + +This document covers using the export, import and anonymization tools for +OpenVoxDB. + +The export tool will return an archive of all of your OpenVoxDB data which can be +uploaded to another OpenVoxDB via the import tool. The export tool also has the +ability to anonymize the archive before returning it. This is particularly +useful when sharing OpenVoxDB data that contains sensitive items. + +## Using the `export` command + +To create an anonymized OpenVoxDB archive directly, use the `puppet db` command +from any node with the [OpenVoxDB CLI][pdb_client_tools] installed: + +```console +puppet db export my-openvoxdb-export.tar.gz --anonymization moderate +``` + +## Using the `import` command + +To import an anonymized OpenVoxDB tarball, use the `puppet db` command from +any node with the [OpenVoxDB CLI][pdb_client_tools] installed: + +```console +puppet db import my-openvoxdb-export.tar.gz +``` + +## How does it work? + +The tool walks through your entire data set, applying different rules to each of +the leaf data based on the profile you have chosen. The data structure is left +intact, and only the data contents are modified. This maintains the "shape" of +the data without exposing the underlying data you may wish to scrub. + +We do this by always ensuring we replace data consistently. For example, if a +string is replaced with something random, we ensure that all instances of that +original string are replaced with the same random string throughout the data. + +By keeping its original shape, the data can be anonymized based on your needs +but still hold some value to the consumer of your anonymized data. + +## Anonymization profiles + +You may not need to anonymize all data in every case, so we have provided a +number of profiles offering varying levels of anonymization. + +The profile can be specified on the command line when the command is run. For +example, to choose the `low` profile, enter: + +```console +puppet db export ./my-openvoxdb-anonymized-export.tar.gz --anonymization low +``` + +### Profile: full + +The `full` profile will anonymize all data (including node names, resource +types, resource titles, parameter names, values, any log messages, file names, +and file lines) while retaining the data set's shape. The result is a completely +anonymized data set. Report metrics under the `resources` and `events` +categories are left intact, as these can be inferred from the rest of the data, +but names of metrics under the `time` category are anonymized as resource types. + +This is useful if you are really concerned about limiting the data you expose, +but provides the least utility for the consumer depending on the activity they +are trying to test. + +### Profile: moderate + +The `moderate` profile attempts to be a bit smarter about what it anonymizes and +is **the recommended profile for most cases**. It sorts and anonymizes data by +data type: + +* Node name: is anonymized by default. +* Resource type name: the core types that are built into Puppet are not + anonymized, including some common types from the modules: `stdlib`, + `postgresql`, `rabbitmq`, `puppetdb`, `apache`, `mrep`, `f5`, `apt`, + `registry`, `concat`, and `mysql`. Any Puppet Enterprise core type names are + also preserved. The goal here is to anonymize any custom or unknown resource + type names, as these may contain confidential information. +* Resource titles: all titles are anonymized expect for those belonging to + Filebucket, Package, Service, and Stage. +* Parameter names: are never anonymized. +* Parameter values: everything is anonymized except for the values for + `provider`, `ensure`, `noop`, `loglevel`, `audit`, and `schedule`. +* Report log messages: are always anonymized. +* File names: are always anonymized. +* File numbers: are left as they are. +* Log messages: are always anonymized. +* Metrics: metric names in the `time` category are anonymized as resource types. + +### Profile: low + +This profile is aimed at hiding security information specifically, but leaving +most of the data in its original state. The following categories are anonymized: + +* Node name: is always anonymized. +* Parameter values: only values and messages for parameter names containing the + strings `password`, `pwd`, `secret`, `key`, or `private` are anonymized. +* Log messages: are always anonymized. + +## Verifying your anonymized data + +After anonymizing data with the `puppet db export` tool, we **strongly +recommend** that you analyze the anonymized data before sharing it with another +party to ensure that all sensitive data has been scrubbed. + +Simply untar the export file and analyze the contents: + +```console +tar -xzf my-openvoxdb-anonymized-export.tar.gz +cd puppetdb-bak +``` + +Inside this directory there is a directory for each content type (reports, +catalogs, and facts), and each file inside represents a node (and a report +instance for reports). The data is represented as human-readable JSON. You can +open these files and use tools such as `grep` to check the status of specific +information you wish to anonymize. diff --git a/docs/_openvoxdb_9x/api/admin/v1/archive.markdown b/docs/_openvoxdb_9x/api/admin/v1/archive.markdown new file mode 100644 index 000000000..d20b9e761 --- /dev/null +++ b/docs/_openvoxdb_9x/api/admin/v1/archive.markdown @@ -0,0 +1,63 @@ +--- +title: "Archive endpoint" +layout: default +canonical: "/openvoxdb/latest/api/admin/v1/archive.html" +--- +# Archive endpoint + +[curl]: ../../query/curl.html#using-curl-from-localhost-non-sslhttp + +The `/archive` endpoint can be used for importing and exporting OpenVoxDB +archives. + +## `POST /pdb/admin/v1/archive` + +This endpoint can be used for streaming a OpenVoxDB archive into OpenVoxDB. + +### Request format + +The request should be a multipart POST and have `Content-Type: multipart/mixed`. + +### URL parameters + +* `archive`: required. The archive file to import to the OpenVoxDB. This archive + must have a file called `openvoxdb-bak/metadata.json` as the first entry in the + tarfile with a key `command_versions` which is a JSON object mapping OpenVoxDB + command names to their version. + +### Response format + +The response will be in `application/json`, and will return a JSON map upon +successful completion of the importation: + + {"ok": true} + +### Example + +[Using `curl` from localhost][curl]: + + curl -X POST http://localhost:8080/pdb/admin/v1/archive \ + -F "archive=@example_backup_archive.tgz" + + {"ok": true} + +## `GET /pdb/admin/v1/archive` + +This endpoint can be used to stream a tarred, gzipped backup archive of OpenVoxDB +to your local machine. + +### URL parameters + +* `anonymization_profile`: optional. The level of anonymization applied to the + archive files. + +### Response format + +The response will be a `application/octet-stream`, and will return a `tar.gz` +archive. + +### Example + +[Using `curl` from localhost][curl]: + + curl -X GET http://localhost:8080/pdb/admin/v1/archive -o puppetdb-export.tgz diff --git a/docs/_openvoxdb_9x/api/admin/v1/cmd.markdown b/docs/_openvoxdb_9x/api/admin/v1/cmd.markdown new file mode 100644 index 000000000..a4a5d62af --- /dev/null +++ b/docs/_openvoxdb_9x/api/admin/v1/cmd.markdown @@ -0,0 +1,143 @@ +--- +title: "Admin commands endpoint" +layout: default +canonical: "/openvoxdb/latest/api/admin/v1/cmd.html" +--- + +# Admin commands endpoint + +[curl]: ../../query/curl.html#using-curl-from-localhost-non-sslhttp +[config-purge-limit]: ../../../configure.html#node-purge-gc-batch-limit + + +The `/cmd` endpoint can be used to trigger OpenVoxDB maintenance +operations or to directly delete a node. Admin commands are processed +synchronously seperate from other OpenVoxDB commands. + +## `POST /pdb/admin/v1/cmd` + +Admin commands must be triggered by a POST. + +### Request format + +The POST request should specify `Content-Type: application/json` and +the request body should look like this: + +``` json +{"command": "...", + "version": 123, + "payload": } +``` + +`command` is a string identifying the command. + +`version` is a JSON integer describing what version of the given +command you're attempting to invoke. The version of the command +also indicates the version of the wire format to use for the command. + +`payload` must be a valid JSON object of any sort. It's up to an +individual handler function to determine how to interpret that object. + +### URL parameters + +* The POST endpoint accepts no URL parameters. + +## List of admin commands + +## "clean", version 1 + +``` json +{"command" : "clean", + "version" : 1, + "payload" : [REQUESTED_OPERATION, ...]} +``` + +where valid `REQUESTED_OPERATION`s are `"expire_nodes"`, +`"purge_nodes"`, `"purge_reports"`, `"gc_packages"`, and `"other"`. +In addition, a purge_nodes operation can be structured like this to +specify a batch_limit: + +``` json +["purge_nodes" {"batch_limit" : 50}] +``` + +When specified, the `batch_limit` restricts the maximum number of +nodes purged to the value specified, and if not specified, the limit +will be the [`node-purge-gc-batch-limit`][config-purge-limit]. + +An empty payload vector requests all maintenance operations. + +### Response format + +The response type will be `application/json`, and upon success will +include this JSON map: + +``` json +{"ok": true} +``` + +Only one maintenance operation can be running at a time. If any other +maintenance operation is already in progress the HTTP response status will be +409 (conflict), will include a map like this + +``` json +{"kind": "conflict", + "msg": "Another cleanup is already in progress", + "details": null} +``` + +and no additional maintenance will be performed. The `msg` and +`details` may or may not vary, but the `kind` will always be +"conflict". + +### Example + +[Using `curl` from localhost][curl]: + +``` sh +$ curl -X POST http://localhost:8080/pdb/admin/v1/cmd \ + -H 'Accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{"command": "clean", + "version": 1, + "payload": ["expire_nodes", "purge_nodes"]}' +{"ok": true} +``` + +## "delete", version 1 + +``` json +{"command" : "delete", + "version" : 1, + "payload" : {"certname" : }} +``` + +The `"delete"` command can be used to trigger the immediate deletion of all data +associated with a certname. It is important to note that the delete operation +doesn't account for commands which may be in the command queue but not yet +processed by OpenVoxDB. This could cause a node targeted for deletion to +reappear when the command in the queue gets processed after the deletion +operation has run. + +### Response format + +The response type will be `application/json`, and upon success will +include this JSON map: + +``` json +{"deleted": "certname"} +``` + +### Example + +[Using `curl` from localhost][curl]: + +``` sh +$ curl -X POST http://localhost:8080/pdb/admin/v1/cmd \ + -H 'Accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{"command": "delete", + "version": 1, + "payload": {"certname" : "node-1"}}' +{"deleted": "node-1"} +``` diff --git a/docs/_openvoxdb_9x/api/admin/v1/summary-stats.markdown b/docs/_openvoxdb_9x/api/admin/v1/summary-stats.markdown new file mode 100644 index 000000000..fa06a9b5e --- /dev/null +++ b/docs/_openvoxdb_9x/api/admin/v1/summary-stats.markdown @@ -0,0 +1,77 @@ +--- +title: "Summary-stats endpoint" +layout: default +--- + +# Summary-stats endpoint + +> **Experimental Endpoint**: The summary-stats endpoint is designated +> as experimental. It may be altered or removed in a future release. +> +> **Warning**: This endpoint will execute a number of relatively expensive SQL +> commands against your database. It will not meaningfully impede performance +> of a running PDB instance, but the request may take several minutes to +> complete. + +## `/pdb/admin/v1/summary-stats` + +The `/summary-stats` endpoint is used to generate information about the way +your PDB installation is using postgres. Its intended purpose at this time is +to aid in diagnosis of support issues, though users may find it independently +useful. + +### Response format + +The response is a JSON map containing the following keys: + +* `table_usage` (json): Postgres statistics related to table usage. Equivalent + to the postgres query + select * from pg_stat_user_tables; +* `index_usage` (json): Postgres statistics related to index usage. Equivalent + to the postgres query + select * from pg_stat_user_indexes; +* `database_usage` (json): Postgres statistics related to usage of the puppetdb + database. Equivalent to the postgres query + select now() as current_time, * from pg_stat_database where datname=current_database(); +* `node_activity` (json): Counts of active and inactive nodes. +* `fact_path_counts_by_depth` (json): Number of fact paths for each fact path + depth represented in the database. +* `num_shared_value_path_combos` (json): The number of fact value/fact path + combinations that are shared across multiple nodes. +* `num_shared_name_value_combos` (json): The number of fact name/fact value + combinations that are shared across multiple nodes. +* `num_unshared_value_path_combos` (json): The number of fact value/fact path + combinations that are only present on one node. +* `num_unshared_name_value_combos` (json): The number of fact name/fact value + combinations that are only present on one node. +* `num_times_paths_values_shared_given_sharing` (json): Across fact path/fact + value combinations shared across multiple nodes, the 0th through 20th + 20-quantiles of the number of nodes sharing. +* `num_unique_fact_values_over_nodes` (json): Across all nodes, the 0th through + 20th 20-quantiles of the number of unique fact values. +* `string_fact_value_character_lengths` (json): 0th through 20th 20-quantiles of the + character length of string-valued facts. +* `structured_fact_value_character_lengths` (json): 0th through 20th 20-quantiles of the + character length of structured facts. +* `report_metric_size_dist` (json): 0th through 20th 20-quantiles of the + character lengths of report metrics. +* `report_log_size_dist` (json): 0th through 20th 20-quantiles of the character + lengths of report logs. +* `fact_values_by_type` (json): Number of fact values of each type present. +* `num_associated_factsets_over_fact_paths` (json): 0th through 20th + 20-quantiles of the number of nodes sharing a given fact. +* `num_resources_per_node` (json): 0th through 20th 20-quantiles over the + number of resources per node. +* `num_resources_per_file` (json): 0th through 20th 20-quantiles over the + number of resources per manifest. +* `num_distinct_edges_source_target` (json): Number of distinct source/target + vertex pairs in the resource graph. +* `file_resource_per_catalog` (json): 0th through 20th 20-quantiles of the + number of resources of type File over the most recent catalogs for all nodes. +* `file_resources_per_catalog_with_source` (json): 0th through 20th + 20-quantiles of the number of file resources with 'source' parameters across + the most recent catalogs for all nodes. + +### Parameters + +This endpoint supports no parameters. diff --git a/docs/_openvoxdb_9x/api/command/v1/commands.markdown b/docs/_openvoxdb_9x/api/command/v1/commands.markdown new file mode 100644 index 000000000..bf8d79fcd --- /dev/null +++ b/docs/_openvoxdb_9x/api/command/v1/commands.markdown @@ -0,0 +1,312 @@ +--- +title: "Commands endpoint" +layout: default +canonical: "/openvoxdb/latest/api/command/v1/commands.html" +--- + +# Commands endpoint + +[factsv4]: ../../wire_format/facts_format_v4.html +[factsv5]: ../../wire_format/facts_format_v5.html +[catalogv6]: ../../wire_format/catalog_format_v6.html +[catalogv7]: ../../wire_format/catalog_format_v7.html +[catalogv8]: ../../wire_format/catalog_format_v8.html +[catalogv9]: ../../wire_format/catalog_format_v9.html +[reportv5]: ../../wire_format/report_format_v5.html +[reportv6]: ../../wire_format/report_format_v6.html +[reportv7]: ../../wire_format/report_format_v7.html +[reportv8]: ../../wire_format/report_format_v8.html +[deactivatev3]: ../../wire_format/deactivate_node_format_v3.html +[expirev1]: ../../wire_format/configure_expiration_format_v1.html +[inputsv1]: ../../wire_format/catalog_inputs_format_v1.html + +Commands are used to change OpenVoxDB's model of a population. Commands are +specified using these values: + +`command` is a string identifying the name of the command. + +`version` is an integer describing what version of the given +command you're attempting to invoke. The version of the command +also indicates the version of the wire format to use for the command. + +`payload` is a valid JSON object of any sort. Each command requires it's own +type of `payload`. This is referred to as the command's *wire format*. + +`certname` is a string identifying the name of the node for which the command +should be applied to. + +`checksum` (optional) is a SHA-1 hash of the payload which will be used for +content verification with the server. This value is not required. + +The OpenVoxDB termini for Puppet Servers use this command API to update facts, +catalogs, and reports for nodes, and will always include the checksum. + +## Command submission + +The entire command **must** be encoded as UTF-8. + +Commands must be submitted via HTTP to the `/pdb/cmd/v1` endpoint via one of +two mechanisms: + +1. Query parameters alongside `payload` JSON body: + +This method entails POSTing the command name, version, certname and optionally +the checksum as query parameters, with the POST body containing the given +command's payload according to its wire format. Providing this information via +query parameters allows OpenVoxDB to provide better validation and feedback at +time of POSTing without having to parse the JSON body. This is the preferred +method of command submission. + +> **Note**: every command must provide the certname both as a query parameter and +> as a JSON value in the payload. +> +> **Note**: when providing a command name as a query parameter, any spaces in the +> command name must be replaced with underscores. + +```text +Request: +POST http://localhost:8080/pdb/cmd/v1?command=&version=&certname=&checksum= + +JSON Body: + +``` + +1. All-encompassing JSON body (DEPRECATED): + +This method entails POSTing a single JSON body containing all the command +information. The JSON body contains the command name, command version, +certname, and payload. The checksum can optionally be provided as another +top-level key. + +```text +Request: +POST http://localhost:8080/pdb/cmd/v1 + +JSON Body: +{"command": , + "version": , + "certname": , + "checksum": , + "payload": } +``` + +## Response + +When a command is successfully submitted, the submitter will receive the +following: + +* A response code of 200. +* A content-type of `application/json`. +* A response body in the form of a JSON object, containing a single key, + `"uuid"`, whose value is a UUID corresponding to the submitted command. This + can be used, for example, by clients to correlate submitted commands with + server-side logs. + +> **Note**: a successful response only means that the command has been been +> successfully queued. The command may still fail once OpenVoxDB tries to +> execute it sometime in the future. Check the OpenVoxDB logs if you suspect a +> command is failing. + +When a command submission fails, the submitter will receive the following: + +* A response code of 400. +* A content-type of `application/json`. +* A response body in the form of a JSON object, containing a single key, + `"error"`, whose value is a string describing the issue with the command. + +### Blocking command submission + +>**Experimental feature:** This is an experimental feature, and it may be changed or removed at any +>time. Although convenient, it should be used with caution. Always prefer +>non-blocking command submission. + +When submitting a command, you may specify the "secondsToWaitForCompletion" +query parameter. If you do, OpenVoxDB will block the request until the command +has been processed, or until the specified timeout has passed, whichever comes +first. The response will contain the following additional keys: + +* `timed_out`: true when your timeout was hit before the command finished processing. + +* `processed`: true when the command has been processed, successfully or not. + Will be set to false if the timeout was hit first. + +* `error`, `exception`: If the command was processed but an error occurred, + these two fields provide the specifics of what went wrong. + +### Custom headers + +When submitting a compressed command body you should indicate the uncompressed +command size by setting the following custom header: +`X-Uncompressed-Length: `. +This header is used to update command size metrics and compared against +`max-command-size` when `reject-large-commands` is set to true. All commands +sent from the OpenVoxDB termini now include this header by default. + +## Command semantics + +Commands are processed *asynchronously*. If OpenVoxDB returns a 200 +when you submit a command, that only indicates that the command has +been *accepted* for processing. There are no guarantees as to when +that command will be processed, nor that when it is processed it will +be successful. + +Commands that fail processing will be stored in files in the "dead +letter office", located under the MQ data directory in +`discarded/`. These files contain the command and diagnostic +information that may be used to determine why the command failed to be +processed. + +## List of commands + +### "replace catalog", version 9 + +* The nullable `producer` property has been added. + +The payload is expected to be a Puppet catalog, as a JSON object, +conforming exactly to the [catalog wire format v9][catalogv9]. Extra +or missing fields are an error. + +### "replace facts", version 5 + +* The nullable `producer` property has been added. + +See [fact wire format v5][factsv5] for more information on the +payload of this command. + +### "deactivate node", version 3 + +* Previous versions of deactivate node required only the certname, as a raw JSON + string. It is now formatted as a JSON map, and the `producer_timestamp` + property has been added. + +See [deactivate node wire format v3][deactivatev3] for more information on the +payload of this command. + +### "store report", version 8 + +* The nullable `producer`, `noop_pending`, `corrective_change`, and optional + `type` fields have been added. + +The payload is expected to be a report, containing events that occurred on +Puppet resources. It is structured as a JSON object, conforming to the +[report wire format v8][reportv8]. + +## Deprecated commands + +### "replace catalog", version 8 + +* The nullable `catalog_uuid` property has been added. + +The payload is expected to be a Puppet catalog, as a JSON object, +conforming exactly to the [catalog wire format v8][catalogv8]. Extra +or missing fields are an error. + +### "replace catalog", version 7 + +* The nullable `code_id` property has been added. + +The payload is expected to be a Puppet catalog, as a JSON object, +conforming exactly to the [catalog wire format v7][catalogv7]. Extra +or missing fields are an error. + +### "replace catalog", version 6 + +* All field names that were previously separated by dashes are now + separated by underscores. + +* The catalog `name` field has been renamed to `certname`. + +The payload is expected to be a Puppet catalog, as a JSON object, +conforming exactly to the [catalog wire format v6][catalogv6]. Extra +or missing fields are an error. + +### "replace facts", version 4 + +* Similar to version 6 of replace catalog, previously dashed fields are now + underscore-separated. + +* The `name` field has been renamed to `certname`, for consistency. + +See [fact wire format v4][factsv4] for more information on the +payload of this command. + +### "store report", version 7 + +* The nullable `catalog_uuid`, `code_id`, and `cached_catalog_status` +properties have been added. + +The payload is expected to be a report, containing events that occurred on +Puppet resources. It is structured as a JSON object, conforming to the +[report wire format v7][reportv7]. + +### "store report", version 6 + +The version 6 store report command differs from previous versions by changing +from a `resource_events` property to a `resources` property. The +`resource_events` property was a merged version of `resources` and their +associated events `events`. This new version moves the command to use a similar +format to a raw Puppet report, with a list of `resources`, each with an `events` +property containing a list of the resource's events. + +The payload is expected to be a report, containing events that occurred on +Puppet resources. It is structured as a JSON object, conforming to the +[report wire format v6][reportv6]. + +### "store report", version 5 + +The version 5 store report command differs from version 4 in the addition of a +"noop" flag, which is a Boolean indicating whether the report was produced by a +puppet run with `--noop`, as well as in the conversion of dash-separated fields to +underscored. + +The payload is expected to be a report, containing events that occurred on +Puppet resources. It is structured as a JSON object, conforming to the +[report wire format v5][reportv5]. + +### "configure expiration", version 1 (experimental) + +The payload should be a JSON format command, conforming to the +[configure_expiration wire format v1][expirev1], indicating whether or +not facts should be expired for a given `certname`. + +> *Note*: this is an experimental command, which might be altered or +> removed in a future release, and for the time being, OpenVoxDB +> exports will not include this information. + +### "replace catalog inputs", version 1 + +Submit a set of information about "inputs" to a catalog such as hiera data, external function calls, etc. +Each piece of information has a type and a name such as `["hiera", "openvoxdb::globals::version"]`, see the +[replace catalog inputs wire format v1][inputsv1] for more details. + +## Examples using `curl` + +To post a `replace facts` command you can use the following curl command: + +```console +curl -X POST \ + -H 'Content-Type:application/json' \ + -H 'Accept:application/json' \ + -d '{"certname":"test1","environment":"DEV","values":{"myfact":"myvalue"},"producer_timestamp":"2015-01-01", "producer":"server1"}' \ + "http://localhost:8080/pdb/cmd/v1?command=replace_facts&version=5&certname=test1" +``` + +An example of `deactivate node`: + +```console +curl -X POST \ + -H 'Content-Type:application/json' \ + -H 'Accept:application/json' \ + -d '{"certname":"test1","producer_timestamp":"2015-01-01"}' \ + "http://localhost:8080/pdb/cmd/v1?certname=test1&command=deactivate_node&version=3" +``` + +To `configure expiration` for facts: + +```console +curl -X POST \ + -H 'Content-Type:application/json' \ + -H 'Accept:application/json' \ + -d '{"certname":"test1","producer_timestamp":"2019-01-01","expire":{"facts":false}}' \ + "http://localhost:8080/pdb/cmd/v1?certname=test1&command=configure_expiration&version=1" +``` diff --git a/docs/_openvoxdb_9x/api/meta/v1/server-time.markdown b/docs/_openvoxdb_9x/api/meta/v1/server-time.markdown new file mode 100644 index 000000000..1211b65d3 --- /dev/null +++ b/docs/_openvoxdb_9x/api/meta/v1/server-time.markdown @@ -0,0 +1,37 @@ +--- +title: "Server time endpoint" +layout: default +canonical: "/openvoxdb/latest/api/meta/v1/server-time.html" +--- + +# Server time endpoint + +[curl]: ../../query/curl.html#using-curl-from-localhost-non-sslhttp + +The `/server-time` endpoint can be used to retrieve the server time from the OpenVoxDB server. + +## `/pdb/meta/v1/server-time` + +This metadata endpoint will return the current time of the clock on the OpenVoxDB +server. This can be useful as input to other time-based queries against OpenVoxDB, +to eliminate the possibility of time differences between the clocks on client +machines. + +This endpoint does not use any URL parameters or query strings. + +### Response format + +The response will be in `application/json`, and will return a JSON map with a +single key: `server_time`, whose value is an ISO-8601 representation of the +current time on the OpenVoxDB server. + + {"server_time": "2013-09-20T20:54:27.472Z"} + +### Example + +[Using `curl` from localhost][curl]: + + curl -X GET http://localhost:8080/pdb/meta/v1/server-time + + {"server_time": "2013-09-20T20:54:27.472Z"} + diff --git a/docs/_openvoxdb_9x/api/meta/v1/version.markdown b/docs/_openvoxdb_9x/api/meta/v1/version.markdown new file mode 100644 index 000000000..9027c0a97 --- /dev/null +++ b/docs/_openvoxdb_9x/api/meta/v1/version.markdown @@ -0,0 +1,48 @@ +--- +title: "Version endpoint" +layout: default +canonical: "/openvoxdb/latest/api/meta/v1/version.html" +--- + +# Version endpoint + +[curl]: ../../query/curl.html#using-curl-from-localhost-non-sslhttp + +The `/version` endpoint can be used to retrieve version information from the OpenVoxDB server. + +## `/pdb/meta/v1/version` + +This query endpoint will return version information about the running OpenVoxDB +server. + +This endpoint does not use any URL parameters or query strings. + +## `/pdb/meta/v1/version/latest` + +This query will display a message describing the latest version of OpenVoxDB. + +### Response format + +The response will be in `application/json`, and will return a JSON map with a +single key: `version`, whose value is a string representation of the version +of the running OpenVoxDB server. + + {"version": "X.Y.Z"} + +### Examples + +[Using `curl` from localhost][curl]: + + curl -X GET http://localhost:8080/pdb/meta/v1/version + + {"version": "X.Y.Z"} + + curl -X GET http://localhost:8080/pdb/meta/v1/version/latest + + { + "newer" : false, + "product" : "puppetdb", + "link" : "https://docs.puppetlabs.com/puppetdb/2.3/release_notes.markdown", + "message" : "Version 2.3.4 is now available!", + "version" : "2.3.4" + } diff --git a/docs/_openvoxdb_9x/api/metrics/v2/jolokia.markdown b/docs/_openvoxdb_9x/api/metrics/v2/jolokia.markdown new file mode 100644 index 000000000..0550060f9 --- /dev/null +++ b/docs/_openvoxdb_9x/api/metrics/v2/jolokia.markdown @@ -0,0 +1,365 @@ +--- +layout: default +title: "OpenVoxDB: Metrics API v2" +canonical: "/openvoxdb/latest/api/metrics/v2/jolokia.html" +--- + +# Metrics API v2 + +The Jolokia API is enabled by default. You must use `https://` to access `metrics/v2` for any +service, and you must present authorization in the form of a Puppet certificate. + +## Jolokia endpoints + +The v2 metrics endpoint uses the [Jolokia](https://jolokia.org) library, an +extensive open-source metrics library with its own documentation. + +The documentation below provides only the information you need to use the metrics +as configured by default for OpenVoxDB, but Jolokia offers more features than +are described below. Consult the [Jolokia documentation](https://jolokia.org/documentation.html) +for more information. + +For security reasons, we enable only the read-access Jolokia interface by default: + +- `read` +- `list` +- `version` +- `search` + +### Creating a metrics.conf file + +To configure Jolokia metrics, create the `/etc/puppetlabs/puppetdb/conf.d/metrics.conf` +file if one doesn't exist. This file should contain a section like the example shown +below. + +```text +metrics { + metrics-webservice: { + jolokia: { + #enabled: false # uncomment this line to disable jolokia metrics + #servlet-init-params: { + ## Specify a custom security policy: + ## https://jolokia.org/reference/html/security.html + #policyLocation: "file:///etc/puppetlabs/puppetdb/jolokia-access.xml" + #} + } + } +} +``` + +### Configuring Jolokia + +To change the security access policy, create the `/etc/puppetlabs/puppetdb/jolokia-access.xml` +file with contents that follow the [Jolokia access policy](https://jolokia.org/reference/html/security.html) +and uncomment the `metrics.metrics-webservice.jolokia.servlet-init-params.policyLocation` +parameter before restarting puppetdb. + +The `metrics.metrics-webservice.jolokia.servlet-init-params` table within the +`/etc/puppetlabs/puppetdb/conf.d/metrics.conf` file provides more configuration options. +See Jolokia's [agent initialization documentation](https://jolokia.org/reference/html/agents.html#agent-war-init-params) +for all of the available options. + +### Disabling the endpoints + +To disable the v2 endpoints, set the `metrics.metrics-webservice.jolokia.enabled` parameter in `metrics.conf` to `false`. + +## Usage + +You can query the metrics v2 API using `GET` or `POST` requests. + +### `GET /metrics/v2/` + +This endpoint requires an operation, and depending on the operation can accept or might require an additional query: + +```text +GET /metrics/v2// +``` + +#### Response + +A successful request returns a JSON document. + +#### Examples + +To list all valid mbeans querying the metrics endpoint + +```text +GET /metrics/v2/list +``` + +Which should return a response similar to + +```json +{ + "request": { + "type": "list" + }, + "value": { + "java.util.logging": { + "type=Logging": { + "op": { + "getLoggerLevel": { + ... + }, + ... + }, + "attr": { + "LoggerNames": { + "rw": false, + "type": "[Ljava.lang.String;", + "desc": "LoggerNames" + }, + "ObjectName": { + "rw": false, + "type": "javax.management.ObjectName", + "desc": "ObjectName" + } + }, + "desc": "Information on the management interface of the MBean" + } + }, + ... + } +} +``` + +So, from the example above we could query for the registered logger names with +this HTTP call: + +```text +GET /metrics/v2/read/java.util.logging:type=Logging/LoggerNames +``` + +Which would return the JSON document + +```json +{ + "request": { + "mbean": "java.util.logging:type=Logging", + "attribute": "LoggerNames", + "type": "read" + }, + "value": [ + "javax.management.snmp", + "global", + "javax.management.notification", + "javax.management.modelmbean", + "javax.management.timer", + "javax.management", + "javax.management.mlet", + "javax.management.mbeanserver", + "javax.management.snmp.daemon", + "javax.management.relation", + "javax.management.monitor", + "javax.management.misc", + "" + ], + "timestamp": 1497977258, + "status": 200 +} +``` + +The MBean names can then be created by joining the the first two keys of the +value table with a colon (the `domain` and `prop list` in Jolokia parlance). +Querying the MBeans is achieved via the `read` operation. The `read` operation +has as its GET signature: + +```text +GET /metrics/v2/read/// +``` + +### `POST /metrics/v2/` + +You can also submit a POST request with the query as a JSON document in the body of the POST. + +## Filtering + +The new Jolokia-based metrics API also provides globbing (wildcard selection) and response filtering features. + +### Example + +You can combine both of these features to query garbage collection data, but return only the collection counts and times. + +```text +GET metrics/v2/read/java.lang:name=*,type=GarbageCollector/CollectionCount,CollectionTime +``` + +This returns a JSON response: + +```json +{ + "request": { + "mbean": "java.lang:name=*,type=GarbageCollector", + "attribute": [ + "CollectionCount", + "CollectionTime" + ], + "type": "read" + }, + "value": { + "java.lang:name=PS Scavenge,type=GarbageCollector": { + "CollectionTime": 1314, + "CollectionCount": 27 + }, + "java.lang:name=PS MarkSweep,type=GarbageCollector": { + "CollectionTime": 580, + "CollectionCount": 5 + } + }, + "timestamp": 1497977710, + "status": 200 +} +``` + +Refer to the +[Jolokia protocol documentation](https://jolokia.org/reference/html/protocol.html) +for more advanced usage. + +## Curl example + +The jolokia endpoint requires cert-based authentication, which can be done in +curl with the following command. + +```console +curl https://localhost:8081/metrics/v2/list \ + --cert path/to/localhost.pem \ + --key path/to/localhost.key \ + --cacert path/to/ca.pem +``` + +Puppet's configuration also has enough information to construct the command for +you. From the appropriate server with a Puppet Agent configured, the following +command should populate the necessary information. For repeated querying, you +should save the output of each command because printing the necessary configs +is _much_ slower than a simple curl command. + +```console +curl "https://$(puppet config print server):8081/metrics/v2/list" \ + --cert "$(puppet config print hostcert)" \ + --key "$(puppet config print hostprivkey)" \ + --cacert "$(puppet config print localcacert)" +``` + +## Useful metrics + +### Population metrics + +- `puppetlabs.puppetdb.population:name=num-nodes`: + the number of nodes in your population. +- `puppetlabs.puppetdb.population:name=num-resources`: + the number of resources in your population. +- `puppetlabs.puppetdb.population:name=avg-resources-per-node`: + the average number of resources per node in your population. +- `puppetlabs.puppetdb.population:name=pct-resource-dupes`: + the percentage of resources that exist on more than one node. + +### Database Metrics + +OpenVoxDB relies on the HikariCP connection pool. The complete list of +HikariCP metrics and their names can be found in +[their documentation](https://github.com/brettwooldridge/HikariCP/wiki/Dropwizard-Metrics). +All the database metrics have the following naming convention: + +```text +puppetlabs.puppetdb.database:PDBWritePool. +puppetlabs.puppetdb.database:PDBReadPool. +``` + +### Message queue metrics + +OpenVoxDB maintains various command processing metrics, all computed +with respect to the last restart. There are `global` statistics, +aggregated across all commands, and individual statistics, computed +for each version of each command. + +#### Global metrics + +Each of these metrics can be accessed as +`puppetlabs.puppetdb.mq:name=global.`, using any of the +following ``s: + +- `seen`: meter measuring commands received (valid or invalid) +- `processed`: meter measuring commands successfully processed +- `fatal`: meter measuring fatal processing errors +- `retried`: meter measuring commands scheduled for retrial +- `awaiting-retry`: number of commands waiting to be retried +- `retry-counts`: histogram of retry counts (until success or discard) +- `discarded`: meter measuring commands discarded as invalid +- `processing-time`: timing statistics for the processing of + previously enqueued commands +- `queue-time`: histogram of the time commands have spent waiting in the queue +- `depth`: number of currently enqueued commands +- `ignored`: number of obsolete commands that have been ignored +- `size`: histogram of submitted command sizes (i.e. HTTP Content-Lengths) + +For example: `puppetlabs.puppetdb.mq:name=global.seen`. + +#### Metrics for each command version + +Each of the command-specific metrics can be accessed as +`puppetlabs.puppetdb.mq:name=..`, where +`` must be a valid command name, `` must be the +integer command version, and `` must be one of the following: + +- `seen`: meter measuring commands received (valid or invalid) +- `processed`: meter measuring commands successfully processed +- `fatal`: meter measuring fatal processing errors +- `retried`: meter measuring commands scheduled for retrial +- `retry-counts`: histogram of retry counts (until success or discard) +- `discarded`: meter measuring commands discarded as invalid +- `ignored`: number of obsolete commands that have been ignored +- `processing-time`: timing statistics for the processing of + previously enqueued commands + +For example: `puppetlabs.puppetdb.mq:name=replace catalog.9.processed`. + +### HTTP metrics + +OpenVoxDB automatically collects metrics about every URL it has served +to clients. You can see things like the average response time on a +per-URL basis, or see how many requests against a particular URL +resulted in a HTTP 400 response code. Each of the following metrics is +available for each URL. The list of automatically generated metrics is +available via the `/metrics/v2` endpoint. + +Additionally, we also support the following explicit names: + +>**Note:** The use of these explicit names is deprecated; please use, for example, `/pdb/cmd/v1` instead. + +- `commands`: stats relating to the command processing REST + endpoint. The OpenVoxDB-termini in Puppet talk to this endpoint to + submit new catalogs, facts, etc. +- `metrics`: stats relating to the metrics REST endpoint. This is the + endpoint you're reading about right now! +- `facts`: stats relating to fact querying. +- `resources`: stats relating to resource querying. This is the + endpoint used when collecting exported resources. + +In addition to customizing ``, the following metrics are +available for each HTTP status code (``). For example, you can +see the stats for all `200` responses for the `resources` +endpoint. This allows you to see, per endpoint and per response, +independent counters and statistics. + +- `puppetlabs.puppetdb.http:name=.service-time`: + stats about how long it takes to service all HTTP requests to this endpoint +- `puppetlabs.puppetdb.http:name=.`: + stats about how often we're returning this response code + +### Storage metrics + +Metrics involving the OpenVoxDB storage subsystem all begin with the +`puppetlabs.puppetdb.storage:name=` prefix. There are +a number of metrics concerned with individual storage operations (storing +resources, storing edges, etc.). Metrics of particular note include: + +- `puppetlabs.puppetdb.storage:name=duplicate-pct`: + the percentage of catalogs that OpenVoxDB determines to be + duplicates of existing catalogs. +- `puppetlabs.puppetdb.storage:name=gc-time`: states + about how long it takes to do storage compaction. + +### JVM metrics + +- `java.lang:type=Memory`: memory usage statistics. +- `java.lang:type=Threading`: stats about JVM threads. diff --git a/docs/_openvoxdb_9x/api/overview.markdown b/docs/_openvoxdb_9x/api/overview.markdown new file mode 100644 index 000000000..e87325f42 --- /dev/null +++ b/docs/_openvoxdb_9x/api/overview.markdown @@ -0,0 +1,78 @@ +--- +title: "API overview" +layout: default +canonical: "/openvoxdb/latest/api/overview.html" +--- + +# API overview + +[commands]: ./command/v1/commands.html +[termini]: ../connect_puppet_server.html +[ast]: ./query/v4/ast.html +[pql]: ./query/v4/pql.html + +Because OpenVoxDB collects lots of data from Puppet, it's an ideal platform for new tools and applications that use that data. You can use the HTTP API described in these pages to interact with OpenVoxDB's data. + +## Summary + +OpenVoxDB's API uses a Command/Query Responsibility Separation (CQRS) pattern. This means: + +- Data can be **queried** using a standard REST-style API. Queries are processed immediately. +- When **making changes** to data (facts, catalogs, etc.), you must send an explicit **command** (as opposed to submitting data without comment and letting the receiver determine intent). Commands are processed + asynchronously in FIFO order. + +The OpenVoxDB API consists of the following parts: + +- [The REST interface for queries](#queries) +- [The HTTP command submission interface](#commands) +- [The wire formats that OpenVoxDB requires for incoming data](#wire-formats) + +## Queries + +OpenVoxDB's data can be queried with a REST API. + +- [Specification of the general query structure](./query/v4/query.html) +- [AST query language][ast] +- [Puppet query language][pql] +- [Query tutorial](./query/tutorial.html) +- [Curl tips](./query/curl.html) + +The available query endpoints are documented in the pages linked below. + +### Query endpoints + +#### Version 4 + +This is the current stable API. + +- [Root Endpoint](./query/v4/overview.html) +- [Entity Endpoints](./query/v4/entities.html) +- [Metrics Endpoint](./metrics/v2/jolokia.html) +- [Server Time Endpoint](./meta/v1/server-time.html) +- [Version Endpoint](./meta/v1/version.html) + +#### Version 3 (Retired) + +Version 3 of the query API has been retired. Please use v4. + +#### Version 2 (Retired) + +Version 2 of the query API has been retired. Please use v4. + +## Commands + +Commands are sent via HTTP but do not use a REST-style interface. + +OpenVoxDB supports a relatively small number of commands. The command submission interface and all available commands are described at the [commands page][commands]. + +Unlike the query API, these commands are generally only useful to Puppet itself, and all format conversion and command submission is handled by the [OpenVoxDB-termini][termini] on your Puppet Server. + +The "replace" commands all require data in one of the wire formats described below. + +## Wire formats + +All of OpenVoxDB's "replace" commands contain payload data, which must be in one of the following formats. These formats are also linked from the [commands](#commands) that use them. + +- [Facts wire format version 4](./wire_format/facts_format_v4.html) +- [Catalog wire format version 6](./wire_format/catalog_format_v6.html) +- [Report wire format version 5](./wire_format/report_format_v5.html) diff --git a/docs/_openvoxdb_9x/api/query/curl.markdown b/docs/_openvoxdb_9x/api/query/curl.markdown new file mode 100644 index 000000000..bca960566 --- /dev/null +++ b/docs/_openvoxdb_9x/api/query/curl.markdown @@ -0,0 +1,130 @@ +--- +layout: default +title: "API curl tips" +canonical: "/openvoxdb/latest/api/query/curl.html" +--- + +# API curl tips + +[curl]: http://curl.haxx.se/docs/manpage.html +[dashboard]: ../../maintain_and_tune.html#monitor-the-performance-dashboard +[allowlist]: ../../configure.html#certificate-allowlist +[entities]: ./v4/entities.html +[pql]: ./tutorial-pql.html +[pdb-cli]: ../../pdb_client_tools.html + +You can use [`curl`][curl] to directly interact with OpenVoxDB's REST API. This is useful for testing, prototyping, and quickly fetching arbitrary data. + +The instructions below are simplified. For full usage details, see [the curl man page][curl]. For additional examples, please see the user guides for the individual [query REST endpoints][entities], or the other REST API services available. + +## Using `curl` From `localhost` (non-SSL/HTTP) + +With its default settings, OpenVoxDB accepts unsecured HTTP connections at port 8080 on `localhost`. This allows you to SSH into the OpenVoxDB server and run curl commands without specifying certificate information: + +```sh +curl http://localhost:8080/pdb/query/v4/nodes +``` + +If you have allowed unsecured access to other hosts in order to [monitor the dashboard][dashboard], these hosts can also use plain HTTP curl commands. + +## Using `curl` from remote hosts (SSL/HTTPS) + +### Using a certificate/private key pair + +To make secured requests from other hosts, you will need to supply the following +via the command line: + +* Your site's CA certificate (`--cacert`) +* An SSL certificate signed by your site's Puppet CA (`--cert`) +* The private key for that certificate (`--key`) + +Any node managed by Puppet agent will already have all of these, and you can +reuse them for contacting OpenVoxDB. You can also generate a new cert on the CA +Puppet Server with the `puppetserver ca generate` command. + +> **Note:** If you have turned on [certificate allowlisting][allowlist], you must +make sure to authorize the certificate you are using: +> +> ```console +> curl 'https://:8081/pdb/query/v4/nodes' \ +> --tlsv1 \ +> --cacert /etc/puppetlabs/puppet/ssl/certs/ca.pem \ +> --cert /etc/puppetlabs/puppet/ssl/certs/.pem \ +> --key /etc/puppetlabs/puppet/ssl/private_keys/.pem +> ``` + +### Locating Puppet certificate files + +Locate Puppet's `ssldir` as follows: + +```console +sudo puppet config print ssldir +``` + +Within this directory: + +* The CA certificate is found at `certs/ca.pem` +* The corresponding private key is found at `private_keys/.pem` +* Other certificates are found at `certs/.pem` + +## Dealing with complex query strings + +Many query strings will contain characters like `[` and `]`, which must be URL-encoded. To handle this, you can use `curl`'s `--data-urlencode` option. + +If you do this with an endpoint that accepts `GET` requests, **you must also use the `-G` or `--get` option.** This is because `curl` defaults to `POST` requests when the `--data-urlencode` option is present. + +```console +curl -G http://localhost:8080/pdb/query/v4/nodes \ + --data-urlencode 'query=["=", "node_state", "active"]' +``` + +## Pretty querying of OpenVoxDB + +OpenVoxDB returns unprettified JSON by default. OpenVoxDB provides the option of +prettifying your JSON responses with the `pretty` parameter. This parameter +accepts a Boolean value (`true` or `false`) to indicate whether the response +should be pretty-printed. Note that pretty printing comes at the cost of +performance on some of our endpoints, such as `/v4/catalogs`, `/v4/reports` and +`/v4/factsets`, due to the storage of some of their data as JSON/JSONB in PostgreSQL. + +```console +curl -X GET http://localhost:8080/pdb/query/v4/nodes \ + --data-urlencode 'pretty=true' +``` + +## Querying OpenVoxDB with POST + +OpenVoxDB supports querying by POST, which is useful for large +queries (exact limits depend on the client and webserver used). POST queries allow you to limit the number of entries in the response. The example below limits the query to return one entry. + +POST queries use the following syntax: + +```console +curl -X POST http://localhost:8080/pdb/query/v4/nodes \ + -H 'Content-Type:application/json' \ + -d '{"query":["~","certname",".*.com"],"order_by":[{"field":"certname"}],"limit":1}' +``` + +## Querying OpenVoxDB based on specific resource attributes + +You can use POST to query for a specific resource attribute. Note that this +requires you to escape your quotes (`"`). Alternatively, use the [OpenVoxDB +CLI][pdb-cli], together with the [Puppet Query Language (PQL)][pql] to make +queries without having to escape characters. + +To query for the following resource attributes: + +```text +resources { + tag = "foo" and + exported = true +} +``` + +Use the following CURL command: + +```console +curl -X POST http://localhost:8080/pdb/query/v4 \ + -H 'Content-Type:application/json' \ + -d '{"query": "resources { tag = \"foo\" and exported = true }"}' +``` diff --git a/docs/_openvoxdb_9x/api/query/examples-pql.markdown b/docs/_openvoxdb_9x/api/query/examples-pql.markdown new file mode 100644 index 000000000..15831d1b7 --- /dev/null +++ b/docs/_openvoxdb_9x/api/query/examples-pql.markdown @@ -0,0 +1,486 @@ +--- +title: "Puppet Query Language (PQL) examples" +layout: default +--- + +# Examples + +[pg-regex]: https://www.postgresql.org/docs/11/functions-matching.html#FUNCTIONS-POSIX-REGEXP +[tutorial]: ./tutorial-pql.html +[reference]: ./v4/pql.html + +## Example Queries + +Below is a list of example PQL queries that you may find useful. + +Other resources you may also find useful include: + +* [PQL tutorial][tutorial] +* [PQL reference guide][reference] + +*** + +### Filtering on node names + +Query nodes with `green` in their name. + +```text +nodes { certname ~ 'green' } +``` + +or [regexp features][pg-regex] + +```text +nodes { certname ~ '(?i)green' } +``` + +*Output:* + +```json +[ + { + "cached_catalog_status": "not_used", + "catalog_environment": "production", + "catalog_timestamp": "2016-08-15T11:06:26.275Z", + "certname": "greenserver.vm", + "deactivated": null, + "expired": null, + "facts_environment": "production", + "facts_timestamp": "2016-08-15T11:06:26.140Z", + "latest_report_corrective_change": null, + "latest_report_hash": "4a956674b016d95a7b77c99513ba26e4a744f8d1", + "latest_report_noop": false, + "latest_report_noop_pending": null, + "latest_report_status": "changed", + "report_environment": "production", + "report_timestamp": "2016-08-15T11:06:18.393Z" + } +] +``` + +*** + +### Querying for inactive nodes + +```text +nodes { node_state = "inactive" } +``` + +*Output:* + +```json +[ + { + "cached_catalog_status": "not_used", + "catalog_environment": "production", + "catalog_timestamp": "2016-08-15T11:06:26.275Z", + "certname": "foo.com", + "deactivated": "2016-08-17T13:04:41.421Z", + "expired": null, + "facts_environment": "production", + "facts_timestamp": "2016-08-15T11:06:26.140Z", + "latest_report_corrective_change": null, + "latest_report_hash": "az956674b016d95a7b77c99513ba26e4a744f8d1", + "latest_report_noop": false, + "latest_report_noop_pending": null, + "latest_report_status": "changed", + "report_environment": "production", + "report_timestamp": "2016-08-15T11:06:18.393Z" + } +] +``` + +*** + +### Basic fact filtering + +Nodes with operating system name `CentOS`. + +```text +inventory { facts.os.name = "CentOS" } +``` + +*Output (abbreviated):* + +```json +[ + { + "certname": "centos70.vm", + "environment": "production", + "facts": { + ... + "operatingsystem": "CentOS", + "operatingsystemmajrelease": "7", + "operatingsystemrelease": "7.0.1406", + "os": { + "architecture": "x86_64", + "family": "RedHat", + "hardware": "x86_64", + "name": "CentOS", + "release": { + "full": "7.0.1406", + "major": "7", + "minor": "0" + }, + "selinux": { + "enabled": false + } + }, + "osfamily": "RedHat", + ... + }, + "timestamp": "2016-08-15T11:06:26.140Z", + "trusted": { + "authenticated": "remote", + "certname": "centos70.vm", + "domain": "vm", + "extensions": {}, + "hostname": "centos70" + } + } +] +``` + +*** + +### Fact and resource filtering + +Nodes with `CentOS` operating system, and with a declared `httpd` service resource. + +```text +inventory[certname] { facts.operatingsystem = "CentOS" and + resources { type = "Service" and title = "httpd" } } +``` + +*Output:* + +```json +[ + { + "certname": "greenserver.vm" + } +] +``` + +*** + +### Fact, resource and resource parameter filtering + +`RedHat` boxes in the `PDX` datacenter, that have their `java` package forced to `1.7.0`. + +```text +inventory[certname] { facts.osfamily = "RedHat" and + facts.datacentre = "PDX" and + resources { type = "Package" and + title = "java" and + parameters.ensure = "1.7.0" } } +``` + +*Output:* + +```json +[ + { + "certname": "greenserver.vm" + } +] +``` + +*** + +### Fact and resource filtering for classes + +CentOS boxes with the `apache` Puppet class. + +```text +inventory[certname] { facts.operatingsystem = "CentOS" and + resources { type = "Class" and + title = "Apache" } } +``` + +*Output:* + +```json +[ + { + "certname": "greenserver.vm" + } +] +``` + +*** + +### Fact, resource and environment filtering + +All windows servers, with service `sqlserver` in a particular feature branch environment. + +```text +nodes { certname in inventory[certname] { facts.osfamily = "Windows" } and + resources { type = "Service" and + title = "sqlserver" } and + report_environment = "feature_SYS-4926" } +``` + +*Output:* + +```json +[ { + "deactivated" : null, + "latest_report_hash" : "4a956674b016d95a7b77c99513ba26e4a744f8d1", + "facts_environment" : "laboratory", + "cached_catalog_status" : "not_used", + "report_environment" : "laboratory", + "latest_report_corrective_change" : false, + "catalog_environment" : "laboratory", + "facts_timestamp" : "2016-07-18T04:12:12.912Z", + "latest_report_noop" : false, + "expired" : null, + "latest_report_noop_pending" : null, + "report_timestamp" : "2016-07-18T04:12:15.907Z", + "certname" : "windowserver.vm", + "catalog_timestamp" : "2016-07-18T04:12:12.917Z", + "latest_report_status" : "success" +} ] +``` + +### Fact, report status filtering with dot notation + +Get only the `certname`, `os.family` and `puppetversion` for all nodes whose most recent +report indicated a failure. + +```text +inventory[certname, facts.os.family, facts.puppetversion] { + certname in nodes[certname] { latest_report_status = "failed" } +} +``` + +Output: + +```json +[ { + "certname" : "server.vm", + "facts.os.family": "Debian", + "facts.puppetversion": "6.8.1" +} ] +``` + +*** + +### Timestamp filtering + +Nodes that haven't checked in for 7 days. + +```text +nodes { report_timestamp <= "2016-08-03 00:00:00" } +``` + +*Output:* + +```json +[ + { + "cached_catalog_status": "not_used", + "catalog_environment": "production", + "catalog_timestamp": "2016-08-15T11:06:26.275Z", + "certname": "greenserver.vm", + "deactivated": null, + "expired": null, + "facts_environment": "production", + "facts_timestamp": "2016-08-15T11:06:26.140Z", + "latest_report_corrective_change": null, + "latest_report_hash": "4a956674b016d95a7b77c99513ba26e4a744f8d1", + "latest_report_noop": false, + "latest_report_noop_pending": null, + "latest_report_status": "changed", + "report_environment": "production", + "report_timestamp": "2016-08-15T11:06:18.393Z" + } +] +``` + +*** + +### Profile querying + +Show active nodes that have the profile class `Profile::Remote_mgmt` applied to it. + +```text +nodes { resources { type = "Class" and title = "Profile::Remote_mgmt" } } +``` + +*Output:* + +```json +[ + { + "cached_catalog_status": "not_used", + "catalog_environment": "production", + "catalog_timestamp": "2016-08-15T11:06:26.275Z", + "certname": "greenserver.vm", + "deactivated": null, + "expired": null, + "facts_environment": "production", + "facts_timestamp": "2016-08-15T11:06:26.140Z", + "latest_report_corrective_change": null, + "latest_report_hash": "4a956674b016d95a7b77c99513ba26e4a744f8d1", + "latest_report_noop": false, + "latest_report_noop_pending": null, + "latest_report_status": "changed", + "report_environment": "production", + "report_timestamp": "2016-08-15T11:06:18.393Z" + } +] +``` + +*** + +### Querying catalog submission time + +Check for nodes that haven't had a catalog applied since a certain time. + +```text +nodes { catalog_timestamp < "2016-08-15T11:37:00.000Z" } +``` + +*Output:* + +```json +[ + { + "cached_catalog_status": "not_used", + "catalog_environment": "production", + "catalog_timestamp": "2016-08-15T11:06:26.275Z", + "certname": "greenserver.vm", + "deactivated": null, + "expired": null, + "facts_environment": "production", + "facts_timestamp": "2016-08-15T11:06:26.140Z", + "latest_report_corrective_change": null, + "latest_report_hash": "4a956674b016d95a7b77c99513ba26e4a744f8d1", + "latest_report_noop": false, + "latest_report_noop_pending": null, + "latest_report_status": "changed", + "report_environment": "production", + "report_timestamp": "2016-08-15T11:06:18.393Z" + } +] +``` + +*** + +### List of nodes with report status failed + +List all nodes that have had a failure on their last run. + +```text +nodes { latest_report_status = 'failed' } +``` + +*Output:* + +```json +[ + { + "cached_catalog_status": "not_used", + "catalog_environment": "production", + "catalog_timestamp": "2016-08-15T11:03:26.275Z", + "certname": "redserver.vm", + "deactivated": null, + "expired": null, + "facts_environment": "production", + "facts_timestamp": "2016-08-15T11:03:26.140Z", + "latest_report_corrective_change": null, + "latest_report_hash": "68f56674b016d95a7b77c99513ba26e4a744f001", + "latest_report_noop": false, + "latest_report_noop_pending": null, + "latest_report_status": "failed", + "report_environment": "production", + "report_timestamp": "2016-08-15T11:03:18.393Z" + } +] +``` + +*** + +### Query code_id from latest reports + +Query all latest reports and show the certname and code_id. + +```text +reports[certname, code_id] { latest_report? = true } +``` + +*Output:* + +```json +[ + { + "certname": "greenserver.vm", + "code_id": "urn:puppet:code-id:1:519e404a1b6217b010cc543494c2dc50df8a53e3;production" + }, + { + "certname": "yellowserver.vm", + "code_id": "urn:puppet:code-id:1:519e404a1b6217b010cc543494c2dc50df8a53e3;production" + } +] +``` + +*** + +### Reports that have not applied a code_id + +Show reports that have not had a particular code_id applied. + +```text +reports[certname, receive_time] { + latest_report? = true and + ! code_id = 'urn:puppet:code-id:1:519e404a1b6217b010cc543494c2dc50df8a53e3;production' +} +``` + +*Output:* + +```json +[ + { + "certname": "whiteserver.vm", + "receive_time": "2016-08-15T10:33:07.130Z" + }, + { + "certname": "brownserver.vm", + "receive_time": "2016-08-15T11:06:26.553Z" + } +] +``` + +*** + +### Show all exported resources + +Show all exported resources + +```text +resources[certname, type, title] { exported = true } +``` + +*Output:* + +```json +[ + { + "certname": "purpleserver.vm", + "title": "purpleserver.vm-mysql", + "type": "Monitor" + }, + { + "certname": "purpleserver.vm", + "title": "purpleserver.vm-httpd", + "type": "Monitor" + }, + { + "certname": "purpleserver.vm", + "title": "purpleserver.vm-/etc", + "type": "Backup" + } +] +``` diff --git a/docs/_openvoxdb_9x/api/query/tutorial-pql.markdown b/docs/_openvoxdb_9x/api/query/tutorial-pql.markdown new file mode 100644 index 000000000..49920e098 --- /dev/null +++ b/docs/_openvoxdb_9x/api/query/tutorial-pql.markdown @@ -0,0 +1,538 @@ +--- +title: "Puppet Query Language (PQL) tutorial" +layout: default +--- + +# Tutorial + +[lists]: ./v4/pql.html#lists +[curl]: ./curl.html +[config_jetty]: ../../configure.html#jetty-http-settings +[resources]: ./v4/resources.html +[entities]: ./v4/entities.html +[pql]: ./v4/pql.html +[projection]: ./v4/pql.html#projection +[regexp]: ./v4/pql.html#regexp- +[in]: ./v4/pql.html#array-match-in +[implicit]: ./v4/pql.html#implicit-subqueries +[cli_install]: ../../pdb_client_tools.html +[examples]: ./examples-pql.html + +This page walks through the construction of several types of Puppet +Query Language (PQL) queries. We use the **version 4 API** in all examples. + +Other resources you may also find useful include: + +* [PQL examples][examples] +* [PQL reference guide][pql] + +## How to query + +Queries are performed by issuing an HTTP GET or POST request to an endpoint URL +and specifying a `query` URL parameter (in the GET case) or a JSON-valued +payload in the POST case, which contains the query to execute. Results are +always returned in `application/json` form. + +Queries are usually issued from code, but you can easily issue them from the +command line by using the [OpenVoxDB CLI][cli_install] or using [curl][curl]. + +### Querying with the OpenVoxDB CLI + +[See the OpenVoxDB CLI installation page for more information about using the OpenVoxDB CLI.][cli_install] + +**Without SSL:** + +```console +puppet query '' \ + --urls http://openvoxdb.example.com:8080 +``` + +This requires that OpenVoxDB be +[configured to accept non-SSL connections][config_jetty]. By default, it will +only accept unencrypted traffic from `localhost`. + +**With SSL:** + +```console +puppet query '' \ + --urls https://openvoxdb.example.com:8081 \ + --cacert /etc/puppetlabs/puppet/ssl/certs/ca.pem \ + --cert /etc/puppetlabs/puppet/ssl/certs/thisnode.pem \ + --key /etc/puppetlabs/puppet/ssl/private_keys/thisnode.pem +``` + +This requires that you specify a certificate (issued by the same CA OpenVoxDB +trusts), a private key, and a CA certificate. + +> **Note**: The OpenVoxDB CLI can be configured using a config file at +`$HOME/.puppetlabs/client-tools/puppetdb.conf` with default values for the +server urls and SSL credentials. + +### Querying with curl + +[See the curl tips page for more information about constructing curl commands.][curl] + +**Without SSL:** + +```console +curl -X GET http://openvoxdb.example.com:8080/pdb/query/v4 \ + --data-urlencode 'query=' + +curl -X POST http://openvoxdb.example.com:8080/pdb/query/v4 \ + -H 'Content-Type:application/json' + -d '{"query":""}' +``` + +This requires that OpenVoxDB be +[configured to accept non-SSL connections][config_jetty]. By default, it will +only accept unencrypted traffic from `localhost`. + +**With SSL:** + +```console +curl -X GET https://openvoxdb.example.com:8081/pdb/query/v4 \ + --tlsv1 \ + --cacert /etc/puppetlabs/puppet/ssl/certs/ca.pem \ + --cert /etc/puppetlabs/puppet/ssl/certs/thisnode.pem \ + --key /etc/puppetlabs/puppet/ssl/private_keys/thisnode.pem \ + --data-urlencode 'query=' +``` + +This requires that you specify a certificate (issued by the same CA OpenVoxDB +trusts), a private key, and a CA certificate. + +### Querying with Puppet code + +The OpenVoxDB terminus includes the `puppetdb_query` function, which can be used +to query OpenVoxDB from within a Puppet manifest. For example, + +```puppet +$debian_nodes_query = 'nodes[certname]{facts{name = "operatingsystem" and value = "Debian"}}' +$debian_nodes = puppetdb_query($debian_nodes_query).map |$value| { $value['certname'] } +notify { 'Debian nodes': + message => "Your debian nodes are ${join($debian_nodes, ', ')}", +} +``` + +## Resources Walkthrough + +### Our first query + +Let's start by taking a look at a simple resource query. + +```text +resources {} +``` + +Executing that query will return all resources for all nodes, however it is rare +that you will want all this information. In this case, we use filters to reduce +the results. + +Now suppose we want to find the user `nick` on every node. We can use this +query: + +```text +resources { type = "User" and title = "nick" } +``` + +This query has two `=` clauses, both of which must be true. + +The `=` operator follows a specific structure: + +```text + = +``` + +In this case, the attributes are `type` and `title`, and the values are `User` +and `nick`. + +The `and` operator also has a well-defined structure: + +```text + and +``` + +The query clauses can be any legal query (including another `and`). At least one +clause must be specified, and all the clauses must be true for the `and` clause +to be true. An `or` operator is also available, which looks just like the `and` +operator, except that, as you'd expect, it's true if *any* specified clause is +true. + +If we execute this query, we get results that look something like this: + +```json +[{ + "parameters" : { + "comment" : "Nick Lewis", + "uid" : "1115", + "shell" : "/bin/bash", + "managehome" : false, + "gid" : "allstaff", + "home" : "/home/nick", + "groups" : "developers", + "ensure" : "present" + }, + "line" : 111, + "file" : "/etc/puppetlabs/code/environments/production/manifests/user.pp", + "exported" : false, + "tags" : [ "firewall", "default", "node", "nick", "role::base", "users", "virtual", "user", "account", "base", "role::firewall::office", "role", "role::firewall", "class", "account::user", "office", "virtual::users", "allstaff" ], + "title" : "nick", + "type" : "User", + "resource" : "0ae7e1230e4d540caa451d0ade2424f316bfbf39", + "certname" : "foo.example.com" +}] +``` + +Our results are an array of `resources`, where each resource is an object with a +particular set of keys. + +* `parameters`: this field is itself an object, containing all the parameters + and values of the resource +* `line`: the line the resource was declared on +* `file`: the file the resource was specified in +* `exported`: true if the resource was exported by this node, or false otherwise +* `tags`: all the tags on the resource +* `title`: the resource title +* `type`: the resource type +* `resource`: this is an internal identifier for the resource used by OpenVoxDB +* `certname`: the node that the resource came from + +There will be an entry in the list for every resource. A resource is specific to +a single node, so if the resource is on 100 nodes, there will be 100 copies of +the resource (each with at least a different certname field). + +*Note:* More information about entities and their available fields can be +accessed from the [entities document][entities]. + +### Excluding results + +We know this instance of the user `nick` is defined on line 111 of +`/etc/puppetlabs/code/environments/production/manifests/user.pp`. What if we +want to check whether or not we define the same resource somewhere else? After +all, if we're repeating ourselves, something may be wrong! Fortunately, there's +an operator to help us: + +```text +resources { + type = "User" and + title = "nick" and + !(file = "/etc/puppetlabs/code/environments/production/manifests/user.pp" and line = 111) +} +``` + +The `!` operator wraps another clause, and returns results for which the clause +is *not* true. In this case, we want resources which aren't defined on line 111 +of `/etc/puppetlabs/code/environments/production/manifests/user.pp`. + +Another thing to note is the way we have grouped `file` and `line`. This +grouping enforces the `!` operator to act on both parameter filters. + +### Resource attributes + +So far we've seen that we can query for resources based on their `certname`, +`type`, `title`, `file`, and `line`. There are a few more available: + +```text +resources { + tag = "foo" and + exported = true +} +``` + +This query returns resources whose set of tags *contains* the tag `foo`, and +which are exported. + +For easy reference, the full set of queryable attributes can be found in +[the resource endpoint documentation][resources]. + +### Regular expressions + +What if we want to restrict our results to a certain subset of nodes? We could +use something like this: + +```text +resources { + certname = "www1.example.com" or + certname = "www2.example.com" or + certname = "www3.example.com" +} +``` + +And this works great if we know exactly the set of nodes we want. But what if we +want all the 'www' servers, regardless of how many we have? In this case, we can +use the regular expression match operator `~`: + +```text +resources { + certname ~ 'www\d+.example.com' +} +``` + +For more information regarding the regular expression operator, +[consult the reference guide][regexp]. + +### Choosing fields to return (projection) + +When you execute a resources query, all available fields for that query are +returned by default. Using the projection syntax however, you can limit what +fields are returned. + +For example, if you want to only respond with the `certname`, `type`, and +`title` of each resource where `Class[Apache]` is defined, you can do the +folllowing: + +```text +resources[certname, type, title] { + type = "Class" and + title = "Apache" +} +``` + +For more information regarding projection, consult the +[reference guide][projection]. + +## Facts walkthrough + +In addition to resources, we can also query for facts. This looks similar, +though the available fields and operators are a bit different. Some things are +the same, though. For instance, suppose you want all the facts for a certain +node: + +```text +facts { certname = "foo.example.com" } +``` + +This gives results that look something like this: + +```json +[ { + "certname" : "foo.example.com", + "name" : "architecture", + "value" : "amd64", + "environment" : "production" +}, { + "certname" : "foo.example.com", + "name" : "fqdn", + "value" : "foo.example.com", + "environment" : "production" +}, { + "certname" : "foo.example.com", + "name" : "hostname", + "value" : "foo", + "environment" : "production" +}, { + "certname" : "foo.example.com", + "name" : "ipaddress", + "value" : "192.168.100.102", + "environment" : "production" +}, { + "certname" : "foo.example.com", + "name" : "kernel", + "value" : "Linux", + "environment" : "production" +}, { + "certname" : "foo.example.com", + "name" : "kernelversion", + "value" : "2.6.32", + "environment" : "production" +} ] +``` + +### Fact attributes + +In the last query, we saw that a "fact" consists of a "certname", a "name", and +a "value". As you might expect, we can query using "name" or "value". + +```text +facts { + name = "operatingsystem" and + value = "Debian" +} +``` + +This will find all the `operatingsystem = Debian` facts, and their corresponding +nodes. As you see, `and` is supported for facts, as are `or` and `!`. + +### Fact operators + +As with resources, facts also support the `~` regular expression match operator +for all their fields. In addition, numeric comparisons are supported for fact +values: + +```text +facts { + name = "uptime_seconds" and + value >= 100000 and + value < 1000000 +} +``` + +This will find nodes for which the `uptime_seconds` fact is in the range 100000 +to 1000000. Numeric comparisons will *always be false* for fact values which are +not numeric. Importantly, version numbers such as 2.6.12 are not numeric, and +numeric comparison operators can't be used with them at this time. + +## Nodes walkthrough + +We can also query for nodes. Again, this is similar to resource and +fact queries: + +```text +nodes { + certname = "foo.example.com" +} +``` + +The result of this query is: + +```json +[ { + "deactivated" : null, + "facts_environment" : "production", + "report_environment" : "production", + "catalog_environment" : "production", + "facts_timestamp" : "2015-06-22T17:25:11.886Z", + "expired" : null, + "report_timestamp" : "2015-06-22T17:25:07.484Z", + "certname" : "foo.example.com", + "catalog_timestamp" : "2015-06-22T17:25:12.023Z", + "latest_report_hash" : "754b0b87af9ee647507b5aa3001f44f8e8843216", + "latest_report_noop": true, + "cached_catalog_status": "not_used", + "latest_report_status" : "unchanged" +} ] +``` + +This will return an object containing the certname `foo.example.com`, as well as +some metadata detailing deactivation status and the most recent fact, report, +and catalog updates from that node. + +## Subquery walkthrough + +### Explicit subqueries + +The queries we've looked at so far are quite powerful and useful, but what if +your query needs to consider both resources *and* facts? + +For instance, suppose you're configuring a load balancer, and need the IP +addresses of your Apache servers. You could find those servers by using this +resource query: + +```text +resources { + type = "Class" and + title = "Apache" +} +``` + +This will find all the `Class[Apache]` resources, which each knows the certname +of the node it came from. Then you could put all those certnames into a fact +query: + +```text +facts { + name = "ipaddress" and + (certname = "a.example.com" or + certname = "b.example.com" or + certname = "c.example.com" or + certname = "d.example.com" or + certname = "e.example.com") +} +``` + +But this query is lengthy, and it requires some logic to assemble and run the +second query. There has to be a better way! What if we could find the +`Class[Apache]` servers and use the results of that query to find the certname? +We can, with this fact query: + +```text +facts { + name = "ipaddress" and + certname in resources[certname] { type = "Class" and title = "Apache" } +} +``` + +This may appear a little daunting, so we'll look at it piece by piece. + +Let's start with `resources[certname] { }`. This query will return the +certname for the results that match the filter as specified. + +We then use an operator called `in` to turn our list of resources into just a +list of certnames. So we now conceptually have something like: + +```text +certname in ["foo.example.com", "bar.example.com", "baz.example.com"] +``` + +The `in` operator matches facts whose `certname` is in the supplied list. (In +our case, this list is generated by a subquery. To use a literal list, you must +use the the syntax described in the [PQL lists section][lists].) + +For more information regarding the `in` operator, consult the +[reference guide][in]. + +### Implicit subqueries + +Explicit subqueries allow you to query across related entities, but they require +you to specify the columns that you wish to join on. Some relationships between +entities are well known to OpenVoxDB. We can use this information ourselves in a +query avoiding the need for specifying how entities relate. + +If you take this example query: + +```text +facts { + name = "operatingsystem" and + value = "Debian" +} +``` + +This will return all `facts` that match the filter. But if we wanted to return +the `nodes` entity results for nodes that have facts that match this filter, we +can utilize implicit subqueries instead, by embedding the query inside a filter: + +```text +nodes { + facts { + name = "operatingsystem" and + value = "Debian" + } +} +``` + +In this case, node information is returned, even though the filter was done +across facts relating to the node: + +```json +[ { + "deactivated" : null, + "facts_environment" : "production", + "report_environment" : "production", + "catalog_environment" : "production", + "facts_timestamp" : "2015-06-22T17:25:11.886Z", + "expired" : null, + "report_timestamp" : "2015-06-22T17:25:07.484Z", + "certname" : "foo.example.com", + "catalog_timestamp" : "2015-06-22T17:25:12.023Z", + "latest_report_hash" : "754b0b87af9ee647507b5aa3001f44f8e8843216", + "latest_report_noop": true, + "cached_catalog_status": "not_used", + "latest_report_status" : "unchanged" +} ] +``` + +Implicit subqueries are then just real queries embedded in the filtering part of +the query itself. You can mix and match subquery filters also like so: + +```text +nodes { + facts { name = "operatingsystem" and value = "Debian" } and + resources { type = "Class" and title = "Apache" } +} +``` + +This query will query all `nodes` that are Debian and have the class +`Class[Apache]` defined. + +For more information regarding implicit subqueries, consult the +[reference guide][implicit]. diff --git a/docs/_openvoxdb_9x/api/query/tutorial.markdown b/docs/_openvoxdb_9x/api/query/tutorial.markdown new file mode 100644 index 000000000..e897b01a7 --- /dev/null +++ b/docs/_openvoxdb_9x/api/query/tutorial.markdown @@ -0,0 +1,319 @@ +--- +title: "API query tutorial" +layout: default +canonical: "/openvoxdb/latest/api/query/tutorial.html" +--- + +# API query tutorial + +[array]: ./v4/ast.html#array +[curl]: ./curl.html +[select]: ./v4/ast.html#select_entity-subquery-statements +[config_jetty]: ../../configure.html#jetty-http-settings + +This page walks through the construction of several types of OpenVoxDB queries. We use the **version 4 API** in all examples. + +## How to query + +Queries are performed by issuing an HTTP GET or POST request to an endpoint URL and specifying a `query` URL parameter (in the GET case) or a JSON-valued payload in the POST case, which contains the query to +execute. Results are always returned in `application/json` form. + +Queries are usually issued from code, but you can easily issue them from the command line by using curl. + +### Querying with curl + +[See the curl tips page for more information about constructing curl commands.][curl] + +**Without SSL:** + + curl -X GET http://openvoxdb.example.com:8080/pdb/query/v4/resources \ + --data-urlencode query@ + + curl -X POST http://openvoxdb.example.com:8080/pdb/query/v4/resources \ + -H 'Content-Type:application/json' + -d '{"query":["=","certname","foo.com"]}' + +This requires that OpenVoxDB be [configured to accept non-SSL connections][config_jetty]. By default, it will only accept unencrypted traffic from `localhost`. + +**With SSL:** + + curl -X GET https://openvoxdb.example.com:8081/pdb/query/v4/resources \ + --tlsv1 \ + --cacert /etc/puppetlabs/puppet/ssl/certs/ca.pem \ + --cert /etc/puppetlabs/puppet/ssl/certs/thisnode.pem \ + --key /etc/puppetlabs/puppet/ssl/private_keys/thisnode.pem \ + --data-urlencode query@ + +This requires that you specify a certificate (issued by the same CA OpenVoxDB trusts), a private key, and a CA certificate. + +In both examples, `` should be a file that contains the query to execute. + +### Querying with Puppet code + +The OpenVoxDB terminus includes the `puppetdb_query` function, which can be used to query OpenVoxDB from within a Puppet manifest. For example, + + $debian_nodes_query = '["from", "nodes", ["=", ["fact", "operatingsystem"], "Debian"]]' + $debian_nodes = puppetdb_query($debian_nodes_query).each |$value| { $value["certname"] } + notify { 'Debian nodes': + message => "Your debian nodes are ${join($debian_nodes, ', ')}", + } + +## Resources Walkthrough + +### Our first query + +Let's start by taking a look at a simple resource query. Suppose we want to find the user "nick" on every node. We can use this query: + + ["and", + ["=", "type", "User"], + ["=", "title", "nick"]] + +This query has two `"="` clauses, both of which must be true. + +In general, the `"="` operator follows a specific structure: + +`["=", , ]` + +In this case, the attributes are "type" and "title", and the values are "User" and "nick". + +The `"and"` operator also has a well-defined structure: + +`["and", , , , ...]` + +The query clauses can be any legal query (including another `"and"`). At least one clause must be specified, and all the clauses must be true for the `"and"` clause to be true. An `"or"` operator is also +available, which looks just like the `"and"` operator, except that, as you'd expect, it's true if _any_ specified clause is true. + +The query format is declarative: it describes conditions the results must satisfy, not how to find them. This means that the order of the clauses is irrelevant. + +You can list either the type clause or the title clause first without impacting the performance or the results of the query. + +If we execute this query against the `/resources` route, and assuming that we're using the production environment, we get results that look something like this: + + [{ + "parameters" : { + "comment" : "Nick Lewis", + "uid" : "1115", + "shell" : "/bin/bash", + "managehome" : false, + "gid" : "allstaff", + "home" : "/home/nick", + "groups" : "developers", + "ensure" : "present" + }, + "line" : 111, + "file" : "/etc/puppetlabs/code/environments/production/manifests/user.pp", + "exported" : false, + "tags" : [ + "firewall", "default", "node", "nick", "role::base", "users", "virtual", "user", "account", "base", "role::firewall::office", "role", "role::firewall", "class", "account::user", "office", + "virtual::users", "allstaff" + ], + "title" : "nick", + "type" : "User", + "resource" : "0ae7e1230e4d540caa451d0ade2424f316bfbf39", + "certname" : "foo.example.com" + }] + +Our results are an array of "resources", where each resource is an object with a particular set of keys. + +- `parameters`: this field is itself an object, containing all the parameters and values of the resource +- `line`: the line the resource was declared on +- `file`: the file the resource was specified in +- `exported`: true if the resource was exported by this node, or false otherwise +- `tags`: all the tags on the resource +- `title`: the resource title +- `type`: the resource type +- `resources`: this is an internal identifier for the resource used by OpenVoxDB +- `certname`: the node that the resource came from + +There will be an entry in the list for every resource. A resource is specific to a single node, so if the resource is on 100 nodes, there will be 100 copies of the resource (each with at least a different +certname field). + +### Excluding results + +We know this instance of the user "nick" is defined on line 111 of `/etc/puppetlabs/code/environments/production/manifests/user.pp`. What if we want to check whether or not we define the same resource somewhere +else? After all, if we're repeating ourselves, something may be wrong! Fortunately, there's an operator to help us: + + ["and", + ["=", "type", "User"], + ["=", "title", "nick"], + ["not", + ["and", + ["=", "file", "/etc/puppetlabs/code/environments/production/manifests/user.pp"], + ["=", "line", 111]]]] + +The `"not"` operator wraps another clause, and returns results for which the clause is _not_ true. In this case, we want resources which aren't defined on line 111 of +`/etc/puppetlabs/code/environments/production/manifests/user.pp`. + +### Resource attributes + +So far we've seen that we can query for resources based on their `certname`, `type`, `title`, `file`, and `line`. There are a few more available: + + ["and", + ["=", "tag", "foo"], + ["=", "exported", true], + ["=", ["parameter", "ensure"], "present"]] + +This query returns resources whose set of tags _contains_ the tag "foo", and which are exported, and whose "ensure" parameter is "present". Because the parameter name can take any value (including that of +another attribute), it must be namespaced using `["parameter", ]`. + +For easy reference, the full set of queryable attributes can be found in [the resource endpoint documentation](./v4/resources.html). + +### Regular expressions + +What if we want to restrict our results to a certain subset of nodes? We could use something like this: + + ["or", + ["=", "certname", "www1.example.com"], + ["=", "certname", "www2.example.com"], + ["=", "certname", "www3.example.com"]] + +And this works great if we know exactly the set of nodes we want. But what if we want all the 'www' servers, regardless of how many we have? In this case, we can use the regular expression match operator `~`: + + ["~", "certname", "www\\d+\\.example\\.com"] + +Because our regular expression is specified inside a string, the backslash characters must be escaped. The rules for which constructs can be used in the regular expression depend on which database is in use, so +common features should be used for interoperability. The regular expression operator can be used on every field of resources except for `parameters` and `exported`. + +## Facts walkthrough + +In addition to resources, we can also query for facts. This looks similar, though the available fields and operators are a bit different. Some things are the same, though. For instance, suppose you want all the +facts for a certain node: + + ["=", "certname", "foo.example.com"] + +This gives results that look something like this: + + [ { + "certname" : "foo.example.com", + "name" : "architecture", + "value" : "amd64", + "environment" : "production" + }, { + "certname" : "foo.example.com", + "name" : "fqdn", + "value" : "foo.example.com", + "environment" : "production" + }, { + "certname" : "foo.example.com", + "name" : "hostname", + "value" : "foo", + "environment" : "production" + }, { + "certname" : "foo.example.com", + "name" : "ipaddress", + "value" : "192.168.100.102", + "environment" : "production" + }, { + "certname" : "foo.example.com", + "name" : "kernel", + "value" : "Linux", + "environment" : "production" + }, { + "certname" : "foo.example.com", + "name" : "kernelversion", + "value" : "2.6.32", + "environment" : "production" + } ] + +### Fact attributes + +In the last query, we saw that a "fact" consists of a "certname", a "name", and a "value". As you might expect, we can query using "name" or "value". + + ["and", + ["=", "name", "operatingsystem"], + ["=", "value", "Debian"]] + +This will find all the "operatingsystem = Debian" facts, and their corresponding nodes. As you see, "and" is supported for facts, as are "or" and "not". + +### Fact operators + +As with resources, facts also support the `~` regular expression match operator for all their fields. In addition, numeric comparisons are supported for fact values: + + ["and", + ["=", "name", "uptime_seconds"], + [">=", "value", 100000], + ["<", "value", 1000000]] + +This will find nodes for which the "uptime_seconds" fact is in the range 100000 to 1000000. Numeric comparisons will _always be false_ for fact values which are not numeric. Importantly, version numbers such as +2.6.12 are not numeric, and numeric comparison operators can't be used with them at this time. + +## Nodes walkthrough + +We can also query for nodes. Again, this is similar to resource and fact queries: + + ["=", "certname", "foo.example.com"] + +The result of this query is: + + { + "deactivated" : null, + "facts_environment" : "production", + "report_environment" : "production", + "catalog_environment" : "production", + "facts_timestamp" : "2015-06-22T17:25:11.886Z", + "expired" : null, + "report_timestamp" : "2015-06-22T17:25:07.484Z", + "certname" : "foo.example.com", + "catalog_timestamp" : "2015-06-22T17:25:12.023Z" + } + +This will return an object containing the certname "foo.example.com", as well as some metadata detailing deactivation status and the most recent fact, report, and catalog updates from that node. + +### Querying on facts + +Nodes can also be queried based on their facts, using the same operators as for fact queries: + + ["and", + ["=", ["fact", "operatingsystem"], "Debian"], + ["<", ["fact", "uptime_seconds"], 10000]] + +This will return Debian nodes with "uptime_seconds" less than 10,000. + +## Subquery walkthrough + +The queries we've looked at so far are quite powerful and useful, but what if your query needs to consider both resources _and_ facts? + +For instance, suppose you're configuring a load balancer, and need the IP addresses of your Apache servers. You could find those servers by using this resource query: + + ["and", + ["=", "type", "Class"], + ["=", "title", "Apache"]] + +This will find all the `Class[Apache]` resources, which each knows the certname of the node it came from. Then you could put all those certnames into a fact query: + + ["and", + ["=", "name", "ipaddress"], + ["or", + ["=", "certname", "a.example.com"], + ["=", "certname", "b.example.com"], + ["=", "certname", "c.example.com"], + ["=", "certname", "d.example.com"], + ["=", "certname", "e.example.com"]]] + +But this query is lengthy, and it requires some logic to assemble and run the second query. There has to be a better way! What if we could find the `Class[Apache]` servers and use the results of that query to +find the certname? We can, with this fact query: + + ["and", + ["=", "name", "ipaddress"], + ["in", "certname", + ["extract", "certname", ["select_resources", + ["and", + ["=", "type", "Class"], + ["=", "title", "Apache"]]]] + +This may appear a little daunting, so we'll look at it piece by piece. + +Let's start with "select_resources". This operator takes one argument, which is a resource query, and returns the results of that query in exactly the form you would expect to see them if you did a plain +resource query. + +We then use an operator called "extract" to turn our list of resources into just a list of certnames. So we now conceptually have something like: + + ["in", "certname", ["foo.example.com", "bar.example.com", "baz.example.com"]] + +The "in" operator matches facts whose "certname" is in the supplied list. (In our case, this list is generated by a subquery. To use a literal list, you must use the the "array" syntax described in the [AST +array documentation][array].) + +At this point, our query seems a lot like the one above, except we didn't have to specify exactly which certnames to use, and instead we get them in the same query. + +Similarly, there are "select*facts", "select_nodes", and "select_fact_contents" operators, which will perform subqueries against the facts, nodes, and fact-contents endpoints. Any subquery operator is usable +from any queryable endpoint. Subqueries may be nested, and multiple subqueries may be used in a single query. For more information see the [`select*` documentation][select]. diff --git a/docs/_openvoxdb_9x/api/query/v4/aggregate-event-counts.markdown b/docs/_openvoxdb_9x/api/query/v4/aggregate-event-counts.markdown new file mode 100644 index 000000000..138034c9f --- /dev/null +++ b/docs/_openvoxdb_9x/api/query/v4/aggregate-event-counts.markdown @@ -0,0 +1,128 @@ +--- +title: "Aggregate event counts endpoint" +layout: default +canonical: "/openvoxdb/latest/api/query/v4/aggregate-event-counts.html" +--- + +# Aggregate event counts endpoint + +[event-counts]: ./event-counts.html +[events]: ./events.html +[curl]: ../curl.html +[query]: query.html + +> **Experimental endpoint**: The aggregate-event-counts endpoint is designated as experimental. It may be altered or removed in a future release. + +Puppet agent nodes submit reports after their runs, and the Puppet Server forwards these to OpenVoxDB. Each report includes: + +- Data about the entire run. +- Metadata about the report. +- Many _events,_ describing what happened during the run. + +After this information is stored in OpenVoxDB, it can be queried in various ways. + +- You can query **data about the run** and **report metadata** by making an HTTP request to the [`/reports`](./reports.html) endpoint. +- You can query **data about individual events** by making an HTTP request to the [`/events`][events] endpoint. +- You can query **summaries of event data** by making an HTTP request to the [`/event-counts`][event-counts] or `aggregate-event-counts` endpoints. + +## `/pdb/query/v4/aggregate-event-counts` + +Returns aggregated count information about all of the resource events matching the given query. + +This endpoint is built entirely on the [`event-counts`][event-counts] endpoint and will aggregate those results into a single map. + +### URL parameters + +This endpoint builds on top of the [`event-counts`][event-counts] endpoint, and it uses all of the same URL parameters. The supported parameters are listed below for easy reference. + +- `summarize_by`: required. A string specifying which object types you'd like counted. Supported values are `resource`, `containing_class`, and `certname`, or any comma-separated combination thereof. + +- `query`: optional. A JSON array of query predicates in prefix form (`["", "", ""]`). This query is forwarded to the [`events`][events] endpoint - see there for additional + documentation. For general info about queries, see [our guide to query structure.][query] + +- `count_by`: optional. A string specifying what type of object is counted when building up the counts of `successes`, `failures`, `noops`, and `skips`. Supported values are `resource` (default) and `certname`. + +- `counts_filter`: optional. A JSON array of query predicates in the usual prefix form. This query is applied to the final event-counts output, but before the results are aggregated. Supported operators are + `=`, `>`, `<`, `>=`, and `<=`. Supported fields are `failures`, `successes`, `noops`, and `skips`. + +- `distinct_resources`: optional. (**Experimental: it is possible that the behavior of this parameter may change in future releases.**) This parameter is passed along to the `events` query. See the [`events` + documentation][events] for more information. + +### Query operators + +This endpoint builds on top of the [`event-counts`][event-counts] and [`events`][events] endpoints, and supports all of the [same operators.](./events.html#query-operators) + +### Query fields + +This endpoint builds on top of the [`event-counts`][event-counts] and [`events`][events] endpoints, and supports all of the [same fields.](./events.html#query-fields) + +### Response format + +The response is an array of JSON maps containing the summarize_by parameter, aggregated event-count information, and a `total` field expressing the number of event-count results that were aggregated. + + [ { + "summarize_by": "containing_class", + "successes": 2, + "failures": 0, + "noops": 0, + "skips": 1, + "total": 3 + } ] + +#### Puppet Enterprise + +In PE, the `successes` and `noops` counts are subdivided into intentional and corrective parts. Events are mapped to the corresponding counts based on the value of `corrective_change` flag. + + [ { + "summarize_by": "containing_class", + "intentional_successes": 2, + "corrective_successes": 0, + "failures": 0, + "intentional_noops": 0, + "corrective_noops": 0, + "skips": 1, + "total": 3 + } ] + +`intentional_successes`, `corrective_successes`, `intentional_noops`, and `corrective_noops` fields can be used in `counts_filter` too. + +### Examples + +You can use [`curl`][curl] to query information about aggregated resource event counts: + + curl -G 'http://localhost:8080/pdb/query/v4/aggregate-event-counts' \ + --data-urlencode 'query=["=", "certname", "foo.local"]' \ + --data-urlencode 'summarize_by=containing_class' + + [ { + "summarize_by" : "containing_class", + "successes" : 2, + "failures" : 0, + "noops" : 0, + "skips" : 0, + "total" : 2 + } ] + + curl -G 'http://localhost:8080/pdb/query/v4/aggregate-event-counts' \ + -d 'query=["=","certname","foo.local"]' \ + -d 'summarize_by=containing_class,certname' + + [ { + "summarize_by" : "containing_class", + "successes" : 2, + "failures" : 0, + "noops" : 0, + "skips" : 0, + "total" : 2 + }, { + "summarize_by" : "certname", + "successes" : 1, + "failures" : 0, + "noops" : 0, + "skips" : 0, + "total" : 1 + } ] + +## No paging + +This endpoint does not support paging options, and results are unordered. diff --git a/docs/_openvoxdb_9x/api/query/v4/ast.markdown b/docs/_openvoxdb_9x/api/query/v4/ast.markdown new file mode 100644 index 000000000..a3a15ab31 --- /dev/null +++ b/docs/_openvoxdb_9x/api/query/v4/ast.markdown @@ -0,0 +1,875 @@ +--- +title: "AST query language" +layout: default +canonical: "/openvoxdb/latest/api/query/v4/ast.html" +--- + +# AST query language + +[root]: ./overview.html +[catalogs]: ./catalogs.html +[contact]: ../../../pdb_support_guide.html#contact-us +[edges]: ./edges.html +[environments]: ./environments.html +[events]: ./events.html +[facts]: ./facts.html +[fact-contents]: ./fact-contents.html +[fact-paths]: ./fact-paths.html +[inventory]: ./inventory.html +[nodes]: ./nodes.html +[pg-regex]: https://www.postgresql.org/docs/current/functions-matching.html#FUNCTIONS-POSIX-REGEXP +[producers]: ./producers.html +[reports]: ./reports.html +[resources]: ./resources.html +[entities]: ./entities.html +[pql]: ./pql.html +[urlencode]: https://en.wikipedia.org/wiki/Percent-encoding +[to-char]: https://www.postgresql.org/docs/current/functions-formatting.html + +## Summary + +The AST (abstract syntax tree) query language for OpenVoxDB is a language that presents +itself as a raw AST format. It can be used to provide complex querying via REST on each of +OpenVoxDB's query [endpoints][entities]. + +This document outlines the operator syntax for this query language. + +An easier to use alternative to this query language is the [Puppet query language][pql], which is +largely based on the AST query language. + +## Query strings + +An AST query string passed to the `query` URL parameter of a REST endpoint must be a [URL-encoded][urlencode] +JSON array, which may contain scalar data types (usually strings) and additional arrays, that describes a +complex _comparison operation_ in _prefix notation_ with an **operator** first and its **arguments** following. + +That is, before being URL-encoded, all AST query strings follow this form: + +```text +[ "", "", (...""...) ] +``` + +Different operators may take different numbers (and types) of arguments. + +## Binary operators + +Each of these operators accepts two arguments: a **field** and a +**value.** These operators are **non-transitive,** which means that their syntax must always be: + +```text +["", "", ""] +``` + +The available fields for each endpoint are listed in that endpoint's documentation. + +### `=` (equality) + +**Works with:** strings, numbers, timestamps, Booleans, arrays, multi, path. + +**Matches if:** the field's actual value is exactly the same as the provided value. + +* Most fields are strings. +* Some fields are Booleans. +* Arrays match if any **one** of their elements matches. +* Path matches are a special kind of array, and must be exactly matched with this operator. + +### `>` (greater than) + +**Works with:** numbers, timestamps, multi. + +**Matches if:** the field is greater than the provided value. + +### `<` (less than) + +**Works with:** numbers, timestamps, multi. + +**Matches if:** the field is less than the provided value. + +### `>=` (greater than or equal to) + +**Works with:** numbers, timestamps, multi. + +**Matches if:** the field is greater than or equal to the provided value. + +### `<=` (less than or equal to) + +**Works with:** numbers, timestamps, multi. + +**Matches if:** the field is less than or equal to the provided value. + +### `~` (regexp match) + +**Works with:** strings, multi. + +**Matches if:** the field's actual value matches the provided regular expression. The provided value must be a regular expression represented as a JSON string: + +* The regexp **must not** be surrounded by the slash characters (`/rexegp/`) that delimit regexps in many languages. +* Every backslash character **must** be escaped with an additional backslash. Thus, a sequence like `\d` + would be represented as `\\d`, and a literal backslash (represented in a regexp as `\\`) would be + represented as a quadruple-backslash (`\\\\`). + +The following example would match if the `certname` field's actual value resembled something like `www03.example.com`: + +```text +["~", "certname", "www\\d+\\.example\\.com"] +``` + +> **Note:** Regular expression matching is performed by the database +> backend, so the available [regexp features][pg-regex] are +> determined by PostgreSQL. For best results, use the simplest and +> most common features that can accomplish your task. + +### `~>` (regexp array match) + +**Works with:** paths. + +**Matches if:** each array element, which must be a PostgreSQL regular +expression or an integer, matches each element of the path. Integers +only match array indexes, regular expressions that only contain +integer digits like `"123"` do not match array indexes, and all other +regular expressions, including something like `"[12]3"`, match both +array indexes and map keys. + +The following example would match any network interface names starting with "eth": + +```text +["~>", "path", ["networking", "eth.*", "macaddress"]] +``` + +If you want to match any index for an array path element, you can use regular expressions, as the element acts like a string: + +```text +["~>", "path", [, ".*"]] +``` + +> Limitations: with the current implementation an anchored expression +> like `"^sda.*"` may never match an array element. Currently +> those expressions will match for queries against the +> [fact-contents][fact-contents], but for now, that should not be +> considered reliable across OpenVoxDB upgrades. + +### `null?` (is null) + +**Works with:** fields that may be null. + +**Matches if:** the field's value is null (when second argument is `true`) or the field is **not** null, or has a real value (when second argument is `false`). + +The following example would return events that do not have an associated line number: + +```text +["null?", "line", true] +``` + +Similarly, the below query would return events that do have a specified line number: + +```text +["null?", "line", false] +``` + +## Boolean operators + +Every argument of these operators should be a **complete query string** in its own right. These operators are **transitive:** the order of their arguments does not matter. + +### `and` + +**Matches if:** **all** of its arguments would match. Accepts any number of query strings as its arguments. + +### `or` + +**Matches if:** **at least one** of its arguments would match. Accepts any number of query strings as its arguments. + +### `not` + +**Matches if:** its argument **would not** match. Accepts a **single** query string as its argument. + +## Projection operators + +### `extract` + +To reduce the keypairs returned for each result in the response, you can use **extract**: + +```text +["extract", ["hash", "certname", "transaction_uuid"], + ["=", "certname", "foo.com"]] +``` + +When only extracting a single column, the `[]` are optional: + +```text +["extract", "transaction_uuid", + ["=", "certname", "foo.com"]] +``` + +When applying an aggregate function over a `group_by` clause, an extract +statement takes the form: + +```text +["extract", [["function", "count"], "status"], + ["=", "certname", "foo.com"], + ["group_by", "status"]] +``` + +Extract can also be used with a standalone function application: + +```text +["extract", [["function", "count"]], ["~", "certname", ".\*.com"]] +``` + +or + +```text +["extract", [["function", "count"]]] +``` + +#### Extracting a subtree + +The JSON fields that support dot notation for hash descendance also support +dot notation for extracting a subtree. See the Dot notation section below +for more information. + +```text +["extract", ["facts.os.family"]] +``` + +### `function` + +The **function** operator is used to call a function on the result of a +subquery. Supported functions are described below. + +#### `avg`, `sum`, `min`, `max` + +These functions operate on any numeric column and they take the column +name as an argument, as in the examples above. + +#### `count` + +The `count` function can be used with or without a column. When no column is +supplied, it will return the number of results in the associated subquery. +Using the function with a column will return the number of results where the +specified column is not null. + +#### `to_string` + +The `to_string` function operates on timestamps and integers, allowing them to +be formatted in a user-defined manner before being returned from puppetdb. +Available formats are the same as those documented for [PostgreSQL's `to_char` +function][to-char]. For instance, to get the full lower case month name of the +`producer_timestamp`, you can query the reports endpoint with: + +```text +["extract", [["function", "to_string", "producer_timestamp", "month"]]] +``` + +To get the last 2 digits of the year a report was submitted from the Puppet Server: + +```text +["extract", [["function", "to_string", "producer_timestamp", "YY"]]]] +``` + +To get the uptime_seconds fact's value as a string, the following query can be used on +facts or fact-contents endpoint: + +```text +["extract", [["function", "to_string", "value", "999999999"]], ["=","name", "uptime_seconds"]] +``` + +Please note that in order for `to_string` function to work with integer values, a mask +must be provided. For more information about masks and how to provide them, please read +the documentation for [PostgreSQL's `to_char`function][to-char]. + +### `group_by` + +The **group_by** operator must be applied as the last argument of an extract, +and takes one or more column names as arguments. For instance, to get event +status counts for active certname by status, you can query the events endpoint +with: + +```text +["extract", [["function", "count"], "status", "certname"], + ["group_by", "status", "certname"]] +``` + +To get the average uptime for your nodes: + +```text +["extract", [["function", "avg", "value"]], ["=", "name", "uptime_seconds"]] +``` + +## Dot notation + +_Note_: Dot notation for hash descendence is under development. Currently it has +full support on the `facts` and `trusted` response keys of the `inventory` +endpoint, and partial support on the `parameters` column of the resources +endpoint. It may be expanded to other endpoints in the future based on demand. + +Certain types of JSON data returned by OpenVoxDB can be queried in a structured +way using `dot notation`. The rules for dot notation are: + +* Hash descendence is represented by a period-separated sequence of key names +* Array indexing (`inventory` only) is represented with brackets (`[]`) on the + end of a key. +* Regular expression matching ([inventory](./inventory.html) only) is + represented with the `match` operator, but note that [`match` in its + current form has been deprecated](#dotted-field-syntax), and is + likely to be removed or altered in a backward-incompatible way in a + future release. + +For example, given the inventory response + +```json +{ + "certname" : "mbp.local", + "timestamp" : "2016-07-11T20:02:33.190Z", + "environment" : "production", + "facts" : { + "kernel" : "Darwin", + "operatingsystem" : "Darwin", + "macaddress_p2p0" : "0e:15:c2:d6:f8:4e", + "system_uptime" : { + "days" : 0, + "hours" : 1, + "uptime" : "1:52 hours", + "seconds" : 6733 + }, + "macaddress_awdl0" : "6e:31:ef:e6:36:54", + "processors": { + "models": [ + "Intel(R) Core(TM) i7-4790 CPU @ 3.60GHz", + "Intel(R) Core(TM) i7-4790 CPU @ 3.60GHz", + "Intel(R) Core(TM) i7-4790 CPU @ 3.60GHz", + "Intel(R) Core(TM) i7-4790 CPU @ 3.60GHz"], + "count": 4, + "physicalcount": 1 + }, + ... + }, + "trusted" : { + "domain" : "local", + "certname" : "mbp.local", + "hostname" : "mbp", + "extensions" : { }, + "authenticated" : "remote" + } +} +``` + +valid queries would include + +* `["=", "facts.kernel", "Darwin"]` + +* `["=", "facts.system_uptime.days", 0]` + +* `[">", "facts.system_uptime.hours", 0]` + +* `["~", "facts.processors.models[0]", "Intel.*"]` + +### Dotted Projections + +Dot notation is also supported for extracting a subtree of JSON fields. +For example you can query the inventory endpoint with + +```text +["extract", ["trusted.certname", "facts.system_uptime"]] +``` + +To get a response with only the elements you've asked for + +```json +{ + "trusted.certname": "mbp.local", + "facts.system_uptime.uptime": { + "days" : 0, + "hours" : 1, + "uptime" : "1:52 hours", + "seconds" : 6733 + } +} +``` + +### Dotted field syntax + +A dotted field, which repseents a path into a JSON tree is made up of +components separated by dots (`.`), for example `facts.kernel`. Any +path component can be double-quoted, for example `facts."x.y".z`, in +which case the name will include all of the characters after the first +double-quote, and before the next double-quote that is itself not +preceded by a backslash and is followed by either a dot, or the end of +the field. So the previous example `facts."x.y".z` represents the +three components, `facts`, `x.y`, and `z`. In AST queries, any +double-quotes will have to be properly JSON escaped. So in an +`extract` the path `x."y.z"` becomes `[extract "x.\"y.z\"", ...]`. + +There is currently no way to represent a field component that contains +a dot and ends in a backslash. For example, a fact named `x.y\` must be +quoted, given the dot, but as just mentioned, quoted fields cannot end +in a backslash. + +> **Note:** the `match()` operator described here is deprecated and is +> likely to be retired or altered in a backward-incompatible way in a +> future release. + +In some cases (e.g. [inventory endpoint][inventory]) dotted fields +can also contain a `match()` component, for example +`facts.partitions.match("sd.*")` The match pattern must be a +[PostgreSQL regular expression][pg-regex], and must begin with +`match`, open paren, double quote, and it will end at the next double +quote, close paren that is not preceded by a backslash and is followed +by either a dot, or the end of the field. The regex then, has +essentially the same syntax as a double quoted field. And similarly, +there is currently no way to specify a match regular expression that +ends in a backslash. + +With the current implementation, the `match()` component's behavior is +not well defined, likley to be surprising, and likely to change in the +future, so we recommend avoiding it for now, but please do +[contact us][contact] if you are currently using it, or would like +to use an operator with better semantics, so we can incorporate that +information into future plans. + +As an example of the potentially surprising behavior, the appearance +of any `match()` operator in a dotted field can cause the entire +field, not just the `match()` segment, to be handled as a regular +expression in an awkward manner. + +## Context operators + +_Note:_ Setting the context at the top of the query is only supported on the +[root][root] endpoint. + +Setting context in a query allows you to choose the entity you are querying +on. This augments the endpoint support we have today, whereby the endpoint +decides the context. For example, `/pdb/query/v4/nodes` sets the context of the query +to `nodes`. + +### `from` + +The `from` operator allows you to choose the [entity][entities] that you want to query and +provide optional query and paging clauses to filter those results. This operator can +be used at the top-level context of a query: + +```text +["from", "nodes", ["=", "certname", "myserver"]] +``` + +The `from` operator can also be used in a subquery for setting the context when +using the [`in` operator](#subquery-operators). + +When querying a particular endpoint, such as `/pdb/query/v4/nodes`, the endpoint provides +the context for the query. Querying the [root] endpoint requires specifying a +context explicitly. + +## Paging operators (`limit`, `offset`, `order_by`) + +OpenVoxDB allows specification of paging clauses within a "from" clause in a +query or subquery. The `limit` and `offset` operators both accept an +integer-valued argument, and `order_by` accepts a vector of either column names +or vector pairs containing a column name and an ordering of "asc" or "desc". +For example, + +```text +["limit", 1] + +["offset", 1] + +["order_by", ["certname"]] + +["order_by", ["certname", ["producer_timestamp", "desc"]]] +``` + +When no ordering is explicitly specified, as in the case of "certname" in the +example above, ascending order is assumed. Here are a few examples of queries +using paging operators: + +Return the most recent ten reports for a certname: + +```text +["from", "reports", + ["=", "certname", "myserver"], + ["order_by", [["producer_timestamp", "desc"]]], + ["limit", 10]] +``` + +Return the next page of ten reports: + +```text +["from", "reports", + ["=", "certname", "myserver"], + ["order_by", [["receive_time", "desc"]]], + ["limit", 10], + ["offset", 10]] +``` + +Return the most recent ten reports for any certname: + +```text +["from", "reports", + ["order_by", [["producer_timestamp", "desc"]]], + ["limit", 10]] +``` + +Return the nodes represented in the ten most recent reports: + +```text +["from", "nodes", + ["in", "certname", + ["from", "reports", + ["extract", "certname"], + ["limit", 10], + ["order_by", [["certname", "desc"]]]]]] +``` + +The order in which paging operators are supplied does not matter. + +## Subquery operators + +Subqueries allow you to correlate data from multiple sources or multiple +rows. For instance, a query such as "fetch the IP addresses of all nodes with +`Class[Apache]`" would have to use both facts and resources to return a list of facts. + +There are two forms of subqueries, implicit and explicit, and both forms work the +same under the hood. Note, however, that the implicit form only requires you to specify the related entity, while the explicit form requires you to be specify exactly how +data should be joined during the subquery. + +### `subquery` (implicit subqueries) + +Implicit queries work like most operators, and simply require you to specify the +related entity and the query to use: + +```text +["subquery", "", ] +``` + +The [``][entities] is the particular entity you are subquerying on, however not +all entities are implicitly relatable to all other entities, as not every relationship makes sense. +Consult the documentation for the chosen [``][entities] for details on what +implicit relationships are supported. + +In OpenVoxDB, we keep a map of how different entities relate to each +other, and therefore no data beyond the entity is needed in this case. This is +different from explicit subqueries, where you must specify how +two entities are related. Implicit subqueries can be used to join any two +entities that have a `certname` field. Additional relationships are described +in the endpoint-specific documentation as applicable. + +#### Implicit subquery examples + +A query string like the following on the [`nodes`][nodes] endpoint will return the list +of all nodes with the `Package[Tomcat]` resource in their catalog, and a certname starting +with `web1`: + +```text +["and", + ["~", "certname", "^web1"], + ["subquery", "resources", + ["and", + ["=", "type", "Package"], + ["=", "title", "Tomcat"]]]] +``` + +If you want to display the entire `networking` fact, and the host's interface uses a certain mac address, +you can do the following on the [`facts`][facts] endpoint: + +```text +["and", + ["=", "name", "networking"], + ["subquery", "fact_contents", + ["and", + ["~>", "path", ["networking", ".*", "macaddress", ".*"]], + ["=", "value", "aa:bb:cc:dd:ee:00"]]]] +``` + +### Explicit subqueries + +While implicit subqueries can make your syntax succinct, not all relationships are +mapped internally. For these more advanced subqueries, you need to specify exactly the fields that +a subquery should join on. This is where an explicit subquery can be useful. + +Explicit subqueries are unlike the other operators listed above. They always appear +together in one of the following forms: + +```text +["in", [""], ["extract", [""], ] ] +``` + +The second new methodology uses `from` to set the context, and now looks like this: + +```text +["in", [""], ["from", , ["extract", [""], ] ] ] +``` + +That is: + +* The `in` operator results in a complete query string. The `extract` operator and the subqueries do not. +* An `in` statement **must** contain one or more fields and an `extract` statement. +* An `extract` statement **must** contain one or more fields and a subquery statement. + +These statements work together as follows (working "outward" and starting with the subquery): + +* The subquery collects a group of OpenVoxDB objects (specifically, a group of [resources][resources], [facts][facts], [fact-contents][fact-contents], or [nodes][nodes]). Each of these objects has many **fields.** +* The `extract` statement collects the value of one or more **fields** across every object returned by the subquery. +* The `in` statement **matches** if its field values are present in the list returned by the `extract` statement. + +* **Subquery:** Every resource whose type is "Class" and title is "Apache." (Note that all resource + objects have a `certname` field, among other fields.) +* **Extract:** Every `certname` field from the results of the subquery. +* **In:** Matches if the `certname` field is present in the list from the `extract` statement. + +The complete `in` statement described above would match any object that shares a `certname` with a +node that has `Class[Apache]`. This could be combined with a Boolean operator to get a specific fact +from every node that matches the `in` statement. + +#### `in` + +An `in` statement constitutes a full query string, which can be used alone or as an argument for a [Boolean operator](#boolean-operators). + +"In" statements are **non-transitive** and take two arguments: + +* The first argument **must** consist of one or more **fields** for the endpoint + or entity **being queried.**. This is a string or vector of strings. +* The second argument **must** be either: + * an **`extract` statement,** which acts as a list of fields to extract during + the subquery for matching against the **fields** in the `in` clause. + * a **`from` statement,** which sets the context, and allows for an extract + statement to be provided. + * an **`array` statement,** which acts as a list of values to match against the + **field** in the `in` clause. + +**Matches if:** the field values are included in the list of values created by the `extract` or `from` statement. + +##### `array` + +An `in` statement also accepts an `array` statement as a second argument. + +"Array" statements take a single vector argument of values to match the first +argument of `in` against. + +The following query filters for the nodes, `foo.local`, `bar.local`, and +`baz.local`: + +```text +["in", "certname", + ["array", + ["foo.local", + "bar.local", + "baz.local"]]] +``` + +which is equivalent to the following query: + +```text +["or", + ["=","certname","foo.local"], + ["=","certname","bar.local"], + ["=","certname","baz.local"]] +``` + +The `in`-`array` operators support much of the same syntax as the `=` operator. +For example, the following query on the `/nodes` endpoint is valid: + +```text +["in", ["fact", "uptime_seconds"], + ["array", + [20000.0, + 150.0, + 30000.0]]] +``` + +#### `from` + +This statement works like the top-level [`from`](#context-operators) operator, +and expects an [entity][entities] as the first argument and an optional query in +the second argument. However, when used within an `in` clause, an `extract` +statement is expected to choose the fields: + +```text +["in", "certname", + ["from", "facts", + ["extract", "certname", + []]]] +``` + +#### `extract` + +"Extract" statements are **non-transitive** and take two arguments: + +* The first argument **must** be a valid set of **fields** for the endpoint + **being subqueried** (see second argument). This is a string or vector of + strings. +* The second argument: + * **must** contain a **subquery statement** + * or when used with the new `from` operator, **may** contain an optional query. + +As the second argument of an `in` statement, an `extract` statement acts as a +list of possible values. This list is compiled by extracting the value of the +requested field from every result of the subquery. + +#### `select_` subquery statements + +A subquery statement **does not** constitute a full query string. It may only be used as the second argument of an `extract` statement. + +Subquery statements are **non-transitive** and take two arguments: + +* The first argument **must** be the **name** of one of the available subqueries (listed below). +* The second argument **must** be a **full query string** that makes sense for the endpoint being subqueried. + +As the second argument of an `extract` statement, a subquery statement acts as a collection of +OpenVoxDB objects. Each of the objects returned by the subquery has many fields; the `extract` +statement takes the value of one field from each of those objects, and passes that list of values to +the `in` statement that contains it. + +Each subquery acts as a normal query to one of the OpenVoxDB endpoints. For info on constructing useful queries, see the docs page for the endpoint matching the subquery: + +* [`select_catalogs`][catalogs] +* [`select_edges`][edges] +* [`select_environments`][environments] +* [`select_events`][events] +* [`select_facts`][facts] +* [`select_fact_contents`][fact-contents] +* [`select_fact_paths`][fact-paths] +* [`select_nodes`][nodes] +* [`select_producers`][producers] +* [`select_reports`][reports] +* [`select_resources`][resources] + +#### Explicit subquery examples + +This query string queries the `/facts` endpoint for the IP address of +all nodes with `Class[Apache]`: + +```text +["and", + ["=", "name", "ipaddress"], + ["in", "certname", + ["extract", "certname", + ["select_resources", + ["and", + ["=", "type", "Class"], + ["=", "title", "Apache"]]]]]] +``` + +This query string queries the `/nodes` endpoint for all nodes with `Class[Apache]`: + +```text +["in", "certname", + ["extract", "certname", + ["select_resources", + ["and", + ["=", "type", "Class"], + ["=", "title", "Apache"]]]]] +``` + +This query string queries the `/facts` endpoint for the IP address of +all Debian nodes. + +```text +["and", + ["=", "name", "ipaddress"], + ["in", "certname", + ["extract", "certname", + ["select_facts", + ["and", + ["=", "name", "operatingsystem"], + ["=", "value", "Debian"]]]]]] +``` + +This query string queries the `/facts` endpoint for uptime_hours of all nodes with +facts_environment `production`: + +```text +["and", + ["=", "name", "uptime_hours"], + ["in", "certname", + ["extract", "certname", + ["select_nodes", + ["=", "facts_environment", "production"]]]]] +``` + +To find node information for a host that has a macaddress of `aa:bb:cc:dd:ee:00` as +its first macaddress on the interface `eth0`, you could use this query on '/nodes': + +```text +["in", "certname", + ["extract", "certname", + ["select_fact_contents", + ["and", + ["=", "path", ["networking", "eth0", "macaddress", 0]], + ["=", "value", "aa:bb:cc:dd:ee:00"]]]]] +``` + +To exhibit a subquery using multiple fields, you could use the following +on '/facts' to list all top-level facts containing fact contents with paths +starting with "up" and value less than 100: + +```text +["in", ["certname", "name"], + ["extract", ["certname", "name"], + ["select_fact_contents", + ["and", + ["~>", "path", ["up.*"]], + ["<", "value", 100]]]]] +``` + +Queries are restricted to active nodes by default; to make this explicit, the +special "node_state" field may be queried using the values "active", "inactive", +or "any". For example, to list all catalogs from inactive nodes, use this on the +/catalogs endpoint: + +```text +["=", "node_state", "inactive"] +``` + +This expands internally into comparisons against each node's deactivation and +expiration time; a node is consider inactive if either field is set. + +#### Explicit subquery examples (with the `from` operator) + +Additions to the query language in support of PQL introduced new ways to +express subqueries using the `from` operator. For example, a query such as this: + +```text +["and", + ["=", "name", "ipaddress"], + ["in", "certname", + ["extract", "certname", + ["select_resources", + ["and", + ["=", "type", "Class"], + ["=", "title", "Apache"]]]]]] +``` + +will now look like this: + +```text +["and", + ["=", "name", "ipaddress"], + ["in", "certname", + ["from", "resources", + ["extract", "certname", + ["and", + ["=", "type", "Class"], + ["=", "title", "Apache"]]]]]] +``` + +Executing this query on the `/facts` endpoint would filter for `uptime_hours` for all nodes with +`facts_environment` set to `production`: + +```text +["and", + ["=", "name", "uptime_hours"], + ["in", "certname", + ["from", "nodes", + ["extract", "certname", + ["=", "facts_environment", "production"]]]]] +``` + +To find node information for a host that has a macaddress of `aa:bb:cc:dd:ee:00` as +its first macaddress on the interface `eth0`, you could use this query on `/nodes`: + +```text +["in", "certname", + ["from", "fact_contents", + ["extract", "certname", + ["and", + ["=", "path", ["networking", "eth0", "macaddress", 0]], + ["=", "value", "aa:bb:cc:dd:ee:00"]]]]] +``` diff --git a/docs/_openvoxdb_9x/api/query/v4/catalog_input_contents.markdown b/docs/_openvoxdb_9x/api/query/v4/catalog_input_contents.markdown new file mode 100644 index 000000000..9d3dd5d63 --- /dev/null +++ b/docs/_openvoxdb_9x/api/query/v4/catalog_input_contents.markdown @@ -0,0 +1,111 @@ +--- +title: "OpenVoxDB: Catalog input contents endpoint" +layout: default +canonical: "/openvoxdb/latest/api/query/v4/catalog_input_contents.html" +--- + +# Catalog input contents endpoint + +[paging]: ./paging.html +[query]: query.html +[subqueries]: ./ast.html#subquery-operators +[ast]: ./ast.html +[nodes]: ./nodes.html + +> **Experimental Endpoint**: This endpoint is designated as +> experimental. It may be altered or removed in a future release. + +## `/pdb/query/v4/catalog-input-contents` + +The `/catalog-input-contents` endpoint returns a JSON array containing +all of the most recent catalog inputs for the catalogs in your +infrastructure. + +### URL parameters + +* `query`: optional. A JSON array containing the query in prefix + notation (`["", "", ""]`). See the sections + below for the supported operators and fields. For general info about + queries, see [our guide to query structure.][query] + +If a query parameter is not provided, all catalog inputs will be returned. + +### Query operators + +See [the AST query language page][ast]. + +### Query fields + +* `certname` (string): the certname associated with the input. +* `producer_timestamp` (string): a string representing the time at + which the `replace catalog inputs` command containing the input was + submitted from the Puppet Server. +* `catalog_uuid` (string): the unique ID of the catalog to which the + input corresponds. +* `type` (string): the input type (ie. `"hiera"`). +* `name` (string): the unique name of the input for the type (for a + hiera key it would just be its full key, + `"openvoxdb::globals::version"`. + +### Subquery Relationships + +Here is a list of related entities that can be used to constrain the result set +using implicit subqueries. For more information consult the documentation for +[subqueries][subqueries]. + +* [`nodes`][nodes]: Node for a catalog. + +### Response format + +Successful responses will be in `application/json`. + +The result will be a JSON array with one entry per certname. Each entry is of +the form: + + { + "certname" : , + "producer_timestamp":