diff --git a/README.md b/README.md index 87c3de66..f95e5e20 100644 --- a/README.md +++ b/README.md @@ -311,7 +311,7 @@ For further Information see the [license file](https://misp.github.io/misp-modul * [Cortex XDR IOC Publish](https://misp.github.io/misp-modules/action_mod/#cortex-xdr-ioc-publish) - Upload missing Cortex XDR IOC rules from IDS-marked MISP event and object attributes. * [Cortex XSOAR Incident Create](https://misp.github.io/misp-modules/action_mod/#cortex-xsoar-incident-create) - Create a Cortex XSOAR 8 incident with a MISP event reference and selected-data counts. * [CrowdStrike IOC Publish](https://misp.github.io/misp-modules/action_mod/#crowdstrike-ioc-publish) - Create and renew module-owned Falcon custom IOCs from MISP event and object attributes. -* [Export to Sentinel or Defender](https://misp.github.io/misp-modules/action_mod/#export-to-sentinel-or-defender) - Export indicators to Microsoft Sentinel or Microsoft Defender. Requires an existing installation of MISP2Sentinel or MISP2Defender. +* [Export to Sentinel or Defender](https://misp.github.io/misp-modules/action_mod/#export-to-sentinel-or-defender) - Export to Sentinel or Defender using the existing script wrapper, or directly to Sentinel's STIX API. * [IRC](https://misp.github.io/misp-modules/action_mod/#irc) - Send a message to an IRC channel. * [Matrix](https://misp.github.io/misp-modules/action_mod/#matrix) - Send a message to an unencrypted Matrix room. * [Mattermost](https://misp.github.io/misp-modules/action_mod/#mattermost) - Simplistic module to send message to a Mattermost channel. @@ -319,6 +319,7 @@ For further Information see the [license file](https://misp.github.io/misp-modul * [Nextcloud talk](https://misp.github.io/misp-modules/action_mod/#nextcloud-talk) - Simplistic module to send a message to a Nextcloud talk conversation. * [OSQuery Pack Publish](https://misp.github.io/misp-modules/action_mod/#osquery-pack-publish) - Publish a MISP query pack to a configured local osquery pack directory. * [QRadar Reference Set Publish](https://misp.github.io/misp-modules/action_mod/#qradar-reference-set-publish) - Publish typed MISP indicators to an existing QRadar reference set and explicit domain. +* [Microsoft Sentinel Publish](https://misp.github.io/misp-modules/action_mod/#microsoft-sentinel-publish) - Publish MISP indicators directly to Microsoft Sentinel's STIX upload API with per-object results. * [SentinelOne IOC Publish](https://misp.github.io/misp-modules/action_mod/#sentinelone-ioc-publish) - Publish MISP event and object indicators to SentinelOne Threat Intelligence with explicit scope. * [Shuffle Workflow Execute](https://misp.github.io/misp-modules/action_mod/#shuffle-workflow-execute) - Start a Shuffle workflow with selected MISP event and object attributes as its JSON execution argument. * [Signal](https://misp.github.io/misp-modules/action_mod/#signal) - Send a message to Signal recipients or a group through signal-cli-rest-api. diff --git a/documentation/README.md b/documentation/README.md index 84bdfe35..d4430cf5 100644 --- a/documentation/README.md +++ b/documentation/README.md @@ -5455,6 +5455,33 @@ Create and renew module-owned Falcon custom IOCs from MISP event and object attr ----- +#### [Export to Sentinel or Defender](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/export_sentinel.py) + +Export to Sentinel or Defender using the existing script wrapper, or directly to Sentinel's STIX API. +[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/export_sentinel.py)] + +- **features**: +>Preserves wrapper mode and its boolean success response by default. Direct mode publishes supplied IDS attributes to Microsoft Sentinel without an external script, using the same implementation as sentinel_publish. Filtered/extracted payloads are rejected because neither path applies workflow filters. See the [usage guide](https://github.com/MISP/misp-modules/blob/main/documentation/action_mod/sentinel_publish.md). + +- **config**: +>{'params': {'mode': {'type': 'string', 'description': 'wrapper (existing MISP2Sentinel/MISP2Defender) or direct (Sentinel STIX API)', 'value': 'wrapper'}, 'misp2_venv': {'type': 'string', 'description': 'MISP2Sentinel or MISP2Defender Python virtual environment', 'value': '/var/www/MISP/misp-custom/misp2sentinel/venv/'}, 'misp2_script': {'type': 'string', 'description': 'MISP2Sentinel or MISP2Defender script path', 'value': '/var/www/MISP/misp-custom/misp2sentinel/script.py'}, 'workspace_id': {'type': 'string', 'description': 'Required: target Log Analytics workspace UUID'}, 'source': {'type': 'string', 'description': 'Stable source-system namespace', 'value': 'misp-modules'}, 'expiration_days': {'type': 'string', 'description': 'Days until indicator expiry (1-365)', 'value': '30'}, 'confidence': {'type': 'string', 'description': 'STIX confidence (0-100)', 'value': '50'}, 'dry_run': {'type': 'string', 'description': 'true previews STIX payloads without network access', 'value': 'false'}}, 'blocking': False, 'support_filters': False, 'expect_misp_core_format': True} + +- **input**: +>An event UUID in wrapper mode, or supplied MISP events/object attributes in direct mode. + +- **output**: +>Boolean wrapper success, or a direct publishing summary with per-indicator outcomes. + +- **references**: +> - https://learn.microsoft.com/en-us/azure/sentinel/stix-objects-api +> - https://github.com/cudeso/misp2sentinel + +- **requirements**: +> - requests +> - MISP2Sentinel/MISP2Defender for wrapper mode, or misp-stix (all extra) and Entra credentials for direct mode + +----- + #### [IRC](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/irc.py) Send a message to an IRC channel. @@ -5608,6 +5635,33 @@ Publish typed MISP indicators to an existing QRadar reference set and explicit d ----- +#### [Microsoft Sentinel Publish](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/sentinel_publish.py) + +Publish MISP indicators directly to Microsoft Sentinel's STIX upload API with per-object results. +[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/sentinel_publish.py)] + +- **features**: +>Uses misp-stix to convert IDS-marked event and object attributes into STIX 2.1 indicator patterns, preserving composite constraints, and uploads batches of up to 100 objects. Supports dry runs, explicit workspace selection, expiry and checked per-object results. Entra credentials come from the module server environment. No external script installation is required. See the [usage guide](https://github.com/MISP/misp-modules/blob/main/documentation/action_mod/sentinel_publish.md). + +- **config**: +>{'params': {'workspace_id': {'type': 'string', 'description': 'Required: target Log Analytics workspace UUID'}, 'source': {'type': 'string', 'description': 'Stable source-system namespace', 'value': 'misp-modules'}, 'expiration_days': {'type': 'string', 'description': 'Days until indicator expiry (1-365)', 'value': '30'}, 'confidence': {'type': 'string', 'description': 'STIX confidence (0-100)', 'value': '50'}, 'dry_run': {'type': 'string', 'description': 'true previews STIX payloads without network access', 'value': 'false'}}, 'blocking': True, 'support_filters': False, 'expect_misp_core_format': True} + +- **input**: +>MISP events and object attributes, including attribute UUIDs and Unix timestamps. + +- **output**: +>Upload counts and per-indicator outcomes; errors signal incomplete publication. + +- **references**: +>https://learn.microsoft.com/en-us/azure/sentinel/stix-objects-api + +- **requirements**: +> - requests +> - misp-stix (included in the all extra) +> - Microsoft Sentinel Contributor at the target workspace + +----- + #### [SentinelOne IOC Publish](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/sentinelone_ioc_publish.py) Publish MISP event and object indicators to SentinelOne Threat Intelligence with explicit scope. diff --git a/documentation/action_mod/sentinel_publish.md b/documentation/action_mod/sentinel_publish.md new file mode 100644 index 00000000..4c1167c7 --- /dev/null +++ b/documentation/action_mod/sentinel_publish.md @@ -0,0 +1,257 @@ +# Microsoft Sentinel direct publishing + +Microsoft Sentinel indicators can now be published directly from supplied MISP +events and object attributes, without installing MISP2Sentinel or executing a +local integration script. Two action entry points share the same implementation: + +- **`sentinel_publish`** is the dedicated, blocking direct-publishing action. +- **`export_sentinel`** supports direct publishing with `mode=direct`. Its default + remains `mode=wrapper`, for existing MISP2Sentinel/MISP2Defender installations. + +Direct publishing targets Microsoft's current **STIX objects upload API**, version +`2024-02-01-preview`, in the commercial Azure cloud. Microsoft documents this API +as preview. Direct publishing requires `misp-stix`, included in the `all` extra +(`pip install 'misp-modules[all]'`), for STIX 2.1 pattern generation and validation, +and the existing `requests` dependency for HTTP. The legacy wrapper does not +require `misp-stix`. The module does not use +the legacy upload-indicators endpoint or the retired Graph TI submission path. +Direct mode publishes to Sentinel; Defender publishing remains available through +the existing wrapper mode. + +## Configure direct publishing + +1. Register a Microsoft Entra application and create a client secret. Grant that + application's service principal **Microsoft Sentinel Contributor** at the + intended Log Analytics workspace. +2. Configure `SENTINEL_TENANT_ID`, `SENTINEL_CLIENT_ID` and + `SENTINEL_CLIENT_SECRET` in the environment of the **misp-modules service**, + then restart it. Tenant/client IDs must be non-nil UUIDs. Credentials are not + accepted as workflow parameters. +3. Enable **Microsoft Sentinel Publish**, or choose **Export to Sentinel or + Defender** with `mode=direct`. Supply the full intended events and objects in + workflow data and set the target workspace UUID. +4. Preview with `dry_run=true`, inspect the STIX objects and scope, then set it to + `false` to publish. Dry runs need no credentials and make no network calls; + they cannot validate Entra roles or tenant-specific API behavior. + +The workspace ID is the **Log Analytics workspace UUID**, not its Azure resource +ID, subscription ID or display name. Endpoint URLs and token audience are fixed: + +```text +POST https://login.microsoftonline.com/{tenantId}/oauth2/v2.0/token +scope = https://management.azure.com/.default + +POST https://api.ti.sentinel.azure.com/workspaces/{workspaceId}/threat-intelligence-stix-objects:upload +api-version = 2024-02-01-preview +``` + +The Entra request uses OAuth client credentials. A fresh token is obtained once +per invocation; it must have enough remaining lifetime for the publication +budget. The token and secret are never returned or included in upload payloads. +TLS verification is enabled and redirects are disabled on both endpoints. This +implementation does not support sovereign cloud endpoints, interactive login, +certificate credentials or managed identity. + +| Direct parameter | Default | Meaning | +| --- | --- | --- | +| `workspace_id` | Required | Explicit target workspace UUID. | +| `source` | `misp-modules` | Stable `sourcesystem` label, 1–64 ASCII letters, digits, spaces or `._:/-`, beginning with a letter/digit. The API reserves `Microsoft Sentinel`. | +| `expiration_days` | `30` | Integer 1–365; set each indicator's `valid_until` from the publication time. | +| `confidence` | `50` | STIX confidence, integer 0–100. | +| `dry_run` | `false` | Return planned request bodies and outcomes without authentication or upload. | + +Boolean values accept JSON booleans, integers 0/1 and strings `true`, `false`, `1`, +`0`. Empty optional values use the documented defaults. MISP displays node +defaults as placeholders; the module also applies them when values are omitted. + +## Inputs and conversion + +`data` accepts one standard (`{"Event": {...}}`), bare or legacy event envelope, +or a list of those events. Object attributes are traversed. Only supplied +`to_ids=true`, non-deleted attributes are eligible; deleted events and objects +are excluded. The direct publisher does not fetch additional MISP data. + +Every eligible attribute needs a valid **UUID** and **Unix-seconds `timestamp`**. +These are present in normal MISP core event exports and identify a source +revision. Missing, invalid or future-dated timestamps and malformed UUIDs are +counted as malformed rather than assigned unstable fallback identities. Ensure +upstream data includes those fields when supplying custom event JSON. + +| MISP type | STIX pattern path | Conversion | +| --- | --- | --- | +| `md5`, `sha1`, `sha256`, `sha512` | `file:hashes.MD5`, `file:hashes.'SHA-1'`, `'SHA-256'`, `'SHA-512'` | Validate hexadecimal length and lowercase. | +| `domain`, `hostname` | `domain-name:value` | IDNA encode, lowercase, remove one trailing DNS dot, validate labels and the 253-character normalized name limit. | +| `ip-src`, `ip-dst` | `network-traffic:src_ref` or `network-traffic:dst_ref` type and value | Validate and canonicalize; preserve traffic direction. CIDRs and scoped IPv6 are excluded. | +| `url` | `url:value` | Require absolute HTTP(S) without credentials, whitespace or controls; retain the exact value. | +| `email`, `email-src`, `email-dst` | `email-addr:value`, `email-message:from_ref.value`, `email-message:to_refs[*].value` | Accept a bare ASCII local part and valid IDNA domain; the converter lowercases email addresses and preserves sender/recipient direction. | +| `domain\|ip` | Domain value AND resolving IP | Validate both components; publish one conjunctive indicator. | +| `filename\|md5`, `filename\|sha1`, `filename\|sha256`, `filename\|sha512`, `malware-sample` | File name AND hash | Preserve both constraints in one indicator. Filenames containing pipes are skipped as ambiguous. Binary sample data is never passed to the converter or uploaded. | + +Patterns come from the public `misp-stix` `MISPtoSTIX21Parser.parse_misp_attribute` +API, using only the normalized type/value and `to_ids=true`. Only a single STIX +indicator pattern is extracted; converter warnings, errors, unsupported output +and invalid STIX 2.1 patterns cause the attribute to be skipped. Full event bundles, +observables, identities and custom MISP objects produced by the converter are +not uploaded. Sentinel-specific identities, expiry, confidence and upload results +remain the publisher's responsibility. + +This reuses the same mappings as the STIX pattern generator/export modules and +avoids maintaining separate STIX paths, composite logic and escaping rules. +The supported input allowlist and value checks remain specific to this publisher; +installing the converter does not enable every MISP type for Sentinel uploads. + +Values are quoted and escaped by the converter. Unsupported types, including +filenames alone, port-qualified composites and other rule languages, are skipped. +Port constraints are not discarded to create broader indicators. Malformed values +are skipped with counts; malformed containers and configuration fail before +network access. Existing `file` and `domain-ip` objects are supported, so no new +MISP object templates are required. + +Exportable event, object and attribute tags are combined as STIX labels. +Local-only tags are excluded from both labels and markings, including the `local` +flag on `EventTag`/`AttributeTag` associations and flattened `Tag` records. Standard TLP +white/clear, green, amber and red tags also produce the corresponding STIX TLP +marking references; clear maps to the equivalent white definition. Other handling +tags remain labels, including TLP extensions for which this implementation has no +marking-definition mapping. The destination's treatment of markings is not an +access-control guarantee: select the workspace and input according to MISP +sharing and handling policy. + +Source event/attribute/object identifiers are retained in descriptions and result +`sources`, and attributes have MISP external references. Free-text event titles, +comments and binary sample data are not copied. Duplicate representations of the +same source revision and normalized pattern are combined within an invocation. +Different MISP attribute UUIDs remain independent indicators, even if they contain +the same observable. + +## Identity and lifecycle + +A deterministic STIX `indicator--UUID` is derived from workspace, source namespace, +attribute UUID, attribute timestamp and normalized pattern. The source revision's +timestamp supplies `created` and `valid_from`; publication time supplies `modified`. +Re-running an unchanged source revision keeps its ID and creation time while +submitting a newer modification time and expiry. Changing confidence or expiry +does not change that revision's identity. + +Compared with the earlier implementation of this PR, `misp-stix` changes patterns +for hashes, directional IP/email attributes and composites. Those changed patterns +produce new IDs. If an earlier version was deployed, its records remain until +expiry or explicit removal; the publisher does not revoke them automatically. + +Changing the attribute timestamp, UUID, normalized pattern, source or workspace +creates a different identity. This makes source revisions independent and avoids +changing the immutable creation timestamp of a reused STIX ID. Older revisions +are not automatically revoked, and separate attributes with equal observables +can coexist. Keep source labels stable and use the configured expiry or separate +Sentinel administration to withdraw old records. Deleted/non-IDS input is not a +deletion instruction. The module does not query existing indicators, claim +ownership of existing records, or guarantee exactly-once ingestion. + +## API example + +POST to the misp-modules `/query` endpoint for a preview: + +```json +{ + "module": "sentinel_publish", + "params": { + "workspace_id": "22549473-0938-4938-8a04-225494730938", + "source": "MISP production", + "expiration_days": "30", + "dry_run": "true" + }, + "data": { + "Event": { + "uuid": "f1ef0409-c80a-4012-b401-b7d1050f4ed4", + "Attribute": [{ + "uuid": "aa09819b-f2d8-4661-83ea-d4897cafebc8", + "timestamp": "1767225600", + "type": "domain", + "value": "example.org", + "to_ids": true + }] + } + } +} +``` + +To use the existing entry point, change `module` to `export_sentinel` and add +`"mode": "direct"` to `params`. Both direct entry points return +`result_key=sentinel_publish` and the same `data` summary: `success`, `dry_run`, +`workspace_id`, `uploaded`, `failed`, `planned`, `skipped` and `items`. Items include +the STIX ID, sources, status and sanitized failure information. Dry runs include +the exact planned API bodies under `payloads`. + +Successful workflow summaries are stored under +`_env.misp_module_results.sentinel_publish`. A top-level `error` signals failure; +MISP may then retain only the error, while standalone API callers receive the +full per-item summary. Prefer the dedicated **blocking** `sentinel_publish` action +when downstream workflow steps depend on complete publication. `export_sentinel` +keeps its existing non-blocking configuration for compatibility. + +### Upload acknowledgements and limits + +Microsoft documents a maximum of **100 objects per request** and **100 requests +per minute**. The module accepts at most **1,000 unique source-revision indicators** +per invocation and splits them into batches of up to 100. A local **1 MiB per-batch** +limit is checked before authentication; split large inputs or reduce tag metadata +if needed. Calls have a 15-second timeout and a 240-second publication budget. + +An HTTP 200 with an empty body confirms validation/publication of that batch. +HTTP 200 can also contain an `errors` array whose zero-based `recordIndex` values +identify failed STIX objects. The module validates those indices and maps failures +back to the matching items; other objects in that batch are counted as uploaded. +An empty error list is accepted. Unexpected response shapes, duplicate/out-of-range +indices, malformed JSON and non-200 responses fail without treating the batch as +successful. + +Unknown/unsuccessful batch outcomes stop subsequent batches, which are reported +as not attempted in their error messages. Indexed validation failures can coexist +with successful objects and do not prevent later batches from being submitted. +There are no automatic retries or rollbacks. **Publication can be partial.** Check +Sentinel before retrying an uncertain request. Counts describe upload API +acceptance, not immediate query visibility, newly created versus updated records, +analytics-rule matches or blocking enforcement. Provider error bodies and Entra +exception details are not exposed in results. + +## Existing wrapper compatibility + +`export_sentinel` keeps `mode=wrapper` as its default and continues to call the +configured Python executable with the event UUID as a single argument: + +```text +{misp2_venv}/bin/python {misp2_script} {event_uuid} +``` + +The existing `misp2_venv` and `misp2_script` parameters retain their default paths: +`/var/www/MISP/misp-custom/misp2sentinel/venv/` and +`/var/www/MISP/misp-custom/misp2sentinel/script.py`. Empty values now apply those +defaults. Paths must be absolute, the executable/script must exist, and the input +must identify one event with a valid UUID. The script's directory remains the +working directory. Successful wrapper calls retain `{"data": true}`. + +Wrapper failures now return `data=false` **and a top-level error**, so failures +are visible to MISP instead of appearing as a successfully executed action. The +timeout is 240 seconds to fit inside the module server's normal timeout. Script +stdout/stderr are suppressed rather than copied into service logs; configure the +integration script's own logging for diagnostics. Wrapper mode has no dry run. +Its installed integration still controls refetching, filters and remote API use. + +Both entry points advertise `support_filters=false` and reject `filteredItems` or +`matchingData` payloads. The old wrapper advertised filter support but never +applied it; it refetched the event by UUID. Select intended events upstream and +configure any wrapper-side selection in MISP2Sentinel/MISP2Defender itself. + +## Validation + +Tests validate STIX objects and patterns, current OAuth/upload wire contracts with +mocked HTTP transport, batching and indexed partial failures, credential handling, +source revision identities, objects and workflow discovery/query endpoints. +Wrapper tests cover the original argument/working-directory contract and execute +a harmless local script through a test virtual environment. No live Sentinel +workspace or Defender tenant was changed; verify credentials, workspace roles, +marking treatment and preview API availability in your deployment. + +References: [Microsoft STIX upload API](https://learn.microsoft.com/en-us/azure/sentinel/stix-objects-api), +[application/workspace setup](https://learn.microsoft.com/en-us/azure/sentinel/connect-threat-intelligence-upload-api), +and [MISP2Sentinel](https://github.com/cudeso/misp2sentinel). diff --git a/documentation/mkdocs/action_mod.md b/documentation/mkdocs/action_mod.md index 7926bbdf..30ba7ca1 100644 --- a/documentation/mkdocs/action_mod.md +++ b/documentation/mkdocs/action_mod.md @@ -81,6 +81,33 @@ Create and renew module-owned Falcon custom IOCs from MISP event and object attr ----- +#### [Export to Sentinel or Defender](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/export_sentinel.py) + +Export to Sentinel or Defender using the existing script wrapper, or directly to Sentinel's STIX API. +[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/export_sentinel.py)] + +- **features**: +>Preserves wrapper mode and its boolean success response by default. Direct mode publishes supplied IDS attributes to Microsoft Sentinel without an external script, using the same implementation as sentinel_publish. Filtered/extracted payloads are rejected because neither path applies workflow filters. See the [usage guide](https://github.com/MISP/misp-modules/blob/main/documentation/action_mod/sentinel_publish.md). + +- **config**: +>{'params': {'mode': {'type': 'string', 'description': 'wrapper (existing MISP2Sentinel/MISP2Defender) or direct (Sentinel STIX API)', 'value': 'wrapper'}, 'misp2_venv': {'type': 'string', 'description': 'MISP2Sentinel or MISP2Defender Python virtual environment', 'value': '/var/www/MISP/misp-custom/misp2sentinel/venv/'}, 'misp2_script': {'type': 'string', 'description': 'MISP2Sentinel or MISP2Defender script path', 'value': '/var/www/MISP/misp-custom/misp2sentinel/script.py'}, 'workspace_id': {'type': 'string', 'description': 'Required: target Log Analytics workspace UUID'}, 'source': {'type': 'string', 'description': 'Stable source-system namespace', 'value': 'misp-modules'}, 'expiration_days': {'type': 'string', 'description': 'Days until indicator expiry (1-365)', 'value': '30'}, 'confidence': {'type': 'string', 'description': 'STIX confidence (0-100)', 'value': '50'}, 'dry_run': {'type': 'string', 'description': 'true previews STIX payloads without network access', 'value': 'false'}}, 'blocking': False, 'support_filters': False, 'expect_misp_core_format': True} + +- **input**: +>An event UUID in wrapper mode, or supplied MISP events/object attributes in direct mode. + +- **output**: +>Boolean wrapper success, or a direct publishing summary with per-indicator outcomes. + +- **references**: +> - https://learn.microsoft.com/en-us/azure/sentinel/stix-objects-api +> - https://github.com/cudeso/misp2sentinel + +- **requirements**: +> - requests +> - MISP2Sentinel/MISP2Defender for wrapper mode, or misp-stix (all extra) and Entra credentials for direct mode + +----- + #### [IRC](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/irc.py) Send a message to an IRC channel. @@ -234,6 +261,33 @@ Publish typed MISP indicators to an existing QRadar reference set and explicit d ----- +#### [Microsoft Sentinel Publish](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/sentinel_publish.py) + +Publish MISP indicators directly to Microsoft Sentinel's STIX upload API with per-object results. +[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/sentinel_publish.py)] + +- **features**: +>Uses misp-stix to convert IDS-marked event and object attributes into STIX 2.1 indicator patterns, preserving composite constraints, and uploads batches of up to 100 objects. Supports dry runs, explicit workspace selection, expiry and checked per-object results. Entra credentials come from the module server environment. No external script installation is required. See the [usage guide](https://github.com/MISP/misp-modules/blob/main/documentation/action_mod/sentinel_publish.md). + +- **config**: +>{'params': {'workspace_id': {'type': 'string', 'description': 'Required: target Log Analytics workspace UUID'}, 'source': {'type': 'string', 'description': 'Stable source-system namespace', 'value': 'misp-modules'}, 'expiration_days': {'type': 'string', 'description': 'Days until indicator expiry (1-365)', 'value': '30'}, 'confidence': {'type': 'string', 'description': 'STIX confidence (0-100)', 'value': '50'}, 'dry_run': {'type': 'string', 'description': 'true previews STIX payloads without network access', 'value': 'false'}}, 'blocking': True, 'support_filters': False, 'expect_misp_core_format': True} + +- **input**: +>MISP events and object attributes, including attribute UUIDs and Unix timestamps. + +- **output**: +>Upload counts and per-indicator outcomes; errors signal incomplete publication. + +- **references**: +>https://learn.microsoft.com/en-us/azure/sentinel/stix-objects-api + +- **requirements**: +> - requests +> - misp-stix (included in the all extra) +> - Microsoft Sentinel Contributor at the target workspace + +----- + #### [SentinelOne IOC Publish](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/sentinelone_ioc_publish.py) Publish MISP event and object indicators to SentinelOne Threat Intelligence with explicit scope. diff --git a/documentation/mkdocs/index.md b/documentation/mkdocs/index.md index 67b993d4..7e53fc39 100644 --- a/documentation/mkdocs/index.md +++ b/documentation/mkdocs/index.md @@ -222,6 +222,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj * [Cortex XDR IOC Publish](https://misp.github.io/misp-modules/action_mod/#cortex-xdr-ioc-publish) - Upload missing Cortex XDR IOC rules from IDS-marked MISP event and object attributes. * [Cortex XSOAR Incident Create](https://misp.github.io/misp-modules/action_mod/#cortex-xsoar-incident-create) - Create a Cortex XSOAR 8 incident with a MISP event reference and selected-data counts. * [CrowdStrike IOC Publish](https://misp.github.io/misp-modules/action_mod/#crowdstrike-ioc-publish) - Create and renew module-owned Falcon custom IOCs from MISP event and object attributes. +* [Export to Sentinel or Defender](https://misp.github.io/misp-modules/action_mod/#export-to-sentinel-or-defender) - Export to Sentinel or Defender using the existing script wrapper, or directly to Sentinel's STIX API. * [IRC](https://misp.github.io/misp-modules/action_mod/#irc) - Send a message to an IRC channel. * [Matrix](https://misp.github.io/misp-modules/action_mod/#matrix) - Send a message to an unencrypted Matrix room. * [Mattermost](https://misp.github.io/misp-modules/action_mod/#mattermost) - Simplistic module to send message to a Mattermost channel. @@ -229,6 +230,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj * [Nextcloud talk](https://misp.github.io/misp-modules/action_mod/#nextcloud-talk) - Simplistic module to send a message to a Nextcloud talk conversation. * [OSQuery Pack Publish](https://misp.github.io/misp-modules/action_mod/#osquery-pack-publish) - Publish a MISP query pack to a configured local osquery pack directory. * [QRadar Reference Set Publish](https://misp.github.io/misp-modules/action_mod/#qradar-reference-set-publish) - Publish typed MISP indicators to an existing QRadar reference set and explicit domain. +* [Microsoft Sentinel Publish](https://misp.github.io/misp-modules/action_mod/#microsoft-sentinel-publish) - Publish MISP indicators directly to Microsoft Sentinel's STIX upload API with per-object results. * [SentinelOne IOC Publish](https://misp.github.io/misp-modules/action_mod/#sentinelone-ioc-publish) - Publish MISP event and object indicators to SentinelOne Threat Intelligence with explicit scope. * [Shuffle Workflow Execute](https://misp.github.io/misp-modules/action_mod/#shuffle-workflow-execute) - Start a Shuffle workflow with selected MISP event and object attributes as its JSON execution argument. * [Signal](https://misp.github.io/misp-modules/action_mod/#signal) - Send a message to Signal recipients or a group through signal-cli-rest-api. diff --git a/misp_modules/modules/action_mod/export_sentinel.py b/misp_modules/modules/action_mod/export_sentinel.py index d479635e..6cac8da5 100644 --- a/misp_modules/modules/action_mod/export_sentinel.py +++ b/misp_modules/modules/action_mod/export_sentinel.py @@ -1,123 +1,141 @@ +"""Microsoft Sentinel direct publishing or the existing MISP2* script wrapper.""" + +import copy import json -import subprocess import os +import subprocess -''' - Synchronise with Microsoft Sentinel or Defender: A wrapper around an existing installation of MISP2Sentinel or MISP2Defender - - Calls the script from MISP2Sentinel or MISP2Defender (misp2_script) using the Python virtual environment (misp2_venv) - Uses the event UUID from the MISP event to export the indicators - Preferably run as Ad-Hoc workflow -''' - -misperrors = {"error": "Error"} +from misp_modules.modules.action_mod import sentinel_publish +DEFAULT_VENV = "/var/www/MISP/misp-custom/misp2sentinel/venv/" +DEFAULT_SCRIPT = "/var/www/MISP/misp-custom/misp2sentinel/script.py" moduleconfig = { "params": { - "misp2_venv": {"type": "string", "description": "Where is the MISP2Sentinel or MISP2Defender Python virtual environment?", "value": "/var/www/MISP/misp-custom/misp2sentinel/venv/"}, - "misp2_script": {"type": "string", "description": "Where is the MISP2Sentinel or MISP2Defender installation?", "value": "/var/www/MISP/misp-custom/misp2sentinel/script.py"}, + "mode": { + "type": "string", + "description": "wrapper (existing MISP2Sentinel/MISP2Defender) or direct (Sentinel STIX API)", + "value": "wrapper", + }, + "misp2_venv": { + "type": "string", + "description": "MISP2Sentinel or MISP2Defender Python virtual environment", + "value": DEFAULT_VENV, + }, + "misp2_script": { + "type": "string", + "description": "MISP2Sentinel or MISP2Defender script path", + "value": DEFAULT_SCRIPT, + }, + **copy.deepcopy(sentinel_publish.moduleconfig["params"]), }, - # Blocking modules break the exection of the current of action "blocking": False, - # Indicates whether parts of the data passed to this module should be extracted. Extracted data can be found under the `filteredItems` key - "support_filters": True, - # Indicates whether the data passed to this module should be compliant with the MISP core format - "expect_misp_core_format": False, + "support_filters": False, + "expect_misp_core_format": True, } - -returns = "boolean" - +returns = "data" moduleinfo = { - "version": "0.1", - "author": "Koen Van Impe", - "description": "Export indicators to Microsoft Sentinel or Microsoft Defender. Requires an existing installation of MISP2Sentinel or MISP2Defender.", + "version": "0.2", + "author": "Koen Van Impe, MISP Project", + "description": ( + "Export to Sentinel or Defender using the existing script wrapper, or directly to Sentinel's STIX API." + ), "module-type": ["action"], "name": "Export to Sentinel or Defender", "logo": "", - "requirements": [], - "features": "", - "references": [], - "input": "", - "output": "", + "requirements": [ + "requests", + "MISP2Sentinel/MISP2Defender for wrapper mode, or misp-stix (all extra) and Entra credentials for direct mode", + ], + "features": ( + "Preserves wrapper mode and its boolean success response by default. Direct mode publishes supplied IDS" + " attributes to Microsoft Sentinel without an external script, using the same implementation as" + " sentinel_publish. Filtered/extracted payloads are rejected because neither path applies workflow filters. See" + " the [usage" + " guide](https://github.com/MISP/misp-modules/blob/main/documentation/action_mod/sentinel_publish.md)." + ), + "references": [ + "https://learn.microsoft.com/en-us/azure/sentinel/stix-objects-api", + "https://github.com/cudeso/misp2sentinel", + ], + "input": "An event UUID in wrapper mode, or supplied MISP events/object attributes in direct mode.", + "output": "Boolean wrapper success, or a direct publishing summary with per-indicator outcomes.", } -def export_to_sentinel(request, event_uuid): - params = request["params"] - misp2_venv = params["misp2_venv"] - misp2_script = params["misp2_script"] - - script_dir = os.path.dirname(misp2_script) - python_executable = os.path.join(misp2_venv, "bin", "python") - - if not os.path.exists(python_executable): - print(f"Error: Python executable not found at {python_executable}") - return False - - if not os.path.exists(misp2_script): - print(f"Error: Script not found at {misp2_script}") - return False +def _paths(params): + venv = params.get("misp2_venv") + script = params.get("misp2_script") + if venv is None or venv == "": + venv = DEFAULT_VENV + if script is None or script == "": + script = DEFAULT_SCRIPT + if not isinstance(venv, str) or not isinstance(script, str) or not os.path.isabs(venv) or not os.path.isabs(script): + raise ValueError("Wrapper virtual-environment and script paths must be absolute.") + if "\x00" in venv or "\x00" in script: + raise ValueError("Wrapper paths must not contain NUL characters.") + return os.path.join(venv, "bin", "python"), script - print(f"Exporting event {event_uuid} to Sentinel or Defender...") - print(f"Using Python: {python_executable}") - print(f"Running script: {misp2_script}") - print(f"Working directory: {script_dir}") +def export_to_sentinel(request, event_uuid): + """Keep the existing wrapper helper's boolean contract without logging secrets.""" try: + python, script = _paths(request.get("params", {})) + if not os.path.isfile(python) or not os.path.isfile(script): + return False result = subprocess.run( - [python_executable, misp2_script, event_uuid], - cwd=script_dir, - capture_output=True, - text=True, - timeout=300 + [python, script, event_uuid], + cwd=os.path.dirname(script), + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=240, + check=False, ) - - if result.returncode == 0: - print(f"Successfully exported event {event_uuid}") - if result.stdout: - print(f"Output: {result.stdout}") - return True - else: - print(f"Export failed with return code {result.returncode}") - if result.stderr: - print(f"Error: {result.stderr}") - if result.stdout: - print(f"Output: {result.stdout}") - return False - - except subprocess.TimeoutExpired: - print("Export timed out") - return False - except Exception as e: - print(f"Error executing script: {str(e)}") + return result.returncode == 0 + except (ValueError, TypeError, OSError, subprocess.SubprocessError): return False def handler(q=False): - if q is False: + if not q: return False - - request = json.loads(q) - success = False - - event_uuid = request.get("data", {}).get("Event", {}).get("uuid") - if event_uuid: - success = export_to_sentinel(request, event_uuid) - else: - print("Error: No event UUID found in request data") - - return {"data": success} + try: + request = json.loads(q) + if not isinstance(request, dict): + raise ValueError("Request must be an object.") + if "filteredItems" in request or "matchingData" in request: + raise ValueError("Workflow filters/extracted data are unsupported; select the intended events upstream.") + params = request.get("params", {}) + if not isinstance(params, dict): + raise ValueError("params must be an object.") + mode = params.get("mode") + if mode is None or mode == "": + mode = "wrapper" + if mode == "direct": + return sentinel_publish.handler(q) + if mode != "wrapper": + raise ValueError("mode must be wrapper or direct.") + if sentinel_publish._boolean(params.get("dry_run")): + raise ValueError("dry_run is available in direct mode; wrapper mode runs the configured script.") + data = request.get("data") + if not isinstance(data, dict) or not isinstance(event := data.get("Event", data), dict): + raise ValueError("Wrapper mode requires one event; use direct mode for event collections.") + identifier = sentinel_publish._uuid(event.get("uuid"), "event UUID") + _paths(params) + except (ValueError, TypeError) as exc: + return {"data": False, "error": f"Sentinel/Defender export: {exc}"} + success = export_to_sentinel(request, identifier) + if not success: + return { + "data": False, + "error": "Sentinel/Defender wrapper failed, timed out or is unavailable; check its installation and logs.", + } + return {"data": True} def introspection(): - modulesetup = {} - try: - modulesetup["config"] = moduleconfig - except NameError: - pass - return modulesetup + return {"config": moduleconfig} def version(): - moduleinfo["config"] = moduleconfig - return moduleinfo + return dict(moduleinfo, config=moduleconfig) diff --git a/misp_modules/modules/action_mod/sentinel_publish.py b/misp_modules/modules/action_mod/sentinel_publish.py new file mode 100644 index 00000000..0e931f08 --- /dev/null +++ b/misp_modules/modules/action_mod/sentinel_publish.py @@ -0,0 +1,506 @@ +"""Publish MISP indicators directly to Microsoft Sentinel's STIX upload API.""" + +import ipaddress +import json +import os +import re +import time +import uuid +from collections import Counter +from datetime import datetime, timedelta, timezone +from urllib.parse import urlsplit + +import requests + +try: + from misp_stix_converter import MISPtoSTIX21Parser + from stix2patterns.validator import run_validator +except ImportError: + # Keep the existing script wrapper available without the optional converter. + MISPtoSTIX21Parser = None + +moduleinfo = { + "version": "1.0", + "author": "MISP Project", + "name": "Microsoft Sentinel Publish", + "description": "Publish MISP indicators directly to Microsoft Sentinel's STIX upload API with per-object results.", + "module-type": ["action"], + "logo": "", + "requirements": [ + "requests", + "misp-stix (included in the all extra)", + "Microsoft Sentinel Contributor at the target workspace", + ], + "features": ( + "Uses misp-stix to convert IDS-marked event and object attributes into STIX 2.1 indicator patterns, preserving" + " composite constraints, and uploads batches of up to 100" + " objects. Supports dry runs, explicit workspace selection, expiry and checked per-object results. Entra" + " credentials come from the module server environment. No external script installation is required. See the" + " [usage guide](https://github.com/MISP/misp-modules/blob/main/documentation/action_mod/sentinel_publish.md)." + ), + "references": ["https://learn.microsoft.com/en-us/azure/sentinel/stix-objects-api"], + "input": "MISP events and object attributes, including attribute UUIDs and Unix timestamps.", + "output": "Upload counts and per-indicator outcomes; errors signal incomplete publication.", +} +moduleconfig = { + "params": { + "workspace_id": {"type": "string", "description": "Required: target Log Analytics workspace UUID"}, + "source": {"type": "string", "description": "Stable source-system namespace", "value": "misp-modules"}, + "expiration_days": {"type": "string", "description": "Days until indicator expiry (1-365)", "value": "30"}, + "confidence": {"type": "string", "description": "STIX confidence (0-100)", "value": "50"}, + "dry_run": { + "type": "string", + "description": "true previews STIX payloads without network access", + "value": "false", + }, + }, + "blocking": True, + "support_filters": False, + "expect_misp_core_format": True, +} +returns = "data" +API_VERSION = "2024-02-01-preview" +UPLOAD_ORIGIN = "https://api.ti.sentinel.azure.com" +AUTH_ORIGIN = "https://login.microsoftonline.com" +SCOPE = "https://management.azure.com/.default" +MAX_INDICATORS = 1000 +BATCH_SIZE = 100 +MAX_BATCH_BYTES = 1024 * 1024 +DEADLINE_SECONDS = 240 +HASH_LENGTHS = {"md5": ("MD5", 32), "sha1": ("SHA-1", 40), "sha256": ("SHA-256", 64), "sha512": ("SHA-512", 128)} +EMAIL_TYPES = {"email", "email-src", "email-dst"} +SUPPORTED_TYPES = ( + {"domain", "hostname", "url", "ip-src", "ip-dst", "domain|ip", "malware-sample"} + | EMAIL_TYPES + | HASH_LENGTHS.keys() + | {f"filename|{kind}" for kind in HASH_LENGTHS} +) +TLP_MARKINGS = { + "tlp:white": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", + "tlp:clear": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", + "tlp:green": "marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da", + "tlp:amber": "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82", + "tlp:red": "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed", +} + + +class PublishError(Exception): + """An error safe to return without provider bodies or credential details.""" + + +def _boolean(value): + if value is None or value == "": + return False + if isinstance(value, (bool, int)) and value in (0, 1): + return bool(value) + if isinstance(value, str) and value.lower() in ("true", "false", "1", "0"): + return value.lower() in ("true", "1") + raise ValueError("Boolean values must be true, false, 1 or 0.") + + +def _uuid(value, name): + if not isinstance(value, str): + raise ValueError(f"{name} must be a UUID string.") + try: + identifier = uuid.UUID(value) + except ValueError: + raise ValueError(f"{name} must be a UUID string.") from None + if not identifier.int: + raise ValueError(f"{name} must be a non-nil UUID.") + return str(identifier) + + +def _integer(value, default, minimum, maximum, name): + if value is None or value == "": + return default + if isinstance(value, bool) or not re.fullmatch(r"[0-9]{1,3}", str(value)): + raise ValueError(f"{name} must be an integer.") + number = int(value) + if not minimum <= number <= maximum: + raise ValueError(f"{name} is outside its supported range.") + return number + + +def _settings(params): + if not isinstance(params, dict): + raise ValueError("params must be an object.") + source = params.get("source") + if source is None or source == "": + source = "misp-modules" + if not isinstance(source, str) or not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9 ._:/-]{0,63}", source): + raise ValueError("source must be 1-64 ASCII letters, digits, spaces or ._:/- and start with a letter or digit.") + if source.lower().strip() == "microsoft sentinel": + raise ValueError("The source-system name Microsoft Sentinel is reserved by the API.") + return { + "workspace_id": _uuid(params.get("workspace_id"), "workspace_id"), + "source": source, + "days": _integer(params.get("expiration_days"), 30, 1, 365, "expiration_days"), + "confidence": _integer(params.get("confidence"), 50, 0, 100, "confidence"), + "dry_run": _boolean(params.get("dry_run")), + } + + +def _records(container, key): + records = container.get(key, []) + if not isinstance(records, list) or any(not isinstance(record, dict) for record in records): + raise ValueError(f"{key} must be a list of objects.") + return records + + +def _tags(*containers): + names = set() + for container in containers: + for key in ("Tag", "EventTag", "AttributeTag"): + for tag in _records(container, key): + if _boolean(tag.get("local")): + continue + tag = tag.get("Tag", tag) + if isinstance(tag, dict) and not _boolean(tag.get("local")) and isinstance(tag.get("name"), str): + tag["name"].encode("utf-8") + if tag["name"]: + names.add(tag["name"]) + return names + + +def _domain(value): + domain = value.removesuffix(".").encode("idna").decode("ascii").lower() + if len(domain) > 253 or any( + not re.fullmatch(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?", label) for label in domain.split(".") + ): + raise ValueError("Invalid domain.") + return domain + + +def _atomic(kind, value): + value.encode("utf-8") + if any(ord(char) < 32 or ord(char) == 127 for char in value): + raise ValueError("Control characters are unsupported.") + if kind in HASH_LENGTHS: + _, length = HASH_LENGTHS[kind] + if not re.fullmatch(rf"[0-9a-fA-F]{{{length}}}", value): + raise ValueError("Invalid hash.") + return value.lower() + if kind in ("ip-src", "ip-dst"): + if "%" in value: + raise ValueError("Scoped IPv6 is unsupported.") + address = ipaddress.ip_address(value) + return str(address) + if kind in ("domain", "hostname"): + return _domain(value) + if kind in EMAIL_TYPES: + local, separator, domain = value.rpartition("@") + if not separator or not re.fullmatch(r"[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+", local): + raise ValueError("Unsupported email address.") + return local + "@" + _domain(domain) + parsed = urlsplit(value) + if parsed.scheme not in ("http", "https") or not parsed.hostname or any(char.isspace() for char in value): + raise ValueError("URL must be absolute HTTP(S) with a host and no whitespace.") + if parsed.username is not None or parsed.password is not None: + raise ValueError("Credential-bearing URLs are unsupported.") + parsed.port + return value + + +def _normalized_value(kind, value): + if "|" not in kind and kind != "malware-sample": + return _atomic(kind, value) + first, separator, second = value.rpartition("|") + if not separator or not first.strip() or not second.strip(): + raise ValueError("Invalid composite.") + if kind == "domain|ip": + return _atomic("domain", first) + "|" + _atomic("ip-dst", second) + first.encode("utf-8") + if "|" in first or any(ord(char) < 32 or ord(char) == 127 for char in first): + raise ValueError("Unsupported composite filename.") + return first + "|" + _atomic("md5" if kind == "malware-sample" else kind.split("|")[1], second) + + +def _patterns(kind, value): + try: + parser = MISPtoSTIX21Parser() + # Only the value is needed. Never pass private metadata or sample data to the converter. + parser.parse_misp_attribute({"type": kind, "value": _normalized_value(kind, value), "to_ids": True}) + if parser.errors or parser.warnings: + raise ValueError("Incomplete conversion.") + patterns = [ + obj["pattern"] + for obj in parser.stix_objects + if obj["type"] == "indicator" and obj.get("pattern_type") == "stix" + ] + if len(patterns) != 1 or run_validator(patterns[0], stix_version="2.1"): + raise ValueError("No single valid STIX 2.1 pattern.") + return patterns + except Exception: + # Converter diagnostics may contain the source value; expose only a skipped count. + raise ValueError("misp-stix could not completely convert the attribute.") from None + + +def _stamp(value): + return value.isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +def _collect(data, settings, now): + if isinstance(data, dict): + data = [data] + if not isinstance(data, list) or any(not isinstance(envelope, dict) for envelope in data): + raise ValueError("data must contain an event or list of events.") + indicators, skipped = {}, Counter() + for envelope in data: + event = envelope.get("Event", envelope) + if not isinstance(event, dict): + raise ValueError("Event must be an object.") + if _boolean(event.get("deleted")): + continue + collections = dict(event, **envelope) + attributes = [(attribute, {}) for attribute in _records(collections, "Attribute")] + for obj in _records(collections, "Object"): + if not _boolean(obj.get("deleted")): + attributes.extend((attribute, obj) for attribute in _records(obj, "Attribute")) + for attribute, obj in attributes: + if _boolean(attribute.get("deleted")) or not _boolean(attribute.get("to_ids")): + skipped["deleted or non-IDS"] += 1 + continue + kind, value = attribute.get("type"), attribute.get("value") + if not isinstance(kind, str) or kind not in SUPPORTED_TYPES: + skipped["unsupported"] += 1 + continue + try: + identifier = _uuid(attribute.get("uuid"), "attribute UUID") + timestamp = attribute.get("timestamp") + if isinstance(timestamp, bool) or not re.fullmatch(r"[0-9]{1,10}", str(timestamp)): + raise ValueError("Attribute timestamp must be Unix seconds.") + created = datetime.fromtimestamp(int(timestamp), timezone.utc) + if created > now: + raise ValueError("Attribute timestamp is in the future.") + if not isinstance(value, str) or not value.strip(): + raise ValueError("Empty value.") + patterns = _patterns(kind, value) + except (ValueError, OverflowError, OSError): + skipped["malformed value or identity/timestamp"] += 1 + continue + labels = _tags(event, envelope, obj, attribute) + ref = { + "event": str(event.get("uuid") or event.get("id") or "unknown"), + "attribute": identifier, + "misp_type": kind, + } + if obj: + ref["object"] = str(obj.get("uuid") or obj.get("id") or obj.get("name") or "unknown") + for pattern in patterns: + stix_id = "indicator--" + str( + uuid.uuid5( + uuid.NAMESPACE_URL, + json.dumps( + [ + "misp-sentinel-publish", + settings["workspace_id"], + settings["source"], + identifier, + str(int(timestamp)), + pattern, + ] + ), + ) + ) + entry = indicators.setdefault( + stix_id, {"pattern": pattern, "created": _stamp(created), "sources": [], "labels": set()} + ) + if ref not in entry["sources"]: + entry["sources"].append(ref) + entry["labels"].update(labels) + if len(indicators) > MAX_INDICATORS: + raise ValueError(f"At most {MAX_INDICATORS} unique indicators are allowed; split the input.") + objects, outcomes = [], [] + for identifier, entry in sorted(indicators.items()): + refs = sorted(entry["sources"], key=lambda ref: json.dumps(ref, sort_keys=True)) + item = { + "type": "indicator", + "spec_version": "2.1", + "id": identifier, + "created": entry["created"], + "modified": _stamp(now), + "pattern_type": "stix", + "pattern_version": "2.1", + "pattern": entry["pattern"], + "valid_from": entry["created"], + "valid_until": _stamp(now + timedelta(days=settings["days"])), + "confidence": settings["confidence"], + "name": "MISP " + identifier, + "description": json.dumps(refs, sort_keys=True), + "external_references": [{"source_name": "MISP", "external_id": ref["attribute"]} for ref in refs], + } + if entry["labels"]: + item["labels"] = sorted(entry["labels"]) + markings = {TLP_MARKINGS[label.lower()] for label in entry["labels"] if label.lower() in TLP_MARKINGS} + if markings: + item["object_marking_refs"] = sorted(markings) + objects.append(item) + outcomes.append({"id": identifier, "sources": refs}) + return objects, outcomes, dict(skipped) + + +def _request(session, endpoint, **kwargs): + try: + response = session.post(endpoint, timeout=15, verify=True, allow_redirects=False, **kwargs) + except (requests.RequestException, OSError): + raise PublishError("Request failed; remote outcome may be unknown. Check Sentinel before retrying.") from None + if response.status_code != 200: + raise PublishError(f"HTTP {response.status_code}; remote outcome may be unknown.") + return response + + +def _authenticate(session): + try: + tenant = _uuid(os.getenv("SENTINEL_TENANT_ID"), "tenant ID") + client = _uuid(os.getenv("SENTINEL_CLIENT_ID"), "client ID") + secret = os.getenv("SENTINEL_CLIENT_SECRET", "") + if not secret or any(ord(char) < 32 or ord(char) == 127 for char in secret): + raise ValueError("Invalid client secret.") + except ValueError: + raise PublishError( + "Configure SENTINEL_TENANT_ID, SENTINEL_CLIENT_ID and SENTINEL_CLIENT_SECRET on the module server." + ) from None + response = _request( + session, + f"{AUTH_ORIGIN}/{tenant}/oauth2/v2.0/token", + data={ + "grant_type": "client_credentials", + "client_id": client, + "client_secret": secret, + "scope": SCOPE, + }, + ) + try: + body = response.json() + token = body["access_token"] + if ( + body.get("token_type", "").lower() != "bearer" + or not isinstance(token, str) + or not token + or any(not 33 <= ord(char) <= 126 for char in token) + or type(body.get("expires_in")) is not int + or body["expires_in"] <= DEADLINE_SECONDS + 15 + ): + raise ValueError("Invalid token response.") + except (ValueError, KeyError, TypeError, AttributeError): + raise PublishError("Invalid or short-lived Entra token response; no indicators submitted.") from None + session.headers.update({"Authorization": f"Bearer {token}"}) + + +def _upload(session, endpoint, payload): + response = _request(session, endpoint, params={"api-version": API_VERSION}, json=payload) + if not response.content.strip(): + return set() + try: + body = response.json() + if not isinstance(body, dict) or set(body) != {"errors"} or not isinstance(body["errors"], list): + raise ValueError("Invalid response.") + failures = set() + for error in body["errors"]: + index = error["recordIndex"] + messages = error["errorMessages"] + if ( + type(index) is not int + or not 0 <= index < len(payload["stixobjects"]) + or index in failures + or not isinstance(messages, list) + or not messages + or any(not isinstance(message, str) for message in messages) + ): + raise ValueError("Invalid per-object result.") + failures.add(index) + return failures + except (ValueError, TypeError, KeyError): + raise PublishError( + "Invalid upload acknowledgement; remote outcome may be unknown. Check Sentinel before retrying." + ) from None + + +def handler(q=False): + if not q: + return False + try: + request = json.loads(q) + if not isinstance(request, dict): + raise ValueError("Request must be an object.") + if "filteredItems" in request or "matchingData" in request: + raise ValueError("Workflow filters/extracted data are unsupported; supply selected events in data.") + settings = _settings(request.get("params", {})) + if MISPtoSTIX21Parser is None: + raise ValueError("Direct publishing requires misp-stix; install misp-modules[all].") + now = datetime.now(timezone.utc) + objects, outcomes, skipped = _collect(request.get("data"), settings, now) + batches = [ + {"sourcesystem": settings["source"], "stixobjects": objects[offset : offset + BATCH_SIZE]} + for offset in range(0, len(objects), BATCH_SIZE) + ] + if any(len(json.dumps(batch).encode("utf-8")) > MAX_BATCH_BYTES for batch in batches): + raise ValueError("A batch exceeds the local 1 MiB limit; split the input or reduce metadata.") + except (ValueError, TypeError) as exc: + return {"error": f"Sentinel publish: {exc}"} + summary = { + "success": True, + "dry_run": settings["dry_run"], + "workspace_id": settings["workspace_id"], + "uploaded": 0, + "failed": 0, + "planned": 0, + "skipped": skipped, + "items": outcomes, + } + result = {"result_key": "sentinel_publish", "data": summary} + if settings["dry_run"]: + summary["planned"] = len(objects) + summary["payloads"] = batches + for item in outcomes: + item["status"] = "planned" + return result + if not objects: + return result + endpoint = f"{UPLOAD_ORIGIN}/workspaces/{settings['workspace_id']}/threat-intelligence-stix-objects:upload" + deadline = time.monotonic() + DEADLINE_SECONDS + with requests.Session() as session: + session.headers.update({"Accept": "application/json"}) + try: + _authenticate(session) + except PublishError as exc: + summary["success"] = False + result["error"] = f"Sentinel authentication: {exc}" + for item in outcomes: + item["status"] = "not_attempted" + return result + stopped = None + for batch_index, batch in enumerate(batches): + batch_items = outcomes[batch_index * BATCH_SIZE : (batch_index + 1) * BATCH_SIZE] + try: + if stopped is not None: + raise PublishError(stopped) + if time.monotonic() >= deadline: + raise PublishError("Publication time budget exceeded; batch not attempted.") + failures = _upload(session, endpoint, batch) + for index, item in enumerate(batch_items): + if index in failures: + item.update(status="failed", error="Sentinel rejected this indicator.") + summary["failed"] += 1 + else: + item["status"] = "uploaded" + summary["uploaded"] += 1 + except PublishError as exc: + for item in batch_items: + item.update(status="failed", error=str(exc)) + summary["failed"] += 1 + stopped = "Publication stopped after an uncertain or unsuccessful batch; batch not attempted." + if summary["failed"]: + summary["success"] = False + result["error"] = ( + f"Sentinel publish: {summary['failed']} failed; {summary['uploaded']} uploaded. Publication may be partial." + ) + return result + + +def introspection(): + return {"config": moduleconfig} + + +def version(): + return dict(moduleinfo, config=moduleconfig) diff --git a/tests/test_sentinel_publish.py b/tests/test_sentinel_publish.py new file mode 100644 index 00000000..4c9f69e5 --- /dev/null +++ b/tests/test_sentinel_publish.py @@ -0,0 +1,681 @@ +import copy +import json +import subprocess +import sys +import uuid +from datetime import datetime, timedelta, timezone +from unittest.mock import MagicMock +from urllib.parse import parse_qs + +import pytest +import requests +from misp_stix_converter import MISPtoSTIX21Parser +from pymisp import MISPObject +from stix2patterns.v21.pattern import Pattern +from tornado.testing import AsyncHTTPTestCase +from tornado.web import Application + +from misp_modules import __main__ as service +from misp_modules.modules.action_mod import export_sentinel as wrapper +from misp_modules.modules.action_mod import sentinel_publish as publish + +WORKSPACE = "22549473-0938-4938-8a04-225494730938" +TENANT = "c6a7d64d-8d28-4c7d-ae30-8efcbb28804d" +CLIENT = "fdb91a8a-e7bd-4c6e-b265-b286c3321879" +EVENT = "f1ef0409-c80a-4012-b401-b7d1050f4ed4" +ATTRIBUTE = "aa09819b-f2d8-4661-83ea-d4897cafebc8" + + +def attribute(kind="domain", value="example.org", **metadata): + return {"uuid": ATTRIBUTE, "timestamp": "1767225600", "type": kind, "value": value, "to_ids": True, **metadata} + + +def event(attributes=None, objects=None, **metadata): + return {"uuid": EVENT, "Attribute": attributes or [], "Object": objects or [], **metadata} + + +def request(data=None, **params): + return json.dumps( + {"data": event([attribute()]) if data is None else data, "params": {"workspace_id": WORKSPACE, **params}} + ) + + +def run(data=None, **params): + return publish.handler(request(data, **params)) + + +def response(body=None, status=200, raw=None): + result = requests.Response() + result.status_code = status + result.headers["Content-Type"] = "application/json" + result._content = (b"" if body is None else json.dumps(body).encode()) if raw is None else raw + return result + + +@pytest.fixture +def client(monkeypatch): + session = MagicMock(spec=requests.Session) + session.headers = {} + session.__enter__.return_value = session + + def post(url, **kwargs): + if "oauth2" in url: + return response({"access_token": "access-token", "token_type": "Bearer", "expires_in": 3600}) + return response() + + session.post.side_effect = post + session.constructor = MagicMock(return_value=session) + monkeypatch.setattr(publish.requests, "Session", session.constructor) + monkeypatch.setenv("SENTINEL_TENANT_ID", TENANT) + monkeypatch.setenv("SENTINEL_CLIENT_ID", CLIENT) + monkeypatch.setenv("SENTINEL_CLIENT_SECRET", "client-secret") + return session + + +def payloads(result): + assert "error" not in result, result + return [obj for batch in result["data"]["payloads"] for obj in batch["stixobjects"]] + + +def test_direct_contract_uses_correct_authority_scope_endpoint_and_empty_body_ack(client): + result = run() + assert result["data"]["uploaded"] == 1 and result["data"]["success"] is True + assert result["result_key"] == "sentinel_publish" + auth, upload = client.post.call_args_list + assert auth.args == (f"https://login.microsoftonline.com/{TENANT}/oauth2/v2.0/token",) + assert auth.kwargs["data"] == { + "grant_type": "client_credentials", + "client_id": CLIENT, + "client_secret": "client-secret", + "scope": "https://management.azure.com/.default", + } + assert upload.args == ( + f"https://api.ti.sentinel.azure.com/workspaces/{WORKSPACE}/threat-intelligence-stix-objects:upload", + ) + assert upload.kwargs["params"] == {"api-version": "2024-02-01-preview"} + assert set(upload.kwargs["json"]) == {"sourcesystem", "stixobjects"} + assert upload.kwargs["json"]["sourcesystem"] == "misp-modules" + obj = upload.kwargs["json"]["stixobjects"][0] + expiry = datetime.fromisoformat(obj["valid_until"].replace("Z", "+00:00")) + assert abs((expiry - datetime.now(timezone.utc) - timedelta(days=30)).total_seconds()) < 5 + assert client.headers["Authorization"] == "Bearer access-token" + assert all( + call.kwargs["verify"] is True and call.kwargs["allow_redirects"] is False and call.kwargs["timeout"] == 15 + for call in [auth, upload] + ) + client.__exit__.assert_called_once() + + +@pytest.mark.parametrize( + "kind,value,pattern", + [ + ("domain", "EXAMPLE.org.", "[domain-name:value = 'example.org']"), + ("hostname", "bücher.example", "[domain-name:value = 'xn--bcher-kva.example']"), + ( + "ip-src", + "192.0.2.1", + "[network-traffic:src_ref.type = 'ipv4-addr' AND network-traffic:src_ref.value = '192.0.2.1']", + ), + ( + "ip-dst", + "2001:0DB8:0000::1", + "[network-traffic:dst_ref.type = 'ipv6-addr' AND network-traffic:dst_ref.value = '2001:db8::1']", + ), + ("url", "https://example.org/A?q=1", "[url:value = 'https://example.org/A?q=1']"), + ("email", "User@EXAMPLE.org", "[email-addr:value = 'user@example.org']"), + ("email-src", "User@EXAMPLE.org", "[email-message:from_ref.value = 'user@example.org']"), + ("email-dst", "User@EXAMPLE.org", "[email-message:to_refs[*].value = 'user@example.org']"), + ("url", "https://example.org/a'b", "[url:value = 'https://example.org/a\\'b']"), + ("url", "https://example.org/a\\b", "[url:value = 'https://example.org/a\\\\b']"), + *[ + ( + kind, + "A" * length, + f"[file:hashes.{algorithm if kind == 'md5' else repr(algorithm)} = '" + "a" * length + "']", + ) + for kind, (algorithm, length) in publish.HASH_LENGTHS.items() + ], + *[ + ( + f"filename|{kind}", + "file.exe|" + "a" * length, + f"[file:name = 'file.exe' AND file:hashes.{algorithm if kind == 'md5' else repr(algorithm)} = '" + + "a" * length + + "']", + ) + for kind, (algorithm, length) in publish.HASH_LENGTHS.items() + ], + ( + "malware-sample", + "file.exe|" + "a" * 32, + "[file:name = 'file.exe' AND file:hashes.MD5 = '" + "a" * 32 + "']", + ), + ], +) +def test_supported_values_generate_valid_stix_patterns(kind, value, pattern): + import stix2 + + obj = payloads(run(event([attribute(kind, value)]), dry_run=True))[0] + assert obj["pattern"] == pattern + parsed = stix2.parse(obj) + assert parsed.type == "indicator" and parsed.spec_version == "2.1" + + +def test_domain_ip_and_object_provenance_are_preserved_without_mutation(): + obj = {"uuid": "object-1", "name": "domain-ip", "Attribute": [attribute("domain|ip", "example.org|192.0.2.1")]} + data = [{"Event": event(objects=[obj])}, {"Event": {"uuid": EVENT}, "Object": [obj]}, event(objects=[obj])] + original = copy.deepcopy(data) + result = run(data, dry_run=True) + objects = payloads(result) + assert len(objects) == 1 + assert ( + objects[0]["pattern"] + == "[domain-name:value = 'example.org' AND domain-name:resolves_to_refs[*].value = '192.0.2.1']" + ) + assert all( + len(item["sources"]) == 1 and item["sources"][0]["object"] == "object-1" for item in result["data"]["items"] + ) + assert data == original + + +@pytest.mark.parametrize( + "template,relation,value", [("file", "sha256", "a" * 64), ("domain-ip", "domain", "example.org")] +) +def test_existing_misp_object_templates_suffice(template, relation, value): + obj = MISPObject(template, strict=True) + obj.add_attribute(relation, value=value, to_ids=True, uuid=ATTRIBUTE, timestamp=1767225600) + obj._validate() + assert len(payloads(run(event(objects=[json.loads(obj.to_json())]), dry_run=True))) == 1 + + +def test_source_revision_ids_and_created_are_stable_but_change_with_content_timestamp_or_namespace(): + first = payloads(run(dry_run=True))[0] + again = payloads(run(dry_run=True, expiration_days=45, confidence=75))[0] + assert first["id"] == again["id"] and first["created"] == again["created"] == "2026-01-01T00:00:00.000Z" + assert first["modified"] <= again["modified"] + assert again["confidence"] == 75 and again["valid_until"] > first["valid_until"] + for data, params in [ + (event([attribute(value="other.example")]), {}), + (event([attribute(timestamp="1767225601")]), {}), + (event([attribute(uuid=str(uuid.uuid4()))]), {}), + (event([attribute()]), {"source": "another publisher"}), + (event([attribute()]), {"workspace_id": str(uuid.uuid4())}), + ]: + assert payloads(run(data, dry_run=True, **params))[0]["id"] != first["id"] + + +def test_tags_and_supported_tlp_markings_are_preserved_without_private_comments_or_samples(): + data = event( + [attribute(Tag=[{"name": "tlp:green"}], comment="private-comment", data="private-sample")], + Tag=[{"name": "event-tag"}], + EventTag=[{"Tag": {"name": "tlp:amber"}}], + info="private-title", + ) + obj = payloads(run(data, dry_run=True))[0] + assert obj["labels"] == ["event-tag", "tlp:amber", "tlp:green"] + assert set(obj["object_marking_refs"]) == {publish.TLP_MARKINGS["tlp:green"], publish.TLP_MARKINGS["tlp:amber"]} + assert not any(word in json.dumps(obj) for word in ["private-comment", "private-sample", "private-title"]) + + +@pytest.mark.parametrize("local", [True, 1, "1", "true"]) +@pytest.mark.parametrize("key", ["Tag", "EventTag", "AttributeTag"]) +@pytest.mark.parametrize("level", ["event", "envelope", "object", "attribute"]) +def test_local_tag_associations_never_leave_misp(local, key, level): + tags = { + key: [ + {"local": local, "Tag": {"name": "private-routing"}}, + {"local": local, "name": "tlp:red"}, + {"Tag": {"name": "private-nested", "local": local}}, + {"local": "0", "Tag": {"name": "public-tag"}}, + ] + } + attr = attribute(**(tags if level == "attribute" else {})) + obj = {"Attribute": [attr], **(tags if level == "object" else {})} + data = {"Event": event(objects=[obj], **(tags if level == "event" else {}))} + if level == "envelope": + data.update(tags) + exported = payloads(run(data, dry_run=True))[0] + assert exported["labels"] == ["public-tag"] + assert "object_marking_refs" not in exported + assert "private-" not in json.dumps(exported) + + +@pytest.mark.parametrize( + "kind,value,object_type,count", + [ + ("url", "https://example.org/a'b\\c", "url", 1), + ("filename|sha256", r"C:\O'Brien\test.exe' OR file:name = 'other|" + "a" * 64, "file", 2), + ], +) +def test_converter_escapes_literals_without_adding_conditions(kind, value, object_type, count): + obj = payloads(run(event([attribute(kind, value)]), dry_run=True))[0] + assert len(Pattern(obj["pattern"]).inspect().comparisons[object_type]) == count + + +@pytest.mark.parametrize("failure", ["errors", "warnings", "exception", "empty", "invalid", "multiple"]) +def test_incomplete_conversions_are_skipped_before_network(client, monkeypatch, failure): + parser = MagicMock() + parser.errors = {"private": ["private-value"]} if failure == "errors" else {} + parser.warnings = {"private": ["private-value"]} if failure == "warnings" else {} + pattern = "invalid" if failure == "invalid" else "[domain-name:value = 'example.org']" + parser.stix_objects = [{"type": "indicator", "pattern_type": "stix", "pattern": pattern}] + if failure == "empty": + parser.stix_objects = [{"type": "x-misp-attribute", "value": "private-value"}] + if failure == "multiple": + parser.stix_objects *= 2 + if failure == "exception": + parser.parse_misp_attribute.side_effect = RuntimeError("private-value") + monkeypatch.setattr(publish, "MISPtoSTIX21Parser", lambda: parser) + result = run() + assert result["data"]["uploaded"] == 0 + assert result["data"]["skipped"] == {"malformed value or identity/timestamp": 1} + assert "private-value" not in json.dumps(result) + client.constructor.assert_not_called() + + +def test_only_the_pattern_is_taken_from_the_public_converter(monkeypatch): + parser = MISPtoSTIX21Parser() + factory = MagicMock(return_value=parser) + parse = MagicMock(wraps=parser.parse_misp_attribute) + monkeypatch.setattr(publish, "MISPtoSTIX21Parser", factory) + monkeypatch.setattr(parser, "parse_misp_attribute", parse) + obj = payloads(run(event([attribute(comment="private-comment", data="private-sample")]), dry_run=True))[0] + parse.assert_called_once_with({"type": "domain", "value": "example.org", "to_ids": True}) + assert obj["pattern"] == next(item["pattern"] for item in parser.stix_objects if item["type"] == "indicator") + assert not any(key.startswith("x_misp") for key in obj) + + +def test_missing_converter_blocks_direct_mode_but_preserves_legacy_wrapper(client, monkeypatch): + monkeypatch.setattr(publish, "MISPtoSTIX21Parser", None) + for module in (publish, wrapper): + result = module.handler(request(mode="direct", dry_run=True)) + assert "install misp-modules[all]" in result["error"] + client.constructor.assert_not_called() + monkeypatch.setattr(wrapper.os.path, "isfile", lambda path: True) + process = MagicMock(return_value=MagicMock(returncode=0)) + monkeypatch.setattr(wrapper.subprocess, "run", process) + assert wrapper.handler(json.dumps({"data": {"Event": {"uuid": EVENT}}})) == {"data": True} + process.assert_called_once() + + +@pytest.mark.parametrize( + "kind,value", + [ + ("filename|sha256", "a|b.exe|" + "a" * 64), + ("domain|ip", "example.org|not-an-ip"), + ("ip-dst", "192.0.2.0/24"), + ("ip-src", "fe80::1%eth0"), + ("url", "https://user:password@example.org/path"), + ("sha256", "bad"), + ], +) +def test_invalid_or_ambiguous_values_do_not_become_broader_patterns(client, kind, value): + result = run(event([attribute(kind, value)])) + assert result["data"]["skipped"] == {"malformed value or identity/timestamp": 1} + client.constructor.assert_not_called() + + +def test_dry_run_needs_no_credentials_or_requests_session(client, monkeypatch): + for name in ["SENTINEL_TENANT_ID", "SENTINEL_CLIENT_ID", "SENTINEL_CLIENT_SECRET"]: + monkeypatch.delenv(name) + assert run(dry_run=True)["data"]["planned"] == 1 + client.constructor.assert_not_called() + + +def test_batches_are_limited_to_one_hundred_and_share_one_authentication(client): + data = event( + [ + attribute(value=f"host{index}.example", uuid=str(uuid.uuid5(uuid.NAMESPACE_URL, str(index)))) + for index in range(205) + ] + ) + result = run(data) + assert result["data"]["uploaded"] == 205 + assert [len(call.kwargs["json"]["stixobjects"]) for call in client.post.call_args_list[1:]] == [100, 100, 5] + assert client.post.call_count == 4 + + +@pytest.mark.parametrize( + "body", + [ + {"errors": [{"recordIndex": 0, "errorMessages": ["client-secret"]}]}, + {"errors": [{"recordIndex": 1, "errorMessages": ["access-token"]}]}, + ], +) +def test_http_200_partial_failures_are_mapped_by_record_index_without_exposing_messages(client, body): + client.post.side_effect = [ + response({"access_token": "access-token", "token_type": "Bearer", "expires_in": 3600}), + response(body), + ] + result = run(event([attribute(), attribute(value="other.example", uuid=str(uuid.uuid4()))])) + assert result["data"]["uploaded"] == 1 and result["data"]["failed"] == 1 + assert result["data"]["success"] is False and "partial" in result["error"] + assert result["data"]["items"][body["errors"][0]["recordIndex"]]["status"] == "failed" + assert "client-secret" not in json.dumps(result) and "access-token" not in json.dumps(result) + + +@pytest.mark.parametrize( + "body", + [ + {}, + [], + {"unexpected": True}, + {"errors": {}}, + {"errors": [{}]}, + {"errors": [{"recordIndex": True, "errorMessages": ["bad"]}]}, + {"errors": [{"recordIndex": -1, "errorMessages": ["bad"]}]}, + {"errors": [{"recordIndex": 1, "errorMessages": ["bad"]}]}, + {"errors": [{"recordIndex": 0, "errorMessages": []}]}, + {"errors": [{"recordIndex": 0, "errorMessages": "bad"}]}, + {"errors": [{"recordIndex": 0, "errorMessages": ["bad"]}] * 2}, + {"errors": [], "warnings": ["bad"]}, + ], +) +def test_malformed_upload_acknowledgements_are_uncertain_failures(client, body): + client.post.side_effect = [ + response({"access_token": "access-token", "token_type": "Bearer", "expires_in": 3600}), + response(body), + ] + result = run() + assert result["data"]["failed"] == 1 and result["data"]["uploaded"] == 0 + assert "unknown" in result["data"]["items"][0]["error"] + assert client.post.call_count == 2 + + +def test_empty_validation_error_list_is_an_acknowledgement(client): + client.post.side_effect = [ + response({"access_token": "access-token", "token_type": "Bearer", "expires_in": 3600}), + response({"errors": []}), + ] + assert run()["data"]["uploaded"] == 1 + + +@pytest.mark.parametrize("status", [202, 204, 301, 302, 400, 401, 403, 404, 429, 500]) +def test_upload_http_failure_stops_later_batches_without_retry_or_echoing_secrets(client, status): + client.post.side_effect = [ + response({"access_token": "access-token", "token_type": "Bearer", "expires_in": 3600}), + response({"message": "client-secret"}, status), + ] + result = run(event([attribute(value=f"host{index}.example") for index in range(101)])) + assert result["data"]["failed"] == 101 and result["data"]["uploaded"] == 0 + assert "not attempted" in result["data"]["items"][-1]["error"] + assert "client-secret" not in json.dumps(result) and client.post.call_count == 2 + + +@pytest.mark.parametrize( + "failure", + [ + requests.Timeout("client-secret"), + requests.ConnectionError("access-token"), + OSError("client-secret"), + response(raw=b"not JSON"), + ], +) +def test_unknown_write_outcomes_preserve_prior_success(client, failure): + client.post.side_effect = [ + response({"access_token": "access-token", "token_type": "Bearer", "expires_in": 3600}), + response(), + failure, + ] + result = run(event([attribute(value=f"host{index}.example") for index in range(101)])) + assert result["data"]["uploaded"] == 100 and result["data"]["failed"] == 1 + assert "client-secret" not in json.dumps(result) and "access-token" not in json.dumps(result) + + +@pytest.mark.parametrize( + "body", + [ + {}, + [], + {"access_token": "access-token"}, + {"access_token": "access-token", "token_type": "other", "expires_in": 3600}, + {"access_token": "access-token\r\nHeader: bad", "token_type": "Bearer", "expires_in": 3600}, + {"access_token": "access-token", "token_type": "Bearer", "expires_in": 10}, + {"access_token": "access-token", "token_type": "Bearer", "expires_in": True}, + ], +) +def test_invalid_token_responses_do_not_upload_or_expose_secrets(client, body): + client.post.side_effect = [response(body)] + result = run() + assert "error" in result and result["data"]["success"] is False + assert result["data"]["items"][0]["status"] == "not_attempted" + assert "access-token" not in json.dumps(result) and client.post.call_count == 1 + + +@pytest.mark.parametrize( + "key,value", + [ + ("SENTINEL_TENANT_ID", "bad"), + ("SENTINEL_CLIENT_ID", "bad"), + ("SENTINEL_CLIENT_SECRET", ""), + ("SENTINEL_CLIENT_SECRET", "secret\r\n"), + ], +) +def test_invalid_server_credentials_never_reach_network(client, monkeypatch, key, value): + monkeypatch.setenv(key, value) + assert "error" in run() + client.post.assert_not_called() + + +@pytest.mark.parametrize( + "params", + [ + {"workspace_id": "bad/path"}, + {"workspace_id": "00000000-0000-0000-0000-000000000000"}, + {"source": "Microsoft Sentinel"}, + {"source": []}, + {"source": "a" * 65}, + {"expiration_days": 0}, + {"expiration_days": 366}, + {"confidence": -1}, + {"confidence": 101}, + {"confidence": True}, + {"dry_run": "maybe"}, + ], +) +def test_invalid_direct_configuration_precedes_network(client, params): + assert "error" in run(**params) + client.constructor.assert_not_called() + + +@pytest.mark.parametrize( + "metadata", + [ + {"uuid": None}, + {"uuid": "bad"}, + {"timestamp": None}, + {"timestamp": True}, + {"timestamp": "9999999999"}, + {"value": "bad..example"}, + {"value": "\ud800"}, + ], +) +def test_bad_source_identity_timestamp_or_value_is_skipped(client, metadata): + result = run(event([attribute(**metadata)])) + assert result["data"]["skipped"] == {"malformed value or identity/timestamp": 1} + client.constructor.assert_not_called() + + +def test_deleted_non_ids_and_unsupported_constraints_are_excluded(client): + data = [ + event( + [attribute(deleted=True), attribute(to_ids="false"), attribute("ip-dst|port", "192.0.2.1|443")], + [{"deleted": True, "Attribute": [attribute()]}], + ), + event([attribute()], deleted=True), + ] + assert run(data)["data"]["skipped"] == {"deleted or non-IDS": 2, "unsupported": 1} + client.constructor.assert_not_called() + + +def test_limits_and_malformed_containers_are_checked_before_network(client, monkeypatch): + assert "error" in run(event([attribute(value=f"host{index}.example") for index in range(1001)])) + monkeypatch.setattr(publish, "MAX_BATCH_BYTES", 10) + assert "error" in run() + for payload in [None, "bad", [1], {"Event": "bad"}, {"Object": [1]}]: + assert "error" in publish.handler(json.dumps({"data": payload, "params": {"workspace_id": WORKSPACE}})) + client.constructor.assert_not_called() + + +def test_deadline_is_checked_after_authentication(client, monkeypatch): + values = iter([0, publish.DEADLINE_SECONDS + 1]) + monkeypatch.setattr(publish.time, "monotonic", lambda: next(values)) + assert run()["data"]["failed"] == 1 + assert client.post.call_count == 1 + + +@pytest.mark.parametrize("module", [publish, wrapper]) +def test_request_validation_and_unapplied_filters(module): + assert module.handler() is False + for payload in [ + "invalid JSON", + "[]", + "null", + json.dumps({"params": []}), + json.dumps({"filteredItems": []}), + json.dumps({"matchingData": []}), + ]: + assert "error" in module.handler(payload) + + +def test_wrapper_direct_mode_uses_same_publisher_and_does_not_spawn(client, monkeypatch): + process = MagicMock() + monkeypatch.setattr(wrapper.subprocess, "run", process) + result = wrapper.handler(request(mode="direct")) + assert result["data"]["uploaded"] == 1 and result["result_key"] == "sentinel_publish" + process.assert_not_called() + + +@pytest.mark.parametrize( + "params,data", + [ + ({"mode": "unknown"}, {"Event": {"uuid": EVENT}}), + ({"dry_run": True}, {"Event": {"uuid": EVENT}}), + ({"misp2_venv": "relative"}, {"Event": {"uuid": EVENT}}), + ({"misp2_script": "/bad\x00path"}, {"Event": {"uuid": EVENT}}), + ({}, {"Event": {"uuid": "bad"}}), + ({}, [{"Event": {"uuid": EVENT}}]), + ], +) +def test_wrapper_invalid_mode_paths_or_input_never_execute(monkeypatch, params, data): + process = MagicMock() + monkeypatch.setattr(wrapper.subprocess, "run", process) + assert "error" in wrapper.handler(json.dumps({"data": data, "params": params})) + process.assert_not_called() + + +def test_missing_wrapper_installation_returns_a_workflow_error(monkeypatch): + monkeypatch.setattr(wrapper.os.path, "isfile", lambda path: False) + assert wrapper.handler(json.dumps({"data": {"Event": {"uuid": EVENT}}}))["data"] is False + + +def test_legacy_wrapper_default_calls_python_with_event_uuid_and_boolean_success(monkeypatch): + monkeypatch.setattr(wrapper.os.path, "isfile", lambda path: True) + process = MagicMock(return_value=MagicMock(returncode=0)) + monkeypatch.setattr(wrapper.subprocess, "run", process) + result = wrapper.handler( + json.dumps( + { + "data": {"Event": {"uuid": EVENT}}, + "params": {"misp2_venv": "/opt/env", "misp2_script": "/opt/app/script.py"}, + } + ) + ) + assert result == {"data": True} + assert process.call_args.args[0] == ["/opt/env/bin/python", "/opt/app/script.py", EVENT] + assert process.call_args.kwargs["cwd"] == "/opt/app" + assert process.call_args.kwargs["stdout"] == process.call_args.kwargs["stderr"] == subprocess.DEVNULL + + +@pytest.mark.parametrize( + "outcome", [MagicMock(returncode=1), subprocess.TimeoutExpired("script", 240), OSError("client-secret")] +) +def test_wrapper_failures_signal_workflow_error_without_logging_provider_output(monkeypatch, capsys, outcome): + monkeypatch.setattr(wrapper.os.path, "isfile", lambda path: True) + process = MagicMock() + if isinstance(outcome, Exception): + process.side_effect = outcome + else: + process.return_value = outcome + monkeypatch.setattr(wrapper.subprocess, "run", process) + result = wrapper.handler(json.dumps({"data": {"Event": {"uuid": EVENT}}, "params": {}})) + assert result["data"] is False and "error" in result and "client-secret" not in json.dumps(result) + assert capsys.readouterr().out == "" + + +def test_real_wrapper_process_remains_compatible(tmp_path): + venv = tmp_path / "venv" + (venv / "bin").mkdir(parents=True) + (venv / "bin" / "python").symlink_to(sys.executable) + script = tmp_path / "script.py" + script.write_text("import sys\nfrom pathlib import Path\nPath('received.txt').write_text(sys.argv[1])\n") + result = wrapper.handler( + json.dumps( + {"data": {"Event": {"uuid": EVENT}}, "params": {"misp2_venv": str(venv), "misp2_script": str(script)}} + ) + ) + assert result == {"data": True} and (tmp_path / "received.txt").read_text() == EVENT + + +def test_actual_requests_wire_contract_has_form_auth_and_json_upload(monkeypatch): + captured = [] + + def send(self, req, **kwargs): + captured.append(req) + if "oauth2" in req.url: + assert "Authorization" not in req.headers + assert parse_qs(req.body)["client_secret"] == ["client-secret"] + result = response({"access_token": "access-token", "token_type": "Bearer", "expires_in": 3600}) + else: + assert ( + req.headers["Authorization"] == "Bearer access-token" + and req.headers["Content-Type"] == "application/json" + ) + assert "api-version=2024-02-01-preview" in req.url + assert len(json.loads(req.body)["stixobjects"]) == 1 + result = response() + result.request, result.url = req, req.url + return result + + for name, value in [ + ("SENTINEL_TENANT_ID", TENANT), + ("SENTINEL_CLIENT_ID", CLIENT), + ("SENTINEL_CLIENT_SECRET", "client-secret"), + ]: + monkeypatch.setenv(name, value) + monkeypatch.setattr(requests.adapters.HTTPAdapter, "send", send) + assert run()["data"]["uploaded"] == 1 and len(captured) == 2 + + +class TestSentinelPublishHTTP(AsyncHTTPTestCase): + def get_app(self): + self.previous, self.cache = service.MODULES_HANDLERS.copy(), service.ListModules.CACHE + service.MODULES_HANDLERS.clear() + service.MODULES_HANDLERS.update( + { + "sentinel_publish": publish, + "type:sentinel_publish": "action_mod", + "export_sentinel": wrapper, + "type:export_sentinel": "action_mod", + } + ) + service.ListModules.CACHE = None + return Application([(r"/modules", service.ListModules), (r"/query", service.QueryModule)]) + + def tearDown(self): + service.MODULES_HANDLERS.clear() + service.MODULES_HANDLERS.update(self.previous) + service.ListModules.CACHE = self.cache + super().tearDown() + + def test_discovery_and_direct_mode_query(self): + descriptors = {item["name"]: item for item in json.loads(self.fetch("/modules").body)} + assert set(descriptors) == {"sentinel_publish", "export_sentinel"} + assert descriptors["sentinel_publish"]["meta"]["config"]["blocking"] is True + for name in descriptors: + payload = json.loads(request(mode="direct", dry_run=True)) + payload["module"] = name + result = self.fetch( + "/query", method="POST", headers={"Content-Type": "application/json"}, body=json.dumps(payload) + ) + assert result.code == 200 and json.loads(result.body)["data"]["planned"] == 1