From f43e904b4c00d7ac0d7910f7e39ea2ada7e00e48 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Wed, 8 Oct 2025 20:39:31 +0200 Subject: [PATCH 01/23] Initial commit of a CLI tool to help with NiFi flow migrations --- .cargo/config.toml | 5 + .github/workflows/nifi_migrate_pr.yaml | 67 ++++ .github/workflows/nifi_migrate_release.yaml | 61 ++++ .github/workflows/pre_commit.yaml | 22 ++ .gitignore | 6 + .markdownlint.yaml | 27 ++ .pre-commit-config.yaml | 59 ++++ .yamllint.yaml | 19 ++ CLAUDE.md | 53 +++ Cargo.lock | 342 ++++++++++++++++++++ Cargo.toml | 16 + Justfile | 45 +++ LICENSE | 29 +- LICENSES/Apache-2.0.txt | 73 +++++ README.md | 162 ++++++++++ REUSE.toml | 12 + deny.toml | 60 ++++ renovate.json | 7 + rust-toolchain.toml | 6 + src/cli.rs | 23 ++ src/lib.rs | 295 +++++++++++++++++ src/main.rs | 33 ++ src/rules/jolt_transform.rs | 217 +++++++++++++ src/rules/jolt_transform_record.rs | 108 +++++++ src/rules/mod.rs | 22 ++ 25 files changed, 1742 insertions(+), 27 deletions(-) create mode 100644 .cargo/config.toml create mode 100644 .github/workflows/nifi_migrate_pr.yaml create mode 100644 .github/workflows/nifi_migrate_release.yaml create mode 100644 .github/workflows/pre_commit.yaml create mode 100644 .gitignore create mode 100644 .markdownlint.yaml create mode 100644 .pre-commit-config.yaml create mode 100644 .yamllint.yaml create mode 100644 CLAUDE.md create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 Justfile create mode 100644 LICENSES/Apache-2.0.txt create mode 100644 README.md create mode 100644 REUSE.toml create mode 100644 deny.toml create mode 100644 renovate.json create mode 100644 rust-toolchain.toml create mode 100644 src/cli.rs create mode 100644 src/lib.rs create mode 100644 src/main.rs create mode 100644 src/rules/jolt_transform.rs create mode 100644 src/rules/jolt_transform_record.rs create mode 100644 src/rules/mod.rs diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..a3910ce --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2025 Stackable GmbH +# SPDX-License-Identifier: Apache-2.0 + +[alias] +nifi-migrate = ["run", "--"] diff --git a/.github/workflows/nifi_migrate_pr.yaml b/.github/workflows/nifi_migrate_pr.yaml new file mode 100644 index 0000000..4d1d337 --- /dev/null +++ b/.github/workflows/nifi_migrate_pr.yaml @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: 2025 Stackable GmbH +# SPDX-License-Identifier: Apache-2.0 + +name: Build nifi-migrate + +on: + pull_request: + paths: + - ".github/workflows/nifi_migrate_pr.yaml" + - "rust-toolchain.toml" + - "src/**.rs" + - "Cargo.*" + +env: + RUST_VERSION: 1.87.0 + +jobs: + # This job is always run to ensure we don't miss any new upstream advisories + cargo-deny: + name: Run cargo-deny + runs-on: ubuntu-latest + # Prevent sudden announcement of a new advisory from failing CI + continue-on-error: ${{ matrix.checks == 'advisories' }} + strategy: + matrix: + checks: + - advisories + - bans licenses sources + steps: + - name: Checkout Repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + submodules: recursive + + - name: Run cargo-deny + uses: EmbarkStudios/cargo-deny-action@f2ba7abc2abebaf185c833c3961145a3c275caad # v2.0.13 + with: + command: check ${{ matrix.checks }} + + build: + name: Build nifi-migrate + needs: + - cargo-deny + strategy: + fail-fast: false + matrix: + targets: + - { target: aarch64-unknown-linux-gnu, os: ubuntu-24.04-arm } + - { target: x86_64-unknown-linux-gnu, os: ubuntu-latest } + - { target: aarch64-apple-darwin, os: macos-latest } + runs-on: ${{ matrix.targets.os }} + steps: + - name: Checkout + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + + - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 + with: + toolchain: ${{ env.RUST_VERSION }} + targets: ${{ matrix.targets.target }} + + - name: Build Binary + env: + TARGET: ${{ matrix.targets.target }} + run: cargo build --target "$TARGET" --package nifi-migrate diff --git a/.github/workflows/nifi_migrate_release.yaml b/.github/workflows/nifi_migrate_release.yaml new file mode 100644 index 0000000..e429853 --- /dev/null +++ b/.github/workflows/nifi_migrate_release.yaml @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: 2025 Stackable GmbH +# SPDX-License-Identifier: Apache-2.0 + +name: Release nifi-migrate + +on: + push: + tags: + - "nifi-migrate-[0-9]+.[0-9]+.[0-9]+**" + +env: + RUST_VERSION: 1.87.0 + +jobs: + create-release: + name: Create Draft Release + runs-on: ubuntu-latest + steps: + - name: Create Draft Release + uses: softprops/action-gh-release@72f2c25fcb47643c292f7107632f7a47c1df5cd8 # v2.3.2 + with: + draft: true + + build: + name: Build nifi-migrate + needs: + - create-release + strategy: + fail-fast: false + matrix: + targets: + - { target: aarch64-unknown-linux-gnu, os: ubuntu-24.04-arm } + - { target: x86_64-unknown-linux-gnu, os: ubuntu-latest } + - { target: aarch64-apple-darwin, os: macos-latest } + runs-on: ${{ matrix.targets.os }} + steps: + - name: Checkout + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + + - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 + with: + toolchain: ${{ env.RUST_VERSION }} + targets: ${{ matrix.targets.target }} + + - name: Build Binary + env: + TARGET: ${{ matrix.targets.target }} + run: cargo build --target "$TARGET" --release --package nifi-migrate + + - name: Rename Binary + env: + TARGET: ${{ matrix.targets.target }} + run: mv "target/$TARGET/release/nifi-migrate" "nifi-migrate-$TARGET" + + - name: Upload Artifact to Release + uses: softprops/action-gh-release@72f2c25fcb47643c292f7107632f7a47c1df5cd8 # v2.3.2 + with: + draft: false + files: nifi-migrate-${{ matrix.targets.target }} diff --git a/.github/workflows/pre_commit.yaml b/.github/workflows/pre_commit.yaml new file mode 100644 index 0000000..fc4ebf3 --- /dev/null +++ b/.github/workflows/pre_commit.yaml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: 2025 Stackable GmbH +# SPDX-License-Identifier: Apache-2.0 + +name: pre-commit + +on: + pull_request: + +env: + RUST_TOOLCHAIN_VERSION: "1.87.0" + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + fetch-depth: 0 + - uses: stackabletech/actions/run-pre-commit@a5d39a4eb109bb6af3c152800701c86e98bfe1a5 # v0.10.1 + with: + rust: ${{ env.RUST_TOOLCHAIN_VERSION }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..11d2e57 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +# Rust build artifacts +/target +Cargo.lock + +# Claude Code local settings +.claude/ diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 0000000..3b47e81 --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,27 @@ +--- +# SPDX-FileCopyrightText: 2025 Stackable GmbH +# SPDX-License-Identifier: Apache-2.0 + +# All defaults or options can be checked here: +# https://github.com/DavidAnson/markdownlint/blob/main/schema/.markdownlint.yaml + +# Default state for all rules +default: true + +# MD013/line-length - Line length +MD013: + # Number of characters + line_length: 9999 + # Number of characters for headings + heading_line_length: 9999 + # Number of characters for code blocks + code_block_line_length: 9999 + +# MD024/no-duplicate-heading/no-duplicate-header - Multiple headings with the same content +MD024: + # Only check sibling headings + siblings_only: true + +# MD033/no-inline-html Inline HTML +MD033: + allowed_elements: [details, summary, img] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..e52e93c --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: 2025 Stackable GmbH +# SPDX-License-Identifier: Apache-2.0 +--- +fail_fast: false +exclude: \.patch$ + +default_language_version: + node: system + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # 6.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: detect-aws-credentials + args: ["--allow-missing-credentials"] + - id: detect-private-key + + - repo: https://github.com/adrienverge/yamllint + rev: 79a6b2b1392eaf49cdd32ac4f14be1a809bbd8f7 # 1.37.1 + hooks: + - id: yamllint + + - repo: https://github.com/igorshubovych/markdownlint-cli + rev: 192ad822316c3a22fb3d3cc8aa6eafa0b8488360 # 0.45.0 + hooks: + - id: markdownlint + + - repo: https://github.com/rhysd/actionlint + rev: 03d0035246f3e81f36aed592ffb4bebf33a03106 # 1.7.7 + hooks: + - id: actionlint + + - repo: local + hooks: + - id: cargo-test + name: cargo-test + language: system + entry: cargo test + stages: [pre-commit, pre-merge-commit] + pass_filenames: false + files: \.rs$|Cargo\.(toml|lock) + + - id: cargo-rustfmt + name: cargo-rustfmt + language: system + entry: cargo fmt --all -- --check + stages: [pre-commit, pre-merge-commit] + pass_filenames: false + files: \.rs$ + + - id: cargo-clippy + name: cargo-clippy + language: system + entry: cargo clippy --all-targets -- -D warnings + stages: [pre-commit, pre-merge-commit] + pass_filenames: false + files: \.rs$ diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 0000000..de61902 --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,19 @@ +--- +# SPDX-FileCopyrightText: 2025 Stackable GmbH +# SPDX-License-Identifier: Apache-2.0 + +extends: default + +ignore: | + deploy/helm/**/templates + +rules: + line-length: disable + truthy: + check-keys: false + comments: + min-spaces-from-content: 1 # Needed due to https://github.com/adrienverge/yamllint/issues/443 + indentation: disable + braces: + max-spaces-inside: 1 + max-spaces-inside-empty: 0 diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..5b6f307 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,53 @@ + + +# Claude Code Instructions + +When making changes to this project, always run the following checks in order: + +1. **Format code**: `cargo fmt` +2. **Lint code**: `cargo clippy --all-targets -- -D warnings` +3. **Run tests**: `cargo test` +4. **Check REUSE compliance**: `reuse lint` +5. **Lint GitHub Actions** (after changes to `.github/workflows/*.yaml`): `actionlint` +6. **Check dependencies** (after changes to `Cargo.toml` or `Cargo.lock`): `cargo deny check` + +All checks must pass before considering the work complete. + +## Convenient Commands + +You can run all checks at once using: + +- `just all` - Run all checks individually (fmt, clippy, test, reuse, actionlint, deny) +- `just pre-commit` - Run pre-commit hooks on all files + +To install pre-commit git hooks: + +- `just pre-commit-install` + +## Project-Specific Notes + +- This project follows FSFE REUSE 3.3 specification +- All source files must have SPDX headers +- Files covered by `REUSE.toml` don't need individual headers +- License: Apache-2.0 +- Copyright holder: Stackable GmbH + +## Adding New Migration Rules + +When adding new migration rules, follow these steps in order: + +1. Create a new file in `src/rules/` (e.g., `my_rule.rs`) +2. Implement the `MigrationRule` trait with SPDX headers +3. Add the module to `src/rules/mod.rs` and export it +4. Register it in `Migrator::new()` in `src/lib.rs` +5. Add comprehensive tests in the rule file +6. **Update README.md** in the "Supported Migrations" section: + - Add a new subsection describing the migration + - Include the old and new type/bundle values + - Explain why the migration is needed +7. Run all checks listed above (fmt, clippy, test, reuse lint) + +All steps must be completed before considering the migration rule complete. diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..6871000 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,342 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anstream" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + +[[package]] +name = "clap" +version = "4.5.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2134bb3ea021b78629caa971416385309e0131b351b25e01dc16fb54e1b5fae" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2ba64afa3c0a6df7fa517765e31314e983f51dda798ffba27b988194fb65dc9" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfd7eae0b0f1a6e63d4b13c9c478de77c2eb546fba158ad50b4203dc24b9f9c" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "nifi-migrate" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "pretty_assertions", + "serde_json", +] + +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" + +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + +[[package]] +name = "proc-macro2" +version = "1.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", + "serde_core", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..3715b91 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "nifi-migrate" +version = "0.1.0" +edition = "2024" +license = "Apache-2.0" +authors = ["Stackable GmbH"] +description = "CLI tool for migrating Apache NiFi flow.json files between versions" +repository = "https://github.com/stackabletech/nifi-migrate" + +[dependencies] +clap = { version = "4.5", features = ["derive"] } +serde_json = "1.0" +anyhow = "1.0" + +[dev-dependencies] +pretty_assertions = "1.4" diff --git a/Justfile b/Justfile new file mode 100644 index 0000000..318b092 --- /dev/null +++ b/Justfile @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: 2025 Stackable GmbH +# SPDX-License-Identifier: Apache-2.0 + +# Run all checks (same as pre-commit) +all: fmt clippy test reuse actionlint deny + +# Format code +fmt: + cargo fmt + +# Lint code +clippy: + cargo clippy --all-targets -- -D warnings + +# Run tests +test: + cargo test + +# Check REUSE compliance +reuse: + reuse lint + +# Lint GitHub Actions workflows +actionlint: + actionlint + +# Check dependencies +deny: + cargo deny check + +# Run pre-commit hooks on all files +pre-commit: + pre-commit run --all-files + +# Install pre-commit hooks +pre-commit-install: + pre-commit install + +# Build release binary +build: + cargo build --release + +# Run the tool +run input output="flow-migrated.json" *args="": + cargo run -- --input {{input}} --output {{output}} {{args}} diff --git a/LICENSE b/LICENSE index 261eeb9..7fb5aea 100644 --- a/LICENSE +++ b/LICENSE @@ -153,7 +153,7 @@ 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be + negligent acts) or agreed in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the @@ -162,7 +162,7 @@ other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. - 9. Accepting Warranty or Additional Liability. While redistributing + 9. Accepting Warranty or Additional Support. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this @@ -174,28 +174,3 @@ of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/LICENSES/Apache-2.0.txt b/LICENSES/Apache-2.0.txt new file mode 100644 index 0000000..137069b --- /dev/null +++ b/LICENSES/Apache-2.0.txt @@ -0,0 +1,73 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + + (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + + You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..cbe7a76 --- /dev/null +++ b/README.md @@ -0,0 +1,162 @@ + + +# nifi-migrate + +A Rust CLI tool for migrating Apache NiFi flow.json files between versions. + +## Features + +- **Non-destructive**: Reads from input file and writes to a separate output file +- **Recursive**: Processes nested process groups automatically +- **Extensible**: Easy to add new migration rules via the trait-based system +- **Safe**: Distinguishes between processors and controller services to avoid incorrect migrations + +## Supported Migrations + +### JoltTransformJSON Processor + +- **Type**: `org.apache.nifi.processors.standard.JoltTransformJSON` → `org.apache.nifi.processors.jolt.JoltTransformJSON` +- **Bundle artifact**: `nifi-standard-nar` → `nifi-jolt-nar` +- **Reason**: In NiFi 2.x, Jolt processors were moved to a separate bundle +- **Reference**: [NIFI-12554](https://issues.apache.org/jira/browse/NIFI-12554) + +### JoltTransformRecord Processor + +- **Type**: `org.apache.nifi.processors.jolt.record.JoltTransformRecord` → `org.apache.nifi.processors.jolt.JoltTransformRecord` +- **Bundle artifact**: `nifi-jolt-record-nar` → `nifi-jolt-nar` +- **Reason**: Consolidated into the main jolt bundle +- **Reference**: [NIFI-12554](https://issues.apache.org/jira/browse/NIFI-12554) + +## Installation + +```bash +cargo build --release +``` + +The binary will be available at `target/release/nifi-migrate` + +## Usage + +Basic usage: + +```bash +nifi-migrate --input flow.json --output flow-migrated.json +``` + +Or using the cargo alias: + +```bash +cargo nifi-migrate --input flow.json --output flow-migrated.json +``` + +With pretty-printed JSON output: + +```bash +nifi-migrate --input flow.json --output flow-migrated.json --pretty +``` + +### Options + +- `-i, --input `: Input flow.json file (required) +- `-o, --output `: Output flow.json file (required) +- `-p, --pretty`: Pretty-print the output JSON (optional, default is compact) +- `-h, --help`: Show help information +- `-V, --version`: Show version information + +## Important Notes + +### JSON Formatting + +⚠️ **The tool will reformat your JSON file.** By default, output is compact (single line). Use `--pretty` flag for human-readable formatting with indentation. + +The order of JSON keys may also change as the file is parsed and reserialized. While this doesn't affect NiFi's ability to read the file, it may make git diffs larger. + +## Adding New Migration Rules + +To add a new migration rule: + +1. Create a new file in `src/rules/` (e.g., `my_rule.rs`) +1. Implement the `MigrationRule` trait: + +```rust +use super::MigrationRule; +use serde_json::Value; + +pub struct MyMigrationRule; + +impl MigrationRule for MyMigrationRule { + fn applies(&self, processor: &Value) -> bool { + // Check if this rule applies + processor.get("type") + .and_then(|t| t.as_str()) + .map(|t| t == "org.apache.nifi.processors.old.Processor") + .unwrap_or(false) + } + + fn apply(&self, processor: &mut Value) -> bool { + // Apply the migration + if let Some(type_field) = processor.get_mut("type") { + *type_field = Value::String("org.apache.nifi.processors.new.Processor".to_string()); + return true; + } + false + } + + fn description(&self) -> String { + "Migrate Processor from old to new package".to_string() + } +} +``` + +1. Add your rule to `src/rules/mod.rs`: + +```rust +mod my_rule; +pub use my_rule::MyMigrationRule; +``` + +1. Register it in `Migrator::new()` in `src/lib.rs`: + +```rust +pub fn new() -> Self { + Self { + rules: vec![ + Box::new(JoltTransformMigration), + Box::new(MyMigrationRule), // Add your rule here + ], + } +} +``` + +1. Add tests to verify your rule works correctly + +## License + +Licensed under the Apache License, Version 2.0. See [LICENSE](LICENSE) for details. + +## Contributing + +Contributions are welcome! Please ensure: + +- No new clippy lints (`cargo clippy --all-targets -- -D warnings`) +- All tests pass (`cargo test`) +- Code is formatted (`cargo fmt`) +- REUSE compliance (`reuse lint`) +- Add tests for new migration rules + +### Running All Checks + +You can run all checks at once using: + +```bash +just all +``` + +Or run pre-commit hooks: + +```bash +just pre-commit +``` diff --git a/REUSE.toml b/REUSE.toml new file mode 100644 index 0000000..fb5ca5f --- /dev/null +++ b/REUSE.toml @@ -0,0 +1,12 @@ +version = 1 + +[[annotations]] +path = [ + "Cargo.toml", + "Cargo.lock", + ".gitignore", + "**/*.json", +] +precedence = "aggregate" +SPDX-FileCopyrightText = "2025 Stackable GmbH" +SPDX-License-Identifier = "Apache-2.0" diff --git a/deny.toml b/deny.toml new file mode 100644 index 0000000..d0a31ed --- /dev/null +++ b/deny.toml @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: 2025 Stackable GmbH +# SPDX-License-Identifier: Apache-2.0 + +# This file is the source of truth for all our repos! +# This includes repos not templated by operator-templating, please copy/paste the file for this repos. + +# TIP: Use "cargo deny check" to check if everything is fine + +[graph] +targets = [ + { triple = "x86_64-unknown-linux-gnu" }, + { triple = "aarch64-unknown-linux-gnu" }, + { triple = "x86_64-unknown-linux-musl" }, + { triple = "aarch64-apple-darwin" }, + { triple = "x86_64-apple-darwin" }, +] + +[advisories] +yanked = "deny" + +[bans] +multiple-versions = "allow" + +[licenses] +unused-allowed-license = "allow" +confidence-threshold = 1.0 +allow = [ + "Apache-2.0", + "BSD-2-Clause", + "BSD-3-Clause", + "CC0-1.0", + "ISC", + "LicenseRef-ring", + "LicenseRef-webpki", + "MIT", + "MPL-2.0", + "OpenSSL", # Needed for the ring and/or aws-lc-sys crate. See https://github.com/stackabletech/operator-templating/pull/464 for details + "Unicode-3.0", + "Unicode-DFS-2016", + "Zlib", + "Unlicense", +] +private = { ignore = true } + +[[licenses.clarify]] +name = "ring" +expression = "LicenseRef-ring" +license-files = [{ path = "LICENSE", hash = 0xbd0eed23 }] + +[[licenses.clarify]] +name = "webpki" +expression = "LicenseRef-webpki" +license-files = [{ path = "LICENSE", hash = 0x001c7e6c }] + +[sources] +unknown-registry = "deny" +unknown-git = "deny" + +[sources.allow-org] +github = ["stackabletech"] diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..43b32fb --- /dev/null +++ b/renovate.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "github>stackabletech/.github:renovate-config", + "docker:pinDigests" + ] +} diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..3a7d4c2 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: 2025 Stackable GmbH +# SPDX-License-Identifier: Apache-2.0 + +[toolchain] +channel = "1.87.0" +profile = "default" diff --git a/src/cli.rs b/src/cli.rs new file mode 100644 index 0000000..a7eb364 --- /dev/null +++ b/src/cli.rs @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2025 Stackable GmbH +// SPDX-License-Identifier: Apache-2.0 + +use clap::Parser; +use std::path::PathBuf; + +#[derive(Parser)] +#[command(name = "nifi-migrate")] +#[command(version)] +#[command(about = "Migrate NiFi 1.x flow.json files to NiFi 2.x format", long_about = None)] +pub struct Args { + /// Input flow.json file + #[arg(short, long)] + pub input: PathBuf, + + /// Output flow.json file + #[arg(short, long)] + pub output: PathBuf, + + /// Pretty-print the output JSON (default: compact) + #[arg(short, long)] + pub pretty: bool, +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..8f01b5f --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,295 @@ +// SPDX-FileCopyrightText: 2025 Stackable GmbH +// SPDX-License-Identifier: Apache-2.0 + +mod rules; + +use anyhow::{Context, Result}; +use rules::{JoltTransformMigration, JoltTransformRecordMigration, MigrationRule}; +use serde_json::Value; +use std::fs; +use std::path::Path; + +/// Represents a change that would be made during migration +#[derive(Debug, Clone)] +pub struct MigrationChange { + pub processor_id: String, + pub processor_name: String, + pub rule_description: String, +} + +/// Main migration engine +pub struct Migrator { + rules: Vec>, +} + +impl Migrator { + /// Create a new migrator with default rules + pub fn new() -> Self { + Self { + rules: vec![ + Box::new(JoltTransformMigration), + Box::new(JoltTransformRecordMigration), + ], + } + } + + /// Migrate a flow JSON file + pub fn migrate_file( + &self, + input_path: &Path, + output_path: &Path, + pretty: bool, + ) -> Result> { + // Validate input exists + if !input_path.exists() { + anyhow::bail!("Input file does not exist: {}", input_path.display()); + } + + // Warn if input and output are the same + let canonical_input = input_path + .canonicalize() + .with_context(|| format!("Failed to resolve input path: {}", input_path.display()))?; + + if let Ok(canonical_output) = output_path.canonicalize() { + if canonical_input == canonical_output { + anyhow::bail!( + "Input and output paths are the same. This would overwrite the original file." + ); + } + } + + // Validate output directory exists + if let Some(parent) = output_path.parent() { + if !parent.as_os_str().is_empty() && !parent.exists() { + anyhow::bail!("Output directory does not exist: {}", parent.display()); + } + } + + // It's not perfect reading it all in memory, but I decided it's fine for now. + // I tried it on a reasonably large file and it was fine. + // We can switch to streaming if it's ever needed. + let content = fs::read_to_string(input_path) + .with_context(|| format!("Failed to read input file: {}", input_path.display()))?; + + let mut flow: Value = serde_json::from_str(&content) + .with_context(|| format!("Failed to parse JSON from: {}", input_path.display()))?; + + let changes = self.migrate_flow(&mut flow)?; + + if !changes.is_empty() { + let output = if pretty { + serde_json::to_string_pretty(&flow) + } else { + serde_json::to_string(&flow) + } + .context("Failed to serialize output JSON")?; + + fs::write(output_path, output).with_context(|| { + format!("Failed to write output file: {}", output_path.display()) + })?; + } + + Ok(changes) + } + + /// Migrate a flow JSON value in-place + fn migrate_flow(&self, flow: &mut Value) -> Result> { + let mut changes = Vec::new(); + self.process_value(flow, &mut changes); + Ok(changes) + } + + /// Recursively process a JSON value looking for processors + fn process_value(&self, value: &mut Value, changes: &mut Vec) { + self.process_value_with_context(value, None, changes); + } + + /// Recursively process a JSON value with parent key context. + /// The NiFi JSON is not very deep so recursive should not cause any issues here. + fn process_value_with_context( + &self, + value: &mut Value, + parent_key: Option<&str>, + changes: &mut Vec, + ) { + match value { + Value::Object(map) => { + // Check if this object is a processor (but not a controller service) + // Controller services have the same structure as processors (type + bundle) + // but appear under "controllerServices" key instead of "processors" key + // This entire matching thing (as well as the migration rules) can be made smarter + // as needed. For now, we only have two rules and both are for processors so it's + // fine as is. + let is_processor = map.contains_key("type") + && map.contains_key("bundle") + && parent_key != Some("controllerServices"); + + if is_processor { + self.process_processor(value, changes); + } + + // Recursively process all nested values + // Need to re-borrow to avoid double mutable borrow + if let Value::Object(map) = value { + for (key, val) in map.iter_mut() { + self.process_value_with_context(val, Some(key), changes); + } + } + } + Value::Array(arr) => { + for item in arr.iter_mut() { + self.process_value_with_context(item, parent_key, changes); + } + } + _ => {} + } + } + + /// Process a single processor object + fn process_processor(&self, processor: &mut Value, changes: &mut Vec) { + for rule in &self.rules { + if rule.applies(processor) && rule.apply(processor) { + let processor_id = processor + .get("identifier") + .or_else(|| processor.get("id")) + .and_then(|v| v.as_str()) + .unwrap_or("unknown") + .to_string(); + + let processor_name = processor + .get("name") + .and_then(|v| v.as_str()) + .unwrap_or("unnamed") + .to_string(); + + changes.push(MigrationChange { + processor_id, + processor_name, + rule_description: rule.description(), + }); + } + } + } +} + +impl Default for Migrator { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use serde_json::json; + + #[test] + fn test_jolt_transform_migration() { + let mut processor = json!({ + "identifier": "test-id-123", + "name": "JoltTransform", + "type": "org.apache.nifi.processors.standard.JoltTransformJSON", + "bundle": { + "artifact": "nifi-standard-nar", + "group": "org.apache.nifi", + "version": "1.25.0" + } + }); + + let rule = JoltTransformMigration; + assert!(rule.applies(&processor)); + assert!(rule.apply(&mut processor)); + + assert_eq!( + processor.get("type").and_then(|v| v.as_str()), + Some("org.apache.nifi.processors.jolt.JoltTransformJSON") + ); + assert_eq!( + processor + .get("bundle") + .and_then(|b| b.get("artifact")) + .and_then(|v| v.as_str()), + Some("nifi-jolt-nar") + ); + } + + #[test] + fn test_nested_process_groups() { + let mut flow = json!({ + "processGroups": [ + { + "processors": [ + { + "identifier": "proc-1", + "name": "Jolt1", + "type": "org.apache.nifi.processors.standard.JoltTransformJSON", + "bundle": { + "artifact": "nifi-standard-nar" + } + } + ], + "processGroups": [ + { + "processors": [ + { + "identifier": "proc-2", + "name": "Jolt2", + "type": "org.apache.nifi.processors.standard.JoltTransformJSON", + "bundle": { + "artifact": "nifi-standard-nar" + } + } + ] + } + ] + } + ] + }); + + let migrator = Migrator::new(); + let changes = migrator.migrate_flow(&mut flow).unwrap(); + + assert_eq!(changes.len(), 2); + assert!(changes.iter().any(|c| c.processor_id == "proc-1")); + assert!(changes.iter().any(|c| c.processor_id == "proc-2")); + } + + #[test] + fn test_only_migrates_jolt_processors() { + let mut flow = json!({ + "processors": [ + { + "identifier": "other-proc", + "name": "SomeOtherProcessor", + "type": "org.apache.nifi.processors.standard.LogAttribute", + "bundle": { + "artifact": "nifi-standard-nar" + } + }, + { + "identifier": "jolt-proc", + "name": "JoltProcessor", + "type": "org.apache.nifi.processors.standard.JoltTransformJSON", + "bundle": { + "artifact": "nifi-standard-nar" + } + } + ] + }); + + let migrator = Migrator::new(); + let changes = migrator.migrate_flow(&mut flow).unwrap(); + + assert_eq!(changes.len(), 1); + assert_eq!(changes[0].processor_id, "jolt-proc"); + + // Verify other processor unchanged + assert_eq!( + flow["processors"][0]["bundle"]["artifact"], + "nifi-standard-nar" + ); + // Verify jolt processor changed + assert_eq!(flow["processors"][1]["bundle"]["artifact"], "nifi-jolt-nar"); + } +} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..b2d967b --- /dev/null +++ b/src/main.rs @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: 2025 Stackable GmbH +// SPDX-License-Identifier: Apache-2.0 + +mod cli; + +use anyhow::Result; +use clap::Parser; +use cli::Args; +use nifi_migrate::Migrator; + +fn main() -> Result<()> { + let args = Args::parse(); + + let migrator = Migrator::new(); + let changes = migrator.migrate_file(&args.input, &args.output, args.pretty)?; + + if changes.is_empty() { + println!("No migrations needed."); + } else { + println!("Migration complete. Changes made:"); + + for change in &changes { + println!( + " - {} ({}): {}", + change.processor_name, change.processor_id, change.rule_description + ); + } + + println!("\nOutput written to: {}", args.output.display()); + } + + Ok(()) +} diff --git a/src/rules/jolt_transform.rs b/src/rules/jolt_transform.rs new file mode 100644 index 0000000..e0fb3b5 --- /dev/null +++ b/src/rules/jolt_transform.rs @@ -0,0 +1,217 @@ +// SPDX-FileCopyrightText: 2025 Stackable GmbH +// SPDX-License-Identifier: Apache-2.0 + +use super::MigrationRule; +use serde_json::Value; + +/// Migration rule for JoltTransformJSON processor +/// +/// Migrates `org.apache.nifi.processors.standard.JoltTransformJSON` to +/// `org.apache.nifi.processors.jolt.JoltTransformJSON` and updates the +/// bundle from `nifi-standard-nar` to `nifi-jolt-nar`. +/// +/// Reference: +pub struct JoltTransformMigration; + +impl MigrationRule for JoltTransformMigration { + fn applies(&self, processor: &Value) -> bool { + processor + .get("type") + .and_then(|t| t.as_str()) + .map(|t| t == "org.apache.nifi.processors.standard.JoltTransformJSON") + .unwrap_or(false) + } + + fn apply(&self, processor: &mut Value) -> bool { + let mut changed = false; + + // Update the type field + if let Some(type_field) = processor.get_mut("type") { + if type_field.as_str() == Some("org.apache.nifi.processors.standard.JoltTransformJSON") + { + *type_field = + Value::String("org.apache.nifi.processors.jolt.JoltTransformJSON".to_string()); + changed = true; + } + } + + // Update the bundle artifact field + if let Some(bundle) = processor.get_mut("bundle") { + if let Some(artifact) = bundle.get_mut("artifact") { + if artifact.as_str() == Some("nifi-standard-nar") { + *artifact = Value::String("nifi-jolt-nar".to_string()); + changed = true; + } + } + } + + changed + } + + fn description(&self) -> String { + "Migrate JoltTransformJSON from standard to jolt bundle".to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use serde_json::json; + + #[test] + fn test_jolt_transform_migration() { + let mut processor = json!({ + "identifier": "test-id-123", + "name": "JoltTransform", + "type": "org.apache.nifi.processors.standard.JoltTransformJSON", + "bundle": { + "artifact": "nifi-standard-nar", + "group": "org.apache.nifi", + "version": "1.25.0" + } + }); + + let rule = JoltTransformMigration; + assert!(rule.applies(&processor)); + assert!(rule.apply(&mut processor)); + + assert_eq!( + processor.get("type").and_then(|v| v.as_str()), + Some("org.apache.nifi.processors.jolt.JoltTransformJSON") + ); + assert_eq!( + processor + .get("bundle") + .and_then(|b| b.get("artifact")) + .and_then(|v| v.as_str()), + Some("nifi-jolt-nar") + ); + } + + #[test] + fn test_full_real_world_processor() { + let mut processor = json!({ + "identifier": "347aaa3b-2b3a-30c5-932d-58a109f9478f", + "instanceIdentifier": "e70d7d94-e92a-3472-b1c7-338e291af5e9", + "name": "JoltTransformJSON", + "comments": "", + "position": { + "x": 3368.0, + "y": 200.0 + }, + "type": "org.apache.nifi.processors.standard.JoltTransformJSON", + "bundle": { + "group": "org.apache.nifi", + "artifact": "nifi-standard-nar", + "version": "1.18.0" + }, + "properties": {}, + "propertyDescriptors": { + "jolt-spec": { + "name": "jolt-spec", + "displayName": "jolt-spec", + "identifiesControllerService": false, + "sensitive": true, + "dynamic": false + }, + "jolt-transform": { + "name": "jolt-transform", + "displayName": "jolt-transform", + "identifiesControllerService": false, + "sensitive": true, + "dynamic": false + }, + "pretty_print": { + "name": "pretty_print", + "displayName": "pretty_print", + "identifiesControllerService": false, + "sensitive": true, + "dynamic": false + }, + "Transform Cache Size": { + "name": "Transform Cache Size", + "displayName": "Transform Cache Size", + "identifiesControllerService": false, + "sensitive": true, + "dynamic": false + } + }, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": ["failure"], + "scheduledState": "ENABLED", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "5e61ca9d-43f8-3176-b706-2404009bcc5b" + }); + + let rule = JoltTransformMigration; + + // Verify it applies to this processor + assert!( + rule.applies(&processor), + "Rule should apply to JoltTransformJSON processor" + ); + + // Apply the migration + assert!(rule.apply(&mut processor), "Migration should make changes"); + + // Verify the type was changed + assert_eq!( + processor.get("type").and_then(|v| v.as_str()), + Some("org.apache.nifi.processors.jolt.JoltTransformJSON"), + "Processor type should be updated" + ); + + // Verify the bundle artifact was changed + assert_eq!( + processor + .get("bundle") + .and_then(|b| b.get("artifact")) + .and_then(|v| v.as_str()), + Some("nifi-jolt-nar"), + "Bundle artifact should be updated to nifi-jolt-nar" + ); + + // Verify other bundle fields remain unchanged + assert_eq!( + processor + .get("bundle") + .and_then(|b| b.get("group")) + .and_then(|v| v.as_str()), + Some("org.apache.nifi"), + "Bundle group should remain unchanged" + ); + assert_eq!( + processor + .get("bundle") + .and_then(|b| b.get("version")) + .and_then(|v| v.as_str()), + Some("1.18.0"), + "Bundle version should remain unchanged" + ); + + // Verify other fields remain unchanged + assert_eq!( + processor.get("identifier").and_then(|v| v.as_str()), + Some("347aaa3b-2b3a-30c5-932d-58a109f9478f"), + "Identifier should remain unchanged" + ); + assert_eq!( + processor.get("name").and_then(|v| v.as_str()), + Some("JoltTransformJSON"), + "Name should remain unchanged" + ); + } +} diff --git a/src/rules/jolt_transform_record.rs b/src/rules/jolt_transform_record.rs new file mode 100644 index 0000000..90e8d38 --- /dev/null +++ b/src/rules/jolt_transform_record.rs @@ -0,0 +1,108 @@ +// SPDX-FileCopyrightText: 2025 Stackable GmbH +// SPDX-License-Identifier: Apache-2.0 + +use super::MigrationRule; +use serde_json::Value; + +/// Migration rule for JoltTransformRecord processor +/// +/// Migrates `org.apache.nifi.processors.jolt.record.JoltTransformRecord` to +/// `org.apache.nifi.processors.jolt.JoltTransformRecord` and updates the +/// bundle from `nifi-jolt-record-nar` to `nifi-jolt-nar`. +/// +/// Reference: +pub struct JoltTransformRecordMigration; + +impl MigrationRule for JoltTransformRecordMigration { + fn applies(&self, processor: &Value) -> bool { + processor + .get("type") + .and_then(|t| t.as_str()) + .map(|t| t == "org.apache.nifi.processors.jolt.record.JoltTransformRecord") + .unwrap_or(false) + } + + fn apply(&self, processor: &mut Value) -> bool { + let mut changed = false; + + // Update the type field + if let Some(type_field) = processor.get_mut("type") { + if type_field.as_str() + == Some("org.apache.nifi.processors.jolt.record.JoltTransformRecord") + { + *type_field = Value::String( + "org.apache.nifi.processors.jolt.JoltTransformRecord".to_string(), + ); + changed = true; + } + } + + // Update the bundle artifact field + if let Some(bundle) = processor.get_mut("bundle") { + if let Some(artifact) = bundle.get_mut("artifact") { + if artifact.as_str() == Some("nifi-jolt-record-nar") { + *artifact = Value::String("nifi-jolt-nar".to_string()); + changed = true; + } + } + } + + changed + } + + fn description(&self) -> String { + "Migrate JoltTransformRecord from jolt-record to jolt bundle".to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use serde_json::json; + + #[test] + fn test_jolt_transform_record_migration() { + let mut processor = json!({ + "identifier": "test-id-456", + "name": "JoltTransformRecord", + "type": "org.apache.nifi.processors.jolt.record.JoltTransformRecord", + "bundle": { + "artifact": "nifi-jolt-record-nar", + "group": "org.apache.nifi", + "version": "1.27.0" + } + }); + + let rule = JoltTransformRecordMigration; + assert!(rule.applies(&processor)); + assert!(rule.apply(&mut processor)); + + assert_eq!( + processor.get("type").and_then(|v| v.as_str()), + Some("org.apache.nifi.processors.jolt.JoltTransformRecord") + ); + assert_eq!( + processor + .get("bundle") + .and_then(|b| b.get("artifact")) + .and_then(|v| v.as_str()), + Some("nifi-jolt-nar") + ); + } + + #[test] + fn test_does_not_apply_to_other_processors() { + let processor = json!({ + "identifier": "other-proc", + "name": "SomeOtherProcessor", + "type": "org.apache.nifi.processors.standard.LogAttribute", + "bundle": { + "artifact": "nifi-standard-nar" + } + }); + + let rule = JoltTransformRecordMigration; + assert!(!rule.applies(&processor)); + } +} diff --git a/src/rules/mod.rs b/src/rules/mod.rs new file mode 100644 index 0000000..eb02eb6 --- /dev/null +++ b/src/rules/mod.rs @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: 2025 Stackable GmbH +// SPDX-License-Identifier: Apache-2.0 + +mod jolt_transform; +mod jolt_transform_record; + +pub use jolt_transform::JoltTransformMigration; +pub use jolt_transform_record::JoltTransformRecordMigration; + +use serde_json::Value; + +/// Represents a migration rule that can be applied to processors +pub trait MigrationRule { + /// Check if this rule applies to the given processor + fn applies(&self, processor: &Value) -> bool; + + /// Apply the migration to the processor, returning true if changes were made + fn apply(&self, processor: &mut Value) -> bool; + + /// Get a description of what this rule does + fn description(&self) -> String; +} From 487462e4842ee6c3ddbf5ba4339731b134d8fee2 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Wed, 8 Oct 2025 22:46:29 +0200 Subject: [PATCH 02/23] Fix Justfile to properly quote filenames --- Justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Justfile b/Justfile index 318b092..14e2728 100644 --- a/Justfile +++ b/Justfile @@ -42,4 +42,4 @@ build: # Run the tool run input output="flow-migrated.json" *args="": - cargo run -- --input {{input}} --output {{output}} {{args}} + cargo run -- --input "{{input}}" --output "{{output}}" {{args}} From 63a492b7699e6e4e21850ed1b1374e820afb8c30 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Wed, 8 Oct 2025 23:06:01 +0200 Subject: [PATCH 03/23] Fix pre-commit warnings and make sure pre-commit shows the same warnings when running it locally as in GitHub --- .github/workflows/nifi_migrate_pr.yaml | 2 +- .github/workflows/nifi_migrate_release.yaml | 2 +- .github/workflows/pre_commit.yaml | 2 +- Justfile | 6 +++++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nifi_migrate_pr.yaml b/.github/workflows/nifi_migrate_pr.yaml index 4d1d337..3c9f98c 100644 --- a/.github/workflows/nifi_migrate_pr.yaml +++ b/.github/workflows/nifi_migrate_pr.yaml @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: 2025 Stackable GmbH # SPDX-License-Identifier: Apache-2.0 - +--- name: Build nifi-migrate on: diff --git a/.github/workflows/nifi_migrate_release.yaml b/.github/workflows/nifi_migrate_release.yaml index e429853..fbabe36 100644 --- a/.github/workflows/nifi_migrate_release.yaml +++ b/.github/workflows/nifi_migrate_release.yaml @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: 2025 Stackable GmbH # SPDX-License-Identifier: Apache-2.0 - +--- name: Release nifi-migrate on: diff --git a/.github/workflows/pre_commit.yaml b/.github/workflows/pre_commit.yaml index fc4ebf3..584b4cd 100644 --- a/.github/workflows/pre_commit.yaml +++ b/.github/workflows/pre_commit.yaml @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: 2025 Stackable GmbH # SPDX-License-Identifier: Apache-2.0 - +--- name: pre-commit on: diff --git a/Justfile b/Justfile index 14e2728..3a9a416 100644 --- a/Justfile +++ b/Justfile @@ -30,7 +30,11 @@ deny: # Run pre-commit hooks on all files pre-commit: - pre-commit run --all-files + pre-commit run \ + --all-files \ + --verbose \ + --show-diff-on-failure \ + --color always # Install pre-commit hooks pre-commit-install: From 594c6cbb2b756b616ea672572322da49dcdd7257 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 9 Oct 2025 08:29:07 +0200 Subject: [PATCH 04/23] Add a format-only mode --- src/cli.rs | 6 ++++++ src/lib.rs | 11 +++++++++-- src/main.rs | 11 ++++++++--- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index a7eb364..2dbe824 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -20,4 +20,10 @@ pub struct Args { /// Pretty-print the output JSON (default: compact) #[arg(short, long)] pub pretty: bool, + + /// Format-only mode: rewrite the file without applying migrations. + /// This can help in diffing a file to see the changes as the formatting will have changed + /// from the input. This way you'll have two consistently formatted files. + #[arg(short, long)] + pub format_only: bool, } diff --git a/src/lib.rs b/src/lib.rs index 8f01b5f..b99ed87 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -39,6 +39,7 @@ impl Migrator { input_path: &Path, output_path: &Path, pretty: bool, + format_only: bool, ) -> Result> { // Validate input exists if !input_path.exists() { @@ -74,9 +75,15 @@ impl Migrator { let mut flow: Value = serde_json::from_str(&content) .with_context(|| format!("Failed to parse JSON from: {}", input_path.display()))?; - let changes = self.migrate_flow(&mut flow)?; + let changes = if format_only { + Vec::new() + } else { + self.migrate_flow(&mut flow)? + }; - if !changes.is_empty() { + // In format-only mode, always write output even if no migrations + // In normal mode, only write if changes were made + if format_only || !changes.is_empty() { let output = if pretty { serde_json::to_string_pretty(&flow) } else { diff --git a/src/main.rs b/src/main.rs index b2d967b..0c6f9bf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,9 +12,14 @@ fn main() -> Result<()> { let args = Args::parse(); let migrator = Migrator::new(); - let changes = migrator.migrate_file(&args.input, &args.output, args.pretty)?; - - if changes.is_empty() { + // Should we get more arguments we can pass along a struct...this kinda grew organically :) + let changes = + migrator.migrate_file(&args.input, &args.output, args.pretty, args.format_only)?; + + if args.format_only { + println!("Format-only mode: File reformatted without migrations."); + println!("Output written to: {}", args.output.display()); + } else if changes.is_empty() { println!("No migrations needed."); } else { println!("Migration complete. Changes made:"); From bbb13f5ddb51fca183b1b813c35df2d7671a9ee8 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 9 Oct 2025 13:13:01 +0200 Subject: [PATCH 05/23] Update .yamllint.yaml Co-authored-by: Techassi --- .yamllint.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.yamllint.yaml b/.yamllint.yaml index de61902..ada065d 100644 --- a/.yamllint.yaml +++ b/.yamllint.yaml @@ -4,8 +4,6 @@ extends: default -ignore: | - deploy/helm/**/templates rules: line-length: disable From ff9190fe6c6071ed191d46c9550b26fd7102376b Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 9 Oct 2025 13:20:08 +0200 Subject: [PATCH 06/23] Update .cargo/config.toml Co-authored-by: Techassi --- .cargo/config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index a3910ce..39d8feb 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -2,4 +2,4 @@ # SPDX-License-Identifier: Apache-2.0 [alias] -nifi-migrate = ["run", "--"] +nifi-migrate = ["run", "--bin", "nifi-migrate", "--"] From a4497a21c510da53ab8c4f2b069229a15687bff4 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 9 Oct 2025 15:34:20 +0200 Subject: [PATCH 07/23] Add periods to end of comments --- CLAUDE.md | 1 + src/cli.rs | 6 +++--- src/lib.rs | 14 +++++++------- src/rules/jolt_transform.rs | 2 +- src/rules/jolt_transform_record.rs | 2 +- src/rules/mod.rs | 8 ++++---- 6 files changed, 17 insertions(+), 16 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 5b6f307..fc1220f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,6 +34,7 @@ To install pre-commit git hooks: - Files covered by `REUSE.toml` don't need individual headers - License: Apache-2.0 - Copyright holder: Stackable GmbH +- **All full sentences in comments (`//` and `///`) must end with a period** ## Adding New Migration Rules diff --git a/src/cli.rs b/src/cli.rs index 2dbe824..0889c42 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -9,15 +9,15 @@ use std::path::PathBuf; #[command(version)] #[command(about = "Migrate NiFi 1.x flow.json files to NiFi 2.x format", long_about = None)] pub struct Args { - /// Input flow.json file + /// Input flow.json file. #[arg(short, long)] pub input: PathBuf, - /// Output flow.json file + /// Output flow.json file. #[arg(short, long)] pub output: PathBuf, - /// Pretty-print the output JSON (default: compact) + /// Pretty-print the output JSON (default: compact). #[arg(short, long)] pub pretty: bool, diff --git a/src/lib.rs b/src/lib.rs index b99ed87..ebabcf0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,7 +9,7 @@ use serde_json::Value; use std::fs; use std::path::Path; -/// Represents a change that would be made during migration +/// Represents a change that would be made during migration. #[derive(Debug, Clone)] pub struct MigrationChange { pub processor_id: String, @@ -17,13 +17,13 @@ pub struct MigrationChange { pub rule_description: String, } -/// Main migration engine +/// Main migration engine. pub struct Migrator { rules: Vec>, } impl Migrator { - /// Create a new migrator with default rules + /// Create a new migrator with default rules. pub fn new() -> Self { Self { rules: vec![ @@ -33,7 +33,7 @@ impl Migrator { } } - /// Migrate a flow JSON file + /// Migrate a flow JSON file. pub fn migrate_file( &self, input_path: &Path, @@ -99,14 +99,14 @@ impl Migrator { Ok(changes) } - /// Migrate a flow JSON value in-place + /// Migrate a flow JSON value in-place. fn migrate_flow(&self, flow: &mut Value) -> Result> { let mut changes = Vec::new(); self.process_value(flow, &mut changes); Ok(changes) } - /// Recursively process a JSON value looking for processors + /// Recursively process a JSON value looking for processors. fn process_value(&self, value: &mut Value, changes: &mut Vec) { self.process_value_with_context(value, None, changes); } @@ -152,7 +152,7 @@ impl Migrator { } } - /// Process a single processor object + /// Process a single processor object. fn process_processor(&self, processor: &mut Value, changes: &mut Vec) { for rule in &self.rules { if rule.applies(processor) && rule.apply(processor) { diff --git a/src/rules/jolt_transform.rs b/src/rules/jolt_transform.rs index e0fb3b5..daa89aa 100644 --- a/src/rules/jolt_transform.rs +++ b/src/rules/jolt_transform.rs @@ -4,7 +4,7 @@ use super::MigrationRule; use serde_json::Value; -/// Migration rule for JoltTransformJSON processor +/// Migration rule for JoltTransformJSON processor. /// /// Migrates `org.apache.nifi.processors.standard.JoltTransformJSON` to /// `org.apache.nifi.processors.jolt.JoltTransformJSON` and updates the diff --git a/src/rules/jolt_transform_record.rs b/src/rules/jolt_transform_record.rs index 90e8d38..39eecf3 100644 --- a/src/rules/jolt_transform_record.rs +++ b/src/rules/jolt_transform_record.rs @@ -4,7 +4,7 @@ use super::MigrationRule; use serde_json::Value; -/// Migration rule for JoltTransformRecord processor +/// Migration rule for JoltTransformRecord processor. /// /// Migrates `org.apache.nifi.processors.jolt.record.JoltTransformRecord` to /// `org.apache.nifi.processors.jolt.JoltTransformRecord` and updates the diff --git a/src/rules/mod.rs b/src/rules/mod.rs index eb02eb6..bd7af91 100644 --- a/src/rules/mod.rs +++ b/src/rules/mod.rs @@ -9,14 +9,14 @@ pub use jolt_transform_record::JoltTransformRecordMigration; use serde_json::Value; -/// Represents a migration rule that can be applied to processors +/// Represents a migration rule that can be applied to processors. pub trait MigrationRule { - /// Check if this rule applies to the given processor + /// Check if this rule applies to the given processor. fn applies(&self, processor: &Value) -> bool; - /// Apply the migration to the processor, returning true if changes were made + /// Apply the migration to the processor, returning true if changes were made. fn apply(&self, processor: &mut Value) -> bool; - /// Get a description of what this rule does + /// Get a description of what this rule does. fn description(&self) -> String; } From 8b172b34195843aa0c564718d948a16bf3113e0b Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 9 Oct 2025 15:45:39 +0200 Subject: [PATCH 08/23] Address review comments --- .github/workflows/nifi_migrate_pr.yaml | 1 - .github/workflows/pre_commit.yaml | 2 +- src/lib.rs | 4 ++-- src/main.rs | 15 +++++++++------ src/rules/jolt_transform.rs | 6 +++--- src/rules/jolt_transform_record.rs | 9 ++++----- 6 files changed, 19 insertions(+), 18 deletions(-) diff --git a/.github/workflows/nifi_migrate_pr.yaml b/.github/workflows/nifi_migrate_pr.yaml index 3c9f98c..712bd70 100644 --- a/.github/workflows/nifi_migrate_pr.yaml +++ b/.github/workflows/nifi_migrate_pr.yaml @@ -31,7 +31,6 @@ jobs: uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: persist-credentials: false - submodules: recursive - name: Run cargo-deny uses: EmbarkStudios/cargo-deny-action@f2ba7abc2abebaf185c833c3961145a3c275caad # v2.0.13 diff --git a/.github/workflows/pre_commit.yaml b/.github/workflows/pre_commit.yaml index 584b4cd..f63c36f 100644 --- a/.github/workflows/pre_commit.yaml +++ b/.github/workflows/pre_commit.yaml @@ -17,6 +17,6 @@ jobs: with: persist-credentials: false fetch-depth: 0 - - uses: stackabletech/actions/run-pre-commit@a5d39a4eb109bb6af3c152800701c86e98bfe1a5 # v0.10.1 + - uses: stackabletech/actions/run-pre-commit@9a70678a34ec4e4f6927d0f8842b74aa857a9577 # v0.10.2 with: rust: ${{ env.RUST_TOOLCHAIN_VERSION }} diff --git a/src/lib.rs b/src/lib.rs index ebabcf0..7376497 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -161,13 +161,13 @@ impl Migrator { .or_else(|| processor.get("id")) .and_then(|v| v.as_str()) .unwrap_or("unknown") - .to_string(); + .to_owned(); let processor_name = processor .get("name") .and_then(|v| v.as_str()) .unwrap_or("unnamed") - .to_string(); + .to_owned(); changes.push(MigrationChange { processor_id, diff --git a/src/main.rs b/src/main.rs index 0c6f9bf..f087802 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,7 +6,7 @@ mod cli; use anyhow::Result; use clap::Parser; use cli::Args; -use nifi_migrate::Migrator; +use nifi_migrate::{MigrationChange, Migrator}; fn main() -> Result<()> { let args = Args::parse(); @@ -24,11 +24,14 @@ fn main() -> Result<()> { } else { println!("Migration complete. Changes made:"); - for change in &changes { - println!( - " - {} ({}): {}", - change.processor_name, change.processor_id, change.rule_description - ); + for MigrationChange { + processor_name, + processor_id, + rule_description, + .. + } in &changes + { + println!(" - {processor_name} ({processor_id}): {rule_description}"); } println!("\nOutput written to: {}", args.output.display()); diff --git a/src/rules/jolt_transform.rs b/src/rules/jolt_transform.rs index daa89aa..cc05bfa 100644 --- a/src/rules/jolt_transform.rs +++ b/src/rules/jolt_transform.rs @@ -30,7 +30,7 @@ impl MigrationRule for JoltTransformMigration { if type_field.as_str() == Some("org.apache.nifi.processors.standard.JoltTransformJSON") { *type_field = - Value::String("org.apache.nifi.processors.jolt.JoltTransformJSON".to_string()); + Value::String("org.apache.nifi.processors.jolt.JoltTransformJSON".to_owned()); changed = true; } } @@ -39,7 +39,7 @@ impl MigrationRule for JoltTransformMigration { if let Some(bundle) = processor.get_mut("bundle") { if let Some(artifact) = bundle.get_mut("artifact") { if artifact.as_str() == Some("nifi-standard-nar") { - *artifact = Value::String("nifi-jolt-nar".to_string()); + *artifact = Value::String("nifi-jolt-nar".to_owned()); changed = true; } } @@ -49,7 +49,7 @@ impl MigrationRule for JoltTransformMigration { } fn description(&self) -> String { - "Migrate JoltTransformJSON from standard to jolt bundle".to_string() + "Migrate JoltTransformJSON from standard to jolt bundle".to_owned() } } diff --git a/src/rules/jolt_transform_record.rs b/src/rules/jolt_transform_record.rs index 39eecf3..3680518 100644 --- a/src/rules/jolt_transform_record.rs +++ b/src/rules/jolt_transform_record.rs @@ -30,9 +30,8 @@ impl MigrationRule for JoltTransformRecordMigration { if type_field.as_str() == Some("org.apache.nifi.processors.jolt.record.JoltTransformRecord") { - *type_field = Value::String( - "org.apache.nifi.processors.jolt.JoltTransformRecord".to_string(), - ); + *type_field = + Value::String("org.apache.nifi.processors.jolt.JoltTransformRecord".to_owned()); changed = true; } } @@ -41,7 +40,7 @@ impl MigrationRule for JoltTransformRecordMigration { if let Some(bundle) = processor.get_mut("bundle") { if let Some(artifact) = bundle.get_mut("artifact") { if artifact.as_str() == Some("nifi-jolt-record-nar") { - *artifact = Value::String("nifi-jolt-nar".to_string()); + *artifact = Value::String("nifi-jolt-nar".to_owned()); changed = true; } } @@ -51,7 +50,7 @@ impl MigrationRule for JoltTransformRecordMigration { } fn description(&self) -> String { - "Migrate JoltTransformRecord from jolt-record to jolt bundle".to_string() + "Migrate JoltTransformRecord from jolt-record to jolt bundle".to_owned() } } From 3e214b800c2bac9ad1dfcd837bb948953c08d7a9 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 9 Oct 2025 16:47:32 +0200 Subject: [PATCH 09/23] Address review comments - Remove `new` for Migrator - Rustdoc improvement - Rename lib.rs and move to a migration module --- src/cli.rs | 2 +- src/main.rs | 9 ++-- src/{lib.rs => migration/mod.rs} | 44 +++++++------------ src/{ => migration}/rules/jolt_transform.rs | 0 .../rules/jolt_transform_record.rs | 0 src/{ => migration}/rules/mod.rs | 2 +- 6 files changed, 23 insertions(+), 34 deletions(-) rename src/{lib.rs => migration/mod.rs} (89%) rename src/{ => migration}/rules/jolt_transform.rs (100%) rename src/{ => migration}/rules/jolt_transform_record.rs (100%) rename src/{ => migration}/rules/mod.rs (87%) diff --git a/src/cli.rs b/src/cli.rs index 0889c42..218ed2a 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -8,7 +8,7 @@ use std::path::PathBuf; #[command(name = "nifi-migrate")] #[command(version)] #[command(about = "Migrate NiFi 1.x flow.json files to NiFi 2.x format", long_about = None)] -pub struct Args { +pub struct Cli { /// Input flow.json file. #[arg(short, long)] pub input: PathBuf, diff --git a/src/main.rs b/src/main.rs index f087802..fc1af2f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,16 +2,17 @@ // SPDX-License-Identifier: Apache-2.0 mod cli; +mod migration; use anyhow::Result; use clap::Parser; -use cli::Args; -use nifi_migrate::{MigrationChange, Migrator}; +use cli::Cli; +use migration::{MigrationChange, Migrator}; fn main() -> Result<()> { - let args = Args::parse(); + let args = Cli::parse(); - let migrator = Migrator::new(); + let migrator = Migrator::default(); // Should we get more arguments we can pass along a struct...this kinda grew organically :) let changes = migrator.migrate_file(&args.input, &args.output, args.pretty, args.format_only)?; diff --git a/src/lib.rs b/src/migration/mod.rs similarity index 89% rename from src/lib.rs rename to src/migration/mod.rs index 7376497..169ccee 100644 --- a/src/lib.rs +++ b/src/migration/mod.rs @@ -22,9 +22,8 @@ pub struct Migrator { rules: Vec>, } -impl Migrator { - /// Create a new migrator with default rules. - pub fn new() -> Self { +impl Default for Migrator { + fn default() -> Self { Self { rules: vec![ Box::new(JoltTransformMigration), @@ -32,7 +31,9 @@ impl Migrator { ], } } +} +impl Migrator { /// Migrate a flow JSON file. pub fn migrate_file( &self, @@ -66,19 +67,17 @@ impl Migrator { } } - // It's not perfect reading it all in memory, but I decided it's fine for now. - // I tried it on a reasonably large file and it was fine. - // We can switch to streaming if it's ever needed. - let content = fs::read_to_string(input_path) - .with_context(|| format!("Failed to read input file: {}", input_path.display()))?; + let file = fs::File::open(input_path) + .with_context(|| format!("Failed to open input file: {}", input_path.display()))?; + let reader = std::io::BufReader::new(file); - let mut flow: Value = serde_json::from_str(&content) + let mut flow: Value = serde_json::from_reader(reader) .with_context(|| format!("Failed to parse JSON from: {}", input_path.display()))?; let changes = if format_only { Vec::new() } else { - self.migrate_flow(&mut flow)? + self.migrate_flow(&mut flow) }; // In format-only mode, always write output even if no migrations @@ -100,15 +99,10 @@ impl Migrator { } /// Migrate a flow JSON value in-place. - fn migrate_flow(&self, flow: &mut Value) -> Result> { + fn migrate_flow(&self, flow: &mut Value) -> Vec { let mut changes = Vec::new(); - self.process_value(flow, &mut changes); - Ok(changes) - } - - /// Recursively process a JSON value looking for processors. - fn process_value(&self, value: &mut Value, changes: &mut Vec) { - self.process_value_with_context(value, None, changes); + self.process_value_with_context(flow, None, &mut changes); + changes } /// Recursively process a JSON value with parent key context. @@ -179,12 +173,6 @@ impl Migrator { } } -impl Default for Migrator { - fn default() -> Self { - Self::new() - } -} - #[cfg(test)] mod tests { use super::*; @@ -254,8 +242,8 @@ mod tests { ] }); - let migrator = Migrator::new(); - let changes = migrator.migrate_flow(&mut flow).unwrap(); + let migrator = Migrator::default(); + let changes = migrator.migrate_flow(&mut flow); assert_eq!(changes.len(), 2); assert!(changes.iter().any(|c| c.processor_id == "proc-1")); @@ -285,8 +273,8 @@ mod tests { ] }); - let migrator = Migrator::new(); - let changes = migrator.migrate_flow(&mut flow).unwrap(); + let migrator = Migrator::default(); + let changes = migrator.migrate_flow(&mut flow); assert_eq!(changes.len(), 1); assert_eq!(changes[0].processor_id, "jolt-proc"); diff --git a/src/rules/jolt_transform.rs b/src/migration/rules/jolt_transform.rs similarity index 100% rename from src/rules/jolt_transform.rs rename to src/migration/rules/jolt_transform.rs diff --git a/src/rules/jolt_transform_record.rs b/src/migration/rules/jolt_transform_record.rs similarity index 100% rename from src/rules/jolt_transform_record.rs rename to src/migration/rules/jolt_transform_record.rs diff --git a/src/rules/mod.rs b/src/migration/rules/mod.rs similarity index 87% rename from src/rules/mod.rs rename to src/migration/rules/mod.rs index bd7af91..0ad7992 100644 --- a/src/rules/mod.rs +++ b/src/migration/rules/mod.rs @@ -14,7 +14,7 @@ pub trait MigrationRule { /// Check if this rule applies to the given processor. fn applies(&self, processor: &Value) -> bool; - /// Apply the migration to the processor, returning true if changes were made. + /// Apply the migration to the processor, returning `true` if changes were made. fn apply(&self, processor: &mut Value) -> bool; /// Get a description of what this rule does. From 74b18e651225566b50806865a092d620b299a539 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Fri, 10 Oct 2025 17:19:40 +0200 Subject: [PATCH 10/23] Update Cargo.toml Co-authored-by: Techassi --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 3715b91..d4392bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ name = "nifi-migrate" version = "0.1.0" edition = "2024" license = "Apache-2.0" -authors = ["Stackable GmbH"] +authors = ["Stackable GmbH "] description = "CLI tool for migrating Apache NiFi flow.json files between versions" repository = "https://github.com/stackabletech/nifi-migrate" From b4d49de2090edbd5576ec734a80e901a4c59d2d6 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Fri, 10 Oct 2025 17:20:02 +0200 Subject: [PATCH 11/23] Update Cargo.toml Co-authored-by: Techassi --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index d4392bd..b45996e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2024" license = "Apache-2.0" authors = ["Stackable GmbH "] -description = "CLI tool for migrating Apache NiFi flow.json files between versions" +description = "A CLI tool for migrating Apache NiFi flow.json files between versions" repository = "https://github.com/stackabletech/nifi-migrate" [dependencies] From fb06a67b956cef6c35f543acc601e8e215f65deb Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Fri, 10 Oct 2025 22:16:29 +0200 Subject: [PATCH 12/23] Move migration module to the new standard without mod.rs files --- src/{migration/mod.rs => migration.rs} | 0 src/migration/{rules/mod.rs => rules.rs} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/{migration/mod.rs => migration.rs} (100%) rename src/migration/{rules/mod.rs => rules.rs} (100%) diff --git a/src/migration/mod.rs b/src/migration.rs similarity index 100% rename from src/migration/mod.rs rename to src/migration.rs diff --git a/src/migration/rules/mod.rs b/src/migration/rules.rs similarity index 100% rename from src/migration/rules/mod.rs rename to src/migration/rules.rs From 5bb9e7b2d5ef21b9ee59fcd4aa850c5b677683ce Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Fri, 10 Oct 2025 22:23:05 +0200 Subject: [PATCH 13/23] Change CLI to take positional arguments for input & output --- Justfile | 2 +- README.md | 21 +++++---------------- src/cli.rs | 6 +++--- 3 files changed, 9 insertions(+), 20 deletions(-) diff --git a/Justfile b/Justfile index 3a9a416..be2d10e 100644 --- a/Justfile +++ b/Justfile @@ -46,4 +46,4 @@ build: # Run the tool run input output="flow-migrated.json" *args="": - cargo run -- --input "{{input}}" --output "{{output}}" {{args}} + cargo run -- "{{input}}" "{{output}}" {{args}} diff --git a/README.md b/README.md index cbe7a76..104b712 100644 --- a/README.md +++ b/README.md @@ -7,13 +7,6 @@ SPDX-License-Identifier: Apache-2.0 A Rust CLI tool for migrating Apache NiFi flow.json files between versions. -## Features - -- **Non-destructive**: Reads from input file and writes to a separate output file -- **Recursive**: Processes nested process groups automatically -- **Extensible**: Easy to add new migration rules via the trait-based system -- **Safe**: Distinguishes between processors and controller services to avoid incorrect migrations - ## Supported Migrations ### JoltTransformJSON Processor @@ -30,7 +23,7 @@ A Rust CLI tool for migrating Apache NiFi flow.json files between versions. - **Reason**: Consolidated into the main jolt bundle - **Reference**: [NIFI-12554](https://issues.apache.org/jira/browse/NIFI-12554) -## Installation +## Build ```bash cargo build --release @@ -43,28 +36,24 @@ The binary will be available at `target/release/nifi-migrate` Basic usage: ```bash -nifi-migrate --input flow.json --output flow-migrated.json +nifi-migrate flow.json flow-migrated.json ``` Or using the cargo alias: ```bash -cargo nifi-migrate --input flow.json --output flow-migrated.json +cargo nifi-migrate flow.json flow-migrated.json ``` With pretty-printed JSON output: ```bash -nifi-migrate --input flow.json --output flow-migrated.json --pretty +nifi-migrate flow.json flow-migrated.json --pretty ``` ### Options -- `-i, --input `: Input flow.json file (required) -- `-o, --output `: Output flow.json file (required) -- `-p, --pretty`: Pretty-print the output JSON (optional, default is compact) -- `-h, --help`: Show help information -- `-V, --version`: Show version information +Call `nifi-migrate --help` to see all its options. ## Important Notes diff --git a/src/cli.rs b/src/cli.rs index 218ed2a..3a6c38c 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: 2025 Stackable GmbH // SPDX-License-Identifier: Apache-2.0 -use clap::Parser; +use clap::{Parser, ValueHint}; use std::path::PathBuf; #[derive(Parser)] @@ -10,11 +10,11 @@ use std::path::PathBuf; #[command(about = "Migrate NiFi 1.x flow.json files to NiFi 2.x format", long_about = None)] pub struct Cli { /// Input flow.json file. - #[arg(short, long)] + #[arg(value_hint = ValueHint::FilePath)] pub input: PathBuf, /// Output flow.json file. - #[arg(short, long)] + #[arg(value_hint = ValueHint::FilePath)] pub output: PathBuf, /// Pretty-print the output JSON (default: compact). From f1e4426e872619368d0372dcae55ef9539e43549 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Fri, 10 Oct 2025 22:29:56 +0200 Subject: [PATCH 14/23] README cleanup --- README.md | 42 ++++-------------------------------------- 1 file changed, 4 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 104b712..fd719cf 100644 --- a/README.md +++ b/README.md @@ -67,47 +67,15 @@ The order of JSON keys may also change as the file is parsed and reserialized. W To add a new migration rule: -1. Create a new file in `src/rules/` (e.g., `my_rule.rs`) -1. Implement the `MigrationRule` trait: - -```rust -use super::MigrationRule; -use serde_json::Value; - -pub struct MyMigrationRule; - -impl MigrationRule for MyMigrationRule { - fn applies(&self, processor: &Value) -> bool { - // Check if this rule applies - processor.get("type") - .and_then(|t| t.as_str()) - .map(|t| t == "org.apache.nifi.processors.old.Processor") - .unwrap_or(false) - } - - fn apply(&self, processor: &mut Value) -> bool { - // Apply the migration - if let Some(type_field) = processor.get_mut("type") { - *type_field = Value::String("org.apache.nifi.processors.new.Processor".to_string()); - return true; - } - false - } - - fn description(&self) -> String { - "Migrate Processor from old to new package".to_string() - } -} -``` - -1. Add your rule to `src/rules/mod.rs`: +1. look at one of the existing ones in `src/rules` and follow the pattern (basically: Implement the `MigrationRule` trait). +1. Add your rule to `src/rules/rules.rs`: ```rust mod my_rule; pub use my_rule::MyMigrationRule; ``` -1. Register it in `Migrator::new()` in `src/lib.rs`: +1. Register it in `Migrator::new()` in `src/migration.rs`: ```rust pub fn new() -> Self { @@ -120,8 +88,6 @@ pub fn new() -> Self { } ``` -1. Add tests to verify your rule works correctly - ## License Licensed under the Apache License, Version 2.0. See [LICENSE](LICENSE) for details. @@ -144,7 +110,7 @@ You can run all checks at once using: just all ``` -Or run pre-commit hooks: +And run pre-commit hooks: ```bash just pre-commit From fb2b0a101f9c84be268bbf3b18d7a6ac01618355 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Fri, 10 Oct 2025 22:03:05 +0200 Subject: [PATCH 15/23] Support for migration of Jolt* properties as well --- README.md | 4 +- src/migration.rs | 6 +- src/migration/rules.rs | 4 +- src/migration/rules/jolt_transform.rs | 217 --------- src/migration/rules/jolt_transform_json.rs | 436 +++++++++++++++++++ src/migration/rules/jolt_transform_record.rs | 182 ++++++++ 6 files changed, 625 insertions(+), 224 deletions(-) delete mode 100644 src/migration/rules/jolt_transform.rs create mode 100644 src/migration/rules/jolt_transform_json.rs diff --git a/README.md b/README.md index fd719cf..2731a55 100644 --- a/README.md +++ b/README.md @@ -13,14 +13,14 @@ A Rust CLI tool for migrating Apache NiFi flow.json files between versions. - **Type**: `org.apache.nifi.processors.standard.JoltTransformJSON` → `org.apache.nifi.processors.jolt.JoltTransformJSON` - **Bundle artifact**: `nifi-standard-nar` → `nifi-jolt-nar` -- **Reason**: In NiFi 2.x, Jolt processors were moved to a separate bundle +- **Reason**: In NiFi 2.x, Jolt processors were moved to a separate bundle and properties were renamed - **Reference**: [NIFI-12554](https://issues.apache.org/jira/browse/NIFI-12554) ### JoltTransformRecord Processor - **Type**: `org.apache.nifi.processors.jolt.record.JoltTransformRecord` → `org.apache.nifi.processors.jolt.JoltTransformRecord` - **Bundle artifact**: `nifi-jolt-record-nar` → `nifi-jolt-nar` -- **Reason**: Consolidated into the main jolt bundle +- **Reason**: In NiFi 2.x, Jolt processors were moved to a separate bundle and properties were renamed - **Reference**: [NIFI-12554](https://issues.apache.org/jira/browse/NIFI-12554) ## Build diff --git a/src/migration.rs b/src/migration.rs index 169ccee..b1c433d 100644 --- a/src/migration.rs +++ b/src/migration.rs @@ -4,7 +4,7 @@ mod rules; use anyhow::{Context, Result}; -use rules::{JoltTransformMigration, JoltTransformRecordMigration, MigrationRule}; +use rules::{JoltTransformJsonMigration, JoltTransformRecordMigration, MigrationRule}; use serde_json::Value; use std::fs; use std::path::Path; @@ -26,7 +26,7 @@ impl Default for Migrator { fn default() -> Self { Self { rules: vec![ - Box::new(JoltTransformMigration), + Box::new(JoltTransformJsonMigration), Box::new(JoltTransformRecordMigration), ], } @@ -192,7 +192,7 @@ mod tests { } }); - let rule = JoltTransformMigration; + let rule = JoltTransformJsonMigration; assert!(rule.applies(&processor)); assert!(rule.apply(&mut processor)); diff --git a/src/migration/rules.rs b/src/migration/rules.rs index 0ad7992..ff59edc 100644 --- a/src/migration/rules.rs +++ b/src/migration/rules.rs @@ -1,10 +1,10 @@ // SPDX-FileCopyrightText: 2025 Stackable GmbH // SPDX-License-Identifier: Apache-2.0 -mod jolt_transform; +mod jolt_transform_json; mod jolt_transform_record; -pub use jolt_transform::JoltTransformMigration; +pub use jolt_transform_json::JoltTransformJsonMigration; pub use jolt_transform_record::JoltTransformRecordMigration; use serde_json::Value; diff --git a/src/migration/rules/jolt_transform.rs b/src/migration/rules/jolt_transform.rs deleted file mode 100644 index cc05bfa..0000000 --- a/src/migration/rules/jolt_transform.rs +++ /dev/null @@ -1,217 +0,0 @@ -// SPDX-FileCopyrightText: 2025 Stackable GmbH -// SPDX-License-Identifier: Apache-2.0 - -use super::MigrationRule; -use serde_json::Value; - -/// Migration rule for JoltTransformJSON processor. -/// -/// Migrates `org.apache.nifi.processors.standard.JoltTransformJSON` to -/// `org.apache.nifi.processors.jolt.JoltTransformJSON` and updates the -/// bundle from `nifi-standard-nar` to `nifi-jolt-nar`. -/// -/// Reference: -pub struct JoltTransformMigration; - -impl MigrationRule for JoltTransformMigration { - fn applies(&self, processor: &Value) -> bool { - processor - .get("type") - .and_then(|t| t.as_str()) - .map(|t| t == "org.apache.nifi.processors.standard.JoltTransformJSON") - .unwrap_or(false) - } - - fn apply(&self, processor: &mut Value) -> bool { - let mut changed = false; - - // Update the type field - if let Some(type_field) = processor.get_mut("type") { - if type_field.as_str() == Some("org.apache.nifi.processors.standard.JoltTransformJSON") - { - *type_field = - Value::String("org.apache.nifi.processors.jolt.JoltTransformJSON".to_owned()); - changed = true; - } - } - - // Update the bundle artifact field - if let Some(bundle) = processor.get_mut("bundle") { - if let Some(artifact) = bundle.get_mut("artifact") { - if artifact.as_str() == Some("nifi-standard-nar") { - *artifact = Value::String("nifi-jolt-nar".to_owned()); - changed = true; - } - } - } - - changed - } - - fn description(&self) -> String { - "Migrate JoltTransformJSON from standard to jolt bundle".to_owned() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use pretty_assertions::assert_eq; - use serde_json::json; - - #[test] - fn test_jolt_transform_migration() { - let mut processor = json!({ - "identifier": "test-id-123", - "name": "JoltTransform", - "type": "org.apache.nifi.processors.standard.JoltTransformJSON", - "bundle": { - "artifact": "nifi-standard-nar", - "group": "org.apache.nifi", - "version": "1.25.0" - } - }); - - let rule = JoltTransformMigration; - assert!(rule.applies(&processor)); - assert!(rule.apply(&mut processor)); - - assert_eq!( - processor.get("type").and_then(|v| v.as_str()), - Some("org.apache.nifi.processors.jolt.JoltTransformJSON") - ); - assert_eq!( - processor - .get("bundle") - .and_then(|b| b.get("artifact")) - .and_then(|v| v.as_str()), - Some("nifi-jolt-nar") - ); - } - - #[test] - fn test_full_real_world_processor() { - let mut processor = json!({ - "identifier": "347aaa3b-2b3a-30c5-932d-58a109f9478f", - "instanceIdentifier": "e70d7d94-e92a-3472-b1c7-338e291af5e9", - "name": "JoltTransformJSON", - "comments": "", - "position": { - "x": 3368.0, - "y": 200.0 - }, - "type": "org.apache.nifi.processors.standard.JoltTransformJSON", - "bundle": { - "group": "org.apache.nifi", - "artifact": "nifi-standard-nar", - "version": "1.18.0" - }, - "properties": {}, - "propertyDescriptors": { - "jolt-spec": { - "name": "jolt-spec", - "displayName": "jolt-spec", - "identifiesControllerService": false, - "sensitive": true, - "dynamic": false - }, - "jolt-transform": { - "name": "jolt-transform", - "displayName": "jolt-transform", - "identifiesControllerService": false, - "sensitive": true, - "dynamic": false - }, - "pretty_print": { - "name": "pretty_print", - "displayName": "pretty_print", - "identifiesControllerService": false, - "sensitive": true, - "dynamic": false - }, - "Transform Cache Size": { - "name": "Transform Cache Size", - "displayName": "Transform Cache Size", - "identifiesControllerService": false, - "sensitive": true, - "dynamic": false - } - }, - "style": {}, - "schedulingPeriod": "0 sec", - "schedulingStrategy": "TIMER_DRIVEN", - "executionNode": "ALL", - "penaltyDuration": "30 sec", - "yieldDuration": "1 sec", - "bulletinLevel": "WARN", - "runDurationMillis": 0, - "concurrentlySchedulableTaskCount": 1, - "autoTerminatedRelationships": ["failure"], - "scheduledState": "ENABLED", - "retryCount": 10, - "retriedRelationships": [], - "backoffMechanism": "PENALIZE_FLOWFILE", - "maxBackoffPeriod": "10 mins", - "componentType": "PROCESSOR", - "groupIdentifier": "5e61ca9d-43f8-3176-b706-2404009bcc5b" - }); - - let rule = JoltTransformMigration; - - // Verify it applies to this processor - assert!( - rule.applies(&processor), - "Rule should apply to JoltTransformJSON processor" - ); - - // Apply the migration - assert!(rule.apply(&mut processor), "Migration should make changes"); - - // Verify the type was changed - assert_eq!( - processor.get("type").and_then(|v| v.as_str()), - Some("org.apache.nifi.processors.jolt.JoltTransformJSON"), - "Processor type should be updated" - ); - - // Verify the bundle artifact was changed - assert_eq!( - processor - .get("bundle") - .and_then(|b| b.get("artifact")) - .and_then(|v| v.as_str()), - Some("nifi-jolt-nar"), - "Bundle artifact should be updated to nifi-jolt-nar" - ); - - // Verify other bundle fields remain unchanged - assert_eq!( - processor - .get("bundle") - .and_then(|b| b.get("group")) - .and_then(|v| v.as_str()), - Some("org.apache.nifi"), - "Bundle group should remain unchanged" - ); - assert_eq!( - processor - .get("bundle") - .and_then(|b| b.get("version")) - .and_then(|v| v.as_str()), - Some("1.18.0"), - "Bundle version should remain unchanged" - ); - - // Verify other fields remain unchanged - assert_eq!( - processor.get("identifier").and_then(|v| v.as_str()), - Some("347aaa3b-2b3a-30c5-932d-58a109f9478f"), - "Identifier should remain unchanged" - ); - assert_eq!( - processor.get("name").and_then(|v| v.as_str()), - Some("JoltTransformJSON"), - "Name should remain unchanged" - ); - } -} diff --git a/src/migration/rules/jolt_transform_json.rs b/src/migration/rules/jolt_transform_json.rs new file mode 100644 index 0000000..0cfa0eb --- /dev/null +++ b/src/migration/rules/jolt_transform_json.rs @@ -0,0 +1,436 @@ +// SPDX-FileCopyrightText: 2025 Stackable GmbH +// SPDX-License-Identifier: Apache-2.0 + +use super::MigrationRule; +use serde_json::Value; + +/// Migration rule for JoltTransformJSON processor. +/// +/// Migrates `org.apache.nifi.processors.standard.JoltTransformJSON` to +/// `org.apache.nifi.processors.jolt.JoltTransformJSON` and updates the +/// bundle from `nifi-standard-nar` to `nifi-jolt-nar`. +/// +/// Also migrates property names: +/// - `jolt-spec` → `Jolt Specification` +/// - `jolt-transform` → `Jolt Transform` +/// - `pretty_print` → `Pretty Print` +/// - `jolt-custom-class` → `Custom Transformation Class Name` +/// - `jolt-custom-modules` → `Custom Module Directory` +/// +/// Reference: +pub struct JoltTransformJsonMigration; + +const PROPERTY_MIGRATIONS: [(&str, &str); 5] = [ + ("jolt-spec", "Jolt Specification"), + ("jolt-transform", "Jolt Transform"), + ("pretty_print", "Pretty Print"), + ("jolt-custom-class", "Custom Transformation Class Name"), + ("jolt-custom-modules", "Custom Module Directory"), +]; + +impl MigrationRule for JoltTransformJsonMigration { + fn applies(&self, processor: &Value) -> bool { + processor + .get("type") + .and_then(|t| t.as_str()) + .map(|t| t == "org.apache.nifi.processors.standard.JoltTransformJSON") + .unwrap_or(false) + } + + fn apply(&self, processor: &mut Value) -> bool { + let mut changed = false; + + // Update the type field + if let Some(type_field) = processor.get_mut("type") { + if type_field.as_str() == Some("org.apache.nifi.processors.standard.JoltTransformJSON") + { + *type_field = + Value::String("org.apache.nifi.processors.jolt.JoltTransformJSON".to_owned()); + changed = true; + } + } + + // Update the bundle artifact field + if let Some(bundle) = processor.get_mut("bundle") { + if let Some(artifact) = bundle.get_mut("artifact") { + if artifact.as_str() == Some("nifi-standard-nar") { + *artifact = Value::String("nifi-jolt-nar".to_owned()); + changed = true; + } + } + } + + // Migrate properties: rename old property keys to new ones + if let Some(properties) = processor + .get_mut("properties") + .and_then(|p| p.as_object_mut()) + { + for (old_name, new_name) in PROPERTY_MIGRATIONS { + if let Some(value) = properties.remove(old_name) { + properties.insert(new_name.to_owned(), value); + changed = true; + } + } + } + + // Migrate propertyDescriptors: rename old property descriptor keys to new ones + if let Some(descriptors) = processor + .get_mut("propertyDescriptors") + .and_then(|p| p.as_object_mut()) + { + for (old_name, new_name) in PROPERTY_MIGRATIONS { + if let Some(mut descriptor) = descriptors.remove(old_name) { + // Update the name and displayName fields within the descriptor + if let Some(descriptor_obj) = descriptor.as_object_mut() { + descriptor_obj + .insert("name".to_owned(), Value::String(new_name.to_owned())); + descriptor_obj + .insert("displayName".to_owned(), Value::String(new_name.to_owned())); + } + descriptors.insert(new_name.to_owned(), descriptor); + changed = true; + } + } + } + + changed + } + + fn description(&self) -> String { + "Migrate JoltTransformJSON from standard to jolt bundle".to_owned() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use serde_json::json; + + #[test] + fn test_jolt_transform_migration() { + let mut processor = json!({ + "identifier": "test-id-123", + "name": "JoltTransform", + "type": "org.apache.nifi.processors.standard.JoltTransformJSON", + "bundle": { + "artifact": "nifi-standard-nar", + "group": "org.apache.nifi", + "version": "1.25.0" + } + }); + + let rule = JoltTransformJsonMigration; + assert!(rule.applies(&processor)); + assert!(rule.apply(&mut processor)); + + assert_eq!( + processor.get("type").and_then(|v| v.as_str()), + Some("org.apache.nifi.processors.jolt.JoltTransformJSON") + ); + assert_eq!( + processor + .get("bundle") + .and_then(|b| b.get("artifact")) + .and_then(|v| v.as_str()), + Some("nifi-jolt-nar") + ); + } + + #[test] + fn test_full_real_world_processor() { + let mut processor = json!({ + "identifier": "347aaa3b-2b3a-30c5-932d-58a109f9478f", + "instanceIdentifier": "e70d7d94-e92a-3472-b1c7-338e291af5e9", + "name": "JoltTransformJSON", + "comments": "", + "position": { + "x": 3368.0, + "y": 200.0 + }, + "type": "org.apache.nifi.processors.standard.JoltTransformJSON", + "bundle": { + "group": "org.apache.nifi", + "artifact": "nifi-standard-nar", + "version": "1.18.0" + }, + "properties": {}, + "propertyDescriptors": { + "jolt-spec": { + "name": "jolt-spec", + "displayName": "jolt-spec", + "identifiesControllerService": false, + "sensitive": true, + "dynamic": false + }, + "jolt-transform": { + "name": "jolt-transform", + "displayName": "jolt-transform", + "identifiesControllerService": false, + "sensitive": true, + "dynamic": false + }, + "pretty_print": { + "name": "pretty_print", + "displayName": "pretty_print", + "identifiesControllerService": false, + "sensitive": true, + "dynamic": false + }, + "Transform Cache Size": { + "name": "Transform Cache Size", + "displayName": "Transform Cache Size", + "identifiesControllerService": false, + "sensitive": true, + "dynamic": false + } + }, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": ["failure"], + "scheduledState": "ENABLED", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "5e61ca9d-43f8-3176-b706-2404009bcc5b" + }); + + let rule = JoltTransformJsonMigration; + + // Verify it applies to this processor + assert!( + rule.applies(&processor), + "Rule should apply to JoltTransformJSON processor" + ); + + // Apply the migration + assert!(rule.apply(&mut processor), "Migration should make changes"); + + // Verify the type was changed + assert_eq!( + processor.get("type").and_then(|v| v.as_str()), + Some("org.apache.nifi.processors.jolt.JoltTransformJSON"), + "Processor type should be updated" + ); + + // Verify the bundle artifact was changed + assert_eq!( + processor + .get("bundle") + .and_then(|b| b.get("artifact")) + .and_then(|v| v.as_str()), + Some("nifi-jolt-nar"), + "Bundle artifact should be updated to nifi-jolt-nar" + ); + + // Verify other bundle fields remain unchanged + assert_eq!( + processor + .get("bundle") + .and_then(|b| b.get("group")) + .and_then(|v| v.as_str()), + Some("org.apache.nifi"), + "Bundle group should remain unchanged" + ); + assert_eq!( + processor + .get("bundle") + .and_then(|b| b.get("version")) + .and_then(|v| v.as_str()), + Some("1.18.0"), + "Bundle version should remain unchanged" + ); + + // Verify other fields remain unchanged + assert_eq!( + processor.get("identifier").and_then(|v| v.as_str()), + Some("347aaa3b-2b3a-30c5-932d-58a109f9478f"), + "Identifier should remain unchanged" + ); + assert_eq!( + processor.get("name").and_then(|v| v.as_str()), + Some("JoltTransformJSON"), + "Name should remain unchanged" + ); + } + + #[test] + fn test_property_migrations() { + let mut processor = json!({ + "identifier": "test-id-123", + "name": "JoltTransform", + "type": "org.apache.nifi.processors.standard.JoltTransformJSON", + "bundle": { + "artifact": "nifi-standard-nar", + "group": "org.apache.nifi", + "version": "1.27.0" + }, + "properties": { + "jolt-spec": "[{\"operation\": \"shift\"}]", + "jolt-transform": "jolt-transform-chain", + "pretty_print": "false", + "jolt-custom-class": "com.example.CustomTransform", + "jolt-custom-modules": "/path/to/modules", + "Other Property": "should remain" + }, + "propertyDescriptors": { + "jolt-spec": { + "name": "jolt-spec", + "displayName": "jolt-spec", + "identifiesControllerService": false, + "sensitive": false, + "dynamic": false + }, + "jolt-transform": { + "name": "jolt-transform", + "displayName": "jolt-transform", + "identifiesControllerService": false, + "sensitive": false, + "dynamic": false + }, + "pretty_print": { + "name": "pretty_print", + "displayName": "pretty_print", + "identifiesControllerService": false, + "sensitive": false, + "dynamic": false + } + } + }); + + let rule = JoltTransformJsonMigration; + assert!(rule.applies(&processor)); + assert!(rule.apply(&mut processor)); + + // Verify properties were migrated + let properties = processor.get("properties").unwrap(); + assert_eq!( + properties + .get("Jolt Specification") + .and_then(|v| v.as_str()), + Some("[{\"operation\": \"shift\"}]"), + "jolt-spec should be migrated to Jolt Specification" + ); + assert_eq!( + properties.get("Jolt Transform").and_then(|v| v.as_str()), + Some("jolt-transform-chain"), + "jolt-transform should be migrated to Jolt Transform" + ); + assert_eq!( + properties.get("Pretty Print").and_then(|v| v.as_str()), + Some("false"), + "pretty_print should be migrated to Pretty Print" + ); + assert_eq!( + properties + .get("Custom Transformation Class Name") + .and_then(|v| v.as_str()), + Some("com.example.CustomTransform"), + "jolt-custom-class should be migrated to Custom Transformation Class Name" + ); + assert_eq!( + properties + .get("Custom Module Directory") + .and_then(|v| v.as_str()), + Some("/path/to/modules"), + "jolt-custom-modules should be migrated to Custom Module Directory" + ); + + // Verify old property names are removed + assert!( + properties.get("jolt-spec").is_none(), + "Old property name jolt-spec should be removed" + ); + assert!( + properties.get("jolt-transform").is_none(), + "Old property name jolt-transform should be removed" + ); + assert!( + properties.get("pretty_print").is_none(), + "Old property name pretty_print should be removed" + ); + + // Verify other properties remain unchanged + assert_eq!( + properties.get("Other Property").and_then(|v| v.as_str()), + Some("should remain"), + "Other properties should remain unchanged" + ); + + // Verify propertyDescriptors were migrated + let descriptors = processor.get("propertyDescriptors").unwrap(); + let jolt_spec_descriptor = descriptors.get("Jolt Specification").unwrap(); + assert_eq!( + jolt_spec_descriptor.get("name").and_then(|v| v.as_str()), + Some("Jolt Specification"), + "Descriptor name should be updated" + ); + assert_eq!( + jolt_spec_descriptor + .get("displayName") + .and_then(|v| v.as_str()), + Some("Jolt Specification"), + "Descriptor displayName should be updated" + ); + + // Verify old descriptor keys are removed + assert!( + descriptors.get("jolt-spec").is_none(), + "Old descriptor key jolt-spec should be removed" + ); + assert!( + descriptors.get("jolt-transform").is_none(), + "Old descriptor key jolt-transform should be removed" + ); + assert!( + descriptors.get("pretty_print").is_none(), + "Old descriptor key pretty_print should be removed" + ); + } + + #[test] + fn test_migration_with_missing_properties() { + let mut processor = json!({ + "identifier": "test-id-123", + "name": "JoltTransform", + "type": "org.apache.nifi.processors.standard.JoltTransformJSON", + "bundle": { + "artifact": "nifi-standard-nar", + "group": "org.apache.nifi", + "version": "1.27.0" + }, + "properties": { + "jolt-spec": "[{\"operation\": \"shift\"}]" + } + }); + + let rule = JoltTransformJsonMigration; + assert!(rule.applies(&processor)); + assert!(rule.apply(&mut processor)); + + // Verify the property that exists was migrated + let properties = processor.get("properties").unwrap(); + assert_eq!( + properties + .get("Jolt Specification") + .and_then(|v| v.as_str()), + Some("[{\"operation\": \"shift\"}]"), + "jolt-spec should be migrated" + ); + + // Verify old property name is removed + assert!( + properties.get("jolt-spec").is_none(), + "Old property name should be removed" + ); + } +} diff --git a/src/migration/rules/jolt_transform_record.rs b/src/migration/rules/jolt_transform_record.rs index 3680518..1298ff2 100644 --- a/src/migration/rules/jolt_transform_record.rs +++ b/src/migration/rules/jolt_transform_record.rs @@ -10,9 +10,27 @@ use serde_json::Value; /// `org.apache.nifi.processors.jolt.JoltTransformRecord` and updates the /// bundle from `nifi-jolt-record-nar` to `nifi-jolt-nar`. /// +/// Also migrates property names: +/// - `jolt-record-transform` → `Jolt Transform` +/// - `jolt-record-spec` → `Jolt Specification` +/// - `jolt-record-custom-class` → `Custom Transformation Class Name` +/// - `jolt-record-custom-modules` → `Custom Module Directory` +/// - `jolt-record-transform-cache-size` → `Transform Cache Size` +/// /// Reference: pub struct JoltTransformRecordMigration; +const PROPERTY_MIGRATIONS: [(&str, &str); 5] = [ + ("jolt-record-transform", "Jolt Transform"), + ("jolt-record-spec", "Jolt Specification"), + ( + "jolt-record-custom-class", + "Custom Transformation Class Name", + ), + ("jolt-record-custom-modules", "Custom Module Directory"), + ("jolt-record-transform-cache-size", "Transform Cache Size"), +]; + impl MigrationRule for JoltTransformRecordMigration { fn applies(&self, processor: &Value) -> bool { processor @@ -46,6 +64,39 @@ impl MigrationRule for JoltTransformRecordMigration { } } + // Migrate properties: rename old property keys to new ones + if let Some(properties) = processor + .get_mut("properties") + .and_then(|p| p.as_object_mut()) + { + for (old_name, new_name) in PROPERTY_MIGRATIONS { + if let Some(value) = properties.remove(old_name) { + properties.insert(new_name.to_owned(), value); + changed = true; + } + } + } + + // Migrate propertyDescriptors: rename old property descriptor keys to new ones + if let Some(descriptors) = processor + .get_mut("propertyDescriptors") + .and_then(|p| p.as_object_mut()) + { + for (old_name, new_name) in PROPERTY_MIGRATIONS { + if let Some(mut descriptor) = descriptors.remove(old_name) { + // Update the name and displayName fields within the descriptor + if let Some(descriptor_obj) = descriptor.as_object_mut() { + descriptor_obj + .insert("name".to_owned(), Value::String(new_name.to_owned())); + descriptor_obj + .insert("displayName".to_owned(), Value::String(new_name.to_owned())); + } + descriptors.insert(new_name.to_owned(), descriptor); + changed = true; + } + } + } + changed } @@ -104,4 +155,135 @@ mod tests { let rule = JoltTransformRecordMigration; assert!(!rule.applies(&processor)); } + + #[test] + fn test_property_migrations() { + let mut processor = json!({ + "identifier": "test-id-789", + "name": "JoltRecord", + "type": "org.apache.nifi.processors.jolt.record.JoltTransformRecord", + "bundle": { + "artifact": "nifi-jolt-record-nar", + "group": "org.apache.nifi", + "version": "1.27.0" + }, + "properties": { + "jolt-record-transform": "jolt-transform-chain", + "jolt-record-spec": "[{\"operation\": \"shift\"}]", + "jolt-record-custom-class": "com.example.CustomTransform", + "jolt-record-custom-modules": "/path/to/modules", + "jolt-record-transform-cache-size": "10", + "jolt-record-record-reader": "reader-service-id", + "jolt-record-record-writer": "writer-service-id" + }, + "propertyDescriptors": { + "jolt-record-transform": { + "name": "jolt-record-transform", + "displayName": "Jolt Transformation DSL", + "identifiesControllerService": false, + "sensitive": false, + "dynamic": false + }, + "jolt-record-spec": { + "name": "jolt-record-spec", + "displayName": "Jolt Specification", + "identifiesControllerService": false, + "sensitive": false, + "dynamic": false + } + } + }); + + let rule = JoltTransformRecordMigration; + assert!(rule.applies(&processor)); + assert!(rule.apply(&mut processor)); + + // Verify properties were migrated + let properties = processor.get("properties").unwrap(); + assert_eq!( + properties.get("Jolt Transform").and_then(|v| v.as_str()), + Some("jolt-transform-chain"), + "jolt-record-transform should be migrated to Jolt Transform" + ); + assert_eq!( + properties + .get("Jolt Specification") + .and_then(|v| v.as_str()), + Some("[{\"operation\": \"shift\"}]"), + "jolt-record-spec should be migrated to Jolt Specification" + ); + assert_eq!( + properties + .get("Custom Transformation Class Name") + .and_then(|v| v.as_str()), + Some("com.example.CustomTransform"), + "jolt-record-custom-class should be migrated" + ); + assert_eq!( + properties + .get("Custom Module Directory") + .and_then(|v| v.as_str()), + Some("/path/to/modules"), + "jolt-record-custom-modules should be migrated" + ); + assert_eq!( + properties + .get("Transform Cache Size") + .and_then(|v| v.as_str()), + Some("10"), + "jolt-record-transform-cache-size should be migrated" + ); + + // Verify old property names are removed + assert!( + properties.get("jolt-record-transform").is_none(), + "Old property name should be removed" + ); + assert!( + properties.get("jolt-record-spec").is_none(), + "Old property name should be removed" + ); + + // Verify properties that should NOT be migrated remain unchanged + assert_eq!( + properties + .get("jolt-record-record-reader") + .and_then(|v| v.as_str()), + Some("reader-service-id"), + "jolt-record-record-reader should remain unchanged" + ); + assert_eq!( + properties + .get("jolt-record-record-writer") + .and_then(|v| v.as_str()), + Some("writer-service-id"), + "jolt-record-record-writer should remain unchanged" + ); + + // Verify propertyDescriptors were migrated + let descriptors = processor.get("propertyDescriptors").unwrap(); + let transform_descriptor = descriptors.get("Jolt Transform").unwrap(); + assert_eq!( + transform_descriptor.get("name").and_then(|v| v.as_str()), + Some("Jolt Transform"), + "Descriptor name should be updated" + ); + assert_eq!( + transform_descriptor + .get("displayName") + .and_then(|v| v.as_str()), + Some("Jolt Transform"), + "Descriptor displayName should be updated" + ); + + // Verify old descriptor keys are removed + assert!( + descriptors.get("jolt-record-transform").is_none(), + "Old descriptor key should be removed" + ); + assert!( + descriptors.get("jolt-record-spec").is_none(), + "Old descriptor key should be removed" + ); + } } From 3003db880cfe0bc1f834762275e176f193854218 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Wed, 8 Oct 2025 20:39:31 +0200 Subject: [PATCH 16/23] Initial commit of a CLI tool to help with NiFi flow migrations --- src/lib.rs | 295 +++++++++++++++++++++++++++++ src/rules/jolt_transform.rs | 217 +++++++++++++++++++++ src/rules/jolt_transform_record.rs | 108 +++++++++++ src/rules/mod.rs | 22 +++ 4 files changed, 642 insertions(+) create mode 100644 src/lib.rs create mode 100644 src/rules/jolt_transform.rs create mode 100644 src/rules/jolt_transform_record.rs create mode 100644 src/rules/mod.rs diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..8f01b5f --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,295 @@ +// SPDX-FileCopyrightText: 2025 Stackable GmbH +// SPDX-License-Identifier: Apache-2.0 + +mod rules; + +use anyhow::{Context, Result}; +use rules::{JoltTransformMigration, JoltTransformRecordMigration, MigrationRule}; +use serde_json::Value; +use std::fs; +use std::path::Path; + +/// Represents a change that would be made during migration +#[derive(Debug, Clone)] +pub struct MigrationChange { + pub processor_id: String, + pub processor_name: String, + pub rule_description: String, +} + +/// Main migration engine +pub struct Migrator { + rules: Vec>, +} + +impl Migrator { + /// Create a new migrator with default rules + pub fn new() -> Self { + Self { + rules: vec![ + Box::new(JoltTransformMigration), + Box::new(JoltTransformRecordMigration), + ], + } + } + + /// Migrate a flow JSON file + pub fn migrate_file( + &self, + input_path: &Path, + output_path: &Path, + pretty: bool, + ) -> Result> { + // Validate input exists + if !input_path.exists() { + anyhow::bail!("Input file does not exist: {}", input_path.display()); + } + + // Warn if input and output are the same + let canonical_input = input_path + .canonicalize() + .with_context(|| format!("Failed to resolve input path: {}", input_path.display()))?; + + if let Ok(canonical_output) = output_path.canonicalize() { + if canonical_input == canonical_output { + anyhow::bail!( + "Input and output paths are the same. This would overwrite the original file." + ); + } + } + + // Validate output directory exists + if let Some(parent) = output_path.parent() { + if !parent.as_os_str().is_empty() && !parent.exists() { + anyhow::bail!("Output directory does not exist: {}", parent.display()); + } + } + + // It's not perfect reading it all in memory, but I decided it's fine for now. + // I tried it on a reasonably large file and it was fine. + // We can switch to streaming if it's ever needed. + let content = fs::read_to_string(input_path) + .with_context(|| format!("Failed to read input file: {}", input_path.display()))?; + + let mut flow: Value = serde_json::from_str(&content) + .with_context(|| format!("Failed to parse JSON from: {}", input_path.display()))?; + + let changes = self.migrate_flow(&mut flow)?; + + if !changes.is_empty() { + let output = if pretty { + serde_json::to_string_pretty(&flow) + } else { + serde_json::to_string(&flow) + } + .context("Failed to serialize output JSON")?; + + fs::write(output_path, output).with_context(|| { + format!("Failed to write output file: {}", output_path.display()) + })?; + } + + Ok(changes) + } + + /// Migrate a flow JSON value in-place + fn migrate_flow(&self, flow: &mut Value) -> Result> { + let mut changes = Vec::new(); + self.process_value(flow, &mut changes); + Ok(changes) + } + + /// Recursively process a JSON value looking for processors + fn process_value(&self, value: &mut Value, changes: &mut Vec) { + self.process_value_with_context(value, None, changes); + } + + /// Recursively process a JSON value with parent key context. + /// The NiFi JSON is not very deep so recursive should not cause any issues here. + fn process_value_with_context( + &self, + value: &mut Value, + parent_key: Option<&str>, + changes: &mut Vec, + ) { + match value { + Value::Object(map) => { + // Check if this object is a processor (but not a controller service) + // Controller services have the same structure as processors (type + bundle) + // but appear under "controllerServices" key instead of "processors" key + // This entire matching thing (as well as the migration rules) can be made smarter + // as needed. For now, we only have two rules and both are for processors so it's + // fine as is. + let is_processor = map.contains_key("type") + && map.contains_key("bundle") + && parent_key != Some("controllerServices"); + + if is_processor { + self.process_processor(value, changes); + } + + // Recursively process all nested values + // Need to re-borrow to avoid double mutable borrow + if let Value::Object(map) = value { + for (key, val) in map.iter_mut() { + self.process_value_with_context(val, Some(key), changes); + } + } + } + Value::Array(arr) => { + for item in arr.iter_mut() { + self.process_value_with_context(item, parent_key, changes); + } + } + _ => {} + } + } + + /// Process a single processor object + fn process_processor(&self, processor: &mut Value, changes: &mut Vec) { + for rule in &self.rules { + if rule.applies(processor) && rule.apply(processor) { + let processor_id = processor + .get("identifier") + .or_else(|| processor.get("id")) + .and_then(|v| v.as_str()) + .unwrap_or("unknown") + .to_string(); + + let processor_name = processor + .get("name") + .and_then(|v| v.as_str()) + .unwrap_or("unnamed") + .to_string(); + + changes.push(MigrationChange { + processor_id, + processor_name, + rule_description: rule.description(), + }); + } + } + } +} + +impl Default for Migrator { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use serde_json::json; + + #[test] + fn test_jolt_transform_migration() { + let mut processor = json!({ + "identifier": "test-id-123", + "name": "JoltTransform", + "type": "org.apache.nifi.processors.standard.JoltTransformJSON", + "bundle": { + "artifact": "nifi-standard-nar", + "group": "org.apache.nifi", + "version": "1.25.0" + } + }); + + let rule = JoltTransformMigration; + assert!(rule.applies(&processor)); + assert!(rule.apply(&mut processor)); + + assert_eq!( + processor.get("type").and_then(|v| v.as_str()), + Some("org.apache.nifi.processors.jolt.JoltTransformJSON") + ); + assert_eq!( + processor + .get("bundle") + .and_then(|b| b.get("artifact")) + .and_then(|v| v.as_str()), + Some("nifi-jolt-nar") + ); + } + + #[test] + fn test_nested_process_groups() { + let mut flow = json!({ + "processGroups": [ + { + "processors": [ + { + "identifier": "proc-1", + "name": "Jolt1", + "type": "org.apache.nifi.processors.standard.JoltTransformJSON", + "bundle": { + "artifact": "nifi-standard-nar" + } + } + ], + "processGroups": [ + { + "processors": [ + { + "identifier": "proc-2", + "name": "Jolt2", + "type": "org.apache.nifi.processors.standard.JoltTransformJSON", + "bundle": { + "artifact": "nifi-standard-nar" + } + } + ] + } + ] + } + ] + }); + + let migrator = Migrator::new(); + let changes = migrator.migrate_flow(&mut flow).unwrap(); + + assert_eq!(changes.len(), 2); + assert!(changes.iter().any(|c| c.processor_id == "proc-1")); + assert!(changes.iter().any(|c| c.processor_id == "proc-2")); + } + + #[test] + fn test_only_migrates_jolt_processors() { + let mut flow = json!({ + "processors": [ + { + "identifier": "other-proc", + "name": "SomeOtherProcessor", + "type": "org.apache.nifi.processors.standard.LogAttribute", + "bundle": { + "artifact": "nifi-standard-nar" + } + }, + { + "identifier": "jolt-proc", + "name": "JoltProcessor", + "type": "org.apache.nifi.processors.standard.JoltTransformJSON", + "bundle": { + "artifact": "nifi-standard-nar" + } + } + ] + }); + + let migrator = Migrator::new(); + let changes = migrator.migrate_flow(&mut flow).unwrap(); + + assert_eq!(changes.len(), 1); + assert_eq!(changes[0].processor_id, "jolt-proc"); + + // Verify other processor unchanged + assert_eq!( + flow["processors"][0]["bundle"]["artifact"], + "nifi-standard-nar" + ); + // Verify jolt processor changed + assert_eq!(flow["processors"][1]["bundle"]["artifact"], "nifi-jolt-nar"); + } +} diff --git a/src/rules/jolt_transform.rs b/src/rules/jolt_transform.rs new file mode 100644 index 0000000..e0fb3b5 --- /dev/null +++ b/src/rules/jolt_transform.rs @@ -0,0 +1,217 @@ +// SPDX-FileCopyrightText: 2025 Stackable GmbH +// SPDX-License-Identifier: Apache-2.0 + +use super::MigrationRule; +use serde_json::Value; + +/// Migration rule for JoltTransformJSON processor +/// +/// Migrates `org.apache.nifi.processors.standard.JoltTransformJSON` to +/// `org.apache.nifi.processors.jolt.JoltTransformJSON` and updates the +/// bundle from `nifi-standard-nar` to `nifi-jolt-nar`. +/// +/// Reference: +pub struct JoltTransformMigration; + +impl MigrationRule for JoltTransformMigration { + fn applies(&self, processor: &Value) -> bool { + processor + .get("type") + .and_then(|t| t.as_str()) + .map(|t| t == "org.apache.nifi.processors.standard.JoltTransformJSON") + .unwrap_or(false) + } + + fn apply(&self, processor: &mut Value) -> bool { + let mut changed = false; + + // Update the type field + if let Some(type_field) = processor.get_mut("type") { + if type_field.as_str() == Some("org.apache.nifi.processors.standard.JoltTransformJSON") + { + *type_field = + Value::String("org.apache.nifi.processors.jolt.JoltTransformJSON".to_string()); + changed = true; + } + } + + // Update the bundle artifact field + if let Some(bundle) = processor.get_mut("bundle") { + if let Some(artifact) = bundle.get_mut("artifact") { + if artifact.as_str() == Some("nifi-standard-nar") { + *artifact = Value::String("nifi-jolt-nar".to_string()); + changed = true; + } + } + } + + changed + } + + fn description(&self) -> String { + "Migrate JoltTransformJSON from standard to jolt bundle".to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use serde_json::json; + + #[test] + fn test_jolt_transform_migration() { + let mut processor = json!({ + "identifier": "test-id-123", + "name": "JoltTransform", + "type": "org.apache.nifi.processors.standard.JoltTransformJSON", + "bundle": { + "artifact": "nifi-standard-nar", + "group": "org.apache.nifi", + "version": "1.25.0" + } + }); + + let rule = JoltTransformMigration; + assert!(rule.applies(&processor)); + assert!(rule.apply(&mut processor)); + + assert_eq!( + processor.get("type").and_then(|v| v.as_str()), + Some("org.apache.nifi.processors.jolt.JoltTransformJSON") + ); + assert_eq!( + processor + .get("bundle") + .and_then(|b| b.get("artifact")) + .and_then(|v| v.as_str()), + Some("nifi-jolt-nar") + ); + } + + #[test] + fn test_full_real_world_processor() { + let mut processor = json!({ + "identifier": "347aaa3b-2b3a-30c5-932d-58a109f9478f", + "instanceIdentifier": "e70d7d94-e92a-3472-b1c7-338e291af5e9", + "name": "JoltTransformJSON", + "comments": "", + "position": { + "x": 3368.0, + "y": 200.0 + }, + "type": "org.apache.nifi.processors.standard.JoltTransformJSON", + "bundle": { + "group": "org.apache.nifi", + "artifact": "nifi-standard-nar", + "version": "1.18.0" + }, + "properties": {}, + "propertyDescriptors": { + "jolt-spec": { + "name": "jolt-spec", + "displayName": "jolt-spec", + "identifiesControllerService": false, + "sensitive": true, + "dynamic": false + }, + "jolt-transform": { + "name": "jolt-transform", + "displayName": "jolt-transform", + "identifiesControllerService": false, + "sensitive": true, + "dynamic": false + }, + "pretty_print": { + "name": "pretty_print", + "displayName": "pretty_print", + "identifiesControllerService": false, + "sensitive": true, + "dynamic": false + }, + "Transform Cache Size": { + "name": "Transform Cache Size", + "displayName": "Transform Cache Size", + "identifiesControllerService": false, + "sensitive": true, + "dynamic": false + } + }, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": ["failure"], + "scheduledState": "ENABLED", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "5e61ca9d-43f8-3176-b706-2404009bcc5b" + }); + + let rule = JoltTransformMigration; + + // Verify it applies to this processor + assert!( + rule.applies(&processor), + "Rule should apply to JoltTransformJSON processor" + ); + + // Apply the migration + assert!(rule.apply(&mut processor), "Migration should make changes"); + + // Verify the type was changed + assert_eq!( + processor.get("type").and_then(|v| v.as_str()), + Some("org.apache.nifi.processors.jolt.JoltTransformJSON"), + "Processor type should be updated" + ); + + // Verify the bundle artifact was changed + assert_eq!( + processor + .get("bundle") + .and_then(|b| b.get("artifact")) + .and_then(|v| v.as_str()), + Some("nifi-jolt-nar"), + "Bundle artifact should be updated to nifi-jolt-nar" + ); + + // Verify other bundle fields remain unchanged + assert_eq!( + processor + .get("bundle") + .and_then(|b| b.get("group")) + .and_then(|v| v.as_str()), + Some("org.apache.nifi"), + "Bundle group should remain unchanged" + ); + assert_eq!( + processor + .get("bundle") + .and_then(|b| b.get("version")) + .and_then(|v| v.as_str()), + Some("1.18.0"), + "Bundle version should remain unchanged" + ); + + // Verify other fields remain unchanged + assert_eq!( + processor.get("identifier").and_then(|v| v.as_str()), + Some("347aaa3b-2b3a-30c5-932d-58a109f9478f"), + "Identifier should remain unchanged" + ); + assert_eq!( + processor.get("name").and_then(|v| v.as_str()), + Some("JoltTransformJSON"), + "Name should remain unchanged" + ); + } +} diff --git a/src/rules/jolt_transform_record.rs b/src/rules/jolt_transform_record.rs new file mode 100644 index 0000000..90e8d38 --- /dev/null +++ b/src/rules/jolt_transform_record.rs @@ -0,0 +1,108 @@ +// SPDX-FileCopyrightText: 2025 Stackable GmbH +// SPDX-License-Identifier: Apache-2.0 + +use super::MigrationRule; +use serde_json::Value; + +/// Migration rule for JoltTransformRecord processor +/// +/// Migrates `org.apache.nifi.processors.jolt.record.JoltTransformRecord` to +/// `org.apache.nifi.processors.jolt.JoltTransformRecord` and updates the +/// bundle from `nifi-jolt-record-nar` to `nifi-jolt-nar`. +/// +/// Reference: +pub struct JoltTransformRecordMigration; + +impl MigrationRule for JoltTransformRecordMigration { + fn applies(&self, processor: &Value) -> bool { + processor + .get("type") + .and_then(|t| t.as_str()) + .map(|t| t == "org.apache.nifi.processors.jolt.record.JoltTransformRecord") + .unwrap_or(false) + } + + fn apply(&self, processor: &mut Value) -> bool { + let mut changed = false; + + // Update the type field + if let Some(type_field) = processor.get_mut("type") { + if type_field.as_str() + == Some("org.apache.nifi.processors.jolt.record.JoltTransformRecord") + { + *type_field = Value::String( + "org.apache.nifi.processors.jolt.JoltTransformRecord".to_string(), + ); + changed = true; + } + } + + // Update the bundle artifact field + if let Some(bundle) = processor.get_mut("bundle") { + if let Some(artifact) = bundle.get_mut("artifact") { + if artifact.as_str() == Some("nifi-jolt-record-nar") { + *artifact = Value::String("nifi-jolt-nar".to_string()); + changed = true; + } + } + } + + changed + } + + fn description(&self) -> String { + "Migrate JoltTransformRecord from jolt-record to jolt bundle".to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use serde_json::json; + + #[test] + fn test_jolt_transform_record_migration() { + let mut processor = json!({ + "identifier": "test-id-456", + "name": "JoltTransformRecord", + "type": "org.apache.nifi.processors.jolt.record.JoltTransformRecord", + "bundle": { + "artifact": "nifi-jolt-record-nar", + "group": "org.apache.nifi", + "version": "1.27.0" + } + }); + + let rule = JoltTransformRecordMigration; + assert!(rule.applies(&processor)); + assert!(rule.apply(&mut processor)); + + assert_eq!( + processor.get("type").and_then(|v| v.as_str()), + Some("org.apache.nifi.processors.jolt.JoltTransformRecord") + ); + assert_eq!( + processor + .get("bundle") + .and_then(|b| b.get("artifact")) + .and_then(|v| v.as_str()), + Some("nifi-jolt-nar") + ); + } + + #[test] + fn test_does_not_apply_to_other_processors() { + let processor = json!({ + "identifier": "other-proc", + "name": "SomeOtherProcessor", + "type": "org.apache.nifi.processors.standard.LogAttribute", + "bundle": { + "artifact": "nifi-standard-nar" + } + }); + + let rule = JoltTransformRecordMigration; + assert!(!rule.applies(&processor)); + } +} diff --git a/src/rules/mod.rs b/src/rules/mod.rs new file mode 100644 index 0000000..eb02eb6 --- /dev/null +++ b/src/rules/mod.rs @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: 2025 Stackable GmbH +// SPDX-License-Identifier: Apache-2.0 + +mod jolt_transform; +mod jolt_transform_record; + +pub use jolt_transform::JoltTransformMigration; +pub use jolt_transform_record::JoltTransformRecordMigration; + +use serde_json::Value; + +/// Represents a migration rule that can be applied to processors +pub trait MigrationRule { + /// Check if this rule applies to the given processor + fn applies(&self, processor: &Value) -> bool; + + /// Apply the migration to the processor, returning true if changes were made + fn apply(&self, processor: &mut Value) -> bool; + + /// Get a description of what this rule does + fn description(&self) -> String; +} From 552cea096350859d0ec3f120e36192fec2a12797 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 9 Oct 2025 08:29:07 +0200 Subject: [PATCH 17/23] Add a format-only mode --- src/lib.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 8f01b5f..b99ed87 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -39,6 +39,7 @@ impl Migrator { input_path: &Path, output_path: &Path, pretty: bool, + format_only: bool, ) -> Result> { // Validate input exists if !input_path.exists() { @@ -74,9 +75,15 @@ impl Migrator { let mut flow: Value = serde_json::from_str(&content) .with_context(|| format!("Failed to parse JSON from: {}", input_path.display()))?; - let changes = self.migrate_flow(&mut flow)?; + let changes = if format_only { + Vec::new() + } else { + self.migrate_flow(&mut flow)? + }; - if !changes.is_empty() { + // In format-only mode, always write output even if no migrations + // In normal mode, only write if changes were made + if format_only || !changes.is_empty() { let output = if pretty { serde_json::to_string_pretty(&flow) } else { From 37a35cc9c51e9e22fcd88e261cefe61e61658b77 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 9 Oct 2025 15:34:20 +0200 Subject: [PATCH 18/23] Add periods to end of comments --- src/lib.rs | 14 +++++++------- src/rules/jolt_transform.rs | 2 +- src/rules/jolt_transform_record.rs | 2 +- src/rules/mod.rs | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index b99ed87..ebabcf0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,7 +9,7 @@ use serde_json::Value; use std::fs; use std::path::Path; -/// Represents a change that would be made during migration +/// Represents a change that would be made during migration. #[derive(Debug, Clone)] pub struct MigrationChange { pub processor_id: String, @@ -17,13 +17,13 @@ pub struct MigrationChange { pub rule_description: String, } -/// Main migration engine +/// Main migration engine. pub struct Migrator { rules: Vec>, } impl Migrator { - /// Create a new migrator with default rules + /// Create a new migrator with default rules. pub fn new() -> Self { Self { rules: vec![ @@ -33,7 +33,7 @@ impl Migrator { } } - /// Migrate a flow JSON file + /// Migrate a flow JSON file. pub fn migrate_file( &self, input_path: &Path, @@ -99,14 +99,14 @@ impl Migrator { Ok(changes) } - /// Migrate a flow JSON value in-place + /// Migrate a flow JSON value in-place. fn migrate_flow(&self, flow: &mut Value) -> Result> { let mut changes = Vec::new(); self.process_value(flow, &mut changes); Ok(changes) } - /// Recursively process a JSON value looking for processors + /// Recursively process a JSON value looking for processors. fn process_value(&self, value: &mut Value, changes: &mut Vec) { self.process_value_with_context(value, None, changes); } @@ -152,7 +152,7 @@ impl Migrator { } } - /// Process a single processor object + /// Process a single processor object. fn process_processor(&self, processor: &mut Value, changes: &mut Vec) { for rule in &self.rules { if rule.applies(processor) && rule.apply(processor) { diff --git a/src/rules/jolt_transform.rs b/src/rules/jolt_transform.rs index e0fb3b5..daa89aa 100644 --- a/src/rules/jolt_transform.rs +++ b/src/rules/jolt_transform.rs @@ -4,7 +4,7 @@ use super::MigrationRule; use serde_json::Value; -/// Migration rule for JoltTransformJSON processor +/// Migration rule for JoltTransformJSON processor. /// /// Migrates `org.apache.nifi.processors.standard.JoltTransformJSON` to /// `org.apache.nifi.processors.jolt.JoltTransformJSON` and updates the diff --git a/src/rules/jolt_transform_record.rs b/src/rules/jolt_transform_record.rs index 90e8d38..39eecf3 100644 --- a/src/rules/jolt_transform_record.rs +++ b/src/rules/jolt_transform_record.rs @@ -4,7 +4,7 @@ use super::MigrationRule; use serde_json::Value; -/// Migration rule for JoltTransformRecord processor +/// Migration rule for JoltTransformRecord processor. /// /// Migrates `org.apache.nifi.processors.jolt.record.JoltTransformRecord` to /// `org.apache.nifi.processors.jolt.JoltTransformRecord` and updates the diff --git a/src/rules/mod.rs b/src/rules/mod.rs index eb02eb6..bd7af91 100644 --- a/src/rules/mod.rs +++ b/src/rules/mod.rs @@ -9,14 +9,14 @@ pub use jolt_transform_record::JoltTransformRecordMigration; use serde_json::Value; -/// Represents a migration rule that can be applied to processors +/// Represents a migration rule that can be applied to processors. pub trait MigrationRule { - /// Check if this rule applies to the given processor + /// Check if this rule applies to the given processor. fn applies(&self, processor: &Value) -> bool; - /// Apply the migration to the processor, returning true if changes were made + /// Apply the migration to the processor, returning true if changes were made. fn apply(&self, processor: &mut Value) -> bool; - /// Get a description of what this rule does + /// Get a description of what this rule does. fn description(&self) -> String; } From f99f4e0b983ed5d84450cb032c276ab631614393 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 9 Oct 2025 15:45:39 +0200 Subject: [PATCH 19/23] Address review comments --- src/lib.rs | 4 ++-- src/rules/jolt_transform.rs | 6 +++--- src/rules/jolt_transform_record.rs | 9 ++++----- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index ebabcf0..7376497 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -161,13 +161,13 @@ impl Migrator { .or_else(|| processor.get("id")) .and_then(|v| v.as_str()) .unwrap_or("unknown") - .to_string(); + .to_owned(); let processor_name = processor .get("name") .and_then(|v| v.as_str()) .unwrap_or("unnamed") - .to_string(); + .to_owned(); changes.push(MigrationChange { processor_id, diff --git a/src/rules/jolt_transform.rs b/src/rules/jolt_transform.rs index daa89aa..cc05bfa 100644 --- a/src/rules/jolt_transform.rs +++ b/src/rules/jolt_transform.rs @@ -30,7 +30,7 @@ impl MigrationRule for JoltTransformMigration { if type_field.as_str() == Some("org.apache.nifi.processors.standard.JoltTransformJSON") { *type_field = - Value::String("org.apache.nifi.processors.jolt.JoltTransformJSON".to_string()); + Value::String("org.apache.nifi.processors.jolt.JoltTransformJSON".to_owned()); changed = true; } } @@ -39,7 +39,7 @@ impl MigrationRule for JoltTransformMigration { if let Some(bundle) = processor.get_mut("bundle") { if let Some(artifact) = bundle.get_mut("artifact") { if artifact.as_str() == Some("nifi-standard-nar") { - *artifact = Value::String("nifi-jolt-nar".to_string()); + *artifact = Value::String("nifi-jolt-nar".to_owned()); changed = true; } } @@ -49,7 +49,7 @@ impl MigrationRule for JoltTransformMigration { } fn description(&self) -> String { - "Migrate JoltTransformJSON from standard to jolt bundle".to_string() + "Migrate JoltTransformJSON from standard to jolt bundle".to_owned() } } diff --git a/src/rules/jolt_transform_record.rs b/src/rules/jolt_transform_record.rs index 39eecf3..3680518 100644 --- a/src/rules/jolt_transform_record.rs +++ b/src/rules/jolt_transform_record.rs @@ -30,9 +30,8 @@ impl MigrationRule for JoltTransformRecordMigration { if type_field.as_str() == Some("org.apache.nifi.processors.jolt.record.JoltTransformRecord") { - *type_field = Value::String( - "org.apache.nifi.processors.jolt.JoltTransformRecord".to_string(), - ); + *type_field = + Value::String("org.apache.nifi.processors.jolt.JoltTransformRecord".to_owned()); changed = true; } } @@ -41,7 +40,7 @@ impl MigrationRule for JoltTransformRecordMigration { if let Some(bundle) = processor.get_mut("bundle") { if let Some(artifact) = bundle.get_mut("artifact") { if artifact.as_str() == Some("nifi-jolt-record-nar") { - *artifact = Value::String("nifi-jolt-nar".to_string()); + *artifact = Value::String("nifi-jolt-nar".to_owned()); changed = true; } } @@ -51,7 +50,7 @@ impl MigrationRule for JoltTransformRecordMigration { } fn description(&self) -> String { - "Migrate JoltTransformRecord from jolt-record to jolt bundle".to_string() + "Migrate JoltTransformRecord from jolt-record to jolt bundle".to_owned() } } From 9fbf0ac17482bca560dfd8af196815a5913fde65 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 9 Oct 2025 16:47:32 +0200 Subject: [PATCH 20/23] Address review comments - Remove `new` for Migrator - Rustdoc improvement - Rename lib.rs and move to a migration module --- src/{lib.rs => migration/mod.rs} | 44 +++--- src/{ => migration}/rules/mod.rs | 2 +- src/rules/jolt_transform.rs | 217 ----------------------------- src/rules/jolt_transform_record.rs | 107 -------------- 4 files changed, 17 insertions(+), 353 deletions(-) rename src/{lib.rs => migration/mod.rs} (89%) rename src/{ => migration}/rules/mod.rs (87%) delete mode 100644 src/rules/jolt_transform.rs delete mode 100644 src/rules/jolt_transform_record.rs diff --git a/src/lib.rs b/src/migration/mod.rs similarity index 89% rename from src/lib.rs rename to src/migration/mod.rs index 7376497..169ccee 100644 --- a/src/lib.rs +++ b/src/migration/mod.rs @@ -22,9 +22,8 @@ pub struct Migrator { rules: Vec>, } -impl Migrator { - /// Create a new migrator with default rules. - pub fn new() -> Self { +impl Default for Migrator { + fn default() -> Self { Self { rules: vec![ Box::new(JoltTransformMigration), @@ -32,7 +31,9 @@ impl Migrator { ], } } +} +impl Migrator { /// Migrate a flow JSON file. pub fn migrate_file( &self, @@ -66,19 +67,17 @@ impl Migrator { } } - // It's not perfect reading it all in memory, but I decided it's fine for now. - // I tried it on a reasonably large file and it was fine. - // We can switch to streaming if it's ever needed. - let content = fs::read_to_string(input_path) - .with_context(|| format!("Failed to read input file: {}", input_path.display()))?; + let file = fs::File::open(input_path) + .with_context(|| format!("Failed to open input file: {}", input_path.display()))?; + let reader = std::io::BufReader::new(file); - let mut flow: Value = serde_json::from_str(&content) + let mut flow: Value = serde_json::from_reader(reader) .with_context(|| format!("Failed to parse JSON from: {}", input_path.display()))?; let changes = if format_only { Vec::new() } else { - self.migrate_flow(&mut flow)? + self.migrate_flow(&mut flow) }; // In format-only mode, always write output even if no migrations @@ -100,15 +99,10 @@ impl Migrator { } /// Migrate a flow JSON value in-place. - fn migrate_flow(&self, flow: &mut Value) -> Result> { + fn migrate_flow(&self, flow: &mut Value) -> Vec { let mut changes = Vec::new(); - self.process_value(flow, &mut changes); - Ok(changes) - } - - /// Recursively process a JSON value looking for processors. - fn process_value(&self, value: &mut Value, changes: &mut Vec) { - self.process_value_with_context(value, None, changes); + self.process_value_with_context(flow, None, &mut changes); + changes } /// Recursively process a JSON value with parent key context. @@ -179,12 +173,6 @@ impl Migrator { } } -impl Default for Migrator { - fn default() -> Self { - Self::new() - } -} - #[cfg(test)] mod tests { use super::*; @@ -254,8 +242,8 @@ mod tests { ] }); - let migrator = Migrator::new(); - let changes = migrator.migrate_flow(&mut flow).unwrap(); + let migrator = Migrator::default(); + let changes = migrator.migrate_flow(&mut flow); assert_eq!(changes.len(), 2); assert!(changes.iter().any(|c| c.processor_id == "proc-1")); @@ -285,8 +273,8 @@ mod tests { ] }); - let migrator = Migrator::new(); - let changes = migrator.migrate_flow(&mut flow).unwrap(); + let migrator = Migrator::default(); + let changes = migrator.migrate_flow(&mut flow); assert_eq!(changes.len(), 1); assert_eq!(changes[0].processor_id, "jolt-proc"); diff --git a/src/rules/mod.rs b/src/migration/rules/mod.rs similarity index 87% rename from src/rules/mod.rs rename to src/migration/rules/mod.rs index bd7af91..0ad7992 100644 --- a/src/rules/mod.rs +++ b/src/migration/rules/mod.rs @@ -14,7 +14,7 @@ pub trait MigrationRule { /// Check if this rule applies to the given processor. fn applies(&self, processor: &Value) -> bool; - /// Apply the migration to the processor, returning true if changes were made. + /// Apply the migration to the processor, returning `true` if changes were made. fn apply(&self, processor: &mut Value) -> bool; /// Get a description of what this rule does. diff --git a/src/rules/jolt_transform.rs b/src/rules/jolt_transform.rs deleted file mode 100644 index cc05bfa..0000000 --- a/src/rules/jolt_transform.rs +++ /dev/null @@ -1,217 +0,0 @@ -// SPDX-FileCopyrightText: 2025 Stackable GmbH -// SPDX-License-Identifier: Apache-2.0 - -use super::MigrationRule; -use serde_json::Value; - -/// Migration rule for JoltTransformJSON processor. -/// -/// Migrates `org.apache.nifi.processors.standard.JoltTransformJSON` to -/// `org.apache.nifi.processors.jolt.JoltTransformJSON` and updates the -/// bundle from `nifi-standard-nar` to `nifi-jolt-nar`. -/// -/// Reference: -pub struct JoltTransformMigration; - -impl MigrationRule for JoltTransformMigration { - fn applies(&self, processor: &Value) -> bool { - processor - .get("type") - .and_then(|t| t.as_str()) - .map(|t| t == "org.apache.nifi.processors.standard.JoltTransformJSON") - .unwrap_or(false) - } - - fn apply(&self, processor: &mut Value) -> bool { - let mut changed = false; - - // Update the type field - if let Some(type_field) = processor.get_mut("type") { - if type_field.as_str() == Some("org.apache.nifi.processors.standard.JoltTransformJSON") - { - *type_field = - Value::String("org.apache.nifi.processors.jolt.JoltTransformJSON".to_owned()); - changed = true; - } - } - - // Update the bundle artifact field - if let Some(bundle) = processor.get_mut("bundle") { - if let Some(artifact) = bundle.get_mut("artifact") { - if artifact.as_str() == Some("nifi-standard-nar") { - *artifact = Value::String("nifi-jolt-nar".to_owned()); - changed = true; - } - } - } - - changed - } - - fn description(&self) -> String { - "Migrate JoltTransformJSON from standard to jolt bundle".to_owned() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use pretty_assertions::assert_eq; - use serde_json::json; - - #[test] - fn test_jolt_transform_migration() { - let mut processor = json!({ - "identifier": "test-id-123", - "name": "JoltTransform", - "type": "org.apache.nifi.processors.standard.JoltTransformJSON", - "bundle": { - "artifact": "nifi-standard-nar", - "group": "org.apache.nifi", - "version": "1.25.0" - } - }); - - let rule = JoltTransformMigration; - assert!(rule.applies(&processor)); - assert!(rule.apply(&mut processor)); - - assert_eq!( - processor.get("type").and_then(|v| v.as_str()), - Some("org.apache.nifi.processors.jolt.JoltTransformJSON") - ); - assert_eq!( - processor - .get("bundle") - .and_then(|b| b.get("artifact")) - .and_then(|v| v.as_str()), - Some("nifi-jolt-nar") - ); - } - - #[test] - fn test_full_real_world_processor() { - let mut processor = json!({ - "identifier": "347aaa3b-2b3a-30c5-932d-58a109f9478f", - "instanceIdentifier": "e70d7d94-e92a-3472-b1c7-338e291af5e9", - "name": "JoltTransformJSON", - "comments": "", - "position": { - "x": 3368.0, - "y": 200.0 - }, - "type": "org.apache.nifi.processors.standard.JoltTransformJSON", - "bundle": { - "group": "org.apache.nifi", - "artifact": "nifi-standard-nar", - "version": "1.18.0" - }, - "properties": {}, - "propertyDescriptors": { - "jolt-spec": { - "name": "jolt-spec", - "displayName": "jolt-spec", - "identifiesControllerService": false, - "sensitive": true, - "dynamic": false - }, - "jolt-transform": { - "name": "jolt-transform", - "displayName": "jolt-transform", - "identifiesControllerService": false, - "sensitive": true, - "dynamic": false - }, - "pretty_print": { - "name": "pretty_print", - "displayName": "pretty_print", - "identifiesControllerService": false, - "sensitive": true, - "dynamic": false - }, - "Transform Cache Size": { - "name": "Transform Cache Size", - "displayName": "Transform Cache Size", - "identifiesControllerService": false, - "sensitive": true, - "dynamic": false - } - }, - "style": {}, - "schedulingPeriod": "0 sec", - "schedulingStrategy": "TIMER_DRIVEN", - "executionNode": "ALL", - "penaltyDuration": "30 sec", - "yieldDuration": "1 sec", - "bulletinLevel": "WARN", - "runDurationMillis": 0, - "concurrentlySchedulableTaskCount": 1, - "autoTerminatedRelationships": ["failure"], - "scheduledState": "ENABLED", - "retryCount": 10, - "retriedRelationships": [], - "backoffMechanism": "PENALIZE_FLOWFILE", - "maxBackoffPeriod": "10 mins", - "componentType": "PROCESSOR", - "groupIdentifier": "5e61ca9d-43f8-3176-b706-2404009bcc5b" - }); - - let rule = JoltTransformMigration; - - // Verify it applies to this processor - assert!( - rule.applies(&processor), - "Rule should apply to JoltTransformJSON processor" - ); - - // Apply the migration - assert!(rule.apply(&mut processor), "Migration should make changes"); - - // Verify the type was changed - assert_eq!( - processor.get("type").and_then(|v| v.as_str()), - Some("org.apache.nifi.processors.jolt.JoltTransformJSON"), - "Processor type should be updated" - ); - - // Verify the bundle artifact was changed - assert_eq!( - processor - .get("bundle") - .and_then(|b| b.get("artifact")) - .and_then(|v| v.as_str()), - Some("nifi-jolt-nar"), - "Bundle artifact should be updated to nifi-jolt-nar" - ); - - // Verify other bundle fields remain unchanged - assert_eq!( - processor - .get("bundle") - .and_then(|b| b.get("group")) - .and_then(|v| v.as_str()), - Some("org.apache.nifi"), - "Bundle group should remain unchanged" - ); - assert_eq!( - processor - .get("bundle") - .and_then(|b| b.get("version")) - .and_then(|v| v.as_str()), - Some("1.18.0"), - "Bundle version should remain unchanged" - ); - - // Verify other fields remain unchanged - assert_eq!( - processor.get("identifier").and_then(|v| v.as_str()), - Some("347aaa3b-2b3a-30c5-932d-58a109f9478f"), - "Identifier should remain unchanged" - ); - assert_eq!( - processor.get("name").and_then(|v| v.as_str()), - Some("JoltTransformJSON"), - "Name should remain unchanged" - ); - } -} diff --git a/src/rules/jolt_transform_record.rs b/src/rules/jolt_transform_record.rs deleted file mode 100644 index 3680518..0000000 --- a/src/rules/jolt_transform_record.rs +++ /dev/null @@ -1,107 +0,0 @@ -// SPDX-FileCopyrightText: 2025 Stackable GmbH -// SPDX-License-Identifier: Apache-2.0 - -use super::MigrationRule; -use serde_json::Value; - -/// Migration rule for JoltTransformRecord processor. -/// -/// Migrates `org.apache.nifi.processors.jolt.record.JoltTransformRecord` to -/// `org.apache.nifi.processors.jolt.JoltTransformRecord` and updates the -/// bundle from `nifi-jolt-record-nar` to `nifi-jolt-nar`. -/// -/// Reference: -pub struct JoltTransformRecordMigration; - -impl MigrationRule for JoltTransformRecordMigration { - fn applies(&self, processor: &Value) -> bool { - processor - .get("type") - .and_then(|t| t.as_str()) - .map(|t| t == "org.apache.nifi.processors.jolt.record.JoltTransformRecord") - .unwrap_or(false) - } - - fn apply(&self, processor: &mut Value) -> bool { - let mut changed = false; - - // Update the type field - if let Some(type_field) = processor.get_mut("type") { - if type_field.as_str() - == Some("org.apache.nifi.processors.jolt.record.JoltTransformRecord") - { - *type_field = - Value::String("org.apache.nifi.processors.jolt.JoltTransformRecord".to_owned()); - changed = true; - } - } - - // Update the bundle artifact field - if let Some(bundle) = processor.get_mut("bundle") { - if let Some(artifact) = bundle.get_mut("artifact") { - if artifact.as_str() == Some("nifi-jolt-record-nar") { - *artifact = Value::String("nifi-jolt-nar".to_owned()); - changed = true; - } - } - } - - changed - } - - fn description(&self) -> String { - "Migrate JoltTransformRecord from jolt-record to jolt bundle".to_owned() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use pretty_assertions::assert_eq; - use serde_json::json; - - #[test] - fn test_jolt_transform_record_migration() { - let mut processor = json!({ - "identifier": "test-id-456", - "name": "JoltTransformRecord", - "type": "org.apache.nifi.processors.jolt.record.JoltTransformRecord", - "bundle": { - "artifact": "nifi-jolt-record-nar", - "group": "org.apache.nifi", - "version": "1.27.0" - } - }); - - let rule = JoltTransformRecordMigration; - assert!(rule.applies(&processor)); - assert!(rule.apply(&mut processor)); - - assert_eq!( - processor.get("type").and_then(|v| v.as_str()), - Some("org.apache.nifi.processors.jolt.JoltTransformRecord") - ); - assert_eq!( - processor - .get("bundle") - .and_then(|b| b.get("artifact")) - .and_then(|v| v.as_str()), - Some("nifi-jolt-nar") - ); - } - - #[test] - fn test_does_not_apply_to_other_processors() { - let processor = json!({ - "identifier": "other-proc", - "name": "SomeOtherProcessor", - "type": "org.apache.nifi.processors.standard.LogAttribute", - "bundle": { - "artifact": "nifi-standard-nar" - } - }); - - let rule = JoltTransformRecordMigration; - assert!(!rule.applies(&processor)); - } -} From 4d7fe26898b0c2463e39e756dc162cd28c99d5c3 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Fri, 10 Oct 2025 22:16:29 +0200 Subject: [PATCH 21/23] Move migration module to the new standard without mod.rs files --- src/migration/mod.rs | 290 ------------------------------------- src/migration/rules/mod.rs | 22 --- 2 files changed, 312 deletions(-) delete mode 100644 src/migration/mod.rs delete mode 100644 src/migration/rules/mod.rs diff --git a/src/migration/mod.rs b/src/migration/mod.rs deleted file mode 100644 index 169ccee..0000000 --- a/src/migration/mod.rs +++ /dev/null @@ -1,290 +0,0 @@ -// SPDX-FileCopyrightText: 2025 Stackable GmbH -// SPDX-License-Identifier: Apache-2.0 - -mod rules; - -use anyhow::{Context, Result}; -use rules::{JoltTransformMigration, JoltTransformRecordMigration, MigrationRule}; -use serde_json::Value; -use std::fs; -use std::path::Path; - -/// Represents a change that would be made during migration. -#[derive(Debug, Clone)] -pub struct MigrationChange { - pub processor_id: String, - pub processor_name: String, - pub rule_description: String, -} - -/// Main migration engine. -pub struct Migrator { - rules: Vec>, -} - -impl Default for Migrator { - fn default() -> Self { - Self { - rules: vec![ - Box::new(JoltTransformMigration), - Box::new(JoltTransformRecordMigration), - ], - } - } -} - -impl Migrator { - /// Migrate a flow JSON file. - pub fn migrate_file( - &self, - input_path: &Path, - output_path: &Path, - pretty: bool, - format_only: bool, - ) -> Result> { - // Validate input exists - if !input_path.exists() { - anyhow::bail!("Input file does not exist: {}", input_path.display()); - } - - // Warn if input and output are the same - let canonical_input = input_path - .canonicalize() - .with_context(|| format!("Failed to resolve input path: {}", input_path.display()))?; - - if let Ok(canonical_output) = output_path.canonicalize() { - if canonical_input == canonical_output { - anyhow::bail!( - "Input and output paths are the same. This would overwrite the original file." - ); - } - } - - // Validate output directory exists - if let Some(parent) = output_path.parent() { - if !parent.as_os_str().is_empty() && !parent.exists() { - anyhow::bail!("Output directory does not exist: {}", parent.display()); - } - } - - let file = fs::File::open(input_path) - .with_context(|| format!("Failed to open input file: {}", input_path.display()))?; - let reader = std::io::BufReader::new(file); - - let mut flow: Value = serde_json::from_reader(reader) - .with_context(|| format!("Failed to parse JSON from: {}", input_path.display()))?; - - let changes = if format_only { - Vec::new() - } else { - self.migrate_flow(&mut flow) - }; - - // In format-only mode, always write output even if no migrations - // In normal mode, only write if changes were made - if format_only || !changes.is_empty() { - let output = if pretty { - serde_json::to_string_pretty(&flow) - } else { - serde_json::to_string(&flow) - } - .context("Failed to serialize output JSON")?; - - fs::write(output_path, output).with_context(|| { - format!("Failed to write output file: {}", output_path.display()) - })?; - } - - Ok(changes) - } - - /// Migrate a flow JSON value in-place. - fn migrate_flow(&self, flow: &mut Value) -> Vec { - let mut changes = Vec::new(); - self.process_value_with_context(flow, None, &mut changes); - changes - } - - /// Recursively process a JSON value with parent key context. - /// The NiFi JSON is not very deep so recursive should not cause any issues here. - fn process_value_with_context( - &self, - value: &mut Value, - parent_key: Option<&str>, - changes: &mut Vec, - ) { - match value { - Value::Object(map) => { - // Check if this object is a processor (but not a controller service) - // Controller services have the same structure as processors (type + bundle) - // but appear under "controllerServices" key instead of "processors" key - // This entire matching thing (as well as the migration rules) can be made smarter - // as needed. For now, we only have two rules and both are for processors so it's - // fine as is. - let is_processor = map.contains_key("type") - && map.contains_key("bundle") - && parent_key != Some("controllerServices"); - - if is_processor { - self.process_processor(value, changes); - } - - // Recursively process all nested values - // Need to re-borrow to avoid double mutable borrow - if let Value::Object(map) = value { - for (key, val) in map.iter_mut() { - self.process_value_with_context(val, Some(key), changes); - } - } - } - Value::Array(arr) => { - for item in arr.iter_mut() { - self.process_value_with_context(item, parent_key, changes); - } - } - _ => {} - } - } - - /// Process a single processor object. - fn process_processor(&self, processor: &mut Value, changes: &mut Vec) { - for rule in &self.rules { - if rule.applies(processor) && rule.apply(processor) { - let processor_id = processor - .get("identifier") - .or_else(|| processor.get("id")) - .and_then(|v| v.as_str()) - .unwrap_or("unknown") - .to_owned(); - - let processor_name = processor - .get("name") - .and_then(|v| v.as_str()) - .unwrap_or("unnamed") - .to_owned(); - - changes.push(MigrationChange { - processor_id, - processor_name, - rule_description: rule.description(), - }); - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use pretty_assertions::assert_eq; - use serde_json::json; - - #[test] - fn test_jolt_transform_migration() { - let mut processor = json!({ - "identifier": "test-id-123", - "name": "JoltTransform", - "type": "org.apache.nifi.processors.standard.JoltTransformJSON", - "bundle": { - "artifact": "nifi-standard-nar", - "group": "org.apache.nifi", - "version": "1.25.0" - } - }); - - let rule = JoltTransformMigration; - assert!(rule.applies(&processor)); - assert!(rule.apply(&mut processor)); - - assert_eq!( - processor.get("type").and_then(|v| v.as_str()), - Some("org.apache.nifi.processors.jolt.JoltTransformJSON") - ); - assert_eq!( - processor - .get("bundle") - .and_then(|b| b.get("artifact")) - .and_then(|v| v.as_str()), - Some("nifi-jolt-nar") - ); - } - - #[test] - fn test_nested_process_groups() { - let mut flow = json!({ - "processGroups": [ - { - "processors": [ - { - "identifier": "proc-1", - "name": "Jolt1", - "type": "org.apache.nifi.processors.standard.JoltTransformJSON", - "bundle": { - "artifact": "nifi-standard-nar" - } - } - ], - "processGroups": [ - { - "processors": [ - { - "identifier": "proc-2", - "name": "Jolt2", - "type": "org.apache.nifi.processors.standard.JoltTransformJSON", - "bundle": { - "artifact": "nifi-standard-nar" - } - } - ] - } - ] - } - ] - }); - - let migrator = Migrator::default(); - let changes = migrator.migrate_flow(&mut flow); - - assert_eq!(changes.len(), 2); - assert!(changes.iter().any(|c| c.processor_id == "proc-1")); - assert!(changes.iter().any(|c| c.processor_id == "proc-2")); - } - - #[test] - fn test_only_migrates_jolt_processors() { - let mut flow = json!({ - "processors": [ - { - "identifier": "other-proc", - "name": "SomeOtherProcessor", - "type": "org.apache.nifi.processors.standard.LogAttribute", - "bundle": { - "artifact": "nifi-standard-nar" - } - }, - { - "identifier": "jolt-proc", - "name": "JoltProcessor", - "type": "org.apache.nifi.processors.standard.JoltTransformJSON", - "bundle": { - "artifact": "nifi-standard-nar" - } - } - ] - }); - - let migrator = Migrator::default(); - let changes = migrator.migrate_flow(&mut flow); - - assert_eq!(changes.len(), 1); - assert_eq!(changes[0].processor_id, "jolt-proc"); - - // Verify other processor unchanged - assert_eq!( - flow["processors"][0]["bundle"]["artifact"], - "nifi-standard-nar" - ); - // Verify jolt processor changed - assert_eq!(flow["processors"][1]["bundle"]["artifact"], "nifi-jolt-nar"); - } -} diff --git a/src/migration/rules/mod.rs b/src/migration/rules/mod.rs deleted file mode 100644 index 0ad7992..0000000 --- a/src/migration/rules/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-FileCopyrightText: 2025 Stackable GmbH -// SPDX-License-Identifier: Apache-2.0 - -mod jolt_transform; -mod jolt_transform_record; - -pub use jolt_transform::JoltTransformMigration; -pub use jolt_transform_record::JoltTransformRecordMigration; - -use serde_json::Value; - -/// Represents a migration rule that can be applied to processors. -pub trait MigrationRule { - /// Check if this rule applies to the given processor. - fn applies(&self, processor: &Value) -> bool; - - /// Apply the migration to the processor, returning `true` if changes were made. - fn apply(&self, processor: &mut Value) -> bool; - - /// Get a description of what this rule does. - fn description(&self) -> String; -} From 922a612d3e3b958fa2587c97f9900a6ae0be96d1 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Fri, 10 Oct 2025 22:03:05 +0200 Subject: [PATCH 22/23] Support for migration of Jolt* properties as well --- README.md | 4 +- src/migration.rs | 6 +- src/migration/rules.rs | 4 +- src/migration/rules/jolt_transform.rs | 217 --------- src/migration/rules/jolt_transform_json.rs | 436 +++++++++++++++++++ src/migration/rules/jolt_transform_record.rs | 182 ++++++++ 6 files changed, 625 insertions(+), 224 deletions(-) delete mode 100644 src/migration/rules/jolt_transform.rs create mode 100644 src/migration/rules/jolt_transform_json.rs diff --git a/README.md b/README.md index 6cee114..ac53a06 100644 --- a/README.md +++ b/README.md @@ -13,14 +13,14 @@ A Rust CLI tool for migrating Apache NiFi flow.json files between versions. - **Type**: `org.apache.nifi.processors.standard.JoltTransformJSON` → `org.apache.nifi.processors.jolt.JoltTransformJSON` - **Bundle artifact**: `nifi-standard-nar` → `nifi-jolt-nar` -- **Reason**: In NiFi 2.x, Jolt processors were moved to a separate bundle +- **Reason**: In NiFi 2.x, Jolt processors were moved to a separate bundle and properties were renamed - **Reference**: [NIFI-12554](https://issues.apache.org/jira/browse/NIFI-12554) ### JoltTransformRecord Processor - **Type**: `org.apache.nifi.processors.jolt.record.JoltTransformRecord` → `org.apache.nifi.processors.jolt.JoltTransformRecord` - **Bundle artifact**: `nifi-jolt-record-nar` → `nifi-jolt-nar` -- **Reason**: Consolidated into the main jolt bundle +- **Reason**: In NiFi 2.x, Jolt processors were moved to a separate bundle and properties were renamed - **Reference**: [NIFI-12554](https://issues.apache.org/jira/browse/NIFI-12554) ## Build diff --git a/src/migration.rs b/src/migration.rs index 169ccee..b1c433d 100644 --- a/src/migration.rs +++ b/src/migration.rs @@ -4,7 +4,7 @@ mod rules; use anyhow::{Context, Result}; -use rules::{JoltTransformMigration, JoltTransformRecordMigration, MigrationRule}; +use rules::{JoltTransformJsonMigration, JoltTransformRecordMigration, MigrationRule}; use serde_json::Value; use std::fs; use std::path::Path; @@ -26,7 +26,7 @@ impl Default for Migrator { fn default() -> Self { Self { rules: vec![ - Box::new(JoltTransformMigration), + Box::new(JoltTransformJsonMigration), Box::new(JoltTransformRecordMigration), ], } @@ -192,7 +192,7 @@ mod tests { } }); - let rule = JoltTransformMigration; + let rule = JoltTransformJsonMigration; assert!(rule.applies(&processor)); assert!(rule.apply(&mut processor)); diff --git a/src/migration/rules.rs b/src/migration/rules.rs index 0ad7992..ff59edc 100644 --- a/src/migration/rules.rs +++ b/src/migration/rules.rs @@ -1,10 +1,10 @@ // SPDX-FileCopyrightText: 2025 Stackable GmbH // SPDX-License-Identifier: Apache-2.0 -mod jolt_transform; +mod jolt_transform_json; mod jolt_transform_record; -pub use jolt_transform::JoltTransformMigration; +pub use jolt_transform_json::JoltTransformJsonMigration; pub use jolt_transform_record::JoltTransformRecordMigration; use serde_json::Value; diff --git a/src/migration/rules/jolt_transform.rs b/src/migration/rules/jolt_transform.rs deleted file mode 100644 index cc05bfa..0000000 --- a/src/migration/rules/jolt_transform.rs +++ /dev/null @@ -1,217 +0,0 @@ -// SPDX-FileCopyrightText: 2025 Stackable GmbH -// SPDX-License-Identifier: Apache-2.0 - -use super::MigrationRule; -use serde_json::Value; - -/// Migration rule for JoltTransformJSON processor. -/// -/// Migrates `org.apache.nifi.processors.standard.JoltTransformJSON` to -/// `org.apache.nifi.processors.jolt.JoltTransformJSON` and updates the -/// bundle from `nifi-standard-nar` to `nifi-jolt-nar`. -/// -/// Reference: -pub struct JoltTransformMigration; - -impl MigrationRule for JoltTransformMigration { - fn applies(&self, processor: &Value) -> bool { - processor - .get("type") - .and_then(|t| t.as_str()) - .map(|t| t == "org.apache.nifi.processors.standard.JoltTransformJSON") - .unwrap_or(false) - } - - fn apply(&self, processor: &mut Value) -> bool { - let mut changed = false; - - // Update the type field - if let Some(type_field) = processor.get_mut("type") { - if type_field.as_str() == Some("org.apache.nifi.processors.standard.JoltTransformJSON") - { - *type_field = - Value::String("org.apache.nifi.processors.jolt.JoltTransformJSON".to_owned()); - changed = true; - } - } - - // Update the bundle artifact field - if let Some(bundle) = processor.get_mut("bundle") { - if let Some(artifact) = bundle.get_mut("artifact") { - if artifact.as_str() == Some("nifi-standard-nar") { - *artifact = Value::String("nifi-jolt-nar".to_owned()); - changed = true; - } - } - } - - changed - } - - fn description(&self) -> String { - "Migrate JoltTransformJSON from standard to jolt bundle".to_owned() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use pretty_assertions::assert_eq; - use serde_json::json; - - #[test] - fn test_jolt_transform_migration() { - let mut processor = json!({ - "identifier": "test-id-123", - "name": "JoltTransform", - "type": "org.apache.nifi.processors.standard.JoltTransformJSON", - "bundle": { - "artifact": "nifi-standard-nar", - "group": "org.apache.nifi", - "version": "1.25.0" - } - }); - - let rule = JoltTransformMigration; - assert!(rule.applies(&processor)); - assert!(rule.apply(&mut processor)); - - assert_eq!( - processor.get("type").and_then(|v| v.as_str()), - Some("org.apache.nifi.processors.jolt.JoltTransformJSON") - ); - assert_eq!( - processor - .get("bundle") - .and_then(|b| b.get("artifact")) - .and_then(|v| v.as_str()), - Some("nifi-jolt-nar") - ); - } - - #[test] - fn test_full_real_world_processor() { - let mut processor = json!({ - "identifier": "347aaa3b-2b3a-30c5-932d-58a109f9478f", - "instanceIdentifier": "e70d7d94-e92a-3472-b1c7-338e291af5e9", - "name": "JoltTransformJSON", - "comments": "", - "position": { - "x": 3368.0, - "y": 200.0 - }, - "type": "org.apache.nifi.processors.standard.JoltTransformJSON", - "bundle": { - "group": "org.apache.nifi", - "artifact": "nifi-standard-nar", - "version": "1.18.0" - }, - "properties": {}, - "propertyDescriptors": { - "jolt-spec": { - "name": "jolt-spec", - "displayName": "jolt-spec", - "identifiesControllerService": false, - "sensitive": true, - "dynamic": false - }, - "jolt-transform": { - "name": "jolt-transform", - "displayName": "jolt-transform", - "identifiesControllerService": false, - "sensitive": true, - "dynamic": false - }, - "pretty_print": { - "name": "pretty_print", - "displayName": "pretty_print", - "identifiesControllerService": false, - "sensitive": true, - "dynamic": false - }, - "Transform Cache Size": { - "name": "Transform Cache Size", - "displayName": "Transform Cache Size", - "identifiesControllerService": false, - "sensitive": true, - "dynamic": false - } - }, - "style": {}, - "schedulingPeriod": "0 sec", - "schedulingStrategy": "TIMER_DRIVEN", - "executionNode": "ALL", - "penaltyDuration": "30 sec", - "yieldDuration": "1 sec", - "bulletinLevel": "WARN", - "runDurationMillis": 0, - "concurrentlySchedulableTaskCount": 1, - "autoTerminatedRelationships": ["failure"], - "scheduledState": "ENABLED", - "retryCount": 10, - "retriedRelationships": [], - "backoffMechanism": "PENALIZE_FLOWFILE", - "maxBackoffPeriod": "10 mins", - "componentType": "PROCESSOR", - "groupIdentifier": "5e61ca9d-43f8-3176-b706-2404009bcc5b" - }); - - let rule = JoltTransformMigration; - - // Verify it applies to this processor - assert!( - rule.applies(&processor), - "Rule should apply to JoltTransformJSON processor" - ); - - // Apply the migration - assert!(rule.apply(&mut processor), "Migration should make changes"); - - // Verify the type was changed - assert_eq!( - processor.get("type").and_then(|v| v.as_str()), - Some("org.apache.nifi.processors.jolt.JoltTransformJSON"), - "Processor type should be updated" - ); - - // Verify the bundle artifact was changed - assert_eq!( - processor - .get("bundle") - .and_then(|b| b.get("artifact")) - .and_then(|v| v.as_str()), - Some("nifi-jolt-nar"), - "Bundle artifact should be updated to nifi-jolt-nar" - ); - - // Verify other bundle fields remain unchanged - assert_eq!( - processor - .get("bundle") - .and_then(|b| b.get("group")) - .and_then(|v| v.as_str()), - Some("org.apache.nifi"), - "Bundle group should remain unchanged" - ); - assert_eq!( - processor - .get("bundle") - .and_then(|b| b.get("version")) - .and_then(|v| v.as_str()), - Some("1.18.0"), - "Bundle version should remain unchanged" - ); - - // Verify other fields remain unchanged - assert_eq!( - processor.get("identifier").and_then(|v| v.as_str()), - Some("347aaa3b-2b3a-30c5-932d-58a109f9478f"), - "Identifier should remain unchanged" - ); - assert_eq!( - processor.get("name").and_then(|v| v.as_str()), - Some("JoltTransformJSON"), - "Name should remain unchanged" - ); - } -} diff --git a/src/migration/rules/jolt_transform_json.rs b/src/migration/rules/jolt_transform_json.rs new file mode 100644 index 0000000..0cfa0eb --- /dev/null +++ b/src/migration/rules/jolt_transform_json.rs @@ -0,0 +1,436 @@ +// SPDX-FileCopyrightText: 2025 Stackable GmbH +// SPDX-License-Identifier: Apache-2.0 + +use super::MigrationRule; +use serde_json::Value; + +/// Migration rule for JoltTransformJSON processor. +/// +/// Migrates `org.apache.nifi.processors.standard.JoltTransformJSON` to +/// `org.apache.nifi.processors.jolt.JoltTransformJSON` and updates the +/// bundle from `nifi-standard-nar` to `nifi-jolt-nar`. +/// +/// Also migrates property names: +/// - `jolt-spec` → `Jolt Specification` +/// - `jolt-transform` → `Jolt Transform` +/// - `pretty_print` → `Pretty Print` +/// - `jolt-custom-class` → `Custom Transformation Class Name` +/// - `jolt-custom-modules` → `Custom Module Directory` +/// +/// Reference: +pub struct JoltTransformJsonMigration; + +const PROPERTY_MIGRATIONS: [(&str, &str); 5] = [ + ("jolt-spec", "Jolt Specification"), + ("jolt-transform", "Jolt Transform"), + ("pretty_print", "Pretty Print"), + ("jolt-custom-class", "Custom Transformation Class Name"), + ("jolt-custom-modules", "Custom Module Directory"), +]; + +impl MigrationRule for JoltTransformJsonMigration { + fn applies(&self, processor: &Value) -> bool { + processor + .get("type") + .and_then(|t| t.as_str()) + .map(|t| t == "org.apache.nifi.processors.standard.JoltTransformJSON") + .unwrap_or(false) + } + + fn apply(&self, processor: &mut Value) -> bool { + let mut changed = false; + + // Update the type field + if let Some(type_field) = processor.get_mut("type") { + if type_field.as_str() == Some("org.apache.nifi.processors.standard.JoltTransformJSON") + { + *type_field = + Value::String("org.apache.nifi.processors.jolt.JoltTransformJSON".to_owned()); + changed = true; + } + } + + // Update the bundle artifact field + if let Some(bundle) = processor.get_mut("bundle") { + if let Some(artifact) = bundle.get_mut("artifact") { + if artifact.as_str() == Some("nifi-standard-nar") { + *artifact = Value::String("nifi-jolt-nar".to_owned()); + changed = true; + } + } + } + + // Migrate properties: rename old property keys to new ones + if let Some(properties) = processor + .get_mut("properties") + .and_then(|p| p.as_object_mut()) + { + for (old_name, new_name) in PROPERTY_MIGRATIONS { + if let Some(value) = properties.remove(old_name) { + properties.insert(new_name.to_owned(), value); + changed = true; + } + } + } + + // Migrate propertyDescriptors: rename old property descriptor keys to new ones + if let Some(descriptors) = processor + .get_mut("propertyDescriptors") + .and_then(|p| p.as_object_mut()) + { + for (old_name, new_name) in PROPERTY_MIGRATIONS { + if let Some(mut descriptor) = descriptors.remove(old_name) { + // Update the name and displayName fields within the descriptor + if let Some(descriptor_obj) = descriptor.as_object_mut() { + descriptor_obj + .insert("name".to_owned(), Value::String(new_name.to_owned())); + descriptor_obj + .insert("displayName".to_owned(), Value::String(new_name.to_owned())); + } + descriptors.insert(new_name.to_owned(), descriptor); + changed = true; + } + } + } + + changed + } + + fn description(&self) -> String { + "Migrate JoltTransformJSON from standard to jolt bundle".to_owned() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use serde_json::json; + + #[test] + fn test_jolt_transform_migration() { + let mut processor = json!({ + "identifier": "test-id-123", + "name": "JoltTransform", + "type": "org.apache.nifi.processors.standard.JoltTransformJSON", + "bundle": { + "artifact": "nifi-standard-nar", + "group": "org.apache.nifi", + "version": "1.25.0" + } + }); + + let rule = JoltTransformJsonMigration; + assert!(rule.applies(&processor)); + assert!(rule.apply(&mut processor)); + + assert_eq!( + processor.get("type").and_then(|v| v.as_str()), + Some("org.apache.nifi.processors.jolt.JoltTransformJSON") + ); + assert_eq!( + processor + .get("bundle") + .and_then(|b| b.get("artifact")) + .and_then(|v| v.as_str()), + Some("nifi-jolt-nar") + ); + } + + #[test] + fn test_full_real_world_processor() { + let mut processor = json!({ + "identifier": "347aaa3b-2b3a-30c5-932d-58a109f9478f", + "instanceIdentifier": "e70d7d94-e92a-3472-b1c7-338e291af5e9", + "name": "JoltTransformJSON", + "comments": "", + "position": { + "x": 3368.0, + "y": 200.0 + }, + "type": "org.apache.nifi.processors.standard.JoltTransformJSON", + "bundle": { + "group": "org.apache.nifi", + "artifact": "nifi-standard-nar", + "version": "1.18.0" + }, + "properties": {}, + "propertyDescriptors": { + "jolt-spec": { + "name": "jolt-spec", + "displayName": "jolt-spec", + "identifiesControllerService": false, + "sensitive": true, + "dynamic": false + }, + "jolt-transform": { + "name": "jolt-transform", + "displayName": "jolt-transform", + "identifiesControllerService": false, + "sensitive": true, + "dynamic": false + }, + "pretty_print": { + "name": "pretty_print", + "displayName": "pretty_print", + "identifiesControllerService": false, + "sensitive": true, + "dynamic": false + }, + "Transform Cache Size": { + "name": "Transform Cache Size", + "displayName": "Transform Cache Size", + "identifiesControllerService": false, + "sensitive": true, + "dynamic": false + } + }, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": ["failure"], + "scheduledState": "ENABLED", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "5e61ca9d-43f8-3176-b706-2404009bcc5b" + }); + + let rule = JoltTransformJsonMigration; + + // Verify it applies to this processor + assert!( + rule.applies(&processor), + "Rule should apply to JoltTransformJSON processor" + ); + + // Apply the migration + assert!(rule.apply(&mut processor), "Migration should make changes"); + + // Verify the type was changed + assert_eq!( + processor.get("type").and_then(|v| v.as_str()), + Some("org.apache.nifi.processors.jolt.JoltTransformJSON"), + "Processor type should be updated" + ); + + // Verify the bundle artifact was changed + assert_eq!( + processor + .get("bundle") + .and_then(|b| b.get("artifact")) + .and_then(|v| v.as_str()), + Some("nifi-jolt-nar"), + "Bundle artifact should be updated to nifi-jolt-nar" + ); + + // Verify other bundle fields remain unchanged + assert_eq!( + processor + .get("bundle") + .and_then(|b| b.get("group")) + .and_then(|v| v.as_str()), + Some("org.apache.nifi"), + "Bundle group should remain unchanged" + ); + assert_eq!( + processor + .get("bundle") + .and_then(|b| b.get("version")) + .and_then(|v| v.as_str()), + Some("1.18.0"), + "Bundle version should remain unchanged" + ); + + // Verify other fields remain unchanged + assert_eq!( + processor.get("identifier").and_then(|v| v.as_str()), + Some("347aaa3b-2b3a-30c5-932d-58a109f9478f"), + "Identifier should remain unchanged" + ); + assert_eq!( + processor.get("name").and_then(|v| v.as_str()), + Some("JoltTransformJSON"), + "Name should remain unchanged" + ); + } + + #[test] + fn test_property_migrations() { + let mut processor = json!({ + "identifier": "test-id-123", + "name": "JoltTransform", + "type": "org.apache.nifi.processors.standard.JoltTransformJSON", + "bundle": { + "artifact": "nifi-standard-nar", + "group": "org.apache.nifi", + "version": "1.27.0" + }, + "properties": { + "jolt-spec": "[{\"operation\": \"shift\"}]", + "jolt-transform": "jolt-transform-chain", + "pretty_print": "false", + "jolt-custom-class": "com.example.CustomTransform", + "jolt-custom-modules": "/path/to/modules", + "Other Property": "should remain" + }, + "propertyDescriptors": { + "jolt-spec": { + "name": "jolt-spec", + "displayName": "jolt-spec", + "identifiesControllerService": false, + "sensitive": false, + "dynamic": false + }, + "jolt-transform": { + "name": "jolt-transform", + "displayName": "jolt-transform", + "identifiesControllerService": false, + "sensitive": false, + "dynamic": false + }, + "pretty_print": { + "name": "pretty_print", + "displayName": "pretty_print", + "identifiesControllerService": false, + "sensitive": false, + "dynamic": false + } + } + }); + + let rule = JoltTransformJsonMigration; + assert!(rule.applies(&processor)); + assert!(rule.apply(&mut processor)); + + // Verify properties were migrated + let properties = processor.get("properties").unwrap(); + assert_eq!( + properties + .get("Jolt Specification") + .and_then(|v| v.as_str()), + Some("[{\"operation\": \"shift\"}]"), + "jolt-spec should be migrated to Jolt Specification" + ); + assert_eq!( + properties.get("Jolt Transform").and_then(|v| v.as_str()), + Some("jolt-transform-chain"), + "jolt-transform should be migrated to Jolt Transform" + ); + assert_eq!( + properties.get("Pretty Print").and_then(|v| v.as_str()), + Some("false"), + "pretty_print should be migrated to Pretty Print" + ); + assert_eq!( + properties + .get("Custom Transformation Class Name") + .and_then(|v| v.as_str()), + Some("com.example.CustomTransform"), + "jolt-custom-class should be migrated to Custom Transformation Class Name" + ); + assert_eq!( + properties + .get("Custom Module Directory") + .and_then(|v| v.as_str()), + Some("/path/to/modules"), + "jolt-custom-modules should be migrated to Custom Module Directory" + ); + + // Verify old property names are removed + assert!( + properties.get("jolt-spec").is_none(), + "Old property name jolt-spec should be removed" + ); + assert!( + properties.get("jolt-transform").is_none(), + "Old property name jolt-transform should be removed" + ); + assert!( + properties.get("pretty_print").is_none(), + "Old property name pretty_print should be removed" + ); + + // Verify other properties remain unchanged + assert_eq!( + properties.get("Other Property").and_then(|v| v.as_str()), + Some("should remain"), + "Other properties should remain unchanged" + ); + + // Verify propertyDescriptors were migrated + let descriptors = processor.get("propertyDescriptors").unwrap(); + let jolt_spec_descriptor = descriptors.get("Jolt Specification").unwrap(); + assert_eq!( + jolt_spec_descriptor.get("name").and_then(|v| v.as_str()), + Some("Jolt Specification"), + "Descriptor name should be updated" + ); + assert_eq!( + jolt_spec_descriptor + .get("displayName") + .and_then(|v| v.as_str()), + Some("Jolt Specification"), + "Descriptor displayName should be updated" + ); + + // Verify old descriptor keys are removed + assert!( + descriptors.get("jolt-spec").is_none(), + "Old descriptor key jolt-spec should be removed" + ); + assert!( + descriptors.get("jolt-transform").is_none(), + "Old descriptor key jolt-transform should be removed" + ); + assert!( + descriptors.get("pretty_print").is_none(), + "Old descriptor key pretty_print should be removed" + ); + } + + #[test] + fn test_migration_with_missing_properties() { + let mut processor = json!({ + "identifier": "test-id-123", + "name": "JoltTransform", + "type": "org.apache.nifi.processors.standard.JoltTransformJSON", + "bundle": { + "artifact": "nifi-standard-nar", + "group": "org.apache.nifi", + "version": "1.27.0" + }, + "properties": { + "jolt-spec": "[{\"operation\": \"shift\"}]" + } + }); + + let rule = JoltTransformJsonMigration; + assert!(rule.applies(&processor)); + assert!(rule.apply(&mut processor)); + + // Verify the property that exists was migrated + let properties = processor.get("properties").unwrap(); + assert_eq!( + properties + .get("Jolt Specification") + .and_then(|v| v.as_str()), + Some("[{\"operation\": \"shift\"}]"), + "jolt-spec should be migrated" + ); + + // Verify old property name is removed + assert!( + properties.get("jolt-spec").is_none(), + "Old property name should be removed" + ); + } +} diff --git a/src/migration/rules/jolt_transform_record.rs b/src/migration/rules/jolt_transform_record.rs index 3680518..1298ff2 100644 --- a/src/migration/rules/jolt_transform_record.rs +++ b/src/migration/rules/jolt_transform_record.rs @@ -10,9 +10,27 @@ use serde_json::Value; /// `org.apache.nifi.processors.jolt.JoltTransformRecord` and updates the /// bundle from `nifi-jolt-record-nar` to `nifi-jolt-nar`. /// +/// Also migrates property names: +/// - `jolt-record-transform` → `Jolt Transform` +/// - `jolt-record-spec` → `Jolt Specification` +/// - `jolt-record-custom-class` → `Custom Transformation Class Name` +/// - `jolt-record-custom-modules` → `Custom Module Directory` +/// - `jolt-record-transform-cache-size` → `Transform Cache Size` +/// /// Reference: pub struct JoltTransformRecordMigration; +const PROPERTY_MIGRATIONS: [(&str, &str); 5] = [ + ("jolt-record-transform", "Jolt Transform"), + ("jolt-record-spec", "Jolt Specification"), + ( + "jolt-record-custom-class", + "Custom Transformation Class Name", + ), + ("jolt-record-custom-modules", "Custom Module Directory"), + ("jolt-record-transform-cache-size", "Transform Cache Size"), +]; + impl MigrationRule for JoltTransformRecordMigration { fn applies(&self, processor: &Value) -> bool { processor @@ -46,6 +64,39 @@ impl MigrationRule for JoltTransformRecordMigration { } } + // Migrate properties: rename old property keys to new ones + if let Some(properties) = processor + .get_mut("properties") + .and_then(|p| p.as_object_mut()) + { + for (old_name, new_name) in PROPERTY_MIGRATIONS { + if let Some(value) = properties.remove(old_name) { + properties.insert(new_name.to_owned(), value); + changed = true; + } + } + } + + // Migrate propertyDescriptors: rename old property descriptor keys to new ones + if let Some(descriptors) = processor + .get_mut("propertyDescriptors") + .and_then(|p| p.as_object_mut()) + { + for (old_name, new_name) in PROPERTY_MIGRATIONS { + if let Some(mut descriptor) = descriptors.remove(old_name) { + // Update the name and displayName fields within the descriptor + if let Some(descriptor_obj) = descriptor.as_object_mut() { + descriptor_obj + .insert("name".to_owned(), Value::String(new_name.to_owned())); + descriptor_obj + .insert("displayName".to_owned(), Value::String(new_name.to_owned())); + } + descriptors.insert(new_name.to_owned(), descriptor); + changed = true; + } + } + } + changed } @@ -104,4 +155,135 @@ mod tests { let rule = JoltTransformRecordMigration; assert!(!rule.applies(&processor)); } + + #[test] + fn test_property_migrations() { + let mut processor = json!({ + "identifier": "test-id-789", + "name": "JoltRecord", + "type": "org.apache.nifi.processors.jolt.record.JoltTransformRecord", + "bundle": { + "artifact": "nifi-jolt-record-nar", + "group": "org.apache.nifi", + "version": "1.27.0" + }, + "properties": { + "jolt-record-transform": "jolt-transform-chain", + "jolt-record-spec": "[{\"operation\": \"shift\"}]", + "jolt-record-custom-class": "com.example.CustomTransform", + "jolt-record-custom-modules": "/path/to/modules", + "jolt-record-transform-cache-size": "10", + "jolt-record-record-reader": "reader-service-id", + "jolt-record-record-writer": "writer-service-id" + }, + "propertyDescriptors": { + "jolt-record-transform": { + "name": "jolt-record-transform", + "displayName": "Jolt Transformation DSL", + "identifiesControllerService": false, + "sensitive": false, + "dynamic": false + }, + "jolt-record-spec": { + "name": "jolt-record-spec", + "displayName": "Jolt Specification", + "identifiesControllerService": false, + "sensitive": false, + "dynamic": false + } + } + }); + + let rule = JoltTransformRecordMigration; + assert!(rule.applies(&processor)); + assert!(rule.apply(&mut processor)); + + // Verify properties were migrated + let properties = processor.get("properties").unwrap(); + assert_eq!( + properties.get("Jolt Transform").and_then(|v| v.as_str()), + Some("jolt-transform-chain"), + "jolt-record-transform should be migrated to Jolt Transform" + ); + assert_eq!( + properties + .get("Jolt Specification") + .and_then(|v| v.as_str()), + Some("[{\"operation\": \"shift\"}]"), + "jolt-record-spec should be migrated to Jolt Specification" + ); + assert_eq!( + properties + .get("Custom Transformation Class Name") + .and_then(|v| v.as_str()), + Some("com.example.CustomTransform"), + "jolt-record-custom-class should be migrated" + ); + assert_eq!( + properties + .get("Custom Module Directory") + .and_then(|v| v.as_str()), + Some("/path/to/modules"), + "jolt-record-custom-modules should be migrated" + ); + assert_eq!( + properties + .get("Transform Cache Size") + .and_then(|v| v.as_str()), + Some("10"), + "jolt-record-transform-cache-size should be migrated" + ); + + // Verify old property names are removed + assert!( + properties.get("jolt-record-transform").is_none(), + "Old property name should be removed" + ); + assert!( + properties.get("jolt-record-spec").is_none(), + "Old property name should be removed" + ); + + // Verify properties that should NOT be migrated remain unchanged + assert_eq!( + properties + .get("jolt-record-record-reader") + .and_then(|v| v.as_str()), + Some("reader-service-id"), + "jolt-record-record-reader should remain unchanged" + ); + assert_eq!( + properties + .get("jolt-record-record-writer") + .and_then(|v| v.as_str()), + Some("writer-service-id"), + "jolt-record-record-writer should remain unchanged" + ); + + // Verify propertyDescriptors were migrated + let descriptors = processor.get("propertyDescriptors").unwrap(); + let transform_descriptor = descriptors.get("Jolt Transform").unwrap(); + assert_eq!( + transform_descriptor.get("name").and_then(|v| v.as_str()), + Some("Jolt Transform"), + "Descriptor name should be updated" + ); + assert_eq!( + transform_descriptor + .get("displayName") + .and_then(|v| v.as_str()), + Some("Jolt Transform"), + "Descriptor displayName should be updated" + ); + + // Verify old descriptor keys are removed + assert!( + descriptors.get("jolt-record-transform").is_none(), + "Old descriptor key should be removed" + ); + assert!( + descriptors.get("jolt-record-spec").is_none(), + "Old descriptor key should be removed" + ); + } } From 622c0fd0d56a553fd91fdb07f0993f0e285b8dda Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Mon, 13 Oct 2025 11:09:58 +0200 Subject: [PATCH 23/23] Revert accidental changes --- CLAUDE.md | 27 +++++++++++++++++++++------ README.md | 43 +++++++++++++++++++++++-------------------- 2 files changed, 44 insertions(+), 26 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index fc1220f..df05029 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,6 +5,8 @@ SPDX-License-Identifier: Apache-2.0 # Claude Code Instructions +Always read the README.md file to understand what this project is about. + When making changes to this project, always run the following checks in order: 1. **Format code**: `cargo fmt` @@ -34,21 +36,34 @@ To install pre-commit git hooks: - Files covered by `REUSE.toml` don't need individual headers - License: Apache-2.0 - Copyright holder: Stackable GmbH -- **All full sentences in comments (`//` and `///`) must end with a period** +- All full sentences in comments (`//` and `///`) must end with a period ## Adding New Migration Rules When adding new migration rules, follow these steps in order: -1. Create a new file in `src/rules/` (e.g., `my_rule.rs`) +1. Create a new file in `src/migration/rules/` (e.g., `my_rule.rs`) 2. Implement the `MigrationRule` trait with SPDX headers -3. Add the module to `src/rules/mod.rs` and export it -4. Register it in `Migrator::new()` in `src/lib.rs` -5. Add comprehensive tests in the rule file + - Rules can apply to processors, controller services, or both + - The trait checks for `type` and `bundle` fields, not component type +3. Add the module to `src/migration/rules.rs` and export it +4. Register it in `Migrator::default()` in `src/migration.rs` + - Add to the `rules` vec with appropriate comment (processor/controller service) +5. Add comprehensive unit tests in the rule file + - Test both the rule in isolation and in the full migration flow 6. **Update README.md** in the "Supported Migrations" section: - Add a new subsection describing the migration - Include the old and new type/bundle values - - Explain why the migration is needed + - Explain why the migration is needed (link to JIRA ticket if available) 7. Run all checks listed above (fmt, clippy, test, reuse lint) All steps must be completed before considering the migration rule complete. + +## Project Structure + +- `src/main.rs` - CLI entry point, displays version detection and migration results +- `src/cli.rs` - Command-line argument parsing with clap +- `src/migration.rs` - Core migration engine with `Migrator` struct +- `src/migration/rules.rs` - Module declarations for all migration rules +- `src/migration/rules/*.rs` - Individual migration rule implementations +- There might be other files but these are the core ones diff --git a/README.md b/README.md index 2731a55..ac53a06 100644 --- a/README.md +++ b/README.md @@ -59,34 +59,37 @@ Call `nifi-migrate --help` to see all its options. ### JSON Formatting -⚠️ **The tool will reformat your JSON file.** By default, output is compact (single line). Use `--pretty` flag for human-readable formatting with indentation. +> [!WARNING] +> The output file will be reformatted. By default, output is compact (single line). Use `--pretty` flag for human-readable formatting with indentation. -The order of JSON keys may also change as the file is parsed and reserialized. While this doesn't affect NiFi's ability to read the file, it may make git diffs larger. +The order of JSON keys in the output may also change as the file is parsed and reserialized. While this doesn't affect NiFi's ability to read the file, it may make git diffs larger. + +The input file is never modified. ## Adding New Migration Rules To add a new migration rule: 1. look at one of the existing ones in `src/rules` and follow the pattern (basically: Implement the `MigrationRule` trait). -1. Add your rule to `src/rules/rules.rs`: - -```rust -mod my_rule; -pub use my_rule::MyMigrationRule; -``` - -1. Register it in `Migrator::new()` in `src/migration.rs`: - -```rust -pub fn new() -> Self { - Self { - rules: vec![ - Box::new(JoltTransformMigration), - Box::new(MyMigrationRule), // Add your rule here - ], +2. Add your rule to `src/rules/rules.rs`: + + ```rust + mod my_rule; + pub use my_rule::MyMigrationRule; + ``` + +3. Register it in `Migrator::new()` in `src/migration.rs`: + + ```rust + pub fn new() -> Self { + Self { + rules: vec![ + Box::new(JoltTransformMigration), + Box::new(MyMigrationRule), // Add your rule here + ], + } } -} -``` + ``` ## License