#122 spark4 upgrade

.github/workflows/build.yml

@@ -24,24 +24,30 @@ on:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
-        scala: [2.12.18, 2.13.11]
-    name: Scala ${{matrix.scala}}
+        #  sbt per project  (Scala, Spark) row of the projectMatrix in build.sbt, Spark 4 with Java 17
+        include:
+          - { scala: 2.12.20, spark: 3.5.7, java: '11', project: standardization2_12Spark35 }
+          - { scala: 2.12.20, spark: 3.5.7, java: '17', project: standardization2_12Spark35 }
+          - { scala: 2.13.17, spark: 3.5.7, java: '11', project: standardizationSpark35 }
+          - { scala: 2.13.17, spark: 3.5.7, java: '17', project: standardizationSpark35 }
+          - { scala: 2.13.17, spark: 4.1.2, java: '17', project: standardizationSpark41 }
+    name: Scala ${{matrix.scala}}, Spark ${{matrix.spark}}, Java ${{matrix.java}}
     steps:
       - name: Checkout code
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
         with:
           persist-credentials: false
 
-      - uses: coursier/cache-action@95e5b1029b6b86e7bac033ee44a0697d8a527d2d
-
-      - name: Setup Scala
-        uses: olafurpg/setup-scala@32ffa16635ff8f19cc21ea253a987f0fdf29844c
+      - name: Setup JDK
+        uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1
         with:
-          java-version: "adopt@1.11"
+          java-version: ${{ matrix.java }}
+          distribution: temurin
+          cache: "sbt"
 
       - name: Build and run tests
-        run: sbt ++${{matrix.scala}} test doc
+        run: sbt ${{matrix.project}}/test ${{matrix.project}}/doc

.github/workflows/format_check.yml

@@ -22,7 +22,7 @@ on:
 
 jobs:
   scalafmt:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     name: Scalafmt Check
     steps:
       - name: Checkout code
@@ -34,10 +34,11 @@ jobs:
 
       - uses: coursier/cache-action@95e5b1029b6b86e7bac033ee44a0697d8a527d2d
 
-      - name: Setup Scala
-        uses: olafurpg/setup-scala@32ffa16635ff8f19cc21ea253a987f0fdf29844c
+      - name: Setup JDK
+        uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1
         with:
-          java-version: "adopt@1.11"
+          distribution: temurin
+          java-version: '17'
 
       - name: Run scalafmt And Print Diff
         continue-on-error: true

.github/workflows/jacoco_check.yml

@@ -23,21 +23,34 @@ on:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
+      fail-fast: false
       matrix:
         include:
-          # The project supports Scala 2.12, 2.13
-          # The CI runs all tests suites for all supported Scala versions at build.yml
-          # The codebase for all Scala versions is the same, so the coverage is calculated only once
-          # Scala 2.12 is chosen since it is supported by the most wide range of Spark versions and
-          # vendor distributions.
+          # The project is built for (Scala 2.12, Spark 3.5), (Scala 2.13, Spark 3.5) and (Scala 2.13, Spark 4.1)
+          # The CI runs all tests suites for all of them at build.yml
+          # The codebase is shared but the runtime paths differ per Spark major version ( ANSI mode is on by default
+          # in Spark 4) so coverage is checked once per Spark major version:
+          # - Scala 2.12 / Spark 3.5: supported by the wide range of Spark versions and vendor distributions
+          # - Scala 2.13 / Spark 4.1: the only Spark 4 row
           - scala: 2.12.20
             scalaShort: "2.12"
+            spark: 3.5.7
+            project: standardization2_12Spark35
             overall: 80.0
             changed: 80.0
             per-changed-file: 60.0
 
+          - scala: 2.13.17
+            scalaShort: "2.13"
+            spark: 4.1.2
+            project: standardizationSpark41
+            overall: 75.0
+            changed: 80.0
+            per-changed-file: 60.0
+    name: JaCoCo - Scala ${{matrix.scala}}, Spark ${{matrix.spark}}
+
     steps:
       - name: Checkout code
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
@@ -46,18 +59,19 @@ jobs:
 
       - uses: coursier/cache-action@95e5b1029b6b86e7bac033ee44a0697d8a527d2d
 
-      - name: Setup Scala
-        uses: olafurpg/setup-scala@32ffa16635ff8f19cc21ea253a987f0fdf29844c
+      - name: Setup JDK
+        uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1
         with:
-          java-version: "adopt@1.11"
+          distribution: temurin
+          java-version: '17'
 
       - name: Setup Python
         uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
         with:
           python-version: "3.14"
 
       - name: Build and run tests
-        run: sbt ++${{matrix.scala}} jacoco
+        run: sbt "project ${{matrix.project}}" jacoco
 
       - name: Add JaCoCo Report in PR comments
         id: jacoco
@@ -68,7 +82,7 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           global-thresholds: '${{ matrix.overall }}*${{ matrix.changed }}'
           report-thresholds-default: '${{ matrix.overall }}*${{ matrix.changed }}*${{ matrix.per-changed-file }}'
-          title: JaCoCo code coverage report - scala ${{ matrix.scala }}
+          title: JaCoCo code coverage report - scala ${{ matrix.scala }}, spark ${{ matrix.spark }}
           update-comment: true
 
       - name: Get the Coverage info

.github/workflows/licence_check.yml

@@ -24,17 +24,18 @@ on:
 
 jobs:
   license-test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout code
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
         with:
           persist-credentials: false
 
-      - name: Setup Scala
-        uses: olafurpg/setup-scala@32ffa16635ff8f19cc21ea253a987f0fdf29844c
+      - name: Setup JDK
+        uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1
         with:
-          java-version: "adopt@1.11"
+          distribution: temurin
+          java-version: '17'
 
       # note, that task "headerCheck" defaults to just "compile:headerCheck" - see https://github.com/sbt/sbt-header/issues/14
       - name: SBT src licence header check

.github/workflows/release.yml

@@ -20,22 +20,41 @@ on:
 
 jobs:
   publish:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout code
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
         with:
           fetch-depth: 0
           persist-credentials: false
 
-      - name: Setup Scala
-        uses: olafurpg/setup-scala@32ffa16635ff8f19cc21ea253a987f0fdf29844c
+      # Spark 4.1 requires Java 17, the Spark 3.5 rows stay Java 11 compatible
+      # build.sbt pins -release 11 / -source 11 -target 11.
+      - name: Setup JDK
+        uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1
         with:
-          java-version: "adopt@1.11"
+          distribution: temurin
+          java-version: '17'
+          cache: sbt
 
-      - run: sbt ci-release
+      # Making the release auditable: printing every module ci-release will publish,
+      # ensuring the Spark 4.1 / Scala 2.13 artifact are visible in the log.
+      # Expected: spark-data-standardization-spark3.5_2.12,
+      #           spark-data-standardization-spark3.5_2.13,
+      #           spark-data-standardization-spark4.1_2.13
+      - name: List modules to be published
+        run: |
+          sbt -batch -no-colors \
+            'show standardization2_12Spark35 / moduleName' \
+            'show standardizationSpark35 / moduleName' \
+            'show standardizationSpark41 / moduleName' \
+            'show root / publish / skip'
+
+      - name: Publish all cross-build artifacts
+        run: sbt ci-release
         env:
           PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }}
           PGP_SECRET: ${{ secrets.PGP_SECRET }}
           SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
           SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
+

README.md

@@ -3,6 +3,7 @@
 [![License](http://img.shields.io/:license-apache-blue.svg)](http://www.apache.org/licenses/LICENSE-2.0.html)
 [![Release](https://github.com/AbsaOSS/spark-data-standardization/actions/workflows/release.yml/badge.svg)](https://github.com/AbsaOSS/spark-data-standardization/actions/workflows/release.yml)
 ![Java 11](https://img.shields.io/badge/Java_11-ED8B00?style=flat&logo=openjdk&logoColor=black)
+![Java 17](https://img.shields.io/badge/Java_17-ED8B00?style=flat&logo=openjdk&logoColor=black)
 
 - Dataframe in 
 - Standardized Dataframe out
@@ -16,50 +17,99 @@ The library needs following dependencies to be included in your project
 ```sbt
 "org.apache.spark" %% "spark-core" % SPARK_VERSION,
 "org.apache.spark" %% "spark-sql" % SPARK_VERSION,
-"za.co.absa" %% s"spark-commons-spark${SPARK_MAJOR}.${SPARK_MINOR}" % "0.6.3",
+"za.co.absa" %% s"spark-commons-spark${SPARK_MAJOR}.${SPARK_MINOR}" % "1.0.0",
 ```
 
 ### Usage in SBT:
+
+The artifact name carries the Spark `major.minor` version it is built against, pick the one matching your Spark:
+
 ```sbt
-"za.co.absa" %% "spark-data-standardization" % VERSION 
+"za.co.absa" %% s"spark-data-standardization-spark${SPARK_MAJOR}.${SPARK_MINOR}" % VERSION
 ```
+e.g. `"za.co.absa" %% "spark-data-standardization-spark4.1" % VERSION` for Spark 4.1.x on Scala 2.13.
 
 ### Usage in Maven
 
-### Scala 2.12 [![Maven Central](https://maven-badges.herokuapp.com/maven-central/za.co.absa/spark-data-standardization_2.12/badge.svg)](https://maven-badges.herokuapp.com/maven-central/za.co.absa/spark-data-standardization_2.12)
+### Spark 3.5, Scala 2.12 [![Maven Central](https://maven-badges.herokuapp.com/maven-central/za.co.absa/spark-data-standardization-spark3.5_2.12/badge.svg)](https://maven-badges.herokuapp.com/maven-central/za.co.absa/spark-data-standardization-spark3.5_2.12)
+
+```xml
+<dependency>
+   <groupId>za.co.absa</groupId>
+   <artifactId>spark-data-standardization-spark3.5_2.12</artifactId>
+   <version>${latest_version}</version>
+</dependency>
+```
+
+### Spark 3.5, Scala 2.13 [![Maven Central](https://maven-badges.herokuapp.com/maven-central/za.co.absa/spark-data-standardization-spark3.5_2.13/badge.svg)](https://maven-badges.herokuapp.com/maven-central/za.co.absa/spark-data-standardization-spark3.5_2.13)
 
 ```xml
 <dependency>
    <groupId>za.co.absa</groupId>
-   <artifactId>spark-data-standardization_2.12</artifactId>
+   <artifactId>spark-data-standardization-spark3.5_2.13</artifactId>
    <version>${latest_version}</version>
 </dependency>
 ```
 
-### Scala 2.13 [![Maven Central](https://maven-badges.herokuapp.com/maven-central/za.co.absa/spark-data-standardization_2.13/badge.svg)](https://maven-badges.herokuapp.com/maven-central/za.co.absa/spark-data-standardization_2.13)
+### Spark 4.1, Scala 2.13 [![Maven Central](https://maven-badges.herokuapp.com/maven-central/za.co.absa/spark-data-standardization-spark4.1_2.13/badge.svg)](https://maven-badges.herokuapp.com/maven-central/za.co.absa/spark-data-standardization-spark4.1_2.13)
 
 ```xml
 <dependency>
    <groupId>za.co.absa</groupId>
-   <artifactId>spark-data-standardization_2.13</artifactId>
+   <artifactId>spark-data-standardization-spark4.1_2.13</artifactId>
    <version>${latest_version}</version>
 </dependency>
 ```
 
+> **Upgrading from versions without the Spark suffix:** the previous `spark-data-standardization_2.12` (Spark 3.5)
+> and `spark-data-standardization_2.13` artifacts are replaced by the `-spark<major>.<minor>` artifacts above,
+> so update the artifact id in your dependency coordinates.
+
 Spark and Scala compatibility
 >| | Scala 2.12 | Scala 2.13 |
 >|---|---|---|
->|Spark| 3.5.x | 3.5.x |
+>|Spark| 3.5.x | 3.5.x / 4.1.x |
+
+Java compatibility
+
+>| | Scala 2.12 (Spark 3.5.x) | Scala 2.13 (Spark 3.5.x) | Scala 2.13 (Spark 4.1.x) |
+>|---|---|---|---|
+>|Java| 11 / 17 | 11 / 17 | 17 |
 
+### ANSI mode
+
+Standardization reports values that can't be cast (unparseable, overflowing, invalid dates, ...) as `null` plus an
+entry in the error column, never as a failing query. On Spark 4, where `spark.sql.ansi.enabled` defaults to `true`,
+`standardize` therefore turns ANSI mode off on the session while the standardized DataFrame is being built (Spark
+fixes a cast's evaluation mode when the plan is analysed, this is enough also for actions run later) and restores
+the original value afterwards, even with concurrent `standardize` calls on the same session, see [ANSI handling](src/main/scala/za/co/absa/standardization/adr/002-ansi-handling/ansi-handling.md).
 ## How to Release
 
 Please see [this file](RELEASE.md) for more details.
 
+## How to build and test
+
+The build is an [sbt-projectmatrix](https://github.com/sbt/sbt-projectmatrix), with one sbt project per supported
+(Scala, Spark) combination:
+
+| sbt project                  | Scala | Spark | Artifact                                    |
+|------------------------------|-------|-------|---------------------------------------------|
+| `standardization2_12Spark35` | 2.12  | 3.5.x | `spark-data-standardization-spark3.5_2.12`  |
+| `standardizationSpark35`     | 2.13  | 3.5.x | `spark-data-standardization-spark3.5_2.13`  |
+| `standardizationSpark41`     | 2.13  | 4.1.x | `spark-data-standardization-spark4.1_2.13`  |
+
+```sbt
+sbt test                              # all combinations
+sbt standardizationSpark41/test       # single combination
+```
+Spark 4 requires Java 17, run sbt on Java 17 to test the Spark 4.1 project.
+
 ## How to generate Code coverage report
 ```sbt
-sbt ++<scala.version> jacoco
+sbt "project <sbt project>" jacoco
 ```
 Code coverage will be generated on path:
 ```
-{project-root}/target/scala-{scala_version}/jacoco/report/html
+{project-root}/target/jvm-{scala_version}-spark{spark_version}/jacoco-report
 ```
+e.g. `target/jvm-2.13-spark4.1/jacoco-report` for `standardizationSpark41`.