From 9493eb8fa10a7666f96e3b77efd7f90d3dc621de Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 5 Jan 2026 14:59:35 +0100 Subject: [PATCH 01/13] add cli wrapper --- .github/workflows/conformance.yml | 29 ++++++++++++++ pom.xml | 25 ++++++++++++ src/main/java/dev/zarr/zarrjava/cli/Main.java | 38 +++++++++++++++++++ 3 files changed, 92 insertions(+) create mode 100644 .github/workflows/conformance.yml create mode 100644 src/main/java/dev/zarr/zarrjava/cli/Main.java diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 0000000..927c3ff --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,29 @@ +name: Zarr Conformance Tests + +on: + workflow_dispatch: + schedule: + - cron: '17 3 * * 0' + +jobs: + conformance-tests: + runs-on: ubuntu-latest + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v5 + + - name: Set up JDK + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + cache: maven + + - name: Build + run: mvn package -DskipTests + + - name: Run Conformance Tests + uses: Bisaloo/zarr-conformance-tests@v0.0.1 + with: + zarr-cli: "java -jar target/zarr-java-0.0.9.jar" diff --git a/pom.xml b/pom.xml index 3df9b78..819f13e 100644 --- a/pom.xml +++ b/pom.xml @@ -121,6 +121,11 @@ 4.13.1 test + + info.picocli + picocli + 4.7.6 + @@ -231,6 +236,26 @@ true + + org.apache.maven.plugins + maven-shade-plugin + 3.5.0 + + + package + + shade + + + + + dev.zarr.zarrjava.cli.Main + + + + + + diff --git a/src/main/java/dev/zarr/zarrjava/cli/Main.java b/src/main/java/dev/zarr/zarrjava/cli/Main.java new file mode 100644 index 0000000..3a9eae8 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/cli/Main.java @@ -0,0 +1,38 @@ +package dev.zarr.zarrjava.cli; + +import dev.zarr.zarrjava.core.Array; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.Callable; + +@Command(name = "zarr-java-cli", mixinStandardHelpOptions = true, version = "1.0", description = "CLI wrapper for zarr-java conformance tests.") +public class Main implements Callable { + + @Option(names = { "--array_path" }, description = "Path to the Zarr array", required = true) + private String arrayPath; + + @Override + public Integer call() throws Exception { + try { + Path path = Paths.get(arrayPath); + // Attempt to open the array. This should throw if the array is invalid or + // cannot be opened. + Array.open(path); + // If we get here, it means we successfully opened the array. + return 0; + } catch (Exception e) { + System.err.println("Failed to open array at " + arrayPath); + e.printStackTrace(); + return 1; + } + } + + public static void main(String[] args) { + int exitCode = new CommandLine(new Main()).execute(args); + System.exit(exitCode); + } +} From 2ec5bda12f6187dbdba3c5463eeb014b9f9b00b3 Mon Sep 17 00:00:00 2001 From: Hannes Spitz <44113112+brokkoli71@users.noreply.github.com> Date: Mon, 5 Jan 2026 15:03:51 +0100 Subject: [PATCH 02/13] Add pull request trigger to conformance workflow --- .github/workflows/conformance.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 927c3ff..8bf9afc 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -4,6 +4,8 @@ on: workflow_dispatch: schedule: - cron: '17 3 * * 0' + pull_request: + branches: [ "main" ] jobs: conformance-tests: From 5e491af03abea63614d983acadd4b97082025ea9 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 5 Jan 2026 15:13:36 +0100 Subject: [PATCH 03/13] fix pom dependencies --- pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pom.xml b/pom.xml index 819f13e..1119bad 100644 --- a/pom.xml +++ b/pom.xml @@ -121,6 +121,11 @@ 4.13.1 test + + org.apache.commons + commons-compress + 1.28.0 + info.picocli picocli From 909da8bc873f051e637f28e5824f448f2f265114 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 5 Jan 2026 15:23:17 +0100 Subject: [PATCH 04/13] print array --- .gitignore | 3 ++- pom.xml | 6 +----- src/main/java/dev/zarr/zarrjava/cli/Main.java | 10 ++++++++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index fcafa91..4c52107 100644 --- a/.gitignore +++ b/.gitignore @@ -45,4 +45,5 @@ build/ /main.py /pyproject.toml /uv.lock -**/__pycache__ \ No newline at end of file +**/__pycache__ +/dependency-reduced-pom.xml diff --git a/pom.xml b/pom.xml index 1119bad..5adcaa9 100644 --- a/pom.xml +++ b/pom.xml @@ -121,11 +121,7 @@ 4.13.1 test - - org.apache.commons - commons-compress - 1.28.0 - + info.picocli picocli diff --git a/src/main/java/dev/zarr/zarrjava/cli/Main.java b/src/main/java/dev/zarr/zarrjava/cli/Main.java index 3a9eae8..1d18cfb 100644 --- a/src/main/java/dev/zarr/zarrjava/cli/Main.java +++ b/src/main/java/dev/zarr/zarrjava/cli/Main.java @@ -21,8 +21,14 @@ public Integer call() throws Exception { Path path = Paths.get(arrayPath); // Attempt to open the array. This should throw if the array is invalid or // cannot be opened. - Array.open(path); - // If we get here, it means we successfully opened the array. + Array array = Array.open(path); + + // Read the entire array + ucar.ma2.Array data = array.read(); + + // Print the array values using ucar.ma2.Array's string representation. + System.out.println(data.toString()); + return 0; } catch (Exception e) { System.err.println("Failed to open array at " + arrayPath); From 41ca7b60d4a358ed8f49bef61059a105349dbf0f Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Sat, 7 Feb 2026 12:59:22 +0100 Subject: [PATCH 05/13] use fork for debugging --- .github/workflows/conformance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 8bf9afc..a33ebba 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -26,6 +26,6 @@ jobs: run: mvn package -DskipTests - name: Run Conformance Tests - uses: Bisaloo/zarr-conformance-tests@v0.0.1 + uses: brokkoli71/zarr-conformance-tests@main with: zarr-cli: "java -jar target/zarr-java-0.0.9.jar" From a786b943ca3652e7aaed5f8a6e53cc19bf38cc50 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Sat, 7 Feb 2026 13:24:02 +0100 Subject: [PATCH 06/13] add zstd to v2 --- .../zarrjava/core/codec/core/ZstdCodec.java | 40 +++++++++++++++++ .../zarr/zarrjava/v2/codec/CodecRegistry.java | 2 + .../zarrjava/v2/codec/core/ZstdCodec.java | 43 +++++++++++++++++++ .../zarrjava/v3/codec/core/ZstdCodec.java | 29 +++---------- 4 files changed, 90 insertions(+), 24 deletions(-) create mode 100644 src/main/java/dev/zarr/zarrjava/core/codec/core/ZstdCodec.java create mode 100644 src/main/java/dev/zarr/zarrjava/v2/codec/core/ZstdCodec.java diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/core/ZstdCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/core/ZstdCodec.java new file mode 100644 index 0000000..bbafa5e --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/core/codec/core/ZstdCodec.java @@ -0,0 +1,40 @@ +package dev.zarr.zarrjava.core.codec.core; + +import com.github.luben.zstd.Zstd; +import com.github.luben.zstd.ZstdCompressCtx; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.codec.BytesBytesCodec; + +import java.nio.ByteBuffer; + +public abstract class ZstdCodec extends BytesBytesCodec { + + protected abstract int getLevel(); + + protected abstract boolean getChecksum(); + + @Override + public ByteBuffer decode(ByteBuffer compressedBytes) throws ZarrException { + byte[] compressedArray = compressedBytes.array(); + + long originalSize = Zstd.getFrameContentSize(compressedArray); + if (originalSize == 0) { + throw new ZarrException("Failed to get decompressed size"); + } + + byte[] decompressed = Zstd.decompress(compressedArray, (int) originalSize); + return ByteBuffer.wrap(decompressed); + } + + @Override + public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException { + byte[] arr = chunkBytes.array(); + byte[] compressed; + try (ZstdCompressCtx ctx = new ZstdCompressCtx()) { + ctx.setLevel(getLevel()); + ctx.setChecksum(getChecksum()); + compressed = ctx.compress(arr); + } + return ByteBuffer.wrap(compressed); + } +} diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/CodecRegistry.java b/src/main/java/dev/zarr/zarrjava/v2/codec/CodecRegistry.java index 2a1a9fa..f0cb7fa 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/CodecRegistry.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/CodecRegistry.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.databind.jsontype.NamedType; import dev.zarr.zarrjava.v2.codec.core.BloscCodec; import dev.zarr.zarrjava.v2.codec.core.ZlibCodec; +import dev.zarr.zarrjava.v2.codec.core.ZstdCodec; import java.util.HashMap; import java.util.Map; @@ -14,6 +15,7 @@ public class CodecRegistry { static { addType("blosc", BloscCodec.class); addType("zlib", ZlibCodec.class); + addType("zstd", ZstdCodec.class); } public static void addType(String name, Class codecClass) { diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZstdCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZstdCodec.java new file mode 100644 index 0000000..d3f23eb --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZstdCodec.java @@ -0,0 +1,43 @@ +package dev.zarr.zarrjava.v2.codec.core; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.ArrayMetadata; +import dev.zarr.zarrjava.v2.codec.Codec; + +public class ZstdCodec extends dev.zarr.zarrjava.core.codec.core.ZstdCodec implements Codec { + + @JsonIgnore + public final String id = "zstd"; + public final int level; + public final boolean checksum; + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public ZstdCodec( + @JsonProperty(value = "level", defaultValue = "0") int level, + @JsonProperty(value = "checksum", defaultValue = "false") boolean checksum) + throws ZarrException { + if (level < -131072 || level > 22) { + throw new ZarrException("'level' needs to be between -131072 and 22."); + } + this.level = level; + this.checksum = checksum; + } + + @Override + protected int getLevel() { + return level; + } + + @Override + protected boolean getChecksum() { + return checksum; + } + + @Override + public Codec evolveFromCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) { + return this; + } +} diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java index d7599bc..3a0a2eb 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java @@ -3,17 +3,13 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; -import com.github.luben.zstd.Zstd; -import com.github.luben.zstd.ZstdCompressCtx; import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.core.codec.BytesBytesCodec; import dev.zarr.zarrjava.v3.ArrayMetadata; import dev.zarr.zarrjava.v3.codec.Codec; import javax.annotation.Nonnull; -import java.nio.ByteBuffer; -public class ZstdCodec extends BytesBytesCodec implements Codec { +public class ZstdCodec extends dev.zarr.zarrjava.core.codec.core.ZstdCodec implements Codec { @JsonIgnore public final String name = "zstd"; @@ -27,28 +23,13 @@ public ZstdCodec( } @Override - public ByteBuffer decode(ByteBuffer compressedBytes) throws ZarrException { - byte[] compressedArray = compressedBytes.array(); - - long originalSize = Zstd.getFrameContentSize(compressedArray); - if (originalSize == 0) { - throw new ZarrException("Failed to get decompressed size"); - } - - byte[] decompressed = Zstd.decompress(compressedArray, (int) originalSize); - return ByteBuffer.wrap(decompressed); + protected int getLevel() { + return configuration.level; } @Override - public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException { - byte[] arr = chunkBytes.array(); - byte[] compressed; - try (ZstdCompressCtx ctx = new ZstdCompressCtx()) { - ctx.setLevel(configuration.level); - ctx.setChecksum(configuration.checksum); - compressed = ctx.compress(arr); - } - return ByteBuffer.wrap(compressed); + protected boolean getChecksum() { + return configuration.checksum; } @Override From 41b092918a0ea966ad553e3396fcdc268aba2e4e Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Sat, 7 Feb 2026 13:24:08 +0100 Subject: [PATCH 07/13] update zarr-cli version to 0.0.10 in conformance tests --- .github/workflows/conformance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index a33ebba..4c535fb 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -28,4 +28,4 @@ jobs: - name: Run Conformance Tests uses: brokkoli71/zarr-conformance-tests@main with: - zarr-cli: "java -jar target/zarr-java-0.0.9.jar" + zarr-cli: "java -jar target/zarr-java-0.0.10.jar" From f27dae81994581237c500352cf597210da874446 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Sat, 7 Feb 2026 13:27:53 +0100 Subject: [PATCH 08/13] update zarr-cli path in conformance tests --- .github/workflows/conformance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 4c535fb..bad068a 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -28,4 +28,4 @@ jobs: - name: Run Conformance Tests uses: brokkoli71/zarr-conformance-tests@main with: - zarr-cli: "java -jar target/zarr-java-0.0.10.jar" + zarr-cli: "java -jar ${{ github.workspace }}/target/zarr-java-0.0.10.jar" From 6ba852e0621cea491d90400788c493d2d372d0f4 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Sat, 7 Feb 2026 13:31:34 +0100 Subject: [PATCH 09/13] set dynamic JAR path for conformance tests --- .github/workflows/conformance.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index bad068a..f4c0be7 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -25,7 +25,14 @@ jobs: - name: Build run: mvn package -DskipTests + - name: Set JAR path + id: jar + run: | + JAR_PATH=$(ls target/zarr-java-*.jar | grep -v original | grep -v javadoc | grep -v sources | head -1) + echo "path=${GITHUB_WORKSPACE}/${JAR_PATH}" >> $GITHUB_OUTPUT + echo "Found JAR at: ${GITHUB_WORKSPACE}/${JAR_PATH}" + - name: Run Conformance Tests uses: brokkoli71/zarr-conformance-tests@main with: - zarr-cli: "java -jar ${{ github.workspace }}/target/zarr-java-0.0.10.jar" + zarr-cli: "java -jar ${{ steps.jar.outputs.path }}" From 39bc251bcbcbf2cbde6bb2922485ec7e561943a5 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Sat, 7 Feb 2026 13:35:43 +0100 Subject: [PATCH 10/13] Improve JAR path handling in conformance workflow - Use GITHUB_ENV instead of GITHUB_OUTPUT for better compatibility - Add verification step to ensure JAR exists before running tests - Add human-readable confirmation message --- .github/workflows/conformance.yml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index f4c0be7..1eaef8d 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -25,14 +25,23 @@ jobs: - name: Build run: mvn package -DskipTests - - name: Set JAR path - id: jar + - name: Prepare JAR for testing run: | + # Find the JAR JAR_PATH=$(ls target/zarr-java-*.jar | grep -v original | grep -v javadoc | grep -v sources | head -1) - echo "path=${GITHUB_WORKSPACE}/${JAR_PATH}" >> $GITHUB_OUTPUT - echo "Found JAR at: ${GITHUB_WORKSPACE}/${JAR_PATH}" + FULL_JAR_PATH="${GITHUB_WORKSPACE}/${JAR_PATH}" + echo "JAR_PATH=${FULL_JAR_PATH}" >> $GITHUB_ENV + echo "Found JAR at: ${FULL_JAR_PATH}" + # Verify it exists + if [ -f "$FULL_JAR_PATH" ]; then + echo "✓ JAR file exists and is accessible" + ls -lh "$FULL_JAR_PATH" + else + echo "✗ JAR file not found!" + exit 1 + fi - name: Run Conformance Tests uses: brokkoli71/zarr-conformance-tests@main with: - zarr-cli: "java -jar ${{ steps.jar.outputs.path }}" + zarr-cli: "java -jar ${{ env.JAR_PATH }}" From 592731e6f670fc1d8c9165db762a9877f4066d74 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Sat, 7 Feb 2026 13:42:53 +0100 Subject: [PATCH 11/13] set conformance tests back to Bisaloo/zarr-conformance-tests@main --- .github/workflows/conformance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 1eaef8d..4f0f1ce 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -42,6 +42,6 @@ jobs: fi - name: Run Conformance Tests - uses: brokkoli71/zarr-conformance-tests@main + uses: Bisaloo/zarr-conformance-tests@main with: zarr-cli: "java -jar ${{ env.JAR_PATH }}" From ab8244d9c68007c3466dabccb400b8907348e789 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Sat, 7 Feb 2026 13:50:02 +0100 Subject: [PATCH 12/13] Revert "set conformance tests back to Bisaloo/zarr-conformance-tests@main" This reverts commit 592731e6f670fc1d8c9165db762a9877f4066d74. --- .github/workflows/conformance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 4f0f1ce..1eaef8d 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -42,6 +42,6 @@ jobs: fi - name: Run Conformance Tests - uses: Bisaloo/zarr-conformance-tests@main + uses: brokkoli71/zarr-conformance-tests@main with: zarr-cli: "java -jar ${{ env.JAR_PATH }}" From 0dbb851190126813868e55085c1f415c5798ffdd Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 9 Feb 2026 13:41:07 +0100 Subject: [PATCH 13/13] remove debug statements --- .github/workflows/conformance.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 1eaef8d..8dd0725 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -32,14 +32,6 @@ jobs: FULL_JAR_PATH="${GITHUB_WORKSPACE}/${JAR_PATH}" echo "JAR_PATH=${FULL_JAR_PATH}" >> $GITHUB_ENV echo "Found JAR at: ${FULL_JAR_PATH}" - # Verify it exists - if [ -f "$FULL_JAR_PATH" ]; then - echo "✓ JAR file exists and is accessible" - ls -lh "$FULL_JAR_PATH" - else - echo "✗ JAR file not found!" - exit 1 - fi - name: Run Conformance Tests uses: brokkoli71/zarr-conformance-tests@main