diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 0000000..8dd0725 --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,39 @@ +name: Zarr Conformance Tests + +on: + workflow_dispatch: + schedule: + - cron: '17 3 * * 0' + pull_request: + branches: [ "main" ] + +jobs: + conformance-tests: + runs-on: ubuntu-latest + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v5 + + - name: Set up JDK + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + cache: maven + + - name: Build + run: mvn package -DskipTests + + - name: Prepare JAR for testing + run: | + # Find the JAR + JAR_PATH=$(ls target/zarr-java-*.jar | grep -v original | grep -v javadoc | grep -v sources | head -1) + FULL_JAR_PATH="${GITHUB_WORKSPACE}/${JAR_PATH}" + echo "JAR_PATH=${FULL_JAR_PATH}" >> $GITHUB_ENV + echo "Found JAR at: ${FULL_JAR_PATH}" + + - name: Run Conformance Tests + uses: brokkoli71/zarr-conformance-tests@main + with: + zarr-cli: "java -jar ${{ env.JAR_PATH }}" diff --git a/pom.xml b/pom.xml index 988b944..03057cc 100644 --- a/pom.xml +++ b/pom.xml @@ -126,6 +126,12 @@ commons-compress 1.28.0 + + + info.picocli + picocli + 4.7.6 + @@ -240,6 +246,26 @@ true + + org.apache.maven.plugins + maven-shade-plugin + 3.5.0 + + + package + + shade + + + + + dev.zarr.zarrjava.cli.Main + + + + + + diff --git a/src/main/java/dev/zarr/zarrjava/cli/Main.java b/src/main/java/dev/zarr/zarrjava/cli/Main.java new file mode 100644 index 0000000..1d18cfb --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/cli/Main.java @@ -0,0 +1,44 @@ +package dev.zarr.zarrjava.cli; + +import dev.zarr.zarrjava.core.Array; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.Callable; + +@Command(name = "zarr-java-cli", mixinStandardHelpOptions = true, version = "1.0", description = "CLI wrapper for zarr-java conformance tests.") +public class Main implements Callable { + + @Option(names = { "--array_path" }, description = "Path to the Zarr array", required = true) + private String arrayPath; + + @Override + public Integer call() throws Exception { + try { + Path path = Paths.get(arrayPath); + // Attempt to open the array. This should throw if the array is invalid or + // cannot be opened. + Array array = Array.open(path); + + // Read the entire array + ucar.ma2.Array data = array.read(); + + // Print the array values using ucar.ma2.Array's string representation. + System.out.println(data.toString()); + + return 0; + } catch (Exception e) { + System.err.println("Failed to open array at " + arrayPath); + e.printStackTrace(); + return 1; + } + } + + public static void main(String[] args) { + int exitCode = new CommandLine(new Main()).execute(args); + System.exit(exitCode); + } +} diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/core/ZstdCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/core/ZstdCodec.java new file mode 100644 index 0000000..b19973d --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/core/codec/core/ZstdCodec.java @@ -0,0 +1,41 @@ +package dev.zarr.zarrjava.core.codec.core; + +import com.github.luben.zstd.Zstd; +import com.github.luben.zstd.ZstdCompressCtx; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.codec.BytesBytesCodec; +import dev.zarr.zarrjava.utils.Utils; + +import java.nio.ByteBuffer; + +public abstract class ZstdCodec extends BytesBytesCodec { + + protected abstract int getLevel(); + + protected abstract boolean getChecksum(); + + @Override + public ByteBuffer decode(ByteBuffer compressedBytes) throws ZarrException { + byte[] compressedArray = Utils.toArray(compressedBytes); + + long originalSize = Zstd.getFrameContentSize(compressedArray); + if (originalSize == 0) { + throw new ZarrException("Failed to get decompressed size"); + } + + byte[] decompressed = Zstd.decompress(compressedArray, (int) originalSize); + return ByteBuffer.wrap(decompressed); + } + + @Override + public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException { + byte[] arr = Utils.toArray(chunkBytes); + byte[] compressed; + try (ZstdCompressCtx ctx = new ZstdCompressCtx()) { + ctx.setLevel(getLevel()); + ctx.setChecksum(getChecksum()); + compressed = ctx.compress(arr); + } + return ByteBuffer.wrap(compressed); + } +} diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/CodecRegistry.java b/src/main/java/dev/zarr/zarrjava/v2/codec/CodecRegistry.java index 2a1a9fa..f0cb7fa 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/CodecRegistry.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/CodecRegistry.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.databind.jsontype.NamedType; import dev.zarr.zarrjava.v2.codec.core.BloscCodec; import dev.zarr.zarrjava.v2.codec.core.ZlibCodec; +import dev.zarr.zarrjava.v2.codec.core.ZstdCodec; import java.util.HashMap; import java.util.Map; @@ -14,6 +15,7 @@ public class CodecRegistry { static { addType("blosc", BloscCodec.class); addType("zlib", ZlibCodec.class); + addType("zstd", ZstdCodec.class); } public static void addType(String name, Class codecClass) { diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZstdCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZstdCodec.java new file mode 100644 index 0000000..d3f23eb --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZstdCodec.java @@ -0,0 +1,43 @@ +package dev.zarr.zarrjava.v2.codec.core; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.ArrayMetadata; +import dev.zarr.zarrjava.v2.codec.Codec; + +public class ZstdCodec extends dev.zarr.zarrjava.core.codec.core.ZstdCodec implements Codec { + + @JsonIgnore + public final String id = "zstd"; + public final int level; + public final boolean checksum; + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public ZstdCodec( + @JsonProperty(value = "level", defaultValue = "0") int level, + @JsonProperty(value = "checksum", defaultValue = "false") boolean checksum) + throws ZarrException { + if (level < -131072 || level > 22) { + throw new ZarrException("'level' needs to be between -131072 and 22."); + } + this.level = level; + this.checksum = checksum; + } + + @Override + protected int getLevel() { + return level; + } + + @Override + protected boolean getChecksum() { + return checksum; + } + + @Override + public Codec evolveFromCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) { + return this; + } +} diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java index 7d4b336..3a0a2eb 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java @@ -3,18 +3,13 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; -import com.github.luben.zstd.Zstd; -import com.github.luben.zstd.ZstdCompressCtx; import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.core.codec.BytesBytesCodec; -import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v3.ArrayMetadata; import dev.zarr.zarrjava.v3.codec.Codec; import javax.annotation.Nonnull; -import java.nio.ByteBuffer; -public class ZstdCodec extends BytesBytesCodec implements Codec { +public class ZstdCodec extends dev.zarr.zarrjava.core.codec.core.ZstdCodec implements Codec { @JsonIgnore public final String name = "zstd"; @@ -28,28 +23,13 @@ public ZstdCodec( } @Override - public ByteBuffer decode(ByteBuffer compressedBytes) throws ZarrException { - byte[] compressedArray = Utils.toArray(compressedBytes); - - long originalSize = Zstd.getFrameContentSize(compressedArray); - if (originalSize == 0) { - throw new ZarrException("Failed to get decompressed size"); - } - - byte[] decompressed = Zstd.decompress(compressedArray, (int) originalSize); - return ByteBuffer.wrap(decompressed); + protected int getLevel() { + return configuration.level; } @Override - public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException { - byte[] arr = Utils.toArray(chunkBytes); - byte[] compressed; - try (ZstdCompressCtx ctx = new ZstdCompressCtx()) { - ctx.setLevel(configuration.level); - ctx.setChecksum(configuration.checksum); - compressed = ctx.compress(arr); - } - return ByteBuffer.wrap(compressed); + protected boolean getChecksum() { + return configuration.checksum; } @Override