Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 39 additions & 0 deletions .github/workflows/conformance.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
name: Zarr Conformance Tests

on:
workflow_dispatch:
schedule:
- cron: '17 3 * * 0'
pull_request:
branches: [ "main" ]

jobs:
conformance-tests:
runs-on: ubuntu-latest
env:
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
steps:
- uses: actions/checkout@v5

- name: Set up JDK
uses: actions/setup-java@v4
with:
java-version: '11'
distribution: 'temurin'
cache: maven

- name: Build
run: mvn package -DskipTests

- name: Prepare JAR for testing
run: |
# Find the JAR
JAR_PATH=$(ls target/zarr-java-*.jar | grep -v original | grep -v javadoc | grep -v sources | head -1)
FULL_JAR_PATH="${GITHUB_WORKSPACE}/${JAR_PATH}"
echo "JAR_PATH=${FULL_JAR_PATH}" >> $GITHUB_ENV
echo "Found JAR at: ${FULL_JAR_PATH}"

- name: Run Conformance Tests
uses: brokkoli71/zarr-conformance-tests@main
with:
zarr-cli: "java -jar ${{ env.JAR_PATH }}"
26 changes: 26 additions & 0 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,12 @@
<artifactId>commons-compress</artifactId>
<version>1.28.0</version>
</dependency>

<dependency>
<groupId>info.picocli</groupId>
<artifactId>picocli</artifactId>
<version>4.7.6</version>
</dependency>
</dependencies>

<repositories>
Expand Down Expand Up @@ -240,6 +246,26 @@
<tokenAuth>true</tokenAuth>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.5.0</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>dev.zarr.zarrjava.cli.Main</mainClass>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
44 changes: 44 additions & 0 deletions src/main/java/dev/zarr/zarrjava/cli/Main.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package dev.zarr.zarrjava.cli;

import dev.zarr.zarrjava.core.Array;
import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;

import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.concurrent.Callable;

@Command(name = "zarr-java-cli", mixinStandardHelpOptions = true, version = "1.0", description = "CLI wrapper for zarr-java conformance tests.")
public class Main implements Callable<Integer> {

@Option(names = { "--array_path" }, description = "Path to the Zarr array", required = true)
private String arrayPath;

@Override
public Integer call() throws Exception {
try {
Path path = Paths.get(arrayPath);
// Attempt to open the array. This should throw if the array is invalid or
// cannot be opened.
Array array = Array.open(path);

// Read the entire array
ucar.ma2.Array data = array.read();

// Print the array values using ucar.ma2.Array's string representation.
System.out.println(data.toString());

return 0;
} catch (Exception e) {
System.err.println("Failed to open array at " + arrayPath);
e.printStackTrace();
return 1;
}
}

public static void main(String[] args) {
int exitCode = new CommandLine(new Main()).execute(args);
System.exit(exitCode);
}
}
41 changes: 41 additions & 0 deletions src/main/java/dev/zarr/zarrjava/core/codec/core/ZstdCodec.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package dev.zarr.zarrjava.core.codec.core;

import com.github.luben.zstd.Zstd;
import com.github.luben.zstd.ZstdCompressCtx;
import dev.zarr.zarrjava.ZarrException;
import dev.zarr.zarrjava.core.codec.BytesBytesCodec;
import dev.zarr.zarrjava.utils.Utils;

import java.nio.ByteBuffer;

public abstract class ZstdCodec extends BytesBytesCodec {

protected abstract int getLevel();

protected abstract boolean getChecksum();

@Override
public ByteBuffer decode(ByteBuffer compressedBytes) throws ZarrException {
byte[] compressedArray = Utils.toArray(compressedBytes);

long originalSize = Zstd.getFrameContentSize(compressedArray);
if (originalSize == 0) {
throw new ZarrException("Failed to get decompressed size");
}

byte[] decompressed = Zstd.decompress(compressedArray, (int) originalSize);
return ByteBuffer.wrap(decompressed);
}

@Override
public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException {
byte[] arr = Utils.toArray(chunkBytes);
byte[] compressed;
try (ZstdCompressCtx ctx = new ZstdCompressCtx()) {
ctx.setLevel(getLevel());
ctx.setChecksum(getChecksum());
compressed = ctx.compress(arr);
}
return ByteBuffer.wrap(compressed);
}
}
2 changes: 2 additions & 0 deletions src/main/java/dev/zarr/zarrjava/v2/codec/CodecRegistry.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import com.fasterxml.jackson.databind.jsontype.NamedType;
import dev.zarr.zarrjava.v2.codec.core.BloscCodec;
import dev.zarr.zarrjava.v2.codec.core.ZlibCodec;
import dev.zarr.zarrjava.v2.codec.core.ZstdCodec;

import java.util.HashMap;
import java.util.Map;
Expand All @@ -14,6 +15,7 @@ public class CodecRegistry {
static {
addType("blosc", BloscCodec.class);
addType("zlib", ZlibCodec.class);
addType("zstd", ZstdCodec.class);
}

public static void addType(String name, Class<? extends Codec> codecClass) {
Expand Down
43 changes: 43 additions & 0 deletions src/main/java/dev/zarr/zarrjava/v2/codec/core/ZstdCodec.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package dev.zarr.zarrjava.v2.codec.core;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import dev.zarr.zarrjava.ZarrException;
import dev.zarr.zarrjava.core.ArrayMetadata;
import dev.zarr.zarrjava.v2.codec.Codec;

public class ZstdCodec extends dev.zarr.zarrjava.core.codec.core.ZstdCodec implements Codec {

@JsonIgnore
public final String id = "zstd";
public final int level;
public final boolean checksum;

@JsonCreator(mode = JsonCreator.Mode.PROPERTIES)
public ZstdCodec(
@JsonProperty(value = "level", defaultValue = "0") int level,
@JsonProperty(value = "checksum", defaultValue = "false") boolean checksum)
throws ZarrException {
if (level < -131072 || level > 22) {
throw new ZarrException("'level' needs to be between -131072 and 22.");
}
this.level = level;
this.checksum = checksum;
}

@Override
protected int getLevel() {
return level;
}

@Override
protected boolean getChecksum() {
return checksum;
}

@Override
public Codec evolveFromCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) {
return this;
}
}
30 changes: 5 additions & 25 deletions src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,13 @@
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.github.luben.zstd.Zstd;
import com.github.luben.zstd.ZstdCompressCtx;
import dev.zarr.zarrjava.ZarrException;
import dev.zarr.zarrjava.core.codec.BytesBytesCodec;
import dev.zarr.zarrjava.utils.Utils;
import dev.zarr.zarrjava.v3.ArrayMetadata;
import dev.zarr.zarrjava.v3.codec.Codec;

import javax.annotation.Nonnull;
import java.nio.ByteBuffer;

public class ZstdCodec extends BytesBytesCodec implements Codec {
public class ZstdCodec extends dev.zarr.zarrjava.core.codec.core.ZstdCodec implements Codec {

@JsonIgnore
public final String name = "zstd";
Expand All @@ -28,28 +23,13 @@ public ZstdCodec(
}

@Override
public ByteBuffer decode(ByteBuffer compressedBytes) throws ZarrException {
byte[] compressedArray = Utils.toArray(compressedBytes);

long originalSize = Zstd.getFrameContentSize(compressedArray);
if (originalSize == 0) {
throw new ZarrException("Failed to get decompressed size");
}

byte[] decompressed = Zstd.decompress(compressedArray, (int) originalSize);
return ByteBuffer.wrap(decompressed);
protected int getLevel() {
return configuration.level;
}

@Override
public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException {
byte[] arr = Utils.toArray(chunkBytes);
byte[] compressed;
try (ZstdCompressCtx ctx = new ZstdCompressCtx()) {
ctx.setLevel(configuration.level);
ctx.setChecksum(configuration.checksum);
compressed = ctx.compress(arr);
}
return ByteBuffer.wrap(compressed);
protected boolean getChecksum() {
return configuration.checksum;
}

@Override
Expand Down
Loading