Open
Conversation
diff --git c/src/main/java/dev/zarr/zarrjava/v2/Array.java i/src/main/java/dev/zarr/zarrjava/v2/Array.java index a34c85a..c5ed44b 100644 --- c/src/main/java/dev/zarr/zarrjava/v2/Array.java +++ i/src/main/java/dev/zarr/zarrjava/v2/Array.java @@ -248,7 +248,8 @@ public class Array extends dev.zarr.zarrjava.core.Array implements Node { * @throws IOException throws IOException if the new metadata cannot be serialized */ public Array updateAttributes(Function<Attributes, Attributes> attributeMapper) throws ZarrException, IOException { - return setAttributes(attributeMapper.apply(metadata.attributes)); + Attributes currentAttributes = metadata.attributes != null ? new Attributes(metadata.attributes) : new Attributes(); + return setAttributes(attributeMapper.apply(currentAttributes)); } @OverRide diff --git c/src/main/java/dev/zarr/zarrjava/v2/Group.java i/src/main/java/dev/zarr/zarrjava/v2/Group.java index d3229e4..73b8cdf 100644 --- c/src/main/java/dev/zarr/zarrjava/v2/Group.java +++ i/src/main/java/dev/zarr/zarrjava/v2/Group.java @@ -283,7 +283,8 @@ public class Group extends dev.zarr.zarrjava.core.Group implements Node { */ public Group updateAttributes(Function<Attributes, Attributes> attributeMapper) throws ZarrException, IOException { - return setAttributes(attributeMapper.apply(metadata.attributes)); + Attributes currentAttributes = metadata.attributes != null ? new Attributes(metadata.attributes) : new Attributes(); + return setAttributes(attributeMapper.apply(currentAttributes)); } diff --git c/src/main/java/dev/zarr/zarrjava/v3/Array.java i/src/main/java/dev/zarr/zarrjava/v3/Array.java index ea029db..45060e9 100644 --- c/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ i/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -248,7 +248,8 @@ public class Array extends dev.zarr.zarrjava.core.Array implements Node { * @throws IOException throws IOException if the new metadata cannot be serialized */ public Array updateAttributes(Function<Attributes, Attributes> attributeMapper) throws ZarrException, IOException { - return setAttributes(attributeMapper.apply(metadata.attributes)); + Attributes currentAttributes = metadata.attributes != null ? new Attributes(metadata.attributes) : new Attributes(); + return setAttributes(attributeMapper.apply(currentAttributes)); } @OverRide diff --git c/src/main/java/dev/zarr/zarrjava/v3/Group.java i/src/main/java/dev/zarr/zarrjava/v3/Group.java index 1d3aa6b..8b1a81b 100644 --- c/src/main/java/dev/zarr/zarrjava/v3/Group.java +++ i/src/main/java/dev/zarr/zarrjava/v3/Group.java @@ -289,7 +289,8 @@ public class Group extends dev.zarr.zarrjava.core.Group implements Node { * @throws IOException if the metadata cannot be serialized */ public Group updateAttributes(Function<Attributes, Attributes> attributeMapper) throws ZarrException, IOException { - return setAttributes(attributeMapper.apply(metadata.attributes)); + Attributes currentAttributes = metadata.attributes != null ? new Attributes(metadata.attributes) : new Attributes(); + return setAttributes(attributeMapper.apply(currentAttributes)); } /** diff --git c/src/test/java/dev/zarr/zarrjava/ParallelWriteTest.java i/src/test/java/dev/zarr/zarrjava/ParallelWriteTest.java new file mode 100644 index 0000000..60db2ee --- /dev/null +++ i/src/test/java/dev/zarr/zarrjava/ParallelWriteTest.java @@ -0,0 +1,154 @@ +package dev.zarr.zarrjava; + +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.v3.Array; +import dev.zarr.zarrjava.v3.DataType; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.*; + +public class ParallelWriteTest extends ZarrTest { + + @test + public void testParallelWriteDataSafety() throws IOException, ZarrException { + // Test internal parallelism of write method (using parallel=true) + Path path = TESTOUTPUT.resolve("parallel_write_safety"); + StoreHandle storeHandle = new FilesystemStore(path).resolve(); + + int shape = 1000; + int chunk = 100; + + Array array = Array.create(storeHandle, Array.metadataBuilder() + .withShape(shape, shape) + .withDataType(DataType.INT32) + .withChunkShape(chunk, chunk) + .withFillValue(0) + .build()); + + int[] data = new int[shape * shape]; + // Fill with some deterministic pattern + for (int i = 0; i < shape * shape; i++) { + data[i] = i; + } + + ucar.ma2.Array outputData = ucar.ma2.Array.factory(ucar.ma2.DataType.INT, new int[]{shape, shape}, data); + + // Write in parallel + array.write(outputData, true); + + // Read back + ucar.ma2.Array readData = array.read(); + int[] readArr = (int[]) readData.get1DJavaArray(ucar.ma2.DataType.INT); + + Assertions.assertArrayEquals(data, readArr, "Data read back should match data written in parallel"); + } + + @test + public void testParallelWriteWithSharding() throws IOException, ZarrException { + // Test internal parallelism with Sharding (nested chunks + shared codec state potential) + Path path = TESTOUTPUT.resolve("parallel_write_sharding"); + StoreHandle storeHandle = new FilesystemStore(path).resolve(); + + int shape = 128; // 128x128 + int shardSize = 64; // Shards are 64x64 + int innerChunk = 32; // Inner chunks 32x32 + + // Metadata with sharding + // With shape 128 and shardSize 64, we have 2x2 = 4 shards. + // Array.write(parallel=true) will likely process these shards concurrently. + dev.zarr.zarrjava.v3.ArrayMetadata metadata = Array.metadataBuilder() + .withShape(shape, shape) + .withDataType(DataType.INT32) + .withChunkShape(shardSize, shardSize) // This sets the shard shape (outer chunks) + .withCodecs(c -> c.withSharding(new int[]{innerChunk, innerChunk}, c2 -> c2.withBytes("LITTLE"))) + .withFillValue(0) + .build(); + + Array array = Array.create(storeHandle, metadata); + + int[] data = new int[shape * shape]; + for (int i = 0; i < shape * shape; i++) { + data[i] = i; + } + + ucar.ma2.Array outputData = ucar.ma2.Array.factory(ucar.ma2.DataType.INT, new int[]{shape, shape}, data); + + // Write in parallel + array.write(outputData, true); + + ucar.ma2.Array readData = array.read(); + int[] readArr = (int[]) readData.get1DJavaArray(ucar.ma2.DataType.INT); + + Assertions.assertArrayEquals(data, readArr, "Sharded data written in parallel should match"); + } + + @test + public void testConcurrentWritesDifferentChunks() throws IOException, ZarrException, InterruptedException, ExecutionException { + // Test external parallelism (multiple threads calling write on same Array instance) + Path path = TESTOUTPUT.resolve("concurrent_write_safety"); + StoreHandle storeHandle = new FilesystemStore(path).resolve(); + + int chunksX = 10; + int chunksY = 10; + int chunkSize = 50; + int shapeX = chunksX * chunkSize; + int shapeY = chunksY * chunkSize; + + Array array = Array.create(storeHandle, Array.metadataBuilder() + .withShape(shapeX, shapeY) + .withDataType(DataType.INT32) + .withChunkShape(chunkSize, chunkSize) + .withFillValue(-1) + .build()); + + ExecutorService executor = Executors.newFixedThreadPool(8); + List<Callable<Void>> tasks = new ArrayList<>(); + + for (int i = 0; i < chunksX; i++) { + for (int j = 0; j < chunksY; j++) { + final int cx = i; + final int cy = j; + tasks.add(() -> { + int[] chunkData = new int[chunkSize * chunkSize]; + int val = cx * chunksY + cy; // Unique value per chunk + java.util.Arrays.fill(chunkData, val); + + ucar.ma2.Array ucarArray = ucar.ma2.Array.factory(ucar.ma2.DataType.INT, new int[]{chunkSize, chunkSize}, chunkData); + + // Write to specific chunk offset + long[] offset = new long[]{cx * chunkSize, cy * chunkSize}; + // Use internal parallelism false to isolate external concurrency test mechanism + array.write(offset, ucarArray, false); + return null; + }); + } + } + + List<Future<Void>> futures = executor.invokeAll(tasks); + + for (Future<Void> f : futures) { + f.get(); // Check for exceptions + } + executor.shutdown(); + + // Verification + ucar.ma2.Array readData = array.read(); + for (int i = 0; i < chunksX; i++) { + for (int j = 0; j < chunksY; j++) { + int expectedVal = i * chunksY + j; + int originX = i * chunkSize; + int originY = j * chunkSize; + + // Verify a pixel in the chunk + int val = readData.getInt(readData.getIndex().set(originX, originY)); + Assertions.assertEquals(expectedVal, val, "Value at chunk " + i + "," + j + " mismatch"); + } + } + } +} diff --git c/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java i/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java index c3e498a..779824c 100644 --- c/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java +++ i/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -332,6 +332,31 @@ public class ZarrV2Test extends ZarrTest { assertContainsTestAttributes(array.metadata().attributes()); } + @test + public void testUpdateAttributesBehavior() throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testUpdateAttributesBehaviorV2"); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunks(5, 5) + .withAttributes(new Attributes(b -> b.set("key1", "val1"))) + .build(); + + Array array1 = Array.create(storeHandle, arrayMetadata); + Array array2 = array1.updateAttributes(attrs -> attrs.set("key2", "val2")); + + Assertions.assertNotSame(array1, array2); + Assertions.assertEquals("val1", array1.metadata().attributes().get("key1")); + Assertions.assertNull(array1.metadata().attributes().get("key2")); + + Assertions.assertEquals("val1", array2.metadata().attributes().get("key1")); + Assertions.assertEquals("val2", array2.metadata().attributes().get("key2")); + + // Re-opening should show the updated attributes + Array array3 = Array.open(storeHandle); + Assertions.assertEquals("val2", array3.metadata().attributes().get("key2")); + } + @test public void testResizeArray() throws IOException, ZarrException { int[] testData = new int[10 * 10]; @@ -360,6 +385,34 @@ public class ZarrV2Test extends ZarrTest { Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType)); } + @test + public void testResizeArrayShrink() throws IOException, ZarrException { + int[] testData = new int[10 * 10]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testResizeArrayShrinkV2"); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT32) + .withChunks(5, 5) + .build(); + ucar.ma2.DataType ma2DataType = arrayMetadata.dataType.getMA2DataType(); + Array array = Array.create(storeHandle, arrayMetadata); + array.write(new long[]{0, 0}, ucar.ma2.Array.factory(ma2DataType, new int[]{10, 10}, testData)); + + array = array.resize(new long[]{5, 5}); + Assertions.assertArrayEquals(new int[]{5, 5}, array.read().getShape()); + + ucar.ma2.Array data = array.read(); + int[] expectedData = new int[5 * 5]; + for (int i = 0; i < 5; i++) { + for (int j = 0; j < 5; j++) { + expectedData[i * 5 + j] = testData[i * 10 + j]; + } + } + Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType)); + } + @test public void testGroupAttributes() throws IOException, ZarrException { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testGroupAttributesV2"); diff --git c/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java i/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index 3ed3c10..4e67204 100644 --- c/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ i/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -708,6 +708,31 @@ public class ZarrV3Test extends ZarrTest { assertContainsTestAttributes(array.metadata().attributes()); } + @test + public void testUpdateAttributesBehavior() throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testUpdateAttributesBehaviorV3"); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunkShape(5, 5) + .withAttributes(new Attributes(b -> b.set("key1", "val1"))) + .build(); + + Array array1 = Array.create(storeHandle, arrayMetadata); + Array array2 = array1.updateAttributes(attrs -> attrs.set("key2", "val2")); + + Assertions.assertNotSame(array1, array2); + Assertions.assertEquals("val1", array1.metadata().attributes().get("key1")); + Assertions.assertNull(array1.metadata().attributes().get("key2")); + + Assertions.assertEquals("val1", array2.metadata().attributes().get("key1")); + Assertions.assertEquals("val2", array2.metadata().attributes().get("key2")); + + // Re-opening should show the updated attributes + Array array3 = Array.open(storeHandle); + Assertions.assertEquals("val2", array3.metadata().attributes().get("key2")); + } + @test public void testResizeArray() throws IOException, ZarrException { int[] testData = new int[10 * 10]; @@ -736,6 +761,34 @@ public class ZarrV3Test extends ZarrTest { Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType)); } + @test + public void testResizeArrayShrink() throws IOException, ZarrException { + int[] testData = new int[10 * 10]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testResizeArrayShrinkV3"); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT32) + .withChunkShape(5, 5) + .build(); + ucar.ma2.DataType ma2DataType = arrayMetadata.dataType.getMA2DataType(); + Array array = Array.create(storeHandle, arrayMetadata); + array.write(new long[]{0, 0}, ucar.ma2.Array.factory(ma2DataType, new int[]{10, 10}, testData)); + + array = array.resize(new long[]{5, 5}); + Assertions.assertArrayEquals(new int[]{5, 5}, array.read().getShape()); + + ucar.ma2.Array data = array.read(); + int[] expectedData = new int[5 * 5]; + for (int i = 0; i < 5; i++) { + for (int j = 0; j < 5; j++) { + expectedData[i * 5 + j] = testData[i * 10 + j]; + } + } + Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType)); + } + @test public void testGroupAttributes() throws IOException, ZarrException { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testGroupAttributesV3");
diff --git c/src/main/java/dev/zarr/zarrjava/core/Array.java i/src/main/java/dev/zarr/zarrjava/core/Array.java index c450cee..7e41e7c 100644 --- c/src/main/java/dev/zarr/zarrjava/core/Array.java +++ i/src/main/java/dev/zarr/zarrjava/core/Array.java @@ -21,6 +21,7 @@ import java.util.stream.Stream; public abstract class Array extends AbstractNode { protected CodecPipeline codecPipeline; + public static final boolean DEFAULT_PARALLELISM = true; protected Array(StoreHandle storeHandle) throws ZarrException { super(storeHandle); @@ -299,7 +300,7 @@ public abstract class Array extends AbstractNode { * @param array the data to write */ public void write(long[] offset, ucar.ma2.Array array) { - write(offset, array, false); + write(offset, array, DEFAULT_PARALLELISM); } /** @@ -334,7 +335,7 @@ public abstract class Array extends AbstractNode { */ @nonnull public ucar.ma2.Array read(final long[] offset, final long[] shape) throws ZarrException { - return read(offset, shape, false); + return read(offset, shape, DEFAULT_PARALLELISM); } /**
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
1 participant
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
This PR adds a USERGUIDE on how to get started with zarr-java