diff --git a/.vscode/settings.json b/.vscode/settings.json index 16667e414..6f3bd17c1 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -21,5 +21,6 @@ }, }, "java.debug.settings.onBuildFailureProceed": true, - "java.compile.nullAnalysis.mode": "disabled" + "java.compile.nullAnalysis.mode": "disabled", + "java.configuration.updateBuildConfiguration": "interactive" } diff --git a/agents/semantickernel-agents-core/pom.xml b/agents/semantickernel-agents-core/pom.xml index 9d40d75d4..cafee140a 100644 --- a/agents/semantickernel-agents-core/pom.xml +++ b/agents/semantickernel-agents-core/pom.xml @@ -1,5 +1,6 @@ - + 4.0.0 com.microsoft.semantic-kernel @@ -18,6 +19,18 @@ com.microsoft.semantic-kernel semantickernel-api + + com.microsoft.semantic-kernel + semantickernel-api-builders + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + \ No newline at end of file diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index a205a60bb..390c3bc11 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -34,6 +34,22 @@ com.microsoft.semantic-kernel semantickernel-api + + com.microsoft.semantic-kernel + semantickernel-api-builders + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-localization + com.fasterxml.jackson.core diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index aaed6aad6..4ce69b952 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -31,6 +31,14 @@ com.microsoft.semantic-kernel semantickernel-api + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + com.azure azure-core diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index 4a93b679b..04b31ca39 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -19,6 +19,30 @@ com.microsoft.semantic-kernel semantickernel-api + + com.microsoft.semantic-kernel + semantickernel-api-data + provided + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + provided + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + provided + + + com.microsoft.semantic-kernel + semantickernel-api-builders + + + com.microsoft.semantic-kernel + semantickernel-api-textembedding-services + + com.azure azure-ai-openai diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index 433e14a8a..94e577f1c 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -61,6 +61,21 @@ semantickernel-data-jdbc test + + com.microsoft.semantic-kernel + semantickernel-data-mysql + test + + + com.microsoft.semantic-kernel + semantickernel-data-hsqldb + test + + + com.microsoft.semantic-kernel + semantickernel-data-sqlite + test + com.microsoft.semantic-kernel semantickernel-data-redis diff --git a/data/semantickernel-data-azureaisearch/pom.xml b/data/semantickernel-data-azureaisearch/pom.xml index 74dc84bcd..038cc038c 100644 --- a/data/semantickernel-data-azureaisearch/pom.xml +++ b/data/semantickernel-data-azureaisearch/pom.xml @@ -21,6 +21,14 @@ com.microsoft.semantic-kernel semantickernel-api + + com.microsoft.semantic-kernel + semantickernel-api-data + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + com.azure diff --git a/data/semantickernel-data-hsqldb/pom.xml b/data/semantickernel-data-hsqldb/pom.xml new file mode 100644 index 000000000..42245185f --- /dev/null +++ b/data/semantickernel-data-hsqldb/pom.xml @@ -0,0 +1,55 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../../pom.xml + + + com.microsoft.semantic-kernel + semantickernel-data-hsqldb + Semantic Kernel HLSQLDB connector + Provides a HLSQLDB connector for the Semantic Kernel + + + + com.microsoft.semantic-kernel + semantickernel-api + + + com.microsoft.semantic-kernel + semantickernel-data-jdbc + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-data + + + com.microsoft.semantic-kernel + semantickernel-api-builders + + + + com.fasterxml.jackson.core + jackson-databind + compile + + + com.fasterxml.jackson.core + jackson-core + compile + + + com.github.spotbugs + spotbugs-annotations + + + \ No newline at end of file diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/hsqldb/HSQLDBVectorStoreQueryProvider.java b/data/semantickernel-data-hsqldb/src/main/java/com/microsoft/semantickernel/data/jdbc/hsqldb/HSQLDBVectorStoreQueryProvider.java similarity index 100% rename from data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/hsqldb/HSQLDBVectorStoreQueryProvider.java rename to data/semantickernel-data-hsqldb/src/main/java/com/microsoft/semantickernel/data/jdbc/hsqldb/HSQLDBVectorStoreQueryProvider.java diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 4cf608b5d..f1e62c6a4 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -15,9 +15,19 @@ com.microsoft.semantic-kernel - semantickernel-api + semantickernel-api-data + provided + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + provided + + + com.microsoft.semantic-kernel + semantickernel-api-builders + provided - org.slf4j slf4j-api @@ -63,5 +73,10 @@ sqlite-jdbc 3.47.0.0 + + com.oracle.database.jdbc + ojdbc11 + 23.7.0.25.01 + \ No newline at end of file diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java index ac21c61fc..2fcc2d5b7 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -106,7 +106,7 @@ public JDBCVectorStoreQueryProvider( @SuppressFBWarnings("EI_EXPOSE_REP2") @Nonnull DataSource dataSource, @Nonnull String collectionsTable, @Nonnull String prefixForCollectionTables, - @Nonnull HashMap, String> supportedKeyTypes, + @Nonnull Map, String> supportedKeyTypes, @Nonnull Map, String> supportedDataTypes, @Nonnull Map, String> supportedVectorTypes) { this.dataSource = dataSource; @@ -220,7 +220,7 @@ public Map, String> getSupportedVectorTypes() { @Override public void prepareVectorStore() { String createCollectionsTable = formatQuery( - "CREATE TABLE IF NOT EXISTS %s (collectionId VARCHAR(255) PRIMARY KEY);", + "CREATE TABLE IF NOT EXISTS %s (collectionId VARCHAR(255) PRIMARY KEY)", validateSQLidentifier(collectionsTable)); try (Connection connection = dataSource.getConnection(); @@ -701,6 +701,16 @@ public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { return String.format("%s LIKE ?", fieldName); } + @Override + public VectorStoreRecordMapper getVectorStoreRecordMapper( + Class recordClass, + VectorStoreRecordDefinition recordDefinition) { + return JDBCVectorStoreRecordMapper.builder() + .withRecordClass(recordClass) + .withVectorStoreRecordDefinition(recordDefinition) + .build(); + } + /** * The builder for {@link JDBCVectorStoreQueryProvider}. */ @@ -755,4 +765,5 @@ public JDBCVectorStoreQueryProvider build() { prefixForCollectionTables); } } + } diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java index 1c1cea5df..1d6b3e09b 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -2,9 +2,6 @@ package com.microsoft.semantickernel.data.jdbc; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; -import com.microsoft.semantickernel.data.jdbc.mysql.MySQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; @@ -37,10 +34,10 @@ public class JDBCVectorStoreRecordCollection implements SQLVectorStoreRecordCollection { private final String collectionName; - private final VectorStoreRecordDefinition recordDefinition; - private final VectorStoreRecordMapper vectorStoreRecordMapper; + protected final VectorStoreRecordDefinition recordDefinition; + protected final VectorStoreRecordMapper vectorStoreRecordMapper; private final JDBCVectorStoreRecordCollectionOptions options; - private final SQLVectorStoreQueryProvider queryProvider; + protected final SQLVectorStoreQueryProvider queryProvider; /** * Creates a new instance of the {@link JDBCVectorStoreRecordCollection}. @@ -73,25 +70,9 @@ public JDBCVectorStoreRecordCollection( // If mapper is not provided, set a default one if (options.getVectorStoreRecordMapper() == null) { - // Default mapper for PostgreSQL - if (this.queryProvider instanceof PostgreSQLVectorStoreQueryProvider) { - vectorStoreRecordMapper = PostgreSQLVectorStoreRecordMapper.builder() - .withRecordClass(options.getRecordClass()) - .withVectorStoreRecordDefinition(recordDefinition) - .build(); - // Default mapper for MySQL - } else if (this.queryProvider instanceof MySQLVectorStoreQueryProvider) { - vectorStoreRecordMapper = JDBCVectorStoreRecordMapper.builder() - .withRecordClass(options.getRecordClass()) - .withVectorStoreRecordDefinition(recordDefinition) - .build(); - // Default mapper for other databases - } else { - vectorStoreRecordMapper = JDBCVectorStoreRecordMapper.builder() - .withRecordClass(options.getRecordClass()) - .withVectorStoreRecordDefinition(recordDefinition) - .build(); - } + vectorStoreRecordMapper = options.getQueryProvider() + .getVectorStoreRecordMapper(options.getRecordClass(), + recordDefinition); } else { vectorStoreRecordMapper = options.getVectorStoreRecordMapper(); } @@ -214,7 +195,7 @@ protected String getKeyFromRecord(Record data) { Field keyField = data.getClass() .getDeclaredField(recordDefinition.getKeyField().getName()); keyField.setAccessible(true); - return (String) keyField.get(data); + return keyField.get(data).toString(); } catch (NoSuchFieldException | IllegalAccessException e) { throw new SKException("Failed to get key from record", e); } diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreQueryProvider.java index 18806c5d7..30a535fb0 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreQueryProvider.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreQueryProvider.java @@ -151,6 +151,18 @@ VectorSearchResults search(String collectionName, VectorStoreRecordDefinition recordDefinition, VectorStoreRecordMapper mapper); + /** + * Gets the record mapper for the given record class and definition. + * + * @param the record type + * @param recordClass the record class + * @param recordDefinition the record definition + * @return the record mapper that maps JDBC result sets to the given record. + */ + VectorStoreRecordMapper getVectorStoreRecordMapper( + final Class recordClass, + final VectorStoreRecordDefinition recordDefinition); + /** * The builder for the JDBC vector store query provider. */ diff --git a/data/semantickernel-data-mysql/pom.xml b/data/semantickernel-data-mysql/pom.xml new file mode 100644 index 000000000..899f1f2aa --- /dev/null +++ b/data/semantickernel-data-mysql/pom.xml @@ -0,0 +1,55 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../../pom.xml + + + com.microsoft.semantic-kernel + semantickernel-data-mysql + Semantic Kernel MySQL connector + Provides a MySQL connector for the Semantic Kernel + + + + com.microsoft.semantic-kernel + semantickernel-api + + + com.microsoft.semantic-kernel + semantickernel-data-jdbc + + + com.microsoft.semantic-kernel + semantickernel-api-data + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-builders + + + + com.fasterxml.jackson.core + jackson-databind + compile + + + com.fasterxml.jackson.core + jackson-core + compile + + + com.github.spotbugs + spotbugs-annotations + + + \ No newline at end of file diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/mysql/MySQLVectorStoreQueryProvider.java b/data/semantickernel-data-mysql/src/main/java/com/microsoft/semantickernel/data/jdbc/mysql/MySQLVectorStoreQueryProvider.java similarity index 100% rename from data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/mysql/MySQLVectorStoreQueryProvider.java rename to data/semantickernel-data-mysql/src/main/java/com/microsoft/semantickernel/data/jdbc/mysql/MySQLVectorStoreQueryProvider.java diff --git a/data/semantickernel-data-oracle/pom.xml b/data/semantickernel-data-oracle/pom.xml new file mode 100644 index 000000000..aa9538702 --- /dev/null +++ b/data/semantickernel-data-oracle/pom.xml @@ -0,0 +1,124 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../../pom.xml + + + semantickernel-data-oracle + Semantic Kernel Oracle connector + Provides a Oracle connector for the Semantic Kernel + + + 1.20.4 + + + + + + org.testcontainers + testcontainers-bom + ${testcontainers.version} + pom + import + + + + + + + com.microsoft.semantic-kernel + semantickernel-api-data + provided + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + provided + + + com.microsoft.semantic-kernel + semantickernel-api-builders + provided + + + com.microsoft.semantic-kernel + semantickernel-data-jdbc + + + com.fasterxml.jackson.core + jackson-databind + compile + + + com.fasterxml.jackson.core + jackson-core + compile + + + com.oracle.database.jdbc + ojdbc11 + 23.7.0.25.01 + + + com.oracle.database.jdbc + ojdbc-provider-jackson-oson + 1.0.4 + + + org.junit.jupiter + junit-jupiter + test + + + org.junit.jupiter + junit-jupiter-api + test + + + org.testcontainers + testcontainers + test + + + org.testcontainers + junit-jupiter + test + + + org.testcontainers + oracle-free + test + + + + + + org.codehaus.mojo + animal-sniffer-maven-plugin + + + android + test + + check + + + + + true + + + + com.diffplug.spotless + spotless-maven-plugin + + true + + + + + \ No newline at end of file diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java new file mode 100644 index 000000000..1d5db527f --- /dev/null +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java @@ -0,0 +1,93 @@ +/* + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. + */ +package com.microsoft.semantickernel.data.jdbc.oracle; + +/** + * Defines oracle database type constants for supported java types. + */ +public class OracleDataTypesMapping { + + /** + * Oracle database type used when strings are mapped to VARCHAR + */ + public static final String STRING_VARCHAR = "VARCHAR2(%s)"; + /** + * Oracle database type used when strings are mapped to CLOB + */ + public static final String STRING_CLOB = "CLOB"; + /** + * Oracle database type used to map booleans + */ + public static final String BOOLEAN = "BOOLEAN"; + /** + * Oracle database type used to map bytes + */ + public static final String BYTE = "NUMBER(3)"; + /** + * Oracle database type used to map byte arrays + */ + public static final String BYTE_ARRAY = "RAW(2000)"; + /** + * Oracle database type used to map shorts + */ + public static final String SHORT = "NUMBER(5)"; + /** + * Oracle database type used to map ints + */ + public static final String INTEGER = "NUMBER(10)"; + /** + * Oracle database type used to map longs + */ + public static final String LONG = "NUMBER(19)"; + /** + * Oracle database type used to map float + */ + public static final String FLOAT = "BINARY_FLOAT"; + /** + * Oracle database type used to map double + */ + public static final String DOUBLE = "BINARY_DOUBLE"; + /** + * Oracle database type used to map BigDecimal + */ + public static final String DECIMAL = "NUMBER"; + /** + * Oracle database type used to map offset date time + */ + public static final String OFFSET_DATE_TIME = "TIMESTAMP(9) WITH TIME ZONE"; + /** + * Oracle database type used to map UUID + */ + public static final String UUID = "VARCHAR2(36)"; + /** + * Oracle database type used to map lists + */ + public static final String JSON = "JSON"; + /** + * Oracle database type used to map vectors (the parameter is the dimension of the vector) + */ + public static final String VECTOR_FLOAT = "VECTOR(%s, FLOAT32)"; +} diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java new file mode 100644 index 000000000..bf4dd2952 --- /dev/null +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java @@ -0,0 +1,287 @@ +/* + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. + */ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.microsoft.semantickernel.data.jdbc.oracle.OracleVectorStoreQueryProvider.StringTypeMapping; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.exceptions.SKException; +import java.math.BigDecimal; +import java.time.OffsetDateTime; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Helper class for field operations. Handles mapping between field java types to DB types and + * generating SQL statement to create field indexes. + */ +class OracleVectorStoreFieldHelper { + + /** + * Object naming regular expression + */ + private static final String OBJECT_NAMING_REGEXP = "[a-zA-Z_][a-zA-Z0-9_]{1,128}"; + /** + * The logger + */ + private static final Logger LOGGER = Logger.getLogger(OracleVectorStoreFieldHelper.class.getName()); + + /** + * Maps supported key java classes to Oracle database types + */ + private static final HashMap, String> supportedKeyTypes = new HashMap(); + static { + supportedKeyTypes.put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, 255)); + } + + /** + * Maps supported vector java classes to Oracle database types + */ + private static final Map, String> supportedVectorTypes = new HashMap(); + static { + supportedVectorTypes.put(String.class, OracleDataTypesMapping.VECTOR_FLOAT); + supportedVectorTypes.put(List.class, OracleDataTypesMapping.VECTOR_FLOAT); + supportedVectorTypes.put(Collection.class, OracleDataTypesMapping.VECTOR_FLOAT); + supportedVectorTypes.put(float[].class, OracleDataTypesMapping.VECTOR_FLOAT); + supportedVectorTypes.put(Float[].class, OracleDataTypesMapping.VECTOR_FLOAT); + } + + /** + * Maps supported data java classes to Oracle database types + */ + private static final HashMap, String> supportedDataTypes = new HashMap(); + static { + supportedDataTypes.put(byte.class, OracleDataTypesMapping.BYTE); + supportedDataTypes.put(Byte.class, OracleDataTypesMapping.BYTE); + supportedDataTypes.put(short.class, OracleDataTypesMapping.SHORT); + supportedDataTypes.put(Short.class, OracleDataTypesMapping.SHORT); + supportedDataTypes.put(int.class, OracleDataTypesMapping.INTEGER); + supportedDataTypes.put(Integer.class, OracleDataTypesMapping.INTEGER); + supportedDataTypes.put(long.class, OracleDataTypesMapping.LONG); + supportedDataTypes.put(Long.class, OracleDataTypesMapping.LONG); + supportedDataTypes.put(Float.class, OracleDataTypesMapping.FLOAT); + supportedDataTypes.put(float.class, OracleDataTypesMapping.FLOAT); + supportedDataTypes.put(Double.class, OracleDataTypesMapping.DOUBLE); + supportedDataTypes.put(double.class, OracleDataTypesMapping.DOUBLE); + supportedDataTypes.put(BigDecimal.class, OracleDataTypesMapping.DECIMAL); + supportedDataTypes.put(Boolean.class, OracleDataTypesMapping.BOOLEAN); + supportedDataTypes.put(boolean.class, OracleDataTypesMapping.BOOLEAN); + supportedDataTypes.put(OffsetDateTime.class, OracleDataTypesMapping.OFFSET_DATE_TIME); + supportedDataTypes.put(UUID.class, OracleDataTypesMapping.UUID); + supportedDataTypes.put(byte[].class, OracleDataTypesMapping.BYTE_ARRAY); + supportedDataTypes.put(List.class, OracleDataTypesMapping.JSON); + } + + /** + * Suffix added to the effective column name to generate the index name for a vector column. + */ + public static final String VECTOR_INDEX_SUFFIX = "_VECTOR_INDEX"; + + /** + * Gets the mapping between the supported Java key types and the Oracle database type. + * + * @return the mapping between the supported Java key types and the Oracle database type. + */ + static Map, String> getSupportedKeyTypes() { + + return Collections.unmodifiableMap(supportedKeyTypes); + } + + /** + * Gets the mapping between the supported Java data types and the Oracle database type. + * + * @return the mapping between the supported Java data types and the Oracle database type. + */ + static Map, String> getSupportedDataTypes( + StringTypeMapping stringTypeMapping, int defaultVarCharLength) { + String stringType = stringTypeMapping.equals(StringTypeMapping.USE_VARCHAR) + ? String.format(OracleDataTypesMapping.STRING_VARCHAR, defaultVarCharLength) + : OracleDataTypesMapping.STRING_CLOB; + supportedDataTypes.put(String.class, stringType); + LOGGER.finest("Mapping String columns to " + stringType); + return Collections.unmodifiableMap(supportedDataTypes); + } + + /** + * Gets the mapping between the supported Java data types and the Oracle database type. + * + * @return the mapping between the supported Java data types and the Oracle database type. + */ + static Map, String> getSupportedVectorTypes() { + + return Collections.unmodifiableMap(supportedVectorTypes); + } + + /** + * Generates the statement to create the index according to the vector field definition. + * + * @return the CREATE VECTOR INDEX statement to create the index according to the vector + * field definition. + */ + static String getCreateVectorIndexStatement(VectorStoreRecordVectorField field, String collectionTableName) { + switch (field.getIndexKind()) { + case IVFFLAT: + return "CREATE VECTOR INDEX IF NOT EXISTS " + + validateObjectNaming(getIndexName(field.getEffectiveStorageName())) + + " ON " + + validateObjectNaming(collectionTableName) + + "( " + validateObjectNaming(field.getEffectiveStorageName()) + " ) " + + " ORGANIZATION NEIGHBOR PARTITIONS " + + " WITH DISTANCE COSINE " + + "PARAMETERS ( TYPE IVF )"; + case HNSW: + return "CREATE VECTOR INDEX IF NOT EXISTS " + + validateObjectNaming(getIndexName(field.getEffectiveStorageName())) + + " ON " + + validateObjectNaming(collectionTableName) + + "( " + validateObjectNaming(field.getEffectiveStorageName()) + " ) " + + "ORGANIZATION INMEMORY GRAPH " + + "WITH DISTANCE COSINE " + + "PARAMETERS (TYPE HNSW)"; + case UNDEFINED: + return null; + default: + LOGGER.warning("Unsupported index kind: " + field.getIndexKind()); + return null; + } + } + + /** + * Generates the statement to create the index according to the field definition. + * + * @return the CREATE INDEX statement to create the index according to the field definition. + */ + static String createIndexForDataField(String collectionTableName, VectorStoreRecordDataField dataField, Map, String> supportedDataTypes) { + if (supportedDataTypes.get(dataField.getFieldType()) == "JSON") { + String dataFieldIndex = "CREATE MULTIVALUE INDEX IF NOT EXISTS %s ON %s t (t.%s.%s)"; + return String.format(dataFieldIndex, + validateObjectNaming(collectionTableName + "_" + dataField.getEffectiveStorageName()), + validateObjectNaming(collectionTableName), + validateObjectNaming(dataField.getEffectiveStorageName()), + getFunctionForType(supportedDataTypes.get(dataField.getFieldSubType()))); + } else { + String dataFieldIndex = "CREATE INDEX IF NOT EXISTS %s ON %s (%s ASC)"; + return String.format(dataFieldIndex, + validateObjectNaming(collectionTableName + "_" + dataField.getEffectiveStorageName()), + validateObjectNaming(collectionTableName), + validateObjectNaming(dataField.getEffectiveStorageName()) + ); + } + } + + /** + * Returns vector columns names and types for CREATE TABLE statement + * @param fields list of vector record fields. + * @return comma separated list of columns and types for CREATE TABLE statement. + */ + static String getVectorColumnNamesAndTypes(List fields) { + List columns = fields.stream() + .map(field -> validateObjectNaming(field.getEffectiveStorageName()) + " " + + OracleVectorStoreFieldHelper.getTypeForVectorField(field) + ).collect(Collectors.toList()); + + return String.join(", ", columns); + } + + /** + * Returns key column names and type for key column for CREATE TABLE statement + * @param field the key field. + * @return column name and type of the key field for CREATE TABLE statement. + */ + static String getKeyColumnNameAndType(VectorStoreRecordKeyField field) { + return validateObjectNaming(field.getEffectiveStorageName()) + " " + supportedKeyTypes.get(field.getFieldType()); + } + + + /** + * Generates the index name given the field name. by suffixing "_VECTOR_INDEX" to the field name. + * @param effectiveStorageName the field name. + * @return the index name. + */ + static String getIndexName(String effectiveStorageName) { + return effectiveStorageName + VECTOR_INDEX_SUFFIX; + } + + /** + * Gets the type of the vector given the field definition. This method is not needed if only + * + * @param field the vector field definition. + * @return returns the type of vector for the given field type. + */ + private static String getTypeForVectorField(VectorStoreRecordVectorField field) { + String dimension = field.getDimensions() > 0 ? String.valueOf(field.getDimensions()) : "*"; + return String.format(supportedVectorTypes.get(field.getFieldType()), dimension); + } + + /** + * Gets the function that allows to return the function that converts the JSON value to the + * data type. + * @param jdbcType The JDBC type. + * @return the function that allows to return the function that converts the JSON value to the + * data type. + */ + private static String getFunctionForType(String jdbcType) { + switch (jdbcType) { + case OracleDataTypesMapping.BOOLEAN: + return "boolean()"; + case OracleDataTypesMapping.BYTE: + case OracleDataTypesMapping.SHORT: + case OracleDataTypesMapping.INTEGER: + case OracleDataTypesMapping.LONG: + case OracleDataTypesMapping.FLOAT: + case OracleDataTypesMapping.DOUBLE: + case OracleDataTypesMapping.DECIMAL: + return "numberOnly()"; + case OracleDataTypesMapping.OFFSET_DATE_TIME: + return "timestamp()"; + default: + return "string()"; + } + } + + + /** + * Validates an SQL identifier. + * + * @param identifier the identifier + * @return the identifier if it is valid + * @throws SKException if the identifier is invalid + */ + static String validateObjectNaming(String identifier) { + if (identifier.matches(OBJECT_NAMING_REGEXP)) { + return identifier; + } + throw new SKException("Invalid SQL identifier: " + identifier); + } + +} diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java new file mode 100644 index 000000000..e3fa157b9 --- /dev/null +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -0,0 +1,866 @@ +/* + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. + */ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; +import com.microsoft.semantickernel.data.filter.EqualToFilterClause; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import com.microsoft.semantickernel.exceptions.SKException; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.math.BigDecimal; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.locks.ReentrantLock; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.concurrent.GuardedBy; +import javax.sql.DataSource; +import oracle.jdbc.OraclePreparedStatement; +import oracle.jdbc.OracleStatement; +import oracle.jdbc.OracleTypes; +import oracle.jdbc.provider.oson.OsonFactory; +import oracle.sql.TIMESTAMPTZ; + +/** + * JDBC Vector Store for the Oracle Database + */ +public class OracleVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider { + + // This could be removed if super.collectionTable made protected + private final String collectionsTable; + + // This could be common to all query providers + private final ObjectMapper objectMapper; + + /** + * Lock used to ensure that only one thread can create a collection at a time. + */ + private static final ReentrantLock dbCreationLock = new ReentrantLock(); + + /** + * The logger + */ + private static final Logger LOGGER = Logger.getLogger(OracleVectorStoreQueryProvider.class.getName()); + + public enum StringTypeMapping { + /** + * Maps String to CLOB + */ + USE_CLOB, + /** + * Maps String to VARCHAR2(4000) + */ + USE_VARCHAR + } + + /** + * Create an instance of OracleVectorStoreQueryProvider. + * + * @param dataSource the datasource + * @param collectionsTable the collections table name + * @param prefixForCollectionTables the prefix for the collection table name + * @param defaultVarcharSize the size of VARCHAR columns + * @param stringTypeMapping the storage type of string columns (VARCHAR or CLOB) + * @param objectMapper the object mapper. + */ + private OracleVectorStoreQueryProvider( + @Nonnull DataSource dataSource, + @Nonnull String collectionsTable, + @Nonnull String prefixForCollectionTables, + int defaultVarcharSize, + @Nonnull StringTypeMapping stringTypeMapping, + ObjectMapper objectMapper) { + super( + dataSource, + collectionsTable, + prefixForCollectionTables, + OracleVectorStoreFieldHelper.getSupportedKeyTypes(), + OracleVectorStoreFieldHelper.getSupportedDataTypes(stringTypeMapping, defaultVarcharSize), + OracleVectorStoreFieldHelper.getSupportedVectorTypes()); + this.collectionsTable = collectionsTable; + this.objectMapper = objectMapper; + // The JavaTimeModule must be registered to handle OffsetDateTime. To make sure that it is + // registered enable the feature IGNORE_DUPLICATE_MODULE_REGISTRATIONS and register the + // module. + this.objectMapper.enable(MapperFeature.IGNORE_DUPLICATE_MODULE_REGISTRATIONS); + this.objectMapper.registerModule(new JavaTimeModule()); + } + + /** + *

+ * Creates a collection with the given name and record definition. + *

+ * A collection is represented as a table in an Oracle DB containing columns + * that match the record definition. The table name is the name of the collection + * prefixed by the provided collection prefix. If no prefix was provided the default + * prefix will be used. + *

+ * @param collectionName the name of the collection + * @param recordDefinition the record definition + */ + @Override + @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") + @GuardedBy("dbCreationLock") + public void createCollection(String collectionName, + VectorStoreRecordDefinition recordDefinition) { + + dbCreationLock.lock(); + try { + + List vectorFields = recordDefinition.getVectorFields(); + String createStorageTable = formatQuery("CREATE TABLE IF NOT EXISTS %s (" + + "%s PRIMARY KEY, " + + "%s, " + + "%s)", + getCollectionTableName(collectionName), + OracleVectorStoreFieldHelper.getKeyColumnNameAndType(recordDefinition.getKeyField()), + getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getDataFields()), + getSupportedDataTypes()), + OracleVectorStoreFieldHelper.getVectorColumnNamesAndTypes( + new ArrayList<>(vectorFields))); + + String insertCollectionQuery = this.getInsertCollectionQuery(collectionsTable); + + try (Connection connection = dataSource.getConnection()) { + // set auto commit of, either all statements should be executed or none + connection.setAutoCommit(false); + try (Statement statement = connection.createStatement()) { + // Create table + statement.addBatch(createStorageTable); + LOGGER.finest("Creating collection " + collectionName + + " using statement: " + createStorageTable); + + // Index filterable data columns + for (VectorStoreRecordDataField dataField : recordDefinition.getDataFields()) { + if (dataField.isFilterable()) { + String dataFieldIndex = OracleVectorStoreFieldHelper.createIndexForDataField( + getCollectionTableName(collectionName), dataField, supportedDataTypes); + statement.addBatch(dataFieldIndex); + LOGGER.finest("Creating index on column " + + dataField.getEffectiveStorageName() + " using the statement: " + + dataFieldIndex); + } + } + + // Create index for vectorFields + for (VectorStoreRecordVectorField vectorField : vectorFields) { + String createVectorIndex = OracleVectorStoreFieldHelper.getCreateVectorIndexStatement( + vectorField, getCollectionTableName(collectionName)); + if (createVectorIndex != null) { + statement.addBatch(createVectorIndex); + LOGGER.finest("Creating index on vector column " + + vectorField.getEffectiveStorageName() + " using the statement: " + + createVectorIndex); + + } + } + statement.executeBatch(); + + // Insert the collection to the store (collections table) using MERGE statement + try (PreparedStatement insert = connection.prepareStatement( + insertCollectionQuery)) { + insert.setString(1, collectionName); + insert.execute(); + LOGGER.finest("Inserting collection to store using statement: " + + insertCollectionQuery); + } + + connection.commit(); + } catch (SQLException e) { + connection.rollback(); + throw new SKException("Failed to create collection", e); + } + } catch (SQLException e) { + throw new SKException("Failed to create collection", e); + } + } finally { + dbCreationLock.unlock(); + } + } + + /** + *

+ * Inserts or updates record of a collection given the collection name, the records, the record + * definition and the upsert options. + *

+ * @Note At the moment {@link UpsertRecordOptions} is an empty class. No options are available. + *

+ * + * @param collectionName the collection name + * @param records the records to update or insert + * @param recordDefinition the record definition + * @param options the options + */ + @Override + @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") + public void upsertRecords(String collectionName, + List records, + VectorStoreRecordDefinition recordDefinition, + UpsertRecordOptions options) { + + final String NEW_VALUE = "new"; + final String EXISTING_VALUE = "existing"; + + // generate the comma separated list of new fields + // Ex.: new.field1, new.field2 ... new.fieldn + String insertNewFieldList = recordDefinition.getAllFields().stream() + .map(f -> NEW_VALUE + "." + f.getEffectiveStorageName()) + .collect(Collectors.joining(", ")); + + // generate the comma separated list of existing fields + // Ex.: existing.field1, existing.field2 ... existing.fieldn + String insertExistingFieldList = recordDefinition.getAllFields().stream() + .map(f -> EXISTING_VALUE + "." + f.getEffectiveStorageName()) + .collect(Collectors.joining(", ")); + + // generate the comma separated list for setting new values on fields + // Ex.: new.field1 = existing.field1, new.field2 = existing.field2 ... new.fieldn = existing.fieldn + String updateFieldList = recordDefinition.getAllFields().stream() + .filter(f -> f != recordDefinition.getKeyField()) + .map(f -> EXISTING_VALUE + "." + f.getEffectiveStorageName() + " = " + NEW_VALUE + "." + f.getEffectiveStorageName()) + .collect(Collectors.joining(", ")); + + // generate the comma separated list of placeholders "?" for each field + // Ex.: ? field1, ? field2 ... ? fieldn + String namedPlaceholders = recordDefinition.getAllFields().stream().map(f -> "? " + f.getEffectiveStorageName()) + .collect(Collectors.joining(", ")); + + // Generate the MERGE statement to perform the upsert. + String upsertStatement = formatQuery("MERGE INTO %s existing "+ + "USING (SELECT %s FROM DUAL) new ON (existing.%s = new.%s) " + + "WHEN MATCHED THEN UPDATE SET %s " + + "WHEN NOT MATCHED THEN INSERT (%s) VALUES (%s)", + getCollectionTableName(collectionName), + namedPlaceholders, + getKeyColumnName(recordDefinition.getKeyField()), + getKeyColumnName(recordDefinition.getKeyField()), + updateFieldList, + insertExistingFieldList, + insertNewFieldList); + + LOGGER.finest("Generated upsert statement: " + upsertStatement); + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(upsertStatement)) { + // Loop through records, set values and add values to batch + for (Object record : records) { + setUpsertStatementValues(statement, record, recordDefinition.getAllFields()); + statement.addBatch(); + } + + // Execute the upsert statement + statement.executeBatch(); + } catch (SQLException e) { + throw new SKException("Failed to upsert records", e); + } + } + + /** + * Generates the MERGE statement to add the given collection to the store. + * + * @param collectionsTable the name of the DB table containing all collections. + * @return a SQL statement that inserts a collection to the store if it does not exist. + */ + @Override + protected String getInsertCollectionQuery(String collectionsTable) { + return formatQuery( + "MERGE INTO %s existing "+ + "USING (SELECT ? AS collectionId FROM DUAL) new ON (existing.collectionId = new.collectionId) " + + "WHEN NOT MATCHED THEN INSERT (existing.collectionId) VALUES (new.collectionId)", + collectionsTable); + } + + /** + * The {@link OracleVectorStoreQueryProvider#upsertRecords(String, List, VectorStoreRecordDefinition, UpsertRecordOptions)} + * method adds a placeholder for each field. This method sets the value of each field on the + * MERGE statement with the value of the record. The placeholder and values are set in the order + * of the fields in the list. + * + * @param upsertStatement the MERGE statement + * @param record the record containing the values + * @param fields the list of fields. + */ + private void setUpsertStatementValues(PreparedStatement upsertStatement, Object record, + List fields) { + + // use the object mapper to convert the record to an equivalent tree mode JsonNode value, + // this allows to retrieve the values using the effective storage name of the fields and + // avoids the use of introspection. + JsonNode jsonNode = objectMapper.valueToTree(record); + + for (int i = 0; i < fields.size(); ++i) { + VectorStoreRecordField field = fields.get(i); + try { + + JsonNode valueNode = jsonNode.get(field.getEffectiveStorageName()); + + // Some field types require special treatment to convert the java type to the + // DB type + if (field instanceof VectorStoreRecordVectorField) { + // If the vector field is not set as a string convert to an array of floats + // and set the value + if (!field.getFieldType().equals(String.class)) { + if (valueNode != null && !valueNode.isNull() && valueNode.isArray()) { + final float[] values = new float[valueNode.size()]; + for (int j = 0; j < ((ArrayNode)valueNode).size(); j++) { + values[j] = ((ArrayNode)valueNode).get(j).floatValue(); + } + upsertStatement.setObject(i + 1, values, OracleTypes.VECTOR_FLOAT32); + } else { + upsertStatement.setNull(i + 1, OracleTypes.VECTOR_FLOAT32); + } + continue; + } + } else if (field instanceof VectorStoreRecordDataField) { + // Lists are stored as JSON objects, write the list using the JDBC OSON + // extensions. + if (field.getFieldType().equals(List.class)) { + JsonFactory osonFactory = new OsonFactory(); + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + try (JsonGenerator osonGen = osonFactory.createGenerator(out)) { + objectMapper.writeValue(osonGen, valueNode); + } + upsertStatement.setBytes(i + 1, out.toByteArray()); + } catch (IOException ioEx) { + throw new SKException("Failed to convert list to JSON value", ioEx); + } + continue; + } + // Convert UUID to string before setting the value. + if (field.getFieldType().equals(UUID.class)) { + upsertStatement.setObject(i + 1, valueNode.isNull() ? null : valueNode.asText()); + continue; + } + // Convert value node (its representations depends on Jackson JSON features) + // to OffsetDateTime before setting the value. + if (field.getFieldType().equals(OffsetDateTime.class)) { + if (valueNode == null || valueNode.isNull()) { + upsertStatement.setNull(i + 1, OracleTypes.TIMESTAMPTZ); + } else { + OffsetDateTime offsetDateTime = (OffsetDateTime) objectMapper.convertValue(valueNode, field.getFieldType()); + upsertStatement.setObject(i + 1, offsetDateTime); + } + continue; + } + } + + // For all other field type use setObject with the field value + upsertStatement.setObject(i + 1, + objectMapper.convertValue(valueNode,field.getFieldType())); + } catch (SQLException e) { + throw new SKException(e); + } + } + } + + /** + *

+ * Executes a vector search query, using the search options and returns the results. The results + * are mapped to the specified record type using the provided mapper. The query is executed + * against the specified collection. + *

+ * + *

+ * @param collectionName the collection name + * @param vector the vector to search with + * @param options the search options + * @param recordDefinition the record definition + * @param mapper the mapper, responsible for mapping the result set to the record + * type. + * @return the search results + * @param the record type + */ + @Override + @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") + public VectorSearchResults search(String collectionName, List vector, + VectorSearchOptions options, VectorStoreRecordDefinition recordDefinition, + VectorStoreRecordMapper mapper) { + + + if (vector != null && recordDefinition.getVectorFields().isEmpty()) { + throw new SKException("Record definition must contain at least one vector field" + + " to perform a vector search"); + } + + // Gets the search vector field and its distance function. If not vector field was provided, + // use the first one + VectorStoreRecordVectorField vectorField = null; + if (vector != null) { + vectorField = getVectorFieldByName(recordDefinition, options.getVectorFieldName()); + } + + + + // get list of fields that should be returned by the query + List fields = (options.isIncludeVectors()) + ? recordDefinition.getAllFields() + : recordDefinition.getNonVectorFields(); + + // get search filters and get the list of parameters for the filters + String filter = getFilter(options.getVectorSearchFilter(), recordDefinition); + List parameters = getFilterParameters(options.getVectorSearchFilter()); + + // generate SQL statement + String selectQuery = "SELECT " + + (vector == null ? "0 as distance, " : + formatQuery("VECTOR_DISTANCE(%s, ?, %s) distance, ", + OracleVectorStoreFieldHelper.validateObjectNaming(vectorField.getEffectiveStorageName()), + toOracleDistanceFunction(vectorField.getDistanceFunction()))) + + getQueryColumnsFromFields(fields) + + " FROM " + getCollectionTableName(collectionName) + + (filter != null && !filter.isEmpty() ? " WHERE " + filter : "") + + " ORDER BY distance" + + (options.getSkip() > 0 ? " OFFSET " + options.getSkip() + " ROWS" : "") + + (options.getTop() > 0 ? " FETCH " + (options.getSkip() > 0 ? "NEXT " : "FIRST ") + options.getTop() + " ROWS ONLY" : ""); + LOGGER.finest("Search using statement: " + selectQuery); + + // Execute the statement + List> records = new ArrayList<>(); + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(selectQuery)) { + // set parameters from filters + int parameterIndex = 1; + // if a vector was provided for similarity search set the value of the vector + if (vector != null) { + float[] arrayVector = new float[vector.size()]; + for (int i = 0; i < vector.size(); i++){ + arrayVector[i] = vector.get(i).floatValue(); + } + statement.setObject(parameterIndex++, arrayVector, OracleTypes.VECTOR_FLOAT32); + } + // set all parameters. + for (Object parameter : parameters) { + if (parameter != null) { + setSearchParameter(statement, parameterIndex++, parameter.getClass(), parameter); + } + } + + // Calls to defineColumnType reduce the number of network requests. When Oracle JDBC knows that it is + // fetching VECTOR, CLOB, and/or JSON columns, the first request it sends to the database can include a LOB + // prefetch size (VECTOR and JSON are value-based-lobs). If defineColumnType is not called, then JDBC needs + // to send an additional request with the LOB prefetch size, after the first request has the database + // respond with the column data types. To request all data, the prefetch size is Integer.MAX_VALUE. + OracleStatement oracleStatement = statement.unwrap(OracleStatement.class); + int columnIndex = 1; + // define distance column as double + defineDataColumnType(columnIndex++, oracleStatement, Double.class); + // define columns for returned fields + for (VectorStoreRecordField field : fields) { + if (!(field instanceof VectorStoreRecordVectorField)) + defineDataColumnType(columnIndex++, oracleStatement, field.getFieldType()); + else + oracleStatement.defineColumnType(columnIndex++, OracleTypes.VECTOR_FLOAT32, + Integer.MAX_VALUE); + } + oracleStatement.setLobPrefetchSize(Integer.MAX_VALUE); // Workaround for Oracle JDBC bug 37030121 + + // Execute the statement and get the results + try (ResultSet rs = statement.executeQuery()) { + GetRecordOptions getRecordOptions = new GetRecordOptions(options.isIncludeVectors()); + while (rs.next()) { + // Cosine distance function. 1 - cosine similarity. + double score = Math.abs(rs.getDouble("distance")); + if (vector != null && vectorField.getDistanceFunction() == DistanceFunction.COSINE_SIMILARITY) { + score = 1d - score; + } + // Use the mapper to convert to result set to records + records.add(new VectorSearchResult<>(mapper.mapStorageModelToRecord(rs, getRecordOptions), score)); + } + } + } catch (SQLException e) { + throw new SKException("Search failed", e); + } + + return new VectorSearchResults<>(records); + } + + private VectorStoreRecordVectorField getVectorFieldByName( + VectorStoreRecordDefinition recordDefinition, + String name) { + VectorStoreRecordField vectorField; + if (name != null) { + vectorField = recordDefinition.getField(name); + if (vectorField == null) { + throw new SKException("Vector field not found in record definition"); + } + if (!(vectorField instanceof VectorStoreRecordVectorField)) { + throw new SKException("Invalid type"); + } + } else { + if (recordDefinition.getVectorFields().isEmpty()) { + throw new SKException("Record definition should contain at least one vector field"); + } + vectorField = recordDefinition.getVectorFields().get(0); + } + return (VectorStoreRecordVectorField)vectorField; + } + + /** + * Sets the parameter value + * @param statement the statement + * @param index the parameter index + * @param type the parameter type + * @param value the value + */ + private void setSearchParameter(PreparedStatement statement, int index, Class type, Object value) { + + try { + // Use JSON string to set lists + if (List.class.equals(type)) { + statement.setObject(index, objectMapper.writeValueAsString(value)); + return; + } + // convert UUID to string + if (UUID.class.equals(type)) { + statement.setString(index, value.toString()); + return; + } + // convert OffsetDateType to TIMESTAMPTZ + if (OffsetDateTime.class.equals(type)) { + if (value == null) { + statement.setNull(index, OracleTypes.TIMESTAMPTZ); + } else { + OffsetDateTime offsetDateTime = (OffsetDateTime) value; + ((OraclePreparedStatement) statement).setTIMESTAMPTZ(index, + TIMESTAMPTZ.of(offsetDateTime)); + } + return; + } + // use setBigDecimal to set BigDecimal value + if (BigDecimal.class.equals(type)) { + if (value == null) { + statement.setNull(index, OracleTypes.DECIMAL); + } else { + BigDecimal bigDecimal = (BigDecimal) value; + ((OraclePreparedStatement) statement).setBigDecimal(index, + bigDecimal); + } + return; + } + + // for all other types set object with the given value + statement.setObject(index, value); + + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + + /** + * Defines the type that will be used to retrieve data from a given database table column. + * @param columnIndex the index of the column + * @param statement the statement + * @param fieldType the java field type + * @throws SQLException if an error occurs while defining the column type + */ + private void defineDataColumnType(int columnIndex, OracleStatement statement, Class fieldType) throws SQLException { + // switch between supported classes and define the column type on the statement + switch (supportedDataTypes.get(fieldType)) { + case OracleDataTypesMapping.STRING_CLOB: + statement.defineColumnType(columnIndex, OracleTypes.CLOB, Integer.MAX_VALUE); + break; + case OracleDataTypesMapping.BYTE: + statement.defineColumnType(columnIndex, OracleTypes.NUMBER); + break; + case OracleDataTypesMapping.SHORT: + statement.defineColumnType(columnIndex, OracleTypes.NUMBER); + break; + case OracleDataTypesMapping.INTEGER: + statement.defineColumnType(columnIndex, OracleTypes.INTEGER); + break; + case OracleDataTypesMapping.LONG: + statement.defineColumnType(columnIndex, OracleTypes.BIGINT); + break; + case OracleDataTypesMapping.FLOAT: + statement.defineColumnType(columnIndex, OracleTypes.BINARY_FLOAT); + break; + case OracleDataTypesMapping.DOUBLE: + statement.defineColumnType(columnIndex, OracleTypes.BINARY_DOUBLE); + break; + case OracleDataTypesMapping.DECIMAL: + statement.defineColumnType(columnIndex, OracleTypes.BINARY_DOUBLE); + break; + case OracleDataTypesMapping.BOOLEAN: + statement.defineColumnType(columnIndex, OracleTypes.BOOLEAN); + break; + case OracleDataTypesMapping.OFFSET_DATE_TIME: + statement.defineColumnType(columnIndex, OracleTypes.TIMESTAMPTZ); + break; + case OracleDataTypesMapping.JSON: + statement.defineColumnType(columnIndex, OracleTypes.JSON, Integer.MAX_VALUE); + break; + case OracleDataTypesMapping.BYTE_ARRAY: + statement.defineColumnType(columnIndex, OracleTypes.RAW); + default: + statement.defineColumnType(columnIndex, OracleTypes.VARCHAR); + } + } + + /** + * Converts a {@link DistanceFunction} to the equivalent Oracle distance function. + * @param distanceFunction the distance function + * @return the Oracle distance function + */ + private String toOracleDistanceFunction(DistanceFunction distanceFunction) { + switch (distanceFunction) { + case DOT_PRODUCT: + return "DOT"; + case COSINE_SIMILARITY: + case COSINE_DISTANCE: + return "COSINE"; + case EUCLIDEAN_DISTANCE: + return "EUCLIDEAN"; + default: + return "COSINE"; + } + } + + /** + * Gets the filter parameters for the given vector search filter to associate with the filter + * string generated by the getFilter method. + * + * @param filter The filter to get the filter parameters for. + * @return The filter parameters. + */ + @Override + public List getFilterParameters(VectorSearchFilter filter) { + // TODO: this method should be protected, not public + if (filter == null + || filter.getFilterClauses().isEmpty()) { + return Collections.emptyList(); + } + + return filter.getFilterClauses().stream().map(filterClause -> { + if (filterClause instanceof EqualToFilterClause) { + EqualToFilterClause equalToFilterClause = (EqualToFilterClause) filterClause; + return equalToFilterClause.getValue(); + } else if (filterClause instanceof AnyTagEqualToFilterClause) { + AnyTagEqualToFilterClause anyTagEqualToFilterClause = (AnyTagEqualToFilterClause) filterClause; + return anyTagEqualToFilterClause.getValue(); + } else { + throw new SKException("Unsupported filter clause type '" + + filterClause.getClass().getSimpleName() + "'."); + } + }).collect(Collectors.toList()); + } + + /** + * Gets the filter clause for an equal to filter + * @param filterClause The equal to filter clause to get the filter string for. + * @return the filter clause + */ + @Override + public String getEqualToFilter(EqualToFilterClause filterClause) { + String fieldName = JDBCVectorStoreQueryProvider + .validateSQLidentifier(filterClause.getFieldName()); + Object value = filterClause.getValue(); + + if (value == null) { + return String.format("%s is NULL", fieldName); + } else { + return String.format("%s = ?", fieldName); + } + } + + /** + * Gets the filter clause for an any tag equal to filter + * @param filterClause The any tag equal to filter clause to get the filter string for. + * @return the filter clause + */ + @Override + public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { + String fieldName = JDBCVectorStoreQueryProvider + .validateSQLidentifier(filterClause.getFieldName()); + + return String.format("JSON_EXISTS(%s, '$[*]?(@ == $v_%s)' PASSING ? AS \"v_%s\")", + fieldName, fieldName, fieldName); + } + + /** + * Gets the mapper used to map a ResultSet to records + * @param recordClass the record class + * @param vectorStoreRecordDefinition the record definition + * @return the vector store record mapper + * @param the type of the records + */ + @Override + public VectorStoreRecordMapper getVectorStoreRecordMapper( + Class recordClass, + VectorStoreRecordDefinition vectorStoreRecordDefinition) { + return OracleVectorStoreRecordMapper.builder() + .withRecordClass(recordClass) + .withVectorStoreRecordDefinition(vectorStoreRecordDefinition) + .withSupportedDataTypesMapping(getSupportedDataTypes()) + .build(); + } + + /** + * Gets a builder that allows to build an OracleVectorStoreQueryProvider + * @return the builder + */ + public static Builder builder() { + return new Builder(); + } + + /** + * OracleVectorStoreQueryProvider builder. + */ + public static class Builder + extends JDBCVectorStoreQueryProvider.Builder { + + /** + * The data source + */ + private DataSource dataSource; + + /** + * The collections table + */ + private String collectionsTable = DEFAULT_COLLECTIONS_TABLE; + + /** + * The prefix for collection table names + */ + private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; + + /** + * The object mapper + */ + private ObjectMapper objectMapper = new ObjectMapper(); + + /** + * The string type mapping choice + */ + private StringTypeMapping stringTypeMapping = StringTypeMapping.USE_VARCHAR; + + /** + * The size of varchar columns + */ + private int defaultVarcharSize = 2000; + + + @SuppressFBWarnings("EI_EXPOSE_REP2") + public Builder withDataSource(DataSource dataSource) { + this.dataSource = dataSource; + return this; + } + + /** + * Sets the collections table name. + * @param collectionsTable the collections table name + * @return the builder + */ + public Builder withCollectionsTable(String collectionsTable) { + this.collectionsTable = validateSQLidentifier(collectionsTable); + return this; + } + + /** + * Sets the prefix for collection tables. + * @param prefixForCollectionTables the prefix for collection tables + * @return the builder + */ + public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { + this.prefixForCollectionTables = validateSQLidentifier(prefixForCollectionTables); + return this; + } + + /** + * Sets the object mapper used to map records to and from results + * @param objectMapper the object mapper + * @return the builder + */ + @SuppressFBWarnings("EI_EXPOSE_REP2") + public Builder withObjectMapper( + ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + return this; + } + + /** + * Sets the desired String type mapping. + * @param stringTypeMapping the desired String type mapping. The default value is + * {@link StringTypeMapping#USE_VARCHAR} + * @return the builder + */ + public Builder withStringTypeMapping (StringTypeMapping stringTypeMapping) { + this.stringTypeMapping = stringTypeMapping; + return this; + } + + /** + * Sets the default size of the VARHCHAR fields. + * @param defaultVarcharSize the default size of the VARHCHAR fields. By default, the size + * is 2000. + * @return then builder + */ + public Builder withDefaultVarcharSize (int defaultVarcharSize) { + this.defaultVarcharSize = defaultVarcharSize; + return this; + } + + /** + * Builds and Oracle vector store query provider. + * @return the query provider + */ + @Override + public OracleVectorStoreQueryProvider build() { + return new OracleVectorStoreQueryProvider(dataSource, collectionsTable, + prefixForCollectionTables, defaultVarcharSize, stringTypeMapping, objectMapper); + } + } +} + diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java new file mode 100644 index 000000000..c75eb6631 --- /dev/null +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java @@ -0,0 +1,264 @@ +/* + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. + */ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.microsoft.semantickernel.builders.SemanticKernelBuilder; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.exceptions.SKException; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import oracle.jdbc.provider.oson.OsonModule; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Map; +import java.util.HashMap; +import java.util.UUID; +import java.util.function.BiFunction; + +/** + * Maps a Oracle result set to a record. + * + * @param the record type + */ +public class OracleVectorStoreRecordMapper + extends VectorStoreRecordMapper { + + /** + * Constructs a new instance of the VectorStoreRecordMapper. + * + * @param storageModelToRecordMapper the function to convert a storage model to a record + */ + protected OracleVectorStoreRecordMapper( + BiFunction storageModelToRecordMapper) { + super(null, storageModelToRecordMapper); + } + + /** + * Creates a new builder. + * + * @param the record type + * @return the builder + */ + public static Builder builder() { + return new Builder<>(); + } + + /** + * Operation not supported. + */ + @Override + public ResultSet mapRecordToStorageModel(Record record) { + throw new UnsupportedOperationException("Not implemented"); + } + + /** + * Builder for {@link OracleVectorStoreRecordMapper}. + * + * @param the record type + */ + public static class Builder + implements SemanticKernelBuilder> { + private Class recordClass; + private VectorStoreRecordDefinition vectorStoreRecordDefinition; + private Map, String> supportedDataTypesMapping; + private ObjectMapper objectMapper = new ObjectMapper(); + + /** + * Sets the record class. + * + * @param recordClass the record class + * @return the builder + */ + public Builder withRecordClass(Class recordClass) { + this.recordClass = recordClass; + return this; + } + + /** + * Sets the vector store record definition. + * + * @param vectorStoreRecordDefinition the vector store record definition + * @return the builder + */ + public Builder withVectorStoreRecordDefinition( + VectorStoreRecordDefinition vectorStoreRecordDefinition) { + this.vectorStoreRecordDefinition = vectorStoreRecordDefinition; + return this; + } + + /** + * Sets the object mapper. + * + * @param objectMapper the object mapper + * @return the builder + */ + @SuppressFBWarnings("EI_EXPOSE_REP2") + public Builder withObjectMapper(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + return this; + } + + /** + * Sets the Map of supported data types and their database representation + * + * @param supportedDataTypesMapping the Map of supported data types and their + * database representation + * @return the builder + */ + public Builder withSupportedDataTypesMapping( + Map, String> supportedDataTypesMapping) { + this.supportedDataTypesMapping = new HashMap<>(supportedDataTypesMapping); + return this; + } + + /** + * Builds the {@link OracleVectorStoreRecordMapper}. + * + * @return the {@link OracleVectorStoreRecordMapper} + */ + public OracleVectorStoreRecordMapper build() { + if (recordClass == null) { + throw new SKException("recordClass is required"); + } + if (vectorStoreRecordDefinition == null) { + throw new SKException("vectorStoreRecordDefinition is required"); + } + + return new OracleVectorStoreRecordMapper<>( + (resultSet, options) -> { + return mapResultSetToRecord(resultSet, options); + }); + } + + private Record mapResultSetToRecord(ResultSet resultSet, GetRecordOptions options) { + try { + objectMapper.registerModule(new OsonModule()); + // Create an ObjectNode to hold the values + ObjectNode objectNode = objectMapper.createObjectNode(); + + // Read non vector fields + for (VectorStoreRecordField field : vectorStoreRecordDefinition.getNonVectorFields()) { + Class fieldType = field.getFieldType(); + + Object value; + switch (supportedDataTypesMapping.get(fieldType)) { + case OracleDataTypesMapping.STRING_CLOB: + value = resultSet.getString(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.BYTE: + value = resultSet.getByte(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.SHORT: + value = resultSet.getShort(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.INTEGER: + value = resultSet.getInt(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.LONG: + value = resultSet.getLong(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.FLOAT: + value = resultSet.getFloat(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.DOUBLE: + value = resultSet.getDouble(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.DECIMAL: + value = resultSet.getBigDecimal(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.BOOLEAN: + value = resultSet.getBoolean(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.OFFSET_DATE_TIME: + value = resultSet.getObject(field.getEffectiveStorageName(), fieldType); + break; + case OracleDataTypesMapping.BYTE_ARRAY: + value = resultSet.getBytes(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.UUID: + String uuidValue = resultSet.getString(field.getEffectiveStorageName()); + value = uuidValue == null ? null : UUID.fromString(uuidValue); + break; + case OracleDataTypesMapping.JSON: + value = resultSet.getObject(field.getEffectiveStorageName(), fieldType); + break; + default: + value = resultSet.getString(field.getEffectiveStorageName()); + } + // Result set getter method sometimes returns a default value when NULL, + // set value to null in that case. + if (resultSet.wasNull()) { + value = null; + } + + JsonNode genericNode = objectMapper.valueToTree(value); + + objectNode.set(field.getEffectiveStorageName(), genericNode); + } + if (options != null && options.isIncludeVectors()) { + for (VectorStoreRecordVectorField field : vectorStoreRecordDefinition.getVectorFields()) { + + // String vector + if (field.getFieldType().equals(String.class)) { + float[] arr = resultSet.getObject(field.getEffectiveStorageName(), float[].class); + String str = (arr == null) + ? null + : objectMapper.writeValueAsString(arr); + objectNode.put(field.getEffectiveStorageName(), str); + continue; + } + + Object value = resultSet.getObject(field.getEffectiveStorageName(), float[].class); + JsonNode genericNode = objectMapper.valueToTree(value); + objectNode.set(field.getEffectiveStorageName(), genericNode); + } + } else { + for (VectorStoreRecordVectorField field : vectorStoreRecordDefinition.getVectorFields()) { + JsonNode genericNode = objectMapper.valueToTree(null); + objectNode.set(field.getEffectiveStorageName(), genericNode); + } + } + + // Deserialize the object node to the record class + return objectMapper.convertValue(objectNode, recordClass); + } catch (SQLException e) { + throw new SKException( + "Failure to serialize object, by default the JDBC connector uses Jackson, ensure your model object can be serialized by Jackson, i.e the class is visible, has getters, constructor, annotations etc.", + e); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + } + +} diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllBoxedTypes.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllBoxedTypes.java new file mode 100644 index 000000000..14b96c142 --- /dev/null +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllBoxedTypes.java @@ -0,0 +1,162 @@ +/* + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. + */ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; +import java.math.BigDecimal; +import java.time.OffsetDateTime; +import java.util.List; +import java.util.UUID; + +public class ClassWithAllBoxedTypes { + + @VectorStoreRecordKey + private final String id; + + @VectorStoreRecordData(isFilterable = true) + private final Boolean booleanValue; + + @VectorStoreRecordData(isFilterable = true) + private final Byte byteValue; + + @VectorStoreRecordData(isFilterable = true) + private final Short shortValue; + + @VectorStoreRecordData(isFilterable = true) + private final Integer integerValue; + + @VectorStoreRecordData(isFilterable = true) + private final Long longValue; + + @VectorStoreRecordData(isFilterable = true) + private final Float floatValue; + + @VectorStoreRecordData(isFilterable = true) + private final Double doubleValue; + + @VectorStoreRecordData(isFilterable = true) + private final BigDecimal decimalValue; + + @VectorStoreRecordData(isFilterable = true) + private final OffsetDateTime offsetDateTimeValue; + + @VectorStoreRecordData(isFilterable = true) + private final UUID uuidValue; + + @VectorStoreRecordData(isFilterable = true) + private final byte[] byteArrayValue; + + @VectorStoreRecordData(isFilterable = true) + private final List listOfFloatValue; + + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.IVFFLAT) + private final Float[] vectorValue; + + + public ClassWithAllBoxedTypes() { + this(null, false, Byte.MIN_VALUE,Short.MIN_VALUE, 0, 0l, 0f, 0d, null, null, null, null, null, null); + }; + public ClassWithAllBoxedTypes(String id, Boolean booleanValue, Byte byteValue, + Short shortValue, Integer integerValue, Long longValue, Float floatValue, Double doubleValue, + BigDecimal decimalValue, OffsetDateTime offsetDateTimeValue, UUID uuidValue, + byte[] byteArrayValue, List listOfFloatValue, Float[] vectorValue) { + this.id = id; + this.booleanValue = booleanValue; + this.byteValue = byteValue; + this.shortValue = shortValue; + this.integerValue = integerValue; + this.longValue = longValue; + this.floatValue = floatValue; + this.doubleValue = doubleValue; + this.decimalValue = decimalValue; + this.offsetDateTimeValue = offsetDateTimeValue; + this.uuidValue = uuidValue; + this.byteArrayValue = byteArrayValue; + this.listOfFloatValue = listOfFloatValue; + this.vectorValue = vectorValue; + } + + public String getId() { + return id; + } + + public Boolean getBooleanValue() { + return booleanValue; + } + + public Byte getByteValue() { + return byteValue; + } + + public Short getShortValue() { + return shortValue; + } + + public Integer getIntegerValue() { + return integerValue; + } + + public Long getLongValue() { + return longValue; + } + + public Float getFloatValue() { + return floatValue; + } + + public Double getDoubleValue() { + return doubleValue; + } + + + public BigDecimal getDecimalValue() { + return decimalValue; + } + + public OffsetDateTime getOffsetDateTimeValue() { + return offsetDateTimeValue; + } + + public UUID getUuidValue() { + return uuidValue; + } + + public byte[] getByteArrayValue() { + return byteArrayValue; + } + + public List getListOfFloatValue() { + return listOfFloatValue; + } + + public Float[] getVectorValue() { + return vectorValue; + } +} diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllPrimitiveTypes.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllPrimitiveTypes.java new file mode 100644 index 000000000..6a8d76d07 --- /dev/null +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllPrimitiveTypes.java @@ -0,0 +1,163 @@ +/* + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. + */ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; + +import java.math.BigDecimal; +import java.time.OffsetDateTime; +import java.util.List; +import java.util.UUID; + +public class ClassWithAllPrimitiveTypes { + + @VectorStoreRecordKey + private final String id; + + @VectorStoreRecordData(isFilterable = true) + private final Boolean booleanValue; + + @VectorStoreRecordData(isFilterable = true) + private final byte byteValue; + + @VectorStoreRecordData(isFilterable = true) + private final short shortValue; + + @VectorStoreRecordData(isFilterable = true) + private final int integerValue; + + @VectorStoreRecordData(isFilterable = true) + private final long longValue; + + @VectorStoreRecordData(isFilterable = true) + private final float floatValue; + + @VectorStoreRecordData(isFilterable = true) + private final double doubleValue; + + @VectorStoreRecordData(isFilterable = true) + private final BigDecimal decimalValue; + + @VectorStoreRecordData(isFilterable = true) + private final OffsetDateTime offsetDateTimeValue; + + @VectorStoreRecordData(isFilterable = true) + private final UUID uuidValue; + + @VectorStoreRecordData(isFilterable = true) + private final byte[] byteArrayValue; + + @VectorStoreRecordData(isFilterable = true) + private final List listOfFloatValue; + + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.IVFFLAT) + private final float[] vectorValue; + + + public ClassWithAllPrimitiveTypes() { + this(null, false, Byte.MIN_VALUE,Short.MIN_VALUE, 0, 0l, 0f, 0d, null, null, null, null, null, null); + }; + public ClassWithAllPrimitiveTypes(String id, boolean booleanValue, byte byteValue, + short shortValue, int integerValue, long longValue, float floatValue, double doubleValue, + BigDecimal decimalValue, OffsetDateTime offsetDateTimeValue, UUID uuidValue, + byte[] byteArrayValue, List listOfFloatValue, float[] vectorValue) { + this.id = id; + this.booleanValue = booleanValue; + this.byteValue = byteValue; + this.shortValue = shortValue; + this.integerValue = integerValue; + this.longValue = longValue; + this.floatValue = floatValue; + this.doubleValue = doubleValue; + this.decimalValue = decimalValue; + this.offsetDateTimeValue = offsetDateTimeValue; + this.uuidValue = uuidValue; + this.byteArrayValue = byteArrayValue; + this.listOfFloatValue = listOfFloatValue; + this.vectorValue = vectorValue; + } + + public String getId() { + return id; + } + + public boolean getBooleanValue() { + return booleanValue; + } + + public byte getByteValue() { + return byteValue; + } + + public short getShortValue() { + return shortValue; + } + + public int getIntegerValue() { + return integerValue; + } + + public long getLongValue() { + return longValue; + } + + public float getFloatValue() { + return floatValue; + } + + public double getDoubleValue() { + return doubleValue; + } + + + public BigDecimal getDecimalValue() { + return decimalValue; + } + + public OffsetDateTime getOffsetDateTimeValue() { + return offsetDateTimeValue; + } + + public UUID getUuidValue() { + return uuidValue; + } + + public byte[] getByteArrayValue() { + return byteArrayValue; + } + + public List getListOfFloatValue() { + return listOfFloatValue; + } + + public float[] getVectorValue() { + return vectorValue; + } +} diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java new file mode 100644 index 000000000..75190debf --- /dev/null +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java @@ -0,0 +1,91 @@ +/* + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. + */ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo.As; + +import java.math.BigDecimal; +import java.time.OffsetDateTime; +import java.util.List; +import java.util.UUID; + +public class ClassWithAnnotatedTypes { + + private final String id; + + @JsonProperty("value_type") + private final String valueType; + + @JsonProperty("value_field") + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = As.EXTERNAL_PROPERTY , property = "value_type") + @JsonSubTypes({ + @JsonSubTypes.Type(value = String.class, name="string"), + @JsonSubTypes.Type(value = Boolean.class, name="boolean"), + @JsonSubTypes.Type(value = Byte.class, name="byte"), + @JsonSubTypes.Type(value = Short.class, name="short"), + @JsonSubTypes.Type(value = Integer.class, name="integer"), + @JsonSubTypes.Type(value = Long.class, name="long"), + @JsonSubTypes.Type(value = Float.class, name="float"), + @JsonSubTypes.Type(value = Double.class, name="double"), + @JsonSubTypes.Type(value = BigDecimal.class, name="decimal"), + @JsonSubTypes.Type(value = OffsetDateTime.class, name="timestamp"), + @JsonSubTypes.Type(value = UUID.class, name="uuid"), + @JsonSubTypes.Type(value = byte[].class, name="byte_array"), + @JsonSubTypes.Type(value = List.class, name="json") + }) + private Object value; + + private final Float[] vectorValue; + + + public ClassWithAnnotatedTypes() { + this(null, null, null, null); + }; + public ClassWithAnnotatedTypes(String id, String valueType, Object value, Float[] vectorValue) { + this.id = id; + this.valueType = valueType; + this.value = value; + this.vectorValue = vectorValue; + } + + public String getId() { + return id; + } + + public String getValueType() { return valueType; } + + public Object getValue() { + return value; + } + + public Float[] getVectorValue() { + return vectorValue; + } +} diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java new file mode 100644 index 000000000..0f93ff7f5 --- /dev/null +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java @@ -0,0 +1,125 @@ + +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; + +import java.util.List; + +import static com.fasterxml.jackson.annotation.JsonCreator.Mode.DELEGATING; +import static com.fasterxml.jackson.annotation.JsonCreator.Mode.PROPERTIES; + +public class Hotel { + @VectorStoreRecordKey + private final String id; + + @VectorStoreRecordData(isFilterable = true) + private final String name; + + @VectorStoreRecordData + private final int code; + + @VectorStoreRecordData + private final double price; + + @VectorStoreRecordData(isFilterable = true) + private final List tags; + + @JsonProperty("summary") + @VectorStoreRecordData( isFilterable = true, isFullTextSearchable = true ) + private final String description; + + @JsonProperty("summaryEmbedding1") + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE, indexKind = IndexKind.IVFFLAT) + private final List euclidean; + + @JsonProperty("summaryEmbedding2") + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.HNSW) + private final float[] cosineDistance; + + @JsonProperty("summaryEmbedding3") + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_SIMILARITY, indexKind = IndexKind.IVFFLAT) + private final float[] cosineSimilarity; + + @JsonProperty("summaryEmbedding4") + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.DOT_PRODUCT, indexKind = IndexKind.IVFFLAT) + private final Float[] dotProduct; + @VectorStoreRecordData + private double rating; + + @JsonCreator(mode = DELEGATING) + public Hotel() { + this(null, null, 0, 0d, null, null, null, null, null, null, 0.0); + } + + @JsonCreator(mode = PROPERTIES) + protected Hotel( + @JsonProperty("id") String id, + @JsonProperty("name") String name, + @JsonProperty("code") int code, + @JsonProperty("price") double price, + @JsonProperty("tags") List tags, + @JsonProperty("summary") String description, + @JsonProperty("summaryEmbedding1") List euclidean, + @JsonProperty("summaryEmbedding2") float[] cosineDistance, + @JsonProperty("summaryEmbedding3") float[] cosineSimilarity, + @JsonProperty("summaryEmbedding4") Float[] dotProduct, + @JsonProperty("rating") double rating) { + this.id = id; + this.name = name; + this.code = code; + this.price = price; + this.tags = tags; + this.description = description; + this.euclidean = euclidean; + this.cosineDistance = cosineDistance; + this.cosineSimilarity = cosineSimilarity; + this.dotProduct = dotProduct; + this.rating = rating; + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public int getCode() { + return code; + } + + public double getPrice() { return price; } + + public List getTags() { return tags; } + + public String getDescription() { + return description; + } + + public List getEuclidean() { + return euclidean; + } + + public float[] getCosineDistance() { + return cosineDistance; + } + + public Float[] getDotProduct() { + return dotProduct; + } + + public double getRating() { + return rating; + } + + public void setRating(double rating) { + this.rating = rating; + } +} diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleCommonVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleCommonVectorStoreRecordCollectionTest.java new file mode 100644 index 000000000..ce260c77e --- /dev/null +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleCommonVectorStoreRecordCollectionTest.java @@ -0,0 +1,91 @@ +/* + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. + */ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import oracle.jdbc.OracleConnection; +import oracle.jdbc.datasource.impl.OracleDataSource; +import org.testcontainers.oracle.OracleContainer; +import org.testcontainers.utility.MountableFile; +import java.sql.SQLException; +import java.time.Duration; + +public class OracleCommonVectorStoreRecordCollectionTest { + + protected static final String ORACLE_IMAGE_NAME = "gvenzl/oracle-free:23.7-slim-faststart"; + protected static final OracleDataSource DATA_SOURCE; + protected static final OracleDataSource SYSDBA_DATA_SOURCE; + + static { + + try { + DATA_SOURCE = new oracle.jdbc.datasource.impl.OracleDataSource(); + SYSDBA_DATA_SOURCE = new oracle.jdbc.datasource.impl.OracleDataSource(); + String urlFromEnv = System.getenv("ORACLE_JDBC_URL"); + + if (urlFromEnv == null) { + // The Ryuk component is relied upon to stop this container. + OracleContainer oracleContainer = new OracleContainer(ORACLE_IMAGE_NAME) + .withCopyFileToContainer(MountableFile.forClasspathResource("/initialize.sql"), + "/container-entrypoint-initdb.d/initialize.sql") + .withStartupTimeout(Duration.ofSeconds(600)) + .withConnectTimeoutSeconds(600) + .withDatabaseName("pdb1") + .withUsername("testuser") + .withPassword("testpwd"); + oracleContainer.start(); + + initDataSource( + DATA_SOURCE, + oracleContainer.getJdbcUrl(), + oracleContainer.getUsername(), + oracleContainer.getPassword()); + initDataSource(SYSDBA_DATA_SOURCE, oracleContainer.getJdbcUrl(), "sys", oracleContainer.getPassword()); + } else { + initDataSource( + DATA_SOURCE, + urlFromEnv, + System.getenv("ORACLE_JDBC_USER"), + System.getenv("ORACLE_JDBC_PASSWORD")); + initDataSource( + SYSDBA_DATA_SOURCE, + urlFromEnv, + System.getenv("ORACLE_JDBC_USER"), + System.getenv("ORACLE_JDBC_PASSWORD")); + } + SYSDBA_DATA_SOURCE.setConnectionProperty(OracleConnection.CONNECTION_PROPERTY_INTERNAL_LOGON, "SYSDBA"); + + } catch (SQLException sqlException) { + throw new AssertionError(sqlException); + } + } + + static void initDataSource(OracleDataSource dataSource, String url, String username, String password) { + dataSource.setURL(url); + dataSource.setUser(username); + dataSource.setPassword(password); + } + +} diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java new file mode 100644 index 000000000..e0332950b --- /dev/null +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java @@ -0,0 +1,163 @@ +/* + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. + */ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.*; + +public class OracleVectorStoreAnnotatedTypeTest extends OracleCommonVectorStoreRecordCollectionTest { + + @ParameterizedTest + @MethodSource("supportedDataTypes") + void testDataTypes(String dataFieldName, Class dataFieldType, Object dataFieldValue, Class fieldSubType) { + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(String.class) + .build(); + + VectorStoreRecordDataField dataField; + if (fieldSubType != null) { + dataField = VectorStoreRecordDataField.builder() + .withName("value") + .withStorageName("value_field") + .withFieldType(dataFieldType, fieldSubType) + .isFilterable(true) + .build(); + } else { + dataField = VectorStoreRecordDataField.builder() + .withName("value") + .withStorageName("value_field") + .withFieldType(dataFieldType) + .isFilterable(true) + .build(); + } + VectorStoreRecordDataField dataTypeField; + dataTypeField = VectorStoreRecordDataField.builder() + .withName("valueType") + .withStorageName("value_type") + .withFieldType(String.class) + .isFilterable(false) + .build(); + + + VectorStoreRecordVectorField dummyVector = VectorStoreRecordVectorField.builder() + .withName("vectorValue") + .withStorageName("vectorValue") + .withFieldType(Float[].class) + .withDimensions(8) + .withDistanceFunction(DistanceFunction.COSINE_DISTANCE) + .withIndexKind(IndexKind.IVFFLAT) + .build(); + + VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField, dataTypeField, dataField, dummyVector) + ); + + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + String collectionName = "test_datatype_" + dataFieldName; + + VectorStoreRecordCollection collection = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions. builder() + .withRecordClass(ClassWithAnnotatedTypes.class) + .withRecordDefinition(definition).build()); + + collection.createCollectionAsync().block(); + + String key = "testid"; + + ClassWithAnnotatedTypes record = + new ClassWithAnnotatedTypes(key, dataFieldName, dataFieldValue, new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f }); + + collection.upsertAsync(record, null).block(); + + ClassWithAnnotatedTypes result = collection.getAsync(key, null).block(); + assertNotNull(result); + if (record.getValue().getClass().equals(OffsetDateTime.class)) { + assertTrue(((OffsetDateTime)dataFieldValue).isEqual((OffsetDateTime)record.getValue())); + } else if (dataFieldName == "byte_array") { + assertArrayEquals((byte[]) dataFieldValue, (byte[])record.getValue()); + } else { + assertEquals(dataFieldValue, result.getValue()); + } + + collection.deleteCollectionAsync().block(); + } + + private static Stream supportedDataTypes() { + return Stream.of( + Arguments.of("string", String.class, "asd123", null), + Arguments.of("boolean", Boolean.class, true, null), + Arguments.of("boolean", Boolean.class, false, null), + Arguments.of("byte", Byte.class, (byte) 127, null), + Arguments.of("short", Short.class, (short) 3, null), + Arguments.of("integer", Integer.class, 321, null), + Arguments.of("long", Long.class, 5L, null), + Arguments.of("float", Float.class, 3.14f, null), + Arguments.of("double", Double.class, 3.14159265358d, null), + Arguments.of("decimal", BigDecimal.class, new BigDecimal("12345.67"), null), + Arguments.of("timestamp", OffsetDateTime.class, OffsetDateTime.now(), null), + Arguments.of("uuid", UUID.class, UUID.randomUUID(), null), + Arguments.of("byte_array", byte[].class, new byte[] {1, 2, 3}, String.class), + Arguments.of("json", List.class, Arrays.asList("a", "s", "d"), String.class) + ); + } + +} diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java new file mode 100644 index 000000000..bac0ee4c3 --- /dev/null +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java @@ -0,0 +1,321 @@ +/* + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. + */ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.util.Arrays; +import java.util.UUID; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class OracleVectorStoreDataTypeSearchTest extends OracleCommonVectorStoreRecordCollectionTest { + private static final double MIN_DOUBLE = 1.0E-130; + private static final double MIN_DECIMAL = -1.0E125; + private static final BigDecimal BIG_NUMBER = BigDecimal.valueOf(9999999999999999.99); + + + + @ParameterizedTest + @MethodSource("supportedDataTypes") + void testDataTypesSearch (ClassWithAllBoxedTypes record) { + VectorStoreRecordCollection collection = setupBoxed(); + + collection.upsertAsync(record, null).block(); + + // boolean + VectorSearchResults results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("booleanValue", record.getBooleanValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getBooleanValue(), results.getResults().get(0).getRecord().getBooleanValue()); + + // byte + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("byteValue", record.getByteValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getByteValue(), results.getResults().get(0).getRecord().getByteValue()); + + // short + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("shortValue", record.getShortValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getShortValue(), results.getResults().get(0).getRecord().getShortValue()); + + // integer + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("integerValue", record.getIntegerValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getIntegerValue(), results.getResults().get(0).getRecord().getIntegerValue()); + + // long + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("longValue", record.getLongValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getLongValue(), results.getResults().get(0).getRecord().getLongValue()); + + // float + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("floatValue", record.getFloatValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getFloatValue(), results.getResults().get(0).getRecord().getFloatValue()); + + // double + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("doubleValue", record.getDoubleValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getDoubleValue(), results.getResults().get(0).getRecord().getDoubleValue()); + + // decimal + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("decimalValue", record.getDecimalValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + if (record.getDecimalValue() != null) { + assertEquals(0, record.getDecimalValue() + .compareTo(results.getResults().get(0).getRecord().getDecimalValue())); + } else { + assertEquals(record.getDecimalValue(), + results.getResults().get(0).getRecord().getDecimalValue()); + } + + // offset date time + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("offsetDateTimeValue", record.getOffsetDateTimeValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + if (record.getOffsetDateTimeValue() != null) { + assertTrue(record.getOffsetDateTimeValue() + .isEqual(results.getResults().get(0).getRecord().getOffsetDateTimeValue())); + } else { + assertEquals(record.getOffsetDateTimeValue(), + results.getResults().get(0).getRecord().getOffsetDateTimeValue()); + } + + // UUID + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("uuidValue", record.getUuidValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getUuidValue(), results.getResults().get(0).getRecord().getUuidValue()); + + // byte array + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("byteArrayValue", record.getByteArrayValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertArrayEquals(record.getByteArrayValue(), results.getResults().get(0).getRecord().getByteArrayValue()); + + collection.deleteCollectionAsync().block(); + + } + + + public VectorStoreRecordCollection setupBoxed() { + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + VectorStoreRecordCollection collection = + vectorStore.getCollection("BoxedTypes", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(ClassWithAllBoxedTypes.class) + .build()).createCollectionAsync().block(); + + collection.createCollectionAsync().block(); + + return collection; + } + + + private static Stream supportedDataTypes() { + return Stream.of( + Arguments.of( + new ClassWithAllBoxedTypes( + "ID1", true, (byte) 127, (short) 3, 321, 5L, + 3.14f, 3.14159265358d, new BigDecimal("12345.67"), + OffsetDateTime.now(), UUID.randomUUID(), "abc".getBytes(StandardCharsets.UTF_8), + Arrays.asList(1.0f, 2.6f), + new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f } + ) + ), + Arguments.of( + new ClassWithAllBoxedTypes( + "ID2", false, Byte.MIN_VALUE, Short.MIN_VALUE, Integer.MIN_VALUE, Long.MIN_VALUE, + Float.MIN_VALUE, MIN_DOUBLE, BigDecimal.valueOf(MIN_DECIMAL), + OffsetDateTime.now(), UUID.randomUUID(), new byte[] {Byte.MIN_VALUE, -10, 0, 10, Byte.MAX_VALUE}, + Arrays.asList(Float.MIN_VALUE, -10f, 0f, 10f, Float.MAX_VALUE), + new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f } + ) + ), + Arguments.of( + new ClassWithAllBoxedTypes( + "ID3", false, Byte.MAX_VALUE, Short.MAX_VALUE, Integer.MAX_VALUE, Long.MAX_VALUE, + Float.MAX_VALUE, BIG_NUMBER.doubleValue(), BIG_NUMBER.subtract(BigDecimal.valueOf(0.01d)), + OffsetDateTime.now(), UUID.randomUUID(), null, + null, + new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f } + ) + ), + Arguments.of( + new ClassWithAllBoxedTypes( + "ID3", null, null, null, null, null, + null, null, null, + null, null, null, + null, + null + ) + ) + ); + } + + private static Stream supportedDataPrimitiveTypes() { + return Stream.of( + Arguments.of( + new ClassWithAllPrimitiveTypes( + "ID1", true, (byte) 127, (short) 3, 321, 5L, + 3.14f, 3.14159265358d, new BigDecimal("12345.67"), + OffsetDateTime.now(), UUID.randomUUID(), "abc".getBytes(StandardCharsets.UTF_8), + Arrays.asList(1.0f, 2.6f), + new float[]{0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f} + ) + ), + Arguments.of( + new ClassWithAllPrimitiveTypes( + "ID2", false, Byte.MIN_VALUE, Short.MIN_VALUE, Integer.MIN_VALUE, + Long.MIN_VALUE, + Float.MIN_VALUE, MIN_DOUBLE, BigDecimal.valueOf(MIN_DECIMAL), + OffsetDateTime.now(), UUID.randomUUID(), + new byte[]{Byte.MIN_VALUE, -10, 0, 10, Byte.MAX_VALUE}, + Arrays.asList(Float.MIN_VALUE, -10f, 0f, 10f, Float.MAX_VALUE), + new float[]{0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f} + ) + ), + Arguments.of( + new ClassWithAllPrimitiveTypes( + "ID3", false, Byte.MAX_VALUE, Short.MAX_VALUE, Integer.MAX_VALUE, + Long.MAX_VALUE, + Float.MAX_VALUE, BIG_NUMBER.doubleValue(), + BIG_NUMBER.subtract(BigDecimal.valueOf(0.01d)), + OffsetDateTime.now(), UUID.randomUUID(), null, + null, + new float[]{0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f} + ) + ), + Arguments.of( + new ClassWithAllPrimitiveTypes( + "ID3", false, (byte) 0, (short) 0, 0, 0l, + 0f, 0d, null, + null, null, null, + null, + null + ) + ) + ); + } +} diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java new file mode 100644 index 000000000..eeb6a2e7b --- /dev/null +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java @@ -0,0 +1,234 @@ +/* + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. + */ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.util.Arrays; +import java.util.UUID; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class OracleVectorStoreDataTypeTest extends OracleCommonVectorStoreRecordCollectionTest { + private static final double MIN_NUMBER = 1.0E-130; + private static final BigDecimal BIG_NUMBER = BigDecimal.valueOf(9999999999999999.99); + + @ParameterizedTest + @MethodSource("supportedDataTypes") + void testDataTypes(ClassWithAllBoxedTypes values) { + + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + VectorStoreRecordCollection collection = + vectorStore.getCollection("BoxedTypes", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(ClassWithAllBoxedTypes.class) + .build()).createCollectionAsync().block(); + + collection.createCollectionAsync().block(); + + ClassWithAllBoxedTypes record = values; + + collection.upsertAsync(record, null).block(); + + ClassWithAllBoxedTypes result = collection.getAsync(values.getId(), null).block(); + assertNotNull(result); + + assertEquals(values.getBooleanValue(), result.getBooleanValue()); + assertArrayEquals(values.getByteArrayValue(), result.getByteArrayValue()); + assertEquals(values.getByteValue(), result.getByteValue()); + assertEquals(values.getDoubleValue(), result.getDoubleValue()); + assertEquals(values.getFloatValue(), result.getFloatValue()); + assertEquals(values.getIntegerValue(), result.getIntegerValue()); + assertEquals(values.getListOfFloatValue(), result.getListOfFloatValue()); + assertEquals(values.getLongValue(), result.getLongValue()); + if (values.getOffsetDateTimeValue() != null) { + assertTrue(values.getOffsetDateTimeValue().isEqual(result.getOffsetDateTimeValue())); + } else { + assertTrue(result.getOffsetDateTimeValue() == null); + } + assertEquals(values.getShortValue(), result.getShortValue()); + assertEquals(values.getUuidValue(), result.getUuidValue()); + + collection.deleteCollectionAsync().block(); + } + + @ParameterizedTest + @MethodSource("supportedDataPrimitiveTypes") + void testPrimitiveDataTypes(ClassWithAllPrimitiveTypes values) { + + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + VectorStoreRecordCollection collection = + vectorStore.getCollection("PrimitiveTypes", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(ClassWithAllPrimitiveTypes.class) + .build()).createCollectionAsync().block(); + + collection.createCollectionAsync().block(); + + ClassWithAllPrimitiveTypes record = values; + + collection.upsertAsync(record, null).block(); + + ClassWithAllPrimitiveTypes result = collection.getAsync(values.getId(), null).block(); + assertNotNull(result); + + assertEquals(values.getBooleanValue(), result.getBooleanValue()); + assertArrayEquals(values.getByteArrayValue(), result.getByteArrayValue()); + assertEquals(values.getByteValue(), result.getByteValue()); + assertEquals(values.getDoubleValue(), result.getDoubleValue()); + assertEquals(values.getFloatValue(), result.getFloatValue()); + assertEquals(values.getIntegerValue(), result.getIntegerValue()); + assertEquals(values.getListOfFloatValue(), result.getListOfFloatValue()); + assertEquals(values.getLongValue(), result.getLongValue()); + if (values.getOffsetDateTimeValue() != null) { + assertTrue(values.getOffsetDateTimeValue().isEqual(result.getOffsetDateTimeValue())); + } else { + assertTrue(result.getOffsetDateTimeValue() == null); + } + assertEquals(values.getShortValue(), result.getShortValue()); + assertEquals(values.getUuidValue(), result.getUuidValue()); + + collection.deleteCollectionAsync().block(); + } + + + private static Stream supportedDataTypes() { + return Stream.of( + Arguments.of( + new ClassWithAllBoxedTypes( + "ID1", true, (byte) 127, (short) 3, 321, 5L, + 3.14f, 3.14159265358d, new BigDecimal("12345.67"), + OffsetDateTime.now(), UUID.randomUUID(), "abc".getBytes(StandardCharsets.UTF_8), + Arrays.asList(1.0f, 2.6f), + new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f } + ) + ), + Arguments.of( + new ClassWithAllBoxedTypes( + "ID2", false, Byte.MIN_VALUE, Short.MIN_VALUE, Integer.MIN_VALUE, Long.MIN_VALUE, + Float.MIN_VALUE, MIN_NUMBER, BigDecimal.valueOf(MIN_NUMBER), + OffsetDateTime.now(), UUID.randomUUID(), new byte[] {Byte.MIN_VALUE, -10, 0, 10, Byte.MAX_VALUE}, + Arrays.asList(Float.MIN_VALUE, -10f, 0f, 10f, Float.MAX_VALUE), + new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f } + ) + ), + Arguments.of( + new ClassWithAllBoxedTypes( + "ID3", false, Byte.MAX_VALUE, Short.MAX_VALUE, Integer.MAX_VALUE, Long.MAX_VALUE, + Float.MAX_VALUE, BIG_NUMBER.doubleValue(), BIG_NUMBER.subtract(BigDecimal.valueOf(0.01d)), + OffsetDateTime.now(), UUID.randomUUID(), null, + null, + new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f } + ) + ), + Arguments.of( + new ClassWithAllBoxedTypes( + "ID3", null, null, null, null, null, + null, null, null, + null, null, null, + null, + null + ) + ) + ); + } + + private static Stream supportedDataPrimitiveTypes() { + return Stream.of( + Arguments.of( + new ClassWithAllPrimitiveTypes( + "ID1", true, (byte) 127, (short) 3, 321, 5L, + 3.14f, 3.14159265358d, new BigDecimal("12345.67"), + OffsetDateTime.now(), UUID.randomUUID(), "abc".getBytes(StandardCharsets.UTF_8), + Arrays.asList(1.0f, 2.6f), + new float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f } + ) + ), + Arguments.of( + new ClassWithAllPrimitiveTypes( + "ID2", false, Byte.MIN_VALUE, Short.MIN_VALUE, Integer.MIN_VALUE, Long.MIN_VALUE, + Float.MIN_VALUE, MIN_NUMBER, BigDecimal.valueOf(MIN_NUMBER), + OffsetDateTime.now(), UUID.randomUUID(), new byte[] {Byte.MIN_VALUE, -10, 0, 10, Byte.MAX_VALUE}, + Arrays.asList(Float.MIN_VALUE, -10f, 0f, 10f, Float.MAX_VALUE), + new float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f } + ) + ), + Arguments.of( + new ClassWithAllPrimitiveTypes( + "ID3", false, Byte.MAX_VALUE, Short.MAX_VALUE, Integer.MAX_VALUE, Long.MAX_VALUE, + Float.MAX_VALUE, BIG_NUMBER.doubleValue(), BIG_NUMBER.subtract(BigDecimal.valueOf(0.01d)), + OffsetDateTime.now(), UUID.randomUUID(), null, + null, + new float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f } + ) + ), + Arguments.of( + new ClassWithAllPrimitiveTypes( + "ID3", false, (byte)0, (short)0, 0, 0l, + 0f, 0d, null, + null, null, null, + null, + null + ) + ) + ); + } + +} diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java new file mode 100644 index 000000000..af2e28fd8 --- /dev/null +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java @@ -0,0 +1,502 @@ +/* + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. + */ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.jdbc.oracle.OracleVectorStoreQueryProvider.StringTypeMapping; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import com.microsoft.semantickernel.exceptions.SKException; + +import org.junit.jupiter.api.Test; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class OracleVectorStoreExtendedTest extends OracleCommonVectorStoreRecordCollectionTest { + + // Test vector types + @Test + void testUseStringVec() { + VectorStoreRecordCollection collection = + createCollection( + "use_string_vec", + DummyRecordForVecString.class, + null); + + DummyRecordForVecString d1 = new DummyRecordForVecString("id1", "description1", "[1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8]"); + DummyRecordForVecString d2 = new DummyRecordForVecString("id2", "description2", "[1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8]"); + + collection.upsertBatchAsync(Arrays.asList(d1,d2), null).block(); + + DummyRecordForVecString rec = collection.getAsync("id1", + GetRecordOptions.builder().includeVectors(true).build()).block(); + + assertNotNull(rec); + assertEquals("[1.1,2.2,3.3,4.4,5.5,6.6,7.7,8.8]", rec.getVec()); + + collection.deleteCollectionAsync().block(); + } + + @Test + void testUseCollectionVec() { + VectorStoreRecordCollection collection = + createCollection( + "use_collection_vec", + DummyRecordForVecCollection.class, + null); + + List v1 = Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10f, -1.3f, 5.5f); + List v2 = Arrays.asList(-2f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f); + DummyRecordForVecCollection d1 = new DummyRecordForVecCollection("id1", "", v1); + DummyRecordForVecCollection d2 = new DummyRecordForVecCollection("id2", "", v2); + + collection.upsertBatchAsync(Arrays.asList(d1,d2), null).block(); + + DummyRecordForVecCollection rec = collection.getAsync("id1", + GetRecordOptions.builder().includeVectors(true).build()).block(); + + assertNotNull(rec); + assertEquals(8, rec.getVec().size()); + assertIterableEquals(v1, rec.getVec()); + + collection.deleteCollectionAsync().block(); + } + + // Test corner-case + @Test + void testUseCLOB() { + VectorStoreRecordCollection collection = + createCollection( + "use_clob", + DummyRecordForCLOB.class, + OracleVectorStoreQueryProvider.StringTypeMapping.USE_CLOB); + + DummyRecordForCLOB d1 = new DummyRecordForCLOB("id1", "clob-description", null); + DummyRecordForCLOB d2 = new DummyRecordForCLOB("id2", "clob-description2", vec(0)); + + collection.upsertBatchAsync(Arrays.asList(d1,d2), null).block(); + + try (Connection c = DATA_SOURCE.getConnection()) { + PreparedStatement st = c.prepareStatement( + "SELECT DATA_TYPE FROM USER_TAB_COLUMNS " + + "WHERE TABLE_NAME = 'SKCOLLECTION_USE_CLOB' AND COLUMN_NAME = 'DESCRIPTION'" + ); + ResultSet rs = st.executeQuery(); + rs.next(); + assertEquals("CLOB", rs.getString(1)); + } catch (SQLException e) { + throw new RuntimeException(e); + } finally { + collection.deleteCollectionAsync().block(); + } + } + + @Test + void testClobLongText() { + VectorStoreRecordCollection collection = + createCollection( + "clob_long_text", + DummyRecordForCLOB.class, + OracleVectorStoreQueryProvider.StringTypeMapping.USE_CLOB); + + String longText = String.join("", java.util.Collections.nCopies(6000, "a")); + DummyRecordForCLOB r = new DummyRecordForCLOB("big", longText, vec(0)); + collection.upsertAsync(r, null).block(); + + DummyRecordForCLOB out = collection.getAsync("big", null).block(); + assertEquals(longText.length(), out.getDescription().length()); + assertTrue(out.getDescription().startsWith("aaaa")); + + collection.deleteCollectionAsync().block(); + } + + @Test + void testMultipleFilter() { + VectorStoreRecordCollection collection = + createCollection( + "multiple_filter", + DummyRecordForMultipleFilter.class, + null); + + DummyRecordForMultipleFilter d1 = new DummyRecordForMultipleFilter("id1", 4, 120, floatVec(0f)); + DummyRecordForMultipleFilter d2 = new DummyRecordForMultipleFilter("id2", 4, 100, floatVec(0f)); + DummyRecordForMultipleFilter d3 = new DummyRecordForMultipleFilter("id3", 3, 100, floatVec(0f)); + + collection.upsertBatchAsync(Arrays.asList(d1,d2,d3), null).block(); + + VectorSearchFilter filter = VectorSearchFilter.builder() + .equalTo("price",100) + .equalTo("stars", 4) + .build(); + + VectorSearchResults results = + collection.searchAsync(null, + VectorSearchOptions.builder() + .withVectorSearchFilter(filter) + .build() + ).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals("id2", results.getResults().get(0).getRecord().getId()); + + collection.deleteCollectionAsync().block(); + } + + @Test + void testVectorDimensionMismatch() { + VectorStoreRecordCollection collection = + createCollection( + "vector_dimension_mismatch", + DummyRecord.class, + null); + + // Empty vector rejected + DummyRecord d1 = new DummyRecord("id1", 4, 120d, new float[]{}); + SKException ex = assertThrows(SKException.class, + () -> collection.upsertBatchAsync(Arrays.asList(d1), null).block()); + assertTrue(ex.getCause().getMessage().contains("ORA-51803")); + + // Vector dimension mismatch + DummyRecord d2 = new DummyRecord("id1", 4, 120d, new float[]{1.1f,2.2f,3.3f,4.4f,5.5f}); + SKException ex2 = assertThrows(SKException.class, + () -> collection.upsertBatchAsync(Arrays.asList(d2), null).block()); + assertTrue(ex2.getCause().getMessage().contains("ORA-51803")); + + collection.deleteCollectionAsync().block(); + } + + @Test + void testNullFieldValue() { + VectorStoreRecordCollection collection = + createCollection("test_null", DummyRecord.class, null); + + DummyRecord d1 = new DummyRecord("id1", 4, null, floatVec(1)); + collection.upsertBatchAsync(Arrays.asList(d1), null).block(); + + VectorSearchFilter filter = VectorSearchFilter.builder() + .equalTo("price",null)// + .build(); + + VectorSearchResults results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter(filter) + .build() + ).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals("id1", results.getResults().get(0).getRecord().getId()); + + collection.deleteCollectionAsync().block(); + } + + @Test + void testSkipAndTop() { + VectorStoreRecordCollection collection = + createCollection( + "test_skip_and_top", + DummyRecord.class, + null); + + List l1 = new ArrayList<>(); + for (int i = 1; i <= 10; i++) { + l1.add(new DummyRecord("id" + i, i, (double) i, floatVec(i))); + } + collection.upsertBatchAsync(l1, null).block(); + + VectorSearchResults results = collection.searchAsync( + Collections.nCopies(8,0f), + VectorSearchOptions.builder() + .withIncludeVectors(true) + .withSkip(5) + .withTop(3) + .build() + ).block(); + + assertEquals(3, results.getResults().size()); + List ids = results.getResults().stream().map(r -> r.getRecord().getId()).collect( + Collectors.toList()); + assertEquals(Arrays.asList("id6","id7","id8"), ids); + + collection.deleteCollectionAsync().block(); + } + + // corner case for OracleVectorStoreRecordMapper + @Test + void testMapRecordToStorageModel_throws() { + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(String.class) + .build(); + + VectorStoreRecordDefinition definition = + VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField) + ); + + OracleVectorStoreRecordMapper mapper = + OracleVectorStoreRecordMapper. builder() + .withRecordClass(DummyRecord.class) + .withVectorStoreRecordDefinition(definition) + .build(); + + UnsupportedOperationException ex = assertThrows( + UnsupportedOperationException.class, + () -> mapper.mapRecordToStorageModel(new DummyRecord())); + assertEquals("Not implemented", ex.getMessage()); + } + + private VectorStoreRecordCollection createCollection( + String collectionName, + Class recordClass, + OracleVectorStoreQueryProvider.StringTypeMapping stringTypeMapping) { + + OracleVectorStoreQueryProvider.Builder builder = + OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE); + + if (stringTypeMapping != null) { + builder.withStringTypeMapping(stringTypeMapping); + } + OracleVectorStoreQueryProvider queryProvider = builder.build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + VectorStoreRecordCollection collection = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(recordClass) + .build()).createCollectionAsync().block(); + + return collection; + } + + private List vec(float x) { + return Arrays.asList(x, x+1, x+2, x+3, x+4, x+5, x+6, x+7); + } + + private float[] floatVec(float x) { + return new float[] { x, x+1, x+2, x+3, x+4, x+5, x+6, x+7 }; + } + + private static class DummyRecordForVecString { + @VectorStoreRecordKey + private final String id; + + @VectorStoreRecordData(isFilterable = false) + private final String description; + + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.IVFFLAT) + private final String vec; + + public DummyRecordForVecString() { + this(null, null, null); + } + public DummyRecordForVecString(String id, String description, String vec) { + this.id = id; + this.description = description; + this.vec = vec; + } + + public String getId() { + return id; + } + public String getDescription() { + return description; + } + public String getVec() { + return vec; + } + } + + private static class DummyRecordForVecCollection{ + @VectorStoreRecordKey + private String id; + + @VectorStoreRecordData(isFilterable = false) + private String description; + + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.IVFFLAT) + private Collection vec; + + public DummyRecordForVecCollection() { + this(null, null, null); + } + public DummyRecordForVecCollection(String id, String description, Collection vec) { + this.id = id; + this.description = description; + this.vec = vec; + } + + public String getId() { + return id; + } + public String getDescription() { + return description; + } + public Collection getVec() { + return vec; + } + } + + private static class DummyRecordForCLOB { + @VectorStoreRecordKey + private String id; + + @VectorStoreRecordData(isFilterable = false) + private String description; + + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.IVFFLAT) + private List vec; + + private DummyRecordForCLOB() { + this(null, null, null); + } + private DummyRecordForCLOB(String id, String description, List vec) { + this.id = id; + this.description = description; + this.vec = vec; + } + + public String getId() { + return id; + } + public String getDescription() { + return description; + } + public List getVec() { + return vec; + } + } + + private static class DummyRecordForMultipleFilter { + @VectorStoreRecordKey + private String id; + + @VectorStoreRecordData(isFilterable = true) + private int stars; + + @VectorStoreRecordData(isFilterable = true) + private double price; + + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.IVFFLAT) + private float[] vec; + + public DummyRecordForMultipleFilter() { + this(null, 0, 0d, null); + } + + public DummyRecordForMultipleFilter(String id, int stars, double price, float[] vec) { + this.id = id; + this.stars = stars; + this.price = price; + this.vec = vec; + } + + public String getId() { + return id; + } + public int getStars() { + return stars; + } + public double getPrice() { + return price; + } + public float[] getVec() { + return vec; + } + } + + private static class DummyRecord { + @VectorStoreRecordKey + private String id; + + @VectorStoreRecordData(isFilterable = true) + private int stars; + + @VectorStoreRecordData(isFilterable = true) + private Double price; + + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.IVFFLAT) + private float[] vec; + + public DummyRecord() { + this(null, 0, 0d, null); + } + + public DummyRecord(String id, int stars, Double price, float[] vec) { + this.id = id; + this.stars = stars; + this.price = price; + this.vec = vec; + } + + public String getId() { + return id; + } + public int getStars() { + return stars; + } + public Double getPrice() { + return price; + } + public float[] getVec() { + return vec; + } + } +} diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java new file mode 100644 index 000000000..850ae9238 --- /dev/null +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -0,0 +1,592 @@ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class OracleVectorStoreRecordCollectionTest extends OracleCommonVectorStoreRecordCollectionTest { + private static VectorStoreRecordCollection recordCollection; + + @BeforeAll + public static void setup() throws Exception { + + // Build a query provider + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + // Build a vector store + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + // Get a collection from the vector store + recordCollection = + vectorStore.getCollection("skhotels", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build()); + + recordCollection.createCollectionIfNotExistsAsync().block(); + } + + @BeforeEach + public void clearCollection() { + recordCollection.deleteCollectionAsync().block(); + recordCollection.createCollectionAsync().block(); + } + + private static List getHotels() { + List vec1 = Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f); + float[] arrayf1 = new float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f }; + Float[] arrayF1 = new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f }; + List vec2 = Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f); + float[] arrayf2 = new float[] { -2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f }; + Float[] arrayF2 = new Float[] { -2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f }; + List vec3 = Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f); + float[] arrayf3 = new float[] { 4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f }; + Float[] arrayF3 = new Float[] { 4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f }; + List vec4 = Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f); + float[] arrayf4 = new float[] { 7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f }; + Float[] arrayF4 = new Float[] { 7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f }; + List vec5 =Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f); + float[] arrayf5 = new float[] { -3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f }; + Float[] arrayF5 = new Float[] { -3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f }; + return Arrays.asList( + new Hotel("id_1", "Hotel 1", 1, 1.49d, Arrays.asList("one", "two"), "Hotel 1 description", + vec1, arrayf1, arrayf1, arrayF1, + 4.0), + new Hotel("id_2", "Hotel 2", 2, 1.44d, Arrays.asList("three", "four"), "Hotel 2 description with free-text search", + vec2, arrayf2, arrayf2, arrayF2, + 4.0), + new Hotel("id_3", "Hotel 3", 3, 1.53d, Arrays.asList("five", "six"), "Hotel 3 description", + vec3, arrayf3, arrayf3, arrayF3, + 5.0), + new Hotel("id_4", "Hotel 4", 4, 1.35d, Arrays.asList("seven", "eight"), "Hotel 4 description", + vec4, arrayf4, arrayf4, arrayF4, + 4.0), + new Hotel("id_5", "Hotel 5", 5, 1.89d, Arrays.asList("nine", "ten"),"Hotel 5 description", + vec5, arrayf5, arrayf5, arrayF5, + 4.0)); + } + + /** + * Search embeddings similar to the third hotel embeddings. + * In order of similarity: + * 1. Hotel 3 + * 2. Hotel 1 + * 3. Hotel 4 + */ + private static final List SEARCH_EMBEDDINGS = Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, + -0.8f, 1.1f, -2.2f, 8.2f); + + @Test + public void createAndDeleteCollectionAsync() { + assertEquals(true, recordCollection.collectionExistsAsync().block()); + + recordCollection.deleteCollectionAsync().block(); + assertEquals(false, recordCollection.collectionExistsAsync().block()); + + recordCollection.createCollectionAsync().block(); + assertEquals(true, recordCollection.collectionExistsAsync().block()); + } + + @Test + public void upsertRecordAsync() { + List hotels = getHotels(); + for (Hotel hotel : hotels) { + recordCollection.upsertAsync(hotel, null).block(); + } + + for (Hotel hotel : hotels) { + Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), null).block(); + assertNotNull(retrievedHotel); + assertEquals(hotel.getId(), retrievedHotel.getId()); + assertEquals(hotel.getName(), retrievedHotel.getName()); + assertEquals(hotel.getDescription(), retrievedHotel.getDescription()); + } + } + + @Test + public void upsertBatchAsync() { + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + for (Hotel hotel : hotels) { + Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), null).block(); + assertNotNull(retrievedHotel); + assertEquals(hotel.getId(), retrievedHotel.getId()); + assertEquals(hotel.getName(), retrievedHotel.getName()); + assertEquals(hotel.getDescription(), retrievedHotel.getDescription()); + } + } + + @Test + public void getBatchAsync() { + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + List keys = hotels.stream().map(Hotel::getId).collect(Collectors.toList()); + List retrievedHotels = recordCollection.getBatchAsync(keys, null).block(); + + assertNotNull(retrievedHotels); + assertEquals(keys.size(), retrievedHotels.size()); + for (Hotel hotel : retrievedHotels) { + assertTrue(keys.contains(hotel.getId())); + } + } + + @Test + public void deleteRecordAsync() { + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + for (Hotel hotel : hotels) { + recordCollection.deleteAsync(hotel.getId(), null).block(); + assertNull(recordCollection.getAsync(hotel.getId(), null).block()); + } + } + + @Test + public void deleteBatchAsync() { + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + List keys = hotels.stream().map(Hotel::getId).collect(Collectors.toList()); + recordCollection.deleteBatchAsync(keys, null).block(); + + for (String key : keys) { + assertNull(recordCollection.getAsync(key, null).block()); + } + } + + @ParameterizedTest + @MethodSource("parametersExactSearch") + public void exactSearch(DistanceFunction distanceFunction, List expectedDistance) { + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + VectorSearchOptions options = VectorSearchOptions.builder() + .withVectorFieldName(distanceFunction.getValue()) + .withTop(3) + .build(); + + // Embeddings similar to the third hotel + List> results = recordCollection + .searchAsync(SEARCH_EMBEDDINGS, options).block().getResults(); + assertNotNull(results); + assertEquals(3, results.size()); + // The third hotel should be the most similar + assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); + assertEquals(expectedDistance.get(0).doubleValue(), results.get(0).getScore(), 0.0002d); + assertEquals(hotels.get(0).getId(), results.get(1).getRecord().getId()); + assertEquals(expectedDistance.get(1).doubleValue(), results.get(1).getScore(), 0.0002d); + assertEquals(hotels.get(3).getId(), results.get(2).getRecord().getId()); + assertEquals(expectedDistance.get(2).doubleValue(), results.get(2).getScore(), 0.0002d); + + options = VectorSearchOptions.builder() + .withVectorFieldName(distanceFunction.getValue()) + .withSkip(1) + .withTop(-100) + .build(); + + // Skip the first result + results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, options).block().getResults(); + assertNotNull(results); + assertEquals(1, results.size()); + // The first hotel should be the most similar + assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId()); + assertEquals(results.get(0).getScore(), expectedDistance.get(1), 0.001d); + } + + @ParameterizedTest + @MethodSource("distanceFunctionAndDistance") + public void searchWithFilter(DistanceFunction distanceFunction, double expectedDistance) { + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + VectorSearchOptions options = VectorSearchOptions.builder() + .withVectorFieldName(distanceFunction.getValue()) + .withTop(3) + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("rating", 4.0).build()) + .build(); + + // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned + List> results = recordCollection + .searchAsync(SEARCH_EMBEDDINGS, options).block().getResults(); + assertNotNull(results); + assertEquals(3, results.size()); + // The first hotel should be the most similar + assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId()); + assertEquals(results.get(0).getScore(), expectedDistance, 0.0002d); + } + + + @Test + public void searchWithTagFilter() { + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + VectorSearchOptions options = VectorSearchOptions.builder() +// .withVectorFieldName("") + .withTop(3) + .withVectorSearchFilter( + VectorSearchFilter.builder() + .anyTagEqualTo("tags", "three") + .build()) + .build(); + + // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned + List> results = recordCollection + .searchAsync(SEARCH_EMBEDDINGS, options).block().getResults(); + assertNotNull(results); + assertEquals(1, results.size()); + // The second hotel contains the tag we are searching for + assertEquals(hotels.get(1).getId(), results.get(0).getRecord().getId()); + } + + @ParameterizedTest + @MethodSource("supportedKeyTypes") + void testKeyTypes(String suffix, Class keyType, Object keyValue) { + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(keyType) + .build(); + + VectorStoreRecordDataField dummyField = VectorStoreRecordDataField.builder() + .withName("dummy") + .withStorageName("dummy") + .withFieldType(String.class) + .build(); + + VectorStoreRecordVectorField dummyVector = VectorStoreRecordVectorField.builder() + .withName("vec") + .withStorageName("vec") + .withFieldType(List.class) + .withDimensions(2) + .withDistanceFunction(DistanceFunction.EUCLIDEAN_DISTANCE) + .withIndexKind(IndexKind.UNDEFINED) + .build(); + + VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField, dummyField, dummyVector) + ); + + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + String collectionName = "test_keytype_" + suffix; + + VectorStoreRecordCollection collectionRaw = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(DummyRecordForKeyTypes.class) + .withRecordDefinition(definition) + .build()); + + VectorStoreRecordCollection collection = + (VectorStoreRecordCollection) collectionRaw; + + collection.createCollectionAsync().block(); + + DummyRecordForKeyTypes record = new DummyRecordForKeyTypes(keyValue, "dummyValue", Arrays.asList(1.0f, 2.0f)); + collection.upsertAsync(record, null).block(); + + DummyRecordForKeyTypes result = collection.getAsync(keyValue, null).block(); + assertNotNull(result); + assertEquals("dummyValue", result.getDummy()); + + collection.deleteCollectionAsync().block(); + } + + + @Nested + class HNSWIndexTests { + @Test + void testHNSWIndexIsCreatedSuccessfully() throws Exception { + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(String.class) + .build(); + + VectorStoreRecordDataField dummyField = VectorStoreRecordDataField.builder() + .withName("dummy") + .withStorageName("dummy") + .withFieldType(String.class) + .isFilterable(false) + .build(); + + VectorStoreRecordVectorField hnswVector= VectorStoreRecordVectorField.builder() + .withName("hnsw") + .withStorageName("hnsw") + .withFieldType(List.class) + .withDimensions(8) + .withDistanceFunction(DistanceFunction.COSINE_SIMILARITY) + .withIndexKind(IndexKind.HNSW) + .build(); + + VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField, dummyField, hnswVector) + ); + + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + String collectionName = "skhotels_hnsw"; + VectorStoreRecordCollection collection = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Object.class) + .withRecordDefinition(definition) + .build()); + + // create collection + collection.createCollectionAsync().block(); + + String expectedIndexName = hnswVector.getEffectiveStorageName().toUpperCase() + "_VECTOR_INDEX"; + + // check if index exist + try (Connection conn = DATA_SOURCE.getConnection(); + PreparedStatement stmt = conn.prepareStatement( + "SELECT COUNT(*) FROM USER_INDEXES WHERE INDEX_NAME=?")) { + stmt.setString(1, expectedIndexName); + ResultSet rs = stmt.executeQuery(); + rs.next(); + int count = rs.getInt(1); + + assertEquals(1, count, "hnsw vector index should have been created"); + } finally { + // clean up + try (Connection conn = DATA_SOURCE.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP TABLE " + "SKCOLLECTION_" + collectionName); + } + } + } + } + + @Nested + class UndefinedIndexTests { + @Test + void testNoIndexIsCreatedForUndefined() throws Exception { + // create key field + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(String.class) + .build(); + + // create vector field, set IndexKind to UNDEFINED + VectorStoreRecordVectorField undefinedVector= VectorStoreRecordVectorField.builder() + .withName("undef") + .withStorageName("undef") + .withFieldType(List.class) + .withDimensions(8) + .withDistanceFunction(DistanceFunction.COSINE_SIMILARITY) + .withIndexKind(IndexKind.UNDEFINED) + .build(); + + VectorStoreRecordDataField dummyField = VectorStoreRecordDataField.builder() + .withName("dummy") + .withStorageName("dummy") + .withFieldType(String.class) + .isFilterable(false) + .build(); + + VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField, dummyField, undefinedVector) + ); + + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + String collectionName = "skhotels_undefined"; + VectorStoreRecordCollection collection = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Object.class) + .withRecordDefinition(definition) + .build()); + + // create collection + collection.createCollectionAsync().block(); + + // check if index exist + String expectedIndexName = undefinedVector.getEffectiveStorageName().toUpperCase() + "_VETCOR_INDEX"; + try (Connection conn = DATA_SOURCE.getConnection(); + PreparedStatement stmt = conn.prepareStatement( + "SELECT COUNT(*) FROM USER_INDEXES WHERE INDEX_NAME = ?")) { + stmt.setString(1, expectedIndexName); + ResultSet rs = stmt.executeQuery(); + rs.next(); + int count = rs.getInt(1); + + assertEquals(0,count,"Vector index should not be created for IndexKind.UNDEFINED"); + } finally { + // clean up + try (Connection conn = DATA_SOURCE.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP TABLE " + "SKCOLLECTION_" + collectionName); + } + } + } + } + + private static Stream distanceFunctionAndDistance() { + return Stream.of( + Arguments.of (DistanceFunction.COSINE_DISTANCE, 0.8548d), + Arguments.of (DistanceFunction.COSINE_SIMILARITY, 0.1451d), + Arguments.of (DistanceFunction.DOT_PRODUCT, 30.3399d), + Arguments.of (DistanceFunction.EUCLIDEAN_DISTANCE, 18.9081d), + Arguments.of (DistanceFunction.UNDEFINED, 18.9081d) + ); + } + + private static Stream parametersExactSearch() { + return Stream.of( + Arguments.of (DistanceFunction.COSINE_SIMILARITY, Arrays.asList(0.9999d, 0.1451d, 0.0178d)), + Arguments.of (DistanceFunction.COSINE_DISTANCE, Arrays.asList(1.6422E-5d, 0.8548d, 0.9821d)), + Arguments.of (DistanceFunction.DOT_PRODUCT, Arrays.asList(202.3399d, 30.3399d, 3.6199d)), + Arguments.of (DistanceFunction.EUCLIDEAN_DISTANCE, Arrays.asList(0.1000d, 18.9081d, 19.9669d)), + Arguments.of (DistanceFunction.UNDEFINED, Arrays.asList(0.1000d, 18.9081d, 19.9669d)) + ); + } + + // commented out temporarily because only String type key is supported in + // JDBCVectorStoreRecordCollection#getKeyFromRecord: + // ... + // return (String) keyField.get(data); + // ... + // thus upsertAync/getAsync won't work + private static Stream supportedKeyTypes() { + return Stream.of( + Arguments.of("string", String.class, "asd123") /*, + Arguments.of("integer", Integer.class, 321), + Arguments.of("long", Long.class, 5L), + Arguments.of("short", Short.class, (short) 3), + Arguments.of("uuid", UUID.class, UUID.randomUUID())*/ + ); + } + + private static class DummyRecordForKeyTypes { + private final Object id; + private final String dummy; + private final List vec; + @JsonCreator + public DummyRecordForKeyTypes( + @JsonProperty("id")Object id, + @JsonProperty("dummy") String dummy, + @JsonProperty("vec") List vec) { + this.id = id; + this.dummy = dummy; + this.vec = vec; + } + + public Object getId() { + return id; + } + + public String getDummy() { + return dummy; + } + + @Override + public String toString() { + return String.valueOf(id); + } + } + + private static class DummyRecordForDataTypes { + private final String id; + private final Object dummy; + private final List vec; + @JsonCreator + public DummyRecordForDataTypes( + @JsonProperty("id") String id, + @JsonProperty("dummy") Object dummy, + @JsonProperty("vec") List vec) { + this.id = id; + this.dummy = dummy; + this.vec = vec; + } + + public String getId() { + return id; + } + + public Object getDummy() { + return dummy; + } + + @Override + public String toString() { + return String.valueOf(id); + } + } +} diff --git a/data/semantickernel-data-oracle/src/test/resources/initialize.sql b/data/semantickernel-data-oracle/src/test/resources/initialize.sql new file mode 100644 index 000000000..8756f121d --- /dev/null +++ b/data/semantickernel-data-oracle/src/test/resources/initialize.sql @@ -0,0 +1,12 @@ +-- Exit on any errors +WHENEVER SQLERROR EXIT SQL.SQLCODEAdd commentMore actions + +-- Configure the size of the Vector Pool to 1 GiB. +ALTER SYSTEM SET vector_memory_size=1G SCOPE=SPFILE; + +sqlplus / as sysdba + +SHUTDOWN ABORT; +STARTUP; + +exit \ No newline at end of file diff --git a/data/semantickernel-data-postgres/pom.xml b/data/semantickernel-data-postgres/pom.xml new file mode 100644 index 000000000..2d01ed81f --- /dev/null +++ b/data/semantickernel-data-postgres/pom.xml @@ -0,0 +1,59 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../../pom.xml + + + semantickernel-data-postgres + Semantic Kernel PostreSQL connector + Provides a PostreSQL connector for the Semantic Kernel + + + + com.microsoft.semantic-kernel + semantickernel-api + + + com.microsoft.semantic-kernel + semantickernel-data-jdbc + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-data + + + com.microsoft.semantic-kernel + semantickernel-api-builders + + + + com.fasterxml.jackson.core + jackson-databind + compile + + + com.fasterxml.jackson.core + jackson-core + compile + + + com.github.spotbugs + spotbugs-annotations + + + org.postgresql + postgresql + 42.7.4 + + + \ No newline at end of file diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorDistanceFunction.java b/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorDistanceFunction.java similarity index 100% rename from data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorDistanceFunction.java rename to data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorDistanceFunction.java diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorIndexKind.java b/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorIndexKind.java similarity index 100% rename from data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorIndexKind.java rename to data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorIndexKind.java diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java b/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java similarity index 98% rename from data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java rename to data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java index 4734f4847..1f3273eb3 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -447,6 +447,16 @@ public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { return String.format("%s @> ?::jsonb", fieldName); } + @Override + public VectorStoreRecordMapper getVectorStoreRecordMapper( + Class recordClass, + VectorStoreRecordDefinition recordDefinition) { + return PostgreSQLVectorStoreRecordMapper.builder() + .withRecordClass(recordClass) + .withVectorStoreRecordDefinition(recordDefinition) + .build(); + } + /** * A builder for the PostgreSQLVectorStoreQueryProvider class. */ diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreRecordMapper.java b/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreRecordMapper.java similarity index 100% rename from data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreRecordMapper.java rename to data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreRecordMapper.java diff --git a/data/semantickernel-data-redis/pom.xml b/data/semantickernel-data-redis/pom.xml index a8e582ebf..4af2dc6a4 100644 --- a/data/semantickernel-data-redis/pom.xml +++ b/data/semantickernel-data-redis/pom.xml @@ -17,6 +17,18 @@ com.microsoft.semantic-kernel semantickernel-api + + com.microsoft.semantic-kernel + semantickernel-api-data + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-builders + org.slf4j diff --git a/data/semantickernel-data-sqlite/pom.xml b/data/semantickernel-data-sqlite/pom.xml new file mode 100644 index 000000000..64b4a6dbb --- /dev/null +++ b/data/semantickernel-data-sqlite/pom.xml @@ -0,0 +1,60 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../../pom.xml + + + com.microsoft.semantic-kernel + semantickernel-data-sqlite + Semantic Kernel SQLite JDBC driver connector + Provides a SQLite connector for the Semantic Kernel + + + + com.microsoft.semantic-kernel + semantickernel-api + + + com.microsoft.semantic-kernel + semantickernel-data-jdbc + + + com.microsoft.semantic-kernel + semantickernel-api-data + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-builders + + + + com.fasterxml.jackson.core + jackson-databind + compile + + + com.fasterxml.jackson.core + jackson-core + compile + + + com.github.spotbugs + spotbugs-annotations + + + org.xerial + sqlite-jdbc + 3.47.0.0 + + + \ No newline at end of file diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/sqlite/SQLiteVectorStoreQueryProvider.java b/data/semantickernel-data-sqlite/src/main/java/com/microsoft/semantickernel/data/jdbc/sqlite/SQLiteVectorStoreQueryProvider.java similarity index 100% rename from data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/sqlite/SQLiteVectorStoreQueryProvider.java rename to data/semantickernel-data-sqlite/src/main/java/com/microsoft/semantickernel/data/jdbc/sqlite/SQLiteVectorStoreQueryProvider.java diff --git a/pom.xml b/pom.xml index 8f8ce2869..c50eda63b 100644 --- a/pom.xml +++ b/pom.xml @@ -77,7 +77,18 @@ data/semantickernel-data-azureaisearch data/semantickernel-data-jdbc data/semantickernel-data-redis + data/semantickernel-data-mysql + data/semantickernel-data-hsqldb + data/semantickernel-data-postgres + data/semantickernel-data-sqlite + data/semantickernel-data-oracle agents/semantickernel-agents-core + semantickernel-api-data + semantickernel-api-exceptions + semantickernel-api-builders + semantickernel-api-textembedding-services + semantickernel-api-localization + semantickernel-api-ai-services @@ -135,6 +146,36 @@ semantickernel-connectors-ai-openai ${project.version} + + com.microsoft.semantic-kernel + semantickernel-api-builders + ${project.version} + provided + + + com.microsoft.semantic-kernel + semantickernel-api-data + ${project.version} + provided + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + ${project.version} + provided + + + com.microsoft.semantic-kernel + semantickernel-api-localization + ${project.version} + provided + + + com.microsoft.semantic-kernel + semantickernel-api-textembedding-services + ${project.version} + provided + com.microsoft.semantic-kernel.extensions semantickernel-sequentialplanner-extension @@ -145,6 +186,13 @@ semantickernel-actionplanner-extension ${project.version} + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + ${project.version} + provided + + com.github.spotbugs @@ -215,6 +263,7 @@ ${maven.compiler.release} ${maven.compiler.release} 8 + -Xlint:unchecked diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index ffc2adae1..177a27126 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -136,6 +136,29 @@ com.github.victools jsonschema-module-jackson + + com.microsoft.semantic-kernel + semantickernel-data-jdbc + ${project.version} + + + com.microsoft.semantic-kernel + semantickernel-learn-resources + ${project.version} + compile + + + com.microsoft.semantic-kernel + semantickernel-data-postgres + ${project.version} + compile + + + com.microsoft.semantic-kernel + semantickernel-data-oracle + ${project.version} + compile + diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java new file mode 100644 index 000000000..e32131050 --- /dev/null +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.samples.syntaxexamples.memory; + +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollection; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.jdbc.oracle.OracleVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.samples.documentationexamples.data.index.Hotel; +import java.sql.SQLException; +import java.util.Collections; +import oracle.jdbc.datasource.impl.OracleDataSource; + +public class VectorStoreWithOracle { + + public static void main(String[] args) throws SQLException { + System.out.println("=============================================================="); + System.out.println("============== Oracle Vector Store Example ==================="); + System.out.println("=============================================================="); + + // Configure the data source + OracleDataSource dataSource = new OracleDataSource(); + dataSource.setURL("jdbc:oracle:thin:@localhost:1521/FREEPDB1"); + dataSource.setUser("scott"); + dataSource.setPassword("tiger"); + + // Build a query provider + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(dataSource) + .build(); + + // Build a vector store + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(dataSource) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + // Get a collection from the vector store + VectorStoreRecordCollection collection = vectorStore.getCollection( + "skhotels", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build()); + + // Create the collection if it doesn't exist yet. + collection.createCollectionAsync().block(); + + collection.upsertAsync(new Hotel("1", + "HotelOne", + "Desc for HotelOne", + Collections.emptyList(), Collections.emptyList()), + null) + .block(); + + } + +} diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index f6f8d2a85..d9fd6eeee 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -43,6 +43,10 @@ com.microsoft.semantic-kernel semantickernel-aiservices-openai + + com.microsoft.semantic-kernel + semantickernel-api-builders + com.azure diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 68062bf41..73cdac1d4 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -41,7 +41,16 @@ com.microsoft.semantic-kernel semantickernel-data-redis - + + com.microsoft.semantic-kernel + semantickernel-data-oracle + ${project.version} + + + com.microsoft.semantic-kernel + semantickernel-data-postgres + ${project.version} + org.apache.logging.log4j log4j-api @@ -85,6 +94,12 @@ 9.0.0 compile + + com.microsoft.semantic-kernel + semantickernel-data-postgres + ${project.version} + compile + diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java new file mode 100644 index 000000000..77b034bf4 --- /dev/null +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java @@ -0,0 +1,125 @@ +/* + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. + */ +package com.microsoft.semantickernel.samples.documentationexamples.data.vectorstores.oracle; + +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; +import java.util.List; + +public class Book { + + public Book() { + } + + public Book(String isbn, String title, String author, int pages, + List tags, String summary, List summaryEmbedding) { + this.isbn = isbn; + this.title = title; + this.author = author; + this.pages = pages; + this.tags = tags; + this.summary = summary; + this.summaryEmbedding = summaryEmbedding; + } + + @VectorStoreRecordKey + private String isbn; + + @VectorStoreRecordData(isFilterable = true) + private String title; + + @VectorStoreRecordData(isFilterable = true) + private String author; + + @VectorStoreRecordData + private int pages; + + @VectorStoreRecordData(isFilterable = true) + private List tags; + + @VectorStoreRecordData(isFilterable = true, isFullTextSearchable = true) + private String summary; + + @VectorStoreRecordVector(dimensions = 2) + private List summaryEmbedding; + + public String getIsbn() { + return isbn; + } + + public String getTitle() { + return title; + } + + public String getAuthor() { + return author; + } + + public int getPages() { + return pages; + } + + public List getTags() { + return tags; + } + + public String getSummary() { + return summary; + } + + public List getSummaryEmbedding() { + return summaryEmbedding; + } + + public void setIsbn(String isbn) { + this.isbn = isbn; + } + + public void setTitle(String title) { + this.title = title; + } + + public void setAuthor(String author) { + this.author = author; + } + + public void setPages(int pages) { + this.pages = pages; + } + + public void setTags(List tags) { + this.tags = tags; + } + + public void setSummaryEmbedding(List summaryEmbedding) { + this.summaryEmbedding = summaryEmbedding; + } + + public void setSummary(String summary) { + this.summary = summary; + } +} \ No newline at end of file diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java new file mode 100644 index 000000000..c03f3eb30 --- /dev/null +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java @@ -0,0 +1,101 @@ +/* + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. + */ +package com.microsoft.semantickernel.samples.documentationexamples.data.vectorstores.oracle; + +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.jdbc.oracle.OracleVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import oracle.jdbc.datasource.impl.OracleDataSource; +import reactor.core.publisher.Mono; + +public class Main { + public static void main(String[] args) throws SQLException { + + // Configure the data source + OracleDataSource dataSource = new OracleDataSource(); + dataSource.setURL("jdbc:oracle:thin:@localhost:1521/FREEPDB1"); + dataSource.setUser("scott"); + dataSource.setPassword("tiger"); + + // Build a query provider + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(dataSource) + .build(); + + // Build a vector store + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(dataSource) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + VectorStoreRecordCollection collection = vectorStore.getCollection( + "books", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Book.class) + .build()); + + // Create the collection if it doesn't exist yet. + collection.createCollectionIfNotExistsAsync().block(); + + collection.upsertBatchAsync(books, null).block(); + + // Retrieve the upserted record. + Book retrievedBook = collection.getAsync("2", null).block(); + + System.out.println(retrievedBook.getAuthor()); + + // Generate a vector for your search text, using your chosen embedding generation implementation. + // Just showing a placeholder method here for brevity. + List searchVector = generateEmbeddingsAsync( + "I'm looking for a horror book.").block(); + + // Do the search. + VectorSearchResults searchResult = collection.searchAsync( + searchVector, VectorSearchOptions.builder().withTop(1).build()).block(); + + retrievedBook = searchResult.getResults().get(0).getRecord(); + System.out.println("Found Book: " + retrievedBook.getIsbn()); + + } + + static List books = Arrays.asList( + new Book("1", "one", "sking", 0, null, "horror", List.of(1f, 1f)), + new Book("2", "two", "squeen", 0, null, "non-fiction", List.of(-1f, -1f))); + + private static Mono> generateEmbeddingsAsync(String text) { + return Mono.just(List.of(-0.1f, -0.1f)); + } + +} \ No newline at end of file diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml index 10fb9a30c..be88deb69 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml @@ -29,6 +29,10 @@ com.microsoft.semantic-kernel semantickernel-api + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + org.apache.logging.log4j diff --git a/semantickernel-api-ai-services/pom.xml b/semantickernel-api-ai-services/pom.xml new file mode 100644 index 000000000..9b4d76f67 --- /dev/null +++ b/semantickernel-api-ai-services/pom.xml @@ -0,0 +1,39 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../pom.xml + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + Semantic Kernel AI Services API + Defines the public interface for the Semantic Kernel Services + + + + com.google.code.findbugs + jsr305 + provided + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + 1 + + + + + + \ No newline at end of file diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/AIService.java b/semantickernel-api-ai-services/src/main/java/com/microsoft/semantickernel/services/AIService.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/services/AIService.java rename to semantickernel-api-ai-services/src/main/java/com/microsoft/semantickernel/services/AIService.java diff --git a/semantickernel-api-builders/pom.xml b/semantickernel-api-builders/pom.xml new file mode 100644 index 000000000..5bc0e11ba --- /dev/null +++ b/semantickernel-api-builders/pom.xml @@ -0,0 +1,30 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + + + com.microsoft.semantic-kernel + semantickernel-api-builders + Semantic Kernel Builders API + Defines the public interface for the Semantic Kernel Builders + + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + 1 + + + + + + \ No newline at end of file diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/builders/SemanticKernelBuilder.java b/semantickernel-api-builders/src/main/java/com/microsoft/semantickernel/builders/SemanticKernelBuilder.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/builders/SemanticKernelBuilder.java rename to semantickernel-api-builders/src/main/java/com/microsoft/semantickernel/builders/SemanticKernelBuilder.java diff --git a/semantickernel-api-data/pom.xml b/semantickernel-api-data/pom.xml new file mode 100644 index 000000000..822b7c574 --- /dev/null +++ b/semantickernel-api-data/pom.xml @@ -0,0 +1,67 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../pom.xml + + + com.microsoft.semantic-kernel + semantickernel-api-data + Semantic Kernel Data API + Defines the public interface for the Semantic Kernel Data + + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + provided + + + com.microsoft.semantic-kernel + semantickernel-api-builders + provided + + + com.microsoft.semantic-kernel + semantickernel-api-textembedding-services + provided + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + provided + + + com.fasterxml.jackson.core + jackson-databind + compile + + + com.fasterxml.jackson.core + jackson-core + compile + + + io.projectreactor + reactor-core + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + 1 + + + + + \ No newline at end of file diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearch.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearch.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearch.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearch.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearchOptions.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearchOptions.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearchOptions.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearchOptions.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/FilterMapping.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/filter/FilterMapping.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/FilterMapping.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/filter/FilterMapping.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchResultMapper.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchResultMapper.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchResultMapper.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchResultMapper.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchStringMapper.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchStringMapper.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchStringMapper.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchStringMapper.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/KernelSearchResults.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/KernelSearchResults.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/KernelSearchResults.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/KernelSearchResults.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearch.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearch.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearch.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearch.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchFilter.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchFilter.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchFilter.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchFilter.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchOptions.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchOptions.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchOptions.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchOptions.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResult.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResult.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResult.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResult.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultLink.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultLink.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultLink.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultLink.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultMapper.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultMapper.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultMapper.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultMapper.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultName.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultName.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultName.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultName.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultValue.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultValue.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultValue.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultValue.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchStringMapper.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchStringMapper.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchStringMapper.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchStringMapper.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResults.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResults.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResults.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResults.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java similarity index 88% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java index 9e5aea11a..713cae29f 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java +++ b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java @@ -39,6 +39,18 @@ public VectorStoreRecordDataField( this.isFullTextSearchable = isFullTextSearchable; } + public VectorStoreRecordDataField( + @Nonnull String name, + @Nullable String storageName, + @Nonnull Class fieldType, + @Nonnull Class fieldSubType, + boolean isFilterable, + boolean isFullTextSearchable) { + super(name, storageName, fieldType, fieldSubType); + this.isFilterable = isFilterable; + this.isFullTextSearchable = isFullTextSearchable; + } + /** * Gets a value indicating whether the field is filterable. * @@ -105,6 +117,7 @@ public VectorStoreRecordDataField build() { name, storageName, fieldType, + fieldSubType, isFilterable, isFullTextSearchable); } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java similarity index 93% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java index 54b2bf2b6..abb061112 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java +++ b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java @@ -7,6 +7,7 @@ import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; import com.microsoft.semantickernel.exceptions.SKException; import java.lang.reflect.Field; +import java.lang.reflect.ParameterizedType; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -193,7 +194,11 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) dataFields.add(VectorStoreRecordDataField.builder() .withName(field.getName()) .withStorageName(storageName) - .withFieldType(field.getType()) + .withFieldType(field.getType(), + List.class.equals(field.getType()) + ? (Class) ((ParameterizedType) field.getGenericType()) + .getActualTypeArguments()[0] + : null) .isFilterable(dataAttribute.isFilterable()) .build()); } @@ -209,7 +214,11 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) vectorFields.add(VectorStoreRecordVectorField.builder() .withName(field.getName()) .withStorageName(storageName) - .withFieldType(field.getType()) + .withFieldType(field.getType(), + List.class.equals(field.getType()) + ? (Class) ((ParameterizedType) field.getGenericType()) + .getActualTypeArguments()[0] + : null) .withDimensions(vectorAttribute.dimensions()) .withIndexKind(vectorAttribute.indexKind()) .withDistanceFunction(vectorAttribute.distanceFunction()) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java similarity index 74% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java index 0ba377af8..f777bd5c1 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java +++ b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java @@ -14,6 +14,7 @@ public class VectorStoreRecordField { @Nullable private final String storageName; private final Class fieldType; + private final Class fieldSubType; /** * Creates a new instance of the VectorStoreRecordField class. @@ -29,6 +30,27 @@ public VectorStoreRecordField( this.name = name; this.storageName = storageName; this.fieldType = fieldType; + this.fieldSubType = null; + } + + /** + * Creates a new instance of the VectorStoreRecordField class. + * + * @param name the name of the field + * @param storageName the storage name of the field + * @param fieldType the field type + * @param fieldSubType if the field type is a list, the type of + * the list elements, otherwise null + */ + public VectorStoreRecordField( + @Nonnull String name, + @Nullable String storageName, + @Nonnull Class fieldType, + @Nonnull Class fieldSubType) { + this.name = name; + this.storageName = storageName; + this.fieldType = fieldType; + this.fieldSubType = fieldSubType; } /** @@ -68,6 +90,10 @@ public Class getFieldType() { return fieldType; } + public Class getFieldSubType() { + return fieldSubType; + } + /** * A builder for the VectorStoreRecordField class. * @param the type of the field @@ -83,6 +109,9 @@ public abstract static class Builder> @Nullable protected Class fieldType; + @Nullable + protected Class fieldSubType; + /** * Sets the name of the field. * @@ -116,6 +145,12 @@ public U withFieldType(Class fieldType) { return (U) this; } + public U withFieldType(Class fieldType, Class fieldSubType) { + this.fieldType = fieldType; + this.fieldSubType = fieldSubType; + return (U) this; + } + /** * Builds the field. * diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java similarity index 96% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java index 00b7627ac..b708d2fbb 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java +++ b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java @@ -34,10 +34,11 @@ public VectorStoreRecordVectorField( @Nonnull String name, @Nullable String storageName, @Nonnull Class fieldType, + Class fieldSubType, int dimensions, @Nullable IndexKind indexKind, @Nullable DistanceFunction distanceFunction) { - super(name, storageName, fieldType); + super(name, storageName, fieldType, fieldSubType); this.dimensions = dimensions; this.indexKind = indexKind == null ? IndexKind.UNDEFINED : indexKind; this.distanceFunction = distanceFunction == null ? DistanceFunction.UNDEFINED @@ -130,7 +131,8 @@ public VectorStoreRecordVectorField build() { throw new IllegalArgumentException("dimensions must be greater than 0"); } - return new VectorStoreRecordVectorField(name, storageName, fieldType, dimensions, + return new VectorStoreRecordVectorField(name, storageName, fieldType, fieldSubType, + dimensions, indexKind, distanceFunction); } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/DeleteRecordOptions.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/DeleteRecordOptions.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/DeleteRecordOptions.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/DeleteRecordOptions.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/UpsertRecordOptions.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/UpsertRecordOptions.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/UpsertRecordOptions.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/UpsertRecordOptions.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java diff --git a/semantickernel-api-exceptions/pom.xml b/semantickernel-api-exceptions/pom.xml new file mode 100644 index 000000000..b459db13e --- /dev/null +++ b/semantickernel-api-exceptions/pom.xml @@ -0,0 +1,44 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../pom.xml + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + Semantic Kernel Exceptions API + Defines the public interface for the Semantic Kernel Exceptions + + + + com.microsoft.semantic-kernel + semantickernel-api-localization + provided + + + com.google.code.findbugs + jsr305 + provided + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + 1 + + + + + + \ No newline at end of file diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/AIException.java b/semantickernel-api-exceptions/src/main/java/com/microsoft/semantickernel/exceptions/AIException.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/AIException.java rename to semantickernel-api-exceptions/src/main/java/com/microsoft/semantickernel/exceptions/AIException.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/ConfigurationException.java b/semantickernel-api-exceptions/src/main/java/com/microsoft/semantickernel/exceptions/ConfigurationException.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/ConfigurationException.java rename to semantickernel-api-exceptions/src/main/java/com/microsoft/semantickernel/exceptions/ConfigurationException.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKCheckedException.java b/semantickernel-api-exceptions/src/main/java/com/microsoft/semantickernel/exceptions/SKCheckedException.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKCheckedException.java rename to semantickernel-api-exceptions/src/main/java/com/microsoft/semantickernel/exceptions/SKCheckedException.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKException.java b/semantickernel-api-exceptions/src/main/java/com/microsoft/semantickernel/exceptions/SKException.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKException.java rename to semantickernel-api-exceptions/src/main/java/com/microsoft/semantickernel/exceptions/SKException.java diff --git a/semantickernel-api-localization/pom.xml b/semantickernel-api-localization/pom.xml new file mode 100644 index 000000000..84be7960d --- /dev/null +++ b/semantickernel-api-localization/pom.xml @@ -0,0 +1,31 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../pom.xml + + + com.microsoft.semantic-kernel + semantickernel-api-localization + Semantic Kernel Localization API + Defines the public interface for the Semantic Kernel Localization + + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + 1 + + + + + + \ No newline at end of file diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/localization/SemanticKernelResources.java b/semantickernel-api-localization/src/main/java/com/microsoft/semantickernel/localization/SemanticKernelResources.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/localization/SemanticKernelResources.java rename to semantickernel-api-localization/src/main/java/com/microsoft/semantickernel/localization/SemanticKernelResources.java diff --git a/semantickernel-api/src/main/resources/com/microsoft/semantickernel/localization/ResourceBundle.properties b/semantickernel-api-localization/src/main/resources/com/microsoft/semantickernel/localization/ResourceBundle.properties similarity index 100% rename from semantickernel-api/src/main/resources/com/microsoft/semantickernel/localization/ResourceBundle.properties rename to semantickernel-api-localization/src/main/resources/com/microsoft/semantickernel/localization/ResourceBundle.properties diff --git a/semantickernel-api-textembedding-services/pom.xml b/semantickernel-api-textembedding-services/pom.xml new file mode 100644 index 000000000..48a524eda --- /dev/null +++ b/semantickernel-api-textembedding-services/pom.xml @@ -0,0 +1,51 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../pom.xml + + + com.microsoft.semantic-kernel + semantickernel-api-textembedding-services + Semantic Kernel Services API + Defines the public interface for the Semantic Kernel Services + + + + io.projectreactor + reactor-core + + + com.google.code.findbugs + jsr305 + provided + + + com.github.spotbugs + spotbugs-annotations + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + 1 + + + + + + \ No newline at end of file diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/Embedding.java b/semantickernel-api-textembedding-services/src/main/java/com/microsoft/semantickernel/services/textembedding/Embedding.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/Embedding.java rename to semantickernel-api-textembedding-services/src/main/java/com/microsoft/semantickernel/services/textembedding/Embedding.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/EmbeddingGenerationService.java b/semantickernel-api-textembedding-services/src/main/java/com/microsoft/semantickernel/services/textembedding/EmbeddingGenerationService.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/EmbeddingGenerationService.java rename to semantickernel-api-textembedding-services/src/main/java/com/microsoft/semantickernel/services/textembedding/EmbeddingGenerationService.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/TextEmbeddingGenerationService.java b/semantickernel-api-textembedding-services/src/main/java/com/microsoft/semantickernel/services/textembedding/TextEmbeddingGenerationService.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/TextEmbeddingGenerationService.java rename to semantickernel-api-textembedding-services/src/main/java/com/microsoft/semantickernel/services/textembedding/TextEmbeddingGenerationService.java diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index e10ed0975..0911e7055 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -1,6 +1,7 @@ - + 4.0.0 @@ -15,6 +16,30 @@ Semantic Kernel API Defines the public interface for the Semantic Kernel + + com.microsoft.semantic-kernel + semantickernel-api-data + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-builders + + + com.microsoft.semantic-kernel + semantickernel-api-localization + + + com.microsoft.semantic-kernel + semantickernel-api-textembedding-services + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + io.opentelemetry.instrumentation opentelemetry-reactor-3.1 @@ -103,6 +128,45 @@ 1 + + org.apache.maven.plugins + maven-dependency-plugin + + + unpack-dependencies + process-resources + + unpack-dependencies + + + + semantickernel-api-exceptions,semantickernel-api-builders,semantickernel-api-localization,semantickernel-api-textembedding-services,semantickernel-api-ai-services,semantickernel-api-data + + ${project.build.directory}/lib + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.6.0 + + + src/assembly/custom.xml + + false + + + + make-assembly + package + + single + + + + diff --git a/semantickernel-api/src/assembly/custom.xml b/semantickernel-api/src/assembly/custom.xml new file mode 100644 index 000000000..48becfffa --- /dev/null +++ b/semantickernel-api/src/assembly/custom.xml @@ -0,0 +1,22 @@ + + with-exceptions + + jar + + false + + + ${project.build.outputDirectory} + / + + + ${project.build.directory}/lib + / + + **/** + + + + \ No newline at end of file diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index ad6d7180e..990eb4339 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -1,6 +1,7 @@ - + 4.0.0 com.microsoft.semantic-kernel @@ -106,6 +107,26 @@ semantickernel-data-jdbc ${project.version} + + com.microsoft.semantic-kernel + semantickernel-data-mysql + ${project.version} + + + com.microsoft.semantic-kernel + semantickernel-data-sqlite + ${project.version} + + + com.microsoft.semantic-kernel + semantickernel-data-hsqldb + ${project.version} + + + com.microsoft.semantic-kernel + semantickernel-data-oracle + ${project.version} + com.microsoft.semantic-kernel semantickernel-data-redis @@ -232,6 +253,11 @@ 4.36.0 true + + io.projectreactor + reactor-core + 3.7.8 + diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 98ce0918b..538c9498c 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -88,6 +88,11 @@ com.microsoft.semantic-kernel semantickernel-api + + com.microsoft.semantic-kernel + semantickernel-api-builders + provided +