Skip to content

Commit e86ce73

Browse files
dev-jonghoonparkdev-jonghoonpark
authored
Make the meta field prefix configurable in Weaviate vector store properties (#3585)
Signed-off-by: jonghoonpark <[email protected]>
1 parent a9fc130 commit e86ce73

File tree

10 files changed

+152
-9
lines changed

10 files changed

+152
-9
lines changed

‎auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-weaviate/src/main/java/org/springframework/ai/vectorstore/weaviate/autoconfigure/WeaviateVectorStoreAutoConfiguration.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ WeaviateVectorStoreOptions mappingPropertiesToOptions(WeaviateVectorStorePropert
105105
PropertyMapper mapper = PropertyMapper.get();
106106
mapper.from(properties::getContentFieldName).whenHasText().to(weaviateVectorStoreOptions::setContentFieldName);
107107
mapper.from(properties::getObjectClass).whenHasText().to(weaviateVectorStoreOptions::setObjectClass);
108+
mapper.from(properties::getMetaFieldPrefix).whenHasText().to(weaviateVectorStoreOptions::setMetaFieldPrefix);
108109

109110
return weaviateVectorStoreOptions;
110111
}

‎auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-weaviate/src/main/java/org/springframework/ai/vectorstore/weaviate/autoconfigure/WeaviateVectorStoreProperties.java

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ public class WeaviateVectorStoreProperties {
4444

4545
private String contentFieldName = "content";
4646

47+
private String metaFieldPrefix = "meta_";
48+
4749
private ConsistentLevel consistencyLevel = WeaviateVectorStore.ConsistentLevel.ONE;
4850

4951
/**
@@ -99,6 +101,20 @@ public void setContentFieldName(String contentFieldName) {
99101
this.contentFieldName = contentFieldName;
100102
}
101103

104+
/**
105+
* @since 1.1.0
106+
*/
107+
public String getMetaFieldPrefix() {
108+
return metaFieldPrefix;
109+
}
110+
111+
/**
112+
* @since 1.1.0
113+
*/
114+
public void setMetaFieldPrefix(String metaFieldPrefix) {
115+
this.metaFieldPrefix = metaFieldPrefix;
116+
}
117+
102118
public ConsistentLevel getConsistencyLevel() {
103119
return this.consistencyLevel;
104120
}

‎auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-weaviate/src/test/java/org/springframework/ai/vectorstore/weaviate/autoconfigure/WeaviateVectorStoreAutoConfigurationIT.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,8 @@ public void autoConfigurationEnabledWhenTypeIsWeaviate() {
180180
public void testMappingPropertiesToOptions() {
181181
this.contextRunner
182182
.withPropertyValues("spring.ai.vectorstore.weaviate.object-class=CustomObjectClass",
183-
"spring.ai.vectorstore.weaviate.content-field-name=customContentFieldName")
183+
"spring.ai.vectorstore.weaviate.content-field-name=customContentFieldName",
184+
"spring.ai.vectorstore.weaviate.meta-field-prefix=custom_")
184185
.run(context -> {
185186
WeaviateVectorStoreAutoConfiguration autoConfiguration = context
186187
.getBean(WeaviateVectorStoreAutoConfiguration.class);
@@ -189,6 +190,7 @@ public void testMappingPropertiesToOptions() {
189190

190191
assertThat(options.getObjectClass()).isEqualTo("CustomObjectClass");
191192
assertThat(options.getContentFieldName()).isEqualTo("customContentFieldName");
193+
assertThat(options.getMetaFieldPrefix()).isEqualTo("custom_");
192194
});
193195
}
194196

‎spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs/weaviate.adoc

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,7 @@ You can use the following properties in your Spring Boot configuration to custom
264264
|`spring.ai.vectorstore.weaviate.api-key`|The API key for authentication|
265265
|`spring.ai.vectorstore.weaviate.object-class`|The class name for storing documents. |SpringAiWeaviate
266266
|`spring.ai.vectorstore.weaviate.content-field-name`|The field name for content|content
267+
|`spring.ai.vectorstore.weaviate.meta-field-prefix`|The field prefix for metadata|meta_
267268
|`spring.ai.vectorstore.weaviate.consistency-level`|Desired tradeoff between consistency and speed|ConsistentLevel.ONE
268269
|`spring.ai.vectorstore.weaviate.filter-field`|Configures metadata fields that can be used in filters. Format: spring.ai.vectorstore.weaviate.filter-field.<field-name>=<field-type>|
269270
|===

‎vector-stores/spring-ai-weaviate-store/src/main/java/org/springframework/ai/vectorstore/weaviate/WeaviateFilterExpressionConverter.java

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2023-2024 the original author or authors.
2+
* Copyright 2023-2025 the original author or authors.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -35,19 +35,42 @@
3535
* (https://weaviate.io/developers/weaviate/api/graphql/filters)
3636
*
3737
* @author Christian Tzolov
38+
* @author Jonghoon Park
3839
*/
3940
public class WeaviateFilterExpressionConverter extends AbstractFilterExpressionConverter {
4041

4142
// https://weaviate.io/developers/weaviate/api/graphql/filters#special-cases
4243
private static final List<String> SYSTEM_IDENTIFIERS = List.of("id", "_creationTimeUnix", "_lastUpdateTimeUnix");
4344

45+
private static final String DEFAULT_META_FIELD_PREFIX = "meta_";
46+
4447
private boolean mapIntegerToNumberValue = true;
4548

4649
private List<String> allowedIdentifierNames;
4750

51+
private final String metaFieldPrefix;
52+
53+
/**
54+
* Constructs a new instance of the {@code WeaviateFilterExpressionConverter} class.
55+
* This constructor uses the default meta field prefix
56+
* ({@link #DEFAULT_META_FIELD_PREFIX}).
57+
* @param allowedIdentifierNames A {@code List} of allowed identifier names.
58+
*/
4859
public WeaviateFilterExpressionConverter(List<String> allowedIdentifierNames) {
60+
this(allowedIdentifierNames, DEFAULT_META_FIELD_PREFIX);
61+
}
62+
63+
/**
64+
* Constructs a new instance of the {@code WeaviateFilterExpressionConverter} class.
65+
* @param allowedIdentifierNames A {@code List} of allowed identifier names.
66+
* @param metaFieldPrefix the prefix for meta fields
67+
* @since 1.1.0
68+
*/
69+
public WeaviateFilterExpressionConverter(List<String> allowedIdentifierNames, String metaFieldPrefix) {
4970
Assert.notNull(allowedIdentifierNames, "List can be empty but not null.");
71+
Assert.notNull(metaFieldPrefix, "metaFieldPrefix can be empty but not null.");
5072
this.allowedIdentifierNames = allowedIdentifierNames;
73+
this.metaFieldPrefix = metaFieldPrefix;
5174
}
5275

5376
public void setAllowedIdentifierNames(List<String> allowedIdentifierNames) {
@@ -112,7 +135,7 @@ public String withMetaPrefix(String identifier) {
112135
}
113136

114137
if (this.allowedIdentifierNames.contains(identifier)) {
115-
return "meta_" + identifier;
138+
return this.metaFieldPrefix + identifier;
116139
}
117140

118141
throw new IllegalArgumentException("Not allowed filter identifier name: " + identifier

‎vector-stores/spring-ai-weaviate-store/src/main/java/org/springframework/ai/vectorstore/weaviate/WeaviateVectorStore.java

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -96,8 +96,6 @@ public class WeaviateVectorStore extends AbstractObservationVectorStore {
9696

9797
private static final Logger logger = LoggerFactory.getLogger(WeaviateVectorStore.class);
9898

99-
private static final String METADATA_FIELD_PREFIX = "meta_";
100-
10199
private static final String METADATA_FIELD_NAME = "metadata";
102100

103101
private static final String ADDITIONAL_FIELD_NAME = "_additional";
@@ -162,7 +160,8 @@ protected WeaviateVectorStore(Builder builder) {
162160
this.consistencyLevel = builder.consistencyLevel;
163161
this.filterMetadataFields = builder.filterMetadataFields;
164162
this.filterExpressionConverter = new WeaviateFilterExpressionConverter(
165-
this.filterMetadataFields.stream().map(MetadataField::name).toList());
163+
this.filterMetadataFields.stream().map(MetadataField::name).toList(),
164+
this.options.getMetaFieldPrefix());
166165
this.weaviateSimilaritySearchFields = buildWeaviateSimilaritySearchFields();
167166
}
168167

@@ -182,7 +181,7 @@ private Field[] buildWeaviateSimilaritySearchFields() {
182181
searchWeaviateFieldList.add(Field.builder().name(this.options.getContentFieldName()).build());
183182
searchWeaviateFieldList.add(Field.builder().name(METADATA_FIELD_NAME).build());
184183
searchWeaviateFieldList.addAll(this.filterMetadataFields.stream()
185-
.map(mf -> Field.builder().name(METADATA_FIELD_PREFIX + mf.name()).build())
184+
.map(mf -> Field.builder().name(this.options.getMetaFieldPrefix() + mf.name()).build())
186185
.toList());
187186
searchWeaviateFieldList.add(Field.builder()
188187
.name(ADDITIONAL_FIELD_NAME)
@@ -260,7 +259,7 @@ private WeaviateObject toWeaviateObject(Document document, List<Document> docume
260259
// expressions on them.
261260
for (MetadataField mf : this.filterMetadataFields) {
262261
if (document.getMetadata().containsKey(mf.name())) {
263-
fields.put(METADATA_FIELD_PREFIX + mf.name(), document.getMetadata().get(mf.name()));
262+
fields.put(this.options.getMetaFieldPrefix() + mf.name(), document.getMetadata().get(mf.name()));
264263
}
265264
}
266265

‎vector-stores/spring-ai-weaviate-store/src/main/java/org/springframework/ai/vectorstore/weaviate/WeaviateVectorStoreOptions.java

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,16 @@
2222
* Provided Weaviate vector option configuration.
2323
*
2424
* @author Jonghoon Park
25-
* @since 1.1.0.
25+
* @since 1.1.0
2626
*/
2727
public class WeaviateVectorStoreOptions {
2828

2929
private String objectClass = "SpringAiWeaviate";
3030

3131
private String contentFieldName = "content";
3232

33+
private String metaFieldPrefix = "meta_";
34+
3335
public String getObjectClass() {
3436
return objectClass;
3537
}
@@ -48,4 +50,13 @@ public void setContentFieldName(String contentFieldName) {
4850
this.contentFieldName = contentFieldName;
4951
}
5052

53+
public String getMetaFieldPrefix() {
54+
return metaFieldPrefix;
55+
}
56+
57+
public void setMetaFieldPrefix(String metaFieldPrefix) {
58+
Assert.notNull(metaFieldPrefix, "metaFieldPrefix can be empty but not null");
59+
this.metaFieldPrefix = metaFieldPrefix;
60+
}
61+
5162
}

‎vector-stores/spring-ai-weaviate-store/src/test/java/org/springframework/ai/vectorstore/weaviate/WeaviateVectorStoreBuilderTests.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ void shouldBuildWithCustomConfiguration() {
6060
WeaviateVectorStoreOptions options = new WeaviateVectorStoreOptions();
6161
options.setObjectClass("CustomObjectClass");
6262
options.setContentFieldName("customContentFieldName");
63+
options.setMetaFieldPrefix("custom_");
6364

6465
WeaviateVectorStore vectorStore = WeaviateVectorStore.builder(weaviateClient, this.embeddingModel)
6566
.options(options)

‎vector-stores/spring-ai-weaviate-store/src/test/java/org/springframework/ai/vectorstore/weaviate/WeaviateVectorStoreIT.java

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@
2828
import io.weaviate.client.Config;
2929
import io.weaviate.client.WeaviateClient;
3030
import org.junit.jupiter.api.Test;
31+
import org.junit.jupiter.params.ParameterizedTest;
32+
import org.junit.jupiter.params.provider.ValueSource;
3133
import org.testcontainers.containers.wait.strategy.Wait;
3234
import org.testcontainers.junit.jupiter.Container;
3335
import org.testcontainers.junit.jupiter.Testcontainers;
@@ -359,6 +361,77 @@ public void addAndSearchWithCustomContentFieldName() {
359361
});
360362
}
361363

364+
@ParameterizedTest(name = "{0} : {displayName} ")
365+
@ValueSource(strings = { "custom_", "" })
366+
public void addAndSearchWithCustomMetaFieldPrefix(String metaFieldPrefix) {
367+
WeaviateVectorStoreOptions optionsWithCustomContentFieldName = new WeaviateVectorStoreOptions();
368+
optionsWithCustomContentFieldName.setMetaFieldPrefix(metaFieldPrefix);
369+
370+
this.contextRunner.run(context -> {
371+
VectorStore vectorStore = context.getBean(VectorStore.class);
372+
resetCollection(vectorStore);
373+
});
374+
375+
this.contextRunner.run(context -> {
376+
WeaviateClient weaviateClient = context.getBean(WeaviateClient.class);
377+
EmbeddingModel embeddingModel = context.getBean(EmbeddingModel.class);
378+
379+
VectorStore customVectorStore = WeaviateVectorStore.builder(weaviateClient, embeddingModel)
380+
.filterMetadataFields(List.of(WeaviateVectorStore.MetadataField.text("country")))
381+
.options(optionsWithCustomContentFieldName)
382+
.build();
383+
384+
var bgDocument = new Document("The World is Big and Salvation Lurks Around the Corner",
385+
Map.of("country", "BG", "year", 2020));
386+
var nlDocument = new Document("The World is Big and Salvation Lurks Around the Corner",
387+
Map.of("country", "NL"));
388+
var bgDocument2 = new Document("The World is Big and Salvation Lurks Around the Corner",
389+
Map.of("country", "BG", "year", 2023));
390+
391+
customVectorStore.add(List.of(bgDocument, nlDocument, bgDocument2));
392+
393+
List<Document> results = customVectorStore
394+
.similaritySearch(SearchRequest.builder().query("The World").topK(5).build());
395+
assertThat(results).hasSize(3);
396+
397+
results = customVectorStore.similaritySearch(SearchRequest.builder()
398+
.query("The World")
399+
.topK(5)
400+
.similarityThresholdAll()
401+
.filterExpression("country == 'NL'")
402+
.build());
403+
assertThat(results).hasSize(1);
404+
assertThat(results.get(0).getId()).isEqualTo(nlDocument.getId());
405+
});
406+
407+
this.contextRunner.run(context -> {
408+
VectorStore vectorStore = context.getBean(VectorStore.class);
409+
List<Document> results = vectorStore.similaritySearch(SearchRequest.builder()
410+
.query("The World")
411+
.topK(5)
412+
.similarityThresholdAll()
413+
.filterExpression("country == 'NL'")
414+
.build());
415+
assertThat(results).hasSize(0);
416+
});
417+
418+
// remove documents for parameterized test
419+
this.contextRunner.run(context -> {
420+
WeaviateClient weaviateClient = context.getBean(WeaviateClient.class);
421+
EmbeddingModel embeddingModel = context.getBean(EmbeddingModel.class);
422+
423+
VectorStore customVectorStore = WeaviateVectorStore.builder(weaviateClient, embeddingModel)
424+
.filterMetadataFields(List.of(WeaviateVectorStore.MetadataField.text("country")))
425+
.options(optionsWithCustomContentFieldName)
426+
.build();
427+
428+
List<Document> results = customVectorStore
429+
.similaritySearch(SearchRequest.builder().query("The World").topK(5).build());
430+
431+
customVectorStore.delete(results.stream().map(Document::getId).toList());
432+
});
433+
}
434+
362435
@SpringBootConfiguration
363436
@EnableAutoConfiguration
364437
public static class TestApplication {

‎vector-stores/spring-ai-weaviate-store/src/test/java/org/springframework/ai/vectorstore/weaviate/WeaviateVectorStoreOptionsTests.java

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
import org.junit.jupiter.api.Test;
2020

21+
import static org.assertj.core.api.Assertions.assertThat;
2122
import static org.assertj.core.api.Assertions.assertThatThrownBy;
2223

2324
/**
@@ -66,4 +67,19 @@ void shouldFailWithEmptyContentFieldName() {
6667
.hasMessage("contentFieldName cannot be null or empty");
6768
}
6869

70+
@Test
71+
void shouldFailWithNullMetaFieldPrefix() {
72+
WeaviateVectorStoreOptions options = new WeaviateVectorStoreOptions();
73+
74+
assertThatThrownBy(() -> options.setMetaFieldPrefix(null)).isInstanceOf(IllegalArgumentException.class)
75+
.hasMessage("metaFieldPrefix can be empty but not null");
76+
}
77+
78+
@Test
79+
void shouldPassWithEmptyMetaFieldPrefix() {
80+
WeaviateVectorStoreOptions options = new WeaviateVectorStoreOptions();
81+
options.setMetaFieldPrefix("");
82+
assertThat(options.getMetaFieldPrefix()).isEqualTo("");
83+
}
84+
6985
}

0 commit comments

Comments
 (0)