Skip to content

Commit 790be7d

Browse files
TyCodingTyCoding
committed
fix: 修复超时、向量化等一些问题，更新依赖
1 parent 9d9e9d9 commit 790be7d

File tree

25 files changed

+226
-274
lines changed

25 files changed

+226
-274
lines changed

‎langchat-ai/langchat-ai-core/pom.xml‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -63,12 +63,12 @@
6363
</dependency>
6464
<dependency>
6565
<groupId>dev.langchain4j</groupId>
66-
<artifactId>langchain4j-qianfan</artifactId>
66+
<artifactId>langchain4j-community-qianfan</artifactId>
6767
<version>${langchain4j.version}</version>
6868
</dependency>
6969
<dependency>
7070
<groupId>dev.langchain4j</groupId>
71-
<artifactId>langchain4j-dashscope</artifactId>
71+
<artifactId>langchain4j-community-dashscope</artifactId>
7272
<version>${langchain4j.version}</version>
7373
<exclusions>
7474
<exclusion>
@@ -79,7 +79,7 @@
7979
</dependency>
8080
<dependency>
8181
<groupId>dev.langchain4j</groupId>
82-
<artifactId>langchain4j-zhipu-ai</artifactId>
82+
<artifactId>langchain4j-community-zhipu-ai</artifactId>
8383
<version>${langchain4j.version}</version>
8484
</dependency>
8585
<dependency>
@@ -89,7 +89,7 @@
8989
</dependency>
9090
<dependency>
9191
<groupId>dev.langchain4j</groupId>
92-
<artifactId>langchain4j-redis</artifactId>
92+
<artifactId>langchain4j-community-redis</artifactId>
9393
<version>${langchain4j.version}</version>
9494
</dependency>
9595
<dependency>
@@ -102,11 +102,6 @@
102102
<artifactId>langchain4j-milvus</artifactId>
103103
<version>${langchain4j.version}</version>
104104
</dependency>
105-
<dependency>
106-
<groupId>dev.langchain4j</groupId>
107-
<artifactId>langchain4j-elasticsearch</artifactId>
108-
<version>${langchain4j.version}</version>
109-
</dependency>
110105
<dependency>
111106
<groupId>dev.langchain4j</groupId>
112107
<artifactId>langchain4j-embedding-store-filter-parser-sql</artifactId>

‎langchat-ai/langchat-ai-core/src/main/java/cn/tycoding/langchat/ai/core/consts/EmbedStoreEnum.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,5 @@ public enum EmbedStoreEnum {
2828
REDIS,
2929
PGVECTOR,
3030
MILVUS,
31-
ELASTICSEARCH,
3231
;
3332
}

‎langchat-ai/langchat-ai-core/src/main/java/cn/tycoding/langchat/ai/core/provider/EmbeddingProvider.java‎

Lines changed: 19 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -17,17 +17,14 @@
1717
package cn.tycoding.langchat.ai.core.provider;
1818

1919
import cn.tycoding.langchat.ai.biz.entity.AigcKnowledge;
20-
import cn.tycoding.langchat.ai.core.consts.ProviderEnum;
2120
import cn.tycoding.langchat.common.core.exception.ServiceException;
2221
import dev.langchain4j.data.document.DocumentSplitter;
2322
import dev.langchain4j.data.document.splitter.DocumentSplitters;
2423
import dev.langchain4j.data.segment.TextSegment;
2524
import dev.langchain4j.model.embedding.EmbeddingModel;
26-
import dev.langchain4j.model.openai.OpenAiTokenizer;
2725
import dev.langchain4j.store.embedding.EmbeddingStore;
2826
import lombok.AllArgsConstructor;
2927
import lombok.extern.slf4j.Slf4j;
30-
import org.springframework.context.ApplicationContext;
3128
import org.springframework.stereotype.Component;
3229

3330
import java.util.ArrayList;
@@ -43,20 +40,19 @@
4340
@AllArgsConstructor
4441
public class EmbeddingProvider {
4542

46-
private final ApplicationContext context;
43+
private final EmbeddingStoreFactory embeddingStoreFactory;
44+
private final KnowledgeStoreFactory knowledgeStoreFactory;
45+
private final ModelStoreFactory modelStoreFactory;
4746

48-
public static DocumentSplitter splitter(String modelName, String modelProvider) {
49-
if (ProviderEnum.OPENAI.name().equals(modelProvider)) {
50-
return DocumentSplitters.recursive(100, 0, new OpenAiTokenizer(modelName));
51-
}
52-
return DocumentSplitters.recursive(100, 0);
47+
public static DocumentSplitter splitter() {
48+
return DocumentSplitters.recursive(300, 20);
5349
}
5450

5551
public EmbeddingModel getEmbeddingModel(List<String> knowledgeIds) {
5652
List<String> storeIds = new ArrayList<>();
5753
knowledgeIds.forEach(id -> {
58-
if (context.containsBean(id)) {
59-
AigcKnowledge data = (AigcKnowledge) context.getBean(id);
54+
if (knowledgeStoreFactory.containsKnowledge(id)) {
55+
AigcKnowledge data = knowledgeStoreFactory.getKnowledge(id);
6056
if (data.getEmbedModelId() != null) {
6157
storeIds.add(data.getEmbedModelId());
6258
}
@@ -71,24 +67,24 @@ public EmbeddingModel getEmbeddingModel(List<String> knowledgeIds) {
7167
throw new ServiceException("存在多个不同Embedding Model的知识库，请先检查配置");
7268
}
7369

74-
return (EmbeddingModel) context.getBean(storeIds.get(0));
70+
return modelStoreFactory.getEmbeddingModel(storeIds.get(0));
7571
}
7672

7773
public EmbeddingModel getEmbeddingModel(String knowledgeId) {
78-
if (context.containsBean(knowledgeId)) {
79-
AigcKnowledge data = (AigcKnowledge) context.getBean(knowledgeId);
80-
if (context.containsBean(data.getEmbedModelId())) {
81-
return (EmbeddingModel) context.getBean(data.getEmbedModelId());
74+
if (knowledgeStoreFactory.containsKnowledge(knowledgeId)) {
75+
AigcKnowledge data = knowledgeStoreFactory.getKnowledge(knowledgeId);
76+
if (modelStoreFactory.containsEmbeddingModel(data.getEmbedModelId())) {
77+
return modelStoreFactory.getEmbeddingModel(data.getEmbedModelId());
8278
}
8379
}
8480
throw new ServiceException("没有找到匹配的Embedding向量数据库");
8581
}
8682

8783
public EmbeddingStore<TextSegment> getEmbeddingStore(String knowledgeId) {
88-
if (context.containsBean(knowledgeId)) {
89-
AigcKnowledge data = (AigcKnowledge) context.getBean(knowledgeId);
90-
if (context.containsBean(data.getEmbedStoreId())) {
91-
return (EmbeddingStore<TextSegment>) context.getBean(data.getEmbedStoreId());
84+
if (knowledgeStoreFactory.containsKnowledge(knowledgeId)) {
85+
AigcKnowledge data = knowledgeStoreFactory.getKnowledge(knowledgeId);
86+
if (embeddingStoreFactory.containsEmbeddingStore(data.getEmbedStoreId())) {
87+
return embeddingStoreFactory.getEmbeddingStore(data.getEmbedStoreId());
9288
}
9389
}
9490
throw new ServiceException("没有找到匹配的Embedding向量数据库");
@@ -97,8 +93,8 @@ public EmbeddingStore<TextSegment> getEmbeddingStore(String knowledgeId) {
9793
public EmbeddingStore<TextSegment> getEmbeddingStore(List<String> knowledgeIds) {
9894
List<String> storeIds = new ArrayList<>();
9995
knowledgeIds.forEach(id -> {
100-
if (context.containsBean(id)) {
101-
AigcKnowledge data = (AigcKnowledge) context.getBean(id);
96+
if (knowledgeStoreFactory.containsKnowledge(id)) {
97+
AigcKnowledge data = knowledgeStoreFactory.getKnowledge(id);
10298
if (data.getEmbedStoreId() != null) {
10399
storeIds.add(data.getEmbedStoreId());
104100
}
@@ -113,7 +109,7 @@ public EmbeddingStore<TextSegment> getEmbeddingStore(List<String> knowledgeIds)
113109
throw new ServiceException("存在多个不同Embedding Store数据源的知识库，请先检查配置");
114110
}
115111

116-
return (EmbeddingStore<TextSegment>) context.getBean(storeIds.get(0));
112+
return embeddingStoreFactory.getEmbeddingStore(storeIds.get(0));
117113
}
118114

119115
}

‎langchat-ai/langchat-ai-core/src/main/java/cn/tycoding/langchat/ai/core/provider/EmbeddingStoreInitialize.java‎ renamed to ‎langchat-ai/langchat-ai-core/src/main/java/cn/tycoding/langchat/ai/core/provider/EmbeddingStoreFactory.java‎

Lines changed: 32 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -20,46 +20,40 @@
2020
import cn.tycoding.langchat.ai.biz.entity.AigcEmbedStore;
2121
import cn.tycoding.langchat.ai.biz.service.AigcEmbedStoreService;
2222
import cn.tycoding.langchat.ai.core.consts.EmbedStoreEnum;
23-
import cn.tycoding.langchat.common.core.component.SpringContextHolder;
24-
import dev.langchain4j.store.embedding.elasticsearch.ElasticsearchEmbeddingStore;
23+
import dev.langchain4j.community.store.embedding.redis.RedisEmbeddingStore;
24+
import dev.langchain4j.data.segment.TextSegment;
25+
import dev.langchain4j.store.embedding.EmbeddingStore;
2526
import dev.langchain4j.store.embedding.milvus.MilvusEmbeddingStore;
2627
import dev.langchain4j.store.embedding.pgvector.PgVectorEmbeddingStore;
27-
import dev.langchain4j.store.embedding.redis.RedisEmbeddingStore;
28-
import lombok.AllArgsConstructor;
28+
import jakarta.annotation.PostConstruct;
2929
import lombok.extern.slf4j.Slf4j;
30-
import org.apache.http.Header;
31-
import org.apache.http.HttpHost;
32-
import org.apache.http.message.BasicHeader;
33-
import org.elasticsearch.client.RestClient;
34-
import org.springframework.beans.BeansException;
35-
import org.springframework.context.ApplicationContext;
36-
import org.springframework.context.ApplicationContextAware;
30+
import org.springframework.beans.factory.annotation.Autowired;
31+
import org.springframework.scheduling.annotation.Async;
3732
import org.springframework.stereotype.Component;
3833

3934
import java.util.ArrayList;
4035
import java.util.List;
36+
import java.util.Map;
37+
import java.util.concurrent.ConcurrentHashMap;
4138

4239
/**
4340
* @author tycoding
4441
* @since 2024/10/28
4542
*/
4643
@Slf4j
4744
@Component
48-
@AllArgsConstructor
49-
public class EmbeddingStoreInitialize implements ApplicationContextAware {
45+
public class EmbeddingStoreFactory {
5046

51-
private final AigcEmbedStoreService aigcEmbedStoreService;
52-
private final SpringContextHolder contextHolder;
53-
private List<AigcEmbedStore> modelStore = new ArrayList<>();
47+
@Autowired
48+
private AigcEmbedStoreService aigcEmbedStoreService;
5449

55-
@Override
56-
public void setApplicationContext(ApplicationContext context) throws BeansException {
57-
init();
58-
59-
modelStore.forEach(i -> log.info("已成功注册Embedding Store：{}， 配置信息：{}", i.getProvider(), i));
60-
}
50+
private final List<AigcEmbedStore> modelStore = new ArrayList<>();
51+
private final Map<String, EmbeddingStore<TextSegment>> embedStoreMap = new ConcurrentHashMap<>();
6152

53+
@Async
54+
@PostConstruct
6255
public void init() {
56+
modelStore.clear();
6357
List<AigcEmbedStore> list = aigcEmbedStoreService.list();
6458
list.forEach(embed -> {
6559
try {
@@ -72,11 +66,11 @@ public void init() {
7266
if (StrUtil.isNotBlank(embed.getUsername()) && StrUtil.isNotBlank(embed.getPassword())) {
7367
builder.user(embed.getUsername()).password(embed.getPassword());
7468
}
75-
RedisEmbeddingStore store = builder.build();
76-
contextHolder.registerBean(embed.getId(), store);
69+
EmbeddingStore<TextSegment> store = builder.build();
70+
embedStoreMap.put(embed.getId(), store);
7771
}
7872
if (EmbedStoreEnum.PGVECTOR.name().equalsIgnoreCase(embed.getProvider())) {
79-
PgVectorEmbeddingStore store = PgVectorEmbeddingStore.builder()
73+
EmbeddingStore<TextSegment> store = PgVectorEmbeddingStore.builder()
8074
.host(embed.getHost())
8175
.port(embed.getPort())
8276
.database(embed.getDatabaseName())
@@ -89,10 +83,10 @@ public void init() {
8983
.createTable(true)
9084
.dropTableFirst(false)
9185
.build();
92-
contextHolder.registerBean(embed.getId(), store);
86+
embedStoreMap.put(embed.getId(), store);
9387
}
9488
if (EmbedStoreEnum.MILVUS.name().equalsIgnoreCase(embed.getProvider())) {
95-
MilvusEmbeddingStore store = MilvusEmbeddingStore.builder()
89+
EmbeddingStore<TextSegment> store = MilvusEmbeddingStore.builder()
9690
.host(embed.getHost())
9791
.port(embed.getPort())
9892
.databaseName(embed.getDatabaseName())
@@ -101,24 +95,23 @@ public void init() {
10195
.password(embed.getPassword())
10296
.collectionName(embed.getTableName())
10397
.build();
104-
contextHolder.registerBean(embed.getId(), store);
105-
}
106-
if (EmbedStoreEnum.ELASTICSEARCH.name().equalsIgnoreCase(embed.getProvider())) {
107-
RestClient restClient = RestClient.builder(HttpHost.create(embed.getHost()))
108-
.setDefaultHeaders(new Header[]{
109-
new BasicHeader("Authorization", "ApiKey " + embed.getPassword())
110-
}).build();
111-
ElasticsearchEmbeddingStore store = ElasticsearchEmbeddingStore.builder()
112-
.restClient(restClient)
113-
.indexName(embed.getDatabaseName())
114-
.build();
115-
contextHolder.registerBean(embed.getId(), store);
98+
embedStoreMap.put(embed.getId(), store);
11699
}
117100
modelStore.add(embed);
118101
} catch (Exception e) {
119102
e.printStackTrace();
120103
log.error("向量数据库初始化失败：[{}] --- [{}]，数据库配置信息：[{}]", embed.getName(), embed.getProvider(), embed);
121104
}
122105
});
106+
107+
modelStore.forEach(i -> log.info("已成功注册Embedding Store：{}， 配置信息：{}", i.getProvider(), i));
108+
}
109+
110+
public EmbeddingStore<TextSegment> getEmbeddingStore(String embeddingId) {
111+
return embedStoreMap.get(embeddingId);
112+
}
113+
114+
public boolean containsEmbeddingStore(String embeddingId) {
115+
return embedStoreMap.containsKey(embeddingId);
123116
}
124117
}

‎langchat-ai/langchat-ai-core/src/main/java/cn/tycoding/langchat/ai/core/provider/KnowledgeStore.java‎ renamed to ‎langchat-ai/langchat-ai-core/src/main/java/cn/tycoding/langchat/ai/core/provider/KnowledgeStoreFactory.java‎

Lines changed: 24 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -22,16 +22,15 @@
2222
import cn.tycoding.langchat.ai.biz.service.AigcEmbedStoreService;
2323
import cn.tycoding.langchat.ai.biz.service.AigcKnowledgeService;
2424
import cn.tycoding.langchat.ai.biz.service.AigcModelService;
25-
import cn.tycoding.langchat.common.core.component.SpringContextHolder;
26-
import lombok.AllArgsConstructor;
25+
import jakarta.annotation.PostConstruct;
2726
import lombok.extern.slf4j.Slf4j;
28-
import org.springframework.beans.BeansException;
29-
import org.springframework.context.ApplicationContext;
30-
import org.springframework.context.ApplicationContextAware;
27+
import org.springframework.beans.factory.annotation.Autowired;
28+
import org.springframework.scheduling.annotation.Async;
3129
import org.springframework.stereotype.Component;
3230

3331
import java.util.List;
3432
import java.util.Map;
33+
import java.util.concurrent.ConcurrentHashMap;
3534
import java.util.stream.Collectors;
3635

3736
/**
@@ -40,20 +39,21 @@
4039
*/
4140
@Slf4j
4241
@Component
43-
@AllArgsConstructor
44-
public class KnowledgeStore implements ApplicationContextAware {
42+
public class KnowledgeStoreFactory {
4543

46-
private final AigcKnowledgeService knowledgeService;
47-
private final SpringContextHolder contextHolder;
48-
private final AigcModelService modelService;
49-
private final AigcEmbedStoreService embedStoreService;
44+
@Autowired
45+
private AigcKnowledgeService knowledgeService;
46+
@Autowired
47+
private AigcModelService modelService;
48+
@Autowired
49+
private AigcEmbedStoreService embedStoreService;
5050

51-
@Override
52-
public void setApplicationContext(ApplicationContext context) throws BeansException {
53-
init();
54-
}
51+
private final Map<String, AigcKnowledge> knowledgeMap = new ConcurrentHashMap<>();
5552

53+
@Async
54+
@PostConstruct
5655
public void init() {
56+
knowledgeMap.clear();
5757
List<AigcKnowledge> list = knowledgeService.list();
5858
Map<String, List<AigcModel>> modelMap = modelService.list().stream().collect(Collectors.groupingBy(AigcModel::getId));
5959
Map<String, List<AigcEmbedStore>> storeMap = embedStoreService.list().stream().collect(Collectors.groupingBy(AigcEmbedStore::getId));
@@ -66,7 +66,15 @@ public void init() {
6666
List<AigcEmbedStore> stores = storeMap.get(know.getEmbedStoreId());
6767
know.setEmbedStore(stores == null ? null : stores.get(0));
6868
}
69-
contextHolder.registerBean(know.getId(), know);
69+
knowledgeMap.put(know.getId(), know);
7070
});
7171
}
72+
73+
public AigcKnowledge getKnowledge(String knowledgeId) {
74+
return knowledgeMap.get(knowledgeId);
75+
}
76+
77+
public boolean containsKnowledge(String knowledgeId) {
78+
return knowledgeMap.containsKey(knowledgeId);
79+
}
7280
}

0 commit comments

Comments
 (0)