updatd indexing param

changliu2 · changliu2 · commit 22fe48ad0574 · 2024-06-19T23:36:34.000Z
diff --git a/src/indexing/build_index.py b/src/indexing/build_index.py
@@ -51,8 +51,8 @@ def build_aisearch_index(index_name, path_to_data):
           )
       ),
       embeddings_cache_path="./indexing",
-      tokens_per_chunk = 5096, # Optional field - Maximum number of tokens per chunk
-      token_overlap_across_chunks = 1024, # Optional field - Number of tokens to overlap between chunks
+      tokens_per_chunk = 800, # Optional field - Maximum number of tokens per chunk
+      token_overlap_across_chunks = 0, # Optional field - Number of tokens to overlap between chunks
   )
   print(f"Local Path: {index_path}")
 

Original file line number	Diff line number	Diff line change
`@@ -51,8 +51,8 @@ def build_aisearch_index(index_name, path_to_data):`
`51`	`51`	`)`
`52`	`52`	`),`
`53`	`53`	`embeddings_cache_path="./indexing",`
`54`		`- tokens_per_chunk = 5096, # Optional field - Maximum number of tokens per chunk`
`55`		`- token_overlap_across_chunks = 1024, # Optional field - Number of tokens to overlap between chunks`
	`54`	`+ tokens_per_chunk = 800, # Optional field - Maximum number of tokens per chunk`
	`55`	`+ token_overlap_across_chunks = 0, # Optional field - Number of tokens to overlap between chunks`
`56`	`56`	`)`
`57`	`57`	`print(f"Local Path: {index_path}")`
`58`	`58`