Skip to content

Commit 6f9a10e

Browse files
ogabrielluizogabrielluiz
committed
Merge remote-tracking branch 'origin/main' into v2
2 parents 4e147b0 + 8b40217 commit 6f9a10e

File tree

16 files changed

+674
-66
lines changed

16 files changed

+674
-66
lines changed

‎.devcontainer/devcontainer.json‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
// "forwardPorts": [],
1616

1717
// Use 'postCreateCommand' to run commands after the container is created.
18-
"postCreateCommand": "make install_frontend && make install_backend",
18+
"postCreateCommand": "make setup_devcontainer",
1919

2020
"containerEnv": {
2121
"POETRY_VIRTUALENVS_IN_PROJECT": "true"
@@ -31,11 +31,13 @@
3131
"sourcery.sourcery",
3232
"eamodio.gitlens",
3333
"ms-vscode.makefile-tools",
34-
"GitHub.vscode-pull-request-github"
34+
"GitHub.vscode-pull-request-github",
35+
"Codium.codium",
36+
"ms-azuretools.vscode-docker"
3537
]
3638
}
3739
}
3840

3941
// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
4042
// "remoteUser": "root"
41-
}
43+
}

‎Makefile‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,15 +49,16 @@ run_frontend:
4949
cd src/frontend && npm start
5050

5151
run_cli:
52-
poetry run langflow run --path src/frontend/build
52+
poetry run langflow --path src/frontend/build
5353

5454
run_cli_debug:
55-
poetry run langflow run --path src/frontend/build --log-level debug
55+
poetry run langflow --path src/frontend/build --log-level debug
5656

5757
setup_devcontainer:
5858
make init
5959
make build_frontend
60-
poetry run langflow --path src/frontend/build
60+
@echo 'Run Cli'
61+
make run_cli
6162

6263
frontend:
6364
@-make install_frontend || (echo "An error occurred while installing frontend dependencies. Attempting to fix." && make install_frontendc)

‎poetry.lock‎

Lines changed: 333 additions & 33 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pyproject.toml‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[tool.poetry]
22
name = "langflow"
3-
version = "0.5.5"
3+
version = "0.5.6"
44
description = "A Python package with a built-in web application"
55
authors = ["Logspace <[email protected]>"]
66
maintainers = [
@@ -35,8 +35,8 @@ google-search-results = "^2.4.1"
3535
google-api-python-client = "^2.79.0"
3636
typer = "^0.9.0"
3737
gunicorn = "^21.2.0"
38-
langchain = "~0.0.320"
39-
openai = "^1.0.1"
38+
langchain = "~0.0.327"
39+
openai = "^0.27.8"
4040
pandas = "2.0.3"
4141
chromadb = "^0.4.0"
4242
huggingface-hub = { version = "^0.16.0", extras = ["inference"] }
@@ -78,7 +78,7 @@ google-cloud-aiplatform = "^1.36.0"
7878
psycopg = "^3.1.9"
7979
psycopg-binary = "^3.1.9"
8080
fastavro = "^1.8.0"
81-
langchain-experimental = "^0.0.34"
81+
langchain-experimental = "*"
8282
celery = { extras = ["redis"], version = "^5.3.1", optional = true }
8383
redis = { version = "^4.6.0", optional = true }
8484
flower = { version = "^2.0.0", optional = true }
@@ -91,10 +91,12 @@ pydantic-settings = "^2.0.3"
9191
zep-python = { version = "^1.3.0", allow-prereleases = true }
9292
pywin32 = { version = "^306", markers = "sys_platform == 'win32'" }
9393
loguru = "^0.7.1"
94-
langfuse = "^1.1.1"
94+
langfuse = "^1.1.11"
9595
pillow = "^10.0.0"
9696
metal-sdk = "^2.4.0"
9797
markupsafe = "^2.1.3"
98+
extract-msg = "^0.45.0"
99+
jq = "^1.6.0"
98100
boto3 = "^1.28.63"
99101
numexpr = "^2.8.6"
100102
qianfan = "0.0.5"

‎src/backend/langflow/components/documentloaders/FileLoader.py‎

Lines changed: 232 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,232 @@
1+
from langflow import CustomComponent
2+
from langchain.schema import Document
3+
from typing import Any, Dict, List
4+
5+
loaders_info: List[Dict[str, Any]] = [
6+
{
7+
"loader": "AirbyteJSONLoader",
8+
"name": "Airbyte JSON (.jsonl)",
9+
"import": "langchain.document_loaders.AirbyteJSONLoader",
10+
"defaultFor": ["jsonl"],
11+
"allowdTypes": ["jsonl"],
12+
},
13+
{
14+
"loader": "JSONLoader",
15+
"name": "JSON (.json)",
16+
"import": "langchain.document_loaders.JSONLoader",
17+
"defaultFor": ["json"],
18+
"allowdTypes": ["json"],
19+
},
20+
{
21+
"loader": "BSHTMLLoader",
22+
"name": "BeautifulSoup4 HTML (.html, .htm)",
23+
"import": "langchain.document_loaders.BSHTMLLoader",
24+
"allowdTypes": ["html", "htm"],
25+
},
26+
{
27+
"loader": "CSVLoader",
28+
"name": "CSV (.csv)",
29+
"import": "langchain.document_loaders.CSVLoader",
30+
"defaultFor": ["csv"],
31+
"allowdTypes": ["csv"],
32+
},
33+
{
34+
"loader": "CoNLLULoader",
35+
"name": "CoNLL-U (.conllu)",
36+
"import": "langchain.document_loaders.CoNLLULoader",
37+
"defaultFor": ["conllu"],
38+
"allowdTypes": ["conllu"],
39+
},
40+
{
41+
"loader": "EverNoteLoader",
42+
"name": "EverNote (.enex)",
43+
"import": "langchain.document_loaders.EverNoteLoader",
44+
"defaultFor": ["enex"],
45+
"allowdTypes": ["enex"],
46+
},
47+
{
48+
"loader": "FacebookChatLoader",
49+
"name": "Facebook Chat (.json)",
50+
"import": "langchain.document_loaders.FacebookChatLoader",
51+
"allowdTypes": ["json"],
52+
},
53+
{
54+
"loader": "OutlookMessageLoader",
55+
"name": "Outlook Message (.msg)",
56+
"import": "langchain.document_loaders.OutlookMessageLoader",
57+
"defaultFor": ["msg"],
58+
"allowdTypes": ["msg"],
59+
},
60+
{
61+
"loader": "PyPDFLoader",
62+
"name": "PyPDF (.pdf)",
63+
"import": "langchain.document_loaders.PyPDFLoader",
64+
"defaultFor": ["pdf"],
65+
"allowdTypes": ["pdf"],
66+
},
67+
{
68+
"loader": "STRLoader",
69+
"name": "Subtitle (.str)",
70+
"import": "langchain.document_loaders.STRLoader",
71+
"defaultFor": ["str"],
72+
"allowdTypes": ["str"],
73+
},
74+
{
75+
"loader": "TextLoader",
76+
"name": "Text (.txt)",
77+
"import": "langchain.document_loaders.TextLoader",
78+
"defaultFor": ["txt"],
79+
"allowdTypes": ["txt"],
80+
},
81+
{
82+
"loader": "UnstructuredEmailLoader",
83+
"name": "Unstructured Email (.eml)",
84+
"import": "langchain.document_loaders.UnstructuredEmailLoader",
85+
"defaultFor": ["eml"],
86+
"allowdTypes": ["eml"],
87+
},
88+
{
89+
"loader": "UnstructuredHTMLLoader",
90+
"name": "Unstructured HTML (.html, .htm)",
91+
"import": "langchain.document_loaders.UnstructuredHTMLLoader",
92+
"defaultFor": ["html", "htm"],
93+
"allowdTypes": ["html", "htm"],
94+
},
95+
{
96+
"loader": "UnstructuredMarkdownLoader",
97+
"name": "Unstructured Markdown (.md)",
98+
"import": "langchain.document_loaders.UnstructuredMarkdownLoader",
99+
"defaultFor": ["md"],
100+
"allowdTypes": ["md"],
101+
},
102+
{
103+
"loader": "UnstructuredPowerPointLoader",
104+
"name": "Unstructured PowerPoint (.pptx)",
105+
"import": "langchain.document_loaders.UnstructuredPowerPointLoader",
106+
"defaultFor": ["pptx"],
107+
"allowdTypes": ["pptx"],
108+
},
109+
{
110+
"loader": "UnstructuredWordLoader",
111+
"name": "Unstructured Word (.docx)",
112+
"import": "langchain.document_loaders.UnstructuredWordLoader",
113+
"defaultFor": ["docx"],
114+
"allowdTypes": ["docx"],
115+
},
116+
]
117+
118+
119+
class FileLoaderComponent(CustomComponent):
120+
display_name: str = "File Loader"
121+
description: str = "Generic File Loader"
122+
beta = True
123+
124+
def build_config(self):
125+
loader_options = ["Automatic"] + [
126+
loader_info["name"] for loader_info in loaders_info
127+
]
128+
129+
file_types = []
130+
suffixes = []
131+
132+
for loader_info in loaders_info:
133+
if "allowedTypes" in loader_info:
134+
file_types.extend(loader_info["allowedTypes"])
135+
suffixes.extend([f".{ext}" for ext in loader_info["allowedTypes"]])
136+
137+
return {
138+
"file_path": {
139+
"display_name": "File Path",
140+
"required": True,
141+
"field_type": "file",
142+
"file_types": [
143+
"json",
144+
"txt",
145+
"csv",
146+
"jsonl",
147+
"html",
148+
"htm",
149+
"conllu",
150+
"enex",
151+
"msg",
152+
"pdf",
153+
"srt",
154+
"eml",
155+
"md",
156+
"pptx",
157+
"docx",
158+
],
159+
"suffixes": [
160+
".json",
161+
".txt",
162+
".csv",
163+
".jsonl",
164+
".html",
165+
".htm",
166+
".conllu",
167+
".enex",
168+
".msg",
169+
".pdf",
170+
".srt",
171+
".eml",
172+
".md",
173+
".pptx",
174+
".docx",
175+
],
176+
# "file_types" : file_types,
177+
# "suffixes": suffixes,
178+
},
179+
"loader": {
180+
"display_name": "Loader",
181+
"is_list": True,
182+
"required": True,
183+
"options": loader_options,
184+
"value": "Automatic",
185+
},
186+
"code": {"show": False},
187+
}
188+
189+
def build(self, file_path: str, loader: str) -> Document:
190+
file_type = file_path.split(".")[-1]
191+
192+
# Mapeie o nome do loader selecionado para suas informações
193+
selected_loader_info = None
194+
for loader_info in loaders_info:
195+
if loader_info["name"] == loader:
196+
selected_loader_info = loader_info
197+
break
198+
199+
if selected_loader_info is None and loader != "Automatic":
200+
raise ValueError(f"Loader {loader} not found in the loader info list")
201+
202+
if loader == "Automatic":
203+
# Determine o loader automaticamente com base na extensão do arquivo
204+
default_loader_info = None
205+
for info in loaders_info:
206+
if "defaultFor" in info and file_type in info["defaultFor"]:
207+
default_loader_info = info
208+
break
209+
210+
if default_loader_info is None:
211+
raise ValueError(f"No default loader found for file type: {file_type}")
212+
213+
selected_loader_info = default_loader_info
214+
if isinstance(selected_loader_info, dict):
215+
loader_import: str = selected_loader_info["import"]
216+
else:
217+
raise ValueError(
218+
f"Loader info for {loader} is not a dict\nLoader info:\n{selected_loader_info}"
219+
)
220+
module_name, class_name = loader_import.rsplit(".", 1)
221+
222+
try:
223+
# Importe o loader dinamicamente
224+
loader_module = __import__(module_name, fromlist=[class_name])
225+
loader_instance = getattr(loader_module, class_name)
226+
except ImportError as e:
227+
raise ValueError(
228+
f"Loader {loader} could not be imported\nLoader info:\n{selected_loader_info}"
229+
) from e
230+
231+
result = loader_instance(file_path=file_path)
232+
return result.load()

‎src/backend/langflow/components/documentloaders/UrlLoader.py‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
from typing import List
2+
from langflow import CustomComponent
3+
from langchain.document_loaders import AZLyricsLoader
4+
from langchain.document_loaders import CollegeConfidentialLoader
5+
from langchain.document_loaders import GitbookLoader
6+
from langchain.document_loaders import HNLoader
7+
from langchain.document_loaders import IFixitLoader
8+
from langchain.document_loaders import IMSDbLoader
9+
from langchain.document_loaders import WebBaseLoader
10+
11+
12+
from langchain.schema import Document
13+
14+
15+
class UrlLoaderComponent(CustomComponent):
16+
display_name: str = "Url Loader"
17+
description: str = "Generic Url Loader Component"
18+
19+
def build_config(self):
20+
return {
21+
"web_path": {
22+
"display_name": "Url",
23+
"required": True,
24+
},
25+
"loader": {
26+
"display_name": "Loader",
27+
"is_list": True,
28+
"required": True,
29+
"options": [
30+
"AZLyricsLoader",
31+
"CollegeConfidentialLoader",
32+
"GitbookLoader",
33+
"HNLoader",
34+
"IFixitLoader",
35+
"IMSDbLoader",
36+
"WebBaseLoader",
37+
],
38+
"value": "WebBaseLoader",
39+
},
40+
"code": {"show": False},
41+
}
42+
43+
def build(self, web_path: str, loader: str) -> List[Document]:
44+
if loader == "AZLyricsLoader":
45+
loader_instance = AZLyricsLoader(web_path=web_path) # type: ignore
46+
elif loader == "CollegeConfidentialLoader":
47+
loader_instance = CollegeConfidentialLoader(web_path=web_path) # type: ignore
48+
elif loader == "GitbookLoader":
49+
loader_instance = GitbookLoader(web_page=web_path) # type: ignore
50+
elif loader == "HNLoader":
51+
loader_instance = HNLoader(web_path=web_path) # type: ignore
52+
elif loader == "IFixitLoader":
53+
loader_instance = IFixitLoader(web_path=web_path) # type: ignore
54+
elif loader == "IMSDbLoader":
55+
loader_instance = IMSDbLoader(web_path=web_path) # type: ignore
56+
elif loader == "WebBaseLoader":
57+
loader_instance = WebBaseLoader(web_path=web_path) # type: ignore
58+
59+
if loader_instance is None:
60+
raise ValueError(f"No loader found for: {web_path}")
61+
62+
return loader_instance.load()

‎src/backend/langflow/components/documentloaders/__init__.py‎

Whitespace-only changes.

‎src/backend/langflow/graph/vertex/types.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,10 @@ def build(
206206
) -> Any:
207207
if not self._built or force:
208208
# Check if the chain requires a PromptVertex
209+
210+
# Temporarily remove "code" from the params
211+
self.params.pop("code", None)
212+
209213
for key, value in self.params.items():
210214
if isinstance(value, PromptVertex):
211215
# Build the PromptVertex, passing the tools if available

0 commit comments

Comments
 (0)