Skip to content

Commit 2e57376

Browse files
adchiaadchia
authored
fix: Enable registry caching in SQL Registry (feast-dev#3395)
* fix: Enable registry caching in SQL Registry Signed-off-by: Danny Chiao <[email protected]> * docs Signed-off-by: Danny Chiao <[email protected]> * fix regular file registry docs too Signed-off-by: Danny Chiao <[email protected]> * fix new file lint Signed-off-by: Danny Chiao <[email protected]> * fix new file lint Signed-off-by: Danny Chiao <[email protected]> Signed-off-by: Danny Chiao <[email protected]>
1 parent 6bcf77c commit 2e57376

File tree

5 files changed

+400
-128
lines changed

5 files changed

+400
-128
lines changed

‎docs/getting-started/concepts/registry.md‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,9 @@ a remote file registry, you need to create a GCS / S3 bucket that Feast can unde
1616
```yaml
1717
project: feast_demo_aws
1818
provider: aws
19-
registry: s3://[YOUR BUCKET YOU CREATED]/registry.pb
19+
registry:
20+
path: s3://[YOUR BUCKET YOU CREATED]/registry.pb
21+
cache_ttl_seconds: 60
2022
online_store: null
2123
offline_store:
2224
type: file
@@ -27,7 +29,9 @@ offline_store:
2729
```yaml
2830
project: feast_demo_gcp
2931
provider: gcp
30-
registry: gs://[YOUR BUCKET YOU CREATED]/registry.pb
32+
registry:
33+
path: gs://[YOUR BUCKET YOU CREATED]/registry.pb
34+
cache_ttl_seconds: 60
3135
online_store: null
3236
offline_store:
3337
type: file
@@ -43,6 +47,18 @@ multiple feature views or time ranges concurrently).
4347
#### SQL Registry
4448
Alternatively, a [SQL Registry](../../tutorials/using-scalable-registry.md) can be used for a more scalable registry.
4549
50+
The configuration roughly looks like:
51+
```yaml
52+
project: <your project name>
53+
provider: <provider name>
54+
online_store: redis
55+
offline_store: file
56+
registry:
57+
registry_type: sql
58+
path: postgresql://postgres:[email protected]:55001/feast
59+
cache_ttl_seconds: 60
60+
```
61+
4662
This supports any SQLAlchemy compatible database as a backend. The exact schema can be seen in [sql.py](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/infra/registry/sql.py)
4763
4864
### Updating the registry

‎docs/tutorials/using-scalable-registry.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ offline_store: file
2828
registry:
2929
registry_type: sql
3030
path: postgresql://postgres:[email protected]:55001/feast
31+
cache_ttl_seconds: 60
3132
```
3233
3334
Specifically, the registry_type needs to be set to sql in the registry config block. On doing so, the path should refer to the [Database URL](https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls) for the database to be used, as expected by SQLAlchemy. No other additional commands are currently needed to configure this registry.

‎sdk/python/feast/infra/registry/proto_registry_utils.py‎

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
from typing import List
2+
3+
from feast.data_source import DataSource
4+
from feast.entity import Entity
5+
from feast.errors import (
6+
DataSourceObjectNotFoundException,
7+
EntityNotFoundException,
8+
FeatureServiceNotFoundException,
9+
FeatureViewNotFoundException,
10+
OnDemandFeatureViewNotFoundException,
11+
SavedDatasetNotFound,
12+
ValidationReferenceNotFound,
13+
)
14+
from feast.feature_service import FeatureService
15+
from feast.feature_view import FeatureView
16+
from feast.on_demand_feature_view import OnDemandFeatureView
17+
from feast.project_metadata import ProjectMetadata
18+
from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto
19+
from feast.request_feature_view import RequestFeatureView
20+
from feast.saved_dataset import SavedDataset, ValidationReference
21+
from feast.stream_feature_view import StreamFeatureView
22+
23+
24+
def get_feature_service(
25+
registry_proto: RegistryProto, name: str, project: str
26+
) -> FeatureService:
27+
for feature_service_proto in registry_proto.feature_services:
28+
if (
29+
feature_service_proto.spec.project == project
30+
and feature_service_proto.spec.name == name
31+
):
32+
return FeatureService.from_proto(feature_service_proto)
33+
raise FeatureServiceNotFoundException(name, project=project)
34+
35+
36+
def get_feature_view(
37+
registry_proto: RegistryProto, name: str, project: str
38+
) -> FeatureView:
39+
for feature_view_proto in registry_proto.feature_views:
40+
if (
41+
feature_view_proto.spec.name == name
42+
and feature_view_proto.spec.project == project
43+
):
44+
return FeatureView.from_proto(feature_view_proto)
45+
raise FeatureViewNotFoundException(name, project)
46+
47+
48+
def get_stream_feature_view(
49+
registry_proto: RegistryProto, name: str, project: str
50+
) -> StreamFeatureView:
51+
for feature_view_proto in registry_proto.stream_feature_views:
52+
if (
53+
feature_view_proto.spec.name == name
54+
and feature_view_proto.spec.project == project
55+
):
56+
return StreamFeatureView.from_proto(feature_view_proto)
57+
raise FeatureViewNotFoundException(name, project)
58+
59+
60+
def get_request_feature_view(registry_proto: RegistryProto, name: str, project: str):
61+
for feature_view_proto in registry_proto.feature_views:
62+
if (
63+
feature_view_proto.spec.name == name
64+
and feature_view_proto.spec.project == project
65+
):
66+
return RequestFeatureView.from_proto(feature_view_proto)
67+
raise FeatureViewNotFoundException(name, project)
68+
69+
70+
def get_on_demand_feature_view(
71+
registry_proto: RegistryProto, name: str, project: str
72+
) -> OnDemandFeatureView:
73+
for on_demand_feature_view in registry_proto.on_demand_feature_views:
74+
if (
75+
on_demand_feature_view.spec.project == project
76+
and on_demand_feature_view.spec.name == name
77+
):
78+
return OnDemandFeatureView.from_proto(on_demand_feature_view)
79+
raise OnDemandFeatureViewNotFoundException(name, project=project)
80+
81+
82+
def get_data_source(
83+
registry_proto: RegistryProto, name: str, project: str
84+
) -> DataSource:
85+
for data_source in registry_proto.data_sources:
86+
if data_source.project == project and data_source.name == name:
87+
return DataSource.from_proto(data_source)
88+
raise DataSourceObjectNotFoundException(name, project=project)
89+
90+
91+
def get_entity(registry_proto: RegistryProto, name: str, project: str) -> Entity:
92+
for entity_proto in registry_proto.entities:
93+
if entity_proto.spec.name == name and entity_proto.spec.project == project:
94+
return Entity.from_proto(entity_proto)
95+
raise EntityNotFoundException(name, project=project)
96+
97+
98+
def get_saved_dataset(
99+
registry_proto: RegistryProto, name: str, project: str
100+
) -> SavedDataset:
101+
for saved_dataset in registry_proto.saved_datasets:
102+
if saved_dataset.spec.name == name and saved_dataset.spec.project == project:
103+
return SavedDataset.from_proto(saved_dataset)
104+
raise SavedDatasetNotFound(name, project=project)
105+
106+
107+
def get_validation_reference(
108+
registry_proto: RegistryProto, name: str, project: str
109+
) -> ValidationReference:
110+
for validation_reference in registry_proto.validation_references:
111+
if (
112+
validation_reference.name == name
113+
and validation_reference.project == project
114+
):
115+
return ValidationReference.from_proto(validation_reference)
116+
raise ValidationReferenceNotFound(name, project=project)
117+
118+
119+
def list_feature_services(
120+
registry_proto: RegistryProto, project: str, allow_cache: bool = False
121+
) -> List[FeatureService]:
122+
feature_services = []
123+
for feature_service_proto in registry_proto.feature_services:
124+
if feature_service_proto.spec.project == project:
125+
feature_services.append(FeatureService.from_proto(feature_service_proto))
126+
return feature_services
127+
128+
129+
def list_feature_views(
130+
registry_proto: RegistryProto, project: str
131+
) -> List[FeatureView]:
132+
feature_views: List[FeatureView] = []
133+
for feature_view_proto in registry_proto.feature_views:
134+
if feature_view_proto.spec.project == project:
135+
feature_views.append(FeatureView.from_proto(feature_view_proto))
136+
return feature_views
137+
138+
139+
def list_request_feature_views(
140+
registry_proto: RegistryProto, project: str
141+
) -> List[RequestFeatureView]:
142+
feature_views: List[RequestFeatureView] = []
143+
for request_feature_view_proto in registry_proto.request_feature_views:
144+
if request_feature_view_proto.spec.project == project:
145+
feature_views.append(
146+
RequestFeatureView.from_proto(request_feature_view_proto)
147+
)
148+
return feature_views
149+
150+
151+
def list_stream_feature_views(
152+
registry_proto: RegistryProto, project: str
153+
) -> List[StreamFeatureView]:
154+
stream_feature_views = []
155+
for stream_feature_view in registry_proto.stream_feature_views:
156+
if stream_feature_view.spec.project == project:
157+
stream_feature_views.append(
158+
StreamFeatureView.from_proto(stream_feature_view)
159+
)
160+
return stream_feature_views
161+
162+
163+
def list_on_demand_feature_views(
164+
registry_proto: RegistryProto, project: str
165+
) -> List[OnDemandFeatureView]:
166+
on_demand_feature_views = []
167+
for on_demand_feature_view in registry_proto.on_demand_feature_views:
168+
if on_demand_feature_view.spec.project == project:
169+
on_demand_feature_views.append(
170+
OnDemandFeatureView.from_proto(on_demand_feature_view)
171+
)
172+
return on_demand_feature_views
173+
174+
175+
def list_entities(registry_proto: RegistryProto, project: str) -> List[Entity]:
176+
entities = []
177+
for entity_proto in registry_proto.entities:
178+
if entity_proto.spec.project == project:
179+
entities.append(Entity.from_proto(entity_proto))
180+
return entities
181+
182+
183+
def list_data_sources(registry_proto: RegistryProto, project: str) -> List[DataSource]:
184+
data_sources = []
185+
for data_source_proto in registry_proto.data_sources:
186+
if data_source_proto.project == project:
187+
data_sources.append(DataSource.from_proto(data_source_proto))
188+
return data_sources
189+
190+
191+
def list_saved_datasets(
192+
registry_proto: RegistryProto, project: str, allow_cache: bool = False
193+
) -> List[SavedDataset]:
194+
return [
195+
SavedDataset.from_proto(saved_dataset)
196+
for saved_dataset in registry_proto.saved_datasets
197+
if saved_dataset.spec.project == project
198+
]
199+
200+
201+
def list_project_metadata(
202+
registry_proto: RegistryProto, project: str
203+
) -> List[ProjectMetadata]:
204+
return [
205+
ProjectMetadata.from_proto(project_metadata)
206+
for project_metadata in registry_proto.project_metadata
207+
if project_metadata.project == project
208+
]

0 commit comments

Comments
 (0)