feat: Support pagination and sorting on registry apis (feast-dev#5495)

Signed-off-by: ntkathole <[email protected]>

Author: ntkathole

protos/feast/registry/RegistryServer.proto

@@ -94,6 +94,26 @@ service RegistryServer{
 
 }
 
+// Common pagination and sorting messages
+message PaginationParams {
+  int32 page = 1;       // 1-based page number
+  int32 limit = 2;      // Number of items per page
+}
+
+message SortingParams {
+  string sort_by = 1;     // Field to sort by (supports dot notation)
+  string sort_order = 2;  // "asc" or "desc"
+}
+
+message PaginationMetadata {
+  int32 page = 1;
+  int32 limit = 2;
+  int32 total_count = 3;
+  int32 total_pages = 4;
+  bool has_next = 5;
+  bool has_previous = 6;
+}
+
 message RefreshRequest {
   string project = 1;
 }
@@ -142,10 +162,13 @@ message ListEntitiesRequest {
     string project = 1;
     bool allow_cache = 2;
     map<string,string> tags = 3;
+    PaginationParams pagination = 4;
+    SortingParams sorting = 5;
 }
 
 message ListEntitiesResponse {
     repeated feast.core.Entity entities = 1;
+    PaginationMetadata pagination = 2;
 }
 
 message DeleteEntityRequest {
@@ -172,10 +195,13 @@ message ListDataSourcesRequest {
     string project = 1;
     bool allow_cache = 2;
     map<string,string> tags = 3;
+    PaginationParams pagination = 4;
+    SortingParams sorting = 5;
 }
 
 message ListDataSourcesResponse {
     repeated feast.core.DataSource data_sources = 1;
+    PaginationMetadata pagination = 2;
 }
 
 message DeleteDataSourceRequest {
@@ -206,10 +232,13 @@ message ListFeatureViewsRequest {
     string project = 1;
     bool allow_cache = 2;
     map<string,string> tags = 3;
+    PaginationParams pagination = 4;
+    SortingParams sorting = 5;
 }
 
 message ListFeatureViewsResponse {
     repeated feast.core.FeatureView feature_views = 1;
+    PaginationMetadata pagination = 2;
 }
 
 message DeleteFeatureViewRequest {
@@ -240,10 +269,13 @@ message ListAllFeatureViewsRequest {
     string project = 1;
     bool allow_cache = 2;
     map<string,string> tags = 3;
+    PaginationParams pagination = 4;
+    SortingParams sorting = 5;
 }
 
 message ListAllFeatureViewsResponse {
     repeated AnyFeatureView feature_views = 1;
+    PaginationMetadata pagination = 2;
 }
 
 
@@ -259,10 +291,13 @@ message ListStreamFeatureViewsRequest {
     string project = 1;
     bool allow_cache = 2;
     map<string,string> tags = 3;
+    PaginationParams pagination = 4;
+    SortingParams sorting = 5;
 }
 
 message ListStreamFeatureViewsResponse {
     repeated feast.core.StreamFeatureView stream_feature_views = 1;
+    PaginationMetadata pagination = 2;
 }
 
 // OnDemandFeatureView
@@ -277,10 +312,13 @@ message ListOnDemandFeatureViewsRequest {
     string project = 1;
     bool allow_cache = 2;
     map<string,string> tags = 3;
+    PaginationParams pagination = 4;
+    SortingParams sorting = 5;
 }
 
 message ListOnDemandFeatureViewsResponse {
     repeated feast.core.OnDemandFeatureView on_demand_feature_views = 1;
+    PaginationMetadata pagination = 2;
 }
 
 // FeatureServices
@@ -301,10 +339,13 @@ message ListFeatureServicesRequest {
     string project = 1;
     bool allow_cache = 2;
     map<string,string> tags = 3;
+    PaginationParams pagination = 4;
+    SortingParams sorting = 5;
 }
 
 message ListFeatureServicesResponse {
     repeated feast.core.FeatureService feature_services = 1;
+    PaginationMetadata pagination = 2;
 }
 
 message DeleteFeatureServiceRequest {
@@ -331,10 +372,13 @@ message ListSavedDatasetsRequest {
     string project = 1;
     bool allow_cache = 2;
     map<string,string> tags = 3;
+    PaginationParams pagination = 4;
+    SortingParams sorting = 5;
 }
 
 message ListSavedDatasetsResponse {
     repeated feast.core.SavedDataset saved_datasets = 1;
+    PaginationMetadata pagination = 2;
 }
 
 message DeleteSavedDatasetRequest {
@@ -361,10 +405,13 @@ message ListValidationReferencesRequest {
     string project = 1;
     bool allow_cache = 2;
     map<string,string> tags = 3;
+    PaginationParams pagination = 4;
+    SortingParams sorting = 5;
 }
 
 message ListValidationReferencesResponse {
     repeated feast.core.ValidationReference validation_references = 1;
+    PaginationMetadata pagination = 2;
 }
 
 message DeleteValidationReferenceRequest {
@@ -391,10 +438,13 @@ message ListPermissionsRequest {
   string project = 1;
   bool allow_cache = 2;
   map<string,string> tags = 3;
+  PaginationParams pagination = 4;
+  SortingParams sorting = 5;
 }
 
 message ListPermissionsResponse {
   repeated feast.core.Permission permissions = 1;
+  PaginationMetadata pagination = 2;
 }
 
 message DeletePermissionRequest {
@@ -418,10 +468,13 @@ message GetProjectRequest {
 message ListProjectsRequest {
   bool allow_cache = 1;
   map<string,string> tags = 2;
+  PaginationParams pagination = 3;
+  SortingParams sorting = 4;
 }
 
 message ListProjectsResponse {
   repeated feast.core.Project projects = 1;
+  PaginationMetadata pagination = 2;
 }
 
 message DeleteProjectRequest {
@@ -446,11 +499,15 @@ message GetRegistryLineageRequest {
   bool allow_cache = 2;
   string filter_object_type = 3;
   string filter_object_name = 4;
+  PaginationParams pagination = 5;
+  SortingParams sorting = 6;
 }
 
 message GetRegistryLineageResponse {
   repeated EntityRelation relationships = 1;
   repeated EntityRelation indirect_relationships = 2;
+  PaginationMetadata relationships_pagination = 3;
+  PaginationMetadata indirect_relationships_pagination = 4;
 }
 
 message GetObjectRelationshipsRequest {
@@ -459,8 +516,11 @@ message GetObjectRelationshipsRequest {
   string object_name = 3;
   bool include_indirect = 4;
   bool allow_cache = 5;
+  PaginationParams pagination = 6;
+  SortingParams sorting = 7;
 }
 
 message GetObjectRelationshipsResponse {
   repeated EntityRelation relationships = 1;
+  PaginationMetadata pagination = 2;
 }

sdk/python/feast/api/registry/rest/data_sources.py

@@ -3,7 +3,14 @@
 
 from fastapi import APIRouter, Depends, Query
 
-from feast.api.registry.rest.rest_utils import grpc_call, parse_tags
+from feast.api.registry.rest.rest_utils import (
+    create_grpc_pagination_params,
+    create_grpc_sorting_params,
+    get_pagination_params,
+    get_sorting_params,
+    grpc_call,
+    parse_tags,
+)
 from feast.protos.feast.registry import RegistryServer_pb2
 
 logger = logging.getLogger(__name__)
@@ -17,14 +24,22 @@ def list_data_sources(
         project: str = Query(...),
         allow_cache: bool = Query(default=True),
         tags: Dict[str, str] = Depends(parse_tags),
+        pagination_params: dict = Depends(get_pagination_params),
+        sorting_params: dict = Depends(get_sorting_params),
     ):
         req = RegistryServer_pb2.ListDataSourcesRequest(
             project=project,
             allow_cache=allow_cache,
             tags=tags,
+            pagination=create_grpc_pagination_params(pagination_params),
+            sorting=create_grpc_sorting_params(sorting_params),
         )
+
         response = grpc_call(grpc_handler.ListDataSources, req)
-        return {"data_sources": response.get("dataSources", [])}
+        return {
+            "data_sources": response.get("dataSources", []),
+            "pagination": response.get("pagination", {}),
+        }
 
     @router.get("/data_sources/{name}")
     def get_data_source(

sdk/python/feast/api/registry/rest/entities.py

@@ -1,8 +1,14 @@
 import logging
 
-from fastapi import APIRouter, Query
-
-from feast.api.registry.rest.rest_utils import grpc_call
+from fastapi import APIRouter, Depends, Query
+
+from feast.api.registry.rest.rest_utils import (
+    create_grpc_pagination_params,
+    create_grpc_sorting_params,
+    get_pagination_params,
+    get_sorting_params,
+    grpc_call,
+)
 from feast.protos.feast.registry import RegistryServer_pb2
 
 logger = logging.getLogger(__name__)
@@ -14,10 +20,18 @@ def get_entity_router(grpc_handler) -> APIRouter:
     @router.get("/entities")
     def list_entities(
         project: str = Query(...),
+        allow_cache: bool = Query(default=True),
+        pagination_params: dict = Depends(get_pagination_params),
+        sorting_params: dict = Depends(get_sorting_params),
     ):
-        req = RegistryServer_pb2.ListEntitiesRequest(project=project)
-        response = grpc_call(grpc_handler.ListEntities, req)
-        return {"entities": response.get("entities", [])}
+        req = RegistryServer_pb2.ListEntitiesRequest(
+            project=project,
+            allow_cache=allow_cache,
+            pagination=create_grpc_pagination_params(pagination_params),
+            sorting=create_grpc_sorting_params(sorting_params),
+        )
+
+        return grpc_call(grpc_handler.ListEntities, req)
 
     @router.get("/entities/{name}")
     def get_entity(

sdk/python/feast/api/registry/rest/feature_services.py

@@ -2,7 +2,14 @@
 
 from fastapi import APIRouter, Depends, Query
 
-from feast.api.registry.rest.rest_utils import grpc_call, parse_tags
+from feast.api.registry.rest.rest_utils import (
+    create_grpc_pagination_params,
+    create_grpc_sorting_params,
+    get_pagination_params,
+    get_sorting_params,
+    grpc_call,
+    parse_tags,
+)
 from feast.protos.feast.registry import RegistryServer_pb2
 
 
@@ -14,11 +21,15 @@ def list_feature_services(
         project: str = Query(...),
         allow_cache: bool = Query(default=True),
         tags: Dict[str, str] = Depends(parse_tags),
+        pagination_params: dict = Depends(get_pagination_params),
+        sorting_params: dict = Depends(get_sorting_params),
     ):
         req = RegistryServer_pb2.ListFeatureServicesRequest(
             project=project,
             allow_cache=allow_cache,
             tags=tags,
+            pagination=create_grpc_pagination_params(pagination_params),
+            sorting=create_grpc_sorting_params(sorting_params),
         )
         return grpc_call(grpc_handler.ListFeatureServices, req)
 

sdk/python/feast/api/registry/rest/feature_views.py

@@ -2,7 +2,14 @@
 
 from fastapi import APIRouter, Depends, Query
 
-from feast.api.registry.rest.rest_utils import grpc_call, parse_tags
+from feast.api.registry.rest.rest_utils import (
+    create_grpc_pagination_params,
+    create_grpc_sorting_params,
+    get_pagination_params,
+    get_sorting_params,
+    grpc_call,
+    parse_tags,
+)
 from feast.registry_server import RegistryServer_pb2
 
 
@@ -21,18 +28,29 @@ def get_any_feature_view(
             allow_cache=allow_cache,
         )
         response = grpc_call(grpc_handler.GetAnyFeatureView, req)
-        return response.get("anyFeatureView", {})
+        any_feature_view = response.get("anyFeatureView", {})
+        feature_view = (
+            any_feature_view.get("featureView")
+            or any_feature_view.get("onDemandFeatureView")
+            or any_feature_view.get("streamFeatureView")
+            or {}
+        )
+        return {"featureView": feature_view}
 
     @router.get("/feature_views")
     def list_all_feature_views(
         project: str = Query(...),
-        allow_cache: bool = Query(True),
+        allow_cache: bool = Query(default=True),
         tags: Dict[str, str] = Depends(parse_tags),
+        pagination_params: dict = Depends(get_pagination_params),
+        sorting_params: dict = Depends(get_sorting_params),
     ):
         req = RegistryServer_pb2.ListAllFeatureViewsRequest(
             project=project,
             allow_cache=allow_cache,
             tags=tags,
+            pagination=create_grpc_pagination_params(pagination_params),
+            sorting=create_grpc_sorting_params(sorting_params),
         )
         return grpc_call(grpc_handler.ListAllFeatureViews, req)