From a7abc8eea6bba473eebf0a34b0f3df45eac17f59 Mon Sep 17 00:00:00 2001 From: Gyubong Date: Mon, 20 Jul 2026 17:49:46 +0900 Subject: [PATCH] feat(BA-6892): add kernel scheduling-history adapter and REST v2 endpoints Co-Authored-By: Claude Opus 4.8 (1M context) --- changes/12870.feature.md | 1 + docs/manager/rest-reference/openapi.json | 530 ++++++++++++++++++ .../manager/v2/scheduling_history/__init__.py | 10 +- .../manager/v2/scheduling_history/request.py | 18 +- .../manager/v2/scheduling_history/response.py | 4 +- .../manager/v2/scheduling_history/types.py | 7 + .../adapters/scheduling_history/adapter.py | 232 ++++++++ .../api/rest/v2/scheduling_history/handler.py | 24 + .../rest/v2/scheduling_history/registry.py | 16 +- 9 files changed, 835 insertions(+), 7 deletions(-) create mode 100644 changes/12870.feature.md diff --git a/changes/12870.feature.md b/changes/12870.feature.md new file mode 100644 index 00000000000..3599870fae2 --- /dev/null +++ b/changes/12870.feature.md @@ -0,0 +1 @@ +Add the kernel scheduling-history adapter and its REST v2 endpoints: a super-admin-only system-wide search, and a kernel-scoped search whose access is checked against the RBAC scope. diff --git a/docs/manager/rest-reference/openapi.json b/docs/manager/rest-reference/openapi.json index 5f488c80f20..c3b57c8d24a 100644 --- a/docs/manager/rest-reference/openapi.json +++ b/docs/manager/rest-reference/openapi.json @@ -24909,6 +24909,478 @@ "title": "AdminSearchSessionHistoriesInput", "type": "object" }, + "KernelHistoryFilter": { + "description": "Filter conditions for kernel scheduling history search.", + "properties": { + "id": { + "anyOf": [ + { + "$ref": "#/components/schemas/UUIDFilter" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Filter by history record ID" + }, + "kernel_id": { + "anyOf": [ + { + "$ref": "#/components/schemas/UUIDFilter" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Filter by kernel ID" + }, + "session_id": { + "anyOf": [ + { + "$ref": "#/components/schemas/UUIDFilter" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Filter by session ID" + }, + "phase": { + "anyOf": [ + { + "$ref": "#/components/schemas/StringFilter" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Filter by scheduling phase" + }, + "from_status": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Filter by from_status values", + "title": "From Status" + }, + "to_status": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Filter by to_status values", + "title": "To Status" + }, + "result": { + "anyOf": [ + { + "$ref": "#/components/schemas/SchedulingResultFilter" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Filter by scheduling result" + }, + "error_code": { + "anyOf": [ + { + "$ref": "#/components/schemas/StringFilter" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Filter by error code" + }, + "message": { + "anyOf": [ + { + "$ref": "#/components/schemas/StringFilter" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Filter by message" + }, + "created_at": { + "anyOf": [ + { + "$ref": "#/components/schemas/DateTimeFilter" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Filter by created_at" + }, + "updated_at": { + "anyOf": [ + { + "$ref": "#/components/schemas/DateTimeFilter" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Filter by updated_at" + }, + "AND": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/KernelHistoryFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AND conjunction.", + "title": "And" + }, + "OR": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/KernelHistoryFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "OR conjunction.", + "title": "Or" + }, + "NOT": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/KernelHistoryFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "NOT negation.", + "title": "Not" + } + }, + "title": "KernelHistoryFilter", + "type": "object" + }, + "KernelHistoryOrder": { + "description": "Order specification for kernel scheduling history.", + "properties": { + "field": { + "$ref": "#/components/schemas/KernelHistoryOrderField", + "description": "Field to order by" + }, + "direction": { + "$ref": "#/components/schemas/OrderDirection", + "default": "DESC", + "description": "Order direction" + } + }, + "required": [ + "field" + ], + "title": "KernelHistoryOrder", + "type": "object" + }, + "KernelHistoryOrderField": { + "description": "Fields available for ordering kernel scheduling history.", + "enum": [ + "created_at", + "updated_at" + ], + "title": "KernelHistoryOrderField", + "type": "string" + }, + "AdminSearchKernelHistoriesInput": { + "description": "Input for admin search of kernel scheduling histories.", + "properties": { + "filter": { + "anyOf": [ + { + "$ref": "#/components/schemas/KernelHistoryFilter" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Filter conditions" + }, + "order": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/KernelHistoryOrder" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Order specifications", + "title": "Order" + }, + "first": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Cursor pagination: number of items", + "title": "First" + }, + "after": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Cursor pagination: after cursor", + "title": "After" + }, + "last": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Cursor pagination: last N items", + "title": "Last" + }, + "before": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Cursor pagination: before cursor", + "title": "Before" + }, + "limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Offset pagination: maximum items", + "title": "Limit" + }, + "offset": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Offset pagination: number to skip", + "title": "Offset" + } + }, + "title": "AdminSearchKernelHistoriesInput", + "type": "object" + }, + "KernelHistoryScopeDTO": { + "description": "Scope for kernel scheduling history queries.", + "properties": { + "kernel_id": { + "description": "Kernel ID to get history for.", + "format": "uuid", + "title": "Kernel Id", + "type": "string" + } + }, + "required": [ + "kernel_id" + ], + "title": "KernelHistoryScopeDTO", + "type": "object" + }, + "ScopedSearchKernelHistoriesInput": { + "description": "Input for searching kernel scheduling histories under a non-admin scope.", + "properties": { + "scope": { + "$ref": "#/components/schemas/KernelHistoryScopeDTO", + "description": "Scope restricting the rows returned" + }, + "filter": { + "anyOf": [ + { + "$ref": "#/components/schemas/KernelHistoryFilter" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Filter conditions" + }, + "order": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/KernelHistoryOrder" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Order specifications", + "title": "Order" + }, + "first": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Cursor pagination: number of items", + "title": "First" + }, + "after": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Cursor pagination: after cursor", + "title": "After" + }, + "last": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Cursor pagination: last N items", + "title": "Last" + }, + "before": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Cursor pagination: before cursor", + "title": "Before" + }, + "limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Offset pagination: maximum items", + "title": "Limit" + }, + "offset": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Offset pagination: number to skip", + "title": "Offset" + } + }, + "required": [ + "scope" + ], + "title": "ScopedSearchKernelHistoriesInput", + "type": "object" + }, "DeploymentHistoryFilter": { "description": "Filter for deployment history.", "properties": { @@ -47767,6 +48239,64 @@ "description": "Search session scheduling histories scoped to a specific session.\n\n**Preconditions:**\n* Superadmin privilege required.\n" } }, + "/v2/scheduling-history/kernels/admin/search": { + "post": { + "operationId": "v2/scheduling-history.admin_search_kernel_history", + "tags": [ + "v2/scheduling-history" + ], + "responses": { + "200": { + "description": "Successful response" + } + }, + "security": [ + { + "TokenAuth": [] + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AdminSearchKernelHistoriesInput" + } + } + } + }, + "parameters": [], + "description": "Search kernel scheduling histories with admin scope.\n\n**Preconditions:**\n* Superadmin privilege required.\n" + } + }, + "/v2/scheduling-history/kernels/scoped/search": { + "post": { + "operationId": "v2/scheduling-history.scoped_search_kernel_history", + "tags": [ + "v2/scheduling-history" + ], + "responses": { + "200": { + "description": "Successful response" + } + }, + "security": [ + { + "TokenAuth": [] + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ScopedSearchKernelHistoriesInput" + } + } + } + }, + "parameters": [], + "description": "Search kernel scheduling histories under a non-admin scope.\n\n**Preconditions:**\n* User privilege required.\n" + } + }, "/v2/scheduling-history/deployments/search": { "post": { "operationId": "v2/scheduling-history.admin_search_deployment_history", diff --git a/src/ai/backend/common/dto/manager/v2/scheduling_history/__init__.py b/src/ai/backend/common/dto/manager/v2/scheduling_history/__init__.py index d2da4324249..340a41f4f02 100644 --- a/src/ai/backend/common/dto/manager/v2/scheduling_history/__init__.py +++ b/src/ai/backend/common/dto/manager/v2/scheduling_history/__init__.py @@ -13,6 +13,7 @@ KernelHistoryOrder, RouteHistoryFilter, RouteHistoryOrder, + ScopedSearchKernelHistoriesInput, SearchDeploymentHistoryInput, SearchRouteHistoryInput, SearchSessionHistoryInput, @@ -21,7 +22,6 @@ ) from ai.backend.common.dto.manager.v2.scheduling_history.response import ( AdminSearchDeploymentHistoriesPayload, - AdminSearchKernelHistoriesPayload, AdminSearchRouteHistoriesPayload, AdminSearchSessionHistoriesPayload, DeploymentHistoryNode, @@ -30,12 +30,14 @@ ListRouteHistoryPayload, ListSessionHistoryPayload, RouteHistoryNode, + SearchKernelHistoriesPayload, SessionHistoryNode, ) from ai.backend.common.dto.manager.v2.scheduling_history.types import ( DeploymentHistoryOrderField, DeploymentHistoryScopeDTO, KernelHistoryOrderField, + KernelHistoryScopeDTO, OrderDirection, RouteHistoryOrderField, RouteHistoryScopeDTO, @@ -50,6 +52,7 @@ "DeploymentHistoryOrderField", "DeploymentHistoryScopeDTO", "KernelHistoryOrderField", + "KernelHistoryScopeDTO", "OrderDirection", "RouteHistoryOrderField", "RouteHistoryScopeDTO", @@ -59,7 +62,6 @@ "SubStepResultInfo", # Input models (request) "AdminSearchDeploymentHistoriesInput", - "AdminSearchKernelHistoriesInput", "AdminSearchRouteHistoriesInput", "AdminSearchSessionHistoriesInput", "DeploymentHistoryFilter", @@ -68,14 +70,15 @@ "KernelHistoryOrder", "RouteHistoryFilter", "RouteHistoryOrder", + "ScopedSearchKernelHistoriesInput", "SearchDeploymentHistoryInput", + "AdminSearchKernelHistoriesInput", "SearchRouteHistoryInput", "SearchSessionHistoryInput", "SessionHistoryFilter", "SessionHistoryOrder", # Response models "AdminSearchDeploymentHistoriesPayload", - "AdminSearchKernelHistoriesPayload", "AdminSearchRouteHistoriesPayload", "AdminSearchSessionHistoriesPayload", "DeploymentHistoryNode", @@ -84,5 +87,6 @@ "ListRouteHistoryPayload", "ListSessionHistoryPayload", "RouteHistoryNode", + "SearchKernelHistoriesPayload", "SessionHistoryNode", ) diff --git a/src/ai/backend/common/dto/manager/v2/scheduling_history/request.py b/src/ai/backend/common/dto/manager/v2/scheduling_history/request.py index 7cb91afc557..61fd5bdf5cd 100644 --- a/src/ai/backend/common/dto/manager/v2/scheduling_history/request.py +++ b/src/ai/backend/common/dto/manager/v2/scheduling_history/request.py @@ -12,6 +12,7 @@ from .types import ( DeploymentHistoryOrderField, KernelHistoryOrderField, + KernelHistoryScopeDTO, OrderDirection, RouteHistoryOrderField, SchedulingResultType, @@ -20,7 +21,6 @@ __all__ = ( "AdminSearchDeploymentHistoriesInput", - "AdminSearchKernelHistoriesInput", "AdminSearchRouteHistoriesInput", "AdminSearchSessionHistoriesInput", "DeploymentHistoryFilter", @@ -30,7 +30,9 @@ "RouteHistoryFilter", "RouteHistoryOrder", "SchedulingResultFilter", + "ScopedSearchKernelHistoriesInput", "SearchDeploymentHistoryInput", + "AdminSearchKernelHistoriesInput", "SearchRouteHistoryInput", "SearchSessionHistoryInput", "SessionHistoryFilter", @@ -230,6 +232,20 @@ class AdminSearchKernelHistoriesInput(BaseRequestModel): offset: int | None = Field(default=None, description="Offset pagination: number to skip") +class ScopedSearchKernelHistoriesInput(BaseRequestModel): + """Input for searching kernel scheduling histories under a non-admin scope.""" + + scope: KernelHistoryScopeDTO = Field(description="Scope restricting the rows returned") + filter: KernelHistoryFilter | None = Field(default=None, description="Filter conditions") + order: list[KernelHistoryOrder] | None = Field(default=None, description="Order specifications") + first: int | None = Field(default=None, description="Cursor pagination: number of items") + after: str | None = Field(default=None, description="Cursor pagination: after cursor") + last: int | None = Field(default=None, description="Cursor pagination: last N items") + before: str | None = Field(default=None, description="Cursor pagination: before cursor") + limit: int | None = Field(default=None, description="Offset pagination: maximum items") + offset: int | None = Field(default=None, description="Offset pagination: number to skip") + + class AdminSearchDeploymentHistoriesInput(BaseRequestModel): """Input for admin search of deployment histories.""" diff --git a/src/ai/backend/common/dto/manager/v2/scheduling_history/response.py b/src/ai/backend/common/dto/manager/v2/scheduling_history/response.py index f16a3556114..b2bc7664f59 100644 --- a/src/ai/backend/common/dto/manager/v2/scheduling_history/response.py +++ b/src/ai/backend/common/dto/manager/v2/scheduling_history/response.py @@ -16,7 +16,6 @@ __all__ = ( "AdminSearchDeploymentHistoriesPayload", - "AdminSearchKernelHistoriesPayload", "AdminSearchRouteHistoriesPayload", "AdminSearchSessionHistoriesPayload", "DeploymentHistoryNode", @@ -25,6 +24,7 @@ "ListRouteHistoryPayload", "ListSessionHistoryPayload", "RouteHistoryNode", + "SearchKernelHistoriesPayload", "SessionHistoryNode", ) @@ -149,7 +149,7 @@ class AdminSearchSessionHistoriesPayload(BaseResponseModel): has_previous_page: bool = Field(description="Whether there is a previous page.") -class AdminSearchKernelHistoriesPayload(BaseResponseModel): +class SearchKernelHistoriesPayload(BaseResponseModel): """Payload for admin and scoped search of kernel scheduling histories.""" items: list[KernelHistoryNode] = Field(description="List of kernel history nodes.") diff --git a/src/ai/backend/common/dto/manager/v2/scheduling_history/types.py b/src/ai/backend/common/dto/manager/v2/scheduling_history/types.py index afd11626da6..3a56b0b17c2 100644 --- a/src/ai/backend/common/dto/manager/v2/scheduling_history/types.py +++ b/src/ai/backend/common/dto/manager/v2/scheduling_history/types.py @@ -17,6 +17,7 @@ "DeploymentHistoryOrderField", "DeploymentHistoryScopeDTO", "KernelHistoryOrderField", + "KernelHistoryScopeDTO", "OrderDirection", "RouteHistoryOrderField", "RouteHistoryScopeDTO", @@ -84,6 +85,12 @@ class SessionHistoryScopeDTO(BaseRequestModel): session_id: UUID = Field(description="Session ID to get history for.") +class KernelHistoryScopeDTO(BaseRequestModel): + """Scope for kernel scheduling history queries.""" + + kernel_id: UUID = Field(description="Kernel ID to get history for.") + + class DeploymentHistoryScopeDTO(BaseRequestModel): """Scope for deployment scheduling history queries.""" diff --git a/src/ai/backend/manager/api/adapters/scheduling_history/adapter.py b/src/ai/backend/manager/api/adapters/scheduling_history/adapter.py index e3bdbc2d91a..5a3832f0712 100644 --- a/src/ai/backend/manager/api/adapters/scheduling_history/adapter.py +++ b/src/ai/backend/manager/api/adapters/scheduling_history/adapter.py @@ -7,12 +7,16 @@ from ai.backend.common.dto.manager.v2.scheduling_history.request import ( AdminSearchDeploymentHistoriesInput, + AdminSearchKernelHistoriesInput, AdminSearchRouteHistoriesInput, AdminSearchSessionHistoriesInput, DeploymentHistoryFilter, DeploymentHistoryOrder, + KernelHistoryFilter, + KernelHistoryOrder, RouteHistoryFilter, RouteHistoryOrder, + ScopedSearchKernelHistoriesInput, SessionHistoryFilter, SessionHistoryOrder, ) @@ -21,14 +25,18 @@ AdminSearchRouteHistoriesPayload, AdminSearchSessionHistoriesPayload, DeploymentHistoryNode, + KernelHistoryNode, RouteHistoryNode, + SearchKernelHistoriesPayload, SessionHistoryNode, ) from ai.backend.common.dto.manager.v2.scheduling_history.types import SubStepResultInfo from ai.backend.common.identifier.replica import ReplicaID +from ai.backend.common.types import KernelId from ai.backend.manager.api.adapter_options.pagination.pagination import PaginationSpec from ai.backend.manager.api.adapters.base import BaseAdapter from ai.backend.manager.data.deployment.types import DeploymentHistoryData, RouteHistoryData +from ai.backend.manager.data.kernel.types import KernelSchedulingHistoryData from ai.backend.manager.data.session.types import ( SchedulingResult, SessionSchedulingHistoryData, @@ -37,6 +45,7 @@ from ai.backend.manager.models.clauses import QueryCondition, QueryOrder from ai.backend.manager.models.scheduling_history.conditions import ( DeploymentHistoryConditions, + KernelSchedulingHistoryConditions, RouteHistoryConditions, SessionSchedulingHistoryConditions, ) @@ -44,6 +53,9 @@ DEPLOYMENT_DEFAULT_BACKWARD_ORDER, DEPLOYMENT_DEFAULT_FORWARD_ORDER, DEPLOYMENT_TIEBREAKER_ORDER, + KERNEL_DEFAULT_BACKWARD_ORDER, + KERNEL_DEFAULT_FORWARD_ORDER, + KERNEL_TIEBREAKER_ORDER, ROUTE_DEFAULT_BACKWARD_ORDER, ROUTE_DEFAULT_FORWARD_ORDER, ROUTE_TIEBREAKER_ORDER, @@ -51,6 +63,7 @@ SESSION_DEFAULT_FORWARD_ORDER, SESSION_TIEBREAKER_ORDER, resolve_deployment_order, + resolve_kernel_order, resolve_route_order, resolve_session_order, ) @@ -71,6 +84,12 @@ from ai.backend.manager.services.scheduling_history.actions.search_deployment_scoped_history import ( SearchDeploymentScopedHistoryAction, ) +from ai.backend.manager.services.scheduling_history.actions.search_kernel_history import ( + SearchKernelHistoryAction, +) +from ai.backend.manager.services.scheduling_history.actions.search_kernel_scoped_history import ( + SearchKernelScopedHistoryAction, +) from ai.backend.manager.services.scheduling_history.actions.search_route_history import ( SearchRouteHistoryAction, ) @@ -92,6 +111,14 @@ tiebreaker_order=SESSION_TIEBREAKER_ORDER, ) +_KERNEL_HISTORY_PAGINATION_SPEC = PaginationSpec( + forward_order=KERNEL_DEFAULT_FORWARD_ORDER, + backward_order=KERNEL_DEFAULT_BACKWARD_ORDER, + forward_condition_factory=KernelSchedulingHistoryConditions.by_cursor_forward, + backward_condition_factory=KernelSchedulingHistoryConditions.by_cursor_backward, + tiebreaker_order=KERNEL_TIEBREAKER_ORDER, +) + _DEPLOYMENT_HISTORY_PAGINATION_SPEC = PaginationSpec( forward_order=DEPLOYMENT_DEFAULT_FORWARD_ORDER, backward_order=DEPLOYMENT_DEFAULT_BACKWARD_ORDER, @@ -348,6 +375,194 @@ def _convert_session_filter(self, filter: SessionHistoryFilter) -> list[QueryCon def _convert_session_orders(order: list[SessionHistoryOrder]) -> list[QueryOrder]: return [resolve_session_order(o.field, o.direction) for o in order] + # ========== Kernel History ========== + + async def admin_search_kernel_history( + self, + input: AdminSearchKernelHistoriesInput, + ) -> SearchKernelHistoriesPayload: + """Search kernel scheduling histories (admin, no scope).""" + conditions = self._convert_kernel_filter(input.filter) if input.filter else [] + orders = self._convert_kernel_orders(input.order) if input.order else [] + querier = self._build_querier( + conditions=conditions, + orders=orders, + pagination_spec=_KERNEL_HISTORY_PAGINATION_SPEC, + first=input.first, + after=input.after, + last=input.last, + before=input.before, + limit=input.limit, + offset=input.offset, + ) + action_result = ( + await self._processors.scheduling_history.search_kernel_history.wait_for_complete( + SearchKernelHistoryAction(querier=querier) + ) + ) + return SearchKernelHistoriesPayload( + items=[self._kernel_data_to_dto(h) for h in action_result.items], + total_count=action_result.total_count, + has_next_page=action_result.has_next_page, + has_previous_page=action_result.has_previous_page, + ) + + async def scoped_search_kernel_history( + self, + input: ScopedSearchKernelHistoriesInput, + ) -> SearchKernelHistoriesPayload: + """Search kernel scheduling histories under a non-admin scope.""" + conditions = self._convert_kernel_filter(input.filter) if input.filter else [] + orders = self._convert_kernel_orders(input.order) if input.order else [] + querier = self._build_querier( + conditions=conditions, + orders=orders, + pagination_spec=_KERNEL_HISTORY_PAGINATION_SPEC, + first=input.first, + after=input.after, + last=input.last, + before=input.before, + limit=input.limit, + offset=input.offset, + ) + action_result = await self._processors.scheduling_history.search_kernel_scoped_history.wait_for_complete( + SearchKernelScopedHistoryAction( + kernel_id=KernelId(input.scope.kernel_id), querier=querier + ) + ) + return SearchKernelHistoriesPayload( + items=[self._kernel_data_to_dto(h) for h in action_result.items], + total_count=action_result.total_count, + has_next_page=action_result.has_next_page, + has_previous_page=action_result.has_previous_page, + ) + + def _convert_kernel_filter(self, filter: KernelHistoryFilter) -> list[QueryCondition]: + conditions: list[QueryCondition] = [] + if filter.id is not None: + condition = self.convert_uuid_filter( + filter.id, + equals_factory=KernelSchedulingHistoryConditions.by_id_filter, + in_factory=KernelSchedulingHistoryConditions.by_id_in, + ) + if condition is not None: + conditions.append(condition) + if filter.kernel_id is not None: + condition = self.convert_uuid_filter( + filter.kernel_id, + equals_factory=KernelSchedulingHistoryConditions.by_kernel_id_filter, + in_factory=KernelSchedulingHistoryConditions.by_kernel_id_in, + ) + if condition is not None: + conditions.append(condition) + if filter.session_id is not None: + condition = self.convert_uuid_filter( + filter.session_id, + equals_factory=KernelSchedulingHistoryConditions.by_session_id_filter, + in_factory=KernelSchedulingHistoryConditions.by_session_id_in, + ) + if condition is not None: + conditions.append(condition) + if filter.phase is not None: + condition = self.convert_string_filter( + filter.phase, + contains_factory=KernelSchedulingHistoryConditions.by_phase_contains, + equals_factory=KernelSchedulingHistoryConditions.by_phase_equals, + starts_with_factory=KernelSchedulingHistoryConditions.by_phase_starts_with, + ends_with_factory=KernelSchedulingHistoryConditions.by_phase_ends_with, + in_factory=KernelSchedulingHistoryConditions.by_phase_in, + ) + if condition is not None: + conditions.append(condition) + if filter.from_status is not None and filter.from_status: + conditions.append( + KernelSchedulingHistoryConditions.by_from_statuses(filter.from_status) + ) + if filter.to_status is not None and filter.to_status: + conditions.append(KernelSchedulingHistoryConditions.by_to_statuses(filter.to_status)) + if filter.result is not None: + r = filter.result + if r.equals is not None: + conditions.append( + KernelSchedulingHistoryConditions.by_result(SchedulingResult(r.equals)) + ) + if r.in_ is not None and r.in_: + conditions.append( + KernelSchedulingHistoryConditions.by_results([ + SchedulingResult(v) for v in r.in_ + ]) + ) + if r.not_equals is not None: + conditions.append( + KernelSchedulingHistoryConditions.by_result_not_equals( + SchedulingResult(r.not_equals) + ) + ) + if r.not_in is not None and r.not_in: + conditions.append( + KernelSchedulingHistoryConditions.by_result_not_in([ + SchedulingResult(v) for v in r.not_in + ]) + ) + if filter.error_code is not None: + condition = self.convert_string_filter( + filter.error_code, + contains_factory=KernelSchedulingHistoryConditions.by_error_code_contains, + equals_factory=KernelSchedulingHistoryConditions.by_error_code_equals, + starts_with_factory=KernelSchedulingHistoryConditions.by_error_code_starts_with, + ends_with_factory=KernelSchedulingHistoryConditions.by_error_code_ends_with, + in_factory=KernelSchedulingHistoryConditions.by_error_code_in, + ) + if condition is not None: + conditions.append(condition) + if filter.message is not None: + condition = self.convert_string_filter( + filter.message, + contains_factory=KernelSchedulingHistoryConditions.by_message_contains, + equals_factory=KernelSchedulingHistoryConditions.by_message_equals, + starts_with_factory=KernelSchedulingHistoryConditions.by_message_starts_with, + ends_with_factory=KernelSchedulingHistoryConditions.by_message_ends_with, + in_factory=KernelSchedulingHistoryConditions.by_message_in, + ) + if condition is not None: + conditions.append(condition) + if filter.created_at is not None: + condition = filter.created_at.build_query_condition( + before_factory=KernelSchedulingHistoryConditions.by_created_at_before, + after_factory=KernelSchedulingHistoryConditions.by_created_at_after, + equals_factory=KernelSchedulingHistoryConditions.by_created_at_equals, + ) + if condition is not None: + conditions.append(condition) + if filter.updated_at is not None: + condition = filter.updated_at.build_query_condition( + before_factory=KernelSchedulingHistoryConditions.by_updated_at_before, + after_factory=KernelSchedulingHistoryConditions.by_updated_at_after, + equals_factory=KernelSchedulingHistoryConditions.by_updated_at_equals, + ) + if condition is not None: + conditions.append(condition) + if filter.AND: + for sub in filter.AND: + conditions.extend(self._convert_kernel_filter(sub)) + if filter.OR: + or_conds: list[QueryCondition] = [] + for sub in filter.OR: + or_conds.extend(self._convert_kernel_filter(sub)) + if or_conds: + conditions.append(combine_conditions_or(or_conds)) + if filter.NOT: + not_conds: list[QueryCondition] = [] + for sub in filter.NOT: + not_conds.extend(self._convert_kernel_filter(sub)) + if not_conds: + conditions.append(negate_conditions(not_conds)) + return conditions + + @staticmethod + def _convert_kernel_orders(order: list[KernelHistoryOrder]) -> list[QueryOrder]: + return [resolve_kernel_order(o.field, o.direction) for o in order] + # ========== Deployment History ========== async def admin_search_deployment_history( @@ -722,6 +937,23 @@ def _session_data_to_dto(data: SessionSchedulingHistoryData) -> SessionHistoryNo updated_at=data.updated_at, ) + @staticmethod + def _kernel_data_to_dto(data: KernelSchedulingHistoryData) -> KernelHistoryNode: + return KernelHistoryNode( + id=data.id, + kernel_id=data.kernel_id, + session_id=data.session_id, + phase=data.phase, + from_status=data.from_status.value if data.from_status else None, + to_status=data.to_status.value if data.to_status else None, + result=data.result.value, + error_code=data.error_code, + message=data.message, + attempts=data.attempts, + created_at=data.created_at, + updated_at=data.updated_at, + ) + @staticmethod def _deployment_data_to_dto(data: DeploymentHistoryData) -> DeploymentHistoryNode: return DeploymentHistoryNode( diff --git a/src/ai/backend/manager/api/rest/v2/scheduling_history/handler.py b/src/ai/backend/manager/api/rest/v2/scheduling_history/handler.py index 0fe87b2a417..60a969be281 100644 --- a/src/ai/backend/manager/api/rest/v2/scheduling_history/handler.py +++ b/src/ai/backend/manager/api/rest/v2/scheduling_history/handler.py @@ -9,8 +9,10 @@ from ai.backend.common.api_handlers import APIResponse, BodyParam, PathParam from ai.backend.common.dto.manager.v2.scheduling_history.request import ( AdminSearchDeploymentHistoriesInput, + AdminSearchKernelHistoriesInput, AdminSearchRouteHistoriesInput, AdminSearchSessionHistoriesInput, + ScopedSearchKernelHistoriesInput, ) from ai.backend.logging import BraceStyleAdapter from ai.backend.manager.api.rest.v2.path_params import ( @@ -55,6 +57,28 @@ async def admin_session_scoped_search( ) return APIResponse.build(status_code=HTTPStatus.OK, response_model=result) + # ========== Kernel History ========== + + async def admin_search_kernel_history( + self, + body: BodyParam[AdminSearchKernelHistoriesInput], + ) -> APIResponse: + """Search kernel scheduling histories with admin scope.""" + result = await self._adapter.admin_search_kernel_history( + body.parsed, + ) + return APIResponse.build(status_code=HTTPStatus.OK, response_model=result) + + async def scoped_search_kernel_history( + self, + body: BodyParam[ScopedSearchKernelHistoriesInput], + ) -> APIResponse: + """Search kernel scheduling histories under a non-admin scope.""" + result = await self._adapter.scoped_search_kernel_history( + body.parsed, + ) + return APIResponse.build(status_code=HTTPStatus.OK, response_model=result) + # ========== Deployment History ========== async def admin_search_deployment_history( diff --git a/src/ai/backend/manager/api/rest/v2/scheduling_history/registry.py b/src/ai/backend/manager/api/rest/v2/scheduling_history/registry.py index 2f616161f21..84282277470 100644 --- a/src/ai/backend/manager/api/rest/v2/scheduling_history/registry.py +++ b/src/ai/backend/manager/api/rest/v2/scheduling_history/registry.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING -from ai.backend.manager.api.rest.middleware.auth import superadmin_required +from ai.backend.manager.api.rest.middleware.auth import auth_required, superadmin_required from ai.backend.manager.api.rest.routing import RouteRegistry from .handler import V2SchedulingHistoryHandler @@ -34,6 +34,20 @@ def register_v2_scheduling_history_routes( middlewares=[superadmin_required], ) + # Kernel history + registry.add( + "POST", + "/kernels/admin/search", + handler.admin_search_kernel_history, + middlewares=[superadmin_required], + ) + registry.add( + "POST", + "/kernels/scoped/search", + handler.scoped_search_kernel_history, + middlewares=[auth_required], + ) + # Deployment history registry.add( "POST",