diff --git a/changes/12989.feature.md b/changes/12989.feature.md new file mode 100644 index 00000000000..ff3d2cb2229 --- /dev/null +++ b/changes/12989.feature.md @@ -0,0 +1 @@ +Add `adminKernelSchedulingHistories` and `scopedKernelSchedulingHistories` GraphQL queries, and a `KernelV2.schedulingHistories` field. diff --git a/docs/manager/graphql-reference/supergraph.graphql b/docs/manager/graphql-reference/supergraph.graphql index e5dffaa0ed4..e3c42799bd5 100644 --- a/docs/manager/graphql-reference/supergraph.graphql +++ b/docs/manager/graphql-reference/supergraph.graphql @@ -8508,6 +8508,100 @@ enum KernelResourceAllocationOrderField USED @join__enumValue(graph: STRAWBERRY) } +"""Added in 26.8.0. Kernel scheduling history record.""" +type KernelSchedulingHistory implements Node + @join__implements(graph: STRAWBERRY, interface: "Node") + @join__type(graph: STRAWBERRY) +{ + """The Globally Unique ID of this object""" + id: ID! + kernelId: ID! + sessionId: ID! + phase: String! + fromStatus: String + toStatus: String + result: SchedulingResult! + errorCode: String + message: String + attempts: Int! + createdAt: DateTime! + updatedAt: DateTime! +} + +"""Added in 26.8.0. Kernel scheduling history connection.""" +type KernelSchedulingHistoryConnection + @join__type(graph: STRAWBERRY) +{ + """Pagination data for this connection""" + pageInfo: PageInfo! + + """Contains the nodes in this connection""" + edges: [KernelSchedulingHistoryEdge!]! + count: Int! +} + +"""An edge in a connection.""" +type KernelSchedulingHistoryEdge + @join__type(graph: STRAWBERRY) +{ + """A cursor for use in pagination""" + cursor: String! + + """The item at the end of the edge""" + node: KernelSchedulingHistory! +} + +"""Added in 26.8.0. Filter for kernel scheduling history""" +input KernelSchedulingHistoryFilter + @join__type(graph: STRAWBERRY) +{ + id: UUIDFilter = null + kernelId: UUIDFilter = null + sessionId: UUIDFilter = null + phase: StringFilter = null + fromStatus: [String!] = null + toStatus: [String!] = null + result: SchedulingResultFilter = null + errorCode: StringFilter = null + message: StringFilter = null + createdAt: DateTimeFilter = null + updatedAt: DateTimeFilter = null + AND: [KernelSchedulingHistoryFilter!] = null + OR: [KernelSchedulingHistoryFilter!] = null + NOT: [KernelSchedulingHistoryFilter!] = null +} + +"""Added in 26.8.0. Order by specification for kernel scheduling history""" +input KernelSchedulingHistoryOrderBy + @join__type(graph: STRAWBERRY) +{ + field: KernelSchedulingHistoryOrderField! + direction: OrderDirection! = DESC +} + +""" +Added in 26.8.0. Fields available for ordering kernel scheduling history +""" +enum KernelSchedulingHistoryOrderField + @join__type(graph: STRAWBERRY) +{ + CREATED_AT @join__enumValue(graph: STRAWBERRY) + UPDATED_AT @join__enumValue(graph: STRAWBERRY) + PHASE @join__enumValue(graph: STRAWBERRY) + FROM_STATUS @join__enumValue(graph: STRAWBERRY) + TO_STATUS @join__enumValue(graph: STRAWBERRY) + RESULT @join__enumValue(graph: STRAWBERRY) + ATTEMPTS @join__enumValue(graph: STRAWBERRY) +} + +"""Added in 26.8.0. Scope for kernel scheduling history query""" +input KernelScope + @join__type(graph: STRAWBERRY) +{ + """Kernel ID to get history for""" + kernelId: UUID! +} + """ Added in 26.2.0. Represents a kernel (compute container) in Backend.AI. """ @@ -8572,6 +8666,11 @@ type KernelV2 implements Node """Added in 26.3.0. Per-slot resource allocation for this kernel.""" resourceAllocations(filter: KernelResourceAllocationFilter = null, orderBy: [KernelResourceAllocationOrderBy!] = null, first: Int = null, after: String = null, last: Int = null, before: String = null, limit: Int = null, offset: Int = null): ResourceAllocationConnection + + """ + Added in 26.8.0. Scheduling history of this kernel with pagination support. + """ + schedulingHistories(filter: KernelSchedulingHistoryFilter = null, orderBy: [KernelSchedulingHistoryOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): KernelSchedulingHistoryConnection } """ @@ -14709,6 +14808,9 @@ type Query """Added in 26.4.2. List session scheduling history (admin only)""" adminSessionSchedulingHistories(filter: SessionSchedulingHistoryFilter = null, orderBy: [SessionSchedulingHistoryOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): SessionSchedulingHistoryConnection @join__field(graph: STRAWBERRY) + """Added in 26.8.0. List kernel scheduling history (superadmin only)""" + adminKernelSchedulingHistories(filter: KernelSchedulingHistoryFilter = null, orderBy: [KernelSchedulingHistoryOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): KernelSchedulingHistoryConnection @join__field(graph: STRAWBERRY) + """Added in 25.16.0. List all deployments (superadmin only).""" adminDeployments(filter: DeploymentFilter = null, orderBy: [DeploymentOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): ModelDeploymentConnection @join__field(graph: STRAWBERRY) @@ -14990,6 +15092,9 @@ type Query """Added in 26.2.0. Get scheduling history for a specific session.""" sessionScopedSchedulingHistories(scope: SessionScope!, filter: SessionSchedulingHistoryFilter = null, orderBy: [SessionSchedulingHistoryOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): SessionSchedulingHistoryConnection @join__field(graph: STRAWBERRY) + """Added in 26.8.0. Get scheduling history for a specific kernel.""" + scopedKernelSchedulingHistories(scope: KernelScope!, filter: KernelSchedulingHistoryFilter = null, orderBy: [KernelSchedulingHistoryOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): KernelSchedulingHistoryConnection @join__field(graph: STRAWBERRY) + """Added in 26.2.0. Get scheduling history for a specific deployment.""" deploymentScopedSchedulingHistories(scope: DeploymentScope!, filter: DeploymentHistoryFilter = null, orderBy: [DeploymentHistoryOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): DeploymentHistoryConnection @join__field(graph: STRAWBERRY) diff --git a/docs/manager/graphql-reference/v2-schema.graphql b/docs/manager/graphql-reference/v2-schema.graphql index 65e9cb0e8fe..922b9681712 100644 --- a/docs/manager/graphql-reference/v2-schema.graphql +++ b/docs/manager/graphql-reference/v2-schema.graphql @@ -5632,6 +5632,85 @@ enum KernelResourceAllocationOrderField { USED } +"""Added in 26.8.0. Kernel scheduling history record.""" +type KernelSchedulingHistory implements Node { + """The Globally Unique ID of this object""" + id: ID! + kernelId: ID! + sessionId: ID! + phase: String! + fromStatus: String + toStatus: String + result: SchedulingResult! + errorCode: String + message: String + attempts: Int! + createdAt: DateTime! + updatedAt: DateTime! +} + +"""Added in 26.8.0. Kernel scheduling history connection.""" +type KernelSchedulingHistoryConnection { + """Pagination data for this connection""" + pageInfo: PageInfo! + + """Contains the nodes in this connection""" + edges: [KernelSchedulingHistoryEdge!]! + count: Int! +} + +"""An edge in a connection.""" +type KernelSchedulingHistoryEdge { + """A cursor for use in pagination""" + cursor: String! + + """The item at the end of the edge""" + node: KernelSchedulingHistory! +} + +"""Added in 26.8.0. Filter for kernel scheduling history""" +input KernelSchedulingHistoryFilter { + id: UUIDFilter = null + kernelId: UUIDFilter = null + sessionId: UUIDFilter = null + phase: StringFilter = null + fromStatus: [String!] = null + toStatus: [String!] = null + result: SchedulingResultFilter = null + errorCode: StringFilter = null + message: StringFilter = null + createdAt: DateTimeFilter = null + updatedAt: DateTimeFilter = null + AND: [KernelSchedulingHistoryFilter!] = null + OR: [KernelSchedulingHistoryFilter!] = null + NOT: [KernelSchedulingHistoryFilter!] = null +} + +"""Added in 26.8.0. Order by specification for kernel scheduling history""" +input KernelSchedulingHistoryOrderBy { + field: KernelSchedulingHistoryOrderField! + direction: OrderDirection! = DESC +} + +""" +Added in 26.8.0. Fields available for ordering kernel scheduling history +""" +enum KernelSchedulingHistoryOrderField { + CREATED_AT + UPDATED_AT + PHASE + FROM_STATUS + TO_STATUS + RESULT + ATTEMPTS +} + +"""Added in 26.8.0. Scope for kernel scheduling history query""" +input KernelScope { + """Kernel ID to get history for""" + kernelId: UUID! +} + """ Added in 26.2.0. Represents a kernel (compute container) in Backend.AI. """ @@ -5693,6 +5772,11 @@ type KernelV2 implements Node { """Added in 26.3.0. Per-slot resource allocation for this kernel.""" resourceAllocations(filter: KernelResourceAllocationFilter = null, orderBy: [KernelResourceAllocationOrderBy!] = null, first: Int = null, after: String = null, last: Int = null, before: String = null, limit: Int = null, offset: Int = null): ResourceAllocationConnection + + """ + Added in 26.8.0. Scheduling history of this kernel with pagination support. + """ + schedulingHistories(filter: KernelSchedulingHistoryFilter = null, orderBy: [KernelSchedulingHistoryOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): KernelSchedulingHistoryConnection } """ @@ -9748,6 +9832,9 @@ type Query { """Added in 26.4.2. List session scheduling history (admin only)""" adminSessionSchedulingHistories(filter: SessionSchedulingHistoryFilter = null, orderBy: [SessionSchedulingHistoryOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): SessionSchedulingHistoryConnection + """Added in 26.8.0. List kernel scheduling history (superadmin only)""" + adminKernelSchedulingHistories(filter: KernelSchedulingHistoryFilter = null, orderBy: [KernelSchedulingHistoryOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): KernelSchedulingHistoryConnection + """Added in 25.16.0. List all deployments (superadmin only).""" adminDeployments(filter: DeploymentFilter = null, orderBy: [DeploymentOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): ModelDeploymentConnection @@ -10029,6 +10116,9 @@ type Query { """Added in 26.2.0. Get scheduling history for a specific session.""" sessionScopedSchedulingHistories(scope: SessionScope!, filter: SessionSchedulingHistoryFilter = null, orderBy: [SessionSchedulingHistoryOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): SessionSchedulingHistoryConnection + """Added in 26.8.0. Get scheduling history for a specific kernel.""" + scopedKernelSchedulingHistories(scope: KernelScope!, filter: KernelSchedulingHistoryFilter = null, orderBy: [KernelSchedulingHistoryOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): KernelSchedulingHistoryConnection + """Added in 26.2.0. Get scheduling history for a specific deployment.""" deploymentScopedSchedulingHistories(scope: DeploymentScope!, filter: DeploymentHistoryFilter = null, orderBy: [DeploymentHistoryOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): DeploymentHistoryConnection diff --git a/docs/manager/rest-reference/openapi.json b/docs/manager/rest-reference/openapi.json index c3b57c8d24a..b466a7c980f 100644 --- a/docs/manager/rest-reference/openapi.json +++ b/docs/manager/rest-reference/openapi.json @@ -25127,7 +25127,12 @@ "description": "Fields available for ordering kernel scheduling history.", "enum": [ "created_at", - "updated_at" + "updated_at", + "phase", + "from_status", + "to_status", + "result", + "attempts" ], "title": "KernelHistoryOrderField", "type": "string" diff --git a/src/ai/backend/common/dto/manager/v2/scheduling_history/types.py b/src/ai/backend/common/dto/manager/v2/scheduling_history/types.py index 3a56b0b17c2..b706494d945 100644 --- a/src/ai/backend/common/dto/manager/v2/scheduling_history/types.py +++ b/src/ai/backend/common/dto/manager/v2/scheduling_history/types.py @@ -52,6 +52,11 @@ class KernelHistoryOrderField(StrEnum): CREATED_AT = "created_at" UPDATED_AT = "updated_at" + PHASE = "phase" + FROM_STATUS = "from_status" + TO_STATUS = "to_status" + RESULT = "result" + ATTEMPTS = "attempts" class DeploymentHistoryOrderField(StrEnum): diff --git a/src/ai/backend/manager/api/adapters/scheduling_history/adapter.py b/src/ai/backend/manager/api/adapters/scheduling_history/adapter.py index 5a3832f0712..96e4a48ba58 100644 --- a/src/ai/backend/manager/api/adapters/scheduling_history/adapter.py +++ b/src/ai/backend/manager/api/adapters/scheduling_history/adapter.py @@ -31,6 +31,7 @@ SessionHistoryNode, ) from ai.backend.common.dto.manager.v2.scheduling_history.types import SubStepResultInfo +from ai.backend.common.identifier.kernel_scheduling_history import KernelSchedulingHistoryID from ai.backend.common.identifier.replica import ReplicaID from ai.backend.common.types import KernelId from ai.backend.manager.api.adapter_options.pagination.pagination import PaginationSpec @@ -162,6 +163,27 @@ async def batch_load_session_histories_by_ids( history_map = {h.id: self._session_data_to_dto(h) for h in action_result.histories} return [history_map.get(history_id) for history_id in ids] + async def batch_load_kernel_histories_by_ids( + self, ids: Sequence[KernelSchedulingHistoryID] + ) -> list[KernelHistoryNode | None]: + """Batch load kernel scheduling histories by their IDs for DataLoader use. + + Returns KernelHistoryNode DTOs in the same order as the input ids list. + """ + if not ids: + return [] + querier = BatchQuerier( + pagination=OffsetPagination(limit=len(ids)), + conditions=[KernelSchedulingHistoryConditions.by_ids(ids)], + ) + action_result = ( + await self._processors.scheduling_history.search_kernel_history.wait_for_complete( + SearchKernelHistoryAction(querier=querier) + ) + ) + history_map = {h.id: self._kernel_data_to_dto(h) for h in action_result.items} + return [history_map.get(history_id) for history_id in ids] + async def batch_load_deployment_histories_by_ids( self, ids: Sequence[UUID] ) -> list[DeploymentHistoryNode | None]: diff --git a/src/ai/backend/manager/api/gql/data_loader/data_loaders.py b/src/ai/backend/manager/api/gql/data_loader/data_loaders.py index 19b10b4d58f..4912dd950c4 100644 --- a/src/ai/backend/manager/api/gql/data_loader/data_loaders.py +++ b/src/ai/backend/manager/api/gql/data_loader/data_loaders.py @@ -9,6 +9,7 @@ from ai.backend.common.identifier.app_config_allow_list import AppConfigAllowListID from ai.backend.common.identifier.app_config_definition import AppConfigDefinitionID from ai.backend.common.identifier.deployment import DeploymentID +from ai.backend.common.identifier.kernel_scheduling_history import KernelSchedulingHistoryID from ai.backend.common.types import AgentId, ImageID, KernelId, SessionId from ai.backend.manager.data.permission.id import ObjectId @@ -102,6 +103,7 @@ ) from ai.backend.manager.api.gql.scheduling_history.types import ( # pants: no-infer-dep DeploymentHistory, + KernelSchedulingHistoryGQL, RouteHistory, SessionSchedulingHistory, ) @@ -718,6 +720,24 @@ async def load_fn(ids: list[uuid.UUID]) -> list[SessionSchedulingHistory | None] return DataLoader(load_fn=load_fn) + @cached_property + def kernel_history_loader( + self, + ) -> DataLoader[KernelSchedulingHistoryID, KernelSchedulingHistoryGQL | None]: + adapter = self._adapters.scheduling_history + + async def load_fn( + ids: list[KernelSchedulingHistoryID], + ) -> list[KernelSchedulingHistoryGQL | None]: + from ai.backend.manager.api.gql.scheduling_history.types import ( # pants: no-infer-dep + KernelSchedulingHistoryGQL as KSH, + ) + + dtos = await adapter.batch_load_kernel_histories_by_ids(ids) + return [KSH.from_pydantic(dto) if dto is not None else None for dto in dtos] + + return DataLoader(load_fn=load_fn) + @cached_property def deployment_history_loader( self, diff --git a/src/ai/backend/manager/api/gql/kernel/types.py b/src/ai/backend/manager/api/gql/kernel/types.py index 06021de4d3b..b2acb7a8336 100644 --- a/src/ai/backend/manager/api/gql/kernel/types.py +++ b/src/ai/backend/manager/api/gql/kernel/types.py @@ -47,6 +47,13 @@ KernelResourceAllocationOrderByGQL, ResourceAllocationConnectionGQL, ) + from ai.backend.manager.api.gql.scheduling_history.resolver import ( + KernelSchedulingHistoryConnectionGQL, + ) + from ai.backend.manager.api.gql.scheduling_history.types import ( + KernelSchedulingHistoryFilterGQL, + KernelSchedulingHistoryOrderByGQL, + ) from ai.backend.manager.api.gql.session.types import SessionV2GQL from ai.backend.common.types import ImageID @@ -555,6 +562,87 @@ async def resource_allocations( ), ) + @gql_added_field( + BackendAIGQLMeta( + added_version=NEXT_RELEASE_VERSION, + description="Scheduling history of this kernel with pagination support.", + ) + ) # type: ignore[misc] + async def scheduling_histories( + self, + info: Info[StrawberryGQLContext], + filter: Annotated[ + KernelSchedulingHistoryFilterGQL, + strawberry.lazy("ai.backend.manager.api.gql.scheduling_history.types"), + ] + | None = None, + order_by: list[ + Annotated[ + KernelSchedulingHistoryOrderByGQL, + strawberry.lazy("ai.backend.manager.api.gql.scheduling_history.types"), + ] + ] + | None = None, + before: str | None = None, + after: str | None = None, + first: int | None = None, + last: int | None = None, + limit: int | None = None, + offset: int | None = None, + ) -> ( + Annotated[ + KernelSchedulingHistoryConnectionGQL, + strawberry.lazy("ai.backend.manager.api.gql.scheduling_history.resolver"), + ] + | None + ): + """Fetch the scheduling history of this kernel.""" + from uuid import UUID + + from ai.backend.common.dto.manager.v2.scheduling_history.request import ( + ScopedSearchKernelHistoriesInput, + ) + from ai.backend.common.dto.manager.v2.scheduling_history.types import ( + KernelHistoryScopeDTO, + ) + from ai.backend.manager.api.gql.base import encode_cursor + from ai.backend.manager.api.gql.scheduling_history.resolver import ( + KernelSchedulingHistoryConnectionGQL, + KernelSchedulingHistoryEdgeGQL, + ) + from ai.backend.manager.api.gql.scheduling_history.types import ( + KernelSchedulingHistoryGQL, + ) + + result = await info.context.adapters.scheduling_history.scoped_search_kernel_history( + ScopedSearchKernelHistoriesInput( + scope=KernelHistoryScopeDTO(kernel_id=UUID(str(self.id))), + filter=filter.to_pydantic() if filter else None, + order=[o.to_pydantic() for o in order_by] if order_by else None, + first=first, + after=after, + last=last, + before=before, + limit=limit, + offset=offset, + ) + ) + nodes = [KernelSchedulingHistoryGQL.from_pydantic(item) for item in result.items] + edges = [ + KernelSchedulingHistoryEdgeGQL(node=node, cursor=encode_cursor(str(node.id))) + for node in nodes + ] + return KernelSchedulingHistoryConnectionGQL( + edges=edges, + page_info=strawberry.relay.PageInfo( + has_next_page=result.has_next_page, + has_previous_page=result.has_previous_page, + start_cursor=edges[0].cursor if edges else None, + end_cursor=edges[-1].cursor if edges else None, + ), + count=result.total_count, + ) + @classmethod @override async def resolve_nodes( # type: ignore[override] # Strawberry Node uses AwaitableOrValue overloads incompatible with async def diff --git a/src/ai/backend/manager/api/gql/scheduling_history/__init__.py b/src/ai/backend/manager/api/gql/scheduling_history/__init__.py index e261d3180b6..f8f09ba9158 100644 --- a/src/ai/backend/manager/api/gql/scheduling_history/__init__.py +++ b/src/ai/backend/manager/api/gql/scheduling_history/__init__.py @@ -2,15 +2,18 @@ from .resolver import ( DeploymentHistoryConnection, + KernelSchedulingHistoryConnectionGQL, RouteHistoryConnection, SessionSchedulingHistoryConnection, admin_deployment_histories, + admin_kernel_scheduling_histories, admin_route_histories, admin_session_scheduling_histories, deployment_histories, deployment_scoped_scheduling_histories, route_histories, route_scoped_scheduling_histories, + scoped_kernel_scheduling_histories, session_scheduling_histories, session_scoped_scheduling_histories, ) @@ -19,6 +22,10 @@ DeploymentHistoryFilter, DeploymentHistoryOrderBy, DeploymentScope, + KernelSchedulingHistoryFilterGQL, + KernelSchedulingHistoryGQL, + KernelSchedulingHistoryOrderByGQL, + KernelScopeGQL, RouteHistory, RouteHistoryFilter, RouteHistoryOrderBy, @@ -37,29 +44,36 @@ # Types "SubStepResultGQL", "SessionSchedulingHistory", + "KernelSchedulingHistoryGQL", "DeploymentHistory", "RouteHistory", # Filters "SessionSchedulingHistoryFilter", "SessionSchedulingHistoryOrderBy", + "KernelSchedulingHistoryFilterGQL", + "KernelSchedulingHistoryOrderByGQL", "DeploymentHistoryFilter", "DeploymentHistoryOrderBy", "RouteHistoryFilter", "RouteHistoryOrderBy", # Scope types (added in 26.2.0) "SessionScope", + "KernelScopeGQL", "DeploymentScope", "RouteScope", # Connections "SessionSchedulingHistoryConnection", + "KernelSchedulingHistoryConnectionGQL", "DeploymentHistoryConnection", "RouteHistoryConnection", # Queries - Admin "admin_session_scheduling_histories", + "admin_kernel_scheduling_histories", "admin_deployment_histories", "admin_route_histories", # Queries - Scoped (added in 26.2.0) "session_scoped_scheduling_histories", + "scoped_kernel_scheduling_histories", "deployment_scoped_scheduling_histories", "route_scoped_scheduling_histories", # Queries - Legacy (deprecated) diff --git a/src/ai/backend/manager/api/gql/scheduling_history/resolver.py b/src/ai/backend/manager/api/gql/scheduling_history/resolver.py index 6519dc499ed..2b41baa55b1 100644 --- a/src/ai/backend/manager/api/gql/scheduling_history/resolver.py +++ b/src/ai/backend/manager/api/gql/scheduling_history/resolver.py @@ -12,9 +12,12 @@ from ai.backend.common.contexts.user import current_user from ai.backend.common.dto.manager.v2.scheduling_history.request import ( AdminSearchDeploymentHistoriesInput, + AdminSearchKernelHistoriesInput, AdminSearchRouteHistoriesInput, AdminSearchSessionHistoriesInput, + ScopedSearchKernelHistoriesInput, ) +from ai.backend.common.meta.meta import NEXT_RELEASE_VERSION from ai.backend.manager.api.gql.base import encode_cursor from ai.backend.manager.api.gql.decorators import ( BackendAIGQLMeta, @@ -29,6 +32,10 @@ DeploymentHistoryFilter, DeploymentHistoryOrderBy, DeploymentScope, + KernelSchedulingHistoryFilterGQL, + KernelSchedulingHistoryGQL, + KernelSchedulingHistoryOrderByGQL, + KernelScopeGQL, RouteHistory, RouteHistoryFilter, RouteHistoryOrderBy, @@ -56,6 +63,24 @@ def __init__(self, *args: Any, count: int, **kwargs: Any) -> None: self.count = count +KernelSchedulingHistoryEdgeGQL = Edge[KernelSchedulingHistoryGQL] + + +@gql_connection_type( + BackendAIGQLMeta( + added_version=NEXT_RELEASE_VERSION, + description="Kernel scheduling history connection.", + ), + name="KernelSchedulingHistoryConnection", +) +class KernelSchedulingHistoryConnectionGQL(Connection[KernelSchedulingHistoryGQL]): + count: int + + def __init__(self, *args: Any, count: int, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.count = count + + DeploymentHistoryEdge = Edge[DeploymentHistory] @@ -184,6 +209,53 @@ async def session_scheduling_histories( ) +@gql_root_field( + BackendAIGQLMeta( + added_version=NEXT_RELEASE_VERSION, + description="List kernel scheduling history (superadmin only)", + ) +) # type: ignore[misc] +async def admin_kernel_scheduling_histories( + info: Info[StrawberryGQLContext], + filter: KernelSchedulingHistoryFilterGQL | None = None, + order_by: list[KernelSchedulingHistoryOrderByGQL] | None = None, + before: str | None = None, + after: str | None = None, + first: int | None = None, + last: int | None = None, + limit: int | None = None, + offset: int | None = None, +) -> KernelSchedulingHistoryConnectionGQL | None: + check_admin_only() + result = await info.context.adapters.scheduling_history.admin_search_kernel_history( + AdminSearchKernelHistoriesInput( + filter=filter.to_pydantic() if filter else None, + order=[o.to_pydantic() for o in order_by] if order_by else None, + first=first, + after=after, + last=last, + before=before, + limit=limit, + offset=offset, + ) + ) + nodes = [KernelSchedulingHistoryGQL.from_pydantic(item) for item in result.items] + edges = [ + KernelSchedulingHistoryEdgeGQL(node=node, cursor=encode_cursor(str(node.id))) + for node in nodes + ] + return KernelSchedulingHistoryConnectionGQL( + edges=edges, + page_info=strawberry.relay.PageInfo( + has_next_page=result.has_next_page, + has_previous_page=result.has_previous_page, + start_cursor=edges[0].cursor if edges else None, + end_cursor=edges[-1].cursor if edges else None, + ), + count=result.total_count, + ) + + @gql_root_field( BackendAIGQLMeta(added_version="26.4.2", description="List deployment history (admin only)") ) # type: ignore[misc] @@ -495,3 +567,52 @@ async def route_scoped_scheduling_histories( ), count=result.total_count, ) + + +@gql_root_field( + BackendAIGQLMeta( + added_version=NEXT_RELEASE_VERSION, + description="Get scheduling history for a specific kernel.", + ) +) # type: ignore[misc] +async def scoped_kernel_scheduling_histories( + info: Info[StrawberryGQLContext], + scope: KernelScopeGQL, + filter: KernelSchedulingHistoryFilterGQL | None = None, + order_by: list[KernelSchedulingHistoryOrderByGQL] | None = None, + before: str | None = None, + after: str | None = None, + first: int | None = None, + last: int | None = None, + limit: int | None = None, + offset: int | None = None, +) -> KernelSchedulingHistoryConnectionGQL | None: + """Get scheduling history for a specific kernel.""" + result = await info.context.adapters.scheduling_history.scoped_search_kernel_history( + ScopedSearchKernelHistoriesInput( + scope=scope.to_pydantic(), + filter=filter.to_pydantic() if filter else None, + order=[o.to_pydantic() for o in order_by] if order_by else None, + first=first, + after=after, + last=last, + before=before, + limit=limit, + offset=offset, + ) + ) + nodes = [KernelSchedulingHistoryGQL.from_pydantic(item) for item in result.items] + edges = [ + KernelSchedulingHistoryEdgeGQL(node=node, cursor=encode_cursor(str(node.id))) + for node in nodes + ] + return KernelSchedulingHistoryConnectionGQL( + edges=edges, + page_info=strawberry.relay.PageInfo( + has_next_page=result.has_next_page, + has_previous_page=result.has_previous_page, + start_cursor=edges[0].cursor if edges else None, + end_cursor=edges[-1].cursor if edges else None, + ), + count=result.total_count, + ) diff --git a/src/ai/backend/manager/api/gql/scheduling_history/types.py b/src/ai/backend/manager/api/gql/scheduling_history/types.py index 4138dab71be..932be891842 100644 --- a/src/ai/backend/manager/api/gql/scheduling_history/types.py +++ b/src/ai/backend/manager/api/gql/scheduling_history/types.py @@ -18,6 +18,12 @@ from ai.backend.common.dto.manager.v2.scheduling_history.request import ( DeploymentHistoryOrder as DeploymentHistoryOrderDTO, ) +from ai.backend.common.dto.manager.v2.scheduling_history.request import ( + KernelHistoryFilter as KernelHistoryFilterDTO, +) +from ai.backend.common.dto.manager.v2.scheduling_history.request import ( + KernelHistoryOrder as KernelHistoryOrderDTO, +) from ai.backend.common.dto.manager.v2.scheduling_history.request import ( RouteHistoryFilter as RouteHistoryFilterDTO, ) @@ -35,15 +41,19 @@ ) from ai.backend.common.dto.manager.v2.scheduling_history.response import ( DeploymentHistoryNode, + KernelHistoryNode, RouteHistoryNode, SessionHistoryNode, ) from ai.backend.common.dto.manager.v2.scheduling_history.types import ( DeploymentHistoryScopeDTO, + KernelHistoryScopeDTO, RouteHistoryScopeDTO, SessionHistoryScopeDTO, SubStepResultInfo, ) +from ai.backend.common.identifier.kernel_scheduling_history import KernelSchedulingHistoryID +from ai.backend.common.meta.meta import NEXT_RELEASE_VERSION from ai.backend.manager.api.gql.base import ( DateTimeFilter, OrderDirection, @@ -77,22 +87,27 @@ # Filter wrappers "SchedulingResultFilterGQL", "SessionSchedulingHistoryOrderField", + "KernelSchedulingHistoryOrderFieldGQL", "DeploymentHistoryOrderField", "RouteHistoryOrderField", # Types "SubStepResultGQL", "SessionSchedulingHistory", + "KernelSchedulingHistoryGQL", "DeploymentHistory", "RouteHistory", # Filters "SessionSchedulingHistoryFilter", "SessionSchedulingHistoryOrderBy", + "KernelSchedulingHistoryFilterGQL", + "KernelSchedulingHistoryOrderByGQL", "DeploymentHistoryFilter", "DeploymentHistoryOrderBy", "RouteHistoryFilter", "RouteHistoryOrderBy", # Scope types (added in 26.2.0) "SessionScope", + "KernelScopeGQL", "DeploymentScope", "RouteScope", ) @@ -126,6 +141,23 @@ class SessionSchedulingHistoryOrderField(StrEnum): UPDATED_AT = "updated_at" +@gql_enum( + BackendAIGQLMeta( + added_version=NEXT_RELEASE_VERSION, + description="Fields available for ordering kernel scheduling history", + ), + name="KernelSchedulingHistoryOrderField", +) +class KernelSchedulingHistoryOrderFieldGQL(StrEnum): + CREATED_AT = "created_at" + UPDATED_AT = "updated_at" + PHASE = "phase" + FROM_STATUS = "from_status" + TO_STATUS = "to_status" + RESULT = "result" + ATTEMPTS = "attempts" + + @gql_enum( BackendAIGQLMeta( added_version="26.3.0", @@ -225,6 +257,42 @@ async def resolve_nodes( # type: ignore[override] # Strawberry Node uses Await return cast(list[Self | None], results) +@gql_node_type( + BackendAIGQLMeta( + added_version=NEXT_RELEASE_VERSION, description="Kernel scheduling history record." + ), + name="KernelSchedulingHistory", +) +class KernelSchedulingHistoryGQL(PydanticNodeMixin[KernelHistoryNode]): + id: NodeID[str] + kernel_id: ID + session_id: ID + phase: str + from_status: str | None + to_status: str | None + result: SchedulingResultGQL + error_code: str | None + message: str | None + attempts: int + created_at: datetime + updated_at: datetime + + @classmethod + @override + async def resolve_nodes( # type: ignore[override] # Strawberry Node uses AwaitableOrValue overloads incompatible with async def + cls, + *, + info: Info[StrawberryGQLContext], + node_ids: Iterable[str], + required: bool = False, + ) -> Iterable[Self | None]: + # DataLoader returns GQL type instances directly via from_pydantic adapter. + results = await info.context.data_loaders.kernel_history_loader.load_many([ + KernelSchedulingHistoryID(UUID(nid)) for nid in node_ids + ]) + return cast(list[Self | None], results) + + @gql_node_type(BackendAIGQLMeta(added_version="26.3.0", description="Deployment history record.")) class DeploymentHistory(PydanticNodeMixin[DeploymentHistoryNode]): id: NodeID[str] @@ -368,6 +436,19 @@ class SessionScope(PydanticInputMixin[SessionHistoryScopeDTO]): session_id: UUID = gql_field(description="Session ID to get history for") +@gql_pydantic_input( + BackendAIGQLMeta( + description="Scope for kernel scheduling history query", + added_version=NEXT_RELEASE_VERSION, + ), + name="KernelScope", +) +class KernelScopeGQL(PydanticInputMixin[KernelHistoryScopeDTO]): + """Scope for kernel-level scheduling history queries.""" + + kernel_id: UUID = gql_field(description="Kernel ID to get history for") + + @gql_pydantic_input( BackendAIGQLMeta( description="Scope for deployment scheduling history query", added_version="24.09.0" @@ -442,6 +523,41 @@ class SessionSchedulingHistoryOrderBy(PydanticInputMixin[SessionHistoryOrderDTO] direction: OrderDirection = OrderDirection.DESC +@gql_pydantic_input( + BackendAIGQLMeta( + description="Filter for kernel scheduling history", added_version=NEXT_RELEASE_VERSION + ), + name="KernelSchedulingHistoryFilter", +) +class KernelSchedulingHistoryFilterGQL(PydanticInputMixin[KernelHistoryFilterDTO]): + id: UUIDFilter | None = None + kernel_id: UUIDFilter | None = None + session_id: UUIDFilter | None = None + phase: StringFilter | None = None + from_status: list[str] | None = None + to_status: list[str] | None = None + result: SchedulingResultFilterGQL | None = None + error_code: StringFilter | None = None + message: StringFilter | None = None + created_at: DateTimeFilter | None = None + updated_at: DateTimeFilter | None = None + AND: list[Self] | None = None + OR: list[Self] | None = None + NOT: list[Self] | None = None + + +@gql_pydantic_input( + BackendAIGQLMeta( + description="Order by specification for kernel scheduling history", + added_version=NEXT_RELEASE_VERSION, + ), + name="KernelSchedulingHistoryOrderBy", +) +class KernelSchedulingHistoryOrderByGQL(PydanticInputMixin[KernelHistoryOrderDTO]): + field: KernelSchedulingHistoryOrderFieldGQL + direction: OrderDirection = OrderDirection.DESC + + @gql_pydantic_input( BackendAIGQLMeta(description="Filter for deployment history", added_version="24.09.0"), name="DeploymentHistoryFilter", diff --git a/src/ai/backend/manager/api/gql/schema.py b/src/ai/backend/manager/api/gql/schema.py index 5ea7e902353..80aa40c9ad9 100644 --- a/src/ai/backend/manager/api/gql/schema.py +++ b/src/ai/backend/manager/api/gql/schema.py @@ -425,12 +425,14 @@ from .scheduling_handler import scheduling_handlers from .scheduling_history import ( admin_deployment_histories, + admin_kernel_scheduling_histories, admin_route_histories, admin_session_scheduling_histories, deployment_histories, deployment_scoped_scheduling_histories, route_histories, route_scoped_scheduling_histories, + scoped_kernel_scheduling_histories, session_scheduling_histories, session_scoped_scheduling_histories, ) @@ -542,6 +544,7 @@ class Query: admin_allowed_projects_for_resource_group_v2 = admin_allowed_projects_for_resource_group_v2 admin_service_catalogs = admin_service_catalogs admin_session_scheduling_histories = admin_session_scheduling_histories + admin_kernel_scheduling_histories = admin_kernel_scheduling_histories admin_deployments = admin_deployments admin_deployment_histories = admin_deployment_histories admin_route_histories = admin_route_histories @@ -626,6 +629,7 @@ class Query: image_scoped_aliases = image_scoped_aliases # Entity Scoped APIs (added in 26.2.0) session_scoped_scheduling_histories = session_scoped_scheduling_histories + scoped_kernel_scheduling_histories = scoped_kernel_scheduling_histories deployment_scoped_scheduling_histories = deployment_scoped_scheduling_histories route_scoped_scheduling_histories = route_scoped_scheduling_histories # Legacy APIs (deprecated) diff --git a/src/ai/backend/manager/models/scheduling_history/orders.py b/src/ai/backend/manager/models/scheduling_history/orders.py index beba5b5ad96..f87e542de1c 100644 --- a/src/ai/backend/manager/models/scheduling_history/orders.py +++ b/src/ai/backend/manager/models/scheduling_history/orders.py @@ -67,6 +67,11 @@ def resolve_session_order(field: SessionHistoryOrderField, direction: OrderDirec KERNEL_ORDER_FIELD_MAP: dict[KernelHistoryOrderField, _OrderColumn] = { KernelHistoryOrderField.CREATED_AT: KernelSchedulingHistoryRow.created_at, KernelHistoryOrderField.UPDATED_AT: KernelSchedulingHistoryRow.updated_at, + KernelHistoryOrderField.PHASE: KernelSchedulingHistoryRow.phase, + KernelHistoryOrderField.FROM_STATUS: KernelSchedulingHistoryRow.from_status, + KernelHistoryOrderField.TO_STATUS: KernelSchedulingHistoryRow.to_status, + KernelHistoryOrderField.RESULT: KernelSchedulingHistoryRow.result, + KernelHistoryOrderField.ATTEMPTS: KernelSchedulingHistoryRow.attempts, } KERNEL_DEFAULT_FORWARD_ORDER: QueryOrder = KernelSchedulingHistoryRow.created_at.desc()