diff --git a/drivers/accel/amdxdna/aie2_pci.c b/drivers/accel/amdxdna/aie2_pci.c index 1b07705c2..3cb1d4ed7 100644 --- a/drivers/accel/amdxdna/aie2_pci.c +++ b/drivers/accel/amdxdna/aie2_pci.c @@ -1236,6 +1236,8 @@ const struct amdxdna_dev_ops aie2_ops = { .fini = aie2_fini, .resume = aie2_hw_resume, .suspend = aie2_hw_suspend, + .runtime_resume = aie2_hw_resume, + .runtime_suspend = aie2_hw_suspend, .get_aie_info = aie2_get_info, .set_aie_state = aie2_set_state, .hwctx_init = aie2_hwctx_init, diff --git a/drivers/accel/amdxdna/aie4_pci.c b/drivers/accel/amdxdna/aie4_pci.c index de39515b6..f864a2330 100644 --- a/drivers/accel/amdxdna/aie4_pci.c +++ b/drivers/accel/amdxdna/aie4_pci.c @@ -716,9 +716,7 @@ static int aie4m_pcidev_init(struct amdxdna_dev *xdna) amdxdna_vbnv_init(xdna); - pm_runtime_disable(&pdev->dev); XDNA_DBG(xdna, "init finished"); - return 0; } @@ -1134,6 +1132,41 @@ static int aie4_hwctx_resume_all(struct amdxdna_dev_hdl *ndev) return ret; } +/* + * AIE4 Suspend/Resume steps per device type. + * -> SUSPEND + * <- RESUME + * + * PF = PF S3/S4 PF-RTM = PF Runtime PM + * VF = VF S3/S4 VF-RTM = VF Runtime PM + * + * | PF | PF-RTM | VF | VF-RTM | Classic + * -----------------------------|-----|--------|-----|--------|-------- + * -> vfs_alive | - | Y | - | - | - + * -> hwctx_suspend_all | - | - | Y | Y | Y + * -> partition_fini | - | - | (1) | Y | Y + * -> suspend_fw | Y | Y | - | - | Y + * -> mailbox_fini | Y | Y | Y | Y | Y + * -> free_async_buffer | - | - | Y | Y | Y + * -> stop_psp/smu | Y | Y | - | - | Y + * ---- power boundary ---- + * <- pci_enable + set_master | Y | Y | Y | Y | Y + * <- start_smu/psp | Y | Y | (2) | (2) | Y + * <- mailbox_init | Y | Y | Y | Y | Y + * <- calibrate_clock | Y | Y | - | - | Y + * <- attach_work_buffer | Y | Y | - | - | Y + * <- query_fw | Y | Y | - | - | Y + * <- query_aie | - | - | Y | Y | Y + * <- partition_init | - | - | Y | Y | Y + * <- alloc_async_event | - | - | Y | Y | Y + * <- hwctx_resume_all | - | - | Y | Y | Y + * <- restore VFs | Y | Y | - | - | - + * + * (1) VF S3/S4 skips partition_fini: PF tears down stateless FW. + * VF RTM must send partition_fini to firmware explicitly. + * (2) VF skips start_smu/psp: PF boots firmware on behalf of all VFs. + */ + static int aie4_pf_suspend(struct amdxdna_dev *xdna) { struct amdxdna_dev_hdl *ndev = xdna->dev_handle; @@ -1146,6 +1179,17 @@ static int aie4_pf_suspend(struct amdxdna_dev *xdna) return 0; } +static int aie4_pf_runtime_suspend(struct amdxdna_dev *xdna) +{ + int ret; + + ret = aie4_vfs_alive(xdna); + if (ret) + return ret; + + return aie4_pf_suspend(xdna); +} + static int aie4_vf_suspend(struct amdxdna_dev *xdna) { struct amdxdna_dev_hdl *ndev = xdna->dev_handle; @@ -1165,6 +1209,18 @@ static int aie4_vf_suspend(struct amdxdna_dev *xdna) return 0; } +static int aie4_vf_runtime_suspend(struct amdxdna_dev *xdna) +{ + struct amdxdna_dev_hdl *ndev = xdna->dev_handle; + + drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); + aie4_hwctx_suspend_all(ndev); + aie4_vf_hw_stop(ndev); + + XDNA_DBG(xdna, "vf runtime suspend done"); + return 0; +} + static int aie4_classic_suspend(struct amdxdna_dev *xdna) { struct amdxdna_dev_hdl *ndev = xdna->dev_handle; @@ -1317,6 +1373,7 @@ static int aie4_pf_init(struct amdxdna_dev *xdna) aie4_msg_init(xdna->dev_handle); amdxdna_dpt_init(&xdna->dev_handle->aie); + amdxdna_pm_init(xdna); return 0; free_work_buf: @@ -1338,6 +1395,7 @@ static int aie4_vf_init(struct amdxdna_dev *xdna) aie4_msg_init(xdna->dev_handle); amdxdna_dpt_init(&xdna->dev_handle->aie); + amdxdna_pm_init(xdna); return 0; } @@ -1359,6 +1417,7 @@ static int aie4_classic_init(struct amdxdna_dev *xdna) aie4_msg_init(xdna->dev_handle); amdxdna_dpt_init(&xdna->dev_handle->aie); + amdxdna_pm_init(xdna); return 0; free_work_buf: @@ -1370,6 +1429,7 @@ static void aie4_pf_fini(struct amdxdna_dev *xdna) { int ret; + amdxdna_pm_fini(xdna); amdxdna_dpt_fini(&xdna->dev_handle->aie); ret = aie4_sriov_stop(xdna->dev_handle); @@ -1384,12 +1444,14 @@ static void aie4_pf_fini(struct amdxdna_dev *xdna) static void aie4_vf_fini(struct amdxdna_dev *xdna) { + amdxdna_pm_fini(xdna); amdxdna_dpt_fini(&xdna->dev_handle->aie); aie4_vf_hw_stop(xdna->dev_handle); } static void aie4_classic_fini(struct amdxdna_dev *xdna) { + amdxdna_pm_fini(xdna); amdxdna_dpt_fini(&xdna->dev_handle->aie); aie4_classic_hw_stop(xdna->dev_handle); aie4_free_work_buffer(xdna->dev_handle); @@ -1571,6 +1633,8 @@ const struct amdxdna_dev_ops aie4_pf_ops = { .sriov_configure = aie4_sriov_configure, .resume = aie4_pf_resume, .suspend = aie4_pf_suspend, + .runtime_resume = aie4_pf_resume, + .runtime_suspend = aie4_pf_runtime_suspend, /* additional check on VM passthrough */ .register_async_event = aie4_async_event_register, .handle_dev_async_event = aie4_handle_dev_event, }; @@ -1590,6 +1654,8 @@ const struct amdxdna_dev_ops aie4_vf_ops = { .get_array = aie4_get_array, .resume = aie4_vf_resume, .suspend = aie4_vf_suspend, + .runtime_resume = aie4_vf_resume, + .runtime_suspend = aie4_vf_runtime_suspend, /* each VF needs to clean up itself */ .register_async_event = aie4_async_event_register, .handle_dev_async_event = aie4_handle_dev_event, }; @@ -1609,6 +1675,8 @@ const struct amdxdna_dev_ops aie4_classic_ops = { .get_array = aie4_get_array, .resume = aie4_classic_resume, .suspend = aie4_classic_suspend, + .runtime_resume = aie4_classic_resume, + .runtime_suspend = aie4_classic_suspend, .register_async_event = aie4_async_event_register, .handle_dev_async_event = aie4_handle_dev_event, }; diff --git a/drivers/accel/amdxdna/aie4_pci.h b/drivers/accel/amdxdna/aie4_pci.h index b461709e2..2a77cd7c2 100644 --- a/drivers/accel/amdxdna/aie4_pci.h +++ b/drivers/accel/amdxdna/aie4_pci.h @@ -149,10 +149,12 @@ void aie4_fill_health_data(struct amdxdna_gem_obj *cmd_abo, struct amdxdna_hwctx int aie4_sriov_configure(struct amdxdna_dev *xdna, int num_vfs); int aie4_create_vfs(struct amdxdna_dev_hdl *ndev, int num_vfs); int aie4_sriov_stop(struct amdxdna_dev_hdl *ndev); +int aie4_vfs_alive(struct amdxdna_dev *xdna); #else #define aie4_sriov_configure NULL static inline int aie4_sriov_stop(struct amdxdna_dev_hdl *ndev) { return 0; } static inline int aie4_create_vfs(struct amdxdna_dev_hdl *ndev, int num_vfs) { return 0; } +static inline int aie4_vfs_alive(struct amdxdna_dev *xdna) { return 0; } #endif /* aie4_message.c */ diff --git a/drivers/accel/amdxdna/aie4_sriov.c b/drivers/accel/amdxdna/aie4_sriov.c index 4c15c8143..c095a8a22 100644 --- a/drivers/accel/amdxdna/aie4_sriov.c +++ b/drivers/accel/amdxdna/aie4_sriov.c @@ -6,6 +6,7 @@ #include "drm/amdxdna_accel.h" #include #include +#include #include "aie.h" #include "aie4_msg_priv.h" @@ -54,27 +55,31 @@ int aie4_sriov_stop(struct amdxdna_dev_hdl *ndev) return -EPERM; } - ret = aie4_destroy_vfs(ndev); - ndev->num_vfs = 0; pci_disable_sriov(pdev); - return ret; + ndev->num_vfs = 0; + + /* + * pci_disable_sriov() removes VF drivers first; call destroy_vfs after + * so firmware VF contexts are not cleared before VF drivers finish cleanup. + */ + return aie4_destroy_vfs(ndev); } -static void aie4_link_vfs(struct amdxdna_dev *xdna) +static int aie4_for_each_vfs(struct amdxdna_dev *xdna, + int (*cb)(struct amdxdna_dev *, struct pci_dev *)) { struct pci_dev *pdev_pf = to_pci_dev(xdna->ddev.dev); - struct device_link *link; struct pci_dev *pdev_vf; int pos, ret; u16 vf_did; pos = pci_find_ext_capability(pdev_pf, PCI_EXT_CAP_ID_SRIOV); if (!pos) - return; + return 0; ret = pci_read_config_word(pdev_pf, pos + PCI_SRIOV_VF_DID, &vf_did); if (ret) { XDNA_ERR(xdna, "read VF Device ID failed %d", ret); - return; + return -ENODEV; } for (pdev_vf = pci_get_device(pdev_pf->vendor, vf_did, NULL); @@ -83,14 +88,65 @@ static void aie4_link_vfs(struct amdxdna_dev *xdna) if (!pdev_vf->is_virtfn || pdev_vf->physfn != pdev_pf) continue; - link = device_link_add(&pdev_vf->dev, /* consumer = VF */ - &pdev_pf->dev, /* supplier = PF */ - DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_CONSUMER); - if (!link) - XDNA_WARN(xdna, "Failed to link VF %s", pci_name(pdev_vf)); - else - XDNA_DBG(xdna, "Linked VF %s", pci_name(pdev_vf)); + ret = cb(xdna, pdev_vf); + if (ret) { + /* + * On early return the next iteration never runs, so + * release the current device's ref manually. + * On normal loop exit pci_get_device() returning NULL + * already releases the last device's ref internally. + */ + pci_dev_put(pdev_vf); + return ret; + } } + + return 0; +} + +static int aie4_link_vf(struct amdxdna_dev *xdna, struct pci_dev *pdev_vf) +{ + struct pci_dev *pdev_pf = to_pci_dev(xdna->ddev.dev); + struct device_link *link; + + link = device_link_add(&pdev_vf->dev, /* consumer = VF */ + &pdev_pf->dev, /* supplier = PF */ + DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_CONSUMER); + if (!link) + XDNA_WARN(xdna, "Failed to link VF %s", pci_name(pdev_vf)); + else + XDNA_DBG(xdna, "Linked VF %s", pci_name(pdev_vf)); + return 0; +} + +static int aie4_check_vf_alive(struct amdxdna_dev *xdna, struct pci_dev *pdev_vf) +{ + struct device_driver *drv = READ_ONCE(pdev_vf->dev.driver); + + if (drv && drv->owner != THIS_MODULE) { + XDNA_WARN(xdna, "VF:%s is in passthrough", pci_name(pdev_vf)); + return -EBUSY; + } + + if (!pm_runtime_suspended(&pdev_vf->dev)) { + XDNA_WARN(xdna, "VF:%s is busy", pci_name(pdev_vf)); + return -EBUSY; + } + return 0; +} + +int aie4_vfs_alive(struct amdxdna_dev *xdna) +{ + if (pci_vfs_assigned(to_pci_dev(xdna->ddev.dev))) { + XDNA_WARN(xdna, "VF devices are being used in VMs, cannot suspend"); + return -EBUSY; + } + return aie4_for_each_vfs(xdna, aie4_check_vf_alive); +} + +static void aie4_link_vfs(struct amdxdna_dev *xdna) +{ + aie4_for_each_vfs(xdna, aie4_link_vf); } static int aie4_sriov_start(struct amdxdna_dev_hdl *ndev, int num_vfs) diff --git a/drivers/accel/amdxdna/amdxdna_pci_drv.c b/drivers/accel/amdxdna/amdxdna_pci_drv.c index 3fe878b49..32c004eae 100644 --- a/drivers/accel/amdxdna/amdxdna_pci_drv.c +++ b/drivers/accel/amdxdna/amdxdna_pci_drv.c @@ -560,18 +560,27 @@ static void amdxdna_remove(struct pci_dev *pdev) static const struct dev_pm_ops amdxdna_pm_ops = { SYSTEM_SLEEP_PM_OPS(amdxdna_pm_suspend, amdxdna_pm_resume) - RUNTIME_PM_OPS(amdxdna_pm_suspend, amdxdna_pm_resume, NULL) + RUNTIME_PM_OPS(amdxdna_pm_runtime_suspend, amdxdna_pm_runtime_resume, NULL) }; static int amdxdna_sriov_configure(struct pci_dev *pdev, int num_vfs) { struct amdxdna_dev *xdna = pci_get_drvdata(pdev); + int ret; guard(mutex)(&xdna->dev_lock); + + ret = amdxdna_pm_resume_get_locked(xdna); + if (ret) + return ret; + if (xdna->dev_info->ops->sriov_configure) - return xdna->dev_info->ops->sriov_configure(xdna, num_vfs); + ret = xdna->dev_info->ops->sriov_configure(xdna, num_vfs); + else + ret = -EOPNOTSUPP; - return -ENOENT; + amdxdna_pm_suspend_put(xdna); + return ret; } static struct pci_driver amdxdna_pci_driver = { diff --git a/drivers/accel/amdxdna/amdxdna_pci_drv.h b/drivers/accel/amdxdna/amdxdna_pci_drv.h index 863a9dd0a..f973956b0 100644 --- a/drivers/accel/amdxdna/amdxdna_pci_drv.h +++ b/drivers/accel/amdxdna/amdxdna_pci_drv.h @@ -65,6 +65,8 @@ struct amdxdna_dev_ops { void (*debugfs_init)(struct amdxdna_dev *xdna); int (*resume)(struct amdxdna_dev *xdna); int (*suspend)(struct amdxdna_dev *xdna); + int (*runtime_resume)(struct amdxdna_dev *xdna); + int (*runtime_suspend)(struct amdxdna_dev *xdna); int (*sriov_configure)(struct amdxdna_dev *xdna, int num_vfs); int (*mmap)(struct amdxdna_client *client, struct vm_area_struct *vma); int (*hwctx_init)(struct amdxdna_hwctx *hwctx); diff --git a/drivers/accel/amdxdna/amdxdna_pm.c b/drivers/accel/amdxdna/amdxdna_pm.c index e7300e9b9..855856c1f 100644 --- a/drivers/accel/amdxdna/amdxdna_pm.c +++ b/drivers/accel/amdxdna/amdxdna_pm.c @@ -65,6 +65,46 @@ int amdxdna_pm_resume(struct device *dev) return ret; } +int amdxdna_pm_runtime_suspend(struct device *dev) +{ + struct amdxdna_dev *xdna = to_xdna_dev(dev_get_drvdata(dev)); + int ret = -EOPNOTSUPP; + + guard(mutex)(&xdna->dev_lock); + if (xdna->dev_info->ops->runtime_suspend) + ret = xdna->dev_info->ops->runtime_suspend(xdna); + + if (!ret) { + int dpt_ret = amdxdna_dpt_suspend(xdna); + + if (dpt_ret) + XDNA_WARN(xdna, "DPT drain/pause on suspend failed: %d", dpt_ret); + } + + XDNA_DBG(xdna, "Runtime suspend done ret %d", ret); + return ret; +} + +int amdxdna_pm_runtime_resume(struct device *dev) +{ + struct amdxdna_dev *xdna = to_xdna_dev(dev_get_drvdata(dev)); + int ret = -EOPNOTSUPP; + + guard(mutex)(&xdna->dev_lock); + if (xdna->dev_info->ops->runtime_resume) + ret = xdna->dev_info->ops->runtime_resume(xdna); + + if (!ret) { + int dpt_ret = amdxdna_dpt_resume(xdna); + + if (dpt_ret) + XDNA_WARN(xdna, "DPT re-arm on resume failed: %d", dpt_ret); + } + + XDNA_DBG(xdna, "Runtime resume done ret %d", ret); + return ret; +} + int amdxdna_pm_resume_get(struct amdxdna_dev *xdna) { struct device *dev = xdna->ddev.dev; diff --git a/drivers/accel/amdxdna/amdxdna_pm.h b/drivers/accel/amdxdna/amdxdna_pm.h index 3d26b973e..26df3d50d 100644 --- a/drivers/accel/amdxdna/amdxdna_pm.h +++ b/drivers/accel/amdxdna/amdxdna_pm.h @@ -9,7 +9,9 @@ #include "amdxdna_pci_drv.h" int amdxdna_pm_suspend(struct device *dev); -int amdxdna_pm_resume(struct device *dev); +int amdxdna_pm_resume(struct device *dev); +int amdxdna_pm_runtime_suspend(struct device *dev); +int amdxdna_pm_runtime_resume(struct device *dev); int amdxdna_pm_resume_get(struct amdxdna_dev *xdna); void amdxdna_pm_suspend_put(struct amdxdna_dev *xdna); void amdxdna_pm_init(struct amdxdna_dev *xdna);