accel/amdxdna: Enable AIE4 runtime suspend/resume

drivers/accel/amdxdna/aie2_pci.c

@@ -1236,6 +1236,8 @@ const struct amdxdna_dev_ops aie2_ops = {
 	.fini = aie2_fini,
 	.resume = aie2_hw_resume,
 	.suspend = aie2_hw_suspend,
+	.runtime_resume = aie2_hw_resume,
+	.runtime_suspend = aie2_hw_suspend,
 	.get_aie_info = aie2_get_info,
 	.set_aie_state = aie2_set_state,
 	.hwctx_init = aie2_hwctx_init,

drivers/accel/amdxdna/aie4_pci.c

@@ -716,9 +716,7 @@ static int aie4m_pcidev_init(struct amdxdna_dev *xdna)
 
 	amdxdna_vbnv_init(xdna);
 
-	pm_runtime_disable(&pdev->dev);
 	XDNA_DBG(xdna, "init finished");
-
 	return 0;
 }
 
@@ -1134,6 +1132,41 @@ static int aie4_hwctx_resume_all(struct amdxdna_dev_hdl *ndev)
 	return ret;
 }
 
+/*
+ * AIE4 Suspend/Resume steps per device type.
+ * -> SUSPEND
+ * <- RESUME
+ *
+ * PF     = PF S3/S4      PF-RTM = PF Runtime PM
+ * VF     = VF S3/S4      VF-RTM = VF Runtime PM
+ *
+ *                              | PF  | PF-RTM | VF  | VF-RTM | Classic
+ * -----------------------------|-----|--------|-----|--------|--------
+ * -> vfs_alive                 |  -  |   Y    |  -  |   -    |   -
+ * -> hwctx_suspend_all         |  -  |   -    |  Y  |   Y    |   Y
+ * -> partition_fini            |  -  |   -    | (1) |   Y    |   Y
+ * -> suspend_fw                |  Y  |   Y    |  -  |   -    |   Y
+ * -> mailbox_fini              |  Y  |   Y    |  Y  |   Y    |   Y
+ * -> free_async_buffer         |  -  |   -    |  Y  |   Y    |   Y
+ * -> stop_psp/smu              |  Y  |   Y    |  -  |   -    |   Y
+ *    ---- power boundary ----
+ * <- pci_enable + set_master   |  Y  |   Y    |  Y  |   Y    |   Y
+ * <- start_smu/psp             |  Y  |   Y    | (2) |  (2)   |   Y
+ * <- mailbox_init              |  Y  |   Y    |  Y  |   Y    |   Y
+ * <- calibrate_clock           |  Y  |   Y    |  -  |   -    |   Y
+ * <- attach_work_buffer        |  Y  |   Y    |  -  |   -    |   Y
+ * <- query_fw                  |  Y  |   Y    |  -  |   -    |   Y
+ * <- query_aie                 |  -  |   -    |  Y  |   Y    |   Y
+ * <- partition_init            |  -  |   -    |  Y  |   Y    |   Y
+ * <- alloc_async_event         |  -  |   -    |  Y  |   Y    |   Y
+ * <- hwctx_resume_all          |  -  |   -    |  Y  |   Y    |   Y
+ * <- restore VFs               |  Y  |   Y    |  -  |   -    |   -
+ *
+ * (1) VF S3/S4 skips partition_fini: PF tears down stateless FW.
+ *     VF RTM must send partition_fini to firmware explicitly.
+ * (2) VF skips start_smu/psp: PF boots firmware on behalf of all VFs.
+ */
+
 static int aie4_pf_suspend(struct amdxdna_dev *xdna)
 {
 	struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
@@ -1146,6 +1179,17 @@ static int aie4_pf_suspend(struct amdxdna_dev *xdna)
 	return 0;
 }
 
+static int aie4_pf_runtime_suspend(struct amdxdna_dev *xdna)
+{
+	int ret;
+
+	ret = aie4_vfs_alive(xdna);
+	if (ret)
+		return ret;
+
+	return aie4_pf_suspend(xdna);
+}
+
 static int aie4_vf_suspend(struct amdxdna_dev *xdna)
 {
 	struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
@@ -1165,6 +1209,18 @@ static int aie4_vf_suspend(struct amdxdna_dev *xdna)
 	return 0;
 }
 
+static int aie4_vf_runtime_suspend(struct amdxdna_dev *xdna)
+{
+	struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
+
+	drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
+	aie4_hwctx_suspend_all(ndev);
+	aie4_vf_hw_stop(ndev);
+
+	XDNA_DBG(xdna, "vf runtime suspend done");
+	return 0;
+}
+
 static int aie4_classic_suspend(struct amdxdna_dev *xdna)
 {
 	struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
@@ -1317,6 +1373,7 @@ static int aie4_pf_init(struct amdxdna_dev *xdna)
 
 	aie4_msg_init(xdna->dev_handle);
 	amdxdna_dpt_init(&xdna->dev_handle->aie);
+	amdxdna_pm_init(xdna);
 	return 0;
 
 free_work_buf:
@@ -1338,6 +1395,7 @@ static int aie4_vf_init(struct amdxdna_dev *xdna)
 
 	aie4_msg_init(xdna->dev_handle);
 	amdxdna_dpt_init(&xdna->dev_handle->aie);
+	amdxdna_pm_init(xdna);
 	return 0;
 }
 
@@ -1359,6 +1417,7 @@ static int aie4_classic_init(struct amdxdna_dev *xdna)
 
 	aie4_msg_init(xdna->dev_handle);
 	amdxdna_dpt_init(&xdna->dev_handle->aie);
+	amdxdna_pm_init(xdna);
 	return 0;
 
 free_work_buf:
@@ -1370,6 +1429,7 @@ static void aie4_pf_fini(struct amdxdna_dev *xdna)
 {
 	int ret;
 
+	amdxdna_pm_fini(xdna);
 	amdxdna_dpt_fini(&xdna->dev_handle->aie);
 
 	ret = aie4_sriov_stop(xdna->dev_handle);
@@ -1384,12 +1444,14 @@ static void aie4_pf_fini(struct amdxdna_dev *xdna)
 
 static void aie4_vf_fini(struct amdxdna_dev *xdna)
 {
+	amdxdna_pm_fini(xdna);
 	amdxdna_dpt_fini(&xdna->dev_handle->aie);
 	aie4_vf_hw_stop(xdna->dev_handle);
 }
 
 static void aie4_classic_fini(struct amdxdna_dev *xdna)
 {
+	amdxdna_pm_fini(xdna);
 	amdxdna_dpt_fini(&xdna->dev_handle->aie);
 	aie4_classic_hw_stop(xdna->dev_handle);
 	aie4_free_work_buffer(xdna->dev_handle);
@@ -1571,6 +1633,8 @@ const struct amdxdna_dev_ops aie4_pf_ops = {
 	.sriov_configure        = aie4_sriov_configure,
 	.resume			= aie4_pf_resume,
 	.suspend		= aie4_pf_suspend,
+	.runtime_resume		= aie4_pf_resume,
+	.runtime_suspend	= aie4_pf_runtime_suspend, /* additional check on VM passthrough */
 	.register_async_event	= aie4_async_event_register,
 	.handle_dev_async_event	= aie4_handle_dev_event,
 };
@@ -1590,6 +1654,8 @@ const struct amdxdna_dev_ops aie4_vf_ops = {
 	.get_array		= aie4_get_array,
 	.resume			= aie4_vf_resume,
 	.suspend		= aie4_vf_suspend,
+	.runtime_resume		= aie4_vf_resume,
+	.runtime_suspend	= aie4_vf_runtime_suspend, /* each VF needs to clean up itself */
 	.register_async_event	= aie4_async_event_register,
 	.handle_dev_async_event	= aie4_handle_dev_event,
 };
@@ -1609,6 +1675,8 @@ const struct amdxdna_dev_ops aie4_classic_ops = {
 	.get_array		= aie4_get_array,
 	.resume			= aie4_classic_resume,
 	.suspend		= aie4_classic_suspend,
+	.runtime_resume		= aie4_classic_resume,
+	.runtime_suspend	= aie4_classic_suspend,
 	.register_async_event	= aie4_async_event_register,
 	.handle_dev_async_event	= aie4_handle_dev_event,
 };

drivers/accel/amdxdna/aie4_pci.h

@@ -149,10 +149,12 @@ void aie4_fill_health_data(struct amdxdna_gem_obj *cmd_abo, struct amdxdna_hwctx
 int aie4_sriov_configure(struct amdxdna_dev *xdna, int num_vfs);
 int aie4_create_vfs(struct amdxdna_dev_hdl *ndev, int num_vfs);
 int aie4_sriov_stop(struct amdxdna_dev_hdl *ndev);
+int aie4_vfs_alive(struct amdxdna_dev *xdna);
 #else
 #define aie4_sriov_configure NULL
 static inline int aie4_sriov_stop(struct amdxdna_dev_hdl *ndev) { return 0; }
 static inline int aie4_create_vfs(struct amdxdna_dev_hdl *ndev, int num_vfs) { return 0; }
+static inline int aie4_vfs_alive(struct amdxdna_dev *xdna) { return 0; }
 #endif
 
 /* aie4_message.c */

drivers/accel/amdxdna/aie4_sriov.c

@@ -6,6 +6,7 @@
 #include "drm/amdxdna_accel.h"
 #include <drm/drm_print.h>
 #include <linux/pci.h>
+#include <linux/pm_runtime.h>
 
 #include "aie.h"
 #include "aie4_msg_priv.h"
@@ -54,27 +55,31 @@ int aie4_sriov_stop(struct amdxdna_dev_hdl *ndev)
 		return -EPERM;
 	}
 
-	ret = aie4_destroy_vfs(ndev);
-	ndev->num_vfs = 0;
 	pci_disable_sriov(pdev);
-	return ret;
+	ndev->num_vfs = 0;
+
+	/*
+	 * pci_disable_sriov() removes VF drivers first; call destroy_vfs after
+	 * so firmware VF contexts are not cleared before VF drivers finish cleanup.
+	 */
+	return aie4_destroy_vfs(ndev);
 }
 
-static void aie4_link_vfs(struct amdxdna_dev *xdna)
+static int aie4_for_each_vfs(struct amdxdna_dev *xdna,
+			     int (*cb)(struct amdxdna_dev *, struct pci_dev *))
 {
 	struct pci_dev *pdev_pf = to_pci_dev(xdna->ddev.dev);
-	struct device_link *link;
 	struct pci_dev *pdev_vf;
 	int pos, ret;
 	u16 vf_did;
 
 	pos = pci_find_ext_capability(pdev_pf, PCI_EXT_CAP_ID_SRIOV);
 	if (!pos)
-		return;
+		return 0;
 	ret = pci_read_config_word(pdev_pf, pos + PCI_SRIOV_VF_DID, &vf_did);
 	if (ret) {
 		XDNA_ERR(xdna, "read VF Device ID failed %d", ret);
-		return;
+		return -ENODEV;
 	}
 
 	for (pdev_vf = pci_get_device(pdev_pf->vendor, vf_did, NULL);
@@ -83,14 +88,65 @@ static void aie4_link_vfs(struct amdxdna_dev *xdna)
 		if (!pdev_vf->is_virtfn || pdev_vf->physfn != pdev_pf)
 			continue;
 
-		link = device_link_add(&pdev_vf->dev,   /* consumer = VF */
-				       &pdev_pf->dev,   /* supplier = PF */
-				       DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_CONSUMER);
-		if (!link)
-			XDNA_WARN(xdna, "Failed to link VF %s", pci_name(pdev_vf));
-		else
-			XDNA_DBG(xdna, "Linked VF %s", pci_name(pdev_vf));
+		ret = cb(xdna, pdev_vf);
+		if (ret) {
+			/*
+			 * On early return the next iteration never runs, so
+			 * release the current device's ref manually.
+			 * On normal loop exit pci_get_device() returning NULL
+			 * already releases the last device's ref internally.
+			 */
+			pci_dev_put(pdev_vf);
+			return ret;
+		}
 	}
+
+	return 0;
+}
+
+static int aie4_link_vf(struct amdxdna_dev *xdna, struct pci_dev *pdev_vf)
+{
+	struct pci_dev *pdev_pf = to_pci_dev(xdna->ddev.dev);
+	struct device_link *link;
+
+	link = device_link_add(&pdev_vf->dev,   /* consumer = VF */
+			       &pdev_pf->dev,   /* supplier = PF */
+			       DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_CONSUMER);
+	if (!link)
+		XDNA_WARN(xdna, "Failed to link VF %s", pci_name(pdev_vf));
+	else
+		XDNA_DBG(xdna, "Linked VF %s", pci_name(pdev_vf));
+	return 0;
+}
+
+static int aie4_check_vf_alive(struct amdxdna_dev *xdna, struct pci_dev *pdev_vf)
+{
+	struct device_driver *drv = READ_ONCE(pdev_vf->dev.driver);
+
+	if (drv && drv->owner != THIS_MODULE) {
+		XDNA_WARN(xdna, "VF:%s is in passthrough", pci_name(pdev_vf));
+		return -EBUSY;
+	}
+
+	if (!pm_runtime_suspended(&pdev_vf->dev)) {
+		XDNA_WARN(xdna, "VF:%s is busy", pci_name(pdev_vf));
+		return -EBUSY;
+	}
+	return 0;
+}
+
+int aie4_vfs_alive(struct amdxdna_dev *xdna)
+{
+	if (pci_vfs_assigned(to_pci_dev(xdna->ddev.dev))) {
+		XDNA_WARN(xdna, "VF devices are being used in VMs, cannot suspend");
+		return -EBUSY;
+	}
+	return aie4_for_each_vfs(xdna, aie4_check_vf_alive);
+}
+
+static void aie4_link_vfs(struct amdxdna_dev *xdna)
+{
+	aie4_for_each_vfs(xdna, aie4_link_vf);
 }
 
 static int aie4_sriov_start(struct amdxdna_dev_hdl *ndev, int num_vfs)

drivers/accel/amdxdna/amdxdna_pci_drv.c

@@ -560,18 +560,27 @@ static void amdxdna_remove(struct pci_dev *pdev)
 
 static const struct dev_pm_ops amdxdna_pm_ops = {
 	SYSTEM_SLEEP_PM_OPS(amdxdna_pm_suspend, amdxdna_pm_resume)
-	RUNTIME_PM_OPS(amdxdna_pm_suspend, amdxdna_pm_resume, NULL)
+	RUNTIME_PM_OPS(amdxdna_pm_runtime_suspend, amdxdna_pm_runtime_resume, NULL)
 };
 
 static int amdxdna_sriov_configure(struct pci_dev *pdev, int num_vfs)
 {
 	struct amdxdna_dev *xdna = pci_get_drvdata(pdev);
+	int ret;
 
 	guard(mutex)(&xdna->dev_lock);
+
+	ret = amdxdna_pm_resume_get_locked(xdna);
+	if (ret)
+		return ret;
+
 	if (xdna->dev_info->ops->sriov_configure)
-		return xdna->dev_info->ops->sriov_configure(xdna, num_vfs);
+		ret = xdna->dev_info->ops->sriov_configure(xdna, num_vfs);
+	else
+		ret = -EOPNOTSUPP;
 
-	return -ENOENT;
+	amdxdna_pm_suspend_put(xdna);
+	return ret;
 }
 
 static struct pci_driver amdxdna_pci_driver = {

drivers/accel/amdxdna/amdxdna_pci_drv.h

@@ -65,6 +65,8 @@ struct amdxdna_dev_ops {
 	void (*debugfs_init)(struct amdxdna_dev *xdna);
 	int (*resume)(struct amdxdna_dev *xdna);
 	int (*suspend)(struct amdxdna_dev *xdna);
+	int (*runtime_resume)(struct amdxdna_dev *xdna);
+	int (*runtime_suspend)(struct amdxdna_dev *xdna);
 	int (*sriov_configure)(struct amdxdna_dev *xdna, int num_vfs);
 	int (*mmap)(struct amdxdna_client *client, struct vm_area_struct *vma);
 	int (*hwctx_init)(struct amdxdna_hwctx *hwctx);

drivers/accel/amdxdna/amdxdna_pm.c

@@ -65,6 +65,46 @@ int amdxdna_pm_resume(struct device *dev)
 	return ret;
 }
 
+int amdxdna_pm_runtime_suspend(struct device *dev)
+{
+	struct amdxdna_dev *xdna = to_xdna_dev(dev_get_drvdata(dev));
+	int ret = -EOPNOTSUPP;
+
+	guard(mutex)(&xdna->dev_lock);
+	if (xdna->dev_info->ops->runtime_suspend)
+		ret = xdna->dev_info->ops->runtime_suspend(xdna);
+
+	if (!ret) {
+		int dpt_ret = amdxdna_dpt_suspend(xdna);
+
+		if (dpt_ret)
+			XDNA_WARN(xdna, "DPT drain/pause on suspend failed: %d", dpt_ret);
+	}
+
+	XDNA_DBG(xdna, "Runtime suspend done ret %d", ret);
+	return ret;
+}
+
+int amdxdna_pm_runtime_resume(struct device *dev)
+{
+	struct amdxdna_dev *xdna = to_xdna_dev(dev_get_drvdata(dev));
+	int ret = -EOPNOTSUPP;
+
+	guard(mutex)(&xdna->dev_lock);
+	if (xdna->dev_info->ops->runtime_resume)
+		ret = xdna->dev_info->ops->runtime_resume(xdna);
+
+	if (!ret) {
+		int dpt_ret = amdxdna_dpt_resume(xdna);
+
+		if (dpt_ret)
+			XDNA_WARN(xdna, "DPT re-arm on resume failed: %d", dpt_ret);
+	}
+
+	XDNA_DBG(xdna, "Runtime resume done ret %d", ret);
+	return ret;
+}
+
 int amdxdna_pm_resume_get(struct amdxdna_dev *xdna)
 {
 	struct device *dev = xdna->ddev.dev;