diff --git a/drivers/media/i2c/imx500.c b/drivers/media/i2c/imx500.c
index c5a0e1d1491bb6..5fc593340e9588 100644
--- a/drivers/media/i2c/imx500.c
+++ b/drivers/media/i2c/imx500.c
@@ -22,6 +22,7 @@
 #include <media/v4l2-ctrls.h>
 #include <media/v4l2-device.h>
 #include <media/v4l2-event.h>
+#include <media/v4l2-fh.h>
 #include <media/v4l2-fwnode.h>
 #include <media/v4l2-mediabus.h>
 
@@ -1924,9 +1925,17 @@ static int imx500_clear_weights(struct imx500 *imx500)
 
 static void imx500_clear_fw_network(struct imx500 *imx500)
 {
+	struct i2c_client *client = v4l2_get_subdevdata(&imx500->sd);
+
 	/* Remove any previous firmware blob. */
-	if (imx500->fw_network)
+	if (imx500->fw_network) {
 		vfree(imx500->fw_network);
+		/*
+		 * Release the PM ref taken when the fw was loaded in
+		 * imx500_set_ctrl().
+		 */
+		pm_runtime_put_noidle(&client->dev);
+	}
 
 	imx500->fw_network = NULL;
 	imx500->network_written = false;
@@ -2138,6 +2147,13 @@ static int imx500_set_ctrl(struct v4l2_ctrl *ctrl)
 		}
 
 		imx500_clear_fw_network(imx500);
+
+		/*
+		 * Take a pm_runtime reference before the read.  This keeps
+		 * the device active as long as a NW firmware has been loaded.
+		 */
+		pm_runtime_resume_and_get(&client->dev);
+
 		ret = kernel_read_file_from_fd(ctrl->val, 0,
 					       (void **)&imx500->fw_network, INT_MAX,
 					       &imx500->fw_network_size,
@@ -2151,7 +2167,8 @@ static int imx500_set_ctrl(struct v4l2_ctrl *ctrl)
 		if (ret < 0) {
 			dev_err(&client->dev, "%s failed to read fw image: %d\n",
 				__func__, ret);
-			imx500_clear_fw_network(imx500);
+			/* Release the ref we took above. */
+			pm_runtime_put_noidle(&client->dev);
 			return ret;
 		}
 		if (ret != imx500->fw_network_size) {
@@ -2315,6 +2332,26 @@ static const struct v4l2_ctrl_ops imx500_ctrl_ops = {
 	.s_ctrl = imx500_set_ctrl,
 };
 
+static int imx500_subdev_close(struct v4l2_subdev *sd,
+			       struct v4l2_subdev_fh *fh)
+{
+	struct imx500 *imx500 = to_imx500(sd);
+	/*
+	 * Release the cached rpk when the last userspace handle on the subdev
+	 * is closed. This also drops the pm_runtime reference ta taken in the
+	 * NETWORK_FW_FD branch of imx500_set_ctrl(), so the device can
+	 * autosuspend between sessions.
+	 */
+	if (v4l2_fh_is_singular(&fh->vfh))
+		imx500_clear_fw_network(imx500);
+
+	return 0;
+}
+
+static const struct v4l2_subdev_internal_ops imx500_internal_ops = {
+	.close = imx500_subdev_close,
+};
+
 static const struct v4l2_ctrl_config imx500_notify_gains_ctrl = {
 	.ops = &imx500_ctrl_ops,
 	.id = V4L2_CID_NOTIFY_GAINS,
@@ -3616,6 +3653,7 @@ static int imx500_probe(struct i2c_client *client)
 		goto error_power_off;
 
 	/* Initialize subdev */
+	imx500->sd.internal_ops = &imx500_internal_ops;
 	imx500->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE |
 			    V4L2_SUBDEV_FL_HAS_EVENTS;
 	imx500->sd.entity.function = MEDIA_ENT_F_CAM_SENSOR;