crypto: qat - support fast ACKs in the PFVF protocol

The original design and current implementation of the PFVF protocol
expects the sender to both acquire and relinquish the ownership of the
shared CSR by setting and clearing the "in use" pattern on the remote
half of the register when sending a message. This happens regardless of
the acknowledgment of the reception, to guarantee changes, including
collisions, are surely detected.

However, in the case of a request that requires a response, collisions
can also be detected by the lack of a reply. This can be exploited to
speed up and simplify the above behaviour, letting the receiver both
acknowledge the message and release the CSR in a single transaction:

1) the sender can return as soon as the message has been acknowledged
2) the receiver doesn't have to wait long before acquiring ownership
of the CSR for the response message, greatly improving the overall
throughput.

Howerver, this improvement cannot be leveraged for fire-and-forget
notifications, as it would be impossible for the sender to clearly
distinguish between a collision and an ack immediately followed by a new
message.

This patch implements this optimization in a new version of the protocol
(v3), which applies the fast-ack logic only whenever possible and
guarantees backward compatibility with older versions. For requests, a
new retry loop guarantees a correct behaviour.

Signed-off-by: Marco Chiappero <marco.chiappero@intel.com>
Reviewed-by: Giovanni Cabiddu <giovanni.cabiddu@intel.com>
Reviewed-by: Fiona Trahe <fiona.trahe@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

drivers/crypto/qat/qat_common/adf_accel_devices.h

@@ -158,7 +158,7 @@ struct adf_pfvf_ops {
 	int (*send_msg)(struct adf_accel_dev *accel_dev, struct pfvf_message msg,
 			u32 pfvf_offset, struct mutex *csr_lock);
 	struct pfvf_message (*recv_msg)(struct adf_accel_dev *accel_dev,
-					u32 pfvf_offset);
+					u32 pfvf_offset, u8 compat_ver);
 };
 
 struct adf_hw_device_data {

drivers/crypto/qat/qat_common/adf_gen2_pfvf.c

@@ -124,11 +124,34 @@ static bool is_legacy_user_pfvf_message(u32 msg)
 	return !(msg & ADF_PFVF_MSGORIGIN_SYSTEM);
 }
 
+static bool is_pf2vf_notification(u8 msg_type)
+{
+	switch (msg_type) {
+	case ADF_PF2VF_MSGTYPE_RESTARTING:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool is_vf2pf_notification(u8 msg_type)
+{
+	switch (msg_type) {
+	case ADF_VF2PF_MSGTYPE_INIT:
+	case ADF_VF2PF_MSGTYPE_SHUTDOWN:
+		return true;
+	default:
+		return false;
+	}
+}
+
 struct pfvf_gen2_params {
 	u32 pfvf_offset;
 	struct mutex *csr_lock; /* lock preventing concurrent access of CSR */
 	enum gen2_csr_pos local_offset;
 	enum gen2_csr_pos remote_offset;
+	bool (*is_notification_message)(u8 msg_type);
+	u8 compat_ver;
 };
 
 static int adf_gen2_pfvf_send(struct adf_accel_dev *accel_dev,
@@ -190,15 +213,27 @@ start:
 		csr_val &= ~int_bit;
 	}
 
-	if (csr_val != csr_msg) {
-		dev_dbg(&GET_DEV(accel_dev),
-			"Collision - PFVF CSR overwritten by remote function\n");
+	/* For fire-and-forget notifications, the receiver does not clear
+	 * the in-use pattern. This is used to detect collisions.
+	 */
+	if (params->is_notification_message(msg.type) && csr_val != csr_msg) {
+		/* Collision must have overwritten the message */
+		dev_err(&GET_DEV(accel_dev),
+			"Collision on notification - PFVF CSR overwritten by remote function\n");
 		goto retry;
 	}
 
-	/* Finished with the PFVF CSR; relinquish it and leave msg in CSR */
-	gen2_csr_clear_in_use(&csr_val, remote_offset);
-	ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val);
+	/* If the far side did not clear the in-use pattern it is either
+	 * 1) Notification - message left intact to detect collision
+	 * 2) Older protocol (compatibility version < 3) on the far side
+	 *    where the sender is responsible for clearing the in-use
+	 *    pattern after the received has acknowledged receipt.
+	 * In either case, clear the in-use pattern now.
+	 */
+	if (gen2_csr_is_in_use(csr_val, remote_offset)) {
+		gen2_csr_clear_in_use(&csr_val, remote_offset);
+		ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val);
+	}
 
 out:
 	mutex_unlock(lock);
@@ -218,6 +253,7 @@ static struct pfvf_message adf_gen2_pfvf_recv(struct adf_accel_dev *accel_dev,
 					      struct pfvf_gen2_params *params)
 {
 	void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
+	enum gen2_csr_pos remote_offset = params->remote_offset;
 	enum gen2_csr_pos local_offset = params->local_offset;
 	u32 pfvf_offset = params->pfvf_offset;
 	struct pfvf_message msg = { 0 };
@@ -242,12 +278,22 @@ static struct pfvf_message adf_gen2_pfvf_recv(struct adf_accel_dev *accel_dev,
 	if (unlikely(is_legacy_user_pfvf_message(csr_msg))) {
 		dev_dbg(&GET_DEV(accel_dev),
 			"Ignored non-system message (0x%.8x);\n", csr_val);
+		/* Because this must be a legacy message, the far side
+		 * must clear the in-use pattern, so don't do it.
+		 */
 		return msg;
 	}
 
 	/* Return the pfvf_message format */
 	msg = adf_pfvf_message_of(accel_dev, csr_msg, &csr_gen2_fmt);
 
+	/* The in-use pattern is not cleared for notifications (so that
+	 * it can be used for collision detection) or older implementations
+	 */
+	if (params->compat_ver >= ADF_PFVF_COMPAT_FAST_ACK &&
+	    !params->is_notification_message(msg.type))
+		gen2_csr_clear_in_use(&csr_val, remote_offset);
+
 	/* To ACK, clear the INT bit */
 	csr_val &= ~int_bit;
 	ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val);
@@ -263,6 +309,7 @@ static int adf_gen2_pf2vf_send(struct adf_accel_dev *accel_dev, struct pfvf_mess
 		.pfvf_offset = pfvf_offset,
 		.local_offset = ADF_GEN2_CSR_PF2VF_OFFSET,
 		.remote_offset = ADF_GEN2_CSR_VF2PF_OFFSET,
+		.is_notification_message = is_pf2vf_notification,
 	};
 
 	return adf_gen2_pfvf_send(accel_dev, msg, &params);
@@ -276,30 +323,35 @@ static int adf_gen2_vf2pf_send(struct adf_accel_dev *accel_dev, struct pfvf_mess
 		.pfvf_offset = pfvf_offset,
 		.local_offset = ADF_GEN2_CSR_VF2PF_OFFSET,
 		.remote_offset = ADF_GEN2_CSR_PF2VF_OFFSET,
+		.is_notification_message = is_vf2pf_notification,
 	};
 
 	return adf_gen2_pfvf_send(accel_dev, msg, &params);
 }
 
 static struct pfvf_message adf_gen2_pf2vf_recv(struct adf_accel_dev *accel_dev,
-					       u32 pfvf_offset)
+					       u32 pfvf_offset, u8 compat_ver)
 {
 	struct pfvf_gen2_params params = {
 		.pfvf_offset = pfvf_offset,
 		.local_offset = ADF_GEN2_CSR_PF2VF_OFFSET,
 		.remote_offset = ADF_GEN2_CSR_VF2PF_OFFSET,
+		.is_notification_message = is_pf2vf_notification,
+		.compat_ver = compat_ver,
 	};
 
 	return adf_gen2_pfvf_recv(accel_dev, &params);
 }
 
 static struct pfvf_message adf_gen2_vf2pf_recv(struct adf_accel_dev *accel_dev,
-					       u32 pfvf_offset)
+					       u32 pfvf_offset, u8 compat_ver)
 {
 	struct pfvf_gen2_params params = {
 		.pfvf_offset = pfvf_offset,
 		.local_offset = ADF_GEN2_CSR_VF2PF_OFFSET,
 		.remote_offset = ADF_GEN2_CSR_PF2VF_OFFSET,
+		.is_notification_message = is_vf2pf_notification,
+		.compat_ver = compat_ver,
 	};
 
 	return adf_gen2_pfvf_recv(accel_dev, &params);

drivers/crypto/qat/qat_common/adf_pfvf_msg.h

@@ -89,8 +89,10 @@ enum vf2pf_msgtype {
 enum pfvf_compatibility_version {
 	/* Support for extended capabilities */
 	ADF_PFVF_COMPAT_CAPABILITIES		= 0x02,
+	/* In-use pattern cleared by receiver */
+	ADF_PFVF_COMPAT_FAST_ACK		= 0x03,
 	/* Reference to the latest version */
-	ADF_PFVF_COMPAT_THIS_VERSION		= 0x02,
+	ADF_PFVF_COMPAT_THIS_VERSION		= 0x03,
 };
 
 /* PF->VF Version Response */

drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c

@@ -48,10 +48,11 @@ int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, struct pfvf_me
  */
 static struct pfvf_message adf_recv_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr)
 {
+	struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr];
 	struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev);
 	u32 pfvf_offset = pfvf_ops->get_vf2pf_offset(vf_nr);
 
-	return pfvf_ops->recv_msg(accel_dev, pfvf_offset);
+	return pfvf_ops->recv_msg(accel_dev, pfvf_offset, vf_info->vf_compat_ver);
 }
 
 static adf_pf2vf_blkmsg_provider get_blkmsg_response_provider(u8 type)

drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c

@@ -15,6 +15,8 @@
 #define ADF_PFVF_MSG_ACK_DELAY			2
 #define ADF_PFVF_MSG_ACK_MAX_RETRY		100
 
+/* How often to retry if there is no response */
+#define ADF_PFVF_MSG_RESP_RETRIES	5
 #define ADF_PFVF_MSG_RESP_TIMEOUT	(ADF_PFVF_MSG_ACK_DELAY * \
 					 ADF_PFVF_MSG_ACK_MAX_RETRY + \
 					 ADF_PFVF_MSG_COLLISION_DETECT_DELAY)
@@ -50,7 +52,7 @@ static struct pfvf_message adf_recv_pf2vf_msg(struct adf_accel_dev *accel_dev)
 	struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev);
 	u32 pfvf_offset = pfvf_ops->get_pf2vf_offset(0);
 
-	return pfvf_ops->recv_msg(accel_dev, pfvf_offset);
+	return pfvf_ops->recv_msg(accel_dev, pfvf_offset, accel_dev->vf.pf_compat_ver);
 }
 
 /**
@@ -68,33 +70,37 @@ int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, struct pfvf_message msg,
 		       struct pfvf_message *resp)
 {
 	unsigned long timeout = msecs_to_jiffies(ADF_PFVF_MSG_RESP_TIMEOUT);
+	unsigned int retries = ADF_PFVF_MSG_RESP_RETRIES;
 	int ret;
 
 	reinit_completion(&accel_dev->vf.msg_received);
 
 	/* Send request from VF to PF */
-	ret = adf_send_vf2pf_msg(accel_dev, msg);
-	if (ret) {
-		dev_err(&GET_DEV(accel_dev),
-			"Failed to send request msg to PF\n");
-		return ret;
-	}
+	do {
+		ret = adf_send_vf2pf_msg(accel_dev, msg);
+		if (ret) {
+			dev_err(&GET_DEV(accel_dev),
+				"Failed to send request msg to PF\n");
+			return ret;
+		}
 
-	/* Wait for response */
-	if (!wait_for_completion_timeout(&accel_dev->vf.msg_received,
-					 timeout)) {
-		dev_err(&GET_DEV(accel_dev),
-			"PFVF request/response message timeout expired\n");
-		return -EIO;
-	}
+		/* Wait for response, if it times out retry */
+		ret = wait_for_completion_timeout(&accel_dev->vf.msg_received,
+						  timeout);
+		if (ret) {
+			if (likely(resp))
+				*resp = accel_dev->vf.response;
 
-	if (likely(resp))
-		*resp = accel_dev->vf.response;
+			/* Once copied, set to an invalid value */
+			accel_dev->vf.response.type = 0;
 
-	/* Once copied, set to an invalid value */
-	accel_dev->vf.response.type = 0;
+			return 0;
+		}
 
-	return 0;
+		dev_err(&GET_DEV(accel_dev), "PFVF response message timeout\n");
+	} while (--retries);
+
+	return -EIO;
 }
 
 static int adf_vf2pf_blkmsg_data_req(struct adf_accel_dev *accel_dev, bool crc,