isci: Make the driver copy data directly from and to sg for PIO

We can copy the data directly to and from sg for SATA PIO read operations.
There is no reason to involve the hardware SGL. In the process we also need
to kmap the sg because we don't know where that can come from.

We also do to not call phys_to_virt(). The driver already has the information.
We can just calculcate the appropriate offets.

Signed-off-by: Dave Jiang <dave.jiang@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>

7 files changed, 84 insertions, 109 deletions

diff --git a/drivers/scsi/isci/core/sci_util.c b/drivers/scsi/isci/core/sci_util.c
index 5cdd96f29a9..03f919ffb41 100644
--- a/drivers/scsi/isci/core/sci_util.c
+++ b/drivers/scsi/isci/core/sci_util.c
@@ -53,7 +53,9 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include <linux/kernel.h>
 #include "sci_util.h"
+#include "sci_environment.h"
 
 void scic_word_copy_with_swap(
 	u32 *destination,
@@ -68,3 +70,17 @@ void scic_word_copy_with_swap(
 	}
 }
 
+void *scic_request_get_virt_addr(struct scic_sds_request *sci_req, dma_addr_t phys_addr)
+{
+	struct isci_request *ireq = sci_object_get_association(sci_req);
+	dma_addr_t offset;
+
+	BUG_ON(phys_addr < ireq->request_daddr);
+
+	offset = phys_addr - ireq->request_daddr;
+
+	BUG_ON(offset >= ireq->request_alloc_size);
+
+	return (char *)ireq + offset;
+}
+
diff --git a/drivers/scsi/isci/core/sci_util.h b/drivers/scsi/isci/core/sci_util.h
index 9748c76df38..50c272ebcde 100644
--- a/drivers/scsi/isci/core/sci_util.h
+++ b/drivers/scsi/isci/core/sci_util.h
@@ -57,6 +57,7 @@
 #define _SCI_UTIL_H_
 
 #include <linux/string.h>
+#include "scic_sds_request.h"
 
 /**
  * SCIC_SWAP_DWORD() -
@@ -96,9 +97,9 @@
  *    byte swap.
  *
  */
-void scic_word_copy_with_swap(
-	u32 *destination,
-	u32 *source,
-	u32 word_count);
+void scic_word_copy_with_swap(u32 *destination, u32 *source, u32 word_count);
+
+void *scic_request_get_virt_addr(struct scic_sds_request *sds_request,
+				 dma_addr_t phys_addr);
 
 #endif /* _SCI_UTIL_H_ */
diff --git a/drivers/scsi/isci/core/scic_sds_request.c b/drivers/scsi/isci/core/scic_sds_request.c
index 2bd47f49e53..9fa45cd0fc4 100644
--- a/drivers/scsi/isci/core/scic_sds_request.c
+++ b/drivers/scsi/isci/core/scic_sds_request.c
@@ -100,15 +100,6 @@
  * **************************************************************************** */
 
 /**
- * scic_sds_request_get_user_request() -
- *
- * This is a helper macro to return the os handle for this request object.
- */
-#define scic_sds_request_get_user_request(request) \
-	((request)->user_request)
-
-
-/**
  * scic_ssp_io_request_get_object_size() -
  *
  * This macro returns the sizeof memory required to store the an SSP IO
diff --git a/drivers/scsi/isci/core/scic_sds_request.h b/drivers/scsi/isci/core/scic_sds_request.h
index 0691a7552a2..60337e6981c 100644
--- a/drivers/scsi/isci/core/scic_sds_request.h
+++ b/drivers/scsi/isci/core/scic_sds_request.h
@@ -400,6 +400,14 @@ extern const struct scic_sds_io_request_state_handler scic_sds_smp_request_start
 		(scu_sge).address_modifier = 0;	\
 	}
 
+/**
+ * scic_sds_request_get_user_request() -
+ *
+ * This is a helper macro to return the os handle for this request object.
+ */
+#define scic_sds_request_get_user_request(request) \
+	((request)->user_request)
+
 /*
  * *****************************************************************************
  * * CORE REQUEST PROTOTYPES
diff --git a/drivers/scsi/isci/core/scic_sds_stp_request.c b/drivers/scsi/isci/core/scic_sds_stp_request.c
index 6d280c0a2fc..10f160e6ab4 100644
--- a/drivers/scsi/isci/core/scic_sds_stp_request.c
+++ b/drivers/scsi/isci/core/scic_sds_stp_request.c
@@ -486,45 +486,34 @@ void *scic_stp_io_request_get_d2h_reg_address(
  * - if there are more SGL element pairs - advance to the next pair and return
  * element A struct scu_sgl_element*
  */
-struct scu_sgl_element *scic_sds_stp_request_pio_get_next_sgl(
-	struct scic_sds_stp_request *this_request
-	) {
+struct scu_sgl_element *scic_sds_stp_request_pio_get_next_sgl(struct scic_sds_stp_request *stp_req)
+{
 	struct scu_sgl_element *current_sgl;
+	struct scic_sds_request *sci_req = &stp_req->parent;
+	struct scic_sds_request_pio_sgl *pio_sgl = &stp_req->type.pio.request_current;
 
-	if (this_request->type.pio.request_current.sgl_set == SCU_SGL_ELEMENT_PAIR_A) {
-		if (
-			(this_request->type.pio.request_current.sgl_pair->B.address_lower == 0)
-			&& (this_request->type.pio.request_current.sgl_pair->B.address_upper == 0)
-			) {
+	if (pio_sgl->sgl_set == SCU_SGL_ELEMENT_PAIR_A) {
+		if (pio_sgl->sgl_pair->B.address_lower == 0 &&
+		    pio_sgl->sgl_pair->B.address_upper == 0) {
 			current_sgl = NULL;
 		} else {
-			this_request->type.pio.request_current.sgl_set = SCU_SGL_ELEMENT_PAIR_B;
-			current_sgl = &(this_request->type.pio.request_current.sgl_pair->B);
+			pio_sgl->sgl_set = SCU_SGL_ELEMENT_PAIR_B;
+			current_sgl = &pio_sgl->sgl_pair->B;
 		}
 	} else {
-		if (
-			(this_request->type.pio.request_current.sgl_pair->next_pair_lower == 0)
-			&& (this_request->type.pio.request_current.sgl_pair->next_pair_upper == 0)
-			) {
+		if (pio_sgl->sgl_pair->next_pair_lower == 0 &&
+		    pio_sgl->sgl_pair->next_pair_upper == 0) {
 			current_sgl = NULL;
 		} else {
-			dma_addr_t physical_address;
-
-			sci_cb_make_physical_address(
-				physical_address,
-				this_request->type.pio.request_current.sgl_pair->next_pair_upper,
-				this_request->type.pio.request_current.sgl_pair->next_pair_lower
-				);
+			u64 phys_addr;
 
-			this_request->type.pio.request_current.sgl_pair =
-				(struct scu_sgl_element_pair *)scic_cb_get_virtual_address(
-					this_request->parent.owning_controller,
-					physical_address
-					);
+			phys_addr = pio_sgl->sgl_pair->next_pair_upper;
+			phys_addr <<= 32;
+			phys_addr |= pio_sgl->sgl_pair->next_pair_lower;
 
-			this_request->type.pio.request_current.sgl_set = SCU_SGL_ELEMENT_PAIR_A;
-
-			current_sgl = &(this_request->type.pio.request_current.sgl_pair->A);
+			pio_sgl->sgl_pair = scic_request_get_virt_addr(sci_req, phys_addr);
+			pio_sgl->sgl_set = SCU_SGL_ELEMENT_PAIR_A;
+			current_sgl = &pio_sgl->sgl_pair->A;
 		}
 	}
 
@@ -882,82 +871,51 @@ static enum sci_status scic_sds_stp_request_pio_data_out_transmit_data(
 
 /**
  *
- * @this_request: The request that is used for the SGL processing.
+ * @stp_request: The request that is used for the SGL processing.
  * @data_buffer: The buffer of data to be copied.
  * @length: The length of the data transfer.
  *
  * Copy the data from the buffer for the length specified to the IO reqeust SGL
  * specified data region. enum sci_status
  */
-static enum sci_status scic_sds_stp_request_pio_data_in_copy_data_buffer(
-	struct scic_sds_stp_request *this_request,
-	u8 *data_buffer,
-	u32 length)
+static enum sci_status
+scic_sds_stp_request_pio_data_in_copy_data_buffer(struct scic_sds_stp_request *stp_req,
+						  u8 *data_buf, u32 len)
 {
-	enum sci_status status;
-	struct scu_sgl_element *current_sgl;
-	u32 sgl_offset;
-	u32 data_offset;
-	u8 *source_address;
-	u8 *destination_address;
-	u32 copy_length;
-
-	/* Initial setup to get the current working SGL and the offset within the buffer */
-	current_sgl =
-		(this_request->type.pio.request_current.sgl_set == SCU_SGL_ELEMENT_PAIR_A) ?
-		&(this_request->type.pio.request_current.sgl_pair->A) :
-		&(this_request->type.pio.request_current.sgl_pair->B);
-
-	sgl_offset = this_request->type.pio.request_current.sgl_offset;
-
-	source_address = data_buffer;
-	data_offset = 0;
-
-	status = SCI_SUCCESS;
-
-	/* While we are still doing Ok and there is more data to transfer */
-	while (
-		(length > 0)
-		&& (status == SCI_SUCCESS)
-		) {
-		if (current_sgl->length == sgl_offset) {
-			/* This SGL has been exauhasted so we need to get the next SGL */
-			current_sgl = scic_sds_stp_request_pio_get_next_sgl(this_request);
-
-			if (current_sgl == NULL)
-				status = SCI_FAILURE;
-			else
-				sgl_offset = 0;
-		} else {
-			dma_addr_t physical_address;
-
-			sci_cb_make_physical_address(
-				physical_address,
-				current_sgl->address_upper,
-				current_sgl->address_lower
-				);
-
-			destination_address = (u8 *)scic_cb_get_virtual_address(
-				this_request->parent.owning_controller,
-				physical_address
-				);
-
-			source_address += data_offset;
-			destination_address += sgl_offset;
-
-			copy_length = min(length, current_sgl->length - sgl_offset);
-
-			memcpy(destination_address, source_address, copy_length);
-
-			length -= copy_length;
-			sgl_offset += copy_length;
-			data_offset += copy_length;
+	struct scic_sds_request *sci_req;
+	struct isci_request *ireq;
+	u8 *src_addr;
+	int copy_len;
+	struct sas_task *task;
+	struct scatterlist *sg;
+	void *kaddr;
+	int total_len = len;
+
+	sci_req = &stp_req->parent;
+	ireq = scic_sds_request_get_user_request(sci_req);
+	task = isci_request_access_task(ireq);
+	src_addr = data_buf;
+
+	if (task->num_scatter > 0) {
+		sg = task->scatter;
+
+		while (total_len > 0) {
+			struct page *page = sg_page(sg);
+
+			copy_len = min_t(int, total_len, sg_dma_len(sg));
+			kaddr = kmap_atomic(page, KM_IRQ0);
+			memcpy(kaddr + sg->offset, src_addr, copy_len);
+			kunmap_atomic(kaddr, KM_IRQ0);
+			total_len -= copy_len;
+			src_addr += copy_len;
+			sg = sg_next(sg);
 		}
+	} else {
+		BUG_ON(task->total_xfer_len < total_len);
+		memcpy(task->scatter, src_addr, total_len);
 	}
 
-	this_request->type.pio.request_current.sgl_offset = sgl_offset;
-
-	return status;
+	return SCI_SUCCESS;
 }
 
 /**
diff --git a/drivers/scsi/isci/core/scic_sds_stp_request.h b/drivers/scsi/isci/core/scic_sds_stp_request.h
index 4a4c68aa56d..c950bb33ee0 100644
--- a/drivers/scsi/isci/core/scic_sds_stp_request.h
+++ b/drivers/scsi/isci/core/scic_sds_stp_request.h
@@ -116,7 +116,7 @@ struct scic_sds_stp_request {
 			 */
 			u8 sat_protocol;
 
-			struct {
+			struct scic_sds_request_pio_sgl {
 				struct scu_sgl_element_pair *sgl_pair;
 				u8 sgl_set;
 				u32 sgl_offset;
diff --git a/drivers/scsi/isci/isci.h b/drivers/scsi/isci/isci.h
index 6c79b29f310..225764f9d93 100644
--- a/drivers/scsi/isci/isci.h
+++ b/drivers/scsi/isci/isci.h
@@ -84,6 +84,7 @@
 #include "host.h"
 #include "timers.h"
 #include "sci_status.h"
+#include "request.h"
 
 extern struct kmem_cache *isci_kmem_cache;
 extern struct isci_firmware *isci_firmware;