From 3fabe979f9a9a45501765e4027dc8fcf96a9b480 Mon Sep 17 00:00:00 2001
From: Tom <support@vamrs.com>
Date: Fri, 8 Jan 2021 19:51:05 +0800
Subject: [PATCH] drivers/usb: Add dcache flush(VIC7100 ONLY)
drivers/usb/cdns3/
drivers/usb/core/
drivers/usb/host/
include/linux/usb.h
---
drivers/usb/cdns3/debug.h | 3 +
drivers/usb/cdns3/ep0.c | 127 ++++++++++-
drivers/usb/cdns3/gadget.c | 207 +++++++++++++++++-
drivers/usb/cdns3/gadget.h | 8 +
drivers/usb/cdns3/trace.h | 7 +
drivers/usb/core/devio.c | 22 ++
drivers/usb/core/hcd.c | 81 ++++++-
drivers/usb/core/urb.c | 4 +
drivers/usb/core/usb.c | 10 +
drivers/usb/host/xhci-dbg.c | 3 +
drivers/usb/host/xhci-dbgcap.c | 28 +++
drivers/usb/host/xhci-debugfs.c | 12 ++
drivers/usb/host/xhci-hub.c | 9 +-
drivers/usb/host/xhci-mem.c | 141 ++++++++++++-
drivers/usb/host/xhci-ring.c | 359 ++++++++++++++++++++++++++++++--
drivers/usb/host/xhci.c | 233 ++++++++++++++++++++-
include/linux/usb.h | 23 ++
17 files changed, 1226 insertions(+), 51 deletions(-)
diff --git a/drivers/usb/cdns3/debug.h b/drivers/usb/cdns3/debug.h
index a5c6a29e1340..ba4143280a23 100644
--- a/drivers/usb/cdns3/debug.h
+++ b/drivers/usb/cdns3/debug.h
@@ -152,6 +152,9 @@ static inline char *cdns3_dbg_ring(struct cdns3_endpoint *priv_ep,
le32_to_cpu(trb->buffer),
le32_to_cpu(trb->length),
le32_to_cpu(trb->control));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(trb, sizeof(struct cdns3_trb));
+#endif
addr += sizeof(*trb);
}
diff --git a/drivers/usb/cdns3/ep0.c b/drivers/usb/cdns3/ep0.c
index d3121a32cc68..cb4237c60540 100644
--- a/drivers/usb/cdns3/ep0.c
+++ b/drivers/usb/cdns3/ep0.c
@@ -53,6 +53,11 @@ static void cdns3_ep0_run_transfer(struct cdns3_device *priv_dev,
priv_ep->trb_pool[1].control = 0;
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma),
+ 2 * TRB_SIZE);
+#endif
+
trace_cdns3_prepare_trb(priv_ep, priv_ep->trb_pool);
cdns3_select_ep(priv_dev, priv_dev->ep0_data_dir);
@@ -88,6 +93,9 @@ static int cdns3_ep0_delegate_req(struct cdns3_device *priv_dev,
spin_unlock(&priv_dev->lock);
priv_dev->setup_pending = 1;
ret = priv_dev->gadget_driver->setup(&priv_dev->gadget, ctrl_req);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_req, sizeof(struct usb_ctrlrequest));
+#endif
priv_dev->setup_pending = 0;
spin_lock(&priv_dev->lock);
return ret;
@@ -97,6 +105,12 @@ static void cdns3_prepare_setup_packet(struct cdns3_device *priv_dev)
{
priv_dev->ep0_data_dir = 0;
priv_dev->ep0_stage = CDNS3_SETUP_STAGE;
+
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ gadget_flush_dcache(priv_dev->setup_dma,
+ sizeof(struct usb_ctrlrequest));
+#endif
+
cdns3_ep0_run_transfer(priv_dev, priv_dev->setup_dma,
sizeof(struct usb_ctrlrequest), 0, 0);
}
@@ -140,6 +154,9 @@ static int cdns3_req_ep0_set_configuration(struct cdns3_device *priv_dev,
u32 config = le16_to_cpu(ctrl_req->wValue);
int result = 0;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_req, sizeof(struct usb_ctrlrequest));
+#endif
switch (device_state) {
case USB_STATE_ADDRESS:
result = cdns3_ep0_delegate_req(priv_dev, ctrl_req);
@@ -185,7 +202,9 @@ static int cdns3_req_ep0_set_address(struct cdns3_device *priv_dev,
u32 addr;
addr = le16_to_cpu(ctrl_req->wValue);
-
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_req, sizeof(struct usb_ctrlrequest));
+#endif
if (addr > USB_DEVICE_MAX_ADDRESS) {
dev_err(priv_dev->dev,
"Device address (%d) cannot be greater than %d\n",
@@ -225,9 +244,14 @@ static int cdns3_req_ep0_get_status(struct cdns3_device *priv_dev,
u16 usb_status = 0;
u32 recip;
u8 index;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ u32 tmp_ind;
+#endif
recip = ctrl->bRequestType & USB_RECIP_MASK;
-
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl, sizeof(struct usb_ctrlrequest));
+#endif
switch (recip) {
case USB_RECIP_DEVICE:
/* self powered */
@@ -253,8 +277,17 @@ static int cdns3_req_ep0_get_status(struct cdns3_device *priv_dev,
index = cdns3_ep_addr_to_index(le16_to_cpu(ctrl->wIndex));
priv_ep = priv_dev->eps[index];
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ tmp_ind = ctrl->wIndex;
+ cdns_virt_flush_dcache(ctrl, sizeof(struct usb_ctrlrequest));
+
+ /* check if endpoint is stalled or stall is pending */
+ cdns3_select_ep(priv_dev, tmp_ind);
+#else
+
/* check if endpoint is stalled or stall is pending */
cdns3_select_ep(priv_dev, le16_to_cpu(ctrl->wIndex));
+#endif
if (EP_STS_STALL(readl(&priv_dev->regs->ep_sts)) ||
(priv_ep->flags & EP_STALL_PENDING))
usb_status = BIT(USB_ENDPOINT_HALT);
@@ -266,6 +299,10 @@ static int cdns3_req_ep0_get_status(struct cdns3_device *priv_dev,
response_pkt = (__le16 *)priv_dev->setup_buf;
*response_pkt = cpu_to_le16(usb_status);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(priv_dev->setup_dma, sizeof(*response_pkt));
+#endif
+
cdns3_ep0_run_transfer(priv_dev, priv_dev->setup_dma,
sizeof(*response_pkt), 1, 0);
return 0;
@@ -282,6 +319,9 @@ static int cdns3_ep0_feature_handle_device(struct cdns3_device *priv_dev,
u16 tmode;
wValue = le16_to_cpu(ctrl->wValue);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl, sizeof(struct usb_ctrlrequest));
+#endif
state = priv_dev->gadget.state;
speed = priv_dev->gadget.speed;
@@ -309,7 +349,9 @@ static int cdns3_ep0_feature_handle_device(struct cdns3_device *priv_dev,
return -EINVAL;
tmode = le16_to_cpu(ctrl->wIndex);
-
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl, sizeof(struct usb_ctrlrequest));
+#endif
if (!set || (tmode & 0xff) != 0)
return -EINVAL;
@@ -342,7 +384,9 @@ static int cdns3_ep0_feature_handle_intf(struct cdns3_device *priv_dev,
int ret = 0;
wValue = le16_to_cpu(ctrl->wValue);
-
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl, sizeof(struct usb_ctrlrequest));
+#endif
switch (wValue) {
case USB_INTRF_FUNC_SUSPEND:
break;
@@ -360,17 +404,38 @@ static int cdns3_ep0_feature_handle_endpoint(struct cdns3_device *priv_dev,
struct cdns3_endpoint *priv_ep;
int ret = 0;
u8 index;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ u32 tmp_ind;
+#endif
- if (le16_to_cpu(ctrl->wValue) != USB_ENDPOINT_HALT)
+ if (le16_to_cpu(ctrl->wValue) != USB_ENDPOINT_HALT) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl, sizeof(struct usb_ctrlrequest));
+#endif
return -EINVAL;
+ }
- if (!(ctrl->wIndex & ~USB_DIR_IN))
+ if (!(ctrl->wIndex & ~USB_DIR_IN)) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl, sizeof(struct usb_ctrlrequest));
+#endif
return 0;
+ }
+
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl, sizeof(struct usb_ctrlrequest));
+#endif
index = cdns3_ep_addr_to_index(le16_to_cpu(ctrl->wIndex));
priv_ep = priv_dev->eps[index];
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ tmp_ind = ctrl->wIndex;
+ cdns_virt_flush_dcache(ctrl, sizeof(struct usb_ctrlrequest));
+ cdns3_select_ep(priv_dev, tmp_ind);
+#else
cdns3_select_ep(priv_dev, le16_to_cpu(ctrl->wIndex));
+#endif
if (set)
__cdns3_gadget_ep_set_halt(priv_ep);
@@ -400,7 +465,9 @@ static int cdns3_req_ep0_handle_feature(struct cdns3_device *priv_dev,
u32 recip;
recip = ctrl->bRequestType & USB_RECIP_MASK;
-
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl, sizeof(struct usb_ctrlrequest));
+#endif
switch (recip) {
case USB_RECIP_DEVICE:
ret = cdns3_ep0_feature_handle_device(priv_dev, ctrl, set);
@@ -434,9 +501,17 @@ static int cdns3_req_ep0_set_sel(struct cdns3_device *priv_dev,
if (le16_to_cpu(ctrl_req->wLength) != 6) {
dev_err(priv_dev->dev, "Set SEL should be 6 bytes, got %d\n",
ctrl_req->wLength);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_req, sizeof(struct usb_ctrlrequest));
+#endif
return -EINVAL;
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_req, sizeof(struct usb_ctrlrequest));
+ cdns_flush_dcache(priv_dev->setup_dma, 6);
+#endif
+
cdns3_ep0_run_transfer(priv_dev, priv_dev->setup_dma, 6, 1, 0);
return 0;
}
@@ -452,11 +527,19 @@ static int cdns3_req_ep0_set_sel(struct cdns3_device *priv_dev,
static int cdns3_req_ep0_set_isoch_delay(struct cdns3_device *priv_dev,
struct usb_ctrlrequest *ctrl_req)
{
- if (ctrl_req->wIndex || ctrl_req->wLength)
+ if (ctrl_req->wIndex || ctrl_req->wLength) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_req, sizeof(struct usb_ctrlrequest));
+#endif
return -EINVAL;
+ }
priv_dev->isoch_delay = le16_to_cpu(ctrl_req->wValue);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_req, sizeof(struct usb_ctrlrequest));
+#endif
+
return 0;
}
@@ -472,7 +555,13 @@ static int cdns3_ep0_standard_request(struct cdns3_device *priv_dev,
{
int ret;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ u8 bReq = ctrl_req->bRequest;
+ cdns_virt_flush_dcache(ctrl_req, sizeof(struct usb_ctrlrequest));
+ switch (bReq) {
+#else
switch (ctrl_req->bRequest) {
+#endif
case USB_REQ_SET_ADDRESS:
ret = cdns3_req_ep0_set_address(priv_dev, ctrl_req);
break;
@@ -535,7 +624,9 @@ static void cdns3_ep0_setup_phase(struct cdns3_device *priv_dev)
int result;
priv_dev->ep0_data_dir = ctrl->bRequestType & USB_DIR_IN;
-
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl, sizeof(struct usb_ctrlrequest));
+#endif
trace_cdns3_ctrl_req(ctrl);
if (!list_empty(&priv_ep->pending_req_list)) {
@@ -552,10 +643,17 @@ static void cdns3_ep0_setup_phase(struct cdns3_device *priv_dev)
else
priv_dev->ep0_stage = CDNS3_STATUS_STAGE;
- if ((ctrl->bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD)
+ if ((ctrl->bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl, sizeof(struct usb_ctrlrequest));
+#endif
result = cdns3_ep0_standard_request(priv_dev, ctrl);
- else
+ } else {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl, sizeof(struct usb_ctrlrequest));
+#endif
result = cdns3_ep0_delegate_req(priv_dev, ctrl);
+ }
if (result == USB_GADGET_DELAYED_STATUS)
return;
@@ -579,6 +677,10 @@ static void cdns3_transfer_completed(struct cdns3_device *priv_dev)
request->actual =
TRB_LEN(le32_to_cpu(priv_ep->trb_pool->length));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma),
+ sizeof(struct cdns3_trb));
+#endif
priv_ep->dir = priv_dev->ep0_data_dir;
cdns3_gadget_giveback(priv_ep, to_cdns3_request(request), 0);
}
@@ -764,6 +866,9 @@ static int cdns3_gadget_ep0_queue(struct usb_ep *ep,
(request->length % ep->maxpacket == 0))
zlp = 1;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ gadget_flush_dcache(request->dma, request->length);
+#endif
cdns3_ep0_run_transfer(priv_dev, request->dma, request->length, 1, zlp);
spin_unlock_irqrestore(&priv_dev->lock, flags);
diff --git a/drivers/usb/cdns3/gadget.c b/drivers/usb/cdns3/gadget.c
index 0aa85cc07ff1..7279af6c4d3c 100644
--- a/drivers/usb/cdns3/gadget.c
+++ b/drivers/usb/cdns3/gadget.c
@@ -245,6 +245,9 @@ int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
}
memset(priv_ep->trb_pool, 0, ring_size);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(priv_ep->trb_pool_dma, ring_size);
+#endif
priv_ep->num_trbs = num_trbs;
@@ -264,6 +267,11 @@ int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma));
link_trb->control = cpu_to_le32(TRB_CYCLE | TRB_TYPE(TRB_LINK) | TRB_TOGGLE);
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(
+ EP_TRADDR_TRADDR(cdns3_trb_virt_to_dma(priv_ep, link_trb)),
+ TRB_SIZE);
+#endif
return 0;
}
@@ -479,6 +487,11 @@ static void __cdns3_descmiss_copy_data(struct usb_request *request,
memcpy(&((u8 *)request->buf)[request->actual],
descmiss_req->buf,
descmiss_req->actual);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(
+ &((u8 *)request->buf)[request->actual],
+ descmiss_req->actual);
+#endif
request->actual = length;
} else {
/* It should never occures */
@@ -834,9 +847,14 @@ void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep,
priv_ep->dir);
if ((priv_req->flags & REQUEST_UNALIGNED) &&
- priv_ep->dir == USB_DIR_OUT && !request->status)
+ priv_ep->dir == USB_DIR_OUT && !request->status) {
memcpy(request->buf, priv_req->aligned_buf->buf,
request->length);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(priv_req->aligned_buf->buf,
+ request->length);
+#endif
+ }
priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED);
/* All TRBs have finished, clear the counter */
@@ -936,6 +954,10 @@ static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req)
return -ENOMEM;
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(buf->dma, buf->size);
+#endif
+
if (priv_req->aligned_buf) {
trace_cdns3_free_aligned_request(priv_req);
priv_req->aligned_buf->in_use = 0;
@@ -953,6 +975,10 @@ static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req)
if (priv_ep->dir == USB_DIR_IN) {
memcpy(buf->buf, priv_req->request.buf,
priv_req->request.length);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(buf->dma, buf->size);
+ cdns_virt_flush_dcache(priv_req->request.buf, buf->size);
+#endif
}
priv_req->flags |= REQUEST_UNALIGNED;
@@ -1015,10 +1041,18 @@ static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
priv_ep->flags |= EP_PENDING_REQUEST;
/* must allocate buffer aligned to 8 */
- if (priv_req->flags & REQUEST_UNALIGNED)
+ if (priv_req->flags & REQUEST_UNALIGNED){
trb_dma = priv_req->aligned_buf->dma;
- else
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ gadget_flush_dcache(priv_req->aligned_buf->dma,
+ priv_req->aligned_buf->size);
+#endif
+ }else{
trb_dma = request->dma;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ gadget_flush_dcache(request->dma, request->length);
+#endif
+ }
/* For stream capable endpoints driver use only single TD. */
trb = priv_ep->trb_pool + priv_ep->enqueue;
@@ -1034,15 +1068,34 @@ static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
if (!request->num_sgs) {
trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma));
length = request->length;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(
+ EP_TRADDR_TRADDR(cdns3_trb_virt_to_dma(priv_ep, trb)),
+ sizeof(struct cdns3_trb));
+#endif
} else {
trb->buffer = cpu_to_le32(TRB_BUFFER(request->sg[sg_idx].dma_address));
length = request->sg[sg_idx].length;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(TRB_BUFFER(request->sg[sg_idx].dma_address),
+ request->sg[sg_idx].length);
+#endif
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ gadget_flush_dcache(TRB_BUFFER(request->sg[sg_idx].dma_address),
+ request->sg[sg_idx].length);
+#endif
}
tdl = DIV_ROUND_UP(length, priv_ep->endpoint.maxpacket);
trb->length = cpu_to_le32(TRB_BURST_LEN(16) | TRB_LEN(length));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(EP_TRADDR_TRADDR(cdns3_trb_virt_to_dma(priv_ep,
+ trb)),
+ sizeof(struct cdns3_trb));
+#endif
+
/*
* For DEV_VER_V2 controller version we have enabled
* USB_CONF2_EN_TDL_TRB in DMULT configuration.
@@ -1055,6 +1108,11 @@ static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
priv_req->flags |= REQUEST_PENDING;
trb->control = cpu_to_le32(control);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(EP_TRADDR_TRADDR(cdns3_trb_virt_to_dma(priv_ep,
+ trb)),
+ sizeof(struct cdns3_trb));
+#endif
trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
@@ -1062,6 +1120,10 @@ static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
* Memory barrier - Cycle Bit must be set before trb->length and
* trb->buffer fields.
*/
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ gadget_flush_dcache(cdns3_trb_virt_to_dma(priv_ep, trb),
+ sizeof(struct cdns3_trb));
+#endif
wmb();
/* always first element */
@@ -1123,6 +1185,9 @@ static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
u32 control;
int pcs;
u16 total_tdl = 0;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ int number = 0;
+#endif
struct scatterlist *s = NULL;
bool sg_supported = !!(request->num_mapped_sgs);
@@ -1142,10 +1207,18 @@ static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
priv_ep->flags |= EP_PENDING_REQUEST;
/* must allocate buffer aligned to 8 */
- if (priv_req->flags & REQUEST_UNALIGNED)
+ if (priv_req->flags & REQUEST_UNALIGNED){
trb_dma = priv_req->aligned_buf->dma;
- else
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ gadget_flush_dcache(priv_req->aligned_buf->dma,
+ priv_req->aligned_buf->size);
+#endif
+ }else{
trb_dma = request->dma;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ gadget_flush_dcache(request->dma, request->length);
+#endif
+ }
trb = priv_ep->trb_pool + priv_ep->enqueue;
priv_req->start_trb = priv_ep->enqueue;
@@ -1183,6 +1256,12 @@ static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
link_trb->control = cpu_to_le32(((priv_ep->pcs) ? TRB_CYCLE : 0) |
TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(
+ EP_TRADDR_TRADDR(cdns3_trb_virt_to_dma(priv_ep,
+ link_trb)),
+ sizeof(struct cdns3_trb));
+#endif
}
if (priv_dev->dev_ver <= DEV_VER_V2)
@@ -1218,12 +1297,26 @@ static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
length = request->length;
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(
+ EP_TRADDR_TRADDR(cdns3_trb_virt_to_dma(priv_ep, trb)),
+ sizeof(struct cdns3_trb));
+ if(request->num_sgs)
+ gadget_flush_dcache(request->sg[sg_iter].dma_address,
+ request->sg[sg_iter].length);
+#endif
+
if (priv_ep->flags & EP_TDLCHK_EN)
total_tdl += DIV_ROUND_UP(length,
priv_ep->endpoint.maxpacket);
trb->length |= cpu_to_le32(TRB_BURST_LEN(priv_ep->trb_burst_size) |
TRB_LEN(length));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(
+ EP_TRADDR_TRADDR(cdns3_trb_virt_to_dma(priv_ep, trb)),
+ sizeof(struct cdns3_trb));
+#endif
pcs = priv_ep->pcs ? TRB_CYCLE : 0;
/*
@@ -1255,12 +1348,23 @@ static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
s = sg_next(s);
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(
+ EP_TRADDR_TRADDR(cdns3_trb_virt_to_dma(priv_ep, trb)),
+ sizeof(struct cdns3_trb));
+#endif
+
control = 0;
++sg_iter;
priv_req->end_trb = priv_ep->enqueue;
cdns3_ep_inc_enq(priv_ep);
trb = priv_ep->trb_pool + priv_ep->enqueue;
trb->length = 0;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(
+ EP_TRADDR_TRADDR(cdns3_trb_virt_to_dma(priv_ep, trb)),
+ sizeof(struct cdns3_trb));
+#endif
} while (sg_iter < num_trb);
trb = priv_req->trb;
@@ -1270,6 +1374,11 @@ static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
if (sg_iter == 1)
trb->control |= cpu_to_le32(TRB_IOC | TRB_ISP);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(
+ EP_TRADDR_TRADDR(cdns3_trb_virt_to_dma(priv_ep, trb)),
+ sizeof(struct cdns3_trb));
+#endif
if (priv_dev->dev_ver < DEV_VER_V2 &&
(priv_ep->flags & EP_TDLCHK_EN)) {
@@ -1294,8 +1403,14 @@ static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
wmb();
/* give the TD to the consumer*/
- if (togle_pcs)
+ if (togle_pcs) {
trb->control = trb->control ^ cpu_to_le32(1);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(
+ EP_TRADDR_TRADDR(cdns3_trb_virt_to_dma(priv_ep, trb)),
+ sizeof(struct cdns3_trb));
+#endif
+ }
if (priv_dev->dev_ver <= DEV_VER_V2)
cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep);
@@ -1323,6 +1438,22 @@ static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
*/
wmb();
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ if((priv_req->start_trb + num_trb) > (priv_ep->num_trbs - 1)) {
+ number = priv_ep->num_trbs - 1 - priv_req->start_trb;
+ gadget_flush_dcache(priv_ep->trb_pool_dma +
+ (priv_req->start_trb * TRB_SIZE),
+ (number + 1) * TRB_SIZE);
+ gadget_flush_dcache(priv_ep->trb_pool_dma,
+ (num_trb - number)* TRB_SIZE);
+ } else {
+ gadget_flush_dcache(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma +
+ priv_req->start_trb *
+ TRB_SIZE),
+ num_trb * TRB_SIZE);
+ }
+#endif
+
/*
* For DMULT mode we can set address to transfer ring only once after
* enabling endpoint.
@@ -1507,9 +1638,18 @@ static void cdns3_transfer_completed(struct cdns3_device *priv_dev,
/* Request was dequeued and TRB was changed to TRB_LINK. */
if (TRB_FIELD_TO_TYPE(le32_to_cpu(trb->control)) == TRB_LINK) {
trace_cdns3_complete_trb(priv_ep, trb);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ gadget_flush_dcache(EP_TRADDR_TRADDR(cdns3_trb_virt_to_dma(priv_ep, trb)),
+ sizeof(struct cdns3_trb));
+#endif
cdns3_move_deq_to_next_trb(priv_req);
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(
+ EP_TRADDR_TRADDR(cdns3_trb_virt_to_dma(priv_ep, trb)),
+ sizeof(struct cdns3_trb));
+#endif
if (!request->stream_id) {
/* Re-select endpoint. It could be changed by other CPU
* during handling usb_gadget_giveback_request.
@@ -1553,6 +1693,11 @@ static void cdns3_transfer_completed(struct cdns3_device *priv_dev,
cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
trb = priv_ep->trb_pool;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(
+ EP_TRADDR_TRADDR(cdns3_trb_virt_to_dma(priv_ep, trb)),
+ sizeof(struct cdns3_trb));
+#endif
trace_cdns3_complete_trb(priv_ep, trb);
if (trb != priv_req->trb)
@@ -1561,6 +1706,12 @@ static void cdns3_transfer_completed(struct cdns3_device *priv_dev,
priv_req->trb, trb);
request->actual += TRB_LEN(le32_to_cpu(trb->length));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(
+ EP_TRADDR_TRADDR(cdns3_trb_virt_to_dma(priv_ep,
+ trb)),
+ sizeof(struct cdns3_trb));
+#endif
if (!request->num_sgs ||
(request->num_sgs == (priv_ep->stream_sg_idx + 1))) {
@@ -1768,6 +1919,10 @@ static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev,
__must_hold(&priv_dev->lock)
{
int speed = 0;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ int i;
+ struct cdns3_endpoint *priv_ep;
+#endif
trace_cdns3_usb_irq(priv_dev, usb_ists);
if (usb_ists & USB_ISTS_L1ENTI) {
@@ -1796,6 +1951,18 @@ __must_hold(&priv_dev->lock)
priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
cdns3_hw_reset_eps_config(priv_dev);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ /* clean TRB*/
+ for(i = 0;i < CDNS3_ENDPOINTS_MAX_COUNT; i++){
+ priv_ep = priv_dev->eps[i];
+ if(priv_ep && priv_ep->trb_pool){
+ memset(priv_ep->trb_pool, 0,
+ priv_ep->alloc_ring_size);
+ gadget_flush_dcache(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma),
+ priv_ep->alloc_ring_size);
+ }
+ }
+#endif
}
if (usb_ists & (USB_ISTS_L2ENTI | USB_ISTS_U3ENTI)) {
@@ -2641,6 +2808,12 @@ found:
((priv_req->end_trb + 1) * TRB_SIZE)));
link_trb->control = cpu_to_le32((le32_to_cpu(link_trb->control) & TRB_CYCLE) |
TRB_TYPE(TRB_LINK) | TRB_CHAIN);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(
+ EP_TRADDR_TRADDR(cdns3_trb_virt_to_dma(priv_ep,
+ link_trb)),
+ sizeof(struct cdns3_trb));
+#endif
if (priv_ep->wa1_trb == priv_req->trb)
cdns3_wa1_restore_cycle_bit(priv_ep);
@@ -2694,8 +2867,15 @@ int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
if (request) {
priv_req = to_cdns3_request(request);
trb = priv_req->trb;
- if (trb)
+ if (trb) {
trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(
+ EP_TRADDR_TRADDR(cdns3_trb_virt_to_dma(priv_ep,
+ trb)),
+ sizeof(struct cdns3_trb));
+#endif
+ }
}
writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
@@ -2709,9 +2889,16 @@ int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING);
if (request) {
- if (trb)
+ if (trb) {
trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(
+ EP_TRADDR_TRADDR(cdns3_trb_virt_to_dma(priv_ep,
+ trb)),
+ sizeof(struct cdns3_trb));
+#endif
+ }
cdns3_rearm_transfer(priv_ep, 1);
}
@@ -3199,7 +3386,9 @@ static int cdns3_gadget_start(struct cdns3 *cdns)
ret = -ENOMEM;
goto err2;
}
-
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(priv_dev->setup_dma, 8);
+#endif
priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6);
dev_dbg(priv_dev->dev, "Device Controller version: %08x\n",
diff --git a/drivers/usb/cdns3/gadget.h b/drivers/usb/cdns3/gadget.h
index 21fa461c518e..1493a4a3fab0 100644
--- a/drivers/usb/cdns3/gadget.h
+++ b/drivers/usb/cdns3/gadget.h
@@ -1365,4 +1365,12 @@ int cdns3_ep_config(struct cdns3_endpoint *priv_ep, bool enable);
void cdns3_check_ep0_interrupt_proceed(struct cdns3_device *priv_dev, int dir);
int __cdns3_gadget_wakeup(struct cdns3_device *priv_dev);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+#include <soc/starfive/vic7100.h>
+static inline void gadget_flush_dcache(unsigned long start, unsigned long len)
+{
+ starfive_flush_dcache(_ALIGN_DOWN(start, 64), len + start % 64);
+}
+#endif
+
#endif /* __LINUX_CDNS3_GADGET */
diff --git a/drivers/usb/cdns3/trace.h b/drivers/usb/cdns3/trace.h
index 0a2a3269bfac..ca368bed1f01 100644
--- a/drivers/usb/cdns3/trace.h
+++ b/drivers/usb/cdns3/trace.h
@@ -187,6 +187,9 @@ DECLARE_EVENT_CLASS(cdns3_log_ctrl,
__entry->wIndex = le16_to_cpu(ctrl->wIndex);
__entry->wLength = le16_to_cpu(ctrl->wLength);
),
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl, sizeof(struct usb_ctrlrequest));
+#endif
TP_printk("%s", usb_decode_ctrl(__get_str(str), CDNS3_MSG_MAX,
__entry->bRequestType,
__entry->bRequest, __entry->wValue,
@@ -410,6 +413,10 @@ DECLARE_EVENT_CLASS(cdns3_log_trb,
__entry->type = usb_endpoint_type(priv_ep->endpoint.desc);
__entry->last_stream_id = priv_ep->last_stream_id;
),
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(EP_TRADDR_TRADDR(cdns3_trb_virt_to_dma(priv_ep, trb)),
+ sizeof(struct cdns3_trb));
+#endif
TP_printk("%s: trb %p, dma buf: 0x%08x, size: %ld, burst: %d ctrl: 0x%08x (%s%s%s%s%s%s%s) SID:%lu LAST_SID:%u",
__get_str(name), __entry->trb, __entry->buffer,
TRB_LEN(__entry->length),
diff --git a/drivers/usb/core/devio.c b/drivers/usb/core/devio.c
index 533236366a03..40e96a56928e 100644
--- a/drivers/usb/core/devio.c
+++ b/drivers/usb/core/devio.c
@@ -251,6 +251,10 @@ static int usbdev_mmap(struct file *file, struct vm_area_struct *vma)
usbm->vma_use_count = 1;
INIT_LIST_HEAD(&usbm->memlist);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(dma_handle, size);
+#endif
+
if (hcd->localmem_pool || !hcd_uses_dma(hcd)) {
if (remap_pfn_range(vma, vma->vm_start,
virt_to_phys(usbm->mem) >> PAGE_SHIFT,
@@ -262,6 +266,9 @@ static int usbdev_mmap(struct file *file, struct vm_area_struct *vma)
if (dma_mmap_coherent(hcd->self.sysdev, vma, mem, dma_handle,
size)) {
dec_usb_memory_use_count(usbm, &usbm->vma_use_count);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(dma_handle, size);
+#endif
return -EAGAIN;
}
}
@@ -542,6 +549,9 @@ static int copy_urb_data_to_user(u8 __user *userbuffer, struct urb *urb)
if (urb->num_sgs == 0) {
if (copy_to_user(userbuffer, urb->transfer_buffer, len))
return -EFAULT;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(urb->transfer_buffer, len);
+#endif
return 0;
}
@@ -1729,6 +1739,12 @@ static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb
as->urb->transfer_buffer = as->usbm->mem +
(uurb_start - as->usbm->vm_start);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(as->usbm->dma_handle +
+ (uurb_start - as->usbm->vm_start),
+ as->usbm->size -
+ (uurb_start - as->usbm->vm_start));
+#endif
} else {
as->urb->transfer_buffer = kmalloc(uurb->buffer_length,
GFP_KERNEL);
@@ -1815,6 +1831,12 @@ static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb
as->urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
as->urb->transfer_dma = as->usbm->dma_handle +
(uurb_start - as->usbm->vm_start);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(as->usbm->dma_handle +
+ (uurb_start - as->usbm->vm_start),
+ as->usbm->size -
+ (uurb_start - as->usbm->vm_start));
+#endif
} else if (is_in && uurb->buffer_length > 0)
as->userbuffer = uurb->buffer;
as->signr = uurb->signr;
diff --git a/drivers/usb/core/hcd.c b/drivers/usb/core/hcd.c
index 2c6b9578a7d3..a6708e791158 100644
--- a/drivers/usb/core/hcd.c
+++ b/drivers/usb/core/hcd.c
@@ -419,6 +419,9 @@ ascii2desc(char const *s, u8 *buf, unsigned len)
*buf++ = t >> 8;
t = (unsigned char)*s++;
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(buf, len);
+#endif
return len;
}
@@ -450,6 +453,9 @@ rh_string(int id, struct usb_hcd const *hcd, u8 *data, unsigned len)
if (len > 4)
len = 4;
memcpy(data, langids, len);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(data, len);
+#endif
return len;
case 1:
/* Serial number */
@@ -502,6 +508,9 @@ static int rh_call_control (struct usb_hcd *hcd, struct urb *urb)
wValue = le16_to_cpu (cmd->wValue);
wIndex = le16_to_cpu (cmd->wIndex);
wLength = le16_to_cpu (cmd->wLength);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(cmd, sizeof(struct usb_ctrlrequest));
+#endif
if (wLength > urb->transfer_buffer_length)
goto error;
@@ -727,6 +736,9 @@ error:
bDeviceProtocol))
((struct usb_device_descriptor *) ubuf)->
bDeviceProtocol = USB_HUB_PR_HS_SINGLE_TT;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ubuf, len);
+#endif
}
kfree(tbuf);
@@ -774,6 +786,9 @@ void usb_hcd_poll_rh_status(struct usb_hcd *hcd)
urb->actual_length = length;
memcpy(urb->transfer_buffer, buffer, length);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(urb->transfer_buffer, length);
+#endif
usb_hcd_unlink_urb_from_ep(hcd, urb);
usb_hcd_giveback_urb(hcd, urb, 0);
} else {
@@ -1300,6 +1315,9 @@ static int hcd_alloc_coherent(struct usb_bus *bus,
memcpy(vaddr, *vaddr_handle, size);
*vaddr_handle = vaddr;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(*dma_handle, size + sizeof(vaddr));
+#endif
return 0;
}
@@ -1311,9 +1329,13 @@ static void hcd_free_coherent(struct usb_bus *bus, dma_addr_t *dma_handle,
vaddr = (void *)get_unaligned((unsigned long *)(vaddr + size));
- if (dir == DMA_FROM_DEVICE)
+ if (dir == DMA_FROM_DEVICE) {
memcpy(vaddr, *vaddr_handle, size);
-
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(vaddr, size);
+ cdns_virt_flush_dcache(*vaddr_handle, size);
+#endif
+ }
hcd_buffer_free(bus, size + sizeof(vaddr), *vaddr_handle, *dma_handle);
*vaddr_handle = vaddr;
@@ -1323,12 +1345,16 @@ static void hcd_free_coherent(struct usb_bus *bus, dma_addr_t *dma_handle,
void usb_hcd_unmap_urb_setup_for_dma(struct usb_hcd *hcd, struct urb *urb)
{
if (IS_ENABLED(CONFIG_HAS_DMA) &&
- (urb->transfer_flags & URB_SETUP_MAP_SINGLE))
+ (urb->transfer_flags & URB_SETUP_MAP_SINGLE)) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(urb->setup_dma,
+ sizeof(struct usb_ctrlrequest));
+#endif
dma_unmap_single(hcd->self.sysdev,
urb->setup_dma,
sizeof(struct usb_ctrlrequest),
DMA_TO_DEVICE);
- else if (urb->transfer_flags & URB_SETUP_MAP_LOCAL)
+ } else if (urb->transfer_flags & URB_SETUP_MAP_LOCAL)
hcd_free_coherent(urb->dev->bus,
&urb->setup_dma,
(void **) &urb->setup_packet,
@@ -1362,23 +1388,36 @@ void usb_hcd_unmap_urb_for_dma(struct usb_hcd *hcd, struct urb *urb)
urb->num_sgs,
dir);
else if (IS_ENABLED(CONFIG_HAS_DMA) &&
- (urb->transfer_flags & URB_DMA_MAP_PAGE))
+ (urb->transfer_flags & URB_DMA_MAP_PAGE)) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(urb->transfer_dma,
+ urb->transfer_buffer_length);
+#endif
dma_unmap_page(hcd->self.sysdev,
urb->transfer_dma,
urb->transfer_buffer_length,
dir);
- else if (IS_ENABLED(CONFIG_HAS_DMA) &&
- (urb->transfer_flags & URB_DMA_MAP_SINGLE))
+ } else if (IS_ENABLED(CONFIG_HAS_DMA) &&
+ (urb->transfer_flags & URB_DMA_MAP_SINGLE)) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(urb->transfer_dma,
+ urb->transfer_buffer_length);
+#endif
dma_unmap_single(hcd->self.sysdev,
urb->transfer_dma,
urb->transfer_buffer_length,
dir);
- else if (urb->transfer_flags & URB_MAP_LOCAL)
+ } else if (urb->transfer_flags & URB_MAP_LOCAL) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(urb->transfer_dma,
+ urb->transfer_buffer_length);
+#endif
hcd_free_coherent(urb->dev->bus,
&urb->transfer_dma,
&urb->transfer_buffer,
urb->transfer_buffer_length,
dir);
+ }
/* Make it safe to call this routine more than once */
urb->transfer_flags &= ~(URB_DMA_MAP_SG | URB_DMA_MAP_PAGE |
@@ -1417,6 +1456,10 @@ int usb_hcd_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
(void **)&urb->setup_packet,
sizeof(struct usb_ctrlrequest),
DMA_TO_DEVICE);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(urb->setup_dma,
+ sizeof(struct usb_ctrlrequest));
+#endif
if (ret)
return ret;
urb->transfer_flags |= URB_SETUP_MAP_LOCAL;
@@ -1434,6 +1477,10 @@ int usb_hcd_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
if (dma_mapping_error(hcd->self.sysdev,
urb->setup_dma))
return -EAGAIN;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(urb->setup_dma,
+ sizeof(struct usb_ctrlrequest));
+#endif
urb->transfer_flags |= URB_SETUP_MAP_SINGLE;
}
}
@@ -1448,6 +1495,10 @@ int usb_hcd_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
&urb->transfer_buffer,
urb->transfer_buffer_length,
dir);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(urb->transfer_dma,
+ urb->transfer_buffer_length + 8);
+#endif
if (ret == 0)
urb->transfer_flags |= URB_MAP_LOCAL;
} else if (hcd_uses_dma(hcd)) {
@@ -1486,6 +1537,10 @@ int usb_hcd_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
ret = -EAGAIN;
else
urb->transfer_flags |= URB_DMA_MAP_PAGE;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(urb->transfer_dma,
+ urb->transfer_buffer_length);
+#endif
} else if (object_is_on_stack(urb->transfer_buffer)) {
WARN_ONCE(1, "transfer buffer is on stack\n");
ret = -EAGAIN;
@@ -1500,6 +1555,10 @@ int usb_hcd_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
ret = -EAGAIN;
else
urb->transfer_flags |= URB_DMA_MAP_SINGLE;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(urb->transfer_dma,
+ urb->transfer_buffer_length);
+#endif
}
}
if (ret && (urb->transfer_flags & (URB_SETUP_MAP_SINGLE |
@@ -2932,6 +2991,9 @@ int usb_hcd_setup_local_mem(struct usb_hcd *hcd, phys_addr_t phys_addr,
if (IS_ERR(local_mem))
return PTR_ERR(local_mem);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(phys_addr,size);
+#endif
/*
* Here we pass a dma_addr_t but the arg type is a phys_addr_t.
* It's not backed by system memory and thus there's no kernel mapping
@@ -2945,6 +3007,9 @@ int usb_hcd_setup_local_mem(struct usb_hcd *hcd, phys_addr_t phys_addr,
return err;
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(dma,size);
+#endif
return 0;
}
EXPORT_SYMBOL_GPL(usb_hcd_setup_local_mem);
diff --git a/drivers/usb/core/urb.c b/drivers/usb/core/urb.c
index 357b149b20d3..3df361946a14 100644
--- a/drivers/usb/core/urb.c
+++ b/drivers/usb/core/urb.c
@@ -407,6 +407,10 @@ int usb_submit_urb(struct urb *urb, gfp_t mem_flags)
return -ENOEXEC;
is_out = !(setup->bRequestType & USB_DIR_IN) ||
!setup->wLength;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(setup,
+ sizeof(struct usb_ctrlrequest));
+#endif
} else {
is_out = usb_endpoint_dir_out(&ep->desc);
}
diff --git a/drivers/usb/core/usb.c b/drivers/usb/core/usb.c
index 9b4ac4415f1a..ab2759817dc7 100644
--- a/drivers/usb/core/usb.c
+++ b/drivers/usb/core/usb.c
@@ -892,9 +892,19 @@ EXPORT_SYMBOL_GPL(__usb_get_extra_descriptor);
void *usb_alloc_coherent(struct usb_device *dev, size_t size, gfp_t mem_flags,
dma_addr_t *dma)
{
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ void *ret;
+#endif
if (!dev || !dev->bus)
return NULL;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ ret = hcd_buffer_alloc(dev->bus, size, mem_flags, dma);
+ if(ret)
+ cdns_flush_dcache(*dma, size);
+ return ret;
+#else
return hcd_buffer_alloc(dev->bus, size, mem_flags, dma);
+#endif
}
EXPORT_SYMBOL_GPL(usb_alloc_coherent);
diff --git a/drivers/usb/host/xhci-dbg.c b/drivers/usb/host/xhci-dbg.c
index 386abf26641d..5a840db12aa8 100644
--- a/drivers/usb/host/xhci-dbg.c
+++ b/drivers/usb/host/xhci-dbg.c
@@ -16,6 +16,9 @@ char *xhci_get_slot_state(struct xhci_hcd *xhci,
struct xhci_slot_ctx *slot_ctx = xhci_get_slot_ctx(xhci, ctx);
int state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
return xhci_slot_state_string(state);
}
diff --git a/drivers/usb/host/xhci-dbgcap.c b/drivers/usb/host/xhci-dbgcap.c
index ccb0156fcebe..683758190869 100644
--- a/drivers/usb/host/xhci-dbgcap.c
+++ b/drivers/usb/host/xhci-dbgcap.c
@@ -105,6 +105,9 @@ static void xhci_dbc_init_contexts(struct xhci_dbc *dbc, u32 string_length)
info->product = cpu_to_le64(dma + DBC_MAX_STRING_LENGTH * 2);
info->serial = cpu_to_le64(dma + DBC_MAX_STRING_LENGTH * 3);
info->length = cpu_to_le32(string_length);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(dma, string_length);
+#endif
/* Populate bulk out endpoint context: */
ep_ctx = dbc_bulkout_ctx(dbc);
@@ -113,6 +116,9 @@ static void xhci_dbc_init_contexts(struct xhci_dbc *dbc, u32 string_length)
ep_ctx->ep_info = 0;
ep_ctx->ep_info2 = dbc_epctx_info2(BULK_OUT_EP, 1024, max_burst);
ep_ctx->deq = cpu_to_le64(deq | dbc->ring_out->cycle_state);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(deq, sizeof(union xhci_trb) * TRBS_PER_SEGMENT);
+#endif
/* Populate bulk in endpoint context: */
ep_ctx = dbc_bulkin_ctx(dbc);
@@ -120,6 +126,9 @@ static void xhci_dbc_init_contexts(struct xhci_dbc *dbc, u32 string_length)
ep_ctx->ep_info = 0;
ep_ctx->ep_info2 = dbc_epctx_info2(BULK_IN_EP, 1024, max_burst);
ep_ctx->deq = cpu_to_le64(deq | dbc->ring_in->cycle_state);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(deq, sizeof(union xhci_trb) * TRBS_PER_SEGMENT);
+#endif
/* Set DbC context and info registers: */
lo_hi_writeq(dbc->ctx->dma, &dbc->regs->dccp);
@@ -279,6 +288,11 @@ static int xhci_dbc_queue_bulk_tx(struct dbc_ep *dep,
* Add a barrier between writes of trb fields and flipping
* the cycle bit:
*/
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(req->dma, req->length);
+ cdns_flush_dcache(req->trb_dma,
+ sizeof(union xhci_trb) * TRBS_PER_SEGMENT);
+#endif
wmb();
if (cycle)
@@ -286,6 +300,10 @@ static int xhci_dbc_queue_bulk_tx(struct dbc_ep *dep,
else
trb->generic.field[3] &= cpu_to_le32(~TRB_CYCLE);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(req->trb_dma,
+ sizeof(union xhci_trb) * TRBS_PER_SEGMENT);
+#endif
writel(DBC_DOOR_BELL_TARGET(dep->direction), &dbc->regs->doorbell);
return 0;
@@ -501,12 +519,19 @@ static int xhci_dbc_mem_init(struct xhci_dbc *dbc, gfp_t flags)
if (!dbc->string)
goto string_fail;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(dbc->string_dma, dbc->string_size);
+#endif
+
/* Setup ERST register: */
writel(dbc->erst.erst_size, &dbc->regs->ersts);
lo_hi_writeq(dbc->erst.erst_dma_addr, &dbc->regs->erstba);
deq = xhci_trb_virt_to_dma(dbc->ring_evt->deq_seg,
dbc->ring_evt->dequeue);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(deq, sizeof(union xhci_trb) * TRBS_PER_SEGMENT);
+#endif
lo_hi_writeq(deq, &dbc->regs->erdp);
/* Setup strings and contexts: */
@@ -877,6 +902,9 @@ static enum evtreturn xhci_dbc_do_handle_events(struct xhci_dbc *dbc)
if (update_erdp) {
deq = xhci_trb_virt_to_dma(dbc->ring_evt->deq_seg,
dbc->ring_evt->dequeue);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(deq, sizeof(union xhci_trb));
+#endif
lo_hi_writeq(deq, &dbc->regs->erdp);
}
diff --git a/drivers/usb/host/xhci-debugfs.c b/drivers/usb/host/xhci-debugfs.c
index 2c0fda57869e..0a6a351c1b32 100644
--- a/drivers/usb/host/xhci-debugfs.c
+++ b/drivers/usb/host/xhci-debugfs.c
@@ -207,6 +207,9 @@ static void xhci_ring_dump_segment(struct seq_file *s,
le32_to_cpu(trb->generic.field[1]),
le32_to_cpu(trb->generic.field[2]),
le32_to_cpu(trb->generic.field[3])));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(dma,sizeof(*trb));
+#endif
}
}
@@ -268,6 +271,9 @@ static int xhci_slot_context_show(struct seq_file *s, void *unused)
le32_to_cpu(slot_ctx->dev_info2),
le32_to_cpu(slot_ctx->tt_info),
le32_to_cpu(slot_ctx->dev_state)));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
return 0;
}
@@ -291,6 +297,9 @@ static int xhci_endpoint_context_show(struct seq_file *s, void *unused)
le32_to_cpu(ep_ctx->ep_info2),
le64_to_cpu(ep_ctx->deq),
le32_to_cpu(ep_ctx->tx_info)));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
}
return 0;
@@ -551,6 +560,9 @@ static int xhci_stream_context_array_show(struct seq_file *s, void *unused)
else
seq_printf(s, "%pad stream context entry not used deq %016llx\n",
&dma, le64_to_cpu(stream_ctx->stream_ring));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(dma,16);
+#endif
}
return 0;
diff --git a/drivers/usb/host/xhci-hub.c b/drivers/usb/host/xhci-hub.c
index 74c497fd3476..23a941c8e3d8 100644
--- a/drivers/usb/host/xhci-hub.c
+++ b/drivers/usb/host/xhci-hub.c
@@ -413,8 +413,15 @@ static int xhci_stop_device(struct xhci_hcd *xhci, int slot_id, int suspend)
ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->out_ctx, i);
/* Check ep is running, required by AMD SNPS 3.1 xHC */
- if (GET_EP_CTX_STATE(ep_ctx) != EP_STATE_RUNNING)
+ if (GET_EP_CTX_STATE(ep_ctx) != EP_STATE_RUNNING) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
continue;
+ }
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
command = xhci_alloc_command(xhci, false, GFP_NOWAIT);
if (!command) {
diff --git a/drivers/usb/host/xhci-mem.c b/drivers/usb/host/xhci-mem.c
index 138ba4528dd3..2b63d1a7dc77 100644
--- a/drivers/usb/host/xhci-mem.c
+++ b/drivers/usb/host/xhci-mem.c
@@ -44,6 +44,9 @@ static struct xhci_segment *xhci_segment_alloc(struct xhci_hcd *xhci,
kfree(seg);
return NULL;
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(dma, sizeof(union xhci_trb) * TRBS_PER_SEGMENT);
+#endif
if (max_packet) {
seg->bounce_buf = kzalloc_node(max_packet, flags,
@@ -56,8 +59,13 @@ static struct xhci_segment *xhci_segment_alloc(struct xhci_hcd *xhci,
}
/* If the cycle state is 0, set the cycle bit to 1 for all the TRBs */
if (cycle_state == 0) {
- for (i = 0; i < TRBS_PER_SEGMENT; i++)
+ for (i = 0; i < TRBS_PER_SEGMENT; i++) {
seg->trbs[i].link.control |= cpu_to_le32(TRB_CYCLE);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(&seg->trbs[i],
+ sizeof(union xhci_trb));
+#endif
+ }
}
seg->dma = dma;
seg->next = NULL;
@@ -68,6 +76,9 @@ static struct xhci_segment *xhci_segment_alloc(struct xhci_hcd *xhci,
static void xhci_segment_free(struct xhci_hcd *xhci, struct xhci_segment *seg)
{
if (seg->trbs) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(seg->trbs, sizeof(union xhci_trb));
+#endif
dma_pool_free(xhci->segment_pool, seg->trbs, seg->dma);
seg->trbs = NULL;
}
@@ -111,11 +122,19 @@ static void xhci_link_segments(struct xhci_segment *prev,
/* Set the last TRB in the segment to have a TRB type ID of Link TRB */
val = le32_to_cpu(prev->trbs[TRBS_PER_SEGMENT-1].link.control);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(&prev->trbs[TRBS_PER_SEGMENT - 1],
+ sizeof(union xhci_trb));
+#endif
val &= ~TRB_TYPE_BITMASK;
val |= TRB_TYPE(TRB_LINK);
if (chain_links)
val |= TRB_CHAIN;
prev->trbs[TRBS_PER_SEGMENT-1].link.control = cpu_to_le32(val);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(&prev->trbs[TRBS_PER_SEGMENT - 1],
+ sizeof(union xhci_trb));
+#endif
}
}
@@ -149,7 +168,15 @@ static void xhci_link_rings(struct xhci_hcd *xhci, struct xhci_ring *ring,
&= ~cpu_to_le32(LINK_TOGGLE);
last->trbs[TRBS_PER_SEGMENT-1].link.control
|= cpu_to_le32(LINK_TOGGLE);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(&ring->last_seg->trbs[TRBS_PER_SEGMENT - 1],
+ sizeof(union xhci_trb));
+#endif
ring->last_seg = last;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(&last->trbs[TRBS_PER_SEGMENT - 1],
+ sizeof(union xhci_trb));
+#endif
}
}
@@ -265,6 +292,10 @@ static void xhci_remove_stream_mapping(struct xhci_ring *ring)
seg = ring->first_seg;
do {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(seg->dma,
+ sizeof(union xhci_trb) * TRBS_PER_SEGMENT);
+#endif
xhci_remove_segment_mapping(ring->trb_address_map, seg);
seg = seg->next;
} while (seg != ring->first_seg);
@@ -398,6 +429,10 @@ struct xhci_ring *xhci_ring_alloc(struct xhci_hcd *xhci,
/* See section 4.9.2.1 and 6.4.4.1 */
ring->last_seg->trbs[TRBS_PER_SEGMENT - 1].link.control |=
cpu_to_le32(LINK_TOGGLE);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(&ring->last_seg->trbs[TRBS_PER_SEGMENT - 1],
+ sizeof(union xhci_trb));
+#endif
}
xhci_initialize_ring_info(ring, cycle_state);
trace_xhci_ring_alloc(ring);
@@ -489,6 +524,9 @@ struct xhci_container_ctx *xhci_alloc_container_ctx(struct xhci_hcd *xhci,
kfree(ctx);
return NULL;
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(ctx->dma, ctx->size);
+#endif
return ctx;
}
@@ -662,6 +700,10 @@ struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci,
goto cleanup_ctx;
memset(stream_info->stream_ctx_array, 0,
sizeof(struct xhci_stream_ctx)*num_stream_ctxs);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(stream_info->ctx_array_dma,
+ sizeof(struct xhci_stream_ctx) * num_stream_ctxs);
+#endif
/* Allocate everything needed to free the stream rings later */
stream_info->free_streams_command =
@@ -691,6 +733,10 @@ struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci,
cur_ring->cycle_state;
stream_info->stream_ctx_array[cur_stream].stream_ring =
cpu_to_le64(addr);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(&stream_info->stream_ctx_array[cur_stream],
+ sizeof(struct xhci_stream_ctx));
+#endif
xhci_dbg(xhci, "Setting stream %d ring ptr to 0x%08llx\n",
cur_stream, (unsigned long long) addr);
@@ -748,6 +794,9 @@ void xhci_setup_streams_ep_input_ctx(struct xhci_hcd *xhci,
ep_ctx->ep_info |= cpu_to_le32(EP_MAXPSTREAMS(max_primary_streams)
| EP_HAS_LSA);
ep_ctx->deq = cpu_to_le64(stream_info->ctx_array_dma);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
}
/*
@@ -762,6 +811,9 @@ void xhci_setup_no_streams_ep_input_ctx(struct xhci_ep_ctx *ep_ctx,
ep_ctx->ep_info &= cpu_to_le32(~(EP_MAXPSTREAMS_MASK | EP_HAS_LSA));
addr = xhci_trb_virt_to_dma(ep->ring->deq_seg, ep->ring->dequeue);
ep_ctx->deq = cpu_to_le64(addr | ep->ring->cycle_state);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx,sizeof(*ep_ctx));
+#endif
}
/* Frees all stream contexts associated with the endpoint,
@@ -1024,12 +1076,19 @@ int xhci_alloc_virt_device(struct xhci_hcd *xhci, int slot_id,
dev->udev = udev;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(dev->out_ctx->dma, dev->out_ctx->size);
+#endif
/* Point to output device context in dcbaa. */
xhci->dcbaa->dev_context_ptrs[slot_id] = cpu_to_le64(dev->out_ctx->dma);
xhci_dbg(xhci, "Set slot id %d dcbaa entry %p to 0x%llx\n",
slot_id,
&xhci->dcbaa->dev_context_ptrs[slot_id],
le64_to_cpu(xhci->dcbaa->dev_context_ptrs[slot_id]));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(xhci->dcbaa->dma,
+ sizeof(struct xhci_device_context_array));
+#endif
trace_xhci_alloc_virt_device(dev);
@@ -1067,6 +1126,9 @@ void xhci_copy_ep0_dequeue_into_input_ctx(struct xhci_hcd *xhci,
ep0_ctx->deq = cpu_to_le64(xhci_trb_virt_to_dma(ep_ring->enq_seg,
ep_ring->enqueue)
| ep_ring->cycle_state);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep0_ctx, sizeof(*ep0_ctx));
+#endif
}
/*
@@ -1119,6 +1181,9 @@ int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *ud
/* 3) Only the control endpoint is valid - one endpoint context */
slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1) | udev->route);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(slot_ctx,sizeof(*slot_ctx));
+#endif
switch (udev->speed) {
case USB_SPEED_SUPER_PLUS:
slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_SSP);
@@ -1150,10 +1215,16 @@ int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *ud
return -EINVAL;
}
/* Find the root hub port this device is under */
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
port_num = xhci_find_real_port_number(xhci, udev);
if (!port_num)
return -EINVAL;
slot_ctx->dev_info2 |= cpu_to_le32(ROOT_HUB_PORT(port_num));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
/* Set the port number in the virtual_device to the faked port number */
for (top_dev = udev; top_dev->parent && top_dev->parent->parent;
top_dev = top_dev->parent)
@@ -1199,6 +1270,9 @@ int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *ud
(udev->ttport << 8));
if (udev->tt->multi)
slot_ctx->dev_info |= cpu_to_le32(DEV_MTT);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
}
xhci_dbg(xhci, "udev->tt = %p\n", udev->tt);
xhci_dbg(xhci, "udev->ttport = 0x%x\n", udev->ttport);
@@ -1213,6 +1287,9 @@ int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *ud
ep0_ctx->deq = cpu_to_le64(dev->eps[0].ring->first_seg->dma |
dev->eps[0].ring->cycle_state);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep0_ctx, sizeof(*ep0_ctx));
+#endif
trace_xhci_setup_addressable_virt_device(dev);
@@ -1522,6 +1599,9 @@ int xhci_endpoint_init(struct xhci_hcd *xhci,
ep_ctx->tx_info = cpu_to_le32(EP_MAX_ESIT_PAYLOAD_LO(max_esit_payload) |
EP_AVG_TRB_LENGTH(avg_trb_len));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
return 0;
}
@@ -1543,6 +1623,9 @@ void xhci_endpoint_zero(struct xhci_hcd *xhci,
/* Don't free the endpoint ring until the set interface or configuration
* request succeeds.
*/
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
}
void xhci_clear_endpoint_bw_info(struct xhci_bw_info *bw_info)
@@ -1574,14 +1657,29 @@ void xhci_update_bw_info(struct xhci_hcd *xhci,
* set in the first place.
*/
if (!EP_IS_ADDED(ctrl_ctx, i) && EP_IS_DROPPED(ctrl_ctx, i)) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx,
+ sizeof(struct xhci_input_control_ctx));
+#endif
/* Dropped endpoint */
xhci_clear_endpoint_bw_info(bw_info);
continue;
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx,
+ sizeof(struct xhci_input_control_ctx));
+#endif
if (EP_IS_ADDED(ctrl_ctx, i)) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx,
+ sizeof(struct xhci_input_control_ctx));
+#endif
ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, i);
ep_type = CTX_TO_EP_TYPE(le32_to_cpu(ep_ctx->ep_info2));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
/* Ignore non-periodic endpoints */
if (ep_type != ISOC_OUT_EP && ep_type != INT_OUT_EP &&
@@ -1605,7 +1703,14 @@ void xhci_update_bw_info(struct xhci_hcd *xhci,
bw_info->type = ep_type;
bw_info->max_esit_payload = CTX_TO_MAX_ESIT_PAYLOAD(
le32_to_cpu(ep_ctx->tx_info));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ else
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(struct xhci_input_control_ctx));
+#endif
}
}
@@ -1632,6 +1737,10 @@ void xhci_endpoint_copy(struct xhci_hcd *xhci,
in_ep_ctx->reserved[0] = out_ep_ctx->reserved[0];
in_ep_ctx->reserved[1] = out_ep_ctx->reserved[1];
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(in_ep_ctx, sizeof(*in_ep_ctx));
+ cdns_virt_flush_dcache(out_ep_ctx, sizeof(*out_ep_ctx));
+#endif
}
/* Copy output xhci_slot_ctx to the input xhci_slot_ctx.
@@ -1653,6 +1762,10 @@ void xhci_slot_copy(struct xhci_hcd *xhci,
in_slot_ctx->dev_info2 = out_slot_ctx->dev_info2;
in_slot_ctx->tt_info = out_slot_ctx->tt_info;
in_slot_ctx->dev_state = out_slot_ctx->dev_state;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(in_slot_ctx, sizeof(*in_slot_ctx));
+ cdns_virt_flush_dcache(out_slot_ctx, sizeof(*out_slot_ctx));
+#endif
}
/* Set up the scratchpad buffer array and scratchpad buffers, if needed. */
@@ -1678,6 +1791,9 @@ static int scratchpad_alloc(struct xhci_hcd *xhci, gfp_t flags)
&xhci->scratchpad->sp_dma, flags);
if (!xhci->scratchpad->sp_array)
goto fail_sp2;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(xhci->scratchpad->sp_dma, num_sp * sizeof(u64));
+#endif
xhci->scratchpad->sp_buffers = kcalloc_node(num_sp, sizeof(void *),
flags, dev_to_node(dev));
@@ -1694,7 +1810,13 @@ static int scratchpad_alloc(struct xhci_hcd *xhci, gfp_t flags)
xhci->scratchpad->sp_array[i] = dma;
xhci->scratchpad->sp_buffers[i] = buf;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(dma, xhci->page_size);
+#endif
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(xhci->scratchpad->sp_dma, num_sp * sizeof(u64));
+#endif
return 0;
@@ -1818,6 +1940,9 @@ int xhci_alloc_erst(struct xhci_hcd *xhci,
size, &erst->erst_dma_addr, flags);
if (!erst->entries)
return -ENOMEM;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(erst->erst_dma_addr, size);
+#endif
erst->num_entries = evt_ring->num_segs;
@@ -1829,6 +1954,9 @@ int xhci_alloc_erst(struct xhci_hcd *xhci,
entry->rsvd = 0;
seg = seg->next;
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(erst->erst_dma_addr, size);
+#endif
return 0;
}
@@ -2123,6 +2251,9 @@ static void xhci_set_hc_event_deq(struct xhci_hcd *xhci)
xhci_dbg_trace(xhci, trace_xhci_dbg_init,
"// Write event ring dequeue pointer, "
"preserving EHB bit");
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(deq, sizeof(union xhci_trb));
+#endif
xhci_write_64(xhci, ((u64) deq & (u64) ~ERST_PTR_MASK) | temp,
&xhci->ir_set->erst_dequeue);
}
@@ -2434,6 +2565,9 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags)
if (!xhci->dcbaa)
goto fail;
xhci->dcbaa->dma = dma;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(xhci->dcbaa->dma, sizeof(*xhci->dcbaa));
+#endif
xhci_dbg_trace(xhci, trace_xhci_dbg_init,
"// Device context base array address = 0x%llx (DMA), %p (virt)",
(unsigned long long)xhci->dcbaa->dma, xhci->dcbaa);
@@ -2542,6 +2676,11 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags)
val_64 = xhci_read_64(xhci, &xhci->ir_set->erst_base);
val_64 &= ERST_PTR_MASK;
val_64 |= (xhci->erst.erst_dma_addr & (u64) ~ERST_PTR_MASK);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(xhci->erst.erst_dma_addr,
+ xhci->event_ring->num_segs *
+ sizeof(struct xhci_erst_entry));
+#endif
xhci_write_64(xhci, val_64, &xhci->ir_set->erst_base);
/* Set the event ring dequeue address */
diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c
index 167dae117f73..32c0e703e01e 100644
--- a/drivers/usb/host/xhci-ring.c
+++ b/drivers/usb/host/xhci-ring.c
@@ -79,12 +79,26 @@ dma_addr_t xhci_trb_virt_to_dma(struct xhci_segment *seg,
static bool trb_is_noop(union xhci_trb *trb)
{
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ bool ret;
+ ret = TRB_TYPE_NOOP_LE32(trb->generic.field[3]);
+ cdns_virt_flush_dcache(trb, sizeof(union xhci_trb));
+ return ret;
+#else
return TRB_TYPE_NOOP_LE32(trb->generic.field[3]);
+#endif
}
static bool trb_is_link(union xhci_trb *trb)
{
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ bool ret;
+ ret = TRB_TYPE_LINK_LE32(trb->link.control);
+ cdns_virt_flush_dcache(trb, sizeof(union xhci_trb));
+ return ret;
+#else
return TRB_TYPE_LINK_LE32(trb->link.control);
+#endif
}
static bool last_trb_on_seg(struct xhci_segment *seg, union xhci_trb *trb)
@@ -100,7 +114,14 @@ static bool last_trb_on_ring(struct xhci_ring *ring,
static bool link_trb_toggles_cycle(union xhci_trb *trb)
{
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ bool ret;
+ ret = le32_to_cpu(trb->link.control) & LINK_TOGGLE;
+ cdns_virt_flush_dcache(trb, sizeof(union xhci_trb));
+ return ret;
+#else
return le32_to_cpu(trb->link.control) & LINK_TOGGLE;
+#endif
}
static bool last_td_in_urb(struct xhci_td *td)
@@ -130,6 +151,9 @@ static void trb_to_noop(union xhci_trb *trb, u32 noop_type)
trb->generic.field[3] &= cpu_to_le32(TRB_CYCLE);
trb->generic.field[3] |= cpu_to_le32(TRB_TYPE(noop_type));
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(trb, sizeof(union xhci_trb));
+#endif
}
/* Updates trb to point to the next TRB in the ring, and updates seg if the next
@@ -211,6 +235,9 @@ static void inc_enq(struct xhci_hcd *xhci, struct xhci_ring *ring,
/* If this is not event ring, there is one less usable TRB */
if (!trb_is_link(ring->enqueue))
ring->num_trbs_free--;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ring->enqueue, sizeof(union xhci_trb));
+#endif
next = ++(ring->enqueue);
/* Update the dequeue pointer further if that was a link TRB */
@@ -236,6 +263,9 @@ static void inc_enq(struct xhci_hcd *xhci, struct xhci_ring *ring,
next->link.control &= cpu_to_le32(~TRB_CHAIN);
next->link.control |= cpu_to_le32(chain);
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(next,sizeof(union xhci_trb));
+#endif
/* Give this link TRB to the hardware */
wmb();
next->link.control ^= cpu_to_le32(TRB_CYCLE);
@@ -243,6 +273,9 @@ static void inc_enq(struct xhci_hcd *xhci, struct xhci_ring *ring,
/* Toggle the cycle bit after the last ring segment. */
if (link_trb_toggles_cycle(next))
ring->cycle_state ^= 1;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(next,sizeof(union xhci_trb));
+#endif
ring->enq_seg = ring->enq_seg->next;
ring->enqueue = ring->enq_seg->trbs;
@@ -495,15 +528,30 @@ static u64 xhci_get_hw_deq(struct xhci_hcd *xhci, struct xhci_virt_device *vdev,
struct xhci_ep_ctx *ep_ctx;
struct xhci_stream_ctx *st_ctx;
struct xhci_virt_ep *ep;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ u64 ret;
+#endif
ep = &vdev->eps[ep_index];
if (ep->ep_state & EP_HAS_STREAMS) {
st_ctx = &ep->stream_info->stream_ctx_array[stream_id];
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ ret = le64_to_cpu(st_ctx->stream_ring);
+ cdns_virt_flush_dcache(st_ctx, sizeof(*st_ctx));
+ return ret;
+#else
return le64_to_cpu(st_ctx->stream_ring);
+#endif
}
ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ ret = le64_to_cpu(ep_ctx->deq);
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+ return ret;
+#else
return le64_to_cpu(ep_ctx->deq);
+#endif
}
/*
@@ -637,8 +685,12 @@ static void td_to_noop(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
trb_to_noop(trb, TRB_TR_NOOP);
/* flip cycle if asked to */
- if (flip_cycle && trb != td->first_trb && trb != td->last_trb)
+ if (flip_cycle && trb != td->first_trb && trb != td->last_trb) {
trb->generic.field[3] ^= cpu_to_le32(TRB_CYCLE);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(trb, sizeof(union xhci_trb));
+#endif
+ }
if (trb == td->last_trb)
break;
@@ -691,16 +743,25 @@ static void xhci_unmap_td_bounce_buffer(struct xhci_hcd *xhci,
return;
if (usb_urb_dir_out(urb)) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(seg->bounce_dma, ring->bounce_buf_len);
+#endif
dma_unmap_single(dev, seg->bounce_dma, ring->bounce_buf_len,
DMA_TO_DEVICE);
return;
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(seg->bounce_dma, ring->bounce_buf_len);
+#endif
dma_unmap_single(dev, seg->bounce_dma, ring->bounce_buf_len,
DMA_FROM_DEVICE);
/* for in tranfers we need to copy the data from bounce to sg */
len = sg_pcopy_from_buffer(urb->sg, urb->num_sgs, seg->bounce_buf,
seg->bounce_len, seg->bounce_offs);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(seg->bounce_dma, ring->bounce_buf_len);
+#endif
if (len != seg->bounce_len)
xhci_warn(xhci, "WARN Wrong bounce buffer read length: %zu != %d\n",
len, seg->bounce_len);
@@ -732,6 +793,9 @@ static void xhci_handle_cmd_stop_ep(struct xhci_hcd *xhci, int slot_id,
struct xhci_dequeue_state deq_state;
if (unlikely(TRB_TO_SUSPEND_PORT(le32_to_cpu(trb->generic.field[3])))) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(trb, sizeof(union xhci_trb));
+#endif
if (!xhci->devs[slot_id])
xhci_warn(xhci, "Stop endpoint command "
"completion for disabled slot %u\n",
@@ -741,10 +805,16 @@ static void xhci_handle_cmd_stop_ep(struct xhci_hcd *xhci, int slot_id,
memset(&deq_state, 0, sizeof(deq_state));
ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(trb, sizeof(union xhci_trb));
+#endif
vdev = xhci->devs[slot_id];
ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index);
trace_xhci_handle_cmd_stop_ep(ep_ctx);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
ep = &xhci->devs[slot_id]->eps[ep_index];
last_unlinked_td = list_last_entry(&ep->cancelled_td_list,
@@ -1071,6 +1141,9 @@ static void xhci_handle_cmd_set_deq(struct xhci_hcd *xhci, int slot_id,
ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
stream_id = TRB_TO_STREAM_ID(le32_to_cpu(trb->generic.field[2]));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(trb, sizeof(union xhci_trb));
+#endif
dev = xhci->devs[slot_id];
ep = &dev->eps[ep_index];
@@ -1086,6 +1159,10 @@ static void xhci_handle_cmd_set_deq(struct xhci_hcd *xhci, int slot_id,
slot_ctx = xhci_get_slot_ctx(xhci, dev->out_ctx);
trace_xhci_handle_cmd_set_deq(slot_ctx);
trace_xhci_handle_cmd_set_deq_ep(ep_ctx);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
if (cmd_comp_code != COMP_SUCCESS) {
unsigned int ep_state;
@@ -1103,6 +1180,10 @@ static void xhci_handle_cmd_set_deq(struct xhci_hcd *xhci, int slot_id,
xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
"Slot state = %u, EP state = %u",
slot_state, ep_state);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
break;
case COMP_SLOT_NOT_ENABLED_ERROR:
xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed because slot %u was not enabled.\n",
@@ -1126,8 +1207,14 @@ static void xhci_handle_cmd_set_deq(struct xhci_hcd *xhci, int slot_id,
struct xhci_stream_ctx *ctx =
&ep->stream_info->stream_ctx_array[stream_id];
deq = le64_to_cpu(ctx->stream_ring) & SCTX_DEQ_MASK;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctx, sizeof(*ctx));
+#endif
} else {
deq = le64_to_cpu(ep_ctx->deq) & ~EP_CTX_CYCLE_MASK;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
}
xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
"Successful Set TR Deq Ptr cmd, deq = @%08llx", deq);
@@ -1161,9 +1248,15 @@ static void xhci_handle_cmd_reset_ep(struct xhci_hcd *xhci, int slot_id,
unsigned int ep_index;
ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(trb, sizeof(union xhci_trb));
+#endif
vdev = xhci->devs[slot_id];
ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index);
trace_xhci_handle_cmd_reset_ep(ep_ctx);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
/* This command will only fail if the endpoint wasn't halted,
* but we don't care.
@@ -1194,8 +1287,16 @@ static void xhci_handle_cmd_reset_ep(struct xhci_hcd *xhci, int slot_id,
}
/* if this was a soft reset, then restart */
- if ((le32_to_cpu(trb->generic.field[3])) & TRB_TSP)
+ if ((le32_to_cpu(trb->generic.field[3])) & TRB_TSP) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(trb, sizeof(union xhci_trb));
+#endif
ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
+ }
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ else
+ cdns_virt_flush_dcache(trb, sizeof(union xhci_trb));
+#endif
}
static void xhci_handle_cmd_enable_slot(struct xhci_hcd *xhci, int slot_id,
@@ -1218,6 +1319,9 @@ static void xhci_handle_cmd_disable_slot(struct xhci_hcd *xhci, int slot_id)
slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
trace_xhci_handle_cmd_disable_slot(slot_ctx);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
if (xhci->quirks & XHCI_EP_LIMIT_QUIRK)
/* Delete default control endpoint resources */
@@ -1252,11 +1356,17 @@ static void xhci_handle_cmd_config_ep(struct xhci_hcd *xhci, int slot_id,
add_flags = le32_to_cpu(ctrl_ctx->add_flags);
drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+#endif
/* Input ctx add_flags are the endpoint index plus one */
ep_index = xhci_last_valid_endpoint(add_flags) - 1;
ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->out_ctx, ep_index);
trace_xhci_handle_cmd_config_ep(ep_ctx);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
/* A usb_set_interface() call directly after clearing a halted
* condition may race on this quirky hardware. Not worth
@@ -1290,6 +1400,9 @@ static void xhci_handle_cmd_addr_dev(struct xhci_hcd *xhci, int slot_id)
vdev = xhci->devs[slot_id];
slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx);
trace_xhci_handle_cmd_addr_dev(slot_ctx);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
}
static void xhci_handle_cmd_reset_dev(struct xhci_hcd *xhci, int slot_id,
@@ -1301,6 +1414,9 @@ static void xhci_handle_cmd_reset_dev(struct xhci_hcd *xhci, int slot_id,
vdev = xhci->devs[slot_id];
slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx);
trace_xhci_handle_cmd_reset_dev(slot_ctx);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
xhci_dbg(xhci, "Completed reset device command.\n");
if (!xhci->devs[slot_id])
@@ -1319,6 +1435,9 @@ static void xhci_handle_cmd_nec_get_fw(struct xhci_hcd *xhci,
"NEC firmware version %2x.%02x",
NEC_FW_MAJOR(le32_to_cpu(event->status)),
NEC_FW_MINOR(le32_to_cpu(event->status)));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(event, sizeof(struct xhci_event_cmd));
+#endif
}
static void xhci_complete_del_and_free_cmd(struct xhci_command *cmd, u32 status)
@@ -1407,6 +1526,9 @@ static void handle_cmd_completion(struct xhci_hcd *xhci,
u32 cmd_type;
cmd_dma = le64_to_cpu(event->cmd_trb);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(event, sizeof(struct xhci_event_cmd));
+#endif
cmd_trb = xhci->cmd_ring->dequeue;
trace_xhci_handle_command(xhci->cmd_ring, &cmd_trb->generic);
@@ -1428,6 +1550,9 @@ static void handle_cmd_completion(struct xhci_hcd *xhci,
cancel_delayed_work(&xhci->cmd_timer);
cmd_comp_code = GET_COMP_CODE(le32_to_cpu(event->status));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(event, sizeof(struct xhci_event_cmd));
+#endif
/* If CMD ring stopped we own the trbs between enqueue and dequeue */
if (cmd_comp_code == COMP_COMMAND_RING_STOPPED) {
@@ -1457,6 +1582,9 @@ static void handle_cmd_completion(struct xhci_hcd *xhci,
}
cmd_type = TRB_FIELD_TO_TYPE(le32_to_cpu(cmd_trb->generic.field[3]));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(cmd_trb, sizeof(union xhci_trb));
+#endif
switch (cmd_type) {
case TRB_ENABLE_SLOT:
xhci_handle_cmd_enable_slot(xhci, slot_id, cmd, cmd_comp_code);
@@ -1477,12 +1605,18 @@ static void handle_cmd_completion(struct xhci_hcd *xhci,
case TRB_STOP_RING:
WARN_ON(slot_id != TRB_TO_SLOT_ID(
le32_to_cpu(cmd_trb->generic.field[3])));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(cmd_trb, sizeof(union xhci_trb));
+#endif
if (!cmd->completion)
xhci_handle_cmd_stop_ep(xhci, slot_id, cmd_trb, event);
break;
case TRB_SET_DEQ:
WARN_ON(slot_id != TRB_TO_SLOT_ID(
le32_to_cpu(cmd_trb->generic.field[3])));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(cmd_trb, sizeof(union xhci_trb));
+#endif
xhci_handle_cmd_set_deq(xhci, slot_id, cmd_trb, cmd_comp_code);
break;
case TRB_CMD_NOOP:
@@ -1493,6 +1627,9 @@ static void handle_cmd_completion(struct xhci_hcd *xhci,
case TRB_RESET_EP:
WARN_ON(slot_id != TRB_TO_SLOT_ID(
le32_to_cpu(cmd_trb->generic.field[3])));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(cmd_trb, sizeof(union xhci_trb));
+#endif
xhci_handle_cmd_reset_ep(xhci, slot_id, cmd_trb, cmd_comp_code);
break;
case TRB_RESET_DEV:
@@ -1501,6 +1638,9 @@ static void handle_cmd_completion(struct xhci_hcd *xhci,
*/
slot_id = TRB_TO_SLOT_ID(
le32_to_cpu(cmd_trb->generic.field[3]));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(cmd_trb, sizeof(union xhci_trb));
+#endif
xhci_handle_cmd_reset_dev(xhci, slot_id, event);
break;
case TRB_NEC_GET_FW:
@@ -1533,6 +1673,9 @@ static void handle_vendor_event(struct xhci_hcd *xhci,
u32 trb_type;
trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(event->generic.field[3]));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(event, sizeof(union xhci_trb));
+#endif
xhci_dbg(xhci, "Vendor specific event TRB type = %u\n", trb_type);
if (trb_type == TRB_NEC_CMD_COMP && (xhci->quirks & XHCI_NEC_HOST))
handle_cmd_completion(xhci, &event->event_cmd);
@@ -1545,6 +1688,9 @@ static void handle_device_notification(struct xhci_hcd *xhci,
struct usb_device *udev;
slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->generic.field[3]));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(event, sizeof(union xhci_trb));
+#endif
if (!xhci->devs[slot_id]) {
xhci_warn(xhci, "Device Notification event for "
"unused slot %u\n", slot_id);
@@ -1607,6 +1753,9 @@ static void handle_port_status(struct xhci_hcd *xhci,
port_id = GET_PORT_ID(le32_to_cpu(event->generic.field[0]));
max_ports = HCS_MAX_PORTS(xhci->hcs_params1);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(event, sizeof(union xhci_trb));
+#endif
if ((port_id <= 0) || (port_id > max_ports)) {
xhci_warn(xhci, "Port change event with invalid port ID %d\n",
@@ -1893,15 +2042,24 @@ static int xhci_requires_manual_halt_cleanup(struct xhci_hcd *xhci,
/* TRB completion codes that may require a manual halt cleanup */
if (trb_comp_code == COMP_USB_TRANSACTION_ERROR ||
trb_comp_code == COMP_BABBLE_DETECTED_ERROR ||
- trb_comp_code == COMP_SPLIT_TRANSACTION_ERROR)
+ trb_comp_code == COMP_SPLIT_TRANSACTION_ERROR) {
/* The 0.95 spec says a babbling control endpoint
* is not halted. The 0.96 spec says it is. Some HW
* claims to be 0.95 compliant, but it halts the control
* endpoint anyway. Check if a babble halted the
* endpoint.
*/
- if (GET_EP_CTX_STATE(ep_ctx) == EP_STATE_HALTED)
+ if (GET_EP_CTX_STATE(ep_ctx) == EP_STATE_HALTED) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
return 1;
+ }
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ else
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
+ }
return 0;
}
@@ -1983,6 +2141,9 @@ static int finish_td(struct xhci_hcd *xhci, struct xhci_td *td,
ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(event, sizeof(union xhci_trb));
+#endif
if (trb_comp_code == COMP_STOPPED_LENGTH_INVALID ||
trb_comp_code == COMP_STOPPED ||
@@ -1996,6 +2157,9 @@ static int finish_td(struct xhci_hcd *xhci, struct xhci_td *td,
if (trb_comp_code == COMP_STALL_ERROR ||
xhci_requires_manual_halt_cleanup(xhci, ep_ctx,
trb_comp_code)) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
/*
* xhci internal endpoint state will go to a "halt" state for
* any stall, including default control pipe protocol stall.
@@ -2011,6 +2175,9 @@ static int finish_td(struct xhci_hcd *xhci, struct xhci_td *td,
xhci_cleanup_halted_endpoint(xhci, slot_id, ep_index,
ep_ring->stream_id, td, EP_HARD_RESET);
} else {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
/* Update ring dequeue pointer */
while (ep_ring->dequeue != td->last_trb)
inc_deq(xhci, ep_ring);
@@ -2031,6 +2198,9 @@ static int sum_trb_lengths(struct xhci_hcd *xhci, struct xhci_ring *ring,
for (sum = 0; trb != stop_trb; next_trb(xhci, ring, &seg, &trb)) {
if (!trb_is_noop(trb) && !trb_is_link(trb))
sum += TRB_LEN(le32_to_cpu(trb->generic.field[2]));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(trb, sizeof(union xhci_trb));
+#endif
}
return sum;
}
@@ -2051,6 +2221,9 @@ static int process_ctrl_td(struct xhci_hcd *xhci, struct xhci_td *td,
u32 trb_type;
trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(ep_trb->generic.field[3]));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_trb, sizeof(union xhci_trb));
+#endif
slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
xdev = xhci->devs[slot_id];
ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
@@ -2058,6 +2231,9 @@ static int process_ctrl_td(struct xhci_hcd *xhci, struct xhci_td *td,
trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
requested = td->urb->transfer_buffer_length;
remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(event, sizeof(union xhci_trb));
+#endif
switch (trb_comp_code) {
case COMP_SUCCESS:
@@ -2161,6 +2337,10 @@ static int process_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td,
requested = frame->length;
remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2]));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(event, sizeof(union xhci_trb));
+ cdns_virt_flush_dcache(ep_trb, sizeof(union xhci_trb));
+#endif
short_framestatus = td->urb->transfer_flags & URB_SHORT_NOT_OK ?
-EREMOTEIO : 0;
@@ -2234,6 +2414,9 @@ static int skip_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td,
int idx;
ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(event, sizeof(union xhci_trb));
+#endif
urb_priv = td->urb->hcpriv;
idx = urb_priv->num_tds_done;
frame = &td->urb->iso_frame_desc[idx];
@@ -2273,6 +2456,10 @@ static int process_bulk_intr_td(struct xhci_hcd *xhci, struct xhci_td *td,
trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2]));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(event, sizeof(union xhci_trb));
+ cdns_virt_flush_dcache(ep_trb, sizeof(union xhci_trb));
+#endif
requested = td->urb->transfer_buffer_length;
switch (trb_comp_code) {
@@ -2303,8 +2490,15 @@ static int process_bulk_intr_td(struct xhci_hcd *xhci, struct xhci_td *td,
break;
case COMP_USB_TRANSACTION_ERROR:
if ((ep_ring->err_count++ > MAX_SOFT_RETRY) ||
- le32_to_cpu(slot_ctx->tt_info) & TT_SLOT)
+ le32_to_cpu(slot_ctx->tt_info) & TT_SLOT) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
break;
+ }
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
*status = 0;
xhci_cleanup_halted_endpoint(xhci, slot_id, ep_index,
ep_ring->stream_id, td, EP_SOFT_RESET);
@@ -2357,6 +2551,9 @@ static int handle_tx_event(struct xhci_hcd *xhci,
ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
ep_trb_dma = le64_to_cpu(event->buffer);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(event, sizeof(union xhci_trb));
+#endif
xdev = xhci->devs[slot_id];
if (!xdev) {
@@ -2373,8 +2570,14 @@ static int handle_tx_event(struct xhci_hcd *xhci,
xhci_err(xhci,
"ERROR Transfer event for disabled endpoint slot %u ep %u\n",
slot_id, ep_index);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
goto err_out;
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
/* Some transfer events don't always point to a trb, see xhci 4.17.4 */
if (!ep_ring) {
@@ -2409,8 +2612,16 @@ static int handle_tx_event(struct xhci_hcd *xhci,
* transfer type
*/
case COMP_SUCCESS:
- if (EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)) == 0)
+ if (EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)) == 0) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(event, sizeof(union xhci_trb));
+#endif
break;
+ }
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ else
+ cdns_virt_flush_dcache(event, sizeof(union xhci_trb));
+#endif
if (xhci->quirks & XHCI_TRUST_TX_LENGTH ||
ep_ring->last_td_was_short)
trb_comp_code = COMP_SHORT_PACKET;
@@ -2488,19 +2699,27 @@ static int handle_tx_event(struct xhci_hcd *xhci,
* Underrun Event for OUT Isoch endpoint.
*/
xhci_dbg(xhci, "underrun event on endpoint\n");
- if (!list_empty(&ep_ring->td_list))
+ if (!list_empty(&ep_ring->td_list)) {
xhci_dbg(xhci, "Underrun Event for slot %d ep %d "
"still with TDs queued?\n",
TRB_TO_SLOT_ID(le32_to_cpu(event->flags)),
ep_index);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(event, sizeof(union xhci_trb));
+#endif
+ }
goto cleanup;
case COMP_RING_OVERRUN:
xhci_dbg(xhci, "overrun event on endpoint\n");
- if (!list_empty(&ep_ring->td_list))
+ if (!list_empty(&ep_ring->td_list)) {
xhci_dbg(xhci, "Overrun Event for slot %d ep %d "
"still with TDs queued?\n",
TRB_TO_SLOT_ID(le32_to_cpu(event->flags)),
ep_index);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(event, sizeof(union xhci_trb));
+#endif
+ }
goto cleanup;
case COMP_MISSED_SERVICE_ERROR:
/*
@@ -2558,6 +2777,9 @@ static int handle_tx_event(struct xhci_hcd *xhci,
xhci_warn(xhci, "WARN Event TRB for slot %d ep %d with no TDs queued?\n",
TRB_TO_SLOT_ID(le32_to_cpu(event->flags)),
ep_index);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(event, sizeof(union xhci_trb));
+#endif
}
if (ep->skip) {
ep->skip = false;
@@ -2650,6 +2872,9 @@ static int handle_tx_event(struct xhci_hcd *xhci,
trace_xhci_handle_transfer(ep_ring,
(struct xhci_generic_trb *) ep_trb);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_trb, sizeof(union xhci_trb));
+#endif
/*
* No-op TRB could trigger interrupts in a case where
@@ -2732,9 +2957,16 @@ static int xhci_handle_event(struct xhci_hcd *xhci)
event = xhci->event_ring->dequeue;
/* Does the HC or OS own the TRB? */
if ((le32_to_cpu(event->event_cmd.flags) & TRB_CYCLE) !=
- xhci->event_ring->cycle_state)
+ xhci->event_ring->cycle_state) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(event, sizeof(union xhci_trb));
+#endif
return 0;
-
+ }
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ else
+ cdns_virt_flush_dcache(event, sizeof(union xhci_trb));
+#endif
trace_xhci_handle_event(xhci->event_ring, &event->generic);
/*
@@ -2761,12 +2993,19 @@ static int xhci_handle_event(struct xhci_hcd *xhci)
break;
default:
if ((le32_to_cpu(event->event_cmd.flags) & TRB_TYPE_BITMASK) >=
- TRB_TYPE(48))
+ TRB_TYPE(48)){
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(event, sizeof(union xhci_trb));
+#endif
handle_vendor_event(xhci, event);
- else
+ }else{
xhci_warn(xhci, "ERROR unknown event type %d\n",
TRB_FIELD_TO_TYPE(
le32_to_cpu(event->event_cmd.flags)));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(event, sizeof(union xhci_trb));
+#endif
+ }
}
/* Any of the above functions may drop and re-acquire the lock, so check
* to make sure a watchdog timer didn't mark the host as non-responsive.
@@ -2816,6 +3055,9 @@ static void xhci_update_erst_dequeue(struct xhci_hcd *xhci,
/* Update HC event ring dequeue pointer */
temp_64 &= ERST_PTR_MASK;
temp_64 |= ((u64) deq & (u64) ~ERST_PTR_MASK);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(deq, sizeof(union xhci_trb));
+#endif
}
/* Clear the event handler busy flag (RW1C) */
@@ -2931,8 +3173,14 @@ static void queue_trb(struct xhci_hcd *xhci, struct xhci_ring *ring,
trb->field[1] = cpu_to_le32(field2);
trb->field[2] = cpu_to_le32(field3);
trb->field[3] = cpu_to_le32(field4);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(trb, sizeof(union xhci_trb));
+#endif
trace_xhci_queue_trb(ring, trb);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(trb, sizeof(union xhci_trb));
+#endif
inc_enq(xhci, ring, more_trbs_coming);
}
@@ -3005,10 +3253,16 @@ static int prepare_ring(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
else
ep_ring->enqueue->link.control |=
cpu_to_le32(TRB_CHAIN);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(&ep_ring->enqueue->link, sizeof(union xhci_trb));
+#endif
wmb();
ep_ring->enqueue->link.control ^= cpu_to_le32(TRB_CYCLE);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(&ep_ring->enqueue->link, sizeof(union xhci_trb));
+#endif
/* Toggle the cycle bit after the last ring segment. */
if (link_trb_toggles_cycle(ep_ring->enqueue))
ep_ring->cycle_state ^= 1;
@@ -3033,6 +3287,9 @@ static int prepare_transfer(struct xhci_hcd *xhci,
struct xhci_td *td;
struct xhci_ring *ep_ring;
struct xhci_ep_ctx *ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ u32 ep_state;
+#endif
ep_ring = xhci_stream_id_to_ring(xdev, ep_index, stream_id);
if (!ep_ring) {
@@ -3041,7 +3298,13 @@ static int prepare_transfer(struct xhci_hcd *xhci,
return -EINVAL;
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ ep_state = GET_EP_CTX_STATE(ep_ctx);
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+ ret = prepare_ring(xhci, ep_ring, ep_state,
+#else
ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx),
+#endif
num_trbs, mem_flags);
if (ret)
return ret;
@@ -3138,6 +3401,9 @@ static void giveback_first_trb(struct xhci_hcd *xhci, int slot_id,
start_trb->field[3] |= cpu_to_le32(start_cycle);
else
start_trb->field[3] &= cpu_to_le32(~TRB_CYCLE);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(start_trb, sizeof(union xhci_trb));
+#endif
xhci_ring_ep_doorbell(xhci, slot_id, ep_index, stream_id);
}
@@ -3148,6 +3414,9 @@ static void check_interval(struct xhci_hcd *xhci, struct urb *urb,
int ep_interval;
xhci_interval = EP_INTERVAL_TO_UFRAMES(le32_to_cpu(ep_ctx->ep_info));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
ep_interval = urb->interval;
/* Convert to microframes */
@@ -3281,6 +3550,9 @@ static int xhci_align_td(struct xhci_hcd *xhci, struct urb *urb, u32 enqd_len,
len, new_buff_len);
seg->bounce_dma = dma_map_single(dev, seg->bounce_buf,
max_pkt, DMA_TO_DEVICE);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(seg->bounce_buf, new_buff_len);
+#endif
} else {
seg->bounce_dma = dma_map_single(dev, seg->bounce_buf,
max_pkt, DMA_FROM_DEVICE);
@@ -3291,6 +3563,9 @@ static int xhci_align_td(struct xhci_hcd *xhci, struct urb *urb, u32 enqd_len,
xhci_warn(xhci, "Failed mapping bounce buffer, not aligning\n");
return 0;
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(seg->bounce_dma, max_pkt);
+#endif
*trb_buff_len = new_buff_len;
seg->bounce_len = new_buff_len;
seg->bounce_offs = enqd_len;
@@ -3336,6 +3611,9 @@ int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
addr = (u64) urb->transfer_dma;
block_len = full_len;
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(addr, block_len);
+#endif
ret = prepare_transfer(xhci, xhci->devs[slot_id],
ep_index, urb->stream_id,
num_trbs, urb, 0, mem_flags);
@@ -3405,6 +3683,10 @@ int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
trb_buff_len);
le64_to_cpus(&send_addr);
field |= TRB_IDT;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(urb->transfer_buffer, trb_buff_len);
+ cdns_flush_dcache(send_addr, trb_buff_len);
+#endif
}
}
@@ -3419,6 +3701,9 @@ int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
length_field = TRB_LEN(trb_buff_len) |
TRB_TD_SIZE(remainder) |
TRB_INTR_TARGET(0);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(send_addr, trb_buff_len);
+#endif
queue_trb(xhci, ring, more_trbs_coming | need_zero_pkt,
lower_32_bits(send_addr),
@@ -3525,6 +3810,9 @@ int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
field |= TRB_TX_TYPE(TRB_DATA_IN);
else
field |= TRB_TX_TYPE(TRB_DATA_OUT);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(setup, sizeof(struct usb_ctrlrequest));
+#endif
}
}
@@ -3534,6 +3822,9 @@ int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
TRB_LEN(8) | TRB_INTR_TARGET(0),
/* Immediate data in pointer */
field);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(setup, sizeof(struct usb_ctrlrequest));
+#endif
/* If there's data, queue data TRBs */
/* Only set interrupt on short packet for IN endpoints */
@@ -3551,6 +3842,12 @@ int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
urb->transfer_buffer_length);
le64_to_cpus(&addr);
field |= TRB_IDT;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(urb->transfer_buffer,
+ urb->transfer_buffer_length);
+ cdns_flush_dcache(addr,
+ urb->transfer_buffer_length);
+#endif
} else {
addr = (u64) urb->transfer_dma;
}
@@ -3564,6 +3861,10 @@ int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
TRB_INTR_TARGET(0);
if (setup->bRequestType & USB_DIR_IN)
field |= TRB_DIR_IN;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(setup, sizeof(struct usb_ctrlrequest));
+ cdns_flush_dcache(addr, urb->transfer_buffer_length);
+#endif
queue_trb(xhci, ep_ring, true,
lower_32_bits(addr),
upper_32_bits(addr),
@@ -3580,6 +3881,9 @@ int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
field = 0;
else
field = TRB_DIR_IN;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(setup, sizeof(struct usb_ctrlrequest));
+#endif
queue_trb(xhci, ep_ring, false,
0,
0,
@@ -3883,7 +4187,9 @@ static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
else
length_field |= TRB_TD_SIZE(remainder);
first_trb = false;
-
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(addr, trb_buff_len);
+#endif
queue_trb(xhci, ep_ring, more_trbs_coming,
lower_32_bits(addr),
upper_32_bits(addr),
@@ -3958,6 +4264,9 @@ int xhci_queue_isoc_tx_prepare(struct xhci_hcd *xhci, gfp_t mem_flags,
int ret;
struct xhci_virt_ep *xep;
int ist;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ u32 ep_state;
+#endif
xdev = xhci->devs[slot_id];
xep = &xhci->devs[slot_id]->eps[ep_index];
@@ -3972,8 +4281,14 @@ int xhci_queue_isoc_tx_prepare(struct xhci_hcd *xhci, gfp_t mem_flags,
/* Check the ring to guarantee there is enough room for the whole urb.
* Do not insert any td of the urb to the ring if the check failed.
*/
+#if defined(CONFIG_USB_CDNS3_HOST_FLUSH_DMA)
+ ep_state = GET_EP_CTX_STATE(ep_ctx);
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+ ret = prepare_ring(xhci, ep_ring, ep_state, num_trbs, mem_flags);
+#else
ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx),
num_trbs, mem_flags);
+#endif
if (ret)
return ret;
@@ -3986,9 +4301,15 @@ int xhci_queue_isoc_tx_prepare(struct xhci_hcd *xhci, gfp_t mem_flags,
/* Calculate the start frame and put it in urb->start_frame. */
if (HCC_CFC(xhci->hcc_params) && !list_empty(&ep_ring->td_list)) {
if (GET_EP_CTX_STATE(ep_ctx) == EP_STATE_RUNNING) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
urb->start_frame = xep->next_frame_id;
goto skip_start_over;
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
}
start_frame = readl(&xhci->run_regs->microframe_index);
@@ -4085,6 +4406,9 @@ int xhci_queue_slot_control(struct xhci_hcd *xhci, struct xhci_command *cmd,
int xhci_queue_address_device(struct xhci_hcd *xhci, struct xhci_command *cmd,
dma_addr_t in_ctx_ptr, u32 slot_id, enum xhci_setup_dev setup)
{
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(in_ctx_ptr, sizeof(struct xhci_ep_ctx));
+#endif
return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr),
upper_32_bits(in_ctx_ptr), 0,
TRB_TYPE(TRB_ADDR_DEV) | SLOT_ID_FOR_TRB(slot_id)
@@ -4111,6 +4435,9 @@ int xhci_queue_configure_endpoint(struct xhci_hcd *xhci,
struct xhci_command *cmd, dma_addr_t in_ctx_ptr,
u32 slot_id, bool command_must_succeed)
{
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(in_ctx_ptr, sizeof(struct xhci_ep_ctx));
+#endif
return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr),
upper_32_bits(in_ctx_ptr), 0,
TRB_TYPE(TRB_CONFIG_EP) | SLOT_ID_FOR_TRB(slot_id),
@@ -4121,6 +4448,9 @@ int xhci_queue_configure_endpoint(struct xhci_hcd *xhci,
int xhci_queue_evaluate_context(struct xhci_hcd *xhci, struct xhci_command *cmd,
dma_addr_t in_ctx_ptr, u32 slot_id, bool command_must_succeed)
{
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(in_ctx_ptr, sizeof(struct xhci_ep_ctx));
+#endif
return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr),
upper_32_bits(in_ctx_ptr), 0,
TRB_TYPE(TRB_EVAL_CONTEXT) | SLOT_ID_FOR_TRB(slot_id),
@@ -4191,6 +4521,9 @@ void xhci_queue_new_dequeue_state(struct xhci_hcd *xhci,
ep->queued_deq_ptr = deq_state->new_deq_ptr;
if (deq_state->stream_id)
trb_sct = SCT_FOR_TRB(SCT_PRI_TR);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(addr, sizeof(union xhci_trb));
+#endif
ret = queue_command(xhci, cmd,
lower_32_bits(addr) | trb_sct | deq_state->new_cycle_state,
upper_32_bits(addr), trb_stream_id,
diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c
index d4a8d0efbbc4..0f3085d284e8 100644
--- a/drivers/usb/host/xhci.c
+++ b/drivers/usb/host/xhci.c
@@ -857,6 +857,11 @@ static void xhci_clear_command_ring(struct xhci_hcd *xhci)
sizeof(union xhci_trb) * (TRBS_PER_SEGMENT - 1));
seg->trbs[TRBS_PER_SEGMENT - 1].link.control &=
cpu_to_le32(~TRB_CYCLE);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(seg->trbs,
+ sizeof(union xhci_trb) *
+ TRBS_PER_SEGMENT);
+#endif
seg = seg->next;
} while (seg != ring->deq_seg);
@@ -1398,6 +1403,9 @@ static int xhci_check_maxpacket(struct xhci_hcd *xhci, unsigned int slot_id,
out_ctx = xhci->devs[slot_id]->out_ctx;
ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);
hw_max_packet_size = MAX_PACKET_DECODED(le32_to_cpu(ep_ctx->ep_info2));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
max_packet_size = usb_endpoint_maxp(&urb->dev->ep0.desc);
if (hw_max_packet_size != max_packet_size) {
xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
@@ -1437,8 +1445,14 @@ static int xhci_check_maxpacket(struct xhci_hcd *xhci, unsigned int slot_id,
ep_ctx->ep_info2 &= cpu_to_le32(~MAX_PACKET_MASK);
ep_ctx->ep_info2 |= cpu_to_le32(MAX_PACKET(max_packet_size));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
ctrl_ctx->add_flags = cpu_to_le32(EP0_FLAG);
ctrl_ctx->drop_flags = 0;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+#endif
ret = xhci_configure_endpoint(xhci, urb->dev, command,
true, false);
@@ -1447,6 +1461,9 @@ static int xhci_check_maxpacket(struct xhci_hcd *xhci, unsigned int slot_id,
* functions.
*/
ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+#endif
command_cleanup:
kfree(command->completion);
kfree(command);
@@ -1776,18 +1793,29 @@ static int xhci_drop_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
if ((GET_EP_CTX_STATE(ep_ctx) == EP_STATE_DISABLED) ||
le32_to_cpu(ctrl_ctx->drop_flags) &
xhci_get_endpoint_flag(&ep->desc)) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
/* Do not warn when called after a usb_device_reset */
if (xhci->devs[udev->slot_id]->eps[ep_index].ring != NULL)
xhci_warn(xhci, "xHCI %s called with disabled ep %p\n",
__func__, ep);
return 0;
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
ctrl_ctx->drop_flags |= cpu_to_le32(drop_flag);
new_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);
ctrl_ctx->add_flags &= cpu_to_le32(~drop_flag);
new_add_flags = le32_to_cpu(ctrl_ctx->add_flags);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+#endif
xhci_debugfs_remove_endpoint(xhci, xhci->devs[udev->slot_id], ep_index);
@@ -1866,20 +1894,32 @@ static int xhci_add_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
*/
if (virt_dev->eps[ep_index].ring &&
!(le32_to_cpu(ctrl_ctx->drop_flags) & added_ctxs)) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+#endif
xhci_warn(xhci, "Trying to add endpoint 0x%x "
"without dropping it.\n",
(unsigned int) ep->desc.bEndpointAddress);
return -EINVAL;
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+#endif
/* If the HCD has already noted the endpoint is enabled,
* ignore this request.
*/
if (le32_to_cpu(ctrl_ctx->add_flags) & added_ctxs) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+#endif
xhci_warn(xhci, "xHCI %s called with enabled ep %p\n",
__func__, ep);
return 0;
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+#endif
/*
* Configuration and alternate setting changes must be done in
@@ -1911,12 +1951,18 @@ static int xhci_add_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
* drop flags alone.
*/
new_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+#endif
/* Store the usb_device pointer for later use */
ep->hcpriv = udev;
ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index);
trace_xhci_add_endpoint(ep_ctx);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
xhci_dbg(xhci, "add ep 0x%x, slot id %d, new drop flags = %#x, new add flags = %#x\n",
(unsigned int) ep->desc.bEndpointAddress,
@@ -1947,16 +1993,25 @@ static void xhci_zero_in_ctx(struct xhci_hcd *xhci, struct xhci_virt_device *vir
*/
ctrl_ctx->drop_flags = 0;
ctrl_ctx->add_flags = 0;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+#endif
slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
/* Endpoint 0 is always valid */
slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
for (i = 1; i < 31; i++) {
ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, i);
ep_ctx->ep_info = 0;
ep_ctx->ep_info2 = 0;
ep_ctx->deq = 0;
ep_ctx->tx_info = 0;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
}
}
@@ -2072,6 +2127,9 @@ static u32 xhci_count_num_new_endpoints(struct xhci_hcd *xhci,
*/
valid_add_flags = le32_to_cpu(ctrl_ctx->add_flags) >> 2;
valid_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags) >> 2;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+#endif
/* Use hweight32 to count the number of ones in the add flags, or
* number of endpoints added. Don't count endpoints that are changed
@@ -2089,6 +2147,9 @@ static unsigned int xhci_count_num_dropped_endpoints(struct xhci_hcd *xhci,
valid_add_flags = le32_to_cpu(ctrl_ctx->add_flags) >> 2;
valid_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags) >> 2;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+#endif
return hweight32(valid_drop_flags) -
hweight32(valid_add_flags & valid_drop_flags);
@@ -2668,8 +2729,17 @@ static int xhci_reserve_bandwidth(struct xhci_hcd *xhci,
}
for (i = 0; i < 31; i++) {
- if (!EP_IS_ADDED(ctrl_ctx, i) && !EP_IS_DROPPED(ctrl_ctx, i))
+ if (!EP_IS_ADDED(ctrl_ctx, i) && !EP_IS_DROPPED(ctrl_ctx, i)) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx,
+ sizeof(struct xhci_input_control_ctx));
+#endif
continue;
+ }
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx,
+ sizeof(struct xhci_input_control_ctx));
+#endif
/* Make a copy of the BW info in case we need to revert this */
memcpy(&ep_bw_info[i], &virt_dev->eps[i].bw_info,
@@ -2677,25 +2747,45 @@ static int xhci_reserve_bandwidth(struct xhci_hcd *xhci,
/* Drop the endpoint from the interval table if the endpoint is
* being dropped or changed.
*/
- if (EP_IS_DROPPED(ctrl_ctx, i))
+ if (EP_IS_DROPPED(ctrl_ctx, i)){
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx,
+ sizeof(struct xhci_input_control_ctx));
+#endif
xhci_drop_ep_from_interval_table(xhci,
&virt_dev->eps[i].bw_info,
virt_dev->bw_table,
virt_dev->udev,
&virt_dev->eps[i],
virt_dev->tt_info);
+ }
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ else
+ cdns_virt_flush_dcache(ctrl_ctx,
+ sizeof(struct xhci_input_control_ctx));
+#endif
}
/* Overwrite the information stored in the endpoints' bw_info */
xhci_update_bw_info(xhci, virt_dev->in_ctx, ctrl_ctx, virt_dev);
for (i = 0; i < 31; i++) {
/* Add any changed or added endpoints to the interval table */
- if (EP_IS_ADDED(ctrl_ctx, i))
+ if (EP_IS_ADDED(ctrl_ctx, i)){
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx,
+ sizeof(struct xhci_input_control_ctx));
+#endif
xhci_add_ep_to_interval_table(xhci,
&virt_dev->eps[i].bw_info,
virt_dev->bw_table,
virt_dev->udev,
&virt_dev->eps[i],
virt_dev->tt_info);
+ }
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ else
+ cdns_virt_flush_dcache(ctrl_ctx,
+ sizeof(struct xhci_input_control_ctx));
+#endif
}
if (!xhci_check_bw_table(xhci, virt_dev, old_active_eps)) {
@@ -2708,13 +2798,26 @@ static int xhci_reserve_bandwidth(struct xhci_hcd *xhci,
/* We don't have enough bandwidth for this, revert the stored info. */
for (i = 0; i < 31; i++) {
- if (!EP_IS_ADDED(ctrl_ctx, i) && !EP_IS_DROPPED(ctrl_ctx, i))
+ if (!EP_IS_ADDED(ctrl_ctx, i) && !EP_IS_DROPPED(ctrl_ctx, i)) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx,
+ sizeof(struct xhci_input_control_ctx));
+#endif
continue;
+ }
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx,
+ sizeof(struct xhci_input_control_ctx));
+#endif
/* Drop the new copies of any added or changed endpoints from
* the interval table.
*/
if (EP_IS_ADDED(ctrl_ctx, i)) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx,
+ sizeof(struct xhci_input_control_ctx));
+#endif
xhci_drop_ep_from_interval_table(xhci,
&virt_dev->eps[i].bw_info,
virt_dev->bw_table,
@@ -2722,18 +2825,36 @@ static int xhci_reserve_bandwidth(struct xhci_hcd *xhci,
&virt_dev->eps[i],
virt_dev->tt_info);
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ else
+ cdns_virt_flush_dcache(ctrl_ctx,
+ sizeof(struct xhci_input_control_ctx));
+#endif
/* Revert the endpoint back to its old information */
memcpy(&virt_dev->eps[i].bw_info, &ep_bw_info[i],
sizeof(ep_bw_info[i]));
/* Add any changed or dropped endpoints back into the table */
- if (EP_IS_DROPPED(ctrl_ctx, i))
+ if (EP_IS_DROPPED(ctrl_ctx, i)) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx,
+ sizeof(struct xhci_input_control_ctx));
+#endif
xhci_add_ep_to_interval_table(xhci,
&virt_dev->eps[i].bw_info,
virt_dev->bw_table,
virt_dev->udev,
&virt_dev->eps[i],
virt_dev->tt_info);
+ }
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ else
+ cdns_virt_flush_dcache(ctrl_ctx,
+ sizeof(struct xhci_input_control_ctx));
+#endif
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+#endif
return -ENOMEM;
}
@@ -2793,6 +2914,10 @@ static int xhci_configure_endpoint(struct xhci_hcd *xhci,
trace_xhci_configure_endpoint_ctrl_ctx(ctrl_ctx);
trace_xhci_configure_endpoint(slot_ctx);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
if (!ctx_change)
ret = xhci_queue_configure_endpoint(xhci, command,
@@ -2899,13 +3024,22 @@ static int xhci_check_bandwidth(struct usb_hcd *hcd, struct usb_device *udev)
ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
ctrl_ctx->add_flags &= cpu_to_le32(~EP0_FLAG);
ctrl_ctx->drop_flags &= cpu_to_le32(~(SLOT_FLAG | EP0_FLAG));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+#endif
/* Don't issue the command if there's no endpoints to update. */
if (ctrl_ctx->add_flags == cpu_to_le32(SLOT_FLAG) &&
ctrl_ctx->drop_flags == 0) {
ret = 0;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+#endif
goto command_cleanup;
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+#endif
/* Fix up Context Entries field. Minimum value is EP0 == BIT(1). */
slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
for (i = 31; i >= 1; i--) {
@@ -2913,10 +3047,19 @@ static int xhci_check_bandwidth(struct usb_hcd *hcd, struct usb_device *udev)
if ((virt_dev->eps[i-1].ring && !(ctrl_ctx->drop_flags & le32))
|| (ctrl_ctx->add_flags & le32) || i == 1) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+#endif
slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(i));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
break;
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+#endif
}
ret = xhci_configure_endpoint(xhci, udev, command,
@@ -2929,9 +3072,16 @@ static int xhci_check_bandwidth(struct usb_hcd *hcd, struct usb_device *udev)
for (i = 1; i < 31; i++) {
if ((le32_to_cpu(ctrl_ctx->drop_flags) & (1 << (i + 1))) &&
!(le32_to_cpu(ctrl_ctx->add_flags) & (1 << (i + 1)))) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+#endif
xhci_free_endpoint_ring(xhci, virt_dev, i);
xhci_check_bw_drop_ep_streams(xhci, virt_dev, i);
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ else
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+#endif
}
xhci_zero_in_ctx(xhci, virt_dev);
/*
@@ -2993,6 +3143,9 @@ static void xhci_setup_input_ctx_for_config_ep(struct xhci_hcd *xhci,
ctrl_ctx->drop_flags = cpu_to_le32(drop_flags);
xhci_slot_copy(xhci, in_ctx, out_ctx);
ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+#endif
}
static void xhci_setup_input_ctx_for_quirk(struct xhci_hcd *xhci,
@@ -3027,6 +3180,9 @@ static void xhci_setup_input_ctx_for_quirk(struct xhci_hcd *xhci,
return;
}
ep_ctx->deq = cpu_to_le64(addr | deq_state->new_cycle_state);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ep_ctx, sizeof(*ep_ctx));
+#endif
added_ctxs = xhci_get_endpoint_flag_from_index(ep_index);
xhci_setup_input_ctx_for_config_ep(xhci, xhci->devs[slot_id]->in_ctx,
@@ -3748,10 +3904,17 @@ static int xhci_discover_or_reset_device(struct usb_hcd *hcd,
/* If device is not setup, there is no point in resetting it */
slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
if (GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state)) ==
- SLOT_STATE_DISABLED)
+ SLOT_STATE_DISABLED) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
return 0;
+ }
trace_xhci_discover_or_reset_device(slot_ctx);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
xhci_dbg(xhci, "Resetting device with slot ID %u\n", slot_id);
/* Allocate the command structure that holds the struct completion.
@@ -3888,6 +4051,9 @@ static void xhci_free_dev(struct usb_hcd *hcd, struct usb_device *udev)
virt_dev = xhci->devs[udev->slot_id];
slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
trace_xhci_free_dev(slot_ctx);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
/* Stop any wayward timer functions (which may grab the lock) */
for (i = 0; i < 31; i++) {
@@ -4023,6 +4189,9 @@ int xhci_alloc_dev(struct usb_hcd *hcd, struct usb_device *udev)
vdev = xhci->devs[slot_id];
slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx);
trace_xhci_alloc_dev(slot_ctx);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
udev->slot_id = slot_id;
@@ -4099,9 +4268,16 @@ static int xhci_setup_device(struct usb_hcd *hcd, struct usb_device *udev,
if (setup == SETUP_CONTEXT_ONLY) {
if (GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state)) ==
SLOT_STATE_DEFAULT) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
xhci_dbg(xhci, "Slot already in default state\n");
goto out;
}
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ else
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
}
command = xhci_alloc_command(xhci, true, GFP_KERNEL);
@@ -4125,18 +4301,35 @@ static int xhci_setup_device(struct usb_hcd *hcd, struct usb_device *udev,
* virt_device realloaction after a resume with an xHCI power loss,
* then set up the slot context.
*/
- if (!slot_ctx->dev_info)
+ if (!slot_ctx->dev_info) {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
xhci_setup_addressable_virt_dev(xhci, udev);
/* Otherwise, update the control endpoint ring enqueue pointer. */
- else
+ } else {
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
xhci_copy_ep0_dequeue_into_input_ctx(xhci, udev);
+ }
ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG | EP0_FLAG);
ctrl_ctx->drop_flags = 0;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+#endif
trace_xhci_address_ctx(xhci, virt_dev->in_ctx,
le32_to_cpu(slot_ctx->dev_info) >> 27);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
trace_xhci_address_ctrl_ctx(ctrl_ctx);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+#endif
+
spin_lock_irqsave(&xhci->lock, flags);
trace_xhci_setup_device(virt_dev);
ret = xhci_queue_address_device(xhci, command, virt_dev->in_ctx->dma,
@@ -4207,6 +4400,9 @@ static int xhci_setup_device(struct usb_hcd *hcd, struct usb_device *udev,
&xhci->dcbaa->dev_context_ptrs[udev->slot_id],
(unsigned long long)
le64_to_cpu(xhci->dcbaa->dev_context_ptrs[udev->slot_id]));
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_flush_dcache(xhci->dcbaa->dma, sizeof(*xhci->dcbaa));
+#endif
xhci_dbg_trace(xhci, trace_xhci_dbg_address,
"Output Context DMA address = %#08llx",
(unsigned long long)virt_dev->out_ctx->dma);
@@ -4218,9 +4414,15 @@ static int xhci_setup_device(struct usb_hcd *hcd, struct usb_device *udev,
*/
trace_xhci_address_ctx(xhci, virt_dev->out_ctx,
le32_to_cpu(slot_ctx->dev_info) >> 27);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
/* Zero the input context control for later use */
ctrl_ctx->add_flags = 0;
ctrl_ctx->drop_flags = 0;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+#endif
slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
udev->devaddr = (u8)(le32_to_cpu(slot_ctx->dev_state) & DEV_ADDR_MASK);
@@ -4228,6 +4430,9 @@ static int xhci_setup_device(struct usb_hcd *hcd, struct usb_device *udev,
"Internal device address = %d",
le32_to_cpu(slot_ctx->dev_state) & DEV_ADDR_MASK);
out:
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
mutex_unlock(&xhci->mutex);
if (command) {
kfree(command->completion);
@@ -4303,10 +4508,16 @@ static int __maybe_unused xhci_change_max_exit_latency(struct xhci_hcd *xhci,
spin_unlock_irqrestore(&xhci->lock, flags);
ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+#endif
slot_ctx = xhci_get_slot_ctx(xhci, command->in_ctx);
slot_ctx->dev_info2 &= cpu_to_le32(~((u32) MAX_EXIT));
slot_ctx->dev_info2 |= cpu_to_le32(max_exit_latency);
slot_ctx->dev_state = 0;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
"Set up evaluate context for LPM MEL change.");
@@ -5064,6 +5275,9 @@ static int xhci_update_hub_device(struct usb_hcd *hcd, struct usb_device *hdev,
xhci_slot_copy(xhci, config_cmd->in_ctx, vdev->out_ctx);
ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(ctrl_ctx, sizeof(*ctrl_ctx));
+#endif
slot_ctx = xhci_get_slot_ctx(xhci, config_cmd->in_ctx);
slot_ctx->dev_info |= cpu_to_le32(DEV_HUB);
/*
@@ -5100,6 +5314,9 @@ static int xhci_update_hub_device(struct usb_hcd *hcd, struct usb_device *hdev,
(unsigned int) xhci->hci_version);
}
slot_ctx->dev_state = 0;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(slot_ctx, sizeof(*slot_ctx));
+#endif
spin_unlock_irqrestore(&xhci->lock, flags);
xhci_dbg(xhci, "Set up %s for hub device.\n",
diff --git a/include/linux/usb.h b/include/linux/usb.h
index 7d72c4e0713c..91477daef4a9 100644
--- a/include/linux/usb.h
+++ b/include/linux/usb.h
@@ -1597,6 +1597,20 @@ struct urb {
/* (in) ISO ONLY */
};
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+#include <soc/starfive/vic7100.h>
+static inline void cdns_flush_dcache(unsigned long start, unsigned long len)
+{
+ starfive_flush_dcache(_ALIGN_DOWN(start, 64), len + (start & 63));
+}
+
+static inline void cdns_virt_flush_dcache(void *virt_start, unsigned long len)
+{
+ unsigned long start = dw_virt_to_phys(virt_start);
+
+ starfive_flush_dcache(_ALIGN_DOWN(start, 64), len + (start & 63));
+}
+#endif
/* ----------------------------------------------------------------------- */
/**
@@ -1629,6 +1643,9 @@ static inline void usb_fill_control_urb(struct urb *urb,
urb->transfer_buffer_length = buffer_length;
urb->complete = complete_fn;
urb->context = context;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(transfer_buffer, buffer_length);
+#endif
}
/**
@@ -1658,6 +1675,9 @@ static inline void usb_fill_bulk_urb(struct urb *urb,
urb->transfer_buffer_length = buffer_length;
urb->complete = complete_fn;
urb->context = context;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(transfer_buffer, buffer_length);
+#endif
}
/**
@@ -1701,6 +1721,9 @@ static inline void usb_fill_int_urb(struct urb *urb,
urb->complete = complete_fn;
urb->context = context;
+#ifdef CONFIG_USB_CDNS3_HOST_FLUSH_DMA
+ cdns_virt_flush_dcache(transfer_buffer, buffer_length);
+#endif
if (dev->speed == USB_SPEED_HIGH || dev->speed >= USB_SPEED_SUPER) {
/* make sure interval is within allowed range */
interval = clamp(interval, 1, 16);