cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

musb_gadget_ep0.c (26232B)


      1// SPDX-License-Identifier: GPL-2.0
      2/*
      3 * MUSB OTG peripheral driver ep0 handling
      4 *
      5 * Copyright 2005 Mentor Graphics Corporation
      6 * Copyright (C) 2005-2006 by Texas Instruments
      7 * Copyright (C) 2006-2007 Nokia Corporation
      8 * Copyright (C) 2008-2009 MontaVista Software, Inc. <source@mvista.com>
      9 */
     10
     11#include <linux/kernel.h>
     12#include <linux/list.h>
     13#include <linux/timer.h>
     14#include <linux/spinlock.h>
     15#include <linux/device.h>
     16#include <linux/interrupt.h>
     17
     18#include "musb_core.h"
     19
     20/* ep0 is always musb->endpoints[0].ep_in */
     21#define	next_ep0_request(musb)	next_in_request(&(musb)->endpoints[0])
     22
     23/*
     24 * locking note:  we use only the controller lock, for simpler correctness.
     25 * It's always held with IRQs blocked.
     26 *
     27 * It protects the ep0 request queue as well as ep0_state, not just the
     28 * controller and indexed registers.  And that lock stays held unless it
     29 * needs to be dropped to allow reentering this driver ... like upcalls to
     30 * the gadget driver, or adjusting endpoint halt status.
     31 */
     32
     33static char *decode_ep0stage(u8 stage)
     34{
     35	switch (stage) {
     36	case MUSB_EP0_STAGE_IDLE:	return "idle";
     37	case MUSB_EP0_STAGE_SETUP:	return "setup";
     38	case MUSB_EP0_STAGE_TX:		return "in";
     39	case MUSB_EP0_STAGE_RX:		return "out";
     40	case MUSB_EP0_STAGE_ACKWAIT:	return "wait";
     41	case MUSB_EP0_STAGE_STATUSIN:	return "in/status";
     42	case MUSB_EP0_STAGE_STATUSOUT:	return "out/status";
     43	default:			return "?";
     44	}
     45}
     46
     47/* handle a standard GET_STATUS request
     48 * Context:  caller holds controller lock
     49 */
     50static int service_tx_status_request(
     51	struct musb *musb,
     52	const struct usb_ctrlrequest *ctrlrequest)
     53{
     54	void __iomem	*mbase = musb->mregs;
     55	int handled = 1;
     56	u8 result[2], epnum = 0;
     57	const u8 recip = ctrlrequest->bRequestType & USB_RECIP_MASK;
     58
     59	result[1] = 0;
     60
     61	switch (recip) {
     62	case USB_RECIP_DEVICE:
     63		result[0] = musb->g.is_selfpowered << USB_DEVICE_SELF_POWERED;
     64		result[0] |= musb->may_wakeup << USB_DEVICE_REMOTE_WAKEUP;
     65		if (musb->g.is_otg) {
     66			result[0] |= musb->g.b_hnp_enable
     67				<< USB_DEVICE_B_HNP_ENABLE;
     68			result[0] |= musb->g.a_alt_hnp_support
     69				<< USB_DEVICE_A_ALT_HNP_SUPPORT;
     70			result[0] |= musb->g.a_hnp_support
     71				<< USB_DEVICE_A_HNP_SUPPORT;
     72		}
     73		break;
     74
     75	case USB_RECIP_INTERFACE:
     76		result[0] = 0;
     77		break;
     78
     79	case USB_RECIP_ENDPOINT: {
     80		int		is_in;
     81		struct musb_ep	*ep;
     82		u16		tmp;
     83		void __iomem	*regs;
     84
     85		epnum = (u8) ctrlrequest->wIndex;
     86		if (!epnum) {
     87			result[0] = 0;
     88			break;
     89		}
     90
     91		is_in = epnum & USB_DIR_IN;
     92		epnum &= 0x0f;
     93		if (epnum >= MUSB_C_NUM_EPS) {
     94			handled = -EINVAL;
     95			break;
     96		}
     97
     98		if (is_in)
     99			ep = &musb->endpoints[epnum].ep_in;
    100		else
    101			ep = &musb->endpoints[epnum].ep_out;
    102		regs = musb->endpoints[epnum].regs;
    103
    104		if (!ep->desc) {
    105			handled = -EINVAL;
    106			break;
    107		}
    108
    109		musb_ep_select(mbase, epnum);
    110		if (is_in)
    111			tmp = musb_readw(regs, MUSB_TXCSR)
    112						& MUSB_TXCSR_P_SENDSTALL;
    113		else
    114			tmp = musb_readw(regs, MUSB_RXCSR)
    115						& MUSB_RXCSR_P_SENDSTALL;
    116		musb_ep_select(mbase, 0);
    117
    118		result[0] = tmp ? 1 : 0;
    119		} break;
    120
    121	default:
    122		/* class, vendor, etc ... delegate */
    123		handled = 0;
    124		break;
    125	}
    126
    127	/* fill up the fifo; caller updates csr0 */
    128	if (handled > 0) {
    129		u16	len = le16_to_cpu(ctrlrequest->wLength);
    130
    131		if (len > 2)
    132			len = 2;
    133		musb_write_fifo(&musb->endpoints[0], len, result);
    134	}
    135
    136	return handled;
    137}
    138
    139/*
    140 * handle a control-IN request, the end0 buffer contains the current request
    141 * that is supposed to be a standard control request. Assumes the fifo to
    142 * be at least 2 bytes long.
    143 *
    144 * @return 0 if the request was NOT HANDLED,
    145 * < 0 when error
    146 * > 0 when the request is processed
    147 *
    148 * Context:  caller holds controller lock
    149 */
    150static int
    151service_in_request(struct musb *musb, const struct usb_ctrlrequest *ctrlrequest)
    152{
    153	int handled = 0;	/* not handled */
    154
    155	if ((ctrlrequest->bRequestType & USB_TYPE_MASK)
    156			== USB_TYPE_STANDARD) {
    157		switch (ctrlrequest->bRequest) {
    158		case USB_REQ_GET_STATUS:
    159			handled = service_tx_status_request(musb,
    160					ctrlrequest);
    161			break;
    162
    163		/* case USB_REQ_SYNC_FRAME: */
    164
    165		default:
    166			break;
    167		}
    168	}
    169	return handled;
    170}
    171
    172/*
    173 * Context:  caller holds controller lock
    174 */
    175static void musb_g_ep0_giveback(struct musb *musb, struct usb_request *req)
    176{
    177	musb_g_giveback(&musb->endpoints[0].ep_in, req, 0);
    178}
    179
    180/*
    181 * Tries to start B-device HNP negotiation if enabled via sysfs
    182 */
    183static inline void musb_try_b_hnp_enable(struct musb *musb)
    184{
    185	void __iomem	*mbase = musb->mregs;
    186	u8		devctl;
    187
    188	musb_dbg(musb, "HNP: Setting HR");
    189	devctl = musb_readb(mbase, MUSB_DEVCTL);
    190	musb_writeb(mbase, MUSB_DEVCTL, devctl | MUSB_DEVCTL_HR);
    191}
    192
    193/*
    194 * Handle all control requests with no DATA stage, including standard
    195 * requests such as:
    196 * USB_REQ_SET_CONFIGURATION, USB_REQ_SET_INTERFACE, unrecognized
    197 *	always delegated to the gadget driver
    198 * USB_REQ_SET_ADDRESS, USB_REQ_CLEAR_FEATURE, USB_REQ_SET_FEATURE
    199 *	always handled here, except for class/vendor/... features
    200 *
    201 * Context:  caller holds controller lock
    202 */
    203static int
    204service_zero_data_request(struct musb *musb,
    205		struct usb_ctrlrequest *ctrlrequest)
    206__releases(musb->lock)
    207__acquires(musb->lock)
    208{
    209	int handled = -EINVAL;
    210	void __iomem *mbase = musb->mregs;
    211	const u8 recip = ctrlrequest->bRequestType & USB_RECIP_MASK;
    212
    213	/* the gadget driver handles everything except what we MUST handle */
    214	if ((ctrlrequest->bRequestType & USB_TYPE_MASK)
    215			== USB_TYPE_STANDARD) {
    216		switch (ctrlrequest->bRequest) {
    217		case USB_REQ_SET_ADDRESS:
    218			/* change it after the status stage */
    219			musb->set_address = true;
    220			musb->address = (u8) (ctrlrequest->wValue & 0x7f);
    221			handled = 1;
    222			break;
    223
    224		case USB_REQ_CLEAR_FEATURE:
    225			switch (recip) {
    226			case USB_RECIP_DEVICE:
    227				if (ctrlrequest->wValue
    228						!= USB_DEVICE_REMOTE_WAKEUP)
    229					break;
    230				musb->may_wakeup = 0;
    231				handled = 1;
    232				break;
    233			case USB_RECIP_INTERFACE:
    234				break;
    235			case USB_RECIP_ENDPOINT:{
    236				const u8		epnum =
    237					ctrlrequest->wIndex & 0x0f;
    238				struct musb_ep		*musb_ep;
    239				struct musb_hw_ep	*ep;
    240				struct musb_request	*request;
    241				void __iomem		*regs;
    242				int			is_in;
    243				u16			csr;
    244
    245				if (epnum == 0 || epnum >= MUSB_C_NUM_EPS ||
    246				    ctrlrequest->wValue != USB_ENDPOINT_HALT)
    247					break;
    248
    249				ep = musb->endpoints + epnum;
    250				regs = ep->regs;
    251				is_in = ctrlrequest->wIndex & USB_DIR_IN;
    252				if (is_in)
    253					musb_ep = &ep->ep_in;
    254				else
    255					musb_ep = &ep->ep_out;
    256				if (!musb_ep->desc)
    257					break;
    258
    259				handled = 1;
    260				/* Ignore request if endpoint is wedged */
    261				if (musb_ep->wedged)
    262					break;
    263
    264				musb_ep_select(mbase, epnum);
    265				if (is_in) {
    266					csr  = musb_readw(regs, MUSB_TXCSR);
    267					csr |= MUSB_TXCSR_CLRDATATOG |
    268					       MUSB_TXCSR_P_WZC_BITS;
    269					csr &= ~(MUSB_TXCSR_P_SENDSTALL |
    270						 MUSB_TXCSR_P_SENTSTALL |
    271						 MUSB_TXCSR_TXPKTRDY);
    272					musb_writew(regs, MUSB_TXCSR, csr);
    273				} else {
    274					csr  = musb_readw(regs, MUSB_RXCSR);
    275					csr |= MUSB_RXCSR_CLRDATATOG |
    276					       MUSB_RXCSR_P_WZC_BITS;
    277					csr &= ~(MUSB_RXCSR_P_SENDSTALL |
    278						 MUSB_RXCSR_P_SENTSTALL);
    279					musb_writew(regs, MUSB_RXCSR, csr);
    280				}
    281
    282				/* Maybe start the first request in the queue */
    283				request = next_request(musb_ep);
    284				if (!musb_ep->busy && request) {
    285					musb_dbg(musb, "restarting the request");
    286					musb_ep_restart(musb, request);
    287				}
    288
    289				/* select ep0 again */
    290				musb_ep_select(mbase, 0);
    291				} break;
    292			default:
    293				/* class, vendor, etc ... delegate */
    294				handled = 0;
    295				break;
    296			}
    297			break;
    298
    299		case USB_REQ_SET_FEATURE:
    300			switch (recip) {
    301			case USB_RECIP_DEVICE:
    302				handled = 1;
    303				switch (ctrlrequest->wValue) {
    304				case USB_DEVICE_REMOTE_WAKEUP:
    305					musb->may_wakeup = 1;
    306					break;
    307				case USB_DEVICE_TEST_MODE:
    308					if (musb->g.speed != USB_SPEED_HIGH)
    309						goto stall;
    310					if (ctrlrequest->wIndex & 0xff)
    311						goto stall;
    312
    313					switch (ctrlrequest->wIndex >> 8) {
    314					case USB_TEST_J:
    315						pr_debug("USB_TEST_J\n");
    316						musb->test_mode_nr =
    317							MUSB_TEST_J;
    318						break;
    319					case USB_TEST_K:
    320						pr_debug("USB_TEST_K\n");
    321						musb->test_mode_nr =
    322							MUSB_TEST_K;
    323						break;
    324					case USB_TEST_SE0_NAK:
    325						pr_debug("USB_TEST_SE0_NAK\n");
    326						musb->test_mode_nr =
    327							MUSB_TEST_SE0_NAK;
    328						break;
    329					case USB_TEST_PACKET:
    330						pr_debug("USB_TEST_PACKET\n");
    331						musb->test_mode_nr =
    332							MUSB_TEST_PACKET;
    333						break;
    334
    335					case 0xc0:
    336						/* TEST_FORCE_HS */
    337						pr_debug("TEST_FORCE_HS\n");
    338						musb->test_mode_nr =
    339							MUSB_TEST_FORCE_HS;
    340						break;
    341					case 0xc1:
    342						/* TEST_FORCE_FS */
    343						pr_debug("TEST_FORCE_FS\n");
    344						musb->test_mode_nr =
    345							MUSB_TEST_FORCE_FS;
    346						break;
    347					case 0xc2:
    348						/* TEST_FIFO_ACCESS */
    349						pr_debug("TEST_FIFO_ACCESS\n");
    350						musb->test_mode_nr =
    351							MUSB_TEST_FIFO_ACCESS;
    352						break;
    353					case 0xc3:
    354						/* TEST_FORCE_HOST */
    355						pr_debug("TEST_FORCE_HOST\n");
    356						musb->test_mode_nr =
    357							MUSB_TEST_FORCE_HOST;
    358						break;
    359					default:
    360						goto stall;
    361					}
    362
    363					/* enter test mode after irq */
    364					if (handled > 0)
    365						musb->test_mode = true;
    366					break;
    367				case USB_DEVICE_B_HNP_ENABLE:
    368					if (!musb->g.is_otg)
    369						goto stall;
    370					musb->g.b_hnp_enable = 1;
    371					musb_try_b_hnp_enable(musb);
    372					break;
    373				case USB_DEVICE_A_HNP_SUPPORT:
    374					if (!musb->g.is_otg)
    375						goto stall;
    376					musb->g.a_hnp_support = 1;
    377					break;
    378				case USB_DEVICE_A_ALT_HNP_SUPPORT:
    379					if (!musb->g.is_otg)
    380						goto stall;
    381					musb->g.a_alt_hnp_support = 1;
    382					break;
    383				case USB_DEVICE_DEBUG_MODE:
    384					handled = 0;
    385					break;
    386stall:
    387				default:
    388					handled = -EINVAL;
    389					break;
    390				}
    391				break;
    392
    393			case USB_RECIP_INTERFACE:
    394				break;
    395
    396			case USB_RECIP_ENDPOINT:{
    397				const u8		epnum =
    398					ctrlrequest->wIndex & 0x0f;
    399				struct musb_ep		*musb_ep;
    400				struct musb_hw_ep	*ep;
    401				void __iomem		*regs;
    402				int			is_in;
    403				u16			csr;
    404
    405				if (epnum == 0 || epnum >= MUSB_C_NUM_EPS ||
    406				    ctrlrequest->wValue	!= USB_ENDPOINT_HALT)
    407					break;
    408
    409				ep = musb->endpoints + epnum;
    410				regs = ep->regs;
    411				is_in = ctrlrequest->wIndex & USB_DIR_IN;
    412				if (is_in)
    413					musb_ep = &ep->ep_in;
    414				else
    415					musb_ep = &ep->ep_out;
    416				if (!musb_ep->desc)
    417					break;
    418
    419				musb_ep_select(mbase, epnum);
    420				if (is_in) {
    421					csr = musb_readw(regs, MUSB_TXCSR);
    422					if (csr & MUSB_TXCSR_FIFONOTEMPTY)
    423						csr |= MUSB_TXCSR_FLUSHFIFO;
    424					csr |= MUSB_TXCSR_P_SENDSTALL
    425						| MUSB_TXCSR_CLRDATATOG
    426						| MUSB_TXCSR_P_WZC_BITS;
    427					musb_writew(regs, MUSB_TXCSR, csr);
    428				} else {
    429					csr = musb_readw(regs, MUSB_RXCSR);
    430					csr |= MUSB_RXCSR_P_SENDSTALL
    431						| MUSB_RXCSR_FLUSHFIFO
    432						| MUSB_RXCSR_CLRDATATOG
    433						| MUSB_RXCSR_P_WZC_BITS;
    434					musb_writew(regs, MUSB_RXCSR, csr);
    435				}
    436
    437				/* select ep0 again */
    438				musb_ep_select(mbase, 0);
    439				handled = 1;
    440				} break;
    441
    442			default:
    443				/* class, vendor, etc ... delegate */
    444				handled = 0;
    445				break;
    446			}
    447			break;
    448		default:
    449			/* delegate SET_CONFIGURATION, etc */
    450			handled = 0;
    451		}
    452	} else
    453		handled = 0;
    454	return handled;
    455}
    456
    457/* we have an ep0out data packet
    458 * Context:  caller holds controller lock
    459 */
    460static void ep0_rxstate(struct musb *musb)
    461{
    462	void __iomem		*regs = musb->control_ep->regs;
    463	struct musb_request	*request;
    464	struct usb_request	*req;
    465	u16			count, csr;
    466
    467	request = next_ep0_request(musb);
    468	req = &request->request;
    469
    470	/* read packet and ack; or stall because of gadget driver bug:
    471	 * should have provided the rx buffer before setup() returned.
    472	 */
    473	if (req) {
    474		void		*buf = req->buf + req->actual;
    475		unsigned	len = req->length - req->actual;
    476
    477		/* read the buffer */
    478		count = musb_readb(regs, MUSB_COUNT0);
    479		if (count > len) {
    480			req->status = -EOVERFLOW;
    481			count = len;
    482		}
    483		if (count > 0) {
    484			musb_read_fifo(&musb->endpoints[0], count, buf);
    485			req->actual += count;
    486		}
    487		csr = MUSB_CSR0_P_SVDRXPKTRDY;
    488		if (count < 64 || req->actual == req->length) {
    489			musb->ep0_state = MUSB_EP0_STAGE_STATUSIN;
    490			csr |= MUSB_CSR0_P_DATAEND;
    491		} else
    492			req = NULL;
    493	} else
    494		csr = MUSB_CSR0_P_SVDRXPKTRDY | MUSB_CSR0_P_SENDSTALL;
    495
    496
    497	/* Completion handler may choose to stall, e.g. because the
    498	 * message just received holds invalid data.
    499	 */
    500	if (req) {
    501		musb->ackpend = csr;
    502		musb_g_ep0_giveback(musb, req);
    503		if (!musb->ackpend)
    504			return;
    505		musb->ackpend = 0;
    506	}
    507	musb_ep_select(musb->mregs, 0);
    508	musb_writew(regs, MUSB_CSR0, csr);
    509}
    510
    511/*
    512 * transmitting to the host (IN), this code might be called from IRQ
    513 * and from kernel thread.
    514 *
    515 * Context:  caller holds controller lock
    516 */
    517static void ep0_txstate(struct musb *musb)
    518{
    519	void __iomem		*regs = musb->control_ep->regs;
    520	struct musb_request	*req = next_ep0_request(musb);
    521	struct usb_request	*request;
    522	u16			csr = MUSB_CSR0_TXPKTRDY;
    523	u8			*fifo_src;
    524	u8			fifo_count;
    525
    526	if (!req) {
    527		/* WARN_ON(1); */
    528		musb_dbg(musb, "odd; csr0 %04x", musb_readw(regs, MUSB_CSR0));
    529		return;
    530	}
    531
    532	request = &req->request;
    533
    534	/* load the data */
    535	fifo_src = (u8 *) request->buf + request->actual;
    536	fifo_count = min((unsigned) MUSB_EP0_FIFOSIZE,
    537		request->length - request->actual);
    538	musb_write_fifo(&musb->endpoints[0], fifo_count, fifo_src);
    539	request->actual += fifo_count;
    540
    541	/* update the flags */
    542	if (fifo_count < MUSB_MAX_END0_PACKET
    543			|| (request->actual == request->length
    544				&& !request->zero)) {
    545		musb->ep0_state = MUSB_EP0_STAGE_STATUSOUT;
    546		csr |= MUSB_CSR0_P_DATAEND;
    547	} else
    548		request = NULL;
    549
    550	/* report completions as soon as the fifo's loaded; there's no
    551	 * win in waiting till this last packet gets acked.  (other than
    552	 * very precise fault reporting, needed by USB TMC; possible with
    553	 * this hardware, but not usable from portable gadget drivers.)
    554	 */
    555	if (request) {
    556		musb->ackpend = csr;
    557		musb_g_ep0_giveback(musb, request);
    558		if (!musb->ackpend)
    559			return;
    560		musb->ackpend = 0;
    561	}
    562
    563	/* send it out, triggering a "txpktrdy cleared" irq */
    564	musb_ep_select(musb->mregs, 0);
    565	musb_writew(regs, MUSB_CSR0, csr);
    566}
    567
    568/*
    569 * Read a SETUP packet (struct usb_ctrlrequest) from the hardware.
    570 * Fields are left in USB byte-order.
    571 *
    572 * Context:  caller holds controller lock.
    573 */
    574static void
    575musb_read_setup(struct musb *musb, struct usb_ctrlrequest *req)
    576{
    577	struct musb_request	*r;
    578	void __iomem		*regs = musb->control_ep->regs;
    579
    580	musb_read_fifo(&musb->endpoints[0], sizeof *req, (u8 *)req);
    581
    582	/* NOTE:  earlier 2.6 versions changed setup packets to host
    583	 * order, but now USB packets always stay in USB byte order.
    584	 */
    585	musb_dbg(musb, "SETUP req%02x.%02x v%04x i%04x l%d",
    586		req->bRequestType,
    587		req->bRequest,
    588		le16_to_cpu(req->wValue),
    589		le16_to_cpu(req->wIndex),
    590		le16_to_cpu(req->wLength));
    591
    592	/* clean up any leftover transfers */
    593	r = next_ep0_request(musb);
    594	if (r)
    595		musb_g_ep0_giveback(musb, &r->request);
    596
    597	/* For zero-data requests we want to delay the STATUS stage to
    598	 * avoid SETUPEND errors.  If we read data (OUT), delay accepting
    599	 * packets until there's a buffer to store them in.
    600	 *
    601	 * If we write data, the controller acts happier if we enable
    602	 * the TX FIFO right away, and give the controller a moment
    603	 * to switch modes...
    604	 */
    605	musb->set_address = false;
    606	musb->ackpend = MUSB_CSR0_P_SVDRXPKTRDY;
    607	if (req->wLength == 0) {
    608		if (req->bRequestType & USB_DIR_IN)
    609			musb->ackpend |= MUSB_CSR0_TXPKTRDY;
    610		musb->ep0_state = MUSB_EP0_STAGE_ACKWAIT;
    611	} else if (req->bRequestType & USB_DIR_IN) {
    612		musb->ep0_state = MUSB_EP0_STAGE_TX;
    613		musb_writew(regs, MUSB_CSR0, MUSB_CSR0_P_SVDRXPKTRDY);
    614		while ((musb_readw(regs, MUSB_CSR0)
    615				& MUSB_CSR0_RXPKTRDY) != 0)
    616			cpu_relax();
    617		musb->ackpend = 0;
    618	} else
    619		musb->ep0_state = MUSB_EP0_STAGE_RX;
    620}
    621
    622static int
    623forward_to_driver(struct musb *musb, const struct usb_ctrlrequest *ctrlrequest)
    624__releases(musb->lock)
    625__acquires(musb->lock)
    626{
    627	int retval;
    628	if (!musb->gadget_driver)
    629		return -EOPNOTSUPP;
    630	spin_unlock(&musb->lock);
    631	retval = musb->gadget_driver->setup(&musb->g, ctrlrequest);
    632	spin_lock(&musb->lock);
    633	return retval;
    634}
    635
    636/*
    637 * Handle peripheral ep0 interrupt
    638 *
    639 * Context: irq handler; we won't re-enter the driver that way.
    640 */
    641irqreturn_t musb_g_ep0_irq(struct musb *musb)
    642{
    643	u16		csr;
    644	u16		len;
    645	void __iomem	*mbase = musb->mregs;
    646	void __iomem	*regs = musb->endpoints[0].regs;
    647	irqreturn_t	retval = IRQ_NONE;
    648
    649	musb_ep_select(mbase, 0);	/* select ep0 */
    650	csr = musb_readw(regs, MUSB_CSR0);
    651	len = musb_readb(regs, MUSB_COUNT0);
    652
    653	musb_dbg(musb, "csr %04x, count %d, ep0stage %s",
    654			csr, len, decode_ep0stage(musb->ep0_state));
    655
    656	if (csr & MUSB_CSR0_P_DATAEND) {
    657		/*
    658		 * If DATAEND is set we should not call the callback,
    659		 * hence the status stage is not complete.
    660		 */
    661		return IRQ_HANDLED;
    662	}
    663
    664	/* I sent a stall.. need to acknowledge it now.. */
    665	if (csr & MUSB_CSR0_P_SENTSTALL) {
    666		musb_writew(regs, MUSB_CSR0,
    667				csr & ~MUSB_CSR0_P_SENTSTALL);
    668		retval = IRQ_HANDLED;
    669		musb->ep0_state = MUSB_EP0_STAGE_IDLE;
    670		csr = musb_readw(regs, MUSB_CSR0);
    671	}
    672
    673	/* request ended "early" */
    674	if (csr & MUSB_CSR0_P_SETUPEND) {
    675		musb_writew(regs, MUSB_CSR0, MUSB_CSR0_P_SVDSETUPEND);
    676		retval = IRQ_HANDLED;
    677		/* Transition into the early status phase */
    678		switch (musb->ep0_state) {
    679		case MUSB_EP0_STAGE_TX:
    680			musb->ep0_state = MUSB_EP0_STAGE_STATUSOUT;
    681			break;
    682		case MUSB_EP0_STAGE_RX:
    683			musb->ep0_state = MUSB_EP0_STAGE_STATUSIN;
    684			break;
    685		default:
    686			ERR("SetupEnd came in a wrong ep0stage %s\n",
    687			    decode_ep0stage(musb->ep0_state));
    688		}
    689		csr = musb_readw(regs, MUSB_CSR0);
    690		/* NOTE:  request may need completion */
    691	}
    692
    693	/* docs from Mentor only describe tx, rx, and idle/setup states.
    694	 * we need to handle nuances around status stages, and also the
    695	 * case where status and setup stages come back-to-back ...
    696	 */
    697	switch (musb->ep0_state) {
    698
    699	case MUSB_EP0_STAGE_TX:
    700		/* irq on clearing txpktrdy */
    701		if ((csr & MUSB_CSR0_TXPKTRDY) == 0) {
    702			ep0_txstate(musb);
    703			retval = IRQ_HANDLED;
    704		}
    705		break;
    706
    707	case MUSB_EP0_STAGE_RX:
    708		/* irq on set rxpktrdy */
    709		if (csr & MUSB_CSR0_RXPKTRDY) {
    710			ep0_rxstate(musb);
    711			retval = IRQ_HANDLED;
    712		}
    713		break;
    714
    715	case MUSB_EP0_STAGE_STATUSIN:
    716		/* end of sequence #2 (OUT/RX state) or #3 (no data) */
    717
    718		/* update address (if needed) only @ the end of the
    719		 * status phase per usb spec, which also guarantees
    720		 * we get 10 msec to receive this irq... until this
    721		 * is done we won't see the next packet.
    722		 */
    723		if (musb->set_address) {
    724			musb->set_address = false;
    725			musb_writeb(mbase, MUSB_FADDR, musb->address);
    726		}
    727
    728		/* enter test mode if needed (exit by reset) */
    729		else if (musb->test_mode) {
    730			musb_dbg(musb, "entering TESTMODE");
    731
    732			if (MUSB_TEST_PACKET == musb->test_mode_nr)
    733				musb_load_testpacket(musb);
    734
    735			musb_writeb(mbase, MUSB_TESTMODE,
    736					musb->test_mode_nr);
    737		}
    738		fallthrough;
    739
    740	case MUSB_EP0_STAGE_STATUSOUT:
    741		/* end of sequence #1: write to host (TX state) */
    742		{
    743			struct musb_request	*req;
    744
    745			req = next_ep0_request(musb);
    746			if (req)
    747				musb_g_ep0_giveback(musb, &req->request);
    748		}
    749
    750		/*
    751		 * In case when several interrupts can get coalesced,
    752		 * check to see if we've already received a SETUP packet...
    753		 */
    754		if (csr & MUSB_CSR0_RXPKTRDY)
    755			goto setup;
    756
    757		retval = IRQ_HANDLED;
    758		musb->ep0_state = MUSB_EP0_STAGE_IDLE;
    759		break;
    760
    761	case MUSB_EP0_STAGE_IDLE:
    762		/*
    763		 * This state is typically (but not always) indiscernible
    764		 * from the status states since the corresponding interrupts
    765		 * tend to happen within too little period of time (with only
    766		 * a zero-length packet in between) and so get coalesced...
    767		 */
    768		retval = IRQ_HANDLED;
    769		musb->ep0_state = MUSB_EP0_STAGE_SETUP;
    770		fallthrough;
    771
    772	case MUSB_EP0_STAGE_SETUP:
    773setup:
    774		if (csr & MUSB_CSR0_RXPKTRDY) {
    775			struct usb_ctrlrequest	setup;
    776			int			handled = 0;
    777
    778			if (len != 8) {
    779				ERR("SETUP packet len %d != 8 ?\n", len);
    780				break;
    781			}
    782			musb_read_setup(musb, &setup);
    783			retval = IRQ_HANDLED;
    784
    785			/* sometimes the RESET won't be reported */
    786			if (unlikely(musb->g.speed == USB_SPEED_UNKNOWN)) {
    787				u8	power;
    788
    789				printk(KERN_NOTICE "%s: peripheral reset "
    790						"irq lost!\n",
    791						musb_driver_name);
    792				power = musb_readb(mbase, MUSB_POWER);
    793				musb->g.speed = (power & MUSB_POWER_HSMODE)
    794					? USB_SPEED_HIGH : USB_SPEED_FULL;
    795
    796			}
    797
    798			switch (musb->ep0_state) {
    799
    800			/* sequence #3 (no data stage), includes requests
    801			 * we can't forward (notably SET_ADDRESS and the
    802			 * device/endpoint feature set/clear operations)
    803			 * plus SET_CONFIGURATION and others we must
    804			 */
    805			case MUSB_EP0_STAGE_ACKWAIT:
    806				handled = service_zero_data_request(
    807						musb, &setup);
    808
    809				/*
    810				 * We're expecting no data in any case, so
    811				 * always set the DATAEND bit -- doing this
    812				 * here helps avoid SetupEnd interrupt coming
    813				 * in the idle stage when we're stalling...
    814				 */
    815				musb->ackpend |= MUSB_CSR0_P_DATAEND;
    816
    817				/* status stage might be immediate */
    818				if (handled > 0)
    819					musb->ep0_state =
    820						MUSB_EP0_STAGE_STATUSIN;
    821				break;
    822
    823			/* sequence #1 (IN to host), includes GET_STATUS
    824			 * requests that we can't forward, GET_DESCRIPTOR
    825			 * and others that we must
    826			 */
    827			case MUSB_EP0_STAGE_TX:
    828				handled = service_in_request(musb, &setup);
    829				if (handled > 0) {
    830					musb->ackpend = MUSB_CSR0_TXPKTRDY
    831						| MUSB_CSR0_P_DATAEND;
    832					musb->ep0_state =
    833						MUSB_EP0_STAGE_STATUSOUT;
    834				}
    835				break;
    836
    837			/* sequence #2 (OUT from host), always forward */
    838			default:		/* MUSB_EP0_STAGE_RX */
    839				break;
    840			}
    841
    842			musb_dbg(musb, "handled %d, csr %04x, ep0stage %s",
    843				handled, csr,
    844				decode_ep0stage(musb->ep0_state));
    845
    846			/* unless we need to delegate this to the gadget
    847			 * driver, we know how to wrap this up:  csr0 has
    848			 * not yet been written.
    849			 */
    850			if (handled < 0)
    851				goto stall;
    852			else if (handled > 0)
    853				goto finish;
    854
    855			handled = forward_to_driver(musb, &setup);
    856			if (handled < 0) {
    857				musb_ep_select(mbase, 0);
    858stall:
    859				musb_dbg(musb, "stall (%d)", handled);
    860				musb->ackpend |= MUSB_CSR0_P_SENDSTALL;
    861				musb->ep0_state = MUSB_EP0_STAGE_IDLE;
    862finish:
    863				musb_writew(regs, MUSB_CSR0,
    864						musb->ackpend);
    865				musb->ackpend = 0;
    866			}
    867		}
    868		break;
    869
    870	case MUSB_EP0_STAGE_ACKWAIT:
    871		/* This should not happen. But happens with tusb6010 with
    872		 * g_file_storage and high speed. Do nothing.
    873		 */
    874		retval = IRQ_HANDLED;
    875		break;
    876
    877	default:
    878		/* "can't happen" */
    879		WARN_ON(1);
    880		musb_writew(regs, MUSB_CSR0, MUSB_CSR0_P_SENDSTALL);
    881		musb->ep0_state = MUSB_EP0_STAGE_IDLE;
    882		break;
    883	}
    884
    885	return retval;
    886}
    887
    888
    889static int
    890musb_g_ep0_enable(struct usb_ep *ep, const struct usb_endpoint_descriptor *desc)
    891{
    892	/* always enabled */
    893	return -EINVAL;
    894}
    895
    896static int musb_g_ep0_disable(struct usb_ep *e)
    897{
    898	/* always enabled */
    899	return -EINVAL;
    900}
    901
    902static int
    903musb_g_ep0_queue(struct usb_ep *e, struct usb_request *r, gfp_t gfp_flags)
    904{
    905	struct musb_ep		*ep;
    906	struct musb_request	*req;
    907	struct musb		*musb;
    908	int			status;
    909	unsigned long		lockflags;
    910	void __iomem		*regs;
    911
    912	if (!e || !r)
    913		return -EINVAL;
    914
    915	ep = to_musb_ep(e);
    916	musb = ep->musb;
    917	regs = musb->control_ep->regs;
    918
    919	req = to_musb_request(r);
    920	req->musb = musb;
    921	req->request.actual = 0;
    922	req->request.status = -EINPROGRESS;
    923	req->tx = ep->is_in;
    924
    925	spin_lock_irqsave(&musb->lock, lockflags);
    926
    927	if (!list_empty(&ep->req_list)) {
    928		status = -EBUSY;
    929		goto cleanup;
    930	}
    931
    932	switch (musb->ep0_state) {
    933	case MUSB_EP0_STAGE_RX:		/* control-OUT data */
    934	case MUSB_EP0_STAGE_TX:		/* control-IN data */
    935	case MUSB_EP0_STAGE_ACKWAIT:	/* zero-length data */
    936		status = 0;
    937		break;
    938	default:
    939		musb_dbg(musb, "ep0 request queued in state %d",
    940				musb->ep0_state);
    941		status = -EINVAL;
    942		goto cleanup;
    943	}
    944
    945	/* add request to the list */
    946	list_add_tail(&req->list, &ep->req_list);
    947
    948	musb_dbg(musb, "queue to %s (%s), length=%d",
    949			ep->name, ep->is_in ? "IN/TX" : "OUT/RX",
    950			req->request.length);
    951
    952	musb_ep_select(musb->mregs, 0);
    953
    954	/* sequence #1, IN ... start writing the data */
    955	if (musb->ep0_state == MUSB_EP0_STAGE_TX)
    956		ep0_txstate(musb);
    957
    958	/* sequence #3, no-data ... issue IN status */
    959	else if (musb->ep0_state == MUSB_EP0_STAGE_ACKWAIT) {
    960		if (req->request.length)
    961			status = -EINVAL;
    962		else {
    963			musb->ep0_state = MUSB_EP0_STAGE_STATUSIN;
    964			musb_writew(regs, MUSB_CSR0,
    965					musb->ackpend | MUSB_CSR0_P_DATAEND);
    966			musb->ackpend = 0;
    967			musb_g_ep0_giveback(ep->musb, r);
    968		}
    969
    970	/* else for sequence #2 (OUT), caller provides a buffer
    971	 * before the next packet arrives.  deferred responses
    972	 * (after SETUP is acked) are racey.
    973	 */
    974	} else if (musb->ackpend) {
    975		musb_writew(regs, MUSB_CSR0, musb->ackpend);
    976		musb->ackpend = 0;
    977	}
    978
    979cleanup:
    980	spin_unlock_irqrestore(&musb->lock, lockflags);
    981	return status;
    982}
    983
    984static int musb_g_ep0_dequeue(struct usb_ep *ep, struct usb_request *req)
    985{
    986	/* we just won't support this */
    987	return -EINVAL;
    988}
    989
    990static int musb_g_ep0_halt(struct usb_ep *e, int value)
    991{
    992	struct musb_ep		*ep;
    993	struct musb		*musb;
    994	void __iomem		*base, *regs;
    995	unsigned long		flags;
    996	int			status;
    997	u16			csr;
    998
    999	if (!e || !value)
   1000		return -EINVAL;
   1001
   1002	ep = to_musb_ep(e);
   1003	musb = ep->musb;
   1004	base = musb->mregs;
   1005	regs = musb->control_ep->regs;
   1006	status = 0;
   1007
   1008	spin_lock_irqsave(&musb->lock, flags);
   1009
   1010	if (!list_empty(&ep->req_list)) {
   1011		status = -EBUSY;
   1012		goto cleanup;
   1013	}
   1014
   1015	musb_ep_select(base, 0);
   1016	csr = musb->ackpend;
   1017
   1018	switch (musb->ep0_state) {
   1019
   1020	/* Stalls are usually issued after parsing SETUP packet, either
   1021	 * directly in irq context from setup() or else later.
   1022	 */
   1023	case MUSB_EP0_STAGE_TX:		/* control-IN data */
   1024	case MUSB_EP0_STAGE_ACKWAIT:	/* STALL for zero-length data */
   1025	case MUSB_EP0_STAGE_RX:		/* control-OUT data */
   1026		csr = musb_readw(regs, MUSB_CSR0);
   1027		fallthrough;
   1028
   1029	/* It's also OK to issue stalls during callbacks when a non-empty
   1030	 * DATA stage buffer has been read (or even written).
   1031	 */
   1032	case MUSB_EP0_STAGE_STATUSIN:	/* control-OUT status */
   1033	case MUSB_EP0_STAGE_STATUSOUT:	/* control-IN status */
   1034
   1035		csr |= MUSB_CSR0_P_SENDSTALL;
   1036		musb_writew(regs, MUSB_CSR0, csr);
   1037		musb->ep0_state = MUSB_EP0_STAGE_IDLE;
   1038		musb->ackpend = 0;
   1039		break;
   1040	default:
   1041		musb_dbg(musb, "ep0 can't halt in state %d", musb->ep0_state);
   1042		status = -EINVAL;
   1043	}
   1044
   1045cleanup:
   1046	spin_unlock_irqrestore(&musb->lock, flags);
   1047	return status;
   1048}
   1049
   1050const struct usb_ep_ops musb_g_ep0_ops = {
   1051	.enable		= musb_g_ep0_enable,
   1052	.disable	= musb_g_ep0_disable,
   1053	.alloc_request	= musb_alloc_request,
   1054	.free_request	= musb_free_request,
   1055	.queue		= musb_g_ep0_queue,
   1056	.dequeue	= musb_g_ep0_dequeue,
   1057	.set_halt	= musb_g_ep0_halt,
   1058};