cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

tegra210.c (36589B)


      1// SPDX-License-Identifier: GPL-2.0-only
      2/*
      3 * Copyright (C) 2020 NVIDIA CORPORATION.  All rights reserved.
      4 */
      5
      6/*
      7 * This source file contains Tegra210 supported video formats,
      8 * VI and CSI SoC specific data, operations and registers accessors.
      9 */
     10#include <linux/bitfield.h>
     11#include <linux/clk.h>
     12#include <linux/clk/tegra.h>
     13#include <linux/delay.h>
     14#include <linux/host1x.h>
     15#include <linux/kthread.h>
     16
     17#include "csi.h"
     18#include "vi.h"
     19
     20#define TEGRA_VI_SYNCPT_WAIT_TIMEOUT			msecs_to_jiffies(200)
     21
     22/* Tegra210 VI registers */
     23#define TEGRA_VI_CFG_VI_INCR_SYNCPT			0x000
     24#define   VI_CFG_VI_INCR_SYNCPT_COND(x)			(((x) & 0xff) << 8)
     25#define   VI_CSI_PP_FRAME_START(port)			(5 + (port) * 4)
     26#define   VI_CSI_MW_ACK_DONE(port)			(7 + (port) * 4)
     27#define TEGRA_VI_CFG_VI_INCR_SYNCPT_CNTRL		0x004
     28#define   VI_INCR_SYNCPT_NO_STALL			BIT(8)
     29#define TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR		0x008
     30#define TEGRA_VI_CFG_CG_CTRL				0x0b8
     31#define   VI_CG_2ND_LEVEL_EN				0x1
     32
     33/* Tegra210 VI CSI registers */
     34#define TEGRA_VI_CSI_SW_RESET				0x000
     35#define TEGRA_VI_CSI_SINGLE_SHOT			0x004
     36#define   SINGLE_SHOT_CAPTURE				0x1
     37#define TEGRA_VI_CSI_IMAGE_DEF				0x00c
     38#define   BYPASS_PXL_TRANSFORM_OFFSET			24
     39#define   IMAGE_DEF_FORMAT_OFFSET			16
     40#define   IMAGE_DEF_DEST_MEM				0x1
     41#define TEGRA_VI_CSI_IMAGE_SIZE				0x018
     42#define   IMAGE_SIZE_HEIGHT_OFFSET			16
     43#define TEGRA_VI_CSI_IMAGE_SIZE_WC			0x01c
     44#define TEGRA_VI_CSI_IMAGE_DT				0x020
     45#define TEGRA_VI_CSI_SURFACE0_OFFSET_MSB		0x024
     46#define TEGRA_VI_CSI_SURFACE0_OFFSET_LSB		0x028
     47#define TEGRA_VI_CSI_SURFACE1_OFFSET_MSB		0x02c
     48#define TEGRA_VI_CSI_SURFACE1_OFFSET_LSB		0x030
     49#define TEGRA_VI_CSI_SURFACE2_OFFSET_MSB		0x034
     50#define TEGRA_VI_CSI_SURFACE2_OFFSET_LSB		0x038
     51#define TEGRA_VI_CSI_SURFACE0_STRIDE			0x054
     52#define TEGRA_VI_CSI_SURFACE1_STRIDE			0x058
     53#define TEGRA_VI_CSI_SURFACE2_STRIDE			0x05c
     54#define TEGRA_VI_CSI_SURFACE_HEIGHT0			0x060
     55#define TEGRA_VI_CSI_ERROR_STATUS			0x084
     56
     57/* Tegra210 CSI Pixel Parser registers: Starts from 0x838, offset 0x0 */
     58#define TEGRA_CSI_INPUT_STREAM_CONTROL                  0x000
     59#define   CSI_SKIP_PACKET_THRESHOLD_OFFSET		16
     60#define TEGRA_CSI_PIXEL_STREAM_CONTROL0			0x004
     61#define   CSI_PP_PACKET_HEADER_SENT			BIT(4)
     62#define   CSI_PP_DATA_IDENTIFIER_ENABLE			BIT(5)
     63#define   CSI_PP_WORD_COUNT_SELECT_HEADER		BIT(6)
     64#define   CSI_PP_CRC_CHECK_ENABLE			BIT(7)
     65#define   CSI_PP_WC_CHECK				BIT(8)
     66#define   CSI_PP_OUTPUT_FORMAT_STORE			(0x3 << 16)
     67#define   CSI_PPA_PAD_LINE_NOPAD			(0x2 << 24)
     68#define   CSI_PP_HEADER_EC_DISABLE			(0x1 << 27)
     69#define   CSI_PPA_PAD_FRAME_NOPAD			(0x2 << 28)
     70#define TEGRA_CSI_PIXEL_STREAM_CONTROL1                 0x008
     71#define   CSI_PP_TOP_FIELD_FRAME_OFFSET			0
     72#define   CSI_PP_TOP_FIELD_FRAME_MASK_OFFSET		4
     73#define TEGRA_CSI_PIXEL_STREAM_GAP                      0x00c
     74#define   PP_FRAME_MIN_GAP_OFFSET			16
     75#define TEGRA_CSI_PIXEL_STREAM_PP_COMMAND               0x010
     76#define   CSI_PP_ENABLE					0x1
     77#define   CSI_PP_DISABLE				0x2
     78#define   CSI_PP_RST					0x3
     79#define   CSI_PP_SINGLE_SHOT_ENABLE			(0x1 << 2)
     80#define   CSI_PP_START_MARKER_FRAME_MAX_OFFSET		12
     81#define TEGRA_CSI_PIXEL_STREAM_EXPECTED_FRAME           0x014
     82#define TEGRA_CSI_PIXEL_PARSER_INTERRUPT_MASK           0x018
     83#define TEGRA_CSI_PIXEL_PARSER_STATUS                   0x01c
     84
     85/* Tegra210 CSI PHY registers */
     86/* CSI_PHY_CIL_COMMAND_0 offset 0x0d0 from TEGRA_CSI_PIXEL_PARSER_0_BASE */
     87#define TEGRA_CSI_PHY_CIL_COMMAND                       0x0d0
     88#define   CSI_A_PHY_CIL_NOP				0x0
     89#define   CSI_A_PHY_CIL_ENABLE				0x1
     90#define   CSI_A_PHY_CIL_DISABLE				0x2
     91#define   CSI_A_PHY_CIL_ENABLE_MASK			0x3
     92#define   CSI_B_PHY_CIL_NOP				(0x0 << 8)
     93#define   CSI_B_PHY_CIL_ENABLE				(0x1 << 8)
     94#define   CSI_B_PHY_CIL_DISABLE				(0x2 << 8)
     95#define   CSI_B_PHY_CIL_ENABLE_MASK			(0x3 << 8)
     96
     97#define TEGRA_CSI_CIL_PAD_CONFIG0                       0x000
     98#define   BRICK_CLOCK_A_4X				(0x1 << 16)
     99#define   BRICK_CLOCK_B_4X				(0x2 << 16)
    100#define TEGRA_CSI_CIL_PAD_CONFIG1                       0x004
    101#define TEGRA_CSI_CIL_PHY_CONTROL                       0x008
    102#define   CLK_SETTLE_MASK				GENMASK(13, 8)
    103#define   THS_SETTLE_MASK				GENMASK(5, 0)
    104#define TEGRA_CSI_CIL_INTERRUPT_MASK                    0x00c
    105#define TEGRA_CSI_CIL_STATUS                            0x010
    106#define TEGRA_CSI_CILX_STATUS                           0x014
    107#define TEGRA_CSI_CIL_SW_SENSOR_RESET                   0x020
    108
    109#define TEGRA_CSI_PATTERN_GENERATOR_CTRL		0x000
    110#define   PG_MODE_OFFSET				2
    111#define   PG_ENABLE					0x1
    112#define   PG_DISABLE					0x0
    113#define TEGRA_CSI_PG_BLANK				0x004
    114#define   PG_VBLANK_OFFSET				16
    115#define TEGRA_CSI_PG_PHASE				0x008
    116#define TEGRA_CSI_PG_RED_FREQ				0x00c
    117#define   PG_RED_VERT_INIT_FREQ_OFFSET			16
    118#define   PG_RED_HOR_INIT_FREQ_OFFSET			0
    119#define TEGRA_CSI_PG_RED_FREQ_RATE			0x010
    120#define TEGRA_CSI_PG_GREEN_FREQ				0x014
    121#define   PG_GREEN_VERT_INIT_FREQ_OFFSET		16
    122#define   PG_GREEN_HOR_INIT_FREQ_OFFSET			0
    123#define TEGRA_CSI_PG_GREEN_FREQ_RATE			0x018
    124#define TEGRA_CSI_PG_BLUE_FREQ				0x01c
    125#define   PG_BLUE_VERT_INIT_FREQ_OFFSET			16
    126#define   PG_BLUE_HOR_INIT_FREQ_OFFSET			0
    127#define TEGRA_CSI_PG_BLUE_FREQ_RATE			0x020
    128#define TEGRA_CSI_PG_AOHDR				0x024
    129#define TEGRA_CSI_CSI_SW_STATUS_RESET			0x214
    130#define TEGRA_CSI_CLKEN_OVERRIDE			0x218
    131
    132#define TEGRA210_CSI_PORT_OFFSET			0x34
    133#define TEGRA210_CSI_CIL_OFFSET				0x0f4
    134#define TEGRA210_CSI_TPG_OFFSET				0x18c
    135
    136#define CSI_PP_OFFSET(block)				((block) * 0x800)
    137#define TEGRA210_VI_CSI_BASE(x)				(0x100 + (x) * 0x100)
    138
    139/* Tegra210 VI registers accessors */
    140static void tegra_vi_write(struct tegra_vi_channel *chan, unsigned int addr,
    141			   u32 val)
    142{
    143	writel_relaxed(val, chan->vi->iomem + addr);
    144}
    145
    146static u32 tegra_vi_read(struct tegra_vi_channel *chan, unsigned int addr)
    147{
    148	return readl_relaxed(chan->vi->iomem + addr);
    149}
    150
    151/* Tegra210 VI_CSI registers accessors */
    152static void vi_csi_write(struct tegra_vi_channel *chan, u8 portno,
    153			 unsigned int addr, u32 val)
    154{
    155	void __iomem *vi_csi_base;
    156
    157	vi_csi_base = chan->vi->iomem + TEGRA210_VI_CSI_BASE(portno);
    158
    159	writel_relaxed(val, vi_csi_base + addr);
    160}
    161
    162static u32 vi_csi_read(struct tegra_vi_channel *chan, u8 portno,
    163		       unsigned int addr)
    164{
    165	void __iomem *vi_csi_base;
    166
    167	vi_csi_base = chan->vi->iomem + TEGRA210_VI_CSI_BASE(portno);
    168
    169	return readl_relaxed(vi_csi_base + addr);
    170}
    171
    172/*
    173 * Tegra210 VI channel capture operations
    174 */
    175static int tegra_channel_capture_setup(struct tegra_vi_channel *chan,
    176				       u8 portno)
    177{
    178	u32 height = chan->format.height;
    179	u32 width = chan->format.width;
    180	u32 format = chan->fmtinfo->img_fmt;
    181	u32 data_type = chan->fmtinfo->img_dt;
    182	u32 word_count = (width * chan->fmtinfo->bit_width) / 8;
    183	u32 bypass_pixel_transform = BIT(BYPASS_PXL_TRANSFORM_OFFSET);
    184
    185	/*
    186	 * VI Pixel transformation unit converts source pixels data format
    187	 * into selected destination pixel format and aligns properly while
    188	 * interfacing with memory packer.
    189	 * This pixel transformation should be enabled for YUV and RGB
    190	 * formats and should be bypassed for RAW formats as RAW formats
    191	 * only support direct to memory.
    192	 */
    193	if (chan->pg_mode || data_type == TEGRA_IMAGE_DT_YUV422_8 ||
    194	    data_type == TEGRA_IMAGE_DT_RGB888)
    195		bypass_pixel_transform = 0;
    196
    197	/*
    198	 * For x8 source streaming, the source image is split onto two x4 ports
    199	 * with left half to first x4 port and right half to second x4 port.
    200	 * So, use split width and corresponding word count for each x4 port.
    201	 */
    202	if (chan->numgangports > 1) {
    203		width = width >> 1;
    204		word_count = (width * chan->fmtinfo->bit_width) / 8;
    205	}
    206
    207	vi_csi_write(chan, portno, TEGRA_VI_CSI_ERROR_STATUS, 0xffffffff);
    208	vi_csi_write(chan, portno, TEGRA_VI_CSI_IMAGE_DEF,
    209		     bypass_pixel_transform |
    210		     (format << IMAGE_DEF_FORMAT_OFFSET) |
    211		     IMAGE_DEF_DEST_MEM);
    212	vi_csi_write(chan, portno, TEGRA_VI_CSI_IMAGE_DT, data_type);
    213	vi_csi_write(chan, portno, TEGRA_VI_CSI_IMAGE_SIZE_WC, word_count);
    214	vi_csi_write(chan, portno, TEGRA_VI_CSI_IMAGE_SIZE,
    215		     (height << IMAGE_SIZE_HEIGHT_OFFSET) | width);
    216	return 0;
    217}
    218
    219static void tegra_channel_vi_soft_reset(struct tegra_vi_channel *chan,
    220					u8 portno)
    221{
    222	/* disable clock gating to enable continuous clock */
    223	tegra_vi_write(chan, TEGRA_VI_CFG_CG_CTRL, 0);
    224	/*
    225	 * Soft reset memory client interface, pixel format logic, sensor
    226	 * control logic, and a shadow copy logic to bring VI to clean state.
    227	 */
    228	vi_csi_write(chan, portno, TEGRA_VI_CSI_SW_RESET, 0xf);
    229	usleep_range(100, 200);
    230	vi_csi_write(chan, portno, TEGRA_VI_CSI_SW_RESET, 0x0);
    231
    232	/* enable back VI clock gating */
    233	tegra_vi_write(chan, TEGRA_VI_CFG_CG_CTRL, VI_CG_2ND_LEVEL_EN);
    234}
    235
    236static void tegra_channel_capture_error_recover(struct tegra_vi_channel *chan,
    237						u8 portno)
    238{
    239	struct v4l2_subdev *subdev;
    240	u32 val;
    241
    242	/*
    243	 * Recover VI and CSI hardware blocks in case of missing frame start
    244	 * events due to source not streaming or noisy csi inputs from the
    245	 * external source or many outstanding frame start or MW_ACK_DONE
    246	 * events which can cause CSI and VI hardware hang.
    247	 * This helps to have a clean capture for next frame.
    248	 */
    249	val = vi_csi_read(chan, portno, TEGRA_VI_CSI_ERROR_STATUS);
    250	dev_dbg(&chan->video.dev, "TEGRA_VI_CSI_ERROR_STATUS 0x%08x\n", val);
    251	vi_csi_write(chan, portno, TEGRA_VI_CSI_ERROR_STATUS, val);
    252
    253	val = tegra_vi_read(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR);
    254	dev_dbg(&chan->video.dev,
    255		"TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR 0x%08x\n", val);
    256	tegra_vi_write(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR, val);
    257
    258	/* recover VI by issuing software reset and re-setup for capture */
    259	tegra_channel_vi_soft_reset(chan, portno);
    260	tegra_channel_capture_setup(chan, portno);
    261
    262	/* recover CSI block */
    263	subdev = tegra_channel_get_remote_csi_subdev(chan);
    264	tegra_csi_error_recover(subdev);
    265}
    266
    267static struct tegra_channel_buffer *
    268dequeue_buf_done(struct tegra_vi_channel *chan)
    269{
    270	struct tegra_channel_buffer *buf = NULL;
    271
    272	spin_lock(&chan->done_lock);
    273	if (list_empty(&chan->done)) {
    274		spin_unlock(&chan->done_lock);
    275		return NULL;
    276	}
    277
    278	buf = list_first_entry(&chan->done,
    279			       struct tegra_channel_buffer, queue);
    280	if (buf)
    281		list_del_init(&buf->queue);
    282	spin_unlock(&chan->done_lock);
    283
    284	return buf;
    285}
    286
    287static void release_buffer(struct tegra_vi_channel *chan,
    288			   struct tegra_channel_buffer *buf,
    289			   enum vb2_buffer_state state)
    290{
    291	struct vb2_v4l2_buffer *vb = &buf->buf;
    292
    293	vb->sequence = chan->sequence++;
    294	vb->field = V4L2_FIELD_NONE;
    295	vb->vb2_buf.timestamp = ktime_get_ns();
    296	vb2_buffer_done(&vb->vb2_buf, state);
    297}
    298
    299static void tegra_channel_vi_buffer_setup(struct tegra_vi_channel *chan,
    300					  u8 portno, u32 buf_offset,
    301					  struct tegra_channel_buffer *buf)
    302{
    303	int bytesperline = chan->format.bytesperline;
    304	u32 sizeimage = chan->format.sizeimage;
    305
    306	/* program buffer address by using surface 0 */
    307	vi_csi_write(chan, portno, TEGRA_VI_CSI_SURFACE0_OFFSET_MSB,
    308		     ((u64)buf->addr + buf_offset) >> 32);
    309	vi_csi_write(chan, portno, TEGRA_VI_CSI_SURFACE0_OFFSET_LSB,
    310		     buf->addr + buf_offset);
    311	vi_csi_write(chan, portno, TEGRA_VI_CSI_SURFACE0_STRIDE, bytesperline);
    312
    313	if (chan->fmtinfo->fourcc != V4L2_PIX_FMT_NV16)
    314		return;
    315	/*
    316	 * Program surface 1 for UV plane with offset sizeimage from Y plane.
    317	 */
    318	vi_csi_write(chan, portno, TEGRA_VI_CSI_SURFACE1_OFFSET_MSB,
    319		     (((u64)buf->addr + sizeimage / 2) + buf_offset) >> 32);
    320	vi_csi_write(chan, portno, TEGRA_VI_CSI_SURFACE1_OFFSET_LSB,
    321		     buf->addr + sizeimage / 2 + buf_offset);
    322	vi_csi_write(chan, portno, TEGRA_VI_CSI_SURFACE1_STRIDE, bytesperline);
    323}
    324
    325static int tegra_channel_capture_frame(struct tegra_vi_channel *chan,
    326				       struct tegra_channel_buffer *buf)
    327{
    328	u32 thresh, value, frame_start, mw_ack_done;
    329	u32 fs_thresh[GANG_PORTS_MAX];
    330	u8 *portnos = chan->portnos;
    331	int gang_bpl = (chan->format.width >> 1) * chan->fmtinfo->bpp;
    332	u32 buf_offset;
    333	bool capture_timedout = false;
    334	int err, i;
    335
    336	for (i = 0; i < chan->numgangports; i++) {
    337		/*
    338		 * Align buffers side-by-side for all consecutive x4 ports
    339		 * in gang ports using bytes per line based on source split
    340		 * width.
    341		 */
    342		buf_offset = i * roundup(gang_bpl, SURFACE_ALIGN_BYTES);
    343		tegra_channel_vi_buffer_setup(chan, portnos[i], buf_offset,
    344					      buf);
    345
    346		/*
    347		 * Tegra VI block interacts with host1x syncpt to synchronize
    348		 * programmed condition and hardware operation for capture.
    349		 * Frame start and Memory write acknowledge syncpts has their
    350		 * own FIFO of depth 2.
    351		 *
    352		 * Syncpoint trigger conditions set through VI_INCR_SYNCPT
    353		 * register are added to HW syncpt FIFO and when HW triggers,
    354		 * syncpt condition is removed from the FIFO and counter at
    355		 * syncpoint index will be incremented by the hardware and
    356		 * software can wait for counter to reach threshold to
    357		 * synchronize capturing frame with hardware capture events.
    358		 */
    359
    360		/* increase channel syncpoint threshold for FRAME_START */
    361		thresh = host1x_syncpt_incr_max(chan->frame_start_sp[i], 1);
    362		fs_thresh[i] = thresh;
    363
    364		/* Program FRAME_START trigger condition syncpt request */
    365		frame_start = VI_CSI_PP_FRAME_START(portnos[i]);
    366		value = VI_CFG_VI_INCR_SYNCPT_COND(frame_start) |
    367			host1x_syncpt_id(chan->frame_start_sp[i]);
    368		tegra_vi_write(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT, value);
    369
    370		/* increase channel syncpoint threshold for MW_ACK_DONE */
    371		thresh = host1x_syncpt_incr_max(chan->mw_ack_sp[i], 1);
    372		buf->mw_ack_sp_thresh[i] = thresh;
    373
    374		/* Program MW_ACK_DONE trigger condition syncpt request */
    375		mw_ack_done = VI_CSI_MW_ACK_DONE(portnos[i]);
    376		value = VI_CFG_VI_INCR_SYNCPT_COND(mw_ack_done) |
    377			host1x_syncpt_id(chan->mw_ack_sp[i]);
    378		tegra_vi_write(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT, value);
    379	}
    380
    381	/* enable single shot capture after all ganged ports are ready */
    382	for (i = 0; i < chan->numgangports; i++)
    383		vi_csi_write(chan, portnos[i], TEGRA_VI_CSI_SINGLE_SHOT,
    384			     SINGLE_SHOT_CAPTURE);
    385
    386	for (i = 0; i < chan->numgangports; i++) {
    387		/*
    388		 * Wait for syncpt counter to reach frame start event threshold
    389		 */
    390		err = host1x_syncpt_wait(chan->frame_start_sp[i], fs_thresh[i],
    391					 TEGRA_VI_SYNCPT_WAIT_TIMEOUT, &value);
    392		if (err) {
    393			capture_timedout = true;
    394			/* increment syncpoint counter for timedout events */
    395			host1x_syncpt_incr(chan->frame_start_sp[i]);
    396			spin_lock(&chan->sp_incr_lock[i]);
    397			host1x_syncpt_incr(chan->mw_ack_sp[i]);
    398			spin_unlock(&chan->sp_incr_lock[i]);
    399			/* clear errors and recover */
    400			tegra_channel_capture_error_recover(chan, portnos[i]);
    401		}
    402	}
    403
    404	if (capture_timedout) {
    405		dev_err_ratelimited(&chan->video.dev,
    406				    "frame start syncpt timeout: %d\n", err);
    407		release_buffer(chan, buf, VB2_BUF_STATE_ERROR);
    408		return err;
    409	}
    410
    411	/* move buffer to capture done queue */
    412	spin_lock(&chan->done_lock);
    413	list_add_tail(&buf->queue, &chan->done);
    414	spin_unlock(&chan->done_lock);
    415
    416	/* wait up kthread for capture done */
    417	wake_up_interruptible(&chan->done_wait);
    418
    419	return 0;
    420}
    421
    422static void tegra_channel_capture_done(struct tegra_vi_channel *chan,
    423				       struct tegra_channel_buffer *buf)
    424{
    425	enum vb2_buffer_state state = VB2_BUF_STATE_DONE;
    426	u32 value;
    427	bool capture_timedout = false;
    428	int ret, i;
    429
    430	for (i = 0; i < chan->numgangports; i++) {
    431		/*
    432		 * Wait for syncpt counter to reach MW_ACK_DONE event threshold
    433		 */
    434		ret = host1x_syncpt_wait(chan->mw_ack_sp[i],
    435					 buf->mw_ack_sp_thresh[i],
    436					 TEGRA_VI_SYNCPT_WAIT_TIMEOUT, &value);
    437		if (ret) {
    438			capture_timedout = true;
    439			state = VB2_BUF_STATE_ERROR;
    440			/* increment syncpoint counter for timedout event */
    441			spin_lock(&chan->sp_incr_lock[i]);
    442			host1x_syncpt_incr(chan->mw_ack_sp[i]);
    443			spin_unlock(&chan->sp_incr_lock[i]);
    444		}
    445	}
    446
    447	if (capture_timedout)
    448		dev_err_ratelimited(&chan->video.dev,
    449				    "MW_ACK_DONE syncpt timeout: %d\n", ret);
    450	release_buffer(chan, buf, state);
    451}
    452
    453static int chan_capture_kthread_start(void *data)
    454{
    455	struct tegra_vi_channel *chan = data;
    456	struct tegra_channel_buffer *buf;
    457	unsigned int retries = 0;
    458	int err = 0;
    459
    460	while (1) {
    461		/*
    462		 * Source is not streaming if error is non-zero.
    463		 * So, do not dequeue buffers on error and let the thread sleep
    464		 * till kthread stop signal is received.
    465		 */
    466		wait_event_interruptible(chan->start_wait,
    467					 kthread_should_stop() ||
    468					 (!list_empty(&chan->capture) &&
    469					 !err));
    470
    471		if (kthread_should_stop())
    472			break;
    473
    474		/* dequeue the buffer and start capture */
    475		spin_lock(&chan->start_lock);
    476		if (list_empty(&chan->capture)) {
    477			spin_unlock(&chan->start_lock);
    478			continue;
    479		}
    480
    481		buf = list_first_entry(&chan->capture,
    482				       struct tegra_channel_buffer, queue);
    483		list_del_init(&buf->queue);
    484		spin_unlock(&chan->start_lock);
    485
    486		err = tegra_channel_capture_frame(chan, buf);
    487		if (!err) {
    488			retries = 0;
    489			continue;
    490		}
    491
    492		if (retries++ > chan->syncpt_timeout_retry)
    493			vb2_queue_error(&chan->queue);
    494		else
    495			err = 0;
    496	}
    497
    498	return 0;
    499}
    500
    501static int chan_capture_kthread_finish(void *data)
    502{
    503	struct tegra_vi_channel *chan = data;
    504	struct tegra_channel_buffer *buf;
    505
    506	while (1) {
    507		wait_event_interruptible(chan->done_wait,
    508					 !list_empty(&chan->done) ||
    509					 kthread_should_stop());
    510
    511		/* dequeue buffers and finish capture */
    512		buf = dequeue_buf_done(chan);
    513		while (buf) {
    514			tegra_channel_capture_done(chan, buf);
    515			buf = dequeue_buf_done(chan);
    516		}
    517
    518		if (kthread_should_stop())
    519			break;
    520	}
    521
    522	return 0;
    523}
    524
    525static int tegra210_vi_start_streaming(struct vb2_queue *vq, u32 count)
    526{
    527	struct tegra_vi_channel *chan = vb2_get_drv_priv(vq);
    528	struct media_pipeline *pipe = &chan->video.pipe;
    529	u32 val;
    530	u8 *portnos = chan->portnos;
    531	int ret, i;
    532
    533	tegra_vi_write(chan, TEGRA_VI_CFG_CG_CTRL, VI_CG_2ND_LEVEL_EN);
    534
    535	/* clear syncpt errors */
    536	val = tegra_vi_read(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR);
    537	tegra_vi_write(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR, val);
    538
    539	/*
    540	 * Sync point FIFO full stalls the host interface.
    541	 * Setting NO_STALL will drop INCR_SYNCPT methods when fifos are
    542	 * full and the corresponding condition bits in INCR_SYNCPT_ERROR
    543	 * register will be set.
    544	 * This allows SW to process error recovery.
    545	 */
    546	tegra_vi_write(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT_CNTRL,
    547		       VI_INCR_SYNCPT_NO_STALL);
    548
    549	/* start the pipeline */
    550	ret = media_pipeline_start(&chan->video.entity, pipe);
    551	if (ret < 0)
    552		goto error_pipeline_start;
    553
    554	/* clear csi errors and do capture setup for all ports in gang mode */
    555	for (i = 0; i < chan->numgangports; i++) {
    556		val = vi_csi_read(chan, portnos[i], TEGRA_VI_CSI_ERROR_STATUS);
    557		vi_csi_write(chan, portnos[i], TEGRA_VI_CSI_ERROR_STATUS, val);
    558
    559		tegra_channel_capture_setup(chan, portnos[i]);
    560	}
    561
    562	ret = tegra_channel_set_stream(chan, true);
    563	if (ret < 0)
    564		goto error_set_stream;
    565
    566	chan->sequence = 0;
    567
    568	/* start kthreads to capture data to buffer and return them */
    569	chan->kthread_start_capture = kthread_run(chan_capture_kthread_start,
    570						  chan, "%s:0",
    571						  chan->video.name);
    572	if (IS_ERR(chan->kthread_start_capture)) {
    573		ret = PTR_ERR(chan->kthread_start_capture);
    574		chan->kthread_start_capture = NULL;
    575		dev_err(&chan->video.dev,
    576			"failed to run capture start kthread: %d\n", ret);
    577		goto error_kthread_start;
    578	}
    579
    580	chan->kthread_finish_capture = kthread_run(chan_capture_kthread_finish,
    581						   chan, "%s:1",
    582						   chan->video.name);
    583	if (IS_ERR(chan->kthread_finish_capture)) {
    584		ret = PTR_ERR(chan->kthread_finish_capture);
    585		chan->kthread_finish_capture = NULL;
    586		dev_err(&chan->video.dev,
    587			"failed to run capture finish kthread: %d\n", ret);
    588		goto error_kthread_done;
    589	}
    590
    591	return 0;
    592
    593error_kthread_done:
    594	kthread_stop(chan->kthread_start_capture);
    595error_kthread_start:
    596	tegra_channel_set_stream(chan, false);
    597error_set_stream:
    598	media_pipeline_stop(&chan->video.entity);
    599error_pipeline_start:
    600	tegra_channel_release_buffers(chan, VB2_BUF_STATE_QUEUED);
    601	return ret;
    602}
    603
    604static void tegra210_vi_stop_streaming(struct vb2_queue *vq)
    605{
    606	struct tegra_vi_channel *chan = vb2_get_drv_priv(vq);
    607
    608	if (chan->kthread_start_capture) {
    609		kthread_stop(chan->kthread_start_capture);
    610		chan->kthread_start_capture = NULL;
    611	}
    612
    613	if (chan->kthread_finish_capture) {
    614		kthread_stop(chan->kthread_finish_capture);
    615		chan->kthread_finish_capture = NULL;
    616	}
    617
    618	tegra_channel_release_buffers(chan, VB2_BUF_STATE_ERROR);
    619	tegra_channel_set_stream(chan, false);
    620	media_pipeline_stop(&chan->video.entity);
    621}
    622
    623/*
    624 * Tegra210 VI Pixel memory format enum.
    625 * These format enum value gets programmed into corresponding Tegra VI
    626 * channel register bits.
    627 */
    628enum tegra210_image_format {
    629	TEGRA210_IMAGE_FORMAT_T_L8 = 16,
    630
    631	TEGRA210_IMAGE_FORMAT_T_R16_I = 32,
    632	TEGRA210_IMAGE_FORMAT_T_B5G6R5,
    633	TEGRA210_IMAGE_FORMAT_T_R5G6B5,
    634	TEGRA210_IMAGE_FORMAT_T_A1B5G5R5,
    635	TEGRA210_IMAGE_FORMAT_T_A1R5G5B5,
    636	TEGRA210_IMAGE_FORMAT_T_B5G5R5A1,
    637	TEGRA210_IMAGE_FORMAT_T_R5G5B5A1,
    638	TEGRA210_IMAGE_FORMAT_T_A4B4G4R4,
    639	TEGRA210_IMAGE_FORMAT_T_A4R4G4B4,
    640	TEGRA210_IMAGE_FORMAT_T_B4G4R4A4,
    641	TEGRA210_IMAGE_FORMAT_T_R4G4B4A4,
    642
    643	TEGRA210_IMAGE_FORMAT_T_A8B8G8R8 = 64,
    644	TEGRA210_IMAGE_FORMAT_T_A8R8G8B8,
    645	TEGRA210_IMAGE_FORMAT_T_B8G8R8A8,
    646	TEGRA210_IMAGE_FORMAT_T_R8G8B8A8,
    647	TEGRA210_IMAGE_FORMAT_T_A2B10G10R10,
    648	TEGRA210_IMAGE_FORMAT_T_A2R10G10B10,
    649	TEGRA210_IMAGE_FORMAT_T_B10G10R10A2,
    650	TEGRA210_IMAGE_FORMAT_T_R10G10B10A2,
    651
    652	TEGRA210_IMAGE_FORMAT_T_A8Y8U8V8 = 193,
    653	TEGRA210_IMAGE_FORMAT_T_V8U8Y8A8,
    654
    655	TEGRA210_IMAGE_FORMAT_T_A2Y10U10V10 = 197,
    656	TEGRA210_IMAGE_FORMAT_T_V10U10Y10A2,
    657	TEGRA210_IMAGE_FORMAT_T_Y8_U8__Y8_V8,
    658	TEGRA210_IMAGE_FORMAT_T_Y8_V8__Y8_U8,
    659	TEGRA210_IMAGE_FORMAT_T_U8_Y8__V8_Y8,
    660	TEGRA210_IMAGE_FORMAT_T_V8_Y8__U8_Y8,
    661
    662	TEGRA210_IMAGE_FORMAT_T_Y8__U8__V8_N444 = 224,
    663	TEGRA210_IMAGE_FORMAT_T_Y8__U8V8_N444,
    664	TEGRA210_IMAGE_FORMAT_T_Y8__V8U8_N444,
    665	TEGRA210_IMAGE_FORMAT_T_Y8__U8__V8_N422,
    666	TEGRA210_IMAGE_FORMAT_T_Y8__U8V8_N422,
    667	TEGRA210_IMAGE_FORMAT_T_Y8__V8U8_N422,
    668	TEGRA210_IMAGE_FORMAT_T_Y8__U8__V8_N420,
    669	TEGRA210_IMAGE_FORMAT_T_Y8__U8V8_N420,
    670	TEGRA210_IMAGE_FORMAT_T_Y8__V8U8_N420,
    671	TEGRA210_IMAGE_FORMAT_T_X2LC10LB10LA10,
    672	TEGRA210_IMAGE_FORMAT_T_A2R6R6R6R6R6,
    673};
    674
    675#define TEGRA210_VIDEO_FMT(DATA_TYPE, BIT_WIDTH, MBUS_CODE, BPP,	\
    676			   FORMAT, FOURCC)				\
    677{									\
    678	TEGRA_IMAGE_DT_##DATA_TYPE,					\
    679	BIT_WIDTH,							\
    680	MEDIA_BUS_FMT_##MBUS_CODE,					\
    681	BPP,								\
    682	TEGRA210_IMAGE_FORMAT_##FORMAT,					\
    683	V4L2_PIX_FMT_##FOURCC,						\
    684}
    685
    686/* Tegra210 supported video formats */
    687static const struct tegra_video_format tegra210_video_formats[] = {
    688	/* RAW 8 */
    689	TEGRA210_VIDEO_FMT(RAW8, 8, SRGGB8_1X8, 1, T_L8, SRGGB8),
    690	TEGRA210_VIDEO_FMT(RAW8, 8, SGRBG8_1X8, 1, T_L8, SGRBG8),
    691	TEGRA210_VIDEO_FMT(RAW8, 8, SGBRG8_1X8, 1, T_L8, SGBRG8),
    692	TEGRA210_VIDEO_FMT(RAW8, 8, SBGGR8_1X8, 1, T_L8, SBGGR8),
    693	/* RAW 10 */
    694	TEGRA210_VIDEO_FMT(RAW10, 10, SRGGB10_1X10, 2, T_R16_I, SRGGB10),
    695	TEGRA210_VIDEO_FMT(RAW10, 10, SGRBG10_1X10, 2, T_R16_I, SGRBG10),
    696	TEGRA210_VIDEO_FMT(RAW10, 10, SGBRG10_1X10, 2, T_R16_I, SGBRG10),
    697	TEGRA210_VIDEO_FMT(RAW10, 10, SBGGR10_1X10, 2, T_R16_I, SBGGR10),
    698	/* RAW 12 */
    699	TEGRA210_VIDEO_FMT(RAW12, 12, SRGGB12_1X12, 2, T_R16_I, SRGGB12),
    700	TEGRA210_VIDEO_FMT(RAW12, 12, SGRBG12_1X12, 2, T_R16_I, SGRBG12),
    701	TEGRA210_VIDEO_FMT(RAW12, 12, SGBRG12_1X12, 2, T_R16_I, SGBRG12),
    702	TEGRA210_VIDEO_FMT(RAW12, 12, SBGGR12_1X12, 2, T_R16_I, SBGGR12),
    703	/* RGB888 */
    704	TEGRA210_VIDEO_FMT(RGB888, 24, RGB888_1X24, 4, T_A8R8G8B8, XBGR32),
    705	TEGRA210_VIDEO_FMT(RGB888, 24, RGB888_1X32_PADHI, 4, T_A8B8G8R8,
    706			   RGBX32),
    707	/* YUV422 */
    708	TEGRA210_VIDEO_FMT(YUV422_8, 16, UYVY8_1X16, 2, T_U8_Y8__V8_Y8, YVYU),
    709	TEGRA210_VIDEO_FMT(YUV422_8, 16, VYUY8_1X16, 2, T_V8_Y8__U8_Y8, YUYV),
    710	TEGRA210_VIDEO_FMT(YUV422_8, 16, YUYV8_1X16, 2, T_Y8_U8__Y8_V8, VYUY),
    711	TEGRA210_VIDEO_FMT(YUV422_8, 16, YVYU8_1X16, 2, T_Y8_V8__Y8_U8, UYVY),
    712	TEGRA210_VIDEO_FMT(YUV422_8, 16, UYVY8_1X16, 1, T_Y8__V8U8_N422, NV16),
    713	TEGRA210_VIDEO_FMT(YUV422_8, 16, UYVY8_2X8, 2, T_U8_Y8__V8_Y8, YVYU),
    714	TEGRA210_VIDEO_FMT(YUV422_8, 16, VYUY8_2X8, 2, T_V8_Y8__U8_Y8, YUYV),
    715	TEGRA210_VIDEO_FMT(YUV422_8, 16, YUYV8_2X8, 2, T_Y8_U8__Y8_V8, VYUY),
    716	TEGRA210_VIDEO_FMT(YUV422_8, 16, YVYU8_2X8, 2, T_Y8_V8__Y8_U8, UYVY),
    717};
    718
    719/* Tegra210 VI operations */
    720static const struct tegra_vi_ops tegra210_vi_ops = {
    721	.vi_start_streaming = tegra210_vi_start_streaming,
    722	.vi_stop_streaming = tegra210_vi_stop_streaming,
    723};
    724
    725/* Tegra210 VI SoC data */
    726const struct tegra_vi_soc tegra210_vi_soc = {
    727	.video_formats = tegra210_video_formats,
    728	.nformats = ARRAY_SIZE(tegra210_video_formats),
    729	.ops = &tegra210_vi_ops,
    730	.hw_revision = 3,
    731	.vi_max_channels = 6,
    732#if IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG)
    733	.vi_max_clk_hz = 499200000,
    734#else
    735	.vi_max_clk_hz = 998400000,
    736#endif
    737};
    738
    739/* Tegra210 CSI PHY registers accessors */
    740static void csi_write(struct tegra_csi *csi, u8 portno, unsigned int addr,
    741		      u32 val)
    742{
    743	void __iomem *csi_pp_base;
    744
    745	csi_pp_base = csi->iomem + CSI_PP_OFFSET(portno >> 1);
    746
    747	writel_relaxed(val, csi_pp_base + addr);
    748}
    749
    750/* Tegra210 CSI Pixel parser registers accessors */
    751static void pp_write(struct tegra_csi *csi, u8 portno, u32 addr, u32 val)
    752{
    753	void __iomem *csi_pp_base;
    754	unsigned int offset;
    755
    756	csi_pp_base = csi->iomem + CSI_PP_OFFSET(portno >> 1);
    757	offset = (portno % CSI_PORTS_PER_BRICK) * TEGRA210_CSI_PORT_OFFSET;
    758
    759	writel_relaxed(val, csi_pp_base + offset + addr);
    760}
    761
    762static u32 pp_read(struct tegra_csi *csi, u8 portno, u32 addr)
    763{
    764	void __iomem *csi_pp_base;
    765	unsigned int offset;
    766
    767	csi_pp_base = csi->iomem + CSI_PP_OFFSET(portno >> 1);
    768	offset = (portno % CSI_PORTS_PER_BRICK) * TEGRA210_CSI_PORT_OFFSET;
    769
    770	return readl_relaxed(csi_pp_base + offset + addr);
    771}
    772
    773/* Tegra210 CSI CIL A/B port registers accessors */
    774static void cil_write(struct tegra_csi *csi, u8 portno, u32 addr, u32 val)
    775{
    776	void __iomem *csi_cil_base;
    777	unsigned int offset;
    778
    779	csi_cil_base = csi->iomem + CSI_PP_OFFSET(portno >> 1) +
    780		       TEGRA210_CSI_CIL_OFFSET;
    781	offset = (portno % CSI_PORTS_PER_BRICK) * TEGRA210_CSI_PORT_OFFSET;
    782
    783	writel_relaxed(val, csi_cil_base + offset + addr);
    784}
    785
    786static u32 cil_read(struct tegra_csi *csi, u8 portno, u32 addr)
    787{
    788	void __iomem *csi_cil_base;
    789	unsigned int offset;
    790
    791	csi_cil_base = csi->iomem + CSI_PP_OFFSET(portno >> 1) +
    792		       TEGRA210_CSI_CIL_OFFSET;
    793	offset = (portno % CSI_PORTS_PER_BRICK) * TEGRA210_CSI_PORT_OFFSET;
    794
    795	return readl_relaxed(csi_cil_base + offset + addr);
    796}
    797
    798/* Tegra210 CSI Test pattern generator registers accessor */
    799static void tpg_write(struct tegra_csi *csi, u8 portno, unsigned int addr,
    800		      u32 val)
    801{
    802	void __iomem *csi_pp_base;
    803	unsigned int offset;
    804
    805	csi_pp_base = csi->iomem + CSI_PP_OFFSET(portno >> 1);
    806	offset = (portno % CSI_PORTS_PER_BRICK) * TEGRA210_CSI_PORT_OFFSET +
    807		 TEGRA210_CSI_TPG_OFFSET;
    808
    809	writel_relaxed(val, csi_pp_base + offset + addr);
    810}
    811
    812/*
    813 * Tegra210 CSI operations
    814 */
    815static void tegra210_csi_port_recover(struct tegra_csi_channel *csi_chan,
    816				      u8 portno)
    817{
    818	struct tegra_csi *csi = csi_chan->csi;
    819	u32 val;
    820
    821	/*
    822	 * Recover CSI hardware in case of capture errors by issuing
    823	 * software reset to CSICIL sensor, pixel parser, and clear errors
    824	 * to have clean capture on  next streaming.
    825	 */
    826	val = pp_read(csi, portno, TEGRA_CSI_PIXEL_PARSER_STATUS);
    827	dev_dbg(csi->dev, "TEGRA_CSI_PIXEL_PARSER_STATUS 0x%08x\n", val);
    828
    829	val = cil_read(csi, portno, TEGRA_CSI_CIL_STATUS);
    830	dev_dbg(csi->dev, "TEGRA_CSI_CIL_STATUS 0x%08x\n", val);
    831
    832	val = cil_read(csi, portno, TEGRA_CSI_CILX_STATUS);
    833	dev_dbg(csi->dev, "TEGRA_CSI_CILX_STATUS 0x%08x\n", val);
    834
    835	if (csi_chan->numlanes == 4) {
    836		/* reset CSI CIL sensor */
    837		cil_write(csi, portno, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x1);
    838		cil_write(csi, portno + 1, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x1);
    839		/*
    840		 * SW_STATUS_RESET resets all status bits of PPA, PPB, CILA,
    841		 * CILB status registers and debug counters.
    842		 * So, SW_STATUS_RESET can be used only when CSI brick is in
    843		 * x4 mode.
    844		 */
    845		csi_write(csi, portno, TEGRA_CSI_CSI_SW_STATUS_RESET, 0x1);
    846
    847		/* sleep for 20 clock cycles to drain the FIFO */
    848		usleep_range(10, 20);
    849
    850		cil_write(csi, portno + 1, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x0);
    851		cil_write(csi, portno, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x0);
    852		csi_write(csi, portno, TEGRA_CSI_CSI_SW_STATUS_RESET, 0x0);
    853	} else {
    854		/* reset CSICIL sensor */
    855		cil_write(csi, portno, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x1);
    856		usleep_range(10, 20);
    857		cil_write(csi, portno, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x0);
    858
    859		/* clear the errors */
    860		pp_write(csi, portno, TEGRA_CSI_PIXEL_PARSER_STATUS,
    861			 0xffffffff);
    862		cil_write(csi, portno, TEGRA_CSI_CIL_STATUS, 0xffffffff);
    863		cil_write(csi, portno, TEGRA_CSI_CILX_STATUS, 0xffffffff);
    864	}
    865}
    866
    867static void tegra210_csi_error_recover(struct tegra_csi_channel *csi_chan)
    868{
    869	u8 *portnos = csi_chan->csi_port_nums;
    870	int i;
    871
    872	for (i = 0; i < csi_chan->numgangports; i++)
    873		tegra210_csi_port_recover(csi_chan, portnos[i]);
    874}
    875
    876static int
    877tegra210_csi_port_start_streaming(struct tegra_csi_channel *csi_chan,
    878				  u8 portno)
    879{
    880	struct tegra_csi *csi = csi_chan->csi;
    881	u8 clk_settle_time = 0;
    882	u8 ths_settle_time = 10;
    883	u32 val;
    884
    885	if (!csi_chan->pg_mode)
    886		tegra_csi_calc_settle_time(csi_chan, portno, &clk_settle_time,
    887					   &ths_settle_time);
    888
    889	csi_write(csi, portno, TEGRA_CSI_CLKEN_OVERRIDE, 0);
    890
    891	/* clean up status */
    892	pp_write(csi, portno, TEGRA_CSI_PIXEL_PARSER_STATUS, 0xffffffff);
    893	cil_write(csi, portno, TEGRA_CSI_CIL_STATUS, 0xffffffff);
    894	cil_write(csi, portno, TEGRA_CSI_CILX_STATUS, 0xffffffff);
    895	cil_write(csi, portno, TEGRA_CSI_CIL_INTERRUPT_MASK, 0x0);
    896
    897	/* CIL PHY registers setup */
    898	cil_write(csi, portno, TEGRA_CSI_CIL_PAD_CONFIG0, 0x0);
    899	cil_write(csi, portno, TEGRA_CSI_CIL_PHY_CONTROL,
    900		  FIELD_PREP(CLK_SETTLE_MASK, clk_settle_time) |
    901		  FIELD_PREP(THS_SETTLE_MASK, ths_settle_time));
    902
    903	/*
    904	 * The CSI unit provides for connection of up to six cameras in
    905	 * the system and is organized as three identical instances of
    906	 * two MIPI support blocks, each with a separate 4-lane
    907	 * interface that can be configured as a single camera with 4
    908	 * lanes or as a dual camera with 2 lanes available for each
    909	 * camera.
    910	 */
    911	if (csi_chan->numlanes == 4) {
    912		cil_write(csi, portno + 1, TEGRA_CSI_CIL_STATUS, 0xffffffff);
    913		cil_write(csi, portno + 1, TEGRA_CSI_CILX_STATUS, 0xffffffff);
    914		cil_write(csi, portno + 1, TEGRA_CSI_CIL_INTERRUPT_MASK, 0x0);
    915
    916		cil_write(csi, portno, TEGRA_CSI_CIL_PAD_CONFIG0,
    917			  BRICK_CLOCK_A_4X);
    918		cil_write(csi, portno + 1, TEGRA_CSI_CIL_PAD_CONFIG0, 0x0);
    919		cil_write(csi, portno + 1, TEGRA_CSI_CIL_INTERRUPT_MASK, 0x0);
    920		cil_write(csi, portno + 1, TEGRA_CSI_CIL_PHY_CONTROL,
    921			  FIELD_PREP(CLK_SETTLE_MASK, clk_settle_time) |
    922			  FIELD_PREP(THS_SETTLE_MASK, ths_settle_time));
    923		csi_write(csi, portno, TEGRA_CSI_PHY_CIL_COMMAND,
    924			  CSI_A_PHY_CIL_ENABLE | CSI_B_PHY_CIL_ENABLE);
    925	} else {
    926		val = ((portno & 1) == PORT_A) ?
    927		      CSI_A_PHY_CIL_ENABLE | CSI_B_PHY_CIL_NOP :
    928		      CSI_B_PHY_CIL_ENABLE | CSI_A_PHY_CIL_NOP;
    929		csi_write(csi, portno, TEGRA_CSI_PHY_CIL_COMMAND, val);
    930	}
    931
    932	/* CSI pixel parser registers setup */
    933	pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_PP_COMMAND,
    934		 (0xf << CSI_PP_START_MARKER_FRAME_MAX_OFFSET) |
    935		 CSI_PP_SINGLE_SHOT_ENABLE | CSI_PP_RST);
    936	pp_write(csi, portno, TEGRA_CSI_PIXEL_PARSER_INTERRUPT_MASK, 0x0);
    937	pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_CONTROL0,
    938		 CSI_PP_PACKET_HEADER_SENT |
    939		 CSI_PP_DATA_IDENTIFIER_ENABLE |
    940		 CSI_PP_WORD_COUNT_SELECT_HEADER |
    941		 CSI_PP_CRC_CHECK_ENABLE |  CSI_PP_WC_CHECK |
    942		 CSI_PP_OUTPUT_FORMAT_STORE | CSI_PPA_PAD_LINE_NOPAD |
    943		 CSI_PP_HEADER_EC_DISABLE | CSI_PPA_PAD_FRAME_NOPAD |
    944		 (portno & 1));
    945	pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_CONTROL1,
    946		 (0x1 << CSI_PP_TOP_FIELD_FRAME_OFFSET) |
    947		 (0x1 << CSI_PP_TOP_FIELD_FRAME_MASK_OFFSET));
    948	pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_GAP,
    949		 0x14 << PP_FRAME_MIN_GAP_OFFSET);
    950	pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_EXPECTED_FRAME, 0x0);
    951	pp_write(csi, portno, TEGRA_CSI_INPUT_STREAM_CONTROL,
    952		 (0x3f << CSI_SKIP_PACKET_THRESHOLD_OFFSET) |
    953		 (csi_chan->numlanes - 1));
    954
    955	/* TPG setup */
    956	if (csi_chan->pg_mode) {
    957		tpg_write(csi, portno, TEGRA_CSI_PATTERN_GENERATOR_CTRL,
    958			  ((csi_chan->pg_mode - 1) << PG_MODE_OFFSET) |
    959			  PG_ENABLE);
    960		tpg_write(csi, portno, TEGRA_CSI_PG_BLANK,
    961			  csi_chan->v_blank << PG_VBLANK_OFFSET |
    962			  csi_chan->h_blank);
    963		tpg_write(csi, portno, TEGRA_CSI_PG_PHASE, 0x0);
    964		tpg_write(csi, portno, TEGRA_CSI_PG_RED_FREQ,
    965			  (0x10 << PG_RED_VERT_INIT_FREQ_OFFSET) |
    966			  (0x10 << PG_RED_HOR_INIT_FREQ_OFFSET));
    967		tpg_write(csi, portno, TEGRA_CSI_PG_RED_FREQ_RATE, 0x0);
    968		tpg_write(csi, portno, TEGRA_CSI_PG_GREEN_FREQ,
    969			  (0x10 << PG_GREEN_VERT_INIT_FREQ_OFFSET) |
    970			  (0x10 << PG_GREEN_HOR_INIT_FREQ_OFFSET));
    971		tpg_write(csi, portno, TEGRA_CSI_PG_GREEN_FREQ_RATE, 0x0);
    972		tpg_write(csi, portno, TEGRA_CSI_PG_BLUE_FREQ,
    973			  (0x10 << PG_BLUE_VERT_INIT_FREQ_OFFSET) |
    974			  (0x10 << PG_BLUE_HOR_INIT_FREQ_OFFSET));
    975		tpg_write(csi, portno, TEGRA_CSI_PG_BLUE_FREQ_RATE, 0x0);
    976	}
    977
    978	pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_PP_COMMAND,
    979		 (0xf << CSI_PP_START_MARKER_FRAME_MAX_OFFSET) |
    980		 CSI_PP_SINGLE_SHOT_ENABLE | CSI_PP_ENABLE);
    981
    982	return 0;
    983}
    984
    985static void
    986tegra210_csi_port_stop_streaming(struct tegra_csi_channel *csi_chan, u8 portno)
    987{
    988	struct tegra_csi *csi = csi_chan->csi;
    989	u32 val;
    990
    991	val = pp_read(csi, portno, TEGRA_CSI_PIXEL_PARSER_STATUS);
    992
    993	dev_dbg(csi->dev, "TEGRA_CSI_PIXEL_PARSER_STATUS 0x%08x\n", val);
    994	pp_write(csi, portno, TEGRA_CSI_PIXEL_PARSER_STATUS, val);
    995
    996	val = cil_read(csi, portno, TEGRA_CSI_CIL_STATUS);
    997	dev_dbg(csi->dev, "TEGRA_CSI_CIL_STATUS 0x%08x\n", val);
    998	cil_write(csi, portno, TEGRA_CSI_CIL_STATUS, val);
    999
   1000	val = cil_read(csi, portno, TEGRA_CSI_CILX_STATUS);
   1001	dev_dbg(csi->dev, "TEGRA_CSI_CILX_STATUS 0x%08x\n", val);
   1002	cil_write(csi, portno, TEGRA_CSI_CILX_STATUS, val);
   1003
   1004	pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_PP_COMMAND,
   1005		 (0xf << CSI_PP_START_MARKER_FRAME_MAX_OFFSET) |
   1006		 CSI_PP_DISABLE);
   1007
   1008	if (csi_chan->pg_mode) {
   1009		tpg_write(csi, portno, TEGRA_CSI_PATTERN_GENERATOR_CTRL,
   1010			  PG_DISABLE);
   1011		return;
   1012	}
   1013
   1014	if (csi_chan->numlanes == 4) {
   1015		csi_write(csi, portno, TEGRA_CSI_PHY_CIL_COMMAND,
   1016			  CSI_A_PHY_CIL_DISABLE |
   1017			  CSI_B_PHY_CIL_DISABLE);
   1018	} else {
   1019		val = ((portno & 1) == PORT_A) ?
   1020		      CSI_A_PHY_CIL_DISABLE | CSI_B_PHY_CIL_NOP :
   1021		      CSI_B_PHY_CIL_DISABLE | CSI_A_PHY_CIL_NOP;
   1022		csi_write(csi, portno, TEGRA_CSI_PHY_CIL_COMMAND, val);
   1023	}
   1024}
   1025
   1026static int tegra210_csi_start_streaming(struct tegra_csi_channel *csi_chan)
   1027{
   1028	u8 *portnos = csi_chan->csi_port_nums;
   1029	int ret, i;
   1030
   1031	for (i = 0; i < csi_chan->numgangports; i++) {
   1032		ret = tegra210_csi_port_start_streaming(csi_chan, portnos[i]);
   1033		if (ret)
   1034			goto stream_start_fail;
   1035	}
   1036
   1037	return 0;
   1038
   1039stream_start_fail:
   1040	for (i = i - 1; i >= 0; i--)
   1041		tegra210_csi_port_stop_streaming(csi_chan, portnos[i]);
   1042
   1043	return ret;
   1044}
   1045
   1046static void tegra210_csi_stop_streaming(struct tegra_csi_channel *csi_chan)
   1047{
   1048	u8 *portnos = csi_chan->csi_port_nums;
   1049	int i;
   1050
   1051	for (i = 0; i < csi_chan->numgangports; i++)
   1052		tegra210_csi_port_stop_streaming(csi_chan, portnos[i]);
   1053}
   1054
   1055/*
   1056 * Tegra210 CSI TPG frame rate table with horizontal and vertical
   1057 * blanking intervals for corresponding format and resolution.
   1058 * Blanking intervals are tuned values from design team for max TPG
   1059 * clock rate.
   1060 */
   1061static const struct tpg_framerate tegra210_tpg_frmrate_table[] = {
   1062	{
   1063		.frmsize = { 1280, 720 },
   1064		.code = MEDIA_BUS_FMT_SRGGB10_1X10,
   1065		.framerate = 120,
   1066		.h_blank = 512,
   1067		.v_blank = 8,
   1068	},
   1069	{
   1070		.frmsize = { 1920, 1080 },
   1071		.code = MEDIA_BUS_FMT_SRGGB10_1X10,
   1072		.framerate = 60,
   1073		.h_blank = 512,
   1074		.v_blank = 8,
   1075	},
   1076	{
   1077		.frmsize = { 3840, 2160 },
   1078		.code = MEDIA_BUS_FMT_SRGGB10_1X10,
   1079		.framerate = 20,
   1080		.h_blank = 8,
   1081		.v_blank = 8,
   1082	},
   1083	{
   1084		.frmsize = { 1280, 720 },
   1085		.code = MEDIA_BUS_FMT_RGB888_1X32_PADHI,
   1086		.framerate = 60,
   1087		.h_blank = 512,
   1088		.v_blank = 8,
   1089	},
   1090	{
   1091		.frmsize = { 1920, 1080 },
   1092		.code = MEDIA_BUS_FMT_RGB888_1X32_PADHI,
   1093		.framerate = 30,
   1094		.h_blank = 512,
   1095		.v_blank = 8,
   1096	},
   1097	{
   1098		.frmsize = { 3840, 2160 },
   1099		.code = MEDIA_BUS_FMT_RGB888_1X32_PADHI,
   1100		.framerate = 8,
   1101		.h_blank = 8,
   1102		.v_blank = 8,
   1103	},
   1104};
   1105
   1106static const char * const tegra210_csi_cil_clks[] = {
   1107	"csi",
   1108	"cilab",
   1109	"cilcd",
   1110	"cile",
   1111#if IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG)
   1112	"csi_tpg",
   1113#endif
   1114};
   1115
   1116/* Tegra210 CSI operations */
   1117static const struct tegra_csi_ops tegra210_csi_ops = {
   1118	.csi_start_streaming = tegra210_csi_start_streaming,
   1119	.csi_stop_streaming = tegra210_csi_stop_streaming,
   1120	.csi_err_recover = tegra210_csi_error_recover,
   1121};
   1122
   1123/* Tegra210 CSI SoC data */
   1124const struct tegra_csi_soc tegra210_csi_soc = {
   1125	.ops = &tegra210_csi_ops,
   1126	.csi_max_channels = 6,
   1127	.clk_names = tegra210_csi_cil_clks,
   1128	.num_clks = ARRAY_SIZE(tegra210_csi_cil_clks),
   1129	.tpg_frmrate_table = tegra210_tpg_frmrate_table,
   1130	.tpg_frmrate_table_size = ARRAY_SIZE(tegra210_tpg_frmrate_table),
   1131};