cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

rv770.c (61284B)


      1/*
      2 * Copyright 2008 Advanced Micro Devices, Inc.
      3 * Copyright 2008 Red Hat Inc.
      4 * Copyright 2009 Jerome Glisse.
      5 *
      6 * Permission is hereby granted, free of charge, to any person obtaining a
      7 * copy of this software and associated documentation files (the "Software"),
      8 * to deal in the Software without restriction, including without limitation
      9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
     10 * and/or sell copies of the Software, and to permit persons to whom the
     11 * Software is furnished to do so, subject to the following conditions:
     12 *
     13 * The above copyright notice and this permission notice shall be included in
     14 * all copies or substantial portions of the Software.
     15 *
     16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
     17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
     18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
     19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
     20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
     21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
     22 * OTHER DEALINGS IN THE SOFTWARE.
     23 *
     24 * Authors: Dave Airlie
     25 *          Alex Deucher
     26 *          Jerome Glisse
     27 */
     28
     29#include <linux/firmware.h>
     30#include <linux/pci.h>
     31#include <linux/slab.h>
     32
     33#include <drm/drm_device.h>
     34#include <drm/radeon_drm.h>
     35#include <drm/drm_fourcc.h>
     36
     37#include "atom.h"
     38#include "avivod.h"
     39#include "radeon.h"
     40#include "radeon_asic.h"
     41#include "radeon_audio.h"
     42#include "rv770d.h"
     43#include "rv770.h"
     44
     45#define R700_PFP_UCODE_SIZE 848
     46#define R700_PM4_UCODE_SIZE 1360
     47
     48static void rv770_gpu_init(struct radeon_device *rdev);
     49void rv770_fini(struct radeon_device *rdev);
     50static void rv770_pcie_gen2_enable(struct radeon_device *rdev);
     51int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk);
     52
     53int rv770_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
     54{
     55	unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
     56	int r;
     57
     58	/* RV740 uses evergreen uvd clk programming */
     59	if (rdev->family == CHIP_RV740)
     60		return evergreen_set_uvd_clocks(rdev, vclk, dclk);
     61
     62	/* bypass vclk and dclk with bclk */
     63	WREG32_P(CG_UPLL_FUNC_CNTL_2,
     64		 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
     65		 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
     66
     67	if (!vclk || !dclk) {
     68		/* keep the Bypass mode, put PLL to sleep */
     69		WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
     70		return 0;
     71	}
     72
     73	r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 50000, 160000,
     74					  43663, 0x03FFFFFE, 1, 30, ~0,
     75					  &fb_div, &vclk_div, &dclk_div);
     76	if (r)
     77		return r;
     78
     79	fb_div |= 1;
     80	vclk_div -= 1;
     81	dclk_div -= 1;
     82
     83	/* set UPLL_FB_DIV to 0x50000 */
     84	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(0x50000), ~UPLL_FB_DIV_MASK);
     85
     86	/* deassert UPLL_RESET and UPLL_SLEEP */
     87	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~(UPLL_RESET_MASK | UPLL_SLEEP_MASK));
     88
     89	/* assert BYPASS EN and FB_DIV[0] <- ??? why? */
     90	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
     91	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(1), ~UPLL_FB_DIV(1));
     92
     93	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
     94	if (r)
     95		return r;
     96
     97	/* assert PLL_RESET */
     98	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
     99
    100	/* set the required FB_DIV, REF_DIV, Post divder values */
    101	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_REF_DIV(1), ~UPLL_REF_DIV_MASK);
    102	WREG32_P(CG_UPLL_FUNC_CNTL_2,
    103		 UPLL_SW_HILEN(vclk_div >> 1) |
    104		 UPLL_SW_LOLEN((vclk_div >> 1) + (vclk_div & 1)) |
    105		 UPLL_SW_HILEN2(dclk_div >> 1) |
    106		 UPLL_SW_LOLEN2((dclk_div >> 1) + (dclk_div & 1)),
    107		 ~UPLL_SW_MASK);
    108
    109	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div),
    110		 ~UPLL_FB_DIV_MASK);
    111
    112	/* give the PLL some time to settle */
    113	mdelay(15);
    114
    115	/* deassert PLL_RESET */
    116	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
    117
    118	mdelay(15);
    119
    120	/* deassert BYPASS EN and FB_DIV[0] <- ??? why? */
    121	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
    122	WREG32_P(CG_UPLL_FUNC_CNTL_3, 0, ~UPLL_FB_DIV(1));
    123
    124	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
    125	if (r)
    126		return r;
    127
    128	/* switch VCLK and DCLK selection */
    129	WREG32_P(CG_UPLL_FUNC_CNTL_2,
    130		 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
    131		 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
    132
    133	mdelay(100);
    134
    135	return 0;
    136}
    137
    138static const u32 r7xx_golden_registers[] =
    139{
    140	0x8d00, 0xffffffff, 0x0e0e0074,
    141	0x8d04, 0xffffffff, 0x013a2b34,
    142	0x9508, 0xffffffff, 0x00000002,
    143	0x8b20, 0xffffffff, 0,
    144	0x88c4, 0xffffffff, 0x000000c2,
    145	0x28350, 0xffffffff, 0,
    146	0x9058, 0xffffffff, 0x0fffc40f,
    147	0x240c, 0xffffffff, 0x00000380,
    148	0x733c, 0xffffffff, 0x00000002,
    149	0x2650, 0x00040000, 0,
    150	0x20bc, 0x00040000, 0,
    151	0x7300, 0xffffffff, 0x001000f0
    152};
    153
    154static const u32 r7xx_golden_dyn_gpr_registers[] =
    155{
    156	0x8db0, 0xffffffff, 0x98989898,
    157	0x8db4, 0xffffffff, 0x98989898,
    158	0x8db8, 0xffffffff, 0x98989898,
    159	0x8dbc, 0xffffffff, 0x98989898,
    160	0x8dc0, 0xffffffff, 0x98989898,
    161	0x8dc4, 0xffffffff, 0x98989898,
    162	0x8dc8, 0xffffffff, 0x98989898,
    163	0x8dcc, 0xffffffff, 0x98989898,
    164	0x88c4, 0xffffffff, 0x00000082
    165};
    166
    167static const u32 rv770_golden_registers[] =
    168{
    169	0x562c, 0xffffffff, 0,
    170	0x3f90, 0xffffffff, 0,
    171	0x9148, 0xffffffff, 0,
    172	0x3f94, 0xffffffff, 0,
    173	0x914c, 0xffffffff, 0,
    174	0x9698, 0x18000000, 0x18000000
    175};
    176
    177static const u32 rv770ce_golden_registers[] =
    178{
    179	0x562c, 0xffffffff, 0,
    180	0x3f90, 0xffffffff, 0x00cc0000,
    181	0x9148, 0xffffffff, 0x00cc0000,
    182	0x3f94, 0xffffffff, 0x00cc0000,
    183	0x914c, 0xffffffff, 0x00cc0000,
    184	0x9b7c, 0xffffffff, 0x00fa0000,
    185	0x3f8c, 0xffffffff, 0x00fa0000,
    186	0x9698, 0x18000000, 0x18000000
    187};
    188
    189static const u32 rv770_mgcg_init[] =
    190{
    191	0x8bcc, 0xffffffff, 0x130300f9,
    192	0x5448, 0xffffffff, 0x100,
    193	0x55e4, 0xffffffff, 0x100,
    194	0x160c, 0xffffffff, 0x100,
    195	0x5644, 0xffffffff, 0x100,
    196	0xc164, 0xffffffff, 0x100,
    197	0x8a18, 0xffffffff, 0x100,
    198	0x897c, 0xffffffff, 0x8000100,
    199	0x8b28, 0xffffffff, 0x3c000100,
    200	0x9144, 0xffffffff, 0x100,
    201	0x9a1c, 0xffffffff, 0x10000,
    202	0x9a50, 0xffffffff, 0x100,
    203	0x9a1c, 0xffffffff, 0x10001,
    204	0x9a50, 0xffffffff, 0x100,
    205	0x9a1c, 0xffffffff, 0x10002,
    206	0x9a50, 0xffffffff, 0x100,
    207	0x9a1c, 0xffffffff, 0x10003,
    208	0x9a50, 0xffffffff, 0x100,
    209	0x9a1c, 0xffffffff, 0x0,
    210	0x9870, 0xffffffff, 0x100,
    211	0x8d58, 0xffffffff, 0x100,
    212	0x9500, 0xffffffff, 0x0,
    213	0x9510, 0xffffffff, 0x100,
    214	0x9500, 0xffffffff, 0x1,
    215	0x9510, 0xffffffff, 0x100,
    216	0x9500, 0xffffffff, 0x2,
    217	0x9510, 0xffffffff, 0x100,
    218	0x9500, 0xffffffff, 0x3,
    219	0x9510, 0xffffffff, 0x100,
    220	0x9500, 0xffffffff, 0x4,
    221	0x9510, 0xffffffff, 0x100,
    222	0x9500, 0xffffffff, 0x5,
    223	0x9510, 0xffffffff, 0x100,
    224	0x9500, 0xffffffff, 0x6,
    225	0x9510, 0xffffffff, 0x100,
    226	0x9500, 0xffffffff, 0x7,
    227	0x9510, 0xffffffff, 0x100,
    228	0x9500, 0xffffffff, 0x8,
    229	0x9510, 0xffffffff, 0x100,
    230	0x9500, 0xffffffff, 0x9,
    231	0x9510, 0xffffffff, 0x100,
    232	0x9500, 0xffffffff, 0x8000,
    233	0x9490, 0xffffffff, 0x0,
    234	0x949c, 0xffffffff, 0x100,
    235	0x9490, 0xffffffff, 0x1,
    236	0x949c, 0xffffffff, 0x100,
    237	0x9490, 0xffffffff, 0x2,
    238	0x949c, 0xffffffff, 0x100,
    239	0x9490, 0xffffffff, 0x3,
    240	0x949c, 0xffffffff, 0x100,
    241	0x9490, 0xffffffff, 0x4,
    242	0x949c, 0xffffffff, 0x100,
    243	0x9490, 0xffffffff, 0x5,
    244	0x949c, 0xffffffff, 0x100,
    245	0x9490, 0xffffffff, 0x6,
    246	0x949c, 0xffffffff, 0x100,
    247	0x9490, 0xffffffff, 0x7,
    248	0x949c, 0xffffffff, 0x100,
    249	0x9490, 0xffffffff, 0x8,
    250	0x949c, 0xffffffff, 0x100,
    251	0x9490, 0xffffffff, 0x9,
    252	0x949c, 0xffffffff, 0x100,
    253	0x9490, 0xffffffff, 0x8000,
    254	0x9604, 0xffffffff, 0x0,
    255	0x9654, 0xffffffff, 0x100,
    256	0x9604, 0xffffffff, 0x1,
    257	0x9654, 0xffffffff, 0x100,
    258	0x9604, 0xffffffff, 0x2,
    259	0x9654, 0xffffffff, 0x100,
    260	0x9604, 0xffffffff, 0x3,
    261	0x9654, 0xffffffff, 0x100,
    262	0x9604, 0xffffffff, 0x4,
    263	0x9654, 0xffffffff, 0x100,
    264	0x9604, 0xffffffff, 0x5,
    265	0x9654, 0xffffffff, 0x100,
    266	0x9604, 0xffffffff, 0x6,
    267	0x9654, 0xffffffff, 0x100,
    268	0x9604, 0xffffffff, 0x7,
    269	0x9654, 0xffffffff, 0x100,
    270	0x9604, 0xffffffff, 0x8,
    271	0x9654, 0xffffffff, 0x100,
    272	0x9604, 0xffffffff, 0x9,
    273	0x9654, 0xffffffff, 0x100,
    274	0x9604, 0xffffffff, 0x80000000,
    275	0x9030, 0xffffffff, 0x100,
    276	0x9034, 0xffffffff, 0x100,
    277	0x9038, 0xffffffff, 0x100,
    278	0x903c, 0xffffffff, 0x100,
    279	0x9040, 0xffffffff, 0x100,
    280	0xa200, 0xffffffff, 0x100,
    281	0xa204, 0xffffffff, 0x100,
    282	0xa208, 0xffffffff, 0x100,
    283	0xa20c, 0xffffffff, 0x100,
    284	0x971c, 0xffffffff, 0x100,
    285	0x915c, 0xffffffff, 0x00020001,
    286	0x9160, 0xffffffff, 0x00040003,
    287	0x916c, 0xffffffff, 0x00060005,
    288	0x9170, 0xffffffff, 0x00080007,
    289	0x9174, 0xffffffff, 0x000a0009,
    290	0x9178, 0xffffffff, 0x000c000b,
    291	0x917c, 0xffffffff, 0x000e000d,
    292	0x9180, 0xffffffff, 0x0010000f,
    293	0x918c, 0xffffffff, 0x00120011,
    294	0x9190, 0xffffffff, 0x00140013,
    295	0x9194, 0xffffffff, 0x00020001,
    296	0x9198, 0xffffffff, 0x00040003,
    297	0x919c, 0xffffffff, 0x00060005,
    298	0x91a8, 0xffffffff, 0x00080007,
    299	0x91ac, 0xffffffff, 0x000a0009,
    300	0x91b0, 0xffffffff, 0x000c000b,
    301	0x91b4, 0xffffffff, 0x000e000d,
    302	0x91b8, 0xffffffff, 0x0010000f,
    303	0x91c4, 0xffffffff, 0x00120011,
    304	0x91c8, 0xffffffff, 0x00140013,
    305	0x91cc, 0xffffffff, 0x00020001,
    306	0x91d0, 0xffffffff, 0x00040003,
    307	0x91d4, 0xffffffff, 0x00060005,
    308	0x91e0, 0xffffffff, 0x00080007,
    309	0x91e4, 0xffffffff, 0x000a0009,
    310	0x91e8, 0xffffffff, 0x000c000b,
    311	0x91ec, 0xffffffff, 0x00020001,
    312	0x91f0, 0xffffffff, 0x00040003,
    313	0x91f4, 0xffffffff, 0x00060005,
    314	0x9200, 0xffffffff, 0x00080007,
    315	0x9204, 0xffffffff, 0x000a0009,
    316	0x9208, 0xffffffff, 0x000c000b,
    317	0x920c, 0xffffffff, 0x000e000d,
    318	0x9210, 0xffffffff, 0x0010000f,
    319	0x921c, 0xffffffff, 0x00120011,
    320	0x9220, 0xffffffff, 0x00140013,
    321	0x9224, 0xffffffff, 0x00020001,
    322	0x9228, 0xffffffff, 0x00040003,
    323	0x922c, 0xffffffff, 0x00060005,
    324	0x9238, 0xffffffff, 0x00080007,
    325	0x923c, 0xffffffff, 0x000a0009,
    326	0x9240, 0xffffffff, 0x000c000b,
    327	0x9244, 0xffffffff, 0x000e000d,
    328	0x9248, 0xffffffff, 0x0010000f,
    329	0x9254, 0xffffffff, 0x00120011,
    330	0x9258, 0xffffffff, 0x00140013,
    331	0x925c, 0xffffffff, 0x00020001,
    332	0x9260, 0xffffffff, 0x00040003,
    333	0x9264, 0xffffffff, 0x00060005,
    334	0x9270, 0xffffffff, 0x00080007,
    335	0x9274, 0xffffffff, 0x000a0009,
    336	0x9278, 0xffffffff, 0x000c000b,
    337	0x927c, 0xffffffff, 0x000e000d,
    338	0x9280, 0xffffffff, 0x0010000f,
    339	0x928c, 0xffffffff, 0x00120011,
    340	0x9290, 0xffffffff, 0x00140013,
    341	0x9294, 0xffffffff, 0x00020001,
    342	0x929c, 0xffffffff, 0x00040003,
    343	0x92a0, 0xffffffff, 0x00060005,
    344	0x92a4, 0xffffffff, 0x00080007
    345};
    346
    347static const u32 rv710_golden_registers[] =
    348{
    349	0x3f90, 0x00ff0000, 0x00fc0000,
    350	0x9148, 0x00ff0000, 0x00fc0000,
    351	0x3f94, 0x00ff0000, 0x00fc0000,
    352	0x914c, 0x00ff0000, 0x00fc0000,
    353	0xb4c, 0x00000020, 0x00000020,
    354	0xa180, 0xffffffff, 0x00003f3f
    355};
    356
    357static const u32 rv710_mgcg_init[] =
    358{
    359	0x8bcc, 0xffffffff, 0x13030040,
    360	0x5448, 0xffffffff, 0x100,
    361	0x55e4, 0xffffffff, 0x100,
    362	0x160c, 0xffffffff, 0x100,
    363	0x5644, 0xffffffff, 0x100,
    364	0xc164, 0xffffffff, 0x100,
    365	0x8a18, 0xffffffff, 0x100,
    366	0x897c, 0xffffffff, 0x8000100,
    367	0x8b28, 0xffffffff, 0x3c000100,
    368	0x9144, 0xffffffff, 0x100,
    369	0x9a1c, 0xffffffff, 0x10000,
    370	0x9a50, 0xffffffff, 0x100,
    371	0x9a1c, 0xffffffff, 0x0,
    372	0x9870, 0xffffffff, 0x100,
    373	0x8d58, 0xffffffff, 0x100,
    374	0x9500, 0xffffffff, 0x0,
    375	0x9510, 0xffffffff, 0x100,
    376	0x9500, 0xffffffff, 0x1,
    377	0x9510, 0xffffffff, 0x100,
    378	0x9500, 0xffffffff, 0x8000,
    379	0x9490, 0xffffffff, 0x0,
    380	0x949c, 0xffffffff, 0x100,
    381	0x9490, 0xffffffff, 0x1,
    382	0x949c, 0xffffffff, 0x100,
    383	0x9490, 0xffffffff, 0x8000,
    384	0x9604, 0xffffffff, 0x0,
    385	0x9654, 0xffffffff, 0x100,
    386	0x9604, 0xffffffff, 0x1,
    387	0x9654, 0xffffffff, 0x100,
    388	0x9604, 0xffffffff, 0x80000000,
    389	0x9030, 0xffffffff, 0x100,
    390	0x9034, 0xffffffff, 0x100,
    391	0x9038, 0xffffffff, 0x100,
    392	0x903c, 0xffffffff, 0x100,
    393	0x9040, 0xffffffff, 0x100,
    394	0xa200, 0xffffffff, 0x100,
    395	0xa204, 0xffffffff, 0x100,
    396	0xa208, 0xffffffff, 0x100,
    397	0xa20c, 0xffffffff, 0x100,
    398	0x971c, 0xffffffff, 0x100,
    399	0x915c, 0xffffffff, 0x00020001,
    400	0x9174, 0xffffffff, 0x00000003,
    401	0x9178, 0xffffffff, 0x00050001,
    402	0x917c, 0xffffffff, 0x00030002,
    403	0x918c, 0xffffffff, 0x00000004,
    404	0x9190, 0xffffffff, 0x00070006,
    405	0x9194, 0xffffffff, 0x00050001,
    406	0x9198, 0xffffffff, 0x00030002,
    407	0x91a8, 0xffffffff, 0x00000004,
    408	0x91ac, 0xffffffff, 0x00070006,
    409	0x91e8, 0xffffffff, 0x00000001,
    410	0x9294, 0xffffffff, 0x00000001,
    411	0x929c, 0xffffffff, 0x00000002,
    412	0x92a0, 0xffffffff, 0x00040003,
    413	0x9150, 0xffffffff, 0x4d940000
    414};
    415
    416static const u32 rv730_golden_registers[] =
    417{
    418	0x3f90, 0x00ff0000, 0x00f00000,
    419	0x9148, 0x00ff0000, 0x00f00000,
    420	0x3f94, 0x00ff0000, 0x00f00000,
    421	0x914c, 0x00ff0000, 0x00f00000,
    422	0x900c, 0xffffffff, 0x003b033f,
    423	0xb4c, 0x00000020, 0x00000020,
    424	0xa180, 0xffffffff, 0x00003f3f
    425};
    426
    427static const u32 rv730_mgcg_init[] =
    428{
    429	0x8bcc, 0xffffffff, 0x130300f9,
    430	0x5448, 0xffffffff, 0x100,
    431	0x55e4, 0xffffffff, 0x100,
    432	0x160c, 0xffffffff, 0x100,
    433	0x5644, 0xffffffff, 0x100,
    434	0xc164, 0xffffffff, 0x100,
    435	0x8a18, 0xffffffff, 0x100,
    436	0x897c, 0xffffffff, 0x8000100,
    437	0x8b28, 0xffffffff, 0x3c000100,
    438	0x9144, 0xffffffff, 0x100,
    439	0x9a1c, 0xffffffff, 0x10000,
    440	0x9a50, 0xffffffff, 0x100,
    441	0x9a1c, 0xffffffff, 0x10001,
    442	0x9a50, 0xffffffff, 0x100,
    443	0x9a1c, 0xffffffff, 0x0,
    444	0x9870, 0xffffffff, 0x100,
    445	0x8d58, 0xffffffff, 0x100,
    446	0x9500, 0xffffffff, 0x0,
    447	0x9510, 0xffffffff, 0x100,
    448	0x9500, 0xffffffff, 0x1,
    449	0x9510, 0xffffffff, 0x100,
    450	0x9500, 0xffffffff, 0x2,
    451	0x9510, 0xffffffff, 0x100,
    452	0x9500, 0xffffffff, 0x3,
    453	0x9510, 0xffffffff, 0x100,
    454	0x9500, 0xffffffff, 0x4,
    455	0x9510, 0xffffffff, 0x100,
    456	0x9500, 0xffffffff, 0x5,
    457	0x9510, 0xffffffff, 0x100,
    458	0x9500, 0xffffffff, 0x6,
    459	0x9510, 0xffffffff, 0x100,
    460	0x9500, 0xffffffff, 0x7,
    461	0x9510, 0xffffffff, 0x100,
    462	0x9500, 0xffffffff, 0x8000,
    463	0x9490, 0xffffffff, 0x0,
    464	0x949c, 0xffffffff, 0x100,
    465	0x9490, 0xffffffff, 0x1,
    466	0x949c, 0xffffffff, 0x100,
    467	0x9490, 0xffffffff, 0x2,
    468	0x949c, 0xffffffff, 0x100,
    469	0x9490, 0xffffffff, 0x3,
    470	0x949c, 0xffffffff, 0x100,
    471	0x9490, 0xffffffff, 0x4,
    472	0x949c, 0xffffffff, 0x100,
    473	0x9490, 0xffffffff, 0x5,
    474	0x949c, 0xffffffff, 0x100,
    475	0x9490, 0xffffffff, 0x6,
    476	0x949c, 0xffffffff, 0x100,
    477	0x9490, 0xffffffff, 0x7,
    478	0x949c, 0xffffffff, 0x100,
    479	0x9490, 0xffffffff, 0x8000,
    480	0x9604, 0xffffffff, 0x0,
    481	0x9654, 0xffffffff, 0x100,
    482	0x9604, 0xffffffff, 0x1,
    483	0x9654, 0xffffffff, 0x100,
    484	0x9604, 0xffffffff, 0x2,
    485	0x9654, 0xffffffff, 0x100,
    486	0x9604, 0xffffffff, 0x3,
    487	0x9654, 0xffffffff, 0x100,
    488	0x9604, 0xffffffff, 0x4,
    489	0x9654, 0xffffffff, 0x100,
    490	0x9604, 0xffffffff, 0x5,
    491	0x9654, 0xffffffff, 0x100,
    492	0x9604, 0xffffffff, 0x6,
    493	0x9654, 0xffffffff, 0x100,
    494	0x9604, 0xffffffff, 0x7,
    495	0x9654, 0xffffffff, 0x100,
    496	0x9604, 0xffffffff, 0x80000000,
    497	0x9030, 0xffffffff, 0x100,
    498	0x9034, 0xffffffff, 0x100,
    499	0x9038, 0xffffffff, 0x100,
    500	0x903c, 0xffffffff, 0x100,
    501	0x9040, 0xffffffff, 0x100,
    502	0xa200, 0xffffffff, 0x100,
    503	0xa204, 0xffffffff, 0x100,
    504	0xa208, 0xffffffff, 0x100,
    505	0xa20c, 0xffffffff, 0x100,
    506	0x971c, 0xffffffff, 0x100,
    507	0x915c, 0xffffffff, 0x00020001,
    508	0x916c, 0xffffffff, 0x00040003,
    509	0x9170, 0xffffffff, 0x00000005,
    510	0x9178, 0xffffffff, 0x00050001,
    511	0x917c, 0xffffffff, 0x00030002,
    512	0x918c, 0xffffffff, 0x00000004,
    513	0x9190, 0xffffffff, 0x00070006,
    514	0x9194, 0xffffffff, 0x00050001,
    515	0x9198, 0xffffffff, 0x00030002,
    516	0x91a8, 0xffffffff, 0x00000004,
    517	0x91ac, 0xffffffff, 0x00070006,
    518	0x91b0, 0xffffffff, 0x00050001,
    519	0x91b4, 0xffffffff, 0x00030002,
    520	0x91c4, 0xffffffff, 0x00000004,
    521	0x91c8, 0xffffffff, 0x00070006,
    522	0x91cc, 0xffffffff, 0x00050001,
    523	0x91d0, 0xffffffff, 0x00030002,
    524	0x91e0, 0xffffffff, 0x00000004,
    525	0x91e4, 0xffffffff, 0x00070006,
    526	0x91e8, 0xffffffff, 0x00000001,
    527	0x91ec, 0xffffffff, 0x00050001,
    528	0x91f0, 0xffffffff, 0x00030002,
    529	0x9200, 0xffffffff, 0x00000004,
    530	0x9204, 0xffffffff, 0x00070006,
    531	0x9208, 0xffffffff, 0x00050001,
    532	0x920c, 0xffffffff, 0x00030002,
    533	0x921c, 0xffffffff, 0x00000004,
    534	0x9220, 0xffffffff, 0x00070006,
    535	0x9224, 0xffffffff, 0x00050001,
    536	0x9228, 0xffffffff, 0x00030002,
    537	0x9238, 0xffffffff, 0x00000004,
    538	0x923c, 0xffffffff, 0x00070006,
    539	0x9240, 0xffffffff, 0x00050001,
    540	0x9244, 0xffffffff, 0x00030002,
    541	0x9254, 0xffffffff, 0x00000004,
    542	0x9258, 0xffffffff, 0x00070006,
    543	0x9294, 0xffffffff, 0x00000001,
    544	0x929c, 0xffffffff, 0x00000002,
    545	0x92a0, 0xffffffff, 0x00040003,
    546	0x92a4, 0xffffffff, 0x00000005
    547};
    548
    549static const u32 rv740_golden_registers[] =
    550{
    551	0x88c4, 0xffffffff, 0x00000082,
    552	0x28a50, 0xfffffffc, 0x00000004,
    553	0x2650, 0x00040000, 0,
    554	0x20bc, 0x00040000, 0,
    555	0x733c, 0xffffffff, 0x00000002,
    556	0x7300, 0xffffffff, 0x001000f0,
    557	0x3f90, 0x00ff0000, 0,
    558	0x9148, 0x00ff0000, 0,
    559	0x3f94, 0x00ff0000, 0,
    560	0x914c, 0x00ff0000, 0,
    561	0x240c, 0xffffffff, 0x00000380,
    562	0x8a14, 0x00000007, 0x00000007,
    563	0x8b24, 0xffffffff, 0x00ff0fff,
    564	0x28a4c, 0xffffffff, 0x00004000,
    565	0xa180, 0xffffffff, 0x00003f3f,
    566	0x8d00, 0xffffffff, 0x0e0e003a,
    567	0x8d04, 0xffffffff, 0x013a0e2a,
    568	0x8c00, 0xffffffff, 0xe400000f,
    569	0x8db0, 0xffffffff, 0x98989898,
    570	0x8db4, 0xffffffff, 0x98989898,
    571	0x8db8, 0xffffffff, 0x98989898,
    572	0x8dbc, 0xffffffff, 0x98989898,
    573	0x8dc0, 0xffffffff, 0x98989898,
    574	0x8dc4, 0xffffffff, 0x98989898,
    575	0x8dc8, 0xffffffff, 0x98989898,
    576	0x8dcc, 0xffffffff, 0x98989898,
    577	0x9058, 0xffffffff, 0x0fffc40f,
    578	0x900c, 0xffffffff, 0x003b033f,
    579	0x28350, 0xffffffff, 0,
    580	0x8cf0, 0x1fffffff, 0x08e00420,
    581	0x9508, 0xffffffff, 0x00000002,
    582	0x88c4, 0xffffffff, 0x000000c2,
    583	0x9698, 0x18000000, 0x18000000
    584};
    585
    586static const u32 rv740_mgcg_init[] =
    587{
    588	0x8bcc, 0xffffffff, 0x13030100,
    589	0x5448, 0xffffffff, 0x100,
    590	0x55e4, 0xffffffff, 0x100,
    591	0x160c, 0xffffffff, 0x100,
    592	0x5644, 0xffffffff, 0x100,
    593	0xc164, 0xffffffff, 0x100,
    594	0x8a18, 0xffffffff, 0x100,
    595	0x897c, 0xffffffff, 0x100,
    596	0x8b28, 0xffffffff, 0x100,
    597	0x9144, 0xffffffff, 0x100,
    598	0x9a1c, 0xffffffff, 0x10000,
    599	0x9a50, 0xffffffff, 0x100,
    600	0x9a1c, 0xffffffff, 0x10001,
    601	0x9a50, 0xffffffff, 0x100,
    602	0x9a1c, 0xffffffff, 0x10002,
    603	0x9a50, 0xffffffff, 0x100,
    604	0x9a1c, 0xffffffff, 0x10003,
    605	0x9a50, 0xffffffff, 0x100,
    606	0x9a1c, 0xffffffff, 0x0,
    607	0x9870, 0xffffffff, 0x100,
    608	0x8d58, 0xffffffff, 0x100,
    609	0x9500, 0xffffffff, 0x0,
    610	0x9510, 0xffffffff, 0x100,
    611	0x9500, 0xffffffff, 0x1,
    612	0x9510, 0xffffffff, 0x100,
    613	0x9500, 0xffffffff, 0x2,
    614	0x9510, 0xffffffff, 0x100,
    615	0x9500, 0xffffffff, 0x3,
    616	0x9510, 0xffffffff, 0x100,
    617	0x9500, 0xffffffff, 0x4,
    618	0x9510, 0xffffffff, 0x100,
    619	0x9500, 0xffffffff, 0x5,
    620	0x9510, 0xffffffff, 0x100,
    621	0x9500, 0xffffffff, 0x6,
    622	0x9510, 0xffffffff, 0x100,
    623	0x9500, 0xffffffff, 0x7,
    624	0x9510, 0xffffffff, 0x100,
    625	0x9500, 0xffffffff, 0x8000,
    626	0x9490, 0xffffffff, 0x0,
    627	0x949c, 0xffffffff, 0x100,
    628	0x9490, 0xffffffff, 0x1,
    629	0x949c, 0xffffffff, 0x100,
    630	0x9490, 0xffffffff, 0x2,
    631	0x949c, 0xffffffff, 0x100,
    632	0x9490, 0xffffffff, 0x3,
    633	0x949c, 0xffffffff, 0x100,
    634	0x9490, 0xffffffff, 0x4,
    635	0x949c, 0xffffffff, 0x100,
    636	0x9490, 0xffffffff, 0x5,
    637	0x949c, 0xffffffff, 0x100,
    638	0x9490, 0xffffffff, 0x6,
    639	0x949c, 0xffffffff, 0x100,
    640	0x9490, 0xffffffff, 0x7,
    641	0x949c, 0xffffffff, 0x100,
    642	0x9490, 0xffffffff, 0x8000,
    643	0x9604, 0xffffffff, 0x0,
    644	0x9654, 0xffffffff, 0x100,
    645	0x9604, 0xffffffff, 0x1,
    646	0x9654, 0xffffffff, 0x100,
    647	0x9604, 0xffffffff, 0x2,
    648	0x9654, 0xffffffff, 0x100,
    649	0x9604, 0xffffffff, 0x3,
    650	0x9654, 0xffffffff, 0x100,
    651	0x9604, 0xffffffff, 0x4,
    652	0x9654, 0xffffffff, 0x100,
    653	0x9604, 0xffffffff, 0x5,
    654	0x9654, 0xffffffff, 0x100,
    655	0x9604, 0xffffffff, 0x6,
    656	0x9654, 0xffffffff, 0x100,
    657	0x9604, 0xffffffff, 0x7,
    658	0x9654, 0xffffffff, 0x100,
    659	0x9604, 0xffffffff, 0x80000000,
    660	0x9030, 0xffffffff, 0x100,
    661	0x9034, 0xffffffff, 0x100,
    662	0x9038, 0xffffffff, 0x100,
    663	0x903c, 0xffffffff, 0x100,
    664	0x9040, 0xffffffff, 0x100,
    665	0xa200, 0xffffffff, 0x100,
    666	0xa204, 0xffffffff, 0x100,
    667	0xa208, 0xffffffff, 0x100,
    668	0xa20c, 0xffffffff, 0x100,
    669	0x971c, 0xffffffff, 0x100,
    670	0x915c, 0xffffffff, 0x00020001,
    671	0x9160, 0xffffffff, 0x00040003,
    672	0x916c, 0xffffffff, 0x00060005,
    673	0x9170, 0xffffffff, 0x00080007,
    674	0x9174, 0xffffffff, 0x000a0009,
    675	0x9178, 0xffffffff, 0x000c000b,
    676	0x917c, 0xffffffff, 0x000e000d,
    677	0x9180, 0xffffffff, 0x0010000f,
    678	0x918c, 0xffffffff, 0x00120011,
    679	0x9190, 0xffffffff, 0x00140013,
    680	0x9194, 0xffffffff, 0x00020001,
    681	0x9198, 0xffffffff, 0x00040003,
    682	0x919c, 0xffffffff, 0x00060005,
    683	0x91a8, 0xffffffff, 0x00080007,
    684	0x91ac, 0xffffffff, 0x000a0009,
    685	0x91b0, 0xffffffff, 0x000c000b,
    686	0x91b4, 0xffffffff, 0x000e000d,
    687	0x91b8, 0xffffffff, 0x0010000f,
    688	0x91c4, 0xffffffff, 0x00120011,
    689	0x91c8, 0xffffffff, 0x00140013,
    690	0x91cc, 0xffffffff, 0x00020001,
    691	0x91d0, 0xffffffff, 0x00040003,
    692	0x91d4, 0xffffffff, 0x00060005,
    693	0x91e0, 0xffffffff, 0x00080007,
    694	0x91e4, 0xffffffff, 0x000a0009,
    695	0x91e8, 0xffffffff, 0x000c000b,
    696	0x91ec, 0xffffffff, 0x00020001,
    697	0x91f0, 0xffffffff, 0x00040003,
    698	0x91f4, 0xffffffff, 0x00060005,
    699	0x9200, 0xffffffff, 0x00080007,
    700	0x9204, 0xffffffff, 0x000a0009,
    701	0x9208, 0xffffffff, 0x000c000b,
    702	0x920c, 0xffffffff, 0x000e000d,
    703	0x9210, 0xffffffff, 0x0010000f,
    704	0x921c, 0xffffffff, 0x00120011,
    705	0x9220, 0xffffffff, 0x00140013,
    706	0x9224, 0xffffffff, 0x00020001,
    707	0x9228, 0xffffffff, 0x00040003,
    708	0x922c, 0xffffffff, 0x00060005,
    709	0x9238, 0xffffffff, 0x00080007,
    710	0x923c, 0xffffffff, 0x000a0009,
    711	0x9240, 0xffffffff, 0x000c000b,
    712	0x9244, 0xffffffff, 0x000e000d,
    713	0x9248, 0xffffffff, 0x0010000f,
    714	0x9254, 0xffffffff, 0x00120011,
    715	0x9258, 0xffffffff, 0x00140013,
    716	0x9294, 0xffffffff, 0x00020001,
    717	0x929c, 0xffffffff, 0x00040003,
    718	0x92a0, 0xffffffff, 0x00060005,
    719	0x92a4, 0xffffffff, 0x00080007
    720};
    721
    722static void rv770_init_golden_registers(struct radeon_device *rdev)
    723{
    724	switch (rdev->family) {
    725	case CHIP_RV770:
    726		radeon_program_register_sequence(rdev,
    727						 r7xx_golden_registers,
    728						 (const u32)ARRAY_SIZE(r7xx_golden_registers));
    729		radeon_program_register_sequence(rdev,
    730						 r7xx_golden_dyn_gpr_registers,
    731						 (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
    732		if (rdev->pdev->device == 0x994e)
    733			radeon_program_register_sequence(rdev,
    734							 rv770ce_golden_registers,
    735							 (const u32)ARRAY_SIZE(rv770ce_golden_registers));
    736		else
    737			radeon_program_register_sequence(rdev,
    738							 rv770_golden_registers,
    739							 (const u32)ARRAY_SIZE(rv770_golden_registers));
    740		radeon_program_register_sequence(rdev,
    741						 rv770_mgcg_init,
    742						 (const u32)ARRAY_SIZE(rv770_mgcg_init));
    743		break;
    744	case CHIP_RV730:
    745		radeon_program_register_sequence(rdev,
    746						 r7xx_golden_registers,
    747						 (const u32)ARRAY_SIZE(r7xx_golden_registers));
    748		radeon_program_register_sequence(rdev,
    749						 r7xx_golden_dyn_gpr_registers,
    750						 (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
    751		radeon_program_register_sequence(rdev,
    752						 rv730_golden_registers,
    753						 (const u32)ARRAY_SIZE(rv730_golden_registers));
    754		radeon_program_register_sequence(rdev,
    755						 rv730_mgcg_init,
    756						 (const u32)ARRAY_SIZE(rv730_mgcg_init));
    757		break;
    758	case CHIP_RV710:
    759		radeon_program_register_sequence(rdev,
    760						 r7xx_golden_registers,
    761						 (const u32)ARRAY_SIZE(r7xx_golden_registers));
    762		radeon_program_register_sequence(rdev,
    763						 r7xx_golden_dyn_gpr_registers,
    764						 (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
    765		radeon_program_register_sequence(rdev,
    766						 rv710_golden_registers,
    767						 (const u32)ARRAY_SIZE(rv710_golden_registers));
    768		radeon_program_register_sequence(rdev,
    769						 rv710_mgcg_init,
    770						 (const u32)ARRAY_SIZE(rv710_mgcg_init));
    771		break;
    772	case CHIP_RV740:
    773		radeon_program_register_sequence(rdev,
    774						 rv740_golden_registers,
    775						 (const u32)ARRAY_SIZE(rv740_golden_registers));
    776		radeon_program_register_sequence(rdev,
    777						 rv740_mgcg_init,
    778						 (const u32)ARRAY_SIZE(rv740_mgcg_init));
    779		break;
    780	default:
    781		break;
    782	}
    783}
    784
    785#define PCIE_BUS_CLK                10000
    786#define TCLK                        (PCIE_BUS_CLK / 10)
    787
    788/**
    789 * rv770_get_xclk - get the xclk
    790 *
    791 * @rdev: radeon_device pointer
    792 *
    793 * Returns the reference clock used by the gfx engine
    794 * (r7xx-cayman).
    795 */
    796u32 rv770_get_xclk(struct radeon_device *rdev)
    797{
    798	u32 reference_clock = rdev->clock.spll.reference_freq;
    799	u32 tmp = RREG32(CG_CLKPIN_CNTL);
    800
    801	if (tmp & MUX_TCLK_TO_XCLK)
    802		return TCLK;
    803
    804	if (tmp & XTALIN_DIVIDE)
    805		return reference_clock / 4;
    806
    807	return reference_clock;
    808}
    809
    810void rv770_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base, bool async)
    811{
    812	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
    813	struct drm_framebuffer *fb = radeon_crtc->base.primary->fb;
    814	u32 tmp = RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset);
    815	int i;
    816
    817	/* Lock the graphics update lock */
    818	tmp |= AVIVO_D1GRPH_UPDATE_LOCK;
    819	WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
    820
    821	/* flip at hsync for async, default is vsync */
    822	WREG32(AVIVO_D1GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
    823	       async ? AVIVO_D1GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
    824	/* update pitch */
    825	WREG32(AVIVO_D1GRPH_PITCH + radeon_crtc->crtc_offset,
    826	       fb->pitches[0] / fb->format->cpp[0]);
    827	/* update the scanout addresses */
    828	if (radeon_crtc->crtc_id) {
    829		WREG32(D2GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
    830		WREG32(D2GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
    831	} else {
    832		WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
    833		WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
    834	}
    835	WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
    836	       (u32)crtc_base);
    837	WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
    838	       (u32)crtc_base);
    839
    840	/* Wait for update_pending to go high. */
    841	for (i = 0; i < rdev->usec_timeout; i++) {
    842		if (RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING)
    843			break;
    844		udelay(1);
    845	}
    846	DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
    847
    848	/* Unlock the lock, so double-buffering can take place inside vblank */
    849	tmp &= ~AVIVO_D1GRPH_UPDATE_LOCK;
    850	WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
    851}
    852
    853bool rv770_page_flip_pending(struct radeon_device *rdev, int crtc_id)
    854{
    855	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
    856
    857	/* Return current update_pending status: */
    858	return !!(RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) &
    859		AVIVO_D1GRPH_SURFACE_UPDATE_PENDING);
    860}
    861
    862/* get temperature in millidegrees */
    863int rv770_get_temp(struct radeon_device *rdev)
    864{
    865	u32 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
    866		ASIC_T_SHIFT;
    867	int actual_temp;
    868
    869	if (temp & 0x400)
    870		actual_temp = -256;
    871	else if (temp & 0x200)
    872		actual_temp = 255;
    873	else if (temp & 0x100) {
    874		actual_temp = temp & 0x1ff;
    875		actual_temp |= ~0x1ff;
    876	} else
    877		actual_temp = temp & 0xff;
    878
    879	return (actual_temp * 1000) / 2;
    880}
    881
    882void rv770_pm_misc(struct radeon_device *rdev)
    883{
    884	int req_ps_idx = rdev->pm.requested_power_state_index;
    885	int req_cm_idx = rdev->pm.requested_clock_mode_index;
    886	struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
    887	struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
    888
    889	if ((voltage->type == VOLTAGE_SW) && voltage->voltage) {
    890		/* 0xff01 is a flag rather then an actual voltage */
    891		if (voltage->voltage == 0xff01)
    892			return;
    893		if (voltage->voltage != rdev->pm.current_vddc) {
    894			radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
    895			rdev->pm.current_vddc = voltage->voltage;
    896			DRM_DEBUG("Setting: v: %d\n", voltage->voltage);
    897		}
    898	}
    899}
    900
    901/*
    902 * GART
    903 */
    904static int rv770_pcie_gart_enable(struct radeon_device *rdev)
    905{
    906	u32 tmp;
    907	int r, i;
    908
    909	if (rdev->gart.robj == NULL) {
    910		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
    911		return -EINVAL;
    912	}
    913	r = radeon_gart_table_vram_pin(rdev);
    914	if (r)
    915		return r;
    916	/* Setup L2 cache */
    917	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
    918				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
    919				EFFECTIVE_L2_QUEUE_SIZE(7));
    920	WREG32(VM_L2_CNTL2, 0);
    921	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
    922	/* Setup TLB control */
    923	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
    924		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
    925		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
    926		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
    927	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
    928	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
    929	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
    930	if (rdev->family == CHIP_RV740)
    931		WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
    932	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
    933	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
    934	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
    935	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
    936	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
    937	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
    938	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
    939	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
    940				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
    941	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
    942			(u32)(rdev->dummy_page.addr >> 12));
    943	for (i = 1; i < 7; i++)
    944		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
    945
    946	r600_pcie_gart_tlb_flush(rdev);
    947	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
    948		 (unsigned)(rdev->mc.gtt_size >> 20),
    949		 (unsigned long long)rdev->gart.table_addr);
    950	rdev->gart.ready = true;
    951	return 0;
    952}
    953
    954static void rv770_pcie_gart_disable(struct radeon_device *rdev)
    955{
    956	u32 tmp;
    957	int i;
    958
    959	/* Disable all tables */
    960	for (i = 0; i < 7; i++)
    961		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
    962
    963	/* Setup L2 cache */
    964	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
    965				EFFECTIVE_L2_QUEUE_SIZE(7));
    966	WREG32(VM_L2_CNTL2, 0);
    967	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
    968	/* Setup TLB control */
    969	tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
    970	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
    971	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
    972	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
    973	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
    974	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
    975	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
    976	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
    977	radeon_gart_table_vram_unpin(rdev);
    978}
    979
    980static void rv770_pcie_gart_fini(struct radeon_device *rdev)
    981{
    982	radeon_gart_fini(rdev);
    983	rv770_pcie_gart_disable(rdev);
    984	radeon_gart_table_vram_free(rdev);
    985}
    986
    987
    988static void rv770_agp_enable(struct radeon_device *rdev)
    989{
    990	u32 tmp;
    991	int i;
    992
    993	/* Setup L2 cache */
    994	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
    995				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
    996				EFFECTIVE_L2_QUEUE_SIZE(7));
    997	WREG32(VM_L2_CNTL2, 0);
    998	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
    999	/* Setup TLB control */
   1000	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
   1001		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
   1002		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
   1003		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
   1004	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
   1005	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
   1006	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
   1007	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
   1008	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
   1009	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
   1010	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
   1011	for (i = 0; i < 7; i++)
   1012		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
   1013}
   1014
   1015static void rv770_mc_program(struct radeon_device *rdev)
   1016{
   1017	struct rv515_mc_save save;
   1018	u32 tmp;
   1019	int i, j;
   1020
   1021	/* Initialize HDP */
   1022	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
   1023		WREG32((0x2c14 + j), 0x00000000);
   1024		WREG32((0x2c18 + j), 0x00000000);
   1025		WREG32((0x2c1c + j), 0x00000000);
   1026		WREG32((0x2c20 + j), 0x00000000);
   1027		WREG32((0x2c24 + j), 0x00000000);
   1028	}
   1029	/* r7xx hw bug.  Read from HDP_DEBUG1 rather
   1030	 * than writing to HDP_REG_COHERENCY_FLUSH_CNTL
   1031	 */
   1032	tmp = RREG32(HDP_DEBUG1);
   1033
   1034	rv515_mc_stop(rdev, &save);
   1035	if (r600_mc_wait_for_idle(rdev)) {
   1036		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
   1037	}
   1038	/* Lockout access through VGA aperture*/
   1039	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
   1040	/* Update configuration */
   1041	if (rdev->flags & RADEON_IS_AGP) {
   1042		if (rdev->mc.vram_start < rdev->mc.gtt_start) {
   1043			/* VRAM before AGP */
   1044			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
   1045				rdev->mc.vram_start >> 12);
   1046			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
   1047				rdev->mc.gtt_end >> 12);
   1048		} else {
   1049			/* VRAM after AGP */
   1050			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
   1051				rdev->mc.gtt_start >> 12);
   1052			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
   1053				rdev->mc.vram_end >> 12);
   1054		}
   1055	} else {
   1056		WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
   1057			rdev->mc.vram_start >> 12);
   1058		WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
   1059			rdev->mc.vram_end >> 12);
   1060	}
   1061	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
   1062	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
   1063	tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
   1064	WREG32(MC_VM_FB_LOCATION, tmp);
   1065	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
   1066	WREG32(HDP_NONSURFACE_INFO, (2 << 7));
   1067	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
   1068	if (rdev->flags & RADEON_IS_AGP) {
   1069		WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
   1070		WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
   1071		WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
   1072	} else {
   1073		WREG32(MC_VM_AGP_BASE, 0);
   1074		WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
   1075		WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
   1076	}
   1077	if (r600_mc_wait_for_idle(rdev)) {
   1078		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
   1079	}
   1080	rv515_mc_resume(rdev, &save);
   1081	/* we need to own VRAM, so turn off the VGA renderer here
   1082	 * to stop it overwriting our objects */
   1083	rv515_vga_render_disable(rdev);
   1084}
   1085
   1086
   1087/*
   1088 * CP.
   1089 */
   1090void r700_cp_stop(struct radeon_device *rdev)
   1091{
   1092	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
   1093		radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
   1094	WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
   1095	WREG32(SCRATCH_UMSK, 0);
   1096	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
   1097}
   1098
   1099static int rv770_cp_load_microcode(struct radeon_device *rdev)
   1100{
   1101	const __be32 *fw_data;
   1102	int i;
   1103
   1104	if (!rdev->me_fw || !rdev->pfp_fw)
   1105		return -EINVAL;
   1106
   1107	r700_cp_stop(rdev);
   1108	WREG32(CP_RB_CNTL,
   1109#ifdef __BIG_ENDIAN
   1110	       BUF_SWAP_32BIT |
   1111#endif
   1112	       RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
   1113
   1114	/* Reset cp */
   1115	WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
   1116	RREG32(GRBM_SOFT_RESET);
   1117	mdelay(15);
   1118	WREG32(GRBM_SOFT_RESET, 0);
   1119
   1120	fw_data = (const __be32 *)rdev->pfp_fw->data;
   1121	WREG32(CP_PFP_UCODE_ADDR, 0);
   1122	for (i = 0; i < R700_PFP_UCODE_SIZE; i++)
   1123		WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
   1124	WREG32(CP_PFP_UCODE_ADDR, 0);
   1125
   1126	fw_data = (const __be32 *)rdev->me_fw->data;
   1127	WREG32(CP_ME_RAM_WADDR, 0);
   1128	for (i = 0; i < R700_PM4_UCODE_SIZE; i++)
   1129		WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
   1130
   1131	WREG32(CP_PFP_UCODE_ADDR, 0);
   1132	WREG32(CP_ME_RAM_WADDR, 0);
   1133	WREG32(CP_ME_RAM_RADDR, 0);
   1134	return 0;
   1135}
   1136
   1137void r700_cp_fini(struct radeon_device *rdev)
   1138{
   1139	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
   1140	r700_cp_stop(rdev);
   1141	radeon_ring_fini(rdev, ring);
   1142	radeon_scratch_free(rdev, ring->rptr_save_reg);
   1143}
   1144
   1145void rv770_set_clk_bypass_mode(struct radeon_device *rdev)
   1146{
   1147	u32 tmp, i;
   1148
   1149	if (rdev->flags & RADEON_IS_IGP)
   1150		return;
   1151
   1152	tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
   1153	tmp &= SCLK_MUX_SEL_MASK;
   1154	tmp |= SCLK_MUX_SEL(1) | SCLK_MUX_UPDATE;
   1155	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
   1156
   1157	for (i = 0; i < rdev->usec_timeout; i++) {
   1158		if (RREG32(CG_SPLL_STATUS) & SPLL_CHG_STATUS)
   1159			break;
   1160		udelay(1);
   1161	}
   1162
   1163	tmp &= ~SCLK_MUX_UPDATE;
   1164	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
   1165
   1166	tmp = RREG32(MPLL_CNTL_MODE);
   1167	if ((rdev->family == CHIP_RV710) || (rdev->family == CHIP_RV730))
   1168		tmp &= ~RV730_MPLL_MCLK_SEL;
   1169	else
   1170		tmp &= ~MPLL_MCLK_SEL;
   1171	WREG32(MPLL_CNTL_MODE, tmp);
   1172}
   1173
   1174/*
   1175 * Core functions
   1176 */
   1177static void rv770_gpu_init(struct radeon_device *rdev)
   1178{
   1179	int i, j, num_qd_pipes;
   1180	u32 ta_aux_cntl;
   1181	u32 sx_debug_1;
   1182	u32 smx_dc_ctl0;
   1183	u32 db_debug3;
   1184	u32 num_gs_verts_per_thread;
   1185	u32 vgt_gs_per_es;
   1186	u32 gs_prim_buffer_depth = 0;
   1187	u32 sq_ms_fifo_sizes;
   1188	u32 sq_config;
   1189	u32 sq_thread_resource_mgmt;
   1190	u32 hdp_host_path_cntl;
   1191	u32 sq_dyn_gpr_size_simd_ab_0;
   1192	u32 gb_tiling_config = 0;
   1193	u32 cc_gc_shader_pipe_config = 0;
   1194	u32 mc_arb_ramcfg;
   1195	u32 db_debug4, tmp;
   1196	u32 inactive_pipes, shader_pipe_config;
   1197	u32 disabled_rb_mask;
   1198	unsigned active_number;
   1199
   1200	/* setup chip specs */
   1201	rdev->config.rv770.tiling_group_size = 256;
   1202	switch (rdev->family) {
   1203	case CHIP_RV770:
   1204		rdev->config.rv770.max_pipes = 4;
   1205		rdev->config.rv770.max_tile_pipes = 8;
   1206		rdev->config.rv770.max_simds = 10;
   1207		rdev->config.rv770.max_backends = 4;
   1208		rdev->config.rv770.max_gprs = 256;
   1209		rdev->config.rv770.max_threads = 248;
   1210		rdev->config.rv770.max_stack_entries = 512;
   1211		rdev->config.rv770.max_hw_contexts = 8;
   1212		rdev->config.rv770.max_gs_threads = 16 * 2;
   1213		rdev->config.rv770.sx_max_export_size = 128;
   1214		rdev->config.rv770.sx_max_export_pos_size = 16;
   1215		rdev->config.rv770.sx_max_export_smx_size = 112;
   1216		rdev->config.rv770.sq_num_cf_insts = 2;
   1217
   1218		rdev->config.rv770.sx_num_of_sets = 7;
   1219		rdev->config.rv770.sc_prim_fifo_size = 0xF9;
   1220		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
   1221		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
   1222		break;
   1223	case CHIP_RV730:
   1224		rdev->config.rv770.max_pipes = 2;
   1225		rdev->config.rv770.max_tile_pipes = 4;
   1226		rdev->config.rv770.max_simds = 8;
   1227		rdev->config.rv770.max_backends = 2;
   1228		rdev->config.rv770.max_gprs = 128;
   1229		rdev->config.rv770.max_threads = 248;
   1230		rdev->config.rv770.max_stack_entries = 256;
   1231		rdev->config.rv770.max_hw_contexts = 8;
   1232		rdev->config.rv770.max_gs_threads = 16 * 2;
   1233		rdev->config.rv770.sx_max_export_size = 256;
   1234		rdev->config.rv770.sx_max_export_pos_size = 32;
   1235		rdev->config.rv770.sx_max_export_smx_size = 224;
   1236		rdev->config.rv770.sq_num_cf_insts = 2;
   1237
   1238		rdev->config.rv770.sx_num_of_sets = 7;
   1239		rdev->config.rv770.sc_prim_fifo_size = 0xf9;
   1240		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
   1241		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
   1242		if (rdev->config.rv770.sx_max_export_pos_size > 16) {
   1243			rdev->config.rv770.sx_max_export_pos_size -= 16;
   1244			rdev->config.rv770.sx_max_export_smx_size += 16;
   1245		}
   1246		break;
   1247	case CHIP_RV710:
   1248		rdev->config.rv770.max_pipes = 2;
   1249		rdev->config.rv770.max_tile_pipes = 2;
   1250		rdev->config.rv770.max_simds = 2;
   1251		rdev->config.rv770.max_backends = 1;
   1252		rdev->config.rv770.max_gprs = 256;
   1253		rdev->config.rv770.max_threads = 192;
   1254		rdev->config.rv770.max_stack_entries = 256;
   1255		rdev->config.rv770.max_hw_contexts = 4;
   1256		rdev->config.rv770.max_gs_threads = 8 * 2;
   1257		rdev->config.rv770.sx_max_export_size = 128;
   1258		rdev->config.rv770.sx_max_export_pos_size = 16;
   1259		rdev->config.rv770.sx_max_export_smx_size = 112;
   1260		rdev->config.rv770.sq_num_cf_insts = 1;
   1261
   1262		rdev->config.rv770.sx_num_of_sets = 7;
   1263		rdev->config.rv770.sc_prim_fifo_size = 0x40;
   1264		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
   1265		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
   1266		break;
   1267	case CHIP_RV740:
   1268		rdev->config.rv770.max_pipes = 4;
   1269		rdev->config.rv770.max_tile_pipes = 4;
   1270		rdev->config.rv770.max_simds = 8;
   1271		rdev->config.rv770.max_backends = 4;
   1272		rdev->config.rv770.max_gprs = 256;
   1273		rdev->config.rv770.max_threads = 248;
   1274		rdev->config.rv770.max_stack_entries = 512;
   1275		rdev->config.rv770.max_hw_contexts = 8;
   1276		rdev->config.rv770.max_gs_threads = 16 * 2;
   1277		rdev->config.rv770.sx_max_export_size = 256;
   1278		rdev->config.rv770.sx_max_export_pos_size = 32;
   1279		rdev->config.rv770.sx_max_export_smx_size = 224;
   1280		rdev->config.rv770.sq_num_cf_insts = 2;
   1281
   1282		rdev->config.rv770.sx_num_of_sets = 7;
   1283		rdev->config.rv770.sc_prim_fifo_size = 0x100;
   1284		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
   1285		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
   1286
   1287		if (rdev->config.rv770.sx_max_export_pos_size > 16) {
   1288			rdev->config.rv770.sx_max_export_pos_size -= 16;
   1289			rdev->config.rv770.sx_max_export_smx_size += 16;
   1290		}
   1291		break;
   1292	default:
   1293		break;
   1294	}
   1295
   1296	/* Initialize HDP */
   1297	j = 0;
   1298	for (i = 0; i < 32; i++) {
   1299		WREG32((0x2c14 + j), 0x00000000);
   1300		WREG32((0x2c18 + j), 0x00000000);
   1301		WREG32((0x2c1c + j), 0x00000000);
   1302		WREG32((0x2c20 + j), 0x00000000);
   1303		WREG32((0x2c24 + j), 0x00000000);
   1304		j += 0x18;
   1305	}
   1306
   1307	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
   1308
   1309	/* setup tiling, simd, pipe config */
   1310	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
   1311
   1312	shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG);
   1313	inactive_pipes = (shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> INACTIVE_QD_PIPES_SHIFT;
   1314	for (i = 0, tmp = 1, active_number = 0; i < R7XX_MAX_PIPES; i++) {
   1315		if (!(inactive_pipes & tmp)) {
   1316			active_number++;
   1317		}
   1318		tmp <<= 1;
   1319	}
   1320	if (active_number == 1) {
   1321		WREG32(SPI_CONFIG_CNTL, DISABLE_INTERP_1);
   1322	} else {
   1323		WREG32(SPI_CONFIG_CNTL, 0);
   1324	}
   1325
   1326	cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00;
   1327	tmp = rdev->config.rv770.max_simds -
   1328		r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R7XX_MAX_SIMDS_MASK);
   1329	rdev->config.rv770.active_simds = tmp;
   1330
   1331	switch (rdev->config.rv770.max_tile_pipes) {
   1332	case 1:
   1333	default:
   1334		gb_tiling_config = PIPE_TILING(0);
   1335		break;
   1336	case 2:
   1337		gb_tiling_config = PIPE_TILING(1);
   1338		break;
   1339	case 4:
   1340		gb_tiling_config = PIPE_TILING(2);
   1341		break;
   1342	case 8:
   1343		gb_tiling_config = PIPE_TILING(3);
   1344		break;
   1345	}
   1346	rdev->config.rv770.tiling_npipes = rdev->config.rv770.max_tile_pipes;
   1347
   1348	disabled_rb_mask = (RREG32(CC_RB_BACKEND_DISABLE) >> 16) & R7XX_MAX_BACKENDS_MASK;
   1349	tmp = 0;
   1350	for (i = 0; i < rdev->config.rv770.max_backends; i++)
   1351		tmp |= (1 << i);
   1352	/* if all the backends are disabled, fix it up here */
   1353	if ((disabled_rb_mask & tmp) == tmp) {
   1354		for (i = 0; i < rdev->config.rv770.max_backends; i++)
   1355			disabled_rb_mask &= ~(1 << i);
   1356	}
   1357	tmp = (gb_tiling_config & PIPE_TILING__MASK) >> PIPE_TILING__SHIFT;
   1358	tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.rv770.max_backends,
   1359					R7XX_MAX_BACKENDS, disabled_rb_mask);
   1360	gb_tiling_config |= tmp << 16;
   1361	rdev->config.rv770.backend_map = tmp;
   1362
   1363	if (rdev->family == CHIP_RV770)
   1364		gb_tiling_config |= BANK_TILING(1);
   1365	else {
   1366		if ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT)
   1367			gb_tiling_config |= BANK_TILING(1);
   1368		else
   1369			gb_tiling_config |= BANK_TILING(0);
   1370	}
   1371	rdev->config.rv770.tiling_nbanks = 4 << ((gb_tiling_config >> 4) & 0x3);
   1372	gb_tiling_config |= GROUP_SIZE((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
   1373	if (((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT) > 3) {
   1374		gb_tiling_config |= ROW_TILING(3);
   1375		gb_tiling_config |= SAMPLE_SPLIT(3);
   1376	} else {
   1377		gb_tiling_config |=
   1378			ROW_TILING(((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT));
   1379		gb_tiling_config |=
   1380			SAMPLE_SPLIT(((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT));
   1381	}
   1382
   1383	gb_tiling_config |= BANK_SWAPS(1);
   1384	rdev->config.rv770.tile_config = gb_tiling_config;
   1385
   1386	WREG32(GB_TILING_CONFIG, gb_tiling_config);
   1387	WREG32(DCP_TILING_CONFIG, (gb_tiling_config & 0xffff));
   1388	WREG32(HDP_TILING_CONFIG, (gb_tiling_config & 0xffff));
   1389	WREG32(DMA_TILING_CONFIG, (gb_tiling_config & 0xffff));
   1390	WREG32(DMA_TILING_CONFIG2, (gb_tiling_config & 0xffff));
   1391	if (rdev->family == CHIP_RV730) {
   1392		WREG32(UVD_UDEC_DB_TILING_CONFIG, (gb_tiling_config & 0xffff));
   1393		WREG32(UVD_UDEC_DBW_TILING_CONFIG, (gb_tiling_config & 0xffff));
   1394		WREG32(UVD_UDEC_TILING_CONFIG, (gb_tiling_config & 0xffff));
   1395	}
   1396
   1397	WREG32(CGTS_SYS_TCC_DISABLE, 0);
   1398	WREG32(CGTS_TCC_DISABLE, 0);
   1399	WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
   1400	WREG32(CGTS_USER_TCC_DISABLE, 0);
   1401
   1402
   1403	num_qd_pipes = R7XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
   1404	WREG32(VGT_OUT_DEALLOC_CNTL, (num_qd_pipes * 4) & DEALLOC_DIST_MASK);
   1405	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((num_qd_pipes * 4) - 2) & VTX_REUSE_DEPTH_MASK);
   1406
   1407	/* set HW defaults for 3D engine */
   1408	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
   1409				     ROQ_IB2_START(0x2b)));
   1410
   1411	WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
   1412
   1413	ta_aux_cntl = RREG32(TA_CNTL_AUX);
   1414	WREG32(TA_CNTL_AUX, ta_aux_cntl | DISABLE_CUBE_ANISO);
   1415
   1416	sx_debug_1 = RREG32(SX_DEBUG_1);
   1417	sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
   1418	WREG32(SX_DEBUG_1, sx_debug_1);
   1419
   1420	smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
   1421	smx_dc_ctl0 &= ~CACHE_DEPTH(0x1ff);
   1422	smx_dc_ctl0 |= CACHE_DEPTH((rdev->config.rv770.sx_num_of_sets * 64) - 1);
   1423	WREG32(SMX_DC_CTL0, smx_dc_ctl0);
   1424
   1425	if (rdev->family != CHIP_RV740)
   1426		WREG32(SMX_EVENT_CTL, (ES_FLUSH_CTL(4) |
   1427				       GS_FLUSH_CTL(4) |
   1428				       ACK_FLUSH_CTL(3) |
   1429				       SYNC_FLUSH_CTL));
   1430
   1431	if (rdev->family != CHIP_RV770)
   1432		WREG32(SMX_SAR_CTL0, 0x00003f3f);
   1433
   1434	db_debug3 = RREG32(DB_DEBUG3);
   1435	db_debug3 &= ~DB_CLK_OFF_DELAY(0x1f);
   1436	switch (rdev->family) {
   1437	case CHIP_RV770:
   1438	case CHIP_RV740:
   1439		db_debug3 |= DB_CLK_OFF_DELAY(0x1f);
   1440		break;
   1441	case CHIP_RV710:
   1442	case CHIP_RV730:
   1443	default:
   1444		db_debug3 |= DB_CLK_OFF_DELAY(2);
   1445		break;
   1446	}
   1447	WREG32(DB_DEBUG3, db_debug3);
   1448
   1449	if (rdev->family != CHIP_RV770) {
   1450		db_debug4 = RREG32(DB_DEBUG4);
   1451		db_debug4 |= DISABLE_TILE_COVERED_FOR_PS_ITER;
   1452		WREG32(DB_DEBUG4, db_debug4);
   1453	}
   1454
   1455	WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.rv770.sx_max_export_size / 4) - 1) |
   1456					POSITION_BUFFER_SIZE((rdev->config.rv770.sx_max_export_pos_size / 4) - 1) |
   1457					SMX_BUFFER_SIZE((rdev->config.rv770.sx_max_export_smx_size / 4) - 1)));
   1458
   1459	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.rv770.sc_prim_fifo_size) |
   1460				 SC_HIZ_TILE_FIFO_SIZE(rdev->config.rv770.sc_hiz_tile_fifo_size) |
   1461				 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.rv770.sc_earlyz_tile_fifo_fize)));
   1462
   1463	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
   1464
   1465	WREG32(VGT_NUM_INSTANCES, 1);
   1466
   1467	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
   1468
   1469	WREG32(CP_PERFMON_CNTL, 0);
   1470
   1471	sq_ms_fifo_sizes = (CACHE_FIFO_SIZE(16 * rdev->config.rv770.sq_num_cf_insts) |
   1472			    DONE_FIFO_HIWATER(0xe0) |
   1473			    ALU_UPDATE_FIFO_HIWATER(0x8));
   1474	switch (rdev->family) {
   1475	case CHIP_RV770:
   1476	case CHIP_RV730:
   1477	case CHIP_RV710:
   1478		sq_ms_fifo_sizes |= FETCH_FIFO_HIWATER(0x1);
   1479		break;
   1480	case CHIP_RV740:
   1481	default:
   1482		sq_ms_fifo_sizes |= FETCH_FIFO_HIWATER(0x4);
   1483		break;
   1484	}
   1485	WREG32(SQ_MS_FIFO_SIZES, sq_ms_fifo_sizes);
   1486
   1487	/* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
   1488	 * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
   1489	 */
   1490	sq_config = RREG32(SQ_CONFIG);
   1491	sq_config &= ~(PS_PRIO(3) |
   1492		       VS_PRIO(3) |
   1493		       GS_PRIO(3) |
   1494		       ES_PRIO(3));
   1495	sq_config |= (DX9_CONSTS |
   1496		      VC_ENABLE |
   1497		      EXPORT_SRC_C |
   1498		      PS_PRIO(0) |
   1499		      VS_PRIO(1) |
   1500		      GS_PRIO(2) |
   1501		      ES_PRIO(3));
   1502	if (rdev->family == CHIP_RV710)
   1503		/* no vertex cache */
   1504		sq_config &= ~VC_ENABLE;
   1505
   1506	WREG32(SQ_CONFIG, sq_config);
   1507
   1508	WREG32(SQ_GPR_RESOURCE_MGMT_1,  (NUM_PS_GPRS((rdev->config.rv770.max_gprs * 24)/64) |
   1509					 NUM_VS_GPRS((rdev->config.rv770.max_gprs * 24)/64) |
   1510					 NUM_CLAUSE_TEMP_GPRS(((rdev->config.rv770.max_gprs * 24)/64)/2)));
   1511
   1512	WREG32(SQ_GPR_RESOURCE_MGMT_2,  (NUM_GS_GPRS((rdev->config.rv770.max_gprs * 7)/64) |
   1513					 NUM_ES_GPRS((rdev->config.rv770.max_gprs * 7)/64)));
   1514
   1515	sq_thread_resource_mgmt = (NUM_PS_THREADS((rdev->config.rv770.max_threads * 4)/8) |
   1516				   NUM_VS_THREADS((rdev->config.rv770.max_threads * 2)/8) |
   1517				   NUM_ES_THREADS((rdev->config.rv770.max_threads * 1)/8));
   1518	if (((rdev->config.rv770.max_threads * 1) / 8) > rdev->config.rv770.max_gs_threads)
   1519		sq_thread_resource_mgmt |= NUM_GS_THREADS(rdev->config.rv770.max_gs_threads);
   1520	else
   1521		sq_thread_resource_mgmt |= NUM_GS_THREADS((rdev->config.rv770.max_gs_threads * 1)/8);
   1522	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
   1523
   1524	WREG32(SQ_STACK_RESOURCE_MGMT_1, (NUM_PS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4) |
   1525						     NUM_VS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4)));
   1526
   1527	WREG32(SQ_STACK_RESOURCE_MGMT_2, (NUM_GS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4) |
   1528						     NUM_ES_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4)));
   1529
   1530	sq_dyn_gpr_size_simd_ab_0 = (SIMDA_RING0((rdev->config.rv770.max_gprs * 38)/64) |
   1531				     SIMDA_RING1((rdev->config.rv770.max_gprs * 38)/64) |
   1532				     SIMDB_RING0((rdev->config.rv770.max_gprs * 38)/64) |
   1533				     SIMDB_RING1((rdev->config.rv770.max_gprs * 38)/64));
   1534
   1535	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_0, sq_dyn_gpr_size_simd_ab_0);
   1536	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_1, sq_dyn_gpr_size_simd_ab_0);
   1537	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_2, sq_dyn_gpr_size_simd_ab_0);
   1538	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_3, sq_dyn_gpr_size_simd_ab_0);
   1539	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_4, sq_dyn_gpr_size_simd_ab_0);
   1540	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_5, sq_dyn_gpr_size_simd_ab_0);
   1541	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_6, sq_dyn_gpr_size_simd_ab_0);
   1542	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_7, sq_dyn_gpr_size_simd_ab_0);
   1543
   1544	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
   1545					  FORCE_EOV_MAX_REZ_CNT(255)));
   1546
   1547	if (rdev->family == CHIP_RV710)
   1548		WREG32(VGT_CACHE_INVALIDATION, (CACHE_INVALIDATION(TC_ONLY) |
   1549						AUTO_INVLD_EN(ES_AND_GS_AUTO)));
   1550	else
   1551		WREG32(VGT_CACHE_INVALIDATION, (CACHE_INVALIDATION(VC_AND_TC) |
   1552						AUTO_INVLD_EN(ES_AND_GS_AUTO)));
   1553
   1554	switch (rdev->family) {
   1555	case CHIP_RV770:
   1556	case CHIP_RV730:
   1557	case CHIP_RV740:
   1558		gs_prim_buffer_depth = 384;
   1559		break;
   1560	case CHIP_RV710:
   1561		gs_prim_buffer_depth = 128;
   1562		break;
   1563	default:
   1564		break;
   1565	}
   1566
   1567	num_gs_verts_per_thread = rdev->config.rv770.max_pipes * 16;
   1568	vgt_gs_per_es = gs_prim_buffer_depth + num_gs_verts_per_thread;
   1569	/* Max value for this is 256 */
   1570	if (vgt_gs_per_es > 256)
   1571		vgt_gs_per_es = 256;
   1572
   1573	WREG32(VGT_ES_PER_GS, 128);
   1574	WREG32(VGT_GS_PER_ES, vgt_gs_per_es);
   1575	WREG32(VGT_GS_PER_VS, 2);
   1576
   1577	/* more default values. 2D/3D driver should adjust as needed */
   1578	WREG32(VGT_GS_VERTEX_REUSE, 16);
   1579	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
   1580	WREG32(VGT_STRMOUT_EN, 0);
   1581	WREG32(SX_MISC, 0);
   1582	WREG32(PA_SC_MODE_CNTL, 0);
   1583	WREG32(PA_SC_EDGERULE, 0xaaaaaaaa);
   1584	WREG32(PA_SC_AA_CONFIG, 0);
   1585	WREG32(PA_SC_CLIPRECT_RULE, 0xffff);
   1586	WREG32(PA_SC_LINE_STIPPLE, 0);
   1587	WREG32(SPI_INPUT_Z, 0);
   1588	WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
   1589	WREG32(CB_COLOR7_FRAG, 0);
   1590
   1591	/* clear render buffer base addresses */
   1592	WREG32(CB_COLOR0_BASE, 0);
   1593	WREG32(CB_COLOR1_BASE, 0);
   1594	WREG32(CB_COLOR2_BASE, 0);
   1595	WREG32(CB_COLOR3_BASE, 0);
   1596	WREG32(CB_COLOR4_BASE, 0);
   1597	WREG32(CB_COLOR5_BASE, 0);
   1598	WREG32(CB_COLOR6_BASE, 0);
   1599	WREG32(CB_COLOR7_BASE, 0);
   1600
   1601	WREG32(TCP_CNTL, 0);
   1602
   1603	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
   1604	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
   1605
   1606	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
   1607
   1608	WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA |
   1609					  NUM_CLIP_SEQ(3)));
   1610	WREG32(VC_ENHANCE, 0);
   1611}
   1612
   1613void r700_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
   1614{
   1615	u64 size_bf, size_af;
   1616
   1617	if (mc->mc_vram_size > 0xE0000000) {
   1618		/* leave room for at least 512M GTT */
   1619		dev_warn(rdev->dev, "limiting VRAM\n");
   1620		mc->real_vram_size = 0xE0000000;
   1621		mc->mc_vram_size = 0xE0000000;
   1622	}
   1623	if (rdev->flags & RADEON_IS_AGP) {
   1624		size_bf = mc->gtt_start;
   1625		size_af = mc->mc_mask - mc->gtt_end;
   1626		if (size_bf > size_af) {
   1627			if (mc->mc_vram_size > size_bf) {
   1628				dev_warn(rdev->dev, "limiting VRAM\n");
   1629				mc->real_vram_size = size_bf;
   1630				mc->mc_vram_size = size_bf;
   1631			}
   1632			mc->vram_start = mc->gtt_start - mc->mc_vram_size;
   1633		} else {
   1634			if (mc->mc_vram_size > size_af) {
   1635				dev_warn(rdev->dev, "limiting VRAM\n");
   1636				mc->real_vram_size = size_af;
   1637				mc->mc_vram_size = size_af;
   1638			}
   1639			mc->vram_start = mc->gtt_end + 1;
   1640		}
   1641		mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
   1642		dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
   1643				mc->mc_vram_size >> 20, mc->vram_start,
   1644				mc->vram_end, mc->real_vram_size >> 20);
   1645	} else {
   1646		radeon_vram_location(rdev, &rdev->mc, 0);
   1647		rdev->mc.gtt_base_align = 0;
   1648		radeon_gtt_location(rdev, mc);
   1649	}
   1650}
   1651
   1652static int rv770_mc_init(struct radeon_device *rdev)
   1653{
   1654	u32 tmp;
   1655	int chansize, numchan;
   1656
   1657	/* Get VRAM informations */
   1658	rdev->mc.vram_is_ddr = true;
   1659	tmp = RREG32(MC_ARB_RAMCFG);
   1660	if (tmp & CHANSIZE_OVERRIDE) {
   1661		chansize = 16;
   1662	} else if (tmp & CHANSIZE_MASK) {
   1663		chansize = 64;
   1664	} else {
   1665		chansize = 32;
   1666	}
   1667	tmp = RREG32(MC_SHARED_CHMAP);
   1668	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
   1669	case 0:
   1670	default:
   1671		numchan = 1;
   1672		break;
   1673	case 1:
   1674		numchan = 2;
   1675		break;
   1676	case 2:
   1677		numchan = 4;
   1678		break;
   1679	case 3:
   1680		numchan = 8;
   1681		break;
   1682	}
   1683	rdev->mc.vram_width = numchan * chansize;
   1684	/* Could aper size report 0 ? */
   1685	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
   1686	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
   1687	/* Setup GPU memory space */
   1688	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
   1689	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
   1690	rdev->mc.visible_vram_size = rdev->mc.aper_size;
   1691	r700_vram_gtt_location(rdev, &rdev->mc);
   1692	radeon_update_bandwidth_info(rdev);
   1693
   1694	return 0;
   1695}
   1696
   1697static void rv770_uvd_init(struct radeon_device *rdev)
   1698{
   1699	int r;
   1700
   1701	if (!rdev->has_uvd)
   1702		return;
   1703
   1704	r = radeon_uvd_init(rdev);
   1705	if (r) {
   1706		dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
   1707		/*
   1708		 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
   1709		 * to early fails uvd_v2_2_resume() and thus nothing happens
   1710		 * there. So it is pointless to try to go through that code
   1711		 * hence why we disable uvd here.
   1712		 */
   1713		rdev->has_uvd = false;
   1714		return;
   1715	}
   1716	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
   1717	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
   1718}
   1719
   1720static void rv770_uvd_start(struct radeon_device *rdev)
   1721{
   1722	int r;
   1723
   1724	if (!rdev->has_uvd)
   1725		return;
   1726
   1727	r = uvd_v2_2_resume(rdev);
   1728	if (r) {
   1729		dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
   1730		goto error;
   1731	}
   1732	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
   1733	if (r) {
   1734		dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
   1735		goto error;
   1736	}
   1737	return;
   1738
   1739error:
   1740	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
   1741}
   1742
   1743static void rv770_uvd_resume(struct radeon_device *rdev)
   1744{
   1745	struct radeon_ring *ring;
   1746	int r;
   1747
   1748	if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
   1749		return;
   1750
   1751	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
   1752	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
   1753	if (r) {
   1754		dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
   1755		return;
   1756	}
   1757	r = uvd_v1_0_init(rdev);
   1758	if (r) {
   1759		dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
   1760		return;
   1761	}
   1762}
   1763
   1764static int rv770_startup(struct radeon_device *rdev)
   1765{
   1766	struct radeon_ring *ring;
   1767	int r;
   1768
   1769	/* enable pcie gen2 link */
   1770	rv770_pcie_gen2_enable(rdev);
   1771
   1772	/* scratch needs to be initialized before MC */
   1773	r = r600_vram_scratch_init(rdev);
   1774	if (r)
   1775		return r;
   1776
   1777	rv770_mc_program(rdev);
   1778
   1779	if (rdev->flags & RADEON_IS_AGP) {
   1780		rv770_agp_enable(rdev);
   1781	} else {
   1782		r = rv770_pcie_gart_enable(rdev);
   1783		if (r)
   1784			return r;
   1785	}
   1786
   1787	rv770_gpu_init(rdev);
   1788
   1789	/* allocate wb buffer */
   1790	r = radeon_wb_init(rdev);
   1791	if (r)
   1792		return r;
   1793
   1794	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
   1795	if (r) {
   1796		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
   1797		return r;
   1798	}
   1799
   1800	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
   1801	if (r) {
   1802		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
   1803		return r;
   1804	}
   1805
   1806	rv770_uvd_start(rdev);
   1807
   1808	/* Enable IRQ */
   1809	if (!rdev->irq.installed) {
   1810		r = radeon_irq_kms_init(rdev);
   1811		if (r)
   1812			return r;
   1813	}
   1814
   1815	r = r600_irq_init(rdev);
   1816	if (r) {
   1817		DRM_ERROR("radeon: IH init failed (%d).\n", r);
   1818		radeon_irq_kms_fini(rdev);
   1819		return r;
   1820	}
   1821	r600_irq_set(rdev);
   1822
   1823	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
   1824	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
   1825			     RADEON_CP_PACKET2);
   1826	if (r)
   1827		return r;
   1828
   1829	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
   1830	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
   1831			     DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
   1832	if (r)
   1833		return r;
   1834
   1835	r = rv770_cp_load_microcode(rdev);
   1836	if (r)
   1837		return r;
   1838	r = r600_cp_resume(rdev);
   1839	if (r)
   1840		return r;
   1841
   1842	r = r600_dma_resume(rdev);
   1843	if (r)
   1844		return r;
   1845
   1846	rv770_uvd_resume(rdev);
   1847
   1848	r = radeon_ib_pool_init(rdev);
   1849	if (r) {
   1850		dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
   1851		return r;
   1852	}
   1853
   1854	r = radeon_audio_init(rdev);
   1855	if (r) {
   1856		DRM_ERROR("radeon: audio init failed\n");
   1857		return r;
   1858	}
   1859
   1860	return 0;
   1861}
   1862
   1863int rv770_resume(struct radeon_device *rdev)
   1864{
   1865	int r;
   1866
   1867	/* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
   1868	 * posting will perform necessary task to bring back GPU into good
   1869	 * shape.
   1870	 */
   1871	/* post card */
   1872	atom_asic_init(rdev->mode_info.atom_context);
   1873
   1874	/* init golden registers */
   1875	rv770_init_golden_registers(rdev);
   1876
   1877	if (rdev->pm.pm_method == PM_METHOD_DPM)
   1878		radeon_pm_resume(rdev);
   1879
   1880	rdev->accel_working = true;
   1881	r = rv770_startup(rdev);
   1882	if (r) {
   1883		DRM_ERROR("r600 startup failed on resume\n");
   1884		rdev->accel_working = false;
   1885		return r;
   1886	}
   1887
   1888	return r;
   1889
   1890}
   1891
   1892int rv770_suspend(struct radeon_device *rdev)
   1893{
   1894	radeon_pm_suspend(rdev);
   1895	radeon_audio_fini(rdev);
   1896	if (rdev->has_uvd) {
   1897		radeon_uvd_suspend(rdev);
   1898		uvd_v1_0_fini(rdev);
   1899	}
   1900	r700_cp_stop(rdev);
   1901	r600_dma_stop(rdev);
   1902	r600_irq_suspend(rdev);
   1903	radeon_wb_disable(rdev);
   1904	rv770_pcie_gart_disable(rdev);
   1905
   1906	return 0;
   1907}
   1908
   1909/* Plan is to move initialization in that function and use
   1910 * helper function so that radeon_device_init pretty much
   1911 * do nothing more than calling asic specific function. This
   1912 * should also allow to remove a bunch of callback function
   1913 * like vram_info.
   1914 */
   1915int rv770_init(struct radeon_device *rdev)
   1916{
   1917	int r;
   1918
   1919	/* Read BIOS */
   1920	if (!radeon_get_bios(rdev)) {
   1921		if (ASIC_IS_AVIVO(rdev))
   1922			return -EINVAL;
   1923	}
   1924	/* Must be an ATOMBIOS */
   1925	if (!rdev->is_atom_bios) {
   1926		dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
   1927		return -EINVAL;
   1928	}
   1929	r = radeon_atombios_init(rdev);
   1930	if (r)
   1931		return r;
   1932	/* Post card if necessary */
   1933	if (!radeon_card_posted(rdev)) {
   1934		if (!rdev->bios) {
   1935			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
   1936			return -EINVAL;
   1937		}
   1938		DRM_INFO("GPU not posted. posting now...\n");
   1939		atom_asic_init(rdev->mode_info.atom_context);
   1940	}
   1941	/* init golden registers */
   1942	rv770_init_golden_registers(rdev);
   1943	/* Initialize scratch registers */
   1944	r600_scratch_init(rdev);
   1945	/* Initialize surface registers */
   1946	radeon_surface_init(rdev);
   1947	/* Initialize clocks */
   1948	radeon_get_clock_info(rdev->ddev);
   1949	/* Fence driver */
   1950	radeon_fence_driver_init(rdev);
   1951	/* initialize AGP */
   1952	if (rdev->flags & RADEON_IS_AGP) {
   1953		r = radeon_agp_init(rdev);
   1954		if (r)
   1955			radeon_agp_disable(rdev);
   1956	}
   1957	r = rv770_mc_init(rdev);
   1958	if (r)
   1959		return r;
   1960	/* Memory manager */
   1961	r = radeon_bo_init(rdev);
   1962	if (r)
   1963		return r;
   1964
   1965	if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
   1966		r = r600_init_microcode(rdev);
   1967		if (r) {
   1968			DRM_ERROR("Failed to load firmware!\n");
   1969			return r;
   1970		}
   1971	}
   1972
   1973	/* Initialize power management */
   1974	radeon_pm_init(rdev);
   1975
   1976	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
   1977	r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
   1978
   1979	rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
   1980	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
   1981
   1982	rv770_uvd_init(rdev);
   1983
   1984	rdev->ih.ring_obj = NULL;
   1985	r600_ih_ring_init(rdev, 64 * 1024);
   1986
   1987	r = r600_pcie_gart_init(rdev);
   1988	if (r)
   1989		return r;
   1990
   1991	rdev->accel_working = true;
   1992	r = rv770_startup(rdev);
   1993	if (r) {
   1994		dev_err(rdev->dev, "disabling GPU acceleration\n");
   1995		r700_cp_fini(rdev);
   1996		r600_dma_fini(rdev);
   1997		r600_irq_fini(rdev);
   1998		radeon_wb_fini(rdev);
   1999		radeon_ib_pool_fini(rdev);
   2000		radeon_irq_kms_fini(rdev);
   2001		rv770_pcie_gart_fini(rdev);
   2002		rdev->accel_working = false;
   2003	}
   2004
   2005	return 0;
   2006}
   2007
   2008void rv770_fini(struct radeon_device *rdev)
   2009{
   2010	radeon_pm_fini(rdev);
   2011	r700_cp_fini(rdev);
   2012	r600_dma_fini(rdev);
   2013	r600_irq_fini(rdev);
   2014	radeon_wb_fini(rdev);
   2015	radeon_ib_pool_fini(rdev);
   2016	radeon_irq_kms_fini(rdev);
   2017	uvd_v1_0_fini(rdev);
   2018	radeon_uvd_fini(rdev);
   2019	rv770_pcie_gart_fini(rdev);
   2020	r600_vram_scratch_fini(rdev);
   2021	radeon_gem_fini(rdev);
   2022	radeon_fence_driver_fini(rdev);
   2023	radeon_agp_fini(rdev);
   2024	radeon_bo_fini(rdev);
   2025	radeon_atombios_fini(rdev);
   2026	kfree(rdev->bios);
   2027	rdev->bios = NULL;
   2028}
   2029
   2030static void rv770_pcie_gen2_enable(struct radeon_device *rdev)
   2031{
   2032	u32 link_width_cntl, lanes, speed_cntl, tmp;
   2033	u16 link_cntl2;
   2034
   2035	if (radeon_pcie_gen2 == 0)
   2036		return;
   2037
   2038	if (rdev->flags & RADEON_IS_IGP)
   2039		return;
   2040
   2041	if (!(rdev->flags & RADEON_IS_PCIE))
   2042		return;
   2043
   2044	/* x2 cards have a special sequence */
   2045	if (ASIC_IS_X2(rdev))
   2046		return;
   2047
   2048	if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
   2049		(rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
   2050		return;
   2051
   2052	DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
   2053
   2054	/* advertise upconfig capability */
   2055	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
   2056	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
   2057	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
   2058	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
   2059	if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
   2060		lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
   2061		link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
   2062				     LC_RECONFIG_ARC_MISSING_ESCAPE);
   2063		link_width_cntl |= lanes | LC_RECONFIG_NOW |
   2064			LC_RENEGOTIATE_EN | LC_UPCONFIGURE_SUPPORT;
   2065		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
   2066	} else {
   2067		link_width_cntl |= LC_UPCONFIGURE_DIS;
   2068		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
   2069	}
   2070
   2071	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
   2072	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
   2073	    (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
   2074
   2075		tmp = RREG32(0x541c);
   2076		WREG32(0x541c, tmp | 0x8);
   2077		WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
   2078		link_cntl2 = RREG16(0x4088);
   2079		link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
   2080		link_cntl2 |= 0x2;
   2081		WREG16(0x4088, link_cntl2);
   2082		WREG32(MM_CFGREGS_CNTL, 0);
   2083
   2084		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
   2085		speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
   2086		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
   2087
   2088		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
   2089		speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
   2090		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
   2091
   2092		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
   2093		speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
   2094		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
   2095
   2096		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
   2097		speed_cntl |= LC_GEN2_EN_STRAP;
   2098		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
   2099
   2100	} else {
   2101		link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
   2102		/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
   2103		if (1)
   2104			link_width_cntl |= LC_UPCONFIGURE_DIS;
   2105		else
   2106			link_width_cntl &= ~LC_UPCONFIGURE_DIS;
   2107		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
   2108	}
   2109}