ni_dpm.c - cachepc-linux - Fork of AMDESE/linux with modifications for CachePC side-channel attack

	cachepc-linux Fork of AMDESE/linux with modifications for CachePC side-channel attack
	git clone https://git.sinitax.com/sinitax/cachepc-linux
	Log \| Files \| Refs \| README \| LICENSE \| sfeed.txt
ni_dpm.c (131846B)
      1/*
      2 * Copyright 2012 Advanced Micro Devices, Inc.
      3 *
      4 * Permission is hereby granted, free of charge, to any person obtaining a
      5 * copy of this software and associated documentation files (the "Software"),
      6 * to deal in the Software without restriction, including without limitation
      7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
      8 * and/or sell copies of the Software, and to permit persons to whom the
      9 * Software is furnished to do so, subject to the following conditions:
     10 *
     11 * The above copyright notice and this permission notice shall be included in
     12 * all copies or substantial portions of the Software.
     13 *
     14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
     15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
     16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
     17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
     18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
     19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
     20 * OTHER DEALINGS IN THE SOFTWARE.
     21 *
     22 */
     23
     24#include <linux/math64.h>
     25#include <linux/pci.h>
     26#include <linux/seq_file.h>
     27
     28#include "atom.h"
     29#include "evergreen.h"
     30#include "ni_dpm.h"
     31#include "nid.h"
     32#include "r600_dpm.h"
     33#include "rv770.h"
     34#include "radeon.h"
     35#include "radeon_asic.h"
     36
     37#define MC_CG_ARB_FREQ_F0           0x0a
     38#define MC_CG_ARB_FREQ_F1           0x0b
     39#define MC_CG_ARB_FREQ_F2           0x0c
     40#define MC_CG_ARB_FREQ_F3           0x0d
     41
     42#define SMC_RAM_END 0xC000
     43
     44static const struct ni_cac_weights cac_weights_cayman_xt =
     45{
     46	0x15,
     47	0x2,
     48	0x19,
     49	0x2,
     50	0x8,
     51	0x14,
     52	0x2,
     53	0x16,
     54	0xE,
     55	0x17,
     56	0x13,
     57	0x2B,
     58	0x10,
     59	0x7,
     60	0x5,
     61	0x5,
     62	0x5,
     63	0x2,
     64	0x3,
     65	0x9,
     66	0x10,
     67	0x10,
     68	0x2B,
     69	0xA,
     70	0x9,
     71	0x4,
     72	0xD,
     73	0xD,
     74	0x3E,
     75	0x18,
     76	0x14,
     77	0,
     78	0x3,
     79	0x3,
     80	0x5,
     81	0,
     82	0x2,
     83	0,
     84	0,
     85	0,
     86	0,
     87	0,
     88	0,
     89	0,
     90	0,
     91	0,
     92	0x1CC,
     93	0,
     94	0x164,
     95	1,
     96	1,
     97	1,
     98	1,
     99	12,
    100	12,
    101	12,
    102	0x12,
    103	0x1F,
    104	132,
    105	5,
    106	7,
    107	0,
    108	{ 0, 0, 0, 0, 0, 0, 0, 0 },
    109	{ 0, 0, 0, 0 },
    110	true
    111};
    112
    113static const struct ni_cac_weights cac_weights_cayman_pro =
    114{
    115	0x16,
    116	0x4,
    117	0x10,
    118	0x2,
    119	0xA,
    120	0x16,
    121	0x2,
    122	0x18,
    123	0x10,
    124	0x1A,
    125	0x16,
    126	0x2D,
    127	0x12,
    128	0xA,
    129	0x6,
    130	0x6,
    131	0x6,
    132	0x2,
    133	0x4,
    134	0xB,
    135	0x11,
    136	0x11,
    137	0x2D,
    138	0xC,
    139	0xC,
    140	0x7,
    141	0x10,
    142	0x10,
    143	0x3F,
    144	0x1A,
    145	0x16,
    146	0,
    147	0x7,
    148	0x4,
    149	0x6,
    150	1,
    151	0x2,
    152	0x1,
    153	0,
    154	0,
    155	0,
    156	0,
    157	0,
    158	0,
    159	0x30,
    160	0,
    161	0x1CF,
    162	0,
    163	0x166,
    164	1,
    165	1,
    166	1,
    167	1,
    168	12,
    169	12,
    170	12,
    171	0x15,
    172	0x1F,
    173	132,
    174	6,
    175	6,
    176	0,
    177	{ 0, 0, 0, 0, 0, 0, 0, 0 },
    178	{ 0, 0, 0, 0 },
    179	true
    180};
    181
    182static const struct ni_cac_weights cac_weights_cayman_le =
    183{
    184	0x7,
    185	0xE,
    186	0x1,
    187	0xA,
    188	0x1,
    189	0x3F,
    190	0x2,
    191	0x18,
    192	0x10,
    193	0x1A,
    194	0x1,
    195	0x3F,
    196	0x1,
    197	0xE,
    198	0x6,
    199	0x6,
    200	0x6,
    201	0x2,
    202	0x4,
    203	0x9,
    204	0x1A,
    205	0x1A,
    206	0x2C,
    207	0xA,
    208	0x11,
    209	0x8,
    210	0x19,
    211	0x19,
    212	0x1,
    213	0x1,
    214	0x1A,
    215	0,
    216	0x8,
    217	0x5,
    218	0x8,
    219	0x1,
    220	0x3,
    221	0x1,
    222	0,
    223	0,
    224	0,
    225	0,
    226	0,
    227	0,
    228	0x38,
    229	0x38,
    230	0x239,
    231	0x3,
    232	0x18A,
    233	1,
    234	1,
    235	1,
    236	1,
    237	12,
    238	12,
    239	12,
    240	0x15,
    241	0x22,
    242	132,
    243	6,
    244	6,
    245	0,
    246	{ 0, 0, 0, 0, 0, 0, 0, 0 },
    247	{ 0, 0, 0, 0 },
    248	true
    249};
    250
    251#define NISLANDS_MGCG_SEQUENCE  300
    252
    253static const u32 cayman_cgcg_cgls_default[] =
    254{
    255	0x000008f8, 0x00000010, 0xffffffff,
    256	0x000008fc, 0x00000000, 0xffffffff,
    257	0x000008f8, 0x00000011, 0xffffffff,
    258	0x000008fc, 0x00000000, 0xffffffff,
    259	0x000008f8, 0x00000012, 0xffffffff,
    260	0x000008fc, 0x00000000, 0xffffffff,
    261	0x000008f8, 0x00000013, 0xffffffff,
    262	0x000008fc, 0x00000000, 0xffffffff,
    263	0x000008f8, 0x00000014, 0xffffffff,
    264	0x000008fc, 0x00000000, 0xffffffff,
    265	0x000008f8, 0x00000015, 0xffffffff,
    266	0x000008fc, 0x00000000, 0xffffffff,
    267	0x000008f8, 0x00000016, 0xffffffff,
    268	0x000008fc, 0x00000000, 0xffffffff,
    269	0x000008f8, 0x00000017, 0xffffffff,
    270	0x000008fc, 0x00000000, 0xffffffff,
    271	0x000008f8, 0x00000018, 0xffffffff,
    272	0x000008fc, 0x00000000, 0xffffffff,
    273	0x000008f8, 0x00000019, 0xffffffff,
    274	0x000008fc, 0x00000000, 0xffffffff,
    275	0x000008f8, 0x0000001a, 0xffffffff,
    276	0x000008fc, 0x00000000, 0xffffffff,
    277	0x000008f8, 0x0000001b, 0xffffffff,
    278	0x000008fc, 0x00000000, 0xffffffff,
    279	0x000008f8, 0x00000020, 0xffffffff,
    280	0x000008fc, 0x00000000, 0xffffffff,
    281	0x000008f8, 0x00000021, 0xffffffff,
    282	0x000008fc, 0x00000000, 0xffffffff,
    283	0x000008f8, 0x00000022, 0xffffffff,
    284	0x000008fc, 0x00000000, 0xffffffff,
    285	0x000008f8, 0x00000023, 0xffffffff,
    286	0x000008fc, 0x00000000, 0xffffffff,
    287	0x000008f8, 0x00000024, 0xffffffff,
    288	0x000008fc, 0x00000000, 0xffffffff,
    289	0x000008f8, 0x00000025, 0xffffffff,
    290	0x000008fc, 0x00000000, 0xffffffff,
    291	0x000008f8, 0x00000026, 0xffffffff,
    292	0x000008fc, 0x00000000, 0xffffffff,
    293	0x000008f8, 0x00000027, 0xffffffff,
    294	0x000008fc, 0x00000000, 0xffffffff,
    295	0x000008f8, 0x00000028, 0xffffffff,
    296	0x000008fc, 0x00000000, 0xffffffff,
    297	0x000008f8, 0x00000029, 0xffffffff,
    298	0x000008fc, 0x00000000, 0xffffffff,
    299	0x000008f8, 0x0000002a, 0xffffffff,
    300	0x000008fc, 0x00000000, 0xffffffff,
    301	0x000008f8, 0x0000002b, 0xffffffff,
    302	0x000008fc, 0x00000000, 0xffffffff
    303};
    304#define CAYMAN_CGCG_CGLS_DEFAULT_LENGTH sizeof(cayman_cgcg_cgls_default) / (3 * sizeof(u32))
    305
    306static const u32 cayman_cgcg_cgls_disable[] =
    307{
    308	0x000008f8, 0x00000010, 0xffffffff,
    309	0x000008fc, 0xffffffff, 0xffffffff,
    310	0x000008f8, 0x00000011, 0xffffffff,
    311	0x000008fc, 0xffffffff, 0xffffffff,
    312	0x000008f8, 0x00000012, 0xffffffff,
    313	0x000008fc, 0xffffffff, 0xffffffff,
    314	0x000008f8, 0x00000013, 0xffffffff,
    315	0x000008fc, 0xffffffff, 0xffffffff,
    316	0x000008f8, 0x00000014, 0xffffffff,
    317	0x000008fc, 0xffffffff, 0xffffffff,
    318	0x000008f8, 0x00000015, 0xffffffff,
    319	0x000008fc, 0xffffffff, 0xffffffff,
    320	0x000008f8, 0x00000016, 0xffffffff,
    321	0x000008fc, 0xffffffff, 0xffffffff,
    322	0x000008f8, 0x00000017, 0xffffffff,
    323	0x000008fc, 0xffffffff, 0xffffffff,
    324	0x000008f8, 0x00000018, 0xffffffff,
    325	0x000008fc, 0xffffffff, 0xffffffff,
    326	0x000008f8, 0x00000019, 0xffffffff,
    327	0x000008fc, 0xffffffff, 0xffffffff,
    328	0x000008f8, 0x0000001a, 0xffffffff,
    329	0x000008fc, 0xffffffff, 0xffffffff,
    330	0x000008f8, 0x0000001b, 0xffffffff,
    331	0x000008fc, 0xffffffff, 0xffffffff,
    332	0x000008f8, 0x00000020, 0xffffffff,
    333	0x000008fc, 0x00000000, 0xffffffff,
    334	0x000008f8, 0x00000021, 0xffffffff,
    335	0x000008fc, 0x00000000, 0xffffffff,
    336	0x000008f8, 0x00000022, 0xffffffff,
    337	0x000008fc, 0x00000000, 0xffffffff,
    338	0x000008f8, 0x00000023, 0xffffffff,
    339	0x000008fc, 0x00000000, 0xffffffff,
    340	0x000008f8, 0x00000024, 0xffffffff,
    341	0x000008fc, 0x00000000, 0xffffffff,
    342	0x000008f8, 0x00000025, 0xffffffff,
    343	0x000008fc, 0x00000000, 0xffffffff,
    344	0x000008f8, 0x00000026, 0xffffffff,
    345	0x000008fc, 0x00000000, 0xffffffff,
    346	0x000008f8, 0x00000027, 0xffffffff,
    347	0x000008fc, 0x00000000, 0xffffffff,
    348	0x000008f8, 0x00000028, 0xffffffff,
    349	0x000008fc, 0x00000000, 0xffffffff,
    350	0x000008f8, 0x00000029, 0xffffffff,
    351	0x000008fc, 0x00000000, 0xffffffff,
    352	0x000008f8, 0x0000002a, 0xffffffff,
    353	0x000008fc, 0x00000000, 0xffffffff,
    354	0x000008f8, 0x0000002b, 0xffffffff,
    355	0x000008fc, 0x00000000, 0xffffffff,
    356	0x00000644, 0x000f7902, 0x001f4180,
    357	0x00000644, 0x000f3802, 0x001f4180
    358};
    359#define CAYMAN_CGCG_CGLS_DISABLE_LENGTH sizeof(cayman_cgcg_cgls_disable) / (3 * sizeof(u32))
    360
    361static const u32 cayman_cgcg_cgls_enable[] =
    362{
    363	0x00000644, 0x000f7882, 0x001f4080,
    364	0x000008f8, 0x00000010, 0xffffffff,
    365	0x000008fc, 0x00000000, 0xffffffff,
    366	0x000008f8, 0x00000011, 0xffffffff,
    367	0x000008fc, 0x00000000, 0xffffffff,
    368	0x000008f8, 0x00000012, 0xffffffff,
    369	0x000008fc, 0x00000000, 0xffffffff,
    370	0x000008f8, 0x00000013, 0xffffffff,
    371	0x000008fc, 0x00000000, 0xffffffff,
    372	0x000008f8, 0x00000014, 0xffffffff,
    373	0x000008fc, 0x00000000, 0xffffffff,
    374	0x000008f8, 0x00000015, 0xffffffff,
    375	0x000008fc, 0x00000000, 0xffffffff,
    376	0x000008f8, 0x00000016, 0xffffffff,
    377	0x000008fc, 0x00000000, 0xffffffff,
    378	0x000008f8, 0x00000017, 0xffffffff,
    379	0x000008fc, 0x00000000, 0xffffffff,
    380	0x000008f8, 0x00000018, 0xffffffff,
    381	0x000008fc, 0x00000000, 0xffffffff,
    382	0x000008f8, 0x00000019, 0xffffffff,
    383	0x000008fc, 0x00000000, 0xffffffff,
    384	0x000008f8, 0x0000001a, 0xffffffff,
    385	0x000008fc, 0x00000000, 0xffffffff,
    386	0x000008f8, 0x0000001b, 0xffffffff,
    387	0x000008fc, 0x00000000, 0xffffffff,
    388	0x000008f8, 0x00000020, 0xffffffff,
    389	0x000008fc, 0xffffffff, 0xffffffff,
    390	0x000008f8, 0x00000021, 0xffffffff,
    391	0x000008fc, 0xffffffff, 0xffffffff,
    392	0x000008f8, 0x00000022, 0xffffffff,
    393	0x000008fc, 0xffffffff, 0xffffffff,
    394	0x000008f8, 0x00000023, 0xffffffff,
    395	0x000008fc, 0xffffffff, 0xffffffff,
    396	0x000008f8, 0x00000024, 0xffffffff,
    397	0x000008fc, 0xffffffff, 0xffffffff,
    398	0x000008f8, 0x00000025, 0xffffffff,
    399	0x000008fc, 0xffffffff, 0xffffffff,
    400	0x000008f8, 0x00000026, 0xffffffff,
    401	0x000008fc, 0xffffffff, 0xffffffff,
    402	0x000008f8, 0x00000027, 0xffffffff,
    403	0x000008fc, 0xffffffff, 0xffffffff,
    404	0x000008f8, 0x00000028, 0xffffffff,
    405	0x000008fc, 0xffffffff, 0xffffffff,
    406	0x000008f8, 0x00000029, 0xffffffff,
    407	0x000008fc, 0xffffffff, 0xffffffff,
    408	0x000008f8, 0x0000002a, 0xffffffff,
    409	0x000008fc, 0xffffffff, 0xffffffff,
    410	0x000008f8, 0x0000002b, 0xffffffff,
    411	0x000008fc, 0xffffffff, 0xffffffff
    412};
    413#define CAYMAN_CGCG_CGLS_ENABLE_LENGTH  sizeof(cayman_cgcg_cgls_enable) / (3 * sizeof(u32))
    414
    415static const u32 cayman_mgcg_default[] =
    416{
    417	0x0000802c, 0xc0000000, 0xffffffff,
    418	0x00003fc4, 0xc0000000, 0xffffffff,
    419	0x00005448, 0x00000100, 0xffffffff,
    420	0x000055e4, 0x00000100, 0xffffffff,
    421	0x0000160c, 0x00000100, 0xffffffff,
    422	0x00008984, 0x06000100, 0xffffffff,
    423	0x0000c164, 0x00000100, 0xffffffff,
    424	0x00008a18, 0x00000100, 0xffffffff,
    425	0x0000897c, 0x06000100, 0xffffffff,
    426	0x00008b28, 0x00000100, 0xffffffff,
    427	0x00009144, 0x00800200, 0xffffffff,
    428	0x00009a60, 0x00000100, 0xffffffff,
    429	0x00009868, 0x00000100, 0xffffffff,
    430	0x00008d58, 0x00000100, 0xffffffff,
    431	0x00009510, 0x00000100, 0xffffffff,
    432	0x0000949c, 0x00000100, 0xffffffff,
    433	0x00009654, 0x00000100, 0xffffffff,
    434	0x00009030, 0x00000100, 0xffffffff,
    435	0x00009034, 0x00000100, 0xffffffff,
    436	0x00009038, 0x00000100, 0xffffffff,
    437	0x0000903c, 0x00000100, 0xffffffff,
    438	0x00009040, 0x00000100, 0xffffffff,
    439	0x0000a200, 0x00000100, 0xffffffff,
    440	0x0000a204, 0x00000100, 0xffffffff,
    441	0x0000a208, 0x00000100, 0xffffffff,
    442	0x0000a20c, 0x00000100, 0xffffffff,
    443	0x00009744, 0x00000100, 0xffffffff,
    444	0x00003f80, 0x00000100, 0xffffffff,
    445	0x0000a210, 0x00000100, 0xffffffff,
    446	0x0000a214, 0x00000100, 0xffffffff,
    447	0x000004d8, 0x00000100, 0xffffffff,
    448	0x00009664, 0x00000100, 0xffffffff,
    449	0x00009698, 0x00000100, 0xffffffff,
    450	0x000004d4, 0x00000200, 0xffffffff,
    451	0x000004d0, 0x00000000, 0xffffffff,
    452	0x000030cc, 0x00000104, 0xffffffff,
    453	0x0000d0c0, 0x00000100, 0xffffffff,
    454	0x0000d8c0, 0x00000100, 0xffffffff,
    455	0x0000802c, 0x40000000, 0xffffffff,
    456	0x00003fc4, 0x40000000, 0xffffffff,
    457	0x0000915c, 0x00010000, 0xffffffff,
    458	0x00009160, 0x00030002, 0xffffffff,
    459	0x00009164, 0x00050004, 0xffffffff,
    460	0x00009168, 0x00070006, 0xffffffff,
    461	0x00009178, 0x00070000, 0xffffffff,
    462	0x0000917c, 0x00030002, 0xffffffff,
    463	0x00009180, 0x00050004, 0xffffffff,
    464	0x0000918c, 0x00010006, 0xffffffff,
    465	0x00009190, 0x00090008, 0xffffffff,
    466	0x00009194, 0x00070000, 0xffffffff,
    467	0x00009198, 0x00030002, 0xffffffff,
    468	0x0000919c, 0x00050004, 0xffffffff,
    469	0x000091a8, 0x00010006, 0xffffffff,
    470	0x000091ac, 0x00090008, 0xffffffff,
    471	0x000091b0, 0x00070000, 0xffffffff,
    472	0x000091b4, 0x00030002, 0xffffffff,
    473	0x000091b8, 0x00050004, 0xffffffff,
    474	0x000091c4, 0x00010006, 0xffffffff,
    475	0x000091c8, 0x00090008, 0xffffffff,
    476	0x000091cc, 0x00070000, 0xffffffff,
    477	0x000091d0, 0x00030002, 0xffffffff,
    478	0x000091d4, 0x00050004, 0xffffffff,
    479	0x000091e0, 0x00010006, 0xffffffff,
    480	0x000091e4, 0x00090008, 0xffffffff,
    481	0x000091e8, 0x00000000, 0xffffffff,
    482	0x000091ec, 0x00070000, 0xffffffff,
    483	0x000091f0, 0x00030002, 0xffffffff,
    484	0x000091f4, 0x00050004, 0xffffffff,
    485	0x00009200, 0x00010006, 0xffffffff,
    486	0x00009204, 0x00090008, 0xffffffff,
    487	0x00009208, 0x00070000, 0xffffffff,
    488	0x0000920c, 0x00030002, 0xffffffff,
    489	0x00009210, 0x00050004, 0xffffffff,
    490	0x0000921c, 0x00010006, 0xffffffff,
    491	0x00009220, 0x00090008, 0xffffffff,
    492	0x00009224, 0x00070000, 0xffffffff,
    493	0x00009228, 0x00030002, 0xffffffff,
    494	0x0000922c, 0x00050004, 0xffffffff,
    495	0x00009238, 0x00010006, 0xffffffff,
    496	0x0000923c, 0x00090008, 0xffffffff,
    497	0x00009240, 0x00070000, 0xffffffff,
    498	0x00009244, 0x00030002, 0xffffffff,
    499	0x00009248, 0x00050004, 0xffffffff,
    500	0x00009254, 0x00010006, 0xffffffff,
    501	0x00009258, 0x00090008, 0xffffffff,
    502	0x0000925c, 0x00070000, 0xffffffff,
    503	0x00009260, 0x00030002, 0xffffffff,
    504	0x00009264, 0x00050004, 0xffffffff,
    505	0x00009270, 0x00010006, 0xffffffff,
    506	0x00009274, 0x00090008, 0xffffffff,
    507	0x00009278, 0x00070000, 0xffffffff,
    508	0x0000927c, 0x00030002, 0xffffffff,
    509	0x00009280, 0x00050004, 0xffffffff,
    510	0x0000928c, 0x00010006, 0xffffffff,
    511	0x00009290, 0x00090008, 0xffffffff,
    512	0x000092a8, 0x00070000, 0xffffffff,
    513	0x000092ac, 0x00030002, 0xffffffff,
    514	0x000092b0, 0x00050004, 0xffffffff,
    515	0x000092bc, 0x00010006, 0xffffffff,
    516	0x000092c0, 0x00090008, 0xffffffff,
    517	0x000092c4, 0x00070000, 0xffffffff,
    518	0x000092c8, 0x00030002, 0xffffffff,
    519	0x000092cc, 0x00050004, 0xffffffff,
    520	0x000092d8, 0x00010006, 0xffffffff,
    521	0x000092dc, 0x00090008, 0xffffffff,
    522	0x00009294, 0x00000000, 0xffffffff,
    523	0x0000802c, 0x40010000, 0xffffffff,
    524	0x00003fc4, 0x40010000, 0xffffffff,
    525	0x0000915c, 0x00010000, 0xffffffff,
    526	0x00009160, 0x00030002, 0xffffffff,
    527	0x00009164, 0x00050004, 0xffffffff,
    528	0x00009168, 0x00070006, 0xffffffff,
    529	0x00009178, 0x00070000, 0xffffffff,
    530	0x0000917c, 0x00030002, 0xffffffff,
    531	0x00009180, 0x00050004, 0xffffffff,
    532	0x0000918c, 0x00010006, 0xffffffff,
    533	0x00009190, 0x00090008, 0xffffffff,
    534	0x00009194, 0x00070000, 0xffffffff,
    535	0x00009198, 0x00030002, 0xffffffff,
    536	0x0000919c, 0x00050004, 0xffffffff,
    537	0x000091a8, 0x00010006, 0xffffffff,
    538	0x000091ac, 0x00090008, 0xffffffff,
    539	0x000091b0, 0x00070000, 0xffffffff,
    540	0x000091b4, 0x00030002, 0xffffffff,
    541	0x000091b8, 0x00050004, 0xffffffff,
    542	0x000091c4, 0x00010006, 0xffffffff,
    543	0x000091c8, 0x00090008, 0xffffffff,
    544	0x000091cc, 0x00070000, 0xffffffff,
    545	0x000091d0, 0x00030002, 0xffffffff,
    546	0x000091d4, 0x00050004, 0xffffffff,
    547	0x000091e0, 0x00010006, 0xffffffff,
    548	0x000091e4, 0x00090008, 0xffffffff,
    549	0x000091e8, 0x00000000, 0xffffffff,
    550	0x000091ec, 0x00070000, 0xffffffff,
    551	0x000091f0, 0x00030002, 0xffffffff,
    552	0x000091f4, 0x00050004, 0xffffffff,
    553	0x00009200, 0x00010006, 0xffffffff,
    554	0x00009204, 0x00090008, 0xffffffff,
    555	0x00009208, 0x00070000, 0xffffffff,
    556	0x0000920c, 0x00030002, 0xffffffff,
    557	0x00009210, 0x00050004, 0xffffffff,
    558	0x0000921c, 0x00010006, 0xffffffff,
    559	0x00009220, 0x00090008, 0xffffffff,
    560	0x00009224, 0x00070000, 0xffffffff,
    561	0x00009228, 0x00030002, 0xffffffff,
    562	0x0000922c, 0x00050004, 0xffffffff,
    563	0x00009238, 0x00010006, 0xffffffff,
    564	0x0000923c, 0x00090008, 0xffffffff,
    565	0x00009240, 0x00070000, 0xffffffff,
    566	0x00009244, 0x00030002, 0xffffffff,
    567	0x00009248, 0x00050004, 0xffffffff,
    568	0x00009254, 0x00010006, 0xffffffff,
    569	0x00009258, 0x00090008, 0xffffffff,
    570	0x0000925c, 0x00070000, 0xffffffff,
    571	0x00009260, 0x00030002, 0xffffffff,
    572	0x00009264, 0x00050004, 0xffffffff,
    573	0x00009270, 0x00010006, 0xffffffff,
    574	0x00009274, 0x00090008, 0xffffffff,
    575	0x00009278, 0x00070000, 0xffffffff,
    576	0x0000927c, 0x00030002, 0xffffffff,
    577	0x00009280, 0x00050004, 0xffffffff,
    578	0x0000928c, 0x00010006, 0xffffffff,
    579	0x00009290, 0x00090008, 0xffffffff,
    580	0x000092a8, 0x00070000, 0xffffffff,
    581	0x000092ac, 0x00030002, 0xffffffff,
    582	0x000092b0, 0x00050004, 0xffffffff,
    583	0x000092bc, 0x00010006, 0xffffffff,
    584	0x000092c0, 0x00090008, 0xffffffff,
    585	0x000092c4, 0x00070000, 0xffffffff,
    586	0x000092c8, 0x00030002, 0xffffffff,
    587	0x000092cc, 0x00050004, 0xffffffff,
    588	0x000092d8, 0x00010006, 0xffffffff,
    589	0x000092dc, 0x00090008, 0xffffffff,
    590	0x00009294, 0x00000000, 0xffffffff,
    591	0x0000802c, 0xc0000000, 0xffffffff,
    592	0x00003fc4, 0xc0000000, 0xffffffff,
    593	0x000008f8, 0x00000010, 0xffffffff,
    594	0x000008fc, 0x00000000, 0xffffffff,
    595	0x000008f8, 0x00000011, 0xffffffff,
    596	0x000008fc, 0x00000000, 0xffffffff,
    597	0x000008f8, 0x00000012, 0xffffffff,
    598	0x000008fc, 0x00000000, 0xffffffff,
    599	0x000008f8, 0x00000013, 0xffffffff,
    600	0x000008fc, 0x00000000, 0xffffffff,
    601	0x000008f8, 0x00000014, 0xffffffff,
    602	0x000008fc, 0x00000000, 0xffffffff,
    603	0x000008f8, 0x00000015, 0xffffffff,
    604	0x000008fc, 0x00000000, 0xffffffff,
    605	0x000008f8, 0x00000016, 0xffffffff,
    606	0x000008fc, 0x00000000, 0xffffffff,
    607	0x000008f8, 0x00000017, 0xffffffff,
    608	0x000008fc, 0x00000000, 0xffffffff,
    609	0x000008f8, 0x00000018, 0xffffffff,
    610	0x000008fc, 0x00000000, 0xffffffff,
    611	0x000008f8, 0x00000019, 0xffffffff,
    612	0x000008fc, 0x00000000, 0xffffffff,
    613	0x000008f8, 0x0000001a, 0xffffffff,
    614	0x000008fc, 0x00000000, 0xffffffff,
    615	0x000008f8, 0x0000001b, 0xffffffff,
    616	0x000008fc, 0x00000000, 0xffffffff
    617};
    618#define CAYMAN_MGCG_DEFAULT_LENGTH sizeof(cayman_mgcg_default) / (3 * sizeof(u32))
    619
    620static const u32 cayman_mgcg_disable[] =
    621{
    622	0x0000802c, 0xc0000000, 0xffffffff,
    623	0x000008f8, 0x00000000, 0xffffffff,
    624	0x000008fc, 0xffffffff, 0xffffffff,
    625	0x000008f8, 0x00000001, 0xffffffff,
    626	0x000008fc, 0xffffffff, 0xffffffff,
    627	0x000008f8, 0x00000002, 0xffffffff,
    628	0x000008fc, 0xffffffff, 0xffffffff,
    629	0x000008f8, 0x00000003, 0xffffffff,
    630	0x000008fc, 0xffffffff, 0xffffffff,
    631	0x00009150, 0x00600000, 0xffffffff
    632};
    633#define CAYMAN_MGCG_DISABLE_LENGTH   sizeof(cayman_mgcg_disable) / (3 * sizeof(u32))
    634
    635static const u32 cayman_mgcg_enable[] =
    636{
    637	0x0000802c, 0xc0000000, 0xffffffff,
    638	0x000008f8, 0x00000000, 0xffffffff,
    639	0x000008fc, 0x00000000, 0xffffffff,
    640	0x000008f8, 0x00000001, 0xffffffff,
    641	0x000008fc, 0x00000000, 0xffffffff,
    642	0x000008f8, 0x00000002, 0xffffffff,
    643	0x000008fc, 0x00600000, 0xffffffff,
    644	0x000008f8, 0x00000003, 0xffffffff,
    645	0x000008fc, 0x00000000, 0xffffffff,
    646	0x00009150, 0x96944200, 0xffffffff
    647};
    648
    649#define CAYMAN_MGCG_ENABLE_LENGTH   sizeof(cayman_mgcg_enable) / (3 * sizeof(u32))
    650
    651#define NISLANDS_SYSLS_SEQUENCE  100
    652
    653static const u32 cayman_sysls_default[] =
    654{
    655	/* Register,   Value,     Mask bits */
    656	0x000055e8, 0x00000000, 0xffffffff,
    657	0x0000d0bc, 0x00000000, 0xffffffff,
    658	0x0000d8bc, 0x00000000, 0xffffffff,
    659	0x000015c0, 0x000c1401, 0xffffffff,
    660	0x0000264c, 0x000c0400, 0xffffffff,
    661	0x00002648, 0x000c0400, 0xffffffff,
    662	0x00002650, 0x000c0400, 0xffffffff,
    663	0x000020b8, 0x000c0400, 0xffffffff,
    664	0x000020bc, 0x000c0400, 0xffffffff,
    665	0x000020c0, 0x000c0c80, 0xffffffff,
    666	0x0000f4a0, 0x000000c0, 0xffffffff,
    667	0x0000f4a4, 0x00680fff, 0xffffffff,
    668	0x00002f50, 0x00000404, 0xffffffff,
    669	0x000004c8, 0x00000001, 0xffffffff,
    670	0x000064ec, 0x00000000, 0xffffffff,
    671	0x00000c7c, 0x00000000, 0xffffffff,
    672	0x00008dfc, 0x00000000, 0xffffffff
    673};
    674#define CAYMAN_SYSLS_DEFAULT_LENGTH sizeof(cayman_sysls_default) / (3 * sizeof(u32))
    675
    676static const u32 cayman_sysls_disable[] =
    677{
    678	/* Register,   Value,     Mask bits */
    679	0x0000d0c0, 0x00000000, 0xffffffff,
    680	0x0000d8c0, 0x00000000, 0xffffffff,
    681	0x000055e8, 0x00000000, 0xffffffff,
    682	0x0000d0bc, 0x00000000, 0xffffffff,
    683	0x0000d8bc, 0x00000000, 0xffffffff,
    684	0x000015c0, 0x00041401, 0xffffffff,
    685	0x0000264c, 0x00040400, 0xffffffff,
    686	0x00002648, 0x00040400, 0xffffffff,
    687	0x00002650, 0x00040400, 0xffffffff,
    688	0x000020b8, 0x00040400, 0xffffffff,
    689	0x000020bc, 0x00040400, 0xffffffff,
    690	0x000020c0, 0x00040c80, 0xffffffff,
    691	0x0000f4a0, 0x000000c0, 0xffffffff,
    692	0x0000f4a4, 0x00680000, 0xffffffff,
    693	0x00002f50, 0x00000404, 0xffffffff,
    694	0x000004c8, 0x00000001, 0xffffffff,
    695	0x000064ec, 0x00007ffd, 0xffffffff,
    696	0x00000c7c, 0x0000ff00, 0xffffffff,
    697	0x00008dfc, 0x0000007f, 0xffffffff
    698};
    699#define CAYMAN_SYSLS_DISABLE_LENGTH sizeof(cayman_sysls_disable) / (3 * sizeof(u32))
    700
    701static const u32 cayman_sysls_enable[] =
    702{
    703	/* Register,   Value,     Mask bits */
    704	0x000055e8, 0x00000001, 0xffffffff,
    705	0x0000d0bc, 0x00000100, 0xffffffff,
    706	0x0000d8bc, 0x00000100, 0xffffffff,
    707	0x000015c0, 0x000c1401, 0xffffffff,
    708	0x0000264c, 0x000c0400, 0xffffffff,
    709	0x00002648, 0x000c0400, 0xffffffff,
    710	0x00002650, 0x000c0400, 0xffffffff,
    711	0x000020b8, 0x000c0400, 0xffffffff,
    712	0x000020bc, 0x000c0400, 0xffffffff,
    713	0x000020c0, 0x000c0c80, 0xffffffff,
    714	0x0000f4a0, 0x000000c0, 0xffffffff,
    715	0x0000f4a4, 0x00680fff, 0xffffffff,
    716	0x00002f50, 0x00000903, 0xffffffff,
    717	0x000004c8, 0x00000000, 0xffffffff,
    718	0x000064ec, 0x00000000, 0xffffffff,
    719	0x00000c7c, 0x00000000, 0xffffffff,
    720	0x00008dfc, 0x00000000, 0xffffffff
    721};
    722#define CAYMAN_SYSLS_ENABLE_LENGTH sizeof(cayman_sysls_enable) / (3 * sizeof(u32))
    723
    724extern int ni_mc_load_microcode(struct radeon_device *rdev);
    725
    726struct ni_power_info *ni_get_pi(struct radeon_device *rdev)
    727{
    728	struct ni_power_info *pi = rdev->pm.dpm.priv;
    729
    730	return pi;
    731}
    732
    733struct ni_ps *ni_get_ps(struct radeon_ps *rps)
    734{
    735	struct ni_ps *ps = rps->ps_priv;
    736
    737	return ps;
    738}
    739
    740static void ni_calculate_leakage_for_v_and_t_formula(const struct ni_leakage_coeffients *coeff,
    741						     u16 v, s32 t,
    742						     u32 ileakage,
    743						     u32 *leakage)
    744{
    745	s64 kt, kv, leakage_w, i_leakage, vddc, temperature;
    746
    747	i_leakage = div64_s64(drm_int2fixp(ileakage), 1000);
    748	vddc = div64_s64(drm_int2fixp(v), 1000);
    749	temperature = div64_s64(drm_int2fixp(t), 1000);
    750
    751	kt = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->at), 1000),
    752			  drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bt), 1000), temperature)));
    753	kv = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->av), 1000),
    754			  drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bv), 1000), vddc)));
    755
    756	leakage_w = drm_fixp_mul(drm_fixp_mul(drm_fixp_mul(i_leakage, kt), kv), vddc);
    757
    758	*leakage = drm_fixp2int(leakage_w * 1000);
    759}
    760
    761static void ni_calculate_leakage_for_v_and_t(struct radeon_device *rdev,
    762					     const struct ni_leakage_coeffients *coeff,
    763					     u16 v,
    764					     s32 t,
    765					     u32 i_leakage,
    766					     u32 *leakage)
    767{
    768	ni_calculate_leakage_for_v_and_t_formula(coeff, v, t, i_leakage, leakage);
    769}
    770
    771bool ni_dpm_vblank_too_short(struct radeon_device *rdev)
    772{
    773	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
    774	u32 vblank_time = r600_dpm_get_vblank_time(rdev);
    775	/* we never hit the non-gddr5 limit so disable it */
    776	u32 switch_limit = pi->mem_gddr5 ? 450 : 0;
    777
    778	if (vblank_time < switch_limit)
    779		return true;
    780	else
    781		return false;
    782
    783}
    784
    785static void ni_apply_state_adjust_rules(struct radeon_device *rdev,
    786					struct radeon_ps *rps)
    787{
    788	struct ni_ps *ps = ni_get_ps(rps);
    789	struct radeon_clock_and_voltage_limits *max_limits;
    790	bool disable_mclk_switching;
    791	u32 mclk;
    792	u16 vddci;
    793	int i;
    794
    795	if ((rdev->pm.dpm.new_active_crtc_count > 1) ||
    796	    ni_dpm_vblank_too_short(rdev))
    797		disable_mclk_switching = true;
    798	else
    799		disable_mclk_switching = false;
    800
    801	if (rdev->pm.dpm.ac_power)
    802		max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
    803	else
    804		max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
    805
    806	if (rdev->pm.dpm.ac_power == false) {
    807		for (i = 0; i < ps->performance_level_count; i++) {
    808			if (ps->performance_levels[i].mclk > max_limits->mclk)
    809				ps->performance_levels[i].mclk = max_limits->mclk;
    810			if (ps->performance_levels[i].sclk > max_limits->sclk)
    811				ps->performance_levels[i].sclk = max_limits->sclk;
    812			if (ps->performance_levels[i].vddc > max_limits->vddc)
    813				ps->performance_levels[i].vddc = max_limits->vddc;
    814			if (ps->performance_levels[i].vddci > max_limits->vddci)
    815				ps->performance_levels[i].vddci = max_limits->vddci;
    816		}
    817	}
    818
    819	/* XXX validate the min clocks required for display */
    820
    821	/* adjust low state */
    822	if (disable_mclk_switching) {
    823		ps->performance_levels[0].mclk =
    824			ps->performance_levels[ps->performance_level_count - 1].mclk;
    825		ps->performance_levels[0].vddci =
    826			ps->performance_levels[ps->performance_level_count - 1].vddci;
    827	}
    828
    829	btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
    830				  &ps->performance_levels[0].sclk,
    831				  &ps->performance_levels[0].mclk);
    832
    833	for (i = 1; i < ps->performance_level_count; i++) {
    834		if (ps->performance_levels[i].sclk < ps->performance_levels[i - 1].sclk)
    835			ps->performance_levels[i].sclk = ps->performance_levels[i - 1].sclk;
    836		if (ps->performance_levels[i].vddc < ps->performance_levels[i - 1].vddc)
    837			ps->performance_levels[i].vddc = ps->performance_levels[i - 1].vddc;
    838	}
    839
    840	/* adjust remaining states */
    841	if (disable_mclk_switching) {
    842		mclk = ps->performance_levels[0].mclk;
    843		vddci = ps->performance_levels[0].vddci;
    844		for (i = 1; i < ps->performance_level_count; i++) {
    845			if (mclk < ps->performance_levels[i].mclk)
    846				mclk = ps->performance_levels[i].mclk;
    847			if (vddci < ps->performance_levels[i].vddci)
    848				vddci = ps->performance_levels[i].vddci;
    849		}
    850		for (i = 0; i < ps->performance_level_count; i++) {
    851			ps->performance_levels[i].mclk = mclk;
    852			ps->performance_levels[i].vddci = vddci;
    853		}
    854	} else {
    855		for (i = 1; i < ps->performance_level_count; i++) {
    856			if (ps->performance_levels[i].mclk < ps->performance_levels[i - 1].mclk)
    857				ps->performance_levels[i].mclk = ps->performance_levels[i - 1].mclk;
    858			if (ps->performance_levels[i].vddci < ps->performance_levels[i - 1].vddci)
    859				ps->performance_levels[i].vddci = ps->performance_levels[i - 1].vddci;
    860		}
    861	}
    862
    863	for (i = 1; i < ps->performance_level_count; i++)
    864		btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
    865					  &ps->performance_levels[i].sclk,
    866					  &ps->performance_levels[i].mclk);
    867
    868	for (i = 0; i < ps->performance_level_count; i++)
    869		btc_adjust_clock_combinations(rdev, max_limits,
    870					      &ps->performance_levels[i]);
    871
    872	for (i = 0; i < ps->performance_level_count; i++) {
    873		btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk,
    874						   ps->performance_levels[i].sclk,
    875						   max_limits->vddc,  &ps->performance_levels[i].vddc);
    876		btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
    877						   ps->performance_levels[i].mclk,
    878						   max_limits->vddci, &ps->performance_levels[i].vddci);
    879		btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
    880						   ps->performance_levels[i].mclk,
    881						   max_limits->vddc,  &ps->performance_levels[i].vddc);
    882		btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk,
    883						   rdev->clock.current_dispclk,
    884						   max_limits->vddc,  &ps->performance_levels[i].vddc);
    885	}
    886
    887	for (i = 0; i < ps->performance_level_count; i++) {
    888		btc_apply_voltage_delta_rules(rdev,
    889					      max_limits->vddc, max_limits->vddci,
    890					      &ps->performance_levels[i].vddc,
    891					      &ps->performance_levels[i].vddci);
    892	}
    893
    894	ps->dc_compatible = true;
    895	for (i = 0; i < ps->performance_level_count; i++) {
    896		if (ps->performance_levels[i].vddc > rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc)
    897			ps->dc_compatible = false;
    898
    899		if (ps->performance_levels[i].vddc < rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2)
    900			ps->performance_levels[i].flags &= ~ATOM_PPLIB_R600_FLAGS_PCIEGEN2;
    901	}
    902}
    903
    904static void ni_cg_clockgating_default(struct radeon_device *rdev)
    905{
    906	u32 count;
    907	const u32 *ps = NULL;
    908
    909	ps = (const u32 *)&cayman_cgcg_cgls_default;
    910	count = CAYMAN_CGCG_CGLS_DEFAULT_LENGTH;
    911
    912	btc_program_mgcg_hw_sequence(rdev, ps, count);
    913}
    914
    915static void ni_gfx_clockgating_enable(struct radeon_device *rdev,
    916				      bool enable)
    917{
    918	u32 count;
    919	const u32 *ps = NULL;
    920
    921	if (enable) {
    922		ps = (const u32 *)&cayman_cgcg_cgls_enable;
    923		count = CAYMAN_CGCG_CGLS_ENABLE_LENGTH;
    924	} else {
    925		ps = (const u32 *)&cayman_cgcg_cgls_disable;
    926		count = CAYMAN_CGCG_CGLS_DISABLE_LENGTH;
    927	}
    928
    929	btc_program_mgcg_hw_sequence(rdev, ps, count);
    930}
    931
    932static void ni_mg_clockgating_default(struct radeon_device *rdev)
    933{
    934	u32 count;
    935	const u32 *ps = NULL;
    936
    937	ps = (const u32 *)&cayman_mgcg_default;
    938	count = CAYMAN_MGCG_DEFAULT_LENGTH;
    939
    940	btc_program_mgcg_hw_sequence(rdev, ps, count);
    941}
    942
    943static void ni_mg_clockgating_enable(struct radeon_device *rdev,
    944				     bool enable)
    945{
    946	u32 count;
    947	const u32 *ps = NULL;
    948
    949	if (enable) {
    950		ps = (const u32 *)&cayman_mgcg_enable;
    951		count = CAYMAN_MGCG_ENABLE_LENGTH;
    952	} else {
    953		ps = (const u32 *)&cayman_mgcg_disable;
    954		count = CAYMAN_MGCG_DISABLE_LENGTH;
    955	}
    956
    957	btc_program_mgcg_hw_sequence(rdev, ps, count);
    958}
    959
    960static void ni_ls_clockgating_default(struct radeon_device *rdev)
    961{
    962	u32 count;
    963	const u32 *ps = NULL;
    964
    965	ps = (const u32 *)&cayman_sysls_default;
    966	count = CAYMAN_SYSLS_DEFAULT_LENGTH;
    967
    968	btc_program_mgcg_hw_sequence(rdev, ps, count);
    969}
    970
    971static void ni_ls_clockgating_enable(struct radeon_device *rdev,
    972				     bool enable)
    973{
    974	u32 count;
    975	const u32 *ps = NULL;
    976
    977	if (enable) {
    978		ps = (const u32 *)&cayman_sysls_enable;
    979		count = CAYMAN_SYSLS_ENABLE_LENGTH;
    980	} else {
    981		ps = (const u32 *)&cayman_sysls_disable;
    982		count = CAYMAN_SYSLS_DISABLE_LENGTH;
    983	}
    984
    985	btc_program_mgcg_hw_sequence(rdev, ps, count);
    986
    987}
    988
    989static int ni_patch_single_dependency_table_based_on_leakage(struct radeon_device *rdev,
    990							     struct radeon_clock_voltage_dependency_table *table)
    991{
    992	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
    993	u32 i;
    994
    995	if (table) {
    996		for (i = 0; i < table->count; i++) {
    997			if (0xff01 == table->entries[i].v) {
    998				if (pi->max_vddc == 0)
    999					return -EINVAL;
   1000				table->entries[i].v = pi->max_vddc;
   1001			}
   1002		}
   1003	}
   1004	return 0;
   1005}
   1006
   1007static int ni_patch_dependency_tables_based_on_leakage(struct radeon_device *rdev)
   1008{
   1009	int ret = 0;
   1010
   1011	ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
   1012								&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk);
   1013
   1014	ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
   1015								&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk);
   1016	return ret;
   1017}
   1018
   1019static void ni_stop_dpm(struct radeon_device *rdev)
   1020{
   1021	WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN);
   1022}
   1023
   1024#if 0
   1025static int ni_notify_hw_of_power_source(struct radeon_device *rdev,
   1026					bool ac_power)
   1027{
   1028	if (ac_power)
   1029		return (rv770_send_msg_to_smc(rdev, PPSMC_MSG_RunningOnAC) == PPSMC_Result_OK) ?
   1030			0 : -EINVAL;
   1031
   1032	return 0;
   1033}
   1034#endif
   1035
   1036static PPSMC_Result ni_send_msg_to_smc_with_parameter(struct radeon_device *rdev,
   1037						      PPSMC_Msg msg, u32 parameter)
   1038{
   1039	WREG32(SMC_SCRATCH0, parameter);
   1040	return rv770_send_msg_to_smc(rdev, msg);
   1041}
   1042
   1043static int ni_restrict_performance_levels_before_switch(struct radeon_device *rdev)
   1044{
   1045	if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_NoForcedLevel) != PPSMC_Result_OK)
   1046		return -EINVAL;
   1047
   1048	return (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) == PPSMC_Result_OK) ?
   1049		0 : -EINVAL;
   1050}
   1051
   1052int ni_dpm_force_performance_level(struct radeon_device *rdev,
   1053				   enum radeon_dpm_forced_level level)
   1054{
   1055	if (level == RADEON_DPM_FORCED_LEVEL_HIGH) {
   1056		if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) != PPSMC_Result_OK)
   1057			return -EINVAL;
   1058
   1059		if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 1) != PPSMC_Result_OK)
   1060			return -EINVAL;
   1061	} else if (level == RADEON_DPM_FORCED_LEVEL_LOW) {
   1062		if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
   1063			return -EINVAL;
   1064
   1065		if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) != PPSMC_Result_OK)
   1066			return -EINVAL;
   1067	} else if (level == RADEON_DPM_FORCED_LEVEL_AUTO) {
   1068		if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
   1069			return -EINVAL;
   1070
   1071		if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) != PPSMC_Result_OK)
   1072			return -EINVAL;
   1073	}
   1074
   1075	rdev->pm.dpm.forced_level = level;
   1076
   1077	return 0;
   1078}
   1079
   1080static void ni_stop_smc(struct radeon_device *rdev)
   1081{
   1082	u32 tmp;
   1083	int i;
   1084
   1085	for (i = 0; i < rdev->usec_timeout; i++) {
   1086		tmp = RREG32(LB_SYNC_RESET_SEL) & LB_SYNC_RESET_SEL_MASK;
   1087		if (tmp != 1)
   1088			break;
   1089		udelay(1);
   1090	}
   1091
   1092	udelay(100);
   1093
   1094	r7xx_stop_smc(rdev);
   1095}
   1096
   1097static int ni_process_firmware_header(struct radeon_device *rdev)
   1098{
   1099	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   1100	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   1101	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   1102	u32 tmp;
   1103	int ret;
   1104
   1105	ret = rv770_read_smc_sram_dword(rdev,
   1106					NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
   1107					NISLANDS_SMC_FIRMWARE_HEADER_stateTable,
   1108					&tmp, pi->sram_end);
   1109
   1110	if (ret)
   1111		return ret;
   1112
   1113	pi->state_table_start = (u16)tmp;
   1114
   1115	ret = rv770_read_smc_sram_dword(rdev,
   1116					NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
   1117					NISLANDS_SMC_FIRMWARE_HEADER_softRegisters,
   1118					&tmp, pi->sram_end);
   1119
   1120	if (ret)
   1121		return ret;
   1122
   1123	pi->soft_regs_start = (u16)tmp;
   1124
   1125	ret = rv770_read_smc_sram_dword(rdev,
   1126					NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
   1127					NISLANDS_SMC_FIRMWARE_HEADER_mcRegisterTable,
   1128					&tmp, pi->sram_end);
   1129
   1130	if (ret)
   1131		return ret;
   1132
   1133	eg_pi->mc_reg_table_start = (u16)tmp;
   1134
   1135	ret = rv770_read_smc_sram_dword(rdev,
   1136					NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
   1137					NISLANDS_SMC_FIRMWARE_HEADER_fanTable,
   1138					&tmp, pi->sram_end);
   1139
   1140	if (ret)
   1141		return ret;
   1142
   1143	ni_pi->fan_table_start = (u16)tmp;
   1144
   1145	ret = rv770_read_smc_sram_dword(rdev,
   1146					NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
   1147					NISLANDS_SMC_FIRMWARE_HEADER_mcArbDramAutoRefreshTable,
   1148					&tmp, pi->sram_end);
   1149
   1150	if (ret)
   1151		return ret;
   1152
   1153	ni_pi->arb_table_start = (u16)tmp;
   1154
   1155	ret = rv770_read_smc_sram_dword(rdev,
   1156					NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
   1157					NISLANDS_SMC_FIRMWARE_HEADER_cacTable,
   1158					&tmp, pi->sram_end);
   1159
   1160	if (ret)
   1161		return ret;
   1162
   1163	ni_pi->cac_table_start = (u16)tmp;
   1164
   1165	ret = rv770_read_smc_sram_dword(rdev,
   1166					NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
   1167					NISLANDS_SMC_FIRMWARE_HEADER_spllTable,
   1168					&tmp, pi->sram_end);
   1169
   1170	if (ret)
   1171		return ret;
   1172
   1173	ni_pi->spll_table_start = (u16)tmp;
   1174
   1175
   1176	return ret;
   1177}
   1178
   1179static void ni_read_clock_registers(struct radeon_device *rdev)
   1180{
   1181	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   1182
   1183	ni_pi->clock_registers.cg_spll_func_cntl = RREG32(CG_SPLL_FUNC_CNTL);
   1184	ni_pi->clock_registers.cg_spll_func_cntl_2 = RREG32(CG_SPLL_FUNC_CNTL_2);
   1185	ni_pi->clock_registers.cg_spll_func_cntl_3 = RREG32(CG_SPLL_FUNC_CNTL_3);
   1186	ni_pi->clock_registers.cg_spll_func_cntl_4 = RREG32(CG_SPLL_FUNC_CNTL_4);
   1187	ni_pi->clock_registers.cg_spll_spread_spectrum = RREG32(CG_SPLL_SPREAD_SPECTRUM);
   1188	ni_pi->clock_registers.cg_spll_spread_spectrum_2 = RREG32(CG_SPLL_SPREAD_SPECTRUM_2);
   1189	ni_pi->clock_registers.mpll_ad_func_cntl = RREG32(MPLL_AD_FUNC_CNTL);
   1190	ni_pi->clock_registers.mpll_ad_func_cntl_2 = RREG32(MPLL_AD_FUNC_CNTL_2);
   1191	ni_pi->clock_registers.mpll_dq_func_cntl = RREG32(MPLL_DQ_FUNC_CNTL);
   1192	ni_pi->clock_registers.mpll_dq_func_cntl_2 = RREG32(MPLL_DQ_FUNC_CNTL_2);
   1193	ni_pi->clock_registers.mclk_pwrmgt_cntl = RREG32(MCLK_PWRMGT_CNTL);
   1194	ni_pi->clock_registers.dll_cntl = RREG32(DLL_CNTL);
   1195	ni_pi->clock_registers.mpll_ss1 = RREG32(MPLL_SS1);
   1196	ni_pi->clock_registers.mpll_ss2 = RREG32(MPLL_SS2);
   1197}
   1198
   1199#if 0
   1200static int ni_enter_ulp_state(struct radeon_device *rdev)
   1201{
   1202	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   1203
   1204	if (pi->gfx_clock_gating) {
   1205		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
   1206		WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
   1207		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
   1208		RREG32(GB_ADDR_CONFIG);
   1209	}
   1210
   1211	WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_SwitchToMinimumPower),
   1212		 ~HOST_SMC_MSG_MASK);
   1213
   1214	udelay(25000);
   1215
   1216	return 0;
   1217}
   1218#endif
   1219
   1220static void ni_program_response_times(struct radeon_device *rdev)
   1221{
   1222	u32 voltage_response_time, backbias_response_time, acpi_delay_time, vbi_time_out;
   1223	u32 vddc_dly, bb_dly, acpi_dly, vbi_dly, mclk_switch_limit;
   1224	u32 reference_clock;
   1225
   1226	rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mvdd_chg_time, 1);
   1227
   1228	voltage_response_time = (u32)rdev->pm.dpm.voltage_response_time;
   1229	backbias_response_time = (u32)rdev->pm.dpm.backbias_response_time;
   1230
   1231	if (voltage_response_time == 0)
   1232		voltage_response_time = 1000;
   1233
   1234	if (backbias_response_time == 0)
   1235		backbias_response_time = 1000;
   1236
   1237	acpi_delay_time = 15000;
   1238	vbi_time_out = 100000;
   1239
   1240	reference_clock = radeon_get_xclk(rdev);
   1241
   1242	vddc_dly = (voltage_response_time  * reference_clock) / 1600;
   1243	bb_dly   = (backbias_response_time * reference_clock) / 1600;
   1244	acpi_dly = (acpi_delay_time * reference_clock) / 1600;
   1245	vbi_dly  = (vbi_time_out * reference_clock) / 1600;
   1246
   1247	mclk_switch_limit = (460 * reference_clock) / 100;
   1248
   1249	rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_vreg,  vddc_dly);
   1250	rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_bbias, bb_dly);
   1251	rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_acpi,  acpi_dly);
   1252	rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_chg_timeout, vbi_dly);
   1253	rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mc_block_delay, 0xAA);
   1254	rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_switch_lim, mclk_switch_limit);
   1255}
   1256
   1257static void ni_populate_smc_voltage_table(struct radeon_device *rdev,
   1258					  struct atom_voltage_table *voltage_table,
   1259					  NISLANDS_SMC_STATETABLE *table)
   1260{
   1261	unsigned int i;
   1262
   1263	for (i = 0; i < voltage_table->count; i++) {
   1264		table->highSMIO[i] = 0;
   1265		table->lowSMIO[i] |= cpu_to_be32(voltage_table->entries[i].smio_low);
   1266	}
   1267}
   1268
   1269static void ni_populate_smc_voltage_tables(struct radeon_device *rdev,
   1270					   NISLANDS_SMC_STATETABLE *table)
   1271{
   1272	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   1273	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   1274	unsigned char i;
   1275
   1276	if (eg_pi->vddc_voltage_table.count) {
   1277		ni_populate_smc_voltage_table(rdev, &eg_pi->vddc_voltage_table, table);
   1278		table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] = 0;
   1279		table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] =
   1280			cpu_to_be32(eg_pi->vddc_voltage_table.mask_low);
   1281
   1282		for (i = 0; i < eg_pi->vddc_voltage_table.count; i++) {
   1283			if (pi->max_vddc_in_table <= eg_pi->vddc_voltage_table.entries[i].value) {
   1284				table->maxVDDCIndexInPPTable = i;
   1285				break;
   1286			}
   1287		}
   1288	}
   1289
   1290	if (eg_pi->vddci_voltage_table.count) {
   1291		ni_populate_smc_voltage_table(rdev, &eg_pi->vddci_voltage_table, table);
   1292
   1293		table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] = 0;
   1294		table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] =
   1295			cpu_to_be32(eg_pi->vddci_voltage_table.mask_low);
   1296	}
   1297}
   1298
   1299static int ni_populate_voltage_value(struct radeon_device *rdev,
   1300				     struct atom_voltage_table *table,
   1301				     u16 value,
   1302				     NISLANDS_SMC_VOLTAGE_VALUE *voltage)
   1303{
   1304	unsigned int i;
   1305
   1306	for (i = 0; i < table->count; i++) {
   1307		if (value <= table->entries[i].value) {
   1308			voltage->index = (u8)i;
   1309			voltage->value = cpu_to_be16(table->entries[i].value);
   1310			break;
   1311		}
   1312	}
   1313
   1314	if (i >= table->count)
   1315		return -EINVAL;
   1316
   1317	return 0;
   1318}
   1319
   1320static void ni_populate_mvdd_value(struct radeon_device *rdev,
   1321				   u32 mclk,
   1322				   NISLANDS_SMC_VOLTAGE_VALUE *voltage)
   1323{
   1324	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   1325	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   1326
   1327	if (!pi->mvdd_control) {
   1328		voltage->index = eg_pi->mvdd_high_index;
   1329		voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
   1330		return;
   1331	}
   1332
   1333	if (mclk <= pi->mvdd_split_frequency) {
   1334		voltage->index = eg_pi->mvdd_low_index;
   1335		voltage->value = cpu_to_be16(MVDD_LOW_VALUE);
   1336	} else {
   1337		voltage->index = eg_pi->mvdd_high_index;
   1338		voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
   1339	}
   1340}
   1341
   1342static int ni_get_std_voltage_value(struct radeon_device *rdev,
   1343				    NISLANDS_SMC_VOLTAGE_VALUE *voltage,
   1344				    u16 *std_voltage)
   1345{
   1346	if (rdev->pm.dpm.dyn_state.cac_leakage_table.entries &&
   1347	    ((u32)voltage->index < rdev->pm.dpm.dyn_state.cac_leakage_table.count))
   1348		*std_voltage = rdev->pm.dpm.dyn_state.cac_leakage_table.entries[voltage->index].vddc;
   1349	else
   1350		*std_voltage = be16_to_cpu(voltage->value);
   1351
   1352	return 0;
   1353}
   1354
   1355static void ni_populate_std_voltage_value(struct radeon_device *rdev,
   1356					  u16 value, u8 index,
   1357					  NISLANDS_SMC_VOLTAGE_VALUE *voltage)
   1358{
   1359	voltage->index = index;
   1360	voltage->value = cpu_to_be16(value);
   1361}
   1362
   1363static u32 ni_get_smc_power_scaling_factor(struct radeon_device *rdev)
   1364{
   1365	u32 xclk_period;
   1366	u32 xclk = radeon_get_xclk(rdev);
   1367	u32 tmp = RREG32(CG_CAC_CTRL) & TID_CNT_MASK;
   1368
   1369	xclk_period = (1000000000UL / xclk);
   1370	xclk_period /= 10000UL;
   1371
   1372	return tmp * xclk_period;
   1373}
   1374
   1375static u32 ni_scale_power_for_smc(u32 power_in_watts, u32 scaling_factor)
   1376{
   1377	return (power_in_watts * scaling_factor) << 2;
   1378}
   1379
   1380static u32 ni_calculate_power_boost_limit(struct radeon_device *rdev,
   1381					  struct radeon_ps *radeon_state,
   1382					  u32 near_tdp_limit)
   1383{
   1384	struct ni_ps *state = ni_get_ps(radeon_state);
   1385	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   1386	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   1387	u32 power_boost_limit = 0;
   1388	int ret;
   1389
   1390	if (ni_pi->enable_power_containment &&
   1391	    ni_pi->use_power_boost_limit) {
   1392		NISLANDS_SMC_VOLTAGE_VALUE vddc;
   1393		u16 std_vddc_med;
   1394		u16 std_vddc_high;
   1395		u64 tmp, n, d;
   1396
   1397		if (state->performance_level_count < 3)
   1398			return 0;
   1399
   1400		ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
   1401						state->performance_levels[state->performance_level_count - 2].vddc,
   1402						&vddc);
   1403		if (ret)
   1404			return 0;
   1405
   1406		ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_med);
   1407		if (ret)
   1408			return 0;
   1409
   1410		ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
   1411						state->performance_levels[state->performance_level_count - 1].vddc,
   1412						&vddc);
   1413		if (ret)
   1414			return 0;
   1415
   1416		ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_high);
   1417		if (ret)
   1418			return 0;
   1419
   1420		n = ((u64)near_tdp_limit * ((u64)std_vddc_med * (u64)std_vddc_med) * 90);
   1421		d = ((u64)std_vddc_high * (u64)std_vddc_high * 100);
   1422		tmp = div64_u64(n, d);
   1423
   1424		if (tmp >> 32)
   1425			return 0;
   1426		power_boost_limit = (u32)tmp;
   1427	}
   1428
   1429	return power_boost_limit;
   1430}
   1431
   1432static int ni_calculate_adjusted_tdp_limits(struct radeon_device *rdev,
   1433					    bool adjust_polarity,
   1434					    u32 tdp_adjustment,
   1435					    u32 *tdp_limit,
   1436					    u32 *near_tdp_limit)
   1437{
   1438	if (tdp_adjustment > (u32)rdev->pm.dpm.tdp_od_limit)
   1439		return -EINVAL;
   1440
   1441	if (adjust_polarity) {
   1442		*tdp_limit = ((100 + tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
   1443		*near_tdp_limit = rdev->pm.dpm.near_tdp_limit + (*tdp_limit - rdev->pm.dpm.tdp_limit);
   1444	} else {
   1445		*tdp_limit = ((100 - tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
   1446		*near_tdp_limit = rdev->pm.dpm.near_tdp_limit - (rdev->pm.dpm.tdp_limit - *tdp_limit);
   1447	}
   1448
   1449	return 0;
   1450}
   1451
   1452static int ni_populate_smc_tdp_limits(struct radeon_device *rdev,
   1453				      struct radeon_ps *radeon_state)
   1454{
   1455	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   1456	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   1457
   1458	if (ni_pi->enable_power_containment) {
   1459		NISLANDS_SMC_STATETABLE *smc_table = &ni_pi->smc_statetable;
   1460		u32 scaling_factor = ni_get_smc_power_scaling_factor(rdev);
   1461		u32 tdp_limit;
   1462		u32 near_tdp_limit;
   1463		u32 power_boost_limit;
   1464		int ret;
   1465
   1466		if (scaling_factor == 0)
   1467			return -EINVAL;
   1468
   1469		memset(smc_table, 0, sizeof(NISLANDS_SMC_STATETABLE));
   1470
   1471		ret = ni_calculate_adjusted_tdp_limits(rdev,
   1472						       false, /* ??? */
   1473						       rdev->pm.dpm.tdp_adjustment,
   1474						       &tdp_limit,
   1475						       &near_tdp_limit);
   1476		if (ret)
   1477			return ret;
   1478
   1479		power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state,
   1480								   near_tdp_limit);
   1481
   1482		smc_table->dpm2Params.TDPLimit =
   1483			cpu_to_be32(ni_scale_power_for_smc(tdp_limit, scaling_factor));
   1484		smc_table->dpm2Params.NearTDPLimit =
   1485			cpu_to_be32(ni_scale_power_for_smc(near_tdp_limit, scaling_factor));
   1486		smc_table->dpm2Params.SafePowerLimit =
   1487			cpu_to_be32(ni_scale_power_for_smc((near_tdp_limit * NISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT) / 100,
   1488							   scaling_factor));
   1489		smc_table->dpm2Params.PowerBoostLimit =
   1490			cpu_to_be32(ni_scale_power_for_smc(power_boost_limit, scaling_factor));
   1491
   1492		ret = rv770_copy_bytes_to_smc(rdev,
   1493					      (u16)(pi->state_table_start + offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
   1494						    offsetof(PP_NIslands_DPM2Parameters, TDPLimit)),
   1495					      (u8 *)(&smc_table->dpm2Params.TDPLimit),
   1496					      sizeof(u32) * 4, pi->sram_end);
   1497		if (ret)
   1498			return ret;
   1499	}
   1500
   1501	return 0;
   1502}
   1503
   1504int ni_copy_and_switch_arb_sets(struct radeon_device *rdev,
   1505				u32 arb_freq_src, u32 arb_freq_dest)
   1506{
   1507	u32 mc_arb_dram_timing;
   1508	u32 mc_arb_dram_timing2;
   1509	u32 burst_time;
   1510	u32 mc_cg_config;
   1511
   1512	switch (arb_freq_src) {
   1513	case MC_CG_ARB_FREQ_F0:
   1514		mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING);
   1515		mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
   1516		burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE0_MASK) >> STATE0_SHIFT;
   1517		break;
   1518	case MC_CG_ARB_FREQ_F1:
   1519		mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_1);
   1520		mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_1);
   1521		burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE1_MASK) >> STATE1_SHIFT;
   1522		break;
   1523	case MC_CG_ARB_FREQ_F2:
   1524		mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_2);
   1525		mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_2);
   1526		burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE2_MASK) >> STATE2_SHIFT;
   1527		break;
   1528	case MC_CG_ARB_FREQ_F3:
   1529		mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_3);
   1530		mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_3);
   1531		burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE3_MASK) >> STATE3_SHIFT;
   1532		break;
   1533	default:
   1534		return -EINVAL;
   1535	}
   1536
   1537	switch (arb_freq_dest) {
   1538	case MC_CG_ARB_FREQ_F0:
   1539		WREG32(MC_ARB_DRAM_TIMING, mc_arb_dram_timing);
   1540		WREG32(MC_ARB_DRAM_TIMING2, mc_arb_dram_timing2);
   1541		WREG32_P(MC_ARB_BURST_TIME, STATE0(burst_time), ~STATE0_MASK);
   1542		break;
   1543	case MC_CG_ARB_FREQ_F1:
   1544		WREG32(MC_ARB_DRAM_TIMING_1, mc_arb_dram_timing);
   1545		WREG32(MC_ARB_DRAM_TIMING2_1, mc_arb_dram_timing2);
   1546		WREG32_P(MC_ARB_BURST_TIME, STATE1(burst_time), ~STATE1_MASK);
   1547		break;
   1548	case MC_CG_ARB_FREQ_F2:
   1549		WREG32(MC_ARB_DRAM_TIMING_2, mc_arb_dram_timing);
   1550		WREG32(MC_ARB_DRAM_TIMING2_2, mc_arb_dram_timing2);
   1551		WREG32_P(MC_ARB_BURST_TIME, STATE2(burst_time), ~STATE2_MASK);
   1552		break;
   1553	case MC_CG_ARB_FREQ_F3:
   1554		WREG32(MC_ARB_DRAM_TIMING_3, mc_arb_dram_timing);
   1555		WREG32(MC_ARB_DRAM_TIMING2_3, mc_arb_dram_timing2);
   1556		WREG32_P(MC_ARB_BURST_TIME, STATE3(burst_time), ~STATE3_MASK);
   1557		break;
   1558	default:
   1559		return -EINVAL;
   1560	}
   1561
   1562	mc_cg_config = RREG32(MC_CG_CONFIG) | 0x0000000F;
   1563	WREG32(MC_CG_CONFIG, mc_cg_config);
   1564	WREG32_P(MC_ARB_CG, CG_ARB_REQ(arb_freq_dest), ~CG_ARB_REQ_MASK);
   1565
   1566	return 0;
   1567}
   1568
   1569static int ni_init_arb_table_index(struct radeon_device *rdev)
   1570{
   1571	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   1572	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   1573	u32 tmp;
   1574	int ret;
   1575
   1576	ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
   1577					&tmp, pi->sram_end);
   1578	if (ret)
   1579		return ret;
   1580
   1581	tmp &= 0x00FFFFFF;
   1582	tmp |= ((u32)MC_CG_ARB_FREQ_F1) << 24;
   1583
   1584	return rv770_write_smc_sram_dword(rdev, ni_pi->arb_table_start,
   1585					  tmp, pi->sram_end);
   1586}
   1587
   1588static int ni_initial_switch_from_arb_f0_to_f1(struct radeon_device *rdev)
   1589{
   1590	return ni_copy_and_switch_arb_sets(rdev, MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
   1591}
   1592
   1593static int ni_force_switch_to_arb_f0(struct radeon_device *rdev)
   1594{
   1595	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   1596	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   1597	u32 tmp;
   1598	int ret;
   1599
   1600	ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
   1601					&tmp, pi->sram_end);
   1602	if (ret)
   1603		return ret;
   1604
   1605	tmp = (tmp >> 24) & 0xff;
   1606
   1607	if (tmp == MC_CG_ARB_FREQ_F0)
   1608		return 0;
   1609
   1610	return ni_copy_and_switch_arb_sets(rdev, tmp, MC_CG_ARB_FREQ_F0);
   1611}
   1612
   1613static int ni_populate_memory_timing_parameters(struct radeon_device *rdev,
   1614						struct rv7xx_pl *pl,
   1615						SMC_NIslands_MCArbDramTimingRegisterSet *arb_regs)
   1616{
   1617	u32 dram_timing;
   1618	u32 dram_timing2;
   1619
   1620	arb_regs->mc_arb_rfsh_rate =
   1621		(u8)rv770_calculate_memory_refresh_rate(rdev, pl->sclk);
   1622
   1623
   1624	radeon_atom_set_engine_dram_timings(rdev, pl->sclk, pl->mclk);
   1625
   1626	dram_timing = RREG32(MC_ARB_DRAM_TIMING);
   1627	dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
   1628
   1629	arb_regs->mc_arb_dram_timing  = cpu_to_be32(dram_timing);
   1630	arb_regs->mc_arb_dram_timing2 = cpu_to_be32(dram_timing2);
   1631
   1632	return 0;
   1633}
   1634
   1635static int ni_do_program_memory_timing_parameters(struct radeon_device *rdev,
   1636						  struct radeon_ps *radeon_state,
   1637						  unsigned int first_arb_set)
   1638{
   1639	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   1640	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   1641	struct ni_ps *state = ni_get_ps(radeon_state);
   1642	SMC_NIslands_MCArbDramTimingRegisterSet arb_regs = { 0 };
   1643	int i, ret = 0;
   1644
   1645	for (i = 0; i < state->performance_level_count; i++) {
   1646		ret = ni_populate_memory_timing_parameters(rdev, &state->performance_levels[i], &arb_regs);
   1647		if (ret)
   1648			break;
   1649
   1650		ret = rv770_copy_bytes_to_smc(rdev,
   1651					      (u16)(ni_pi->arb_table_start +
   1652						    offsetof(SMC_NIslands_MCArbDramTimingRegisters, data) +
   1653						    sizeof(SMC_NIslands_MCArbDramTimingRegisterSet) * (first_arb_set + i)),
   1654					      (u8 *)&arb_regs,
   1655					      (u16)sizeof(SMC_NIslands_MCArbDramTimingRegisterSet),
   1656					      pi->sram_end);
   1657		if (ret)
   1658			break;
   1659	}
   1660	return ret;
   1661}
   1662
   1663static int ni_program_memory_timing_parameters(struct radeon_device *rdev,
   1664					       struct radeon_ps *radeon_new_state)
   1665{
   1666	return ni_do_program_memory_timing_parameters(rdev, radeon_new_state,
   1667						      NISLANDS_DRIVER_STATE_ARB_INDEX);
   1668}
   1669
   1670static void ni_populate_initial_mvdd_value(struct radeon_device *rdev,
   1671					   struct NISLANDS_SMC_VOLTAGE_VALUE *voltage)
   1672{
   1673	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   1674
   1675	voltage->index = eg_pi->mvdd_high_index;
   1676	voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
   1677}
   1678
   1679static int ni_populate_smc_initial_state(struct radeon_device *rdev,
   1680					 struct radeon_ps *radeon_initial_state,
   1681					 NISLANDS_SMC_STATETABLE *table)
   1682{
   1683	struct ni_ps *initial_state = ni_get_ps(radeon_initial_state);
   1684	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   1685	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   1686	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   1687	u32 reg;
   1688	int ret;
   1689
   1690	table->initialState.level.mclk.vMPLL_AD_FUNC_CNTL =
   1691		cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl);
   1692	table->initialState.level.mclk.vMPLL_AD_FUNC_CNTL_2 =
   1693		cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl_2);
   1694	table->initialState.level.mclk.vMPLL_DQ_FUNC_CNTL =
   1695		cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl);
   1696	table->initialState.level.mclk.vMPLL_DQ_FUNC_CNTL_2 =
   1697		cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl_2);
   1698	table->initialState.level.mclk.vMCLK_PWRMGT_CNTL =
   1699		cpu_to_be32(ni_pi->clock_registers.mclk_pwrmgt_cntl);
   1700	table->initialState.level.mclk.vDLL_CNTL =
   1701		cpu_to_be32(ni_pi->clock_registers.dll_cntl);
   1702	table->initialState.level.mclk.vMPLL_SS =
   1703		cpu_to_be32(ni_pi->clock_registers.mpll_ss1);
   1704	table->initialState.level.mclk.vMPLL_SS2 =
   1705		cpu_to_be32(ni_pi->clock_registers.mpll_ss2);
   1706	table->initialState.level.mclk.mclk_value =
   1707		cpu_to_be32(initial_state->performance_levels[0].mclk);
   1708
   1709	table->initialState.level.sclk.vCG_SPLL_FUNC_CNTL =
   1710		cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl);
   1711	table->initialState.level.sclk.vCG_SPLL_FUNC_CNTL_2 =
   1712		cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_2);
   1713	table->initialState.level.sclk.vCG_SPLL_FUNC_CNTL_3 =
   1714		cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_3);
   1715	table->initialState.level.sclk.vCG_SPLL_FUNC_CNTL_4 =
   1716		cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_4);
   1717	table->initialState.level.sclk.vCG_SPLL_SPREAD_SPECTRUM =
   1718		cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum);
   1719	table->initialState.level.sclk.vCG_SPLL_SPREAD_SPECTRUM_2 =
   1720		cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum_2);
   1721	table->initialState.level.sclk.sclk_value =
   1722		cpu_to_be32(initial_state->performance_levels[0].sclk);
   1723	table->initialState.level.arbRefreshState =
   1724		NISLANDS_INITIAL_STATE_ARB_INDEX;
   1725
   1726	table->initialState.level.ACIndex = 0;
   1727
   1728	ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
   1729					initial_state->performance_levels[0].vddc,
   1730					&table->initialState.level.vddc);
   1731	if (!ret) {
   1732		u16 std_vddc;
   1733
   1734		ret = ni_get_std_voltage_value(rdev,
   1735					       &table->initialState.level.vddc,
   1736					       &std_vddc);
   1737		if (!ret)
   1738			ni_populate_std_voltage_value(rdev, std_vddc,
   1739						      table->initialState.level.vddc.index,
   1740						      &table->initialState.level.std_vddc);
   1741	}
   1742
   1743	if (eg_pi->vddci_control)
   1744		ni_populate_voltage_value(rdev,
   1745					  &eg_pi->vddci_voltage_table,
   1746					  initial_state->performance_levels[0].vddci,
   1747					  &table->initialState.level.vddci);
   1748
   1749	ni_populate_initial_mvdd_value(rdev, &table->initialState.level.mvdd);
   1750
   1751	reg = CG_R(0xffff) | CG_L(0);
   1752	table->initialState.level.aT = cpu_to_be32(reg);
   1753
   1754	table->initialState.level.bSP = cpu_to_be32(pi->dsp);
   1755
   1756	if (pi->boot_in_gen2)
   1757		table->initialState.level.gen2PCIE = 1;
   1758	else
   1759		table->initialState.level.gen2PCIE = 0;
   1760
   1761	if (pi->mem_gddr5) {
   1762		table->initialState.level.strobeMode =
   1763			cypress_get_strobe_mode_settings(rdev,
   1764							 initial_state->performance_levels[0].mclk);
   1765
   1766		if (initial_state->performance_levels[0].mclk > pi->mclk_edc_enable_threshold)
   1767			table->initialState.level.mcFlags = NISLANDS_SMC_MC_EDC_RD_FLAG | NISLANDS_SMC_MC_EDC_WR_FLAG;
   1768		else
   1769			table->initialState.level.mcFlags =  0;
   1770	}
   1771
   1772	table->initialState.levelCount = 1;
   1773
   1774	table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC;
   1775
   1776	table->initialState.level.dpm2.MaxPS = 0;
   1777	table->initialState.level.dpm2.NearTDPDec = 0;
   1778	table->initialState.level.dpm2.AboveSafeInc = 0;
   1779	table->initialState.level.dpm2.BelowSafeInc = 0;
   1780
   1781	reg = MIN_POWER_MASK | MAX_POWER_MASK;
   1782	table->initialState.level.SQPowerThrottle = cpu_to_be32(reg);
   1783
   1784	reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
   1785	table->initialState.level.SQPowerThrottle_2 = cpu_to_be32(reg);
   1786
   1787	return 0;
   1788}
   1789
   1790static int ni_populate_smc_acpi_state(struct radeon_device *rdev,
   1791				      NISLANDS_SMC_STATETABLE *table)
   1792{
   1793	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   1794	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   1795	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   1796	u32 mpll_ad_func_cntl   = ni_pi->clock_registers.mpll_ad_func_cntl;
   1797	u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
   1798	u32 mpll_dq_func_cntl   = ni_pi->clock_registers.mpll_dq_func_cntl;
   1799	u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
   1800	u32 spll_func_cntl      = ni_pi->clock_registers.cg_spll_func_cntl;
   1801	u32 spll_func_cntl_2    = ni_pi->clock_registers.cg_spll_func_cntl_2;
   1802	u32 spll_func_cntl_3    = ni_pi->clock_registers.cg_spll_func_cntl_3;
   1803	u32 spll_func_cntl_4    = ni_pi->clock_registers.cg_spll_func_cntl_4;
   1804	u32 mclk_pwrmgt_cntl    = ni_pi->clock_registers.mclk_pwrmgt_cntl;
   1805	u32 dll_cntl            = ni_pi->clock_registers.dll_cntl;
   1806	u32 reg;
   1807	int ret;
   1808
   1809	table->ACPIState = table->initialState;
   1810
   1811	table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC;
   1812
   1813	if (pi->acpi_vddc) {
   1814		ret = ni_populate_voltage_value(rdev,
   1815						&eg_pi->vddc_voltage_table,
   1816						pi->acpi_vddc, &table->ACPIState.level.vddc);
   1817		if (!ret) {
   1818			u16 std_vddc;
   1819
   1820			ret = ni_get_std_voltage_value(rdev,
   1821						       &table->ACPIState.level.vddc, &std_vddc);
   1822			if (!ret)
   1823				ni_populate_std_voltage_value(rdev, std_vddc,
   1824							      table->ACPIState.level.vddc.index,
   1825							      &table->ACPIState.level.std_vddc);
   1826		}
   1827
   1828		if (pi->pcie_gen2) {
   1829			if (pi->acpi_pcie_gen2)
   1830				table->ACPIState.level.gen2PCIE = 1;
   1831			else
   1832				table->ACPIState.level.gen2PCIE = 0;
   1833		} else {
   1834			table->ACPIState.level.gen2PCIE = 0;
   1835		}
   1836	} else {
   1837		ret = ni_populate_voltage_value(rdev,
   1838						&eg_pi->vddc_voltage_table,
   1839						pi->min_vddc_in_table,
   1840						&table->ACPIState.level.vddc);
   1841		if (!ret) {
   1842			u16 std_vddc;
   1843
   1844			ret = ni_get_std_voltage_value(rdev,
   1845						       &table->ACPIState.level.vddc,
   1846						       &std_vddc);
   1847			if (!ret)
   1848				ni_populate_std_voltage_value(rdev, std_vddc,
   1849							      table->ACPIState.level.vddc.index,
   1850							      &table->ACPIState.level.std_vddc);
   1851		}
   1852		table->ACPIState.level.gen2PCIE = 0;
   1853	}
   1854
   1855	if (eg_pi->acpi_vddci) {
   1856		if (eg_pi->vddci_control)
   1857			ni_populate_voltage_value(rdev,
   1858						  &eg_pi->vddci_voltage_table,
   1859						  eg_pi->acpi_vddci,
   1860						  &table->ACPIState.level.vddci);
   1861	}
   1862
   1863
   1864	mpll_ad_func_cntl &= ~PDNB;
   1865
   1866	mpll_ad_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN;
   1867
   1868	if (pi->mem_gddr5)
   1869		mpll_dq_func_cntl &= ~PDNB;
   1870	mpll_dq_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN | BYPASS;
   1871
   1872
   1873	mclk_pwrmgt_cntl |= (MRDCKA0_RESET |
   1874			     MRDCKA1_RESET |
   1875			     MRDCKB0_RESET |
   1876			     MRDCKB1_RESET |
   1877			     MRDCKC0_RESET |
   1878			     MRDCKC1_RESET |
   1879			     MRDCKD0_RESET |
   1880			     MRDCKD1_RESET);
   1881
   1882	mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
   1883			      MRDCKA1_PDNB |
   1884			      MRDCKB0_PDNB |
   1885			      MRDCKB1_PDNB |
   1886			      MRDCKC0_PDNB |
   1887			      MRDCKC1_PDNB |
   1888			      MRDCKD0_PDNB |
   1889			      MRDCKD1_PDNB);
   1890
   1891	dll_cntl |= (MRDCKA0_BYPASS |
   1892		     MRDCKA1_BYPASS |
   1893		     MRDCKB0_BYPASS |
   1894		     MRDCKB1_BYPASS |
   1895		     MRDCKC0_BYPASS |
   1896		     MRDCKC1_BYPASS |
   1897		     MRDCKD0_BYPASS |
   1898		     MRDCKD1_BYPASS);
   1899
   1900	spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
   1901	spll_func_cntl_2 |= SCLK_MUX_SEL(4);
   1902
   1903	table->ACPIState.level.mclk.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
   1904	table->ACPIState.level.mclk.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
   1905	table->ACPIState.level.mclk.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
   1906	table->ACPIState.level.mclk.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
   1907	table->ACPIState.level.mclk.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
   1908	table->ACPIState.level.mclk.vDLL_CNTL = cpu_to_be32(dll_cntl);
   1909
   1910	table->ACPIState.level.mclk.mclk_value = 0;
   1911
   1912	table->ACPIState.level.sclk.vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl);
   1913	table->ACPIState.level.sclk.vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2);
   1914	table->ACPIState.level.sclk.vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3);
   1915	table->ACPIState.level.sclk.vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(spll_func_cntl_4);
   1916
   1917	table->ACPIState.level.sclk.sclk_value = 0;
   1918
   1919	ni_populate_mvdd_value(rdev, 0, &table->ACPIState.level.mvdd);
   1920
   1921	if (eg_pi->dynamic_ac_timing)
   1922		table->ACPIState.level.ACIndex = 1;
   1923
   1924	table->ACPIState.level.dpm2.MaxPS = 0;
   1925	table->ACPIState.level.dpm2.NearTDPDec = 0;
   1926	table->ACPIState.level.dpm2.AboveSafeInc = 0;
   1927	table->ACPIState.level.dpm2.BelowSafeInc = 0;
   1928
   1929	reg = MIN_POWER_MASK | MAX_POWER_MASK;
   1930	table->ACPIState.level.SQPowerThrottle = cpu_to_be32(reg);
   1931
   1932	reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
   1933	table->ACPIState.level.SQPowerThrottle_2 = cpu_to_be32(reg);
   1934
   1935	return 0;
   1936}
   1937
   1938static int ni_init_smc_table(struct radeon_device *rdev)
   1939{
   1940	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   1941	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   1942	int ret;
   1943	struct radeon_ps *radeon_boot_state = rdev->pm.dpm.boot_ps;
   1944	NISLANDS_SMC_STATETABLE *table = &ni_pi->smc_statetable;
   1945
   1946	memset(table, 0, sizeof(NISLANDS_SMC_STATETABLE));
   1947
   1948	ni_populate_smc_voltage_tables(rdev, table);
   1949
   1950	switch (rdev->pm.int_thermal_type) {
   1951	case THERMAL_TYPE_NI:
   1952	case THERMAL_TYPE_EMC2103_WITH_INTERNAL:
   1953		table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL;
   1954		break;
   1955	case THERMAL_TYPE_NONE:
   1956		table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE;
   1957		break;
   1958	default:
   1959		table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL;
   1960		break;
   1961	}
   1962
   1963	if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC)
   1964		table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
   1965
   1966	if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT)
   1967		table->systemFlags |= PPSMC_SYSTEMFLAG_REGULATOR_HOT;
   1968
   1969	if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
   1970		table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
   1971
   1972	if (pi->mem_gddr5)
   1973		table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
   1974
   1975	ret = ni_populate_smc_initial_state(rdev, radeon_boot_state, table);
   1976	if (ret)
   1977		return ret;
   1978
   1979	ret = ni_populate_smc_acpi_state(rdev, table);
   1980	if (ret)
   1981		return ret;
   1982
   1983	table->driverState.flags = table->initialState.flags;
   1984	table->driverState.levelCount = table->initialState.levelCount;
   1985	table->driverState.levels[0] = table->initialState.level;
   1986
   1987	table->ULVState = table->initialState;
   1988
   1989	ret = ni_do_program_memory_timing_parameters(rdev, radeon_boot_state,
   1990						     NISLANDS_INITIAL_STATE_ARB_INDEX);
   1991	if (ret)
   1992		return ret;
   1993
   1994	return rv770_copy_bytes_to_smc(rdev, pi->state_table_start, (u8 *)table,
   1995				       sizeof(NISLANDS_SMC_STATETABLE), pi->sram_end);
   1996}
   1997
   1998static int ni_calculate_sclk_params(struct radeon_device *rdev,
   1999				    u32 engine_clock,
   2000				    NISLANDS_SMC_SCLK_VALUE *sclk)
   2001{
   2002	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   2003	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   2004	struct atom_clock_dividers dividers;
   2005	u32 spll_func_cntl = ni_pi->clock_registers.cg_spll_func_cntl;
   2006	u32 spll_func_cntl_2 = ni_pi->clock_registers.cg_spll_func_cntl_2;
   2007	u32 spll_func_cntl_3 = ni_pi->clock_registers.cg_spll_func_cntl_3;
   2008	u32 spll_func_cntl_4 = ni_pi->clock_registers.cg_spll_func_cntl_4;
   2009	u32 cg_spll_spread_spectrum = ni_pi->clock_registers.cg_spll_spread_spectrum;
   2010	u32 cg_spll_spread_spectrum_2 = ni_pi->clock_registers.cg_spll_spread_spectrum_2;
   2011	u64 tmp;
   2012	u32 reference_clock = rdev->clock.spll.reference_freq;
   2013	u32 reference_divider;
   2014	u32 fbdiv;
   2015	int ret;
   2016
   2017	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
   2018					     engine_clock, false, &dividers);
   2019	if (ret)
   2020		return ret;
   2021
   2022	reference_divider = 1 + dividers.ref_div;
   2023
   2024
   2025	tmp = (u64) engine_clock * reference_divider * dividers.post_div * 16834;
   2026	do_div(tmp, reference_clock);
   2027	fbdiv = (u32) tmp;
   2028
   2029	spll_func_cntl &= ~(SPLL_PDIV_A_MASK | SPLL_REF_DIV_MASK);
   2030	spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div);
   2031	spll_func_cntl |= SPLL_PDIV_A(dividers.post_div);
   2032
   2033	spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
   2034	spll_func_cntl_2 |= SCLK_MUX_SEL(2);
   2035
   2036	spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK;
   2037	spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv);
   2038	spll_func_cntl_3 |= SPLL_DITHEN;
   2039
   2040	if (pi->sclk_ss) {
   2041		struct radeon_atom_ss ss;
   2042		u32 vco_freq = engine_clock * dividers.post_div;
   2043
   2044		if (radeon_atombios_get_asic_ss_info(rdev, &ss,
   2045						     ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
   2046			u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate);
   2047			u32 clk_v = 4 * ss.percentage * fbdiv / (clk_s * 10000);
   2048
   2049			cg_spll_spread_spectrum &= ~CLK_S_MASK;
   2050			cg_spll_spread_spectrum |= CLK_S(clk_s);
   2051			cg_spll_spread_spectrum |= SSEN;
   2052
   2053			cg_spll_spread_spectrum_2 &= ~CLK_V_MASK;
   2054			cg_spll_spread_spectrum_2 |= CLK_V(clk_v);
   2055		}
   2056	}
   2057
   2058	sclk->sclk_value = engine_clock;
   2059	sclk->vCG_SPLL_FUNC_CNTL = spll_func_cntl;
   2060	sclk->vCG_SPLL_FUNC_CNTL_2 = spll_func_cntl_2;
   2061	sclk->vCG_SPLL_FUNC_CNTL_3 = spll_func_cntl_3;
   2062	sclk->vCG_SPLL_FUNC_CNTL_4 = spll_func_cntl_4;
   2063	sclk->vCG_SPLL_SPREAD_SPECTRUM = cg_spll_spread_spectrum;
   2064	sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cg_spll_spread_spectrum_2;
   2065
   2066	return 0;
   2067}
   2068
   2069static int ni_populate_sclk_value(struct radeon_device *rdev,
   2070				  u32 engine_clock,
   2071				  NISLANDS_SMC_SCLK_VALUE *sclk)
   2072{
   2073	NISLANDS_SMC_SCLK_VALUE sclk_tmp;
   2074	int ret;
   2075
   2076	ret = ni_calculate_sclk_params(rdev, engine_clock, &sclk_tmp);
   2077	if (!ret) {
   2078		sclk->sclk_value = cpu_to_be32(sclk_tmp.sclk_value);
   2079		sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL);
   2080		sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_2);
   2081		sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_3);
   2082		sclk->vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_4);
   2083		sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM);
   2084		sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM_2);
   2085	}
   2086
   2087	return ret;
   2088}
   2089
   2090static int ni_init_smc_spll_table(struct radeon_device *rdev)
   2091{
   2092	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   2093	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   2094	SMC_NISLANDS_SPLL_DIV_TABLE *spll_table;
   2095	NISLANDS_SMC_SCLK_VALUE sclk_params;
   2096	u32 fb_div;
   2097	u32 p_div;
   2098	u32 clk_s;
   2099	u32 clk_v;
   2100	u32 sclk = 0;
   2101	int i, ret;
   2102	u32 tmp;
   2103
   2104	if (ni_pi->spll_table_start == 0)
   2105		return -EINVAL;
   2106
   2107	spll_table = kzalloc(sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), GFP_KERNEL);
   2108	if (spll_table == NULL)
   2109		return -ENOMEM;
   2110
   2111	for (i = 0; i < 256; i++) {
   2112		ret = ni_calculate_sclk_params(rdev, sclk, &sclk_params);
   2113		if (ret)
   2114			break;
   2115
   2116		p_div = (sclk_params.vCG_SPLL_FUNC_CNTL & SPLL_PDIV_A_MASK) >> SPLL_PDIV_A_SHIFT;
   2117		fb_div = (sclk_params.vCG_SPLL_FUNC_CNTL_3 & SPLL_FB_DIV_MASK) >> SPLL_FB_DIV_SHIFT;
   2118		clk_s = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM & CLK_S_MASK) >> CLK_S_SHIFT;
   2119		clk_v = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM_2 & CLK_V_MASK) >> CLK_V_SHIFT;
   2120
   2121		fb_div &= ~0x00001FFF;
   2122		fb_div >>= 1;
   2123		clk_v >>= 6;
   2124
   2125		if (p_div & ~(SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT))
   2126			ret = -EINVAL;
   2127
   2128		if (clk_s & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT))
   2129			ret = -EINVAL;
   2130
   2131		if (fb_div & ~(SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT))
   2132			ret = -EINVAL;
   2133
   2134		if (clk_v & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT))
   2135			ret = -EINVAL;
   2136
   2137		if (ret)
   2138			break;
   2139
   2140		tmp = ((fb_div << SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_MASK) |
   2141			((p_div << SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK);
   2142		spll_table->freq[i] = cpu_to_be32(tmp);
   2143
   2144		tmp = ((clk_v << SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK) |
   2145			((clk_s << SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK);
   2146		spll_table->ss[i] = cpu_to_be32(tmp);
   2147
   2148		sclk += 512;
   2149	}
   2150
   2151	if (!ret)
   2152		ret = rv770_copy_bytes_to_smc(rdev, ni_pi->spll_table_start, (u8 *)spll_table,
   2153					      sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), pi->sram_end);
   2154
   2155	kfree(spll_table);
   2156
   2157	return ret;
   2158}
   2159
   2160static int ni_populate_mclk_value(struct radeon_device *rdev,
   2161				  u32 engine_clock,
   2162				  u32 memory_clock,
   2163				  NISLANDS_SMC_MCLK_VALUE *mclk,
   2164				  bool strobe_mode,
   2165				  bool dll_state_on)
   2166{
   2167	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   2168	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   2169	u32 mpll_ad_func_cntl = ni_pi->clock_registers.mpll_ad_func_cntl;
   2170	u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
   2171	u32 mpll_dq_func_cntl = ni_pi->clock_registers.mpll_dq_func_cntl;
   2172	u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
   2173	u32 mclk_pwrmgt_cntl = ni_pi->clock_registers.mclk_pwrmgt_cntl;
   2174	u32 dll_cntl = ni_pi->clock_registers.dll_cntl;
   2175	u32 mpll_ss1 = ni_pi->clock_registers.mpll_ss1;
   2176	u32 mpll_ss2 = ni_pi->clock_registers.mpll_ss2;
   2177	struct atom_clock_dividers dividers;
   2178	u32 ibias;
   2179	u32 dll_speed;
   2180	int ret;
   2181	u32 mc_seq_misc7;
   2182
   2183	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
   2184					     memory_clock, strobe_mode, &dividers);
   2185	if (ret)
   2186		return ret;
   2187
   2188	if (!strobe_mode) {
   2189		mc_seq_misc7 = RREG32(MC_SEQ_MISC7);
   2190
   2191		if (mc_seq_misc7 & 0x8000000)
   2192			dividers.post_div = 1;
   2193	}
   2194
   2195	ibias = cypress_map_clkf_to_ibias(rdev, dividers.whole_fb_div);
   2196
   2197	mpll_ad_func_cntl &= ~(CLKR_MASK |
   2198			       YCLK_POST_DIV_MASK |
   2199			       CLKF_MASK |
   2200			       CLKFRAC_MASK |
   2201			       IBIAS_MASK);
   2202	mpll_ad_func_cntl |= CLKR(dividers.ref_div);
   2203	mpll_ad_func_cntl |= YCLK_POST_DIV(dividers.post_div);
   2204	mpll_ad_func_cntl |= CLKF(dividers.whole_fb_div);
   2205	mpll_ad_func_cntl |= CLKFRAC(dividers.frac_fb_div);
   2206	mpll_ad_func_cntl |= IBIAS(ibias);
   2207
   2208	if (dividers.vco_mode)
   2209		mpll_ad_func_cntl_2 |= VCO_MODE;
   2210	else
   2211		mpll_ad_func_cntl_2 &= ~VCO_MODE;
   2212
   2213	if (pi->mem_gddr5) {
   2214		mpll_dq_func_cntl &= ~(CLKR_MASK |
   2215				       YCLK_POST_DIV_MASK |
   2216				       CLKF_MASK |
   2217				       CLKFRAC_MASK |
   2218				       IBIAS_MASK);
   2219		mpll_dq_func_cntl |= CLKR(dividers.ref_div);
   2220		mpll_dq_func_cntl |= YCLK_POST_DIV(dividers.post_div);
   2221		mpll_dq_func_cntl |= CLKF(dividers.whole_fb_div);
   2222		mpll_dq_func_cntl |= CLKFRAC(dividers.frac_fb_div);
   2223		mpll_dq_func_cntl |= IBIAS(ibias);
   2224
   2225		if (strobe_mode)
   2226			mpll_dq_func_cntl &= ~PDNB;
   2227		else
   2228			mpll_dq_func_cntl |= PDNB;
   2229
   2230		if (dividers.vco_mode)
   2231			mpll_dq_func_cntl_2 |= VCO_MODE;
   2232		else
   2233			mpll_dq_func_cntl_2 &= ~VCO_MODE;
   2234	}
   2235
   2236	if (pi->mclk_ss) {
   2237		struct radeon_atom_ss ss;
   2238		u32 vco_freq = memory_clock * dividers.post_div;
   2239
   2240		if (radeon_atombios_get_asic_ss_info(rdev, &ss,
   2241						     ASIC_INTERNAL_MEMORY_SS, vco_freq)) {
   2242			u32 reference_clock = rdev->clock.mpll.reference_freq;
   2243			u32 decoded_ref = rv740_get_decoded_reference_divider(dividers.ref_div);
   2244			u32 clk_s = reference_clock * 5 / (decoded_ref * ss.rate);
   2245			u32 clk_v = ss.percentage *
   2246				(0x4000 * dividers.whole_fb_div + 0x800 * dividers.frac_fb_div) / (clk_s * 625);
   2247
   2248			mpll_ss1 &= ~CLKV_MASK;
   2249			mpll_ss1 |= CLKV(clk_v);
   2250
   2251			mpll_ss2 &= ~CLKS_MASK;
   2252			mpll_ss2 |= CLKS(clk_s);
   2253		}
   2254	}
   2255
   2256	dll_speed = rv740_get_dll_speed(pi->mem_gddr5,
   2257					memory_clock);
   2258
   2259	mclk_pwrmgt_cntl &= ~DLL_SPEED_MASK;
   2260	mclk_pwrmgt_cntl |= DLL_SPEED(dll_speed);
   2261	if (dll_state_on)
   2262		mclk_pwrmgt_cntl |= (MRDCKA0_PDNB |
   2263				     MRDCKA1_PDNB |
   2264				     MRDCKB0_PDNB |
   2265				     MRDCKB1_PDNB |
   2266				     MRDCKC0_PDNB |
   2267				     MRDCKC1_PDNB |
   2268				     MRDCKD0_PDNB |
   2269				     MRDCKD1_PDNB);
   2270	else
   2271		mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
   2272				      MRDCKA1_PDNB |
   2273				      MRDCKB0_PDNB |
   2274				      MRDCKB1_PDNB |
   2275				      MRDCKC0_PDNB |
   2276				      MRDCKC1_PDNB |
   2277				      MRDCKD0_PDNB |
   2278				      MRDCKD1_PDNB);
   2279
   2280
   2281	mclk->mclk_value = cpu_to_be32(memory_clock);
   2282	mclk->vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
   2283	mclk->vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
   2284	mclk->vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
   2285	mclk->vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
   2286	mclk->vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
   2287	mclk->vDLL_CNTL = cpu_to_be32(dll_cntl);
   2288	mclk->vMPLL_SS = cpu_to_be32(mpll_ss1);
   2289	mclk->vMPLL_SS2 = cpu_to_be32(mpll_ss2);
   2290
   2291	return 0;
   2292}
   2293
   2294static void ni_populate_smc_sp(struct radeon_device *rdev,
   2295			       struct radeon_ps *radeon_state,
   2296			       NISLANDS_SMC_SWSTATE *smc_state)
   2297{
   2298	struct ni_ps *ps = ni_get_ps(radeon_state);
   2299	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   2300	int i;
   2301
   2302	for (i = 0; i < ps->performance_level_count - 1; i++)
   2303		smc_state->levels[i].bSP = cpu_to_be32(pi->dsp);
   2304
   2305	smc_state->levels[ps->performance_level_count - 1].bSP =
   2306		cpu_to_be32(pi->psp);
   2307}
   2308
   2309static int ni_convert_power_level_to_smc(struct radeon_device *rdev,
   2310					 struct rv7xx_pl *pl,
   2311					 NISLANDS_SMC_HW_PERFORMANCE_LEVEL *level)
   2312{
   2313	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   2314	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   2315	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   2316	int ret;
   2317	bool dll_state_on;
   2318	u16 std_vddc;
   2319	u32 tmp = RREG32(DC_STUTTER_CNTL);
   2320
   2321	level->gen2PCIE = pi->pcie_gen2 ?
   2322		((pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0) : 0;
   2323
   2324	ret = ni_populate_sclk_value(rdev, pl->sclk, &level->sclk);
   2325	if (ret)
   2326		return ret;
   2327
   2328	level->mcFlags =  0;
   2329	if (pi->mclk_stutter_mode_threshold &&
   2330	    (pl->mclk <= pi->mclk_stutter_mode_threshold) &&
   2331	    !eg_pi->uvd_enabled &&
   2332	    (tmp & DC_STUTTER_ENABLE_A) &&
   2333	    (tmp & DC_STUTTER_ENABLE_B))
   2334		level->mcFlags |= NISLANDS_SMC_MC_STUTTER_EN;
   2335
   2336	if (pi->mem_gddr5) {
   2337		if (pl->mclk > pi->mclk_edc_enable_threshold)
   2338			level->mcFlags |= NISLANDS_SMC_MC_EDC_RD_FLAG;
   2339		if (pl->mclk > eg_pi->mclk_edc_wr_enable_threshold)
   2340			level->mcFlags |= NISLANDS_SMC_MC_EDC_WR_FLAG;
   2341
   2342		level->strobeMode = cypress_get_strobe_mode_settings(rdev, pl->mclk);
   2343
   2344		if (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) {
   2345			if (cypress_get_mclk_frequency_ratio(rdev, pl->mclk, true) >=
   2346			    ((RREG32(MC_SEQ_MISC7) >> 16) & 0xf))
   2347				dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false;
   2348			else
   2349				dll_state_on = ((RREG32(MC_SEQ_MISC6) >> 1) & 0x1) ? true : false;
   2350		} else {
   2351			dll_state_on = false;
   2352			if (pl->mclk > ni_pi->mclk_rtt_mode_threshold)
   2353				level->mcFlags |= NISLANDS_SMC_MC_RTT_ENABLE;
   2354		}
   2355
   2356		ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk,
   2357					     &level->mclk,
   2358					     (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) != 0,
   2359					     dll_state_on);
   2360	} else
   2361		ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk, &level->mclk, 1, 1);
   2362
   2363	if (ret)
   2364		return ret;
   2365
   2366	ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
   2367					pl->vddc, &level->vddc);
   2368	if (ret)
   2369		return ret;
   2370
   2371	ret = ni_get_std_voltage_value(rdev, &level->vddc, &std_vddc);
   2372	if (ret)
   2373		return ret;
   2374
   2375	ni_populate_std_voltage_value(rdev, std_vddc,
   2376				      level->vddc.index, &level->std_vddc);
   2377
   2378	if (eg_pi->vddci_control) {
   2379		ret = ni_populate_voltage_value(rdev, &eg_pi->vddci_voltage_table,
   2380						pl->vddci, &level->vddci);
   2381		if (ret)
   2382			return ret;
   2383	}
   2384
   2385	ni_populate_mvdd_value(rdev, pl->mclk, &level->mvdd);
   2386
   2387	return ret;
   2388}
   2389
   2390static int ni_populate_smc_t(struct radeon_device *rdev,
   2391			     struct radeon_ps *radeon_state,
   2392			     NISLANDS_SMC_SWSTATE *smc_state)
   2393{
   2394	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   2395	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   2396	struct ni_ps *state = ni_get_ps(radeon_state);
   2397	u32 a_t;
   2398	u32 t_l, t_h;
   2399	u32 high_bsp;
   2400	int i, ret;
   2401
   2402	if (state->performance_level_count >= 9)
   2403		return -EINVAL;
   2404
   2405	if (state->performance_level_count < 2) {
   2406		a_t = CG_R(0xffff) | CG_L(0);
   2407		smc_state->levels[0].aT = cpu_to_be32(a_t);
   2408		return 0;
   2409	}
   2410
   2411	smc_state->levels[0].aT = cpu_to_be32(0);
   2412
   2413	for (i = 0; i <= state->performance_level_count - 2; i++) {
   2414		if (eg_pi->uvd_enabled)
   2415			ret = r600_calculate_at(
   2416				1000 * (i * (eg_pi->smu_uvd_hs ? 2 : 8) + 2),
   2417				100 * R600_AH_DFLT,
   2418				state->performance_levels[i + 1].sclk,
   2419				state->performance_levels[i].sclk,
   2420				&t_l,
   2421				&t_h);
   2422		else
   2423			ret = r600_calculate_at(
   2424				1000 * (i + 1),
   2425				100 * R600_AH_DFLT,
   2426				state->performance_levels[i + 1].sclk,
   2427				state->performance_levels[i].sclk,
   2428				&t_l,
   2429				&t_h);
   2430
   2431		if (ret) {
   2432			t_h = (i + 1) * 1000 - 50 * R600_AH_DFLT;
   2433			t_l = (i + 1) * 1000 + 50 * R600_AH_DFLT;
   2434		}
   2435
   2436		a_t = be32_to_cpu(smc_state->levels[i].aT) & ~CG_R_MASK;
   2437		a_t |= CG_R(t_l * pi->bsp / 20000);
   2438		smc_state->levels[i].aT = cpu_to_be32(a_t);
   2439
   2440		high_bsp = (i == state->performance_level_count - 2) ?
   2441			pi->pbsp : pi->bsp;
   2442
   2443		a_t = CG_R(0xffff) | CG_L(t_h * high_bsp / 20000);
   2444		smc_state->levels[i + 1].aT = cpu_to_be32(a_t);
   2445	}
   2446
   2447	return 0;
   2448}
   2449
   2450static int ni_populate_power_containment_values(struct radeon_device *rdev,
   2451						struct radeon_ps *radeon_state,
   2452						NISLANDS_SMC_SWSTATE *smc_state)
   2453{
   2454	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   2455	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   2456	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   2457	struct ni_ps *state = ni_get_ps(radeon_state);
   2458	u32 prev_sclk;
   2459	u32 max_sclk;
   2460	u32 min_sclk;
   2461	int i, ret;
   2462	u32 tdp_limit;
   2463	u32 near_tdp_limit;
   2464	u32 power_boost_limit;
   2465	u8 max_ps_percent;
   2466
   2467	if (ni_pi->enable_power_containment == false)
   2468		return 0;
   2469
   2470	if (state->performance_level_count == 0)
   2471		return -EINVAL;
   2472
   2473	if (smc_state->levelCount != state->performance_level_count)
   2474		return -EINVAL;
   2475
   2476	ret = ni_calculate_adjusted_tdp_limits(rdev,
   2477					       false, /* ??? */
   2478					       rdev->pm.dpm.tdp_adjustment,
   2479					       &tdp_limit,
   2480					       &near_tdp_limit);
   2481	if (ret)
   2482		return ret;
   2483
   2484	power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state, near_tdp_limit);
   2485
   2486	ret = rv770_write_smc_sram_dword(rdev,
   2487					 pi->state_table_start +
   2488					 offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
   2489					 offsetof(PP_NIslands_DPM2Parameters, PowerBoostLimit),
   2490					 ni_scale_power_for_smc(power_boost_limit, ni_get_smc_power_scaling_factor(rdev)),
   2491					 pi->sram_end);
   2492	if (ret)
   2493		power_boost_limit = 0;
   2494
   2495	smc_state->levels[0].dpm2.MaxPS = 0;
   2496	smc_state->levels[0].dpm2.NearTDPDec = 0;
   2497	smc_state->levels[0].dpm2.AboveSafeInc = 0;
   2498	smc_state->levels[0].dpm2.BelowSafeInc = 0;
   2499	smc_state->levels[0].stateFlags |= power_boost_limit ? PPSMC_STATEFLAG_POWERBOOST : 0;
   2500
   2501	for (i = 1; i < state->performance_level_count; i++) {
   2502		prev_sclk = state->performance_levels[i-1].sclk;
   2503		max_sclk  = state->performance_levels[i].sclk;
   2504		max_ps_percent = (i != (state->performance_level_count - 1)) ?
   2505			NISLANDS_DPM2_MAXPS_PERCENT_M : NISLANDS_DPM2_MAXPS_PERCENT_H;
   2506
   2507		if (max_sclk < prev_sclk)
   2508			return -EINVAL;
   2509
   2510		if ((max_ps_percent == 0) || (prev_sclk == max_sclk) || eg_pi->uvd_enabled)
   2511			min_sclk = max_sclk;
   2512		else if (1 == i)
   2513			min_sclk = prev_sclk;
   2514		else
   2515			min_sclk = (prev_sclk * (u32)max_ps_percent) / 100;
   2516
   2517		if (min_sclk < state->performance_levels[0].sclk)
   2518			min_sclk = state->performance_levels[0].sclk;
   2519
   2520		if (min_sclk == 0)
   2521			return -EINVAL;
   2522
   2523		smc_state->levels[i].dpm2.MaxPS =
   2524			(u8)((NISLANDS_DPM2_MAX_PULSE_SKIP * (max_sclk - min_sclk)) / max_sclk);
   2525		smc_state->levels[i].dpm2.NearTDPDec = NISLANDS_DPM2_NEAR_TDP_DEC;
   2526		smc_state->levels[i].dpm2.AboveSafeInc = NISLANDS_DPM2_ABOVE_SAFE_INC;
   2527		smc_state->levels[i].dpm2.BelowSafeInc = NISLANDS_DPM2_BELOW_SAFE_INC;
   2528		smc_state->levels[i].stateFlags |=
   2529			((i != (state->performance_level_count - 1)) && power_boost_limit) ?
   2530			PPSMC_STATEFLAG_POWERBOOST : 0;
   2531	}
   2532
   2533	return 0;
   2534}
   2535
   2536static int ni_populate_sq_ramping_values(struct radeon_device *rdev,
   2537					 struct radeon_ps *radeon_state,
   2538					 NISLANDS_SMC_SWSTATE *smc_state)
   2539{
   2540	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   2541	struct ni_ps *state = ni_get_ps(radeon_state);
   2542	u32 sq_power_throttle;
   2543	u32 sq_power_throttle2;
   2544	bool enable_sq_ramping = ni_pi->enable_sq_ramping;
   2545	int i;
   2546
   2547	if (state->performance_level_count == 0)
   2548		return -EINVAL;
   2549
   2550	if (smc_state->levelCount != state->performance_level_count)
   2551		return -EINVAL;
   2552
   2553	if (rdev->pm.dpm.sq_ramping_threshold == 0)
   2554		return -EINVAL;
   2555
   2556	if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER > (MAX_POWER_MASK >> MAX_POWER_SHIFT))
   2557		enable_sq_ramping = false;
   2558
   2559	if (NISLANDS_DPM2_SQ_RAMP_MIN_POWER > (MIN_POWER_MASK >> MIN_POWER_SHIFT))
   2560		enable_sq_ramping = false;
   2561
   2562	if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA > (MAX_POWER_DELTA_MASK >> MAX_POWER_DELTA_SHIFT))
   2563		enable_sq_ramping = false;
   2564
   2565	if (NISLANDS_DPM2_SQ_RAMP_STI_SIZE > (STI_SIZE_MASK >> STI_SIZE_SHIFT))
   2566		enable_sq_ramping = false;
   2567
   2568	if (NISLANDS_DPM2_SQ_RAMP_LTI_RATIO > (LTI_RATIO_MASK >> LTI_RATIO_SHIFT))
   2569		enable_sq_ramping = false;
   2570
   2571	for (i = 0; i < state->performance_level_count; i++) {
   2572		sq_power_throttle  = 0;
   2573		sq_power_throttle2 = 0;
   2574
   2575		if ((state->performance_levels[i].sclk >= rdev->pm.dpm.sq_ramping_threshold) &&
   2576		    enable_sq_ramping) {
   2577			sq_power_throttle |= MAX_POWER(NISLANDS_DPM2_SQ_RAMP_MAX_POWER);
   2578			sq_power_throttle |= MIN_POWER(NISLANDS_DPM2_SQ_RAMP_MIN_POWER);
   2579			sq_power_throttle2 |= MAX_POWER_DELTA(NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA);
   2580			sq_power_throttle2 |= STI_SIZE(NISLANDS_DPM2_SQ_RAMP_STI_SIZE);
   2581			sq_power_throttle2 |= LTI_RATIO(NISLANDS_DPM2_SQ_RAMP_LTI_RATIO);
   2582		} else {
   2583			sq_power_throttle |= MAX_POWER_MASK | MIN_POWER_MASK;
   2584			sq_power_throttle2 |= MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
   2585		}
   2586
   2587		smc_state->levels[i].SQPowerThrottle   = cpu_to_be32(sq_power_throttle);
   2588		smc_state->levels[i].SQPowerThrottle_2 = cpu_to_be32(sq_power_throttle2);
   2589	}
   2590
   2591	return 0;
   2592}
   2593
   2594static int ni_enable_power_containment(struct radeon_device *rdev,
   2595				       struct radeon_ps *radeon_new_state,
   2596				       bool enable)
   2597{
   2598	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   2599	PPSMC_Result smc_result;
   2600	int ret = 0;
   2601
   2602	if (ni_pi->enable_power_containment) {
   2603		if (enable) {
   2604			if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
   2605				smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingActive);
   2606				if (smc_result != PPSMC_Result_OK) {
   2607					ret = -EINVAL;
   2608					ni_pi->pc_enabled = false;
   2609				} else {
   2610					ni_pi->pc_enabled = true;
   2611				}
   2612			}
   2613		} else {
   2614			smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingInactive);
   2615			if (smc_result != PPSMC_Result_OK)
   2616				ret = -EINVAL;
   2617			ni_pi->pc_enabled = false;
   2618		}
   2619	}
   2620
   2621	return ret;
   2622}
   2623
   2624static int ni_convert_power_state_to_smc(struct radeon_device *rdev,
   2625					 struct radeon_ps *radeon_state,
   2626					 NISLANDS_SMC_SWSTATE *smc_state)
   2627{
   2628	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   2629	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   2630	struct ni_ps *state = ni_get_ps(radeon_state);
   2631	int i, ret;
   2632	u32 threshold = state->performance_levels[state->performance_level_count - 1].sclk * 100 / 100;
   2633
   2634	if (!(radeon_state->caps & ATOM_PPLIB_DISALLOW_ON_DC))
   2635		smc_state->flags |= PPSMC_SWSTATE_FLAG_DC;
   2636
   2637	smc_state->levelCount = 0;
   2638
   2639	if (state->performance_level_count > NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE)
   2640		return -EINVAL;
   2641
   2642	for (i = 0; i < state->performance_level_count; i++) {
   2643		ret = ni_convert_power_level_to_smc(rdev, &state->performance_levels[i],
   2644						    &smc_state->levels[i]);
   2645		smc_state->levels[i].arbRefreshState =
   2646			(u8)(NISLANDS_DRIVER_STATE_ARB_INDEX + i);
   2647
   2648		if (ret)
   2649			return ret;
   2650
   2651		if (ni_pi->enable_power_containment)
   2652			smc_state->levels[i].displayWatermark =
   2653				(state->performance_levels[i].sclk < threshold) ?
   2654				PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
   2655		else
   2656			smc_state->levels[i].displayWatermark = (i < 2) ?
   2657				PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
   2658
   2659		if (eg_pi->dynamic_ac_timing)
   2660			smc_state->levels[i].ACIndex = NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i;
   2661		else
   2662			smc_state->levels[i].ACIndex = 0;
   2663
   2664		smc_state->levelCount++;
   2665	}
   2666
   2667	rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_watermark_threshold,
   2668				      cpu_to_be32(threshold / 512));
   2669
   2670	ni_populate_smc_sp(rdev, radeon_state, smc_state);
   2671
   2672	ret = ni_populate_power_containment_values(rdev, radeon_state, smc_state);
   2673	if (ret)
   2674		ni_pi->enable_power_containment = false;
   2675
   2676	ret = ni_populate_sq_ramping_values(rdev, radeon_state, smc_state);
   2677	if (ret)
   2678		ni_pi->enable_sq_ramping = false;
   2679
   2680	return ni_populate_smc_t(rdev, radeon_state, smc_state);
   2681}
   2682
   2683static int ni_upload_sw_state(struct radeon_device *rdev,
   2684			      struct radeon_ps *radeon_new_state)
   2685{
   2686	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   2687	u16 address = pi->state_table_start +
   2688		offsetof(NISLANDS_SMC_STATETABLE, driverState);
   2689	NISLANDS_SMC_SWSTATE *smc_state;
   2690	size_t state_size = struct_size(smc_state, levels,
   2691			NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE);
   2692	int ret;
   2693
   2694	smc_state = kzalloc(state_size, GFP_KERNEL);
   2695	if (smc_state == NULL)
   2696		return -ENOMEM;
   2697
   2698	ret = ni_convert_power_state_to_smc(rdev, radeon_new_state, smc_state);
   2699	if (ret)
   2700		goto done;
   2701
   2702	ret = rv770_copy_bytes_to_smc(rdev, address, (u8 *)smc_state, state_size, pi->sram_end);
   2703
   2704done:
   2705	kfree(smc_state);
   2706
   2707	return ret;
   2708}
   2709
   2710static int ni_set_mc_special_registers(struct radeon_device *rdev,
   2711				       struct ni_mc_reg_table *table)
   2712{
   2713	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   2714	u8 i, j, k;
   2715	u32 temp_reg;
   2716
   2717	for (i = 0, j = table->last; i < table->last; i++) {
   2718		switch (table->mc_reg_address[i].s1) {
   2719		case MC_SEQ_MISC1 >> 2:
   2720			if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
   2721				return -EINVAL;
   2722			temp_reg = RREG32(MC_PMG_CMD_EMRS);
   2723			table->mc_reg_address[j].s1 = MC_PMG_CMD_EMRS >> 2;
   2724			table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
   2725			for (k = 0; k < table->num_entries; k++)
   2726				table->mc_reg_table_entry[k].mc_data[j] =
   2727					((temp_reg & 0xffff0000)) |
   2728					((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
   2729			j++;
   2730			if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
   2731				return -EINVAL;
   2732
   2733			temp_reg = RREG32(MC_PMG_CMD_MRS);
   2734			table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS >> 2;
   2735			table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS_LP >> 2;
   2736			for(k = 0; k < table->num_entries; k++) {
   2737				table->mc_reg_table_entry[k].mc_data[j] =
   2738					(temp_reg & 0xffff0000) |
   2739					(table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
   2740				if (!pi->mem_gddr5)
   2741					table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
   2742			}
   2743			j++;
   2744			if (j > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
   2745				return -EINVAL;
   2746			break;
   2747		case MC_SEQ_RESERVE_M >> 2:
   2748			temp_reg = RREG32(MC_PMG_CMD_MRS1);
   2749			table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS1 >> 2;
   2750			table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
   2751			for (k = 0; k < table->num_entries; k++)
   2752				table->mc_reg_table_entry[k].mc_data[j] =
   2753					(temp_reg & 0xffff0000) |
   2754					(table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
   2755			j++;
   2756			if (j > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
   2757				return -EINVAL;
   2758			break;
   2759		default:
   2760			break;
   2761		}
   2762	}
   2763
   2764	table->last = j;
   2765
   2766	return 0;
   2767}
   2768
   2769static bool ni_check_s0_mc_reg_index(u16 in_reg, u16 *out_reg)
   2770{
   2771	bool result = true;
   2772
   2773	switch (in_reg) {
   2774	case  MC_SEQ_RAS_TIMING >> 2:
   2775		*out_reg = MC_SEQ_RAS_TIMING_LP >> 2;
   2776		break;
   2777	case MC_SEQ_CAS_TIMING >> 2:
   2778		*out_reg = MC_SEQ_CAS_TIMING_LP >> 2;
   2779		break;
   2780	case MC_SEQ_MISC_TIMING >> 2:
   2781		*out_reg = MC_SEQ_MISC_TIMING_LP >> 2;
   2782		break;
   2783	case MC_SEQ_MISC_TIMING2 >> 2:
   2784		*out_reg = MC_SEQ_MISC_TIMING2_LP >> 2;
   2785		break;
   2786	case MC_SEQ_RD_CTL_D0 >> 2:
   2787		*out_reg = MC_SEQ_RD_CTL_D0_LP >> 2;
   2788		break;
   2789	case MC_SEQ_RD_CTL_D1 >> 2:
   2790		*out_reg = MC_SEQ_RD_CTL_D1_LP >> 2;
   2791		break;
   2792	case MC_SEQ_WR_CTL_D0 >> 2:
   2793		*out_reg = MC_SEQ_WR_CTL_D0_LP >> 2;
   2794		break;
   2795	case MC_SEQ_WR_CTL_D1 >> 2:
   2796		*out_reg = MC_SEQ_WR_CTL_D1_LP >> 2;
   2797		break;
   2798	case MC_PMG_CMD_EMRS >> 2:
   2799		*out_reg = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
   2800		break;
   2801	case MC_PMG_CMD_MRS >> 2:
   2802		*out_reg = MC_SEQ_PMG_CMD_MRS_LP >> 2;
   2803		break;
   2804	case MC_PMG_CMD_MRS1 >> 2:
   2805		*out_reg = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
   2806		break;
   2807	case MC_SEQ_PMG_TIMING >> 2:
   2808		*out_reg = MC_SEQ_PMG_TIMING_LP >> 2;
   2809		break;
   2810	case MC_PMG_CMD_MRS2 >> 2:
   2811		*out_reg = MC_SEQ_PMG_CMD_MRS2_LP >> 2;
   2812		break;
   2813	default:
   2814		result = false;
   2815		break;
   2816	}
   2817
   2818	return result;
   2819}
   2820
   2821static void ni_set_valid_flag(struct ni_mc_reg_table *table)
   2822{
   2823	u8 i, j;
   2824
   2825	for (i = 0; i < table->last; i++) {
   2826		for (j = 1; j < table->num_entries; j++) {
   2827			if (table->mc_reg_table_entry[j-1].mc_data[i] != table->mc_reg_table_entry[j].mc_data[i]) {
   2828				table->valid_flag |= 1 << i;
   2829				break;
   2830			}
   2831		}
   2832	}
   2833}
   2834
   2835static void ni_set_s0_mc_reg_index(struct ni_mc_reg_table *table)
   2836{
   2837	u32 i;
   2838	u16 address;
   2839
   2840	for (i = 0; i < table->last; i++)
   2841		table->mc_reg_address[i].s0 =
   2842			ni_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address) ?
   2843			address : table->mc_reg_address[i].s1;
   2844}
   2845
   2846static int ni_copy_vbios_mc_reg_table(struct atom_mc_reg_table *table,
   2847				      struct ni_mc_reg_table *ni_table)
   2848{
   2849	u8 i, j;
   2850
   2851	if (table->last > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
   2852		return -EINVAL;
   2853	if (table->num_entries > MAX_AC_TIMING_ENTRIES)
   2854		return -EINVAL;
   2855
   2856	for (i = 0; i < table->last; i++)
   2857		ni_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1;
   2858	ni_table->last = table->last;
   2859
   2860	for (i = 0; i < table->num_entries; i++) {
   2861		ni_table->mc_reg_table_entry[i].mclk_max =
   2862			table->mc_reg_table_entry[i].mclk_max;
   2863		for (j = 0; j < table->last; j++)
   2864			ni_table->mc_reg_table_entry[i].mc_data[j] =
   2865				table->mc_reg_table_entry[i].mc_data[j];
   2866	}
   2867	ni_table->num_entries = table->num_entries;
   2868
   2869	return 0;
   2870}
   2871
   2872static int ni_initialize_mc_reg_table(struct radeon_device *rdev)
   2873{
   2874	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   2875	int ret;
   2876	struct atom_mc_reg_table *table;
   2877	struct ni_mc_reg_table *ni_table = &ni_pi->mc_reg_table;
   2878	u8 module_index = rv770_get_memory_module_index(rdev);
   2879
   2880	table = kzalloc(sizeof(struct atom_mc_reg_table), GFP_KERNEL);
   2881	if (!table)
   2882		return -ENOMEM;
   2883
   2884	WREG32(MC_SEQ_RAS_TIMING_LP, RREG32(MC_SEQ_RAS_TIMING));
   2885	WREG32(MC_SEQ_CAS_TIMING_LP, RREG32(MC_SEQ_CAS_TIMING));
   2886	WREG32(MC_SEQ_MISC_TIMING_LP, RREG32(MC_SEQ_MISC_TIMING));
   2887	WREG32(MC_SEQ_MISC_TIMING2_LP, RREG32(MC_SEQ_MISC_TIMING2));
   2888	WREG32(MC_SEQ_PMG_CMD_EMRS_LP, RREG32(MC_PMG_CMD_EMRS));
   2889	WREG32(MC_SEQ_PMG_CMD_MRS_LP, RREG32(MC_PMG_CMD_MRS));
   2890	WREG32(MC_SEQ_PMG_CMD_MRS1_LP, RREG32(MC_PMG_CMD_MRS1));
   2891	WREG32(MC_SEQ_WR_CTL_D0_LP, RREG32(MC_SEQ_WR_CTL_D0));
   2892	WREG32(MC_SEQ_WR_CTL_D1_LP, RREG32(MC_SEQ_WR_CTL_D1));
   2893	WREG32(MC_SEQ_RD_CTL_D0_LP, RREG32(MC_SEQ_RD_CTL_D0));
   2894	WREG32(MC_SEQ_RD_CTL_D1_LP, RREG32(MC_SEQ_RD_CTL_D1));
   2895	WREG32(MC_SEQ_PMG_TIMING_LP, RREG32(MC_SEQ_PMG_TIMING));
   2896	WREG32(MC_SEQ_PMG_CMD_MRS2_LP, RREG32(MC_PMG_CMD_MRS2));
   2897
   2898	ret = radeon_atom_init_mc_reg_table(rdev, module_index, table);
   2899
   2900	if (ret)
   2901		goto init_mc_done;
   2902
   2903	ret = ni_copy_vbios_mc_reg_table(table, ni_table);
   2904
   2905	if (ret)
   2906		goto init_mc_done;
   2907
   2908	ni_set_s0_mc_reg_index(ni_table);
   2909
   2910	ret = ni_set_mc_special_registers(rdev, ni_table);
   2911
   2912	if (ret)
   2913		goto init_mc_done;
   2914
   2915	ni_set_valid_flag(ni_table);
   2916
   2917init_mc_done:
   2918	kfree(table);
   2919
   2920	return ret;
   2921}
   2922
   2923static void ni_populate_mc_reg_addresses(struct radeon_device *rdev,
   2924					 SMC_NIslands_MCRegisters *mc_reg_table)
   2925{
   2926	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   2927	u32 i, j;
   2928
   2929	for (i = 0, j = 0; j < ni_pi->mc_reg_table.last; j++) {
   2930		if (ni_pi->mc_reg_table.valid_flag & (1 << j)) {
   2931			if (i >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
   2932				break;
   2933			mc_reg_table->address[i].s0 =
   2934				cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s0);
   2935			mc_reg_table->address[i].s1 =
   2936				cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s1);
   2937			i++;
   2938		}
   2939	}
   2940	mc_reg_table->last = (u8)i;
   2941}
   2942
   2943
   2944static void ni_convert_mc_registers(struct ni_mc_reg_entry *entry,
   2945				    SMC_NIslands_MCRegisterSet *data,
   2946				    u32 num_entries, u32 valid_flag)
   2947{
   2948	u32 i, j;
   2949
   2950	for (i = 0, j = 0; j < num_entries; j++) {
   2951		if (valid_flag & (1 << j)) {
   2952			data->value[i] = cpu_to_be32(entry->mc_data[j]);
   2953			i++;
   2954		}
   2955	}
   2956}
   2957
   2958static void ni_convert_mc_reg_table_entry_to_smc(struct radeon_device *rdev,
   2959						 struct rv7xx_pl *pl,
   2960						 SMC_NIslands_MCRegisterSet *mc_reg_table_data)
   2961{
   2962	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   2963	u32 i = 0;
   2964
   2965	for (i = 0; i < ni_pi->mc_reg_table.num_entries; i++) {
   2966		if (pl->mclk <= ni_pi->mc_reg_table.mc_reg_table_entry[i].mclk_max)
   2967			break;
   2968	}
   2969
   2970	if ((i == ni_pi->mc_reg_table.num_entries) && (i > 0))
   2971		--i;
   2972
   2973	ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[i],
   2974				mc_reg_table_data,
   2975				ni_pi->mc_reg_table.last,
   2976				ni_pi->mc_reg_table.valid_flag);
   2977}
   2978
   2979static void ni_convert_mc_reg_table_to_smc(struct radeon_device *rdev,
   2980					   struct radeon_ps *radeon_state,
   2981					   SMC_NIslands_MCRegisters *mc_reg_table)
   2982{
   2983	struct ni_ps *state = ni_get_ps(radeon_state);
   2984	int i;
   2985
   2986	for (i = 0; i < state->performance_level_count; i++) {
   2987		ni_convert_mc_reg_table_entry_to_smc(rdev,
   2988						     &state->performance_levels[i],
   2989						     &mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i]);
   2990	}
   2991}
   2992
   2993static int ni_populate_mc_reg_table(struct radeon_device *rdev,
   2994				    struct radeon_ps *radeon_boot_state)
   2995{
   2996	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   2997	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   2998	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   2999	struct ni_ps *boot_state = ni_get_ps(radeon_boot_state);
   3000	SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
   3001
   3002	memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
   3003
   3004	rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_seq_index, 1);
   3005
   3006	ni_populate_mc_reg_addresses(rdev, mc_reg_table);
   3007
   3008	ni_convert_mc_reg_table_entry_to_smc(rdev, &boot_state->performance_levels[0],
   3009					     &mc_reg_table->data[0]);
   3010
   3011	ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[0],
   3012				&mc_reg_table->data[1],
   3013				ni_pi->mc_reg_table.last,
   3014				ni_pi->mc_reg_table.valid_flag);
   3015
   3016	ni_convert_mc_reg_table_to_smc(rdev, radeon_boot_state, mc_reg_table);
   3017
   3018	return rv770_copy_bytes_to_smc(rdev, eg_pi->mc_reg_table_start,
   3019				       (u8 *)mc_reg_table,
   3020				       sizeof(SMC_NIslands_MCRegisters),
   3021				       pi->sram_end);
   3022}
   3023
   3024static int ni_upload_mc_reg_table(struct radeon_device *rdev,
   3025				  struct radeon_ps *radeon_new_state)
   3026{
   3027	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   3028	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   3029	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   3030	struct ni_ps *ni_new_state = ni_get_ps(radeon_new_state);
   3031	SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
   3032	u16 address;
   3033
   3034	memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
   3035
   3036	ni_convert_mc_reg_table_to_smc(rdev, radeon_new_state, mc_reg_table);
   3037
   3038	address = eg_pi->mc_reg_table_start +
   3039		(u16)offsetof(SMC_NIslands_MCRegisters, data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT]);
   3040
   3041	return rv770_copy_bytes_to_smc(rdev, address,
   3042				       (u8 *)&mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT],
   3043				       sizeof(SMC_NIslands_MCRegisterSet) * ni_new_state->performance_level_count,
   3044				       pi->sram_end);
   3045}
   3046
   3047static int ni_init_driver_calculated_leakage_table(struct radeon_device *rdev,
   3048						   PP_NIslands_CACTABLES *cac_tables)
   3049{
   3050	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   3051	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   3052	u32 leakage = 0;
   3053	unsigned int i, j, table_size;
   3054	s32 t;
   3055	u32 smc_leakage, max_leakage = 0;
   3056	u32 scaling_factor;
   3057
   3058	table_size = eg_pi->vddc_voltage_table.count;
   3059
   3060	if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
   3061		table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
   3062
   3063	scaling_factor = ni_get_smc_power_scaling_factor(rdev);
   3064
   3065	for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++) {
   3066		for (j = 0; j < table_size; j++) {
   3067			t = (1000 * ((i + 1) * 8));
   3068
   3069			if (t < ni_pi->cac_data.leakage_minimum_temperature)
   3070				t = ni_pi->cac_data.leakage_minimum_temperature;
   3071
   3072			ni_calculate_leakage_for_v_and_t(rdev,
   3073							 &ni_pi->cac_data.leakage_coefficients,
   3074							 eg_pi->vddc_voltage_table.entries[j].value,
   3075							 t,
   3076							 ni_pi->cac_data.i_leakage,
   3077							 &leakage);
   3078
   3079			smc_leakage = ni_scale_power_for_smc(leakage, scaling_factor) / 1000;
   3080			if (smc_leakage > max_leakage)
   3081				max_leakage = smc_leakage;
   3082
   3083			cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(smc_leakage);
   3084		}
   3085	}
   3086
   3087	for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
   3088		for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
   3089			cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(max_leakage);
   3090	}
   3091	return 0;
   3092}
   3093
   3094static int ni_init_simplified_leakage_table(struct radeon_device *rdev,
   3095					    PP_NIslands_CACTABLES *cac_tables)
   3096{
   3097	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   3098	struct radeon_cac_leakage_table *leakage_table =
   3099		&rdev->pm.dpm.dyn_state.cac_leakage_table;
   3100	u32 i, j, table_size;
   3101	u32 smc_leakage, max_leakage = 0;
   3102	u32 scaling_factor;
   3103
   3104	if (!leakage_table)
   3105		return -EINVAL;
   3106
   3107	table_size = leakage_table->count;
   3108
   3109	if (eg_pi->vddc_voltage_table.count != table_size)
   3110		table_size = (eg_pi->vddc_voltage_table.count < leakage_table->count) ?
   3111			eg_pi->vddc_voltage_table.count : leakage_table->count;
   3112
   3113	if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
   3114		table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
   3115
   3116	if (table_size == 0)
   3117		return -EINVAL;
   3118
   3119	scaling_factor = ni_get_smc_power_scaling_factor(rdev);
   3120
   3121	for (j = 0; j < table_size; j++) {
   3122		smc_leakage = leakage_table->entries[j].leakage;
   3123
   3124		if (smc_leakage > max_leakage)
   3125			max_leakage = smc_leakage;
   3126
   3127		for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
   3128			cac_tables->cac_lkge_lut[i][j] =
   3129				cpu_to_be32(ni_scale_power_for_smc(smc_leakage, scaling_factor));
   3130	}
   3131
   3132	for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
   3133		for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
   3134			cac_tables->cac_lkge_lut[i][j] =
   3135				cpu_to_be32(ni_scale_power_for_smc(max_leakage, scaling_factor));
   3136	}
   3137	return 0;
   3138}
   3139
   3140static int ni_initialize_smc_cac_tables(struct radeon_device *rdev)
   3141{
   3142	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   3143	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   3144	PP_NIslands_CACTABLES *cac_tables = NULL;
   3145	int i, ret;
   3146	u32 reg;
   3147
   3148	if (ni_pi->enable_cac == false)
   3149		return 0;
   3150
   3151	cac_tables = kzalloc(sizeof(PP_NIslands_CACTABLES), GFP_KERNEL);
   3152	if (!cac_tables)
   3153		return -ENOMEM;
   3154
   3155	reg = RREG32(CG_CAC_CTRL) & ~(TID_CNT_MASK | TID_UNIT_MASK);
   3156	reg |= (TID_CNT(ni_pi->cac_weights->tid_cnt) |
   3157		TID_UNIT(ni_pi->cac_weights->tid_unit));
   3158	WREG32(CG_CAC_CTRL, reg);
   3159
   3160	for (i = 0; i < NISLANDS_DCCAC_MAX_LEVELS; i++)
   3161		ni_pi->dc_cac_table[i] = ni_pi->cac_weights->dc_cac[i];
   3162
   3163	for (i = 0; i < SMC_NISLANDS_BIF_LUT_NUM_OF_ENTRIES; i++)
   3164		cac_tables->cac_bif_lut[i] = ni_pi->cac_weights->pcie_cac[i];
   3165
   3166	ni_pi->cac_data.i_leakage = rdev->pm.dpm.cac_leakage;
   3167	ni_pi->cac_data.pwr_const = 0;
   3168	ni_pi->cac_data.dc_cac_value = ni_pi->dc_cac_table[NISLANDS_DCCAC_LEVEL_0];
   3169	ni_pi->cac_data.bif_cac_value = 0;
   3170	ni_pi->cac_data.mc_wr_weight = ni_pi->cac_weights->mc_write_weight;
   3171	ni_pi->cac_data.mc_rd_weight = ni_pi->cac_weights->mc_read_weight;
   3172	ni_pi->cac_data.allow_ovrflw = 0;
   3173	ni_pi->cac_data.l2num_win_tdp = ni_pi->lta_window_size;
   3174	ni_pi->cac_data.num_win_tdp = 0;
   3175	ni_pi->cac_data.lts_truncate_n = ni_pi->lts_truncate;
   3176
   3177	if (ni_pi->driver_calculate_cac_leakage)
   3178		ret = ni_init_driver_calculated_leakage_table(rdev, cac_tables);
   3179	else
   3180		ret = ni_init_simplified_leakage_table(rdev, cac_tables);
   3181
   3182	if (ret)
   3183		goto done_free;
   3184
   3185	cac_tables->pwr_const      = cpu_to_be32(ni_pi->cac_data.pwr_const);
   3186	cac_tables->dc_cacValue    = cpu_to_be32(ni_pi->cac_data.dc_cac_value);
   3187	cac_tables->bif_cacValue   = cpu_to_be32(ni_pi->cac_data.bif_cac_value);
   3188	cac_tables->AllowOvrflw    = ni_pi->cac_data.allow_ovrflw;
   3189	cac_tables->MCWrWeight     = ni_pi->cac_data.mc_wr_weight;
   3190	cac_tables->MCRdWeight     = ni_pi->cac_data.mc_rd_weight;
   3191	cac_tables->numWin_TDP     = ni_pi->cac_data.num_win_tdp;
   3192	cac_tables->l2numWin_TDP   = ni_pi->cac_data.l2num_win_tdp;
   3193	cac_tables->lts_truncate_n = ni_pi->cac_data.lts_truncate_n;
   3194
   3195	ret = rv770_copy_bytes_to_smc(rdev, ni_pi->cac_table_start, (u8 *)cac_tables,
   3196				      sizeof(PP_NIslands_CACTABLES), pi->sram_end);
   3197
   3198done_free:
   3199	if (ret) {
   3200		ni_pi->enable_cac = false;
   3201		ni_pi->enable_power_containment = false;
   3202	}
   3203
   3204	kfree(cac_tables);
   3205
   3206	return 0;
   3207}
   3208
   3209static int ni_initialize_hardware_cac_manager(struct radeon_device *rdev)
   3210{
   3211	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   3212	u32 reg;
   3213
   3214	if (!ni_pi->enable_cac ||
   3215	    !ni_pi->cac_configuration_required)
   3216		return 0;
   3217
   3218	if (ni_pi->cac_weights == NULL)
   3219		return -EINVAL;
   3220
   3221	reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_0) & ~(WEIGHT_TCP_SIG0_MASK |
   3222						      WEIGHT_TCP_SIG1_MASK |
   3223						      WEIGHT_TA_SIG_MASK);
   3224	reg |= (WEIGHT_TCP_SIG0(ni_pi->cac_weights->weight_tcp_sig0) |
   3225		WEIGHT_TCP_SIG1(ni_pi->cac_weights->weight_tcp_sig1) |
   3226		WEIGHT_TA_SIG(ni_pi->cac_weights->weight_ta_sig));
   3227	WREG32_CG(CG_CAC_REGION_1_WEIGHT_0, reg);
   3228
   3229	reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_1) & ~(WEIGHT_TCC_EN0_MASK |
   3230						      WEIGHT_TCC_EN1_MASK |
   3231						      WEIGHT_TCC_EN2_MASK);
   3232	reg |= (WEIGHT_TCC_EN0(ni_pi->cac_weights->weight_tcc_en0) |
   3233		WEIGHT_TCC_EN1(ni_pi->cac_weights->weight_tcc_en1) |
   3234		WEIGHT_TCC_EN2(ni_pi->cac_weights->weight_tcc_en2));
   3235	WREG32_CG(CG_CAC_REGION_1_WEIGHT_1, reg);
   3236
   3237	reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_0) & ~(WEIGHT_CB_EN0_MASK |
   3238						      WEIGHT_CB_EN1_MASK |
   3239						      WEIGHT_CB_EN2_MASK |
   3240						      WEIGHT_CB_EN3_MASK);
   3241	reg |= (WEIGHT_CB_EN0(ni_pi->cac_weights->weight_cb_en0) |
   3242		WEIGHT_CB_EN1(ni_pi->cac_weights->weight_cb_en1) |
   3243		WEIGHT_CB_EN2(ni_pi->cac_weights->weight_cb_en2) |
   3244		WEIGHT_CB_EN3(ni_pi->cac_weights->weight_cb_en3));
   3245	WREG32_CG(CG_CAC_REGION_2_WEIGHT_0, reg);
   3246
   3247	reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_1) & ~(WEIGHT_DB_SIG0_MASK |
   3248						      WEIGHT_DB_SIG1_MASK |
   3249						      WEIGHT_DB_SIG2_MASK |
   3250						      WEIGHT_DB_SIG3_MASK);
   3251	reg |= (WEIGHT_DB_SIG0(ni_pi->cac_weights->weight_db_sig0) |
   3252		WEIGHT_DB_SIG1(ni_pi->cac_weights->weight_db_sig1) |
   3253		WEIGHT_DB_SIG2(ni_pi->cac_weights->weight_db_sig2) |
   3254		WEIGHT_DB_SIG3(ni_pi->cac_weights->weight_db_sig3));
   3255	WREG32_CG(CG_CAC_REGION_2_WEIGHT_1, reg);
   3256
   3257	reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_2) & ~(WEIGHT_SXM_SIG0_MASK |
   3258						      WEIGHT_SXM_SIG1_MASK |
   3259						      WEIGHT_SXM_SIG2_MASK |
   3260						      WEIGHT_SXS_SIG0_MASK |
   3261						      WEIGHT_SXS_SIG1_MASK);
   3262	reg |= (WEIGHT_SXM_SIG0(ni_pi->cac_weights->weight_sxm_sig0) |
   3263		WEIGHT_SXM_SIG1(ni_pi->cac_weights->weight_sxm_sig1) |
   3264		WEIGHT_SXM_SIG2(ni_pi->cac_weights->weight_sxm_sig2) |
   3265		WEIGHT_SXS_SIG0(ni_pi->cac_weights->weight_sxs_sig0) |
   3266		WEIGHT_SXS_SIG1(ni_pi->cac_weights->weight_sxs_sig1));
   3267	WREG32_CG(CG_CAC_REGION_2_WEIGHT_2, reg);
   3268
   3269	reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_0) & ~(WEIGHT_XBR_0_MASK |
   3270						      WEIGHT_XBR_1_MASK |
   3271						      WEIGHT_XBR_2_MASK |
   3272						      WEIGHT_SPI_SIG0_MASK);
   3273	reg |= (WEIGHT_XBR_0(ni_pi->cac_weights->weight_xbr_0) |
   3274		WEIGHT_XBR_1(ni_pi->cac_weights->weight_xbr_1) |
   3275		WEIGHT_XBR_2(ni_pi->cac_weights->weight_xbr_2) |
   3276		WEIGHT_SPI_SIG0(ni_pi->cac_weights->weight_spi_sig0));
   3277	WREG32_CG(CG_CAC_REGION_3_WEIGHT_0, reg);
   3278
   3279	reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_1) & ~(WEIGHT_SPI_SIG1_MASK |
   3280						      WEIGHT_SPI_SIG2_MASK |
   3281						      WEIGHT_SPI_SIG3_MASK |
   3282						      WEIGHT_SPI_SIG4_MASK |
   3283						      WEIGHT_SPI_SIG5_MASK);
   3284	reg |= (WEIGHT_SPI_SIG1(ni_pi->cac_weights->weight_spi_sig1) |
   3285		WEIGHT_SPI_SIG2(ni_pi->cac_weights->weight_spi_sig2) |
   3286		WEIGHT_SPI_SIG3(ni_pi->cac_weights->weight_spi_sig3) |
   3287		WEIGHT_SPI_SIG4(ni_pi->cac_weights->weight_spi_sig4) |
   3288		WEIGHT_SPI_SIG5(ni_pi->cac_weights->weight_spi_sig5));
   3289	WREG32_CG(CG_CAC_REGION_3_WEIGHT_1, reg);
   3290
   3291	reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_0) & ~(WEIGHT_LDS_SIG0_MASK |
   3292						      WEIGHT_LDS_SIG1_MASK |
   3293						      WEIGHT_SC_MASK);
   3294	reg |= (WEIGHT_LDS_SIG0(ni_pi->cac_weights->weight_lds_sig0) |
   3295		WEIGHT_LDS_SIG1(ni_pi->cac_weights->weight_lds_sig1) |
   3296		WEIGHT_SC(ni_pi->cac_weights->weight_sc));
   3297	WREG32_CG(CG_CAC_REGION_4_WEIGHT_0, reg);
   3298
   3299	reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_1) & ~(WEIGHT_BIF_MASK |
   3300						      WEIGHT_CP_MASK |
   3301						      WEIGHT_PA_SIG0_MASK |
   3302						      WEIGHT_PA_SIG1_MASK |
   3303						      WEIGHT_VGT_SIG0_MASK);
   3304	reg |= (WEIGHT_BIF(ni_pi->cac_weights->weight_bif) |
   3305		WEIGHT_CP(ni_pi->cac_weights->weight_cp) |
   3306		WEIGHT_PA_SIG0(ni_pi->cac_weights->weight_pa_sig0) |
   3307		WEIGHT_PA_SIG1(ni_pi->cac_weights->weight_pa_sig1) |
   3308		WEIGHT_VGT_SIG0(ni_pi->cac_weights->weight_vgt_sig0));
   3309	WREG32_CG(CG_CAC_REGION_4_WEIGHT_1, reg);
   3310
   3311	reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_2) & ~(WEIGHT_VGT_SIG1_MASK |
   3312						      WEIGHT_VGT_SIG2_MASK |
   3313						      WEIGHT_DC_SIG0_MASK |
   3314						      WEIGHT_DC_SIG1_MASK |
   3315						      WEIGHT_DC_SIG2_MASK);
   3316	reg |= (WEIGHT_VGT_SIG1(ni_pi->cac_weights->weight_vgt_sig1) |
   3317		WEIGHT_VGT_SIG2(ni_pi->cac_weights->weight_vgt_sig2) |
   3318		WEIGHT_DC_SIG0(ni_pi->cac_weights->weight_dc_sig0) |
   3319		WEIGHT_DC_SIG1(ni_pi->cac_weights->weight_dc_sig1) |
   3320		WEIGHT_DC_SIG2(ni_pi->cac_weights->weight_dc_sig2));
   3321	WREG32_CG(CG_CAC_REGION_4_WEIGHT_2, reg);
   3322
   3323	reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_3) & ~(WEIGHT_DC_SIG3_MASK |
   3324						      WEIGHT_UVD_SIG0_MASK |
   3325						      WEIGHT_UVD_SIG1_MASK |
   3326						      WEIGHT_SPARE0_MASK |
   3327						      WEIGHT_SPARE1_MASK);
   3328	reg |= (WEIGHT_DC_SIG3(ni_pi->cac_weights->weight_dc_sig3) |
   3329		WEIGHT_UVD_SIG0(ni_pi->cac_weights->weight_uvd_sig0) |
   3330		WEIGHT_UVD_SIG1(ni_pi->cac_weights->weight_uvd_sig1) |
   3331		WEIGHT_SPARE0(ni_pi->cac_weights->weight_spare0) |
   3332		WEIGHT_SPARE1(ni_pi->cac_weights->weight_spare1));
   3333	WREG32_CG(CG_CAC_REGION_4_WEIGHT_3, reg);
   3334
   3335	reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_0) & ~(WEIGHT_SQ_VSP_MASK |
   3336						      WEIGHT_SQ_VSP0_MASK);
   3337	reg |= (WEIGHT_SQ_VSP(ni_pi->cac_weights->weight_sq_vsp) |
   3338		WEIGHT_SQ_VSP0(ni_pi->cac_weights->weight_sq_vsp0));
   3339	WREG32_CG(CG_CAC_REGION_5_WEIGHT_0, reg);
   3340
   3341	reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_1) & ~(WEIGHT_SQ_GPR_MASK);
   3342	reg |= WEIGHT_SQ_GPR(ni_pi->cac_weights->weight_sq_gpr);
   3343	WREG32_CG(CG_CAC_REGION_5_WEIGHT_1, reg);
   3344
   3345	reg = RREG32_CG(CG_CAC_REGION_4_OVERRIDE_4) & ~(OVR_MODE_SPARE_0_MASK |
   3346							OVR_VAL_SPARE_0_MASK |
   3347							OVR_MODE_SPARE_1_MASK |
   3348							OVR_VAL_SPARE_1_MASK);
   3349	reg |= (OVR_MODE_SPARE_0(ni_pi->cac_weights->ovr_mode_spare_0) |
   3350		OVR_VAL_SPARE_0(ni_pi->cac_weights->ovr_val_spare_0) |
   3351		OVR_MODE_SPARE_1(ni_pi->cac_weights->ovr_mode_spare_1) |
   3352		OVR_VAL_SPARE_1(ni_pi->cac_weights->ovr_val_spare_1));
   3353	WREG32_CG(CG_CAC_REGION_4_OVERRIDE_4, reg);
   3354
   3355	reg = RREG32(SQ_CAC_THRESHOLD) & ~(VSP_MASK |
   3356					   VSP0_MASK |
   3357					   GPR_MASK);
   3358	reg |= (VSP(ni_pi->cac_weights->vsp) |
   3359		VSP0(ni_pi->cac_weights->vsp0) |
   3360		GPR(ni_pi->cac_weights->gpr));
   3361	WREG32(SQ_CAC_THRESHOLD, reg);
   3362
   3363	reg = (MCDW_WR_ENABLE |
   3364	       MCDX_WR_ENABLE |
   3365	       MCDY_WR_ENABLE |
   3366	       MCDZ_WR_ENABLE |
   3367	       INDEX(0x09D4));
   3368	WREG32(MC_CG_CONFIG, reg);
   3369
   3370	reg = (READ_WEIGHT(ni_pi->cac_weights->mc_read_weight) |
   3371	       WRITE_WEIGHT(ni_pi->cac_weights->mc_write_weight) |
   3372	       ALLOW_OVERFLOW);
   3373	WREG32(MC_CG_DATAPORT, reg);
   3374
   3375	return 0;
   3376}
   3377
   3378static int ni_enable_smc_cac(struct radeon_device *rdev,
   3379			     struct radeon_ps *radeon_new_state,
   3380			     bool enable)
   3381{
   3382	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   3383	int ret = 0;
   3384	PPSMC_Result smc_result;
   3385
   3386	if (ni_pi->enable_cac) {
   3387		if (enable) {
   3388			if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
   3389				smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_CollectCAC_PowerCorreln);
   3390
   3391				if (ni_pi->support_cac_long_term_average) {
   3392					smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgEnable);
   3393					if (PPSMC_Result_OK != smc_result)
   3394						ni_pi->support_cac_long_term_average = false;
   3395				}
   3396
   3397				smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableCac);
   3398				if (PPSMC_Result_OK != smc_result)
   3399					ret = -EINVAL;
   3400
   3401				ni_pi->cac_enabled = (PPSMC_Result_OK == smc_result) ? true : false;
   3402			}
   3403		} else if (ni_pi->cac_enabled) {
   3404			smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_DisableCac);
   3405
   3406			ni_pi->cac_enabled = false;
   3407
   3408			if (ni_pi->support_cac_long_term_average) {
   3409				smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgDisable);
   3410				if (PPSMC_Result_OK != smc_result)
   3411					ni_pi->support_cac_long_term_average = false;
   3412			}
   3413		}
   3414	}
   3415
   3416	return ret;
   3417}
   3418
   3419static int ni_pcie_performance_request(struct radeon_device *rdev,
   3420				       u8 perf_req, bool advertise)
   3421{
   3422#if defined(CONFIG_ACPI)
   3423	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   3424
   3425	if ((perf_req == PCIE_PERF_REQ_PECI_GEN1) ||
   3426	    (perf_req == PCIE_PERF_REQ_PECI_GEN2)) {
   3427		if (eg_pi->pcie_performance_request_registered == false)
   3428			radeon_acpi_pcie_notify_device_ready(rdev);
   3429		eg_pi->pcie_performance_request_registered = true;
   3430		return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
   3431	} else if ((perf_req == PCIE_PERF_REQ_REMOVE_REGISTRY) &&
   3432		    eg_pi->pcie_performance_request_registered) {
   3433		eg_pi->pcie_performance_request_registered = false;
   3434		return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
   3435	}
   3436#endif
   3437	return 0;
   3438}
   3439
   3440static int ni_advertise_gen2_capability(struct radeon_device *rdev)
   3441{
   3442	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   3443	u32 tmp;
   3444
   3445	tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
   3446
   3447	if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
   3448	    (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2))
   3449		pi->pcie_gen2 = true;
   3450	else
   3451		pi->pcie_gen2 = false;
   3452
   3453	if (!pi->pcie_gen2)
   3454		ni_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN2, true);
   3455
   3456	return 0;
   3457}
   3458
   3459static void ni_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev,
   3460					    bool enable)
   3461{
   3462	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   3463	u32 tmp, bif;
   3464
   3465	tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
   3466
   3467	if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
   3468	    (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
   3469		if (enable) {
   3470			if (!pi->boot_in_gen2) {
   3471				bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
   3472				bif |= CG_CLIENT_REQ(0xd);
   3473				WREG32(CG_BIF_REQ_AND_RSP, bif);
   3474			}
   3475			tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
   3476			tmp |= LC_HW_VOLTAGE_IF_CONTROL(1);
   3477			tmp |= LC_GEN2_EN_STRAP;
   3478
   3479			tmp |= LC_CLR_FAILED_SPD_CHANGE_CNT;
   3480			WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
   3481			udelay(10);
   3482			tmp &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
   3483			WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
   3484		} else {
   3485			if (!pi->boot_in_gen2) {
   3486				bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
   3487				bif |= CG_CLIENT_REQ(0xd);
   3488				WREG32(CG_BIF_REQ_AND_RSP, bif);
   3489
   3490				tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
   3491				tmp &= ~LC_GEN2_EN_STRAP;
   3492			}
   3493			WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
   3494		}
   3495	}
   3496}
   3497
   3498static void ni_enable_dynamic_pcie_gen2(struct radeon_device *rdev,
   3499					bool enable)
   3500{
   3501	ni_enable_bif_dynamic_pcie_gen2(rdev, enable);
   3502
   3503	if (enable)
   3504		WREG32_P(GENERAL_PWRMGT, ENABLE_GEN2PCIE, ~ENABLE_GEN2PCIE);
   3505	else
   3506		WREG32_P(GENERAL_PWRMGT, 0, ~ENABLE_GEN2PCIE);
   3507}
   3508
   3509void ni_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
   3510					   struct radeon_ps *new_ps,
   3511					   struct radeon_ps *old_ps)
   3512{
   3513	struct ni_ps *new_state = ni_get_ps(new_ps);
   3514	struct ni_ps *current_state = ni_get_ps(old_ps);
   3515
   3516	if ((new_ps->vclk == old_ps->vclk) &&
   3517	    (new_ps->dclk == old_ps->dclk))
   3518		return;
   3519
   3520	if (new_state->performance_levels[new_state->performance_level_count - 1].sclk >=
   3521	    current_state->performance_levels[current_state->performance_level_count - 1].sclk)
   3522		return;
   3523
   3524	radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
   3525}
   3526
   3527void ni_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
   3528					  struct radeon_ps *new_ps,
   3529					  struct radeon_ps *old_ps)
   3530{
   3531	struct ni_ps *new_state = ni_get_ps(new_ps);
   3532	struct ni_ps *current_state = ni_get_ps(old_ps);
   3533
   3534	if ((new_ps->vclk == old_ps->vclk) &&
   3535	    (new_ps->dclk == old_ps->dclk))
   3536		return;
   3537
   3538	if (new_state->performance_levels[new_state->performance_level_count - 1].sclk <
   3539	    current_state->performance_levels[current_state->performance_level_count - 1].sclk)
   3540		return;
   3541
   3542	radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
   3543}
   3544
   3545void ni_dpm_setup_asic(struct radeon_device *rdev)
   3546{
   3547	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   3548	int r;
   3549
   3550	r = ni_mc_load_microcode(rdev);
   3551	if (r)
   3552		DRM_ERROR("Failed to load MC firmware!\n");
   3553	ni_read_clock_registers(rdev);
   3554	btc_read_arb_registers(rdev);
   3555	rv770_get_memory_type(rdev);
   3556	if (eg_pi->pcie_performance_request)
   3557		ni_advertise_gen2_capability(rdev);
   3558	rv770_get_pcie_gen2_status(rdev);
   3559	rv770_enable_acpi_pm(rdev);
   3560}
   3561
   3562void ni_update_current_ps(struct radeon_device *rdev,
   3563			  struct radeon_ps *rps)
   3564{
   3565	struct ni_ps *new_ps = ni_get_ps(rps);
   3566	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   3567	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   3568
   3569	eg_pi->current_rps = *rps;
   3570	ni_pi->current_ps = *new_ps;
   3571	eg_pi->current_rps.ps_priv = &ni_pi->current_ps;
   3572}
   3573
   3574void ni_update_requested_ps(struct radeon_device *rdev,
   3575			    struct radeon_ps *rps)
   3576{
   3577	struct ni_ps *new_ps = ni_get_ps(rps);
   3578	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   3579	struct ni_power_info *ni_pi = ni_get_pi(rdev);
   3580
   3581	eg_pi->requested_rps = *rps;
   3582	ni_pi->requested_ps = *new_ps;
   3583	eg_pi->requested_rps.ps_priv = &ni_pi->requested_ps;
   3584}
   3585
   3586int ni_dpm_enable(struct radeon_device *rdev)
   3587{
   3588	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   3589	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   3590	struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
   3591	int ret;
   3592
   3593	if (pi->gfx_clock_gating)
   3594		ni_cg_clockgating_default(rdev);
   3595	if (btc_dpm_enabled(rdev))
   3596		return -EINVAL;
   3597	if (pi->mg_clock_gating)
   3598		ni_mg_clockgating_default(rdev);
   3599	if (eg_pi->ls_clock_gating)
   3600		ni_ls_clockgating_default(rdev);
   3601	if (pi->voltage_control) {
   3602		rv770_enable_voltage_control(rdev, true);
   3603		ret = cypress_construct_voltage_tables(rdev);
   3604		if (ret) {
   3605			DRM_ERROR("cypress_construct_voltage_tables failed\n");
   3606			return ret;
   3607		}
   3608	}
   3609	if (eg_pi->dynamic_ac_timing) {
   3610		ret = ni_initialize_mc_reg_table(rdev);
   3611		if (ret)
   3612			eg_pi->dynamic_ac_timing = false;
   3613	}
   3614	if (pi->dynamic_ss)
   3615		cypress_enable_spread_spectrum(rdev, true);
   3616	if (pi->thermal_protection)
   3617		rv770_enable_thermal_protection(rdev, true);
   3618	rv770_setup_bsp(rdev);
   3619	rv770_program_git(rdev);
   3620	rv770_program_tp(rdev);
   3621	rv770_program_tpp(rdev);
   3622	rv770_program_sstp(rdev);
   3623	cypress_enable_display_gap(rdev);
   3624	rv770_program_vc(rdev);
   3625	if (pi->dynamic_pcie_gen2)
   3626		ni_enable_dynamic_pcie_gen2(rdev, true);
   3627	ret = rv770_upload_firmware(rdev);
   3628	if (ret) {
   3629		DRM_ERROR("rv770_upload_firmware failed\n");
   3630		return ret;
   3631	}
   3632	ret = ni_process_firmware_header(rdev);
   3633	if (ret) {
   3634		DRM_ERROR("ni_process_firmware_header failed\n");
   3635		return ret;
   3636	}
   3637	ret = ni_initial_switch_from_arb_f0_to_f1(rdev);
   3638	if (ret) {
   3639		DRM_ERROR("ni_initial_switch_from_arb_f0_to_f1 failed\n");
   3640		return ret;
   3641	}
   3642	ret = ni_init_smc_table(rdev);
   3643	if (ret) {
   3644		DRM_ERROR("ni_init_smc_table failed\n");
   3645		return ret;
   3646	}
   3647	ret = ni_init_smc_spll_table(rdev);
   3648	if (ret) {
   3649		DRM_ERROR("ni_init_smc_spll_table failed\n");
   3650		return ret;
   3651	}
   3652	ret = ni_init_arb_table_index(rdev);
   3653	if (ret) {
   3654		DRM_ERROR("ni_init_arb_table_index failed\n");
   3655		return ret;
   3656	}
   3657	if (eg_pi->dynamic_ac_timing) {
   3658		ret = ni_populate_mc_reg_table(rdev, boot_ps);
   3659		if (ret) {
   3660			DRM_ERROR("ni_populate_mc_reg_table failed\n");
   3661			return ret;
   3662		}
   3663	}
   3664	ret = ni_initialize_smc_cac_tables(rdev);
   3665	if (ret) {
   3666		DRM_ERROR("ni_initialize_smc_cac_tables failed\n");
   3667		return ret;
   3668	}
   3669	ret = ni_initialize_hardware_cac_manager(rdev);
   3670	if (ret) {
   3671		DRM_ERROR("ni_initialize_hardware_cac_manager failed\n");
   3672		return ret;
   3673	}
   3674	ret = ni_populate_smc_tdp_limits(rdev, boot_ps);
   3675	if (ret) {
   3676		DRM_ERROR("ni_populate_smc_tdp_limits failed\n");
   3677		return ret;
   3678	}
   3679	ni_program_response_times(rdev);
   3680	r7xx_start_smc(rdev);
   3681	ret = cypress_notify_smc_display_change(rdev, false);
   3682	if (ret) {
   3683		DRM_ERROR("cypress_notify_smc_display_change failed\n");
   3684		return ret;
   3685	}
   3686	cypress_enable_sclk_control(rdev, true);
   3687	if (eg_pi->memory_transition)
   3688		cypress_enable_mclk_control(rdev, true);
   3689	cypress_start_dpm(rdev);
   3690	if (pi->gfx_clock_gating)
   3691		ni_gfx_clockgating_enable(rdev, true);
   3692	if (pi->mg_clock_gating)
   3693		ni_mg_clockgating_enable(rdev, true);
   3694	if (eg_pi->ls_clock_gating)
   3695		ni_ls_clockgating_enable(rdev, true);
   3696
   3697	rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
   3698
   3699	ni_update_current_ps(rdev, boot_ps);
   3700
   3701	return 0;
   3702}
   3703
   3704void ni_dpm_disable(struct radeon_device *rdev)
   3705{
   3706	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   3707	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   3708	struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
   3709
   3710	if (!btc_dpm_enabled(rdev))
   3711		return;
   3712	rv770_clear_vc(rdev);
   3713	if (pi->thermal_protection)
   3714		rv770_enable_thermal_protection(rdev, false);
   3715	ni_enable_power_containment(rdev, boot_ps, false);
   3716	ni_enable_smc_cac(rdev, boot_ps, false);
   3717	cypress_enable_spread_spectrum(rdev, false);
   3718	rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, false);
   3719	if (pi->dynamic_pcie_gen2)
   3720		ni_enable_dynamic_pcie_gen2(rdev, false);
   3721
   3722	if (rdev->irq.installed &&
   3723	    r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
   3724		rdev->irq.dpm_thermal = false;
   3725		radeon_irq_set(rdev);
   3726	}
   3727
   3728	if (pi->gfx_clock_gating)
   3729		ni_gfx_clockgating_enable(rdev, false);
   3730	if (pi->mg_clock_gating)
   3731		ni_mg_clockgating_enable(rdev, false);
   3732	if (eg_pi->ls_clock_gating)
   3733		ni_ls_clockgating_enable(rdev, false);
   3734	ni_stop_dpm(rdev);
   3735	btc_reset_to_default(rdev);
   3736	ni_stop_smc(rdev);
   3737	ni_force_switch_to_arb_f0(rdev);
   3738
   3739	ni_update_current_ps(rdev, boot_ps);
   3740}
   3741
   3742static int ni_power_control_set_level(struct radeon_device *rdev)
   3743{
   3744	struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps;
   3745	int ret;
   3746
   3747	ret = ni_restrict_performance_levels_before_switch(rdev);
   3748	if (ret)
   3749		return ret;
   3750	ret = rv770_halt_smc(rdev);
   3751	if (ret)
   3752		return ret;
   3753	ret = ni_populate_smc_tdp_limits(rdev, new_ps);
   3754	if (ret)
   3755		return ret;
   3756	ret = rv770_resume_smc(rdev);
   3757	if (ret)
   3758		return ret;
   3759	ret = rv770_set_sw_state(rdev);
   3760	if (ret)
   3761		return ret;
   3762
   3763	return 0;
   3764}
   3765
   3766int ni_dpm_pre_set_power_state(struct radeon_device *rdev)
   3767{
   3768	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   3769	struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps;
   3770	struct radeon_ps *new_ps = &requested_ps;
   3771
   3772	ni_update_requested_ps(rdev, new_ps);
   3773
   3774	ni_apply_state_adjust_rules(rdev, &eg_pi->requested_rps);
   3775
   3776	return 0;
   3777}
   3778
   3779int ni_dpm_set_power_state(struct radeon_device *rdev)
   3780{
   3781	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   3782	struct radeon_ps *new_ps = &eg_pi->requested_rps;
   3783	struct radeon_ps *old_ps = &eg_pi->current_rps;
   3784	int ret;
   3785
   3786	ret = ni_restrict_performance_levels_before_switch(rdev);
   3787	if (ret) {
   3788		DRM_ERROR("ni_restrict_performance_levels_before_switch failed\n");
   3789		return ret;
   3790	}
   3791	ni_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
   3792	ret = ni_enable_power_containment(rdev, new_ps, false);
   3793	if (ret) {
   3794		DRM_ERROR("ni_enable_power_containment failed\n");
   3795		return ret;
   3796	}
   3797	ret = ni_enable_smc_cac(rdev, new_ps, false);
   3798	if (ret) {
   3799		DRM_ERROR("ni_enable_smc_cac failed\n");
   3800		return ret;
   3801	}
   3802	ret = rv770_halt_smc(rdev);
   3803	if (ret) {
   3804		DRM_ERROR("rv770_halt_smc failed\n");
   3805		return ret;
   3806	}
   3807	if (eg_pi->smu_uvd_hs)
   3808		btc_notify_uvd_to_smc(rdev, new_ps);
   3809	ret = ni_upload_sw_state(rdev, new_ps);
   3810	if (ret) {
   3811		DRM_ERROR("ni_upload_sw_state failed\n");
   3812		return ret;
   3813	}
   3814	if (eg_pi->dynamic_ac_timing) {
   3815		ret = ni_upload_mc_reg_table(rdev, new_ps);
   3816		if (ret) {
   3817			DRM_ERROR("ni_upload_mc_reg_table failed\n");
   3818			return ret;
   3819		}
   3820	}
   3821	ret = ni_program_memory_timing_parameters(rdev, new_ps);
   3822	if (ret) {
   3823		DRM_ERROR("ni_program_memory_timing_parameters failed\n");
   3824		return ret;
   3825	}
   3826	ret = rv770_resume_smc(rdev);
   3827	if (ret) {
   3828		DRM_ERROR("rv770_resume_smc failed\n");
   3829		return ret;
   3830	}
   3831	ret = rv770_set_sw_state(rdev);
   3832	if (ret) {
   3833		DRM_ERROR("rv770_set_sw_state failed\n");
   3834		return ret;
   3835	}
   3836	ni_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
   3837	ret = ni_enable_smc_cac(rdev, new_ps, true);
   3838	if (ret) {
   3839		DRM_ERROR("ni_enable_smc_cac failed\n");
   3840		return ret;
   3841	}
   3842	ret = ni_enable_power_containment(rdev, new_ps, true);
   3843	if (ret) {
   3844		DRM_ERROR("ni_enable_power_containment failed\n");
   3845		return ret;
   3846	}
   3847
   3848	/* update tdp */
   3849	ret = ni_power_control_set_level(rdev);
   3850	if (ret) {
   3851		DRM_ERROR("ni_power_control_set_level failed\n");
   3852		return ret;
   3853	}
   3854
   3855	return 0;
   3856}
   3857
   3858void ni_dpm_post_set_power_state(struct radeon_device *rdev)
   3859{
   3860	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   3861	struct radeon_ps *new_ps = &eg_pi->requested_rps;
   3862
   3863	ni_update_current_ps(rdev, new_ps);
   3864}
   3865
   3866#if 0
   3867void ni_dpm_reset_asic(struct radeon_device *rdev)
   3868{
   3869	ni_restrict_performance_levels_before_switch(rdev);
   3870	rv770_set_boot_state(rdev);
   3871}
   3872#endif
   3873
   3874union power_info {
   3875	struct _ATOM_POWERPLAY_INFO info;
   3876	struct _ATOM_POWERPLAY_INFO_V2 info_2;
   3877	struct _ATOM_POWERPLAY_INFO_V3 info_3;
   3878	struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
   3879	struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
   3880	struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
   3881};
   3882
   3883union pplib_clock_info {
   3884	struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
   3885	struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
   3886	struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
   3887	struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
   3888};
   3889
   3890union pplib_power_state {
   3891	struct _ATOM_PPLIB_STATE v1;
   3892	struct _ATOM_PPLIB_STATE_V2 v2;
   3893};
   3894
   3895static void ni_parse_pplib_non_clock_info(struct radeon_device *rdev,
   3896					  struct radeon_ps *rps,
   3897					  struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
   3898					  u8 table_rev)
   3899{
   3900	rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
   3901	rps->class = le16_to_cpu(non_clock_info->usClassification);
   3902	rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
   3903
   3904	if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
   3905		rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
   3906		rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
   3907	} else if (r600_is_uvd_state(rps->class, rps->class2)) {
   3908		rps->vclk = RV770_DEFAULT_VCLK_FREQ;
   3909		rps->dclk = RV770_DEFAULT_DCLK_FREQ;
   3910	} else {
   3911		rps->vclk = 0;
   3912		rps->dclk = 0;
   3913	}
   3914
   3915	if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
   3916		rdev->pm.dpm.boot_ps = rps;
   3917	if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
   3918		rdev->pm.dpm.uvd_ps = rps;
   3919}
   3920
   3921static void ni_parse_pplib_clock_info(struct radeon_device *rdev,
   3922				      struct radeon_ps *rps, int index,
   3923				      union pplib_clock_info *clock_info)
   3924{
   3925	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
   3926	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   3927	struct ni_ps *ps = ni_get_ps(rps);
   3928	struct rv7xx_pl *pl = &ps->performance_levels[index];
   3929
   3930	ps->performance_level_count = index + 1;
   3931
   3932	pl->sclk = le16_to_cpu(clock_info->evergreen.usEngineClockLow);
   3933	pl->sclk |= clock_info->evergreen.ucEngineClockHigh << 16;
   3934	pl->mclk = le16_to_cpu(clock_info->evergreen.usMemoryClockLow);
   3935	pl->mclk |= clock_info->evergreen.ucMemoryClockHigh << 16;
   3936
   3937	pl->vddc = le16_to_cpu(clock_info->evergreen.usVDDC);
   3938	pl->vddci = le16_to_cpu(clock_info->evergreen.usVDDCI);
   3939	pl->flags = le32_to_cpu(clock_info->evergreen.ulFlags);
   3940
   3941	/* patch up vddc if necessary */
   3942	if (pl->vddc == 0xff01) {
   3943		if (pi->max_vddc)
   3944			pl->vddc = pi->max_vddc;
   3945	}
   3946
   3947	if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) {
   3948		pi->acpi_vddc = pl->vddc;
   3949		eg_pi->acpi_vddci = pl->vddci;
   3950		if (ps->performance_levels[0].flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)
   3951			pi->acpi_pcie_gen2 = true;
   3952		else
   3953			pi->acpi_pcie_gen2 = false;
   3954	}
   3955
   3956	if (rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) {
   3957		eg_pi->ulv.supported = true;
   3958		eg_pi->ulv.pl = pl;
   3959	}
   3960
   3961	if (pi->min_vddc_in_table > pl->vddc)
   3962		pi->min_vddc_in_table = pl->vddc;
   3963
   3964	if (pi->max_vddc_in_table < pl->vddc)
   3965		pi->max_vddc_in_table = pl->vddc;
   3966
   3967	/* patch up boot state */
   3968	if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
   3969		u16 vddc, vddci, mvdd;
   3970		radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd);
   3971		pl->mclk = rdev->clock.default_mclk;
   3972		pl->sclk = rdev->clock.default_sclk;
   3973		pl->vddc = vddc;
   3974		pl->vddci = vddci;
   3975	}
   3976
   3977	if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) ==
   3978	    ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) {
   3979		rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk = pl->sclk;
   3980		rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk = pl->mclk;
   3981		rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc = pl->vddc;
   3982		rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci = pl->vddci;
   3983	}
   3984}
   3985
   3986static int ni_parse_power_table(struct radeon_device *rdev)
   3987{
   3988	struct radeon_mode_info *mode_info = &rdev->mode_info;
   3989	struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
   3990	union pplib_power_state *power_state;
   3991	int i, j;
   3992	union pplib_clock_info *clock_info;
   3993	union power_info *power_info;
   3994	int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
   3995	u16 data_offset;
   3996	u8 frev, crev;
   3997	struct ni_ps *ps;
   3998
   3999	if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
   4000				   &frev, &crev, &data_offset))
   4001		return -EINVAL;
   4002	power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
   4003
   4004	rdev->pm.dpm.ps = kcalloc(power_info->pplib.ucNumStates,
   4005				  sizeof(struct radeon_ps),
   4006				  GFP_KERNEL);
   4007	if (!rdev->pm.dpm.ps)
   4008		return -ENOMEM;
   4009
   4010	for (i = 0; i < power_info->pplib.ucNumStates; i++) {
   4011		power_state = (union pplib_power_state *)
   4012			(mode_info->atom_context->bios + data_offset +
   4013			 le16_to_cpu(power_info->pplib.usStateArrayOffset) +
   4014			 i * power_info->pplib.ucStateEntrySize);
   4015		non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
   4016			(mode_info->atom_context->bios + data_offset +
   4017			 le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) +
   4018			 (power_state->v1.ucNonClockStateIndex *
   4019			  power_info->pplib.ucNonClockSize));
   4020		if (power_info->pplib.ucStateEntrySize - 1) {
   4021			u8 *idx;
   4022			ps = kzalloc(sizeof(struct ni_ps), GFP_KERNEL);
   4023			if (ps == NULL) {
   4024				kfree(rdev->pm.dpm.ps);
   4025				return -ENOMEM;
   4026			}
   4027			rdev->pm.dpm.ps[i].ps_priv = ps;
   4028			ni_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
   4029							 non_clock_info,
   4030							 power_info->pplib.ucNonClockSize);
   4031			idx = (u8 *)&power_state->v1.ucClockStateIndices[0];
   4032			for (j = 0; j < (power_info->pplib.ucStateEntrySize - 1); j++) {
   4033				clock_info = (union pplib_clock_info *)
   4034					(mode_info->atom_context->bios + data_offset +
   4035					 le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) +
   4036					 (idx[j] * power_info->pplib.ucClockInfoSize));
   4037				ni_parse_pplib_clock_info(rdev,
   4038							  &rdev->pm.dpm.ps[i], j,
   4039							  clock_info);
   4040			}
   4041		}
   4042	}
   4043	rdev->pm.dpm.num_ps = power_info->pplib.ucNumStates;
   4044	return 0;
   4045}
   4046
   4047int ni_dpm_init(struct radeon_device *rdev)
   4048{
   4049	struct rv7xx_power_info *pi;
   4050	struct evergreen_power_info *eg_pi;
   4051	struct ni_power_info *ni_pi;
   4052	struct atom_clock_dividers dividers;
   4053	int ret;
   4054
   4055	ni_pi = kzalloc(sizeof(struct ni_power_info), GFP_KERNEL);
   4056	if (ni_pi == NULL)
   4057		return -ENOMEM;
   4058	rdev->pm.dpm.priv = ni_pi;
   4059	eg_pi = &ni_pi->eg;
   4060	pi = &eg_pi->rv7xx;
   4061
   4062	rv770_get_max_vddc(rdev);
   4063
   4064	eg_pi->ulv.supported = false;
   4065	pi->acpi_vddc = 0;
   4066	eg_pi->acpi_vddci = 0;
   4067	pi->min_vddc_in_table = 0;
   4068	pi->max_vddc_in_table = 0;
   4069
   4070	ret = r600_get_platform_caps(rdev);
   4071	if (ret)
   4072		return ret;
   4073
   4074	ret = ni_parse_power_table(rdev);
   4075	if (ret)
   4076		return ret;
   4077	ret = r600_parse_extended_power_table(rdev);
   4078	if (ret)
   4079		return ret;
   4080
   4081	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries =
   4082		kcalloc(4,
   4083			sizeof(struct radeon_clock_voltage_dependency_entry),
   4084			GFP_KERNEL);
   4085	if (!rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries) {
   4086		r600_free_extended_power_table(rdev);
   4087		return -ENOMEM;
   4088	}
   4089	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count = 4;
   4090	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].clk = 0;
   4091	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].v = 0;
   4092	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].clk = 36000;
   4093	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].v = 720;
   4094	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].clk = 54000;
   4095	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].v = 810;
   4096	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].clk = 72000;
   4097	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].v = 900;
   4098
   4099	ni_patch_dependency_tables_based_on_leakage(rdev);
   4100
   4101	if (rdev->pm.dpm.voltage_response_time == 0)
   4102		rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT;
   4103	if (rdev->pm.dpm.backbias_response_time == 0)
   4104		rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT;
   4105
   4106	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
   4107					     0, false, &dividers);
   4108	if (ret)
   4109		pi->ref_div = dividers.ref_div + 1;
   4110	else
   4111		pi->ref_div = R600_REFERENCEDIVIDER_DFLT;
   4112
   4113	pi->rlp = RV770_RLP_DFLT;
   4114	pi->rmp = RV770_RMP_DFLT;
   4115	pi->lhp = RV770_LHP_DFLT;
   4116	pi->lmp = RV770_LMP_DFLT;
   4117
   4118	eg_pi->ats[0].rlp = RV770_RLP_DFLT;
   4119	eg_pi->ats[0].rmp = RV770_RMP_DFLT;
   4120	eg_pi->ats[0].lhp = RV770_LHP_DFLT;
   4121	eg_pi->ats[0].lmp = RV770_LMP_DFLT;
   4122
   4123	eg_pi->ats[1].rlp = BTC_RLP_UVD_DFLT;
   4124	eg_pi->ats[1].rmp = BTC_RMP_UVD_DFLT;
   4125	eg_pi->ats[1].lhp = BTC_LHP_UVD_DFLT;
   4126	eg_pi->ats[1].lmp = BTC_LMP_UVD_DFLT;
   4127
   4128	eg_pi->smu_uvd_hs = true;
   4129
   4130	if (rdev->pdev->device == 0x6707) {
   4131		pi->mclk_strobe_mode_threshold = 55000;
   4132		pi->mclk_edc_enable_threshold = 55000;
   4133		eg_pi->mclk_edc_wr_enable_threshold = 55000;
   4134	} else {
   4135		pi->mclk_strobe_mode_threshold = 40000;
   4136		pi->mclk_edc_enable_threshold = 40000;
   4137		eg_pi->mclk_edc_wr_enable_threshold = 40000;
   4138	}
   4139	ni_pi->mclk_rtt_mode_threshold = eg_pi->mclk_edc_wr_enable_threshold;
   4140
   4141	pi->voltage_control =
   4142		radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0);
   4143
   4144	pi->mvdd_control =
   4145		radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_MVDDC, 0);
   4146
   4147	eg_pi->vddci_control =
   4148		radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDCI, 0);
   4149
   4150	rv770_get_engine_memory_ss(rdev);
   4151
   4152	pi->asi = RV770_ASI_DFLT;
   4153	pi->pasi = CYPRESS_HASI_DFLT;
   4154	pi->vrc = CYPRESS_VRC_DFLT;
   4155
   4156	pi->power_gating = false;
   4157
   4158	pi->gfx_clock_gating = true;
   4159
   4160	pi->mg_clock_gating = true;
   4161	pi->mgcgtssm = true;
   4162	eg_pi->ls_clock_gating = false;
   4163	eg_pi->sclk_deep_sleep = false;
   4164
   4165	pi->dynamic_pcie_gen2 = true;
   4166
   4167	if (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE)
   4168		pi->thermal_protection = true;
   4169	else
   4170		pi->thermal_protection = false;
   4171
   4172	pi->display_gap = true;
   4173
   4174	pi->dcodt = true;
   4175
   4176	pi->ulps = true;
   4177
   4178	eg_pi->dynamic_ac_timing = true;
   4179	eg_pi->abm = true;
   4180	eg_pi->mcls = true;
   4181	eg_pi->light_sleep = true;
   4182	eg_pi->memory_transition = true;
   4183#if defined(CONFIG_ACPI)
   4184	eg_pi->pcie_performance_request =
   4185		radeon_acpi_is_pcie_performance_request_supported(rdev);
   4186#else
   4187	eg_pi->pcie_performance_request = false;
   4188#endif
   4189
   4190	eg_pi->dll_default_on = false;
   4191
   4192	eg_pi->sclk_deep_sleep = false;
   4193
   4194	pi->mclk_stutter_mode_threshold = 0;
   4195
   4196	pi->sram_end = SMC_RAM_END;
   4197
   4198	rdev->pm.dpm.dyn_state.mclk_sclk_ratio = 3;
   4199	rdev->pm.dpm.dyn_state.vddc_vddci_delta = 200;
   4200	rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2 = 900;
   4201	rdev->pm.dpm.dyn_state.valid_sclk_values.count = ARRAY_SIZE(btc_valid_sclk);
   4202	rdev->pm.dpm.dyn_state.valid_sclk_values.values = btc_valid_sclk;
   4203	rdev->pm.dpm.dyn_state.valid_mclk_values.count = 0;
   4204	rdev->pm.dpm.dyn_state.valid_mclk_values.values = NULL;
   4205	rdev->pm.dpm.dyn_state.sclk_mclk_delta = 12500;
   4206
   4207	ni_pi->cac_data.leakage_coefficients.at = 516;
   4208	ni_pi->cac_data.leakage_coefficients.bt = 18;
   4209	ni_pi->cac_data.leakage_coefficients.av = 51;
   4210	ni_pi->cac_data.leakage_coefficients.bv = 2957;
   4211
   4212	switch (rdev->pdev->device) {
   4213	case 0x6700:
   4214	case 0x6701:
   4215	case 0x6702:
   4216	case 0x6703:
   4217	case 0x6718:
   4218		ni_pi->cac_weights = &cac_weights_cayman_xt;
   4219		break;
   4220	case 0x6705:
   4221	case 0x6719:
   4222	case 0x671D:
   4223	case 0x671C:
   4224	default:
   4225		ni_pi->cac_weights = &cac_weights_cayman_pro;
   4226		break;
   4227	case 0x6704:
   4228	case 0x6706:
   4229	case 0x6707:
   4230	case 0x6708:
   4231	case 0x6709:
   4232		ni_pi->cac_weights = &cac_weights_cayman_le;
   4233		break;
   4234	}
   4235
   4236	if (ni_pi->cac_weights->enable_power_containment_by_default) {
   4237		ni_pi->enable_power_containment = true;
   4238		ni_pi->enable_cac = true;
   4239		ni_pi->enable_sq_ramping = true;
   4240	} else {
   4241		ni_pi->enable_power_containment = false;
   4242		ni_pi->enable_cac = false;
   4243		ni_pi->enable_sq_ramping = false;
   4244	}
   4245
   4246	ni_pi->driver_calculate_cac_leakage = false;
   4247	ni_pi->cac_configuration_required = true;
   4248
   4249	if (ni_pi->cac_configuration_required) {
   4250		ni_pi->support_cac_long_term_average = true;
   4251		ni_pi->lta_window_size = ni_pi->cac_weights->l2_lta_window_size;
   4252		ni_pi->lts_truncate = ni_pi->cac_weights->lts_truncate;
   4253	} else {
   4254		ni_pi->support_cac_long_term_average = false;
   4255		ni_pi->lta_window_size = 0;
   4256		ni_pi->lts_truncate = 0;
   4257	}
   4258
   4259	ni_pi->use_power_boost_limit = true;
   4260
   4261	/* make sure dc limits are valid */
   4262	if ((rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.sclk == 0) ||
   4263	    (rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.mclk == 0))
   4264		rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc =
   4265			rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
   4266
   4267	return 0;
   4268}
   4269
   4270void ni_dpm_fini(struct radeon_device *rdev)
   4271{
   4272	int i;
   4273
   4274	for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
   4275		kfree(rdev->pm.dpm.ps[i].ps_priv);
   4276	}
   4277	kfree(rdev->pm.dpm.ps);
   4278	kfree(rdev->pm.dpm.priv);
   4279	kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries);
   4280	r600_free_extended_power_table(rdev);
   4281}
   4282
   4283void ni_dpm_print_power_state(struct radeon_device *rdev,
   4284			      struct radeon_ps *rps)
   4285{
   4286	struct ni_ps *ps = ni_get_ps(rps);
   4287	struct rv7xx_pl *pl;
   4288	int i;
   4289
   4290	r600_dpm_print_class_info(rps->class, rps->class2);
   4291	r600_dpm_print_cap_info(rps->caps);
   4292	printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
   4293	for (i = 0; i < ps->performance_level_count; i++) {
   4294		pl = &ps->performance_levels[i];
   4295		if (rdev->family >= CHIP_TAHITI)
   4296			printk("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u pcie gen: %u\n",
   4297			       i, pl->sclk, pl->mclk, pl->vddc, pl->vddci, pl->pcie_gen + 1);
   4298		else
   4299			printk("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
   4300			       i, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
   4301	}
   4302	r600_dpm_print_ps_status(rdev, rps);
   4303}
   4304
   4305void ni_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
   4306						    struct seq_file *m)
   4307{
   4308	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   4309	struct radeon_ps *rps = &eg_pi->current_rps;
   4310	struct ni_ps *ps = ni_get_ps(rps);
   4311	struct rv7xx_pl *pl;
   4312	u32 current_index =
   4313		(RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
   4314		CURRENT_STATE_INDEX_SHIFT;
   4315
   4316	if (current_index >= ps->performance_level_count) {
   4317		seq_printf(m, "invalid dpm profile %d\n", current_index);
   4318	} else {
   4319		pl = &ps->performance_levels[current_index];
   4320		seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
   4321		seq_printf(m, "power level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
   4322			   current_index, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
   4323	}
   4324}
   4325
   4326u32 ni_dpm_get_current_sclk(struct radeon_device *rdev)
   4327{
   4328	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   4329	struct radeon_ps *rps = &eg_pi->current_rps;
   4330	struct ni_ps *ps = ni_get_ps(rps);
   4331	struct rv7xx_pl *pl;
   4332	u32 current_index =
   4333		(RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
   4334		CURRENT_STATE_INDEX_SHIFT;
   4335
   4336	if (current_index >= ps->performance_level_count) {
   4337		return 0;
   4338	} else {
   4339		pl = &ps->performance_levels[current_index];
   4340		return pl->sclk;
   4341	}
   4342}
   4343
   4344u32 ni_dpm_get_current_mclk(struct radeon_device *rdev)
   4345{
   4346	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   4347	struct radeon_ps *rps = &eg_pi->current_rps;
   4348	struct ni_ps *ps = ni_get_ps(rps);
   4349	struct rv7xx_pl *pl;
   4350	u32 current_index =
   4351		(RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
   4352		CURRENT_STATE_INDEX_SHIFT;
   4353
   4354	if (current_index >= ps->performance_level_count) {
   4355		return 0;
   4356	} else {
   4357		pl = &ps->performance_levels[current_index];
   4358		return pl->mclk;
   4359	}
   4360}
   4361
   4362u32 ni_dpm_get_sclk(struct radeon_device *rdev, bool low)
   4363{
   4364	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   4365	struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
   4366
   4367	if (low)
   4368		return requested_state->performance_levels[0].sclk;
   4369	else
   4370		return requested_state->performance_levels[requested_state->performance_level_count - 1].sclk;
   4371}
   4372
   4373u32 ni_dpm_get_mclk(struct radeon_device *rdev, bool low)
   4374{
   4375	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
   4376	struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
   4377
   4378	if (low)
   4379		return requested_state->performance_levels[0].mclk;
   4380	else
   4381		return requested_state->performance_levels[requested_state->performance_level_count - 1].mclk;
   4382}
   4383