wq.h (7489B)
1/* 2 * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33#ifndef __MLX5_WQ_H__ 34#define __MLX5_WQ_H__ 35 36#include <linux/mlx5/mlx5_ifc.h> 37#include <linux/mlx5/cq.h> 38#include <linux/mlx5/qp.h> 39 40struct mlx5_wq_param { 41 int buf_numa_node; 42 int db_numa_node; 43}; 44 45struct mlx5_wq_ctrl { 46 struct mlx5_core_dev *mdev; 47 struct mlx5_frag_buf buf; 48 struct mlx5_db db; 49}; 50 51struct mlx5_wq_cyc { 52 struct mlx5_frag_buf_ctrl fbc; 53 __be32 *db; 54 u16 sz; 55 u16 wqe_ctr; 56 u16 cur_sz; 57}; 58 59struct mlx5_wq_qp { 60 struct mlx5_wq_cyc rq; 61 struct mlx5_wq_cyc sq; 62}; 63 64struct mlx5_cqwq { 65 struct mlx5_frag_buf_ctrl fbc; 66 __be32 *db; 67 u32 cc; /* consumer counter */ 68}; 69 70struct mlx5_wq_ll { 71 struct mlx5_frag_buf_ctrl fbc; 72 __be32 *db; 73 __be16 *tail_next; 74 u16 head; 75 u16 wqe_ctr; 76 u16 cur_sz; 77}; 78 79int mlx5_wq_cyc_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param, 80 void *wqc, struct mlx5_wq_cyc *wq, 81 struct mlx5_wq_ctrl *wq_ctrl); 82void mlx5_wq_cyc_wqe_dump(struct mlx5_wq_cyc *wq, u16 ix, u8 nstrides); 83void mlx5_wq_cyc_reset(struct mlx5_wq_cyc *wq); 84 85int mlx5_wq_qp_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param, 86 void *qpc, struct mlx5_wq_qp *wq, 87 struct mlx5_wq_ctrl *wq_ctrl); 88 89int mlx5_cqwq_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param, 90 void *cqc, struct mlx5_cqwq *wq, 91 struct mlx5_wq_ctrl *wq_ctrl); 92 93int mlx5_wq_ll_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param, 94 void *wqc, struct mlx5_wq_ll *wq, 95 struct mlx5_wq_ctrl *wq_ctrl); 96void mlx5_wq_ll_reset(struct mlx5_wq_ll *wq); 97 98void mlx5_wq_destroy(struct mlx5_wq_ctrl *wq_ctrl); 99 100static inline u32 mlx5_wq_cyc_get_size(struct mlx5_wq_cyc *wq) 101{ 102 return (u32)wq->fbc.sz_m1 + 1; 103} 104 105static inline int mlx5_wq_cyc_is_full(struct mlx5_wq_cyc *wq) 106{ 107 return wq->cur_sz == wq->sz; 108} 109 110static inline int mlx5_wq_cyc_missing(struct mlx5_wq_cyc *wq) 111{ 112 return wq->sz - wq->cur_sz; 113} 114 115static inline int mlx5_wq_cyc_is_empty(struct mlx5_wq_cyc *wq) 116{ 117 return !wq->cur_sz; 118} 119 120static inline void mlx5_wq_cyc_push(struct mlx5_wq_cyc *wq) 121{ 122 wq->wqe_ctr++; 123 wq->cur_sz++; 124} 125 126static inline void mlx5_wq_cyc_push_n(struct mlx5_wq_cyc *wq, u8 n) 127{ 128 wq->wqe_ctr += n; 129 wq->cur_sz += n; 130} 131 132static inline void mlx5_wq_cyc_pop(struct mlx5_wq_cyc *wq) 133{ 134 wq->cur_sz--; 135} 136 137static inline void mlx5_wq_cyc_update_db_record(struct mlx5_wq_cyc *wq) 138{ 139 *wq->db = cpu_to_be32(wq->wqe_ctr); 140} 141 142static inline u16 mlx5_wq_cyc_ctr2ix(struct mlx5_wq_cyc *wq, u16 ctr) 143{ 144 return ctr & wq->fbc.sz_m1; 145} 146 147static inline u16 mlx5_wq_cyc_get_head(struct mlx5_wq_cyc *wq) 148{ 149 return mlx5_wq_cyc_ctr2ix(wq, wq->wqe_ctr); 150} 151 152static inline u16 mlx5_wq_cyc_get_tail(struct mlx5_wq_cyc *wq) 153{ 154 return mlx5_wq_cyc_ctr2ix(wq, wq->wqe_ctr - wq->cur_sz); 155} 156 157static inline void *mlx5_wq_cyc_get_wqe(struct mlx5_wq_cyc *wq, u16 ix) 158{ 159 return mlx5_frag_buf_get_wqe(&wq->fbc, ix); 160} 161 162static inline u16 mlx5_wq_cyc_get_contig_wqebbs(struct mlx5_wq_cyc *wq, u16 ix) 163{ 164 return mlx5_frag_buf_get_idx_last_contig_stride(&wq->fbc, ix) - ix + 1; 165} 166 167static inline int mlx5_wq_cyc_cc_bigger(u16 cc1, u16 cc2) 168{ 169 int equal = (cc1 == cc2); 170 int smaller = 0x8000 & (cc1 - cc2); 171 172 return !equal && !smaller; 173} 174 175static inline u16 mlx5_wq_cyc_get_counter(struct mlx5_wq_cyc *wq) 176{ 177 return wq->wqe_ctr; 178} 179 180static inline u32 mlx5_cqwq_get_size(struct mlx5_cqwq *wq) 181{ 182 return wq->fbc.sz_m1 + 1; 183} 184 185static inline u8 mlx5_cqwq_get_log_stride_size(struct mlx5_cqwq *wq) 186{ 187 return wq->fbc.log_stride; 188} 189 190static inline u32 mlx5_cqwq_ctr2ix(struct mlx5_cqwq *wq, u32 ctr) 191{ 192 return ctr & wq->fbc.sz_m1; 193} 194 195static inline u32 mlx5_cqwq_get_ci(struct mlx5_cqwq *wq) 196{ 197 return mlx5_cqwq_ctr2ix(wq, wq->cc); 198} 199 200static inline struct mlx5_cqe64 *mlx5_cqwq_get_wqe(struct mlx5_cqwq *wq, u32 ix) 201{ 202 struct mlx5_cqe64 *cqe = mlx5_frag_buf_get_wqe(&wq->fbc, ix); 203 204 /* For 128B CQEs the data is in the last 64B */ 205 cqe += wq->fbc.log_stride == 7; 206 207 return cqe; 208} 209 210static inline u32 mlx5_cqwq_get_ctr_wrap_cnt(struct mlx5_cqwq *wq, u32 ctr) 211{ 212 return ctr >> wq->fbc.log_sz; 213} 214 215static inline u32 mlx5_cqwq_get_wrap_cnt(struct mlx5_cqwq *wq) 216{ 217 return mlx5_cqwq_get_ctr_wrap_cnt(wq, wq->cc); 218} 219 220static inline void mlx5_cqwq_pop(struct mlx5_cqwq *wq) 221{ 222 wq->cc++; 223} 224 225static inline void mlx5_cqwq_update_db_record(struct mlx5_cqwq *wq) 226{ 227 *wq->db = cpu_to_be32(wq->cc & 0xffffff); 228} 229 230static inline struct mlx5_cqe64 *mlx5_cqwq_get_cqe(struct mlx5_cqwq *wq) 231{ 232 u32 ci = mlx5_cqwq_get_ci(wq); 233 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(wq, ci); 234 u8 cqe_ownership_bit = cqe->op_own & MLX5_CQE_OWNER_MASK; 235 u8 sw_ownership_val = mlx5_cqwq_get_wrap_cnt(wq) & 1; 236 237 if (cqe_ownership_bit != sw_ownership_val) 238 return NULL; 239 240 /* ensure cqe content is read after cqe ownership bit */ 241 dma_rmb(); 242 243 return cqe; 244} 245 246static inline u32 mlx5_wq_ll_get_size(struct mlx5_wq_ll *wq) 247{ 248 return (u32)wq->fbc.sz_m1 + 1; 249} 250 251static inline int mlx5_wq_ll_is_full(struct mlx5_wq_ll *wq) 252{ 253 return wq->cur_sz == wq->fbc.sz_m1; 254} 255 256static inline int mlx5_wq_ll_is_empty(struct mlx5_wq_ll *wq) 257{ 258 return !wq->cur_sz; 259} 260 261static inline int mlx5_wq_ll_missing(struct mlx5_wq_ll *wq) 262{ 263 return wq->fbc.sz_m1 - wq->cur_sz; 264} 265 266static inline void *mlx5_wq_ll_get_wqe(struct mlx5_wq_ll *wq, u16 ix) 267{ 268 return mlx5_frag_buf_get_wqe(&wq->fbc, ix); 269} 270 271static inline u16 mlx5_wq_ll_get_wqe_next_ix(struct mlx5_wq_ll *wq, u16 ix) 272{ 273 struct mlx5_wqe_srq_next_seg *wqe = mlx5_wq_ll_get_wqe(wq, ix); 274 275 return be16_to_cpu(wqe->next_wqe_index); 276} 277 278static inline void mlx5_wq_ll_push(struct mlx5_wq_ll *wq, u16 head_next) 279{ 280 wq->head = head_next; 281 wq->wqe_ctr++; 282 wq->cur_sz++; 283} 284 285static inline void mlx5_wq_ll_pop(struct mlx5_wq_ll *wq, __be16 ix, 286 __be16 *next_tail_next) 287{ 288 *wq->tail_next = ix; 289 wq->tail_next = next_tail_next; 290 wq->cur_sz--; 291} 292 293static inline void mlx5_wq_ll_update_db_record(struct mlx5_wq_ll *wq) 294{ 295 *wq->db = cpu_to_be32(wq->wqe_ctr); 296} 297 298static inline u16 mlx5_wq_ll_get_head(struct mlx5_wq_ll *wq) 299{ 300 return wq->head; 301} 302 303static inline u16 mlx5_wq_ll_get_counter(struct mlx5_wq_ll *wq) 304{ 305 return wq->wqe_ctr; 306} 307 308#endif /* __MLX5_WQ_H__ */