drivers/crypto/qat/qat_common/qat_hal.c

   1 /*
   2   This file is provided under a dual BSD/GPLv2 license.  When using or
   3   redistributing this file, you may do so under either license.
   4
   5   GPL LICENSE SUMMARY
   6   Copyright(c) 2014 Intel Corporation.
   7   This program is free software; you can redistribute it and/or modify
   8   it under the terms of version 2 of the GNU General Public License as
   9   published by the Free Software Foundation.
  10
  11   This program is distributed in the hope that it will be useful, but
  12   WITHOUT ANY WARRANTY; without even the implied warranty of
  13   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14   General Public License for more details.
  15
  16   Contact Information:
  17   qat-linux@intel.com
  18
  19   BSD LICENSE
  20   Copyright(c) 2014 Intel Corporation.
  21   Redistribution and use in source and binary forms, with or without
  22   modification, are permitted provided that the following conditions
  23   are met:
  24
  25     * Redistributions of source code must retain the above copyright
  26       notice, this list of conditions and the following disclaimer.
  27     * Redistributions in binary form must reproduce the above copyright
  28       notice, this list of conditions and the following disclaimer in
  29       the documentation and/or other materials provided with the
  30       distribution.
  31     * Neither the name of Intel Corporation nor the names of its
  32       contributors may be used to endorse or promote products derived
  33       from this software without specific prior written permission.
  34
  35   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  36   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  37   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  38   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  39   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  40   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  41   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  42   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  43   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  44   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  45   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  46 */
  47 #include <linux/slab.h>
  48
  49 #include "adf_accel_devices.h"
  50 #include "adf_common_drv.h"
  51 #include "icp_qat_hal.h"
  52 #include "icp_qat_uclo.h"
  53
  54 #define BAD_REGADDR               0xffff
  55 #define MAX_RETRY_TIMES           10000
  56 #define INIT_CTX_ARB_VALUE        0x0
  57 #define INIT_CTX_ENABLE_VALUE     0x0
  58 #define INIT_PC_VALUE             0x0
  59 #define INIT_WAKEUP_EVENTS_VALUE  0x1
  60 #define INIT_SIG_EVENTS_VALUE     0x1
  61 #define INIT_CCENABLE_VALUE       0x2000
  62 #define RST_CSR_QAT_LSB           20
  63 #define RST_CSR_AE_LSB            0
  64 #define MC_TIMESTAMP_ENABLE       (0x1 << 7)
  65
  66 #define IGNORE_W1C_MASK ((~(1 << CE_BREAKPOINT_BITPOS)) & \
  67         (~(1 << CE_CNTL_STORE_PARITY_ERROR_BITPOS)) & \
  68         (~(1 << CE_REG_PAR_ERR_BITPOS)))
  69 #define INSERT_IMMED_GPRA_CONST(inst, const_val) \
  70         (inst = ((inst & 0xFFFF00C03FFull) | \
  71                 ((((const_val) << 12) & 0x0FF00000ull) | \
  72                 (((const_val) << 10) & 0x0003FC00ull))))
  73 #define INSERT_IMMED_GPRB_CONST(inst, const_val) \
  74         (inst = ((inst & 0xFFFF00FFF00ull) | \
  75                 ((((const_val) << 12) & 0x0FF00000ull) | \
  76                 (((const_val) <<  0) & 0x000000FFull))))
  77
  78 #define AE(handle, ae) handle->hal_handle->aes[ae]
  79
  80 static const uint64_t inst_4b[] = {
  81         0x0F0400C0000ull, 0x0F4400C0000ull, 0x0F040000300ull, 0x0F440000300ull,
  82         0x0FC066C0000ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
  83         0x0A021000000ull
  84 };
  85
  86 static const uint64_t inst[] = {
  87         0x0F0000C0000ull, 0x0F000000380ull, 0x0D805000011ull, 0x0FC082C0300ull,
  88         0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
  89         0x0A0643C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0F0000C0001ull,
  90         0x0FC066C0001ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
  91         0x0F000400300ull, 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D804400101ull,
  92         0x0A0580C0000ull, 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull,
  93         0x0A0584C0000ull, 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull,
  94         0x0A0588C0000ull, 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull,
  95         0x0A058CC0000ull, 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull,
  96         0x0A05C0C0000ull, 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull,
  97         0x0A05C4C0000ull, 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull,
  98         0x0A05C8C0000ull, 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull,
  99         0x0A05CCC0000ull, 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull,
 100         0x0A0400C0000ull, 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull,
 101         0x0A0402C0000ull, 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull,
 102         0x0A0404C0000ull, 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull,
 103         0x0A0406C0000ull, 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull,
 104         0x0A0408C0000ull, 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull,
 105         0x0A040AC0000ull, 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull,
 106         0x0A040CC0000ull, 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull,
 107         0x0A040EC0000ull, 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull,
 108         0x0D81581C010ull, 0x0E000010000ull, 0x0E000010000ull,
 109 };
 110
 111 void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle,
 112                           unsigned char ae, unsigned int ctx_mask)
 113 {
 114         AE(handle, ae).live_ctx_mask = ctx_mask;
 115 }
 116
 117 #define CSR_RETRY_TIMES 500
 118 static int qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle *handle,
 119                              unsigned char ae, unsigned int csr,
 120                              unsigned int *value)
 121 {
 122         unsigned int iterations = CSR_RETRY_TIMES;
 123
 124         do {
 125                 *value = GET_AE_CSR(handle, ae, csr);
 126                 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
 127                         return 0;
 128         } while (iterations--);
 129
 130         pr_err("QAT: Read CSR timeout\n");
 131         return -EFAULT;
 132 }
 133
 134 static int qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle *handle,
 135                              unsigned char ae, unsigned int csr,
 136                              unsigned int value)
 137 {
 138         unsigned int iterations = CSR_RETRY_TIMES;
 139
 140         do {
 141                 SET_AE_CSR(handle, ae, csr, value);
 142                 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
 143                         return 0;
 144         } while (iterations--);
 145
 146         pr_err("QAT: Write CSR Timeout\n");
 147         return -EFAULT;
 148 }
 149
 150 static void qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle *handle,
 151                                      unsigned char ae, unsigned char ctx,
 152                                      unsigned int *events)
 153 {
 154         unsigned int cur_ctx;
 155
 156         qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
 157         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 158         qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT, events);
 159         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 160 }
 161
 162 static int qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle,
 163                                unsigned char ae, unsigned int cycles,
 164                                int chk_inactive)
 165 {
 166         unsigned int base_cnt = 0, cur_cnt = 0;
 167         unsigned int csr = (1 << ACS_ABO_BITPOS);
 168         int times = MAX_RETRY_TIMES;
 169         int elapsed_cycles = 0;
 170
 171         qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, &base_cnt);
 172         base_cnt &= 0xffff;
 173         while ((int)cycles > elapsed_cycles && times--) {
 174                 if (chk_inactive)
 175                         qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &csr);
 176
 177                 qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, &cur_cnt);
 178                 cur_cnt &= 0xffff;
 179                 elapsed_cycles = cur_cnt - base_cnt;
 180
 181                 if (elapsed_cycles < 0)
 182                         elapsed_cycles += 0x10000;
 183
 184                 /* ensure at least 8 time cycles elapsed in wait_cycles */
 185                 if (elapsed_cycles >= 8 && !(csr & (1 << ACS_ABO_BITPOS)))
 186                         return 0;
 187         }
 188         if (!times) {
 189                 pr_err("QAT: wait_num_cycles time out\n");
 190                 return -EFAULT;
 191         }
 192         return 0;
 193 }
 194
 195 #define CLR_BIT(wrd, bit) (wrd & ~(1 << bit))
 196 #define SET_BIT(wrd, bit) (wrd | 1 << bit)
 197
 198 int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle,
 199                             unsigned char ae, unsigned char mode)
 200 {
 201         unsigned int csr, new_csr;
 202
 203         if ((mode != 4) && (mode != 8)) {
 204                 pr_err("QAT: bad ctx mode=%d\n", mode);
 205                 return -EINVAL;
 206         }
 207
 208         /* Sets the accelaration engine context mode to either four or eight */
 209         qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr);
 210         csr = IGNORE_W1C_MASK & csr;
 211         new_csr = (mode == 4) ?
 212                 SET_BIT(csr, CE_INUSE_CONTEXTS_BITPOS) :
 213                 CLR_BIT(csr, CE_INUSE_CONTEXTS_BITPOS);
 214         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
 215         return 0;
 216 }
 217
 218 int qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle,
 219                            unsigned char ae, unsigned char mode)
 220 {
 221         unsigned int csr, new_csr;
 222
 223         qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr);
 224         csr &= IGNORE_W1C_MASK;
 225
 226         new_csr = (mode) ?
 227                 SET_BIT(csr, CE_NN_MODE_BITPOS) :
 228                 CLR_BIT(csr, CE_NN_MODE_BITPOS);
 229
 230         if (new_csr != csr)
 231                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
 232
 233         return 0;
 234 }
 235
 236 int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle,
 237                            unsigned char ae, enum icp_qat_uof_regtype lm_type,
 238                            unsigned char mode)
 239 {
 240         unsigned int csr, new_csr;
 241
 242         qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr);
 243         csr &= IGNORE_W1C_MASK;
 244         switch (lm_type) {
 245         case ICP_LMEM0:
 246                 new_csr = (mode) ?
 247                         SET_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS) :
 248                         CLR_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS);
 249                 break;
 250         case ICP_LMEM1:
 251                 new_csr = (mode) ?
 252                         SET_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS) :
 253                         CLR_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS);
 254                 break;
 255         default:
 256                 pr_err("QAT: lmType = 0x%x\n", lm_type);
 257                 return -EINVAL;
 258         }
 259
 260         if (new_csr != csr)
 261                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
 262         return 0;
 263 }
 264
 265 static unsigned short qat_hal_get_reg_addr(unsigned int type,
 266                                            unsigned short reg_num)
 267 {
 268         unsigned short reg_addr;
 269
 270         switch (type) {
 271         case ICP_GPA_ABS:
 272         case ICP_GPB_ABS:
 273                 reg_addr = 0x80 | (reg_num & 0x7f);
 274                 break;
 275         case ICP_GPA_REL:
 276         case ICP_GPB_REL:
 277                 reg_addr = reg_num & 0x1f;
 278                 break;
 279         case ICP_SR_RD_REL:
 280         case ICP_SR_WR_REL:
 281         case ICP_SR_REL:
 282                 reg_addr = 0x180 | (reg_num & 0x1f);
 283                 break;
 284         case ICP_SR_ABS:
 285                 reg_addr = 0x140 | ((reg_num & 0x3) << 1);
 286                 break;
 287         case ICP_DR_RD_REL:
 288         case ICP_DR_WR_REL:
 289         case ICP_DR_REL:
 290                 reg_addr = 0x1c0 | (reg_num & 0x1f);
 291                 break;
 292         case ICP_DR_ABS:
 293                 reg_addr = 0x100 | ((reg_num & 0x3) << 1);
 294                 break;
 295         case ICP_NEIGH_REL:
 296                 reg_addr = 0x280 | (reg_num & 0x1f);
 297                 break;
 298         case ICP_LMEM0:
 299                 reg_addr = 0x200;
 300                 break;
 301         case ICP_LMEM1:
 302                 reg_addr = 0x220;
 303                 break;
 304         case ICP_NO_DEST:
 305                 reg_addr = 0x300 | (reg_num & 0xff);
 306                 break;
 307         default:
 308                 reg_addr = BAD_REGADDR;
 309                 break;
 310         }
 311         return reg_addr;
 312 }
 313
 314 void qat_hal_reset(struct icp_qat_fw_loader_handle *handle)
 315 {
 316         unsigned int ae_reset_csr;
 317
 318         ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET);
 319         ae_reset_csr |= handle->hal_handle->ae_mask << RST_CSR_AE_LSB;
 320         ae_reset_csr |= handle->hal_handle->slice_mask << RST_CSR_QAT_LSB;
 321         SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr);
 322 }
 323
 324 static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle,
 325                                 unsigned char ae, unsigned int ctx_mask,
 326                                 unsigned int ae_csr, unsigned int csr_val)
 327 {
 328         unsigned int ctx, cur_ctx;
 329
 330         qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
 331
 332         for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
 333                 if (!(ctx_mask & (1 << ctx)))
 334                         continue;
 335                 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 336                 qat_hal_wr_ae_csr(handle, ae, ae_csr, csr_val);
 337         }
 338
 339         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 340 }
 341
 342 static void qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle,
 343                                 unsigned char ae, unsigned char ctx,
 344                                 unsigned int ae_csr, unsigned int *csr_val)
 345 {
 346         unsigned int cur_ctx;
 347
 348         qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
 349         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 350         qat_hal_rd_ae_csr(handle, ae, ae_csr, csr_val);
 351         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 352 }
 353
 354 static void qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle,
 355                                   unsigned char ae, unsigned int ctx_mask,
 356                                   unsigned int events)
 357 {
 358         unsigned int ctx, cur_ctx;
 359
 360         qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
 361         for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
 362                 if (!(ctx_mask & (1 << ctx)))
 363                         continue;
 364                 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 365                 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_INDIRECT, events);
 366         }
 367         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 368 }
 369
 370 static void qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle,
 371                                      unsigned char ae, unsigned int ctx_mask,
 372                                      unsigned int events)
 373 {
 374         unsigned int ctx, cur_ctx;
 375
 376         qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
 377         for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
 378                 if (!(ctx_mask & (1 << ctx)))
 379                         continue;
 380                 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 381                 qat_hal_wr_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT,
 382                                   events);
 383         }
 384         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 385 }
 386
 387 static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle)
 388 {
 389         unsigned int base_cnt, cur_cnt;
 390         unsigned char ae;
 391         unsigned int times = MAX_RETRY_TIMES;
 392
 393         for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
 394                 if (!(handle->hal_handle->ae_mask & (1 << ae)))
 395                         continue;
 396
 397                 qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT,
 398                                   (unsigned int *)&base_cnt);
 399                 base_cnt &= 0xffff;
 400
 401                 do {
 402                         qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT,
 403                                           (unsigned int *)&cur_cnt);
 404                         cur_cnt &= 0xffff;
 405                 } while (times-- && (cur_cnt == base_cnt));
 406
 407                 if (!times) {
 408                         pr_err("QAT: AE%d is inactive!!\n", ae);
 409                         return -EFAULT;
 410                 }
 411         }
 412
 413         return 0;
 414 }
 415
 416 static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle)
 417 {
 418         unsigned int misc_ctl;
 419         unsigned char ae;
 420
 421         /* stop the timestamp timers */
 422         misc_ctl = GET_GLB_CSR(handle, MISC_CONTROL);
 423         if (misc_ctl & MC_TIMESTAMP_ENABLE)
 424                 SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl &
 425                             (~MC_TIMESTAMP_ENABLE));
 426
 427         for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
 428                 if (!(handle->hal_handle->ae_mask & (1 << ae)))
 429                         continue;
 430                 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0);
 431                 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0);
 432         }
 433         /* start timestamp timers */
 434         SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl | MC_TIMESTAMP_ENABLE);
 435 }
 436
 437 #define ESRAM_AUTO_TINIT        BIT(2)
 438 #define ESRAM_AUTO_TINIT_DONE   BIT(3)
 439 #define ESRAM_AUTO_INIT_USED_CYCLES (1640)
 440 #define ESRAM_AUTO_INIT_CSR_OFFSET 0xC1C
 441 static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle)
 442 {
 443         void __iomem *csr_addr = handle->hal_ep_csr_addr_v +
 444                                  ESRAM_AUTO_INIT_CSR_OFFSET;
 445         unsigned int csr_val, times = 30;
 446
 447         csr_val = ADF_CSR_RD(csr_addr, 0);
 448         if ((csr_val & ESRAM_AUTO_TINIT) && (csr_val & ESRAM_AUTO_TINIT_DONE))
 449                 return 0;
 450
 451         csr_val = ADF_CSR_RD(csr_addr, 0);
 452         csr_val |= ESRAM_AUTO_TINIT;
 453         ADF_CSR_WR(csr_addr, 0, csr_val);
 454
 455         do {
 456                 qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0);
 457                 csr_val = ADF_CSR_RD(csr_addr, 0);
 458         } while (!(csr_val & ESRAM_AUTO_TINIT_DONE) && times--);
 459         if ((!times)) {
 460                 pr_err("QAT: Fail to init eSram!\n");
 461                 return -EFAULT;
 462         }
 463         return 0;
 464 }
 465
 466 #define SHRAM_INIT_CYCLES 2060
 467 int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle)
 468 {
 469         unsigned int ae_reset_csr;
 470         unsigned char ae;
 471         unsigned int clk_csr;
 472         unsigned int times = 100;
 473         unsigned int csr;
 474
 475         /* write to the reset csr */
 476         ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET);
 477         ae_reset_csr &= ~(handle->hal_handle->ae_mask << RST_CSR_AE_LSB);
 478         ae_reset_csr &= ~(handle->hal_handle->slice_mask << RST_CSR_QAT_LSB);
 479         do {
 480                 SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr);
 481                 if (!(times--))
 482                         goto out_err;
 483                 csr = GET_GLB_CSR(handle, ICP_RESET);
 484         } while ((handle->hal_handle->ae_mask |
 485                  (handle->hal_handle->slice_mask << RST_CSR_QAT_LSB)) & csr);
 486         /* enable clock */
 487         clk_csr = GET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE);
 488         clk_csr |= handle->hal_handle->ae_mask << 0;
 489         clk_csr |= handle->hal_handle->slice_mask << 20;
 490         SET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE, clk_csr);
 491         if (qat_hal_check_ae_alive(handle))
 492                 goto out_err;
 493
 494         /* Set undefined power-up/reset states to reasonable default values */
 495         for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
 496                 if (!(handle->hal_handle->ae_mask & (1 << ae)))
 497                         continue;
 498                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,
 499                                   INIT_CTX_ENABLE_VALUE);
 500                 qat_hal_wr_indr_csr(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX,
 501                                     CTX_STS_INDIRECT,
 502                                     handle->hal_handle->upc_mask &
 503                                     INIT_PC_VALUE);
 504                 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
 505                 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);
 506                 qat_hal_put_wakeup_event(handle, ae,
 507                                          ICP_QAT_UCLO_AE_ALL_CTX,
 508                                          INIT_WAKEUP_EVENTS_VALUE);
 509                 qat_hal_put_sig_event(handle, ae,
 510                                       ICP_QAT_UCLO_AE_ALL_CTX,
 511                                       INIT_SIG_EVENTS_VALUE);
 512         }
 513         if (qat_hal_init_esram(handle))
 514                 goto out_err;
 515         if (qat_hal_wait_cycles(handle, 0, SHRAM_INIT_CYCLES, 0))
 516                 goto out_err;
 517         qat_hal_reset_timestamp(handle);
 518
 519         return 0;
 520 out_err:
 521         pr_err("QAT: failed to get device out of reset\n");
 522         return -EFAULT;
 523 }
 524
 525 static void qat_hal_disable_ctx(struct icp_qat_fw_loader_handle *handle,
 526                                 unsigned char ae, unsigned int ctx_mask)
 527 {
 528         unsigned int ctx;
 529
 530         qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx);
 531         ctx &= IGNORE_W1C_MASK &
 532                 (~((ctx_mask & ICP_QAT_UCLO_AE_ALL_CTX) << CE_ENABLE_BITPOS));
 533         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
 534 }
 535
 536 static uint64_t qat_hal_parity_64bit(uint64_t word)
 537 {
 538         word ^= word >> 1;
 539         word ^= word >> 2;
 540         word ^= word >> 4;
 541         word ^= word >> 8;
 542         word ^= word >> 16;
 543         word ^= word >> 32;
 544         return word & 1;
 545 }
 546
 547 static uint64_t qat_hal_set_uword_ecc(uint64_t uword)
 548 {
 549         uint64_t bit0_mask = 0xff800007fffULL, bit1_mask = 0x1f801ff801fULL,
 550                 bit2_mask = 0xe387e0781e1ULL, bit3_mask = 0x7cb8e388e22ULL,
 551                 bit4_mask = 0xaf5b2c93244ULL, bit5_mask = 0xf56d5525488ULL,
 552                 bit6_mask = 0xdaf69a46910ULL;
 553
 554         /* clear the ecc bits */
 555         uword &= ~(0x7fULL << 0x2C);
 556         uword |= qat_hal_parity_64bit(bit0_mask & uword) << 0x2C;
 557         uword |= qat_hal_parity_64bit(bit1_mask & uword) << 0x2D;
 558         uword |= qat_hal_parity_64bit(bit2_mask & uword) << 0x2E;
 559         uword |= qat_hal_parity_64bit(bit3_mask & uword) << 0x2F;
 560         uword |= qat_hal_parity_64bit(bit4_mask & uword) << 0x30;
 561         uword |= qat_hal_parity_64bit(bit5_mask & uword) << 0x31;
 562         uword |= qat_hal_parity_64bit(bit6_mask & uword) << 0x32;
 563         return uword;
 564 }
 565
 566 void qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle,
 567                        unsigned char ae, unsigned int uaddr,
 568                        unsigned int words_num, uint64_t *uword)
 569 {
 570         unsigned int ustore_addr;
 571         unsigned int i;
 572
 573         qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr);
 574         uaddr |= UA_ECS;
 575         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 576         for (i = 0; i < words_num; i++) {
 577                 unsigned int uwrd_lo, uwrd_hi;
 578                 uint64_t tmp;
 579
 580                 tmp = qat_hal_set_uword_ecc(uword[i]);
 581                 uwrd_lo = (unsigned int)(tmp & 0xffffffff);
 582                 uwrd_hi = (unsigned int)(tmp >> 0x20);
 583                 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
 584                 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
 585         }
 586         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
 587 }
 588
 589 static void qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle,
 590                                unsigned char ae, unsigned int ctx_mask)
 591 {
 592         unsigned int ctx;
 593
 594         qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx);
 595         ctx &= IGNORE_W1C_MASK;
 596         ctx_mask &= (ctx & CE_INUSE_CONTEXTS) ? 0x55 : 0xFF;
 597         ctx |= (ctx_mask << CE_ENABLE_BITPOS);
 598         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
 599 }
 600
 601 static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle)
 602 {
 603         unsigned char ae;
 604         unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX;
 605         int times = MAX_RETRY_TIMES;
 606         unsigned int csr_val = 0;
 607         unsigned short reg;
 608         unsigned int savctx = 0;
 609         int ret = 0;
 610
 611         for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
 612                 if (!(handle->hal_handle->ae_mask & (1 << ae)))
 613                         continue;
 614                 for (reg = 0; reg < ICP_QAT_UCLO_MAX_GPR_REG; reg++) {
 615                         qat_hal_init_rd_xfer(handle, ae, 0, ICP_SR_RD_ABS,
 616                                              reg, 0);
 617                         qat_hal_init_rd_xfer(handle, ae, 0, ICP_DR_RD_ABS,
 618                                              reg, 0);
 619                 }
 620                 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val);
 621                 csr_val &= ~(1 << MMC_SHARE_CS_BITPOS);
 622                 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val);
 623                 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr_val);
 624                 csr_val &= IGNORE_W1C_MASK;
 625                 csr_val |= CE_NN_MODE;
 626                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val);
 627                 qat_hal_wr_uwords(handle, ae, 0, ARRAY_SIZE(inst),
 628                                   (uint64_t *)inst);
 629                 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
 630                                     handle->hal_handle->upc_mask &
 631                                     INIT_PC_VALUE);
 632                 qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx);
 633                 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 0);
 634                 qat_hal_put_wakeup_event(handle, ae, ctx_mask, XCWE_VOLUNTARY);
 635                 qat_hal_wr_indr_csr(handle, ae, ctx_mask,
 636                                     CTX_SIG_EVENTS_INDIRECT, 0);
 637                 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
 638                 qat_hal_enable_ctx(handle, ae, ctx_mask);
 639         }
 640         for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
 641                 if (!(handle->hal_handle->ae_mask & (1 << ae)))
 642                         continue;
 643                 /* wait for AE to finish */
 644                 do {
 645                         ret = qat_hal_wait_cycles(handle, ae, 20, 1);
 646                 } while (ret && times--);
 647
 648                 if (!times) {
 649                         pr_err("QAT: clear GPR of AE %d failed", ae);
 650                         return -EINVAL;
 651                 }
 652                 qat_hal_disable_ctx(handle, ae, ctx_mask);
 653                 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
 654                                   savctx & ACS_ACNO);
 655                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,
 656                                   INIT_CTX_ENABLE_VALUE);
 657                 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
 658                                     handle->hal_handle->upc_mask &
 659                                     INIT_PC_VALUE);
 660                 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
 661                 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);
 662                 qat_hal_put_wakeup_event(handle, ae, ctx_mask,
 663                                          INIT_WAKEUP_EVENTS_VALUE);
 664                 qat_hal_put_sig_event(handle, ae, ctx_mask,
 665                                       INIT_SIG_EVENTS_VALUE);
 666         }
 667         return 0;
 668 }
 669
 670 #define ICP_DH895XCC_AE_OFFSET      0x20000
 671 #define ICP_DH895XCC_CAP_OFFSET     (ICP_DH895XCC_AE_OFFSET + 0x10000)
 672 #define LOCAL_TO_XFER_REG_OFFSET    0x800
 673 #define ICP_DH895XCC_EP_OFFSET      0x3a000
 674 int qat_hal_init(struct adf_accel_dev *accel_dev)
 675 {
 676         unsigned char ae;
 677         unsigned int max_en_ae_id = 0;
 678         struct icp_qat_fw_loader_handle *handle;
 679         struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev;
 680         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 681         struct adf_bar *misc_bar =
 682                         &pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)];
 683         struct adf_bar *sram_bar =
 684                         &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)];
 685
 686         handle = kzalloc(sizeof(*handle), GFP_KERNEL);
 687         if (!handle)
 688                 return -ENOMEM;
 689
 690         handle->hal_cap_g_ctl_csr_addr_v = misc_bar->virt_addr +
 691                                                 ICP_DH895XCC_CAP_OFFSET;
 692         handle->hal_cap_ae_xfer_csr_addr_v = misc_bar->virt_addr +
 693                                                 ICP_DH895XCC_AE_OFFSET;
 694         handle->hal_ep_csr_addr_v = misc_bar->virt_addr +
 695                                     ICP_DH895XCC_EP_OFFSET;
 696         handle->hal_cap_ae_local_csr_addr_v =
 697                 handle->hal_cap_ae_xfer_csr_addr_v + LOCAL_TO_XFER_REG_OFFSET;
 698         handle->hal_sram_addr_v = sram_bar->virt_addr;
 699         handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL);
 700         if (!handle->hal_handle)
 701                 goto out_hal_handle;
 702         handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid;
 703         handle->hal_handle->ae_mask = hw_data->ae_mask;
 704         handle->hal_handle->slice_mask = hw_data->accel_mask;
 705         /* create AE objects */
 706         handle->hal_handle->upc_mask = 0x1ffff;
 707         handle->hal_handle->max_ustore = 0x4000;
 708         for (ae = 0; ae < ICP_QAT_UCLO_MAX_AE; ae++) {
 709                 if (!(hw_data->ae_mask & (1 << ae)))
 710                         continue;
 711                 handle->hal_handle->aes[ae].free_addr = 0;
 712                 handle->hal_handle->aes[ae].free_size =
 713                     handle->hal_handle->max_ustore;
 714                 handle->hal_handle->aes[ae].ustore_size =
 715                     handle->hal_handle->max_ustore;
 716                 handle->hal_handle->aes[ae].live_ctx_mask =
 717                                                 ICP_QAT_UCLO_AE_ALL_CTX;
 718                 max_en_ae_id = ae;
 719         }
 720         handle->hal_handle->ae_max_num = max_en_ae_id + 1;
 721         /* take all AEs out of reset */
 722         if (qat_hal_clr_reset(handle)) {
 723                 dev_err(&GET_DEV(accel_dev), "qat_hal_clr_reset error\n");
 724                 goto out_err;
 725         }
 726         if (qat_hal_clear_gpr(handle))
 727                 goto out_err;
 728         /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */
 729         for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
 730                 unsigned int csr_val = 0;
 731
 732                 if (!(hw_data->ae_mask & (1 << ae)))
 733                         continue;
 734                 qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE, &csr_val);
 735                 csr_val |= 0x1;
 736                 qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val);
 737         }
 738         accel_dev->fw_loader->fw_loader = handle;
 739         return 0;
 740
 741 out_err:
 742         kfree(handle->hal_handle);
 743 out_hal_handle:
 744         kfree(handle);
 745         return -EFAULT;
 746 }
 747
 748 void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle)
 749 {
 750         if (!handle)
 751                 return;
 752         kfree(handle->hal_handle);
 753         kfree(handle);
 754 }
 755
 756 void qat_hal_start(struct icp_qat_fw_loader_handle *handle, unsigned char ae,
 757                    unsigned int ctx_mask)
 758 {
 759         qat_hal_put_wakeup_event(handle, ae, (~ctx_mask) &
 760                                  ICP_QAT_UCLO_AE_ALL_CTX, 0x10000);
 761         qat_hal_enable_ctx(handle, ae, ctx_mask);
 762 }
 763
 764 void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae,
 765                   unsigned int ctx_mask)
 766 {
 767         qat_hal_disable_ctx(handle, ae, ctx_mask);
 768 }
 769
 770 void qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle,
 771                     unsigned char ae, unsigned int ctx_mask, unsigned int upc)
 772 {
 773         qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
 774                             handle->hal_handle->upc_mask & upc);
 775 }
 776
 777 static void qat_hal_get_uwords(struct icp_qat_fw_loader_handle *handle,
 778                                unsigned char ae, unsigned int uaddr,
 779                                unsigned int words_num, uint64_t *uword)
 780 {
 781         unsigned int i, uwrd_lo, uwrd_hi;
 782         unsigned int ustore_addr, misc_control;
 783
 784         qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &misc_control);
 785         qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL,
 786                           misc_control & 0xfffffffb);
 787         qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr);
 788         uaddr |= UA_ECS;
 789         for (i = 0; i < words_num; i++) {
 790                 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 791                 uaddr++;
 792                 qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER, &uwrd_lo);
 793                 qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER, &uwrd_hi);
 794                 uword[i] = uwrd_hi;
 795                 uword[i] = (uword[i] << 0x20) | uwrd_lo;
 796         }
 797         qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, misc_control);
 798         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
 799 }
 800
 801 void qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle,
 802                      unsigned char ae, unsigned int uaddr,
 803                      unsigned int words_num, unsigned int *data)
 804 {
 805         unsigned int i, ustore_addr;
 806
 807         qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr);
 808         uaddr |= UA_ECS;
 809         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 810         for (i = 0; i < words_num; i++) {
 811                 unsigned int uwrd_lo, uwrd_hi, tmp;
 812
 813                 uwrd_lo = ((data[i] & 0xfff0000) << 4) | (0x3 << 18) |
 814                           ((data[i] & 0xff00) << 2) |
 815                           (0x3 << 8) | (data[i] & 0xff);
 816                 uwrd_hi = (0xf << 4) | ((data[i] & 0xf0000000) >> 28);
 817                 uwrd_hi |= (hweight32(data[i] & 0xffff) & 0x1) << 8;
 818                 tmp = ((data[i] >> 0x10) & 0xffff);
 819                 uwrd_hi |= (hweight32(tmp) & 0x1) << 9;
 820                 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
 821                 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
 822         }
 823         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
 824 }
 825
 826 #define MAX_EXEC_INST 100
 827 static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle,
 828                                    unsigned char ae, unsigned char ctx,
 829                                    uint64_t *micro_inst, unsigned int inst_num,
 830                                    int code_off, unsigned int max_cycle,
 831                                    unsigned int *endpc)
 832 {
 833         uint64_t savuwords[MAX_EXEC_INST];
 834         unsigned int ind_lm_addr0, ind_lm_addr1;
 835         unsigned int ind_lm_addr_byte0, ind_lm_addr_byte1;
 836         unsigned int ind_cnt_sig;
 837         unsigned int ind_sig, act_sig;
 838         unsigned int csr_val = 0, newcsr_val;
 839         unsigned int savctx;
 840         unsigned int savcc, wakeup_events, savpc;
 841         unsigned int ctxarb_ctl, ctx_enables;
 842
 843         if ((inst_num > handle->hal_handle->max_ustore) || !micro_inst) {
 844                 pr_err("QAT: invalid instruction num %d\n", inst_num);
 845                 return -EINVAL;
 846         }
 847         /* save current context */
 848         qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT, &ind_lm_addr0);
 849         qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT, &ind_lm_addr1);
 850         qat_hal_rd_indr_csr(handle, ae, ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX,
 851                             &ind_lm_addr_byte0);
 852         qat_hal_rd_indr_csr(handle, ae, ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX,
 853                             &ind_lm_addr_byte1);
 854         if (inst_num <= MAX_EXEC_INST)
 855                 qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords);
 856         qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events);
 857         qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT, &savpc);
 858         savpc = (savpc & handle->hal_handle->upc_mask) >> 0;
 859         qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
 860         ctx_enables &= IGNORE_W1C_MASK;
 861         qat_hal_rd_ae_csr(handle, ae, CC_ENABLE, &savcc);
 862         qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx);
 863         qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL, &ctxarb_ctl);
 864         qat_hal_rd_indr_csr(handle, ae, ctx, FUTURE_COUNT_SIGNAL_INDIRECT,
 865                             &ind_cnt_sig);
 866         qat_hal_rd_indr_csr(handle, ae, ctx, CTX_SIG_EVENTS_INDIRECT, &ind_sig);
 867         qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, &act_sig);
 868         /* execute micro codes */
 869         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
 870         qat_hal_wr_uwords(handle, ae, 0, inst_num, micro_inst);
 871         qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, 0);
 872         qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, ctx & ACS_ACNO);
 873         if (code_off)
 874                 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc & 0xffffdfff);
 875         qat_hal_put_wakeup_event(handle, ae, (1 << ctx), XCWE_VOLUNTARY);
 876         qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, 0);
 877         qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
 878         qat_hal_enable_ctx(handle, ae, (1 << ctx));
 879         /* wait for micro codes to finish */
 880         if (qat_hal_wait_cycles(handle, ae, max_cycle, 1) != 0)
 881                 return -EFAULT;
 882         if (endpc) {
 883                 unsigned int ctx_status;
 884
 885                 qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT,
 886                                     &ctx_status);
 887                 *endpc = ctx_status & handle->hal_handle->upc_mask;
 888         }
 889         /* retore to saved context */
 890         qat_hal_disable_ctx(handle, ae, (1 << ctx));
 891         if (inst_num <= MAX_EXEC_INST)
 892                 qat_hal_wr_uwords(handle, ae, 0, inst_num, savuwords);
 893         qat_hal_put_wakeup_event(handle, ae, (1 << ctx), wakeup_events);
 894         qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT,
 895                             handle->hal_handle->upc_mask & savpc);
 896         qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val);
 897         newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS);
 898         qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val);
 899         qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc);
 900         qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, savctx & ACS_ACNO);
 901         qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_ctl);
 902         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
 903                             LM_ADDR_0_INDIRECT, ind_lm_addr0);
 904         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
 905                             LM_ADDR_1_INDIRECT, ind_lm_addr1);
 906         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
 907                             INDIRECT_LM_ADDR_0_BYTE_INDEX, ind_lm_addr_byte0);
 908         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
 909                             INDIRECT_LM_ADDR_1_BYTE_INDEX, ind_lm_addr_byte1);
 910         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
 911                             FUTURE_COUNT_SIGNAL_INDIRECT, ind_cnt_sig);
 912         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
 913                             CTX_SIG_EVENTS_INDIRECT, ind_sig);
 914         qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, act_sig);
 915         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
 916
 917         return 0;
 918 }
 919
 920 static int qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle,
 921                               unsigned char ae, unsigned char ctx,
 922                               enum icp_qat_uof_regtype reg_type,
 923                               unsigned short reg_num, unsigned int *data)
 924 {
 925         unsigned int savctx, uaddr, uwrd_lo, uwrd_hi;
 926         unsigned int ctxarb_cntl, ustore_addr, ctx_enables;
 927         unsigned short reg_addr;
 928         int status = 0;
 929         uint64_t insts, savuword;
 930
 931         reg_addr = qat_hal_get_reg_addr(reg_type, reg_num);
 932         if (reg_addr == BAD_REGADDR) {
 933                 pr_err("QAT: bad regaddr=0x%x\n", reg_addr);
 934                 return -EINVAL;
 935         }
 936         switch (reg_type) {
 937         case ICP_GPA_REL:
 938                 insts = 0xA070000000ull | (reg_addr & 0x3ff);
 939                 break;
 940         default:
 941                 insts = (uint64_t)0xA030000000ull | ((reg_addr & 0x3ff) << 10);
 942                 break;
 943         }
 944         qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx);
 945         qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL, &ctxarb_cntl);
 946         qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
 947         ctx_enables &= IGNORE_W1C_MASK;
 948         if (ctx != (savctx & ACS_ACNO))
 949                 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
 950                                   ctx & ACS_ACNO);
 951         qat_hal_get_uwords(handle, ae, 0, 1, &savuword);
 952         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
 953         qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr);
 954         uaddr = UA_ECS;
 955         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 956         insts = qat_hal_set_uword_ecc(insts);
 957         uwrd_lo = (unsigned int)(insts & 0xffffffff);
 958         uwrd_hi = (unsigned int)(insts >> 0x20);
 959         qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
 960         qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
 961         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 962         /* delay for at least 8 cycles */
 963         qat_hal_wait_cycles(handle, ae, 0x8, 0);
 964         /*
 965          * read ALU output
 966          * the instruction should have been executed
 967          * prior to clearing the ECS in putUwords
 968          */
 969         qat_hal_rd_ae_csr(handle, ae, ALU_OUT, data);
 970         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
 971         qat_hal_wr_uwords(handle, ae, 0, 1, &savuword);
 972         if (ctx != (savctx & ACS_ACNO))
 973                 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
 974                                   savctx & ACS_ACNO);
 975         qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_cntl);
 976         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
 977
 978         return status;
 979 }
 980
 981 static int qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle *handle,
 982                               unsigned char ae, unsigned char ctx,
 983                               enum icp_qat_uof_regtype reg_type,
 984                               unsigned short reg_num, unsigned int data)
 985 {
 986         unsigned short src_hiaddr, src_lowaddr, dest_addr, data16hi, data16lo;
 987         uint64_t insts[] = {
 988                 0x0F440000000ull,
 989                 0x0F040000000ull,
 990                 0x0F0000C0300ull,
 991                 0x0E000010000ull
 992         };
 993         const int num_inst = ARRAY_SIZE(insts), code_off = 1;
 994         const int imm_w1 = 0, imm_w0 = 1;
 995
 996         dest_addr = qat_hal_get_reg_addr(reg_type, reg_num);
 997         if (dest_addr == BAD_REGADDR) {
 998                 pr_err("QAT: bad destAddr=0x%x\n", dest_addr);
 999                 return -EINVAL;
1000         }
1001
1002         data16lo = 0xffff & data;
1003         data16hi = 0xffff & (data >> 0x10);
1004         src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short)
1005                                           (0xff & data16hi));
1006         src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short)
1007                                            (0xff & data16lo));
1008         switch (reg_type) {
1009         case ICP_GPA_REL:
1010                 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1011                     ((src_hiaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1012                 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1013                     ((src_lowaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1014                 break;
1015         default:
1016                 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1017                     ((dest_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1018
1019                 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1020                     ((dest_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1021                 break;
1022         }
1023
1024         return qat_hal_exec_micro_inst(handle, ae, ctx, insts, num_inst,
1025                                        code_off, num_inst * 0x5, NULL);
1026 }
1027
1028 int qat_hal_get_ins_num(void)
1029 {
1030         return ARRAY_SIZE(inst_4b);
1031 }
1032
1033 static int qat_hal_concat_micro_code(uint64_t *micro_inst,
1034                                      unsigned int inst_num, unsigned int size,
1035                                      unsigned int addr, unsigned int *value)
1036 {
1037         int i, val_indx;
1038         unsigned int cur_value;
1039         const uint64_t *inst_arr;
1040         int fixup_offset;
1041         int usize = 0;
1042         int orig_num;
1043
1044         orig_num = inst_num;
1045         val_indx = 0;
1046         cur_value = value[val_indx++];
1047         inst_arr = inst_4b;
1048         usize = ARRAY_SIZE(inst_4b);
1049         fixup_offset = inst_num;
1050         for (i = 0; i < usize; i++)
1051                 micro_inst[inst_num++] = inst_arr[i];
1052         INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], (addr));
1053         fixup_offset++;
1054         INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], 0);
1055         fixup_offset++;
1056         INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0));
1057         fixup_offset++;
1058         INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0x10));
1059
1060         return inst_num - orig_num;
1061 }
1062
1063 static int qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle *handle,
1064                                       unsigned char ae, unsigned char ctx,
1065                                       int *pfirst_exec, uint64_t *micro_inst,
1066                                       unsigned int inst_num)
1067 {
1068         int stat = 0;
1069         unsigned int gpra0 = 0, gpra1 = 0, gpra2 = 0;
1070         unsigned int gprb0 = 0, gprb1 = 0;
1071
1072         if (*pfirst_exec) {
1073                 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, &gpra0);
1074                 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, &gpra1);
1075                 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, &gpra2);
1076                 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, &gprb0);
1077                 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, &gprb1);
1078                 *pfirst_exec = 0;
1079         }
1080         stat = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, inst_num, 1,
1081                                        inst_num * 0x5, NULL);
1082         if (stat != 0)
1083                 return -EFAULT;
1084         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, gpra0);
1085         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, gpra1);
1086         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, gpra2);
1087         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, gprb0);
1088         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, gprb1);
1089
1090         return 0;
1091 }
1092
1093 int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle,
1094                         unsigned char ae,
1095                         struct icp_qat_uof_batch_init *lm_init_header)
1096 {
1097         struct icp_qat_uof_batch_init *plm_init;
1098         uint64_t *micro_inst_arry;
1099         int micro_inst_num;
1100         int alloc_inst_size;
1101         int first_exec = 1;
1102         int stat = 0;
1103
1104         plm_init = lm_init_header->next;
1105         alloc_inst_size = lm_init_header->size;
1106         if ((unsigned int)alloc_inst_size > handle->hal_handle->max_ustore)
1107                 alloc_inst_size = handle->hal_handle->max_ustore;
1108         micro_inst_arry = kmalloc_array(alloc_inst_size, sizeof(uint64_t),
1109                                         GFP_KERNEL);
1110         if (!micro_inst_arry)
1111                 return -ENOMEM;
1112         micro_inst_num = 0;
1113         while (plm_init) {
1114                 unsigned int addr, *value, size;
1115
1116                 ae = plm_init->ae;
1117                 addr = plm_init->addr;
1118                 value = plm_init->value;
1119                 size = plm_init->size;
1120                 micro_inst_num += qat_hal_concat_micro_code(micro_inst_arry,
1121                                                             micro_inst_num,
1122                                                             size, addr, value);
1123                 plm_init = plm_init->next;
1124         }
1125         /* exec micro codes */
1126         if (micro_inst_arry && (micro_inst_num > 0)) {
1127                 micro_inst_arry[micro_inst_num++] = 0x0E000010000ull;
1128                 stat = qat_hal_exec_micro_init_lm(handle, ae, 0, &first_exec,
1129                                                   micro_inst_arry,
1130                                                   micro_inst_num);
1131         }
1132         kfree(micro_inst_arry);
1133         return stat;
1134 }
1135
1136 static int qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1137                                    unsigned char ae, unsigned char ctx,
1138                                    enum icp_qat_uof_regtype reg_type,
1139                                    unsigned short reg_num, unsigned int val)
1140 {
1141         int status = 0;
1142         unsigned int reg_addr;
1143         unsigned int ctx_enables;
1144         unsigned short mask;
1145         unsigned short dr_offset = 0x10;
1146
1147         status = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
1148         if (CE_INUSE_CONTEXTS & ctx_enables) {
1149                 if (ctx & 0x1) {
1150                         pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx);
1151                         return -EINVAL;
1152                 }
1153                 mask = 0x1f;
1154                 dr_offset = 0x20;
1155         } else {
1156                 mask = 0x0f;
1157         }
1158         if (reg_num & ~mask)
1159                 return -EINVAL;
1160         reg_addr = reg_num + (ctx << 0x5);
1161         switch (reg_type) {
1162         case ICP_SR_RD_REL:
1163         case ICP_SR_REL:
1164                 SET_AE_XFER(handle, ae, reg_addr, val);
1165                 break;
1166         case ICP_DR_RD_REL:
1167         case ICP_DR_REL:
1168                 SET_AE_XFER(handle, ae, (reg_addr + dr_offset), val);
1169                 break;
1170         default:
1171                 status = -EINVAL;
1172                 break;
1173         }
1174         return status;
1175 }
1176
1177 static int qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1178                                    unsigned char ae, unsigned char ctx,
1179                                    enum icp_qat_uof_regtype reg_type,
1180                                    unsigned short reg_num, unsigned int data)
1181 {
1182         unsigned int gprval, ctx_enables;
1183         unsigned short src_hiaddr, src_lowaddr, gpr_addr, xfr_addr, data16hi,
1184             data16low;
1185         unsigned short reg_mask;
1186         int status = 0;
1187         uint64_t micro_inst[] = {
1188                 0x0F440000000ull,
1189                 0x0F040000000ull,
1190                 0x0A000000000ull,
1191                 0x0F0000C0300ull,
1192                 0x0E000010000ull
1193         };
1194         const int num_inst = ARRAY_SIZE(micro_inst), code_off = 1;
1195         const unsigned short gprnum = 0, dly = num_inst * 0x5;
1196
1197         qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
1198         if (CE_INUSE_CONTEXTS & ctx_enables) {
1199                 if (ctx & 0x1) {
1200                         pr_err("QAT: 4-ctx mode,ctx=0x%x\n", ctx);
1201                         return -EINVAL;
1202                 }
1203                 reg_mask = (unsigned short)~0x1f;
1204         } else {
1205                 reg_mask = (unsigned short)~0xf;
1206         }
1207         if (reg_num & reg_mask)
1208                 return -EINVAL;
1209         xfr_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1210         if (xfr_addr == BAD_REGADDR) {
1211                 pr_err("QAT: bad xfrAddr=0x%x\n", xfr_addr);
1212                 return -EINVAL;
1213         }
1214         qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval);
1215         gpr_addr = qat_hal_get_reg_addr(ICP_GPB_REL, gprnum);
1216         data16low = 0xffff & data;
1217         data16hi = 0xffff & (data >> 0x10);
1218         src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1219                                           (unsigned short)(0xff & data16hi));
1220         src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1221                                            (unsigned short)(0xff & data16low));
1222         micro_inst[0] = micro_inst[0x0] | ((data16hi >> 8) << 20) |
1223             ((gpr_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1224         micro_inst[1] = micro_inst[0x1] | ((data16low >> 8) << 20) |
1225             ((gpr_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1226         micro_inst[0x2] = micro_inst[0x2] |
1227             ((xfr_addr & 0x3ff) << 20) | ((gpr_addr & 0x3ff) << 10);
1228         status = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, num_inst,
1229                                          code_off, dly, NULL);
1230         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, gprval);
1231         return status;
1232 }
1233
1234 static int qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle *handle,
1235                               unsigned char ae, unsigned char ctx,
1236                               unsigned short nn, unsigned int val)
1237 {
1238         unsigned int ctx_enables;
1239         int stat = 0;
1240
1241         qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
1242         ctx_enables &= IGNORE_W1C_MASK;
1243         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables | CE_NN_MODE);
1244
1245         stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, ICP_NEIGH_REL, nn, val);
1246         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1247         return stat;
1248 }
1249
1250 static int qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle
1251                                       *handle, unsigned char ae,
1252                                       unsigned short absreg_num,
1253                                       unsigned short *relreg,
1254                                       unsigned char *ctx)
1255 {
1256         unsigned int ctx_enables;
1257
1258         qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
1259         if (ctx_enables & CE_INUSE_CONTEXTS) {
1260                 /* 4-ctx mode */
1261                 *relreg = absreg_num & 0x1F;
1262                 *ctx = (absreg_num >> 0x4) & 0x6;
1263         } else {
1264                 /* 8-ctx mode */
1265                 *relreg = absreg_num & 0x0F;
1266                 *ctx = (absreg_num >> 0x4) & 0x7;
1267         }
1268         return 0;
1269 }
1270
1271 int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle,
1272                      unsigned char ae, unsigned char ctx_mask,
1273                      enum icp_qat_uof_regtype reg_type,
1274                      unsigned short reg_num, unsigned int regdata)
1275 {
1276         int stat = 0;
1277         unsigned short reg;
1278         unsigned char ctx = 0;
1279         enum icp_qat_uof_regtype type;
1280
1281         if (reg_num >= ICP_QAT_UCLO_MAX_GPR_REG)
1282                 return -EINVAL;
1283
1284         do {
1285                 if (ctx_mask == 0) {
1286                         qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1287                                                    &ctx);
1288                         type = reg_type - 1;
1289                 } else {
1290                         reg = reg_num;
1291                         type = reg_type;
1292                         if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1293                                 continue;
1294                 }
1295                 stat = qat_hal_wr_rel_reg(handle, ae, ctx, type, reg, regdata);
1296                 if (stat) {
1297                         pr_err("QAT: write gpr fail\n");
1298                         return -EINVAL;
1299                 }
1300         } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1301
1302         return 0;
1303 }
1304
1305 int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1306                          unsigned char ae, unsigned char ctx_mask,
1307                          enum icp_qat_uof_regtype reg_type,
1308                          unsigned short reg_num, unsigned int regdata)
1309 {
1310         int stat = 0;
1311         unsigned short reg;
1312         unsigned char ctx = 0;
1313         enum icp_qat_uof_regtype type;
1314
1315         if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1316                 return -EINVAL;
1317
1318         do {
1319                 if (ctx_mask == 0) {
1320                         qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1321                                                    &ctx);
1322                         type = reg_type - 3;
1323                 } else {
1324                         reg = reg_num;
1325                         type = reg_type;
1326                         if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1327                                 continue;
1328                 }
1329                 stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, type, reg,
1330                                                regdata);
1331                 if (stat) {
1332                         pr_err("QAT: write wr xfer fail\n");
1333                         return -EINVAL;
1334                 }
1335         } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1336
1337         return 0;
1338 }
1339
1340 int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1341                          unsigned char ae, unsigned char ctx_mask,
1342                          enum icp_qat_uof_regtype reg_type,
1343                          unsigned short reg_num, unsigned int regdata)
1344 {
1345         int stat = 0;
1346         unsigned short reg;
1347         unsigned char ctx = 0;
1348         enum icp_qat_uof_regtype type;
1349
1350         if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1351                 return -EINVAL;
1352
1353         do {
1354                 if (ctx_mask == 0) {
1355                         qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1356                                                    &ctx);
1357                         type = reg_type - 3;
1358                 } else {
1359                         reg = reg_num;
1360                         type = reg_type;
1361                         if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1362                                 continue;
1363                 }
1364                 stat = qat_hal_put_rel_rd_xfer(handle, ae, ctx, type, reg,
1365                                                regdata);
1366                 if (stat) {
1367                         pr_err("QAT: write rd xfer fail\n");
1368                         return -EINVAL;
1369                 }
1370         } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1371
1372         return 0;
1373 }
1374
1375 int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle,
1376                     unsigned char ae, unsigned char ctx_mask,
1377                     unsigned short reg_num, unsigned int regdata)
1378 {
1379         int stat = 0;
1380         unsigned char ctx;
1381
1382         if (ctx_mask == 0)
1383                 return -EINVAL;
1384
1385         for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
1386                 if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1387                         continue;
1388                 stat = qat_hal_put_rel_nn(handle, ae, ctx, reg_num, regdata);
1389                 if (stat) {
1390                         pr_err("QAT: write neigh error\n");
1391                         return -EINVAL;
1392                 }
1393         }
1394
1395         return 0;
1396 }