drivers/scsi/megaraid/megaraid_sas_fusion.c

   1 /*
   2  *  Linux MegaRAID driver for SAS based RAID controllers
   3  *
   4  *  Copyright (c) 2009-2013  LSI Corporation
   5  *  Copyright (c) 2013-2014  Avago Technologies
   6  *
   7  *  This program is free software; you can redistribute it and/or
   8  *  modify it under the terms of the GNU General Public License
   9  *  as published by the Free Software Foundation; either version 2
  10  *  of the License, or (at your option) any later version.
  11  *
  12  *  This program is distributed in the hope that it will be useful,
  13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  *  GNU General Public License for more details.
  16  *
  17  *  You should have received a copy of the GNU General Public License
  18  *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19  *
  20  *  FILE: megaraid_sas_fusion.c
  21  *
  22  *  Authors: Avago Technologies
  23  *           Sumant Patro
  24  *           Adam Radford
  25  *           Kashyap Desai <kashyap.desai@avagotech.com>
  26  *           Sumit Saxena <sumit.saxena@avagotech.com>
  27  *
  28  *  Send feedback to: megaraidlinux.pdl@avagotech.com
  29  *
  30  *  Mail to: Avago Technologies, 350 West Trimble Road, Building 90,
  31  *  San Jose, California 95131
  32  */
  33
  34 #include <linux/kernel.h>
  35 #include <linux/types.h>
  36 #include <linux/pci.h>
  37 #include <linux/list.h>
  38 #include <linux/moduleparam.h>
  39 #include <linux/module.h>
  40 #include <linux/spinlock.h>
  41 #include <linux/interrupt.h>
  42 #include <linux/delay.h>
  43 #include <linux/uio.h>
  44 #include <linux/uaccess.h>
  45 #include <linux/fs.h>
  46 #include <linux/compat.h>
  47 #include <linux/blkdev.h>
  48 #include <linux/mutex.h>
  49 #include <linux/poll.h>
  50 #include <linux/vmalloc.h>
  51
  52 #include <scsi/scsi.h>
  53 #include <scsi/scsi_cmnd.h>
  54 #include <scsi/scsi_device.h>
  55 #include <scsi/scsi_host.h>
  56 #include <scsi/scsi_dbg.h>
  57 #include <linux/dmi.h>
  58
  59 #include "megaraid_sas_fusion.h"
  60 #include "megaraid_sas.h"
  61
  62
  63 extern void megasas_free_cmds(struct megasas_instance *instance);
  64 extern struct megasas_cmd *megasas_get_cmd(struct megasas_instance
  65                                            *instance);
  66 extern void
  67 megasas_complete_cmd(struct megasas_instance *instance,
  68                      struct megasas_cmd *cmd, u8 alt_status);
  69 int
  70 wait_and_poll(struct megasas_instance *instance, struct megasas_cmd *cmd,
  71               int seconds);
  72
  73 void
  74 megasas_return_cmd(struct megasas_instance *instance, struct megasas_cmd *cmd);
  75 int megasas_alloc_cmds(struct megasas_instance *instance);
  76 int
  77 megasas_clear_intr_fusion(struct megasas_register_set __iomem *regs);
  78 int
  79 megasas_issue_polled(struct megasas_instance *instance,
  80                      struct megasas_cmd *cmd);
  81 void
  82 megasas_check_and_restore_queue_depth(struct megasas_instance *instance);
  83
  84 int megasas_transition_to_ready(struct megasas_instance *instance, int ocr);
  85 void megaraid_sas_kill_hba(struct megasas_instance *instance);
  86
  87 extern u32 megasas_dbg_lvl;
  88 void megasas_sriov_heartbeat_handler(unsigned long instance_addr);
  89 int megasas_sriov_start_heartbeat(struct megasas_instance *instance,
  90                                   int initial);
  91 void megasas_start_timer(struct megasas_instance *instance,
  92                         struct timer_list *timer,
  93                          void *fn, unsigned long interval);
  94 extern struct megasas_mgmt_info megasas_mgmt_info;
  95 extern unsigned int resetwaittime;
  96 extern unsigned int dual_qdepth_disable;
  97 static void megasas_free_rdpq_fusion(struct megasas_instance *instance);
  98 static void megasas_free_reply_fusion(struct megasas_instance *instance);
  99
 100
 101
 102 /**
 103  * megasas_enable_intr_fusion - Enables interrupts
 104  * @regs:                       MFI register set
 105  */
 106 void
 107 megasas_enable_intr_fusion(struct megasas_instance *instance)
 108 {
 109         struct megasas_register_set __iomem *regs;
 110         regs = instance->reg_set;
 111
 112         instance->mask_interrupts = 0;
 113         /* For Thunderbolt/Invader also clear intr on enable */
 114         writel(~0, &regs->outbound_intr_status);
 115         readl(&regs->outbound_intr_status);
 116
 117         writel(~MFI_FUSION_ENABLE_INTERRUPT_MASK, &(regs)->outbound_intr_mask);
 118
 119         /* Dummy readl to force pci flush */
 120         readl(&regs->outbound_intr_mask);
 121 }
 122
 123 /**
 124  * megasas_disable_intr_fusion - Disables interrupt
 125  * @regs:                        MFI register set
 126  */
 127 void
 128 megasas_disable_intr_fusion(struct megasas_instance *instance)
 129 {
 130         u32 mask = 0xFFFFFFFF;
 131         u32 status;
 132         struct megasas_register_set __iomem *regs;
 133         regs = instance->reg_set;
 134         instance->mask_interrupts = 1;
 135
 136         writel(mask, &regs->outbound_intr_mask);
 137         /* Dummy readl to force pci flush */
 138         status = readl(&regs->outbound_intr_mask);
 139 }
 140
 141 int
 142 megasas_clear_intr_fusion(struct megasas_register_set __iomem *regs)
 143 {
 144         u32 status;
 145         /*
 146          * Check if it is our interrupt
 147          */
 148         status = readl(&regs->outbound_intr_status);
 149
 150         if (status & 1) {
 151                 writel(status, &regs->outbound_intr_status);
 152                 readl(&regs->outbound_intr_status);
 153                 return 1;
 154         }
 155         if (!(status & MFI_FUSION_ENABLE_INTERRUPT_MASK))
 156                 return 0;
 157
 158         return 1;
 159 }
 160
 161 /**
 162  * megasas_get_cmd_fusion -     Get a command from the free pool
 163  * @instance:           Adapter soft state
 164  *
 165  * Returns a blk_tag indexed mpt frame
 166  */
 167 inline struct megasas_cmd_fusion *megasas_get_cmd_fusion(struct megasas_instance
 168                                                   *instance, u32 blk_tag)
 169 {
 170         struct fusion_context *fusion;
 171
 172         fusion = instance->ctrl_context;
 173         return fusion->cmd_list[blk_tag];
 174 }
 175
 176 /**
 177  * megasas_return_cmd_fusion -  Return a cmd to free command pool
 178  * @instance:           Adapter soft state
 179  * @cmd:                Command packet to be returned to free command pool
 180  */
 181 inline void megasas_return_cmd_fusion(struct megasas_instance *instance,
 182         struct megasas_cmd_fusion *cmd)
 183 {
 184         cmd->scmd = NULL;
 185         memset(cmd->io_request, 0, MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE);
 186         cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
 187         cmd->cmd_completed = false;
 188 }
 189
 190 /**
 191  * megasas_fire_cmd_fusion -    Sends command to the FW
 192  * @instance:                   Adapter soft state
 193  * @req_desc:                   32bit or 64bit Request descriptor
 194  *
 195  * Perform PCI Write. Ventura supports 32 bit Descriptor.
 196  * Prior to Ventura (12G) MR controller supports 64 bit Descriptor.
 197  */
 198
 199 static void
 200 megasas_fire_cmd_fusion(struct megasas_instance *instance,
 201                 union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc)
 202 {
 203         if (instance->is_ventura)
 204                 writel(le32_to_cpu(req_desc->u.low),
 205                         &instance->reg_set->inbound_single_queue_port);
 206         else {
 207 #if defined(writeq) && defined(CONFIG_64BIT)
 208                 u64 req_data = (((u64)le32_to_cpu(req_desc->u.high) << 32) |
 209                                 le32_to_cpu(req_desc->u.low));
 210
 211                 writeq(req_data, &instance->reg_set->inbound_low_queue_port);
 212 #else
 213                 unsigned long flags;
 214                 spin_lock_irqsave(&instance->hba_lock, flags);
 215                 writel(le32_to_cpu(req_desc->u.low),
 216                         &instance->reg_set->inbound_low_queue_port);
 217                 writel(le32_to_cpu(req_desc->u.high),
 218                         &instance->reg_set->inbound_high_queue_port);
 219                 mmiowb();
 220                 spin_unlock_irqrestore(&instance->hba_lock, flags);
 221 #endif
 222         }
 223 }
 224
 225 /**
 226  * megasas_fusion_update_can_queue -    Do all Adapter Queue depth related calculations here
 227  * @instance:                                                   Adapter soft state
 228  * fw_boot_context:                                             Whether this function called during probe or after OCR
 229  *
 230  * This function is only for fusion controllers.
 231  * Update host can queue, if firmware downgrade max supported firmware commands.
 232  * Firmware upgrade case will be skiped because underlying firmware has
 233  * more resource than exposed to the OS.
 234  *
 235  */
 236 static void
 237 megasas_fusion_update_can_queue(struct megasas_instance *instance, int fw_boot_context)
 238 {
 239         u16 cur_max_fw_cmds = 0;
 240         u16 ldio_threshold = 0;
 241         struct megasas_register_set __iomem *reg_set;
 242
 243         reg_set = instance->reg_set;
 244
 245         /* ventura FW does not fill outbound_scratch_pad_3 with queue depth */
 246         if (!instance->is_ventura)
 247                 cur_max_fw_cmds =
 248                 readl(&instance->reg_set->outbound_scratch_pad_3) & 0x00FFFF;
 249
 250         if (dual_qdepth_disable || !cur_max_fw_cmds)
 251                 cur_max_fw_cmds = instance->instancet->read_fw_status_reg(reg_set) & 0x00FFFF;
 252         else
 253                 ldio_threshold =
 254                         (instance->instancet->read_fw_status_reg(reg_set) & 0x00FFFF) - MEGASAS_FUSION_IOCTL_CMDS;
 255
 256         dev_info(&instance->pdev->dev,
 257                         "Current firmware maximum commands: %d\t LDIO threshold: %d\n",
 258                         cur_max_fw_cmds, ldio_threshold);
 259
 260         if (fw_boot_context == OCR_CONTEXT) {
 261                 cur_max_fw_cmds = cur_max_fw_cmds - 1;
 262                 if (cur_max_fw_cmds < instance->max_fw_cmds) {
 263                         instance->cur_can_queue =
 264                                 cur_max_fw_cmds - (MEGASAS_FUSION_INTERNAL_CMDS +
 265                                                 MEGASAS_FUSION_IOCTL_CMDS);
 266                         instance->host->can_queue = instance->cur_can_queue;
 267                         instance->ldio_threshold = ldio_threshold;
 268                 }
 269         } else {
 270                 instance->max_fw_cmds = cur_max_fw_cmds;
 271                 instance->ldio_threshold = ldio_threshold;
 272
 273                 if (!instance->is_rdpq)
 274                         instance->max_fw_cmds =
 275                                 min_t(u16, instance->max_fw_cmds, 1024);
 276
 277                 if (reset_devices)
 278                         instance->max_fw_cmds = min(instance->max_fw_cmds,
 279                                                 (u16)MEGASAS_KDUMP_QUEUE_DEPTH);
 280                 /*
 281                 * Reduce the max supported cmds by 1. This is to ensure that the
 282                 * reply_q_sz (1 more than the max cmd that driver may send)
 283                 * does not exceed max cmds that the FW can support
 284                 */
 285                 instance->max_fw_cmds = instance->max_fw_cmds-1;
 286
 287                 instance->max_scsi_cmds = instance->max_fw_cmds -
 288                                 (MEGASAS_FUSION_INTERNAL_CMDS +
 289                                 MEGASAS_FUSION_IOCTL_CMDS);
 290                 instance->cur_can_queue = instance->max_scsi_cmds;
 291                 instance->host->can_queue = instance->cur_can_queue;
 292         }
 293
 294         if (instance->is_ventura)
 295                 instance->max_mpt_cmds =
 296                 instance->max_fw_cmds * RAID_1_PEER_CMDS;
 297         else
 298                 instance->max_mpt_cmds = instance->max_fw_cmds;
 299 }
 300 /**
 301  * megasas_free_cmds_fusion -   Free all the cmds in the free cmd pool
 302  * @instance:           Adapter soft state
 303  */
 304 void
 305 megasas_free_cmds_fusion(struct megasas_instance *instance)
 306 {
 307         int i;
 308         struct fusion_context *fusion = instance->ctrl_context;
 309         struct megasas_cmd_fusion *cmd;
 310
 311         /* SG, Sense */
 312         for (i = 0; i < instance->max_mpt_cmds; i++) {
 313                 cmd = fusion->cmd_list[i];
 314                 if (cmd) {
 315                         if (cmd->sg_frame)
 316                                 pci_pool_free(fusion->sg_dma_pool, cmd->sg_frame,
 317                                       cmd->sg_frame_phys_addr);
 318                         if (cmd->sense)
 319                                 pci_pool_free(fusion->sense_dma_pool, cmd->sense,
 320                                       cmd->sense_phys_addr);
 321                 }
 322         }
 323
 324         if (fusion->sg_dma_pool) {
 325                 pci_pool_destroy(fusion->sg_dma_pool);
 326                 fusion->sg_dma_pool = NULL;
 327         }
 328         if (fusion->sense_dma_pool) {
 329                 pci_pool_destroy(fusion->sense_dma_pool);
 330                 fusion->sense_dma_pool = NULL;
 331         }
 332
 333
 334         /* Reply Frame, Desc*/
 335         if (instance->is_rdpq)
 336                 megasas_free_rdpq_fusion(instance);
 337         else
 338                 megasas_free_reply_fusion(instance);
 339
 340         /* Request Frame, Desc*/
 341         if (fusion->req_frames_desc)
 342                 dma_free_coherent(&instance->pdev->dev,
 343                         fusion->request_alloc_sz, fusion->req_frames_desc,
 344                         fusion->req_frames_desc_phys);
 345         if (fusion->io_request_frames)
 346                 pci_pool_free(fusion->io_request_frames_pool,
 347                         fusion->io_request_frames,
 348                         fusion->io_request_frames_phys);
 349         if (fusion->io_request_frames_pool) {
 350                 pci_pool_destroy(fusion->io_request_frames_pool);
 351                 fusion->io_request_frames_pool = NULL;
 352         }
 353
 354
 355         /* cmd_list */
 356         for (i = 0; i < instance->max_mpt_cmds; i++)
 357                 kfree(fusion->cmd_list[i]);
 358
 359         kfree(fusion->cmd_list);
 360 }
 361
 362 /**
 363  * megasas_create_sg_sense_fusion -     Creates DMA pool for cmd frames
 364  * @instance:                   Adapter soft state
 365  *
 366  */
 367 static int megasas_create_sg_sense_fusion(struct megasas_instance *instance)
 368 {
 369         int i;
 370         u16 max_cmd;
 371         struct fusion_context *fusion;
 372         struct megasas_cmd_fusion *cmd;
 373
 374         fusion = instance->ctrl_context;
 375         max_cmd = instance->max_fw_cmds;
 376
 377
 378         fusion->sg_dma_pool =
 379                         pci_pool_create("mr_sg", instance->pdev,
 380                                 instance->max_chain_frame_sz,
 381                                 MR_DEFAULT_NVME_PAGE_SIZE, 0);
 382         /* SCSI_SENSE_BUFFERSIZE  = 96 bytes */
 383         fusion->sense_dma_pool =
 384                         pci_pool_create("mr_sense", instance->pdev,
 385                                 SCSI_SENSE_BUFFERSIZE, 64, 0);
 386
 387         if (!fusion->sense_dma_pool || !fusion->sg_dma_pool) {
 388                 dev_err(&instance->pdev->dev,
 389                         "Failed from %s %d\n",  __func__, __LINE__);
 390                 return -ENOMEM;
 391         }
 392
 393         /*
 394          * Allocate and attach a frame to each of the commands in cmd_list
 395          */
 396         for (i = 0; i < max_cmd; i++) {
 397                 cmd = fusion->cmd_list[i];
 398                 cmd->sg_frame = pci_pool_alloc(fusion->sg_dma_pool,
 399                                         GFP_KERNEL, &cmd->sg_frame_phys_addr);
 400
 401                 cmd->sense = pci_pool_alloc(fusion->sense_dma_pool,
 402                                         GFP_KERNEL, &cmd->sense_phys_addr);
 403                 if (!cmd->sg_frame || !cmd->sense) {
 404                         dev_err(&instance->pdev->dev,
 405                                 "Failed from %s %d\n",  __func__, __LINE__);
 406                         return -ENOMEM;
 407                 }
 408         }
 409
 410         /* create sense buffer for the raid 1/10 fp */
 411         for (i = max_cmd; i < instance->max_mpt_cmds; i++) {
 412                 cmd = fusion->cmd_list[i];
 413                 cmd->sense = pci_pool_alloc(fusion->sense_dma_pool,
 414                         GFP_KERNEL, &cmd->sense_phys_addr);
 415                 if (!cmd->sense) {
 416                         dev_err(&instance->pdev->dev,
 417                                 "Failed from %s %d\n",  __func__, __LINE__);
 418                         return -ENOMEM;
 419                 }
 420         }
 421
 422         return 0;
 423 }
 424
 425 int
 426 megasas_alloc_cmdlist_fusion(struct megasas_instance *instance)
 427 {
 428         u32 max_mpt_cmd, i;
 429         struct fusion_context *fusion;
 430
 431         fusion = instance->ctrl_context;
 432
 433         max_mpt_cmd = instance->max_mpt_cmds;
 434
 435         /*
 436          * fusion->cmd_list is an array of struct megasas_cmd_fusion pointers.
 437          * Allocate the dynamic array first and then allocate individual
 438          * commands.
 439          */
 440         fusion->cmd_list =
 441                 kzalloc(sizeof(struct megasas_cmd_fusion *) * max_mpt_cmd,
 442                         GFP_KERNEL);
 443         if (!fusion->cmd_list) {
 444                 dev_err(&instance->pdev->dev,
 445                         "Failed from %s %d\n",  __func__, __LINE__);
 446                 return -ENOMEM;
 447         }
 448
 449         for (i = 0; i < max_mpt_cmd; i++) {
 450                 fusion->cmd_list[i] = kzalloc(sizeof(struct megasas_cmd_fusion),
 451                                               GFP_KERNEL);
 452                 if (!fusion->cmd_list[i]) {
 453                         dev_err(&instance->pdev->dev,
 454                                 "Failed from %s %d\n",  __func__, __LINE__);
 455                         return -ENOMEM;
 456                 }
 457         }
 458         return 0;
 459 }
 460 int
 461 megasas_alloc_request_fusion(struct megasas_instance *instance)
 462 {
 463         struct fusion_context *fusion;
 464
 465         fusion = instance->ctrl_context;
 466
 467         fusion->req_frames_desc =
 468                 dma_alloc_coherent(&instance->pdev->dev,
 469                         fusion->request_alloc_sz,
 470                         &fusion->req_frames_desc_phys, GFP_KERNEL);
 471         if (!fusion->req_frames_desc) {
 472                 dev_err(&instance->pdev->dev,
 473                         "Failed from %s %d\n",  __func__, __LINE__);
 474                 return -ENOMEM;
 475         }
 476
 477         fusion->io_request_frames_pool =
 478                         pci_pool_create("mr_ioreq", instance->pdev,
 479                                 fusion->io_frames_alloc_sz, 16, 0);
 480
 481         if (!fusion->io_request_frames_pool) {
 482                 dev_err(&instance->pdev->dev,
 483                         "Failed from %s %d\n",  __func__, __LINE__);
 484                 return -ENOMEM;
 485         }
 486
 487         fusion->io_request_frames =
 488                         pci_pool_alloc(fusion->io_request_frames_pool,
 489                                 GFP_KERNEL, &fusion->io_request_frames_phys);
 490         if (!fusion->io_request_frames) {
 491                 dev_err(&instance->pdev->dev,
 492                         "Failed from %s %d\n",  __func__, __LINE__);
 493                 return -ENOMEM;
 494         }
 495         return 0;
 496 }
 497
 498 int
 499 megasas_alloc_reply_fusion(struct megasas_instance *instance)
 500 {
 501         int i, count;
 502         struct fusion_context *fusion;
 503         union MPI2_REPLY_DESCRIPTORS_UNION *reply_desc;
 504         fusion = instance->ctrl_context;
 505
 506         count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
 507         fusion->reply_frames_desc_pool =
 508                         pci_pool_create("mr_reply", instance->pdev,
 509                                 fusion->reply_alloc_sz * count, 16, 0);
 510
 511         if (!fusion->reply_frames_desc_pool) {
 512                 dev_err(&instance->pdev->dev,
 513                         "Failed from %s %d\n",  __func__, __LINE__);
 514                 return -ENOMEM;
 515         }
 516
 517         fusion->reply_frames_desc[0] =
 518                 pci_pool_alloc(fusion->reply_frames_desc_pool,
 519                         GFP_KERNEL, &fusion->reply_frames_desc_phys[0]);
 520         if (!fusion->reply_frames_desc[0]) {
 521                 dev_err(&instance->pdev->dev,
 522                         "Failed from %s %d\n",  __func__, __LINE__);
 523                 return -ENOMEM;
 524         }
 525         reply_desc = fusion->reply_frames_desc[0];
 526         for (i = 0; i < fusion->reply_q_depth * count; i++, reply_desc++)
 527                 reply_desc->Words = cpu_to_le64(ULLONG_MAX);
 528
 529         /* This is not a rdpq mode, but driver still populate
 530          * reply_frame_desc array to use same msix index in ISR path.
 531          */
 532         for (i = 0; i < (count - 1); i++)
 533                 fusion->reply_frames_desc[i + 1] =
 534                         fusion->reply_frames_desc[i] +
 535                         (fusion->reply_alloc_sz)/sizeof(union MPI2_REPLY_DESCRIPTORS_UNION);
 536
 537         return 0;
 538 }
 539
 540 int
 541 megasas_alloc_rdpq_fusion(struct megasas_instance *instance)
 542 {
 543         int i, j, count;
 544         struct fusion_context *fusion;
 545         union MPI2_REPLY_DESCRIPTORS_UNION *reply_desc;
 546
 547         fusion = instance->ctrl_context;
 548
 549         fusion->rdpq_virt = pci_alloc_consistent(instance->pdev,
 550                                 sizeof(struct MPI2_IOC_INIT_RDPQ_ARRAY_ENTRY) * MAX_MSIX_QUEUES_FUSION,
 551                                 &fusion->rdpq_phys);
 552         if (!fusion->rdpq_virt) {
 553                 dev_err(&instance->pdev->dev,
 554                         "Failed from %s %d\n",  __func__, __LINE__);
 555                 return -ENOMEM;
 556         }
 557
 558         memset(fusion->rdpq_virt, 0,
 559                         sizeof(struct MPI2_IOC_INIT_RDPQ_ARRAY_ENTRY) * MAX_MSIX_QUEUES_FUSION);
 560         count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
 561         fusion->reply_frames_desc_pool = pci_pool_create("mr_rdpq",
 562                                                          instance->pdev, fusion->reply_alloc_sz, 16, 0);
 563
 564         if (!fusion->reply_frames_desc_pool) {
 565                 dev_err(&instance->pdev->dev,
 566                         "Failed from %s %d\n",  __func__, __LINE__);
 567                 return -ENOMEM;
 568         }
 569
 570         for (i = 0; i < count; i++) {
 571                 fusion->reply_frames_desc[i] =
 572                                 pci_pool_alloc(fusion->reply_frames_desc_pool,
 573                                         GFP_KERNEL, &fusion->reply_frames_desc_phys[i]);
 574                 if (!fusion->reply_frames_desc[i]) {
 575                         dev_err(&instance->pdev->dev,
 576                                 "Failed from %s %d\n",  __func__, __LINE__);
 577                         return -ENOMEM;
 578                 }
 579
 580                 fusion->rdpq_virt[i].RDPQBaseAddress =
 581                         cpu_to_le64(fusion->reply_frames_desc_phys[i]);
 582
 583                 reply_desc = fusion->reply_frames_desc[i];
 584                 for (j = 0; j < fusion->reply_q_depth; j++, reply_desc++)
 585                         reply_desc->Words = cpu_to_le64(ULLONG_MAX);
 586         }
 587         return 0;
 588 }
 589
 590 static void
 591 megasas_free_rdpq_fusion(struct megasas_instance *instance) {
 592
 593         int i;
 594         struct fusion_context *fusion;
 595
 596         fusion = instance->ctrl_context;
 597
 598         for (i = 0; i < MAX_MSIX_QUEUES_FUSION; i++) {
 599                 if (fusion->reply_frames_desc[i])
 600                         pci_pool_free(fusion->reply_frames_desc_pool,
 601                                 fusion->reply_frames_desc[i],
 602                                 fusion->reply_frames_desc_phys[i]);
 603         }
 604
 605         if (fusion->reply_frames_desc_pool)
 606                 pci_pool_destroy(fusion->reply_frames_desc_pool);
 607
 608         if (fusion->rdpq_virt)
 609                 pci_free_consistent(instance->pdev,
 610                         sizeof(struct MPI2_IOC_INIT_RDPQ_ARRAY_ENTRY) * MAX_MSIX_QUEUES_FUSION,
 611                         fusion->rdpq_virt, fusion->rdpq_phys);
 612 }
 613
 614 static void
 615 megasas_free_reply_fusion(struct megasas_instance *instance) {
 616
 617         struct fusion_context *fusion;
 618
 619         fusion = instance->ctrl_context;
 620
 621         if (fusion->reply_frames_desc[0])
 622                 pci_pool_free(fusion->reply_frames_desc_pool,
 623                         fusion->reply_frames_desc[0],
 624                         fusion->reply_frames_desc_phys[0]);
 625
 626         if (fusion->reply_frames_desc_pool)
 627                 pci_pool_destroy(fusion->reply_frames_desc_pool);
 628
 629 }
 630
 631
 632 /**
 633  * megasas_alloc_cmds_fusion -  Allocates the command packets
 634  * @instance:           Adapter soft state
 635  *
 636  *
 637  * Each frame has a 32-bit field called context. This context is used to get
 638  * back the megasas_cmd_fusion from the frame when a frame gets completed
 639  * In this driver, the 32 bit values are the indices into an array cmd_list.
 640  * This array is used only to look up the megasas_cmd_fusion given the context.
 641  * The free commands themselves are maintained in a linked list called cmd_pool.
 642  *
 643  * cmds are formed in the io_request and sg_frame members of the
 644  * megasas_cmd_fusion. The context field is used to get a request descriptor
 645  * and is used as SMID of the cmd.
 646  * SMID value range is from 1 to max_fw_cmds.
 647  */
 648 int
 649 megasas_alloc_cmds_fusion(struct megasas_instance *instance)
 650 {
 651         int i;
 652         struct fusion_context *fusion;
 653         struct megasas_cmd_fusion *cmd;
 654         u32 offset;
 655         dma_addr_t io_req_base_phys;
 656         u8 *io_req_base;
 657
 658
 659         fusion = instance->ctrl_context;
 660
 661         if (megasas_alloc_cmdlist_fusion(instance))
 662                 goto fail_exit;
 663
 664         if (megasas_alloc_request_fusion(instance))
 665                 goto fail_exit;
 666
 667         if (instance->is_rdpq) {
 668                 if (megasas_alloc_rdpq_fusion(instance))
 669                         goto fail_exit;
 670         } else
 671                 if (megasas_alloc_reply_fusion(instance))
 672                         goto fail_exit;
 673
 674
 675         /* The first 256 bytes (SMID 0) is not used. Don't add to the cmd list */
 676         io_req_base = fusion->io_request_frames + MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE;
 677         io_req_base_phys = fusion->io_request_frames_phys + MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE;
 678
 679         /*
 680          * Add all the commands to command pool (fusion->cmd_pool)
 681          */
 682
 683         /* SMID 0 is reserved. Set SMID/index from 1 */
 684         for (i = 0; i < instance->max_mpt_cmds; i++) {
 685                 cmd = fusion->cmd_list[i];
 686                 offset = MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE * i;
 687                 memset(cmd, 0, sizeof(struct megasas_cmd_fusion));
 688                 cmd->index = i + 1;
 689                 cmd->scmd = NULL;
 690                 cmd->sync_cmd_idx =
 691                 (i >= instance->max_scsi_cmds && i < instance->max_fw_cmds) ?
 692                                 (i - instance->max_scsi_cmds) :
 693                                 (u32)ULONG_MAX; /* Set to Invalid */
 694                 cmd->instance = instance;
 695                 cmd->io_request =
 696                         (struct MPI2_RAID_SCSI_IO_REQUEST *)
 697                   (io_req_base + offset);
 698                 memset(cmd->io_request, 0,
 699                        sizeof(struct MPI2_RAID_SCSI_IO_REQUEST));
 700                 cmd->io_request_phys_addr = io_req_base_phys + offset;
 701                 cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
 702         }
 703
 704         if (megasas_create_sg_sense_fusion(instance))
 705                 goto fail_exit;
 706
 707         return 0;
 708
 709 fail_exit:
 710         megasas_free_cmds_fusion(instance);
 711         return -ENOMEM;
 712 }
 713
 714 /**
 715  * wait_and_poll -      Issues a polling command
 716  * @instance:                   Adapter soft state
 717  * @cmd:                        Command packet to be issued
 718  *
 719  * For polling, MFI requires the cmd_status to be set to 0xFF before posting.
 720  */
 721 int
 722 wait_and_poll(struct megasas_instance *instance, struct megasas_cmd *cmd,
 723         int seconds)
 724 {
 725         int i;
 726         struct megasas_header *frame_hdr = &cmd->frame->hdr;
 727         struct fusion_context *fusion;
 728
 729         u32 msecs = seconds * 1000;
 730
 731         fusion = instance->ctrl_context;
 732         /*
 733          * Wait for cmd_status to change
 734          */
 735         for (i = 0; (i < msecs) && (frame_hdr->cmd_status == 0xff); i += 20) {
 736                 rmb();
 737                 msleep(20);
 738         }
 739
 740         if (frame_hdr->cmd_status == MFI_STAT_INVALID_STATUS)
 741                 return DCMD_TIMEOUT;
 742         else if (frame_hdr->cmd_status == MFI_STAT_OK)
 743                 return DCMD_SUCCESS;
 744         else
 745                 return DCMD_FAILED;
 746 }
 747
 748 /**
 749  * megasas_ioc_init_fusion -    Initializes the FW
 750  * @instance:           Adapter soft state
 751  *
 752  * Issues the IOC Init cmd
 753  */
 754 int
 755 megasas_ioc_init_fusion(struct megasas_instance *instance)
 756 {
 757         struct megasas_init_frame *init_frame;
 758         struct MPI2_IOC_INIT_REQUEST *IOCInitMessage = NULL;
 759         dma_addr_t      ioc_init_handle;
 760         struct megasas_cmd *cmd;
 761         u8 ret, cur_rdpq_mode;
 762         struct fusion_context *fusion;
 763         union MEGASAS_REQUEST_DESCRIPTOR_UNION req_desc;
 764         int i;
 765         struct megasas_header *frame_hdr;
 766         const char *sys_info;
 767         MFI_CAPABILITIES *drv_ops;
 768         u32 scratch_pad_2;
 769         unsigned long flags;
 770
 771         fusion = instance->ctrl_context;
 772
 773         cmd = megasas_get_cmd(instance);
 774
 775         if (!cmd) {
 776                 dev_err(&instance->pdev->dev, "Could not allocate cmd for INIT Frame\n");
 777                 ret = 1;
 778                 goto fail_get_cmd;
 779         }
 780
 781         scratch_pad_2 = readl
 782                 (&instance->reg_set->outbound_scratch_pad_2);
 783
 784         cur_rdpq_mode = (scratch_pad_2 & MR_RDPQ_MODE_OFFSET) ? 1 : 0;
 785
 786         if (instance->is_rdpq && !cur_rdpq_mode) {
 787                 dev_err(&instance->pdev->dev, "Firmware downgrade *NOT SUPPORTED*"
 788                         " from RDPQ mode to non RDPQ mode\n");
 789                 ret = 1;
 790                 goto fail_fw_init;
 791         }
 792
 793         instance->fw_sync_cache_support = (scratch_pad_2 &
 794                 MR_CAN_HANDLE_SYNC_CACHE_OFFSET) ? 1 : 0;
 795         dev_info(&instance->pdev->dev, "FW supports sync cache\t: %s\n",
 796                  instance->fw_sync_cache_support ? "Yes" : "No");
 797
 798         IOCInitMessage =
 799           dma_alloc_coherent(&instance->pdev->dev,
 800                              sizeof(struct MPI2_IOC_INIT_REQUEST),
 801                              &ioc_init_handle, GFP_KERNEL);
 802
 803         if (!IOCInitMessage) {
 804                 dev_err(&instance->pdev->dev, "Could not allocate memory for "
 805                        "IOCInitMessage\n");
 806                 ret = 1;
 807                 goto fail_fw_init;
 808         }
 809
 810         memset(IOCInitMessage, 0, sizeof(struct MPI2_IOC_INIT_REQUEST));
 811
 812         IOCInitMessage->Function = MPI2_FUNCTION_IOC_INIT;
 813         IOCInitMessage->WhoInit = MPI2_WHOINIT_HOST_DRIVER;
 814         IOCInitMessage->MsgVersion = cpu_to_le16(MPI2_VERSION);
 815         IOCInitMessage->HeaderVersion = cpu_to_le16(MPI2_HEADER_VERSION);
 816         IOCInitMessage->SystemRequestFrameSize = cpu_to_le16(MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE / 4);
 817
 818         IOCInitMessage->ReplyDescriptorPostQueueDepth = cpu_to_le16(fusion->reply_q_depth);
 819         IOCInitMessage->ReplyDescriptorPostQueueAddress = instance->is_rdpq ?
 820                         cpu_to_le64(fusion->rdpq_phys) :
 821                         cpu_to_le64(fusion->reply_frames_desc_phys[0]);
 822         IOCInitMessage->MsgFlags = instance->is_rdpq ?
 823                         MPI2_IOCINIT_MSGFLAG_RDPQ_ARRAY_MODE : 0;
 824         IOCInitMessage->SystemRequestFrameBaseAddress = cpu_to_le64(fusion->io_request_frames_phys);
 825         IOCInitMessage->HostMSIxVectors = instance->msix_vectors;
 826         IOCInitMessage->HostPageSize = MR_DEFAULT_NVME_PAGE_SHIFT;
 827         init_frame = (struct megasas_init_frame *)cmd->frame;
 828         memset(init_frame, 0, MEGAMFI_FRAME_SIZE);
 829
 830         frame_hdr = &cmd->frame->hdr;
 831         frame_hdr->cmd_status = 0xFF;
 832         frame_hdr->flags = cpu_to_le16(
 833                 le16_to_cpu(frame_hdr->flags) |
 834                 MFI_FRAME_DONT_POST_IN_REPLY_QUEUE);
 835
 836         init_frame->cmd = MFI_CMD_INIT;
 837         init_frame->cmd_status = 0xFF;
 838
 839         drv_ops = (MFI_CAPABILITIES *) &(init_frame->driver_operations);
 840
 841         /* driver support Extended MSIX */
 842         if (fusion->adapter_type >= INVADER_SERIES)
 843                 drv_ops->mfi_capabilities.support_additional_msix = 1;
 844         /* driver supports HA / Remote LUN over Fast Path interface */
 845         drv_ops->mfi_capabilities.support_fp_remote_lun = 1;
 846
 847         drv_ops->mfi_capabilities.support_max_255lds = 1;
 848         drv_ops->mfi_capabilities.support_ndrive_r1_lb = 1;
 849         drv_ops->mfi_capabilities.security_protocol_cmds_fw = 1;
 850
 851         if (instance->max_chain_frame_sz > MEGASAS_CHAIN_FRAME_SZ_MIN)
 852                 drv_ops->mfi_capabilities.support_ext_io_size = 1;
 853
 854         drv_ops->mfi_capabilities.support_fp_rlbypass = 1;
 855         if (!dual_qdepth_disable)
 856                 drv_ops->mfi_capabilities.support_ext_queue_depth = 1;
 857
 858         drv_ops->mfi_capabilities.support_qd_throttling = 1;
 859         drv_ops->mfi_capabilities.support_pd_map_target_id = 1;
 860         /* Convert capability to LE32 */
 861         cpu_to_le32s((u32 *)&init_frame->driver_operations.mfi_capabilities);
 862
 863         sys_info = dmi_get_system_info(DMI_PRODUCT_UUID);
 864         if (instance->system_info_buf && sys_info) {
 865                 memcpy(instance->system_info_buf->systemId, sys_info,
 866                         strlen(sys_info) > 64 ? 64 : strlen(sys_info));
 867                 instance->system_info_buf->systemIdLength =
 868                         strlen(sys_info) > 64 ? 64 : strlen(sys_info);
 869                 init_frame->system_info_lo = instance->system_info_h;
 870                 init_frame->system_info_hi = 0;
 871         }
 872
 873         init_frame->queue_info_new_phys_addr_hi =
 874                 cpu_to_le32(upper_32_bits(ioc_init_handle));
 875         init_frame->queue_info_new_phys_addr_lo =
 876                 cpu_to_le32(lower_32_bits(ioc_init_handle));
 877         init_frame->data_xfer_len = cpu_to_le32(sizeof(struct MPI2_IOC_INIT_REQUEST));
 878
 879         req_desc.u.low = cpu_to_le32(lower_32_bits(cmd->frame_phys_addr));
 880         req_desc.u.high = cpu_to_le32(upper_32_bits(cmd->frame_phys_addr));
 881         req_desc.MFAIo.RequestFlags =
 882                 (MEGASAS_REQ_DESCRIPT_FLAGS_MFA <<
 883                 MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
 884
 885         /*
 886          * disable the intr before firing the init frame
 887          */
 888         instance->instancet->disable_intr(instance);
 889
 890         for (i = 0; i < (10 * 1000); i += 20) {
 891                 if (readl(&instance->reg_set->doorbell) & 1)
 892                         msleep(20);
 893                 else
 894                         break;
 895         }
 896
 897         /* For Ventura also IOC INIT required 64 bit Descriptor write. */
 898         spin_lock_irqsave(&instance->hba_lock, flags);
 899         writel(le32_to_cpu(req_desc.u.low),
 900                &instance->reg_set->inbound_low_queue_port);
 901         writel(le32_to_cpu(req_desc.u.high),
 902                &instance->reg_set->inbound_high_queue_port);
 903         mmiowb();
 904         spin_unlock_irqrestore(&instance->hba_lock, flags);
 905
 906         wait_and_poll(instance, cmd, MFI_POLL_TIMEOUT_SECS);
 907
 908         frame_hdr = &cmd->frame->hdr;
 909         if (frame_hdr->cmd_status != 0) {
 910                 ret = 1;
 911                 goto fail_fw_init;
 912         }
 913         dev_info(&instance->pdev->dev, "Init cmd success\n");
 914
 915         ret = 0;
 916
 917 fail_fw_init:
 918         megasas_return_cmd(instance, cmd);
 919         if (IOCInitMessage)
 920                 dma_free_coherent(&instance->pdev->dev,
 921                                   sizeof(struct MPI2_IOC_INIT_REQUEST),
 922                                   IOCInitMessage, ioc_init_handle);
 923 fail_get_cmd:
 924         return ret;
 925 }
 926
 927 /**
 928  * megasas_sync_pd_seq_num -    JBOD SEQ MAP
 929  * @instance:           Adapter soft state
 930  * @pend:               set to 1, if it is pended jbod map.
 931  *
 932  * Issue Jbod map to the firmware. If it is pended command,
 933  * issue command and return. If it is first instance of jbod map
 934  * issue and receive command.
 935  */
 936 int
 937 megasas_sync_pd_seq_num(struct megasas_instance *instance, bool pend) {
 938         int ret = 0;
 939         u32 pd_seq_map_sz;
 940         struct megasas_cmd *cmd;
 941         struct megasas_dcmd_frame *dcmd;
 942         struct fusion_context *fusion = instance->ctrl_context;
 943         struct MR_PD_CFG_SEQ_NUM_SYNC *pd_sync;
 944         dma_addr_t pd_seq_h;
 945
 946         pd_sync = (void *)fusion->pd_seq_sync[(instance->pd_seq_map_id & 1)];
 947         pd_seq_h = fusion->pd_seq_phys[(instance->pd_seq_map_id & 1)];
 948         pd_seq_map_sz = sizeof(struct MR_PD_CFG_SEQ_NUM_SYNC) +
 949                         (sizeof(struct MR_PD_CFG_SEQ) *
 950                         (MAX_PHYSICAL_DEVICES - 1));
 951
 952         cmd = megasas_get_cmd(instance);
 953         if (!cmd) {
 954                 dev_err(&instance->pdev->dev,
 955                         "Could not get mfi cmd. Fail from %s %d\n",
 956                         __func__, __LINE__);
 957                 return -ENOMEM;
 958         }
 959
 960         dcmd = &cmd->frame->dcmd;
 961
 962         memset(pd_sync, 0, pd_seq_map_sz);
 963         memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
 964         dcmd->cmd = MFI_CMD_DCMD;
 965         dcmd->cmd_status = 0xFF;
 966         dcmd->sge_count = 1;
 967         dcmd->timeout = 0;
 968         dcmd->pad_0 = 0;
 969         dcmd->data_xfer_len = cpu_to_le32(pd_seq_map_sz);
 970         dcmd->opcode = cpu_to_le32(MR_DCMD_SYSTEM_PD_MAP_GET_INFO);
 971         dcmd->sgl.sge32[0].phys_addr = cpu_to_le32(pd_seq_h);
 972         dcmd->sgl.sge32[0].length = cpu_to_le32(pd_seq_map_sz);
 973
 974         if (pend) {
 975                 dcmd->mbox.b[0] = MEGASAS_DCMD_MBOX_PEND_FLAG;
 976                 dcmd->flags = cpu_to_le16(MFI_FRAME_DIR_WRITE);
 977                 instance->jbod_seq_cmd = cmd;
 978                 instance->instancet->issue_dcmd(instance, cmd);
 979                 return 0;
 980         }
 981
 982         dcmd->flags = cpu_to_le16(MFI_FRAME_DIR_READ);
 983
 984         /* Below code is only for non pended DCMD */
 985         if (instance->ctrl_context && !instance->mask_interrupts)
 986                 ret = megasas_issue_blocked_cmd(instance, cmd,
 987                         MFI_IO_TIMEOUT_SECS);
 988         else
 989                 ret = megasas_issue_polled(instance, cmd);
 990
 991         if (le32_to_cpu(pd_sync->count) > MAX_PHYSICAL_DEVICES) {
 992                 dev_warn(&instance->pdev->dev,
 993                         "driver supports max %d JBOD, but FW reports %d\n",
 994                         MAX_PHYSICAL_DEVICES, le32_to_cpu(pd_sync->count));
 995                 ret = -EINVAL;
 996         }
 997
 998         if (ret == DCMD_TIMEOUT && instance->ctrl_context)
 999                 megaraid_sas_kill_hba(instance);
1000
1001         if (ret == DCMD_SUCCESS)
1002                 instance->pd_seq_map_id++;
1003
1004         megasas_return_cmd(instance, cmd);
1005         return ret;
1006 }
1007
1008 /*
1009  * megasas_get_ld_map_info -    Returns FW's ld_map structure
1010  * @instance:                           Adapter soft state
1011  * @pend:                               Pend the command or not
1012  * Issues an internal command (DCMD) to get the FW's controller PD
1013  * list structure.  This information is mainly used to find out SYSTEM
1014  * supported by the FW.
1015  * dcmd.mbox value setting for MR_DCMD_LD_MAP_GET_INFO
1016  * dcmd.mbox.b[0]       - number of LDs being sync'd
1017  * dcmd.mbox.b[1]       - 0 - complete command immediately.
1018  *                      - 1 - pend till config change
1019  * dcmd.mbox.b[2]       - 0 - supports max 64 lds and uses legacy MR_FW_RAID_MAP
1020  *                      - 1 - supports max MAX_LOGICAL_DRIVES_EXT lds and
1021  *                              uses extended struct MR_FW_RAID_MAP_EXT
1022  */
1023 static int
1024 megasas_get_ld_map_info(struct megasas_instance *instance)
1025 {
1026         int ret = 0;
1027         struct megasas_cmd *cmd;
1028         struct megasas_dcmd_frame *dcmd;
1029         void *ci;
1030         dma_addr_t ci_h = 0;
1031         u32 size_map_info;
1032         struct fusion_context *fusion;
1033
1034         cmd = megasas_get_cmd(instance);
1035
1036         if (!cmd) {
1037                 dev_printk(KERN_DEBUG, &instance->pdev->dev, "Failed to get cmd for map info\n");
1038                 return -ENOMEM;
1039         }
1040
1041         fusion = instance->ctrl_context;
1042
1043         if (!fusion) {
1044                 megasas_return_cmd(instance, cmd);
1045                 return -ENXIO;
1046         }
1047
1048         dcmd = &cmd->frame->dcmd;
1049
1050         size_map_info = fusion->current_map_sz;
1051
1052         ci = (void *) fusion->ld_map[(instance->map_id & 1)];
1053         ci_h = fusion->ld_map_phys[(instance->map_id & 1)];
1054
1055         if (!ci) {
1056                 dev_printk(KERN_DEBUG, &instance->pdev->dev, "Failed to alloc mem for ld_map_info\n");
1057                 megasas_return_cmd(instance, cmd);
1058                 return -ENOMEM;
1059         }
1060
1061         memset(ci, 0, fusion->max_map_sz);
1062         memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
1063         dcmd->cmd = MFI_CMD_DCMD;
1064         dcmd->cmd_status = 0xFF;
1065         dcmd->sge_count = 1;
1066         dcmd->flags = cpu_to_le16(MFI_FRAME_DIR_READ);
1067         dcmd->timeout = 0;
1068         dcmd->pad_0 = 0;
1069         dcmd->data_xfer_len = cpu_to_le32(size_map_info);
1070         dcmd->opcode = cpu_to_le32(MR_DCMD_LD_MAP_GET_INFO);
1071         dcmd->sgl.sge32[0].phys_addr = cpu_to_le32(ci_h);
1072         dcmd->sgl.sge32[0].length = cpu_to_le32(size_map_info);
1073
1074         if (instance->ctrl_context && !instance->mask_interrupts)
1075                 ret = megasas_issue_blocked_cmd(instance, cmd,
1076                         MFI_IO_TIMEOUT_SECS);
1077         else
1078                 ret = megasas_issue_polled(instance, cmd);
1079
1080         if (ret == DCMD_TIMEOUT && instance->ctrl_context)
1081                 megaraid_sas_kill_hba(instance);
1082
1083         megasas_return_cmd(instance, cmd);
1084
1085         return ret;
1086 }
1087
1088 u8
1089 megasas_get_map_info(struct megasas_instance *instance)
1090 {
1091         struct fusion_context *fusion = instance->ctrl_context;
1092
1093         fusion->fast_path_io = 0;
1094         if (!megasas_get_ld_map_info(instance)) {
1095                 if (MR_ValidateMapInfo(instance)) {
1096                         fusion->fast_path_io = 1;
1097                         return 0;
1098                 }
1099         }
1100         return 1;
1101 }
1102
1103 /*
1104  * megasas_sync_map_info -      Returns FW's ld_map structure
1105  * @instance:                           Adapter soft state
1106  *
1107  * Issues an internal command (DCMD) to get the FW's controller PD
1108  * list structure.  This information is mainly used to find out SYSTEM
1109  * supported by the FW.
1110  */
1111 int
1112 megasas_sync_map_info(struct megasas_instance *instance)
1113 {
1114         int i;
1115         struct megasas_cmd *cmd;
1116         struct megasas_dcmd_frame *dcmd;
1117         u16 num_lds;
1118         u32 size_sync_info;
1119         struct fusion_context *fusion;
1120         struct MR_LD_TARGET_SYNC *ci = NULL;
1121         struct MR_DRV_RAID_MAP_ALL *map;
1122         struct MR_LD_RAID  *raid;
1123         struct MR_LD_TARGET_SYNC *ld_sync;
1124         dma_addr_t ci_h = 0;
1125         u32 size_map_info;
1126
1127         cmd = megasas_get_cmd(instance);
1128
1129         if (!cmd) {
1130                 dev_printk(KERN_DEBUG, &instance->pdev->dev, "Failed to get cmd for sync info\n");
1131                 return -ENOMEM;
1132         }
1133
1134         fusion = instance->ctrl_context;
1135
1136         if (!fusion) {
1137                 megasas_return_cmd(instance, cmd);
1138                 return 1;
1139         }
1140
1141         map = fusion->ld_drv_map[instance->map_id & 1];
1142
1143         num_lds = le16_to_cpu(map->raidMap.ldCount);
1144
1145         dcmd = &cmd->frame->dcmd;
1146
1147         size_sync_info = sizeof(struct MR_LD_TARGET_SYNC) *num_lds;
1148
1149         memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
1150
1151         ci = (struct MR_LD_TARGET_SYNC *)
1152           fusion->ld_map[(instance->map_id - 1) & 1];
1153         memset(ci, 0, fusion->max_map_sz);
1154
1155         ci_h = fusion->ld_map_phys[(instance->map_id - 1) & 1];
1156
1157         ld_sync = (struct MR_LD_TARGET_SYNC *)ci;
1158
1159         for (i = 0; i < num_lds; i++, ld_sync++) {
1160                 raid = MR_LdRaidGet(i, map);
1161                 ld_sync->targetId = MR_GetLDTgtId(i, map);
1162                 ld_sync->seqNum = raid->seqNum;
1163         }
1164
1165         size_map_info = fusion->current_map_sz;
1166
1167         dcmd->cmd = MFI_CMD_DCMD;
1168         dcmd->cmd_status = 0xFF;
1169         dcmd->sge_count = 1;
1170         dcmd->flags = cpu_to_le16(MFI_FRAME_DIR_WRITE);
1171         dcmd->timeout = 0;
1172         dcmd->pad_0 = 0;
1173         dcmd->data_xfer_len = cpu_to_le32(size_map_info);
1174         dcmd->mbox.b[0] = num_lds;
1175         dcmd->mbox.b[1] = MEGASAS_DCMD_MBOX_PEND_FLAG;
1176         dcmd->opcode = cpu_to_le32(MR_DCMD_LD_MAP_GET_INFO);
1177         dcmd->sgl.sge32[0].phys_addr = cpu_to_le32(ci_h);
1178         dcmd->sgl.sge32[0].length = cpu_to_le32(size_map_info);
1179
1180         instance->map_update_cmd = cmd;
1181
1182         instance->instancet->issue_dcmd(instance, cmd);
1183
1184         return 0;
1185 }
1186
1187 /*
1188  * meagasas_display_intel_branding - Display branding string
1189  * @instance: per adapter object
1190  *
1191  * Return nothing.
1192  */
1193 static void
1194 megasas_display_intel_branding(struct megasas_instance *instance)
1195 {
1196         if (instance->pdev->subsystem_vendor != PCI_VENDOR_ID_INTEL)
1197                 return;
1198
1199         switch (instance->pdev->device) {
1200         case PCI_DEVICE_ID_LSI_INVADER:
1201                 switch (instance->pdev->subsystem_device) {
1202                 case MEGARAID_INTEL_RS3DC080_SSDID:
1203                         dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1204                                 instance->host->host_no,
1205                                 MEGARAID_INTEL_RS3DC080_BRANDING);
1206                         break;
1207                 case MEGARAID_INTEL_RS3DC040_SSDID:
1208                         dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1209                                 instance->host->host_no,
1210                                 MEGARAID_INTEL_RS3DC040_BRANDING);
1211                         break;
1212                 case MEGARAID_INTEL_RS3SC008_SSDID:
1213                         dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1214                                 instance->host->host_no,
1215                                 MEGARAID_INTEL_RS3SC008_BRANDING);
1216                         break;
1217                 case MEGARAID_INTEL_RS3MC044_SSDID:
1218                         dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1219                                 instance->host->host_no,
1220                                 MEGARAID_INTEL_RS3MC044_BRANDING);
1221                         break;
1222                 default:
1223                         break;
1224                 }
1225                 break;
1226         case PCI_DEVICE_ID_LSI_FURY:
1227                 switch (instance->pdev->subsystem_device) {
1228                 case MEGARAID_INTEL_RS3WC080_SSDID:
1229                         dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1230                                 instance->host->host_no,
1231                                 MEGARAID_INTEL_RS3WC080_BRANDING);
1232                         break;
1233                 case MEGARAID_INTEL_RS3WC040_SSDID:
1234                         dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1235                                 instance->host->host_no,
1236                                 MEGARAID_INTEL_RS3WC040_BRANDING);
1237                         break;
1238                 default:
1239                         break;
1240                 }
1241                 break;
1242         case PCI_DEVICE_ID_LSI_CUTLASS_52:
1243         case PCI_DEVICE_ID_LSI_CUTLASS_53:
1244                 switch (instance->pdev->subsystem_device) {
1245                 case MEGARAID_INTEL_RMS3BC160_SSDID:
1246                         dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1247                                 instance->host->host_no,
1248                                 MEGARAID_INTEL_RMS3BC160_BRANDING);
1249                         break;
1250                 default:
1251                         break;
1252                 }
1253                 break;
1254         default:
1255                 break;
1256         }
1257 }
1258
1259 /**
1260  * megasas_init_adapter_fusion -        Initializes the FW
1261  * @instance:           Adapter soft state
1262  *
1263  * This is the main function for initializing firmware.
1264  */
1265 u32
1266 megasas_init_adapter_fusion(struct megasas_instance *instance)
1267 {
1268         struct megasas_register_set __iomem *reg_set;
1269         struct fusion_context *fusion;
1270         u16 max_cmd;
1271         u32 scratch_pad_2;
1272         int i = 0, count;
1273
1274         fusion = instance->ctrl_context;
1275
1276         reg_set = instance->reg_set;
1277
1278         megasas_fusion_update_can_queue(instance, PROBE_CONTEXT);
1279
1280         /*
1281          * Only Driver's internal DCMDs and IOCTL DCMDs needs to have MFI frames
1282          */
1283         instance->max_mfi_cmds =
1284                 MEGASAS_FUSION_INTERNAL_CMDS + MEGASAS_FUSION_IOCTL_CMDS;
1285
1286         max_cmd = instance->max_fw_cmds;
1287
1288         fusion->reply_q_depth = 2 * (((max_cmd + 1 + 15)/16)*16);
1289
1290         fusion->request_alloc_sz =
1291         sizeof(union MEGASAS_REQUEST_DESCRIPTOR_UNION) * instance->max_mpt_cmds;
1292         fusion->reply_alloc_sz = sizeof(union MPI2_REPLY_DESCRIPTORS_UNION)
1293                 *(fusion->reply_q_depth);
1294         fusion->io_frames_alloc_sz = MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE +
1295                 (MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE
1296                 * (instance->max_mpt_cmds + 1)); /* Extra 1 for SMID 0 */
1297
1298         scratch_pad_2 = readl(&instance->reg_set->outbound_scratch_pad_2);
1299         /* If scratch_pad_2 & MEGASAS_MAX_CHAIN_SIZE_UNITS_MASK is set,
1300          * Firmware support extended IO chain frame which is 4 times more than
1301          * legacy Firmware.
1302          * Legacy Firmware - Frame size is (8 * 128) = 1K
1303          * 1M IO Firmware  - Frame size is (8 * 128 * 4)  = 4K
1304          */
1305         if (scratch_pad_2 & MEGASAS_MAX_CHAIN_SIZE_UNITS_MASK)
1306                 instance->max_chain_frame_sz =
1307                         ((scratch_pad_2 & MEGASAS_MAX_CHAIN_SIZE_MASK) >>
1308                         MEGASAS_MAX_CHAIN_SHIFT) * MEGASAS_1MB_IO;
1309         else
1310                 instance->max_chain_frame_sz =
1311                         ((scratch_pad_2 & MEGASAS_MAX_CHAIN_SIZE_MASK) >>
1312                         MEGASAS_MAX_CHAIN_SHIFT) * MEGASAS_256K_IO;
1313
1314         if (instance->max_chain_frame_sz < MEGASAS_CHAIN_FRAME_SZ_MIN) {
1315                 dev_warn(&instance->pdev->dev, "frame size %d invalid, fall back to legacy max frame size %d\n",
1316                         instance->max_chain_frame_sz,
1317                         MEGASAS_CHAIN_FRAME_SZ_MIN);
1318                 instance->max_chain_frame_sz = MEGASAS_CHAIN_FRAME_SZ_MIN;
1319         }
1320
1321         fusion->max_sge_in_main_msg =
1322                 (MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE
1323                         - offsetof(struct MPI2_RAID_SCSI_IO_REQUEST, SGL))/16;
1324
1325         fusion->max_sge_in_chain =
1326                 instance->max_chain_frame_sz
1327                         / sizeof(union MPI2_SGE_IO_UNION);
1328
1329         instance->max_num_sge =
1330                 rounddown_pow_of_two(fusion->max_sge_in_main_msg
1331                         + fusion->max_sge_in_chain - 2);
1332
1333         /* Used for pass thru MFI frame (DCMD) */
1334         fusion->chain_offset_mfi_pthru =
1335                 offsetof(struct MPI2_RAID_SCSI_IO_REQUEST, SGL)/16;
1336
1337         fusion->chain_offset_io_request =
1338                 (MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE -
1339                  sizeof(union MPI2_SGE_IO_UNION))/16;
1340
1341         count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
1342         for (i = 0 ; i < count; i++)
1343                 fusion->last_reply_idx[i] = 0;
1344
1345         /*
1346          * For fusion adapters, 3 commands for IOCTL and 8 commands
1347          * for driver's internal DCMDs.
1348          */
1349         instance->max_scsi_cmds = instance->max_fw_cmds -
1350                                 (MEGASAS_FUSION_INTERNAL_CMDS +
1351                                 MEGASAS_FUSION_IOCTL_CMDS);
1352         sema_init(&instance->ioctl_sem, MEGASAS_FUSION_IOCTL_CMDS);
1353
1354         /*
1355          * Allocate memory for descriptors
1356          * Create a pool of commands
1357          */
1358         if (megasas_alloc_cmds(instance))
1359                 goto fail_alloc_mfi_cmds;
1360         if (megasas_alloc_cmds_fusion(instance))
1361                 goto fail_alloc_cmds;
1362
1363         if (megasas_ioc_init_fusion(instance))
1364                 goto fail_ioc_init;
1365
1366         megasas_display_intel_branding(instance);
1367         if (megasas_get_ctrl_info(instance)) {
1368                 dev_err(&instance->pdev->dev,
1369                         "Could not get controller info. Fail from %s %d\n",
1370                         __func__, __LINE__);
1371                 goto fail_ioc_init;
1372         }
1373
1374         instance->flag_ieee = 1;
1375         instance->r1_ldio_hint_default =  MR_R1_LDIO_PIGGYBACK_DEFAULT;
1376         fusion->fast_path_io = 0;
1377
1378         fusion->drv_map_pages = get_order(fusion->drv_map_sz);
1379         for (i = 0; i < 2; i++) {
1380                 fusion->ld_map[i] = NULL;
1381                 fusion->ld_drv_map[i] = (void *)__get_free_pages(GFP_KERNEL,
1382                         fusion->drv_map_pages);
1383                 if (!fusion->ld_drv_map[i]) {
1384                         dev_err(&instance->pdev->dev, "Could not allocate "
1385                                 "memory for local map info for %d pages\n",
1386                                 fusion->drv_map_pages);
1387                         if (i == 1)
1388                                 free_pages((ulong)fusion->ld_drv_map[0],
1389                                         fusion->drv_map_pages);
1390                         goto fail_ioc_init;
1391                 }
1392                 memset(fusion->ld_drv_map[i], 0,
1393                         ((1 << PAGE_SHIFT) << fusion->drv_map_pages));
1394         }
1395
1396         for (i = 0; i < 2; i++) {
1397                 fusion->ld_map[i] = dma_alloc_coherent(&instance->pdev->dev,
1398                                                        fusion->max_map_sz,
1399                                                        &fusion->ld_map_phys[i],
1400                                                        GFP_KERNEL);
1401                 if (!fusion->ld_map[i]) {
1402                         dev_err(&instance->pdev->dev, "Could not allocate memory "
1403                                "for map info\n");
1404                         goto fail_map_info;
1405                 }
1406         }
1407
1408         if (!megasas_get_map_info(instance))
1409                 megasas_sync_map_info(instance);
1410
1411         return 0;
1412
1413 fail_map_info:
1414         if (i == 1)
1415                 dma_free_coherent(&instance->pdev->dev, fusion->max_map_sz,
1416                                   fusion->ld_map[0], fusion->ld_map_phys[0]);
1417 fail_ioc_init:
1418         megasas_free_cmds_fusion(instance);
1419 fail_alloc_cmds:
1420         megasas_free_cmds(instance);
1421 fail_alloc_mfi_cmds:
1422         return 1;
1423 }
1424
1425 /**
1426  * map_cmd_status -     Maps FW cmd status to OS cmd status
1427  * @cmd :               Pointer to cmd
1428  * @status :            status of cmd returned by FW
1429  * @ext_status :        ext status of cmd returned by FW
1430  */
1431
1432 void
1433 map_cmd_status(struct fusion_context *fusion,
1434                 struct scsi_cmnd *scmd, u8 status, u8 ext_status,
1435                 u32 data_length, u8 *sense)
1436 {
1437         u8 cmd_type;
1438         int resid;
1439
1440         cmd_type = megasas_cmd_type(scmd);
1441         switch (status) {
1442
1443         case MFI_STAT_OK:
1444                 scmd->result = DID_OK << 16;
1445                 break;
1446
1447         case MFI_STAT_SCSI_IO_FAILED:
1448         case MFI_STAT_LD_INIT_IN_PROGRESS:
1449                 scmd->result = (DID_ERROR << 16) | ext_status;
1450                 break;
1451
1452         case MFI_STAT_SCSI_DONE_WITH_ERROR:
1453
1454                 scmd->result = (DID_OK << 16) | ext_status;
1455                 if (ext_status == SAM_STAT_CHECK_CONDITION) {
1456                         memset(scmd->sense_buffer, 0,
1457                                SCSI_SENSE_BUFFERSIZE);
1458                         memcpy(scmd->sense_buffer, sense,
1459                                SCSI_SENSE_BUFFERSIZE);
1460                         scmd->result |= DRIVER_SENSE << 24;
1461                 }
1462
1463                 /*
1464                  * If the  IO request is partially completed, then MR FW will
1465                  * update "io_request->DataLength" field with actual number of
1466                  * bytes transferred.Driver will set residual bytes count in
1467                  * SCSI command structure.
1468                  */
1469                 resid = (scsi_bufflen(scmd) - data_length);
1470                 scsi_set_resid(scmd, resid);
1471
1472                 if (resid &&
1473                         ((cmd_type == READ_WRITE_LDIO) ||
1474                         (cmd_type == READ_WRITE_SYSPDIO)))
1475                         scmd_printk(KERN_INFO, scmd, "BRCM Debug mfi stat 0x%x, data len"
1476                                 " requested/completed 0x%x/0x%x\n",
1477                                 status, scsi_bufflen(scmd), data_length);
1478                 break;
1479
1480         case MFI_STAT_LD_OFFLINE:
1481         case MFI_STAT_DEVICE_NOT_FOUND:
1482                 scmd->result = DID_BAD_TARGET << 16;
1483                 break;
1484         case MFI_STAT_CONFIG_SEQ_MISMATCH:
1485                 scmd->result = DID_IMM_RETRY << 16;
1486                 break;
1487         default:
1488                 scmd->result = DID_ERROR << 16;
1489                 break;
1490         }
1491 }
1492
1493 /**
1494  * megasas_is_prp_possible -
1495  * Checks if native NVMe PRPs can be built for the IO
1496  *
1497  * @instance:           Adapter soft state
1498  * @scmd:               SCSI command from the mid-layer
1499  * @sge_count:          scatter gather element count.
1500  *
1501  * Returns:             true: PRPs can be built
1502  *                      false: IEEE SGLs needs to be built
1503  */
1504 static bool
1505 megasas_is_prp_possible(struct megasas_instance *instance,
1506                         struct scsi_cmnd *scmd, int sge_count)
1507 {
1508         struct fusion_context *fusion;
1509         int i;
1510         u32 data_length = 0;
1511         struct scatterlist *sg_scmd;
1512         bool build_prp = false;
1513         u32 mr_nvme_pg_size;
1514
1515         mr_nvme_pg_size = max_t(u32, instance->nvme_page_size,
1516                                 MR_DEFAULT_NVME_PAGE_SIZE);
1517         fusion = instance->ctrl_context;
1518         data_length = scsi_bufflen(scmd);
1519         sg_scmd = scsi_sglist(scmd);
1520
1521         /*
1522          * NVMe uses one PRP for each page (or part of a page)
1523          * look at the data length - if 4 pages or less then IEEE is OK
1524          * if  > 5 pages then we need to build a native SGL
1525          * if > 4 and <= 5 pages, then check physical address of 1st SG entry
1526          * if this first size in the page is >= the residual beyond 4 pages
1527          * then use IEEE, otherwise use native SGL
1528          */
1529
1530         if (data_length > (mr_nvme_pg_size * 5)) {
1531                 build_prp = true;
1532         } else if ((data_length > (mr_nvme_pg_size * 4)) &&
1533                         (data_length <= (mr_nvme_pg_size * 5)))  {
1534                 /* check if 1st SG entry size is < residual beyond 4 pages */
1535                 if (sg_dma_len(sg_scmd) < (data_length - (mr_nvme_pg_size * 4)))
1536                         build_prp = true;
1537         }
1538
1539 /*
1540  * Below code detects gaps/holes in IO data buffers.
1541  * What does holes/gaps mean?
1542  * Any SGE except first one in a SGL starts at non NVME page size
1543  * aligned address OR Any SGE except last one in a SGL ends at
1544  * non NVME page size boundary.
1545  *
1546  * Driver has already informed block layer by setting boundary rules for
1547  * bio merging done at NVME page size boundary calling kernel API
1548  * blk_queue_virt_boundary inside slave_config.
1549  * Still there is possibility of IO coming with holes to driver because of
1550  * IO merging done by IO scheduler.
1551  *
1552  * With SCSI BLK MQ enabled, there will be no IO with holes as there is no
1553  * IO scheduling so no IO merging.
1554  *
1555  * With SCSI BLK MQ disabled, IO scheduler may attempt to merge IOs and
1556  * then sending IOs with holes.
1557  *
1558  * Though driver can request block layer to disable IO merging by calling-
1559  * queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, sdev->request_queue) but
1560  * user may tune sysfs parameter- nomerges again to 0 or 1.
1561  *
1562  * If in future IO scheduling is enabled with SCSI BLK MQ,
1563  * this algorithm to detect holes will be required in driver
1564  * for SCSI BLK MQ enabled case as well.
1565  *
1566  *
1567  */
1568         scsi_for_each_sg(scmd, sg_scmd, sge_count, i) {
1569                 if ((i != 0) && (i != (sge_count - 1))) {
1570                         if (mega_mod64(sg_dma_len(sg_scmd), mr_nvme_pg_size) ||
1571                             mega_mod64(sg_dma_address(sg_scmd),
1572                                        mr_nvme_pg_size)) {
1573                                 build_prp = false;
1574                                 atomic_inc(&instance->sge_holes_type1);
1575                                 break;
1576                         }
1577                 }
1578
1579                 if ((sge_count > 1) && (i == 0)) {
1580                         if ((mega_mod64((sg_dma_address(sg_scmd) +
1581                                         sg_dma_len(sg_scmd)),
1582                                         mr_nvme_pg_size))) {
1583                                 build_prp = false;
1584                                 atomic_inc(&instance->sge_holes_type2);
1585                                 break;
1586                         }
1587                 }
1588
1589                 if ((sge_count > 1) && (i == (sge_count - 1))) {
1590                         if (mega_mod64(sg_dma_address(sg_scmd),
1591                                        mr_nvme_pg_size)) {
1592                                 build_prp = false;
1593                                 atomic_inc(&instance->sge_holes_type3);
1594                                 break;
1595                         }
1596                 }
1597         }
1598
1599         return build_prp;
1600 }
1601
1602 /**
1603  * megasas_make_prp_nvme -
1604  * Prepare PRPs(Physical Region Page)- SGLs specific to NVMe drives only
1605  *
1606  * @instance:           Adapter soft state
1607  * @scmd:               SCSI command from the mid-layer
1608  * @sgl_ptr:            SGL to be filled in
1609  * @cmd:                Fusion command frame
1610  * @sge_count:          scatter gather element count.
1611  *
1612  * Returns:             true: PRPs are built
1613  *                      false: IEEE SGLs needs to be built
1614  */
1615 static bool
1616 megasas_make_prp_nvme(struct megasas_instance *instance, struct scsi_cmnd *scmd,
1617                       struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr,
1618                       struct megasas_cmd_fusion *cmd, int sge_count)
1619 {
1620         int sge_len, offset, num_prp_in_chain = 0;
1621         struct MPI25_IEEE_SGE_CHAIN64 *main_chain_element, *ptr_first_sgl;
1622         u64 *ptr_sgl;
1623         dma_addr_t ptr_sgl_phys;
1624         u64 sge_addr;
1625         u32 page_mask, page_mask_result;
1626         struct scatterlist *sg_scmd;
1627         u32 first_prp_len;
1628         bool build_prp = false;
1629         int data_len = scsi_bufflen(scmd);
1630         struct fusion_context *fusion;
1631         u32 mr_nvme_pg_size = max_t(u32, instance->nvme_page_size,
1632                                         MR_DEFAULT_NVME_PAGE_SIZE);
1633
1634         fusion = instance->ctrl_context;
1635
1636         build_prp = megasas_is_prp_possible(instance, scmd, sge_count);
1637
1638         if (!build_prp)
1639                 return false;
1640
1641         /*
1642          * Nvme has a very convoluted prp format.  One prp is required
1643          * for each page or partial page. Driver need to split up OS sg_list
1644          * entries if it is longer than one page or cross a page
1645          * boundary.  Driver also have to insert a PRP list pointer entry as
1646          * the last entry in each physical page of the PRP list.
1647          *
1648          * NOTE: The first PRP "entry" is actually placed in the first
1649          * SGL entry in the main message as IEEE 64 format.  The 2nd
1650          * entry in the main message is the chain element, and the rest
1651          * of the PRP entries are built in the contiguous pcie buffer.
1652          */
1653         page_mask = mr_nvme_pg_size - 1;
1654         ptr_sgl = (u64 *)cmd->sg_frame;
1655         ptr_sgl_phys = cmd->sg_frame_phys_addr;
1656         memset(ptr_sgl, 0, instance->max_chain_frame_sz);
1657
1658         /* Build chain frame element which holds all prps except first*/
1659         main_chain_element = (struct MPI25_IEEE_SGE_CHAIN64 *)
1660             ((u8 *)sgl_ptr + sizeof(struct MPI25_IEEE_SGE_CHAIN64));
1661
1662         main_chain_element->Address = cpu_to_le64(ptr_sgl_phys);
1663         main_chain_element->NextChainOffset = 0;
1664         main_chain_element->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT |
1665                                         IEEE_SGE_FLAGS_SYSTEM_ADDR |
1666                                         MPI26_IEEE_SGE_FLAGS_NSF_NVME_PRP;
1667
1668         /* Build first prp, sge need not to be page aligned*/
1669         ptr_first_sgl = sgl_ptr;
1670         sg_scmd = scsi_sglist(scmd);
1671         sge_addr = sg_dma_address(sg_scmd);
1672         sge_len = sg_dma_len(sg_scmd);
1673
1674         offset = (u32)(sge_addr & page_mask);
1675         first_prp_len = mr_nvme_pg_size - offset;
1676
1677         ptr_first_sgl->Address = cpu_to_le64(sge_addr);
1678         ptr_first_sgl->Length = cpu_to_le32(first_prp_len);
1679
1680         data_len -= first_prp_len;
1681
1682         if (sge_len > first_prp_len) {
1683                 sge_addr += first_prp_len;
1684                 sge_len -= first_prp_len;
1685         } else if (sge_len == first_prp_len) {
1686                 sg_scmd = sg_next(sg_scmd);
1687                 sge_addr = sg_dma_address(sg_scmd);
1688                 sge_len = sg_dma_len(sg_scmd);
1689         }
1690
1691         for (;;) {
1692                 offset = (u32)(sge_addr & page_mask);
1693
1694                 /* Put PRP pointer due to page boundary*/
1695                 page_mask_result = (uintptr_t)(ptr_sgl + 1) & page_mask;
1696                 if (unlikely(!page_mask_result)) {
1697                         scmd_printk(KERN_NOTICE,
1698                                     scmd, "page boundary ptr_sgl: 0x%p\n",
1699                                     ptr_sgl);
1700                         ptr_sgl_phys += 8;
1701                         *ptr_sgl = cpu_to_le64(ptr_sgl_phys);
1702                         ptr_sgl++;
1703                         num_prp_in_chain++;
1704                 }
1705
1706                 *ptr_sgl = cpu_to_le64(sge_addr);
1707                 ptr_sgl++;
1708                 ptr_sgl_phys += 8;
1709                 num_prp_in_chain++;
1710
1711                 sge_addr += mr_nvme_pg_size;
1712                 sge_len -= mr_nvme_pg_size;
1713                 data_len -= mr_nvme_pg_size;
1714
1715                 if (data_len <= 0)
1716                         break;
1717
1718                 if (sge_len > 0)
1719                         continue;
1720
1721                 sg_scmd = sg_next(sg_scmd);
1722                 sge_addr = sg_dma_address(sg_scmd);
1723                 sge_len = sg_dma_len(sg_scmd);
1724         }
1725
1726         main_chain_element->Length =
1727                         cpu_to_le32(num_prp_in_chain * sizeof(u64));
1728
1729         atomic_inc(&instance->prp_sgl);
1730         return build_prp;
1731 }
1732
1733 /**
1734  * megasas_make_sgl_fusion -    Prepares 32-bit SGL
1735  * @instance:           Adapter soft state
1736  * @scp:                SCSI command from the mid-layer
1737  * @sgl_ptr:            SGL to be filled in
1738  * @cmd:                cmd we are working on
1739  * @sge_count           sge count
1740  *
1741  */
1742 static void
1743 megasas_make_sgl_fusion(struct megasas_instance *instance,
1744                         struct scsi_cmnd *scp,
1745                         struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr,
1746                         struct megasas_cmd_fusion *cmd, int sge_count)
1747 {
1748         int i, sg_processed;
1749         struct scatterlist *os_sgl;
1750         struct fusion_context *fusion;
1751
1752         fusion = instance->ctrl_context;
1753
1754         if (fusion->adapter_type >= INVADER_SERIES) {
1755                 struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr_end = sgl_ptr;
1756                 sgl_ptr_end += fusion->max_sge_in_main_msg - 1;
1757                 sgl_ptr_end->Flags = 0;
1758         }
1759
1760         scsi_for_each_sg(scp, os_sgl, sge_count, i) {
1761                 sgl_ptr->Length = cpu_to_le32(sg_dma_len(os_sgl));
1762                 sgl_ptr->Address = cpu_to_le64(sg_dma_address(os_sgl));
1763                 sgl_ptr->Flags = 0;
1764                 if (fusion->adapter_type >= INVADER_SERIES)
1765                         if (i == sge_count - 1)
1766                                 sgl_ptr->Flags = IEEE_SGE_FLAGS_END_OF_LIST;
1767                 sgl_ptr++;
1768                 sg_processed = i + 1;
1769
1770                 if ((sg_processed ==  (fusion->max_sge_in_main_msg - 1)) &&
1771                     (sge_count > fusion->max_sge_in_main_msg)) {
1772
1773                         struct MPI25_IEEE_SGE_CHAIN64 *sg_chain;
1774                         if (fusion->adapter_type >= INVADER_SERIES) {
1775                                 if ((le16_to_cpu(cmd->io_request->IoFlags) &
1776                                         MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH) !=
1777                                         MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH)
1778                                         cmd->io_request->ChainOffset =
1779                                                 fusion->
1780                                                 chain_offset_io_request;
1781                                 else
1782                                         cmd->io_request->ChainOffset = 0;
1783                         } else
1784                                 cmd->io_request->ChainOffset =
1785                                         fusion->chain_offset_io_request;
1786
1787                         sg_chain = sgl_ptr;
1788                         /* Prepare chain element */
1789                         sg_chain->NextChainOffset = 0;
1790                         if (fusion->adapter_type >= INVADER_SERIES)
1791                                 sg_chain->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT;
1792                         else
1793                                 sg_chain->Flags =
1794                                         (IEEE_SGE_FLAGS_CHAIN_ELEMENT |
1795                                          MPI2_IEEE_SGE_FLAGS_IOCPLBNTA_ADDR);
1796                         sg_chain->Length =  cpu_to_le32((sizeof(union MPI2_SGE_IO_UNION) * (sge_count - sg_processed)));
1797                         sg_chain->Address = cpu_to_le64(cmd->sg_frame_phys_addr);
1798
1799                         sgl_ptr =
1800                           (struct MPI25_IEEE_SGE_CHAIN64 *)cmd->sg_frame;
1801                         memset(sgl_ptr, 0, instance->max_chain_frame_sz);
1802                 }
1803         }
1804         atomic_inc(&instance->ieee_sgl);
1805 }
1806
1807 /**
1808  * megasas_make_sgl -   Build Scatter Gather List(SGLs)
1809  * @scp:                SCSI command pointer
1810  * @instance:           Soft instance of controller
1811  * @cmd:                Fusion command pointer
1812  *
1813  * This function will build sgls based on device type.
1814  * For nvme drives, there is different way of building sgls in nvme native
1815  * format- PRPs(Physical Region Page).
1816  *
1817  * Returns the number of sg lists actually used, zero if the sg lists
1818  * is NULL, or -ENOMEM if the mapping failed
1819  */
1820 static
1821 int megasas_make_sgl(struct megasas_instance *instance, struct scsi_cmnd *scp,
1822                      struct megasas_cmd_fusion *cmd)
1823 {
1824         int sge_count;
1825         bool build_prp = false;
1826         struct MPI25_IEEE_SGE_CHAIN64 *sgl_chain64;
1827
1828         sge_count = scsi_dma_map(scp);
1829
1830         if ((sge_count > instance->max_num_sge) || (sge_count <= 0))
1831                 return sge_count;
1832
1833         sgl_chain64 = (struct MPI25_IEEE_SGE_CHAIN64 *)&cmd->io_request->SGL;
1834         if ((le16_to_cpu(cmd->io_request->IoFlags) &
1835             MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH) &&
1836             (cmd->pd_interface == NVME_PD))
1837                 build_prp = megasas_make_prp_nvme(instance, scp, sgl_chain64,
1838                                                   cmd, sge_count);
1839
1840         if (!build_prp)
1841                 megasas_make_sgl_fusion(instance, scp, sgl_chain64,
1842                                         cmd, sge_count);
1843
1844         return sge_count;
1845 }
1846
1847 /**
1848  * megasas_set_pd_lba - Sets PD LBA
1849  * @cdb:                CDB
1850  * @cdb_len:            cdb length
1851  * @start_blk:          Start block of IO
1852  *
1853  * Used to set the PD LBA in CDB for FP IOs
1854  */
1855 void
1856 megasas_set_pd_lba(struct MPI2_RAID_SCSI_IO_REQUEST *io_request, u8 cdb_len,
1857                    struct IO_REQUEST_INFO *io_info, struct scsi_cmnd *scp,
1858                    struct MR_DRV_RAID_MAP_ALL *local_map_ptr, u32 ref_tag)
1859 {
1860         struct MR_LD_RAID *raid;
1861         u16 ld;
1862         u64 start_blk = io_info->pdBlock;
1863         u8 *cdb = io_request->CDB.CDB32;
1864         u32 num_blocks = io_info->numBlocks;
1865         u8 opcode = 0, flagvals = 0, groupnum = 0, control = 0;
1866
1867         /* Check if T10 PI (DIF) is enabled for this LD */
1868         ld = MR_TargetIdToLdGet(io_info->ldTgtId, local_map_ptr);
1869         raid = MR_LdRaidGet(ld, local_map_ptr);
1870         if (raid->capability.ldPiMode == MR_PROT_INFO_TYPE_CONTROLLER) {
1871                 memset(cdb, 0, sizeof(io_request->CDB.CDB32));
1872                 cdb[0] =  MEGASAS_SCSI_VARIABLE_LENGTH_CMD;
1873                 cdb[7] =  MEGASAS_SCSI_ADDL_CDB_LEN;
1874
1875                 if (scp->sc_data_direction == PCI_DMA_FROMDEVICE)
1876                         cdb[9] = MEGASAS_SCSI_SERVICE_ACTION_READ32;
1877                 else
1878                         cdb[9] = MEGASAS_SCSI_SERVICE_ACTION_WRITE32;
1879                 cdb[10] = MEGASAS_RD_WR_PROTECT_CHECK_ALL;
1880
1881                 /* LBA */
1882                 cdb[12] = (u8)((start_blk >> 56) & 0xff);
1883                 cdb[13] = (u8)((start_blk >> 48) & 0xff);
1884                 cdb[14] = (u8)((start_blk >> 40) & 0xff);
1885                 cdb[15] = (u8)((start_blk >> 32) & 0xff);
1886                 cdb[16] = (u8)((start_blk >> 24) & 0xff);
1887                 cdb[17] = (u8)((start_blk >> 16) & 0xff);
1888                 cdb[18] = (u8)((start_blk >> 8) & 0xff);
1889                 cdb[19] = (u8)(start_blk & 0xff);
1890
1891                 /* Logical block reference tag */
1892                 io_request->CDB.EEDP32.PrimaryReferenceTag =
1893                         cpu_to_be32(ref_tag);
1894                 io_request->CDB.EEDP32.PrimaryApplicationTagMask = cpu_to_be16(0xffff);
1895                 io_request->IoFlags = cpu_to_le16(32); /* Specify 32-byte cdb */
1896
1897                 /* Transfer length */
1898                 cdb[28] = (u8)((num_blocks >> 24) & 0xff);
1899                 cdb[29] = (u8)((num_blocks >> 16) & 0xff);
1900                 cdb[30] = (u8)((num_blocks >> 8) & 0xff);
1901                 cdb[31] = (u8)(num_blocks & 0xff);
1902
1903                 /* set SCSI IO EEDPFlags */
1904                 if (scp->sc_data_direction == PCI_DMA_FROMDEVICE) {
1905                         io_request->EEDPFlags = cpu_to_le16(
1906                                 MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG  |
1907                                 MPI2_SCSIIO_EEDPFLAGS_CHECK_REFTAG |
1908                                 MPI2_SCSIIO_EEDPFLAGS_CHECK_REMOVE_OP |
1909                                 MPI2_SCSIIO_EEDPFLAGS_CHECK_APPTAG |
1910                                 MPI25_SCSIIO_EEDPFLAGS_DO_NOT_DISABLE_MODE |
1911                                 MPI2_SCSIIO_EEDPFLAGS_CHECK_GUARD);
1912                 } else {
1913                         io_request->EEDPFlags = cpu_to_le16(
1914                                 MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG |
1915                                 MPI2_SCSIIO_EEDPFLAGS_INSERT_OP);
1916                 }
1917                 io_request->Control |= cpu_to_le32((0x4 << 26));
1918                 io_request->EEDPBlockSize = cpu_to_le32(scp->device->sector_size);
1919         } else {
1920                 /* Some drives don't support 16/12 byte CDB's, convert to 10 */
1921                 if (((cdb_len == 12) || (cdb_len == 16)) &&
1922                     (start_blk <= 0xffffffff)) {
1923                         if (cdb_len == 16) {
1924                                 opcode = cdb[0] == READ_16 ? READ_10 : WRITE_10;
1925                                 flagvals = cdb[1];
1926                                 groupnum = cdb[14];
1927                                 control = cdb[15];
1928                         } else {
1929                                 opcode = cdb[0] == READ_12 ? READ_10 : WRITE_10;
1930                                 flagvals = cdb[1];
1931                                 groupnum = cdb[10];
1932                                 control = cdb[11];
1933                         }
1934
1935                         memset(cdb, 0, sizeof(io_request->CDB.CDB32));
1936
1937                         cdb[0] = opcode;
1938                         cdb[1] = flagvals;
1939                         cdb[6] = groupnum;
1940                         cdb[9] = control;
1941
1942                         /* Transfer length */
1943                         cdb[8] = (u8)(num_blocks & 0xff);
1944                         cdb[7] = (u8)((num_blocks >> 8) & 0xff);
1945
1946                         io_request->IoFlags = cpu_to_le16(10); /* Specify 10-byte cdb */
1947                         cdb_len = 10;
1948                 } else if ((cdb_len < 16) && (start_blk > 0xffffffff)) {
1949                         /* Convert to 16 byte CDB for large LBA's */
1950                         switch (cdb_len) {
1951                         case 6:
1952                                 opcode = cdb[0] == READ_6 ? READ_16 : WRITE_16;
1953                                 control = cdb[5];
1954                                 break;
1955                         case 10:
1956                                 opcode =
1957                                         cdb[0] == READ_10 ? READ_16 : WRITE_16;
1958                                 flagvals = cdb[1];
1959                                 groupnum = cdb[6];
1960                                 control = cdb[9];
1961                                 break;
1962                         case 12:
1963                                 opcode =
1964                                         cdb[0] == READ_12 ? READ_16 : WRITE_16;
1965                                 flagvals = cdb[1];
1966                                 groupnum = cdb[10];
1967                                 control = cdb[11];
1968                                 break;
1969                         }
1970
1971                         memset(cdb, 0, sizeof(io_request->CDB.CDB32));
1972
1973                         cdb[0] = opcode;
1974                         cdb[1] = flagvals;
1975                         cdb[14] = groupnum;
1976                         cdb[15] = control;
1977
1978                         /* Transfer length */
1979                         cdb[13] = (u8)(num_blocks & 0xff);
1980                         cdb[12] = (u8)((num_blocks >> 8) & 0xff);
1981                         cdb[11] = (u8)((num_blocks >> 16) & 0xff);
1982                         cdb[10] = (u8)((num_blocks >> 24) & 0xff);
1983
1984                         io_request->IoFlags = cpu_to_le16(16); /* Specify 16-byte cdb */
1985                         cdb_len = 16;
1986                 }
1987
1988                 /* Normal case, just load LBA here */
1989                 switch (cdb_len) {
1990                 case 6:
1991                 {
1992                         u8 val = cdb[1] & 0xE0;
1993                         cdb[3] = (u8)(start_blk & 0xff);
1994                         cdb[2] = (u8)((start_blk >> 8) & 0xff);
1995                         cdb[1] = val | ((u8)(start_blk >> 16) & 0x1f);
1996                         break;
1997                 }
1998                 case 10:
1999                         cdb[5] = (u8)(start_blk & 0xff);
2000                         cdb[4] = (u8)((start_blk >> 8) & 0xff);
2001                         cdb[3] = (u8)((start_blk >> 16) & 0xff);
2002                         cdb[2] = (u8)((start_blk >> 24) & 0xff);
2003                         break;
2004                 case 12:
2005                         cdb[5]    = (u8)(start_blk & 0xff);
2006                         cdb[4]    = (u8)((start_blk >> 8) & 0xff);
2007                         cdb[3]    = (u8)((start_blk >> 16) & 0xff);
2008                         cdb[2]    = (u8)((start_blk >> 24) & 0xff);
2009                         break;
2010                 case 16:
2011                         cdb[9]    = (u8)(start_blk & 0xff);
2012                         cdb[8]    = (u8)((start_blk >> 8) & 0xff);
2013                         cdb[7]    = (u8)((start_blk >> 16) & 0xff);
2014                         cdb[6]    = (u8)((start_blk >> 24) & 0xff);
2015                         cdb[5]    = (u8)((start_blk >> 32) & 0xff);
2016                         cdb[4]    = (u8)((start_blk >> 40) & 0xff);
2017                         cdb[3]    = (u8)((start_blk >> 48) & 0xff);
2018                         cdb[2]    = (u8)((start_blk >> 56) & 0xff);
2019                         break;
2020                 }
2021         }
2022 }
2023
2024 /**
2025  * megasas_stream_detect -      stream detection on read and and write IOs
2026  * @instance:           Adapter soft state
2027  * @cmd:                    Command to be prepared
2028  * @io_info:            IO Request info
2029  *
2030  */
2031
2032 /** stream detection on read and and write IOs */
2033 static void megasas_stream_detect(struct megasas_instance *instance,
2034                                   struct megasas_cmd_fusion *cmd,
2035                                   struct IO_REQUEST_INFO *io_info)
2036 {
2037         struct fusion_context *fusion = instance->ctrl_context;
2038         u32 device_id = io_info->ldTgtId;
2039         struct LD_STREAM_DETECT *current_ld_sd
2040                 = fusion->stream_detect_by_ld[device_id];
2041         u32 *track_stream = &current_ld_sd->mru_bit_map, stream_num;
2042         u32 shifted_values, unshifted_values;
2043         u32 index_value_mask, shifted_values_mask;
2044         int i;
2045         bool is_read_ahead = false;
2046         struct STREAM_DETECT *current_sd;
2047         /* find possible stream */
2048         for (i = 0; i < MAX_STREAMS_TRACKED; ++i) {
2049                 stream_num = (*track_stream >>
2050                         (i * BITS_PER_INDEX_STREAM)) &
2051                         STREAM_MASK;
2052                 current_sd = &current_ld_sd->stream_track[stream_num];
2053                 /* if we found a stream, update the raid
2054                  *  context and also update the mruBitMap
2055                  */
2056                 /*      boundary condition */
2057                 if ((current_sd->next_seq_lba) &&
2058                     (io_info->ldStartBlock >= current_sd->next_seq_lba) &&
2059                     (io_info->ldStartBlock <= (current_sd->next_seq_lba + 32)) &&
2060                     (current_sd->is_read == io_info->isRead)) {
2061
2062                         if ((io_info->ldStartBlock != current_sd->next_seq_lba) &&
2063                             ((!io_info->isRead) || (!is_read_ahead)))
2064                                 /*
2065                                  * Once the API availible we need to change this.
2066                                  * At this point we are not allowing any gap
2067                                  */
2068                                 continue;
2069
2070                         SET_STREAM_DETECTED(cmd->io_request->RaidContext.raid_context_g35);
2071                         current_sd->next_seq_lba =
2072                         io_info->ldStartBlock + io_info->numBlocks;
2073                         /*
2074                          *      update the mruBitMap LRU
2075                          */
2076                         shifted_values_mask =
2077                                 (1 <<  i * BITS_PER_INDEX_STREAM) - 1;
2078                         shifted_values = ((*track_stream & shifted_values_mask)
2079                                                 << BITS_PER_INDEX_STREAM);
2080                         index_value_mask =
2081                                 STREAM_MASK << i * BITS_PER_INDEX_STREAM;
2082                         unshifted_values =
2083                                 *track_stream & ~(shifted_values_mask |
2084                                 index_value_mask);
2085                         *track_stream =
2086                                 unshifted_values | shifted_values | stream_num;
2087                         return;
2088                 }
2089         }
2090         /*
2091          * if we did not find any stream, create a new one
2092          * from the least recently used
2093          */
2094         stream_num = (*track_stream >>
2095                 ((MAX_STREAMS_TRACKED - 1) * BITS_PER_INDEX_STREAM)) &
2096                 STREAM_MASK;
2097         current_sd = &current_ld_sd->stream_track[stream_num];
2098         current_sd->is_read = io_info->isRead;
2099         current_sd->next_seq_lba = io_info->ldStartBlock + io_info->numBlocks;
2100         *track_stream = (((*track_stream & ZERO_LAST_STREAM) << 4) | stream_num);
2101         return;
2102 }
2103
2104 /**
2105  * megasas_set_raidflag_cpu_affinity - This function sets the cpu
2106  * affinity (cpu of the controller) and raid_flags in the raid context
2107  * based on IO type.
2108  *
2109  * @praid_context:      IO RAID context
2110  * @raid:               LD raid map
2111  * @fp_possible:        Is fast path possible?
2112  * @is_read:            Is read IO?
2113  *
2114  */
2115 static void
2116 megasas_set_raidflag_cpu_affinity(union RAID_CONTEXT_UNION *praid_context,
2117                                   struct MR_LD_RAID *raid, bool fp_possible,
2118                                   u8 is_read, u32 scsi_buff_len)
2119 {
2120         u8 cpu_sel = MR_RAID_CTX_CPUSEL_0;
2121         struct RAID_CONTEXT_G35 *rctx_g35;
2122
2123         rctx_g35 = &praid_context->raid_context_g35;
2124         if (fp_possible) {
2125                 if (is_read) {
2126                         if ((raid->cpuAffinity.pdRead.cpu0) &&
2127                             (raid->cpuAffinity.pdRead.cpu1))
2128                                 cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2129                         else if (raid->cpuAffinity.pdRead.cpu1)
2130                                 cpu_sel = MR_RAID_CTX_CPUSEL_1;
2131                 } else {
2132                         if ((raid->cpuAffinity.pdWrite.cpu0) &&
2133                             (raid->cpuAffinity.pdWrite.cpu1))
2134                                 cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2135                         else if (raid->cpuAffinity.pdWrite.cpu1)
2136                                 cpu_sel = MR_RAID_CTX_CPUSEL_1;
2137                         /* Fast path cache by pass capable R0/R1 VD */
2138                         if ((raid->level <= 1) &&
2139                             (raid->capability.fp_cache_bypass_capable)) {
2140                                 rctx_g35->routing_flags |=
2141                                         (1 << MR_RAID_CTX_ROUTINGFLAGS_SLD_SHIFT);
2142                                 rctx_g35->raid_flags =
2143                                         (MR_RAID_FLAGS_IO_SUB_TYPE_CACHE_BYPASS
2144                                         << MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT);
2145                         }
2146                 }
2147         } else {
2148                 if (is_read) {
2149                         if ((raid->cpuAffinity.ldRead.cpu0) &&
2150                             (raid->cpuAffinity.ldRead.cpu1))
2151                                 cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2152                         else if (raid->cpuAffinity.ldRead.cpu1)
2153                                 cpu_sel = MR_RAID_CTX_CPUSEL_1;
2154                 } else {
2155                         if ((raid->cpuAffinity.ldWrite.cpu0) &&
2156                             (raid->cpuAffinity.ldWrite.cpu1))
2157                                 cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2158                         else if (raid->cpuAffinity.ldWrite.cpu1)
2159                                 cpu_sel = MR_RAID_CTX_CPUSEL_1;
2160
2161                         if (is_stream_detected(rctx_g35) &&
2162                             ((raid->level == 5) || (raid->level == 6)) &&
2163                             (raid->writeMode == MR_RL_WRITE_THROUGH_MODE) &&
2164                             (cpu_sel == MR_RAID_CTX_CPUSEL_FCFS))
2165                                 cpu_sel = MR_RAID_CTX_CPUSEL_0;
2166                 }
2167         }
2168
2169         rctx_g35->routing_flags |=
2170                 (cpu_sel << MR_RAID_CTX_ROUTINGFLAGS_CPUSEL_SHIFT);
2171
2172         /* Always give priority to MR_RAID_FLAGS_IO_SUB_TYPE_LDIO_BW_LIMIT
2173          * vs MR_RAID_FLAGS_IO_SUB_TYPE_CACHE_BYPASS.
2174          * IO Subtype is not bitmap.
2175          */
2176         if ((raid->level == 1) && (!is_read)) {
2177                 if (scsi_buff_len > MR_LARGE_IO_MIN_SIZE)
2178                         praid_context->raid_context_g35.raid_flags =
2179                                 (MR_RAID_FLAGS_IO_SUB_TYPE_LDIO_BW_LIMIT
2180                                 << MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT);
2181         }
2182 }
2183
2184 /**
2185  * megasas_build_ldio_fusion -  Prepares IOs to devices
2186  * @instance:           Adapter soft state
2187  * @scp:                SCSI command
2188  * @cmd:                Command to be prepared
2189  *
2190  * Prepares the io_request and chain elements (sg_frame) for IO
2191  * The IO can be for PD (Fast Path) or LD
2192  */
2193 void
2194 megasas_build_ldio_fusion(struct megasas_instance *instance,
2195                           struct scsi_cmnd *scp,
2196                           struct megasas_cmd_fusion *cmd)
2197 {
2198         bool fp_possible;
2199         u16 ld;
2200         u32 start_lba_lo, start_lba_hi, device_id, datalength = 0;
2201         u32 scsi_buff_len;
2202         struct MPI2_RAID_SCSI_IO_REQUEST *io_request;
2203         union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
2204         struct IO_REQUEST_INFO io_info;
2205         struct fusion_context *fusion;
2206         struct MR_DRV_RAID_MAP_ALL *local_map_ptr;
2207         u8 *raidLUN;
2208         unsigned long spinlock_flags;
2209         union RAID_CONTEXT_UNION *praid_context;
2210         struct MR_LD_RAID *raid = NULL;
2211         struct MR_PRIV_DEVICE *mrdev_priv;
2212
2213         device_id = MEGASAS_DEV_INDEX(scp);
2214
2215         fusion = instance->ctrl_context;
2216
2217         io_request = cmd->io_request;
2218         io_request->RaidContext.raid_context.virtual_disk_tgt_id =
2219                 cpu_to_le16(device_id);
2220         io_request->RaidContext.raid_context.status = 0;
2221         io_request->RaidContext.raid_context.ex_status = 0;
2222
2223         req_desc = (union MEGASAS_REQUEST_DESCRIPTOR_UNION *)cmd->request_desc;
2224
2225         start_lba_lo = 0;
2226         start_lba_hi = 0;
2227         fp_possible = false;
2228
2229         /*
2230          * 6-byte READ(0x08) or WRITE(0x0A) cdb
2231          */
2232         if (scp->cmd_len == 6) {
2233                 datalength = (u32) scp->cmnd[4];
2234                 start_lba_lo = ((u32) scp->cmnd[1] << 16) |
2235                         ((u32) scp->cmnd[2] << 8) | (u32) scp->cmnd[3];
2236
2237                 start_lba_lo &= 0x1FFFFF;
2238         }
2239
2240         /*
2241          * 10-byte READ(0x28) or WRITE(0x2A) cdb
2242          */
2243         else if (scp->cmd_len == 10) {
2244                 datalength = (u32) scp->cmnd[8] |
2245                         ((u32) scp->cmnd[7] << 8);
2246                 start_lba_lo = ((u32) scp->cmnd[2] << 24) |
2247                         ((u32) scp->cmnd[3] << 16) |
2248                         ((u32) scp->cmnd[4] << 8) | (u32) scp->cmnd[5];
2249         }
2250
2251         /*
2252          * 12-byte READ(0xA8) or WRITE(0xAA) cdb
2253          */
2254         else if (scp->cmd_len == 12) {
2255                 datalength = ((u32) scp->cmnd[6] << 24) |
2256                         ((u32) scp->cmnd[7] << 16) |
2257                         ((u32) scp->cmnd[8] << 8) | (u32) scp->cmnd[9];
2258                 start_lba_lo = ((u32) scp->cmnd[2] << 24) |
2259                         ((u32) scp->cmnd[3] << 16) |
2260                         ((u32) scp->cmnd[4] << 8) | (u32) scp->cmnd[5];
2261         }
2262
2263         /*
2264          * 16-byte READ(0x88) or WRITE(0x8A) cdb
2265          */
2266         else if (scp->cmd_len == 16) {
2267                 datalength = ((u32) scp->cmnd[10] << 24) |
2268                         ((u32) scp->cmnd[11] << 16) |
2269                         ((u32) scp->cmnd[12] << 8) | (u32) scp->cmnd[13];
2270                 start_lba_lo = ((u32) scp->cmnd[6] << 24) |
2271                         ((u32) scp->cmnd[7] << 16) |
2272                         ((u32) scp->cmnd[8] << 8) | (u32) scp->cmnd[9];
2273
2274                 start_lba_hi = ((u32) scp->cmnd[2] << 24) |
2275                         ((u32) scp->cmnd[3] << 16) |
2276                         ((u32) scp->cmnd[4] << 8) | (u32) scp->cmnd[5];
2277         }
2278
2279         memset(&io_info, 0, sizeof(struct IO_REQUEST_INFO));
2280         io_info.ldStartBlock = ((u64)start_lba_hi << 32) | start_lba_lo;
2281         io_info.numBlocks = datalength;
2282         io_info.ldTgtId = device_id;
2283         io_info.r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
2284         scsi_buff_len = scsi_bufflen(scp);
2285         io_request->DataLength = cpu_to_le32(scsi_buff_len);
2286
2287         if (scp->sc_data_direction == PCI_DMA_FROMDEVICE)
2288                 io_info.isRead = 1;
2289
2290         local_map_ptr = fusion->ld_drv_map[(instance->map_id & 1)];
2291         ld = MR_TargetIdToLdGet(device_id, local_map_ptr);
2292
2293         if (ld < instance->fw_supported_vd_count)
2294                 raid = MR_LdRaidGet(ld, local_map_ptr);
2295
2296         if (!raid || (!fusion->fast_path_io)) {
2297                 io_request->RaidContext.raid_context.reg_lock_flags  = 0;
2298                 fp_possible = false;
2299         } else {
2300                 if (MR_BuildRaidContext(instance, &io_info,
2301                                         &io_request->RaidContext.raid_context,
2302                                         local_map_ptr, &raidLUN))
2303                         fp_possible = (io_info.fpOkForIo > 0) ? true : false;
2304         }
2305
2306         /* Use raw_smp_processor_id() for now until cmd->request->cpu is CPU
2307            id by default, not CPU group id, otherwise all MSI-X queues won't
2308            be utilized */
2309         cmd->request_desc->SCSIIO.MSIxIndex = instance->msix_vectors ?
2310                 raw_smp_processor_id() % instance->msix_vectors : 0;
2311
2312         praid_context = &io_request->RaidContext;
2313
2314         if (instance->is_ventura) {
2315                 spin_lock_irqsave(&instance->stream_lock, spinlock_flags);
2316                 megasas_stream_detect(instance, cmd, &io_info);
2317                 spin_unlock_irqrestore(&instance->stream_lock, spinlock_flags);
2318                 /* In ventura if stream detected for a read and it is read ahead
2319                  *  capable make this IO as LDIO
2320                  */
2321                 if (is_stream_detected(&io_request->RaidContext.raid_context_g35) &&
2322                     io_info.isRead && io_info.ra_capable)
2323                         fp_possible = false;
2324
2325                 /* FP for Optimal raid level 1.
2326                  * All large RAID-1 writes (> 32 KiB, both WT and WB modes)
2327                  * are built by the driver as LD I/Os.
2328                  * All small RAID-1 WT writes (<= 32 KiB) are built as FP I/Os
2329                  * (there is never a reason to process these as buffered writes)
2330                  * All small RAID-1 WB writes (<= 32 KiB) are built as FP I/Os
2331                  * with the SLD bit asserted.
2332                  */
2333                 if (io_info.r1_alt_dev_handle != MR_DEVHANDLE_INVALID) {
2334                         mrdev_priv = scp->device->hostdata;
2335
2336                         if (atomic_inc_return(&instance->fw_outstanding) >
2337                                 (instance->host->can_queue)) {
2338                                 fp_possible = false;
2339                                 atomic_dec(&instance->fw_outstanding);
2340                         } else if ((scsi_buff_len > MR_LARGE_IO_MIN_SIZE) ||
2341                                    (atomic_dec_if_positive(&mrdev_priv->r1_ldio_hint) > 0)) {
2342                                 fp_possible = false;
2343                                 atomic_dec(&instance->fw_outstanding);
2344                                 if (scsi_buff_len > MR_LARGE_IO_MIN_SIZE)
2345                                         atomic_set(&mrdev_priv->r1_ldio_hint,
2346                                                    instance->r1_ldio_hint_default);
2347                         }
2348                 }
2349
2350                 /* If raid is NULL, set CPU affinity to default CPU0 */
2351                 if (raid)
2352                         megasas_set_raidflag_cpu_affinity(praid_context,
2353                                 raid, fp_possible, io_info.isRead,
2354                                 scsi_buff_len);
2355                 else
2356                         praid_context->raid_context_g35.routing_flags |=
2357                                 (MR_RAID_CTX_CPUSEL_0 << MR_RAID_CTX_ROUTINGFLAGS_CPUSEL_SHIFT);
2358         }
2359
2360         if (fp_possible) {
2361                 megasas_set_pd_lba(io_request, scp->cmd_len, &io_info, scp,
2362                                    local_map_ptr, start_lba_lo);
2363                 io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
2364                 cmd->request_desc->SCSIIO.RequestFlags =
2365                         (MPI2_REQ_DESCRIPT_FLAGS_FP_IO
2366                          << MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2367                 if (fusion->adapter_type == INVADER_SERIES) {
2368                         if (io_request->RaidContext.raid_context.reg_lock_flags ==
2369                             REGION_TYPE_UNUSED)
2370                                 cmd->request_desc->SCSIIO.RequestFlags =
2371                                         (MEGASAS_REQ_DESCRIPT_FLAGS_NO_LOCK <<
2372                                         MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2373                         io_request->RaidContext.raid_context.type
2374                                 = MPI2_TYPE_CUDA;
2375                         io_request->RaidContext.raid_context.nseg = 0x1;
2376                         io_request->IoFlags |= cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH);
2377                         io_request->RaidContext.raid_context.reg_lock_flags |=
2378                           (MR_RL_FLAGS_GRANT_DESTINATION_CUDA |
2379                            MR_RL_FLAGS_SEQ_NUM_ENABLE);
2380                 } else if (instance->is_ventura) {
2381                         io_request->RaidContext.raid_context_g35.nseg_type |=
2382                                                 (1 << RAID_CONTEXT_NSEG_SHIFT);
2383                         io_request->RaidContext.raid_context_g35.nseg_type |=
2384                                                 (MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT);
2385                         io_request->RaidContext.raid_context_g35.routing_flags |=
2386                                                 (1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT);
2387                         io_request->IoFlags |=
2388                                 cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH);
2389                 }
2390                 if (fusion->load_balance_info &&
2391                         (fusion->load_balance_info[device_id].loadBalanceFlag) &&
2392                         (io_info.isRead)) {
2393                         io_info.devHandle =
2394                                 get_updated_dev_handle(instance,
2395                                         &fusion->load_balance_info[device_id],
2396                                         &io_info, local_map_ptr);
2397                         scp->SCp.Status |= MEGASAS_LOAD_BALANCE_FLAG;
2398                         cmd->pd_r1_lb = io_info.pd_after_lb;
2399                         if (instance->is_ventura)
2400                                 io_request->RaidContext.raid_context_g35.span_arm
2401                                         = io_info.span_arm;
2402                         else
2403                                 io_request->RaidContext.raid_context.span_arm
2404                                         = io_info.span_arm;
2405
2406                 } else
2407                         scp->SCp.Status &= ~MEGASAS_LOAD_BALANCE_FLAG;
2408
2409                 if (instance->is_ventura)
2410                         cmd->r1_alt_dev_handle = io_info.r1_alt_dev_handle;
2411                 else
2412                         cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
2413
2414                 if ((raidLUN[0] == 1) &&
2415                         (local_map_ptr->raidMap.devHndlInfo[io_info.pd_after_lb].validHandles > 1)) {
2416                         instance->dev_handle = !(instance->dev_handle);
2417                         io_info.devHandle =
2418                                 local_map_ptr->raidMap.devHndlInfo[io_info.pd_after_lb].devHandle[instance->dev_handle];
2419                 }
2420
2421                 cmd->request_desc->SCSIIO.DevHandle = io_info.devHandle;
2422                 io_request->DevHandle = io_info.devHandle;
2423                 cmd->pd_interface = io_info.pd_interface;
2424                 /* populate the LUN field */
2425                 memcpy(io_request->LUN, raidLUN, 8);
2426         } else {
2427                 io_request->RaidContext.raid_context.timeout_value =
2428                         cpu_to_le16(local_map_ptr->raidMap.fpPdIoTimeoutSec);
2429                 cmd->request_desc->SCSIIO.RequestFlags =
2430                         (MEGASAS_REQ_DESCRIPT_FLAGS_LD_IO
2431                          << MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2432                 if (fusion->adapter_type == INVADER_SERIES) {
2433                         if (io_info.do_fp_rlbypass ||
2434                         (io_request->RaidContext.raid_context.reg_lock_flags
2435                                         == REGION_TYPE_UNUSED))
2436                                 cmd->request_desc->SCSIIO.RequestFlags =
2437                                         (MEGASAS_REQ_DESCRIPT_FLAGS_NO_LOCK <<
2438                                         MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2439                         io_request->RaidContext.raid_context.type
2440                                 = MPI2_TYPE_CUDA;
2441                         io_request->RaidContext.raid_context.reg_lock_flags |=
2442                                 (MR_RL_FLAGS_GRANT_DESTINATION_CPU0 |
2443                                  MR_RL_FLAGS_SEQ_NUM_ENABLE);
2444                         io_request->RaidContext.raid_context.nseg = 0x1;
2445                 } else if (instance->is_ventura) {
2446                         io_request->RaidContext.raid_context_g35.routing_flags |=
2447                                         (1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT);
2448                         io_request->RaidContext.raid_context_g35.nseg_type |=
2449                                         (1 << RAID_CONTEXT_NSEG_SHIFT);
2450                         io_request->RaidContext.raid_context_g35.nseg_type |=
2451                                         (MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT);
2452                 }
2453                 io_request->Function = MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST;
2454                 io_request->DevHandle = cpu_to_le16(device_id);
2455
2456         } /* Not FP */
2457 }
2458
2459 /**
2460  * megasas_build_ld_nonrw_fusion - prepares non rw ios for virtual disk
2461  * @instance:           Adapter soft state
2462  * @scp:                SCSI command
2463  * @cmd:                Command to be prepared
2464  *
2465  * Prepares the io_request frame for non-rw io cmds for vd.
2466  */
2467 static void megasas_build_ld_nonrw_fusion(struct megasas_instance *instance,
2468                           struct scsi_cmnd *scmd, struct megasas_cmd_fusion *cmd)
2469 {
2470         u32 device_id;
2471         struct MPI2_RAID_SCSI_IO_REQUEST *io_request;
2472         u16 ld;
2473         struct MR_DRV_RAID_MAP_ALL *local_map_ptr;
2474         struct fusion_context *fusion = instance->ctrl_context;
2475         u8                          span, physArm;
2476         __le16                      devHandle;
2477         u32                         arRef, pd;
2478         struct MR_LD_RAID                  *raid;
2479         struct RAID_CONTEXT                *pRAID_Context;
2480         u8 fp_possible = 1;
2481
2482         io_request = cmd->io_request;
2483         device_id = MEGASAS_DEV_INDEX(scmd);
2484         local_map_ptr = fusion->ld_drv_map[(instance->map_id & 1)];
2485         io_request->DataLength = cpu_to_le32(scsi_bufflen(scmd));
2486         /* get RAID_Context pointer */
2487         pRAID_Context = &io_request->RaidContext.raid_context;
2488         /* Check with FW team */
2489         pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);
2490         pRAID_Context->reg_lock_row_lba    = 0;
2491         pRAID_Context->reg_lock_length    = 0;
2492
2493         if (fusion->fast_path_io && (
2494                 device_id < instance->fw_supported_vd_count)) {
2495
2496                 ld = MR_TargetIdToLdGet(device_id, local_map_ptr);
2497                 if (ld >= instance->fw_supported_vd_count)
2498                         fp_possible = 0;
2499                 else {
2500                         raid = MR_LdRaidGet(ld, local_map_ptr);
2501                         if (!(raid->capability.fpNonRWCapable))
2502                                 fp_possible = 0;
2503                 }
2504         } else
2505                 fp_possible = 0;
2506
2507         if (!fp_possible) {
2508                 io_request->Function  = MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST;
2509                 io_request->DevHandle = cpu_to_le16(device_id);
2510                 io_request->LUN[1] = scmd->device->lun;
2511                 pRAID_Context->timeout_value =
2512                         cpu_to_le16 (scmd->request->timeout / HZ);
2513                 cmd->request_desc->SCSIIO.RequestFlags =
2514                         (MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO <<
2515                         MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2516         } else {
2517
2518                 /* set RAID context values */
2519                 pRAID_Context->config_seq_num = raid->seqNum;
2520                 if (!instance->is_ventura)
2521                         pRAID_Context->reg_lock_flags = REGION_TYPE_SHARED_READ;
2522                 pRAID_Context->timeout_value =
2523                         cpu_to_le16(raid->fpIoTimeoutForLd);
2524
2525                 /* get the DevHandle for the PD (since this is
2526                    fpNonRWCapable, this is a single disk RAID0) */
2527                 span = physArm = 0;
2528                 arRef = MR_LdSpanArrayGet(ld, span, local_map_ptr);
2529                 pd = MR_ArPdGet(arRef, physArm, local_map_ptr);
2530                 devHandle = MR_PdDevHandleGet(pd, local_map_ptr);
2531
2532                 /* build request descriptor */
2533                 cmd->request_desc->SCSIIO.RequestFlags =
2534                         (MPI2_REQ_DESCRIPT_FLAGS_FP_IO <<
2535                         MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2536                 cmd->request_desc->SCSIIO.DevHandle = devHandle;
2537
2538                 /* populate the LUN field */
2539                 memcpy(io_request->LUN, raid->LUN, 8);
2540
2541                 /* build the raidScsiIO structure */
2542                 io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
2543                 io_request->DevHandle = devHandle;
2544         }
2545 }
2546
2547 /**
2548  * megasas_build_syspd_fusion - prepares rw/non-rw ios for syspd
2549  * @instance:           Adapter soft state
2550  * @scp:                SCSI command
2551  * @cmd:                Command to be prepared
2552  * @fp_possible:        parameter to detect fast path or firmware path io.
2553  *
2554  * Prepares the io_request frame for rw/non-rw io cmds for syspds
2555  */
2556 static void
2557 megasas_build_syspd_fusion(struct megasas_instance *instance,
2558         struct scsi_cmnd *scmd, struct megasas_cmd_fusion *cmd,
2559         bool fp_possible)
2560 {
2561         u32 device_id;
2562         struct MPI2_RAID_SCSI_IO_REQUEST *io_request;
2563         u16 pd_index = 0;
2564         u16 os_timeout_value;
2565         u16 timeout_limit;
2566         struct MR_DRV_RAID_MAP_ALL *local_map_ptr;
2567         struct RAID_CONTEXT     *pRAID_Context;
2568         struct MR_PD_CFG_SEQ_NUM_SYNC *pd_sync;
2569         struct MR_PRIV_DEVICE *mr_device_priv_data;
2570         struct fusion_context *fusion = instance->ctrl_context;
2571         pd_sync = (void *)fusion->pd_seq_sync[(instance->pd_seq_map_id - 1) & 1];
2572
2573         device_id = MEGASAS_DEV_INDEX(scmd);
2574         pd_index = MEGASAS_PD_INDEX(scmd);
2575         os_timeout_value = scmd->request->timeout / HZ;
2576         mr_device_priv_data = scmd->device->hostdata;
2577         cmd->pd_interface = mr_device_priv_data->interface_type;
2578
2579         io_request = cmd->io_request;
2580         /* get RAID_Context pointer */
2581         pRAID_Context = &io_request->RaidContext.raid_context;
2582         pRAID_Context->reg_lock_flags = 0;
2583         pRAID_Context->reg_lock_row_lba = 0;
2584         pRAID_Context->reg_lock_length = 0;
2585         io_request->DataLength = cpu_to_le32(scsi_bufflen(scmd));
2586         io_request->LUN[1] = scmd->device->lun;
2587         pRAID_Context->raid_flags = MR_RAID_FLAGS_IO_SUB_TYPE_SYSTEM_PD
2588                 << MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT;
2589
2590         /* If FW supports PD sequence number */
2591         if (instance->use_seqnum_jbod_fp &&
2592                 instance->pd_list[pd_index].driveType == TYPE_DISK) {
2593                 /* TgtId must be incremented by 255 as jbod seq number is index
2594                  * below raid map
2595                  */
2596                  /* More than 256 PD/JBOD support for Ventura */
2597                 if (instance->support_morethan256jbod)
2598                         pRAID_Context->virtual_disk_tgt_id =
2599                                 pd_sync->seq[pd_index].pd_target_id;
2600                 else
2601                         pRAID_Context->virtual_disk_tgt_id =
2602                                 cpu_to_le16(device_id + (MAX_PHYSICAL_DEVICES - 1));
2603                 pRAID_Context->config_seq_num = pd_sync->seq[pd_index].seqNum;
2604                 io_request->DevHandle = pd_sync->seq[pd_index].devHandle;
2605                 if (instance->is_ventura) {
2606                         io_request->RaidContext.raid_context_g35.routing_flags |=
2607                                 (1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT);
2608                         io_request->RaidContext.raid_context_g35.nseg_type |=
2609                                                         (1 << RAID_CONTEXT_NSEG_SHIFT);
2610                         io_request->RaidContext.raid_context_g35.nseg_type |=
2611                                                         (MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT);
2612                 } else {
2613                         pRAID_Context->type = MPI2_TYPE_CUDA;
2614                         pRAID_Context->nseg = 0x1;
2615                         pRAID_Context->reg_lock_flags |=
2616                                 (MR_RL_FLAGS_SEQ_NUM_ENABLE|MR_RL_FLAGS_GRANT_DESTINATION_CUDA);
2617                 }
2618         } else if (fusion->fast_path_io) {
2619                 pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);
2620                 pRAID_Context->config_seq_num = 0;
2621                 local_map_ptr = fusion->ld_drv_map[(instance->map_id & 1)];
2622                 io_request->DevHandle =
2623                         local_map_ptr->raidMap.devHndlInfo[device_id].curDevHdl;
2624         } else {
2625                 /* Want to send all IO via FW path */
2626                 pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);
2627                 pRAID_Context->config_seq_num = 0;
2628                 io_request->DevHandle = cpu_to_le16(0xFFFF);
2629         }
2630
2631         cmd->request_desc->SCSIIO.DevHandle = io_request->DevHandle;
2632         cmd->request_desc->SCSIIO.MSIxIndex =
2633                 instance->msix_vectors ?
2634                 (raw_smp_processor_id() % instance->msix_vectors) : 0;
2635
2636
2637         if (!fp_possible) {
2638                 /* system pd firmware path */
2639                 io_request->Function  = MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST;
2640                 cmd->request_desc->SCSIIO.RequestFlags =
2641                         (MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO <<
2642                                 MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2643                 pRAID_Context->timeout_value = cpu_to_le16(os_timeout_value);
2644                 pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);
2645         } else {
2646                 /* system pd Fast Path */
2647                 io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
2648                 timeout_limit = (scmd->device->type == TYPE_DISK) ?
2649                                 255 : 0xFFFF;
2650                 pRAID_Context->timeout_value =
2651                         cpu_to_le16((os_timeout_value > timeout_limit) ?
2652                         timeout_limit : os_timeout_value);
2653                 if (fusion->adapter_type >= INVADER_SERIES)
2654                         io_request->IoFlags |=
2655                                 cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH);
2656
2657                 cmd->request_desc->SCSIIO.RequestFlags =
2658                         (MPI2_REQ_DESCRIPT_FLAGS_FP_IO <<
2659                                 MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2660         }
2661 }
2662
2663 /**
2664  * megasas_build_io_fusion -    Prepares IOs to devices
2665  * @instance:           Adapter soft state
2666  * @scp:                SCSI command
2667  * @cmd:                Command to be prepared
2668  *
2669  * Invokes helper functions to prepare request frames
2670  * and sets flags appropriate for IO/Non-IO cmd
2671  */
2672 int
2673 megasas_build_io_fusion(struct megasas_instance *instance,
2674                         struct scsi_cmnd *scp,
2675                         struct megasas_cmd_fusion *cmd)
2676 {
2677         int sge_count;
2678         u8  cmd_type;
2679         struct MPI2_RAID_SCSI_IO_REQUEST *io_request = cmd->io_request;
2680         struct MR_PRIV_DEVICE *mr_device_priv_data;
2681         mr_device_priv_data = scp->device->hostdata;
2682
2683         /* Zero out some fields so they don't get reused */
2684         memset(io_request->LUN, 0x0, 8);
2685         io_request->CDB.EEDP32.PrimaryReferenceTag = 0;
2686         io_request->CDB.EEDP32.PrimaryApplicationTagMask = 0;
2687         io_request->EEDPFlags = 0;
2688         io_request->Control = 0;
2689         io_request->EEDPBlockSize = 0;
2690         io_request->ChainOffset = 0;
2691         io_request->RaidContext.raid_context.raid_flags = 0;
2692         io_request->RaidContext.raid_context.type = 0;
2693         io_request->RaidContext.raid_context.nseg = 0;
2694
2695         memcpy(io_request->CDB.CDB32, scp->cmnd, scp->cmd_len);
2696         /*
2697          * Just the CDB length,rest of the Flags are zero
2698          * This will be modified for FP in build_ldio_fusion
2699          */
2700         io_request->IoFlags = cpu_to_le16(scp->cmd_len);
2701
2702         switch (cmd_type = megasas_cmd_type(scp)) {
2703         case READ_WRITE_LDIO:
2704                 megasas_build_ldio_fusion(instance, scp, cmd);
2705                 break;
2706         case NON_READ_WRITE_LDIO:
2707                 megasas_build_ld_nonrw_fusion(instance, scp, cmd);
2708                 break;
2709         case READ_WRITE_SYSPDIO:
2710                 megasas_build_syspd_fusion(instance, scp, cmd, true);
2711                 break;
2712         case NON_READ_WRITE_SYSPDIO:
2713                 if (instance->secure_jbod_support ||
2714                     mr_device_priv_data->is_tm_capable)
2715                         megasas_build_syspd_fusion(instance, scp, cmd, false);
2716                 else
2717                         megasas_build_syspd_fusion(instance, scp, cmd, true);
2718                 break;
2719         default:
2720                 break;
2721         }
2722
2723         /*
2724          * Construct SGL
2725          */
2726
2727         sge_count = megasas_make_sgl(instance, scp, cmd);
2728
2729         if (sge_count > instance->max_num_sge || (sge_count < 0)) {
2730                 dev_err(&instance->pdev->dev,
2731                         "%s %d sge_count (%d) is out of range. Range is:  0-%d\n",
2732                         __func__, __LINE__, sge_count, instance->max_num_sge);
2733                 return 1;
2734         }
2735
2736         if (instance->is_ventura) {
2737                 set_num_sge(&io_request->RaidContext.raid_context_g35, sge_count);
2738                 cpu_to_le16s(&io_request->RaidContext.raid_context_g35.routing_flags);
2739                 cpu_to_le16s(&io_request->RaidContext.raid_context_g35.nseg_type);
2740         } else {
2741                 /* numSGE store lower 8 bit of sge_count.
2742                  * numSGEExt store higher 8 bit of sge_count
2743                  */
2744                 io_request->RaidContext.raid_context.num_sge = sge_count;
2745                 io_request->RaidContext.raid_context.num_sge_ext =
2746                         (u8)(sge_count >> 8);
2747         }
2748
2749         io_request->SGLFlags = cpu_to_le16(MPI2_SGE_FLAGS_64_BIT_ADDRESSING);
2750
2751         if (scp->sc_data_direction == PCI_DMA_TODEVICE)
2752                 io_request->Control |= cpu_to_le32(MPI2_SCSIIO_CONTROL_WRITE);
2753         else if (scp->sc_data_direction == PCI_DMA_FROMDEVICE)
2754                 io_request->Control |= cpu_to_le32(MPI2_SCSIIO_CONTROL_READ);
2755
2756         io_request->SGLOffset0 =
2757                 offsetof(struct MPI2_RAID_SCSI_IO_REQUEST, SGL) / 4;
2758
2759         io_request->SenseBufferLowAddress = cpu_to_le32(cmd->sense_phys_addr);
2760         io_request->SenseBufferLength = SCSI_SENSE_BUFFERSIZE;
2761
2762         cmd->scmd = scp;
2763         scp->SCp.ptr = (char *)cmd;
2764
2765         return 0;
2766 }
2767
2768 static union MEGASAS_REQUEST_DESCRIPTOR_UNION *
2769 megasas_get_request_descriptor(struct megasas_instance *instance, u16 index)
2770 {
2771         u8 *p;
2772         struct fusion_context *fusion;
2773
2774         fusion = instance->ctrl_context;
2775         p = fusion->req_frames_desc +
2776                 sizeof(union MEGASAS_REQUEST_DESCRIPTOR_UNION) * index;
2777
2778         return (union MEGASAS_REQUEST_DESCRIPTOR_UNION *)p;
2779 }
2780
2781
2782 /* megasas_prepate_secondRaid1_IO
2783  *  It prepares the raid 1 second IO
2784  */
2785 void megasas_prepare_secondRaid1_IO(struct megasas_instance *instance,
2786                             struct megasas_cmd_fusion *cmd,
2787                             struct megasas_cmd_fusion *r1_cmd)
2788 {
2789         union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc, *req_desc2 = NULL;
2790         struct fusion_context *fusion;
2791         fusion = instance->ctrl_context;
2792         req_desc = cmd->request_desc;
2793         /* copy the io request frame as well as 8 SGEs data for r1 command*/
2794         memcpy(r1_cmd->io_request, cmd->io_request,
2795                (sizeof(struct MPI2_RAID_SCSI_IO_REQUEST)));
2796         memcpy(&r1_cmd->io_request->SGL, &cmd->io_request->SGL,
2797                (fusion->max_sge_in_main_msg * sizeof(union MPI2_SGE_IO_UNION)));
2798         /*sense buffer is different for r1 command*/
2799         r1_cmd->io_request->SenseBufferLowAddress =
2800                         cpu_to_le32(r1_cmd->sense_phys_addr);
2801         r1_cmd->scmd = cmd->scmd;
2802         req_desc2 = megasas_get_request_descriptor(instance,
2803                                                    (r1_cmd->index - 1));
2804         req_desc2->Words = 0;
2805         r1_cmd->request_desc = req_desc2;
2806         req_desc2->SCSIIO.SMID = cpu_to_le16(r1_cmd->index);
2807         req_desc2->SCSIIO.RequestFlags = req_desc->SCSIIO.RequestFlags;
2808         r1_cmd->request_desc->SCSIIO.DevHandle = cmd->r1_alt_dev_handle;
2809         r1_cmd->io_request->DevHandle = cmd->r1_alt_dev_handle;
2810         r1_cmd->r1_alt_dev_handle = cmd->io_request->DevHandle;
2811         cmd->io_request->RaidContext.raid_context_g35.smid.peer_smid =
2812                         cpu_to_le16(r1_cmd->index);
2813         r1_cmd->io_request->RaidContext.raid_context_g35.smid.peer_smid =
2814                         cpu_to_le16(cmd->index);
2815         /*MSIxIndex of both commands request descriptors should be same*/
2816         r1_cmd->request_desc->SCSIIO.MSIxIndex =
2817                         cmd->request_desc->SCSIIO.MSIxIndex;
2818         /*span arm is different for r1 cmd*/
2819         r1_cmd->io_request->RaidContext.raid_context_g35.span_arm =
2820                         cmd->io_request->RaidContext.raid_context_g35.span_arm + 1;
2821 }
2822
2823 /**
2824  * megasas_build_and_issue_cmd_fusion -Main routine for building and
2825  *                                     issuing non IOCTL cmd
2826  * @instance:                   Adapter soft state
2827  * @scmd:                       pointer to scsi cmd from OS
2828  */
2829 static u32
2830 megasas_build_and_issue_cmd_fusion(struct megasas_instance *instance,
2831                                    struct scsi_cmnd *scmd)
2832 {
2833         struct megasas_cmd_fusion *cmd, *r1_cmd = NULL;
2834         union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
2835         u32 index;
2836         struct fusion_context *fusion;
2837
2838         fusion = instance->ctrl_context;
2839
2840         if ((megasas_cmd_type(scmd) == READ_WRITE_LDIO) &&
2841                 instance->ldio_threshold &&
2842                 (atomic_inc_return(&instance->ldio_outstanding) >
2843                 instance->ldio_threshold)) {
2844                 atomic_dec(&instance->ldio_outstanding);
2845                 return SCSI_MLQUEUE_DEVICE_BUSY;
2846         }
2847
2848         if (atomic_inc_return(&instance->fw_outstanding) >
2849                         instance->host->can_queue) {
2850                 atomic_dec(&instance->fw_outstanding);
2851                 return SCSI_MLQUEUE_HOST_BUSY;
2852         }
2853
2854         cmd = megasas_get_cmd_fusion(instance, scmd->request->tag);
2855
2856         if (!cmd) {
2857                 atomic_dec(&instance->fw_outstanding);
2858                 return SCSI_MLQUEUE_HOST_BUSY;
2859         }
2860
2861         index = cmd->index;
2862
2863         req_desc = megasas_get_request_descriptor(instance, index-1);
2864
2865         req_desc->Words = 0;
2866         cmd->request_desc = req_desc;
2867
2868         if (megasas_build_io_fusion(instance, scmd, cmd)) {
2869                 megasas_return_cmd_fusion(instance, cmd);
2870                 dev_err(&instance->pdev->dev, "Error building command\n");
2871                 cmd->request_desc = NULL;
2872                 atomic_dec(&instance->fw_outstanding);
2873                 return SCSI_MLQUEUE_HOST_BUSY;
2874         }
2875
2876         req_desc = cmd->request_desc;
2877         req_desc->SCSIIO.SMID = cpu_to_le16(index);
2878
2879         if (cmd->io_request->ChainOffset != 0 &&
2880             cmd->io_request->ChainOffset != 0xF)
2881                 dev_err(&instance->pdev->dev, "The chain offset value is not "
2882                        "correct : %x\n", cmd->io_request->ChainOffset);
2883         /*
2884          *      if it is raid 1/10 fp write capable.
2885          *      try to get second command from pool and construct it.
2886          *      From FW, it has confirmed that lba values of two PDs
2887          *      corresponds to single R1/10 LD are always same
2888          *
2889          */
2890         /*      driver side count always should be less than max_fw_cmds
2891          *      to get new command
2892          */
2893         if (cmd->r1_alt_dev_handle != MR_DEVHANDLE_INVALID) {
2894                 r1_cmd = megasas_get_cmd_fusion(instance,
2895                                 (scmd->request->tag + instance->max_fw_cmds));
2896                 megasas_prepare_secondRaid1_IO(instance, cmd, r1_cmd);
2897         }
2898
2899
2900         /*
2901          * Issue the command to the FW
2902          */
2903
2904         megasas_fire_cmd_fusion(instance, req_desc);
2905
2906         if (r1_cmd)
2907                 megasas_fire_cmd_fusion(instance, r1_cmd->request_desc);
2908
2909
2910         return 0;
2911 }
2912
2913 /**
2914  * megasas_complete_r1_command -
2915  * completes R1 FP write commands which has valid peer smid
2916  * @instance:                   Adapter soft state
2917  * @cmd_fusion:                 MPT command frame
2918  *
2919  */
2920 static inline void
2921 megasas_complete_r1_command(struct megasas_instance *instance,
2922                             struct megasas_cmd_fusion *cmd)
2923 {
2924         u8 *sense, status, ex_status;
2925         u32 data_length;
2926         u16 peer_smid;
2927         struct fusion_context *fusion;
2928         struct megasas_cmd_fusion *r1_cmd = NULL;
2929         struct scsi_cmnd *scmd_local = NULL;
2930         struct RAID_CONTEXT_G35 *rctx_g35;
2931
2932         rctx_g35 = &cmd->io_request->RaidContext.raid_context_g35;
2933         fusion = instance->ctrl_context;
2934         peer_smid = le16_to_cpu(rctx_g35->smid.peer_smid);
2935
2936         r1_cmd = fusion->cmd_list[peer_smid - 1];
2937         scmd_local = cmd->scmd;
2938         status = rctx_g35->status;
2939         ex_status = rctx_g35->ex_status;
2940         data_length = cmd->io_request->DataLength;
2941         sense = cmd->sense;
2942
2943         cmd->cmd_completed = true;
2944
2945         /* Check if peer command is completed or not*/
2946         if (r1_cmd->cmd_completed) {
2947                 rctx_g35 = &r1_cmd->io_request->RaidContext.raid_context_g35;
2948                 if (rctx_g35->status != MFI_STAT_OK) {
2949                         status = rctx_g35->status;
2950                         ex_status = rctx_g35->ex_status;
2951                         data_length = r1_cmd->io_request->DataLength;
2952                         sense = r1_cmd->sense;
2953                 }
2954
2955                 megasas_return_cmd_fusion(instance, r1_cmd);
2956                 map_cmd_status(fusion, scmd_local, status, ex_status,
2957                                le32_to_cpu(data_length), sense);
2958                 if (instance->ldio_threshold &&
2959                     megasas_cmd_type(scmd_local) == READ_WRITE_LDIO)
2960                         atomic_dec(&instance->ldio_outstanding);
2961                 scmd_local->SCp.ptr = NULL;
2962                 megasas_return_cmd_fusion(instance, cmd);
2963                 scsi_dma_unmap(scmd_local);
2964                 scmd_local->scsi_done(scmd_local);
2965         }
2966 }
2967
2968 /**
2969  * complete_cmd_fusion -        Completes command
2970  * @instance:                   Adapter soft state
2971  * Completes all commands that is in reply descriptor queue
2972  */
2973 int
2974 complete_cmd_fusion(struct megasas_instance *instance, u32 MSIxIndex)
2975 {
2976         union MPI2_REPLY_DESCRIPTORS_UNION *desc;
2977         struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *reply_desc;
2978         struct MPI2_RAID_SCSI_IO_REQUEST *scsi_io_req;
2979         struct fusion_context *fusion;
2980         struct megasas_cmd *cmd_mfi;
2981         struct megasas_cmd_fusion *cmd_fusion;
2982         u16 smid, num_completed;
2983         u8 reply_descript_type, *sense, status, extStatus;
2984         u32 device_id, data_length;
2985         union desc_value d_val;
2986         struct LD_LOAD_BALANCE_INFO *lbinfo;
2987         int threshold_reply_count = 0;
2988         struct scsi_cmnd *scmd_local = NULL;
2989         struct MR_TASK_MANAGE_REQUEST *mr_tm_req;
2990         struct MPI2_SCSI_TASK_MANAGE_REQUEST *mpi_tm_req;
2991
2992         fusion = instance->ctrl_context;
2993
2994         if (atomic_read(&instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR)
2995                 return IRQ_HANDLED;
2996
2997         desc = fusion->reply_frames_desc[MSIxIndex] +
2998                                 fusion->last_reply_idx[MSIxIndex];
2999
3000         reply_desc = (struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *)desc;
3001
3002         d_val.word = desc->Words;
3003
3004         reply_descript_type = reply_desc->ReplyFlags &
3005                 MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
3006
3007         if (reply_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED)
3008                 return IRQ_NONE;
3009
3010         num_completed = 0;
3011
3012         while (d_val.u.low != cpu_to_le32(UINT_MAX) &&
3013                d_val.u.high != cpu_to_le32(UINT_MAX)) {
3014
3015                 smid = le16_to_cpu(reply_desc->SMID);
3016                 cmd_fusion = fusion->cmd_list[smid - 1];
3017                 scsi_io_req = (struct MPI2_RAID_SCSI_IO_REQUEST *)
3018                                                 cmd_fusion->io_request;
3019
3020                 scmd_local = cmd_fusion->scmd;
3021                 status = scsi_io_req->RaidContext.raid_context.status;
3022                 extStatus = scsi_io_req->RaidContext.raid_context.ex_status;
3023                 sense = cmd_fusion->sense;
3024                 data_length = scsi_io_req->DataLength;
3025
3026                 switch (scsi_io_req->Function) {
3027                 case MPI2_FUNCTION_SCSI_TASK_MGMT:
3028                         mr_tm_req = (struct MR_TASK_MANAGE_REQUEST *)
3029                                                 cmd_fusion->io_request;
3030                         mpi_tm_req = (struct MPI2_SCSI_TASK_MANAGE_REQUEST *)
3031                                                 &mr_tm_req->TmRequest;
3032                         dev_dbg(&instance->pdev->dev, "TM completion:"
3033                                 "type: 0x%x TaskMID: 0x%x\n",
3034                                 mpi_tm_req->TaskType, mpi_tm_req->TaskMID);
3035                         complete(&cmd_fusion->done);
3036                         break;
3037                 case MPI2_FUNCTION_SCSI_IO_REQUEST:  /*Fast Path IO.*/
3038                         /* Update load balancing info */
3039                         if (fusion->load_balance_info &&
3040                             (cmd_fusion->scmd->SCp.Status &
3041                             MEGASAS_LOAD_BALANCE_FLAG)) {
3042                                 device_id = MEGASAS_DEV_INDEX(scmd_local);
3043                                 lbinfo = &fusion->load_balance_info[device_id];
3044                                 atomic_dec(&lbinfo->scsi_pending_cmds[cmd_fusion->pd_r1_lb]);
3045                                 cmd_fusion->scmd->SCp.Status &= ~MEGASAS_LOAD_BALANCE_FLAG;
3046                         }
3047                         //Fall thru and complete IO
3048                 case MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST: /* LD-IO Path */
3049                         atomic_dec(&instance->fw_outstanding);
3050                         if (cmd_fusion->r1_alt_dev_handle == MR_DEVHANDLE_INVALID) {
3051                                 map_cmd_status(fusion, scmd_local, status,
3052                                                extStatus, le32_to_cpu(data_length),
3053                                                sense);
3054                                 if (instance->ldio_threshold &&
3055                                     (megasas_cmd_type(scmd_local) == READ_WRITE_LDIO))
3056                                         atomic_dec(&instance->ldio_outstanding);
3057                                 scmd_local->SCp.ptr = NULL;
3058                                 megasas_return_cmd_fusion(instance, cmd_fusion);
3059                                 scsi_dma_unmap(scmd_local);
3060                                 scmd_local->scsi_done(scmd_local);
3061                         } else  /* Optimal VD - R1 FP command completion. */
3062                                 megasas_complete_r1_command(instance, cmd_fusion);
3063                         break;
3064                 case MEGASAS_MPI2_FUNCTION_PASSTHRU_IO_REQUEST: /*MFI command */
3065                         cmd_mfi = instance->cmd_list[cmd_fusion->sync_cmd_idx];
3066                         /* Poll mode. Dummy free.
3067                          * In case of Interrupt mode, caller has reverse check.
3068                          */
3069                         if (cmd_mfi->flags & DRV_DCMD_POLLED_MODE) {
3070                                 cmd_mfi->flags &= ~DRV_DCMD_POLLED_MODE;
3071                                 megasas_return_cmd(instance, cmd_mfi);
3072                         } else
3073                                 megasas_complete_cmd(instance, cmd_mfi, DID_OK);
3074                         break;
3075                 }
3076
3077                 fusion->last_reply_idx[MSIxIndex]++;
3078                 if (fusion->last_reply_idx[MSIxIndex] >=
3079                     fusion->reply_q_depth)
3080                         fusion->last_reply_idx[MSIxIndex] = 0;
3081
3082                 desc->Words = cpu_to_le64(ULLONG_MAX);
3083                 num_completed++;
3084                 threshold_reply_count++;
3085
3086                 /* Get the next reply descriptor */
3087                 if (!fusion->last_reply_idx[MSIxIndex])
3088                         desc = fusion->reply_frames_desc[MSIxIndex];
3089                 else
3090                         desc++;
3091
3092                 reply_desc =
3093                   (struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *)desc;
3094
3095                 d_val.word = desc->Words;
3096
3097                 reply_descript_type = reply_desc->ReplyFlags &
3098                         MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
3099
3100                 if (reply_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED)
3101                         break;
3102                 /*
3103                  * Write to reply post host index register after completing threshold
3104                  * number of reply counts and still there are more replies in reply queue
3105                  * pending to be completed
3106                  */
3107                 if (threshold_reply_count >= THRESHOLD_REPLY_COUNT) {
3108                         if (instance->msix_combined)
3109                                 writel(((MSIxIndex & 0x7) << 24) |
3110                                         fusion->last_reply_idx[MSIxIndex],
3111                                         instance->reply_post_host_index_addr[MSIxIndex/8]);
3112                         else
3113                                 writel((MSIxIndex << 24) |
3114                                         fusion->last_reply_idx[MSIxIndex],
3115                                         instance->reply_post_host_index_addr[0]);
3116                         threshold_reply_count = 0;
3117                 }
3118         }
3119
3120         if (!num_completed)
3121                 return IRQ_NONE;
3122
3123         wmb();
3124         if (instance->msix_combined)
3125                 writel(((MSIxIndex & 0x7) << 24) |
3126                         fusion->last_reply_idx[MSIxIndex],
3127                         instance->reply_post_host_index_addr[MSIxIndex/8]);
3128         else
3129                 writel((MSIxIndex << 24) |
3130                         fusion->last_reply_idx[MSIxIndex],
3131                         instance->reply_post_host_index_addr[0]);
3132         megasas_check_and_restore_queue_depth(instance);
3133         return IRQ_HANDLED;
3134 }
3135
3136 /**
3137  * megasas_sync_irqs -  Synchronizes all IRQs owned by adapter
3138  * @instance:                   Adapter soft state
3139  */
3140 void megasas_sync_irqs(unsigned long instance_addr)
3141 {
3142         u32 count, i;
3143         struct megasas_instance *instance =
3144                 (struct megasas_instance *)instance_addr;
3145
3146         count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
3147
3148         for (i = 0; i < count; i++)
3149                 synchronize_irq(pci_irq_vector(instance->pdev, i));
3150 }
3151
3152 /**
3153  * megasas_complete_cmd_dpc_fusion -    Completes command
3154  * @instance:                   Adapter soft state
3155  *
3156  * Tasklet to complete cmds
3157  */
3158 void
3159 megasas_complete_cmd_dpc_fusion(unsigned long instance_addr)
3160 {
3161         struct megasas_instance *instance =
3162                 (struct megasas_instance *)instance_addr;
3163         unsigned long flags;
3164         u32 count, MSIxIndex;
3165
3166         count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
3167
3168         /* If we have already declared adapter dead, donot complete cmds */
3169         spin_lock_irqsave(&instance->hba_lock, flags);
3170         if (atomic_read(&instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR) {
3171                 spin_unlock_irqrestore(&instance->hba_lock, flags);
3172                 return;
3173         }
3174         spin_unlock_irqrestore(&instance->hba_lock, flags);
3175
3176         for (MSIxIndex = 0 ; MSIxIndex < count; MSIxIndex++)
3177                 complete_cmd_fusion(instance, MSIxIndex);
3178 }
3179
3180 /**
3181  * megasas_isr_fusion - isr entry point
3182  */
3183 irqreturn_t megasas_isr_fusion(int irq, void *devp)
3184 {
3185         struct megasas_irq_context *irq_context = devp;
3186         struct megasas_instance *instance = irq_context->instance;
3187         u32 mfiStatus, fw_state, dma_state;
3188
3189         if (instance->mask_interrupts)
3190                 return IRQ_NONE;
3191
3192         if (!instance->msix_vectors) {
3193                 mfiStatus = instance->instancet->clear_intr(instance->reg_set);
3194                 if (!mfiStatus)
3195                         return IRQ_NONE;
3196         }
3197
3198         /* If we are resetting, bail */
3199         if (test_bit(MEGASAS_FUSION_IN_RESET, &instance->reset_flags)) {
3200                 instance->instancet->clear_intr(instance->reg_set);
3201                 return IRQ_HANDLED;
3202         }
3203
3204         if (!complete_cmd_fusion(instance, irq_context->MSIxIndex)) {
3205                 instance->instancet->clear_intr(instance->reg_set);
3206                 /* If we didn't complete any commands, check for FW fault */
3207                 fw_state = instance->instancet->read_fw_status_reg(
3208                         instance->reg_set) & MFI_STATE_MASK;
3209                 dma_state = instance->instancet->read_fw_status_reg
3210                         (instance->reg_set) & MFI_STATE_DMADONE;
3211                 if (instance->crash_dump_drv_support &&
3212                         instance->crash_dump_app_support) {
3213                         /* Start collecting crash, if DMA bit is done */
3214                         if ((fw_state == MFI_STATE_FAULT) && dma_state)
3215                                 schedule_work(&instance->crash_init);
3216                         else if (fw_state == MFI_STATE_FAULT) {
3217                                 if (instance->unload == 0)
3218                                         schedule_work(&instance->work_init);
3219                         }
3220                 } else if (fw_state == MFI_STATE_FAULT) {
3221                         dev_warn(&instance->pdev->dev, "Iop2SysDoorbellInt"
3222                                "for scsi%d\n", instance->host->host_no);
3223                         if (instance->unload == 0)
3224                                 schedule_work(&instance->work_init);
3225                 }
3226         }
3227
3228         return IRQ_HANDLED;
3229 }
3230
3231 /**
3232  * build_mpt_mfi_pass_thru - builds a cmd fo MFI Pass thru
3233  * @instance:                   Adapter soft state
3234  * mfi_cmd:                     megasas_cmd pointer
3235  *
3236  */
3237 void
3238 build_mpt_mfi_pass_thru(struct megasas_instance *instance,
3239                         struct megasas_cmd *mfi_cmd)
3240 {
3241         struct MPI25_IEEE_SGE_CHAIN64 *mpi25_ieee_chain;
3242         struct MPI2_RAID_SCSI_IO_REQUEST *io_req;
3243         struct megasas_cmd_fusion *cmd;
3244         struct fusion_context *fusion;
3245         struct megasas_header *frame_hdr = &mfi_cmd->frame->hdr;
3246
3247         fusion = instance->ctrl_context;
3248
3249         cmd = megasas_get_cmd_fusion(instance,
3250                         instance->max_scsi_cmds + mfi_cmd->index);
3251
3252         /*  Save the smid. To be used for returning the cmd */
3253         mfi_cmd->context.smid = cmd->index;
3254
3255         /*
3256          * For cmds where the flag is set, store the flag and check
3257          * on completion. For cmds with this flag, don't call
3258          * megasas_complete_cmd
3259          */
3260
3261         if (frame_hdr->flags & cpu_to_le16(MFI_FRAME_DONT_POST_IN_REPLY_QUEUE))
3262                 mfi_cmd->flags |= DRV_DCMD_POLLED_MODE;
3263
3264         io_req = cmd->io_request;
3265
3266         if (fusion->adapter_type >= INVADER_SERIES) {
3267                 struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr_end =
3268                         (struct MPI25_IEEE_SGE_CHAIN64 *)&io_req->SGL;
3269                 sgl_ptr_end += fusion->max_sge_in_main_msg - 1;
3270                 sgl_ptr_end->Flags = 0;
3271         }
3272
3273         mpi25_ieee_chain =
3274           (struct MPI25_IEEE_SGE_CHAIN64 *)&io_req->SGL.IeeeChain;
3275
3276         io_req->Function    = MEGASAS_MPI2_FUNCTION_PASSTHRU_IO_REQUEST;
3277         io_req->SGLOffset0  = offsetof(struct MPI2_RAID_SCSI_IO_REQUEST,
3278                                        SGL) / 4;
3279         io_req->ChainOffset = fusion->chain_offset_mfi_pthru;
3280
3281         mpi25_ieee_chain->Address = cpu_to_le64(mfi_cmd->frame_phys_addr);
3282
3283         mpi25_ieee_chain->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT |
3284                 MPI2_IEEE_SGE_FLAGS_IOCPLBNTA_ADDR;
3285
3286         mpi25_ieee_chain->Length = cpu_to_le32(instance->max_chain_frame_sz);
3287 }
3288
3289 /**
3290  * build_mpt_cmd - Calls helper function to build a cmd MFI Pass thru cmd
3291  * @instance:                   Adapter soft state
3292  * @cmd:                        mfi cmd to build
3293  *
3294  */
3295 union MEGASAS_REQUEST_DESCRIPTOR_UNION *
3296 build_mpt_cmd(struct megasas_instance *instance, struct megasas_cmd *cmd)
3297 {
3298         union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc = NULL;
3299         u16 index;
3300
3301         build_mpt_mfi_pass_thru(instance, cmd);
3302         index = cmd->context.smid;
3303
3304         req_desc = megasas_get_request_descriptor(instance, index - 1);
3305
3306         req_desc->Words = 0;
3307         req_desc->SCSIIO.RequestFlags = (MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO <<
3308                                          MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3309
3310         req_desc->SCSIIO.SMID = cpu_to_le16(index);
3311
3312         return req_desc;
3313 }
3314
3315 /**
3316  * megasas_issue_dcmd_fusion - Issues a MFI Pass thru cmd
3317  * @instance:                   Adapter soft state
3318  * @cmd:                        mfi cmd pointer
3319  *
3320  */
3321 void
3322 megasas_issue_dcmd_fusion(struct megasas_instance *instance,
3323                           struct megasas_cmd *cmd)
3324 {
3325         union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
3326
3327         req_desc = build_mpt_cmd(instance, cmd);
3328
3329         megasas_fire_cmd_fusion(instance, req_desc);
3330         return;
3331 }
3332
3333 /**
3334  * megasas_release_fusion -     Reverses the FW initialization
3335  * @instance:                   Adapter soft state
3336  */
3337 void
3338 megasas_release_fusion(struct megasas_instance *instance)
3339 {
3340         megasas_free_cmds(instance);
3341         megasas_free_cmds_fusion(instance);
3342
3343         iounmap(instance->reg_set);
3344
3345         pci_release_selected_regions(instance->pdev, 1<<instance->bar);
3346 }
3347
3348 /**
3349  * megasas_read_fw_status_reg_fusion - returns the current FW status value
3350  * @regs:                       MFI register set
3351  */
3352 static u32
3353 megasas_read_fw_status_reg_fusion(struct megasas_register_set __iomem *regs)
3354 {
3355         return readl(&(regs)->outbound_scratch_pad);
3356 }
3357
3358 /**
3359  * megasas_alloc_host_crash_buffer -    Host buffers for Crash dump collection from Firmware
3360  * @instance:                           Controller's soft instance
3361  * return:                              Number of allocated host crash buffers
3362  */
3363 static void
3364 megasas_alloc_host_crash_buffer(struct megasas_instance *instance)
3365 {
3366         unsigned int i;
3367
3368         instance->crash_buf_pages = get_order(CRASH_DMA_BUF_SIZE);
3369         for (i = 0; i < MAX_CRASH_DUMP_SIZE; i++) {
3370                 instance->crash_buf[i] = (void  *)__get_free_pages(GFP_KERNEL,
3371                                 instance->crash_buf_pages);
3372                 if (!instance->crash_buf[i]) {
3373                         dev_info(&instance->pdev->dev, "Firmware crash dump "
3374                                 "memory allocation failed at index %d\n", i);
3375                         break;
3376                 }
3377                 memset(instance->crash_buf[i], 0,
3378                         ((1 << PAGE_SHIFT) << instance->crash_buf_pages));
3379         }
3380         instance->drv_buf_alloc = i;
3381 }
3382
3383 /**
3384  * megasas_free_host_crash_buffer -     Host buffers for Crash dump collection from Firmware
3385  * @instance:                           Controller's soft instance
3386  */
3387 void
3388 megasas_free_host_crash_buffer(struct megasas_instance *instance)
3389 {
3390         unsigned int i
3391 ;
3392         for (i = 0; i < instance->drv_buf_alloc; i++) {
3393                 if (instance->crash_buf[i])
3394                         free_pages((ulong)instance->crash_buf[i],
3395                                         instance->crash_buf_pages);
3396         }
3397         instance->drv_buf_index = 0;
3398         instance->drv_buf_alloc = 0;
3399         instance->fw_crash_state = UNAVAILABLE;
3400         instance->fw_crash_buffer_size = 0;
3401 }
3402
3403 /**
3404  * megasas_adp_reset_fusion -   For controller reset
3405  * @regs:                               MFI register set
3406  */
3407 static int
3408 megasas_adp_reset_fusion(struct megasas_instance *instance,
3409                          struct megasas_register_set __iomem *regs)
3410 {
3411         u32 host_diag, abs_state, retry;
3412
3413         /* Now try to reset the chip */
3414         writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
3415         writel(MPI2_WRSEQ_1ST_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
3416         writel(MPI2_WRSEQ_2ND_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
3417         writel(MPI2_WRSEQ_3RD_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
3418         writel(MPI2_WRSEQ_4TH_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
3419         writel(MPI2_WRSEQ_5TH_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
3420         writel(MPI2_WRSEQ_6TH_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
3421
3422         /* Check that the diag write enable (DRWE) bit is on */
3423         host_diag = readl(&instance->reg_set->fusion_host_diag);
3424         retry = 0;
3425         while (!(host_diag & HOST_DIAG_WRITE_ENABLE)) {
3426                 msleep(100);
3427                 host_diag = readl(&instance->reg_set->fusion_host_diag);
3428                 if (retry++ == 100) {
3429                         dev_warn(&instance->pdev->dev,
3430                                 "Host diag unlock failed from %s %d\n",
3431                                 __func__, __LINE__);
3432                         break;
3433                 }
3434         }
3435         if (!(host_diag & HOST_DIAG_WRITE_ENABLE))
3436                 return -1;
3437
3438         /* Send chip reset command */
3439         writel(host_diag | HOST_DIAG_RESET_ADAPTER,
3440                 &instance->reg_set->fusion_host_diag);
3441         msleep(3000);
3442
3443         /* Make sure reset adapter bit is cleared */
3444         host_diag = readl(&instance->reg_set->fusion_host_diag);
3445         retry = 0;
3446         while (host_diag & HOST_DIAG_RESET_ADAPTER) {
3447                 msleep(100);
3448                 host_diag = readl(&instance->reg_set->fusion_host_diag);
3449                 if (retry++ == 1000) {
3450                         dev_warn(&instance->pdev->dev,
3451                                 "Diag reset adapter never cleared %s %d\n",
3452                                 __func__, __LINE__);
3453                         break;
3454                 }
3455         }
3456         if (host_diag & HOST_DIAG_RESET_ADAPTER)
3457                 return -1;
3458
3459         abs_state = instance->instancet->read_fw_status_reg(instance->reg_set)
3460                         & MFI_STATE_MASK;
3461         retry = 0;
3462
3463         while ((abs_state <= MFI_STATE_FW_INIT) && (retry++ < 1000)) {
3464                 msleep(100);
3465                 abs_state = instance->instancet->
3466                         read_fw_status_reg(instance->reg_set) & MFI_STATE_MASK;
3467         }
3468         if (abs_state <= MFI_STATE_FW_INIT) {
3469                 dev_warn(&instance->pdev->dev,
3470                         "fw state < MFI_STATE_FW_INIT, state = 0x%x %s %d\n",
3471                         abs_state, __func__, __LINE__);
3472                 return -1;
3473         }
3474
3475         return 0;
3476 }
3477
3478 /**
3479  * megasas_check_reset_fusion - For controller reset check
3480  * @regs:                               MFI register set
3481  */
3482 static int
3483 megasas_check_reset_fusion(struct megasas_instance *instance,
3484                            struct megasas_register_set __iomem *regs)
3485 {
3486         return 0;
3487 }
3488
3489 /* This function waits for outstanding commands on fusion to complete */
3490 int megasas_wait_for_outstanding_fusion(struct megasas_instance *instance,
3491                                         int reason, int *convert)
3492 {
3493         int i, outstanding, retval = 0, hb_seconds_missed = 0;
3494         u32 fw_state;
3495
3496         for (i = 0; i < resetwaittime; i++) {
3497                 /* Check if firmware is in fault state */
3498                 fw_state = instance->instancet->read_fw_status_reg(
3499                         instance->reg_set) & MFI_STATE_MASK;
3500                 if (fw_state == MFI_STATE_FAULT) {
3501                         dev_warn(&instance->pdev->dev, "Found FW in FAULT state,"
3502                                " will reset adapter scsi%d.\n",
3503                                 instance->host->host_no);
3504                         megasas_complete_cmd_dpc_fusion((unsigned long)instance);
3505                         if (instance->requestorId && reason) {
3506                                 dev_warn(&instance->pdev->dev, "SR-IOV Found FW in FAULT"
3507                                 " state while polling during"
3508                                 " I/O timeout handling for %d\n",
3509                                 instance->host->host_no);
3510                                 *convert = 1;
3511                         }
3512
3513                         retval = 1;
3514                         goto out;
3515                 }
3516
3517                 if (reason == MFI_IO_TIMEOUT_OCR) {
3518                         dev_info(&instance->pdev->dev,
3519                                 "MFI IO is timed out, initiating OCR\n");
3520                         megasas_complete_cmd_dpc_fusion((unsigned long)instance);
3521                         retval = 1;
3522                         goto out;
3523                 }
3524
3525                 /* If SR-IOV VF mode & heartbeat timeout, don't wait */
3526                 if (instance->requestorId && !reason) {
3527                         retval = 1;
3528                         goto out;
3529                 }
3530
3531                 /* If SR-IOV VF mode & I/O timeout, check for HB timeout */
3532                 if (instance->requestorId && (reason == SCSIIO_TIMEOUT_OCR)) {
3533                         if (instance->hb_host_mem->HB.fwCounter !=
3534                             instance->hb_host_mem->HB.driverCounter) {
3535                                 instance->hb_host_mem->HB.driverCounter =
3536                                         instance->hb_host_mem->HB.fwCounter;
3537                                 hb_seconds_missed = 0;
3538                         } else {
3539                                 hb_seconds_missed++;
3540                                 if (hb_seconds_missed ==
3541                                     (MEGASAS_SRIOV_HEARTBEAT_INTERVAL_VF/HZ)) {
3542                                         dev_warn(&instance->pdev->dev, "SR-IOV:"
3543                                                " Heartbeat never completed "
3544                                                " while polling during I/O "
3545                                                " timeout handling for "
3546                                                "scsi%d.\n",
3547                                                instance->host->host_no);
3548                                                *convert = 1;
3549                                                retval = 1;
3550                                                goto out;
3551                                 }
3552                         }
3553                 }
3554
3555                 outstanding = atomic_read(&instance->fw_outstanding);
3556                 if (!outstanding)
3557                         goto out;
3558
3559                 if (!(i % MEGASAS_RESET_NOTICE_INTERVAL)) {
3560                         dev_notice(&instance->pdev->dev, "[%2d]waiting for %d "
3561                                "commands to complete for scsi%d\n", i,
3562                                outstanding, instance->host->host_no);
3563                         megasas_complete_cmd_dpc_fusion(
3564                                 (unsigned long)instance);
3565                 }
3566                 msleep(1000);
3567         }
3568
3569         if (atomic_read(&instance->fw_outstanding)) {
3570                 dev_err(&instance->pdev->dev, "pending commands remain after waiting, "
3571                        "will reset adapter scsi%d.\n",
3572                        instance->host->host_no);
3573                 *convert = 1;
3574                 retval = 1;
3575         }
3576 out:
3577         return retval;
3578 }
3579
3580 void  megasas_reset_reply_desc(struct megasas_instance *instance)
3581 {
3582         int i, j, count;
3583         struct fusion_context *fusion;
3584         union MPI2_REPLY_DESCRIPTORS_UNION *reply_desc;
3585
3586         fusion = instance->ctrl_context;
3587         count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
3588         for (i = 0 ; i < count ; i++) {
3589                 fusion->last_reply_idx[i] = 0;
3590                 reply_desc = fusion->reply_frames_desc[i];
3591                 for (j = 0 ; j < fusion->reply_q_depth; j++, reply_desc++)
3592                         reply_desc->Words = cpu_to_le64(ULLONG_MAX);
3593         }
3594 }
3595
3596 /*
3597  * megasas_refire_mgmt_cmd :    Re-fire management commands
3598  * @instance:                           Controller's soft instance
3599 */
3600 void megasas_refire_mgmt_cmd(struct megasas_instance *instance)
3601 {
3602         int j;
3603         struct megasas_cmd_fusion *cmd_fusion;
3604         struct fusion_context *fusion;
3605         struct megasas_cmd *cmd_mfi;
3606         union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
3607         u16 smid;
3608         bool refire_cmd = 0;
3609
3610         fusion = instance->ctrl_context;
3611
3612         /* Re-fire management commands.
3613          * Do not traverse complet MPT frame pool. Start from max_scsi_cmds.
3614          */
3615         for (j = instance->max_scsi_cmds ; j < instance->max_fw_cmds; j++) {
3616                 cmd_fusion = fusion->cmd_list[j];
3617                 cmd_mfi = instance->cmd_list[cmd_fusion->sync_cmd_idx];
3618                 smid = le16_to_cpu(cmd_mfi->context.smid);
3619
3620                 if (!smid)
3621                         continue;
3622                 req_desc = megasas_get_request_descriptor
3623                                         (instance, smid - 1);
3624                 refire_cmd = req_desc && ((cmd_mfi->frame->dcmd.opcode !=
3625                                 cpu_to_le32(MR_DCMD_LD_MAP_GET_INFO)) &&
3626                                  (cmd_mfi->frame->dcmd.opcode !=
3627                                 cpu_to_le32(MR_DCMD_SYSTEM_PD_MAP_GET_INFO)))
3628                                 && !(cmd_mfi->flags & DRV_DCMD_SKIP_REFIRE);
3629                 if (refire_cmd)
3630                         megasas_fire_cmd_fusion(instance, req_desc);
3631                 else
3632                         megasas_return_cmd(instance, cmd_mfi);
3633         }
3634 }
3635
3636 /*
3637  * megasas_track_scsiio : Track SCSI IOs outstanding to a SCSI device
3638  * @instance: per adapter struct
3639  * @channel: the channel assigned by the OS
3640  * @id: the id assigned by the OS
3641  *
3642  * Returns SUCCESS if no IOs pending to SCSI device, else return FAILED
3643  */
3644
3645 static int megasas_track_scsiio(struct megasas_instance *instance,
3646                 int id, int channel)
3647 {
3648         int i, found = 0;
3649         struct megasas_cmd_fusion *cmd_fusion;
3650         struct fusion_context *fusion;
3651         fusion = instance->ctrl_context;
3652
3653         for (i = 0 ; i < instance->max_scsi_cmds; i++) {
3654                 cmd_fusion = fusion->cmd_list[i];
3655                 if (cmd_fusion->scmd &&
3656                         (cmd_fusion->scmd->device->id == id &&
3657                         cmd_fusion->scmd->device->channel == channel)) {
3658                         dev_info(&instance->pdev->dev,
3659                                 "SCSI commands pending to target"
3660                                 "channel %d id %d \tSMID: 0x%x\n",
3661                                 channel, id, cmd_fusion->index);
3662                         scsi_print_command(cmd_fusion->scmd);
3663                         found = 1;
3664                         break;
3665                 }
3666         }
3667
3668         return found ? FAILED : SUCCESS;
3669 }
3670
3671 /**
3672  * megasas_tm_response_code - translation of device response code
3673  * @ioc: per adapter object
3674  * @mpi_reply: MPI reply returned by firmware
3675  *
3676  * Return nothing.
3677  */
3678 static void
3679 megasas_tm_response_code(struct megasas_instance *instance,
3680                 struct MPI2_SCSI_TASK_MANAGE_REPLY *mpi_reply)
3681 {
3682         char *desc;
3683
3684         switch (mpi_reply->ResponseCode) {
3685         case MPI2_SCSITASKMGMT_RSP_TM_COMPLETE:
3686                 desc = "task management request completed";
3687                 break;
3688         case MPI2_SCSITASKMGMT_RSP_INVALID_FRAME:
3689                 desc = "invalid frame";
3690                 break;
3691         case MPI2_SCSITASKMGMT_RSP_TM_NOT_SUPPORTED:
3692                 desc = "task management request not supported";
3693                 break;
3694         case MPI2_SCSITASKMGMT_RSP_TM_FAILED:
3695                 desc = "task management request failed";
3696                 break;
3697         case MPI2_SCSITASKMGMT_RSP_TM_SUCCEEDED:
3698                 desc = "task management request succeeded";
3699                 break;
3700         case MPI2_SCSITASKMGMT_RSP_TM_INVALID_LUN:
3701                 desc = "invalid lun";
3702                 break;
3703         case 0xA:
3704                 desc = "overlapped tag attempted";
3705                 break;
3706         case MPI2_SCSITASKMGMT_RSP_IO_QUEUED_ON_IOC:
3707                 desc = "task queued, however not sent to target";
3708                 break;
3709         default:
3710                 desc = "unknown";
3711                 break;
3712         }
3713         dev_dbg(&instance->pdev->dev, "response_code(%01x): %s\n",
3714                 mpi_reply->ResponseCode, desc);
3715         dev_dbg(&instance->pdev->dev,
3716                 "TerminationCount/DevHandle/Function/TaskType/IOCStat/IOCLoginfo"
3717                 " 0x%x/0x%x/0x%x/0x%x/0x%x/0x%x\n",
3718                 mpi_reply->TerminationCount, mpi_reply->DevHandle,
3719                 mpi_reply->Function, mpi_reply->TaskType,
3720                 mpi_reply->IOCStatus, mpi_reply->IOCLogInfo);
3721 }
3722
3723 /**
3724  * megasas_issue_tm - main routine for sending tm requests
3725  * @instance: per adapter struct
3726  * @device_handle: device handle
3727  * @channel: the channel assigned by the OS
3728  * @id: the id assigned by the OS
3729  * @type: MPI2_SCSITASKMGMT_TASKTYPE__XXX (defined in megaraid_sas_fusion.c)
3730  * @smid_task: smid assigned to the task
3731  * @m_type: TM_MUTEX_ON or TM_MUTEX_OFF
3732  * Context: user
3733  *
3734  * MegaRaid use MPT interface for Task Magement request.
3735  * A generic API for sending task management requests to firmware.
3736  *
3737  * Return SUCCESS or FAILED.
3738  */
3739 static int
3740 megasas_issue_tm(struct megasas_instance *instance, u16 device_handle,
3741         uint channel, uint id, u16 smid_task, u8 type)
3742 {
3743         struct MR_TASK_MANAGE_REQUEST *mr_request;
3744         struct MPI2_SCSI_TASK_MANAGE_REQUEST *mpi_request;
3745         unsigned long timeleft;
3746         struct megasas_cmd_fusion *cmd_fusion;
3747         struct megasas_cmd *cmd_mfi;
3748         union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
3749         struct fusion_context *fusion;
3750         struct megasas_cmd_fusion *scsi_lookup;
3751         int rc;
3752         struct MPI2_SCSI_TASK_MANAGE_REPLY *mpi_reply;
3753
3754         fusion = instance->ctrl_context;
3755
3756         cmd_mfi = megasas_get_cmd(instance);
3757
3758         if (!cmd_mfi) {
3759                 dev_err(&instance->pdev->dev, "Failed from %s %d\n",
3760                         __func__, __LINE__);
3761                 return -ENOMEM;
3762         }
3763
3764         cmd_fusion = megasas_get_cmd_fusion(instance,
3765                         instance->max_scsi_cmds + cmd_mfi->index);
3766
3767         /*  Save the smid. To be used for returning the cmd */
3768         cmd_mfi->context.smid = cmd_fusion->index;
3769
3770         req_desc = megasas_get_request_descriptor(instance,
3771                         (cmd_fusion->index - 1));
3772
3773         cmd_fusion->request_desc = req_desc;
3774         req_desc->Words = 0;
3775
3776         scsi_lookup = fusion->cmd_list[smid_task - 1];
3777
3778         mr_request = (struct MR_TASK_MANAGE_REQUEST *) cmd_fusion->io_request;
3779         memset(mr_request, 0, sizeof(struct MR_TASK_MANAGE_REQUEST));
3780         mpi_request = (struct MPI2_SCSI_TASK_MANAGE_REQUEST *) &mr_request->TmRequest;
3781         mpi_request->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
3782         mpi_request->DevHandle = cpu_to_le16(device_handle);
3783         mpi_request->TaskType = type;
3784         mpi_request->TaskMID = cpu_to_le16(smid_task);
3785         mpi_request->LUN[1] = 0;
3786
3787
3788         req_desc = cmd_fusion->request_desc;
3789         req_desc->HighPriority.SMID = cpu_to_le16(cmd_fusion->index);
3790         req_desc->HighPriority.RequestFlags =
3791                 (MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY <<
3792                 MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3793         req_desc->HighPriority.MSIxIndex =  0;
3794         req_desc->HighPriority.LMID = 0;
3795         req_desc->HighPriority.Reserved1 = 0;
3796
3797         if (channel < MEGASAS_MAX_PD_CHANNELS)
3798                 mr_request->tmReqFlags.isTMForPD = 1;
3799         else
3800                 mr_request->tmReqFlags.isTMForLD = 1;
3801
3802         init_completion(&cmd_fusion->done);
3803         megasas_fire_cmd_fusion(instance, req_desc);
3804
3805         timeleft = wait_for_completion_timeout(&cmd_fusion->done, 50 * HZ);
3806
3807         if (!timeleft) {
3808                 dev_err(&instance->pdev->dev,
3809                         "task mgmt type 0x%x timed out\n", type);
3810                 cmd_mfi->flags |= DRV_DCMD_SKIP_REFIRE;
3811                 mutex_unlock(&instance->reset_mutex);
3812                 rc = megasas_reset_fusion(instance->host, MFI_IO_TIMEOUT_OCR);
3813                 mutex_lock(&instance->reset_mutex);
3814                 return rc;
3815         }
3816
3817         mpi_reply = (struct MPI2_SCSI_TASK_MANAGE_REPLY *) &mr_request->TMReply;
3818         megasas_tm_response_code(instance, mpi_reply);
3819
3820         megasas_return_cmd(instance, cmd_mfi);
3821         rc = SUCCESS;
3822         switch (type) {
3823         case MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK:
3824                 if (scsi_lookup->scmd == NULL)
3825                         break;
3826                 else {
3827                         instance->instancet->disable_intr(instance);
3828                         megasas_sync_irqs((unsigned long)instance);
3829                         megasas_complete_cmd_dpc_fusion
3830                                         ((unsigned long)instance);
3831                         instance->instancet->enable_intr(instance);
3832                         if (scsi_lookup->scmd == NULL)
3833                                 break;
3834                 }
3835                 rc = FAILED;
3836                 break;
3837
3838         case MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET:
3839                 if ((channel == 0xFFFFFFFF) && (id == 0xFFFFFFFF))
3840                         break;
3841                 instance->instancet->disable_intr(instance);
3842                 msleep(1000);
3843                 megasas_complete_cmd_dpc_fusion
3844                                 ((unsigned long)instance);
3845                 rc = megasas_track_scsiio(instance, id, channel);
3846                 instance->instancet->enable_intr(instance);
3847
3848                 break;
3849         case MPI2_SCSITASKMGMT_TASKTYPE_ABRT_TASK_SET:
3850         case MPI2_SCSITASKMGMT_TASKTYPE_QUERY_TASK:
3851                 break;
3852         default:
3853                 rc = FAILED;
3854                 break;
3855         }
3856
3857         return rc;
3858
3859 }
3860
3861 /*
3862  * megasas_fusion_smid_lookup : Look for fusion command correpspodning to SCSI
3863  * @instance: per adapter struct
3864  *
3865  * Return Non Zero index, if SMID found in outstanding commands
3866  */
3867 static u16 megasas_fusion_smid_lookup(struct scsi_cmnd *scmd)
3868 {
3869         int i, ret = 0;
3870         struct megasas_instance *instance;
3871         struct megasas_cmd_fusion *cmd_fusion;
3872         struct fusion_context *fusion;
3873
3874         instance = (struct megasas_instance *)scmd->device->host->hostdata;
3875
3876         fusion = instance->ctrl_context;
3877
3878         for (i = 0; i < instance->max_scsi_cmds; i++) {
3879                 cmd_fusion = fusion->cmd_list[i];
3880                 if (cmd_fusion->scmd && (cmd_fusion->scmd == scmd)) {
3881                         scmd_printk(KERN_NOTICE, scmd, "Abort request is for"
3882                                 " SMID: %d\n", cmd_fusion->index);
3883                         ret = cmd_fusion->index;
3884                         break;
3885                 }
3886         }
3887
3888         return ret;
3889 }
3890
3891 /*
3892 * megasas_get_tm_devhandle - Get devhandle for TM request
3893 * @sdev-                     OS provided scsi device
3894 *
3895 * Returns-                   devhandle/targetID of SCSI device
3896 */
3897 static u16 megasas_get_tm_devhandle(struct scsi_device *sdev)
3898 {
3899         u16 pd_index = 0;
3900         u32 device_id;
3901         struct megasas_instance *instance;
3902         struct fusion_context *fusion;
3903         struct MR_PD_CFG_SEQ_NUM_SYNC *pd_sync;
3904         u16 devhandle = (u16)ULONG_MAX;
3905
3906         instance = (struct megasas_instance *)sdev->host->hostdata;
3907         fusion = instance->ctrl_context;
3908
3909         if (!MEGASAS_IS_LOGICAL(sdev)) {
3910                 if (instance->use_seqnum_jbod_fp) {
3911                         pd_index = (sdev->channel * MEGASAS_MAX_DEV_PER_CHANNEL)
3912                                     + sdev->id;
3913                         pd_sync = (void *)fusion->pd_seq_sync
3914                                         [(instance->pd_seq_map_id - 1) & 1];
3915                         devhandle = pd_sync->seq[pd_index].devHandle;
3916                 } else
3917                         sdev_printk(KERN_ERR, sdev, "Firmware expose tmCapable"
3918                                 " without JBOD MAP support from %s %d\n", __func__, __LINE__);
3919         } else {
3920                 device_id = ((sdev->channel % 2) * MEGASAS_MAX_DEV_PER_CHANNEL)
3921                                 + sdev->id;
3922                 devhandle = device_id;
3923         }
3924
3925         return devhandle;
3926 }
3927
3928 /*
3929  * megasas_task_abort_fusion : SCSI task abort function for fusion adapters
3930  * @scmd : pointer to scsi command object
3931  *
3932  * Return SUCCESS, if command aborted else FAILED
3933  */
3934
3935 int megasas_task_abort_fusion(struct scsi_cmnd *scmd)
3936 {
3937         struct megasas_instance *instance;
3938         u16 smid, devhandle;
3939         struct fusion_context *fusion;
3940         int ret;
3941         struct MR_PRIV_DEVICE *mr_device_priv_data;
3942         mr_device_priv_data = scmd->device->hostdata;
3943
3944
3945         instance = (struct megasas_instance *)scmd->device->host->hostdata;
3946         fusion = instance->ctrl_context;
3947
3948         scmd_printk(KERN_INFO, scmd, "task abort called for scmd(%p)\n", scmd);
3949         scsi_print_command(scmd);
3950
3951         if (atomic_read(&instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) {
3952                 dev_err(&instance->pdev->dev, "Controller is not OPERATIONAL,"
3953                 "SCSI host:%d\n", instance->host->host_no);
3954                 ret = FAILED;
3955                 return ret;
3956         }
3957
3958         if (!mr_device_priv_data) {
3959                 sdev_printk(KERN_INFO, scmd->device, "device been deleted! "
3960                         "scmd(%p)\n", scmd);
3961                 scmd->result = DID_NO_CONNECT << 16;
3962                 ret = SUCCESS;
3963                 goto out;
3964         }
3965
3966
3967         if (!mr_device_priv_data->is_tm_capable) {
3968                 ret = FAILED;
3969                 goto out;
3970         }
3971
3972         mutex_lock(&instance->reset_mutex);
3973
3974         smid = megasas_fusion_smid_lookup(scmd);
3975
3976         if (!smid) {
3977                 ret = SUCCESS;
3978                 scmd_printk(KERN_NOTICE, scmd, "Command for which abort is"
3979                         " issued is not found in oustanding commands\n");
3980                 mutex_unlock(&instance->reset_mutex);
3981                 goto out;
3982         }
3983
3984         devhandle = megasas_get_tm_devhandle(scmd->device);
3985
3986         if (devhandle == (u16)ULONG_MAX) {
3987                 ret = SUCCESS;
3988                 sdev_printk(KERN_INFO, scmd->device,
3989                         "task abort issued for invalid devhandle\n");
3990                 mutex_unlock(&instance->reset_mutex);
3991                 goto out;
3992         }
3993         sdev_printk(KERN_INFO, scmd->device,
3994                 "attempting task abort! scmd(%p) tm_dev_handle 0x%x\n",
3995                 scmd, devhandle);
3996
3997         mr_device_priv_data->tm_busy = 1;
3998         ret = megasas_issue_tm(instance, devhandle,
3999                         scmd->device->channel, scmd->device->id, smid,
4000                         MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK);
4001         mr_device_priv_data->tm_busy = 0;
4002
4003         mutex_unlock(&instance->reset_mutex);
4004 out:
4005         sdev_printk(KERN_INFO, scmd->device, "task abort: %s scmd(%p)\n",
4006                         ((ret == SUCCESS) ? "SUCCESS" : "FAILED"), scmd);
4007
4008         return ret;
4009 }
4010
4011 /*
4012  * megasas_reset_target_fusion : target reset function for fusion adapters
4013  * scmd: SCSI command pointer
4014  *
4015  * Returns SUCCESS if all commands associated with target aborted else FAILED
4016  */
4017
4018 int megasas_reset_target_fusion(struct scsi_cmnd *scmd)
4019 {
4020
4021         struct megasas_instance *instance;
4022         int ret = FAILED;
4023         u16 devhandle;
4024         struct fusion_context *fusion;
4025         struct MR_PRIV_DEVICE *mr_device_priv_data;
4026         mr_device_priv_data = scmd->device->hostdata;
4027
4028         instance = (struct megasas_instance *)scmd->device->host->hostdata;
4029         fusion = instance->ctrl_context;
4030
4031         sdev_printk(KERN_INFO, scmd->device,
4032                     "target reset called for scmd(%p)\n", scmd);
4033
4034         if (atomic_read(&instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) {
4035                 dev_err(&instance->pdev->dev, "Controller is not OPERATIONAL,"
4036                 "SCSI host:%d\n", instance->host->host_no);
4037                 ret = FAILED;
4038                 return ret;
4039         }
4040
4041         if (!mr_device_priv_data) {
4042                 sdev_printk(KERN_INFO, scmd->device, "device been deleted! "
4043                         "scmd(%p)\n", scmd);
4044                 scmd->result = DID_NO_CONNECT << 16;
4045                 ret = SUCCESS;
4046                 goto out;
4047         }
4048
4049
4050         if (!mr_device_priv_data->is_tm_capable) {
4051                 ret = FAILED;
4052                 goto out;
4053         }
4054
4055         mutex_lock(&instance->reset_mutex);
4056         devhandle = megasas_get_tm_devhandle(scmd->device);
4057
4058         if (devhandle == (u16)ULONG_MAX) {
4059                 ret = SUCCESS;
4060                 sdev_printk(KERN_INFO, scmd->device,
4061                         "target reset issued for invalid devhandle\n");
4062                 mutex_unlock(&instance->reset_mutex);
4063                 goto out;
4064         }
4065
4066         sdev_printk(KERN_INFO, scmd->device,
4067                 "attempting target reset! scmd(%p) tm_dev_handle 0x%x\n",
4068                 scmd, devhandle);
4069         mr_device_priv_data->tm_busy = 1;
4070         ret = megasas_issue_tm(instance, devhandle,
4071                         scmd->device->channel, scmd->device->id, 0,
4072                         MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET);
4073         mr_device_priv_data->tm_busy = 0;
4074         mutex_unlock(&instance->reset_mutex);
4075 out:
4076         scmd_printk(KERN_NOTICE, scmd, "megasas: target reset %s!!\n",
4077                 (ret == SUCCESS) ? "SUCCESS" : "FAILED");
4078
4079         return ret;
4080 }
4081
4082 /*SRIOV get other instance in cluster if any*/
4083 struct megasas_instance *megasas_get_peer_instance(struct megasas_instance *instance)
4084 {
4085         int i;
4086
4087         for (i = 0; i < MAX_MGMT_ADAPTERS; i++) {
4088                 if (megasas_mgmt_info.instance[i] &&
4089                         (megasas_mgmt_info.instance[i] != instance) &&
4090                          megasas_mgmt_info.instance[i]->requestorId &&
4091                          megasas_mgmt_info.instance[i]->peerIsPresent &&
4092                         (memcmp((megasas_mgmt_info.instance[i]->clusterId),
4093                         instance->clusterId, MEGASAS_CLUSTER_ID_SIZE) == 0))
4094                         return megasas_mgmt_info.instance[i];
4095         }
4096         return NULL;
4097 }
4098
4099 /* Check for a second path that is currently UP */
4100 int megasas_check_mpio_paths(struct megasas_instance *instance,
4101         struct scsi_cmnd *scmd)
4102 {
4103         struct megasas_instance *peer_instance = NULL;
4104         int retval = (DID_REQUEUE << 16);
4105
4106         if (instance->peerIsPresent) {
4107                 peer_instance = megasas_get_peer_instance(instance);
4108                 if ((peer_instance) &&
4109                         (atomic_read(&peer_instance->adprecovery) ==
4110                         MEGASAS_HBA_OPERATIONAL))
4111                         retval = (DID_NO_CONNECT << 16);
4112         }
4113         return retval;
4114 }
4115
4116 /* Core fusion reset function */
4117 int megasas_reset_fusion(struct Scsi_Host *shost, int reason)
4118 {
4119         int retval = SUCCESS, i, j, convert = 0;
4120         struct megasas_instance *instance;
4121         struct megasas_cmd_fusion *cmd_fusion, *r1_cmd;
4122         struct fusion_context *fusion;
4123         u32 abs_state, status_reg, reset_adapter;
4124         u32 io_timeout_in_crash_mode = 0;
4125         struct scsi_cmnd *scmd_local = NULL;
4126         struct scsi_device *sdev;
4127
4128         instance = (struct megasas_instance *)shost->hostdata;
4129         fusion = instance->ctrl_context;
4130
4131         mutex_lock(&instance->reset_mutex);
4132
4133         if (atomic_read(&instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR) {
4134                 dev_warn(&instance->pdev->dev, "Hardware critical error, "
4135                        "returning FAILED for scsi%d.\n",
4136                         instance->host->host_no);
4137                 mutex_unlock(&instance->reset_mutex);
4138                 return FAILED;
4139         }
4140         status_reg = instance->instancet->read_fw_status_reg(instance->reg_set);
4141         abs_state = status_reg & MFI_STATE_MASK;
4142
4143         /* IO timeout detected, forcibly put FW in FAULT state */
4144         if (abs_state != MFI_STATE_FAULT && instance->crash_dump_buf &&
4145                 instance->crash_dump_app_support && reason) {
4146                 dev_info(&instance->pdev->dev, "IO/DCMD timeout is detected, "
4147                         "forcibly FAULT Firmware\n");
4148                 atomic_set(&instance->adprecovery, MEGASAS_ADPRESET_SM_INFAULT);
4149                 status_reg = readl(&instance->reg_set->doorbell);
4150                 writel(status_reg | MFI_STATE_FORCE_OCR,
4151                         &instance->reg_set->doorbell);
4152                 readl(&instance->reg_set->doorbell);
4153                 mutex_unlock(&instance->reset_mutex);
4154                 do {
4155                         ssleep(3);
4156                         io_timeout_in_crash_mode++;
4157                         dev_dbg(&instance->pdev->dev, "waiting for [%d] "
4158                                 "seconds for crash dump collection and OCR "
4159                                 "to be done\n", (io_timeout_in_crash_mode * 3));
4160                 } while ((atomic_read(&instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) &&
4161                         (io_timeout_in_crash_mode < 80));
4162
4163                 if (atomic_read(&instance->adprecovery) == MEGASAS_HBA_OPERATIONAL) {
4164                         dev_info(&instance->pdev->dev, "OCR done for IO "
4165                                 "timeout case\n");
4166                         retval = SUCCESS;
4167                 } else {
4168                         dev_info(&instance->pdev->dev, "Controller is not "
4169                                 "operational after 240 seconds wait for IO "
4170                                 "timeout case in FW crash dump mode\n do "
4171                                 "OCR/kill adapter\n");
4172                         retval = megasas_reset_fusion(shost, 0);
4173                 }
4174                 return retval;
4175         }
4176
4177         if (instance->requestorId && !instance->skip_heartbeat_timer_del)
4178                 del_timer_sync(&instance->sriov_heartbeat_timer);
4179         set_bit(MEGASAS_FUSION_IN_RESET, &instance->reset_flags);
4180         atomic_set(&instance->adprecovery, MEGASAS_ADPRESET_SM_POLLING);
4181         instance->instancet->disable_intr(instance);
4182         megasas_sync_irqs((unsigned long)instance);
4183
4184         /* First try waiting for commands to complete */
4185         if (megasas_wait_for_outstanding_fusion(instance, reason,
4186                                                 &convert)) {
4187                 atomic_set(&instance->adprecovery, MEGASAS_ADPRESET_SM_INFAULT);
4188                 dev_warn(&instance->pdev->dev, "resetting fusion "
4189                        "adapter scsi%d.\n", instance->host->host_no);
4190                 if (convert)
4191                         reason = 0;
4192
4193                 if (megasas_dbg_lvl & OCR_LOGS)
4194                         dev_info(&instance->pdev->dev, "\nPending SCSI commands:\n");
4195
4196                 /* Now return commands back to the OS */
4197                 for (i = 0 ; i < instance->max_scsi_cmds; i++) {
4198                         cmd_fusion = fusion->cmd_list[i];
4199                         /*check for extra commands issued by driver*/
4200                         if (instance->is_ventura) {
4201                                 r1_cmd = fusion->cmd_list[i + instance->max_fw_cmds];
4202                                 megasas_return_cmd_fusion(instance, r1_cmd);
4203                         }
4204                         scmd_local = cmd_fusion->scmd;
4205                         if (cmd_fusion->scmd) {
4206                                 if (megasas_dbg_lvl & OCR_LOGS) {
4207                                         sdev_printk(KERN_INFO,
4208                                                 cmd_fusion->scmd->device, "SMID: 0x%x\n",
4209                                                 cmd_fusion->index);
4210                                         scsi_print_command(cmd_fusion->scmd);
4211                                 }
4212
4213                                 scmd_local->result =
4214                                         megasas_check_mpio_paths(instance,
4215                                                         scmd_local);
4216                                 if (instance->ldio_threshold &&
4217                                         megasas_cmd_type(scmd_local) == READ_WRITE_LDIO)
4218                                         atomic_dec(&instance->ldio_outstanding);
4219                                 megasas_return_cmd_fusion(instance, cmd_fusion);
4220                                 scsi_dma_unmap(scmd_local);
4221                                 scmd_local->scsi_done(scmd_local);
4222                         }
4223                 }
4224
4225                 atomic_set(&instance->fw_outstanding, 0);
4226
4227                 status_reg = instance->instancet->read_fw_status_reg(
4228                         instance->reg_set);
4229                 abs_state = status_reg & MFI_STATE_MASK;
4230                 reset_adapter = status_reg & MFI_RESET_ADAPTER;
4231                 if (instance->disableOnlineCtrlReset ||
4232                     (abs_state == MFI_STATE_FAULT && !reset_adapter)) {
4233                         /* Reset not supported, kill adapter */
4234                         dev_warn(&instance->pdev->dev, "Reset not supported"
4235                                ", killing adapter scsi%d.\n",
4236                                 instance->host->host_no);
4237                         megaraid_sas_kill_hba(instance);
4238                         instance->skip_heartbeat_timer_del = 1;
4239                         retval = FAILED;
4240                         goto out;
4241                 }
4242
4243                 /* Let SR-IOV VF & PF sync up if there was a HB failure */
4244                 if (instance->requestorId && !reason) {
4245                         msleep(MEGASAS_OCR_SETTLE_TIME_VF);
4246                         goto transition_to_ready;
4247                 }
4248
4249                 /* Now try to reset the chip */
4250                 for (i = 0; i < MEGASAS_FUSION_MAX_RESET_TRIES; i++) {
4251
4252                         if (instance->instancet->adp_reset
4253                                 (instance, instance->reg_set))
4254                                 continue;
4255 transition_to_ready:
4256                         /* Wait for FW to become ready */
4257                         if (megasas_transition_to_ready(instance, 1)) {
4258                                 dev_warn(&instance->pdev->dev,
4259                                         "Failed to transition controller to ready for "
4260                                         "scsi%d.\n", instance->host->host_no);
4261                                 if (instance->requestorId && !reason)
4262                                         goto fail_kill_adapter;
4263                                 else
4264                                         continue;
4265                         }
4266                         megasas_reset_reply_desc(instance);
4267                         megasas_fusion_update_can_queue(instance, OCR_CONTEXT);
4268
4269                         if (megasas_ioc_init_fusion(instance)) {
4270                                 dev_warn(&instance->pdev->dev,
4271                                        "megasas_ioc_init_fusion() failed! for "
4272                                        "scsi%d\n", instance->host->host_no);
4273                                 if (instance->requestorId && !reason)
4274                                         goto fail_kill_adapter;
4275                                 else
4276                                         continue;
4277                         }
4278
4279                         megasas_refire_mgmt_cmd(instance);
4280
4281                         if (megasas_get_ctrl_info(instance)) {
4282                                 dev_info(&instance->pdev->dev,
4283                                         "Failed from %s %d\n",
4284                                         __func__, __LINE__);
4285                                 megaraid_sas_kill_hba(instance);
4286                                 retval = FAILED;
4287                                 goto out;
4288                         }
4289                         /* Reset load balance info */
4290                         if (fusion->load_balance_info)
4291                                 memset(fusion->load_balance_info, 0,
4292                                        (sizeof(struct LD_LOAD_BALANCE_INFO) *
4293                                        MAX_LOGICAL_DRIVES_EXT));
4294
4295                         if (!megasas_get_map_info(instance))
4296                                 megasas_sync_map_info(instance);
4297
4298                         megasas_setup_jbod_map(instance);
4299
4300                         shost_for_each_device(sdev, shost)
4301                                 megasas_set_dynamic_target_properties(sdev);
4302
4303                         /* reset stream detection array */
4304                         if (instance->is_ventura) {
4305                                 for (j = 0; j < MAX_LOGICAL_DRIVES_EXT; ++j) {
4306                                         memset(fusion->stream_detect_by_ld[j],
4307                                         0, sizeof(struct LD_STREAM_DETECT));
4308                                  fusion->stream_detect_by_ld[j]->mru_bit_map
4309                                                 = MR_STREAM_BITMAP;
4310                                 }
4311                         }
4312
4313                         clear_bit(MEGASAS_FUSION_IN_RESET,
4314                                   &instance->reset_flags);
4315                         instance->instancet->enable_intr(instance);
4316                         atomic_set(&instance->adprecovery, MEGASAS_HBA_OPERATIONAL);
4317
4318                         /* Restart SR-IOV heartbeat */
4319                         if (instance->requestorId) {
4320                                 if (!megasas_sriov_start_heartbeat(instance, 0))
4321                                         megasas_start_timer(instance,
4322                                                             &instance->sriov_heartbeat_timer,
4323                                                             megasas_sriov_heartbeat_handler,
4324                                                             MEGASAS_SRIOV_HEARTBEAT_INTERVAL_VF);
4325                                 else
4326                                         instance->skip_heartbeat_timer_del = 1;
4327                         }
4328
4329                         /* Adapter reset completed successfully */
4330                         dev_warn(&instance->pdev->dev, "Reset "
4331                                "successful for scsi%d.\n",
4332                                 instance->host->host_no);
4333
4334                         if (instance->crash_dump_drv_support &&
4335                                 instance->crash_dump_app_support)
4336                                 megasas_set_crash_dump_params(instance,
4337                                         MR_CRASH_BUF_TURN_ON);
4338                         else
4339                                 megasas_set_crash_dump_params(instance,
4340                                         MR_CRASH_BUF_TURN_OFF);
4341
4342                         retval = SUCCESS;
4343                         goto out;
4344                 }
4345 fail_kill_adapter:
4346                 /* Reset failed, kill the adapter */
4347                 dev_warn(&instance->pdev->dev, "Reset failed, killing "
4348                        "adapter scsi%d.\n", instance->host->host_no);
4349                 megaraid_sas_kill_hba(instance);
4350                 instance->skip_heartbeat_timer_del = 1;
4351                 retval = FAILED;
4352         } else {
4353                 /* For VF: Restart HB timer if we didn't OCR */
4354                 if (instance->requestorId) {
4355                         megasas_start_timer(instance,
4356                                             &instance->sriov_heartbeat_timer,
4357                                             megasas_sriov_heartbeat_handler,
4358                                             MEGASAS_SRIOV_HEARTBEAT_INTERVAL_VF);
4359                 }
4360                 clear_bit(MEGASAS_FUSION_IN_RESET, &instance->reset_flags);
4361                 instance->instancet->enable_intr(instance);
4362                 atomic_set(&instance->adprecovery, MEGASAS_HBA_OPERATIONAL);
4363         }
4364 out:
4365         clear_bit(MEGASAS_FUSION_IN_RESET, &instance->reset_flags);
4366         mutex_unlock(&instance->reset_mutex);
4367         return retval;
4368 }
4369
4370 /* Fusion Crash dump collection work queue */
4371 void  megasas_fusion_crash_dump_wq(struct work_struct *work)
4372 {
4373         struct megasas_instance *instance =
4374                 container_of(work, struct megasas_instance, crash_init);
4375         u32 status_reg;
4376         u8 partial_copy = 0;
4377
4378
4379         status_reg = instance->instancet->read_fw_status_reg(instance->reg_set);
4380
4381         /*
4382          * Allocate host crash buffers to copy data from 1 MB DMA crash buffer
4383          * to host crash buffers
4384          */
4385         if (instance->drv_buf_index == 0) {
4386                 /* Buffer is already allocated for old Crash dump.
4387                  * Do OCR and do not wait for crash dump collection
4388                  */
4389                 if (instance->drv_buf_alloc) {
4390                         dev_info(&instance->pdev->dev, "earlier crash dump is "
4391                                 "not yet copied by application, ignoring this "
4392                                 "crash dump and initiating OCR\n");
4393                         status_reg |= MFI_STATE_CRASH_DUMP_DONE;
4394                         writel(status_reg,
4395                                 &instance->reg_set->outbound_scratch_pad);
4396                         readl(&instance->reg_set->outbound_scratch_pad);
4397                         return;
4398                 }
4399                 megasas_alloc_host_crash_buffer(instance);
4400                 dev_info(&instance->pdev->dev, "Number of host crash buffers "
4401                         "allocated: %d\n", instance->drv_buf_alloc);
4402         }
4403
4404         /*
4405          * Driver has allocated max buffers, which can be allocated
4406          * and FW has more crash dump data, then driver will
4407          * ignore the data.
4408          */
4409         if (instance->drv_buf_index >= (instance->drv_buf_alloc)) {
4410                 dev_info(&instance->pdev->dev, "Driver is done copying "
4411                         "the buffer: %d\n", instance->drv_buf_alloc);
4412                 status_reg |= MFI_STATE_CRASH_DUMP_DONE;
4413                 partial_copy = 1;
4414         } else {
4415                 memcpy(instance->crash_buf[instance->drv_buf_index],
4416                         instance->crash_dump_buf, CRASH_DMA_BUF_SIZE);
4417                 instance->drv_buf_index++;
4418                 status_reg &= ~MFI_STATE_DMADONE;
4419         }
4420
4421         if (status_reg & MFI_STATE_CRASH_DUMP_DONE) {
4422                 dev_info(&instance->pdev->dev, "Crash Dump is available,number "
4423                         "of copied buffers: %d\n", instance->drv_buf_index);
4424                 instance->fw_crash_buffer_size =  instance->drv_buf_index;
4425                 instance->fw_crash_state = AVAILABLE;
4426                 instance->drv_buf_index = 0;
4427                 writel(status_reg, &instance->reg_set->outbound_scratch_pad);
4428                 readl(&instance->reg_set->outbound_scratch_pad);
4429                 if (!partial_copy)
4430                         megasas_reset_fusion(instance->host, 0);
4431         } else {
4432                 writel(status_reg, &instance->reg_set->outbound_scratch_pad);
4433                 readl(&instance->reg_set->outbound_scratch_pad);
4434         }
4435 }
4436
4437
4438 /* Fusion OCR work queue */
4439 void megasas_fusion_ocr_wq(struct work_struct *work)
4440 {
4441         struct megasas_instance *instance =
4442                 container_of(work, struct megasas_instance, work_init);
4443
4444         megasas_reset_fusion(instance->host, 0);
4445 }
4446
4447 /* Allocate fusion context */
4448 int
4449 megasas_alloc_fusion_context(struct megasas_instance *instance)
4450 {
4451         struct fusion_context *fusion;
4452
4453         instance->ctrl_context_pages = get_order(sizeof(struct fusion_context));
4454         instance->ctrl_context = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
4455                 instance->ctrl_context_pages);
4456         if (!instance->ctrl_context) {
4457                 /* fall back to using vmalloc for fusion_context */
4458                 instance->ctrl_context = vzalloc(sizeof(struct fusion_context));
4459                 if (!instance->ctrl_context) {
4460                         dev_err(&instance->pdev->dev, "Failed from %s %d\n", __func__, __LINE__);
4461                         return -ENOMEM;
4462                 }
4463         }
4464
4465         fusion = instance->ctrl_context;
4466
4467         fusion->load_balance_info_pages = get_order(MAX_LOGICAL_DRIVES_EXT *
4468                 sizeof(struct LD_LOAD_BALANCE_INFO));
4469         fusion->load_balance_info =
4470                 (struct LD_LOAD_BALANCE_INFO *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
4471                 fusion->load_balance_info_pages);
4472         if (!fusion->load_balance_info) {
4473                 fusion->load_balance_info = vzalloc(MAX_LOGICAL_DRIVES_EXT *
4474                         sizeof(struct LD_LOAD_BALANCE_INFO));
4475                 if (!fusion->load_balance_info)
4476                         dev_err(&instance->pdev->dev, "Failed to allocate load_balance_info, "
4477                                 "continuing without Load Balance support\n");
4478         }
4479
4480         return 0;
4481 }
4482
4483 void
4484 megasas_free_fusion_context(struct megasas_instance *instance)
4485 {
4486         struct fusion_context *fusion = instance->ctrl_context;
4487
4488         if (fusion) {
4489                 if (fusion->load_balance_info) {
4490                         if (is_vmalloc_addr(fusion->load_balance_info))
4491                                 vfree(fusion->load_balance_info);
4492                         else
4493                                 free_pages((ulong)fusion->load_balance_info,
4494                                         fusion->load_balance_info_pages);
4495                 }
4496
4497                 if (is_vmalloc_addr(fusion))
4498                         vfree(fusion);
4499                 else
4500                         free_pages((ulong)fusion,
4501                                 instance->ctrl_context_pages);
4502         }
4503 }
4504
4505 struct megasas_instance_template megasas_instance_template_fusion = {
4506         .enable_intr = megasas_enable_intr_fusion,
4507         .disable_intr = megasas_disable_intr_fusion,
4508         .clear_intr = megasas_clear_intr_fusion,
4509         .read_fw_status_reg = megasas_read_fw_status_reg_fusion,
4510         .adp_reset = megasas_adp_reset_fusion,
4511         .check_reset = megasas_check_reset_fusion,
4512         .service_isr = megasas_isr_fusion,
4513         .tasklet = megasas_complete_cmd_dpc_fusion,
4514         .init_adapter = megasas_init_adapter_fusion,
4515         .build_and_issue_cmd = megasas_build_and_issue_cmd_fusion,
4516         .issue_dcmd = megasas_issue_dcmd_fusion,
4517 };