drivers/target/iscsi/iscsi_target.c

   1 /*******************************************************************************
   2  * This file contains main functions related to the iSCSI Target Core Driver.
   3  *
   4  * (c) Copyright 2007-2013 Datera, Inc.
   5  *
   6  * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  ******************************************************************************/
  18
  19 #include <crypto/hash.h>
  20 #include <linux/string.h>
  21 #include <linux/kthread.h>
  22 #include <linux/completion.h>
  23 #include <linux/module.h>
  24 #include <linux/vmalloc.h>
  25 #include <linux/idr.h>
  26 #include <linux/delay.h>
  27 #include <linux/sched/signal.h>
  28 #include <asm/unaligned.h>
  29 #include <net/ipv6.h>
  30 #include <scsi/scsi_proto.h>
  31 #include <scsi/iscsi_proto.h>
  32 #include <scsi/scsi_tcq.h>
  33 #include <target/target_core_base.h>
  34 #include <target/target_core_fabric.h>
  35
  36 #include <target/iscsi/iscsi_target_core.h>
  37 #include "iscsi_target_parameters.h"
  38 #include "iscsi_target_seq_pdu_list.h"
  39 #include "iscsi_target_datain_values.h"
  40 #include "iscsi_target_erl0.h"
  41 #include "iscsi_target_erl1.h"
  42 #include "iscsi_target_erl2.h"
  43 #include "iscsi_target_login.h"
  44 #include "iscsi_target_tmr.h"
  45 #include "iscsi_target_tpg.h"
  46 #include "iscsi_target_util.h"
  47 #include "iscsi_target.h"
  48 #include "iscsi_target_device.h"
  49 #include <target/iscsi/iscsi_target_stat.h>
  50
  51 #include <target/iscsi/iscsi_transport.h>
  52
  53 static LIST_HEAD(g_tiqn_list);
  54 static LIST_HEAD(g_np_list);
  55 static DEFINE_SPINLOCK(tiqn_lock);
  56 static DEFINE_MUTEX(np_lock);
  57
  58 static struct idr tiqn_idr;
  59 struct idr sess_idr;
  60 struct mutex auth_id_lock;
  61 spinlock_t sess_idr_lock;
  62
  63 struct iscsit_global *iscsit_global;
  64
  65 struct kmem_cache *lio_qr_cache;
  66 struct kmem_cache *lio_dr_cache;
  67 struct kmem_cache *lio_ooo_cache;
  68 struct kmem_cache *lio_r2t_cache;
  69
  70 static int iscsit_handle_immediate_data(struct iscsi_cmd *,
  71                         struct iscsi_scsi_req *, u32);
  72
  73 struct iscsi_tiqn *iscsit_get_tiqn_for_login(unsigned char *buf)
  74 {
  75         struct iscsi_tiqn *tiqn = NULL;
  76
  77         spin_lock(&tiqn_lock);
  78         list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) {
  79                 if (!strcmp(tiqn->tiqn, buf)) {
  80
  81                         spin_lock(&tiqn->tiqn_state_lock);
  82                         if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) {
  83                                 tiqn->tiqn_access_count++;
  84                                 spin_unlock(&tiqn->tiqn_state_lock);
  85                                 spin_unlock(&tiqn_lock);
  86                                 return tiqn;
  87                         }
  88                         spin_unlock(&tiqn->tiqn_state_lock);
  89                 }
  90         }
  91         spin_unlock(&tiqn_lock);
  92
  93         return NULL;
  94 }
  95
  96 static int iscsit_set_tiqn_shutdown(struct iscsi_tiqn *tiqn)
  97 {
  98         spin_lock(&tiqn->tiqn_state_lock);
  99         if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) {
 100                 tiqn->tiqn_state = TIQN_STATE_SHUTDOWN;
 101                 spin_unlock(&tiqn->tiqn_state_lock);
 102                 return 0;
 103         }
 104         spin_unlock(&tiqn->tiqn_state_lock);
 105
 106         return -1;
 107 }
 108
 109 void iscsit_put_tiqn_for_login(struct iscsi_tiqn *tiqn)
 110 {
 111         spin_lock(&tiqn->tiqn_state_lock);
 112         tiqn->tiqn_access_count--;
 113         spin_unlock(&tiqn->tiqn_state_lock);
 114 }
 115
 116 /*
 117  * Note that IQN formatting is expected to be done in userspace, and
 118  * no explict IQN format checks are done here.
 119  */
 120 struct iscsi_tiqn *iscsit_add_tiqn(unsigned char *buf)
 121 {
 122         struct iscsi_tiqn *tiqn = NULL;
 123         int ret;
 124
 125         if (strlen(buf) >= ISCSI_IQN_LEN) {
 126                 pr_err("Target IQN exceeds %d bytes\n",
 127                                 ISCSI_IQN_LEN);
 128                 return ERR_PTR(-EINVAL);
 129         }
 130
 131         tiqn = kzalloc(sizeof(struct iscsi_tiqn), GFP_KERNEL);
 132         if (!tiqn) {
 133                 pr_err("Unable to allocate struct iscsi_tiqn\n");
 134                 return ERR_PTR(-ENOMEM);
 135         }
 136
 137         sprintf(tiqn->tiqn, "%s", buf);
 138         INIT_LIST_HEAD(&tiqn->tiqn_list);
 139         INIT_LIST_HEAD(&tiqn->tiqn_tpg_list);
 140         spin_lock_init(&tiqn->tiqn_state_lock);
 141         spin_lock_init(&tiqn->tiqn_tpg_lock);
 142         spin_lock_init(&tiqn->sess_err_stats.lock);
 143         spin_lock_init(&tiqn->login_stats.lock);
 144         spin_lock_init(&tiqn->logout_stats.lock);
 145
 146         tiqn->tiqn_state = TIQN_STATE_ACTIVE;
 147
 148         idr_preload(GFP_KERNEL);
 149         spin_lock(&tiqn_lock);
 150
 151         ret = idr_alloc(&tiqn_idr, NULL, 0, 0, GFP_NOWAIT);
 152         if (ret < 0) {
 153                 pr_err("idr_alloc() failed for tiqn->tiqn_index\n");
 154                 spin_unlock(&tiqn_lock);
 155                 idr_preload_end();
 156                 kfree(tiqn);
 157                 return ERR_PTR(ret);
 158         }
 159         tiqn->tiqn_index = ret;
 160         list_add_tail(&tiqn->tiqn_list, &g_tiqn_list);
 161
 162         spin_unlock(&tiqn_lock);
 163         idr_preload_end();
 164
 165         pr_debug("CORE[0] - Added iSCSI Target IQN: %s\n", tiqn->tiqn);
 166
 167         return tiqn;
 168
 169 }
 170
 171 static void iscsit_wait_for_tiqn(struct iscsi_tiqn *tiqn)
 172 {
 173         /*
 174          * Wait for accesses to said struct iscsi_tiqn to end.
 175          */
 176         spin_lock(&tiqn->tiqn_state_lock);
 177         while (tiqn->tiqn_access_count != 0) {
 178                 spin_unlock(&tiqn->tiqn_state_lock);
 179                 msleep(10);
 180                 spin_lock(&tiqn->tiqn_state_lock);
 181         }
 182         spin_unlock(&tiqn->tiqn_state_lock);
 183 }
 184
 185 void iscsit_del_tiqn(struct iscsi_tiqn *tiqn)
 186 {
 187         /*
 188          * iscsit_set_tiqn_shutdown sets tiqn->tiqn_state = TIQN_STATE_SHUTDOWN
 189          * while holding tiqn->tiqn_state_lock.  This means that all subsequent
 190          * attempts to access this struct iscsi_tiqn will fail from both transport
 191          * fabric and control code paths.
 192          */
 193         if (iscsit_set_tiqn_shutdown(tiqn) < 0) {
 194                 pr_err("iscsit_set_tiqn_shutdown() failed\n");
 195                 return;
 196         }
 197
 198         iscsit_wait_for_tiqn(tiqn);
 199
 200         spin_lock(&tiqn_lock);
 201         list_del(&tiqn->tiqn_list);
 202         idr_remove(&tiqn_idr, tiqn->tiqn_index);
 203         spin_unlock(&tiqn_lock);
 204
 205         pr_debug("CORE[0] - Deleted iSCSI Target IQN: %s\n",
 206                         tiqn->tiqn);
 207         kfree(tiqn);
 208 }
 209
 210 int iscsit_access_np(struct iscsi_np *np, struct iscsi_portal_group *tpg)
 211 {
 212         int ret;
 213         /*
 214          * Determine if the network portal is accepting storage traffic.
 215          */
 216         spin_lock_bh(&np->np_thread_lock);
 217         if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) {
 218                 spin_unlock_bh(&np->np_thread_lock);
 219                 return -1;
 220         }
 221         spin_unlock_bh(&np->np_thread_lock);
 222         /*
 223          * Determine if the portal group is accepting storage traffic.
 224          */
 225         spin_lock_bh(&tpg->tpg_state_lock);
 226         if (tpg->tpg_state != TPG_STATE_ACTIVE) {
 227                 spin_unlock_bh(&tpg->tpg_state_lock);
 228                 return -1;
 229         }
 230         spin_unlock_bh(&tpg->tpg_state_lock);
 231
 232         /*
 233          * Here we serialize access across the TIQN+TPG Tuple.
 234          */
 235         ret = down_interruptible(&tpg->np_login_sem);
 236         if (ret != 0)
 237                 return -1;
 238
 239         spin_lock_bh(&tpg->tpg_state_lock);
 240         if (tpg->tpg_state != TPG_STATE_ACTIVE) {
 241                 spin_unlock_bh(&tpg->tpg_state_lock);
 242                 up(&tpg->np_login_sem);
 243                 return -1;
 244         }
 245         spin_unlock_bh(&tpg->tpg_state_lock);
 246
 247         return 0;
 248 }
 249
 250 void iscsit_login_kref_put(struct kref *kref)
 251 {
 252         struct iscsi_tpg_np *tpg_np = container_of(kref,
 253                                 struct iscsi_tpg_np, tpg_np_kref);
 254
 255         complete(&tpg_np->tpg_np_comp);
 256 }
 257
 258 int iscsit_deaccess_np(struct iscsi_np *np, struct iscsi_portal_group *tpg,
 259                        struct iscsi_tpg_np *tpg_np)
 260 {
 261         struct iscsi_tiqn *tiqn = tpg->tpg_tiqn;
 262
 263         up(&tpg->np_login_sem);
 264
 265         if (tpg_np)
 266                 kref_put(&tpg_np->tpg_np_kref, iscsit_login_kref_put);
 267
 268         if (tiqn)
 269                 iscsit_put_tiqn_for_login(tiqn);
 270
 271         return 0;
 272 }
 273
 274 bool iscsit_check_np_match(
 275         struct sockaddr_storage *sockaddr,
 276         struct iscsi_np *np,
 277         int network_transport)
 278 {
 279         struct sockaddr_in *sock_in, *sock_in_e;
 280         struct sockaddr_in6 *sock_in6, *sock_in6_e;
 281         bool ip_match = false;
 282         u16 port, port_e;
 283
 284         if (sockaddr->ss_family == AF_INET6) {
 285                 sock_in6 = (struct sockaddr_in6 *)sockaddr;
 286                 sock_in6_e = (struct sockaddr_in6 *)&np->np_sockaddr;
 287
 288                 if (!memcmp(&sock_in6->sin6_addr.in6_u,
 289                             &sock_in6_e->sin6_addr.in6_u,
 290                             sizeof(struct in6_addr)))
 291                         ip_match = true;
 292
 293                 port = ntohs(sock_in6->sin6_port);
 294                 port_e = ntohs(sock_in6_e->sin6_port);
 295         } else {
 296                 sock_in = (struct sockaddr_in *)sockaddr;
 297                 sock_in_e = (struct sockaddr_in *)&np->np_sockaddr;
 298
 299                 if (sock_in->sin_addr.s_addr == sock_in_e->sin_addr.s_addr)
 300                         ip_match = true;
 301
 302                 port = ntohs(sock_in->sin_port);
 303                 port_e = ntohs(sock_in_e->sin_port);
 304         }
 305
 306         if (ip_match && (port_e == port) &&
 307             (np->np_network_transport == network_transport))
 308                 return true;
 309
 310         return false;
 311 }
 312
 313 /*
 314  * Called with mutex np_lock held
 315  */
 316 static struct iscsi_np *iscsit_get_np(
 317         struct sockaddr_storage *sockaddr,
 318         int network_transport)
 319 {
 320         struct iscsi_np *np;
 321         bool match;
 322
 323         list_for_each_entry(np, &g_np_list, np_list) {
 324                 spin_lock_bh(&np->np_thread_lock);
 325                 if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) {
 326                         spin_unlock_bh(&np->np_thread_lock);
 327                         continue;
 328                 }
 329
 330                 match = iscsit_check_np_match(sockaddr, np, network_transport);
 331                 if (match) {
 332                         /*
 333                          * Increment the np_exports reference count now to
 334                          * prevent iscsit_del_np() below from being called
 335                          * while iscsi_tpg_add_network_portal() is called.
 336                          */
 337                         np->np_exports++;
 338                         spin_unlock_bh(&np->np_thread_lock);
 339                         return np;
 340                 }
 341                 spin_unlock_bh(&np->np_thread_lock);
 342         }
 343
 344         return NULL;
 345 }
 346
 347 struct iscsi_np *iscsit_add_np(
 348         struct sockaddr_storage *sockaddr,
 349         int network_transport)
 350 {
 351         struct iscsi_np *np;
 352         int ret;
 353
 354         mutex_lock(&np_lock);
 355
 356         /*
 357          * Locate the existing struct iscsi_np if already active..
 358          */
 359         np = iscsit_get_np(sockaddr, network_transport);
 360         if (np) {
 361                 mutex_unlock(&np_lock);
 362                 return np;
 363         }
 364
 365         np = kzalloc(sizeof(struct iscsi_np), GFP_KERNEL);
 366         if (!np) {
 367                 pr_err("Unable to allocate memory for struct iscsi_np\n");
 368                 mutex_unlock(&np_lock);
 369                 return ERR_PTR(-ENOMEM);
 370         }
 371
 372         np->np_flags |= NPF_IP_NETWORK;
 373         np->np_network_transport = network_transport;
 374         spin_lock_init(&np->np_thread_lock);
 375         init_completion(&np->np_restart_comp);
 376         INIT_LIST_HEAD(&np->np_list);
 377
 378         ret = iscsi_target_setup_login_socket(np, sockaddr);
 379         if (ret != 0) {
 380                 kfree(np);
 381                 mutex_unlock(&np_lock);
 382                 return ERR_PTR(ret);
 383         }
 384
 385         np->np_thread = kthread_run(iscsi_target_login_thread, np, "iscsi_np");
 386         if (IS_ERR(np->np_thread)) {
 387                 pr_err("Unable to create kthread: iscsi_np\n");
 388                 ret = PTR_ERR(np->np_thread);
 389                 kfree(np);
 390                 mutex_unlock(&np_lock);
 391                 return ERR_PTR(ret);
 392         }
 393         /*
 394          * Increment the np_exports reference count now to prevent
 395          * iscsit_del_np() below from being run while a new call to
 396          * iscsi_tpg_add_network_portal() for a matching iscsi_np is
 397          * active.  We don't need to hold np->np_thread_lock at this
 398          * point because iscsi_np has not been added to g_np_list yet.
 399          */
 400         np->np_exports = 1;
 401         np->np_thread_state = ISCSI_NP_THREAD_ACTIVE;
 402
 403         list_add_tail(&np->np_list, &g_np_list);
 404         mutex_unlock(&np_lock);
 405
 406         pr_debug("CORE[0] - Added Network Portal: %pISpc on %s\n",
 407                 &np->np_sockaddr, np->np_transport->name);
 408
 409         return np;
 410 }
 411
 412 int iscsit_reset_np_thread(
 413         struct iscsi_np *np,
 414         struct iscsi_tpg_np *tpg_np,
 415         struct iscsi_portal_group *tpg,
 416         bool shutdown)
 417 {
 418         spin_lock_bh(&np->np_thread_lock);
 419         if (np->np_thread_state == ISCSI_NP_THREAD_INACTIVE) {
 420                 spin_unlock_bh(&np->np_thread_lock);
 421                 return 0;
 422         }
 423         np->np_thread_state = ISCSI_NP_THREAD_RESET;
 424
 425         if (np->np_thread) {
 426                 spin_unlock_bh(&np->np_thread_lock);
 427                 send_sig(SIGINT, np->np_thread, 1);
 428                 wait_for_completion(&np->np_restart_comp);
 429                 spin_lock_bh(&np->np_thread_lock);
 430         }
 431         spin_unlock_bh(&np->np_thread_lock);
 432
 433         if (tpg_np && shutdown) {
 434                 kref_put(&tpg_np->tpg_np_kref, iscsit_login_kref_put);
 435
 436                 wait_for_completion(&tpg_np->tpg_np_comp);
 437         }
 438
 439         return 0;
 440 }
 441
 442 static void iscsit_free_np(struct iscsi_np *np)
 443 {
 444         if (np->np_socket)
 445                 sock_release(np->np_socket);
 446 }
 447
 448 int iscsit_del_np(struct iscsi_np *np)
 449 {
 450         spin_lock_bh(&np->np_thread_lock);
 451         np->np_exports--;
 452         if (np->np_exports) {
 453                 np->enabled = true;
 454                 spin_unlock_bh(&np->np_thread_lock);
 455                 return 0;
 456         }
 457         np->np_thread_state = ISCSI_NP_THREAD_SHUTDOWN;
 458         spin_unlock_bh(&np->np_thread_lock);
 459
 460         if (np->np_thread) {
 461                 /*
 462                  * We need to send the signal to wakeup Linux/Net
 463                  * which may be sleeping in sock_accept()..
 464                  */
 465                 send_sig(SIGINT, np->np_thread, 1);
 466                 kthread_stop(np->np_thread);
 467                 np->np_thread = NULL;
 468         }
 469
 470         np->np_transport->iscsit_free_np(np);
 471
 472         mutex_lock(&np_lock);
 473         list_del(&np->np_list);
 474         mutex_unlock(&np_lock);
 475
 476         pr_debug("CORE[0] - Removed Network Portal: %pISpc on %s\n",
 477                 &np->np_sockaddr, np->np_transport->name);
 478
 479         iscsit_put_transport(np->np_transport);
 480         kfree(np);
 481         return 0;
 482 }
 483
 484 static void iscsit_get_rx_pdu(struct iscsi_conn *);
 485
 486 int iscsit_queue_rsp(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
 487 {
 488         iscsit_add_cmd_to_response_queue(cmd, cmd->conn, cmd->i_state);
 489         return 0;
 490 }
 491 EXPORT_SYMBOL(iscsit_queue_rsp);
 492
 493 void iscsit_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
 494 {
 495         bool scsi_cmd = (cmd->iscsi_opcode == ISCSI_OP_SCSI_CMD);
 496
 497         spin_lock_bh(&conn->cmd_lock);
 498         if (!list_empty(&cmd->i_conn_node) &&
 499             !(cmd->se_cmd.transport_state & CMD_T_FABRIC_STOP))
 500                 list_del_init(&cmd->i_conn_node);
 501         spin_unlock_bh(&conn->cmd_lock);
 502
 503         __iscsit_free_cmd(cmd, scsi_cmd, true);
 504 }
 505 EXPORT_SYMBOL(iscsit_aborted_task);
 506
 507 static void iscsit_do_crypto_hash_buf(struct ahash_request *, const void *,
 508                                       u32, u32, u8 *, u8 *);
 509 static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *);
 510
 511 static int
 512 iscsit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 513                           const void *data_buf, u32 data_buf_len)
 514 {
 515         struct iscsi_hdr *hdr = (struct iscsi_hdr *)cmd->pdu;
 516         struct kvec *iov;
 517         u32 niov = 0, tx_size = ISCSI_HDR_LEN;
 518         int ret;
 519
 520         iov = &cmd->iov_misc[0];
 521         iov[niov].iov_base      = cmd->pdu;
 522         iov[niov++].iov_len     = ISCSI_HDR_LEN;
 523
 524         if (conn->conn_ops->HeaderDigest) {
 525                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
 526
 527                 iscsit_do_crypto_hash_buf(conn->conn_tx_hash, hdr,
 528                                           ISCSI_HDR_LEN, 0, NULL,
 529                                           (u8 *)header_digest);
 530
 531                 iov[0].iov_len += ISCSI_CRC_LEN;
 532                 tx_size += ISCSI_CRC_LEN;
 533                 pr_debug("Attaching CRC32C HeaderDigest"
 534                          " to opcode 0x%x 0x%08x\n",
 535                          hdr->opcode, *header_digest);
 536         }
 537
 538         if (data_buf_len) {
 539                 u32 padding = ((-data_buf_len) & 3);
 540
 541                 iov[niov].iov_base      = (void *)data_buf;
 542                 iov[niov++].iov_len     = data_buf_len;
 543                 tx_size += data_buf_len;
 544
 545                 if (padding != 0) {
 546                         iov[niov].iov_base = &cmd->pad_bytes;
 547                         iov[niov++].iov_len = padding;
 548                         tx_size += padding;
 549                         pr_debug("Attaching %u additional"
 550                                  " padding bytes.\n", padding);
 551                 }
 552
 553                 if (conn->conn_ops->DataDigest) {
 554                         iscsit_do_crypto_hash_buf(conn->conn_tx_hash,
 555                                                   data_buf, data_buf_len,
 556                                                   padding,
 557                                                   (u8 *)&cmd->pad_bytes,
 558                                                   (u8 *)&cmd->data_crc);
 559
 560                         iov[niov].iov_base = &cmd->data_crc;
 561                         iov[niov++].iov_len = ISCSI_CRC_LEN;
 562                         tx_size += ISCSI_CRC_LEN;
 563                         pr_debug("Attached DataDigest for %u"
 564                                  " bytes opcode 0x%x, CRC 0x%08x\n",
 565                                  data_buf_len, hdr->opcode, cmd->data_crc);
 566                 }
 567         }
 568
 569         cmd->iov_misc_count = niov;
 570         cmd->tx_size = tx_size;
 571
 572         ret = iscsit_send_tx_data(cmd, conn, 1);
 573         if (ret < 0) {
 574                 iscsit_tx_thread_wait_for_tcp(conn);
 575                 return ret;
 576         }
 577
 578         return 0;
 579 }
 580
 581 static int iscsit_map_iovec(struct iscsi_cmd *, struct kvec *, u32, u32);
 582 static void iscsit_unmap_iovec(struct iscsi_cmd *);
 583 static u32 iscsit_do_crypto_hash_sg(struct ahash_request *, struct iscsi_cmd *,
 584                                     u32, u32, u32, u8 *);
 585 static int
 586 iscsit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 587                        const struct iscsi_datain *datain)
 588 {
 589         struct kvec *iov;
 590         u32 iov_count = 0, tx_size = 0;
 591         int ret, iov_ret;
 592
 593         iov = &cmd->iov_data[0];
 594         iov[iov_count].iov_base = cmd->pdu;
 595         iov[iov_count++].iov_len = ISCSI_HDR_LEN;
 596         tx_size += ISCSI_HDR_LEN;
 597
 598         if (conn->conn_ops->HeaderDigest) {
 599                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
 600
 601                 iscsit_do_crypto_hash_buf(conn->conn_tx_hash, cmd->pdu,
 602                                           ISCSI_HDR_LEN, 0, NULL,
 603                                           (u8 *)header_digest);
 604
 605                 iov[0].iov_len += ISCSI_CRC_LEN;
 606                 tx_size += ISCSI_CRC_LEN;
 607
 608                 pr_debug("Attaching CRC32 HeaderDigest for DataIN PDU 0x%08x\n",
 609                          *header_digest);
 610         }
 611
 612         iov_ret = iscsit_map_iovec(cmd, &cmd->iov_data[1],
 613                                    datain->offset, datain->length);
 614         if (iov_ret < 0)
 615                 return -1;
 616
 617         iov_count += iov_ret;
 618         tx_size += datain->length;
 619
 620         cmd->padding = ((-datain->length) & 3);
 621         if (cmd->padding) {
 622                 iov[iov_count].iov_base         = cmd->pad_bytes;
 623                 iov[iov_count++].iov_len        = cmd->padding;
 624                 tx_size += cmd->padding;
 625
 626                 pr_debug("Attaching %u padding bytes\n", cmd->padding);
 627         }
 628
 629         if (conn->conn_ops->DataDigest) {
 630                 cmd->data_crc = iscsit_do_crypto_hash_sg(conn->conn_tx_hash,
 631                                                          cmd, datain->offset,
 632                                                          datain->length,
 633                                                          cmd->padding,
 634                                                          cmd->pad_bytes);
 635
 636                 iov[iov_count].iov_base = &cmd->data_crc;
 637                 iov[iov_count++].iov_len = ISCSI_CRC_LEN;
 638                 tx_size += ISCSI_CRC_LEN;
 639
 640                 pr_debug("Attached CRC32C DataDigest %d bytes, crc 0x%08x\n",
 641                          datain->length + cmd->padding, cmd->data_crc);
 642         }
 643
 644         cmd->iov_data_count = iov_count;
 645         cmd->tx_size = tx_size;
 646
 647         ret = iscsit_fe_sendpage_sg(cmd, conn);
 648
 649         iscsit_unmap_iovec(cmd);
 650
 651         if (ret < 0) {
 652                 iscsit_tx_thread_wait_for_tcp(conn);
 653                 return ret;
 654         }
 655
 656         return 0;
 657 }
 658
 659 static int iscsit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 660                            struct iscsi_datain_req *dr, const void *buf,
 661                            u32 buf_len)
 662 {
 663         if (dr)
 664                 return iscsit_xmit_datain_pdu(conn, cmd, buf);
 665         else
 666                 return iscsit_xmit_nondatain_pdu(conn, cmd, buf, buf_len);
 667 }
 668
 669 static enum target_prot_op iscsit_get_sup_prot_ops(struct iscsi_conn *conn)
 670 {
 671         return TARGET_PROT_NORMAL;
 672 }
 673
 674 static struct iscsit_transport iscsi_target_transport = {
 675         .name                   = "iSCSI/TCP",
 676         .transport_type         = ISCSI_TCP,
 677         .rdma_shutdown          = false,
 678         .owner                  = NULL,
 679         .iscsit_setup_np        = iscsit_setup_np,
 680         .iscsit_accept_np       = iscsit_accept_np,
 681         .iscsit_free_np         = iscsit_free_np,
 682         .iscsit_get_login_rx    = iscsit_get_login_rx,
 683         .iscsit_put_login_tx    = iscsit_put_login_tx,
 684         .iscsit_get_dataout     = iscsit_build_r2ts_for_cmd,
 685         .iscsit_immediate_queue = iscsit_immediate_queue,
 686         .iscsit_response_queue  = iscsit_response_queue,
 687         .iscsit_queue_data_in   = iscsit_queue_rsp,
 688         .iscsit_queue_status    = iscsit_queue_rsp,
 689         .iscsit_aborted_task    = iscsit_aborted_task,
 690         .iscsit_xmit_pdu        = iscsit_xmit_pdu,
 691         .iscsit_get_rx_pdu      = iscsit_get_rx_pdu,
 692         .iscsit_get_sup_prot_ops = iscsit_get_sup_prot_ops,
 693 };
 694
 695 static int __init iscsi_target_init_module(void)
 696 {
 697         int ret = 0, size;
 698
 699         pr_debug("iSCSI-Target "ISCSIT_VERSION"\n");
 700
 701         iscsit_global = kzalloc(sizeof(struct iscsit_global), GFP_KERNEL);
 702         if (!iscsit_global) {
 703                 pr_err("Unable to allocate memory for iscsit_global\n");
 704                 return -1;
 705         }
 706         spin_lock_init(&iscsit_global->ts_bitmap_lock);
 707         mutex_init(&auth_id_lock);
 708         spin_lock_init(&sess_idr_lock);
 709         idr_init(&tiqn_idr);
 710         idr_init(&sess_idr);
 711
 712         ret = target_register_template(&iscsi_ops);
 713         if (ret)
 714                 goto out;
 715
 716         size = BITS_TO_LONGS(ISCSIT_BITMAP_BITS) * sizeof(long);
 717         iscsit_global->ts_bitmap = vzalloc(size);
 718         if (!iscsit_global->ts_bitmap) {
 719                 pr_err("Unable to allocate iscsit_global->ts_bitmap\n");
 720                 goto configfs_out;
 721         }
 722
 723         lio_qr_cache = kmem_cache_create("lio_qr_cache",
 724                         sizeof(struct iscsi_queue_req),
 725                         __alignof__(struct iscsi_queue_req), 0, NULL);
 726         if (!lio_qr_cache) {
 727                 pr_err("nable to kmem_cache_create() for"
 728                                 " lio_qr_cache\n");
 729                 goto bitmap_out;
 730         }
 731
 732         lio_dr_cache = kmem_cache_create("lio_dr_cache",
 733                         sizeof(struct iscsi_datain_req),
 734                         __alignof__(struct iscsi_datain_req), 0, NULL);
 735         if (!lio_dr_cache) {
 736                 pr_err("Unable to kmem_cache_create() for"
 737                                 " lio_dr_cache\n");
 738                 goto qr_out;
 739         }
 740
 741         lio_ooo_cache = kmem_cache_create("lio_ooo_cache",
 742                         sizeof(struct iscsi_ooo_cmdsn),
 743                         __alignof__(struct iscsi_ooo_cmdsn), 0, NULL);
 744         if (!lio_ooo_cache) {
 745                 pr_err("Unable to kmem_cache_create() for"
 746                                 " lio_ooo_cache\n");
 747                 goto dr_out;
 748         }
 749
 750         lio_r2t_cache = kmem_cache_create("lio_r2t_cache",
 751                         sizeof(struct iscsi_r2t), __alignof__(struct iscsi_r2t),
 752                         0, NULL);
 753         if (!lio_r2t_cache) {
 754                 pr_err("Unable to kmem_cache_create() for"
 755                                 " lio_r2t_cache\n");
 756                 goto ooo_out;
 757         }
 758
 759         iscsit_register_transport(&iscsi_target_transport);
 760
 761         if (iscsit_load_discovery_tpg() < 0)
 762                 goto r2t_out;
 763
 764         return ret;
 765 r2t_out:
 766         iscsit_unregister_transport(&iscsi_target_transport);
 767         kmem_cache_destroy(lio_r2t_cache);
 768 ooo_out:
 769         kmem_cache_destroy(lio_ooo_cache);
 770 dr_out:
 771         kmem_cache_destroy(lio_dr_cache);
 772 qr_out:
 773         kmem_cache_destroy(lio_qr_cache);
 774 bitmap_out:
 775         vfree(iscsit_global->ts_bitmap);
 776 configfs_out:
 777         /* XXX: this probably wants it to be it's own unwind step.. */
 778         if (iscsit_global->discovery_tpg)
 779                 iscsit_tpg_disable_portal_group(iscsit_global->discovery_tpg, 1);
 780         target_unregister_template(&iscsi_ops);
 781 out:
 782         kfree(iscsit_global);
 783         return -ENOMEM;
 784 }
 785
 786 static void __exit iscsi_target_cleanup_module(void)
 787 {
 788         iscsit_release_discovery_tpg();
 789         iscsit_unregister_transport(&iscsi_target_transport);
 790         kmem_cache_destroy(lio_qr_cache);
 791         kmem_cache_destroy(lio_dr_cache);
 792         kmem_cache_destroy(lio_ooo_cache);
 793         kmem_cache_destroy(lio_r2t_cache);
 794
 795         /*
 796          * Shutdown discovery sessions and disable discovery TPG
 797          */
 798         if (iscsit_global->discovery_tpg)
 799                 iscsit_tpg_disable_portal_group(iscsit_global->discovery_tpg, 1);
 800
 801         target_unregister_template(&iscsi_ops);
 802
 803         vfree(iscsit_global->ts_bitmap);
 804         kfree(iscsit_global);
 805 }
 806
 807 int iscsit_add_reject(
 808         struct iscsi_conn *conn,
 809         u8 reason,
 810         unsigned char *buf)
 811 {
 812         struct iscsi_cmd *cmd;
 813
 814         cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
 815         if (!cmd)
 816                 return -1;
 817
 818         cmd->iscsi_opcode = ISCSI_OP_REJECT;
 819         cmd->reject_reason = reason;
 820
 821         cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL);
 822         if (!cmd->buf_ptr) {
 823                 pr_err("Unable to allocate memory for cmd->buf_ptr\n");
 824                 iscsit_free_cmd(cmd, false);
 825                 return -1;
 826         }
 827
 828         spin_lock_bh(&conn->cmd_lock);
 829         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
 830         spin_unlock_bh(&conn->cmd_lock);
 831
 832         cmd->i_state = ISTATE_SEND_REJECT;
 833         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
 834
 835         return -1;
 836 }
 837 EXPORT_SYMBOL(iscsit_add_reject);
 838
 839 static int iscsit_add_reject_from_cmd(
 840         struct iscsi_cmd *cmd,
 841         u8 reason,
 842         bool add_to_conn,
 843         unsigned char *buf)
 844 {
 845         struct iscsi_conn *conn;
 846
 847         if (!cmd->conn) {
 848                 pr_err("cmd->conn is NULL for ITT: 0x%08x\n",
 849                                 cmd->init_task_tag);
 850                 return -1;
 851         }
 852         conn = cmd->conn;
 853
 854         cmd->iscsi_opcode = ISCSI_OP_REJECT;
 855         cmd->reject_reason = reason;
 856
 857         cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL);
 858         if (!cmd->buf_ptr) {
 859                 pr_err("Unable to allocate memory for cmd->buf_ptr\n");
 860                 iscsit_free_cmd(cmd, false);
 861                 return -1;
 862         }
 863
 864         if (add_to_conn) {
 865                 spin_lock_bh(&conn->cmd_lock);
 866                 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
 867                 spin_unlock_bh(&conn->cmd_lock);
 868         }
 869
 870         cmd->i_state = ISTATE_SEND_REJECT;
 871         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
 872         /*
 873          * Perform the kref_put now if se_cmd has already been setup by
 874          * scsit_setup_scsi_cmd()
 875          */
 876         if (cmd->se_cmd.se_tfo != NULL) {
 877                 pr_debug("iscsi reject: calling target_put_sess_cmd >>>>>>\n");
 878                 target_put_sess_cmd(&cmd->se_cmd);
 879         }
 880         return -1;
 881 }
 882
 883 static int iscsit_add_reject_cmd(struct iscsi_cmd *cmd, u8 reason,
 884                                  unsigned char *buf)
 885 {
 886         return iscsit_add_reject_from_cmd(cmd, reason, true, buf);
 887 }
 888
 889 int iscsit_reject_cmd(struct iscsi_cmd *cmd, u8 reason, unsigned char *buf)
 890 {
 891         return iscsit_add_reject_from_cmd(cmd, reason, false, buf);
 892 }
 893 EXPORT_SYMBOL(iscsit_reject_cmd);
 894
 895 /*
 896  * Map some portion of the allocated scatterlist to an iovec, suitable for
 897  * kernel sockets to copy data in/out.
 898  */
 899 static int iscsit_map_iovec(
 900         struct iscsi_cmd *cmd,
 901         struct kvec *iov,
 902         u32 data_offset,
 903         u32 data_length)
 904 {
 905         u32 i = 0;
 906         struct scatterlist *sg;
 907         unsigned int page_off;
 908
 909         /*
 910          * We know each entry in t_data_sg contains a page.
 911          */
 912         u32 ent = data_offset / PAGE_SIZE;
 913
 914         if (ent >= cmd->se_cmd.t_data_nents) {
 915                 pr_err("Initial page entry out-of-bounds\n");
 916                 return -1;
 917         }
 918
 919         sg = &cmd->se_cmd.t_data_sg[ent];
 920         page_off = (data_offset % PAGE_SIZE);
 921
 922         cmd->first_data_sg = sg;
 923         cmd->first_data_sg_off = page_off;
 924
 925         while (data_length) {
 926                 u32 cur_len = min_t(u32, data_length, sg->length - page_off);
 927
 928                 iov[i].iov_base = kmap(sg_page(sg)) + sg->offset + page_off;
 929                 iov[i].iov_len = cur_len;
 930
 931                 data_length -= cur_len;
 932                 page_off = 0;
 933                 sg = sg_next(sg);
 934                 i++;
 935         }
 936
 937         cmd->kmapped_nents = i;
 938
 939         return i;
 940 }
 941
 942 static void iscsit_unmap_iovec(struct iscsi_cmd *cmd)
 943 {
 944         u32 i;
 945         struct scatterlist *sg;
 946
 947         sg = cmd->first_data_sg;
 948
 949         for (i = 0; i < cmd->kmapped_nents; i++)
 950                 kunmap(sg_page(&sg[i]));
 951 }
 952
 953 static void iscsit_ack_from_expstatsn(struct iscsi_conn *conn, u32 exp_statsn)
 954 {
 955         LIST_HEAD(ack_list);
 956         struct iscsi_cmd *cmd, *cmd_p;
 957
 958         conn->exp_statsn = exp_statsn;
 959
 960         if (conn->sess->sess_ops->RDMAExtensions)
 961                 return;
 962
 963         spin_lock_bh(&conn->cmd_lock);
 964         list_for_each_entry_safe(cmd, cmd_p, &conn->conn_cmd_list, i_conn_node) {
 965                 spin_lock(&cmd->istate_lock);
 966                 if ((cmd->i_state == ISTATE_SENT_STATUS) &&
 967                     iscsi_sna_lt(cmd->stat_sn, exp_statsn)) {
 968                         cmd->i_state = ISTATE_REMOVE;
 969                         spin_unlock(&cmd->istate_lock);
 970                         list_move_tail(&cmd->i_conn_node, &ack_list);
 971                         continue;
 972                 }
 973                 spin_unlock(&cmd->istate_lock);
 974         }
 975         spin_unlock_bh(&conn->cmd_lock);
 976
 977         list_for_each_entry_safe(cmd, cmd_p, &ack_list, i_conn_node) {
 978                 list_del_init(&cmd->i_conn_node);
 979                 iscsit_free_cmd(cmd, false);
 980         }
 981 }
 982
 983 static int iscsit_allocate_iovecs(struct iscsi_cmd *cmd)
 984 {
 985         u32 iov_count = max(1UL, DIV_ROUND_UP(cmd->se_cmd.data_length, PAGE_SIZE));
 986
 987         iov_count += ISCSI_IOV_DATA_BUFFER;
 988
 989         cmd->iov_data = kzalloc(iov_count * sizeof(struct kvec), GFP_KERNEL);
 990         if (!cmd->iov_data) {
 991                 pr_err("Unable to allocate cmd->iov_data\n");
 992                 return -ENOMEM;
 993         }
 994
 995         cmd->orig_iov_data_count = iov_count;
 996         return 0;
 997 }
 998
 999 int iscsit_setup_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1000                           unsigned char *buf)
1001 {
1002         int data_direction, payload_length;
1003         struct iscsi_scsi_req *hdr;
1004         int iscsi_task_attr;
1005         int sam_task_attr;
1006
1007         atomic_long_inc(&conn->sess->cmd_pdus);
1008
1009         hdr                     = (struct iscsi_scsi_req *) buf;
1010         payload_length          = ntoh24(hdr->dlength);
1011
1012         /* FIXME; Add checks for AdditionalHeaderSegment */
1013
1014         if (!(hdr->flags & ISCSI_FLAG_CMD_WRITE) &&
1015             !(hdr->flags & ISCSI_FLAG_CMD_FINAL)) {
1016                 pr_err("ISCSI_FLAG_CMD_WRITE & ISCSI_FLAG_CMD_FINAL"
1017                                 " not set. Bad iSCSI Initiator.\n");
1018                 return iscsit_add_reject_cmd(cmd,
1019                                              ISCSI_REASON_BOOKMARK_INVALID, buf);
1020         }
1021
1022         if (((hdr->flags & ISCSI_FLAG_CMD_READ) ||
1023              (hdr->flags & ISCSI_FLAG_CMD_WRITE)) && !hdr->data_length) {
1024                 /*
1025                  * From RFC-3720 Section 10.3.1:
1026                  *
1027                  * "Either or both of R and W MAY be 1 when either the
1028                  *  Expected Data Transfer Length and/or Bidirectional Read
1029                  *  Expected Data Transfer Length are 0"
1030                  *
1031                  * For this case, go ahead and clear the unnecssary bits
1032                  * to avoid any confusion with ->data_direction.
1033                  */
1034                 hdr->flags &= ~ISCSI_FLAG_CMD_READ;
1035                 hdr->flags &= ~ISCSI_FLAG_CMD_WRITE;
1036
1037                 pr_warn("ISCSI_FLAG_CMD_READ or ISCSI_FLAG_CMD_WRITE"
1038                         " set when Expected Data Transfer Length is 0 for"
1039                         " CDB: 0x%02x, Fixing up flags\n", hdr->cdb[0]);
1040         }
1041
1042         if (!(hdr->flags & ISCSI_FLAG_CMD_READ) &&
1043             !(hdr->flags & ISCSI_FLAG_CMD_WRITE) && (hdr->data_length != 0)) {
1044                 pr_err("ISCSI_FLAG_CMD_READ and/or ISCSI_FLAG_CMD_WRITE"
1045                         " MUST be set if Expected Data Transfer Length is not 0."
1046                         " Bad iSCSI Initiator\n");
1047                 return iscsit_add_reject_cmd(cmd,
1048                                              ISCSI_REASON_BOOKMARK_INVALID, buf);
1049         }
1050
1051         if ((hdr->flags & ISCSI_FLAG_CMD_READ) &&
1052             (hdr->flags & ISCSI_FLAG_CMD_WRITE)) {
1053                 pr_err("Bidirectional operations not supported!\n");
1054                 return iscsit_add_reject_cmd(cmd,
1055                                              ISCSI_REASON_BOOKMARK_INVALID, buf);
1056         }
1057
1058         if (hdr->opcode & ISCSI_OP_IMMEDIATE) {
1059                 pr_err("Illegally set Immediate Bit in iSCSI Initiator"
1060                                 " Scsi Command PDU.\n");
1061                 return iscsit_add_reject_cmd(cmd,
1062                                              ISCSI_REASON_BOOKMARK_INVALID, buf);
1063         }
1064
1065         if (payload_length && !conn->sess->sess_ops->ImmediateData) {
1066                 pr_err("ImmediateData=No but DataSegmentLength=%u,"
1067                         " protocol error.\n", payload_length);
1068                 return iscsit_add_reject_cmd(cmd,
1069                                              ISCSI_REASON_PROTOCOL_ERROR, buf);
1070         }
1071
1072         if ((be32_to_cpu(hdr->data_length) == payload_length) &&
1073             (!(hdr->flags & ISCSI_FLAG_CMD_FINAL))) {
1074                 pr_err("Expected Data Transfer Length and Length of"
1075                         " Immediate Data are the same, but ISCSI_FLAG_CMD_FINAL"
1076                         " bit is not set protocol error\n");
1077                 return iscsit_add_reject_cmd(cmd,
1078                                              ISCSI_REASON_PROTOCOL_ERROR, buf);
1079         }
1080
1081         if (payload_length > be32_to_cpu(hdr->data_length)) {
1082                 pr_err("DataSegmentLength: %u is greater than"
1083                         " EDTL: %u, protocol error.\n", payload_length,
1084                                 hdr->data_length);
1085                 return iscsit_add_reject_cmd(cmd,
1086                                              ISCSI_REASON_PROTOCOL_ERROR, buf);
1087         }
1088
1089         if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
1090                 pr_err("DataSegmentLength: %u is greater than"
1091                         " MaxXmitDataSegmentLength: %u, protocol error.\n",
1092                         payload_length, conn->conn_ops->MaxXmitDataSegmentLength);
1093                 return iscsit_add_reject_cmd(cmd,
1094                                              ISCSI_REASON_PROTOCOL_ERROR, buf);
1095         }
1096
1097         if (payload_length > conn->sess->sess_ops->FirstBurstLength) {
1098                 pr_err("DataSegmentLength: %u is greater than"
1099                         " FirstBurstLength: %u, protocol error.\n",
1100                         payload_length, conn->sess->sess_ops->FirstBurstLength);
1101                 return iscsit_add_reject_cmd(cmd,
1102                                              ISCSI_REASON_BOOKMARK_INVALID, buf);
1103         }
1104
1105         data_direction = (hdr->flags & ISCSI_FLAG_CMD_WRITE) ? DMA_TO_DEVICE :
1106                          (hdr->flags & ISCSI_FLAG_CMD_READ) ? DMA_FROM_DEVICE :
1107                           DMA_NONE;
1108
1109         cmd->data_direction = data_direction;
1110         iscsi_task_attr = hdr->flags & ISCSI_FLAG_CMD_ATTR_MASK;
1111         /*
1112          * Figure out the SAM Task Attribute for the incoming SCSI CDB
1113          */
1114         if ((iscsi_task_attr == ISCSI_ATTR_UNTAGGED) ||
1115             (iscsi_task_attr == ISCSI_ATTR_SIMPLE))
1116                 sam_task_attr = TCM_SIMPLE_TAG;
1117         else if (iscsi_task_attr == ISCSI_ATTR_ORDERED)
1118                 sam_task_attr = TCM_ORDERED_TAG;
1119         else if (iscsi_task_attr == ISCSI_ATTR_HEAD_OF_QUEUE)
1120                 sam_task_attr = TCM_HEAD_TAG;
1121         else if (iscsi_task_attr == ISCSI_ATTR_ACA)
1122                 sam_task_attr = TCM_ACA_TAG;
1123         else {
1124                 pr_debug("Unknown iSCSI Task Attribute: 0x%02x, using"
1125                         " TCM_SIMPLE_TAG\n", iscsi_task_attr);
1126                 sam_task_attr = TCM_SIMPLE_TAG;
1127         }
1128
1129         cmd->iscsi_opcode       = ISCSI_OP_SCSI_CMD;
1130         cmd->i_state            = ISTATE_NEW_CMD;
1131         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
1132         cmd->immediate_data     = (payload_length) ? 1 : 0;
1133         cmd->unsolicited_data   = ((!(hdr->flags & ISCSI_FLAG_CMD_FINAL) &&
1134                                      (hdr->flags & ISCSI_FLAG_CMD_WRITE)) ? 1 : 0);
1135         if (cmd->unsolicited_data)
1136                 cmd->cmd_flags |= ICF_NON_IMMEDIATE_UNSOLICITED_DATA;
1137
1138         conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt;
1139         if (hdr->flags & ISCSI_FLAG_CMD_READ)
1140                 cmd->targ_xfer_tag = session_get_next_ttt(conn->sess);
1141         else
1142                 cmd->targ_xfer_tag = 0xFFFFFFFF;
1143         cmd->cmd_sn             = be32_to_cpu(hdr->cmdsn);
1144         cmd->exp_stat_sn        = be32_to_cpu(hdr->exp_statsn);
1145         cmd->first_burst_len    = payload_length;
1146
1147         if (!conn->sess->sess_ops->RDMAExtensions &&
1148              cmd->data_direction == DMA_FROM_DEVICE) {
1149                 struct iscsi_datain_req *dr;
1150
1151                 dr = iscsit_allocate_datain_req();
1152                 if (!dr)
1153                         return iscsit_add_reject_cmd(cmd,
1154                                         ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
1155
1156                 iscsit_attach_datain_req(cmd, dr);
1157         }
1158
1159         /*
1160          * Initialize struct se_cmd descriptor from target_core_mod infrastructure
1161          */
1162         transport_init_se_cmd(&cmd->se_cmd, &iscsi_ops,
1163                         conn->sess->se_sess, be32_to_cpu(hdr->data_length),
1164                         cmd->data_direction, sam_task_attr,
1165                         cmd->sense_buffer + 2);
1166
1167         pr_debug("Got SCSI Command, ITT: 0x%08x, CmdSN: 0x%08x,"
1168                 " ExpXferLen: %u, Length: %u, CID: %hu\n", hdr->itt,
1169                 hdr->cmdsn, be32_to_cpu(hdr->data_length), payload_length,
1170                 conn->cid);
1171
1172         target_get_sess_cmd(&cmd->se_cmd, true);
1173
1174         cmd->sense_reason = transport_lookup_cmd_lun(&cmd->se_cmd,
1175                                                      scsilun_to_int(&hdr->lun));
1176         if (cmd->sense_reason)
1177                 goto attach_cmd;
1178
1179         /* only used for printks or comparing with ->ref_task_tag */
1180         cmd->se_cmd.tag = (__force u32)cmd->init_task_tag;
1181         cmd->sense_reason = target_setup_cmd_from_cdb(&cmd->se_cmd, hdr->cdb);
1182         if (cmd->sense_reason) {
1183                 if (cmd->sense_reason == TCM_OUT_OF_RESOURCES) {
1184                         return iscsit_add_reject_cmd(cmd,
1185                                         ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
1186                 }
1187
1188                 goto attach_cmd;
1189         }
1190
1191         if (iscsit_build_pdu_and_seq_lists(cmd, payload_length) < 0) {
1192                 return iscsit_add_reject_cmd(cmd,
1193                                 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
1194         }
1195
1196 attach_cmd:
1197         spin_lock_bh(&conn->cmd_lock);
1198         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
1199         spin_unlock_bh(&conn->cmd_lock);
1200         /*
1201          * Check if we need to delay processing because of ALUA
1202          * Active/NonOptimized primary access state..
1203          */
1204         core_alua_check_nonop_delay(&cmd->se_cmd);
1205
1206         return 0;
1207 }
1208 EXPORT_SYMBOL(iscsit_setup_scsi_cmd);
1209
1210 void iscsit_set_unsoliticed_dataout(struct iscsi_cmd *cmd)
1211 {
1212         iscsit_set_dataout_sequence_values(cmd);
1213
1214         spin_lock_bh(&cmd->dataout_timeout_lock);
1215         iscsit_start_dataout_timer(cmd, cmd->conn);
1216         spin_unlock_bh(&cmd->dataout_timeout_lock);
1217 }
1218 EXPORT_SYMBOL(iscsit_set_unsoliticed_dataout);
1219
1220 int iscsit_process_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1221                             struct iscsi_scsi_req *hdr)
1222 {
1223         int cmdsn_ret = 0;
1224         /*
1225          * Check the CmdSN against ExpCmdSN/MaxCmdSN here if
1226          * the Immediate Bit is not set, and no Immediate
1227          * Data is attached.
1228          *
1229          * A PDU/CmdSN carrying Immediate Data can only
1230          * be processed after the DataCRC has passed.
1231          * If the DataCRC fails, the CmdSN MUST NOT
1232          * be acknowledged. (See below)
1233          */
1234         if (!cmd->immediate_data) {
1235                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
1236                                         (unsigned char *)hdr, hdr->cmdsn);
1237                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1238                         return -1;
1239                 else if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
1240                         target_put_sess_cmd(&cmd->se_cmd);
1241                         return 0;
1242                 }
1243         }
1244
1245         iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
1246
1247         /*
1248          * If no Immediate Data is attached, it's OK to return now.
1249          */
1250         if (!cmd->immediate_data) {
1251                 if (!cmd->sense_reason && cmd->unsolicited_data)
1252                         iscsit_set_unsoliticed_dataout(cmd);
1253                 if (!cmd->sense_reason)
1254                         return 0;
1255
1256                 target_put_sess_cmd(&cmd->se_cmd);
1257                 return 0;
1258         }
1259
1260         /*
1261          * Early CHECK_CONDITIONs with ImmediateData never make it to command
1262          * execution.  These exceptions are processed in CmdSN order using
1263          * iscsit_check_received_cmdsn() in iscsit_get_immediate_data() below.
1264          */
1265         if (cmd->sense_reason) {
1266                 if (cmd->reject_reason)
1267                         return 0;
1268
1269                 return 1;
1270         }
1271         /*
1272          * Call directly into transport_generic_new_cmd() to perform
1273          * the backend memory allocation.
1274          */
1275         cmd->sense_reason = transport_generic_new_cmd(&cmd->se_cmd);
1276         if (cmd->sense_reason)
1277                 return 1;
1278
1279         return 0;
1280 }
1281 EXPORT_SYMBOL(iscsit_process_scsi_cmd);
1282
1283 static int
1284 iscsit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
1285                           bool dump_payload)
1286 {
1287         int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
1288         /*
1289          * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes.
1290          */
1291         if (dump_payload)
1292                 goto after_immediate_data;
1293
1294         immed_ret = iscsit_handle_immediate_data(cmd, hdr,
1295                                         cmd->first_burst_len);
1296 after_immediate_data:
1297         if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) {
1298                 /*
1299                  * A PDU/CmdSN carrying Immediate Data passed
1300                  * DataCRC, check against ExpCmdSN/MaxCmdSN if
1301                  * Immediate Bit is not set.
1302                  */
1303                 cmdsn_ret = iscsit_sequence_cmd(cmd->conn, cmd,
1304                                         (unsigned char *)hdr, hdr->cmdsn);
1305                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1306                         return -1;
1307
1308                 if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
1309                         int rc;
1310
1311                         rc = iscsit_dump_data_payload(cmd->conn,
1312                                                       cmd->first_burst_len, 1);
1313                         target_put_sess_cmd(&cmd->se_cmd);
1314                         return rc;
1315                 } else if (cmd->unsolicited_data)
1316                         iscsit_set_unsoliticed_dataout(cmd);
1317
1318         } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) {
1319                 /*
1320                  * Immediate Data failed DataCRC and ERL>=1,
1321                  * silently drop this PDU and let the initiator
1322                  * plug the CmdSN gap.
1323                  *
1324                  * FIXME: Send Unsolicited NOPIN with reserved
1325                  * TTT here to help the initiator figure out
1326                  * the missing CmdSN, although they should be
1327                  * intelligent enough to determine the missing
1328                  * CmdSN and issue a retry to plug the sequence.
1329                  */
1330                 cmd->i_state = ISTATE_REMOVE;
1331                 iscsit_add_cmd_to_immediate_queue(cmd, cmd->conn, cmd->i_state);
1332         } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */
1333                 return -1;
1334
1335         return 0;
1336 }
1337
1338 static int
1339 iscsit_handle_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1340                            unsigned char *buf)
1341 {
1342         struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf;
1343         int rc, immed_data;
1344         bool dump_payload = false;
1345
1346         rc = iscsit_setup_scsi_cmd(conn, cmd, buf);
1347         if (rc < 0)
1348                 return 0;
1349         /*
1350          * Allocation iovecs needed for struct socket operations for
1351          * traditional iSCSI block I/O.
1352          */
1353         if (iscsit_allocate_iovecs(cmd) < 0) {
1354                 return iscsit_reject_cmd(cmd,
1355                                 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
1356         }
1357         immed_data = cmd->immediate_data;
1358
1359         rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
1360         if (rc < 0)
1361                 return rc;
1362         else if (rc > 0)
1363                 dump_payload = true;
1364
1365         if (!immed_data)
1366                 return 0;
1367
1368         return iscsit_get_immediate_data(cmd, hdr, dump_payload);
1369 }
1370
1371 static u32 iscsit_do_crypto_hash_sg(
1372         struct ahash_request *hash,
1373         struct iscsi_cmd *cmd,
1374         u32 data_offset,
1375         u32 data_length,
1376         u32 padding,
1377         u8 *pad_bytes)
1378 {
1379         u32 data_crc;
1380         struct scatterlist *sg;
1381         unsigned int page_off;
1382
1383         crypto_ahash_init(hash);
1384
1385         sg = cmd->first_data_sg;
1386         page_off = cmd->first_data_sg_off;
1387
1388         while (data_length) {
1389                 u32 cur_len = min_t(u32, data_length, (sg->length - page_off));
1390
1391                 ahash_request_set_crypt(hash, sg, NULL, cur_len);
1392                 crypto_ahash_update(hash);
1393
1394                 data_length -= cur_len;
1395                 page_off = 0;
1396                 /* iscsit_map_iovec has already checked for invalid sg pointers */
1397                 sg = sg_next(sg);
1398         }
1399
1400         if (padding) {
1401                 struct scatterlist pad_sg;
1402
1403                 sg_init_one(&pad_sg, pad_bytes, padding);
1404                 ahash_request_set_crypt(hash, &pad_sg, (u8 *)&data_crc,
1405                                         padding);
1406                 crypto_ahash_finup(hash);
1407         } else {
1408                 ahash_request_set_crypt(hash, NULL, (u8 *)&data_crc, 0);
1409                 crypto_ahash_final(hash);
1410         }
1411
1412         return data_crc;
1413 }
1414
1415 static void iscsit_do_crypto_hash_buf(
1416         struct ahash_request *hash,
1417         const void *buf,
1418         u32 payload_length,
1419         u32 padding,
1420         u8 *pad_bytes,
1421         u8 *data_crc)
1422 {
1423         struct scatterlist sg[2];
1424
1425         sg_init_table(sg, ARRAY_SIZE(sg));
1426         sg_set_buf(sg, buf, payload_length);
1427         sg_set_buf(sg + 1, pad_bytes, padding);
1428
1429         ahash_request_set_crypt(hash, sg, data_crc, payload_length + padding);
1430
1431         crypto_ahash_digest(hash);
1432 }
1433
1434 int
1435 __iscsit_check_dataout_hdr(struct iscsi_conn *conn, void *buf,
1436                            struct iscsi_cmd *cmd, u32 payload_length,
1437                            bool *success)
1438 {
1439         struct iscsi_data *hdr = buf;
1440         struct se_cmd *se_cmd;
1441         int rc;
1442
1443         /* iSCSI write */
1444         atomic_long_add(payload_length, &conn->sess->rx_data_octets);
1445
1446         pr_debug("Got DataOut ITT: 0x%08x, TTT: 0x%08x,"
1447                 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n",
1448                 hdr->itt, hdr->ttt, hdr->datasn, ntohl(hdr->offset),
1449                 payload_length, conn->cid);
1450
1451         if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT) {
1452                 pr_err("Command ITT: 0x%08x received DataOUT after"
1453                         " last DataOUT received, dumping payload\n",
1454                         cmd->init_task_tag);
1455                 return iscsit_dump_data_payload(conn, payload_length, 1);
1456         }
1457
1458         if (cmd->data_direction != DMA_TO_DEVICE) {
1459                 pr_err("Command ITT: 0x%08x received DataOUT for a"
1460                         " NON-WRITE command.\n", cmd->init_task_tag);
1461                 return iscsit_dump_data_payload(conn, payload_length, 1);
1462         }
1463         se_cmd = &cmd->se_cmd;
1464         iscsit_mod_dataout_timer(cmd);
1465
1466         if ((be32_to_cpu(hdr->offset) + payload_length) > cmd->se_cmd.data_length) {
1467                 pr_err("DataOut Offset: %u, Length %u greater than"
1468                         " iSCSI Command EDTL %u, protocol error.\n",
1469                         hdr->offset, payload_length, cmd->se_cmd.data_length);
1470                 return iscsit_reject_cmd(cmd, ISCSI_REASON_BOOKMARK_INVALID, buf);
1471         }
1472
1473         if (cmd->unsolicited_data) {
1474                 int dump_unsolicited_data = 0;
1475
1476                 if (conn->sess->sess_ops->InitialR2T) {
1477                         pr_err("Received unexpected unsolicited data"
1478                                 " while InitialR2T=Yes, protocol error.\n");
1479                         transport_send_check_condition_and_sense(&cmd->se_cmd,
1480                                         TCM_UNEXPECTED_UNSOLICITED_DATA, 0);
1481                         return -1;
1482                 }
1483                 /*
1484                  * Special case for dealing with Unsolicited DataOUT
1485                  * and Unsupported SAM WRITE Opcodes and SE resource allocation
1486                  * failures;
1487                  */
1488
1489                 /* Something's amiss if we're not in WRITE_PENDING state... */
1490                 WARN_ON(se_cmd->t_state != TRANSPORT_WRITE_PENDING);
1491                 if (!(se_cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE))
1492                         dump_unsolicited_data = 1;
1493
1494                 if (dump_unsolicited_data) {
1495                         /*
1496                          * Check if a delayed TASK_ABORTED status needs to
1497                          * be sent now if the ISCSI_FLAG_CMD_FINAL has been
1498                          * received with the unsolicited data out.
1499                          */
1500                         if (hdr->flags & ISCSI_FLAG_CMD_FINAL)
1501                                 iscsit_stop_dataout_timer(cmd);
1502
1503                         transport_check_aborted_status(se_cmd,
1504                                         (hdr->flags & ISCSI_FLAG_CMD_FINAL));
1505                         return iscsit_dump_data_payload(conn, payload_length, 1);
1506                 }
1507         } else {
1508                 /*
1509                  * For the normal solicited data path:
1510                  *
1511                  * Check for a delayed TASK_ABORTED status and dump any
1512                  * incoming data out payload if one exists.  Also, when the
1513                  * ISCSI_FLAG_CMD_FINAL is set to denote the end of the current
1514                  * data out sequence, we decrement outstanding_r2ts.  Once
1515                  * outstanding_r2ts reaches zero, go ahead and send the delayed
1516                  * TASK_ABORTED status.
1517                  */
1518                 if (se_cmd->transport_state & CMD_T_ABORTED) {
1519                         if (hdr->flags & ISCSI_FLAG_CMD_FINAL)
1520                                 if (--cmd->outstanding_r2ts < 1) {
1521                                         iscsit_stop_dataout_timer(cmd);
1522                                         transport_check_aborted_status(
1523                                                         se_cmd, 1);
1524                                 }
1525
1526                         return iscsit_dump_data_payload(conn, payload_length, 1);
1527                 }
1528         }
1529         /*
1530          * Perform DataSN, DataSequenceInOrder, DataPDUInOrder, and
1531          * within-command recovery checks before receiving the payload.
1532          */
1533         rc = iscsit_check_pre_dataout(cmd, buf);
1534         if (rc == DATAOUT_WITHIN_COMMAND_RECOVERY)
1535                 return 0;
1536         else if (rc == DATAOUT_CANNOT_RECOVER)
1537                 return -1;
1538         *success = true;
1539         return 0;
1540 }
1541 EXPORT_SYMBOL(__iscsit_check_dataout_hdr);
1542
1543 int
1544 iscsit_check_dataout_hdr(struct iscsi_conn *conn, void *buf,
1545                          struct iscsi_cmd **out_cmd)
1546 {
1547         struct iscsi_data *hdr = buf;
1548         struct iscsi_cmd *cmd;
1549         u32 payload_length = ntoh24(hdr->dlength);
1550         int rc;
1551         bool success = false;
1552
1553         if (!payload_length) {
1554                 pr_warn_ratelimited("DataOUT payload is ZERO, ignoring.\n");
1555                 return 0;
1556         }
1557
1558         if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
1559                 pr_err_ratelimited("DataSegmentLength: %u is greater than"
1560                         " MaxXmitDataSegmentLength: %u\n", payload_length,
1561                         conn->conn_ops->MaxXmitDataSegmentLength);
1562                 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, buf);
1563         }
1564
1565         cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt, payload_length);
1566         if (!cmd)
1567                 return 0;
1568
1569         rc = __iscsit_check_dataout_hdr(conn, buf, cmd, payload_length, &success);
1570
1571         if (success)
1572                 *out_cmd = cmd;
1573
1574         return rc;
1575 }
1576 EXPORT_SYMBOL(iscsit_check_dataout_hdr);
1577
1578 static int
1579 iscsit_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1580                    struct iscsi_data *hdr)
1581 {
1582         struct kvec *iov;
1583         u32 checksum, iov_count = 0, padding = 0, rx_got = 0, rx_size = 0;
1584         u32 payload_length = ntoh24(hdr->dlength);
1585         int iov_ret, data_crc_failed = 0;
1586
1587         rx_size += payload_length;
1588         iov = &cmd->iov_data[0];
1589
1590         iov_ret = iscsit_map_iovec(cmd, iov, be32_to_cpu(hdr->offset),
1591                                    payload_length);
1592         if (iov_ret < 0)
1593                 return -1;
1594
1595         iov_count += iov_ret;
1596
1597         padding = ((-payload_length) & 3);
1598         if (padding != 0) {
1599                 iov[iov_count].iov_base = cmd->pad_bytes;
1600                 iov[iov_count++].iov_len = padding;
1601                 rx_size += padding;
1602                 pr_debug("Receiving %u padding bytes.\n", padding);
1603         }
1604
1605         if (conn->conn_ops->DataDigest) {
1606                 iov[iov_count].iov_base = &checksum;
1607                 iov[iov_count++].iov_len = ISCSI_CRC_LEN;
1608                 rx_size += ISCSI_CRC_LEN;
1609         }
1610
1611         rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size);
1612
1613         iscsit_unmap_iovec(cmd);
1614
1615         if (rx_got != rx_size)
1616                 return -1;
1617
1618         if (conn->conn_ops->DataDigest) {
1619                 u32 data_crc;
1620
1621                 data_crc = iscsit_do_crypto_hash_sg(conn->conn_rx_hash, cmd,
1622                                                     be32_to_cpu(hdr->offset),
1623                                                     payload_length, padding,
1624                                                     cmd->pad_bytes);
1625
1626                 if (checksum != data_crc) {
1627                         pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"
1628                                 " DataSN: 0x%08x, CRC32C DataDigest 0x%08x"
1629                                 " does not match computed 0x%08x\n",
1630                                 hdr->itt, hdr->offset, payload_length,
1631                                 hdr->datasn, checksum, data_crc);
1632                         data_crc_failed = 1;
1633                 } else {
1634                         pr_debug("Got CRC32C DataDigest 0x%08x for"
1635                                 " %u bytes of Data Out\n", checksum,
1636                                 payload_length);
1637                 }
1638         }
1639
1640         return data_crc_failed;
1641 }
1642
1643 int
1644 iscsit_check_dataout_payload(struct iscsi_cmd *cmd, struct iscsi_data *hdr,
1645                              bool data_crc_failed)
1646 {
1647         struct iscsi_conn *conn = cmd->conn;
1648         int rc, ooo_cmdsn;
1649         /*
1650          * Increment post receive data and CRC values or perform
1651          * within-command recovery.
1652          */
1653         rc = iscsit_check_post_dataout(cmd, (unsigned char *)hdr, data_crc_failed);
1654         if ((rc == DATAOUT_NORMAL) || (rc == DATAOUT_WITHIN_COMMAND_RECOVERY))
1655                 return 0;
1656         else if (rc == DATAOUT_SEND_R2T) {
1657                 iscsit_set_dataout_sequence_values(cmd);
1658                 conn->conn_transport->iscsit_get_dataout(conn, cmd, false);
1659         } else if (rc == DATAOUT_SEND_TO_TRANSPORT) {
1660                 /*
1661                  * Handle extra special case for out of order
1662                  * Unsolicited Data Out.
1663                  */
1664                 spin_lock_bh(&cmd->istate_lock);
1665                 ooo_cmdsn = (cmd->cmd_flags & ICF_OOO_CMDSN);
1666                 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
1667                 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
1668                 spin_unlock_bh(&cmd->istate_lock);
1669
1670                 iscsit_stop_dataout_timer(cmd);
1671                 if (ooo_cmdsn)
1672                         return 0;
1673                 target_execute_cmd(&cmd->se_cmd);
1674                 return 0;
1675         } else /* DATAOUT_CANNOT_RECOVER */
1676                 return -1;
1677
1678         return 0;
1679 }
1680 EXPORT_SYMBOL(iscsit_check_dataout_payload);
1681
1682 static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf)
1683 {
1684         struct iscsi_cmd *cmd = NULL;
1685         struct iscsi_data *hdr = (struct iscsi_data *)buf;
1686         int rc;
1687         bool data_crc_failed = false;
1688
1689         rc = iscsit_check_dataout_hdr(conn, buf, &cmd);
1690         if (rc < 0)
1691                 return 0;
1692         else if (!cmd)
1693                 return 0;
1694
1695         rc = iscsit_get_dataout(conn, cmd, hdr);
1696         if (rc < 0)
1697                 return rc;
1698         else if (rc > 0)
1699                 data_crc_failed = true;
1700
1701         return iscsit_check_dataout_payload(cmd, hdr, data_crc_failed);
1702 }
1703
1704 int iscsit_setup_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1705                          struct iscsi_nopout *hdr)
1706 {
1707         u32 payload_length = ntoh24(hdr->dlength);
1708
1709         if (!(hdr->flags & ISCSI_FLAG_CMD_FINAL)) {
1710                 pr_err("NopOUT Flag's, Left Most Bit not set, protocol error.\n");
1711                 if (!cmd)
1712                         return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1713                                                  (unsigned char *)hdr);
1714
1715                 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
1716                                          (unsigned char *)hdr);
1717         }
1718
1719         if (hdr->itt == RESERVED_ITT && !(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
1720                 pr_err("NOPOUT ITT is reserved, but Immediate Bit is"
1721                         " not set, protocol error.\n");
1722                 if (!cmd)
1723                         return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1724                                                  (unsigned char *)hdr);
1725
1726                 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
1727                                          (unsigned char *)hdr);
1728         }
1729
1730         if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
1731                 pr_err("NOPOUT Ping Data DataSegmentLength: %u is"
1732                         " greater than MaxXmitDataSegmentLength: %u, protocol"
1733                         " error.\n", payload_length,
1734                         conn->conn_ops->MaxXmitDataSegmentLength);
1735                 if (!cmd)
1736                         return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1737                                                  (unsigned char *)hdr);
1738
1739                 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
1740                                          (unsigned char *)hdr);
1741         }
1742
1743         pr_debug("Got NOPOUT Ping %s ITT: 0x%08x, TTT: 0x%08x,"
1744                 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, Length: %u\n",
1745                 hdr->itt == RESERVED_ITT ? "Response" : "Request",
1746                 hdr->itt, hdr->ttt, hdr->cmdsn, hdr->exp_statsn,
1747                 payload_length);
1748         /*
1749          * This is not a response to a Unsolicited NopIN, which means
1750          * it can either be a NOPOUT ping request (with a valid ITT),
1751          * or a NOPOUT not requesting a NOPIN (with a reserved ITT).
1752          * Either way, make sure we allocate an struct iscsi_cmd, as both
1753          * can contain ping data.
1754          */
1755         if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1756                 cmd->iscsi_opcode       = ISCSI_OP_NOOP_OUT;
1757                 cmd->i_state            = ISTATE_SEND_NOPIN;
1758                 cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ?
1759                                                 1 : 0);
1760                 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt;
1761                 cmd->targ_xfer_tag      = 0xFFFFFFFF;
1762                 cmd->cmd_sn             = be32_to_cpu(hdr->cmdsn);
1763                 cmd->exp_stat_sn        = be32_to_cpu(hdr->exp_statsn);
1764                 cmd->data_direction     = DMA_NONE;
1765         }
1766
1767         return 0;
1768 }
1769 EXPORT_SYMBOL(iscsit_setup_nop_out);
1770
1771 int iscsit_process_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1772                            struct iscsi_nopout *hdr)
1773 {
1774         struct iscsi_cmd *cmd_p = NULL;
1775         int cmdsn_ret = 0;
1776         /*
1777          * Initiator is expecting a NopIN ping reply..
1778          */
1779         if (hdr->itt != RESERVED_ITT) {
1780                 if (!cmd)
1781                         return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1782                                                 (unsigned char *)hdr);
1783
1784                 spin_lock_bh(&conn->cmd_lock);
1785                 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
1786                 spin_unlock_bh(&conn->cmd_lock);
1787
1788                 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
1789
1790                 if (hdr->opcode & ISCSI_OP_IMMEDIATE) {
1791                         iscsit_add_cmd_to_response_queue(cmd, conn,
1792                                                          cmd->i_state);
1793                         return 0;
1794                 }
1795
1796                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
1797                                 (unsigned char *)hdr, hdr->cmdsn);
1798                 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP)
1799                         return 0;
1800                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1801                         return -1;
1802
1803                 return 0;
1804         }
1805         /*
1806          * This was a response to a unsolicited NOPIN ping.
1807          */
1808         if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) {
1809                 cmd_p = iscsit_find_cmd_from_ttt(conn, be32_to_cpu(hdr->ttt));
1810                 if (!cmd_p)
1811                         return -EINVAL;
1812
1813                 iscsit_stop_nopin_response_timer(conn);
1814
1815                 cmd_p->i_state = ISTATE_REMOVE;
1816                 iscsit_add_cmd_to_immediate_queue(cmd_p, conn, cmd_p->i_state);
1817
1818                 iscsit_start_nopin_timer(conn);
1819                 return 0;
1820         }
1821         /*
1822          * Otherwise, initiator is not expecting a NOPIN is response.
1823          * Just ignore for now.
1824          */
1825
1826         if (cmd)
1827                 iscsit_free_cmd(cmd, false);
1828
1829         return 0;
1830 }
1831 EXPORT_SYMBOL(iscsit_process_nop_out);
1832
1833 static int iscsit_handle_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1834                                  unsigned char *buf)
1835 {
1836         unsigned char *ping_data = NULL;
1837         struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf;
1838         struct kvec *iov = NULL;
1839         u32 payload_length = ntoh24(hdr->dlength);
1840         int ret;
1841
1842         ret = iscsit_setup_nop_out(conn, cmd, hdr);
1843         if (ret < 0)
1844                 return 0;
1845         /*
1846          * Handle NOP-OUT payload for traditional iSCSI sockets
1847          */
1848         if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1849                 u32 checksum, data_crc, padding = 0;
1850                 int niov = 0, rx_got, rx_size = payload_length;
1851
1852                 ping_data = kzalloc(payload_length + 1, GFP_KERNEL);
1853                 if (!ping_data) {
1854                         pr_err("Unable to allocate memory for"
1855                                 " NOPOUT ping data.\n");
1856                         ret = -1;
1857                         goto out;
1858                 }
1859
1860                 iov = &cmd->iov_misc[0];
1861                 iov[niov].iov_base      = ping_data;
1862                 iov[niov++].iov_len     = payload_length;
1863
1864                 padding = ((-payload_length) & 3);
1865                 if (padding != 0) {
1866                         pr_debug("Receiving %u additional bytes"
1867                                 " for padding.\n", padding);
1868                         iov[niov].iov_base      = &cmd->pad_bytes;
1869                         iov[niov++].iov_len     = padding;
1870                         rx_size += padding;
1871                 }
1872                 if (conn->conn_ops->DataDigest) {
1873                         iov[niov].iov_base      = &checksum;
1874                         iov[niov++].iov_len     = ISCSI_CRC_LEN;
1875                         rx_size += ISCSI_CRC_LEN;
1876                 }
1877
1878                 rx_got = rx_data(conn, &cmd->iov_misc[0], niov, rx_size);
1879                 if (rx_got != rx_size) {
1880                         ret = -1;
1881                         goto out;
1882                 }
1883
1884                 if (conn->conn_ops->DataDigest) {
1885                         iscsit_do_crypto_hash_buf(conn->conn_rx_hash,
1886                                         ping_data, payload_length,
1887                                         padding, cmd->pad_bytes,
1888                                         (u8 *)&data_crc);
1889
1890                         if (checksum != data_crc) {
1891                                 pr_err("Ping data CRC32C DataDigest"
1892                                 " 0x%08x does not match computed 0x%08x\n",
1893                                         checksum, data_crc);
1894                                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1895                                         pr_err("Unable to recover from"
1896                                         " NOPOUT Ping DataCRC failure while in"
1897                                                 " ERL=0.\n");
1898                                         ret = -1;
1899                                         goto out;
1900                                 } else {
1901                                         /*
1902                                          * Silently drop this PDU and let the
1903                                          * initiator plug the CmdSN gap.
1904                                          */
1905                                         pr_debug("Dropping NOPOUT"
1906                                         " Command CmdSN: 0x%08x due to"
1907                                         " DataCRC error.\n", hdr->cmdsn);
1908                                         ret = 0;
1909                                         goto out;
1910                                 }
1911                         } else {
1912                                 pr_debug("Got CRC32C DataDigest"
1913                                 " 0x%08x for %u bytes of ping data.\n",
1914                                         checksum, payload_length);
1915                         }
1916                 }
1917
1918                 ping_data[payload_length] = '\0';
1919                 /*
1920                  * Attach ping data to struct iscsi_cmd->buf_ptr.
1921                  */
1922                 cmd->buf_ptr = ping_data;
1923                 cmd->buf_ptr_size = payload_length;
1924
1925                 pr_debug("Got %u bytes of NOPOUT ping"
1926                         " data.\n", payload_length);
1927                 pr_debug("Ping Data: \"%s\"\n", ping_data);
1928         }
1929
1930         return iscsit_process_nop_out(conn, cmd, hdr);
1931 out:
1932         if (cmd)
1933                 iscsit_free_cmd(cmd, false);
1934
1935         kfree(ping_data);
1936         return ret;
1937 }
1938
1939 static enum tcm_tmreq_table iscsit_convert_tmf(u8 iscsi_tmf)
1940 {
1941         switch (iscsi_tmf) {
1942         case ISCSI_TM_FUNC_ABORT_TASK:
1943                 return TMR_ABORT_TASK;
1944         case ISCSI_TM_FUNC_ABORT_TASK_SET:
1945                 return TMR_ABORT_TASK_SET;
1946         case ISCSI_TM_FUNC_CLEAR_ACA:
1947                 return TMR_CLEAR_ACA;
1948         case ISCSI_TM_FUNC_CLEAR_TASK_SET:
1949                 return TMR_CLEAR_TASK_SET;
1950         case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET:
1951                 return TMR_LUN_RESET;
1952         case ISCSI_TM_FUNC_TARGET_WARM_RESET:
1953                 return TMR_TARGET_WARM_RESET;
1954         case ISCSI_TM_FUNC_TARGET_COLD_RESET:
1955                 return TMR_TARGET_COLD_RESET;
1956         default:
1957                 return TMR_UNKNOWN;
1958         }
1959 }
1960
1961 int
1962 iscsit_handle_task_mgt_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1963                            unsigned char *buf)
1964 {
1965         struct se_tmr_req *se_tmr;
1966         struct iscsi_tmr_req *tmr_req;
1967         struct iscsi_tm *hdr;
1968         int out_of_order_cmdsn = 0, ret;
1969         bool sess_ref = false;
1970         u8 function, tcm_function = TMR_UNKNOWN;
1971
1972         hdr                     = (struct iscsi_tm *) buf;
1973         hdr->flags &= ~ISCSI_FLAG_CMD_FINAL;
1974         function = hdr->flags;
1975
1976         pr_debug("Got Task Management Request ITT: 0x%08x, CmdSN:"
1977                 " 0x%08x, Function: 0x%02x, RefTaskTag: 0x%08x, RefCmdSN:"
1978                 " 0x%08x, CID: %hu\n", hdr->itt, hdr->cmdsn, function,
1979                 hdr->rtt, hdr->refcmdsn, conn->cid);
1980
1981         if ((function != ISCSI_TM_FUNC_ABORT_TASK) &&
1982             ((function != ISCSI_TM_FUNC_TASK_REASSIGN) &&
1983              hdr->rtt != RESERVED_ITT)) {
1984                 pr_err("RefTaskTag should be set to 0xFFFFFFFF.\n");
1985                 hdr->rtt = RESERVED_ITT;
1986         }
1987
1988         if ((function == ISCSI_TM_FUNC_TASK_REASSIGN) &&
1989                         !(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
1990                 pr_err("Task Management Request TASK_REASSIGN not"
1991                         " issued as immediate command, bad iSCSI Initiator"
1992                                 "implementation\n");
1993                 return iscsit_add_reject_cmd(cmd,
1994                                              ISCSI_REASON_PROTOCOL_ERROR, buf);
1995         }
1996         if ((function != ISCSI_TM_FUNC_ABORT_TASK) &&
1997             be32_to_cpu(hdr->refcmdsn) != ISCSI_RESERVED_TAG)
1998                 hdr->refcmdsn = cpu_to_be32(ISCSI_RESERVED_TAG);
1999
2000         cmd->data_direction = DMA_NONE;
2001
2002         cmd->tmr_req = kzalloc(sizeof(struct iscsi_tmr_req), GFP_KERNEL);
2003         if (!cmd->tmr_req) {
2004                 pr_err("Unable to allocate memory for"
2005                         " Task Management command!\n");
2006                 return iscsit_add_reject_cmd(cmd,
2007                                              ISCSI_REASON_BOOKMARK_NO_RESOURCES,
2008                                              buf);
2009         }
2010
2011         /*
2012          * TASK_REASSIGN for ERL=2 / connection stays inside of
2013          * LIO-Target $FABRIC_MOD
2014          */
2015         if (function != ISCSI_TM_FUNC_TASK_REASSIGN) {
2016                 transport_init_se_cmd(&cmd->se_cmd, &iscsi_ops,
2017                                       conn->sess->se_sess, 0, DMA_NONE,
2018                                       TCM_SIMPLE_TAG, cmd->sense_buffer + 2);
2019
2020                 target_get_sess_cmd(&cmd->se_cmd, true);
2021                 sess_ref = true;
2022                 tcm_function = iscsit_convert_tmf(function);
2023                 if (tcm_function == TMR_UNKNOWN) {
2024                         pr_err("Unknown iSCSI TMR Function:"
2025                                " 0x%02x\n", function);
2026                         return iscsit_add_reject_cmd(cmd,
2027                                 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
2028                 }
2029         }
2030         ret = core_tmr_alloc_req(&cmd->se_cmd, cmd->tmr_req, tcm_function,
2031                                  GFP_KERNEL);
2032         if (ret < 0)
2033                 return iscsit_add_reject_cmd(cmd,
2034                                 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
2035
2036         cmd->tmr_req->se_tmr_req = cmd->se_cmd.se_tmr_req;
2037
2038         cmd->iscsi_opcode       = ISCSI_OP_SCSI_TMFUNC;
2039         cmd->i_state            = ISTATE_SEND_TASKMGTRSP;
2040         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
2041         cmd->init_task_tag      = hdr->itt;
2042         cmd->targ_xfer_tag      = 0xFFFFFFFF;
2043         cmd->cmd_sn             = be32_to_cpu(hdr->cmdsn);
2044         cmd->exp_stat_sn        = be32_to_cpu(hdr->exp_statsn);
2045         se_tmr                  = cmd->se_cmd.se_tmr_req;
2046         tmr_req                 = cmd->tmr_req;
2047         /*
2048          * Locate the struct se_lun for all TMRs not related to ERL=2 TASK_REASSIGN
2049          */
2050         if (function != ISCSI_TM_FUNC_TASK_REASSIGN) {
2051                 ret = transport_lookup_tmr_lun(&cmd->se_cmd,
2052                                                scsilun_to_int(&hdr->lun));
2053                 if (ret < 0) {
2054                         se_tmr->response = ISCSI_TMF_RSP_NO_LUN;
2055                         goto attach;
2056                 }
2057         }
2058
2059         switch (function) {
2060         case ISCSI_TM_FUNC_ABORT_TASK:
2061                 se_tmr->response = iscsit_tmr_abort_task(cmd, buf);
2062                 if (se_tmr->response)
2063                         goto attach;
2064                 break;
2065         case ISCSI_TM_FUNC_ABORT_TASK_SET:
2066         case ISCSI_TM_FUNC_CLEAR_ACA:
2067         case ISCSI_TM_FUNC_CLEAR_TASK_SET:
2068         case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET:
2069                 break;
2070         case ISCSI_TM_FUNC_TARGET_WARM_RESET:
2071                 if (iscsit_tmr_task_warm_reset(conn, tmr_req, buf) < 0) {
2072                         se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED;
2073                         goto attach;
2074                 }
2075                 break;
2076         case ISCSI_TM_FUNC_TARGET_COLD_RESET:
2077                 if (iscsit_tmr_task_cold_reset(conn, tmr_req, buf) < 0) {
2078                         se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED;
2079                         goto attach;
2080                 }
2081                 break;
2082         case ISCSI_TM_FUNC_TASK_REASSIGN:
2083                 se_tmr->response = iscsit_tmr_task_reassign(cmd, buf);
2084                 /*
2085                  * Perform sanity checks on the ExpDataSN only if the
2086                  * TASK_REASSIGN was successful.
2087                  */
2088                 if (se_tmr->response)
2089                         break;
2090
2091                 if (iscsit_check_task_reassign_expdatasn(tmr_req, conn) < 0)
2092                         return iscsit_add_reject_cmd(cmd,
2093                                         ISCSI_REASON_BOOKMARK_INVALID, buf);
2094                 break;
2095         default:
2096                 pr_err("Unknown TMR function: 0x%02x, protocol"
2097                         " error.\n", function);
2098                 se_tmr->response = ISCSI_TMF_RSP_NOT_SUPPORTED;
2099                 goto attach;
2100         }
2101
2102         if ((function != ISCSI_TM_FUNC_TASK_REASSIGN) &&
2103             (se_tmr->response == ISCSI_TMF_RSP_COMPLETE))
2104                 se_tmr->call_transport = 1;
2105 attach:
2106         spin_lock_bh(&conn->cmd_lock);
2107         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
2108         spin_unlock_bh(&conn->cmd_lock);
2109
2110         if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
2111                 int cmdsn_ret = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn);
2112                 if (cmdsn_ret == CMDSN_HIGHER_THAN_EXP)
2113                         out_of_order_cmdsn = 1;
2114                 else if (cmdsn_ret == CMDSN_LOWER_THAN_EXP)
2115                         return 0;
2116                 else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
2117                         return -1;
2118         }
2119         iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
2120
2121         if (out_of_order_cmdsn || !(hdr->opcode & ISCSI_OP_IMMEDIATE))
2122                 return 0;
2123         /*
2124          * Found the referenced task, send to transport for processing.
2125          */
2126         if (se_tmr->call_transport)
2127                 return transport_generic_handle_tmr(&cmd->se_cmd);
2128
2129         /*
2130          * Could not find the referenced LUN, task, or Task Management
2131          * command not authorized or supported.  Change state and
2132          * let the tx_thread send the response.
2133          *
2134          * For connection recovery, this is also the default action for
2135          * TMR TASK_REASSIGN.
2136          */
2137         if (sess_ref) {
2138                 pr_debug("Handle TMR, using sess_ref=true check\n");
2139                 target_put_sess_cmd(&cmd->se_cmd);
2140         }
2141
2142         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2143         return 0;
2144 }
2145 EXPORT_SYMBOL(iscsit_handle_task_mgt_cmd);
2146
2147 /* #warning FIXME: Support Text Command parameters besides SendTargets */
2148 int
2149 iscsit_setup_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
2150                       struct iscsi_text *hdr)
2151 {
2152         u32 payload_length = ntoh24(hdr->dlength);
2153
2154         if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
2155                 pr_err("Unable to accept text parameter length: %u"
2156                         "greater than MaxXmitDataSegmentLength %u.\n",
2157                        payload_length, conn->conn_ops->MaxXmitDataSegmentLength);
2158                 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
2159                                          (unsigned char *)hdr);
2160         }
2161
2162         if (!(hdr->flags & ISCSI_FLAG_CMD_FINAL) ||
2163              (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE)) {
2164                 pr_err("Multi sequence text commands currently not supported\n");
2165                 return iscsit_reject_cmd(cmd, ISCSI_REASON_CMD_NOT_SUPPORTED,
2166                                         (unsigned char *)hdr);
2167         }
2168
2169         pr_debug("Got Text Request: ITT: 0x%08x, CmdSN: 0x%08x,"
2170                 " ExpStatSN: 0x%08x, Length: %u\n", hdr->itt, hdr->cmdsn,
2171                 hdr->exp_statsn, payload_length);
2172
2173         cmd->iscsi_opcode       = ISCSI_OP_TEXT;
2174         cmd->i_state            = ISTATE_SEND_TEXTRSP;
2175         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
2176         conn->sess->init_task_tag = cmd->init_task_tag  = hdr->itt;
2177         cmd->targ_xfer_tag      = 0xFFFFFFFF;
2178         cmd->cmd_sn             = be32_to_cpu(hdr->cmdsn);
2179         cmd->exp_stat_sn        = be32_to_cpu(hdr->exp_statsn);
2180         cmd->data_direction     = DMA_NONE;
2181         cmd->text_in_ptr        = NULL;
2182
2183         return 0;
2184 }
2185 EXPORT_SYMBOL(iscsit_setup_text_cmd);
2186
2187 int
2188 iscsit_process_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
2189                         struct iscsi_text *hdr)
2190 {
2191         unsigned char *text_in = cmd->text_in_ptr, *text_ptr;
2192         int cmdsn_ret;
2193
2194         if (!text_in) {
2195                 cmd->targ_xfer_tag = be32_to_cpu(hdr->ttt);
2196                 if (cmd->targ_xfer_tag == 0xFFFFFFFF) {
2197                         pr_err("Unable to locate text_in buffer for sendtargets"
2198                                " discovery\n");
2199                         goto reject;
2200                 }
2201                 goto empty_sendtargets;
2202         }
2203         if (strncmp("SendTargets", text_in, 11) != 0) {
2204                 pr_err("Received Text Data that is not"
2205                         " SendTargets, cannot continue.\n");
2206                 goto reject;
2207         }
2208         text_ptr = strchr(text_in, '=');
2209         if (!text_ptr) {
2210                 pr_err("No \"=\" separator found in Text Data,"
2211                         "  cannot continue.\n");
2212                 goto reject;
2213         }
2214         if (!strncmp("=All", text_ptr, 4)) {
2215                 cmd->cmd_flags |= ICF_SENDTARGETS_ALL;
2216         } else if (!strncmp("=iqn.", text_ptr, 5) ||
2217                    !strncmp("=eui.", text_ptr, 5)) {
2218                 cmd->cmd_flags |= ICF_SENDTARGETS_SINGLE;
2219         } else {
2220                 pr_err("Unable to locate valid SendTargets=%s value\n", text_ptr);
2221                 goto reject;
2222         }
2223
2224         spin_lock_bh(&conn->cmd_lock);
2225         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
2226         spin_unlock_bh(&conn->cmd_lock);
2227
2228 empty_sendtargets:
2229         iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
2230
2231         if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
2232                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
2233                                 (unsigned char *)hdr, hdr->cmdsn);
2234                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
2235                         return -1;
2236
2237                 return 0;
2238         }
2239
2240         return iscsit_execute_cmd(cmd, 0);
2241
2242 reject:
2243         return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
2244                                  (unsigned char *)hdr);
2245 }
2246 EXPORT_SYMBOL(iscsit_process_text_cmd);
2247
2248 static int
2249 iscsit_handle_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
2250                        unsigned char *buf)
2251 {
2252         struct iscsi_text *hdr = (struct iscsi_text *)buf;
2253         char *text_in = NULL;
2254         u32 payload_length = ntoh24(hdr->dlength);
2255         int rx_size, rc;
2256
2257         rc = iscsit_setup_text_cmd(conn, cmd, hdr);
2258         if (rc < 0)
2259                 return 0;
2260
2261         rx_size = payload_length;
2262         if (payload_length) {
2263                 u32 checksum = 0, data_crc = 0;
2264                 u32 padding = 0, pad_bytes = 0;
2265                 int niov = 0, rx_got;
2266                 struct kvec iov[3];
2267
2268                 text_in = kzalloc(payload_length, GFP_KERNEL);
2269                 if (!text_in) {
2270                         pr_err("Unable to allocate memory for"
2271                                 " incoming text parameters\n");
2272                         goto reject;
2273                 }
2274                 cmd->text_in_ptr = text_in;
2275
2276                 memset(iov, 0, 3 * sizeof(struct kvec));
2277                 iov[niov].iov_base      = text_in;
2278                 iov[niov++].iov_len     = payload_length;
2279
2280                 padding = ((-payload_length) & 3);
2281                 if (padding != 0) {
2282                         iov[niov].iov_base = &pad_bytes;
2283                         iov[niov++].iov_len  = padding;
2284                         rx_size += padding;
2285                         pr_debug("Receiving %u additional bytes"
2286                                         " for padding.\n", padding);
2287                 }
2288                 if (conn->conn_ops->DataDigest) {
2289                         iov[niov].iov_base      = &checksum;
2290                         iov[niov++].iov_len     = ISCSI_CRC_LEN;
2291                         rx_size += ISCSI_CRC_LEN;
2292                 }
2293
2294                 rx_got = rx_data(conn, &iov[0], niov, rx_size);
2295                 if (rx_got != rx_size)
2296                         goto reject;
2297
2298                 if (conn->conn_ops->DataDigest) {
2299                         iscsit_do_crypto_hash_buf(conn->conn_rx_hash,
2300                                         text_in, payload_length,
2301                                         padding, (u8 *)&pad_bytes,
2302                                         (u8 *)&data_crc);
2303
2304                         if (checksum != data_crc) {
2305                                 pr_err("Text data CRC32C DataDigest"
2306                                         " 0x%08x does not match computed"
2307                                         " 0x%08x\n", checksum, data_crc);
2308                                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
2309                                         pr_err("Unable to recover from"
2310                                         " Text Data digest failure while in"
2311                                                 " ERL=0.\n");
2312                                         goto reject;
2313                                 } else {
2314                                         /*
2315                                          * Silently drop this PDU and let the
2316                                          * initiator plug the CmdSN gap.
2317                                          */
2318                                         pr_debug("Dropping Text"
2319                                         " Command CmdSN: 0x%08x due to"
2320                                         " DataCRC error.\n", hdr->cmdsn);
2321                                         kfree(text_in);
2322                                         return 0;
2323                                 }
2324                         } else {
2325                                 pr_debug("Got CRC32C DataDigest"
2326                                         " 0x%08x for %u bytes of text data.\n",
2327                                                 checksum, payload_length);
2328                         }
2329                 }
2330                 text_in[payload_length - 1] = '\0';
2331                 pr_debug("Successfully read %d bytes of text"
2332                                 " data.\n", payload_length);
2333         }
2334
2335         return iscsit_process_text_cmd(conn, cmd, hdr);
2336
2337 reject:
2338         kfree(cmd->text_in_ptr);
2339         cmd->text_in_ptr = NULL;
2340         return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, buf);
2341 }
2342
2343 int iscsit_logout_closesession(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2344 {
2345         struct iscsi_conn *conn_p;
2346         struct iscsi_session *sess = conn->sess;
2347
2348         pr_debug("Received logout request CLOSESESSION on CID: %hu"
2349                 " for SID: %u.\n", conn->cid, conn->sess->sid);
2350
2351         atomic_set(&sess->session_logout, 1);
2352         atomic_set(&conn->conn_logout_remove, 1);
2353         conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_SESSION;
2354
2355         iscsit_inc_conn_usage_count(conn);
2356         iscsit_inc_session_usage_count(sess);
2357
2358         spin_lock_bh(&sess->conn_lock);
2359         list_for_each_entry(conn_p, &sess->sess_conn_list, conn_list) {
2360                 if (conn_p->conn_state != TARG_CONN_STATE_LOGGED_IN)
2361                         continue;
2362
2363                 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
2364                 conn_p->conn_state = TARG_CONN_STATE_IN_LOGOUT;
2365         }
2366         spin_unlock_bh(&sess->conn_lock);
2367
2368         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2369
2370         return 0;
2371 }
2372
2373 int iscsit_logout_closeconnection(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2374 {
2375         struct iscsi_conn *l_conn;
2376         struct iscsi_session *sess = conn->sess;
2377
2378         pr_debug("Received logout request CLOSECONNECTION for CID:"
2379                 " %hu on CID: %hu.\n", cmd->logout_cid, conn->cid);
2380
2381         /*
2382          * A Logout Request with a CLOSECONNECTION reason code for a CID
2383          * can arrive on a connection with a differing CID.
2384          */
2385         if (conn->cid == cmd->logout_cid) {
2386                 spin_lock_bh(&conn->state_lock);
2387                 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
2388                 conn->conn_state = TARG_CONN_STATE_IN_LOGOUT;
2389
2390                 atomic_set(&conn->conn_logout_remove, 1);
2391                 conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_CONNECTION;
2392                 iscsit_inc_conn_usage_count(conn);
2393
2394                 spin_unlock_bh(&conn->state_lock);
2395         } else {
2396                 /*
2397                  * Handle all different cid CLOSECONNECTION requests in
2398                  * iscsit_logout_post_handler_diffcid() as to give enough
2399                  * time for any non immediate command's CmdSN to be
2400                  * acknowledged on the connection in question.
2401                  *
2402                  * Here we simply make sure the CID is still around.
2403                  */
2404                 l_conn = iscsit_get_conn_from_cid(sess,
2405                                 cmd->logout_cid);
2406                 if (!l_conn) {
2407                         cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND;
2408                         iscsit_add_cmd_to_response_queue(cmd, conn,
2409                                         cmd->i_state);
2410                         return 0;
2411                 }
2412
2413                 iscsit_dec_conn_usage_count(l_conn);
2414         }
2415
2416         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2417
2418         return 0;
2419 }
2420
2421 int iscsit_logout_removeconnforrecovery(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2422 {
2423         struct iscsi_session *sess = conn->sess;
2424
2425         pr_debug("Received explicit REMOVECONNFORRECOVERY logout for"
2426                 " CID: %hu on CID: %hu.\n", cmd->logout_cid, conn->cid);
2427
2428         if (sess->sess_ops->ErrorRecoveryLevel != 2) {
2429                 pr_err("Received Logout Request REMOVECONNFORRECOVERY"
2430                         " while ERL!=2.\n");
2431                 cmd->logout_response = ISCSI_LOGOUT_RECOVERY_UNSUPPORTED;
2432                 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2433                 return 0;
2434         }
2435
2436         if (conn->cid == cmd->logout_cid) {
2437                 pr_err("Received Logout Request REMOVECONNFORRECOVERY"
2438                         " with CID: %hu on CID: %hu, implementation error.\n",
2439                                 cmd->logout_cid, conn->cid);
2440                 cmd->logout_response = ISCSI_LOGOUT_CLEANUP_FAILED;
2441                 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2442                 return 0;
2443         }
2444
2445         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2446
2447         return 0;
2448 }
2449
2450 int
2451 iscsit_handle_logout_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
2452                         unsigned char *buf)
2453 {
2454         int cmdsn_ret, logout_remove = 0;
2455         u8 reason_code = 0;
2456         struct iscsi_logout *hdr;
2457         struct iscsi_tiqn *tiqn = iscsit_snmp_get_tiqn(conn);
2458
2459         hdr                     = (struct iscsi_logout *) buf;
2460         reason_code             = (hdr->flags & 0x7f);
2461
2462         if (tiqn) {
2463                 spin_lock(&tiqn->logout_stats.lock);
2464                 if (reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION)
2465                         tiqn->logout_stats.normal_logouts++;
2466                 else
2467                         tiqn->logout_stats.abnormal_logouts++;
2468                 spin_unlock(&tiqn->logout_stats.lock);
2469         }
2470
2471         pr_debug("Got Logout Request ITT: 0x%08x CmdSN: 0x%08x"
2472                 " ExpStatSN: 0x%08x Reason: 0x%02x CID: %hu on CID: %hu\n",
2473                 hdr->itt, hdr->cmdsn, hdr->exp_statsn, reason_code,
2474                 hdr->cid, conn->cid);
2475
2476         if (conn->conn_state != TARG_CONN_STATE_LOGGED_IN) {
2477                 pr_err("Received logout request on connection that"
2478                         " is not in logged in state, ignoring request.\n");
2479                 iscsit_free_cmd(cmd, false);
2480                 return 0;
2481         }
2482
2483         cmd->iscsi_opcode       = ISCSI_OP_LOGOUT;
2484         cmd->i_state            = ISTATE_SEND_LOGOUTRSP;
2485         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
2486         conn->sess->init_task_tag = cmd->init_task_tag  = hdr->itt;
2487         cmd->targ_xfer_tag      = 0xFFFFFFFF;
2488         cmd->cmd_sn             = be32_to_cpu(hdr->cmdsn);
2489         cmd->exp_stat_sn        = be32_to_cpu(hdr->exp_statsn);
2490         cmd->logout_cid         = be16_to_cpu(hdr->cid);
2491         cmd->logout_reason      = reason_code;
2492         cmd->data_direction     = DMA_NONE;
2493
2494         /*
2495          * We need to sleep in these cases (by returning 1) until the Logout
2496          * Response gets sent in the tx thread.
2497          */
2498         if ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION) ||
2499            ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION) &&
2500             be16_to_cpu(hdr->cid) == conn->cid))
2501                 logout_remove = 1;
2502
2503         spin_lock_bh(&conn->cmd_lock);
2504         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
2505         spin_unlock_bh(&conn->cmd_lock);
2506
2507         if (reason_code != ISCSI_LOGOUT_REASON_RECOVERY)
2508                 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
2509
2510         /*
2511          * Immediate commands are executed, well, immediately.
2512          * Non-Immediate Logout Commands are executed in CmdSN order.
2513          */
2514         if (cmd->immediate_cmd) {
2515                 int ret = iscsit_execute_cmd(cmd, 0);
2516
2517                 if (ret < 0)
2518                         return ret;
2519         } else {
2520                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn);
2521                 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP)
2522                         logout_remove = 0;
2523                 else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
2524                         return -1;
2525         }
2526
2527         return logout_remove;
2528 }
2529 EXPORT_SYMBOL(iscsit_handle_logout_cmd);
2530
2531 int iscsit_handle_snack(
2532         struct iscsi_conn *conn,
2533         unsigned char *buf)
2534 {
2535         struct iscsi_snack *hdr;
2536
2537         hdr                     = (struct iscsi_snack *) buf;
2538         hdr->flags              &= ~ISCSI_FLAG_CMD_FINAL;
2539
2540         pr_debug("Got ISCSI_INIT_SNACK, ITT: 0x%08x, ExpStatSN:"
2541                 " 0x%08x, Type: 0x%02x, BegRun: 0x%08x, RunLength: 0x%08x,"
2542                 " CID: %hu\n", hdr->itt, hdr->exp_statsn, hdr->flags,
2543                         hdr->begrun, hdr->runlength, conn->cid);
2544
2545         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
2546                 pr_err("Initiator sent SNACK request while in"
2547                         " ErrorRecoveryLevel=0.\n");
2548                 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
2549                                          buf);
2550         }
2551         /*
2552          * SNACK_DATA and SNACK_R2T are both 0,  so check which function to
2553          * call from inside iscsi_send_recovery_datain_or_r2t().
2554          */
2555         switch (hdr->flags & ISCSI_FLAG_SNACK_TYPE_MASK) {
2556         case 0:
2557                 return iscsit_handle_recovery_datain_or_r2t(conn, buf,
2558                         hdr->itt,
2559                         be32_to_cpu(hdr->ttt),
2560                         be32_to_cpu(hdr->begrun),
2561                         be32_to_cpu(hdr->runlength));
2562         case ISCSI_FLAG_SNACK_TYPE_STATUS:
2563                 return iscsit_handle_status_snack(conn, hdr->itt,
2564                         be32_to_cpu(hdr->ttt),
2565                         be32_to_cpu(hdr->begrun), be32_to_cpu(hdr->runlength));
2566         case ISCSI_FLAG_SNACK_TYPE_DATA_ACK:
2567                 return iscsit_handle_data_ack(conn, be32_to_cpu(hdr->ttt),
2568                         be32_to_cpu(hdr->begrun),
2569                         be32_to_cpu(hdr->runlength));
2570         case ISCSI_FLAG_SNACK_TYPE_RDATA:
2571                 /* FIXME: Support R-Data SNACK */
2572                 pr_err("R-Data SNACK Not Supported.\n");
2573                 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
2574                                          buf);
2575         default:
2576                 pr_err("Unknown SNACK type 0x%02x, protocol"
2577                         " error.\n", hdr->flags & 0x0f);
2578                 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
2579                                          buf);
2580         }
2581
2582         return 0;
2583 }
2584 EXPORT_SYMBOL(iscsit_handle_snack);
2585
2586 static void iscsit_rx_thread_wait_for_tcp(struct iscsi_conn *conn)
2587 {
2588         if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) ||
2589             (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) {
2590                 wait_for_completion_interruptible_timeout(
2591                                         &conn->rx_half_close_comp,
2592                                         ISCSI_RX_THREAD_TCP_TIMEOUT * HZ);
2593         }
2594 }
2595
2596 static int iscsit_handle_immediate_data(
2597         struct iscsi_cmd *cmd,
2598         struct iscsi_scsi_req *hdr,
2599         u32 length)
2600 {
2601         int iov_ret, rx_got = 0, rx_size = 0;
2602         u32 checksum, iov_count = 0, padding = 0;
2603         struct iscsi_conn *conn = cmd->conn;
2604         struct kvec *iov;
2605
2606         iov_ret = iscsit_map_iovec(cmd, cmd->iov_data, cmd->write_data_done, length);
2607         if (iov_ret < 0)
2608                 return IMMEDIATE_DATA_CANNOT_RECOVER;
2609
2610         rx_size = length;
2611         iov_count = iov_ret;
2612         iov = &cmd->iov_data[0];
2613
2614         padding = ((-length) & 3);
2615         if (padding != 0) {
2616                 iov[iov_count].iov_base = cmd->pad_bytes;
2617                 iov[iov_count++].iov_len = padding;
2618                 rx_size += padding;
2619         }
2620
2621         if (conn->conn_ops->DataDigest) {
2622                 iov[iov_count].iov_base         = &checksum;
2623                 iov[iov_count++].iov_len        = ISCSI_CRC_LEN;
2624                 rx_size += ISCSI_CRC_LEN;
2625         }
2626
2627         rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size);
2628
2629         iscsit_unmap_iovec(cmd);
2630
2631         if (rx_got != rx_size) {
2632                 iscsit_rx_thread_wait_for_tcp(conn);
2633                 return IMMEDIATE_DATA_CANNOT_RECOVER;
2634         }
2635
2636         if (conn->conn_ops->DataDigest) {
2637                 u32 data_crc;
2638
2639                 data_crc = iscsit_do_crypto_hash_sg(conn->conn_rx_hash, cmd,
2640                                                     cmd->write_data_done, length, padding,
2641                                                     cmd->pad_bytes);
2642
2643                 if (checksum != data_crc) {
2644                         pr_err("ImmediateData CRC32C DataDigest 0x%08x"
2645                                 " does not match computed 0x%08x\n", checksum,
2646                                 data_crc);
2647
2648                         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
2649                                 pr_err("Unable to recover from"
2650                                         " Immediate Data digest failure while"
2651                                         " in ERL=0.\n");
2652                                 iscsit_reject_cmd(cmd,
2653                                                 ISCSI_REASON_DATA_DIGEST_ERROR,
2654                                                 (unsigned char *)hdr);
2655                                 return IMMEDIATE_DATA_CANNOT_RECOVER;
2656                         } else {
2657                                 iscsit_reject_cmd(cmd,
2658                                                 ISCSI_REASON_DATA_DIGEST_ERROR,
2659                                                 (unsigned char *)hdr);
2660                                 return IMMEDIATE_DATA_ERL1_CRC_FAILURE;
2661                         }
2662                 } else {
2663                         pr_debug("Got CRC32C DataDigest 0x%08x for"
2664                                 " %u bytes of Immediate Data\n", checksum,
2665                                 length);
2666                 }
2667         }
2668
2669         cmd->write_data_done += length;
2670
2671         if (cmd->write_data_done == cmd->se_cmd.data_length) {
2672                 spin_lock_bh(&cmd->istate_lock);
2673                 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
2674                 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
2675                 spin_unlock_bh(&cmd->istate_lock);
2676         }
2677
2678         return IMMEDIATE_DATA_NORMAL_OPERATION;
2679 }
2680
2681 /*
2682  *      Called with sess->conn_lock held.
2683  */
2684 /* #warning iscsi_build_conn_drop_async_message() only sends out on connections
2685         with active network interface */
2686 static void iscsit_build_conn_drop_async_message(struct iscsi_conn *conn)
2687 {
2688         struct iscsi_cmd *cmd;
2689         struct iscsi_conn *conn_p;
2690         bool found = false;
2691
2692         /*
2693          * Only send a Asynchronous Message on connections whos network
2694          * interface is still functional.
2695          */
2696         list_for_each_entry(conn_p, &conn->sess->sess_conn_list, conn_list) {
2697                 if (conn_p->conn_state == TARG_CONN_STATE_LOGGED_IN) {
2698                         iscsit_inc_conn_usage_count(conn_p);
2699                         found = true;
2700                         break;
2701                 }
2702         }
2703
2704         if (!found)
2705                 return;
2706
2707         cmd = iscsit_allocate_cmd(conn_p, TASK_RUNNING);
2708         if (!cmd) {
2709                 iscsit_dec_conn_usage_count(conn_p);
2710                 return;
2711         }
2712
2713         cmd->logout_cid = conn->cid;
2714         cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT;
2715         cmd->i_state = ISTATE_SEND_ASYNCMSG;
2716
2717         spin_lock_bh(&conn_p->cmd_lock);
2718         list_add_tail(&cmd->i_conn_node, &conn_p->conn_cmd_list);
2719         spin_unlock_bh(&conn_p->cmd_lock);
2720
2721         iscsit_add_cmd_to_response_queue(cmd, conn_p, cmd->i_state);
2722         iscsit_dec_conn_usage_count(conn_p);
2723 }
2724
2725 static int iscsit_send_conn_drop_async_message(
2726         struct iscsi_cmd *cmd,
2727         struct iscsi_conn *conn)
2728 {
2729         struct iscsi_async *hdr;
2730
2731         cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT;
2732
2733         hdr                     = (struct iscsi_async *) cmd->pdu;
2734         hdr->opcode             = ISCSI_OP_ASYNC_EVENT;
2735         hdr->flags              = ISCSI_FLAG_CMD_FINAL;
2736         cmd->init_task_tag      = RESERVED_ITT;
2737         cmd->targ_xfer_tag      = 0xFFFFFFFF;
2738         put_unaligned_be64(0xFFFFFFFFFFFFFFFFULL, &hdr->rsvd4[0]);
2739         cmd->stat_sn            = conn->stat_sn++;
2740         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
2741         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2742         hdr->max_cmdsn          = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
2743         hdr->async_event        = ISCSI_ASYNC_MSG_DROPPING_CONNECTION;
2744         hdr->param1             = cpu_to_be16(cmd->logout_cid);
2745         hdr->param2             = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Wait);
2746         hdr->param3             = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Retain);
2747
2748         pr_debug("Sending Connection Dropped Async Message StatSN:"
2749                 " 0x%08x, for CID: %hu on CID: %hu\n", cmd->stat_sn,
2750                         cmd->logout_cid, conn->cid);
2751
2752         return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0);
2753 }
2754
2755 static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *conn)
2756 {
2757         if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) ||
2758             (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) {
2759                 wait_for_completion_interruptible_timeout(
2760                                         &conn->tx_half_close_comp,
2761                                         ISCSI_TX_THREAD_TCP_TIMEOUT * HZ);
2762         }
2763 }
2764
2765 void
2766 iscsit_build_datain_pdu(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
2767                         struct iscsi_datain *datain, struct iscsi_data_rsp *hdr,
2768                         bool set_statsn)
2769 {
2770         hdr->opcode             = ISCSI_OP_SCSI_DATA_IN;
2771         hdr->flags              = datain->flags;
2772         if (hdr->flags & ISCSI_FLAG_DATA_STATUS) {
2773                 if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) {
2774                         hdr->flags |= ISCSI_FLAG_DATA_OVERFLOW;
2775                         hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
2776                 } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) {
2777                         hdr->flags |= ISCSI_FLAG_DATA_UNDERFLOW;
2778                         hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
2779                 }
2780         }
2781         hton24(hdr->dlength, datain->length);
2782         if (hdr->flags & ISCSI_FLAG_DATA_ACK)
2783                 int_to_scsilun(cmd->se_cmd.orig_fe_lun,
2784                                 (struct scsi_lun *)&hdr->lun);
2785         else
2786                 put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun);
2787
2788         hdr->itt                = cmd->init_task_tag;
2789
2790         if (hdr->flags & ISCSI_FLAG_DATA_ACK)
2791                 hdr->ttt                = cpu_to_be32(cmd->targ_xfer_tag);
2792         else
2793                 hdr->ttt                = cpu_to_be32(0xFFFFFFFF);
2794         if (set_statsn)
2795                 hdr->statsn             = cpu_to_be32(cmd->stat_sn);
2796         else
2797                 hdr->statsn             = cpu_to_be32(0xFFFFFFFF);
2798
2799         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2800         hdr->max_cmdsn          = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
2801         hdr->datasn             = cpu_to_be32(datain->data_sn);
2802         hdr->offset             = cpu_to_be32(datain->offset);
2803
2804         pr_debug("Built DataIN ITT: 0x%08x, StatSN: 0x%08x,"
2805                 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n",
2806                 cmd->init_task_tag, ntohl(hdr->statsn), ntohl(hdr->datasn),
2807                 ntohl(hdr->offset), datain->length, conn->cid);
2808 }
2809 EXPORT_SYMBOL(iscsit_build_datain_pdu);
2810
2811 static int iscsit_send_datain(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2812 {
2813         struct iscsi_data_rsp *hdr = (struct iscsi_data_rsp *)&cmd->pdu[0];
2814         struct iscsi_datain datain;
2815         struct iscsi_datain_req *dr;
2816         int eodr = 0, ret;
2817         bool set_statsn = false;
2818
2819         memset(&datain, 0, sizeof(struct iscsi_datain));
2820         dr = iscsit_get_datain_values(cmd, &datain);
2821         if (!dr) {
2822                 pr_err("iscsit_get_datain_values failed for ITT: 0x%08x\n",
2823                                 cmd->init_task_tag);
2824                 return -1;
2825         }
2826         /*
2827          * Be paranoid and double check the logic for now.
2828          */
2829         if ((datain.offset + datain.length) > cmd->se_cmd.data_length) {
2830                 pr_err("Command ITT: 0x%08x, datain.offset: %u and"
2831                         " datain.length: %u exceeds cmd->data_length: %u\n",
2832                         cmd->init_task_tag, datain.offset, datain.length,
2833                         cmd->se_cmd.data_length);
2834                 return -1;
2835         }
2836
2837         atomic_long_add(datain.length, &conn->sess->tx_data_octets);
2838         /*
2839          * Special case for successfully execution w/ both DATAIN
2840          * and Sense Data.
2841          */
2842         if ((datain.flags & ISCSI_FLAG_DATA_STATUS) &&
2843             (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE))
2844                 datain.flags &= ~ISCSI_FLAG_DATA_STATUS;
2845         else {
2846                 if ((dr->dr_complete == DATAIN_COMPLETE_NORMAL) ||
2847                     (dr->dr_complete == DATAIN_COMPLETE_CONNECTION_RECOVERY)) {
2848                         iscsit_increment_maxcmdsn(cmd, conn->sess);
2849                         cmd->stat_sn = conn->stat_sn++;
2850                         set_statsn = true;
2851                 } else if (dr->dr_complete ==
2852                            DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY)
2853                         set_statsn = true;
2854         }
2855
2856         iscsit_build_datain_pdu(cmd, conn, &datain, hdr, set_statsn);
2857
2858         ret = conn->conn_transport->iscsit_xmit_pdu(conn, cmd, dr, &datain, 0);
2859         if (ret < 0)
2860                 return ret;
2861
2862         if (dr->dr_complete) {
2863                 eodr = (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ?
2864                                 2 : 1;
2865                 iscsit_free_datain_req(cmd, dr);
2866         }
2867
2868         return eodr;
2869 }
2870
2871 int
2872 iscsit_build_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
2873                         struct iscsi_logout_rsp *hdr)
2874 {
2875         struct iscsi_conn *logout_conn = NULL;
2876         struct iscsi_conn_recovery *cr = NULL;
2877         struct iscsi_session *sess = conn->sess;
2878         /*
2879          * The actual shutting down of Sessions and/or Connections
2880          * for CLOSESESSION and CLOSECONNECTION Logout Requests
2881          * is done in scsi_logout_post_handler().
2882          */
2883         switch (cmd->logout_reason) {
2884         case ISCSI_LOGOUT_REASON_CLOSE_SESSION:
2885                 pr_debug("iSCSI session logout successful, setting"
2886                         " logout response to ISCSI_LOGOUT_SUCCESS.\n");
2887                 cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
2888                 break;
2889         case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION:
2890                 if (cmd->logout_response == ISCSI_LOGOUT_CID_NOT_FOUND)
2891                         break;
2892                 /*
2893                  * For CLOSECONNECTION logout requests carrying
2894                  * a matching logout CID -> local CID, the reference
2895                  * for the local CID will have been incremented in
2896                  * iscsi_logout_closeconnection().
2897                  *
2898                  * For CLOSECONNECTION logout requests carrying
2899                  * a different CID than the connection it arrived
2900                  * on, the connection responding to cmd->logout_cid
2901                  * is stopped in iscsit_logout_post_handler_diffcid().
2902                  */
2903
2904                 pr_debug("iSCSI CID: %hu logout on CID: %hu"
2905                         " successful.\n", cmd->logout_cid, conn->cid);
2906                 cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
2907                 break;
2908         case ISCSI_LOGOUT_REASON_RECOVERY:
2909                 if ((cmd->logout_response == ISCSI_LOGOUT_RECOVERY_UNSUPPORTED) ||
2910                     (cmd->logout_response == ISCSI_LOGOUT_CLEANUP_FAILED))
2911                         break;
2912                 /*
2913                  * If the connection is still active from our point of view
2914                  * force connection recovery to occur.
2915                  */
2916                 logout_conn = iscsit_get_conn_from_cid_rcfr(sess,
2917                                 cmd->logout_cid);
2918                 if (logout_conn) {
2919                         iscsit_connection_reinstatement_rcfr(logout_conn);
2920                         iscsit_dec_conn_usage_count(logout_conn);
2921                 }
2922
2923                 cr = iscsit_get_inactive_connection_recovery_entry(
2924                                 conn->sess, cmd->logout_cid);
2925                 if (!cr) {
2926                         pr_err("Unable to locate CID: %hu for"
2927                         " REMOVECONNFORRECOVERY Logout Request.\n",
2928                                 cmd->logout_cid);
2929                         cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND;
2930                         break;
2931                 }
2932
2933                 iscsit_discard_cr_cmds_by_expstatsn(cr, cmd->exp_stat_sn);
2934
2935                 pr_debug("iSCSI REMOVECONNFORRECOVERY logout"
2936                         " for recovery for CID: %hu on CID: %hu successful.\n",
2937                                 cmd->logout_cid, conn->cid);
2938                 cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
2939                 break;
2940         default:
2941                 pr_err("Unknown cmd->logout_reason: 0x%02x\n",
2942                                 cmd->logout_reason);
2943                 return -1;
2944         }
2945
2946         hdr->opcode             = ISCSI_OP_LOGOUT_RSP;
2947         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
2948         hdr->response           = cmd->logout_response;
2949         hdr->itt                = cmd->init_task_tag;
2950         cmd->stat_sn            = conn->stat_sn++;
2951         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
2952
2953         iscsit_increment_maxcmdsn(cmd, conn->sess);
2954         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2955         hdr->max_cmdsn          = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
2956
2957         pr_debug("Built Logout Response ITT: 0x%08x StatSN:"
2958                 " 0x%08x Response: 0x%02x CID: %hu on CID: %hu\n",
2959                 cmd->init_task_tag, cmd->stat_sn, hdr->response,
2960                 cmd->logout_cid, conn->cid);
2961
2962         return 0;
2963 }
2964 EXPORT_SYMBOL(iscsit_build_logout_rsp);
2965
2966 static int
2967 iscsit_send_logout(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2968 {
2969         int rc;
2970
2971         rc = iscsit_build_logout_rsp(cmd, conn,
2972                         (struct iscsi_logout_rsp *)&cmd->pdu[0]);
2973         if (rc < 0)
2974                 return rc;
2975
2976         return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0);
2977 }
2978
2979 void
2980 iscsit_build_nopin_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
2981                        struct iscsi_nopin *hdr, bool nopout_response)
2982 {
2983         hdr->opcode             = ISCSI_OP_NOOP_IN;
2984         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
2985         hton24(hdr->dlength, cmd->buf_ptr_size);
2986         if (nopout_response)
2987                 put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun);
2988         hdr->itt                = cmd->init_task_tag;
2989         hdr->ttt                = cpu_to_be32(cmd->targ_xfer_tag);
2990         cmd->stat_sn            = (nopout_response) ? conn->stat_sn++ :
2991                                   conn->stat_sn;
2992         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
2993
2994         if (nopout_response)
2995                 iscsit_increment_maxcmdsn(cmd, conn->sess);
2996
2997         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2998         hdr->max_cmdsn          = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
2999
3000         pr_debug("Built NOPIN %s Response ITT: 0x%08x, TTT: 0x%08x,"
3001                 " StatSN: 0x%08x, Length %u\n", (nopout_response) ?
3002                 "Solicited" : "Unsolicited", cmd->init_task_tag,
3003                 cmd->targ_xfer_tag, cmd->stat_sn, cmd->buf_ptr_size);
3004 }
3005 EXPORT_SYMBOL(iscsit_build_nopin_rsp);
3006
3007 /*
3008  *      Unsolicited NOPIN, either requesting a response or not.
3009  */
3010 static int iscsit_send_unsolicited_nopin(
3011         struct iscsi_cmd *cmd,
3012         struct iscsi_conn *conn,
3013         int want_response)
3014 {
3015         struct iscsi_nopin *hdr = (struct iscsi_nopin *)&cmd->pdu[0];
3016         int ret;
3017
3018         iscsit_build_nopin_rsp(cmd, conn, hdr, false);
3019
3020         pr_debug("Sending Unsolicited NOPIN TTT: 0x%08x StatSN:"
3021                 " 0x%08x CID: %hu\n", hdr->ttt, cmd->stat_sn, conn->cid);
3022
3023         ret = conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0);
3024         if (ret < 0)
3025                 return ret;
3026
3027         spin_lock_bh(&cmd->istate_lock);
3028         cmd->i_state = want_response ?
3029                 ISTATE_SENT_NOPIN_WANT_RESPONSE : ISTATE_SENT_STATUS;
3030         spin_unlock_bh(&cmd->istate_lock);
3031
3032         return 0;
3033 }
3034
3035 static int
3036 iscsit_send_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
3037 {
3038         struct iscsi_nopin *hdr = (struct iscsi_nopin *)&cmd->pdu[0];
3039
3040         iscsit_build_nopin_rsp(cmd, conn, hdr, true);
3041
3042         /*
3043          * NOPOUT Ping Data is attached to struct iscsi_cmd->buf_ptr.
3044          * NOPOUT DataSegmentLength is at struct iscsi_cmd->buf_ptr_size.
3045          */
3046         pr_debug("Echoing back %u bytes of ping data.\n", cmd->buf_ptr_size);
3047
3048         return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL,
3049                                                      cmd->buf_ptr,
3050                                                      cmd->buf_ptr_size);
3051 }
3052
3053 static int iscsit_send_r2t(
3054         struct iscsi_cmd *cmd,
3055         struct iscsi_conn *conn)
3056 {
3057         struct iscsi_r2t *r2t;
3058         struct iscsi_r2t_rsp *hdr;
3059         int ret;
3060
3061         r2t = iscsit_get_r2t_from_list(cmd);
3062         if (!r2t)
3063                 return -1;
3064
3065         hdr                     = (struct iscsi_r2t_rsp *) cmd->pdu;
3066         memset(hdr, 0, ISCSI_HDR_LEN);
3067         hdr->opcode             = ISCSI_OP_R2T;
3068         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
3069         int_to_scsilun(cmd->se_cmd.orig_fe_lun,
3070                         (struct scsi_lun *)&hdr->lun);
3071         hdr->itt                = cmd->init_task_tag;
3072         if (conn->conn_transport->iscsit_get_r2t_ttt)
3073                 conn->conn_transport->iscsit_get_r2t_ttt(conn, cmd, r2t);
3074         else
3075                 r2t->targ_xfer_tag = session_get_next_ttt(conn->sess);
3076         hdr->ttt                = cpu_to_be32(r2t->targ_xfer_tag);
3077         hdr->statsn             = cpu_to_be32(conn->stat_sn);
3078         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3079         hdr->max_cmdsn          = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
3080         hdr->r2tsn              = cpu_to_be32(r2t->r2t_sn);
3081         hdr->data_offset        = cpu_to_be32(r2t->offset);
3082         hdr->data_length        = cpu_to_be32(r2t->xfer_len);
3083
3084         pr_debug("Built %sR2T, ITT: 0x%08x, TTT: 0x%08x, StatSN:"
3085                 " 0x%08x, R2TSN: 0x%08x, Offset: %u, DDTL: %u, CID: %hu\n",
3086                 (!r2t->recovery_r2t) ? "" : "Recovery ", cmd->init_task_tag,
3087                 r2t->targ_xfer_tag, ntohl(hdr->statsn), r2t->r2t_sn,
3088                         r2t->offset, r2t->xfer_len, conn->cid);
3089
3090         spin_lock_bh(&cmd->r2t_lock);
3091         r2t->sent_r2t = 1;
3092         spin_unlock_bh(&cmd->r2t_lock);
3093
3094         ret = conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0);
3095         if (ret < 0) {
3096                 return ret;
3097         }
3098
3099         spin_lock_bh(&cmd->dataout_timeout_lock);
3100         iscsit_start_dataout_timer(cmd, conn);
3101         spin_unlock_bh(&cmd->dataout_timeout_lock);
3102
3103         return 0;
3104 }
3105
3106 /*
3107  *      @recovery: If called from iscsi_task_reassign_complete_write() for
3108  *              connection recovery.
3109  */
3110 int iscsit_build_r2ts_for_cmd(
3111         struct iscsi_conn *conn,
3112         struct iscsi_cmd *cmd,
3113         bool recovery)
3114 {
3115         int first_r2t = 1;
3116         u32 offset = 0, xfer_len = 0;
3117
3118         spin_lock_bh(&cmd->r2t_lock);
3119         if (cmd->cmd_flags & ICF_SENT_LAST_R2T) {
3120                 spin_unlock_bh(&cmd->r2t_lock);
3121                 return 0;
3122         }
3123
3124         if (conn->sess->sess_ops->DataSequenceInOrder &&
3125             !recovery)
3126                 cmd->r2t_offset = max(cmd->r2t_offset, cmd->write_data_done);
3127
3128         while (cmd->outstanding_r2ts < conn->sess->sess_ops->MaxOutstandingR2T) {
3129                 if (conn->sess->sess_ops->DataSequenceInOrder) {
3130                         offset = cmd->r2t_offset;
3131
3132                         if (first_r2t && recovery) {
3133                                 int new_data_end = offset +
3134                                         conn->sess->sess_ops->MaxBurstLength -
3135                                         cmd->next_burst_len;
3136
3137                                 if (new_data_end > cmd->se_cmd.data_length)
3138                                         xfer_len = cmd->se_cmd.data_length - offset;
3139                                 else
3140                                         xfer_len =
3141                                                 conn->sess->sess_ops->MaxBurstLength -
3142                                                 cmd->next_burst_len;
3143                         } else {
3144                                 int new_data_end = offset +
3145                                         conn->sess->sess_ops->MaxBurstLength;
3146
3147                                 if (new_data_end > cmd->se_cmd.data_length)
3148                                         xfer_len = cmd->se_cmd.data_length - offset;
3149                                 else
3150                                         xfer_len = conn->sess->sess_ops->MaxBurstLength;
3151                         }
3152                         cmd->r2t_offset += xfer_len;
3153
3154                         if (cmd->r2t_offset == cmd->se_cmd.data_length)
3155                                 cmd->cmd_flags |= ICF_SENT_LAST_R2T;
3156                 } else {
3157                         struct iscsi_seq *seq;
3158
3159                         seq = iscsit_get_seq_holder_for_r2t(cmd);
3160                         if (!seq) {
3161                                 spin_unlock_bh(&cmd->r2t_lock);
3162                                 return -1;
3163                         }
3164
3165                         offset = seq->offset;
3166                         xfer_len = seq->xfer_len;
3167
3168                         if (cmd->seq_send_order == cmd->seq_count)
3169                                 cmd->cmd_flags |= ICF_SENT_LAST_R2T;
3170                 }
3171                 cmd->outstanding_r2ts++;
3172                 first_r2t = 0;
3173
3174                 if (iscsit_add_r2t_to_list(cmd, offset, xfer_len, 0, 0) < 0) {
3175                         spin_unlock_bh(&cmd->r2t_lock);
3176                         return -1;
3177                 }
3178
3179                 if (cmd->cmd_flags & ICF_SENT_LAST_R2T)
3180                         break;
3181         }
3182         spin_unlock_bh(&cmd->r2t_lock);
3183
3184         return 0;
3185 }
3186 EXPORT_SYMBOL(iscsit_build_r2ts_for_cmd);
3187
3188 void iscsit_build_rsp_pdu(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
3189                         bool inc_stat_sn, struct iscsi_scsi_rsp *hdr)
3190 {
3191         if (inc_stat_sn)
3192                 cmd->stat_sn = conn->stat_sn++;
3193
3194         atomic_long_inc(&conn->sess->rsp_pdus);
3195
3196         memset(hdr, 0, ISCSI_HDR_LEN);
3197         hdr->opcode             = ISCSI_OP_SCSI_CMD_RSP;
3198         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
3199         if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) {
3200                 hdr->flags |= ISCSI_FLAG_CMD_OVERFLOW;
3201                 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
3202         } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) {
3203                 hdr->flags |= ISCSI_FLAG_CMD_UNDERFLOW;
3204                 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
3205         }
3206         hdr->response           = cmd->iscsi_response;
3207         hdr->cmd_status         = cmd->se_cmd.scsi_status;
3208         hdr->itt                = cmd->init_task_tag;
3209         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3210
3211         iscsit_increment_maxcmdsn(cmd, conn->sess);
3212         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3213         hdr->max_cmdsn          = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
3214
3215         pr_debug("Built SCSI Response, ITT: 0x%08x, StatSN: 0x%08x,"
3216                 " Response: 0x%02x, SAM Status: 0x%02x, CID: %hu\n",
3217                 cmd->init_task_tag, cmd->stat_sn, cmd->se_cmd.scsi_status,
3218                 cmd->se_cmd.scsi_status, conn->cid);
3219 }
3220 EXPORT_SYMBOL(iscsit_build_rsp_pdu);
3221
3222 static int iscsit_send_response(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
3223 {
3224         struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *)&cmd->pdu[0];
3225         bool inc_stat_sn = (cmd->i_state == ISTATE_SEND_STATUS);
3226         void *data_buf = NULL;
3227         u32 padding = 0, data_buf_len = 0;
3228
3229         iscsit_build_rsp_pdu(cmd, conn, inc_stat_sn, hdr);
3230
3231         /*
3232          * Attach SENSE DATA payload to iSCSI Response PDU
3233          */
3234         if (cmd->se_cmd.sense_buffer &&
3235            ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ||
3236             (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) {
3237                 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, cmd->sense_buffer);
3238                 cmd->se_cmd.scsi_sense_length += sizeof (__be16);
3239
3240                 padding         = -(cmd->se_cmd.scsi_sense_length) & 3;
3241                 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length);
3242                 data_buf = cmd->sense_buffer;
3243                 data_buf_len = cmd->se_cmd.scsi_sense_length + padding;
3244
3245                 if (padding) {
3246                         memset(cmd->sense_buffer +
3247                                 cmd->se_cmd.scsi_sense_length, 0, padding);
3248                         pr_debug("Adding %u bytes of padding to"
3249                                 " SENSE.\n", padding);
3250                 }
3251
3252                 pr_debug("Attaching SENSE DATA: %u bytes to iSCSI"
3253                                 " Response PDU\n",
3254                                 cmd->se_cmd.scsi_sense_length);
3255         }
3256
3257         return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, data_buf,
3258                                                      data_buf_len);
3259 }
3260
3261 static u8 iscsit_convert_tcm_tmr_rsp(struct se_tmr_req *se_tmr)
3262 {
3263         switch (se_tmr->response) {
3264         case TMR_FUNCTION_COMPLETE:
3265                 return ISCSI_TMF_RSP_COMPLETE;
3266         case TMR_TASK_DOES_NOT_EXIST:
3267                 return ISCSI_TMF_RSP_NO_TASK;
3268         case TMR_LUN_DOES_NOT_EXIST:
3269                 return ISCSI_TMF_RSP_NO_LUN;
3270         case TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED:
3271                 return ISCSI_TMF_RSP_NOT_SUPPORTED;
3272         case TMR_FUNCTION_REJECTED:
3273         default:
3274                 return ISCSI_TMF_RSP_REJECTED;
3275         }
3276 }
3277
3278 void
3279 iscsit_build_task_mgt_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
3280                           struct iscsi_tm_rsp *hdr)
3281 {
3282         struct se_tmr_req *se_tmr = cmd->se_cmd.se_tmr_req;
3283
3284         hdr->opcode             = ISCSI_OP_SCSI_TMFUNC_RSP;
3285         hdr->flags              = ISCSI_FLAG_CMD_FINAL;
3286         hdr->response           = iscsit_convert_tcm_tmr_rsp(se_tmr);
3287         hdr->itt                = cmd->init_task_tag;
3288         cmd->stat_sn            = conn->stat_sn++;
3289         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3290
3291         iscsit_increment_maxcmdsn(cmd, conn->sess);
3292         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3293         hdr->max_cmdsn          = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
3294
3295         pr_debug("Built Task Management Response ITT: 0x%08x,"
3296                 " StatSN: 0x%08x, Response: 0x%02x, CID: %hu\n",
3297                 cmd->init_task_tag, cmd->stat_sn, hdr->response, conn->cid);
3298 }
3299 EXPORT_SYMBOL(iscsit_build_task_mgt_rsp);
3300
3301 static int
3302 iscsit_send_task_mgt_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
3303 {
3304         struct iscsi_tm_rsp *hdr = (struct iscsi_tm_rsp *)&cmd->pdu[0];
3305
3306         iscsit_build_task_mgt_rsp(cmd, conn, hdr);
3307
3308         return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0);
3309 }
3310
3311 static bool iscsit_check_inaddr_any(struct iscsi_np *np)
3312 {
3313         bool ret = false;
3314
3315         if (np->np_sockaddr.ss_family == AF_INET6) {
3316                 const struct sockaddr_in6 sin6 = {
3317                         .sin6_addr = IN6ADDR_ANY_INIT };
3318                 struct sockaddr_in6 *sock_in6 =
3319                          (struct sockaddr_in6 *)&np->np_sockaddr;
3320
3321                 if (!memcmp(sock_in6->sin6_addr.s6_addr,
3322                                 sin6.sin6_addr.s6_addr, 16))
3323                         ret = true;
3324         } else {
3325                 struct sockaddr_in * sock_in =
3326                         (struct sockaddr_in *)&np->np_sockaddr;
3327
3328                 if (sock_in->sin_addr.s_addr == htonl(INADDR_ANY))
3329                         ret = true;
3330         }
3331
3332         return ret;
3333 }
3334
3335 #define SENDTARGETS_BUF_LIMIT 32768U
3336
3337 static int
3338 iscsit_build_sendtargets_response(struct iscsi_cmd *cmd,
3339                                   enum iscsit_transport_type network_transport,
3340                                   int skip_bytes, bool *completed)
3341 {
3342         char *payload = NULL;
3343         struct iscsi_conn *conn = cmd->conn;
3344         struct iscsi_portal_group *tpg;
3345         struct iscsi_tiqn *tiqn;
3346         struct iscsi_tpg_np *tpg_np;
3347         int buffer_len, end_of_buf = 0, len = 0, payload_len = 0;
3348         int target_name_printed;
3349         unsigned char buf[ISCSI_IQN_LEN+12]; /* iqn + "TargetName=" + \0 */
3350         unsigned char *text_in = cmd->text_in_ptr, *text_ptr = NULL;
3351         bool active;
3352
3353         buffer_len = min(conn->conn_ops->MaxRecvDataSegmentLength,
3354                          SENDTARGETS_BUF_LIMIT);
3355
3356         payload = kzalloc(buffer_len, GFP_KERNEL);
3357         if (!payload) {
3358                 pr_err("Unable to allocate memory for sendtargets"
3359                                 " response.\n");
3360                 return -ENOMEM;
3361         }
3362         /*
3363          * Locate pointer to iqn./eui. string for ICF_SENDTARGETS_SINGLE
3364          * explicit case..
3365          */
3366         if (cmd->cmd_flags & ICF_SENDTARGETS_SINGLE) {
3367                 text_ptr = strchr(text_in, '=');
3368                 if (!text_ptr) {
3369                         pr_err("Unable to locate '=' string in text_in:"
3370                                " %s\n", text_in);
3371                         kfree(payload);
3372                         return -EINVAL;
3373                 }
3374                 /*
3375                  * Skip over '=' character..
3376                  */
3377                 text_ptr += 1;
3378         }
3379
3380         spin_lock(&tiqn_lock);
3381         list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) {
3382                 if ((cmd->cmd_flags & ICF_SENDTARGETS_SINGLE) &&
3383                      strcmp(tiqn->tiqn, text_ptr)) {
3384                         continue;
3385                 }
3386
3387                 target_name_printed = 0;
3388
3389                 spin_lock(&tiqn->tiqn_tpg_lock);
3390                 list_for_each_entry(tpg, &tiqn->tiqn_tpg_list, tpg_list) {
3391
3392                         /* If demo_mode_discovery=0 and generate_node_acls=0
3393                          * (demo mode dislabed) do not return
3394                          * TargetName+TargetAddress unless a NodeACL exists.
3395                          */
3396
3397                         if ((tpg->tpg_attrib.generate_node_acls == 0) &&
3398                             (tpg->tpg_attrib.demo_mode_discovery == 0) &&
3399                             (!target_tpg_has_node_acl(&tpg->tpg_se_tpg,
3400                                 cmd->conn->sess->sess_ops->InitiatorName))) {
3401                                 continue;
3402                         }
3403
3404                         spin_lock(&tpg->tpg_state_lock);
3405                         active = (tpg->tpg_state == TPG_STATE_ACTIVE);
3406                         spin_unlock(&tpg->tpg_state_lock);
3407
3408                         if (!active && tpg->tpg_attrib.tpg_enabled_sendtargets)
3409                                 continue;
3410
3411                         spin_lock(&tpg->tpg_np_lock);
3412                         list_for_each_entry(tpg_np, &tpg->tpg_gnp_list,
3413                                                 tpg_np_list) {
3414                                 struct iscsi_np *np = tpg_np->tpg_np;
3415                                 bool inaddr_any = iscsit_check_inaddr_any(np);
3416                                 struct sockaddr_storage *sockaddr;
3417
3418                                 if (np->np_network_transport != network_transport)
3419                                         continue;
3420
3421                                 if (!target_name_printed) {
3422                                         len = sprintf(buf, "TargetName=%s",
3423                                                       tiqn->tiqn);
3424                                         len += 1;
3425
3426                                         if ((len + payload_len) > buffer_len) {
3427                                                 spin_unlock(&tpg->tpg_np_lock);
3428                                                 spin_unlock(&tiqn->tiqn_tpg_lock);
3429                                                 end_of_buf = 1;
3430                                                 goto eob;
3431                                         }
3432
3433                                         if (skip_bytes && len <= skip_bytes) {
3434                                                 skip_bytes -= len;
3435                                         } else {
3436                                                 memcpy(payload + payload_len, buf, len);
3437                                                 payload_len += len;
3438                                                 target_name_printed = 1;
3439                                                 if (len > skip_bytes)
3440                                                         skip_bytes = 0;
3441                                         }
3442                                 }
3443
3444                                 if (inaddr_any)
3445                                         sockaddr = &conn->local_sockaddr;
3446                                 else
3447                                         sockaddr = &np->np_sockaddr;
3448
3449                                 len = sprintf(buf, "TargetAddress="
3450                                               "%pISpc,%hu",
3451                                               sockaddr,
3452                                               tpg->tpgt);
3453                                 len += 1;
3454
3455                                 if ((len + payload_len) > buffer_len) {
3456                                         spin_unlock(&tpg->tpg_np_lock);
3457                                         spin_unlock(&tiqn->tiqn_tpg_lock);
3458                                         end_of_buf = 1;
3459                                         goto eob;
3460                                 }
3461
3462                                 if (skip_bytes && len <= skip_bytes) {
3463                                         skip_bytes -= len;
3464                                 } else {
3465                                         memcpy(payload + payload_len, buf, len);
3466                                         payload_len += len;
3467                                         if (len > skip_bytes)
3468                                                 skip_bytes = 0;
3469                                 }
3470                         }
3471                         spin_unlock(&tpg->tpg_np_lock);
3472                 }
3473                 spin_unlock(&tiqn->tiqn_tpg_lock);
3474 eob:
3475                 if (end_of_buf) {
3476                         *completed = false;
3477                         break;
3478                 }
3479
3480                 if (cmd->cmd_flags & ICF_SENDTARGETS_SINGLE)
3481                         break;
3482         }
3483         spin_unlock(&tiqn_lock);
3484
3485         cmd->buf_ptr = payload;
3486
3487         return payload_len;
3488 }
3489
3490 int
3491 iscsit_build_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
3492                       struct iscsi_text_rsp *hdr,
3493                       enum iscsit_transport_type network_transport)
3494 {
3495         int text_length, padding;
3496         bool completed = true;
3497
3498         text_length = iscsit_build_sendtargets_response(cmd, network_transport,
3499                                                         cmd->read_data_done,
3500                                                         &completed);
3501         if (text_length < 0)
3502                 return text_length;
3503
3504         if (completed) {
3505                 hdr->flags |= ISCSI_FLAG_CMD_FINAL;
3506         } else {
3507                 hdr->flags |= ISCSI_FLAG_TEXT_CONTINUE;
3508                 cmd->read_data_done += text_length;
3509                 if (cmd->targ_xfer_tag == 0xFFFFFFFF)
3510                         cmd->targ_xfer_tag = session_get_next_ttt(conn->sess);
3511         }
3512         hdr->opcode = ISCSI_OP_TEXT_RSP;
3513         padding = ((-text_length) & 3);
3514         hton24(hdr->dlength, text_length);
3515         hdr->itt = cmd->init_task_tag;
3516         hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag);
3517         cmd->stat_sn = conn->stat_sn++;
3518         hdr->statsn = cpu_to_be32(cmd->stat_sn);
3519
3520         iscsit_increment_maxcmdsn(cmd, conn->sess);
3521         /*
3522          * Reset maxcmdsn_inc in multi-part text payload exchanges to
3523          * correctly increment MaxCmdSN for each response answering a
3524          * non immediate text request with a valid CmdSN.
3525          */
3526         cmd->maxcmdsn_inc = 0;
3527         hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn);
3528         hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
3529
3530         pr_debug("Built Text Response: ITT: 0x%08x, TTT: 0x%08x, StatSN: 0x%08x,"
3531                 " Length: %u, CID: %hu F: %d C: %d\n", cmd->init_task_tag,
3532                 cmd->targ_xfer_tag, cmd->stat_sn, text_length, conn->cid,
3533                 !!(hdr->flags & ISCSI_FLAG_CMD_FINAL),
3534                 !!(hdr->flags & ISCSI_FLAG_TEXT_CONTINUE));
3535
3536         return text_length + padding;
3537 }
3538 EXPORT_SYMBOL(iscsit_build_text_rsp);
3539
3540 static int iscsit_send_text_rsp(
3541         struct iscsi_cmd *cmd,
3542         struct iscsi_conn *conn)
3543 {
3544         struct iscsi_text_rsp *hdr = (struct iscsi_text_rsp *)cmd->pdu;
3545         int text_length;
3546
3547         text_length = iscsit_build_text_rsp(cmd, conn, hdr,
3548                                 conn->conn_transport->transport_type);
3549         if (text_length < 0)
3550                 return text_length;
3551
3552         return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL,
3553                                                      cmd->buf_ptr,
3554                                                      text_length);
3555 }
3556
3557 void
3558 iscsit_build_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
3559                     struct iscsi_reject *hdr)
3560 {
3561         hdr->opcode             = ISCSI_OP_REJECT;
3562         hdr->reason             = cmd->reject_reason;
3563         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
3564         hton24(hdr->dlength, ISCSI_HDR_LEN);
3565         hdr->ffffffff           = cpu_to_be32(0xffffffff);
3566         cmd->stat_sn            = conn->stat_sn++;
3567         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3568         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3569         hdr->max_cmdsn          = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
3570
3571 }
3572 EXPORT_SYMBOL(iscsit_build_reject);
3573
3574 static int iscsit_send_reject(
3575         struct iscsi_cmd *cmd,
3576         struct iscsi_conn *conn)
3577 {
3578         struct iscsi_reject *hdr = (struct iscsi_reject *)&cmd->pdu[0];
3579
3580         iscsit_build_reject(cmd, conn, hdr);
3581
3582         pr_debug("Built Reject PDU StatSN: 0x%08x, Reason: 0x%02x,"
3583                 " CID: %hu\n", ntohl(hdr->statsn), hdr->reason, conn->cid);
3584
3585         return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL,
3586                                                      cmd->buf_ptr,
3587                                                      ISCSI_HDR_LEN);
3588 }
3589
3590 void iscsit_thread_get_cpumask(struct iscsi_conn *conn)
3591 {
3592         int ord, cpu;
3593         /*
3594          * bitmap_id is assigned from iscsit_global->ts_bitmap from
3595          * within iscsit_start_kthreads()
3596          *
3597          * Here we use bitmap_id to determine which CPU that this
3598          * iSCSI connection's RX/TX threads will be scheduled to
3599          * execute upon.
3600          */
3601         ord = conn->bitmap_id % cpumask_weight(cpu_online_mask);
3602         for_each_online_cpu(cpu) {
3603                 if (ord-- == 0) {
3604                         cpumask_set_cpu(cpu, conn->conn_cpumask);
3605                         return;
3606                 }
3607         }
3608         /*
3609          * This should never be reached..
3610          */
3611         dump_stack();
3612         cpumask_setall(conn->conn_cpumask);
3613 }
3614
3615 int
3616 iscsit_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
3617 {
3618         int ret;
3619
3620         switch (state) {
3621         case ISTATE_SEND_R2T:
3622                 ret = iscsit_send_r2t(cmd, conn);
3623                 if (ret < 0)
3624                         goto err;
3625                 break;
3626         case ISTATE_REMOVE:
3627                 spin_lock_bh(&conn->cmd_lock);
3628                 list_del_init(&cmd->i_conn_node);
3629                 spin_unlock_bh(&conn->cmd_lock);
3630
3631                 iscsit_free_cmd(cmd, false);
3632                 break;
3633         case ISTATE_SEND_NOPIN_WANT_RESPONSE:
3634                 iscsit_mod_nopin_response_timer(conn);
3635                 ret = iscsit_send_unsolicited_nopin(cmd, conn, 1);
3636                 if (ret < 0)
3637                         goto err;
3638                 break;
3639         case ISTATE_SEND_NOPIN_NO_RESPONSE:
3640                 ret = iscsit_send_unsolicited_nopin(cmd, conn, 0);
3641                 if (ret < 0)
3642                         goto err;
3643                 break;
3644         default:
3645                 pr_err("Unknown Opcode: 0x%02x ITT:"
3646                        " 0x%08x, i_state: %d on CID: %hu\n",
3647                        cmd->iscsi_opcode, cmd->init_task_tag, state,
3648                        conn->cid);
3649                 goto err;
3650         }
3651
3652         return 0;
3653
3654 err:
3655         return -1;
3656 }
3657 EXPORT_SYMBOL(iscsit_immediate_queue);
3658
3659 static int
3660 iscsit_handle_immediate_queue(struct iscsi_conn *conn)
3661 {
3662         struct iscsit_transport *t = conn->conn_transport;
3663         struct iscsi_queue_req *qr;
3664         struct iscsi_cmd *cmd;
3665         u8 state;
3666         int ret;
3667
3668         while ((qr = iscsit_get_cmd_from_immediate_queue(conn))) {
3669                 atomic_set(&conn->check_immediate_queue, 0);
3670                 cmd = qr->cmd;
3671                 state = qr->state;
3672                 kmem_cache_free(lio_qr_cache, qr);
3673
3674                 ret = t->iscsit_immediate_queue(conn, cmd, state);
3675                 if (ret < 0)
3676                         return ret;
3677         }
3678
3679         return 0;
3680 }
3681
3682 int
3683 iscsit_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
3684 {
3685         int ret;
3686
3687 check_rsp_state:
3688         switch (state) {
3689         case ISTATE_SEND_DATAIN:
3690                 ret = iscsit_send_datain(cmd, conn);
3691                 if (ret < 0)
3692                         goto err;
3693                 else if (!ret)
3694                         /* more drs */
3695                         goto check_rsp_state;
3696                 else if (ret == 1) {
3697                         /* all done */
3698                         spin_lock_bh(&cmd->istate_lock);
3699                         cmd->i_state = ISTATE_SENT_STATUS;
3700                         spin_unlock_bh(&cmd->istate_lock);
3701
3702                         if (atomic_read(&conn->check_immediate_queue))
3703                                 return 1;
3704
3705                         return 0;
3706                 } else if (ret == 2) {
3707                         /* Still must send status,
3708                            SCF_TRANSPORT_TASK_SENSE was set */
3709                         spin_lock_bh(&cmd->istate_lock);
3710                         cmd->i_state = ISTATE_SEND_STATUS;
3711                         spin_unlock_bh(&cmd->istate_lock);
3712                         state = ISTATE_SEND_STATUS;
3713                         goto check_rsp_state;
3714                 }
3715
3716                 break;
3717         case ISTATE_SEND_STATUS:
3718         case ISTATE_SEND_STATUS_RECOVERY:
3719                 ret = iscsit_send_response(cmd, conn);
3720                 break;
3721         case ISTATE_SEND_LOGOUTRSP:
3722                 ret = iscsit_send_logout(cmd, conn);
3723                 break;
3724         case ISTATE_SEND_ASYNCMSG:
3725                 ret = iscsit_send_conn_drop_async_message(
3726                         cmd, conn);
3727                 break;
3728         case ISTATE_SEND_NOPIN:
3729                 ret = iscsit_send_nopin(cmd, conn);
3730                 break;
3731         case ISTATE_SEND_REJECT:
3732                 ret = iscsit_send_reject(cmd, conn);
3733                 break;
3734         case ISTATE_SEND_TASKMGTRSP:
3735                 ret = iscsit_send_task_mgt_rsp(cmd, conn);
3736                 if (ret != 0)
3737                         break;
3738                 ret = iscsit_tmr_post_handler(cmd, conn);
3739                 if (ret != 0)
3740                         iscsit_fall_back_to_erl0(conn->sess);
3741                 break;
3742         case ISTATE_SEND_TEXTRSP:
3743                 ret = iscsit_send_text_rsp(cmd, conn);
3744                 break;
3745         default:
3746                 pr_err("Unknown Opcode: 0x%02x ITT:"
3747                        " 0x%08x, i_state: %d on CID: %hu\n",
3748                        cmd->iscsi_opcode, cmd->init_task_tag,
3749                        state, conn->cid);
3750                 goto err;
3751         }
3752         if (ret < 0)
3753                 goto err;
3754
3755         switch (state) {
3756         case ISTATE_SEND_LOGOUTRSP:
3757                 if (!iscsit_logout_post_handler(cmd, conn))
3758                         return -ECONNRESET;
3759                 /* fall through */
3760         case ISTATE_SEND_STATUS:
3761         case ISTATE_SEND_ASYNCMSG:
3762         case ISTATE_SEND_NOPIN:
3763         case ISTATE_SEND_STATUS_RECOVERY:
3764         case ISTATE_SEND_TEXTRSP:
3765         case ISTATE_SEND_TASKMGTRSP:
3766         case ISTATE_SEND_REJECT:
3767                 spin_lock_bh(&cmd->istate_lock);
3768                 cmd->i_state = ISTATE_SENT_STATUS;
3769                 spin_unlock_bh(&cmd->istate_lock);
3770                 break;
3771         default:
3772                 pr_err("Unknown Opcode: 0x%02x ITT:"
3773                        " 0x%08x, i_state: %d on CID: %hu\n",
3774                        cmd->iscsi_opcode, cmd->init_task_tag,
3775                        cmd->i_state, conn->cid);
3776                 goto err;
3777         }
3778
3779         if (atomic_read(&conn->check_immediate_queue))
3780                 return 1;
3781
3782         return 0;
3783
3784 err:
3785         return -1;
3786 }
3787 EXPORT_SYMBOL(iscsit_response_queue);
3788
3789 static int iscsit_handle_response_queue(struct iscsi_conn *conn)
3790 {
3791         struct iscsit_transport *t = conn->conn_transport;
3792         struct iscsi_queue_req *qr;
3793         struct iscsi_cmd *cmd;
3794         u8 state;
3795         int ret;
3796
3797         while ((qr = iscsit_get_cmd_from_response_queue(conn))) {
3798                 cmd = qr->cmd;
3799                 state = qr->state;
3800                 kmem_cache_free(lio_qr_cache, qr);
3801
3802                 ret = t->iscsit_response_queue(conn, cmd, state);
3803                 if (ret == 1 || ret < 0)
3804                         return ret;
3805         }
3806
3807         return 0;
3808 }
3809
3810 int iscsi_target_tx_thread(void *arg)
3811 {
3812         int ret = 0;
3813         struct iscsi_conn *conn = arg;
3814         /*
3815          * Allow ourselves to be interrupted by SIGINT so that a
3816          * connection recovery / failure event can be triggered externally.
3817          */
3818         allow_signal(SIGINT);
3819
3820         while (!kthread_should_stop()) {
3821                 /*
3822                  * Ensure that both TX and RX per connection kthreads
3823                  * are scheduled to run on the same CPU.
3824                  */
3825                 iscsit_thread_check_cpumask(conn, current, 1);
3826
3827                 wait_event_interruptible(conn->queues_wq,
3828                                          !iscsit_conn_all_queues_empty(conn));
3829
3830                 if (signal_pending(current))
3831                         goto transport_err;
3832
3833 get_immediate:
3834                 ret = iscsit_handle_immediate_queue(conn);
3835                 if (ret < 0)
3836                         goto transport_err;
3837
3838                 ret = iscsit_handle_response_queue(conn);
3839                 if (ret == 1)
3840                         goto get_immediate;
3841                 else if (ret == -ECONNRESET)
3842                         goto out;
3843                 else if (ret < 0)
3844                         goto transport_err;
3845         }
3846
3847 transport_err:
3848         /*
3849          * Avoid the normal connection failure code-path if this connection
3850          * is still within LOGIN mode, and iscsi_np process context is
3851          * responsible for cleaning up the early connection failure.
3852          */
3853         if (conn->conn_state != TARG_CONN_STATE_IN_LOGIN)
3854                 iscsit_take_action_for_connection_exit(conn);
3855 out:
3856         return 0;
3857 }
3858
3859 static int iscsi_target_rx_opcode(struct iscsi_conn *conn, unsigned char *buf)
3860 {
3861         struct iscsi_hdr *hdr = (struct iscsi_hdr *)buf;
3862         struct iscsi_cmd *cmd;
3863         int ret = 0;
3864
3865         switch (hdr->opcode & ISCSI_OPCODE_MASK) {
3866         case ISCSI_OP_SCSI_CMD:
3867                 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
3868                 if (!cmd)
3869                         goto reject;
3870
3871                 ret = iscsit_handle_scsi_cmd(conn, cmd, buf);
3872                 break;
3873         case ISCSI_OP_SCSI_DATA_OUT:
3874                 ret = iscsit_handle_data_out(conn, buf);
3875                 break;
3876         case ISCSI_OP_NOOP_OUT:
3877                 cmd = NULL;
3878                 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
3879                         cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
3880                         if (!cmd)
3881                                 goto reject;
3882                 }
3883                 ret = iscsit_handle_nop_out(conn, cmd, buf);
3884                 break;
3885         case ISCSI_OP_SCSI_TMFUNC:
3886                 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
3887                 if (!cmd)
3888                         goto reject;
3889
3890                 ret = iscsit_handle_task_mgt_cmd(conn, cmd, buf);
3891                 break;
3892         case ISCSI_OP_TEXT:
3893                 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) {
3894                         cmd = iscsit_find_cmd_from_itt(conn, hdr->itt);
3895                         if (!cmd)
3896                                 goto reject;
3897                 } else {
3898                         cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
3899                         if (!cmd)
3900                                 goto reject;
3901                 }
3902
3903                 ret = iscsit_handle_text_cmd(conn, cmd, buf);
3904                 break;
3905         case ISCSI_OP_LOGOUT:
3906                 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
3907                 if (!cmd)
3908                         goto reject;
3909
3910                 ret = iscsit_handle_logout_cmd(conn, cmd, buf);
3911                 if (ret > 0)
3912                         wait_for_completion_timeout(&conn->conn_logout_comp,
3913                                         SECONDS_FOR_LOGOUT_COMP * HZ);
3914                 break;
3915         case ISCSI_OP_SNACK:
3916                 ret = iscsit_handle_snack(conn, buf);
3917                 break;
3918         default:
3919                 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", hdr->opcode);
3920                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
3921                         pr_err("Cannot recover from unknown"
3922                         " opcode while ERL=0, closing iSCSI connection.\n");
3923                         return -1;
3924                 }
3925                 pr_err("Unable to recover from unknown opcode while OFMarker=No,"
3926                        " closing iSCSI connection.\n");
3927                 ret = -1;
3928                 break;
3929         }
3930
3931         return ret;
3932 reject:
3933         return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
3934 }
3935
3936 static bool iscsi_target_check_conn_state(struct iscsi_conn *conn)
3937 {
3938         bool ret;
3939
3940         spin_lock_bh(&conn->state_lock);
3941         ret = (conn->conn_state != TARG_CONN_STATE_LOGGED_IN);
3942         spin_unlock_bh(&conn->state_lock);
3943
3944         return ret;
3945 }
3946
3947 static void iscsit_get_rx_pdu(struct iscsi_conn *conn)
3948 {
3949         int ret;
3950         u8 buffer[ISCSI_HDR_LEN], opcode;
3951         u32 checksum = 0, digest = 0;
3952         struct kvec iov;
3953
3954         while (!kthread_should_stop()) {
3955                 /*
3956                  * Ensure that both TX and RX per connection kthreads
3957                  * are scheduled to run on the same CPU.
3958                  */
3959                 iscsit_thread_check_cpumask(conn, current, 0);
3960
3961                 memset(buffer, 0, ISCSI_HDR_LEN);
3962                 memset(&iov, 0, sizeof(struct kvec));
3963
3964                 iov.iov_base    = buffer;
3965                 iov.iov_len     = ISCSI_HDR_LEN;
3966
3967                 ret = rx_data(conn, &iov, 1, ISCSI_HDR_LEN);
3968                 if (ret != ISCSI_HDR_LEN) {
3969                         iscsit_rx_thread_wait_for_tcp(conn);
3970                         return;
3971                 }
3972
3973                 if (conn->conn_ops->HeaderDigest) {
3974                         iov.iov_base    = &digest;
3975                         iov.iov_len     = ISCSI_CRC_LEN;
3976
3977                         ret = rx_data(conn, &iov, 1, ISCSI_CRC_LEN);
3978                         if (ret != ISCSI_CRC_LEN) {
3979                                 iscsit_rx_thread_wait_for_tcp(conn);
3980                                 return;
3981                         }
3982
3983                         iscsit_do_crypto_hash_buf(conn->conn_rx_hash,
3984                                         buffer, ISCSI_HDR_LEN,
3985                                         0, NULL, (u8 *)&checksum);
3986
3987                         if (digest != checksum) {
3988                                 pr_err("HeaderDigest CRC32C failed,"
3989                                         " received 0x%08x, computed 0x%08x\n",
3990                                         digest, checksum);
3991                                 /*
3992                                  * Set the PDU to 0xff so it will intentionally
3993                                  * hit default in the switch below.
3994                                  */
3995                                 memset(buffer, 0xff, ISCSI_HDR_LEN);
3996                                 atomic_long_inc(&conn->sess->conn_digest_errors);
3997                         } else {
3998                                 pr_debug("Got HeaderDigest CRC32C"
3999                                                 " 0x%08x\n", checksum);
4000                         }
4001                 }
4002
4003                 if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT)
4004                         return;
4005
4006                 opcode = buffer[0] & ISCSI_OPCODE_MASK;
4007
4008                 if (conn->sess->sess_ops->SessionType &&
4009                    ((!(opcode & ISCSI_OP_TEXT)) ||
4010                     (!(opcode & ISCSI_OP_LOGOUT)))) {
4011                         pr_err("Received illegal iSCSI Opcode: 0x%02x"
4012                         " while in Discovery Session, rejecting.\n", opcode);
4013                         iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
4014                                           buffer);
4015                         return;
4016                 }
4017
4018                 ret = iscsi_target_rx_opcode(conn, buffer);
4019                 if (ret < 0)
4020                         return;
4021         }
4022 }
4023
4024 int iscsi_target_rx_thread(void *arg)
4025 {
4026         int rc;
4027         struct iscsi_conn *conn = arg;
4028
4029         /*
4030          * Allow ourselves to be interrupted by SIGINT so that a
4031          * connection recovery / failure event can be triggered externally.
4032          */
4033         allow_signal(SIGINT);
4034         /*
4035          * Wait for iscsi_post_login_handler() to complete before allowing
4036          * incoming iscsi/tcp socket I/O, and/or failing the connection.
4037          */
4038         rc = wait_for_completion_interruptible(&conn->rx_login_comp);
4039         if (rc < 0 || iscsi_target_check_conn_state(conn))
4040                 return 0;
4041
4042         if (!conn->conn_transport->iscsit_get_rx_pdu)
4043                 return 0;
4044
4045         conn->conn_transport->iscsit_get_rx_pdu(conn);
4046
4047         if (!signal_pending(current))
4048                 atomic_set(&conn->transport_failed, 1);
4049         iscsit_take_action_for_connection_exit(conn);
4050         return 0;
4051 }
4052
4053 static void iscsit_release_commands_from_conn(struct iscsi_conn *conn)
4054 {
4055         LIST_HEAD(tmp_list);
4056         struct iscsi_cmd *cmd = NULL, *cmd_tmp = NULL;
4057         struct iscsi_session *sess = conn->sess;
4058         /*
4059          * We expect this function to only ever be called from either RX or TX
4060          * thread context via iscsit_close_connection() once the other context
4061          * has been reset -> returned sleeping pre-handler state.
4062          */
4063         spin_lock_bh(&conn->cmd_lock);
4064         list_splice_init(&conn->conn_cmd_list, &tmp_list);
4065
4066         list_for_each_entry(cmd, &tmp_list, i_conn_node) {
4067                 struct se_cmd *se_cmd = &cmd->se_cmd;
4068
4069                 if (se_cmd->se_tfo != NULL) {
4070                         spin_lock(&se_cmd->t_state_lock);
4071                         se_cmd->transport_state |= CMD_T_FABRIC_STOP;
4072                         spin_unlock(&se_cmd->t_state_lock);
4073                 }
4074         }
4075         spin_unlock_bh(&conn->cmd_lock);
4076
4077         list_for_each_entry_safe(cmd, cmd_tmp, &tmp_list, i_conn_node) {
4078                 list_del_init(&cmd->i_conn_node);
4079
4080                 iscsit_increment_maxcmdsn(cmd, sess);
4081                 iscsit_free_cmd(cmd, true);
4082
4083         }
4084 }
4085
4086 static void iscsit_stop_timers_for_cmds(
4087         struct iscsi_conn *conn)
4088 {
4089         struct iscsi_cmd *cmd;
4090
4091         spin_lock_bh(&conn->cmd_lock);
4092         list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
4093                 if (cmd->data_direction == DMA_TO_DEVICE)
4094                         iscsit_stop_dataout_timer(cmd);
4095         }
4096         spin_unlock_bh(&conn->cmd_lock);
4097 }
4098
4099 int iscsit_close_connection(
4100         struct iscsi_conn *conn)
4101 {
4102         int conn_logout = (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT);
4103         struct iscsi_session    *sess = conn->sess;
4104
4105         pr_debug("Closing iSCSI connection CID %hu on SID:"
4106                 " %u\n", conn->cid, sess->sid);
4107         /*
4108          * Always up conn_logout_comp for the traditional TCP and HW_OFFLOAD
4109          * case just in case the RX Thread in iscsi_target_rx_opcode() is
4110          * sleeping and the logout response never got sent because the
4111          * connection failed.
4112          *
4113          * However for iser-target, isert_wait4logout() is using conn_logout_comp
4114          * to signal logout response TX interrupt completion.  Go ahead and skip
4115          * this for iser since isert_rx_opcode() does not wait on logout failure,
4116          * and to avoid iscsi_conn pointer dereference in iser-target code.
4117          */
4118         if (!conn->conn_transport->rdma_shutdown)
4119                 complete(&conn->conn_logout_comp);
4120
4121         if (!strcmp(current->comm, ISCSI_RX_THREAD_NAME)) {
4122                 if (conn->tx_thread &&
4123                     cmpxchg(&conn->tx_thread_active, true, false)) {
4124                         send_sig(SIGINT, conn->tx_thread, 1);
4125                         kthread_stop(conn->tx_thread);
4126                 }
4127         } else if (!strcmp(current->comm, ISCSI_TX_THREAD_NAME)) {
4128                 if (conn->rx_thread &&
4129                     cmpxchg(&conn->rx_thread_active, true, false)) {
4130                         send_sig(SIGINT, conn->rx_thread, 1);
4131                         kthread_stop(conn->rx_thread);
4132                 }
4133         }
4134
4135         spin_lock(&iscsit_global->ts_bitmap_lock);
4136         bitmap_release_region(iscsit_global->ts_bitmap, conn->bitmap_id,
4137                               get_order(1));
4138         spin_unlock(&iscsit_global->ts_bitmap_lock);
4139
4140         iscsit_stop_timers_for_cmds(conn);
4141         iscsit_stop_nopin_response_timer(conn);
4142         iscsit_stop_nopin_timer(conn);
4143
4144         if (conn->conn_transport->iscsit_wait_conn)
4145                 conn->conn_transport->iscsit_wait_conn(conn);
4146
4147         /*
4148          * During Connection recovery drop unacknowledged out of order
4149          * commands for this connection, and prepare the other commands
4150          * for reallegiance.
4151          *
4152          * During normal operation clear the out of order commands (but
4153          * do not free the struct iscsi_ooo_cmdsn's) and release all
4154          * struct iscsi_cmds.
4155          */
4156         if (atomic_read(&conn->connection_recovery)) {
4157                 iscsit_discard_unacknowledged_ooo_cmdsns_for_conn(conn);
4158                 iscsit_prepare_cmds_for_reallegiance(conn);
4159         } else {
4160                 iscsit_clear_ooo_cmdsns_for_conn(conn);
4161                 iscsit_release_commands_from_conn(conn);
4162         }
4163         iscsit_free_queue_reqs_for_conn(conn);
4164
4165         /*
4166          * Handle decrementing session or connection usage count if
4167          * a logout response was not able to be sent because the
4168          * connection failed.  Fall back to Session Recovery here.
4169          */
4170         if (atomic_read(&conn->conn_logout_remove)) {
4171                 if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_SESSION) {
4172                         iscsit_dec_conn_usage_count(conn);
4173                         iscsit_dec_session_usage_count(sess);
4174                 }
4175                 if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION)
4176                         iscsit_dec_conn_usage_count(conn);
4177
4178                 atomic_set(&conn->conn_logout_remove, 0);
4179                 atomic_set(&sess->session_reinstatement, 0);
4180                 atomic_set(&sess->session_fall_back_to_erl0, 1);
4181         }
4182
4183         spin_lock_bh(&sess->conn_lock);
4184         list_del(&conn->conn_list);
4185
4186         /*
4187          * Attempt to let the Initiator know this connection failed by
4188          * sending an Connection Dropped Async Message on another
4189          * active connection.
4190          */
4191         if (atomic_read(&conn->connection_recovery))
4192                 iscsit_build_conn_drop_async_message(conn);
4193
4194         spin_unlock_bh(&sess->conn_lock);
4195
4196         /*
4197          * If connection reinstatement is being performed on this connection,
4198          * up the connection reinstatement semaphore that is being blocked on
4199          * in iscsit_cause_connection_reinstatement().
4200          */
4201         spin_lock_bh(&conn->state_lock);
4202         if (atomic_read(&conn->sleep_on_conn_wait_comp)) {
4203                 spin_unlock_bh(&conn->state_lock);
4204                 complete(&conn->conn_wait_comp);
4205                 wait_for_completion(&conn->conn_post_wait_comp);
4206                 spin_lock_bh(&conn->state_lock);
4207         }
4208
4209         /*
4210          * If connection reinstatement is being performed on this connection
4211          * by receiving a REMOVECONNFORRECOVERY logout request, up the
4212          * connection wait rcfr semaphore that is being blocked on
4213          * an iscsit_connection_reinstatement_rcfr().
4214          */
4215         if (atomic_read(&conn->connection_wait_rcfr)) {
4216                 spin_unlock_bh(&conn->state_lock);
4217                 complete(&conn->conn_wait_rcfr_comp);
4218                 wait_for_completion(&conn->conn_post_wait_comp);
4219                 spin_lock_bh(&conn->state_lock);
4220         }
4221         atomic_set(&conn->connection_reinstatement, 1);
4222         spin_unlock_bh(&conn->state_lock);
4223
4224         /*
4225          * If any other processes are accessing this connection pointer we
4226          * must wait until they have completed.
4227          */
4228         iscsit_check_conn_usage_count(conn);
4229
4230         ahash_request_free(conn->conn_tx_hash);
4231         if (conn->conn_rx_hash) {
4232                 struct crypto_ahash *tfm;
4233
4234                 tfm = crypto_ahash_reqtfm(conn->conn_rx_hash);
4235                 ahash_request_free(conn->conn_rx_hash);
4236                 crypto_free_ahash(tfm);
4237         }
4238
4239         free_cpumask_var(conn->conn_cpumask);
4240
4241         kfree(conn->conn_ops);
4242         conn->conn_ops = NULL;
4243
4244         if (conn->sock)
4245                 sock_release(conn->sock);
4246
4247         if (conn->conn_transport->iscsit_free_conn)
4248                 conn->conn_transport->iscsit_free_conn(conn);
4249
4250         iscsit_put_transport(conn->conn_transport);
4251
4252         pr_debug("Moving to TARG_CONN_STATE_FREE.\n");
4253         conn->conn_state = TARG_CONN_STATE_FREE;
4254         kfree(conn);
4255
4256         spin_lock_bh(&sess->conn_lock);
4257         atomic_dec(&sess->nconn);
4258         pr_debug("Decremented iSCSI connection count to %hu from node:"
4259                 " %s\n", atomic_read(&sess->nconn),
4260                 sess->sess_ops->InitiatorName);
4261         /*
4262          * Make sure that if one connection fails in an non ERL=2 iSCSI
4263          * Session that they all fail.
4264          */
4265         if ((sess->sess_ops->ErrorRecoveryLevel != 2) && !conn_logout &&
4266              !atomic_read(&sess->session_logout))
4267                 atomic_set(&sess->session_fall_back_to_erl0, 1);
4268
4269         /*
4270          * If this was not the last connection in the session, and we are
4271          * performing session reinstatement or falling back to ERL=0, call
4272          * iscsit_stop_session() without sleeping to shutdown the other
4273          * active connections.
4274          */
4275         if (atomic_read(&sess->nconn)) {
4276                 if (!atomic_read(&sess->session_reinstatement) &&
4277                     !atomic_read(&sess->session_fall_back_to_erl0)) {
4278                         spin_unlock_bh(&sess->conn_lock);
4279                         return 0;
4280                 }
4281                 if (!atomic_read(&sess->session_stop_active)) {
4282                         atomic_set(&sess->session_stop_active, 1);
4283                         spin_unlock_bh(&sess->conn_lock);
4284                         iscsit_stop_session(sess, 0, 0);
4285                         return 0;
4286                 }
4287                 spin_unlock_bh(&sess->conn_lock);
4288                 return 0;
4289         }
4290
4291         /*
4292          * If this was the last connection in the session and one of the
4293          * following is occurring:
4294          *
4295          * Session Reinstatement is not being performed, and are falling back
4296          * to ERL=0 call iscsit_close_session().
4297          *
4298          * Session Logout was requested.  iscsit_close_session() will be called
4299          * elsewhere.
4300          *
4301          * Session Continuation is not being performed, start the Time2Retain
4302          * handler and check if sleep_on_sess_wait_sem is active.
4303          */
4304         if (!atomic_read(&sess->session_reinstatement) &&
4305              atomic_read(&sess->session_fall_back_to_erl0)) {
4306                 spin_unlock_bh(&sess->conn_lock);
4307                 iscsit_close_session(sess);
4308
4309                 return 0;
4310         } else if (atomic_read(&sess->session_logout)) {
4311                 pr_debug("Moving to TARG_SESS_STATE_FREE.\n");
4312                 sess->session_state = TARG_SESS_STATE_FREE;
4313                 spin_unlock_bh(&sess->conn_lock);
4314
4315                 if (atomic_read(&sess->sleep_on_sess_wait_comp))
4316                         complete(&sess->session_wait_comp);
4317
4318                 return 0;
4319         } else {
4320                 pr_debug("Moving to TARG_SESS_STATE_FAILED.\n");
4321                 sess->session_state = TARG_SESS_STATE_FAILED;
4322
4323                 if (!atomic_read(&sess->session_continuation)) {
4324                         spin_unlock_bh(&sess->conn_lock);
4325                         iscsit_start_time2retain_handler(sess);
4326                 } else
4327                         spin_unlock_bh(&sess->conn_lock);
4328
4329                 if (atomic_read(&sess->sleep_on_sess_wait_comp))
4330                         complete(&sess->session_wait_comp);
4331
4332                 return 0;
4333         }
4334 }
4335
4336 /*
4337  * If the iSCSI Session for the iSCSI Initiator Node exists,
4338  * forcefully shutdown the iSCSI NEXUS.
4339  */
4340 int iscsit_close_session(struct iscsi_session *sess)
4341 {
4342         struct iscsi_portal_group *tpg = sess->tpg;
4343         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
4344
4345         if (atomic_read(&sess->nconn)) {
4346                 pr_err("%d connection(s) still exist for iSCSI session"
4347                         " to %s\n", atomic_read(&sess->nconn),
4348                         sess->sess_ops->InitiatorName);
4349                 BUG();
4350         }
4351
4352         spin_lock_bh(&se_tpg->session_lock);
4353         atomic_set(&sess->session_logout, 1);
4354         atomic_set(&sess->session_reinstatement, 1);
4355         iscsit_stop_time2retain_timer(sess);
4356         spin_unlock_bh(&se_tpg->session_lock);
4357
4358         /*
4359          * transport_deregister_session_configfs() will clear the
4360          * struct se_node_acl->nacl_sess pointer now as a iscsi_np process context
4361          * can be setting it again with __transport_register_session() in
4362          * iscsi_post_login_handler() again after the iscsit_stop_session()
4363          * completes in iscsi_np context.
4364          */
4365         transport_deregister_session_configfs(sess->se_sess);
4366
4367         /*
4368          * If any other processes are accessing this session pointer we must
4369          * wait until they have completed.  If we are in an interrupt (the
4370          * time2retain handler) and contain and active session usage count we
4371          * restart the timer and exit.
4372          */
4373         if (!in_interrupt()) {
4374                 if (iscsit_check_session_usage_count(sess) == 1)
4375                         iscsit_stop_session(sess, 1, 1);
4376         } else {
4377                 if (iscsit_check_session_usage_count(sess) == 2) {
4378                         atomic_set(&sess->session_logout, 0);
4379                         iscsit_start_time2retain_handler(sess);
4380                         return 0;
4381                 }
4382         }
4383
4384         transport_deregister_session(sess->se_sess);
4385
4386         if (sess->sess_ops->ErrorRecoveryLevel == 2)
4387                 iscsit_free_connection_recovery_entires(sess);
4388
4389         iscsit_free_all_ooo_cmdsns(sess);
4390
4391         spin_lock_bh(&se_tpg->session_lock);
4392         pr_debug("Moving to TARG_SESS_STATE_FREE.\n");
4393         sess->session_state = TARG_SESS_STATE_FREE;
4394         pr_debug("Released iSCSI session from node: %s\n",
4395                         sess->sess_ops->InitiatorName);
4396         tpg->nsessions--;
4397         if (tpg->tpg_tiqn)
4398                 tpg->tpg_tiqn->tiqn_nsessions--;
4399
4400         pr_debug("Decremented number of active iSCSI Sessions on"
4401                 " iSCSI TPG: %hu to %u\n", tpg->tpgt, tpg->nsessions);
4402
4403         spin_lock(&sess_idr_lock);
4404         idr_remove(&sess_idr, sess->session_index);
4405         spin_unlock(&sess_idr_lock);
4406
4407         kfree(sess->sess_ops);
4408         sess->sess_ops = NULL;
4409         spin_unlock_bh(&se_tpg->session_lock);
4410
4411         kfree(sess);
4412         return 0;
4413 }
4414
4415 static void iscsit_logout_post_handler_closesession(
4416         struct iscsi_conn *conn)
4417 {
4418         struct iscsi_session *sess = conn->sess;
4419         int sleep = 1;
4420         /*
4421          * Traditional iscsi/tcp will invoke this logic from TX thread
4422          * context during session logout, so clear tx_thread_active and
4423          * sleep if iscsit_close_connection() has not already occured.
4424          *
4425          * Since iser-target invokes this logic from it's own workqueue,
4426          * always sleep waiting for RX/TX thread shutdown to complete
4427          * within iscsit_close_connection().
4428          */
4429         if (!conn->conn_transport->rdma_shutdown)
4430                 sleep = cmpxchg(&conn->tx_thread_active, true, false);
4431
4432         atomic_set(&conn->conn_logout_remove, 0);
4433         complete(&conn->conn_logout_comp);
4434
4435         iscsit_dec_conn_usage_count(conn);
4436         iscsit_stop_session(sess, sleep, sleep);
4437         iscsit_dec_session_usage_count(sess);
4438         iscsit_close_session(sess);
4439 }
4440
4441 static void iscsit_logout_post_handler_samecid(
4442         struct iscsi_conn *conn)
4443 {
4444         int sleep = 1;
4445
4446         if (!conn->conn_transport->rdma_shutdown)
4447                 sleep = cmpxchg(&conn->tx_thread_active, true, false);
4448
4449         atomic_set(&conn->conn_logout_remove, 0);
4450         complete(&conn->conn_logout_comp);
4451
4452         iscsit_cause_connection_reinstatement(conn, sleep);
4453         iscsit_dec_conn_usage_count(conn);
4454 }
4455
4456 static void iscsit_logout_post_handler_diffcid(
4457         struct iscsi_conn *conn,
4458         u16 cid)
4459 {
4460         struct iscsi_conn *l_conn;
4461         struct iscsi_session *sess = conn->sess;
4462         bool conn_found = false;
4463
4464         if (!sess)
4465                 return;
4466
4467         spin_lock_bh(&sess->conn_lock);
4468         list_for_each_entry(l_conn, &sess->sess_conn_list, conn_list) {
4469                 if (l_conn->cid == cid) {
4470                         iscsit_inc_conn_usage_count(l_conn);
4471                         conn_found = true;
4472                         break;
4473                 }
4474         }
4475         spin_unlock_bh(&sess->conn_lock);
4476
4477         if (!conn_found)
4478                 return;
4479
4480         if (l_conn->sock)
4481                 l_conn->sock->ops->shutdown(l_conn->sock, RCV_SHUTDOWN);
4482
4483         spin_lock_bh(&l_conn->state_lock);
4484         pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
4485         l_conn->conn_state = TARG_CONN_STATE_IN_LOGOUT;
4486         spin_unlock_bh(&l_conn->state_lock);
4487
4488         iscsit_cause_connection_reinstatement(l_conn, 1);
4489         iscsit_dec_conn_usage_count(l_conn);
4490 }
4491
4492 /*
4493  *      Return of 0 causes the TX thread to restart.
4494  */
4495 int iscsit_logout_post_handler(
4496         struct iscsi_cmd *cmd,
4497         struct iscsi_conn *conn)
4498 {
4499         int ret = 0;
4500
4501         switch (cmd->logout_reason) {
4502         case ISCSI_LOGOUT_REASON_CLOSE_SESSION:
4503                 switch (cmd->logout_response) {
4504                 case ISCSI_LOGOUT_SUCCESS:
4505                 case ISCSI_LOGOUT_CLEANUP_FAILED:
4506                 default:
4507                         iscsit_logout_post_handler_closesession(conn);
4508                         break;
4509                 }
4510                 ret = 0;
4511                 break;
4512         case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION:
4513                 if (conn->cid == cmd->logout_cid) {
4514                         switch (cmd->logout_response) {
4515                         case ISCSI_LOGOUT_SUCCESS:
4516                         case ISCSI_LOGOUT_CLEANUP_FAILED:
4517                         default:
4518                                 iscsit_logout_post_handler_samecid(conn);
4519                                 break;
4520                         }
4521                         ret = 0;
4522                 } else {
4523                         switch (cmd->logout_response) {
4524                         case ISCSI_LOGOUT_SUCCESS:
4525                                 iscsit_logout_post_handler_diffcid(conn,
4526                                         cmd->logout_cid);
4527                                 break;
4528                         case ISCSI_LOGOUT_CID_NOT_FOUND:
4529                         case ISCSI_LOGOUT_CLEANUP_FAILED:
4530                         default:
4531                                 break;
4532                         }
4533                         ret = 1;
4534                 }
4535                 break;
4536         case ISCSI_LOGOUT_REASON_RECOVERY:
4537                 switch (cmd->logout_response) {
4538                 case ISCSI_LOGOUT_SUCCESS:
4539                 case ISCSI_LOGOUT_CID_NOT_FOUND:
4540                 case ISCSI_LOGOUT_RECOVERY_UNSUPPORTED:
4541                 case ISCSI_LOGOUT_CLEANUP_FAILED:
4542                 default:
4543                         break;
4544                 }
4545                 ret = 1;
4546                 break;
4547         default:
4548                 break;
4549
4550         }
4551         return ret;
4552 }
4553 EXPORT_SYMBOL(iscsit_logout_post_handler);
4554
4555 void iscsit_fail_session(struct iscsi_session *sess)
4556 {
4557         struct iscsi_conn *conn;
4558
4559         spin_lock_bh(&sess->conn_lock);
4560         list_for_each_entry(conn, &sess->sess_conn_list, conn_list) {
4561                 pr_debug("Moving to TARG_CONN_STATE_CLEANUP_WAIT.\n");
4562                 conn->conn_state = TARG_CONN_STATE_CLEANUP_WAIT;
4563         }
4564         spin_unlock_bh(&sess->conn_lock);
4565
4566         pr_debug("Moving to TARG_SESS_STATE_FAILED.\n");
4567         sess->session_state = TARG_SESS_STATE_FAILED;
4568 }
4569
4570 int iscsit_free_session(struct iscsi_session *sess)
4571 {
4572         u16 conn_count = atomic_read(&sess->nconn);
4573         struct iscsi_conn *conn, *conn_tmp = NULL;
4574         int is_last;
4575
4576         spin_lock_bh(&sess->conn_lock);
4577         atomic_set(&sess->sleep_on_sess_wait_comp, 1);
4578
4579         list_for_each_entry_safe(conn, conn_tmp, &sess->sess_conn_list,
4580                         conn_list) {
4581                 if (conn_count == 0)
4582                         break;
4583
4584                 if (list_is_last(&conn->conn_list, &sess->sess_conn_list)) {
4585                         is_last = 1;
4586                 } else {
4587                         iscsit_inc_conn_usage_count(conn_tmp);
4588                         is_last = 0;
4589                 }
4590                 iscsit_inc_conn_usage_count(conn);
4591
4592                 spin_unlock_bh(&sess->conn_lock);
4593                 iscsit_cause_connection_reinstatement(conn, 1);
4594                 spin_lock_bh(&sess->conn_lock);
4595
4596                 iscsit_dec_conn_usage_count(conn);
4597                 if (is_last == 0)
4598                         iscsit_dec_conn_usage_count(conn_tmp);
4599
4600                 conn_count--;
4601         }
4602
4603         if (atomic_read(&sess->nconn)) {
4604                 spin_unlock_bh(&sess->conn_lock);
4605                 wait_for_completion(&sess->session_wait_comp);
4606         } else
4607                 spin_unlock_bh(&sess->conn_lock);
4608
4609         iscsit_close_session(sess);
4610         return 0;
4611 }
4612
4613 void iscsit_stop_session(
4614         struct iscsi_session *sess,
4615         int session_sleep,
4616         int connection_sleep)
4617 {
4618         u16 conn_count = atomic_read(&sess->nconn);
4619         struct iscsi_conn *conn, *conn_tmp = NULL;
4620         int is_last;
4621
4622         spin_lock_bh(&sess->conn_lock);
4623         if (session_sleep)
4624                 atomic_set(&sess->sleep_on_sess_wait_comp, 1);
4625
4626         if (connection_sleep) {
4627                 list_for_each_entry_safe(conn, conn_tmp, &sess->sess_conn_list,
4628                                 conn_list) {
4629                         if (conn_count == 0)
4630                                 break;
4631
4632                         if (list_is_last(&conn->conn_list, &sess->sess_conn_list)) {
4633                                 is_last = 1;
4634                         } else {
4635                                 iscsit_inc_conn_usage_count(conn_tmp);
4636                                 is_last = 0;
4637                         }
4638                         iscsit_inc_conn_usage_count(conn);
4639
4640                         spin_unlock_bh(&sess->conn_lock);
4641                         iscsit_cause_connection_reinstatement(conn, 1);
4642                         spin_lock_bh(&sess->conn_lock);
4643
4644                         iscsit_dec_conn_usage_count(conn);
4645                         if (is_last == 0)
4646                                 iscsit_dec_conn_usage_count(conn_tmp);
4647                         conn_count--;
4648                 }
4649         } else {
4650                 list_for_each_entry(conn, &sess->sess_conn_list, conn_list)
4651                         iscsit_cause_connection_reinstatement(conn, 0);
4652         }
4653
4654         if (session_sleep && atomic_read(&sess->nconn)) {
4655                 spin_unlock_bh(&sess->conn_lock);
4656                 wait_for_completion(&sess->session_wait_comp);
4657         } else
4658                 spin_unlock_bh(&sess->conn_lock);
4659 }
4660
4661 int iscsit_release_sessions_for_tpg(struct iscsi_portal_group *tpg, int force)
4662 {
4663         struct iscsi_session *sess;
4664         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
4665         struct se_session *se_sess, *se_sess_tmp;
4666         LIST_HEAD(free_list);
4667         int session_count = 0;
4668
4669         spin_lock_bh(&se_tpg->session_lock);
4670         if (tpg->nsessions && !force) {
4671                 spin_unlock_bh(&se_tpg->session_lock);
4672                 return -1;
4673         }
4674
4675         list_for_each_entry_safe(se_sess, se_sess_tmp, &se_tpg->tpg_sess_list,
4676                         sess_list) {
4677                 sess = (struct iscsi_session *)se_sess->fabric_sess_ptr;
4678
4679                 spin_lock(&sess->conn_lock);
4680                 if (atomic_read(&sess->session_fall_back_to_erl0) ||
4681                     atomic_read(&sess->session_logout) ||
4682                     (sess->time2retain_timer_flags & ISCSI_TF_EXPIRED)) {
4683                         spin_unlock(&sess->conn_lock);
4684                         continue;
4685                 }
4686                 atomic_set(&sess->session_reinstatement, 1);
4687                 spin_unlock(&sess->conn_lock);
4688
4689                 list_move_tail(&se_sess->sess_list, &free_list);
4690         }
4691         spin_unlock_bh(&se_tpg->session_lock);
4692
4693         list_for_each_entry_safe(se_sess, se_sess_tmp, &free_list, sess_list) {
4694                 sess = (struct iscsi_session *)se_sess->fabric_sess_ptr;
4695
4696                 iscsit_free_session(sess);
4697                 session_count++;
4698         }
4699
4700         pr_debug("Released %d iSCSI Session(s) from Target Portal"
4701                         " Group: %hu\n", session_count, tpg->tpgt);
4702         return 0;
4703 }
4704
4705 MODULE_DESCRIPTION("iSCSI-Target Driver for mainline target infrastructure");
4706 MODULE_VERSION("4.1.x");
4707 MODULE_AUTHOR("nab@Linux-iSCSI.org");
4708 MODULE_LICENSE("GPL");
4709
4710 module_init(iscsi_target_init_module);
4711 module_exit(iscsi_target_cleanup_module);