2 * LSI/Engenio/NetApp E-Series RDAC SCSI Device Handler
4 * Copyright (C) 2005 Mike Christie. All rights reserved.
5 * Copyright (C) Chandra Seetharaman, IBM Corp. 2007
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
22 #include <scsi/scsi.h>
23 #include <scsi/scsi_eh.h>
24 #include <scsi/scsi_dh.h>
25 #include <linux/workqueue.h>
26 #include <linux/slab.h>
27 #include <linux/module.h>
29 #define RDAC_NAME "rdac"
30 #define RDAC_RETRY_COUNT 5
35 * These struct definitions and the forming of the
36 * mode page were taken from the LSI RDAC 2.4 GPL'd
37 * driver, and then converted to Linux conventions.
39 #define RDAC_QUIESCENCE_TIME 20
43 #define RDAC_PAGE_CODE_REDUNDANT_CONTROLLER 0x2c
46 * Controller modes definitions
48 #define RDAC_MODE_TRANSFER_SPECIFIED_LUNS 0x02
53 #define RDAC_FORCED_QUIESENCE 0x02
55 #define RDAC_TIMEOUT (60 * HZ)
56 #define RDAC_RETRIES 3
58 struct rdac_mode_6_hdr {
65 struct rdac_mode_10_hdr {
73 struct rdac_mode_common {
74 u8 controller_serial[16];
75 u8 alt_controller_serial[16];
78 u8 quiescence_timeout;
82 struct rdac_pg_legacy {
83 struct rdac_mode_6_hdr hdr;
86 struct rdac_mode_common common;
87 #define MODE6_MAX_LUN 32
88 u8 lun_table[MODE6_MAX_LUN];
94 struct rdac_pg_expanded {
95 struct rdac_mode_10_hdr hdr;
99 struct rdac_mode_common common;
107 u8 page_code; /* 0xC9 */
110 u8 page_id[4]; /* "vace" */
116 #define SUBSYS_ID_LEN 16
117 #define SLOT_ID_LEN 2
118 #define ARRAY_LABEL_LEN 31
122 u8 page_code; /* 0xC4 */
125 u8 page_id[4]; /* "subs" */
126 u8 subsys_id[SUBSYS_ID_LEN];
128 u8 slot_id[SLOT_ID_LEN];
132 #define UNIQUE_ID_LEN 16
135 u8 page_code; /* 0xC8 */
138 u8 page_id[4]; /* "edid" */
142 u8 vol_user_label_len;
143 u8 vol_user_label[60];
144 u8 array_uniq_id_len;
145 u8 array_unique_id[UNIQUE_ID_LEN];
146 u8 array_user_label_len;
147 u8 array_user_label[60];
151 struct rdac_controller {
152 u8 array_id[UNIQUE_ID_LEN];
155 struct list_head node; /* list of all controllers */
157 struct rdac_pg_legacy legacy;
158 struct rdac_pg_expanded expanded;
161 u8 array_name[ARRAY_LABEL_LEN];
162 struct Scsi_Host *host;
165 struct work_struct ms_work;
166 struct scsi_device *ms_sdev;
167 struct list_head ms_head;
168 struct list_head dh_list;
173 u8 page_code; /* 0xC2 */
176 u8 page_id[4]; /* "swr4" */
180 u8 max_lun_supported;
181 u8 partitions[239]; /* Total allocation length should be 0xFF */
184 struct rdac_dh_data {
185 struct list_head node;
186 struct rdac_controller *ctlr;
187 struct scsi_device *sdev;
188 #define UNINITIALIZED_LUN (1 << 8)
192 #define RDAC_MODE_AVT 1
193 #define RDAC_MODE_IOSHIP 2
196 #define RDAC_STATE_ACTIVE 0
197 #define RDAC_STATE_PASSIVE 1
200 #define RDAC_LUN_UNOWNED 0
201 #define RDAC_LUN_OWNED 1
204 #define RDAC_PREFERRED 0
205 #define RDAC_NON_PREFERRED 1
209 struct c2_inquiry c2;
210 struct c4_inquiry c4;
211 struct c8_inquiry c8;
212 struct c9_inquiry c9;
216 static const char *mode[] = {
221 static const char *lun_state[] =
227 struct rdac_queue_data {
228 struct list_head entry;
229 struct rdac_dh_data *h;
230 activate_complete callback_fn;
234 static LIST_HEAD(ctlr_list);
235 static DEFINE_SPINLOCK(list_lock);
236 static struct workqueue_struct *kmpath_rdacd;
237 static void send_mode_select(struct work_struct *work);
240 * module parameter to enable rdac debug logging.
241 * 2 bits for each type of logging, only two types defined for now
242 * Can be enhanced if required at later point
244 static int rdac_logging = 1;
245 module_param(rdac_logging, int, S_IRUGO|S_IWUSR);
246 MODULE_PARM_DESC(rdac_logging, "A bit mask of rdac logging levels, "
247 "Default is 1 - failover logging enabled, "
248 "set it to 0xF to enable all the logs");
250 #define RDAC_LOG_FAILOVER 0
251 #define RDAC_LOG_SENSE 2
253 #define RDAC_LOG_BITS 2
255 #define RDAC_LOG_LEVEL(SHIFT) \
256 ((rdac_logging >> (SHIFT)) & ((1 << (RDAC_LOG_BITS)) - 1))
258 #define RDAC_LOG(SHIFT, sdev, f, arg...) \
260 if (unlikely(RDAC_LOG_LEVEL(SHIFT))) \
261 sdev_printk(KERN_INFO, sdev, RDAC_NAME ": " f "\n", ## arg); \
264 static unsigned int rdac_failover_get(struct rdac_controller *ctlr,
265 struct list_head *list,
268 struct scsi_device *sdev = ctlr->ms_sdev;
269 struct rdac_dh_data *h = sdev->handler_data;
270 struct rdac_mode_common *common;
272 struct rdac_queue_data *qdata;
275 if (h->ctlr->use_ms10) {
276 struct rdac_pg_expanded *rdac_pg;
278 data_size = sizeof(struct rdac_pg_expanded);
279 rdac_pg = &h->ctlr->mode_select.expanded;
280 memset(rdac_pg, 0, data_size);
281 common = &rdac_pg->common;
282 rdac_pg->page_code = RDAC_PAGE_CODE_REDUNDANT_CONTROLLER + 0x40;
283 rdac_pg->subpage_code = 0x1;
284 rdac_pg->page_len[0] = 0x01;
285 rdac_pg->page_len[1] = 0x28;
286 lun_table = rdac_pg->lun_table;
288 struct rdac_pg_legacy *rdac_pg;
290 data_size = sizeof(struct rdac_pg_legacy);
291 rdac_pg = &h->ctlr->mode_select.legacy;
292 memset(rdac_pg, 0, data_size);
293 common = &rdac_pg->common;
294 rdac_pg->page_code = RDAC_PAGE_CODE_REDUNDANT_CONTROLLER;
295 rdac_pg->page_len = 0x68;
296 lun_table = rdac_pg->lun_table;
298 common->rdac_mode[1] = RDAC_MODE_TRANSFER_SPECIFIED_LUNS;
299 common->quiescence_timeout = RDAC_QUIESCENCE_TIME;
300 common->rdac_options = RDAC_FORCED_QUIESENCE;
302 list_for_each_entry(qdata, list, entry) {
303 lun_table[qdata->h->lun] = 0x81;
306 /* Prepare the command. */
307 if (h->ctlr->use_ms10) {
308 cdb[0] = MODE_SELECT_10;
309 cdb[7] = data_size >> 8;
310 cdb[8] = data_size & 0xff;
312 cdb[0] = MODE_SELECT;
319 static void release_controller(struct kref *kref)
321 struct rdac_controller *ctlr;
322 ctlr = container_of(kref, struct rdac_controller, kref);
324 list_del(&ctlr->node);
328 static struct rdac_controller *get_controller(int index, char *array_name,
329 u8 *array_id, struct scsi_device *sdev)
331 struct rdac_controller *ctlr, *tmp;
333 list_for_each_entry(tmp, &ctlr_list, node) {
334 if ((memcmp(tmp->array_id, array_id, UNIQUE_ID_LEN) == 0) &&
335 (tmp->index == index) &&
336 (tmp->host == sdev->host)) {
337 kref_get(&tmp->kref);
341 ctlr = kmalloc(sizeof(*ctlr), GFP_ATOMIC);
345 /* initialize fields of controller */
346 memcpy(ctlr->array_id, array_id, UNIQUE_ID_LEN);
348 ctlr->host = sdev->host;
349 memcpy(ctlr->array_name, array_name, ARRAY_LABEL_LEN);
351 kref_init(&ctlr->kref);
354 ctlr->ms_sdev = NULL;
355 spin_lock_init(&ctlr->ms_lock);
356 INIT_WORK(&ctlr->ms_work, send_mode_select);
357 INIT_LIST_HEAD(&ctlr->ms_head);
358 list_add(&ctlr->node, &ctlr_list);
359 INIT_LIST_HEAD(&ctlr->dh_list);
364 static int get_lun_info(struct scsi_device *sdev, struct rdac_dh_data *h,
365 char *array_name, u8 *array_id)
367 int err = SCSI_DH_IO, i;
368 struct c8_inquiry *inqp = &h->inq.c8;
370 if (!scsi_get_vpd_page(sdev, 0xC8, (unsigned char *)inqp,
371 sizeof(struct c8_inquiry))) {
372 if (inqp->page_code != 0xc8)
373 return SCSI_DH_NOSYS;
374 if (inqp->page_id[0] != 'e' || inqp->page_id[1] != 'd' ||
375 inqp->page_id[2] != 'i' || inqp->page_id[3] != 'd')
376 return SCSI_DH_NOSYS;
377 h->lun = inqp->lun[7]; /* Uses only the last byte */
379 for(i=0; i<ARRAY_LABEL_LEN-1; ++i)
380 *(array_name+i) = inqp->array_user_label[(2*i)+1];
382 *(array_name+ARRAY_LABEL_LEN-1) = '\0';
383 memset(array_id, 0, UNIQUE_ID_LEN);
384 memcpy(array_id, inqp->array_unique_id, inqp->array_uniq_id_len);
390 static int check_ownership(struct scsi_device *sdev, struct rdac_dh_data *h)
392 int err = SCSI_DH_IO, access_state;
393 struct rdac_dh_data *tmp;
394 struct c9_inquiry *inqp = &h->inq.c9;
396 h->state = RDAC_STATE_ACTIVE;
397 if (!scsi_get_vpd_page(sdev, 0xC9, (unsigned char *)inqp,
398 sizeof(struct c9_inquiry))) {
399 /* detect the operating mode */
400 if ((inqp->avte_cvp >> 5) & 0x1)
401 h->mode = RDAC_MODE_IOSHIP; /* LUN in IOSHIP mode */
402 else if (inqp->avte_cvp >> 7)
403 h->mode = RDAC_MODE_AVT; /* LUN in AVT mode */
405 h->mode = RDAC_MODE; /* LUN in RDAC mode */
407 /* Update ownership */
408 if (inqp->avte_cvp & 0x1) {
409 h->lun_state = RDAC_LUN_OWNED;
410 access_state = SCSI_ACCESS_STATE_OPTIMAL;
412 h->lun_state = RDAC_LUN_UNOWNED;
413 if (h->mode == RDAC_MODE) {
414 h->state = RDAC_STATE_PASSIVE;
415 access_state = SCSI_ACCESS_STATE_STANDBY;
417 access_state = SCSI_ACCESS_STATE_ACTIVE;
420 /* Update path prio*/
421 if (inqp->path_prio & 0x1) {
422 h->preferred = RDAC_PREFERRED;
423 access_state |= SCSI_ACCESS_STATE_PREFERRED;
425 h->preferred = RDAC_NON_PREFERRED;
427 list_for_each_entry_rcu(tmp, &h->ctlr->dh_list, node) {
428 /* h->sdev should always be valid */
430 tmp->sdev->access_state = access_state;
439 static int initialize_controller(struct scsi_device *sdev,
440 struct rdac_dh_data *h, char *array_name, u8 *array_id)
442 int err = SCSI_DH_IO, index;
443 struct c4_inquiry *inqp = &h->inq.c4;
445 if (!scsi_get_vpd_page(sdev, 0xC4, (unsigned char *)inqp,
446 sizeof(struct c4_inquiry))) {
447 /* get the controller index */
448 if (inqp->slot_id[1] == 0x31)
453 spin_lock(&list_lock);
454 h->ctlr = get_controller(index, array_name, array_id, sdev);
456 err = SCSI_DH_RES_TEMP_UNAVAIL;
458 list_add_rcu(&h->node, &h->ctlr->dh_list);
461 spin_unlock(&list_lock);
467 static int set_mode_select(struct scsi_device *sdev, struct rdac_dh_data *h)
469 int err = SCSI_DH_IO;
470 struct c2_inquiry *inqp = &h->inq.c2;
472 if (!scsi_get_vpd_page(sdev, 0xC2, (unsigned char *)inqp,
473 sizeof(struct c2_inquiry))) {
475 * If more than MODE6_MAX_LUN luns are supported, use
478 if (inqp->max_lun_supported >= MODE6_MAX_LUN)
479 h->ctlr->use_ms10 = 1;
481 h->ctlr->use_ms10 = 0;
487 static int mode_select_handle_sense(struct scsi_device *sdev,
488 struct scsi_sense_hdr *sense_hdr)
490 int err = SCSI_DH_IO;
491 struct rdac_dh_data *h = sdev->handler_data;
493 if (!scsi_sense_valid(sense_hdr))
496 switch (sense_hdr->sense_key) {
498 case ABORTED_COMMAND:
503 if (sense_hdr->asc == 0x04 && sense_hdr->ascq == 0x01)
504 /* LUN Not Ready and is in the Process of Becoming
509 case ILLEGAL_REQUEST:
510 if (sense_hdr->asc == 0x91 && sense_hdr->ascq == 0x36)
512 * Command Lock contention
514 err = SCSI_DH_IMM_RETRY;
520 RDAC_LOG(RDAC_LOG_FAILOVER, sdev, "array %s, ctlr %d, "
521 "MODE_SELECT returned with sense %02x/%02x/%02x",
522 (char *) h->ctlr->array_name, h->ctlr->index,
523 sense_hdr->sense_key, sense_hdr->asc, sense_hdr->ascq);
529 static void send_mode_select(struct work_struct *work)
531 struct rdac_controller *ctlr =
532 container_of(work, struct rdac_controller, ms_work);
533 struct scsi_device *sdev = ctlr->ms_sdev;
534 struct rdac_dh_data *h = sdev->handler_data;
535 int err = SCSI_DH_OK, retry_cnt = RDAC_RETRY_COUNT;
536 struct rdac_queue_data *tmp, *qdata;
538 unsigned char cdb[COMMAND_SIZE(MODE_SELECT_10)];
539 struct scsi_sense_hdr sshdr;
540 unsigned int data_size;
541 u64 req_flags = REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT |
544 spin_lock(&ctlr->ms_lock);
545 list_splice_init(&ctlr->ms_head, &list);
547 ctlr->ms_sdev = NULL;
548 spin_unlock(&ctlr->ms_lock);
551 data_size = rdac_failover_get(ctlr, &list, cdb);
553 RDAC_LOG(RDAC_LOG_FAILOVER, sdev, "array %s, ctlr %d, "
554 "%s MODE_SELECT command",
555 (char *) h->ctlr->array_name, h->ctlr->index,
556 (retry_cnt == RDAC_RETRY_COUNT) ? "queueing" : "retrying");
558 if (scsi_execute(sdev, cdb, DMA_TO_DEVICE, &h->ctlr->mode_select,
559 data_size, NULL, &sshdr, RDAC_TIMEOUT * HZ,
560 RDAC_RETRIES, req_flags, 0, NULL)) {
561 err = mode_select_handle_sense(sdev, &sshdr);
562 if (err == SCSI_DH_RETRY && retry_cnt--)
564 if (err == SCSI_DH_IMM_RETRY)
567 if (err == SCSI_DH_OK) {
568 h->state = RDAC_STATE_ACTIVE;
569 RDAC_LOG(RDAC_LOG_FAILOVER, sdev, "array %s, ctlr %d, "
570 "MODE_SELECT completed",
571 (char *) h->ctlr->array_name, h->ctlr->index);
574 list_for_each_entry_safe(qdata, tmp, &list, entry) {
575 list_del(&qdata->entry);
576 if (err == SCSI_DH_OK)
577 qdata->h->state = RDAC_STATE_ACTIVE;
578 if (qdata->callback_fn)
579 qdata->callback_fn(qdata->callback_data, err);
585 static int queue_mode_select(struct scsi_device *sdev,
586 activate_complete fn, void *data)
588 struct rdac_queue_data *qdata;
589 struct rdac_controller *ctlr;
591 qdata = kzalloc(sizeof(*qdata), GFP_KERNEL);
593 return SCSI_DH_RETRY;
595 qdata->h = sdev->handler_data;
596 qdata->callback_fn = fn;
597 qdata->callback_data = data;
599 ctlr = qdata->h->ctlr;
600 spin_lock(&ctlr->ms_lock);
601 list_add_tail(&qdata->entry, &ctlr->ms_head);
602 if (!ctlr->ms_queued) {
604 ctlr->ms_sdev = sdev;
605 queue_work(kmpath_rdacd, &ctlr->ms_work);
607 spin_unlock(&ctlr->ms_lock);
611 static int rdac_activate(struct scsi_device *sdev,
612 activate_complete fn, void *data)
614 struct rdac_dh_data *h = sdev->handler_data;
615 int err = SCSI_DH_OK;
618 err = check_ownership(sdev, h);
619 if (err != SCSI_DH_OK)
624 if (h->lun_state == RDAC_LUN_UNOWNED)
627 case RDAC_MODE_IOSHIP:
628 if ((h->lun_state == RDAC_LUN_UNOWNED) &&
629 (h->preferred == RDAC_PREFERRED))
637 err = queue_mode_select(sdev, fn, data);
638 if (err == SCSI_DH_OK)
647 static int rdac_prep_fn(struct scsi_device *sdev, struct request *req)
649 struct rdac_dh_data *h = sdev->handler_data;
650 int ret = BLKPREP_OK;
652 if (h->state != RDAC_STATE_ACTIVE) {
654 req->rq_flags |= RQF_QUIET;
660 static int rdac_check_sense(struct scsi_device *sdev,
661 struct scsi_sense_hdr *sense_hdr)
663 struct rdac_dh_data *h = sdev->handler_data;
665 RDAC_LOG(RDAC_LOG_SENSE, sdev, "array %s, ctlr %d, "
666 "I/O returned with sense %02x/%02x/%02x",
667 (char *) h->ctlr->array_name, h->ctlr->index,
668 sense_hdr->sense_key, sense_hdr->asc, sense_hdr->ascq);
670 switch (sense_hdr->sense_key) {
672 if (sense_hdr->asc == 0x04 && sense_hdr->ascq == 0x01)
673 /* LUN Not Ready - Logical Unit Not Ready and is in
674 * the process of becoming ready
677 return ADD_TO_MLQUEUE;
678 if (sense_hdr->asc == 0x04 && sense_hdr->ascq == 0x81)
679 /* LUN Not Ready - Storage firmware incompatible
680 * Manual code synchonisation required.
682 * Nothing we can do here. Try to bypass the path.
685 if (sense_hdr->asc == 0x04 && sense_hdr->ascq == 0xA1)
686 /* LUN Not Ready - Quiescense in progress
688 * Just retry and wait.
690 return ADD_TO_MLQUEUE;
691 if (sense_hdr->asc == 0xA1 && sense_hdr->ascq == 0x02)
692 /* LUN Not Ready - Quiescense in progress
693 * or has been achieved
696 return ADD_TO_MLQUEUE;
698 case ILLEGAL_REQUEST:
699 if (sense_hdr->asc == 0x94 && sense_hdr->ascq == 0x01) {
700 /* Invalid Request - Current Logical Unit Ownership.
701 * Controller is not the current owner of the LUN,
702 * Fail the path, so that the other path be used.
704 h->state = RDAC_STATE_PASSIVE;
709 if (sense_hdr->asc == 0x29 && sense_hdr->ascq == 0x00)
711 * Power On, Reset, or Bus Device Reset, just retry.
713 return ADD_TO_MLQUEUE;
714 if (sense_hdr->asc == 0x8b && sense_hdr->ascq == 0x02)
716 * Quiescence in progress , just retry.
718 return ADD_TO_MLQUEUE;
721 /* success just means we do not care what scsi-ml does */
722 return SCSI_RETURN_NOT_HANDLED;
725 static int rdac_bus_attach(struct scsi_device *sdev)
727 struct rdac_dh_data *h;
729 char array_name[ARRAY_LABEL_LEN];
730 char array_id[UNIQUE_ID_LEN];
732 h = kzalloc(sizeof(*h) , GFP_KERNEL);
735 h->lun = UNINITIALIZED_LUN;
736 h->state = RDAC_STATE_ACTIVE;
738 err = get_lun_info(sdev, h, array_name, array_id);
739 if (err != SCSI_DH_OK)
742 err = initialize_controller(sdev, h, array_name, array_id);
743 if (err != SCSI_DH_OK)
746 err = check_ownership(sdev, h);
747 if (err != SCSI_DH_OK)
750 err = set_mode_select(sdev, h);
751 if (err != SCSI_DH_OK)
754 sdev_printk(KERN_NOTICE, sdev,
755 "%s: LUN %d (%s) (%s)\n",
756 RDAC_NAME, h->lun, mode[(int)h->mode],
757 lun_state[(int)h->lun_state]);
759 sdev->handler_data = h;
763 spin_lock(&list_lock);
764 kref_put(&h->ctlr->kref, release_controller);
765 spin_unlock(&list_lock);
772 static void rdac_bus_detach( struct scsi_device *sdev )
774 struct rdac_dh_data *h = sdev->handler_data;
776 if (h->ctlr && h->ctlr->ms_queued)
777 flush_workqueue(kmpath_rdacd);
779 spin_lock(&list_lock);
781 list_del_rcu(&h->node);
783 kref_put(&h->ctlr->kref, release_controller);
785 spin_unlock(&list_lock);
786 sdev->handler_data = NULL;
790 static struct scsi_device_handler rdac_dh = {
792 .module = THIS_MODULE,
793 .prep_fn = rdac_prep_fn,
794 .check_sense = rdac_check_sense,
795 .attach = rdac_bus_attach,
796 .detach = rdac_bus_detach,
797 .activate = rdac_activate,
800 static int __init rdac_init(void)
804 r = scsi_register_device_handler(&rdac_dh);
806 printk(KERN_ERR "Failed to register scsi device handler.");
811 * Create workqueue to handle mode selects for rdac
813 kmpath_rdacd = create_singlethread_workqueue("kmpath_rdacd");
815 scsi_unregister_device_handler(&rdac_dh);
816 printk(KERN_ERR "kmpath_rdacd creation failed.\n");
824 static void __exit rdac_exit(void)
826 destroy_workqueue(kmpath_rdacd);
827 scsi_unregister_device_handler(&rdac_dh);
830 module_init(rdac_init);
831 module_exit(rdac_exit);
833 MODULE_DESCRIPTION("Multipath LSI/Engenio/NetApp E-Series RDAC driver");
834 MODULE_AUTHOR("Mike Christie, Chandra Seetharaman");
835 MODULE_VERSION("01.00.0000.0000");
836 MODULE_LICENSE("GPL");