drivers/xen/xen-pciback/pci_stub.c

   1 /*
   2  * PCI Stub Driver - Grabs devices in backend to be exported later
   3  *
   4  * Ryan Wilson <hap9@epoch.ncsc.mil>
   5  * Chris Bookholt <hap10@epoch.ncsc.mil>
   6  */
   7 #include <linux/module.h>
   8 #include <linux/init.h>
   9 #include <linux/rwsem.h>
  10 #include <linux/list.h>
  11 #include <linux/spinlock.h>
  12 #include <linux/kref.h>
  13 #include <linux/pci.h>
  14 #include <linux/wait.h>
  15 #include <linux/sched.h>
  16 #include <linux/atomic.h>
  17 #include <xen/events.h>
  18 #include <asm/xen/pci.h>
  19 #include <asm/xen/hypervisor.h>
  20 #include "pciback.h"
  21 #include "conf_space.h"
  22 #include "conf_space_quirks.h"
  23
  24 static char *pci_devs_to_hide;
  25 wait_queue_head_t xen_pcibk_aer_wait_queue;
  26 /*Add sem for sync AER handling and xen_pcibk remove/reconfigue ops,
  27 * We want to avoid in middle of AER ops, xen_pcibk devices is being removed
  28 */
  29 static DECLARE_RWSEM(pcistub_sem);
  30 module_param_named(hide, pci_devs_to_hide, charp, 0444);
  31
  32 struct pcistub_device_id {
  33         struct list_head slot_list;
  34         int domain;
  35         unsigned char bus;
  36         unsigned int devfn;
  37 };
  38 static LIST_HEAD(pcistub_device_ids);
  39 static DEFINE_SPINLOCK(device_ids_lock);
  40
  41 struct pcistub_device {
  42         struct kref kref;
  43         struct list_head dev_list;
  44         spinlock_t lock;
  45
  46         struct pci_dev *dev;
  47         struct xen_pcibk_device *pdev;/* non-NULL if struct pci_dev is in use */
  48 };
  49
  50 /* Access to pcistub_devices & seized_devices lists and the initialize_devices
  51  * flag must be locked with pcistub_devices_lock
  52  */
  53 static DEFINE_SPINLOCK(pcistub_devices_lock);
  54 static LIST_HEAD(pcistub_devices);
  55
  56 /* wait for device_initcall before initializing our devices
  57  * (see pcistub_init_devices_late)
  58  */
  59 static int initialize_devices;
  60 static LIST_HEAD(seized_devices);
  61
  62 static struct pcistub_device *pcistub_device_alloc(struct pci_dev *dev)
  63 {
  64         struct pcistub_device *psdev;
  65
  66         dev_dbg(&dev->dev, "pcistub_device_alloc\n");
  67
  68         psdev = kzalloc(sizeof(*psdev), GFP_ATOMIC);
  69         if (!psdev)
  70                 return NULL;
  71
  72         psdev->dev = pci_dev_get(dev);
  73         if (!psdev->dev) {
  74                 kfree(psdev);
  75                 return NULL;
  76         }
  77
  78         kref_init(&psdev->kref);
  79         spin_lock_init(&psdev->lock);
  80
  81         return psdev;
  82 }
  83
  84 /* Don't call this directly as it's called by pcistub_device_put */
  85 static void pcistub_device_release(struct kref *kref)
  86 {
  87         struct pcistub_device *psdev;
  88
  89         psdev = container_of(kref, struct pcistub_device, kref);
  90
  91         dev_dbg(&psdev->dev->dev, "pcistub_device_release\n");
  92
  93         xen_unregister_device_domain_owner(psdev->dev);
  94
  95         /* Clean-up the device */
  96         xen_pcibk_reset_device(psdev->dev);
  97         xen_pcibk_config_free_dyn_fields(psdev->dev);
  98         xen_pcibk_config_free_dev(psdev->dev);
  99         kfree(pci_get_drvdata(psdev->dev));
 100         pci_set_drvdata(psdev->dev, NULL);
 101
 102         pci_dev_put(psdev->dev);
 103
 104         kfree(psdev);
 105 }
 106
 107 static inline void pcistub_device_get(struct pcistub_device *psdev)
 108 {
 109         kref_get(&psdev->kref);
 110 }
 111
 112 static inline void pcistub_device_put(struct pcistub_device *psdev)
 113 {
 114         kref_put(&psdev->kref, pcistub_device_release);
 115 }
 116
 117 static struct pcistub_device *pcistub_device_find(int domain, int bus,
 118                                                   int slot, int func)
 119 {
 120         struct pcistub_device *psdev = NULL;
 121         unsigned long flags;
 122
 123         spin_lock_irqsave(&pcistub_devices_lock, flags);
 124
 125         list_for_each_entry(psdev, &pcistub_devices, dev_list) {
 126                 if (psdev->dev != NULL
 127                     && domain == pci_domain_nr(psdev->dev->bus)
 128                     && bus == psdev->dev->bus->number
 129                     && PCI_DEVFN(slot, func) == psdev->dev->devfn) {
 130                         pcistub_device_get(psdev);
 131                         goto out;
 132                 }
 133         }
 134
 135         /* didn't find it */
 136         psdev = NULL;
 137
 138 out:
 139         spin_unlock_irqrestore(&pcistub_devices_lock, flags);
 140         return psdev;
 141 }
 142
 143 static struct pci_dev *pcistub_device_get_pci_dev(struct xen_pcibk_device *pdev,
 144                                                   struct pcistub_device *psdev)
 145 {
 146         struct pci_dev *pci_dev = NULL;
 147         unsigned long flags;
 148
 149         pcistub_device_get(psdev);
 150
 151         spin_lock_irqsave(&psdev->lock, flags);
 152         if (!psdev->pdev) {
 153                 psdev->pdev = pdev;
 154                 pci_dev = psdev->dev;
 155         }
 156         spin_unlock_irqrestore(&psdev->lock, flags);
 157
 158         if (!pci_dev)
 159                 pcistub_device_put(psdev);
 160
 161         return pci_dev;
 162 }
 163
 164 struct pci_dev *pcistub_get_pci_dev_by_slot(struct xen_pcibk_device *pdev,
 165                                             int domain, int bus,
 166                                             int slot, int func)
 167 {
 168         struct pcistub_device *psdev;
 169         struct pci_dev *found_dev = NULL;
 170         unsigned long flags;
 171
 172         spin_lock_irqsave(&pcistub_devices_lock, flags);
 173
 174         list_for_each_entry(psdev, &pcistub_devices, dev_list) {
 175                 if (psdev->dev != NULL
 176                     && domain == pci_domain_nr(psdev->dev->bus)
 177                     && bus == psdev->dev->bus->number
 178                     && PCI_DEVFN(slot, func) == psdev->dev->devfn) {
 179                         found_dev = pcistub_device_get_pci_dev(pdev, psdev);
 180                         break;
 181                 }
 182         }
 183
 184         spin_unlock_irqrestore(&pcistub_devices_lock, flags);
 185         return found_dev;
 186 }
 187
 188 struct pci_dev *pcistub_get_pci_dev(struct xen_pcibk_device *pdev,
 189                                     struct pci_dev *dev)
 190 {
 191         struct pcistub_device *psdev;
 192         struct pci_dev *found_dev = NULL;
 193         unsigned long flags;
 194
 195         spin_lock_irqsave(&pcistub_devices_lock, flags);
 196
 197         list_for_each_entry(psdev, &pcistub_devices, dev_list) {
 198                 if (psdev->dev == dev) {
 199                         found_dev = pcistub_device_get_pci_dev(pdev, psdev);
 200                         break;
 201                 }
 202         }
 203
 204         spin_unlock_irqrestore(&pcistub_devices_lock, flags);
 205         return found_dev;
 206 }
 207
 208 void pcistub_put_pci_dev(struct pci_dev *dev)
 209 {
 210         struct pcistub_device *psdev, *found_psdev = NULL;
 211         unsigned long flags;
 212
 213         spin_lock_irqsave(&pcistub_devices_lock, flags);
 214
 215         list_for_each_entry(psdev, &pcistub_devices, dev_list) {
 216                 if (psdev->dev == dev) {
 217                         found_psdev = psdev;
 218                         break;
 219                 }
 220         }
 221
 222         spin_unlock_irqrestore(&pcistub_devices_lock, flags);
 223
 224         /*hold this lock for avoiding breaking link between
 225         * pcistub and xen_pcibk when AER is in processing
 226         */
 227         down_write(&pcistub_sem);
 228         /* Cleanup our device
 229          * (so it's ready for the next domain)
 230          */
 231         xen_pcibk_reset_device(found_psdev->dev);
 232         xen_pcibk_config_free_dyn_fields(found_psdev->dev);
 233         xen_pcibk_config_reset_dev(found_psdev->dev);
 234
 235         spin_lock_irqsave(&found_psdev->lock, flags);
 236         found_psdev->pdev = NULL;
 237         spin_unlock_irqrestore(&found_psdev->lock, flags);
 238
 239         pcistub_device_put(found_psdev);
 240         up_write(&pcistub_sem);
 241 }
 242
 243 static int __devinit pcistub_match_one(struct pci_dev *dev,
 244                                        struct pcistub_device_id *pdev_id)
 245 {
 246         /* Match the specified device by domain, bus, slot, func and also if
 247          * any of the device's parent bridges match.
 248          */
 249         for (; dev != NULL; dev = dev->bus->self) {
 250                 if (pci_domain_nr(dev->bus) == pdev_id->domain
 251                     && dev->bus->number == pdev_id->bus
 252                     && dev->devfn == pdev_id->devfn)
 253                         return 1;
 254
 255                 /* Sometimes topmost bridge links to itself. */
 256                 if (dev == dev->bus->self)
 257                         break;
 258         }
 259
 260         return 0;
 261 }
 262
 263 static int __devinit pcistub_match(struct pci_dev *dev)
 264 {
 265         struct pcistub_device_id *pdev_id;
 266         unsigned long flags;
 267         int found = 0;
 268
 269         spin_lock_irqsave(&device_ids_lock, flags);
 270         list_for_each_entry(pdev_id, &pcistub_device_ids, slot_list) {
 271                 if (pcistub_match_one(dev, pdev_id)) {
 272                         found = 1;
 273                         break;
 274                 }
 275         }
 276         spin_unlock_irqrestore(&device_ids_lock, flags);
 277
 278         return found;
 279 }
 280
 281 static int __devinit pcistub_init_device(struct pci_dev *dev)
 282 {
 283         struct xen_pcibk_dev_data *dev_data;
 284         int err = 0;
 285
 286         dev_dbg(&dev->dev, "initializing...\n");
 287
 288         /* The PCI backend is not intended to be a module (or to work with
 289          * removable PCI devices (yet). If it were, xen_pcibk_config_free()
 290          * would need to be called somewhere to free the memory allocated
 291          * here and then to call kfree(pci_get_drvdata(psdev->dev)).
 292          */
 293         dev_data = kzalloc(sizeof(*dev_data) +  strlen(DRV_NAME "[]")
 294                                 + strlen(pci_name(dev)) + 1, GFP_ATOMIC);
 295         if (!dev_data) {
 296                 err = -ENOMEM;
 297                 goto out;
 298         }
 299         pci_set_drvdata(dev, dev_data);
 300
 301         /*
 302          * Setup name for fake IRQ handler. It will only be enabled
 303          * once the device is turned on by the guest.
 304          */
 305         sprintf(dev_data->irq_name, DRV_NAME "[%s]", pci_name(dev));
 306
 307         dev_dbg(&dev->dev, "initializing config\n");
 308
 309         init_waitqueue_head(&xen_pcibk_aer_wait_queue);
 310         err = xen_pcibk_config_init_dev(dev);
 311         if (err)
 312                 goto out;
 313
 314         /* HACK: Force device (& ACPI) to determine what IRQ it's on - we
 315          * must do this here because pcibios_enable_device may specify
 316          * the pci device's true irq (and possibly its other resources)
 317          * if they differ from what's in the configuration space.
 318          * This makes the assumption that the device's resources won't
 319          * change after this point (otherwise this code may break!)
 320          */
 321         dev_dbg(&dev->dev, "enabling device\n");
 322         err = pci_enable_device(dev);
 323         if (err)
 324                 goto config_release;
 325
 326         /* Now disable the device (this also ensures some private device
 327          * data is setup before we export)
 328          */
 329         dev_dbg(&dev->dev, "reset device\n");
 330         xen_pcibk_reset_device(dev);
 331
 332         return 0;
 333
 334 config_release:
 335         xen_pcibk_config_free_dev(dev);
 336
 337 out:
 338         pci_set_drvdata(dev, NULL);
 339         kfree(dev_data);
 340         return err;
 341 }
 342
 343 /*
 344  * Because some initialization still happens on
 345  * devices during fs_initcall, we need to defer
 346  * full initialization of our devices until
 347  * device_initcall.
 348  */
 349 static int __init pcistub_init_devices_late(void)
 350 {
 351         struct pcistub_device *psdev;
 352         unsigned long flags;
 353         int err = 0;
 354
 355         pr_debug(DRV_NAME ": pcistub_init_devices_late\n");
 356
 357         spin_lock_irqsave(&pcistub_devices_lock, flags);
 358
 359         while (!list_empty(&seized_devices)) {
 360                 psdev = container_of(seized_devices.next,
 361                                      struct pcistub_device, dev_list);
 362                 list_del(&psdev->dev_list);
 363
 364                 spin_unlock_irqrestore(&pcistub_devices_lock, flags);
 365
 366                 err = pcistub_init_device(psdev->dev);
 367                 if (err) {
 368                         dev_err(&psdev->dev->dev,
 369                                 "error %d initializing device\n", err);
 370                         kfree(psdev);
 371                         psdev = NULL;
 372                 }
 373
 374                 spin_lock_irqsave(&pcistub_devices_lock, flags);
 375
 376                 if (psdev)
 377                         list_add_tail(&psdev->dev_list, &pcistub_devices);
 378         }
 379
 380         initialize_devices = 1;
 381
 382         spin_unlock_irqrestore(&pcistub_devices_lock, flags);
 383
 384         return 0;
 385 }
 386
 387 static int __devinit pcistub_seize(struct pci_dev *dev)
 388 {
 389         struct pcistub_device *psdev;
 390         unsigned long flags;
 391         int err = 0;
 392
 393         psdev = pcistub_device_alloc(dev);
 394         if (!psdev)
 395                 return -ENOMEM;
 396
 397         spin_lock_irqsave(&pcistub_devices_lock, flags);
 398
 399         if (initialize_devices) {
 400                 spin_unlock_irqrestore(&pcistub_devices_lock, flags);
 401
 402                 /* don't want irqs disabled when calling pcistub_init_device */
 403                 err = pcistub_init_device(psdev->dev);
 404
 405                 spin_lock_irqsave(&pcistub_devices_lock, flags);
 406
 407                 if (!err)
 408                         list_add(&psdev->dev_list, &pcistub_devices);
 409         } else {
 410                 dev_dbg(&dev->dev, "deferring initialization\n");
 411                 list_add(&psdev->dev_list, &seized_devices);
 412         }
 413
 414         spin_unlock_irqrestore(&pcistub_devices_lock, flags);
 415
 416         if (err)
 417                 pcistub_device_put(psdev);
 418
 419         return err;
 420 }
 421
 422 static int __devinit pcistub_probe(struct pci_dev *dev,
 423                                    const struct pci_device_id *id)
 424 {
 425         int err = 0;
 426
 427         dev_dbg(&dev->dev, "probing...\n");
 428
 429         if (pcistub_match(dev)) {
 430
 431                 if (dev->hdr_type != PCI_HEADER_TYPE_NORMAL
 432                     && dev->hdr_type != PCI_HEADER_TYPE_BRIDGE) {
 433                         dev_err(&dev->dev, "can't export pci devices that "
 434                                 "don't have a normal (0) or bridge (1) "
 435                                 "header type!\n");
 436                         err = -ENODEV;
 437                         goto out;
 438                 }
 439
 440                 dev_info(&dev->dev, "seizing device\n");
 441                 err = pcistub_seize(dev);
 442         } else
 443                 /* Didn't find the device */
 444                 err = -ENODEV;
 445
 446 out:
 447         return err;
 448 }
 449
 450 static void pcistub_remove(struct pci_dev *dev)
 451 {
 452         struct pcistub_device *psdev, *found_psdev = NULL;
 453         unsigned long flags;
 454
 455         dev_dbg(&dev->dev, "removing\n");
 456
 457         spin_lock_irqsave(&pcistub_devices_lock, flags);
 458
 459         xen_pcibk_config_quirk_release(dev);
 460
 461         list_for_each_entry(psdev, &pcistub_devices, dev_list) {
 462                 if (psdev->dev == dev) {
 463                         found_psdev = psdev;
 464                         break;
 465                 }
 466         }
 467
 468         spin_unlock_irqrestore(&pcistub_devices_lock, flags);
 469
 470         if (found_psdev) {
 471                 dev_dbg(&dev->dev, "found device to remove - in use? %p\n",
 472                         found_psdev->pdev);
 473
 474                 if (found_psdev->pdev) {
 475                         printk(KERN_WARNING DRV_NAME ": ****** removing device "
 476                                "%s while still in-use! ******\n",
 477                                pci_name(found_psdev->dev));
 478                         printk(KERN_WARNING DRV_NAME ": ****** driver domain may"
 479                                " still access this device's i/o resources!\n");
 480                         printk(KERN_WARNING DRV_NAME ": ****** shutdown driver "
 481                                "domain before binding device\n");
 482                         printk(KERN_WARNING DRV_NAME ": ****** to other drivers "
 483                                "or domains\n");
 484
 485                         xen_pcibk_release_pci_dev(found_psdev->pdev,
 486                                                 found_psdev->dev);
 487                 }
 488
 489                 spin_lock_irqsave(&pcistub_devices_lock, flags);
 490                 list_del(&found_psdev->dev_list);
 491                 spin_unlock_irqrestore(&pcistub_devices_lock, flags);
 492
 493                 /* the final put for releasing from the list */
 494                 pcistub_device_put(found_psdev);
 495         }
 496 }
 497
 498 static DEFINE_PCI_DEVICE_TABLE(pcistub_ids) = {
 499         {
 500          .vendor = PCI_ANY_ID,
 501          .device = PCI_ANY_ID,
 502          .subvendor = PCI_ANY_ID,
 503          .subdevice = PCI_ANY_ID,
 504          },
 505         {0,},
 506 };
 507
 508 #define PCI_NODENAME_MAX 40
 509 static void kill_domain_by_device(struct pcistub_device *psdev)
 510 {
 511         struct xenbus_transaction xbt;
 512         int err;
 513         char nodename[PCI_NODENAME_MAX];
 514
 515         if (!psdev) {
 516                 dev_err(&psdev->dev->dev,
 517                         "device is NULL when do AER recovery/kill_domain\n");
 518                 return;
 519         }
 520
 521         snprintf(nodename, PCI_NODENAME_MAX, "/local/domain/0/backend/pci/%d/0",
 522                 psdev->pdev->xdev->otherend_id);
 523
 524 again:
 525         err = xenbus_transaction_start(&xbt);
 526         if (err) {
 527                 dev_err(&psdev->dev->dev,
 528                         "error %d when start xenbus transaction\n", err);
 529                 return;
 530         }
 531         /*PV AER handlers will set this flag*/
 532         xenbus_printf(xbt, nodename, "aerState" , "aerfail");
 533         err = xenbus_transaction_end(xbt, 0);
 534         if (err) {
 535                 if (err == -EAGAIN)
 536                         goto again;
 537                 dev_err(&psdev->dev->dev,
 538                         "error %d when end xenbus transaction\n", err);
 539                 return;
 540         }
 541 }
 542
 543 /* For each aer recovery step error_detected, mmio_enabled, etc, front_end and
 544  * backend need to have cooperation. In xen_pcibk, those steps will do similar
 545  * jobs: send service request and waiting for front_end response.
 546 */
 547 static pci_ers_result_t common_process(struct pcistub_device *psdev,
 548                                        pci_channel_state_t state, int aer_cmd,
 549                                        pci_ers_result_t result)
 550 {
 551         pci_ers_result_t res = result;
 552         struct xen_pcie_aer_op *aer_op;
 553         int ret;
 554
 555         /*with PV AER drivers*/
 556         aer_op = &(psdev->pdev->sh_info->aer_op);
 557         aer_op->cmd = aer_cmd ;
 558         /*useful for error_detected callback*/
 559         aer_op->err = state;
 560         /*pcifront_end BDF*/
 561         ret = xen_pcibk_get_pcifront_dev(psdev->dev, psdev->pdev,
 562                 &aer_op->domain, &aer_op->bus, &aer_op->devfn);
 563         if (!ret) {
 564                 dev_err(&psdev->dev->dev,
 565                         DRV_NAME ": failed to get pcifront device\n");
 566                 return PCI_ERS_RESULT_NONE;
 567         }
 568         wmb();
 569
 570         dev_dbg(&psdev->dev->dev,
 571                         DRV_NAME ": aer_op %x dom %x bus %x devfn %x\n",
 572                         aer_cmd, aer_op->domain, aer_op->bus, aer_op->devfn);
 573         /*local flag to mark there's aer request, xen_pcibk callback will use
 574         * this flag to judge whether we need to check pci-front give aer
 575         * service ack signal
 576         */
 577         set_bit(_PCIB_op_pending, (unsigned long *)&psdev->pdev->flags);
 578
 579         /*It is possible that a pcifront conf_read_write ops request invokes
 580         * the callback which cause the spurious execution of wake_up.
 581         * Yet it is harmless and better than a spinlock here
 582         */
 583         set_bit(_XEN_PCIB_active,
 584                 (unsigned long *)&psdev->pdev->sh_info->flags);
 585         wmb();
 586         notify_remote_via_irq(psdev->pdev->evtchn_irq);
 587
 588         ret = wait_event_timeout(xen_pcibk_aer_wait_queue,
 589                                  !(test_bit(_XEN_PCIB_active, (unsigned long *)
 590                                  &psdev->pdev->sh_info->flags)), 300*HZ);
 591
 592         if (!ret) {
 593                 if (test_bit(_XEN_PCIB_active,
 594                         (unsigned long *)&psdev->pdev->sh_info->flags)) {
 595                         dev_err(&psdev->dev->dev,
 596                                 "pcifront aer process not responding!\n");
 597                         clear_bit(_XEN_PCIB_active,
 598                           (unsigned long *)&psdev->pdev->sh_info->flags);
 599                         aer_op->err = PCI_ERS_RESULT_NONE;
 600                         return res;
 601                 }
 602         }
 603         clear_bit(_PCIB_op_pending, (unsigned long *)&psdev->pdev->flags);
 604
 605         if (test_bit(_XEN_PCIF_active,
 606                 (unsigned long *)&psdev->pdev->sh_info->flags)) {
 607                 dev_dbg(&psdev->dev->dev,
 608                         "schedule pci_conf service in " DRV_NAME "\n");
 609                 xen_pcibk_test_and_schedule_op(psdev->pdev);
 610         }
 611
 612         res = (pci_ers_result_t)aer_op->err;
 613         return res;
 614 }
 615
 616 /*
 617 * xen_pcibk_slot_reset: it will send the slot_reset request to  pcifront in case
 618 * of the device driver could provide this service, and then wait for pcifront
 619 * ack.
 620 * @dev: pointer to PCI devices
 621 * return value is used by aer_core do_recovery policy
 622 */
 623 static pci_ers_result_t xen_pcibk_slot_reset(struct pci_dev *dev)
 624 {
 625         struct pcistub_device *psdev;
 626         pci_ers_result_t result;
 627
 628         result = PCI_ERS_RESULT_RECOVERED;
 629         dev_dbg(&dev->dev, "xen_pcibk_slot_reset(bus:%x,devfn:%x)\n",
 630                 dev->bus->number, dev->devfn);
 631
 632         down_write(&pcistub_sem);
 633         psdev = pcistub_device_find(pci_domain_nr(dev->bus),
 634                                 dev->bus->number,
 635                                 PCI_SLOT(dev->devfn),
 636                                 PCI_FUNC(dev->devfn));
 637
 638         if (!psdev || !psdev->pdev) {
 639                 dev_err(&dev->dev,
 640                         DRV_NAME " device is not found/assigned\n");
 641                 goto end;
 642         }
 643
 644         if (!psdev->pdev->sh_info) {
 645                 dev_err(&dev->dev, DRV_NAME " device is not connected or owned"
 646                         " by HVM, kill it\n");
 647                 kill_domain_by_device(psdev);
 648                 goto release;
 649         }
 650
 651         if (!test_bit(_XEN_PCIB_AERHANDLER,
 652                 (unsigned long *)&psdev->pdev->sh_info->flags)) {
 653                 dev_err(&dev->dev,
 654                         "guest with no AER driver should have been killed\n");
 655                 goto release;
 656         }
 657         result = common_process(psdev, 1, XEN_PCI_OP_aer_slotreset, result);
 658
 659         if (result == PCI_ERS_RESULT_NONE ||
 660                 result == PCI_ERS_RESULT_DISCONNECT) {
 661                 dev_dbg(&dev->dev,
 662                         "No AER slot_reset service or disconnected!\n");
 663                 kill_domain_by_device(psdev);
 664         }
 665 release:
 666         pcistub_device_put(psdev);
 667 end:
 668         up_write(&pcistub_sem);
 669         return result;
 670
 671 }
 672
 673
 674 /*xen_pcibk_mmio_enabled: it will send the mmio_enabled request to  pcifront
 675 * in case of the device driver could provide this service, and then wait
 676 * for pcifront ack
 677 * @dev: pointer to PCI devices
 678 * return value is used by aer_core do_recovery policy
 679 */
 680
 681 static pci_ers_result_t xen_pcibk_mmio_enabled(struct pci_dev *dev)
 682 {
 683         struct pcistub_device *psdev;
 684         pci_ers_result_t result;
 685
 686         result = PCI_ERS_RESULT_RECOVERED;
 687         dev_dbg(&dev->dev, "xen_pcibk_mmio_enabled(bus:%x,devfn:%x)\n",
 688                 dev->bus->number, dev->devfn);
 689
 690         down_write(&pcistub_sem);
 691         psdev = pcistub_device_find(pci_domain_nr(dev->bus),
 692                                 dev->bus->number,
 693                                 PCI_SLOT(dev->devfn),
 694                                 PCI_FUNC(dev->devfn));
 695
 696         if (!psdev || !psdev->pdev) {
 697                 dev_err(&dev->dev,
 698                         DRV_NAME " device is not found/assigned\n");
 699                 goto end;
 700         }
 701
 702         if (!psdev->pdev->sh_info) {
 703                 dev_err(&dev->dev, DRV_NAME " device is not connected or owned"
 704                         " by HVM, kill it\n");
 705                 kill_domain_by_device(psdev);
 706                 goto release;
 707         }
 708
 709         if (!test_bit(_XEN_PCIB_AERHANDLER,
 710                 (unsigned long *)&psdev->pdev->sh_info->flags)) {
 711                 dev_err(&dev->dev,
 712                         "guest with no AER driver should have been killed\n");
 713                 goto release;
 714         }
 715         result = common_process(psdev, 1, XEN_PCI_OP_aer_mmio, result);
 716
 717         if (result == PCI_ERS_RESULT_NONE ||
 718                 result == PCI_ERS_RESULT_DISCONNECT) {
 719                 dev_dbg(&dev->dev,
 720                         "No AER mmio_enabled service or disconnected!\n");
 721                 kill_domain_by_device(psdev);
 722         }
 723 release:
 724         pcistub_device_put(psdev);
 725 end:
 726         up_write(&pcistub_sem);
 727         return result;
 728 }
 729
 730 /*xen_pcibk_error_detected: it will send the error_detected request to  pcifront
 731 * in case of the device driver could provide this service, and then wait
 732 * for pcifront ack.
 733 * @dev: pointer to PCI devices
 734 * @error: the current PCI connection state
 735 * return value is used by aer_core do_recovery policy
 736 */
 737
 738 static pci_ers_result_t xen_pcibk_error_detected(struct pci_dev *dev,
 739         pci_channel_state_t error)
 740 {
 741         struct pcistub_device *psdev;
 742         pci_ers_result_t result;
 743
 744         result = PCI_ERS_RESULT_CAN_RECOVER;
 745         dev_dbg(&dev->dev, "xen_pcibk_error_detected(bus:%x,devfn:%x)\n",
 746                 dev->bus->number, dev->devfn);
 747
 748         down_write(&pcistub_sem);
 749         psdev = pcistub_device_find(pci_domain_nr(dev->bus),
 750                                 dev->bus->number,
 751                                 PCI_SLOT(dev->devfn),
 752                                 PCI_FUNC(dev->devfn));
 753
 754         if (!psdev || !psdev->pdev) {
 755                 dev_err(&dev->dev,
 756                         DRV_NAME " device is not found/assigned\n");
 757                 goto end;
 758         }
 759
 760         if (!psdev->pdev->sh_info) {
 761                 dev_err(&dev->dev, DRV_NAME " device is not connected or owned"
 762                         " by HVM, kill it\n");
 763                 kill_domain_by_device(psdev);
 764                 goto release;
 765         }
 766
 767         /*Guest owns the device yet no aer handler regiested, kill guest*/
 768         if (!test_bit(_XEN_PCIB_AERHANDLER,
 769                 (unsigned long *)&psdev->pdev->sh_info->flags)) {
 770                 dev_dbg(&dev->dev, "guest may have no aer driver, kill it\n");
 771                 kill_domain_by_device(psdev);
 772                 goto release;
 773         }
 774         result = common_process(psdev, error, XEN_PCI_OP_aer_detected, result);
 775
 776         if (result == PCI_ERS_RESULT_NONE ||
 777                 result == PCI_ERS_RESULT_DISCONNECT) {
 778                 dev_dbg(&dev->dev,
 779                         "No AER error_detected service or disconnected!\n");
 780                 kill_domain_by_device(psdev);
 781         }
 782 release:
 783         pcistub_device_put(psdev);
 784 end:
 785         up_write(&pcistub_sem);
 786         return result;
 787 }
 788
 789 /*xen_pcibk_error_resume: it will send the error_resume request to  pcifront
 790 * in case of the device driver could provide this service, and then wait
 791 * for pcifront ack.
 792 * @dev: pointer to PCI devices
 793 */
 794
 795 static void xen_pcibk_error_resume(struct pci_dev *dev)
 796 {
 797         struct pcistub_device *psdev;
 798
 799         dev_dbg(&dev->dev, "xen_pcibk_error_resume(bus:%x,devfn:%x)\n",
 800                 dev->bus->number, dev->devfn);
 801
 802         down_write(&pcistub_sem);
 803         psdev = pcistub_device_find(pci_domain_nr(dev->bus),
 804                                 dev->bus->number,
 805                                 PCI_SLOT(dev->devfn),
 806                                 PCI_FUNC(dev->devfn));
 807
 808         if (!psdev || !psdev->pdev) {
 809                 dev_err(&dev->dev,
 810                         DRV_NAME " device is not found/assigned\n");
 811                 goto end;
 812         }
 813
 814         if (!psdev->pdev->sh_info) {
 815                 dev_err(&dev->dev, DRV_NAME " device is not connected or owned"
 816                         " by HVM, kill it\n");
 817                 kill_domain_by_device(psdev);
 818                 goto release;
 819         }
 820
 821         if (!test_bit(_XEN_PCIB_AERHANDLER,
 822                 (unsigned long *)&psdev->pdev->sh_info->flags)) {
 823                 dev_err(&dev->dev,
 824                         "guest with no AER driver should have been killed\n");
 825                 kill_domain_by_device(psdev);
 826                 goto release;
 827         }
 828         common_process(psdev, 1, XEN_PCI_OP_aer_resume,
 829                        PCI_ERS_RESULT_RECOVERED);
 830 release:
 831         pcistub_device_put(psdev);
 832 end:
 833         up_write(&pcistub_sem);
 834         return;
 835 }
 836
 837 /*add xen_pcibk AER handling*/
 838 static struct pci_error_handlers xen_pcibk_error_handler = {
 839         .error_detected = xen_pcibk_error_detected,
 840         .mmio_enabled = xen_pcibk_mmio_enabled,
 841         .slot_reset = xen_pcibk_slot_reset,
 842         .resume = xen_pcibk_error_resume,
 843 };
 844
 845 /*
 846  * Note: There is no MODULE_DEVICE_TABLE entry here because this isn't
 847  * for a normal device. I don't want it to be loaded automatically.
 848  */
 849
 850 static struct pci_driver xen_pcibk_pci_driver = {
 851         /* The name should be xen_pciback, but until the tools are updated
 852          * we will keep it as pciback. */
 853         .name = "pciback",
 854         .id_table = pcistub_ids,
 855         .probe = pcistub_probe,
 856         .remove = pcistub_remove,
 857         .err_handler = &xen_pcibk_error_handler,
 858 };
 859
 860 static inline int str_to_slot(const char *buf, int *domain, int *bus,
 861                               int *slot, int *func)
 862 {
 863         int err;
 864
 865         err = sscanf(buf, " %x:%x:%x.%x", domain, bus, slot, func);
 866         if (err == 4)
 867                 return 0;
 868         else if (err < 0)
 869                 return -EINVAL;
 870
 871         /* try again without domain */
 872         *domain = 0;
 873         err = sscanf(buf, " %x:%x.%x", bus, slot, func);
 874         if (err == 3)
 875                 return 0;
 876
 877         return -EINVAL;
 878 }
 879
 880 static inline int str_to_quirk(const char *buf, int *domain, int *bus, int
 881                                *slot, int *func, int *reg, int *size, int *mask)
 882 {
 883         int err;
 884
 885         err =
 886             sscanf(buf, " %04x:%02x:%02x.%1x-%08x:%1x:%08x", domain, bus, slot,
 887                    func, reg, size, mask);
 888         if (err == 7)
 889                 return 0;
 890         return -EINVAL;
 891 }
 892
 893 static int pcistub_device_id_add(int domain, int bus, int slot, int func)
 894 {
 895         struct pcistub_device_id *pci_dev_id;
 896         unsigned long flags;
 897
 898         pci_dev_id = kmalloc(sizeof(*pci_dev_id), GFP_KERNEL);
 899         if (!pci_dev_id)
 900                 return -ENOMEM;
 901
 902         pci_dev_id->domain = domain;
 903         pci_dev_id->bus = bus;
 904         pci_dev_id->devfn = PCI_DEVFN(slot, func);
 905
 906         pr_debug(DRV_NAME ": wants to seize %04x:%02x:%02x.%01x\n",
 907                  domain, bus, slot, func);
 908
 909         spin_lock_irqsave(&device_ids_lock, flags);
 910         list_add_tail(&pci_dev_id->slot_list, &pcistub_device_ids);
 911         spin_unlock_irqrestore(&device_ids_lock, flags);
 912
 913         return 0;
 914 }
 915
 916 static int pcistub_device_id_remove(int domain, int bus, int slot, int func)
 917 {
 918         struct pcistub_device_id *pci_dev_id, *t;
 919         int devfn = PCI_DEVFN(slot, func);
 920         int err = -ENOENT;
 921         unsigned long flags;
 922
 923         spin_lock_irqsave(&device_ids_lock, flags);
 924         list_for_each_entry_safe(pci_dev_id, t, &pcistub_device_ids,
 925                                  slot_list) {
 926                 if (pci_dev_id->domain == domain
 927                     && pci_dev_id->bus == bus && pci_dev_id->devfn == devfn) {
 928                         /* Don't break; here because it's possible the same
 929                          * slot could be in the list more than once
 930                          */
 931                         list_del(&pci_dev_id->slot_list);
 932                         kfree(pci_dev_id);
 933
 934                         err = 0;
 935
 936                         pr_debug(DRV_NAME ": removed %04x:%02x:%02x.%01x from "
 937                                  "seize list\n", domain, bus, slot, func);
 938                 }
 939         }
 940         spin_unlock_irqrestore(&device_ids_lock, flags);
 941
 942         return err;
 943 }
 944
 945 static int pcistub_reg_add(int domain, int bus, int slot, int func, int reg,
 946                            int size, int mask)
 947 {
 948         int err = 0;
 949         struct pcistub_device *psdev;
 950         struct pci_dev *dev;
 951         struct config_field *field;
 952
 953         psdev = pcistub_device_find(domain, bus, slot, func);
 954         if (!psdev || !psdev->dev) {
 955                 err = -ENODEV;
 956                 goto out;
 957         }
 958         dev = psdev->dev;
 959
 960         field = kzalloc(sizeof(*field), GFP_ATOMIC);
 961         if (!field) {
 962                 err = -ENOMEM;
 963                 goto out;
 964         }
 965
 966         field->offset = reg;
 967         field->size = size;
 968         field->mask = mask;
 969         field->init = NULL;
 970         field->reset = NULL;
 971         field->release = NULL;
 972         field->clean = xen_pcibk_config_field_free;
 973
 974         err = xen_pcibk_config_quirks_add_field(dev, field);
 975         if (err)
 976                 kfree(field);
 977 out:
 978         return err;
 979 }
 980
 981 static ssize_t pcistub_slot_add(struct device_driver *drv, const char *buf,
 982                                 size_t count)
 983 {
 984         int domain, bus, slot, func;
 985         int err;
 986
 987         err = str_to_slot(buf, &domain, &bus, &slot, &func);
 988         if (err)
 989                 goto out;
 990
 991         err = pcistub_device_id_add(domain, bus, slot, func);
 992
 993 out:
 994         if (!err)
 995                 err = count;
 996         return err;
 997 }
 998 static DRIVER_ATTR(new_slot, S_IWUSR, NULL, pcistub_slot_add);
 999
1000 static ssize_t pcistub_slot_remove(struct device_driver *drv, const char *buf,
1001                                    size_t count)
1002 {
1003         int domain, bus, slot, func;
1004         int err;
1005
1006         err = str_to_slot(buf, &domain, &bus, &slot, &func);
1007         if (err)
1008                 goto out;
1009
1010         err = pcistub_device_id_remove(domain, bus, slot, func);
1011
1012 out:
1013         if (!err)
1014                 err = count;
1015         return err;
1016 }
1017 static DRIVER_ATTR(remove_slot, S_IWUSR, NULL, pcistub_slot_remove);
1018
1019 static ssize_t pcistub_slot_show(struct device_driver *drv, char *buf)
1020 {
1021         struct pcistub_device_id *pci_dev_id;
1022         size_t count = 0;
1023         unsigned long flags;
1024
1025         spin_lock_irqsave(&device_ids_lock, flags);
1026         list_for_each_entry(pci_dev_id, &pcistub_device_ids, slot_list) {
1027                 if (count >= PAGE_SIZE)
1028                         break;
1029
1030                 count += scnprintf(buf + count, PAGE_SIZE - count,
1031                                    "%04x:%02x:%02x.%01x\n",
1032                                    pci_dev_id->domain, pci_dev_id->bus,
1033                                    PCI_SLOT(pci_dev_id->devfn),
1034                                    PCI_FUNC(pci_dev_id->devfn));
1035         }
1036         spin_unlock_irqrestore(&device_ids_lock, flags);
1037
1038         return count;
1039 }
1040 static DRIVER_ATTR(slots, S_IRUSR, pcistub_slot_show, NULL);
1041
1042 static ssize_t pcistub_irq_handler_show(struct device_driver *drv, char *buf)
1043 {
1044         struct pcistub_device *psdev;
1045         struct xen_pcibk_dev_data *dev_data;
1046         size_t count = 0;
1047         unsigned long flags;
1048
1049         spin_lock_irqsave(&pcistub_devices_lock, flags);
1050         list_for_each_entry(psdev, &pcistub_devices, dev_list) {
1051                 if (count >= PAGE_SIZE)
1052                         break;
1053                 if (!psdev->dev)
1054                         continue;
1055                 dev_data = pci_get_drvdata(psdev->dev);
1056                 if (!dev_data)
1057                         continue;
1058                 count +=
1059                     scnprintf(buf + count, PAGE_SIZE - count,
1060                               "%s:%s:%sing:%ld\n",
1061                               pci_name(psdev->dev),
1062                               dev_data->isr_on ? "on" : "off",
1063                               dev_data->ack_intr ? "ack" : "not ack",
1064                               dev_data->handled);
1065         }
1066         spin_unlock_irqrestore(&pcistub_devices_lock, flags);
1067         return count;
1068 }
1069 static DRIVER_ATTR(irq_handlers, S_IRUSR, pcistub_irq_handler_show, NULL);
1070
1071 static ssize_t pcistub_irq_handler_switch(struct device_driver *drv,
1072                                           const char *buf,
1073                                           size_t count)
1074 {
1075         struct pcistub_device *psdev;
1076         struct xen_pcibk_dev_data *dev_data;
1077         int domain, bus, slot, func;
1078         int err = -ENOENT;
1079
1080         err = str_to_slot(buf, &domain, &bus, &slot, &func);
1081         if (err)
1082                 goto out;
1083
1084         psdev = pcistub_device_find(domain, bus, slot, func);
1085
1086         if (!psdev)
1087                 goto out;
1088
1089         dev_data = pci_get_drvdata(psdev->dev);
1090         if (!dev_data)
1091                 goto out;
1092
1093         dev_dbg(&psdev->dev->dev, "%s fake irq handler: %d->%d\n",
1094                 dev_data->irq_name, dev_data->isr_on,
1095                 !dev_data->isr_on);
1096
1097         dev_data->isr_on = !(dev_data->isr_on);
1098         if (dev_data->isr_on)
1099                 dev_data->ack_intr = 1;
1100 out:
1101         if (!err)
1102                 err = count;
1103         return err;
1104 }
1105 static DRIVER_ATTR(irq_handler_state, S_IWUSR, NULL,
1106                    pcistub_irq_handler_switch);
1107
1108 static ssize_t pcistub_quirk_add(struct device_driver *drv, const char *buf,
1109                                  size_t count)
1110 {
1111         int domain, bus, slot, func, reg, size, mask;
1112         int err;
1113
1114         err = str_to_quirk(buf, &domain, &bus, &slot, &func, &reg, &size,
1115                            &mask);
1116         if (err)
1117                 goto out;
1118
1119         err = pcistub_reg_add(domain, bus, slot, func, reg, size, mask);
1120
1121 out:
1122         if (!err)
1123                 err = count;
1124         return err;
1125 }
1126
1127 static ssize_t pcistub_quirk_show(struct device_driver *drv, char *buf)
1128 {
1129         int count = 0;
1130         unsigned long flags;
1131         struct xen_pcibk_config_quirk *quirk;
1132         struct xen_pcibk_dev_data *dev_data;
1133         const struct config_field *field;
1134         const struct config_field_entry *cfg_entry;
1135
1136         spin_lock_irqsave(&device_ids_lock, flags);
1137         list_for_each_entry(quirk, &xen_pcibk_quirks, quirks_list) {
1138                 if (count >= PAGE_SIZE)
1139                         goto out;
1140
1141                 count += scnprintf(buf + count, PAGE_SIZE - count,
1142                                    "%02x:%02x.%01x\n\t%04x:%04x:%04x:%04x\n",
1143                                    quirk->pdev->bus->number,
1144                                    PCI_SLOT(quirk->pdev->devfn),
1145                                    PCI_FUNC(quirk->pdev->devfn),
1146                                    quirk->devid.vendor, quirk->devid.device,
1147                                    quirk->devid.subvendor,
1148                                    quirk->devid.subdevice);
1149
1150                 dev_data = pci_get_drvdata(quirk->pdev);
1151
1152                 list_for_each_entry(cfg_entry, &dev_data->config_fields, list) {
1153                         field = cfg_entry->field;
1154                         if (count >= PAGE_SIZE)
1155                                 goto out;
1156
1157                         count += scnprintf(buf + count, PAGE_SIZE - count,
1158                                            "\t\t%08x:%01x:%08x\n",
1159                                            cfg_entry->base_offset +
1160                                            field->offset, field->size,
1161                                            field->mask);
1162                 }
1163         }
1164
1165 out:
1166         spin_unlock_irqrestore(&device_ids_lock, flags);
1167
1168         return count;
1169 }
1170 static DRIVER_ATTR(quirks, S_IRUSR | S_IWUSR, pcistub_quirk_show,
1171                    pcistub_quirk_add);
1172
1173 static ssize_t permissive_add(struct device_driver *drv, const char *buf,
1174                               size_t count)
1175 {
1176         int domain, bus, slot, func;
1177         int err;
1178         struct pcistub_device *psdev;
1179         struct xen_pcibk_dev_data *dev_data;
1180         err = str_to_slot(buf, &domain, &bus, &slot, &func);
1181         if (err)
1182                 goto out;
1183         psdev = pcistub_device_find(domain, bus, slot, func);
1184         if (!psdev) {
1185                 err = -ENODEV;
1186                 goto out;
1187         }
1188         if (!psdev->dev) {
1189                 err = -ENODEV;
1190                 goto release;
1191         }
1192         dev_data = pci_get_drvdata(psdev->dev);
1193         /* the driver data for a device should never be null at this point */
1194         if (!dev_data) {
1195                 err = -ENXIO;
1196                 goto release;
1197         }
1198         if (!dev_data->permissive) {
1199                 dev_data->permissive = 1;
1200                 /* Let user know that what they're doing could be unsafe */
1201                 dev_warn(&psdev->dev->dev, "enabling permissive mode "
1202                          "configuration space accesses!\n");
1203                 dev_warn(&psdev->dev->dev,
1204                          "permissive mode is potentially unsafe!\n");
1205         }
1206 release:
1207         pcistub_device_put(psdev);
1208 out:
1209         if (!err)
1210                 err = count;
1211         return err;
1212 }
1213
1214 static ssize_t permissive_show(struct device_driver *drv, char *buf)
1215 {
1216         struct pcistub_device *psdev;
1217         struct xen_pcibk_dev_data *dev_data;
1218         size_t count = 0;
1219         unsigned long flags;
1220         spin_lock_irqsave(&pcistub_devices_lock, flags);
1221         list_for_each_entry(psdev, &pcistub_devices, dev_list) {
1222                 if (count >= PAGE_SIZE)
1223                         break;
1224                 if (!psdev->dev)
1225                         continue;
1226                 dev_data = pci_get_drvdata(psdev->dev);
1227                 if (!dev_data || !dev_data->permissive)
1228                         continue;
1229                 count +=
1230                     scnprintf(buf + count, PAGE_SIZE - count, "%s\n",
1231                               pci_name(psdev->dev));
1232         }
1233         spin_unlock_irqrestore(&pcistub_devices_lock, flags);
1234         return count;
1235 }
1236 static DRIVER_ATTR(permissive, S_IRUSR | S_IWUSR, permissive_show,
1237                    permissive_add);
1238
1239 static void pcistub_exit(void)
1240 {
1241         driver_remove_file(&xen_pcibk_pci_driver.driver, &driver_attr_new_slot);
1242         driver_remove_file(&xen_pcibk_pci_driver.driver,
1243                            &driver_attr_remove_slot);
1244         driver_remove_file(&xen_pcibk_pci_driver.driver, &driver_attr_slots);
1245         driver_remove_file(&xen_pcibk_pci_driver.driver, &driver_attr_quirks);
1246         driver_remove_file(&xen_pcibk_pci_driver.driver,
1247                            &driver_attr_permissive);
1248         driver_remove_file(&xen_pcibk_pci_driver.driver,
1249                            &driver_attr_irq_handlers);
1250         driver_remove_file(&xen_pcibk_pci_driver.driver,
1251                            &driver_attr_irq_handler_state);
1252         pci_unregister_driver(&xen_pcibk_pci_driver);
1253 }
1254
1255 static int __init pcistub_init(void)
1256 {
1257         int pos = 0;
1258         int err = 0;
1259         int domain, bus, slot, func;
1260         int parsed;
1261
1262         if (pci_devs_to_hide && *pci_devs_to_hide) {
1263                 do {
1264                         parsed = 0;
1265
1266                         err = sscanf(pci_devs_to_hide + pos,
1267                                      " (%x:%x:%x.%x) %n",
1268                                      &domain, &bus, &slot, &func, &parsed);
1269                         if (err != 4) {
1270                                 domain = 0;
1271                                 err = sscanf(pci_devs_to_hide + pos,
1272                                              " (%x:%x.%x) %n",
1273                                              &bus, &slot, &func, &parsed);
1274                                 if (err != 3)
1275                                         goto parse_error;
1276                         }
1277
1278                         err = pcistub_device_id_add(domain, bus, slot, func);
1279                         if (err)
1280                                 goto out;
1281
1282                         /* if parsed<=0, we've reached the end of the string */
1283                         pos += parsed;
1284                 } while (parsed > 0 && pci_devs_to_hide[pos]);
1285         }
1286
1287         /* If we're the first PCI Device Driver to register, we're the
1288          * first one to get offered PCI devices as they become
1289          * available (and thus we can be the first to grab them)
1290          */
1291         err = pci_register_driver(&xen_pcibk_pci_driver);
1292         if (err < 0)
1293                 goto out;
1294
1295         err = driver_create_file(&xen_pcibk_pci_driver.driver,
1296                                  &driver_attr_new_slot);
1297         if (!err)
1298                 err = driver_create_file(&xen_pcibk_pci_driver.driver,
1299                                          &driver_attr_remove_slot);
1300         if (!err)
1301                 err = driver_create_file(&xen_pcibk_pci_driver.driver,
1302                                          &driver_attr_slots);
1303         if (!err)
1304                 err = driver_create_file(&xen_pcibk_pci_driver.driver,
1305                                          &driver_attr_quirks);
1306         if (!err)
1307                 err = driver_create_file(&xen_pcibk_pci_driver.driver,
1308                                          &driver_attr_permissive);
1309
1310         if (!err)
1311                 err = driver_create_file(&xen_pcibk_pci_driver.driver,
1312                                          &driver_attr_irq_handlers);
1313         if (!err)
1314                 err = driver_create_file(&xen_pcibk_pci_driver.driver,
1315                                         &driver_attr_irq_handler_state);
1316         if (err)
1317                 pcistub_exit();
1318
1319 out:
1320         return err;
1321
1322 parse_error:
1323         printk(KERN_ERR DRV_NAME ": Error parsing pci_devs_to_hide at \"%s\"\n",
1324                pci_devs_to_hide + pos);
1325         return -EINVAL;
1326 }
1327
1328 #ifndef MODULE
1329 /*
1330  * fs_initcall happens before device_initcall
1331  * so xen_pcibk *should* get called first (b/c we
1332  * want to suck up any device before other drivers
1333  * get a chance by being the first pci device
1334  * driver to register)
1335  */
1336 fs_initcall(pcistub_init);
1337 #endif
1338
1339 static int __init xen_pcibk_init(void)
1340 {
1341         int err;
1342
1343         if (!xen_initial_domain())
1344                 return -ENODEV;
1345
1346         err = xen_pcibk_config_init();
1347         if (err)
1348                 return err;
1349
1350 #ifdef MODULE
1351         err = pcistub_init();
1352         if (err < 0)
1353                 return err;
1354 #endif
1355
1356         pcistub_init_devices_late();
1357         err = xen_pcibk_xenbus_register();
1358         if (err)
1359                 pcistub_exit();
1360
1361         return err;
1362 }
1363
1364 static void __exit xen_pcibk_cleanup(void)
1365 {
1366         xen_pcibk_xenbus_unregister();
1367         pcistub_exit();
1368 }
1369
1370 module_init(xen_pcibk_init);
1371 module_exit(xen_pcibk_cleanup);
1372
1373 MODULE_LICENSE("Dual BSD/GPL");
1374 MODULE_ALIAS("xen-backend:pci");