drivers/iommu/arm-smmu.c

   1 /*
   2  * IOMMU API for ARM architected SMMU implementations.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  16  *
  17  * Copyright (C) 2013 ARM Limited
  18  *
  19  * Author: Will Deacon <will.deacon@arm.com>
  20  *
  21  * This driver currently supports:
  22  *      - SMMUv1 and v2 implementations
  23  *      - Stream-matching and stream-indexing
  24  *      - v7/v8 long-descriptor format
  25  *      - Non-secure access to the SMMU
  26  *      - Context fault reporting
  27  */
  28
  29 #define pr_fmt(fmt) "arm-smmu: " fmt
  30
  31 #include <linux/delay.h>
  32 #include <linux/dma-iommu.h>
  33 #include <linux/dma-mapping.h>
  34 #include <linux/err.h>
  35 #include <linux/interrupt.h>
  36 #include <linux/io.h>
  37 #include <linux/iommu.h>
  38 #include <linux/iopoll.h>
  39 #include <linux/module.h>
  40 #include <linux/of.h>
  41 #include <linux/of_address.h>
  42 #include <linux/pci.h>
  43 #include <linux/platform_device.h>
  44 #include <linux/slab.h>
  45 #include <linux/spinlock.h>
  46
  47 #include <linux/amba/bus.h>
  48
  49 #include "io-pgtable.h"
  50
  51 /* Maximum number of stream IDs assigned to a single device */
  52 #define MAX_MASTER_STREAMIDS            MAX_PHANDLE_ARGS
  53
  54 /* Maximum number of context banks per SMMU */
  55 #define ARM_SMMU_MAX_CBS                128
  56
  57 /* Maximum number of mapping groups per SMMU */
  58 #define ARM_SMMU_MAX_SMRS               128
  59
  60 /* SMMU global address space */
  61 #define ARM_SMMU_GR0(smmu)              ((smmu)->base)
  62 #define ARM_SMMU_GR1(smmu)              ((smmu)->base + (1 << (smmu)->pgshift))
  63
  64 /*
  65  * SMMU global address space with conditional offset to access secure
  66  * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
  67  * nsGFSYNR0: 0x450)
  68  */
  69 #define ARM_SMMU_GR0_NS(smmu)                                           \
  70         ((smmu)->base +                                                 \
  71                 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS)       \
  72                         ? 0x400 : 0))
  73
  74 #ifdef CONFIG_64BIT
  75 #define smmu_writeq     writeq_relaxed
  76 #else
  77 #define smmu_writeq(reg64, addr)                                \
  78         do {                                                    \
  79                 u64 __val = (reg64);                            \
  80                 void __iomem *__addr = (addr);                  \
  81                 writel_relaxed(__val >> 32, __addr + 4);        \
  82                 writel_relaxed(__val, __addr);                  \
  83         } while (0)
  84 #endif
  85
  86 /* Configuration registers */
  87 #define ARM_SMMU_GR0_sCR0               0x0
  88 #define sCR0_CLIENTPD                   (1 << 0)
  89 #define sCR0_GFRE                       (1 << 1)
  90 #define sCR0_GFIE                       (1 << 2)
  91 #define sCR0_GCFGFRE                    (1 << 4)
  92 #define sCR0_GCFGFIE                    (1 << 5)
  93 #define sCR0_USFCFG                     (1 << 10)
  94 #define sCR0_VMIDPNE                    (1 << 11)
  95 #define sCR0_PTM                        (1 << 12)
  96 #define sCR0_FB                         (1 << 13)
  97 #define sCR0_VMID16EN                   (1 << 31)
  98 #define sCR0_BSU_SHIFT                  14
  99 #define sCR0_BSU_MASK                   0x3
 100
 101 /* Identification registers */
 102 #define ARM_SMMU_GR0_ID0                0x20
 103 #define ARM_SMMU_GR0_ID1                0x24
 104 #define ARM_SMMU_GR0_ID2                0x28
 105 #define ARM_SMMU_GR0_ID3                0x2c
 106 #define ARM_SMMU_GR0_ID4                0x30
 107 #define ARM_SMMU_GR0_ID5                0x34
 108 #define ARM_SMMU_GR0_ID6                0x38
 109 #define ARM_SMMU_GR0_ID7                0x3c
 110 #define ARM_SMMU_GR0_sGFSR              0x48
 111 #define ARM_SMMU_GR0_sGFSYNR0           0x50
 112 #define ARM_SMMU_GR0_sGFSYNR1           0x54
 113 #define ARM_SMMU_GR0_sGFSYNR2           0x58
 114
 115 #define ID0_S1TS                        (1 << 30)
 116 #define ID0_S2TS                        (1 << 29)
 117 #define ID0_NTS                         (1 << 28)
 118 #define ID0_SMS                         (1 << 27)
 119 #define ID0_ATOSNS                      (1 << 26)
 120 #define ID0_CTTW                        (1 << 14)
 121 #define ID0_NUMIRPT_SHIFT               16
 122 #define ID0_NUMIRPT_MASK                0xff
 123 #define ID0_NUMSIDB_SHIFT               9
 124 #define ID0_NUMSIDB_MASK                0xf
 125 #define ID0_NUMSMRG_SHIFT               0
 126 #define ID0_NUMSMRG_MASK                0xff
 127
 128 #define ID1_PAGESIZE                    (1 << 31)
 129 #define ID1_NUMPAGENDXB_SHIFT           28
 130 #define ID1_NUMPAGENDXB_MASK            7
 131 #define ID1_NUMS2CB_SHIFT               16
 132 #define ID1_NUMS2CB_MASK                0xff
 133 #define ID1_NUMCB_SHIFT                 0
 134 #define ID1_NUMCB_MASK                  0xff
 135
 136 #define ID2_OAS_SHIFT                   4
 137 #define ID2_OAS_MASK                    0xf
 138 #define ID2_IAS_SHIFT                   0
 139 #define ID2_IAS_MASK                    0xf
 140 #define ID2_UBS_SHIFT                   8
 141 #define ID2_UBS_MASK                    0xf
 142 #define ID2_PTFS_4K                     (1 << 12)
 143 #define ID2_PTFS_16K                    (1 << 13)
 144 #define ID2_PTFS_64K                    (1 << 14)
 145 #define ID2_VMID16                      (1 << 15)
 146
 147 /* Global TLB invalidation */
 148 #define ARM_SMMU_GR0_TLBIVMID           0x64
 149 #define ARM_SMMU_GR0_TLBIALLNSNH        0x68
 150 #define ARM_SMMU_GR0_TLBIALLH           0x6c
 151 #define ARM_SMMU_GR0_sTLBGSYNC          0x70
 152 #define ARM_SMMU_GR0_sTLBGSTATUS        0x74
 153 #define sTLBGSTATUS_GSACTIVE            (1 << 0)
 154 #define TLB_LOOP_TIMEOUT                1000000 /* 1s! */
 155
 156 /* Stream mapping registers */
 157 #define ARM_SMMU_GR0_SMR(n)             (0x800 + ((n) << 2))
 158 #define SMR_VALID                       (1 << 31)
 159 #define SMR_MASK_SHIFT                  16
 160 #define SMR_MASK_MASK                   0x7fff
 161 #define SMR_ID_SHIFT                    0
 162 #define SMR_ID_MASK                     0x7fff
 163
 164 #define ARM_SMMU_GR0_S2CR(n)            (0xc00 + ((n) << 2))
 165 #define S2CR_CBNDX_SHIFT                0
 166 #define S2CR_CBNDX_MASK                 0xff
 167 #define S2CR_TYPE_SHIFT                 16
 168 #define S2CR_TYPE_MASK                  0x3
 169 #define S2CR_TYPE_TRANS                 (0 << S2CR_TYPE_SHIFT)
 170 #define S2CR_TYPE_BYPASS                (1 << S2CR_TYPE_SHIFT)
 171 #define S2CR_TYPE_FAULT                 (2 << S2CR_TYPE_SHIFT)
 172
 173 #define S2CR_PRIVCFG_SHIFT              24
 174 #define S2CR_PRIVCFG_UNPRIV             (2 << S2CR_PRIVCFG_SHIFT)
 175
 176 /* Context bank attribute registers */
 177 #define ARM_SMMU_GR1_CBAR(n)            (0x0 + ((n) << 2))
 178 #define CBAR_VMID_SHIFT                 0
 179 #define CBAR_VMID_MASK                  0xff
 180 #define CBAR_S1_BPSHCFG_SHIFT           8
 181 #define CBAR_S1_BPSHCFG_MASK            3
 182 #define CBAR_S1_BPSHCFG_NSH             3
 183 #define CBAR_S1_MEMATTR_SHIFT           12
 184 #define CBAR_S1_MEMATTR_MASK            0xf
 185 #define CBAR_S1_MEMATTR_WB              0xf
 186 #define CBAR_TYPE_SHIFT                 16
 187 #define CBAR_TYPE_MASK                  0x3
 188 #define CBAR_TYPE_S2_TRANS              (0 << CBAR_TYPE_SHIFT)
 189 #define CBAR_TYPE_S1_TRANS_S2_BYPASS    (1 << CBAR_TYPE_SHIFT)
 190 #define CBAR_TYPE_S1_TRANS_S2_FAULT     (2 << CBAR_TYPE_SHIFT)
 191 #define CBAR_TYPE_S1_TRANS_S2_TRANS     (3 << CBAR_TYPE_SHIFT)
 192 #define CBAR_IRPTNDX_SHIFT              24
 193 #define CBAR_IRPTNDX_MASK               0xff
 194
 195 #define ARM_SMMU_GR1_CBA2R(n)           (0x800 + ((n) << 2))
 196 #define CBA2R_RW64_32BIT                (0 << 0)
 197 #define CBA2R_RW64_64BIT                (1 << 0)
 198 #define CBA2R_VMID_SHIFT                16
 199 #define CBA2R_VMID_MASK                 0xffff
 200
 201 /* Translation context bank */
 202 #define ARM_SMMU_CB_BASE(smmu)          ((smmu)->base + ((smmu)->size >> 1))
 203 #define ARM_SMMU_CB(smmu, n)            ((n) * (1 << (smmu)->pgshift))
 204
 205 #define ARM_SMMU_CB_SCTLR               0x0
 206 #define ARM_SMMU_CB_ACTLR               0x4
 207 #define ARM_SMMU_CB_RESUME              0x8
 208 #define ARM_SMMU_CB_TTBCR2              0x10
 209 #define ARM_SMMU_CB_TTBR0               0x20
 210 #define ARM_SMMU_CB_TTBR1               0x28
 211 #define ARM_SMMU_CB_TTBCR               0x30
 212 #define ARM_SMMU_CB_S1_MAIR0            0x38
 213 #define ARM_SMMU_CB_S1_MAIR1            0x3c
 214 #define ARM_SMMU_CB_PAR_LO              0x50
 215 #define ARM_SMMU_CB_PAR_HI              0x54
 216 #define ARM_SMMU_CB_FSR                 0x58
 217 #define ARM_SMMU_CB_FAR_LO              0x60
 218 #define ARM_SMMU_CB_FAR_HI              0x64
 219 #define ARM_SMMU_CB_FSYNR0              0x68
 220 #define ARM_SMMU_CB_S1_TLBIVA           0x600
 221 #define ARM_SMMU_CB_S1_TLBIASID         0x610
 222 #define ARM_SMMU_CB_S1_TLBIVAL          0x620
 223 #define ARM_SMMU_CB_S2_TLBIIPAS2        0x630
 224 #define ARM_SMMU_CB_S2_TLBIIPAS2L       0x638
 225 #define ARM_SMMU_CB_ATS1PR              0x800
 226 #define ARM_SMMU_CB_ATSR                0x8f0
 227
 228 #define SCTLR_S1_ASIDPNE                (1 << 12)
 229 #define SCTLR_CFCFG                     (1 << 7)
 230 #define SCTLR_CFIE                      (1 << 6)
 231 #define SCTLR_CFRE                      (1 << 5)
 232 #define SCTLR_E                         (1 << 4)
 233 #define SCTLR_AFE                       (1 << 2)
 234 #define SCTLR_TRE                       (1 << 1)
 235 #define SCTLR_M                         (1 << 0)
 236 #define SCTLR_EAE_SBOP                  (SCTLR_AFE | SCTLR_TRE)
 237
 238 #define ARM_MMU500_ACTLR_CPRE           (1 << 1)
 239
 240 #define CB_PAR_F                        (1 << 0)
 241
 242 #define ATSR_ACTIVE                     (1 << 0)
 243
 244 #define RESUME_RETRY                    (0 << 0)
 245 #define RESUME_TERMINATE                (1 << 0)
 246
 247 #define TTBCR2_SEP_SHIFT                15
 248 #define TTBCR2_SEP_UPSTREAM             (0x7 << TTBCR2_SEP_SHIFT)
 249
 250 #define TTBRn_ASID_SHIFT                48
 251
 252 #define FSR_MULTI                       (1 << 31)
 253 #define FSR_SS                          (1 << 30)
 254 #define FSR_UUT                         (1 << 8)
 255 #define FSR_ASF                         (1 << 7)
 256 #define FSR_TLBLKF                      (1 << 6)
 257 #define FSR_TLBMCF                      (1 << 5)
 258 #define FSR_EF                          (1 << 4)
 259 #define FSR_PF                          (1 << 3)
 260 #define FSR_AFF                         (1 << 2)
 261 #define FSR_TF                          (1 << 1)
 262
 263 #define FSR_IGN                         (FSR_AFF | FSR_ASF | \
 264                                          FSR_TLBMCF | FSR_TLBLKF)
 265 #define FSR_FAULT                       (FSR_MULTI | FSR_SS | FSR_UUT | \
 266                                          FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
 267
 268 #define FSYNR0_WNR                      (1 << 4)
 269
 270 static int force_stage;
 271 module_param(force_stage, int, S_IRUGO);
 272 MODULE_PARM_DESC(force_stage,
 273         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
 274 static bool disable_bypass;
 275 module_param(disable_bypass, bool, S_IRUGO);
 276 MODULE_PARM_DESC(disable_bypass,
 277         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
 278
 279 enum arm_smmu_arch_version {
 280         ARM_SMMU_V1 = 1,
 281         ARM_SMMU_V2,
 282 };
 283
 284 enum arm_smmu_implementation {
 285         GENERIC_SMMU,
 286         ARM_MMU500,
 287         CAVIUM_SMMUV2,
 288 };
 289
 290 struct arm_smmu_smr {
 291         u8                              idx;
 292         u16                             mask;
 293         u16                             id;
 294 };
 295
 296 struct arm_smmu_master_cfg {
 297         int                             num_streamids;
 298         u16                             streamids[MAX_MASTER_STREAMIDS];
 299         struct arm_smmu_smr             *smrs;
 300 };
 301
 302 struct arm_smmu_master {
 303         struct device_node              *of_node;
 304         struct rb_node                  node;
 305         struct arm_smmu_master_cfg      cfg;
 306 };
 307
 308 struct arm_smmu_device {
 309         struct device                   *dev;
 310
 311         void __iomem                    *base;
 312         unsigned long                   size;
 313         unsigned long                   pgshift;
 314
 315 #define ARM_SMMU_FEAT_COHERENT_WALK     (1 << 0)
 316 #define ARM_SMMU_FEAT_STREAM_MATCH      (1 << 1)
 317 #define ARM_SMMU_FEAT_TRANS_S1          (1 << 2)
 318 #define ARM_SMMU_FEAT_TRANS_S2          (1 << 3)
 319 #define ARM_SMMU_FEAT_TRANS_NESTED      (1 << 4)
 320 #define ARM_SMMU_FEAT_TRANS_OPS         (1 << 5)
 321 #define ARM_SMMU_FEAT_VMID16            (1 << 6)
 322         u32                             features;
 323
 324 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
 325         u32                             options;
 326         enum arm_smmu_arch_version      version;
 327         enum arm_smmu_implementation    model;
 328
 329         u32                             num_context_banks;
 330         u32                             num_s2_context_banks;
 331         DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
 332         atomic_t                        irptndx;
 333
 334         u32                             num_mapping_groups;
 335         DECLARE_BITMAP(smr_map, ARM_SMMU_MAX_SMRS);
 336
 337         unsigned long                   va_size;
 338         unsigned long                   ipa_size;
 339         unsigned long                   pa_size;
 340
 341         u32                             num_global_irqs;
 342         u32                             num_context_irqs;
 343         unsigned int                    *irqs;
 344
 345         struct list_head                list;
 346         struct rb_root                  masters;
 347
 348         u32                             cavium_id_base; /* Specific to Cavium */
 349 };
 350
 351 struct arm_smmu_cfg {
 352         u8                              cbndx;
 353         u8                              irptndx;
 354         u32                             cbar;
 355 };
 356 #define INVALID_IRPTNDX                 0xff
 357
 358 #define ARM_SMMU_CB_ASID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx)
 359 #define ARM_SMMU_CB_VMID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx + 1)
 360
 361 enum arm_smmu_domain_stage {
 362         ARM_SMMU_DOMAIN_S1 = 0,
 363         ARM_SMMU_DOMAIN_S2,
 364         ARM_SMMU_DOMAIN_NESTED,
 365 };
 366
 367 struct arm_smmu_domain {
 368         struct arm_smmu_device          *smmu;
 369         struct io_pgtable_ops           *pgtbl_ops;
 370         spinlock_t                      pgtbl_lock;
 371         struct arm_smmu_cfg             cfg;
 372         enum arm_smmu_domain_stage      stage;
 373         struct mutex                    init_mutex; /* Protects smmu pointer */
 374         struct iommu_domain             domain;
 375 };
 376
 377 static struct iommu_ops arm_smmu_ops;
 378
 379 static DEFINE_SPINLOCK(arm_smmu_devices_lock);
 380 static LIST_HEAD(arm_smmu_devices);
 381
 382 struct arm_smmu_option_prop {
 383         u32 opt;
 384         const char *prop;
 385 };
 386
 387 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
 388
 389 static struct arm_smmu_option_prop arm_smmu_options[] = {
 390         { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
 391         { 0, NULL},
 392 };
 393
 394 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
 395 {
 396         return container_of(dom, struct arm_smmu_domain, domain);
 397 }
 398
 399 static void parse_driver_options(struct arm_smmu_device *smmu)
 400 {
 401         int i = 0;
 402
 403         do {
 404                 if (of_property_read_bool(smmu->dev->of_node,
 405                                                 arm_smmu_options[i].prop)) {
 406                         smmu->options |= arm_smmu_options[i].opt;
 407                         dev_notice(smmu->dev, "option %s\n",
 408                                 arm_smmu_options[i].prop);
 409                 }
 410         } while (arm_smmu_options[++i].opt);
 411 }
 412
 413 static struct device_node *dev_get_dev_node(struct device *dev)
 414 {
 415         if (dev_is_pci(dev)) {
 416                 struct pci_bus *bus = to_pci_dev(dev)->bus;
 417
 418                 while (!pci_is_root_bus(bus))
 419                         bus = bus->parent;
 420                 return bus->bridge->parent->of_node;
 421         }
 422
 423         return dev->of_node;
 424 }
 425
 426 static struct arm_smmu_master *find_smmu_master(struct arm_smmu_device *smmu,
 427                                                 struct device_node *dev_node)
 428 {
 429         struct rb_node *node = smmu->masters.rb_node;
 430
 431         while (node) {
 432                 struct arm_smmu_master *master;
 433
 434                 master = container_of(node, struct arm_smmu_master, node);
 435
 436                 if (dev_node < master->of_node)
 437                         node = node->rb_left;
 438                 else if (dev_node > master->of_node)
 439                         node = node->rb_right;
 440                 else
 441                         return master;
 442         }
 443
 444         return NULL;
 445 }
 446
 447 static struct arm_smmu_master_cfg *
 448 find_smmu_master_cfg(struct device *dev)
 449 {
 450         struct arm_smmu_master_cfg *cfg = NULL;
 451         struct iommu_group *group = iommu_group_get(dev);
 452
 453         if (group) {
 454                 cfg = iommu_group_get_iommudata(group);
 455                 iommu_group_put(group);
 456         }
 457
 458         return cfg;
 459 }
 460
 461 static int insert_smmu_master(struct arm_smmu_device *smmu,
 462                               struct arm_smmu_master *master)
 463 {
 464         struct rb_node **new, *parent;
 465
 466         new = &smmu->masters.rb_node;
 467         parent = NULL;
 468         while (*new) {
 469                 struct arm_smmu_master *this
 470                         = container_of(*new, struct arm_smmu_master, node);
 471
 472                 parent = *new;
 473                 if (master->of_node < this->of_node)
 474                         new = &((*new)->rb_left);
 475                 else if (master->of_node > this->of_node)
 476                         new = &((*new)->rb_right);
 477                 else
 478                         return -EEXIST;
 479         }
 480
 481         rb_link_node(&master->node, parent, new);
 482         rb_insert_color(&master->node, &smmu->masters);
 483         return 0;
 484 }
 485
 486 static int register_smmu_master(struct arm_smmu_device *smmu,
 487                                 struct device *dev,
 488                                 struct of_phandle_args *masterspec)
 489 {
 490         int i;
 491         struct arm_smmu_master *master;
 492
 493         master = find_smmu_master(smmu, masterspec->np);
 494         if (master) {
 495                 dev_err(dev,
 496                         "rejecting multiple registrations for master device %s\n",
 497                         masterspec->np->name);
 498                 return -EBUSY;
 499         }
 500
 501         if (masterspec->args_count > MAX_MASTER_STREAMIDS) {
 502                 dev_err(dev,
 503                         "reached maximum number (%d) of stream IDs for master device %s\n",
 504                         MAX_MASTER_STREAMIDS, masterspec->np->name);
 505                 return -ENOSPC;
 506         }
 507
 508         master = devm_kzalloc(dev, sizeof(*master), GFP_KERNEL);
 509         if (!master)
 510                 return -ENOMEM;
 511
 512         master->of_node                 = masterspec->np;
 513         master->cfg.num_streamids       = masterspec->args_count;
 514
 515         for (i = 0; i < master->cfg.num_streamids; ++i) {
 516                 u16 streamid = masterspec->args[i];
 517
 518                 if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) &&
 519                      (streamid >= smmu->num_mapping_groups)) {
 520                         dev_err(dev,
 521                                 "stream ID for master device %s greater than maximum allowed (%d)\n",
 522                                 masterspec->np->name, smmu->num_mapping_groups);
 523                         return -ERANGE;
 524                 }
 525                 master->cfg.streamids[i] = streamid;
 526         }
 527         return insert_smmu_master(smmu, master);
 528 }
 529
 530 static struct arm_smmu_device *find_smmu_for_device(struct device *dev)
 531 {
 532         struct arm_smmu_device *smmu;
 533         struct arm_smmu_master *master = NULL;
 534         struct device_node *dev_node = dev_get_dev_node(dev);
 535
 536         spin_lock(&arm_smmu_devices_lock);
 537         list_for_each_entry(smmu, &arm_smmu_devices, list) {
 538                 master = find_smmu_master(smmu, dev_node);
 539                 if (master)
 540                         break;
 541         }
 542         spin_unlock(&arm_smmu_devices_lock);
 543
 544         return master ? smmu : NULL;
 545 }
 546
 547 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
 548 {
 549         int idx;
 550
 551         do {
 552                 idx = find_next_zero_bit(map, end, start);
 553                 if (idx == end)
 554                         return -ENOSPC;
 555         } while (test_and_set_bit(idx, map));
 556
 557         return idx;
 558 }
 559
 560 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
 561 {
 562         clear_bit(idx, map);
 563 }
 564
 565 /* Wait for any pending TLB invalidations to complete */
 566 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
 567 {
 568         int count = 0;
 569         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
 570
 571         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_sTLBGSYNC);
 572         while (readl_relaxed(gr0_base + ARM_SMMU_GR0_sTLBGSTATUS)
 573                & sTLBGSTATUS_GSACTIVE) {
 574                 cpu_relax();
 575                 if (++count == TLB_LOOP_TIMEOUT) {
 576                         dev_err_ratelimited(smmu->dev,
 577                         "TLB sync timed out -- SMMU may be deadlocked\n");
 578                         return;
 579                 }
 580                 udelay(1);
 581         }
 582 }
 583
 584 static void arm_smmu_tlb_sync(void *cookie)
 585 {
 586         struct arm_smmu_domain *smmu_domain = cookie;
 587         __arm_smmu_tlb_sync(smmu_domain->smmu);
 588 }
 589
 590 static void arm_smmu_tlb_inv_context(void *cookie)
 591 {
 592         struct arm_smmu_domain *smmu_domain = cookie;
 593         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 594         struct arm_smmu_device *smmu = smmu_domain->smmu;
 595         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 596         void __iomem *base;
 597
 598         if (stage1) {
 599                 base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 600                 writel_relaxed(ARM_SMMU_CB_ASID(smmu, cfg),
 601                                base + ARM_SMMU_CB_S1_TLBIASID);
 602         } else {
 603                 base = ARM_SMMU_GR0(smmu);
 604                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg),
 605                                base + ARM_SMMU_GR0_TLBIVMID);
 606         }
 607
 608         __arm_smmu_tlb_sync(smmu);
 609 }
 610
 611 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
 612                                           size_t granule, bool leaf, void *cookie)
 613 {
 614         struct arm_smmu_domain *smmu_domain = cookie;
 615         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 616         struct arm_smmu_device *smmu = smmu_domain->smmu;
 617         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 618         void __iomem *reg;
 619
 620         if (stage1) {
 621                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 622                 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
 623
 624                 if (!IS_ENABLED(CONFIG_64BIT) || smmu->version == ARM_SMMU_V1) {
 625                         iova &= ~12UL;
 626                         iova |= ARM_SMMU_CB_ASID(smmu, cfg);
 627                         do {
 628                                 writel_relaxed(iova, reg);
 629                                 iova += granule;
 630                         } while (size -= granule);
 631 #ifdef CONFIG_64BIT
 632                 } else {
 633                         iova >>= 12;
 634                         iova |= (u64)ARM_SMMU_CB_ASID(smmu, cfg) << 48;
 635                         do {
 636                                 writeq_relaxed(iova, reg);
 637                                 iova += granule >> 12;
 638                         } while (size -= granule);
 639 #endif
 640                 }
 641 #ifdef CONFIG_64BIT
 642         } else if (smmu->version == ARM_SMMU_V2) {
 643                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 644                 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
 645                               ARM_SMMU_CB_S2_TLBIIPAS2;
 646                 iova >>= 12;
 647                 do {
 648                         writeq_relaxed(iova, reg);
 649                         iova += granule >> 12;
 650                 } while (size -= granule);
 651 #endif
 652         } else {
 653                 reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID;
 654                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), reg);
 655         }
 656 }
 657
 658 static struct iommu_gather_ops arm_smmu_gather_ops = {
 659         .tlb_flush_all  = arm_smmu_tlb_inv_context,
 660         .tlb_add_flush  = arm_smmu_tlb_inv_range_nosync,
 661         .tlb_sync       = arm_smmu_tlb_sync,
 662 };
 663
 664 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 665 {
 666         int flags, ret;
 667         u32 fsr, far, fsynr, resume;
 668         unsigned long iova;
 669         struct iommu_domain *domain = dev;
 670         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 671         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 672         struct arm_smmu_device *smmu = smmu_domain->smmu;
 673         void __iomem *cb_base;
 674
 675         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 676         fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
 677
 678         if (!(fsr & FSR_FAULT))
 679                 return IRQ_NONE;
 680
 681         if (fsr & FSR_IGN)
 682                 dev_err_ratelimited(smmu->dev,
 683                                     "Unexpected context fault (fsr 0x%x)\n",
 684                                     fsr);
 685
 686         fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
 687         flags = fsynr & FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ;
 688
 689         far = readl_relaxed(cb_base + ARM_SMMU_CB_FAR_LO);
 690         iova = far;
 691 #ifdef CONFIG_64BIT
 692         far = readl_relaxed(cb_base + ARM_SMMU_CB_FAR_HI);
 693         iova |= ((unsigned long)far << 32);
 694 #endif
 695
 696         if (!report_iommu_fault(domain, smmu->dev, iova, flags)) {
 697                 ret = IRQ_HANDLED;
 698                 resume = RESUME_RETRY;
 699         } else {
 700                 dev_err_ratelimited(smmu->dev,
 701                     "Unhandled context fault: iova=0x%08lx, fsynr=0x%x, cb=%d\n",
 702                     iova, fsynr, cfg->cbndx);
 703                 ret = IRQ_NONE;
 704                 resume = RESUME_TERMINATE;
 705         }
 706
 707         /* Clear the faulting FSR */
 708         writel(fsr, cb_base + ARM_SMMU_CB_FSR);
 709
 710         /* Retry or terminate any stalled transactions */
 711         if (fsr & FSR_SS)
 712                 writel_relaxed(resume, cb_base + ARM_SMMU_CB_RESUME);
 713
 714         return ret;
 715 }
 716
 717 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 718 {
 719         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
 720         struct arm_smmu_device *smmu = dev;
 721         void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
 722
 723         gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
 724         gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
 725         gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
 726         gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
 727
 728         if (!gfsr)
 729                 return IRQ_NONE;
 730
 731         dev_err_ratelimited(smmu->dev,
 732                 "Unexpected global fault, this could be serious\n");
 733         dev_err_ratelimited(smmu->dev,
 734                 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
 735                 gfsr, gfsynr0, gfsynr1, gfsynr2);
 736
 737         writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
 738         return IRQ_HANDLED;
 739 }
 740
 741 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 742                                        struct io_pgtable_cfg *pgtbl_cfg)
 743 {
 744         u32 reg;
 745         u64 reg64;
 746         bool stage1;
 747         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 748         struct arm_smmu_device *smmu = smmu_domain->smmu;
 749         void __iomem *cb_base, *gr1_base;
 750
 751         gr1_base = ARM_SMMU_GR1(smmu);
 752         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 753         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 754
 755         if (smmu->version > ARM_SMMU_V1) {
 756 #ifdef CONFIG_64BIT
 757                 reg = CBA2R_RW64_64BIT;
 758 #else
 759                 reg = CBA2R_RW64_32BIT;
 760 #endif
 761                 /* 16-bit VMIDs live in CBA2R */
 762                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
 763                         reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBA2R_VMID_SHIFT;
 764
 765                 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
 766         }
 767
 768         /* CBAR */
 769         reg = cfg->cbar;
 770         if (smmu->version == ARM_SMMU_V1)
 771                 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
 772
 773         /*
 774          * Use the weakest shareability/memory types, so they are
 775          * overridden by the ttbcr/pte.
 776          */
 777         if (stage1) {
 778                 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
 779                         (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
 780         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
 781                 /* 8-bit VMIDs live in CBAR */
 782                 reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBAR_VMID_SHIFT;
 783         }
 784         writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
 785
 786         /* TTBRs */
 787         if (stage1) {
 788                 reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
 789
 790                 reg64 |= ((u64)ARM_SMMU_CB_ASID(smmu, cfg)) << TTBRn_ASID_SHIFT;
 791                 smmu_writeq(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 792
 793                 reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
 794                 reg64 |= ((u64)ARM_SMMU_CB_ASID(smmu, cfg)) << TTBRn_ASID_SHIFT;
 795                 smmu_writeq(reg64, cb_base + ARM_SMMU_CB_TTBR1);
 796         } else {
 797                 reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
 798                 smmu_writeq(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 799         }
 800
 801         /* TTBCR */
 802         if (stage1) {
 803                 reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
 804                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
 805                 if (smmu->version > ARM_SMMU_V1) {
 806                         reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
 807                         reg |= TTBCR2_SEP_UPSTREAM;
 808                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR2);
 809                 }
 810         } else {
 811                 reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
 812                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
 813         }
 814
 815         /* MAIRs (stage-1 only) */
 816         if (stage1) {
 817                 reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
 818                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
 819                 reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
 820                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR1);
 821         }
 822
 823         /* SCTLR */
 824         reg = SCTLR_CFCFG | SCTLR_CFIE | SCTLR_CFRE | SCTLR_M | SCTLR_EAE_SBOP;
 825         if (stage1)
 826                 reg |= SCTLR_S1_ASIDPNE;
 827 #ifdef __BIG_ENDIAN
 828         reg |= SCTLR_E;
 829 #endif
 830         writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
 831 }
 832
 833 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 834                                         struct arm_smmu_device *smmu)
 835 {
 836         int irq, start, ret = 0;
 837         unsigned long ias, oas;
 838         struct io_pgtable_ops *pgtbl_ops;
 839         struct io_pgtable_cfg pgtbl_cfg;
 840         enum io_pgtable_fmt fmt;
 841         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 842         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 843
 844         mutex_lock(&smmu_domain->init_mutex);
 845         if (smmu_domain->smmu)
 846                 goto out_unlock;
 847
 848         /*
 849          * Mapping the requested stage onto what we support is surprisingly
 850          * complicated, mainly because the spec allows S1+S2 SMMUs without
 851          * support for nested translation. That means we end up with the
 852          * following table:
 853          *
 854          * Requested        Supported        Actual
 855          *     S1               N              S1
 856          *     S1             S1+S2            S1
 857          *     S1               S2             S2
 858          *     S1               S1             S1
 859          *     N                N              N
 860          *     N              S1+S2            S2
 861          *     N                S2             S2
 862          *     N                S1             S1
 863          *
 864          * Note that you can't actually request stage-2 mappings.
 865          */
 866         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
 867                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 868         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 869                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 870
 871         switch (smmu_domain->stage) {
 872         case ARM_SMMU_DOMAIN_S1:
 873                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
 874                 start = smmu->num_s2_context_banks;
 875                 ias = smmu->va_size;
 876                 oas = smmu->ipa_size;
 877                 if (IS_ENABLED(CONFIG_64BIT))
 878                         fmt = ARM_64_LPAE_S1;
 879                 else
 880                         fmt = ARM_32_LPAE_S1;
 881                 break;
 882         case ARM_SMMU_DOMAIN_NESTED:
 883                 /*
 884                  * We will likely want to change this if/when KVM gets
 885                  * involved.
 886                  */
 887         case ARM_SMMU_DOMAIN_S2:
 888                 cfg->cbar = CBAR_TYPE_S2_TRANS;
 889                 start = 0;
 890                 ias = smmu->ipa_size;
 891                 oas = smmu->pa_size;
 892                 if (IS_ENABLED(CONFIG_64BIT))
 893                         fmt = ARM_64_LPAE_S2;
 894                 else
 895                         fmt = ARM_32_LPAE_S2;
 896                 break;
 897         default:
 898                 ret = -EINVAL;
 899                 goto out_unlock;
 900         }
 901
 902         ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
 903                                       smmu->num_context_banks);
 904         if (IS_ERR_VALUE(ret))
 905                 goto out_unlock;
 906
 907         cfg->cbndx = ret;
 908         if (smmu->version == ARM_SMMU_V1) {
 909                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
 910                 cfg->irptndx %= smmu->num_context_irqs;
 911         } else {
 912                 cfg->irptndx = cfg->cbndx;
 913         }
 914
 915         pgtbl_cfg = (struct io_pgtable_cfg) {
 916                 .pgsize_bitmap  = arm_smmu_ops.pgsize_bitmap,
 917                 .ias            = ias,
 918                 .oas            = oas,
 919                 .tlb            = &arm_smmu_gather_ops,
 920                 .iommu_dev      = smmu->dev,
 921         };
 922
 923         smmu_domain->smmu = smmu;
 924         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
 925         if (!pgtbl_ops) {
 926                 ret = -ENOMEM;
 927                 goto out_clear_smmu;
 928         }
 929
 930         /* Update our support page sizes to reflect the page table format */
 931         arm_smmu_ops.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 932
 933         /* Initialise the context bank with our page table cfg */
 934         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
 935
 936         /*
 937          * Request context fault interrupt. Do this last to avoid the
 938          * handler seeing a half-initialised domain state.
 939          */
 940         irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 941         ret = request_irq(irq, arm_smmu_context_fault, IRQF_SHARED,
 942                           "arm-smmu-context-fault", domain);
 943         if (IS_ERR_VALUE(ret)) {
 944                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
 945                         cfg->irptndx, irq);
 946                 cfg->irptndx = INVALID_IRPTNDX;
 947         }
 948
 949         mutex_unlock(&smmu_domain->init_mutex);
 950
 951         /* Publish page table ops for map/unmap */
 952         smmu_domain->pgtbl_ops = pgtbl_ops;
 953         return 0;
 954
 955 out_clear_smmu:
 956         smmu_domain->smmu = NULL;
 957 out_unlock:
 958         mutex_unlock(&smmu_domain->init_mutex);
 959         return ret;
 960 }
 961
 962 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 963 {
 964         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 965         struct arm_smmu_device *smmu = smmu_domain->smmu;
 966         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 967         void __iomem *cb_base;
 968         int irq;
 969
 970         if (!smmu)
 971                 return;
 972
 973         /*
 974          * Disable the context bank and free the page tables before freeing
 975          * it.
 976          */
 977         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 978         writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
 979
 980         if (cfg->irptndx != INVALID_IRPTNDX) {
 981                 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 982                 free_irq(irq, domain);
 983         }
 984
 985         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
 986         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
 987 }
 988
 989 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
 990 {
 991         struct arm_smmu_domain *smmu_domain;
 992
 993         if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
 994                 return NULL;
 995         /*
 996          * Allocate the domain and initialise some of its data structures.
 997          * We can't really do anything meaningful until we've added a
 998          * master.
 999          */
1000         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1001         if (!smmu_domain)
1002                 return NULL;
1003
1004         if (type == IOMMU_DOMAIN_DMA &&
1005             iommu_get_dma_cookie(&smmu_domain->domain)) {
1006                 kfree(smmu_domain);
1007                 return NULL;
1008         }
1009
1010         mutex_init(&smmu_domain->init_mutex);
1011         spin_lock_init(&smmu_domain->pgtbl_lock);
1012
1013         return &smmu_domain->domain;
1014 }
1015
1016 static void arm_smmu_domain_free(struct iommu_domain *domain)
1017 {
1018         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1019
1020         /*
1021          * Free the domain resources. We assume that all devices have
1022          * already been detached.
1023          */
1024         iommu_put_dma_cookie(domain);
1025         arm_smmu_destroy_domain_context(domain);
1026         kfree(smmu_domain);
1027 }
1028
1029 static int arm_smmu_master_configure_smrs(struct arm_smmu_device *smmu,
1030                                           struct arm_smmu_master_cfg *cfg)
1031 {
1032         int i;
1033         struct arm_smmu_smr *smrs;
1034         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1035
1036         if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH))
1037                 return 0;
1038
1039         if (cfg->smrs)
1040                 return -EEXIST;
1041
1042         smrs = kmalloc_array(cfg->num_streamids, sizeof(*smrs), GFP_KERNEL);
1043         if (!smrs) {
1044                 dev_err(smmu->dev, "failed to allocate %d SMRs\n",
1045                         cfg->num_streamids);
1046                 return -ENOMEM;
1047         }
1048
1049         /* Allocate the SMRs on the SMMU */
1050         for (i = 0; i < cfg->num_streamids; ++i) {
1051                 int idx = __arm_smmu_alloc_bitmap(smmu->smr_map, 0,
1052                                                   smmu->num_mapping_groups);
1053                 if (IS_ERR_VALUE(idx)) {
1054                         dev_err(smmu->dev, "failed to allocate free SMR\n");
1055                         goto err_free_smrs;
1056                 }
1057
1058                 smrs[i] = (struct arm_smmu_smr) {
1059                         .idx    = idx,
1060                         .mask   = 0, /* We don't currently share SMRs */
1061                         .id     = cfg->streamids[i],
1062                 };
1063         }
1064
1065         /* It worked! Now, poke the actual hardware */
1066         for (i = 0; i < cfg->num_streamids; ++i) {
1067                 u32 reg = SMR_VALID | smrs[i].id << SMR_ID_SHIFT |
1068                           smrs[i].mask << SMR_MASK_SHIFT;
1069                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_SMR(smrs[i].idx));
1070         }
1071
1072         cfg->smrs = smrs;
1073         return 0;
1074
1075 err_free_smrs:
1076         while (--i >= 0)
1077                 __arm_smmu_free_bitmap(smmu->smr_map, smrs[i].idx);
1078         kfree(smrs);
1079         return -ENOSPC;
1080 }
1081
1082 static void arm_smmu_master_free_smrs(struct arm_smmu_device *smmu,
1083                                       struct arm_smmu_master_cfg *cfg)
1084 {
1085         int i;
1086         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1087         struct arm_smmu_smr *smrs = cfg->smrs;
1088
1089         if (!smrs)
1090                 return;
1091
1092         /* Invalidate the SMRs before freeing back to the allocator */
1093         for (i = 0; i < cfg->num_streamids; ++i) {
1094                 u8 idx = smrs[i].idx;
1095
1096                 writel_relaxed(~SMR_VALID, gr0_base + ARM_SMMU_GR0_SMR(idx));
1097                 __arm_smmu_free_bitmap(smmu->smr_map, idx);
1098         }
1099
1100         cfg->smrs = NULL;
1101         kfree(smrs);
1102 }
1103
1104 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1105                                       struct arm_smmu_master_cfg *cfg)
1106 {
1107         int i, ret;
1108         struct arm_smmu_device *smmu = smmu_domain->smmu;
1109         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1110
1111         /* Devices in an IOMMU group may already be configured */
1112         ret = arm_smmu_master_configure_smrs(smmu, cfg);
1113         if (ret)
1114                 return ret == -EEXIST ? 0 : ret;
1115
1116         /*
1117          * FIXME: This won't be needed once we have IOMMU-backed DMA ops
1118          * for all devices behind the SMMU.
1119          */
1120         if (smmu_domain->domain.type == IOMMU_DOMAIN_DMA)
1121                 return 0;
1122
1123         for (i = 0; i < cfg->num_streamids; ++i) {
1124                 u32 idx, s2cr;
1125
1126                 idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i];
1127                 s2cr = S2CR_TYPE_TRANS | S2CR_PRIVCFG_UNPRIV |
1128                        (smmu_domain->cfg.cbndx << S2CR_CBNDX_SHIFT);
1129                 writel_relaxed(s2cr, gr0_base + ARM_SMMU_GR0_S2CR(idx));
1130         }
1131
1132         return 0;
1133 }
1134
1135 static void arm_smmu_domain_remove_master(struct arm_smmu_domain *smmu_domain,
1136                                           struct arm_smmu_master_cfg *cfg)
1137 {
1138         int i;
1139         struct arm_smmu_device *smmu = smmu_domain->smmu;
1140         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1141
1142         /* An IOMMU group is torn down by the first device to be removed */
1143         if ((smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) && !cfg->smrs)
1144                 return;
1145
1146         /*
1147          * We *must* clear the S2CR first, because freeing the SMR means
1148          * that it can be re-allocated immediately.
1149          */
1150         for (i = 0; i < cfg->num_streamids; ++i) {
1151                 u32 idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i];
1152                 u32 reg = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS;
1153
1154                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_S2CR(idx));
1155         }
1156
1157         arm_smmu_master_free_smrs(smmu, cfg);
1158 }
1159
1160 static void arm_smmu_detach_dev(struct device *dev,
1161                                 struct arm_smmu_master_cfg *cfg)
1162 {
1163         struct iommu_domain *domain = dev->archdata.iommu;
1164         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1165
1166         dev->archdata.iommu = NULL;
1167         arm_smmu_domain_remove_master(smmu_domain, cfg);
1168 }
1169
1170 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1171 {
1172         int ret;
1173         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1174         struct arm_smmu_device *smmu;
1175         struct arm_smmu_master_cfg *cfg;
1176
1177         smmu = find_smmu_for_device(dev);
1178         if (!smmu) {
1179                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1180                 return -ENXIO;
1181         }
1182
1183         /* Ensure that the domain is finalised */
1184         ret = arm_smmu_init_domain_context(domain, smmu);
1185         if (IS_ERR_VALUE(ret))
1186                 return ret;
1187
1188         /*
1189          * Sanity check the domain. We don't support domains across
1190          * different SMMUs.
1191          */
1192         if (smmu_domain->smmu != smmu) {
1193                 dev_err(dev,
1194                         "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1195                         dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1196                 return -EINVAL;
1197         }
1198
1199         /* Looks ok, so add the device to the domain */
1200         cfg = find_smmu_master_cfg(dev);
1201         if (!cfg)
1202                 return -ENODEV;
1203
1204         /* Detach the dev from its current domain */
1205         if (dev->archdata.iommu)
1206                 arm_smmu_detach_dev(dev, cfg);
1207
1208         ret = arm_smmu_domain_add_master(smmu_domain, cfg);
1209         if (!ret)
1210                 dev->archdata.iommu = domain;
1211         return ret;
1212 }
1213
1214 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1215                         phys_addr_t paddr, size_t size, int prot)
1216 {
1217         int ret;
1218         unsigned long flags;
1219         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1220         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1221
1222         if (!ops)
1223                 return -ENODEV;
1224
1225         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1226         ret = ops->map(ops, iova, paddr, size, prot);
1227         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1228         return ret;
1229 }
1230
1231 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1232                              size_t size)
1233 {
1234         size_t ret;
1235         unsigned long flags;
1236         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1237         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1238
1239         if (!ops)
1240                 return 0;
1241
1242         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1243         ret = ops->unmap(ops, iova, size);
1244         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1245         return ret;
1246 }
1247
1248 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1249                                               dma_addr_t iova)
1250 {
1251         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1252         struct arm_smmu_device *smmu = smmu_domain->smmu;
1253         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1254         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1255         struct device *dev = smmu->dev;
1256         void __iomem *cb_base;
1257         u32 tmp;
1258         u64 phys;
1259         unsigned long va;
1260
1261         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1262
1263         /* ATS1 registers can only be written atomically */
1264         va = iova & ~0xfffUL;
1265         if (smmu->version == ARM_SMMU_V2)
1266                 smmu_writeq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1267         else
1268                 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1269
1270         if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1271                                       !(tmp & ATSR_ACTIVE), 5, 50)) {
1272                 dev_err(dev,
1273                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1274                         &iova);
1275                 return ops->iova_to_phys(ops, iova);
1276         }
1277
1278         phys = readl_relaxed(cb_base + ARM_SMMU_CB_PAR_LO);
1279         phys |= ((u64)readl_relaxed(cb_base + ARM_SMMU_CB_PAR_HI)) << 32;
1280
1281         if (phys & CB_PAR_F) {
1282                 dev_err(dev, "translation fault!\n");
1283                 dev_err(dev, "PAR = 0x%llx\n", phys);
1284                 return 0;
1285         }
1286
1287         return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1288 }
1289
1290 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1291                                         dma_addr_t iova)
1292 {
1293         phys_addr_t ret;
1294         unsigned long flags;
1295         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1296         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1297
1298         if (!ops)
1299                 return 0;
1300
1301         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1302         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1303                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1304                 ret = arm_smmu_iova_to_phys_hard(domain, iova);
1305         } else {
1306                 ret = ops->iova_to_phys(ops, iova);
1307         }
1308
1309         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1310
1311         return ret;
1312 }
1313
1314 static bool arm_smmu_capable(enum iommu_cap cap)
1315 {
1316         switch (cap) {
1317         case IOMMU_CAP_CACHE_COHERENCY:
1318                 /*
1319                  * Return true here as the SMMU can always send out coherent
1320                  * requests.
1321                  */
1322                 return true;
1323         case IOMMU_CAP_INTR_REMAP:
1324                 return true; /* MSIs are just memory writes */
1325         case IOMMU_CAP_NOEXEC:
1326                 return true;
1327         default:
1328                 return false;
1329         }
1330 }
1331
1332 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
1333 {
1334         *((u16 *)data) = alias;
1335         return 0; /* Continue walking */
1336 }
1337
1338 static void __arm_smmu_release_pci_iommudata(void *data)
1339 {
1340         kfree(data);
1341 }
1342
1343 static int arm_smmu_init_pci_device(struct pci_dev *pdev,
1344                                     struct iommu_group *group)
1345 {
1346         struct arm_smmu_master_cfg *cfg;
1347         u16 sid;
1348         int i;
1349
1350         cfg = iommu_group_get_iommudata(group);
1351         if (!cfg) {
1352                 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
1353                 if (!cfg)
1354                         return -ENOMEM;
1355
1356                 iommu_group_set_iommudata(group, cfg,
1357                                           __arm_smmu_release_pci_iommudata);
1358         }
1359
1360         if (cfg->num_streamids >= MAX_MASTER_STREAMIDS)
1361                 return -ENOSPC;
1362
1363         /*
1364          * Assume Stream ID == Requester ID for now.
1365          * We need a way to describe the ID mappings in FDT.
1366          */
1367         pci_for_each_dma_alias(pdev, __arm_smmu_get_pci_sid, &sid);
1368         for (i = 0; i < cfg->num_streamids; ++i)
1369                 if (cfg->streamids[i] == sid)
1370                         break;
1371
1372         /* Avoid duplicate SIDs, as this can lead to SMR conflicts */
1373         if (i == cfg->num_streamids)
1374                 cfg->streamids[cfg->num_streamids++] = sid;
1375
1376         return 0;
1377 }
1378
1379 static int arm_smmu_init_platform_device(struct device *dev,
1380                                          struct iommu_group *group)
1381 {
1382         struct arm_smmu_device *smmu = find_smmu_for_device(dev);
1383         struct arm_smmu_master *master;
1384
1385         if (!smmu)
1386                 return -ENODEV;
1387
1388         master = find_smmu_master(smmu, dev->of_node);
1389         if (!master)
1390                 return -ENODEV;
1391
1392         iommu_group_set_iommudata(group, &master->cfg, NULL);
1393
1394         return 0;
1395 }
1396
1397 static int arm_smmu_add_device(struct device *dev)
1398 {
1399         struct iommu_group *group;
1400
1401         group = iommu_group_get_for_dev(dev);
1402         if (IS_ERR(group))
1403                 return PTR_ERR(group);
1404
1405         iommu_group_put(group);
1406         return 0;
1407 }
1408
1409 static void arm_smmu_remove_device(struct device *dev)
1410 {
1411         iommu_group_remove_device(dev);
1412 }
1413
1414 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1415 {
1416         struct iommu_group *group;
1417         int ret;
1418
1419         if (dev_is_pci(dev))
1420                 group = pci_device_group(dev);
1421         else
1422                 group = generic_device_group(dev);
1423
1424         if (IS_ERR(group))
1425                 return group;
1426
1427         if (dev_is_pci(dev))
1428                 ret = arm_smmu_init_pci_device(to_pci_dev(dev), group);
1429         else
1430                 ret = arm_smmu_init_platform_device(dev, group);
1431
1432         if (ret) {
1433                 iommu_group_put(group);
1434                 group = ERR_PTR(ret);
1435         }
1436
1437         return group;
1438 }
1439
1440 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1441                                     enum iommu_attr attr, void *data)
1442 {
1443         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1444
1445         switch (attr) {
1446         case DOMAIN_ATTR_NESTING:
1447                 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1448                 return 0;
1449         default:
1450                 return -ENODEV;
1451         }
1452 }
1453
1454 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1455                                     enum iommu_attr attr, void *data)
1456 {
1457         int ret = 0;
1458         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1459
1460         mutex_lock(&smmu_domain->init_mutex);
1461
1462         switch (attr) {
1463         case DOMAIN_ATTR_NESTING:
1464                 if (smmu_domain->smmu) {
1465                         ret = -EPERM;
1466                         goto out_unlock;
1467                 }
1468
1469                 if (*(int *)data)
1470                         smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1471                 else
1472                         smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1473
1474                 break;
1475         default:
1476                 ret = -ENODEV;
1477         }
1478
1479 out_unlock:
1480         mutex_unlock(&smmu_domain->init_mutex);
1481         return ret;
1482 }
1483
1484 static struct iommu_ops arm_smmu_ops = {
1485         .capable                = arm_smmu_capable,
1486         .domain_alloc           = arm_smmu_domain_alloc,
1487         .domain_free            = arm_smmu_domain_free,
1488         .attach_dev             = arm_smmu_attach_dev,
1489         .map                    = arm_smmu_map,
1490         .unmap                  = arm_smmu_unmap,
1491         .map_sg                 = default_iommu_map_sg,
1492         .iova_to_phys           = arm_smmu_iova_to_phys,
1493         .add_device             = arm_smmu_add_device,
1494         .remove_device          = arm_smmu_remove_device,
1495         .device_group           = arm_smmu_device_group,
1496         .domain_get_attr        = arm_smmu_domain_get_attr,
1497         .domain_set_attr        = arm_smmu_domain_set_attr,
1498         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1499 };
1500
1501 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1502 {
1503         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1504         void __iomem *cb_base;
1505         int i = 0;
1506         u32 reg;
1507
1508         /* clear global FSR */
1509         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1510         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1511
1512         /* Mark all SMRn as invalid and all S2CRn as bypass unless overridden */
1513         reg = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS;
1514         for (i = 0; i < smmu->num_mapping_groups; ++i) {
1515                 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_SMR(i));
1516                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_S2CR(i));
1517         }
1518
1519         /* Make sure all context banks are disabled and clear CB_FSR  */
1520         for (i = 0; i < smmu->num_context_banks; ++i) {
1521                 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i);
1522                 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1523                 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1524                 /*
1525                  * Disable MMU-500's not-particularly-beneficial next-page
1526                  * prefetcher for the sake of errata #841119 and #826419.
1527                  */
1528                 if (smmu->model == ARM_MMU500) {
1529                         reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
1530                         reg &= ~ARM_MMU500_ACTLR_CPRE;
1531                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
1532                 }
1533         }
1534
1535         /* Invalidate the TLB, just in case */
1536         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1537         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1538
1539         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1540
1541         /* Enable fault reporting */
1542         reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1543
1544         /* Disable TLB broadcasting. */
1545         reg |= (sCR0_VMIDPNE | sCR0_PTM);
1546
1547         /* Enable client access, handling unmatched streams as appropriate */
1548         reg &= ~sCR0_CLIENTPD;
1549         if (disable_bypass)
1550                 reg |= sCR0_USFCFG;
1551         else
1552                 reg &= ~sCR0_USFCFG;
1553
1554         /* Disable forced broadcasting */
1555         reg &= ~sCR0_FB;
1556
1557         /* Don't upgrade barriers */
1558         reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
1559
1560         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1561                 reg |= sCR0_VMID16EN;
1562
1563         /* Push the button */
1564         __arm_smmu_tlb_sync(smmu);
1565         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1566 }
1567
1568 static int arm_smmu_id_size_to_bits(int size)
1569 {
1570         switch (size) {
1571         case 0:
1572                 return 32;
1573         case 1:
1574                 return 36;
1575         case 2:
1576                 return 40;
1577         case 3:
1578                 return 42;
1579         case 4:
1580                 return 44;
1581         case 5:
1582         default:
1583                 return 48;
1584         }
1585 }
1586
1587 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1588 {
1589         unsigned long size;
1590         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1591         u32 id;
1592         bool cttw_dt, cttw_reg;
1593
1594         dev_notice(smmu->dev, "probing hardware configuration...\n");
1595         dev_notice(smmu->dev, "SMMUv%d with:\n", smmu->version);
1596
1597         /* ID0 */
1598         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1599
1600         /* Restrict available stages based on module parameter */
1601         if (force_stage == 1)
1602                 id &= ~(ID0_S2TS | ID0_NTS);
1603         else if (force_stage == 2)
1604                 id &= ~(ID0_S1TS | ID0_NTS);
1605
1606         if (id & ID0_S1TS) {
1607                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1608                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1609         }
1610
1611         if (id & ID0_S2TS) {
1612                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1613                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1614         }
1615
1616         if (id & ID0_NTS) {
1617                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1618                 dev_notice(smmu->dev, "\tnested translation\n");
1619         }
1620
1621         if (!(smmu->features &
1622                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1623                 dev_err(smmu->dev, "\tno translation support!\n");
1624                 return -ENODEV;
1625         }
1626
1627         if ((id & ID0_S1TS) && ((smmu->version == 1) || !(id & ID0_ATOSNS))) {
1628                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1629                 dev_notice(smmu->dev, "\taddress translation ops\n");
1630         }
1631
1632         /*
1633          * In order for DMA API calls to work properly, we must defer to what
1634          * the DT says about coherency, regardless of what the hardware claims.
1635          * Fortunately, this also opens up a workaround for systems where the
1636          * ID register value has ended up configured incorrectly.
1637          */
1638         cttw_dt = of_dma_is_coherent(smmu->dev->of_node);
1639         cttw_reg = !!(id & ID0_CTTW);
1640         if (cttw_dt)
1641                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1642         if (cttw_dt || cttw_reg)
1643                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1644                            cttw_dt ? "" : "non-");
1645         if (cttw_dt != cttw_reg)
1646                 dev_notice(smmu->dev,
1647                            "\t(IDR0.CTTW overridden by dma-coherent property)\n");
1648
1649         if (id & ID0_SMS) {
1650                 u32 smr, sid, mask;
1651
1652                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1653                 smmu->num_mapping_groups = (id >> ID0_NUMSMRG_SHIFT) &
1654                                            ID0_NUMSMRG_MASK;
1655                 if (smmu->num_mapping_groups == 0) {
1656                         dev_err(smmu->dev,
1657                                 "stream-matching supported, but no SMRs present!\n");
1658                         return -ENODEV;
1659                 }
1660
1661                 smr = SMR_MASK_MASK << SMR_MASK_SHIFT;
1662                 smr |= (SMR_ID_MASK << SMR_ID_SHIFT);
1663                 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1664                 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1665
1666                 mask = (smr >> SMR_MASK_SHIFT) & SMR_MASK_MASK;
1667                 sid = (smr >> SMR_ID_SHIFT) & SMR_ID_MASK;
1668                 if ((mask & sid) != sid) {
1669                         dev_err(smmu->dev,
1670                                 "SMR mask bits (0x%x) insufficient for ID field (0x%x)\n",
1671                                 mask, sid);
1672                         return -ENODEV;
1673                 }
1674
1675                 dev_notice(smmu->dev,
1676                            "\tstream matching with %u register groups, mask 0x%x",
1677                            smmu->num_mapping_groups, mask);
1678         } else {
1679                 smmu->num_mapping_groups = (id >> ID0_NUMSIDB_SHIFT) &
1680                                            ID0_NUMSIDB_MASK;
1681         }
1682
1683         /* ID1 */
1684         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1685         smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1686
1687         /* Check for size mismatch of SMMU address space from mapped region */
1688         size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
1689         size *= 2 << smmu->pgshift;
1690         if (smmu->size != size)
1691                 dev_warn(smmu->dev,
1692                         "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n",
1693                         size, smmu->size);
1694
1695         smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
1696         smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
1697         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1698                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1699                 return -ENODEV;
1700         }
1701         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1702                    smmu->num_context_banks, smmu->num_s2_context_banks);
1703         /*
1704          * Cavium CN88xx erratum #27704.
1705          * Ensure ASID and VMID allocation is unique across all SMMUs in
1706          * the system.
1707          */
1708         if (smmu->model == CAVIUM_SMMUV2) {
1709                 smmu->cavium_id_base =
1710                         atomic_add_return(smmu->num_context_banks,
1711                                           &cavium_smmu_context_count);
1712                 smmu->cavium_id_base -= smmu->num_context_banks;
1713         }
1714
1715         /* ID2 */
1716         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1717         size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
1718         smmu->ipa_size = size;
1719
1720         /* The output mask is also applied for bypass */
1721         size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
1722         smmu->pa_size = size;
1723
1724         if (id & ID2_VMID16)
1725                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1726
1727         /*
1728          * What the page table walker can address actually depends on which
1729          * descriptor format is in use, but since a) we don't know that yet,
1730          * and b) it can vary per context bank, this will have to do...
1731          */
1732         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1733                 dev_warn(smmu->dev,
1734                          "failed to set DMA mask for table walker\n");
1735
1736         if (smmu->version == ARM_SMMU_V1) {
1737                 smmu->va_size = smmu->ipa_size;
1738                 size = SZ_4K | SZ_2M | SZ_1G;
1739         } else {
1740                 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
1741                 smmu->va_size = arm_smmu_id_size_to_bits(size);
1742 #ifndef CONFIG_64BIT
1743                 smmu->va_size = min(32UL, smmu->va_size);
1744 #endif
1745                 size = 0;
1746                 if (id & ID2_PTFS_4K)
1747                         size |= SZ_4K | SZ_2M | SZ_1G;
1748                 if (id & ID2_PTFS_16K)
1749                         size |= SZ_16K | SZ_32M;
1750                 if (id & ID2_PTFS_64K)
1751                         size |= SZ_64K | SZ_512M;
1752         }
1753
1754         arm_smmu_ops.pgsize_bitmap &= size;
1755         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n", size);
1756
1757         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1758                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1759                            smmu->va_size, smmu->ipa_size);
1760
1761         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1762                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1763                            smmu->ipa_size, smmu->pa_size);
1764
1765         return 0;
1766 }
1767
1768 struct arm_smmu_match_data {
1769         enum arm_smmu_arch_version version;
1770         enum arm_smmu_implementation model;
1771 };
1772
1773 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
1774 static struct arm_smmu_match_data name = { .version = ver, .model = imp }
1775
1776 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1777 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1778 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1779 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1780
1781 static const struct of_device_id arm_smmu_of_match[] = {
1782         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1783         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1784         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1785         { .compatible = "arm,mmu-401", .data = &smmu_generic_v1 },
1786         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1787         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1788         { },
1789 };
1790 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1791
1792 static int arm_smmu_device_dt_probe(struct platform_device *pdev)
1793 {
1794         const struct of_device_id *of_id;
1795         const struct arm_smmu_match_data *data;
1796         struct resource *res;
1797         struct arm_smmu_device *smmu;
1798         struct device *dev = &pdev->dev;
1799         struct rb_node *node;
1800         struct of_phandle_args masterspec;
1801         int num_irqs, i, err;
1802
1803         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
1804         if (!smmu) {
1805                 dev_err(dev, "failed to allocate arm_smmu_device\n");
1806                 return -ENOMEM;
1807         }
1808         smmu->dev = dev;
1809
1810         of_id = of_match_node(arm_smmu_of_match, dev->of_node);
1811         data = of_id->data;
1812         smmu->version = data->version;
1813         smmu->model = data->model;
1814
1815         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1816         smmu->base = devm_ioremap_resource(dev, res);
1817         if (IS_ERR(smmu->base))
1818                 return PTR_ERR(smmu->base);
1819         smmu->size = resource_size(res);
1820
1821         if (of_property_read_u32(dev->of_node, "#global-interrupts",
1822                                  &smmu->num_global_irqs)) {
1823                 dev_err(dev, "missing #global-interrupts property\n");
1824                 return -ENODEV;
1825         }
1826
1827         num_irqs = 0;
1828         while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
1829                 num_irqs++;
1830                 if (num_irqs > smmu->num_global_irqs)
1831                         smmu->num_context_irqs++;
1832         }
1833
1834         if (!smmu->num_context_irqs) {
1835                 dev_err(dev, "found %d interrupts but expected at least %d\n",
1836                         num_irqs, smmu->num_global_irqs + 1);
1837                 return -ENODEV;
1838         }
1839
1840         smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs,
1841                                   GFP_KERNEL);
1842         if (!smmu->irqs) {
1843                 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
1844                 return -ENOMEM;
1845         }
1846
1847         for (i = 0; i < num_irqs; ++i) {
1848                 int irq = platform_get_irq(pdev, i);
1849
1850                 if (irq < 0) {
1851                         dev_err(dev, "failed to get irq index %d\n", i);
1852                         return -ENODEV;
1853                 }
1854                 smmu->irqs[i] = irq;
1855         }
1856
1857         err = arm_smmu_device_cfg_probe(smmu);
1858         if (err)
1859                 return err;
1860
1861         i = 0;
1862         smmu->masters = RB_ROOT;
1863         while (!of_parse_phandle_with_args(dev->of_node, "mmu-masters",
1864                                            "#stream-id-cells", i,
1865                                            &masterspec)) {
1866                 err = register_smmu_master(smmu, dev, &masterspec);
1867                 if (err) {
1868                         dev_err(dev, "failed to add master %s\n",
1869                                 masterspec.np->name);
1870                         goto out_put_masters;
1871                 }
1872
1873                 i++;
1874         }
1875         dev_notice(dev, "registered %d master devices\n", i);
1876
1877         parse_driver_options(smmu);
1878
1879         if (smmu->version > ARM_SMMU_V1 &&
1880             smmu->num_context_banks != smmu->num_context_irqs) {
1881                 dev_err(dev,
1882                         "found only %d context interrupt(s) but %d required\n",
1883                         smmu->num_context_irqs, smmu->num_context_banks);
1884                 err = -ENODEV;
1885                 goto out_put_masters;
1886         }
1887
1888         for (i = 0; i < smmu->num_global_irqs; ++i) {
1889                 err = request_irq(smmu->irqs[i],
1890                                   arm_smmu_global_fault,
1891                                   IRQF_SHARED,
1892                                   "arm-smmu global fault",
1893                                   smmu);
1894                 if (err) {
1895                         dev_err(dev, "failed to request global IRQ %d (%u)\n",
1896                                 i, smmu->irqs[i]);
1897                         goto out_free_irqs;
1898                 }
1899         }
1900
1901         INIT_LIST_HEAD(&smmu->list);
1902         spin_lock(&arm_smmu_devices_lock);
1903         list_add(&smmu->list, &arm_smmu_devices);
1904         spin_unlock(&arm_smmu_devices_lock);
1905
1906         arm_smmu_device_reset(smmu);
1907         return 0;
1908
1909 out_free_irqs:
1910         while (i--)
1911                 free_irq(smmu->irqs[i], smmu);
1912
1913 out_put_masters:
1914         for (node = rb_first(&smmu->masters); node; node = rb_next(node)) {
1915                 struct arm_smmu_master *master
1916                         = container_of(node, struct arm_smmu_master, node);
1917                 of_node_put(master->of_node);
1918         }
1919
1920         return err;
1921 }
1922
1923 static int arm_smmu_device_remove(struct platform_device *pdev)
1924 {
1925         int i;
1926         struct device *dev = &pdev->dev;
1927         struct arm_smmu_device *curr, *smmu = NULL;
1928         struct rb_node *node;
1929
1930         spin_lock(&arm_smmu_devices_lock);
1931         list_for_each_entry(curr, &arm_smmu_devices, list) {
1932                 if (curr->dev == dev) {
1933                         smmu = curr;
1934                         list_del(&smmu->list);
1935                         break;
1936                 }
1937         }
1938         spin_unlock(&arm_smmu_devices_lock);
1939
1940         if (!smmu)
1941                 return -ENODEV;
1942
1943         for (node = rb_first(&smmu->masters); node; node = rb_next(node)) {
1944                 struct arm_smmu_master *master
1945                         = container_of(node, struct arm_smmu_master, node);
1946                 of_node_put(master->of_node);
1947         }
1948
1949         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
1950                 dev_err(dev, "removing device with active domains!\n");
1951
1952         for (i = 0; i < smmu->num_global_irqs; ++i)
1953                 free_irq(smmu->irqs[i], smmu);
1954
1955         /* Turn the thing off */
1956         writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1957         return 0;
1958 }
1959
1960 static struct platform_driver arm_smmu_driver = {
1961         .driver = {
1962                 .name           = "arm-smmu",
1963                 .of_match_table = of_match_ptr(arm_smmu_of_match),
1964         },
1965         .probe  = arm_smmu_device_dt_probe,
1966         .remove = arm_smmu_device_remove,
1967 };
1968
1969 static int __init arm_smmu_init(void)
1970 {
1971         struct device_node *np;
1972         int ret;
1973
1974         /*
1975          * Play nice with systems that don't have an ARM SMMU by checking that
1976          * an ARM SMMU exists in the system before proceeding with the driver
1977          * and IOMMU bus operation registration.
1978          */
1979         np = of_find_matching_node(NULL, arm_smmu_of_match);
1980         if (!np)
1981                 return 0;
1982
1983         of_node_put(np);
1984
1985         ret = platform_driver_register(&arm_smmu_driver);
1986         if (ret)
1987                 return ret;
1988
1989         /* Oh, for a proper bus abstraction */
1990         if (!iommu_present(&platform_bus_type))
1991                 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
1992
1993 #ifdef CONFIG_ARM_AMBA
1994         if (!iommu_present(&amba_bustype))
1995                 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
1996 #endif
1997
1998 #ifdef CONFIG_PCI
1999         if (!iommu_present(&pci_bus_type))
2000                 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2001 #endif
2002
2003         return 0;
2004 }
2005
2006 static void __exit arm_smmu_exit(void)
2007 {
2008         return platform_driver_unregister(&arm_smmu_driver);
2009 }
2010
2011 subsys_initcall(arm_smmu_init);
2012 module_exit(arm_smmu_exit);
2013
2014 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2015 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2016 MODULE_LICENSE("GPL v2");