drivers/iommu/arm-smmu.c

   1 /*
   2  * IOMMU API for ARM architected SMMU implementations.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  16  *
  17  * Copyright (C) 2013 ARM Limited
  18  *
  19  * Author: Will Deacon <will.deacon@arm.com>
  20  *
  21  * This driver currently supports:
  22  *      - SMMUv1 and v2 implementations
  23  *      - Stream-matching and stream-indexing
  24  *      - v7/v8 long-descriptor format
  25  *      - Non-secure access to the SMMU
  26  *      - Context fault reporting
  27  *      - Extended Stream ID (16 bit)
  28  */
  29
  30 #define pr_fmt(fmt) "arm-smmu: " fmt
  31
  32 #include <linux/acpi.h>
  33 #include <linux/acpi_iort.h>
  34 #include <linux/atomic.h>
  35 #include <linux/delay.h>
  36 #include <linux/dma-iommu.h>
  37 #include <linux/dma-mapping.h>
  38 #include <linux/err.h>
  39 #include <linux/interrupt.h>
  40 #include <linux/io.h>
  41 #include <linux/io-64-nonatomic-hi-lo.h>
  42 #include <linux/iommu.h>
  43 #include <linux/iopoll.h>
  44 #include <linux/module.h>
  45 #include <linux/of.h>
  46 #include <linux/of_address.h>
  47 #include <linux/of_device.h>
  48 #include <linux/of_iommu.h>
  49 #include <linux/pci.h>
  50 #include <linux/platform_device.h>
  51 #include <linux/slab.h>
  52 #include <linux/spinlock.h>
  53
  54 #include <linux/amba/bus.h>
  55
  56 #include "io-pgtable.h"
  57
  58 /* Maximum number of context banks per SMMU */
  59 #define ARM_SMMU_MAX_CBS                128
  60
  61 /* SMMU global address space */
  62 #define ARM_SMMU_GR0(smmu)              ((smmu)->base)
  63 #define ARM_SMMU_GR1(smmu)              ((smmu)->base + (1 << (smmu)->pgshift))
  64
  65 /*
  66  * SMMU global address space with conditional offset to access secure
  67  * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
  68  * nsGFSYNR0: 0x450)
  69  */
  70 #define ARM_SMMU_GR0_NS(smmu)                                           \
  71         ((smmu)->base +                                                 \
  72                 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS)       \
  73                         ? 0x400 : 0))
  74
  75 /*
  76  * Some 64-bit registers only make sense to write atomically, but in such
  77  * cases all the data relevant to AArch32 formats lies within the lower word,
  78  * therefore this actually makes more sense than it might first appear.
  79  */
  80 #ifdef CONFIG_64BIT
  81 #define smmu_write_atomic_lq            writeq_relaxed
  82 #else
  83 #define smmu_write_atomic_lq            writel_relaxed
  84 #endif
  85
  86 /* Configuration registers */
  87 #define ARM_SMMU_GR0_sCR0               0x0
  88 #define sCR0_CLIENTPD                   (1 << 0)
  89 #define sCR0_GFRE                       (1 << 1)
  90 #define sCR0_GFIE                       (1 << 2)
  91 #define sCR0_EXIDENABLE                 (1 << 3)
  92 #define sCR0_GCFGFRE                    (1 << 4)
  93 #define sCR0_GCFGFIE                    (1 << 5)
  94 #define sCR0_USFCFG                     (1 << 10)
  95 #define sCR0_VMIDPNE                    (1 << 11)
  96 #define sCR0_PTM                        (1 << 12)
  97 #define sCR0_FB                         (1 << 13)
  98 #define sCR0_VMID16EN                   (1 << 31)
  99 #define sCR0_BSU_SHIFT                  14
 100 #define sCR0_BSU_MASK                   0x3
 101
 102 /* Auxiliary Configuration register */
 103 #define ARM_SMMU_GR0_sACR               0x10
 104
 105 /* Identification registers */
 106 #define ARM_SMMU_GR0_ID0                0x20
 107 #define ARM_SMMU_GR0_ID1                0x24
 108 #define ARM_SMMU_GR0_ID2                0x28
 109 #define ARM_SMMU_GR0_ID3                0x2c
 110 #define ARM_SMMU_GR0_ID4                0x30
 111 #define ARM_SMMU_GR0_ID5                0x34
 112 #define ARM_SMMU_GR0_ID6                0x38
 113 #define ARM_SMMU_GR0_ID7                0x3c
 114 #define ARM_SMMU_GR0_sGFSR              0x48
 115 #define ARM_SMMU_GR0_sGFSYNR0           0x50
 116 #define ARM_SMMU_GR0_sGFSYNR1           0x54
 117 #define ARM_SMMU_GR0_sGFSYNR2           0x58
 118
 119 #define ID0_S1TS                        (1 << 30)
 120 #define ID0_S2TS                        (1 << 29)
 121 #define ID0_NTS                         (1 << 28)
 122 #define ID0_SMS                         (1 << 27)
 123 #define ID0_ATOSNS                      (1 << 26)
 124 #define ID0_PTFS_NO_AARCH32             (1 << 25)
 125 #define ID0_PTFS_NO_AARCH32S            (1 << 24)
 126 #define ID0_CTTW                        (1 << 14)
 127 #define ID0_NUMIRPT_SHIFT               16
 128 #define ID0_NUMIRPT_MASK                0xff
 129 #define ID0_NUMSIDB_SHIFT               9
 130 #define ID0_NUMSIDB_MASK                0xf
 131 #define ID0_EXIDS                       (1 << 8)
 132 #define ID0_NUMSMRG_SHIFT               0
 133 #define ID0_NUMSMRG_MASK                0xff
 134
 135 #define ID1_PAGESIZE                    (1 << 31)
 136 #define ID1_NUMPAGENDXB_SHIFT           28
 137 #define ID1_NUMPAGENDXB_MASK            7
 138 #define ID1_NUMS2CB_SHIFT               16
 139 #define ID1_NUMS2CB_MASK                0xff
 140 #define ID1_NUMCB_SHIFT                 0
 141 #define ID1_NUMCB_MASK                  0xff
 142
 143 #define ID2_OAS_SHIFT                   4
 144 #define ID2_OAS_MASK                    0xf
 145 #define ID2_IAS_SHIFT                   0
 146 #define ID2_IAS_MASK                    0xf
 147 #define ID2_UBS_SHIFT                   8
 148 #define ID2_UBS_MASK                    0xf
 149 #define ID2_PTFS_4K                     (1 << 12)
 150 #define ID2_PTFS_16K                    (1 << 13)
 151 #define ID2_PTFS_64K                    (1 << 14)
 152 #define ID2_VMID16                      (1 << 15)
 153
 154 #define ID7_MAJOR_SHIFT                 4
 155 #define ID7_MAJOR_MASK                  0xf
 156
 157 /* Global TLB invalidation */
 158 #define ARM_SMMU_GR0_TLBIVMID           0x64
 159 #define ARM_SMMU_GR0_TLBIALLNSNH        0x68
 160 #define ARM_SMMU_GR0_TLBIALLH           0x6c
 161 #define ARM_SMMU_GR0_sTLBGSYNC          0x70
 162 #define ARM_SMMU_GR0_sTLBGSTATUS        0x74
 163 #define sTLBGSTATUS_GSACTIVE            (1 << 0)
 164 #define TLB_LOOP_TIMEOUT                1000000 /* 1s! */
 165 #define TLB_SPIN_COUNT                  10
 166
 167 /* Stream mapping registers */
 168 #define ARM_SMMU_GR0_SMR(n)             (0x800 + ((n) << 2))
 169 #define SMR_VALID                       (1 << 31)
 170 #define SMR_MASK_SHIFT                  16
 171 #define SMR_ID_SHIFT                    0
 172
 173 #define ARM_SMMU_GR0_S2CR(n)            (0xc00 + ((n) << 2))
 174 #define S2CR_CBNDX_SHIFT                0
 175 #define S2CR_CBNDX_MASK                 0xff
 176 #define S2CR_EXIDVALID                  (1 << 10)
 177 #define S2CR_TYPE_SHIFT                 16
 178 #define S2CR_TYPE_MASK                  0x3
 179 enum arm_smmu_s2cr_type {
 180         S2CR_TYPE_TRANS,
 181         S2CR_TYPE_BYPASS,
 182         S2CR_TYPE_FAULT,
 183 };
 184
 185 #define S2CR_PRIVCFG_SHIFT              24
 186 #define S2CR_PRIVCFG_MASK               0x3
 187 enum arm_smmu_s2cr_privcfg {
 188         S2CR_PRIVCFG_DEFAULT,
 189         S2CR_PRIVCFG_DIPAN,
 190         S2CR_PRIVCFG_UNPRIV,
 191         S2CR_PRIVCFG_PRIV,
 192 };
 193
 194 /* Context bank attribute registers */
 195 #define ARM_SMMU_GR1_CBAR(n)            (0x0 + ((n) << 2))
 196 #define CBAR_VMID_SHIFT                 0
 197 #define CBAR_VMID_MASK                  0xff
 198 #define CBAR_S1_BPSHCFG_SHIFT           8
 199 #define CBAR_S1_BPSHCFG_MASK            3
 200 #define CBAR_S1_BPSHCFG_NSH             3
 201 #define CBAR_S1_MEMATTR_SHIFT           12
 202 #define CBAR_S1_MEMATTR_MASK            0xf
 203 #define CBAR_S1_MEMATTR_WB              0xf
 204 #define CBAR_TYPE_SHIFT                 16
 205 #define CBAR_TYPE_MASK                  0x3
 206 #define CBAR_TYPE_S2_TRANS              (0 << CBAR_TYPE_SHIFT)
 207 #define CBAR_TYPE_S1_TRANS_S2_BYPASS    (1 << CBAR_TYPE_SHIFT)
 208 #define CBAR_TYPE_S1_TRANS_S2_FAULT     (2 << CBAR_TYPE_SHIFT)
 209 #define CBAR_TYPE_S1_TRANS_S2_TRANS     (3 << CBAR_TYPE_SHIFT)
 210 #define CBAR_IRPTNDX_SHIFT              24
 211 #define CBAR_IRPTNDX_MASK               0xff
 212
 213 #define ARM_SMMU_GR1_CBA2R(n)           (0x800 + ((n) << 2))
 214 #define CBA2R_RW64_32BIT                (0 << 0)
 215 #define CBA2R_RW64_64BIT                (1 << 0)
 216 #define CBA2R_VMID_SHIFT                16
 217 #define CBA2R_VMID_MASK                 0xffff
 218
 219 /* Translation context bank */
 220 #define ARM_SMMU_CB(smmu, n)    ((smmu)->cb_base + ((n) << (smmu)->pgshift))
 221
 222 #define ARM_SMMU_CB_SCTLR               0x0
 223 #define ARM_SMMU_CB_ACTLR               0x4
 224 #define ARM_SMMU_CB_RESUME              0x8
 225 #define ARM_SMMU_CB_TTBCR2              0x10
 226 #define ARM_SMMU_CB_TTBR0               0x20
 227 #define ARM_SMMU_CB_TTBR1               0x28
 228 #define ARM_SMMU_CB_TTBCR               0x30
 229 #define ARM_SMMU_CB_CONTEXTIDR          0x34
 230 #define ARM_SMMU_CB_S1_MAIR0            0x38
 231 #define ARM_SMMU_CB_S1_MAIR1            0x3c
 232 #define ARM_SMMU_CB_PAR                 0x50
 233 #define ARM_SMMU_CB_FSR                 0x58
 234 #define ARM_SMMU_CB_FAR                 0x60
 235 #define ARM_SMMU_CB_FSYNR0              0x68
 236 #define ARM_SMMU_CB_S1_TLBIVA           0x600
 237 #define ARM_SMMU_CB_S1_TLBIASID         0x610
 238 #define ARM_SMMU_CB_S1_TLBIVAL          0x620
 239 #define ARM_SMMU_CB_S2_TLBIIPAS2        0x630
 240 #define ARM_SMMU_CB_S2_TLBIIPAS2L       0x638
 241 #define ARM_SMMU_CB_TLBSYNC             0x7f0
 242 #define ARM_SMMU_CB_TLBSTATUS           0x7f4
 243 #define ARM_SMMU_CB_ATS1PR              0x800
 244 #define ARM_SMMU_CB_ATSR                0x8f0
 245
 246 #define SCTLR_S1_ASIDPNE                (1 << 12)
 247 #define SCTLR_CFCFG                     (1 << 7)
 248 #define SCTLR_CFIE                      (1 << 6)
 249 #define SCTLR_CFRE                      (1 << 5)
 250 #define SCTLR_E                         (1 << 4)
 251 #define SCTLR_AFE                       (1 << 2)
 252 #define SCTLR_TRE                       (1 << 1)
 253 #define SCTLR_M                         (1 << 0)
 254
 255 #define ARM_MMU500_ACTLR_CPRE           (1 << 1)
 256
 257 #define ARM_MMU500_ACR_CACHE_LOCK       (1 << 26)
 258 #define ARM_MMU500_ACR_SMTNMB_TLBEN     (1 << 8)
 259
 260 #define CB_PAR_F                        (1 << 0)
 261
 262 #define ATSR_ACTIVE                     (1 << 0)
 263
 264 #define RESUME_RETRY                    (0 << 0)
 265 #define RESUME_TERMINATE                (1 << 0)
 266
 267 #define TTBCR2_SEP_SHIFT                15
 268 #define TTBCR2_SEP_UPSTREAM             (0x7 << TTBCR2_SEP_SHIFT)
 269 #define TTBCR2_AS                       (1 << 4)
 270
 271 #define TTBRn_ASID_SHIFT                48
 272
 273 #define FSR_MULTI                       (1 << 31)
 274 #define FSR_SS                          (1 << 30)
 275 #define FSR_UUT                         (1 << 8)
 276 #define FSR_ASF                         (1 << 7)
 277 #define FSR_TLBLKF                      (1 << 6)
 278 #define FSR_TLBMCF                      (1 << 5)
 279 #define FSR_EF                          (1 << 4)
 280 #define FSR_PF                          (1 << 3)
 281 #define FSR_AFF                         (1 << 2)
 282 #define FSR_TF                          (1 << 1)
 283
 284 #define FSR_IGN                         (FSR_AFF | FSR_ASF | \
 285                                          FSR_TLBMCF | FSR_TLBLKF)
 286 #define FSR_FAULT                       (FSR_MULTI | FSR_SS | FSR_UUT | \
 287                                          FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
 288
 289 #define FSYNR0_WNR                      (1 << 4)
 290
 291 #define MSI_IOVA_BASE                   0x8000000
 292 #define MSI_IOVA_LENGTH                 0x100000
 293
 294 static int force_stage;
 295 module_param(force_stage, int, S_IRUGO);
 296 MODULE_PARM_DESC(force_stage,
 297         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
 298 static bool disable_bypass;
 299 module_param(disable_bypass, bool, S_IRUGO);
 300 MODULE_PARM_DESC(disable_bypass,
 301         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
 302
 303 enum arm_smmu_arch_version {
 304         ARM_SMMU_V1,
 305         ARM_SMMU_V1_64K,
 306         ARM_SMMU_V2,
 307 };
 308
 309 enum arm_smmu_implementation {
 310         GENERIC_SMMU,
 311         ARM_MMU500,
 312         CAVIUM_SMMUV2,
 313 };
 314
 315 /* Until ACPICA headers cover IORT rev. C */
 316 #ifndef ACPI_IORT_SMMU_CORELINK_MMU401
 317 #define ACPI_IORT_SMMU_CORELINK_MMU401  0x4
 318 #endif
 319 #ifndef ACPI_IORT_SMMU_CAVIUM_THUNDERX
 320 #define ACPI_IORT_SMMU_CAVIUM_THUNDERX  0x5
 321 #endif
 322
 323 struct arm_smmu_s2cr {
 324         struct iommu_group              *group;
 325         int                             count;
 326         enum arm_smmu_s2cr_type         type;
 327         enum arm_smmu_s2cr_privcfg      privcfg;
 328         u8                              cbndx;
 329 };
 330
 331 #define s2cr_init_val (struct arm_smmu_s2cr){                           \
 332         .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,    \
 333 }
 334
 335 struct arm_smmu_smr {
 336         u16                             mask;
 337         u16                             id;
 338         bool                            valid;
 339 };
 340
 341 struct arm_smmu_master_cfg {
 342         struct arm_smmu_device          *smmu;
 343         s16                             smendx[];
 344 };
 345 #define INVALID_SMENDX                  -1
 346 #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
 347 #define fwspec_smmu(fw)  (__fwspec_cfg(fw)->smmu)
 348 #define fwspec_smendx(fw, i) \
 349         (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i])
 350 #define for_each_cfg_sme(fw, i, idx) \
 351         for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i)
 352
 353 struct arm_smmu_device {
 354         struct device                   *dev;
 355
 356         void __iomem                    *base;
 357         void __iomem                    *cb_base;
 358         unsigned long                   pgshift;
 359
 360 #define ARM_SMMU_FEAT_COHERENT_WALK     (1 << 0)
 361 #define ARM_SMMU_FEAT_STREAM_MATCH      (1 << 1)
 362 #define ARM_SMMU_FEAT_TRANS_S1          (1 << 2)
 363 #define ARM_SMMU_FEAT_TRANS_S2          (1 << 3)
 364 #define ARM_SMMU_FEAT_TRANS_NESTED      (1 << 4)
 365 #define ARM_SMMU_FEAT_TRANS_OPS         (1 << 5)
 366 #define ARM_SMMU_FEAT_VMID16            (1 << 6)
 367 #define ARM_SMMU_FEAT_FMT_AARCH64_4K    (1 << 7)
 368 #define ARM_SMMU_FEAT_FMT_AARCH64_16K   (1 << 8)
 369 #define ARM_SMMU_FEAT_FMT_AARCH64_64K   (1 << 9)
 370 #define ARM_SMMU_FEAT_FMT_AARCH32_L     (1 << 10)
 371 #define ARM_SMMU_FEAT_FMT_AARCH32_S     (1 << 11)
 372 #define ARM_SMMU_FEAT_EXIDS             (1 << 12)
 373         u32                             features;
 374
 375 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
 376         u32                             options;
 377         enum arm_smmu_arch_version      version;
 378         enum arm_smmu_implementation    model;
 379
 380         u32                             num_context_banks;
 381         u32                             num_s2_context_banks;
 382         DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
 383         atomic_t                        irptndx;
 384
 385         u32                             num_mapping_groups;
 386         u16                             streamid_mask;
 387         u16                             smr_mask_mask;
 388         struct arm_smmu_smr             *smrs;
 389         struct arm_smmu_s2cr            *s2crs;
 390         struct mutex                    stream_map_mutex;
 391
 392         unsigned long                   va_size;
 393         unsigned long                   ipa_size;
 394         unsigned long                   pa_size;
 395         unsigned long                   pgsize_bitmap;
 396
 397         u32                             num_global_irqs;
 398         u32                             num_context_irqs;
 399         unsigned int                    *irqs;
 400
 401         u32                             cavium_id_base; /* Specific to Cavium */
 402
 403         /* IOMMU core code handle */
 404         struct iommu_device             iommu;
 405 };
 406
 407 enum arm_smmu_context_fmt {
 408         ARM_SMMU_CTX_FMT_NONE,
 409         ARM_SMMU_CTX_FMT_AARCH64,
 410         ARM_SMMU_CTX_FMT_AARCH32_L,
 411         ARM_SMMU_CTX_FMT_AARCH32_S,
 412 };
 413
 414 struct arm_smmu_cfg {
 415         u8                              cbndx;
 416         u8                              irptndx;
 417         union {
 418                 u16                     asid;
 419                 u16                     vmid;
 420         };
 421         u32                             cbar;
 422         enum arm_smmu_context_fmt       fmt;
 423 };
 424 #define INVALID_IRPTNDX                 0xff
 425
 426 enum arm_smmu_domain_stage {
 427         ARM_SMMU_DOMAIN_S1 = 0,
 428         ARM_SMMU_DOMAIN_S2,
 429         ARM_SMMU_DOMAIN_NESTED,
 430         ARM_SMMU_DOMAIN_BYPASS,
 431 };
 432
 433 struct arm_smmu_domain {
 434         struct arm_smmu_device          *smmu;
 435         struct io_pgtable_ops           *pgtbl_ops;
 436         spinlock_t                      pgtbl_lock;
 437         struct arm_smmu_cfg             cfg;
 438         enum arm_smmu_domain_stage      stage;
 439         struct mutex                    init_mutex; /* Protects smmu pointer */
 440         struct iommu_domain             domain;
 441 };
 442
 443 struct arm_smmu_option_prop {
 444         u32 opt;
 445         const char *prop;
 446 };
 447
 448 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
 449
 450 static bool using_legacy_binding, using_generic_binding;
 451
 452 static struct arm_smmu_option_prop arm_smmu_options[] = {
 453         { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
 454         { 0, NULL},
 455 };
 456
 457 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
 458 {
 459         return container_of(dom, struct arm_smmu_domain, domain);
 460 }
 461
 462 static void parse_driver_options(struct arm_smmu_device *smmu)
 463 {
 464         int i = 0;
 465
 466         do {
 467                 if (of_property_read_bool(smmu->dev->of_node,
 468                                                 arm_smmu_options[i].prop)) {
 469                         smmu->options |= arm_smmu_options[i].opt;
 470                         dev_notice(smmu->dev, "option %s\n",
 471                                 arm_smmu_options[i].prop);
 472                 }
 473         } while (arm_smmu_options[++i].opt);
 474 }
 475
 476 static struct device_node *dev_get_dev_node(struct device *dev)
 477 {
 478         if (dev_is_pci(dev)) {
 479                 struct pci_bus *bus = to_pci_dev(dev)->bus;
 480
 481                 while (!pci_is_root_bus(bus))
 482                         bus = bus->parent;
 483                 return of_node_get(bus->bridge->parent->of_node);
 484         }
 485
 486         return of_node_get(dev->of_node);
 487 }
 488
 489 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
 490 {
 491         *((__be32 *)data) = cpu_to_be32(alias);
 492         return 0; /* Continue walking */
 493 }
 494
 495 static int __find_legacy_master_phandle(struct device *dev, void *data)
 496 {
 497         struct of_phandle_iterator *it = *(void **)data;
 498         struct device_node *np = it->node;
 499         int err;
 500
 501         of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
 502                             "#stream-id-cells", 0)
 503                 if (it->node == np) {
 504                         *(void **)data = dev;
 505                         return 1;
 506                 }
 507         it->node = np;
 508         return err == -ENOENT ? 0 : err;
 509 }
 510
 511 static struct platform_driver arm_smmu_driver;
 512 static struct iommu_ops arm_smmu_ops;
 513
 514 static int arm_smmu_register_legacy_master(struct device *dev,
 515                                            struct arm_smmu_device **smmu)
 516 {
 517         struct device *smmu_dev;
 518         struct device_node *np;
 519         struct of_phandle_iterator it;
 520         void *data = &it;
 521         u32 *sids;
 522         __be32 pci_sid;
 523         int err;
 524
 525         np = dev_get_dev_node(dev);
 526         if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
 527                 of_node_put(np);
 528                 return -ENODEV;
 529         }
 530
 531         it.node = np;
 532         err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
 533                                      __find_legacy_master_phandle);
 534         smmu_dev = data;
 535         of_node_put(np);
 536         if (err == 0)
 537                 return -ENODEV;
 538         if (err < 0)
 539                 return err;
 540
 541         if (dev_is_pci(dev)) {
 542                 /* "mmu-masters" assumes Stream ID == Requester ID */
 543                 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
 544                                        &pci_sid);
 545                 it.cur = &pci_sid;
 546                 it.cur_count = 1;
 547         }
 548
 549         err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
 550                                 &arm_smmu_ops);
 551         if (err)
 552                 return err;
 553
 554         sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
 555         if (!sids)
 556                 return -ENOMEM;
 557
 558         *smmu = dev_get_drvdata(smmu_dev);
 559         of_phandle_iterator_args(&it, sids, it.cur_count);
 560         err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
 561         kfree(sids);
 562         return err;
 563 }
 564
 565 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
 566 {
 567         int idx;
 568
 569         do {
 570                 idx = find_next_zero_bit(map, end, start);
 571                 if (idx == end)
 572                         return -ENOSPC;
 573         } while (test_and_set_bit(idx, map));
 574
 575         return idx;
 576 }
 577
 578 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
 579 {
 580         clear_bit(idx, map);
 581 }
 582
 583 /* Wait for any pending TLB invalidations to complete */
 584 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu,
 585                                 void __iomem *sync, void __iomem *status)
 586 {
 587         unsigned int spin_cnt, delay;
 588
 589         writel_relaxed(0, sync);
 590         for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
 591                 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
 592                         if (!(readl_relaxed(status) & sTLBGSTATUS_GSACTIVE))
 593                                 return;
 594                         cpu_relax();
 595                 }
 596                 udelay(delay);
 597         }
 598         dev_err_ratelimited(smmu->dev,
 599                             "TLB sync timed out -- SMMU may be deadlocked\n");
 600 }
 601
 602 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
 603 {
 604         void __iomem *base = ARM_SMMU_GR0(smmu);
 605
 606         __arm_smmu_tlb_sync(smmu, base + ARM_SMMU_GR0_sTLBGSYNC,
 607                             base + ARM_SMMU_GR0_sTLBGSTATUS);
 608 }
 609
 610 static void arm_smmu_tlb_sync_context(void *cookie)
 611 {
 612         struct arm_smmu_domain *smmu_domain = cookie;
 613         struct arm_smmu_device *smmu = smmu_domain->smmu;
 614         void __iomem *base = ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx);
 615
 616         __arm_smmu_tlb_sync(smmu, base + ARM_SMMU_CB_TLBSYNC,
 617                             base + ARM_SMMU_CB_TLBSTATUS);
 618 }
 619
 620 static void arm_smmu_tlb_sync_vmid(void *cookie)
 621 {
 622         struct arm_smmu_domain *smmu_domain = cookie;
 623
 624         arm_smmu_tlb_sync_global(smmu_domain->smmu);
 625 }
 626
 627 static void arm_smmu_tlb_inv_context_s1(void *cookie)
 628 {
 629         struct arm_smmu_domain *smmu_domain = cookie;
 630         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 631         void __iomem *base = ARM_SMMU_CB(smmu_domain->smmu, cfg->cbndx);
 632
 633         writel_relaxed(cfg->asid, base + ARM_SMMU_CB_S1_TLBIASID);
 634         arm_smmu_tlb_sync_context(cookie);
 635 }
 636
 637 static void arm_smmu_tlb_inv_context_s2(void *cookie)
 638 {
 639         struct arm_smmu_domain *smmu_domain = cookie;
 640         struct arm_smmu_device *smmu = smmu_domain->smmu;
 641         void __iomem *base = ARM_SMMU_GR0(smmu);
 642
 643         writel_relaxed(smmu_domain->cfg.vmid, base + ARM_SMMU_GR0_TLBIVMID);
 644         arm_smmu_tlb_sync_global(smmu);
 645 }
 646
 647 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
 648                                           size_t granule, bool leaf, void *cookie)
 649 {
 650         struct arm_smmu_domain *smmu_domain = cookie;
 651         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 652         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 653         void __iomem *reg = ARM_SMMU_CB(smmu_domain->smmu, cfg->cbndx);
 654
 655         if (stage1) {
 656                 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
 657
 658                 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
 659                         iova &= ~12UL;
 660                         iova |= cfg->asid;
 661                         do {
 662                                 writel_relaxed(iova, reg);
 663                                 iova += granule;
 664                         } while (size -= granule);
 665                 } else {
 666                         iova >>= 12;
 667                         iova |= (u64)cfg->asid << 48;
 668                         do {
 669                                 writeq_relaxed(iova, reg);
 670                                 iova += granule >> 12;
 671                         } while (size -= granule);
 672                 }
 673         } else {
 674                 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
 675                               ARM_SMMU_CB_S2_TLBIIPAS2;
 676                 iova >>= 12;
 677                 do {
 678                         smmu_write_atomic_lq(iova, reg);
 679                         iova += granule >> 12;
 680                 } while (size -= granule);
 681         }
 682 }
 683
 684 /*
 685  * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
 686  * almost negligible, but the benefit of getting the first one in as far ahead
 687  * of the sync as possible is significant, hence we don't just make this a
 688  * no-op and set .tlb_sync to arm_smmu_inv_context_s2() as you might think.
 689  */
 690 static void arm_smmu_tlb_inv_vmid_nosync(unsigned long iova, size_t size,
 691                                          size_t granule, bool leaf, void *cookie)
 692 {
 693         struct arm_smmu_domain *smmu_domain = cookie;
 694         void __iomem *base = ARM_SMMU_GR0(smmu_domain->smmu);
 695
 696         writel_relaxed(smmu_domain->cfg.vmid, base + ARM_SMMU_GR0_TLBIVMID);
 697 }
 698
 699 static const struct iommu_gather_ops arm_smmu_s1_tlb_ops = {
 700         .tlb_flush_all  = arm_smmu_tlb_inv_context_s1,
 701         .tlb_add_flush  = arm_smmu_tlb_inv_range_nosync,
 702         .tlb_sync       = arm_smmu_tlb_sync_context,
 703 };
 704
 705 static const struct iommu_gather_ops arm_smmu_s2_tlb_ops_v2 = {
 706         .tlb_flush_all  = arm_smmu_tlb_inv_context_s2,
 707         .tlb_add_flush  = arm_smmu_tlb_inv_range_nosync,
 708         .tlb_sync       = arm_smmu_tlb_sync_context,
 709 };
 710
 711 static const struct iommu_gather_ops arm_smmu_s2_tlb_ops_v1 = {
 712         .tlb_flush_all  = arm_smmu_tlb_inv_context_s2,
 713         .tlb_add_flush  = arm_smmu_tlb_inv_vmid_nosync,
 714         .tlb_sync       = arm_smmu_tlb_sync_vmid,
 715 };
 716
 717 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 718 {
 719         u32 fsr, fsynr;
 720         unsigned long iova;
 721         struct iommu_domain *domain = dev;
 722         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 723         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 724         struct arm_smmu_device *smmu = smmu_domain->smmu;
 725         void __iomem *cb_base;
 726
 727         cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
 728         fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
 729
 730         if (!(fsr & FSR_FAULT))
 731                 return IRQ_NONE;
 732
 733         fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
 734         iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
 735
 736         dev_err_ratelimited(smmu->dev,
 737         "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cb=%d\n",
 738                             fsr, iova, fsynr, cfg->cbndx);
 739
 740         writel(fsr, cb_base + ARM_SMMU_CB_FSR);
 741         return IRQ_HANDLED;
 742 }
 743
 744 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 745 {
 746         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
 747         struct arm_smmu_device *smmu = dev;
 748         void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
 749
 750         gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
 751         gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
 752         gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
 753         gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
 754
 755         if (!gfsr)
 756                 return IRQ_NONE;
 757
 758         dev_err_ratelimited(smmu->dev,
 759                 "Unexpected global fault, this could be serious\n");
 760         dev_err_ratelimited(smmu->dev,
 761                 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
 762                 gfsr, gfsynr0, gfsynr1, gfsynr2);
 763
 764         writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
 765         return IRQ_HANDLED;
 766 }
 767
 768 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 769                                        struct io_pgtable_cfg *pgtbl_cfg)
 770 {
 771         u32 reg, reg2;
 772         u64 reg64;
 773         bool stage1;
 774         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 775         struct arm_smmu_device *smmu = smmu_domain->smmu;
 776         void __iomem *cb_base, *gr1_base;
 777
 778         gr1_base = ARM_SMMU_GR1(smmu);
 779         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 780         cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
 781
 782         if (smmu->version > ARM_SMMU_V1) {
 783                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 784                         reg = CBA2R_RW64_64BIT;
 785                 else
 786                         reg = CBA2R_RW64_32BIT;
 787                 /* 16-bit VMIDs live in CBA2R */
 788                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
 789                         reg |= cfg->vmid << CBA2R_VMID_SHIFT;
 790
 791                 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
 792         }
 793
 794         /* CBAR */
 795         reg = cfg->cbar;
 796         if (smmu->version < ARM_SMMU_V2)
 797                 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
 798
 799         /*
 800          * Use the weakest shareability/memory types, so they are
 801          * overridden by the ttbcr/pte.
 802          */
 803         if (stage1) {
 804                 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
 805                         (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
 806         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
 807                 /* 8-bit VMIDs live in CBAR */
 808                 reg |= cfg->vmid << CBAR_VMID_SHIFT;
 809         }
 810         writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
 811
 812         /*
 813          * TTBCR
 814          * We must write this before the TTBRs, since it determines the
 815          * access behaviour of some fields (in particular, ASID[15:8]).
 816          */
 817         if (stage1) {
 818                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 819                         reg = pgtbl_cfg->arm_v7s_cfg.tcr;
 820                         reg2 = 0;
 821                 } else {
 822                         reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
 823                         reg2 = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
 824                         reg2 |= TTBCR2_SEP_UPSTREAM;
 825                         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 826                                 reg2 |= TTBCR2_AS;
 827                 }
 828                 if (smmu->version > ARM_SMMU_V1)
 829                         writel_relaxed(reg2, cb_base + ARM_SMMU_CB_TTBCR2);
 830         } else {
 831                 reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
 832         }
 833         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
 834
 835         /* TTBRs */
 836         if (stage1) {
 837                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 838                         reg = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
 839                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0);
 840                         reg = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
 841                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1);
 842                         writel_relaxed(cfg->asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
 843                 } else {
 844                         reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
 845                         reg64 |= (u64)cfg->asid << TTBRn_ASID_SHIFT;
 846                         writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 847                         reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
 848                         reg64 |= (u64)cfg->asid << TTBRn_ASID_SHIFT;
 849                         writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1);
 850                 }
 851         } else {
 852                 reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
 853                 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 854         }
 855
 856         /* MAIRs (stage-1 only) */
 857         if (stage1) {
 858                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 859                         reg = pgtbl_cfg->arm_v7s_cfg.prrr;
 860                         reg2 = pgtbl_cfg->arm_v7s_cfg.nmrr;
 861                 } else {
 862                         reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
 863                         reg2 = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
 864                 }
 865                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
 866                 writel_relaxed(reg2, cb_base + ARM_SMMU_CB_S1_MAIR1);
 867         }
 868
 869         /* SCTLR */
 870         reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
 871         if (stage1)
 872                 reg |= SCTLR_S1_ASIDPNE;
 873 #ifdef __BIG_ENDIAN
 874         reg |= SCTLR_E;
 875 #endif
 876         writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
 877 }
 878
 879 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 880                                         struct arm_smmu_device *smmu)
 881 {
 882         int irq, start, ret = 0;
 883         unsigned long ias, oas;
 884         struct io_pgtable_ops *pgtbl_ops;
 885         struct io_pgtable_cfg pgtbl_cfg;
 886         enum io_pgtable_fmt fmt;
 887         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 888         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 889         const struct iommu_gather_ops *tlb_ops;
 890
 891         mutex_lock(&smmu_domain->init_mutex);
 892         if (smmu_domain->smmu)
 893                 goto out_unlock;
 894
 895         if (domain->type == IOMMU_DOMAIN_IDENTITY) {
 896                 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
 897                 smmu_domain->smmu = smmu;
 898                 goto out_unlock;
 899         }
 900
 901         /*
 902          * Mapping the requested stage onto what we support is surprisingly
 903          * complicated, mainly because the spec allows S1+S2 SMMUs without
 904          * support for nested translation. That means we end up with the
 905          * following table:
 906          *
 907          * Requested        Supported        Actual
 908          *     S1               N              S1
 909          *     S1             S1+S2            S1
 910          *     S1               S2             S2
 911          *     S1               S1             S1
 912          *     N                N              N
 913          *     N              S1+S2            S2
 914          *     N                S2             S2
 915          *     N                S1             S1
 916          *
 917          * Note that you can't actually request stage-2 mappings.
 918          */
 919         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
 920                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 921         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 922                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 923
 924         /*
 925          * Choosing a suitable context format is even more fiddly. Until we
 926          * grow some way for the caller to express a preference, and/or move
 927          * the decision into the io-pgtable code where it arguably belongs,
 928          * just aim for the closest thing to the rest of the system, and hope
 929          * that the hardware isn't esoteric enough that we can't assume AArch64
 930          * support to be a superset of AArch32 support...
 931          */
 932         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
 933                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
 934         if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
 935             !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
 936             (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
 937             (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
 938                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
 939         if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
 940             (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
 941                                ARM_SMMU_FEAT_FMT_AARCH64_16K |
 942                                ARM_SMMU_FEAT_FMT_AARCH64_4K)))
 943                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
 944
 945         if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
 946                 ret = -EINVAL;
 947                 goto out_unlock;
 948         }
 949
 950         switch (smmu_domain->stage) {
 951         case ARM_SMMU_DOMAIN_S1:
 952                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
 953                 start = smmu->num_s2_context_banks;
 954                 ias = smmu->va_size;
 955                 oas = smmu->ipa_size;
 956                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 957                         fmt = ARM_64_LPAE_S1;
 958                 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
 959                         fmt = ARM_32_LPAE_S1;
 960                         ias = min(ias, 32UL);
 961                         oas = min(oas, 40UL);
 962                 } else {
 963                         fmt = ARM_V7S;
 964                         ias = min(ias, 32UL);
 965                         oas = min(oas, 32UL);
 966                 }
 967                 tlb_ops = &arm_smmu_s1_tlb_ops;
 968                 break;
 969         case ARM_SMMU_DOMAIN_NESTED:
 970                 /*
 971                  * We will likely want to change this if/when KVM gets
 972                  * involved.
 973                  */
 974         case ARM_SMMU_DOMAIN_S2:
 975                 cfg->cbar = CBAR_TYPE_S2_TRANS;
 976                 start = 0;
 977                 ias = smmu->ipa_size;
 978                 oas = smmu->pa_size;
 979                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 980                         fmt = ARM_64_LPAE_S2;
 981                 } else {
 982                         fmt = ARM_32_LPAE_S2;
 983                         ias = min(ias, 40UL);
 984                         oas = min(oas, 40UL);
 985                 }
 986                 if (smmu->version == ARM_SMMU_V2)
 987                         tlb_ops = &arm_smmu_s2_tlb_ops_v2;
 988                 else
 989                         tlb_ops = &arm_smmu_s2_tlb_ops_v1;
 990                 break;
 991         default:
 992                 ret = -EINVAL;
 993                 goto out_unlock;
 994         }
 995         ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
 996                                       smmu->num_context_banks);
 997         if (ret < 0)
 998                 goto out_unlock;
 999
1000         cfg->cbndx = ret;
1001         if (smmu->version < ARM_SMMU_V2) {
1002                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
1003                 cfg->irptndx %= smmu->num_context_irqs;
1004         } else {
1005                 cfg->irptndx = cfg->cbndx;
1006         }
1007
1008         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
1009                 cfg->vmid = cfg->cbndx + 1 + smmu->cavium_id_base;
1010         else
1011                 cfg->asid = cfg->cbndx + smmu->cavium_id_base;
1012
1013         pgtbl_cfg = (struct io_pgtable_cfg) {
1014                 .pgsize_bitmap  = smmu->pgsize_bitmap,
1015                 .ias            = ias,
1016                 .oas            = oas,
1017                 .tlb            = tlb_ops,
1018                 .iommu_dev      = smmu->dev,
1019         };
1020
1021         smmu_domain->smmu = smmu;
1022         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
1023         if (!pgtbl_ops) {
1024                 ret = -ENOMEM;
1025                 goto out_clear_smmu;
1026         }
1027
1028         /* Update the domain's page sizes to reflect the page table format */
1029         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
1030         domain->geometry.aperture_end = (1UL << ias) - 1;
1031         domain->geometry.force_aperture = true;
1032
1033         /* Initialise the context bank with our page table cfg */
1034         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
1035
1036         /*
1037          * Request context fault interrupt. Do this last to avoid the
1038          * handler seeing a half-initialised domain state.
1039          */
1040         irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
1041         ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
1042                                IRQF_SHARED, "arm-smmu-context-fault", domain);
1043         if (ret < 0) {
1044                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
1045                         cfg->irptndx, irq);
1046                 cfg->irptndx = INVALID_IRPTNDX;
1047         }
1048
1049         mutex_unlock(&smmu_domain->init_mutex);
1050
1051         /* Publish page table ops for map/unmap */
1052         smmu_domain->pgtbl_ops = pgtbl_ops;
1053         return 0;
1054
1055 out_clear_smmu:
1056         smmu_domain->smmu = NULL;
1057 out_unlock:
1058         mutex_unlock(&smmu_domain->init_mutex);
1059         return ret;
1060 }
1061
1062 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
1063 {
1064         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1065         struct arm_smmu_device *smmu = smmu_domain->smmu;
1066         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1067         void __iomem *cb_base;
1068         int irq;
1069
1070         if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
1071                 return;
1072
1073         /*
1074          * Disable the context bank and free the page tables before freeing
1075          * it.
1076          */
1077         cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
1078         writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1079
1080         if (cfg->irptndx != INVALID_IRPTNDX) {
1081                 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
1082                 devm_free_irq(smmu->dev, irq, domain);
1083         }
1084
1085         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
1086         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
1087 }
1088
1089 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1090 {
1091         struct arm_smmu_domain *smmu_domain;
1092
1093         if (type != IOMMU_DOMAIN_UNMANAGED &&
1094             type != IOMMU_DOMAIN_DMA &&
1095             type != IOMMU_DOMAIN_IDENTITY)
1096                 return NULL;
1097         /*
1098          * Allocate the domain and initialise some of its data structures.
1099          * We can't really do anything meaningful until we've added a
1100          * master.
1101          */
1102         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1103         if (!smmu_domain)
1104                 return NULL;
1105
1106         if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
1107             iommu_get_dma_cookie(&smmu_domain->domain))) {
1108                 kfree(smmu_domain);
1109                 return NULL;
1110         }
1111
1112         mutex_init(&smmu_domain->init_mutex);
1113         spin_lock_init(&smmu_domain->pgtbl_lock);
1114
1115         return &smmu_domain->domain;
1116 }
1117
1118 static void arm_smmu_domain_free(struct iommu_domain *domain)
1119 {
1120         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1121
1122         /*
1123          * Free the domain resources. We assume that all devices have
1124          * already been detached.
1125          */
1126         iommu_put_dma_cookie(domain);
1127         arm_smmu_destroy_domain_context(domain);
1128         kfree(smmu_domain);
1129 }
1130
1131 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
1132 {
1133         struct arm_smmu_smr *smr = smmu->smrs + idx;
1134         u32 reg = smr->id << SMR_ID_SHIFT | smr->mask << SMR_MASK_SHIFT;
1135
1136         if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
1137                 reg |= SMR_VALID;
1138         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx));
1139 }
1140
1141 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
1142 {
1143         struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
1144         u32 reg = (s2cr->type & S2CR_TYPE_MASK) << S2CR_TYPE_SHIFT |
1145                   (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT |
1146                   (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT;
1147
1148         if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
1149             smmu->smrs[idx].valid)
1150                 reg |= S2CR_EXIDVALID;
1151         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx));
1152 }
1153
1154 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
1155 {
1156         arm_smmu_write_s2cr(smmu, idx);
1157         if (smmu->smrs)
1158                 arm_smmu_write_smr(smmu, idx);
1159 }
1160
1161 /*
1162  * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
1163  * should be called after sCR0 is written.
1164  */
1165 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
1166 {
1167         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1168         u32 smr;
1169
1170         if (!smmu->smrs)
1171                 return;
1172
1173         /*
1174          * SMR.ID bits may not be preserved if the corresponding MASK
1175          * bits are set, so check each one separately. We can reject
1176          * masters later if they try to claim IDs outside these masks.
1177          */
1178         smr = smmu->streamid_mask << SMR_ID_SHIFT;
1179         writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1180         smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1181         smmu->streamid_mask = smr >> SMR_ID_SHIFT;
1182
1183         smr = smmu->streamid_mask << SMR_MASK_SHIFT;
1184         writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1185         smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1186         smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT;
1187 }
1188
1189 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1190 {
1191         struct arm_smmu_smr *smrs = smmu->smrs;
1192         int i, free_idx = -ENOSPC;
1193
1194         /* Stream indexing is blissfully easy */
1195         if (!smrs)
1196                 return id;
1197
1198         /* Validating SMRs is... less so */
1199         for (i = 0; i < smmu->num_mapping_groups; ++i) {
1200                 if (!smrs[i].valid) {
1201                         /*
1202                          * Note the first free entry we come across, which
1203                          * we'll claim in the end if nothing else matches.
1204                          */
1205                         if (free_idx < 0)
1206                                 free_idx = i;
1207                         continue;
1208                 }
1209                 /*
1210                  * If the new entry is _entirely_ matched by an existing entry,
1211                  * then reuse that, with the guarantee that there also cannot
1212                  * be any subsequent conflicting entries. In normal use we'd
1213                  * expect simply identical entries for this case, but there's
1214                  * no harm in accommodating the generalisation.
1215                  */
1216                 if ((mask & smrs[i].mask) == mask &&
1217                     !((id ^ smrs[i].id) & ~smrs[i].mask))
1218                         return i;
1219                 /*
1220                  * If the new entry has any other overlap with an existing one,
1221                  * though, then there always exists at least one stream ID
1222                  * which would cause a conflict, and we can't allow that risk.
1223                  */
1224                 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1225                         return -EINVAL;
1226         }
1227
1228         return free_idx;
1229 }
1230
1231 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1232 {
1233         if (--smmu->s2crs[idx].count)
1234                 return false;
1235
1236         smmu->s2crs[idx] = s2cr_init_val;
1237         if (smmu->smrs)
1238                 smmu->smrs[idx].valid = false;
1239
1240         return true;
1241 }
1242
1243 static int arm_smmu_master_alloc_smes(struct device *dev)
1244 {
1245         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1246         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1247         struct arm_smmu_device *smmu = cfg->smmu;
1248         struct arm_smmu_smr *smrs = smmu->smrs;
1249         struct iommu_group *group;
1250         int i, idx, ret;
1251
1252         mutex_lock(&smmu->stream_map_mutex);
1253         /* Figure out a viable stream map entry allocation */
1254         for_each_cfg_sme(fwspec, i, idx) {
1255                 u16 sid = fwspec->ids[i];
1256                 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1257
1258                 if (idx != INVALID_SMENDX) {
1259                         ret = -EEXIST;
1260                         goto out_err;
1261                 }
1262
1263                 ret = arm_smmu_find_sme(smmu, sid, mask);
1264                 if (ret < 0)
1265                         goto out_err;
1266
1267                 idx = ret;
1268                 if (smrs && smmu->s2crs[idx].count == 0) {
1269                         smrs[idx].id = sid;
1270                         smrs[idx].mask = mask;
1271                         smrs[idx].valid = true;
1272                 }
1273                 smmu->s2crs[idx].count++;
1274                 cfg->smendx[i] = (s16)idx;
1275         }
1276
1277         group = iommu_group_get_for_dev(dev);
1278         if (!group)
1279                 group = ERR_PTR(-ENOMEM);
1280         if (IS_ERR(group)) {
1281                 ret = PTR_ERR(group);
1282                 goto out_err;
1283         }
1284         iommu_group_put(group);
1285
1286         /* It worked! Now, poke the actual hardware */
1287         for_each_cfg_sme(fwspec, i, idx) {
1288                 arm_smmu_write_sme(smmu, idx);
1289                 smmu->s2crs[idx].group = group;
1290         }
1291
1292         mutex_unlock(&smmu->stream_map_mutex);
1293         return 0;
1294
1295 out_err:
1296         while (i--) {
1297                 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1298                 cfg->smendx[i] = INVALID_SMENDX;
1299         }
1300         mutex_unlock(&smmu->stream_map_mutex);
1301         return ret;
1302 }
1303
1304 static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
1305 {
1306         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1307         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1308         int i, idx;
1309
1310         mutex_lock(&smmu->stream_map_mutex);
1311         for_each_cfg_sme(fwspec, i, idx) {
1312                 if (arm_smmu_free_sme(smmu, idx))
1313                         arm_smmu_write_sme(smmu, idx);
1314                 cfg->smendx[i] = INVALID_SMENDX;
1315         }
1316         mutex_unlock(&smmu->stream_map_mutex);
1317 }
1318
1319 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1320                                       struct iommu_fwspec *fwspec)
1321 {
1322         struct arm_smmu_device *smmu = smmu_domain->smmu;
1323         struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1324         u8 cbndx = smmu_domain->cfg.cbndx;
1325         enum arm_smmu_s2cr_type type;
1326         int i, idx;
1327
1328         if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
1329                 type = S2CR_TYPE_BYPASS;
1330         else
1331                 type = S2CR_TYPE_TRANS;
1332
1333         for_each_cfg_sme(fwspec, i, idx) {
1334                 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1335                         continue;
1336
1337                 s2cr[idx].type = type;
1338                 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1339                 s2cr[idx].cbndx = cbndx;
1340                 arm_smmu_write_s2cr(smmu, idx);
1341         }
1342         return 0;
1343 }
1344
1345 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1346 {
1347         int ret;
1348         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1349         struct arm_smmu_device *smmu;
1350         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1351
1352         if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1353                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1354                 return -ENXIO;
1355         }
1356
1357         /*
1358          * FIXME: The arch/arm DMA API code tries to attach devices to its own
1359          * domains between of_xlate() and add_device() - we have no way to cope
1360          * with that, so until ARM gets converted to rely on groups and default
1361          * domains, just say no (but more politely than by dereferencing NULL).
1362          * This should be at least a WARN_ON once that's sorted.
1363          */
1364         if (!fwspec->iommu_priv)
1365                 return -ENODEV;
1366
1367         smmu = fwspec_smmu(fwspec);
1368         /* Ensure that the domain is finalised */
1369         ret = arm_smmu_init_domain_context(domain, smmu);
1370         if (ret < 0)
1371                 return ret;
1372
1373         /*
1374          * Sanity check the domain. We don't support domains across
1375          * different SMMUs.
1376          */
1377         if (smmu_domain->smmu != smmu) {
1378                 dev_err(dev,
1379                         "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1380                         dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1381                 return -EINVAL;
1382         }
1383
1384         /* Looks ok, so add the device to the domain */
1385         return arm_smmu_domain_add_master(smmu_domain, fwspec);
1386 }
1387
1388 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1389                         phys_addr_t paddr, size_t size, int prot)
1390 {
1391         int ret;
1392         unsigned long flags;
1393         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1394         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1395
1396         if (!ops)
1397                 return -ENODEV;
1398
1399         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1400         ret = ops->map(ops, iova, paddr, size, prot);
1401         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1402         return ret;
1403 }
1404
1405 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1406                              size_t size)
1407 {
1408         size_t ret;
1409         unsigned long flags;
1410         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1411         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1412
1413         if (!ops)
1414                 return 0;
1415
1416         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1417         ret = ops->unmap(ops, iova, size);
1418         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1419         return ret;
1420 }
1421
1422 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1423                                               dma_addr_t iova)
1424 {
1425         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1426         struct arm_smmu_device *smmu = smmu_domain->smmu;
1427         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1428         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1429         struct device *dev = smmu->dev;
1430         void __iomem *cb_base;
1431         u32 tmp;
1432         u64 phys;
1433         unsigned long va;
1434
1435         cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
1436
1437         /* ATS1 registers can only be written atomically */
1438         va = iova & ~0xfffUL;
1439         if (smmu->version == ARM_SMMU_V2)
1440                 smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1441         else /* Register is only 32-bit in v1 */
1442                 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1443
1444         if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1445                                       !(tmp & ATSR_ACTIVE), 5, 50)) {
1446                 dev_err(dev,
1447                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1448                         &iova);
1449                 return ops->iova_to_phys(ops, iova);
1450         }
1451
1452         phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
1453         if (phys & CB_PAR_F) {
1454                 dev_err(dev, "translation fault!\n");
1455                 dev_err(dev, "PAR = 0x%llx\n", phys);
1456                 return 0;
1457         }
1458
1459         return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1460 }
1461
1462 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1463                                         dma_addr_t iova)
1464 {
1465         phys_addr_t ret;
1466         unsigned long flags;
1467         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1468         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1469
1470         if (domain->type == IOMMU_DOMAIN_IDENTITY)
1471                 return iova;
1472
1473         if (!ops)
1474                 return 0;
1475
1476         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1477         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1478                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1479                 ret = arm_smmu_iova_to_phys_hard(domain, iova);
1480         } else {
1481                 ret = ops->iova_to_phys(ops, iova);
1482         }
1483
1484         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1485
1486         return ret;
1487 }
1488
1489 static bool arm_smmu_capable(enum iommu_cap cap)
1490 {
1491         switch (cap) {
1492         case IOMMU_CAP_CACHE_COHERENCY:
1493                 /*
1494                  * Return true here as the SMMU can always send out coherent
1495                  * requests.
1496                  */
1497                 return true;
1498         case IOMMU_CAP_NOEXEC:
1499                 return true;
1500         default:
1501                 return false;
1502         }
1503 }
1504
1505 static int arm_smmu_match_node(struct device *dev, void *data)
1506 {
1507         return dev->fwnode == data;
1508 }
1509
1510 static
1511 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1512 {
1513         struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
1514                                                 fwnode, arm_smmu_match_node);
1515         put_device(dev);
1516         return dev ? dev_get_drvdata(dev) : NULL;
1517 }
1518
1519 static int arm_smmu_add_device(struct device *dev)
1520 {
1521         struct arm_smmu_device *smmu;
1522         struct arm_smmu_master_cfg *cfg;
1523         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1524         int i, ret;
1525
1526         if (using_legacy_binding) {
1527                 ret = arm_smmu_register_legacy_master(dev, &smmu);
1528                 fwspec = dev->iommu_fwspec;
1529                 if (ret)
1530                         goto out_free;
1531         } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1532                 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1533         } else {
1534                 return -ENODEV;
1535         }
1536
1537         ret = -EINVAL;
1538         for (i = 0; i < fwspec->num_ids; i++) {
1539                 u16 sid = fwspec->ids[i];
1540                 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1541
1542                 if (sid & ~smmu->streamid_mask) {
1543                         dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1544                                 sid, smmu->streamid_mask);
1545                         goto out_free;
1546                 }
1547                 if (mask & ~smmu->smr_mask_mask) {
1548                         dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1549                                 mask, smmu->smr_mask_mask);
1550                         goto out_free;
1551                 }
1552         }
1553
1554         ret = -ENOMEM;
1555         cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1556                       GFP_KERNEL);
1557         if (!cfg)
1558                 goto out_free;
1559
1560         cfg->smmu = smmu;
1561         fwspec->iommu_priv = cfg;
1562         while (i--)
1563                 cfg->smendx[i] = INVALID_SMENDX;
1564
1565         ret = arm_smmu_master_alloc_smes(dev);
1566         if (ret)
1567                 goto out_free;
1568
1569         iommu_device_link(&smmu->iommu, dev);
1570
1571         return 0;
1572
1573 out_free:
1574         if (fwspec)
1575                 kfree(fwspec->iommu_priv);
1576         iommu_fwspec_free(dev);
1577         return ret;
1578 }
1579
1580 static void arm_smmu_remove_device(struct device *dev)
1581 {
1582         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1583         struct arm_smmu_master_cfg *cfg;
1584         struct arm_smmu_device *smmu;
1585
1586
1587         if (!fwspec || fwspec->ops != &arm_smmu_ops)
1588                 return;
1589
1590         cfg  = fwspec->iommu_priv;
1591         smmu = cfg->smmu;
1592
1593         iommu_device_unlink(&smmu->iommu, dev);
1594         arm_smmu_master_free_smes(fwspec);
1595         iommu_group_remove_device(dev);
1596         kfree(fwspec->iommu_priv);
1597         iommu_fwspec_free(dev);
1598 }
1599
1600 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1601 {
1602         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1603         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1604         struct iommu_group *group = NULL;
1605         int i, idx;
1606
1607         for_each_cfg_sme(fwspec, i, idx) {
1608                 if (group && smmu->s2crs[idx].group &&
1609                     group != smmu->s2crs[idx].group)
1610                         return ERR_PTR(-EINVAL);
1611
1612                 group = smmu->s2crs[idx].group;
1613         }
1614
1615         if (group)
1616                 return iommu_group_ref_get(group);
1617
1618         if (dev_is_pci(dev))
1619                 group = pci_device_group(dev);
1620         else
1621                 group = generic_device_group(dev);
1622
1623         return group;
1624 }
1625
1626 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1627                                     enum iommu_attr attr, void *data)
1628 {
1629         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1630
1631         if (domain->type != IOMMU_DOMAIN_UNMANAGED)
1632                 return -EINVAL;
1633
1634         switch (attr) {
1635         case DOMAIN_ATTR_NESTING:
1636                 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1637                 return 0;
1638         default:
1639                 return -ENODEV;
1640         }
1641 }
1642
1643 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1644                                     enum iommu_attr attr, void *data)
1645 {
1646         int ret = 0;
1647         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1648
1649         if (domain->type != IOMMU_DOMAIN_UNMANAGED)
1650                 return -EINVAL;
1651
1652         mutex_lock(&smmu_domain->init_mutex);
1653
1654         switch (attr) {
1655         case DOMAIN_ATTR_NESTING:
1656                 if (smmu_domain->smmu) {
1657                         ret = -EPERM;
1658                         goto out_unlock;
1659                 }
1660
1661                 if (*(int *)data)
1662                         smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1663                 else
1664                         smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1665
1666                 break;
1667         default:
1668                 ret = -ENODEV;
1669         }
1670
1671 out_unlock:
1672         mutex_unlock(&smmu_domain->init_mutex);
1673         return ret;
1674 }
1675
1676 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1677 {
1678         u32 mask, fwid = 0;
1679
1680         if (args->args_count > 0)
1681                 fwid |= (u16)args->args[0];
1682
1683         if (args->args_count > 1)
1684                 fwid |= (u16)args->args[1] << SMR_MASK_SHIFT;
1685         else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1686                 fwid |= (u16)mask << SMR_MASK_SHIFT;
1687
1688         return iommu_fwspec_add_ids(dev, &fwid, 1);
1689 }
1690
1691 static void arm_smmu_get_resv_regions(struct device *dev,
1692                                       struct list_head *head)
1693 {
1694         struct iommu_resv_region *region;
1695         int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1696
1697         region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1698                                          prot, IOMMU_RESV_SW_MSI);
1699         if (!region)
1700                 return;
1701
1702         list_add_tail(&region->list, head);
1703
1704         iommu_dma_get_resv_regions(dev, head);
1705 }
1706
1707 static void arm_smmu_put_resv_regions(struct device *dev,
1708                                       struct list_head *head)
1709 {
1710         struct iommu_resv_region *entry, *next;
1711
1712         list_for_each_entry_safe(entry, next, head, list)
1713                 kfree(entry);
1714 }
1715
1716 static struct iommu_ops arm_smmu_ops = {
1717         .capable                = arm_smmu_capable,
1718         .domain_alloc           = arm_smmu_domain_alloc,
1719         .domain_free            = arm_smmu_domain_free,
1720         .attach_dev             = arm_smmu_attach_dev,
1721         .map                    = arm_smmu_map,
1722         .unmap                  = arm_smmu_unmap,
1723         .map_sg                 = default_iommu_map_sg,
1724         .iova_to_phys           = arm_smmu_iova_to_phys,
1725         .add_device             = arm_smmu_add_device,
1726         .remove_device          = arm_smmu_remove_device,
1727         .device_group           = arm_smmu_device_group,
1728         .domain_get_attr        = arm_smmu_domain_get_attr,
1729         .domain_set_attr        = arm_smmu_domain_set_attr,
1730         .of_xlate               = arm_smmu_of_xlate,
1731         .get_resv_regions       = arm_smmu_get_resv_regions,
1732         .put_resv_regions       = arm_smmu_put_resv_regions,
1733         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1734 };
1735
1736 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1737 {
1738         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1739         void __iomem *cb_base;
1740         int i;
1741         u32 reg, major;
1742
1743         /* clear global FSR */
1744         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1745         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1746
1747         /*
1748          * Reset stream mapping groups: Initial values mark all SMRn as
1749          * invalid and all S2CRn as bypass unless overridden.
1750          */
1751         for (i = 0; i < smmu->num_mapping_groups; ++i)
1752                 arm_smmu_write_sme(smmu, i);
1753
1754         if (smmu->model == ARM_MMU500) {
1755                 /*
1756                  * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1757                  * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1758                  * bit is only present in MMU-500r2 onwards.
1759                  */
1760                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
1761                 major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
1762                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
1763                 if (major >= 2)
1764                         reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
1765                 /*
1766                  * Allow unmatched Stream IDs to allocate bypass
1767                  * TLB entries for reduced latency.
1768                  */
1769                 reg |= ARM_MMU500_ACR_SMTNMB_TLBEN;
1770                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
1771         }
1772
1773         /* Make sure all context banks are disabled and clear CB_FSR  */
1774         for (i = 0; i < smmu->num_context_banks; ++i) {
1775                 cb_base = ARM_SMMU_CB(smmu, i);
1776                 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1777                 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1778                 /*
1779                  * Disable MMU-500's not-particularly-beneficial next-page
1780                  * prefetcher for the sake of errata #841119 and #826419.
1781                  */
1782                 if (smmu->model == ARM_MMU500) {
1783                         reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
1784                         reg &= ~ARM_MMU500_ACTLR_CPRE;
1785                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
1786                 }
1787         }
1788
1789         /* Invalidate the TLB, just in case */
1790         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1791         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1792
1793         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1794
1795         /* Enable fault reporting */
1796         reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1797
1798         /* Disable TLB broadcasting. */
1799         reg |= (sCR0_VMIDPNE | sCR0_PTM);
1800
1801         /* Enable client access, handling unmatched streams as appropriate */
1802         reg &= ~sCR0_CLIENTPD;
1803         if (disable_bypass)
1804                 reg |= sCR0_USFCFG;
1805         else
1806                 reg &= ~sCR0_USFCFG;
1807
1808         /* Disable forced broadcasting */
1809         reg &= ~sCR0_FB;
1810
1811         /* Don't upgrade barriers */
1812         reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
1813
1814         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1815                 reg |= sCR0_VMID16EN;
1816
1817         if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1818                 reg |= sCR0_EXIDENABLE;
1819
1820         /* Push the button */
1821         arm_smmu_tlb_sync_global(smmu);
1822         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1823 }
1824
1825 static int arm_smmu_id_size_to_bits(int size)
1826 {
1827         switch (size) {
1828         case 0:
1829                 return 32;
1830         case 1:
1831                 return 36;
1832         case 2:
1833                 return 40;
1834         case 3:
1835                 return 42;
1836         case 4:
1837                 return 44;
1838         case 5:
1839         default:
1840                 return 48;
1841         }
1842 }
1843
1844 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1845 {
1846         unsigned long size;
1847         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1848         u32 id;
1849         bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1850         int i;
1851
1852         dev_notice(smmu->dev, "probing hardware configuration...\n");
1853         dev_notice(smmu->dev, "SMMUv%d with:\n",
1854                         smmu->version == ARM_SMMU_V2 ? 2 : 1);
1855
1856         /* ID0 */
1857         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1858
1859         /* Restrict available stages based on module parameter */
1860         if (force_stage == 1)
1861                 id &= ~(ID0_S2TS | ID0_NTS);
1862         else if (force_stage == 2)
1863                 id &= ~(ID0_S1TS | ID0_NTS);
1864
1865         if (id & ID0_S1TS) {
1866                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1867                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1868         }
1869
1870         if (id & ID0_S2TS) {
1871                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1872                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1873         }
1874
1875         if (id & ID0_NTS) {
1876                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1877                 dev_notice(smmu->dev, "\tnested translation\n");
1878         }
1879
1880         if (!(smmu->features &
1881                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1882                 dev_err(smmu->dev, "\tno translation support!\n");
1883                 return -ENODEV;
1884         }
1885
1886         if ((id & ID0_S1TS) &&
1887                 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1888                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1889                 dev_notice(smmu->dev, "\taddress translation ops\n");
1890         }
1891
1892         /*
1893          * In order for DMA API calls to work properly, we must defer to what
1894          * the FW says about coherency, regardless of what the hardware claims.
1895          * Fortunately, this also opens up a workaround for systems where the
1896          * ID register value has ended up configured incorrectly.
1897          */
1898         cttw_reg = !!(id & ID0_CTTW);
1899         if (cttw_fw || cttw_reg)
1900                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1901                            cttw_fw ? "" : "non-");
1902         if (cttw_fw != cttw_reg)
1903                 dev_notice(smmu->dev,
1904                            "\t(IDR0.CTTW overridden by FW configuration)\n");
1905
1906         /* Max. number of entries we have for stream matching/indexing */
1907         if (smmu->version == ARM_SMMU_V2 && id & ID0_EXIDS) {
1908                 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1909                 size = 1 << 16;
1910         } else {
1911                 size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK);
1912         }
1913         smmu->streamid_mask = size - 1;
1914         if (id & ID0_SMS) {
1915                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1916                 size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK;
1917                 if (size == 0) {
1918                         dev_err(smmu->dev,
1919                                 "stream-matching supported, but no SMRs present!\n");
1920                         return -ENODEV;
1921                 }
1922
1923                 /* Zero-initialised to mark as invalid */
1924                 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1925                                           GFP_KERNEL);
1926                 if (!smmu->smrs)
1927                         return -ENOMEM;
1928
1929                 dev_notice(smmu->dev,
1930                            "\tstream matching with %lu register groups", size);
1931         }
1932         /* s2cr->type == 0 means translation, so initialise explicitly */
1933         smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1934                                          GFP_KERNEL);
1935         if (!smmu->s2crs)
1936                 return -ENOMEM;
1937         for (i = 0; i < size; i++)
1938                 smmu->s2crs[i] = s2cr_init_val;
1939
1940         smmu->num_mapping_groups = size;
1941         mutex_init(&smmu->stream_map_mutex);
1942
1943         if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1944                 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1945                 if (!(id & ID0_PTFS_NO_AARCH32S))
1946                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1947         }
1948
1949         /* ID1 */
1950         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1951         smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1952
1953         /* Check for size mismatch of SMMU address space from mapped region */
1954         size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
1955         size <<= smmu->pgshift;
1956         if (smmu->cb_base != gr0_base + size)
1957                 dev_warn(smmu->dev,
1958                         "SMMU address space size (0x%lx) differs from mapped region size (0x%tx)!\n",
1959                         size * 2, (smmu->cb_base - gr0_base) * 2);
1960
1961         smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
1962         smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
1963         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1964                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1965                 return -ENODEV;
1966         }
1967         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1968                    smmu->num_context_banks, smmu->num_s2_context_banks);
1969         /*
1970          * Cavium CN88xx erratum #27704.
1971          * Ensure ASID and VMID allocation is unique across all SMMUs in
1972          * the system.
1973          */
1974         if (smmu->model == CAVIUM_SMMUV2) {
1975                 smmu->cavium_id_base =
1976                         atomic_add_return(smmu->num_context_banks,
1977                                           &cavium_smmu_context_count);
1978                 smmu->cavium_id_base -= smmu->num_context_banks;
1979                 dev_notice(smmu->dev, "\tenabling workaround for Cavium erratum 27704\n");
1980         }
1981
1982         /* ID2 */
1983         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1984         size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
1985         smmu->ipa_size = size;
1986
1987         /* The output mask is also applied for bypass */
1988         size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
1989         smmu->pa_size = size;
1990
1991         if (id & ID2_VMID16)
1992                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1993
1994         /*
1995          * What the page table walker can address actually depends on which
1996          * descriptor format is in use, but since a) we don't know that yet,
1997          * and b) it can vary per context bank, this will have to do...
1998          */
1999         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
2000                 dev_warn(smmu->dev,
2001                          "failed to set DMA mask for table walker\n");
2002
2003         if (smmu->version < ARM_SMMU_V2) {
2004                 smmu->va_size = smmu->ipa_size;
2005                 if (smmu->version == ARM_SMMU_V1_64K)
2006                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
2007         } else {
2008                 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
2009                 smmu->va_size = arm_smmu_id_size_to_bits(size);
2010                 if (id & ID2_PTFS_4K)
2011                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
2012                 if (id & ID2_PTFS_16K)
2013                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
2014                 if (id & ID2_PTFS_64K)
2015                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
2016         }
2017
2018         /* Now we've corralled the various formats, what'll it do? */
2019         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
2020                 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
2021         if (smmu->features &
2022             (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
2023                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
2024         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
2025                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
2026         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
2027                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
2028
2029         if (arm_smmu_ops.pgsize_bitmap == -1UL)
2030                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
2031         else
2032                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
2033         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
2034                    smmu->pgsize_bitmap);
2035
2036
2037         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
2038                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
2039                            smmu->va_size, smmu->ipa_size);
2040
2041         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
2042                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
2043                            smmu->ipa_size, smmu->pa_size);
2044
2045         return 0;
2046 }
2047
2048 struct arm_smmu_match_data {
2049         enum arm_smmu_arch_version version;
2050         enum arm_smmu_implementation model;
2051 };
2052
2053 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
2054 static struct arm_smmu_match_data name = { .version = ver, .model = imp }
2055
2056 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
2057 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
2058 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
2059 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
2060 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
2061
2062 static const struct of_device_id arm_smmu_of_match[] = {
2063         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
2064         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
2065         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
2066         { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
2067         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
2068         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
2069         { },
2070 };
2071 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
2072
2073 #ifdef CONFIG_ACPI
2074 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
2075 {
2076         int ret = 0;
2077
2078         switch (model) {
2079         case ACPI_IORT_SMMU_V1:
2080         case ACPI_IORT_SMMU_CORELINK_MMU400:
2081                 smmu->version = ARM_SMMU_V1;
2082                 smmu->model = GENERIC_SMMU;
2083                 break;
2084         case ACPI_IORT_SMMU_CORELINK_MMU401:
2085                 smmu->version = ARM_SMMU_V1_64K;
2086                 smmu->model = GENERIC_SMMU;
2087                 break;
2088         case ACPI_IORT_SMMU_V2:
2089                 smmu->version = ARM_SMMU_V2;
2090                 smmu->model = GENERIC_SMMU;
2091                 break;
2092         case ACPI_IORT_SMMU_CORELINK_MMU500:
2093                 smmu->version = ARM_SMMU_V2;
2094                 smmu->model = ARM_MMU500;
2095                 break;
2096         case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
2097                 smmu->version = ARM_SMMU_V2;
2098                 smmu->model = CAVIUM_SMMUV2;
2099                 break;
2100         default:
2101                 ret = -ENODEV;
2102         }
2103
2104         return ret;
2105 }
2106
2107 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2108                                       struct arm_smmu_device *smmu)
2109 {
2110         struct device *dev = smmu->dev;
2111         struct acpi_iort_node *node =
2112                 *(struct acpi_iort_node **)dev_get_platdata(dev);
2113         struct acpi_iort_smmu *iort_smmu;
2114         int ret;
2115
2116         /* Retrieve SMMU1/2 specific data */
2117         iort_smmu = (struct acpi_iort_smmu *)node->node_data;
2118
2119         ret = acpi_smmu_get_data(iort_smmu->model, smmu);
2120         if (ret < 0)
2121                 return ret;
2122
2123         /* Ignore the configuration access interrupt */
2124         smmu->num_global_irqs = 1;
2125
2126         if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
2127                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2128
2129         return 0;
2130 }
2131 #else
2132 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2133                                              struct arm_smmu_device *smmu)
2134 {
2135         return -ENODEV;
2136 }
2137 #endif
2138
2139 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
2140                                     struct arm_smmu_device *smmu)
2141 {
2142         const struct arm_smmu_match_data *data;
2143         struct device *dev = &pdev->dev;
2144         bool legacy_binding;
2145
2146         if (of_property_read_u32(dev->of_node, "#global-interrupts",
2147                                  &smmu->num_global_irqs)) {
2148                 dev_err(dev, "missing #global-interrupts property\n");
2149                 return -ENODEV;
2150         }
2151
2152         data = of_device_get_match_data(dev);
2153         smmu->version = data->version;
2154         smmu->model = data->model;
2155
2156         parse_driver_options(smmu);
2157
2158         legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2159         if (legacy_binding && !using_generic_binding) {
2160                 if (!using_legacy_binding)
2161                         pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
2162                 using_legacy_binding = true;
2163         } else if (!legacy_binding && !using_legacy_binding) {
2164                 using_generic_binding = true;
2165         } else {
2166                 dev_err(dev, "not probing due to mismatched DT properties\n");
2167                 return -ENODEV;
2168         }
2169
2170         if (of_dma_is_coherent(dev->of_node))
2171                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2172
2173         return 0;
2174 }
2175
2176 static void arm_smmu_bus_init(void)
2177 {
2178         /* Oh, for a proper bus abstraction */
2179         if (!iommu_present(&platform_bus_type))
2180                 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2181 #ifdef CONFIG_ARM_AMBA
2182         if (!iommu_present(&amba_bustype))
2183                 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2184 #endif
2185 #ifdef CONFIG_PCI
2186         if (!iommu_present(&pci_bus_type)) {
2187                 pci_request_acs();
2188                 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2189         }
2190 #endif
2191 }
2192
2193 static int arm_smmu_device_probe(struct platform_device *pdev)
2194 {
2195         struct resource *res;
2196         resource_size_t ioaddr;
2197         struct arm_smmu_device *smmu;
2198         struct device *dev = &pdev->dev;
2199         int num_irqs, i, err;
2200
2201         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2202         if (!smmu) {
2203                 dev_err(dev, "failed to allocate arm_smmu_device\n");
2204                 return -ENOMEM;
2205         }
2206         smmu->dev = dev;
2207
2208         if (dev->of_node)
2209                 err = arm_smmu_device_dt_probe(pdev, smmu);
2210         else
2211                 err = arm_smmu_device_acpi_probe(pdev, smmu);
2212
2213         if (err)
2214                 return err;
2215
2216         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2217         ioaddr = res->start;
2218         smmu->base = devm_ioremap_resource(dev, res);
2219         if (IS_ERR(smmu->base))
2220                 return PTR_ERR(smmu->base);
2221         smmu->cb_base = smmu->base + resource_size(res) / 2;
2222
2223         num_irqs = 0;
2224         while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2225                 num_irqs++;
2226                 if (num_irqs > smmu->num_global_irqs)
2227                         smmu->num_context_irqs++;
2228         }
2229
2230         if (!smmu->num_context_irqs) {
2231                 dev_err(dev, "found %d interrupts but expected at least %d\n",
2232                         num_irqs, smmu->num_global_irqs + 1);
2233                 return -ENODEV;
2234         }
2235
2236         smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs,
2237                                   GFP_KERNEL);
2238         if (!smmu->irqs) {
2239                 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2240                 return -ENOMEM;
2241         }
2242
2243         for (i = 0; i < num_irqs; ++i) {
2244                 int irq = platform_get_irq(pdev, i);
2245
2246                 if (irq < 0) {
2247                         dev_err(dev, "failed to get irq index %d\n", i);
2248                         return -ENODEV;
2249                 }
2250                 smmu->irqs[i] = irq;
2251         }
2252
2253         err = arm_smmu_device_cfg_probe(smmu);
2254         if (err)
2255                 return err;
2256
2257         if (smmu->version == ARM_SMMU_V2 &&
2258             smmu->num_context_banks != smmu->num_context_irqs) {
2259                 dev_err(dev,
2260                         "found only %d context interrupt(s) but %d required\n",
2261                         smmu->num_context_irqs, smmu->num_context_banks);
2262                 return -ENODEV;
2263         }
2264
2265         for (i = 0; i < smmu->num_global_irqs; ++i) {
2266                 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2267                                        arm_smmu_global_fault,
2268                                        IRQF_SHARED,
2269                                        "arm-smmu global fault",
2270                                        smmu);
2271                 if (err) {
2272                         dev_err(dev, "failed to request global IRQ %d (%u)\n",
2273                                 i, smmu->irqs[i]);
2274                         return err;
2275                 }
2276         }
2277
2278         err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2279                                      "smmu.%pa", &ioaddr);
2280         if (err) {
2281                 dev_err(dev, "Failed to register iommu in sysfs\n");
2282                 return err;
2283         }
2284
2285         iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
2286         iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
2287
2288         err = iommu_device_register(&smmu->iommu);
2289         if (err) {
2290                 dev_err(dev, "Failed to register iommu\n");
2291                 return err;
2292         }
2293
2294         platform_set_drvdata(pdev, smmu);
2295         arm_smmu_device_reset(smmu);
2296         arm_smmu_test_smr_masks(smmu);
2297
2298         /*
2299          * For ACPI and generic DT bindings, an SMMU will be probed before
2300          * any device which might need it, so we want the bus ops in place
2301          * ready to handle default domain setup as soon as any SMMU exists.
2302          */
2303         if (!using_legacy_binding)
2304                 arm_smmu_bus_init();
2305
2306         return 0;
2307 }
2308
2309 /*
2310  * With the legacy DT binding in play, though, we have no guarantees about
2311  * probe order, but then we're also not doing default domains, so we can
2312  * delay setting bus ops until we're sure every possible SMMU is ready,
2313  * and that way ensure that no add_device() calls get missed.
2314  */
2315 static int arm_smmu_legacy_bus_init(void)
2316 {
2317         if (using_legacy_binding)
2318                 arm_smmu_bus_init();
2319         return 0;
2320 }
2321 device_initcall_sync(arm_smmu_legacy_bus_init);
2322
2323 static int arm_smmu_device_remove(struct platform_device *pdev)
2324 {
2325         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2326
2327         if (!smmu)
2328                 return -ENODEV;
2329
2330         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2331                 dev_err(&pdev->dev, "removing device with active domains!\n");
2332
2333         /* Turn the thing off */
2334         writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
2335         return 0;
2336 }
2337
2338 static struct platform_driver arm_smmu_driver = {
2339         .driver = {
2340                 .name           = "arm-smmu",
2341                 .of_match_table = of_match_ptr(arm_smmu_of_match),
2342         },
2343         .probe  = arm_smmu_device_probe,
2344         .remove = arm_smmu_device_remove,
2345 };
2346 module_platform_driver(arm_smmu_driver);
2347
2348 IOMMU_OF_DECLARE(arm_smmuv1, "arm,smmu-v1", NULL);
2349 IOMMU_OF_DECLARE(arm_smmuv2, "arm,smmu-v2", NULL);
2350 IOMMU_OF_DECLARE(arm_mmu400, "arm,mmu-400", NULL);
2351 IOMMU_OF_DECLARE(arm_mmu401, "arm,mmu-401", NULL);
2352 IOMMU_OF_DECLARE(arm_mmu500, "arm,mmu-500", NULL);
2353 IOMMU_OF_DECLARE(cavium_smmuv2, "cavium,smmu-v2", NULL);
2354
2355 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2356 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2357 MODULE_LICENSE("GPL v2");