drivers/iommu/arm-smmu.c

   1 /*
   2  * IOMMU API for ARM architected SMMU implementations.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  16  *
  17  * Copyright (C) 2013 ARM Limited
  18  *
  19  * Author: Will Deacon <will.deacon@arm.com>
  20  *
  21  * This driver currently supports:
  22  *      - SMMUv1 and v2 implementations
  23  *      - Stream-matching and stream-indexing
  24  *      - v7/v8 long-descriptor format
  25  *      - Non-secure access to the SMMU
  26  *      - Context fault reporting
  27  */
  28
  29 #define pr_fmt(fmt) "arm-smmu: " fmt
  30
  31 #include <linux/atomic.h>
  32 #include <linux/delay.h>
  33 #include <linux/dma-iommu.h>
  34 #include <linux/dma-mapping.h>
  35 #include <linux/err.h>
  36 #include <linux/interrupt.h>
  37 #include <linux/io.h>
  38 #include <linux/io-64-nonatomic-hi-lo.h>
  39 #include <linux/iommu.h>
  40 #include <linux/iopoll.h>
  41 #include <linux/module.h>
  42 #include <linux/of.h>
  43 #include <linux/of_address.h>
  44 #include <linux/of_device.h>
  45 #include <linux/pci.h>
  46 #include <linux/platform_device.h>
  47 #include <linux/slab.h>
  48 #include <linux/spinlock.h>
  49
  50 #include <linux/amba/bus.h>
  51
  52 #include "io-pgtable.h"
  53
  54 /* Maximum number of stream IDs assigned to a single device */
  55 #define MAX_MASTER_STREAMIDS            128
  56
  57 /* Maximum number of context banks per SMMU */
  58 #define ARM_SMMU_MAX_CBS                128
  59
  60 /* SMMU global address space */
  61 #define ARM_SMMU_GR0(smmu)              ((smmu)->base)
  62 #define ARM_SMMU_GR1(smmu)              ((smmu)->base + (1 << (smmu)->pgshift))
  63
  64 /*
  65  * SMMU global address space with conditional offset to access secure
  66  * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
  67  * nsGFSYNR0: 0x450)
  68  */
  69 #define ARM_SMMU_GR0_NS(smmu)                                           \
  70         ((smmu)->base +                                                 \
  71                 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS)       \
  72                         ? 0x400 : 0))
  73
  74 /*
  75  * Some 64-bit registers only make sense to write atomically, but in such
  76  * cases all the data relevant to AArch32 formats lies within the lower word,
  77  * therefore this actually makes more sense than it might first appear.
  78  */
  79 #ifdef CONFIG_64BIT
  80 #define smmu_write_atomic_lq            writeq_relaxed
  81 #else
  82 #define smmu_write_atomic_lq            writel_relaxed
  83 #endif
  84
  85 /* Configuration registers */
  86 #define ARM_SMMU_GR0_sCR0               0x0
  87 #define sCR0_CLIENTPD                   (1 << 0)
  88 #define sCR0_GFRE                       (1 << 1)
  89 #define sCR0_GFIE                       (1 << 2)
  90 #define sCR0_GCFGFRE                    (1 << 4)
  91 #define sCR0_GCFGFIE                    (1 << 5)
  92 #define sCR0_USFCFG                     (1 << 10)
  93 #define sCR0_VMIDPNE                    (1 << 11)
  94 #define sCR0_PTM                        (1 << 12)
  95 #define sCR0_FB                         (1 << 13)
  96 #define sCR0_VMID16EN                   (1 << 31)
  97 #define sCR0_BSU_SHIFT                  14
  98 #define sCR0_BSU_MASK                   0x3
  99
 100 /* Auxiliary Configuration register */
 101 #define ARM_SMMU_GR0_sACR               0x10
 102
 103 /* Identification registers */
 104 #define ARM_SMMU_GR0_ID0                0x20
 105 #define ARM_SMMU_GR0_ID1                0x24
 106 #define ARM_SMMU_GR0_ID2                0x28
 107 #define ARM_SMMU_GR0_ID3                0x2c
 108 #define ARM_SMMU_GR0_ID4                0x30
 109 #define ARM_SMMU_GR0_ID5                0x34
 110 #define ARM_SMMU_GR0_ID6                0x38
 111 #define ARM_SMMU_GR0_ID7                0x3c
 112 #define ARM_SMMU_GR0_sGFSR              0x48
 113 #define ARM_SMMU_GR0_sGFSYNR0           0x50
 114 #define ARM_SMMU_GR0_sGFSYNR1           0x54
 115 #define ARM_SMMU_GR0_sGFSYNR2           0x58
 116
 117 #define ID0_S1TS                        (1 << 30)
 118 #define ID0_S2TS                        (1 << 29)
 119 #define ID0_NTS                         (1 << 28)
 120 #define ID0_SMS                         (1 << 27)
 121 #define ID0_ATOSNS                      (1 << 26)
 122 #define ID0_PTFS_NO_AARCH32             (1 << 25)
 123 #define ID0_PTFS_NO_AARCH32S            (1 << 24)
 124 #define ID0_CTTW                        (1 << 14)
 125 #define ID0_NUMIRPT_SHIFT               16
 126 #define ID0_NUMIRPT_MASK                0xff
 127 #define ID0_NUMSIDB_SHIFT               9
 128 #define ID0_NUMSIDB_MASK                0xf
 129 #define ID0_NUMSMRG_SHIFT               0
 130 #define ID0_NUMSMRG_MASK                0xff
 131
 132 #define ID1_PAGESIZE                    (1 << 31)
 133 #define ID1_NUMPAGENDXB_SHIFT           28
 134 #define ID1_NUMPAGENDXB_MASK            7
 135 #define ID1_NUMS2CB_SHIFT               16
 136 #define ID1_NUMS2CB_MASK                0xff
 137 #define ID1_NUMCB_SHIFT                 0
 138 #define ID1_NUMCB_MASK                  0xff
 139
 140 #define ID2_OAS_SHIFT                   4
 141 #define ID2_OAS_MASK                    0xf
 142 #define ID2_IAS_SHIFT                   0
 143 #define ID2_IAS_MASK                    0xf
 144 #define ID2_UBS_SHIFT                   8
 145 #define ID2_UBS_MASK                    0xf
 146 #define ID2_PTFS_4K                     (1 << 12)
 147 #define ID2_PTFS_16K                    (1 << 13)
 148 #define ID2_PTFS_64K                    (1 << 14)
 149 #define ID2_VMID16                      (1 << 15)
 150
 151 #define ID7_MAJOR_SHIFT                 4
 152 #define ID7_MAJOR_MASK                  0xf
 153
 154 /* Global TLB invalidation */
 155 #define ARM_SMMU_GR0_TLBIVMID           0x64
 156 #define ARM_SMMU_GR0_TLBIALLNSNH        0x68
 157 #define ARM_SMMU_GR0_TLBIALLH           0x6c
 158 #define ARM_SMMU_GR0_sTLBGSYNC          0x70
 159 #define ARM_SMMU_GR0_sTLBGSTATUS        0x74
 160 #define sTLBGSTATUS_GSACTIVE            (1 << 0)
 161 #define TLB_LOOP_TIMEOUT                1000000 /* 1s! */
 162
 163 /* Stream mapping registers */
 164 #define ARM_SMMU_GR0_SMR(n)             (0x800 + ((n) << 2))
 165 #define SMR_VALID                       (1 << 31)
 166 #define SMR_MASK_SHIFT                  16
 167 #define SMR_ID_SHIFT                    0
 168
 169 #define ARM_SMMU_GR0_S2CR(n)            (0xc00 + ((n) << 2))
 170 #define S2CR_CBNDX_SHIFT                0
 171 #define S2CR_CBNDX_MASK                 0xff
 172 #define S2CR_TYPE_SHIFT                 16
 173 #define S2CR_TYPE_MASK                  0x3
 174 enum arm_smmu_s2cr_type {
 175         S2CR_TYPE_TRANS,
 176         S2CR_TYPE_BYPASS,
 177         S2CR_TYPE_FAULT,
 178 };
 179
 180 #define S2CR_PRIVCFG_SHIFT              24
 181 #define S2CR_PRIVCFG_MASK               0x3
 182 enum arm_smmu_s2cr_privcfg {
 183         S2CR_PRIVCFG_DEFAULT,
 184         S2CR_PRIVCFG_DIPAN,
 185         S2CR_PRIVCFG_UNPRIV,
 186         S2CR_PRIVCFG_PRIV,
 187 };
 188
 189 /* Context bank attribute registers */
 190 #define ARM_SMMU_GR1_CBAR(n)            (0x0 + ((n) << 2))
 191 #define CBAR_VMID_SHIFT                 0
 192 #define CBAR_VMID_MASK                  0xff
 193 #define CBAR_S1_BPSHCFG_SHIFT           8
 194 #define CBAR_S1_BPSHCFG_MASK            3
 195 #define CBAR_S1_BPSHCFG_NSH             3
 196 #define CBAR_S1_MEMATTR_SHIFT           12
 197 #define CBAR_S1_MEMATTR_MASK            0xf
 198 #define CBAR_S1_MEMATTR_WB              0xf
 199 #define CBAR_TYPE_SHIFT                 16
 200 #define CBAR_TYPE_MASK                  0x3
 201 #define CBAR_TYPE_S2_TRANS              (0 << CBAR_TYPE_SHIFT)
 202 #define CBAR_TYPE_S1_TRANS_S2_BYPASS    (1 << CBAR_TYPE_SHIFT)
 203 #define CBAR_TYPE_S1_TRANS_S2_FAULT     (2 << CBAR_TYPE_SHIFT)
 204 #define CBAR_TYPE_S1_TRANS_S2_TRANS     (3 << CBAR_TYPE_SHIFT)
 205 #define CBAR_IRPTNDX_SHIFT              24
 206 #define CBAR_IRPTNDX_MASK               0xff
 207
 208 #define ARM_SMMU_GR1_CBA2R(n)           (0x800 + ((n) << 2))
 209 #define CBA2R_RW64_32BIT                (0 << 0)
 210 #define CBA2R_RW64_64BIT                (1 << 0)
 211 #define CBA2R_VMID_SHIFT                16
 212 #define CBA2R_VMID_MASK                 0xffff
 213
 214 /* Translation context bank */
 215 #define ARM_SMMU_CB_BASE(smmu)          ((smmu)->base + ((smmu)->size >> 1))
 216 #define ARM_SMMU_CB(smmu, n)            ((n) * (1 << (smmu)->pgshift))
 217
 218 #define ARM_SMMU_CB_SCTLR               0x0
 219 #define ARM_SMMU_CB_ACTLR               0x4
 220 #define ARM_SMMU_CB_RESUME              0x8
 221 #define ARM_SMMU_CB_TTBCR2              0x10
 222 #define ARM_SMMU_CB_TTBR0               0x20
 223 #define ARM_SMMU_CB_TTBR1               0x28
 224 #define ARM_SMMU_CB_TTBCR               0x30
 225 #define ARM_SMMU_CB_CONTEXTIDR          0x34
 226 #define ARM_SMMU_CB_S1_MAIR0            0x38
 227 #define ARM_SMMU_CB_S1_MAIR1            0x3c
 228 #define ARM_SMMU_CB_PAR                 0x50
 229 #define ARM_SMMU_CB_FSR                 0x58
 230 #define ARM_SMMU_CB_FAR                 0x60
 231 #define ARM_SMMU_CB_FSYNR0              0x68
 232 #define ARM_SMMU_CB_S1_TLBIVA           0x600
 233 #define ARM_SMMU_CB_S1_TLBIASID         0x610
 234 #define ARM_SMMU_CB_S1_TLBIVAL          0x620
 235 #define ARM_SMMU_CB_S2_TLBIIPAS2        0x630
 236 #define ARM_SMMU_CB_S2_TLBIIPAS2L       0x638
 237 #define ARM_SMMU_CB_ATS1PR              0x800
 238 #define ARM_SMMU_CB_ATSR                0x8f0
 239
 240 #define SCTLR_S1_ASIDPNE                (1 << 12)
 241 #define SCTLR_CFCFG                     (1 << 7)
 242 #define SCTLR_CFIE                      (1 << 6)
 243 #define SCTLR_CFRE                      (1 << 5)
 244 #define SCTLR_E                         (1 << 4)
 245 #define SCTLR_AFE                       (1 << 2)
 246 #define SCTLR_TRE                       (1 << 1)
 247 #define SCTLR_M                         (1 << 0)
 248
 249 #define ARM_MMU500_ACTLR_CPRE           (1 << 1)
 250
 251 #define ARM_MMU500_ACR_CACHE_LOCK       (1 << 26)
 252
 253 #define CB_PAR_F                        (1 << 0)
 254
 255 #define ATSR_ACTIVE                     (1 << 0)
 256
 257 #define RESUME_RETRY                    (0 << 0)
 258 #define RESUME_TERMINATE                (1 << 0)
 259
 260 #define TTBCR2_SEP_SHIFT                15
 261 #define TTBCR2_SEP_UPSTREAM             (0x7 << TTBCR2_SEP_SHIFT)
 262
 263 #define TTBRn_ASID_SHIFT                48
 264
 265 #define FSR_MULTI                       (1 << 31)
 266 #define FSR_SS                          (1 << 30)
 267 #define FSR_UUT                         (1 << 8)
 268 #define FSR_ASF                         (1 << 7)
 269 #define FSR_TLBLKF                      (1 << 6)
 270 #define FSR_TLBMCF                      (1 << 5)
 271 #define FSR_EF                          (1 << 4)
 272 #define FSR_PF                          (1 << 3)
 273 #define FSR_AFF                         (1 << 2)
 274 #define FSR_TF                          (1 << 1)
 275
 276 #define FSR_IGN                         (FSR_AFF | FSR_ASF | \
 277                                          FSR_TLBMCF | FSR_TLBLKF)
 278 #define FSR_FAULT                       (FSR_MULTI | FSR_SS | FSR_UUT | \
 279                                          FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
 280
 281 #define FSYNR0_WNR                      (1 << 4)
 282
 283 static int force_stage;
 284 module_param(force_stage, int, S_IRUGO);
 285 MODULE_PARM_DESC(force_stage,
 286         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
 287 static bool disable_bypass;
 288 module_param(disable_bypass, bool, S_IRUGO);
 289 MODULE_PARM_DESC(disable_bypass,
 290         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
 291
 292 enum arm_smmu_arch_version {
 293         ARM_SMMU_V1,
 294         ARM_SMMU_V1_64K,
 295         ARM_SMMU_V2,
 296 };
 297
 298 enum arm_smmu_implementation {
 299         GENERIC_SMMU,
 300         ARM_MMU500,
 301         CAVIUM_SMMUV2,
 302 };
 303
 304 struct arm_smmu_s2cr {
 305         enum arm_smmu_s2cr_type         type;
 306         enum arm_smmu_s2cr_privcfg      privcfg;
 307         u8                              cbndx;
 308 };
 309
 310 #define s2cr_init_val (struct arm_smmu_s2cr){                           \
 311         .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,    \
 312 }
 313
 314 struct arm_smmu_smr {
 315         u16                             mask;
 316         u16                             id;
 317         bool                            valid;
 318 };
 319
 320 struct arm_smmu_master_cfg {
 321         struct arm_smmu_device          *smmu;
 322         int                             num_streamids;
 323         u16                             streamids[MAX_MASTER_STREAMIDS];
 324         s16                             smendx[MAX_MASTER_STREAMIDS];
 325 };
 326 #define INVALID_SMENDX                  -1
 327
 328 struct arm_smmu_device {
 329         struct device                   *dev;
 330
 331         void __iomem                    *base;
 332         unsigned long                   size;
 333         unsigned long                   pgshift;
 334
 335 #define ARM_SMMU_FEAT_COHERENT_WALK     (1 << 0)
 336 #define ARM_SMMU_FEAT_STREAM_MATCH      (1 << 1)
 337 #define ARM_SMMU_FEAT_TRANS_S1          (1 << 2)
 338 #define ARM_SMMU_FEAT_TRANS_S2          (1 << 3)
 339 #define ARM_SMMU_FEAT_TRANS_NESTED      (1 << 4)
 340 #define ARM_SMMU_FEAT_TRANS_OPS         (1 << 5)
 341 #define ARM_SMMU_FEAT_VMID16            (1 << 6)
 342 #define ARM_SMMU_FEAT_FMT_AARCH64_4K    (1 << 7)
 343 #define ARM_SMMU_FEAT_FMT_AARCH64_16K   (1 << 8)
 344 #define ARM_SMMU_FEAT_FMT_AARCH64_64K   (1 << 9)
 345 #define ARM_SMMU_FEAT_FMT_AARCH32_L     (1 << 10)
 346 #define ARM_SMMU_FEAT_FMT_AARCH32_S     (1 << 11)
 347         u32                             features;
 348
 349 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
 350         u32                             options;
 351         enum arm_smmu_arch_version      version;
 352         enum arm_smmu_implementation    model;
 353
 354         u32                             num_context_banks;
 355         u32                             num_s2_context_banks;
 356         DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
 357         atomic_t                        irptndx;
 358
 359         u32                             num_mapping_groups;
 360         u16                             streamid_mask;
 361         u16                             smr_mask_mask;
 362         struct arm_smmu_smr             *smrs;
 363         struct arm_smmu_s2cr            *s2crs;
 364
 365         unsigned long                   va_size;
 366         unsigned long                   ipa_size;
 367         unsigned long                   pa_size;
 368         unsigned long                   pgsize_bitmap;
 369
 370         u32                             num_global_irqs;
 371         u32                             num_context_irqs;
 372         unsigned int                    *irqs;
 373
 374         u32                             cavium_id_base; /* Specific to Cavium */
 375 };
 376
 377 enum arm_smmu_context_fmt {
 378         ARM_SMMU_CTX_FMT_NONE,
 379         ARM_SMMU_CTX_FMT_AARCH64,
 380         ARM_SMMU_CTX_FMT_AARCH32_L,
 381         ARM_SMMU_CTX_FMT_AARCH32_S,
 382 };
 383
 384 struct arm_smmu_cfg {
 385         u8                              cbndx;
 386         u8                              irptndx;
 387         u32                             cbar;
 388         enum arm_smmu_context_fmt       fmt;
 389 };
 390 #define INVALID_IRPTNDX                 0xff
 391
 392 #define ARM_SMMU_CB_ASID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx)
 393 #define ARM_SMMU_CB_VMID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx + 1)
 394
 395 enum arm_smmu_domain_stage {
 396         ARM_SMMU_DOMAIN_S1 = 0,
 397         ARM_SMMU_DOMAIN_S2,
 398         ARM_SMMU_DOMAIN_NESTED,
 399 };
 400
 401 struct arm_smmu_domain {
 402         struct arm_smmu_device          *smmu;
 403         struct io_pgtable_ops           *pgtbl_ops;
 404         spinlock_t                      pgtbl_lock;
 405         struct arm_smmu_cfg             cfg;
 406         enum arm_smmu_domain_stage      stage;
 407         struct mutex                    init_mutex; /* Protects smmu pointer */
 408         struct iommu_domain             domain;
 409 };
 410
 411 struct arm_smmu_option_prop {
 412         u32 opt;
 413         const char *prop;
 414 };
 415
 416 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
 417
 418 static struct arm_smmu_option_prop arm_smmu_options[] = {
 419         { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
 420         { 0, NULL},
 421 };
 422
 423 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
 424 {
 425         return container_of(dom, struct arm_smmu_domain, domain);
 426 }
 427
 428 static void parse_driver_options(struct arm_smmu_device *smmu)
 429 {
 430         int i = 0;
 431
 432         do {
 433                 if (of_property_read_bool(smmu->dev->of_node,
 434                                                 arm_smmu_options[i].prop)) {
 435                         smmu->options |= arm_smmu_options[i].opt;
 436                         dev_notice(smmu->dev, "option %s\n",
 437                                 arm_smmu_options[i].prop);
 438                 }
 439         } while (arm_smmu_options[++i].opt);
 440 }
 441
 442 static struct device_node *dev_get_dev_node(struct device *dev)
 443 {
 444         if (dev_is_pci(dev)) {
 445                 struct pci_bus *bus = to_pci_dev(dev)->bus;
 446
 447                 while (!pci_is_root_bus(bus))
 448                         bus = bus->parent;
 449                 return of_node_get(bus->bridge->parent->of_node);
 450         }
 451
 452         return of_node_get(dev->of_node);
 453 }
 454
 455 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
 456 {
 457         *((__be32 *)data) = cpu_to_be32(alias);
 458         return 0; /* Continue walking */
 459 }
 460
 461 static int __find_legacy_master_phandle(struct device *dev, void *data)
 462 {
 463         struct of_phandle_iterator *it = *(void **)data;
 464         struct device_node *np = it->node;
 465         int err;
 466
 467         of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
 468                             "#stream-id-cells", 0)
 469                 if (it->node == np) {
 470                         *(void **)data = dev;
 471                         return 1;
 472                 }
 473         it->node = np;
 474         return err == -ENOENT ? 0 : err;
 475 }
 476
 477 static struct platform_driver arm_smmu_driver;
 478
 479 static int arm_smmu_register_legacy_master(struct device *dev)
 480 {
 481         struct arm_smmu_device *smmu;
 482         struct arm_smmu_master_cfg *cfg;
 483         struct device_node *np;
 484         struct of_phandle_iterator it;
 485         void *data = &it;
 486         __be32 pci_sid;
 487         int err;
 488
 489         np = dev_get_dev_node(dev);
 490         if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
 491                 of_node_put(np);
 492                 return -ENODEV;
 493         }
 494
 495         it.node = np;
 496         err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
 497                                      __find_legacy_master_phandle);
 498         of_node_put(np);
 499         if (err == 0)
 500                 return -ENODEV;
 501         if (err < 0)
 502                 return err;
 503
 504         smmu = dev_get_drvdata(data);
 505
 506         if (it.cur_count > MAX_MASTER_STREAMIDS) {
 507                 dev_err(smmu->dev,
 508                         "reached maximum number (%d) of stream IDs for master device %s\n",
 509                         MAX_MASTER_STREAMIDS, dev_name(dev));
 510                 return -ENOSPC;
 511         }
 512         if (dev_is_pci(dev)) {
 513                 /* "mmu-masters" assumes Stream ID == Requester ID */
 514                 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
 515                                        &pci_sid);
 516                 it.cur = &pci_sid;
 517                 it.cur_count = 1;
 518         }
 519
 520         cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
 521         if (!cfg)
 522                 return -ENOMEM;
 523
 524         cfg->smmu = smmu;
 525         dev->archdata.iommu = cfg;
 526
 527         while (it.cur_count--)
 528                 cfg->streamids[cfg->num_streamids++] = be32_to_cpup(it.cur++);
 529
 530         return 0;
 531 }
 532
 533 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
 534 {
 535         int idx;
 536
 537         do {
 538                 idx = find_next_zero_bit(map, end, start);
 539                 if (idx == end)
 540                         return -ENOSPC;
 541         } while (test_and_set_bit(idx, map));
 542
 543         return idx;
 544 }
 545
 546 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
 547 {
 548         clear_bit(idx, map);
 549 }
 550
 551 /* Wait for any pending TLB invalidations to complete */
 552 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
 553 {
 554         int count = 0;
 555         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
 556
 557         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_sTLBGSYNC);
 558         while (readl_relaxed(gr0_base + ARM_SMMU_GR0_sTLBGSTATUS)
 559                & sTLBGSTATUS_GSACTIVE) {
 560                 cpu_relax();
 561                 if (++count == TLB_LOOP_TIMEOUT) {
 562                         dev_err_ratelimited(smmu->dev,
 563                         "TLB sync timed out -- SMMU may be deadlocked\n");
 564                         return;
 565                 }
 566                 udelay(1);
 567         }
 568 }
 569
 570 static void arm_smmu_tlb_sync(void *cookie)
 571 {
 572         struct arm_smmu_domain *smmu_domain = cookie;
 573         __arm_smmu_tlb_sync(smmu_domain->smmu);
 574 }
 575
 576 static void arm_smmu_tlb_inv_context(void *cookie)
 577 {
 578         struct arm_smmu_domain *smmu_domain = cookie;
 579         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 580         struct arm_smmu_device *smmu = smmu_domain->smmu;
 581         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 582         void __iomem *base;
 583
 584         if (stage1) {
 585                 base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 586                 writel_relaxed(ARM_SMMU_CB_ASID(smmu, cfg),
 587                                base + ARM_SMMU_CB_S1_TLBIASID);
 588         } else {
 589                 base = ARM_SMMU_GR0(smmu);
 590                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg),
 591                                base + ARM_SMMU_GR0_TLBIVMID);
 592         }
 593
 594         __arm_smmu_tlb_sync(smmu);
 595 }
 596
 597 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
 598                                           size_t granule, bool leaf, void *cookie)
 599 {
 600         struct arm_smmu_domain *smmu_domain = cookie;
 601         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 602         struct arm_smmu_device *smmu = smmu_domain->smmu;
 603         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 604         void __iomem *reg;
 605
 606         if (stage1) {
 607                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 608                 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
 609
 610                 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
 611                         iova &= ~12UL;
 612                         iova |= ARM_SMMU_CB_ASID(smmu, cfg);
 613                         do {
 614                                 writel_relaxed(iova, reg);
 615                                 iova += granule;
 616                         } while (size -= granule);
 617                 } else {
 618                         iova >>= 12;
 619                         iova |= (u64)ARM_SMMU_CB_ASID(smmu, cfg) << 48;
 620                         do {
 621                                 writeq_relaxed(iova, reg);
 622                                 iova += granule >> 12;
 623                         } while (size -= granule);
 624                 }
 625         } else if (smmu->version == ARM_SMMU_V2) {
 626                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 627                 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
 628                               ARM_SMMU_CB_S2_TLBIIPAS2;
 629                 iova >>= 12;
 630                 do {
 631                         smmu_write_atomic_lq(iova, reg);
 632                         iova += granule >> 12;
 633                 } while (size -= granule);
 634         } else {
 635                 reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID;
 636                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), reg);
 637         }
 638 }
 639
 640 static struct iommu_gather_ops arm_smmu_gather_ops = {
 641         .tlb_flush_all  = arm_smmu_tlb_inv_context,
 642         .tlb_add_flush  = arm_smmu_tlb_inv_range_nosync,
 643         .tlb_sync       = arm_smmu_tlb_sync,
 644 };
 645
 646 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 647 {
 648         u32 fsr, fsynr;
 649         unsigned long iova;
 650         struct iommu_domain *domain = dev;
 651         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 652         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 653         struct arm_smmu_device *smmu = smmu_domain->smmu;
 654         void __iomem *cb_base;
 655
 656         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 657         fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
 658
 659         if (!(fsr & FSR_FAULT))
 660                 return IRQ_NONE;
 661
 662         fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
 663         iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
 664
 665         dev_err_ratelimited(smmu->dev,
 666         "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cb=%d\n",
 667                             fsr, iova, fsynr, cfg->cbndx);
 668
 669         writel(fsr, cb_base + ARM_SMMU_CB_FSR);
 670         return IRQ_HANDLED;
 671 }
 672
 673 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 674 {
 675         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
 676         struct arm_smmu_device *smmu = dev;
 677         void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
 678
 679         gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
 680         gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
 681         gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
 682         gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
 683
 684         if (!gfsr)
 685                 return IRQ_NONE;
 686
 687         dev_err_ratelimited(smmu->dev,
 688                 "Unexpected global fault, this could be serious\n");
 689         dev_err_ratelimited(smmu->dev,
 690                 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
 691                 gfsr, gfsynr0, gfsynr1, gfsynr2);
 692
 693         writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
 694         return IRQ_HANDLED;
 695 }
 696
 697 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 698                                        struct io_pgtable_cfg *pgtbl_cfg)
 699 {
 700         u32 reg, reg2;
 701         u64 reg64;
 702         bool stage1;
 703         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 704         struct arm_smmu_device *smmu = smmu_domain->smmu;
 705         void __iomem *cb_base, *gr1_base;
 706
 707         gr1_base = ARM_SMMU_GR1(smmu);
 708         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 709         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 710
 711         if (smmu->version > ARM_SMMU_V1) {
 712                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 713                         reg = CBA2R_RW64_64BIT;
 714                 else
 715                         reg = CBA2R_RW64_32BIT;
 716                 /* 16-bit VMIDs live in CBA2R */
 717                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
 718                         reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBA2R_VMID_SHIFT;
 719
 720                 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
 721         }
 722
 723         /* CBAR */
 724         reg = cfg->cbar;
 725         if (smmu->version < ARM_SMMU_V2)
 726                 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
 727
 728         /*
 729          * Use the weakest shareability/memory types, so they are
 730          * overridden by the ttbcr/pte.
 731          */
 732         if (stage1) {
 733                 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
 734                         (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
 735         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
 736                 /* 8-bit VMIDs live in CBAR */
 737                 reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBAR_VMID_SHIFT;
 738         }
 739         writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
 740
 741         /* TTBRs */
 742         if (stage1) {
 743                 u16 asid = ARM_SMMU_CB_ASID(smmu, cfg);
 744
 745                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 746                         reg = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
 747                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0);
 748                         reg = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
 749                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1);
 750                         writel_relaxed(asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
 751                 } else {
 752                         reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
 753                         reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
 754                         writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 755                         reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
 756                         reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
 757                         writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1);
 758                 }
 759         } else {
 760                 reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
 761                 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 762         }
 763
 764         /* TTBCR */
 765         if (stage1) {
 766                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 767                         reg = pgtbl_cfg->arm_v7s_cfg.tcr;
 768                         reg2 = 0;
 769                 } else {
 770                         reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
 771                         reg2 = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
 772                         reg2 |= TTBCR2_SEP_UPSTREAM;
 773                 }
 774                 if (smmu->version > ARM_SMMU_V1)
 775                         writel_relaxed(reg2, cb_base + ARM_SMMU_CB_TTBCR2);
 776         } else {
 777                 reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
 778         }
 779         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
 780
 781         /* MAIRs (stage-1 only) */
 782         if (stage1) {
 783                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 784                         reg = pgtbl_cfg->arm_v7s_cfg.prrr;
 785                         reg2 = pgtbl_cfg->arm_v7s_cfg.nmrr;
 786                 } else {
 787                         reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
 788                         reg2 = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
 789                 }
 790                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
 791                 writel_relaxed(reg2, cb_base + ARM_SMMU_CB_S1_MAIR1);
 792         }
 793
 794         /* SCTLR */
 795         reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
 796         if (stage1)
 797                 reg |= SCTLR_S1_ASIDPNE;
 798 #ifdef __BIG_ENDIAN
 799         reg |= SCTLR_E;
 800 #endif
 801         writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
 802 }
 803
 804 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 805                                         struct arm_smmu_device *smmu)
 806 {
 807         int irq, start, ret = 0;
 808         unsigned long ias, oas;
 809         struct io_pgtable_ops *pgtbl_ops;
 810         struct io_pgtable_cfg pgtbl_cfg;
 811         enum io_pgtable_fmt fmt;
 812         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 813         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 814
 815         mutex_lock(&smmu_domain->init_mutex);
 816         if (smmu_domain->smmu)
 817                 goto out_unlock;
 818
 819         /* We're bypassing these SIDs, so don't allocate an actual context */
 820         if (domain->type == IOMMU_DOMAIN_DMA) {
 821                 smmu_domain->smmu = smmu;
 822                 goto out_unlock;
 823         }
 824
 825         /*
 826          * Mapping the requested stage onto what we support is surprisingly
 827          * complicated, mainly because the spec allows S1+S2 SMMUs without
 828          * support for nested translation. That means we end up with the
 829          * following table:
 830          *
 831          * Requested        Supported        Actual
 832          *     S1               N              S1
 833          *     S1             S1+S2            S1
 834          *     S1               S2             S2
 835          *     S1               S1             S1
 836          *     N                N              N
 837          *     N              S1+S2            S2
 838          *     N                S2             S2
 839          *     N                S1             S1
 840          *
 841          * Note that you can't actually request stage-2 mappings.
 842          */
 843         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
 844                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 845         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 846                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 847
 848         /*
 849          * Choosing a suitable context format is even more fiddly. Until we
 850          * grow some way for the caller to express a preference, and/or move
 851          * the decision into the io-pgtable code where it arguably belongs,
 852          * just aim for the closest thing to the rest of the system, and hope
 853          * that the hardware isn't esoteric enough that we can't assume AArch64
 854          * support to be a superset of AArch32 support...
 855          */
 856         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
 857                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
 858         if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
 859             !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
 860             (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
 861             (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
 862                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
 863         if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
 864             (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
 865                                ARM_SMMU_FEAT_FMT_AARCH64_16K |
 866                                ARM_SMMU_FEAT_FMT_AARCH64_4K)))
 867                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
 868
 869         if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
 870                 ret = -EINVAL;
 871                 goto out_unlock;
 872         }
 873
 874         switch (smmu_domain->stage) {
 875         case ARM_SMMU_DOMAIN_S1:
 876                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
 877                 start = smmu->num_s2_context_banks;
 878                 ias = smmu->va_size;
 879                 oas = smmu->ipa_size;
 880                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 881                         fmt = ARM_64_LPAE_S1;
 882                 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
 883                         fmt = ARM_32_LPAE_S1;
 884                         ias = min(ias, 32UL);
 885                         oas = min(oas, 40UL);
 886                 } else {
 887                         fmt = ARM_V7S;
 888                         ias = min(ias, 32UL);
 889                         oas = min(oas, 32UL);
 890                 }
 891                 break;
 892         case ARM_SMMU_DOMAIN_NESTED:
 893                 /*
 894                  * We will likely want to change this if/when KVM gets
 895                  * involved.
 896                  */
 897         case ARM_SMMU_DOMAIN_S2:
 898                 cfg->cbar = CBAR_TYPE_S2_TRANS;
 899                 start = 0;
 900                 ias = smmu->ipa_size;
 901                 oas = smmu->pa_size;
 902                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 903                         fmt = ARM_64_LPAE_S2;
 904                 } else {
 905                         fmt = ARM_32_LPAE_S2;
 906                         ias = min(ias, 40UL);
 907                         oas = min(oas, 40UL);
 908                 }
 909                 break;
 910         default:
 911                 ret = -EINVAL;
 912                 goto out_unlock;
 913         }
 914
 915         ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
 916                                       smmu->num_context_banks);
 917         if (ret < 0)
 918                 goto out_unlock;
 919
 920         cfg->cbndx = ret;
 921         if (smmu->version < ARM_SMMU_V2) {
 922                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
 923                 cfg->irptndx %= smmu->num_context_irqs;
 924         } else {
 925                 cfg->irptndx = cfg->cbndx;
 926         }
 927
 928         pgtbl_cfg = (struct io_pgtable_cfg) {
 929                 .pgsize_bitmap  = smmu->pgsize_bitmap,
 930                 .ias            = ias,
 931                 .oas            = oas,
 932                 .tlb            = &arm_smmu_gather_ops,
 933                 .iommu_dev      = smmu->dev,
 934         };
 935
 936         smmu_domain->smmu = smmu;
 937         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
 938         if (!pgtbl_ops) {
 939                 ret = -ENOMEM;
 940                 goto out_clear_smmu;
 941         }
 942
 943         /* Update the domain's page sizes to reflect the page table format */
 944         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 945
 946         /* Initialise the context bank with our page table cfg */
 947         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
 948
 949         /*
 950          * Request context fault interrupt. Do this last to avoid the
 951          * handler seeing a half-initialised domain state.
 952          */
 953         irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 954         ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
 955                                IRQF_SHARED, "arm-smmu-context-fault", domain);
 956         if (ret < 0) {
 957                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
 958                         cfg->irptndx, irq);
 959                 cfg->irptndx = INVALID_IRPTNDX;
 960         }
 961
 962         mutex_unlock(&smmu_domain->init_mutex);
 963
 964         /* Publish page table ops for map/unmap */
 965         smmu_domain->pgtbl_ops = pgtbl_ops;
 966         return 0;
 967
 968 out_clear_smmu:
 969         smmu_domain->smmu = NULL;
 970 out_unlock:
 971         mutex_unlock(&smmu_domain->init_mutex);
 972         return ret;
 973 }
 974
 975 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 976 {
 977         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 978         struct arm_smmu_device *smmu = smmu_domain->smmu;
 979         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 980         void __iomem *cb_base;
 981         int irq;
 982
 983         if (!smmu || domain->type == IOMMU_DOMAIN_DMA)
 984                 return;
 985
 986         /*
 987          * Disable the context bank and free the page tables before freeing
 988          * it.
 989          */
 990         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 991         writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
 992
 993         if (cfg->irptndx != INVALID_IRPTNDX) {
 994                 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 995                 devm_free_irq(smmu->dev, irq, domain);
 996         }
 997
 998         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
 999         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
1000 }
1001
1002 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1003 {
1004         struct arm_smmu_domain *smmu_domain;
1005
1006         if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
1007                 return NULL;
1008         /*
1009          * Allocate the domain and initialise some of its data structures.
1010          * We can't really do anything meaningful until we've added a
1011          * master.
1012          */
1013         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1014         if (!smmu_domain)
1015                 return NULL;
1016
1017         if (type == IOMMU_DOMAIN_DMA &&
1018             iommu_get_dma_cookie(&smmu_domain->domain)) {
1019                 kfree(smmu_domain);
1020                 return NULL;
1021         }
1022
1023         mutex_init(&smmu_domain->init_mutex);
1024         spin_lock_init(&smmu_domain->pgtbl_lock);
1025
1026         return &smmu_domain->domain;
1027 }
1028
1029 static void arm_smmu_domain_free(struct iommu_domain *domain)
1030 {
1031         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1032
1033         /*
1034          * Free the domain resources. We assume that all devices have
1035          * already been detached.
1036          */
1037         iommu_put_dma_cookie(domain);
1038         arm_smmu_destroy_domain_context(domain);
1039         kfree(smmu_domain);
1040 }
1041
1042 static int arm_smmu_alloc_smr(struct arm_smmu_device *smmu)
1043 {
1044         int i;
1045
1046         for (i = 0; i < smmu->num_mapping_groups; i++)
1047                 if (!cmpxchg(&smmu->smrs[i].valid, false, true))
1048                         return i;
1049
1050         return INVALID_SMENDX;
1051 }
1052
1053 static void arm_smmu_free_smr(struct arm_smmu_device *smmu, int idx)
1054 {
1055         writel_relaxed(~SMR_VALID, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx));
1056         WRITE_ONCE(smmu->smrs[idx].valid, false);
1057 }
1058
1059 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
1060 {
1061         struct arm_smmu_smr *smr = smmu->smrs + idx;
1062         u32 reg = smr->id << SMR_ID_SHIFT | smr->mask << SMR_MASK_SHIFT;
1063
1064         if (smr->valid)
1065                 reg |= SMR_VALID;
1066         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx));
1067 }
1068
1069 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
1070 {
1071         struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
1072         u32 reg = (s2cr->type & S2CR_TYPE_MASK) << S2CR_TYPE_SHIFT |
1073                   (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT |
1074                   (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT;
1075
1076         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx));
1077 }
1078
1079 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
1080 {
1081         arm_smmu_write_s2cr(smmu, idx);
1082         if (smmu->smrs)
1083                 arm_smmu_write_smr(smmu, idx);
1084 }
1085
1086 static int arm_smmu_master_alloc_smes(struct arm_smmu_device *smmu,
1087                                       struct arm_smmu_master_cfg *cfg)
1088 {
1089         struct arm_smmu_smr *smrs = smmu->smrs;
1090         int i, idx;
1091
1092         /* Allocate the SMRs on the SMMU */
1093         for (i = 0; i < cfg->num_streamids; ++i) {
1094                 if (cfg->smendx[i] != INVALID_SMENDX)
1095                         return -EEXIST;
1096
1097                 /* ...except on stream indexing hardware, of course */
1098                 if (!smrs) {
1099                         cfg->smendx[i] = cfg->streamids[i];
1100                         continue;
1101                 }
1102
1103                 idx = arm_smmu_alloc_smr(smmu);
1104                 if (idx < 0) {
1105                         dev_err(smmu->dev, "failed to allocate free SMR\n");
1106                         goto err_free_smrs;
1107                 }
1108                 cfg->smendx[i] = idx;
1109
1110                 smrs[idx].id = cfg->streamids[i];
1111                 smrs[idx].mask = 0; /* We don't currently share SMRs */
1112         }
1113
1114         if (!smrs)
1115                 return 0;
1116
1117         /* It worked! Now, poke the actual hardware */
1118         for (i = 0; i < cfg->num_streamids; ++i)
1119                 arm_smmu_write_smr(smmu, cfg->smendx[i]);
1120
1121         return 0;
1122
1123 err_free_smrs:
1124         while (i--) {
1125                 arm_smmu_free_smr(smmu, cfg->smendx[i]);
1126                 cfg->smendx[i] = INVALID_SMENDX;
1127         }
1128         return -ENOSPC;
1129 }
1130
1131 static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg)
1132 {
1133         struct arm_smmu_device *smmu = cfg->smmu;
1134         int i;
1135
1136         /*
1137          * We *must* clear the S2CR first, because freeing the SMR means
1138          * that it can be re-allocated immediately.
1139          */
1140         for (i = 0; i < cfg->num_streamids; ++i) {
1141                 int idx = cfg->smendx[i];
1142
1143                 /* An IOMMU group is torn down by the first device to be removed */
1144                 if (idx == INVALID_SMENDX)
1145                         return;
1146
1147                 smmu->s2crs[idx] = s2cr_init_val;
1148                 arm_smmu_write_s2cr(smmu, idx);
1149         }
1150         /* Sync S2CR updates before touching anything else */
1151         __iowmb();
1152
1153         /* Invalidate the SMRs before freeing back to the allocator */
1154         for (i = 0; i < cfg->num_streamids; ++i) {
1155                 if (smmu->smrs)
1156                         arm_smmu_free_smr(smmu, cfg->smendx[i]);
1157
1158                 cfg->smendx[i] = INVALID_SMENDX;
1159         }
1160 }
1161
1162 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1163                                       struct arm_smmu_master_cfg *cfg)
1164 {
1165         int i, ret = 0;
1166         struct arm_smmu_device *smmu = smmu_domain->smmu;
1167         struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1168         enum arm_smmu_s2cr_type type = S2CR_TYPE_TRANS;
1169         u8 cbndx = smmu_domain->cfg.cbndx;
1170
1171         if (cfg->smendx[0] == INVALID_SMENDX)
1172                 ret = arm_smmu_master_alloc_smes(smmu, cfg);
1173         if (ret)
1174                 return ret;
1175
1176         /*
1177          * FIXME: This won't be needed once we have IOMMU-backed DMA ops
1178          * for all devices behind the SMMU. Note that we need to take
1179          * care configuring SMRs for devices both a platform_device and
1180          * and a PCI device (i.e. a PCI host controller)
1181          */
1182         if (smmu_domain->domain.type == IOMMU_DOMAIN_DMA)
1183                 type = S2CR_TYPE_BYPASS;
1184
1185         for (i = 0; i < cfg->num_streamids; ++i) {
1186                 int idx = cfg->smendx[i];
1187
1188                 /* Devices in an IOMMU group may already be configured */
1189                 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1190                         break;
1191
1192                 s2cr[idx].type = type;
1193                 s2cr[idx].privcfg = S2CR_PRIVCFG_UNPRIV;
1194                 s2cr[idx].cbndx = cbndx;
1195                 arm_smmu_write_s2cr(smmu, idx);
1196         }
1197         return 0;
1198 }
1199
1200 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1201 {
1202         int ret;
1203         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1204         struct arm_smmu_master_cfg *cfg = dev->archdata.iommu;
1205
1206         if (!cfg) {
1207                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1208                 return -ENXIO;
1209         }
1210
1211         /* Ensure that the domain is finalised */
1212         ret = arm_smmu_init_domain_context(domain, cfg->smmu);
1213         if (ret < 0)
1214                 return ret;
1215
1216         /*
1217          * Sanity check the domain. We don't support domains across
1218          * different SMMUs.
1219          */
1220         if (smmu_domain->smmu != cfg->smmu) {
1221                 dev_err(dev,
1222                         "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1223                         dev_name(smmu_domain->smmu->dev), dev_name(cfg->smmu->dev));
1224                 return -EINVAL;
1225         }
1226
1227         /* Looks ok, so add the device to the domain */
1228         return arm_smmu_domain_add_master(smmu_domain, cfg);
1229 }
1230
1231 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1232                         phys_addr_t paddr, size_t size, int prot)
1233 {
1234         int ret;
1235         unsigned long flags;
1236         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1237         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1238
1239         if (!ops)
1240                 return -ENODEV;
1241
1242         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1243         ret = ops->map(ops, iova, paddr, size, prot);
1244         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1245         return ret;
1246 }
1247
1248 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1249                              size_t size)
1250 {
1251         size_t ret;
1252         unsigned long flags;
1253         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1254         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1255
1256         if (!ops)
1257                 return 0;
1258
1259         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1260         ret = ops->unmap(ops, iova, size);
1261         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1262         return ret;
1263 }
1264
1265 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1266                                               dma_addr_t iova)
1267 {
1268         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1269         struct arm_smmu_device *smmu = smmu_domain->smmu;
1270         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1271         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1272         struct device *dev = smmu->dev;
1273         void __iomem *cb_base;
1274         u32 tmp;
1275         u64 phys;
1276         unsigned long va;
1277
1278         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1279
1280         /* ATS1 registers can only be written atomically */
1281         va = iova & ~0xfffUL;
1282         if (smmu->version == ARM_SMMU_V2)
1283                 smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1284         else /* Register is only 32-bit in v1 */
1285                 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1286
1287         if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1288                                       !(tmp & ATSR_ACTIVE), 5, 50)) {
1289                 dev_err(dev,
1290                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1291                         &iova);
1292                 return ops->iova_to_phys(ops, iova);
1293         }
1294
1295         phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
1296         if (phys & CB_PAR_F) {
1297                 dev_err(dev, "translation fault!\n");
1298                 dev_err(dev, "PAR = 0x%llx\n", phys);
1299                 return 0;
1300         }
1301
1302         return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1303 }
1304
1305 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1306                                         dma_addr_t iova)
1307 {
1308         phys_addr_t ret;
1309         unsigned long flags;
1310         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1311         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1312
1313         if (!ops)
1314                 return 0;
1315
1316         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1317         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1318                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1319                 ret = arm_smmu_iova_to_phys_hard(domain, iova);
1320         } else {
1321                 ret = ops->iova_to_phys(ops, iova);
1322         }
1323
1324         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1325
1326         return ret;
1327 }
1328
1329 static bool arm_smmu_capable(enum iommu_cap cap)
1330 {
1331         switch (cap) {
1332         case IOMMU_CAP_CACHE_COHERENCY:
1333                 /*
1334                  * Return true here as the SMMU can always send out coherent
1335                  * requests.
1336                  */
1337                 return true;
1338         case IOMMU_CAP_INTR_REMAP:
1339                 return true; /* MSIs are just memory writes */
1340         case IOMMU_CAP_NOEXEC:
1341                 return true;
1342         default:
1343                 return false;
1344         }
1345 }
1346
1347 static int arm_smmu_add_device(struct device *dev)
1348 {
1349         struct arm_smmu_master_cfg *cfg;
1350         struct iommu_group *group;
1351         int i, ret;
1352
1353         ret = arm_smmu_register_legacy_master(dev);
1354         cfg = dev->archdata.iommu;
1355         if (ret)
1356                 goto out_free;
1357
1358         ret = -EINVAL;
1359         for (i = 0; i < cfg->num_streamids; i++) {
1360                 u16 sid = cfg->streamids[i];
1361
1362                 if (sid & ~cfg->smmu->streamid_mask) {
1363                         dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1364                                 sid, cfg->smmu->streamid_mask);
1365                         goto out_free;
1366                 }
1367                 cfg->smendx[i] = INVALID_SMENDX;
1368         }
1369
1370         group = iommu_group_get_for_dev(dev);
1371         if (IS_ERR(group)) {
1372                 ret = PTR_ERR(group);
1373                 goto out_free;
1374         }
1375         iommu_group_put(group);
1376         return 0;
1377
1378 out_free:
1379         kfree(cfg);
1380         dev->archdata.iommu = NULL;
1381         return ret;
1382 }
1383
1384 static void arm_smmu_remove_device(struct device *dev)
1385 {
1386         struct arm_smmu_master_cfg *cfg = dev->archdata.iommu;
1387
1388         if (!cfg)
1389                 return;
1390
1391         arm_smmu_master_free_smes(cfg);
1392         iommu_group_remove_device(dev);
1393         kfree(cfg);
1394         dev->archdata.iommu = NULL;
1395 }
1396
1397 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1398 {
1399         struct iommu_group *group;
1400
1401         if (dev_is_pci(dev))
1402                 group = pci_device_group(dev);
1403         else
1404                 group = generic_device_group(dev);
1405
1406         return group;
1407 }
1408
1409 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1410                                     enum iommu_attr attr, void *data)
1411 {
1412         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1413
1414         switch (attr) {
1415         case DOMAIN_ATTR_NESTING:
1416                 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1417                 return 0;
1418         default:
1419                 return -ENODEV;
1420         }
1421 }
1422
1423 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1424                                     enum iommu_attr attr, void *data)
1425 {
1426         int ret = 0;
1427         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1428
1429         mutex_lock(&smmu_domain->init_mutex);
1430
1431         switch (attr) {
1432         case DOMAIN_ATTR_NESTING:
1433                 if (smmu_domain->smmu) {
1434                         ret = -EPERM;
1435                         goto out_unlock;
1436                 }
1437
1438                 if (*(int *)data)
1439                         smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1440                 else
1441                         smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1442
1443                 break;
1444         default:
1445                 ret = -ENODEV;
1446         }
1447
1448 out_unlock:
1449         mutex_unlock(&smmu_domain->init_mutex);
1450         return ret;
1451 }
1452
1453 static struct iommu_ops arm_smmu_ops = {
1454         .capable                = arm_smmu_capable,
1455         .domain_alloc           = arm_smmu_domain_alloc,
1456         .domain_free            = arm_smmu_domain_free,
1457         .attach_dev             = arm_smmu_attach_dev,
1458         .map                    = arm_smmu_map,
1459         .unmap                  = arm_smmu_unmap,
1460         .map_sg                 = default_iommu_map_sg,
1461         .iova_to_phys           = arm_smmu_iova_to_phys,
1462         .add_device             = arm_smmu_add_device,
1463         .remove_device          = arm_smmu_remove_device,
1464         .device_group           = arm_smmu_device_group,
1465         .domain_get_attr        = arm_smmu_domain_get_attr,
1466         .domain_set_attr        = arm_smmu_domain_set_attr,
1467         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1468 };
1469
1470 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1471 {
1472         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1473         void __iomem *cb_base;
1474         int i;
1475         u32 reg, major;
1476
1477         /* clear global FSR */
1478         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1479         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1480
1481         /*
1482          * Reset stream mapping groups: Initial values mark all SMRn as
1483          * invalid and all S2CRn as bypass unless overridden.
1484          */
1485         for (i = 0; i < smmu->num_mapping_groups; ++i)
1486                 arm_smmu_write_sme(smmu, i);
1487
1488         /*
1489          * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1490          * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1491          * bit is only present in MMU-500r2 onwards.
1492          */
1493         reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
1494         major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
1495         if ((smmu->model == ARM_MMU500) && (major >= 2)) {
1496                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
1497                 reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
1498                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
1499         }
1500
1501         /* Make sure all context banks are disabled and clear CB_FSR  */
1502         for (i = 0; i < smmu->num_context_banks; ++i) {
1503                 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i);
1504                 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1505                 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1506                 /*
1507                  * Disable MMU-500's not-particularly-beneficial next-page
1508                  * prefetcher for the sake of errata #841119 and #826419.
1509                  */
1510                 if (smmu->model == ARM_MMU500) {
1511                         reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
1512                         reg &= ~ARM_MMU500_ACTLR_CPRE;
1513                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
1514                 }
1515         }
1516
1517         /* Invalidate the TLB, just in case */
1518         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1519         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1520
1521         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1522
1523         /* Enable fault reporting */
1524         reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1525
1526         /* Disable TLB broadcasting. */
1527         reg |= (sCR0_VMIDPNE | sCR0_PTM);
1528
1529         /* Enable client access, handling unmatched streams as appropriate */
1530         reg &= ~sCR0_CLIENTPD;
1531         if (disable_bypass)
1532                 reg |= sCR0_USFCFG;
1533         else
1534                 reg &= ~sCR0_USFCFG;
1535
1536         /* Disable forced broadcasting */
1537         reg &= ~sCR0_FB;
1538
1539         /* Don't upgrade barriers */
1540         reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
1541
1542         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1543                 reg |= sCR0_VMID16EN;
1544
1545         /* Push the button */
1546         __arm_smmu_tlb_sync(smmu);
1547         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1548 }
1549
1550 static int arm_smmu_id_size_to_bits(int size)
1551 {
1552         switch (size) {
1553         case 0:
1554                 return 32;
1555         case 1:
1556                 return 36;
1557         case 2:
1558                 return 40;
1559         case 3:
1560                 return 42;
1561         case 4:
1562                 return 44;
1563         case 5:
1564         default:
1565                 return 48;
1566         }
1567 }
1568
1569 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1570 {
1571         unsigned long size;
1572         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1573         u32 id;
1574         bool cttw_dt, cttw_reg;
1575         int i;
1576
1577         dev_notice(smmu->dev, "probing hardware configuration...\n");
1578         dev_notice(smmu->dev, "SMMUv%d with:\n",
1579                         smmu->version == ARM_SMMU_V2 ? 2 : 1);
1580
1581         /* ID0 */
1582         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1583
1584         /* Restrict available stages based on module parameter */
1585         if (force_stage == 1)
1586                 id &= ~(ID0_S2TS | ID0_NTS);
1587         else if (force_stage == 2)
1588                 id &= ~(ID0_S1TS | ID0_NTS);
1589
1590         if (id & ID0_S1TS) {
1591                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1592                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1593         }
1594
1595         if (id & ID0_S2TS) {
1596                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1597                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1598         }
1599
1600         if (id & ID0_NTS) {
1601                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1602                 dev_notice(smmu->dev, "\tnested translation\n");
1603         }
1604
1605         if (!(smmu->features &
1606                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1607                 dev_err(smmu->dev, "\tno translation support!\n");
1608                 return -ENODEV;
1609         }
1610
1611         if ((id & ID0_S1TS) &&
1612                 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1613                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1614                 dev_notice(smmu->dev, "\taddress translation ops\n");
1615         }
1616
1617         /*
1618          * In order for DMA API calls to work properly, we must defer to what
1619          * the DT says about coherency, regardless of what the hardware claims.
1620          * Fortunately, this also opens up a workaround for systems where the
1621          * ID register value has ended up configured incorrectly.
1622          */
1623         cttw_dt = of_dma_is_coherent(smmu->dev->of_node);
1624         cttw_reg = !!(id & ID0_CTTW);
1625         if (cttw_dt)
1626                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1627         if (cttw_dt || cttw_reg)
1628                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1629                            cttw_dt ? "" : "non-");
1630         if (cttw_dt != cttw_reg)
1631                 dev_notice(smmu->dev,
1632                            "\t(IDR0.CTTW overridden by dma-coherent property)\n");
1633
1634         /* Max. number of entries we have for stream matching/indexing */
1635         size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK);
1636         smmu->streamid_mask = size - 1;
1637         if (id & ID0_SMS) {
1638                 u32 smr;
1639
1640                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1641                 size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK;
1642                 if (size == 0) {
1643                         dev_err(smmu->dev,
1644                                 "stream-matching supported, but no SMRs present!\n");
1645                         return -ENODEV;
1646                 }
1647
1648                 /*
1649                  * SMR.ID bits may not be preserved if the corresponding MASK
1650                  * bits are set, so check each one separately. We can reject
1651                  * masters later if they try to claim IDs outside these masks.
1652                  */
1653                 smr = smmu->streamid_mask << SMR_ID_SHIFT;
1654                 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1655                 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1656                 smmu->streamid_mask = smr >> SMR_ID_SHIFT;
1657
1658                 smr = smmu->streamid_mask << SMR_MASK_SHIFT;
1659                 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1660                 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1661                 smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT;
1662
1663                 /* Zero-initialised to mark as invalid */
1664                 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1665                                           GFP_KERNEL);
1666                 if (!smmu->smrs)
1667                         return -ENOMEM;
1668
1669                 dev_notice(smmu->dev,
1670                            "\tstream matching with %lu register groups, mask 0x%x",
1671                            size, smmu->smr_mask_mask);
1672         }
1673         /* s2cr->type == 0 means translation, so initialise explicitly */
1674         smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1675                                          GFP_KERNEL);
1676         if (!smmu->s2crs)
1677                 return -ENOMEM;
1678         for (i = 0; i < size; i++)
1679                 smmu->s2crs[i] = s2cr_init_val;
1680
1681         smmu->num_mapping_groups = size;
1682
1683         if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1684                 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1685                 if (!(id & ID0_PTFS_NO_AARCH32S))
1686                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1687         }
1688
1689         /* ID1 */
1690         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1691         smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1692
1693         /* Check for size mismatch of SMMU address space from mapped region */
1694         size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
1695         size *= 2 << smmu->pgshift;
1696         if (smmu->size != size)
1697                 dev_warn(smmu->dev,
1698                         "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n",
1699                         size, smmu->size);
1700
1701         smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
1702         smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
1703         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1704                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1705                 return -ENODEV;
1706         }
1707         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1708                    smmu->num_context_banks, smmu->num_s2_context_banks);
1709         /*
1710          * Cavium CN88xx erratum #27704.
1711          * Ensure ASID and VMID allocation is unique across all SMMUs in
1712          * the system.
1713          */
1714         if (smmu->model == CAVIUM_SMMUV2) {
1715                 smmu->cavium_id_base =
1716                         atomic_add_return(smmu->num_context_banks,
1717                                           &cavium_smmu_context_count);
1718                 smmu->cavium_id_base -= smmu->num_context_banks;
1719         }
1720
1721         /* ID2 */
1722         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1723         size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
1724         smmu->ipa_size = size;
1725
1726         /* The output mask is also applied for bypass */
1727         size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
1728         smmu->pa_size = size;
1729
1730         if (id & ID2_VMID16)
1731                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1732
1733         /*
1734          * What the page table walker can address actually depends on which
1735          * descriptor format is in use, but since a) we don't know that yet,
1736          * and b) it can vary per context bank, this will have to do...
1737          */
1738         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1739                 dev_warn(smmu->dev,
1740                          "failed to set DMA mask for table walker\n");
1741
1742         if (smmu->version < ARM_SMMU_V2) {
1743                 smmu->va_size = smmu->ipa_size;
1744                 if (smmu->version == ARM_SMMU_V1_64K)
1745                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1746         } else {
1747                 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
1748                 smmu->va_size = arm_smmu_id_size_to_bits(size);
1749                 if (id & ID2_PTFS_4K)
1750                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1751                 if (id & ID2_PTFS_16K)
1752                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1753                 if (id & ID2_PTFS_64K)
1754                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1755         }
1756
1757         /* Now we've corralled the various formats, what'll it do? */
1758         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1759                 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1760         if (smmu->features &
1761             (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1762                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1763         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1764                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1765         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1766                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1767
1768         if (arm_smmu_ops.pgsize_bitmap == -1UL)
1769                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1770         else
1771                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1772         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1773                    smmu->pgsize_bitmap);
1774
1775
1776         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1777                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1778                            smmu->va_size, smmu->ipa_size);
1779
1780         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1781                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1782                            smmu->ipa_size, smmu->pa_size);
1783
1784         return 0;
1785 }
1786
1787 struct arm_smmu_match_data {
1788         enum arm_smmu_arch_version version;
1789         enum arm_smmu_implementation model;
1790 };
1791
1792 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
1793 static struct arm_smmu_match_data name = { .version = ver, .model = imp }
1794
1795 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1796 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1797 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1798 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1799 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1800
1801 static const struct of_device_id arm_smmu_of_match[] = {
1802         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1803         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1804         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1805         { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1806         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1807         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1808         { },
1809 };
1810 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1811
1812 static int arm_smmu_device_dt_probe(struct platform_device *pdev)
1813 {
1814         const struct arm_smmu_match_data *data;
1815         struct resource *res;
1816         struct arm_smmu_device *smmu;
1817         struct device *dev = &pdev->dev;
1818         int num_irqs, i, err;
1819
1820         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
1821         if (!smmu) {
1822                 dev_err(dev, "failed to allocate arm_smmu_device\n");
1823                 return -ENOMEM;
1824         }
1825         smmu->dev = dev;
1826
1827         data = of_device_get_match_data(dev);
1828         smmu->version = data->version;
1829         smmu->model = data->model;
1830
1831         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1832         smmu->base = devm_ioremap_resource(dev, res);
1833         if (IS_ERR(smmu->base))
1834                 return PTR_ERR(smmu->base);
1835         smmu->size = resource_size(res);
1836
1837         if (of_property_read_u32(dev->of_node, "#global-interrupts",
1838                                  &smmu->num_global_irqs)) {
1839                 dev_err(dev, "missing #global-interrupts property\n");
1840                 return -ENODEV;
1841         }
1842
1843         num_irqs = 0;
1844         while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
1845                 num_irqs++;
1846                 if (num_irqs > smmu->num_global_irqs)
1847                         smmu->num_context_irqs++;
1848         }
1849
1850         if (!smmu->num_context_irqs) {
1851                 dev_err(dev, "found %d interrupts but expected at least %d\n",
1852                         num_irqs, smmu->num_global_irqs + 1);
1853                 return -ENODEV;
1854         }
1855
1856         smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs,
1857                                   GFP_KERNEL);
1858         if (!smmu->irqs) {
1859                 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
1860                 return -ENOMEM;
1861         }
1862
1863         for (i = 0; i < num_irqs; ++i) {
1864                 int irq = platform_get_irq(pdev, i);
1865
1866                 if (irq < 0) {
1867                         dev_err(dev, "failed to get irq index %d\n", i);
1868                         return -ENODEV;
1869                 }
1870                 smmu->irqs[i] = irq;
1871         }
1872
1873         err = arm_smmu_device_cfg_probe(smmu);
1874         if (err)
1875                 return err;
1876
1877         parse_driver_options(smmu);
1878
1879         if (smmu->version == ARM_SMMU_V2 &&
1880             smmu->num_context_banks != smmu->num_context_irqs) {
1881                 dev_err(dev,
1882                         "found only %d context interrupt(s) but %d required\n",
1883                         smmu->num_context_irqs, smmu->num_context_banks);
1884                 return -ENODEV;
1885         }
1886
1887         for (i = 0; i < smmu->num_global_irqs; ++i) {
1888                 err = devm_request_irq(smmu->dev, smmu->irqs[i],
1889                                        arm_smmu_global_fault,
1890                                        IRQF_SHARED,
1891                                        "arm-smmu global fault",
1892                                        smmu);
1893                 if (err) {
1894                         dev_err(dev, "failed to request global IRQ %d (%u)\n",
1895                                 i, smmu->irqs[i]);
1896                         return err;
1897                 }
1898         }
1899
1900         platform_set_drvdata(pdev, smmu);
1901         arm_smmu_device_reset(smmu);
1902         return 0;
1903 }
1904
1905 static int arm_smmu_device_remove(struct platform_device *pdev)
1906 {
1907         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
1908
1909         if (!smmu)
1910                 return -ENODEV;
1911
1912         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
1913                 dev_err(&pdev->dev, "removing device with active domains!\n");
1914
1915         /* Turn the thing off */
1916         writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1917         return 0;
1918 }
1919
1920 static struct platform_driver arm_smmu_driver = {
1921         .driver = {
1922                 .name           = "arm-smmu",
1923                 .of_match_table = of_match_ptr(arm_smmu_of_match),
1924         },
1925         .probe  = arm_smmu_device_dt_probe,
1926         .remove = arm_smmu_device_remove,
1927 };
1928
1929 static int __init arm_smmu_init(void)
1930 {
1931         struct device_node *np;
1932         int ret;
1933
1934         /*
1935          * Play nice with systems that don't have an ARM SMMU by checking that
1936          * an ARM SMMU exists in the system before proceeding with the driver
1937          * and IOMMU bus operation registration.
1938          */
1939         np = of_find_matching_node(NULL, arm_smmu_of_match);
1940         if (!np)
1941                 return 0;
1942
1943         of_node_put(np);
1944
1945         ret = platform_driver_register(&arm_smmu_driver);
1946         if (ret)
1947                 return ret;
1948
1949         /* Oh, for a proper bus abstraction */
1950         if (!iommu_present(&platform_bus_type))
1951                 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
1952
1953 #ifdef CONFIG_ARM_AMBA
1954         if (!iommu_present(&amba_bustype))
1955                 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
1956 #endif
1957
1958 #ifdef CONFIG_PCI
1959         if (!iommu_present(&pci_bus_type)) {
1960                 pci_request_acs();
1961                 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
1962         }
1963 #endif
1964
1965         return 0;
1966 }
1967
1968 static void __exit arm_smmu_exit(void)
1969 {
1970         return platform_driver_unregister(&arm_smmu_driver);
1971 }
1972
1973 subsys_initcall(arm_smmu_init);
1974 module_exit(arm_smmu_exit);
1975
1976 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
1977 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
1978 MODULE_LICENSE("GPL v2");