]> git.kernelconcepts.de Git - karo-tx-linux.git/blob - drivers/iommu/fsl_pamu_domain.c
Merge branch 'fix/rt5645' of git://git.kernel.org/pub/scm/linux/kernel/git/broonie...
[karo-tx-linux.git] / drivers / iommu / fsl_pamu_domain.c
1 /*
2  * This program is free software; you can redistribute it and/or modify
3  * it under the terms of the GNU General Public License, version 2, as
4  * published by the Free Software Foundation.
5  *
6  * This program is distributed in the hope that it will be useful,
7  * but WITHOUT ANY WARRANTY; without even the implied warranty of
8  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
9  * GNU General Public License for more details.
10  *
11  * You should have received a copy of the GNU General Public License
12  * along with this program; if not, write to the Free Software
13  * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
14  *
15  * Copyright (C) 2013 Freescale Semiconductor, Inc.
16  * Author: Varun Sethi <varun.sethi@freescale.com>
17  *
18  */
19
20 #define pr_fmt(fmt)    "fsl-pamu-domain: %s: " fmt, __func__
21
22 #include "fsl_pamu_domain.h"
23
24 #include <sysdev/fsl_pci.h>
25
26 /*
27  * Global spinlock that needs to be held while
28  * configuring PAMU.
29  */
30 static DEFINE_SPINLOCK(iommu_lock);
31
32 static struct kmem_cache *fsl_pamu_domain_cache;
33 static struct kmem_cache *iommu_devinfo_cache;
34 static DEFINE_SPINLOCK(device_domain_lock);
35
36 static struct fsl_dma_domain *to_fsl_dma_domain(struct iommu_domain *dom)
37 {
38         return container_of(dom, struct fsl_dma_domain, iommu_domain);
39 }
40
41 static int __init iommu_init_mempool(void)
42 {
43         fsl_pamu_domain_cache = kmem_cache_create("fsl_pamu_domain",
44                                                   sizeof(struct fsl_dma_domain),
45                                                   0,
46                                                   SLAB_HWCACHE_ALIGN,
47                                                   NULL);
48         if (!fsl_pamu_domain_cache) {
49                 pr_debug("Couldn't create fsl iommu_domain cache\n");
50                 return -ENOMEM;
51         }
52
53         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
54                                                 sizeof(struct device_domain_info),
55                                                 0,
56                                                 SLAB_HWCACHE_ALIGN,
57                                                 NULL);
58         if (!iommu_devinfo_cache) {
59                 pr_debug("Couldn't create devinfo cache\n");
60                 kmem_cache_destroy(fsl_pamu_domain_cache);
61                 return -ENOMEM;
62         }
63
64         return 0;
65 }
66
67 static phys_addr_t get_phys_addr(struct fsl_dma_domain *dma_domain, dma_addr_t iova)
68 {
69         u32 win_cnt = dma_domain->win_cnt;
70         struct dma_window *win_ptr = &dma_domain->win_arr[0];
71         struct iommu_domain_geometry *geom;
72
73         geom = &dma_domain->iommu_domain.geometry;
74
75         if (!win_cnt || !dma_domain->geom_size) {
76                 pr_debug("Number of windows/geometry not configured for the domain\n");
77                 return 0;
78         }
79
80         if (win_cnt > 1) {
81                 u64 subwin_size;
82                 dma_addr_t subwin_iova;
83                 u32 wnd;
84
85                 subwin_size = dma_domain->geom_size >> ilog2(win_cnt);
86                 subwin_iova = iova & ~(subwin_size - 1);
87                 wnd = (subwin_iova - geom->aperture_start) >> ilog2(subwin_size);
88                 win_ptr = &dma_domain->win_arr[wnd];
89         }
90
91         if (win_ptr->valid)
92                 return win_ptr->paddr + (iova & (win_ptr->size - 1));
93
94         return 0;
95 }
96
97 static int map_subwins(int liodn, struct fsl_dma_domain *dma_domain)
98 {
99         struct dma_window *sub_win_ptr = &dma_domain->win_arr[0];
100         int i, ret;
101         unsigned long rpn, flags;
102
103         for (i = 0; i < dma_domain->win_cnt; i++) {
104                 if (sub_win_ptr[i].valid) {
105                         rpn = sub_win_ptr[i].paddr >> PAMU_PAGE_SHIFT;
106                         spin_lock_irqsave(&iommu_lock, flags);
107                         ret = pamu_config_spaace(liodn, dma_domain->win_cnt, i,
108                                                  sub_win_ptr[i].size,
109                                                  ~(u32)0,
110                                                  rpn,
111                                                  dma_domain->snoop_id,
112                                                  dma_domain->stash_id,
113                                                  (i > 0) ? 1 : 0,
114                                                  sub_win_ptr[i].prot);
115                         spin_unlock_irqrestore(&iommu_lock, flags);
116                         if (ret) {
117                                 pr_debug("SPAACE configuration failed for liodn %d\n",
118                                          liodn);
119                                 return ret;
120                         }
121                 }
122         }
123
124         return ret;
125 }
126
127 static int map_win(int liodn, struct fsl_dma_domain *dma_domain)
128 {
129         int ret;
130         struct dma_window *wnd = &dma_domain->win_arr[0];
131         phys_addr_t wnd_addr = dma_domain->iommu_domain.geometry.aperture_start;
132         unsigned long flags;
133
134         spin_lock_irqsave(&iommu_lock, flags);
135         ret = pamu_config_ppaace(liodn, wnd_addr,
136                                  wnd->size,
137                                  ~(u32)0,
138                                  wnd->paddr >> PAMU_PAGE_SHIFT,
139                                  dma_domain->snoop_id, dma_domain->stash_id,
140                                  0, wnd->prot);
141         spin_unlock_irqrestore(&iommu_lock, flags);
142         if (ret)
143                 pr_debug("PAACE configuration failed for liodn %d\n", liodn);
144
145         return ret;
146 }
147
148 /* Map the DMA window corresponding to the LIODN */
149 static int map_liodn(int liodn, struct fsl_dma_domain *dma_domain)
150 {
151         if (dma_domain->win_cnt > 1)
152                 return map_subwins(liodn, dma_domain);
153         else
154                 return map_win(liodn, dma_domain);
155 }
156
157 /* Update window/subwindow mapping for the LIODN */
158 static int update_liodn(int liodn, struct fsl_dma_domain *dma_domain, u32 wnd_nr)
159 {
160         int ret;
161         struct dma_window *wnd = &dma_domain->win_arr[wnd_nr];
162         unsigned long flags;
163
164         spin_lock_irqsave(&iommu_lock, flags);
165         if (dma_domain->win_cnt > 1) {
166                 ret = pamu_config_spaace(liodn, dma_domain->win_cnt, wnd_nr,
167                                          wnd->size,
168                                          ~(u32)0,
169                                          wnd->paddr >> PAMU_PAGE_SHIFT,
170                                          dma_domain->snoop_id,
171                                          dma_domain->stash_id,
172                                          (wnd_nr > 0) ? 1 : 0,
173                                          wnd->prot);
174                 if (ret)
175                         pr_debug("Subwindow reconfiguration failed for liodn %d\n",
176                                  liodn);
177         } else {
178                 phys_addr_t wnd_addr;
179
180                 wnd_addr = dma_domain->iommu_domain.geometry.aperture_start;
181
182                 ret = pamu_config_ppaace(liodn, wnd_addr,
183                                          wnd->size,
184                                          ~(u32)0,
185                                          wnd->paddr >> PAMU_PAGE_SHIFT,
186                                          dma_domain->snoop_id, dma_domain->stash_id,
187                                          0, wnd->prot);
188                 if (ret)
189                         pr_debug("Window reconfiguration failed for liodn %d\n",
190                                  liodn);
191         }
192
193         spin_unlock_irqrestore(&iommu_lock, flags);
194
195         return ret;
196 }
197
198 static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain,
199                               u32 val)
200 {
201         int ret = 0, i;
202         unsigned long flags;
203
204         spin_lock_irqsave(&iommu_lock, flags);
205         if (!dma_domain->win_arr) {
206                 pr_debug("Windows not configured, stash destination update failed for liodn %d\n",
207                          liodn);
208                 spin_unlock_irqrestore(&iommu_lock, flags);
209                 return -EINVAL;
210         }
211
212         for (i = 0; i < dma_domain->win_cnt; i++) {
213                 ret = pamu_update_paace_stash(liodn, i, val);
214                 if (ret) {
215                         pr_debug("Failed to update SPAACE %d field for liodn %d\n ",
216                                  i, liodn);
217                         spin_unlock_irqrestore(&iommu_lock, flags);
218                         return ret;
219                 }
220         }
221
222         spin_unlock_irqrestore(&iommu_lock, flags);
223
224         return ret;
225 }
226
227 /* Set the geometry parameters for a LIODN */
228 static int pamu_set_liodn(int liodn, struct device *dev,
229                           struct fsl_dma_domain *dma_domain,
230                           struct iommu_domain_geometry *geom_attr,
231                           u32 win_cnt)
232 {
233         phys_addr_t window_addr, window_size;
234         phys_addr_t subwin_size;
235         int ret = 0, i;
236         u32 omi_index = ~(u32)0;
237         unsigned long flags;
238
239         /*
240          * Configure the omi_index at the geometry setup time.
241          * This is a static value which depends on the type of
242          * device and would not change thereafter.
243          */
244         get_ome_index(&omi_index, dev);
245
246         window_addr = geom_attr->aperture_start;
247         window_size = dma_domain->geom_size;
248
249         spin_lock_irqsave(&iommu_lock, flags);
250         ret = pamu_disable_liodn(liodn);
251         if (!ret)
252                 ret = pamu_config_ppaace(liodn, window_addr, window_size, omi_index,
253                                          0, dma_domain->snoop_id,
254                                          dma_domain->stash_id, win_cnt, 0);
255         spin_unlock_irqrestore(&iommu_lock, flags);
256         if (ret) {
257                 pr_debug("PAACE configuration failed for liodn %d, win_cnt =%d\n",
258                          liodn, win_cnt);
259                 return ret;
260         }
261
262         if (win_cnt > 1) {
263                 subwin_size = window_size >> ilog2(win_cnt);
264                 for (i = 0; i < win_cnt; i++) {
265                         spin_lock_irqsave(&iommu_lock, flags);
266                         ret = pamu_disable_spaace(liodn, i);
267                         if (!ret)
268                                 ret = pamu_config_spaace(liodn, win_cnt, i,
269                                                          subwin_size, omi_index,
270                                                          0, dma_domain->snoop_id,
271                                                          dma_domain->stash_id,
272                                                          0, 0);
273                         spin_unlock_irqrestore(&iommu_lock, flags);
274                         if (ret) {
275                                 pr_debug("SPAACE configuration failed for liodn %d\n",
276                                          liodn);
277                                 return ret;
278                         }
279                 }
280         }
281
282         return ret;
283 }
284
285 static int check_size(u64 size, dma_addr_t iova)
286 {
287         /*
288          * Size must be a power of two and at least be equal
289          * to PAMU page size.
290          */
291         if ((size & (size - 1)) || size < PAMU_PAGE_SIZE) {
292                 pr_debug("Size too small or not a power of two\n");
293                 return -EINVAL;
294         }
295
296         /* iova must be page size aligned */
297         if (iova & (size - 1)) {
298                 pr_debug("Address is not aligned with window size\n");
299                 return -EINVAL;
300         }
301
302         return 0;
303 }
304
305 static struct fsl_dma_domain *iommu_alloc_dma_domain(void)
306 {
307         struct fsl_dma_domain *domain;
308
309         domain = kmem_cache_zalloc(fsl_pamu_domain_cache, GFP_KERNEL);
310         if (!domain)
311                 return NULL;
312
313         domain->stash_id = ~(u32)0;
314         domain->snoop_id = ~(u32)0;
315         domain->win_cnt = pamu_get_max_subwin_cnt();
316         domain->geom_size = 0;
317
318         INIT_LIST_HEAD(&domain->devices);
319
320         spin_lock_init(&domain->domain_lock);
321
322         return domain;
323 }
324
325 static void remove_device_ref(struct device_domain_info *info, u32 win_cnt)
326 {
327         unsigned long flags;
328
329         list_del(&info->link);
330         spin_lock_irqsave(&iommu_lock, flags);
331         if (win_cnt > 1)
332                 pamu_free_subwins(info->liodn);
333         pamu_disable_liodn(info->liodn);
334         spin_unlock_irqrestore(&iommu_lock, flags);
335         spin_lock_irqsave(&device_domain_lock, flags);
336         info->dev->archdata.iommu_domain = NULL;
337         kmem_cache_free(iommu_devinfo_cache, info);
338         spin_unlock_irqrestore(&device_domain_lock, flags);
339 }
340
341 static void detach_device(struct device *dev, struct fsl_dma_domain *dma_domain)
342 {
343         struct device_domain_info *info, *tmp;
344         unsigned long flags;
345
346         spin_lock_irqsave(&dma_domain->domain_lock, flags);
347         /* Remove the device from the domain device list */
348         list_for_each_entry_safe(info, tmp, &dma_domain->devices, link) {
349                 if (!dev || (info->dev == dev))
350                         remove_device_ref(info, dma_domain->win_cnt);
351         }
352         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
353 }
354
355 static void attach_device(struct fsl_dma_domain *dma_domain, int liodn, struct device *dev)
356 {
357         struct device_domain_info *info, *old_domain_info;
358         unsigned long flags;
359
360         spin_lock_irqsave(&device_domain_lock, flags);
361         /*
362          * Check here if the device is already attached to domain or not.
363          * If the device is already attached to a domain detach it.
364          */
365         old_domain_info = dev->archdata.iommu_domain;
366         if (old_domain_info && old_domain_info->domain != dma_domain) {
367                 spin_unlock_irqrestore(&device_domain_lock, flags);
368                 detach_device(dev, old_domain_info->domain);
369                 spin_lock_irqsave(&device_domain_lock, flags);
370         }
371
372         info = kmem_cache_zalloc(iommu_devinfo_cache, GFP_ATOMIC);
373
374         info->dev = dev;
375         info->liodn = liodn;
376         info->domain = dma_domain;
377
378         list_add(&info->link, &dma_domain->devices);
379         /*
380          * In case of devices with multiple LIODNs just store
381          * the info for the first LIODN as all
382          * LIODNs share the same domain
383          */
384         if (!dev->archdata.iommu_domain)
385                 dev->archdata.iommu_domain = info;
386         spin_unlock_irqrestore(&device_domain_lock, flags);
387 }
388
389 static phys_addr_t fsl_pamu_iova_to_phys(struct iommu_domain *domain,
390                                          dma_addr_t iova)
391 {
392         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
393
394         if (iova < domain->geometry.aperture_start ||
395             iova > domain->geometry.aperture_end)
396                 return 0;
397
398         return get_phys_addr(dma_domain, iova);
399 }
400
401 static bool fsl_pamu_capable(enum iommu_cap cap)
402 {
403         return cap == IOMMU_CAP_CACHE_COHERENCY;
404 }
405
406 static void fsl_pamu_domain_free(struct iommu_domain *domain)
407 {
408         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
409
410         /* remove all the devices from the device list */
411         detach_device(NULL, dma_domain);
412
413         dma_domain->enabled = 0;
414         dma_domain->mapped = 0;
415
416         kmem_cache_free(fsl_pamu_domain_cache, dma_domain);
417 }
418
419 static struct iommu_domain *fsl_pamu_domain_alloc(unsigned type)
420 {
421         struct fsl_dma_domain *dma_domain;
422
423         if (type != IOMMU_DOMAIN_UNMANAGED)
424                 return NULL;
425
426         dma_domain = iommu_alloc_dma_domain();
427         if (!dma_domain) {
428                 pr_debug("dma_domain allocation failed\n");
429                 return NULL;
430         }
431         /* defaul geometry 64 GB i.e. maximum system address */
432         dma_domain->iommu_domain. geometry.aperture_start = 0;
433         dma_domain->iommu_domain.geometry.aperture_end = (1ULL << 36) - 1;
434         dma_domain->iommu_domain.geometry.force_aperture = true;
435
436         return &dma_domain->iommu_domain;
437 }
438
439 /* Configure geometry settings for all LIODNs associated with domain */
440 static int pamu_set_domain_geometry(struct fsl_dma_domain *dma_domain,
441                                     struct iommu_domain_geometry *geom_attr,
442                                     u32 win_cnt)
443 {
444         struct device_domain_info *info;
445         int ret = 0;
446
447         list_for_each_entry(info, &dma_domain->devices, link) {
448                 ret = pamu_set_liodn(info->liodn, info->dev, dma_domain,
449                                      geom_attr, win_cnt);
450                 if (ret)
451                         break;
452         }
453
454         return ret;
455 }
456
457 /* Update stash destination for all LIODNs associated with the domain */
458 static int update_domain_stash(struct fsl_dma_domain *dma_domain, u32 val)
459 {
460         struct device_domain_info *info;
461         int ret = 0;
462
463         list_for_each_entry(info, &dma_domain->devices, link) {
464                 ret = update_liodn_stash(info->liodn, dma_domain, val);
465                 if (ret)
466                         break;
467         }
468
469         return ret;
470 }
471
472 /* Update domain mappings for all LIODNs associated with the domain */
473 static int update_domain_mapping(struct fsl_dma_domain *dma_domain, u32 wnd_nr)
474 {
475         struct device_domain_info *info;
476         int ret = 0;
477
478         list_for_each_entry(info, &dma_domain->devices, link) {
479                 ret = update_liodn(info->liodn, dma_domain, wnd_nr);
480                 if (ret)
481                         break;
482         }
483         return ret;
484 }
485
486 static int disable_domain_win(struct fsl_dma_domain *dma_domain, u32 wnd_nr)
487 {
488         struct device_domain_info *info;
489         int ret = 0;
490
491         list_for_each_entry(info, &dma_domain->devices, link) {
492                 if (dma_domain->win_cnt == 1 && dma_domain->enabled) {
493                         ret = pamu_disable_liodn(info->liodn);
494                         if (!ret)
495                                 dma_domain->enabled = 0;
496                 } else {
497                         ret = pamu_disable_spaace(info->liodn, wnd_nr);
498                 }
499         }
500
501         return ret;
502 }
503
504 static void fsl_pamu_window_disable(struct iommu_domain *domain, u32 wnd_nr)
505 {
506         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
507         unsigned long flags;
508         int ret;
509
510         spin_lock_irqsave(&dma_domain->domain_lock, flags);
511         if (!dma_domain->win_arr) {
512                 pr_debug("Number of windows not configured\n");
513                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
514                 return;
515         }
516
517         if (wnd_nr >= dma_domain->win_cnt) {
518                 pr_debug("Invalid window index\n");
519                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
520                 return;
521         }
522
523         if (dma_domain->win_arr[wnd_nr].valid) {
524                 ret = disable_domain_win(dma_domain, wnd_nr);
525                 if (!ret) {
526                         dma_domain->win_arr[wnd_nr].valid = 0;
527                         dma_domain->mapped--;
528                 }
529         }
530
531         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
532 }
533
534 static int fsl_pamu_window_enable(struct iommu_domain *domain, u32 wnd_nr,
535                                   phys_addr_t paddr, u64 size, int prot)
536 {
537         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
538         struct dma_window *wnd;
539         int pamu_prot = 0;
540         int ret;
541         unsigned long flags;
542         u64 win_size;
543
544         if (prot & IOMMU_READ)
545                 pamu_prot |= PAACE_AP_PERMS_QUERY;
546         if (prot & IOMMU_WRITE)
547                 pamu_prot |= PAACE_AP_PERMS_UPDATE;
548
549         spin_lock_irqsave(&dma_domain->domain_lock, flags);
550         if (!dma_domain->win_arr) {
551                 pr_debug("Number of windows not configured\n");
552                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
553                 return -ENODEV;
554         }
555
556         if (wnd_nr >= dma_domain->win_cnt) {
557                 pr_debug("Invalid window index\n");
558                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
559                 return -EINVAL;
560         }
561
562         win_size = dma_domain->geom_size >> ilog2(dma_domain->win_cnt);
563         if (size > win_size) {
564                 pr_debug("Invalid window size\n");
565                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
566                 return -EINVAL;
567         }
568
569         if (dma_domain->win_cnt == 1) {
570                 if (dma_domain->enabled) {
571                         pr_debug("Disable the window before updating the mapping\n");
572                         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
573                         return -EBUSY;
574                 }
575
576                 ret = check_size(size, domain->geometry.aperture_start);
577                 if (ret) {
578                         pr_debug("Aperture start not aligned to the size\n");
579                         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
580                         return -EINVAL;
581                 }
582         }
583
584         wnd = &dma_domain->win_arr[wnd_nr];
585         if (!wnd->valid) {
586                 wnd->paddr = paddr;
587                 wnd->size = size;
588                 wnd->prot = pamu_prot;
589
590                 ret = update_domain_mapping(dma_domain, wnd_nr);
591                 if (!ret) {
592                         wnd->valid = 1;
593                         dma_domain->mapped++;
594                 }
595         } else {
596                 pr_debug("Disable the window before updating the mapping\n");
597                 ret = -EBUSY;
598         }
599
600         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
601
602         return ret;
603 }
604
605 /*
606  * Attach the LIODN to the DMA domain and configure the geometry
607  * and window mappings.
608  */
609 static int handle_attach_device(struct fsl_dma_domain *dma_domain,
610                                 struct device *dev, const u32 *liodn,
611                                 int num)
612 {
613         unsigned long flags;
614         struct iommu_domain *domain = &dma_domain->iommu_domain;
615         int ret = 0;
616         int i;
617
618         spin_lock_irqsave(&dma_domain->domain_lock, flags);
619         for (i = 0; i < num; i++) {
620                 /* Ensure that LIODN value is valid */
621                 if (liodn[i] >= PAACE_NUMBER_ENTRIES) {
622                         pr_debug("Invalid liodn %d, attach device failed for %s\n",
623                                  liodn[i], dev->of_node->full_name);
624                         ret = -EINVAL;
625                         break;
626                 }
627
628                 attach_device(dma_domain, liodn[i], dev);
629                 /*
630                  * Check if geometry has already been configured
631                  * for the domain. If yes, set the geometry for
632                  * the LIODN.
633                  */
634                 if (dma_domain->win_arr) {
635                         u32 win_cnt = dma_domain->win_cnt > 1 ? dma_domain->win_cnt : 0;
636
637                         ret = pamu_set_liodn(liodn[i], dev, dma_domain,
638                                              &domain->geometry, win_cnt);
639                         if (ret)
640                                 break;
641                         if (dma_domain->mapped) {
642                                 /*
643                                  * Create window/subwindow mapping for
644                                  * the LIODN.
645                                  */
646                                 ret = map_liodn(liodn[i], dma_domain);
647                                 if (ret)
648                                         break;
649                         }
650                 }
651         }
652         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
653
654         return ret;
655 }
656
657 static int fsl_pamu_attach_device(struct iommu_domain *domain,
658                                   struct device *dev)
659 {
660         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
661         const u32 *liodn;
662         u32 liodn_cnt;
663         int len, ret = 0;
664         struct pci_dev *pdev = NULL;
665         struct pci_controller *pci_ctl;
666
667         /*
668          * Use LIODN of the PCI controller while attaching a
669          * PCI device.
670          */
671         if (dev_is_pci(dev)) {
672                 pdev = to_pci_dev(dev);
673                 pci_ctl = pci_bus_to_host(pdev->bus);
674                 /*
675                  * make dev point to pci controller device
676                  * so we can get the LIODN programmed by
677                  * u-boot.
678                  */
679                 dev = pci_ctl->parent;
680         }
681
682         liodn = of_get_property(dev->of_node, "fsl,liodn", &len);
683         if (liodn) {
684                 liodn_cnt = len / sizeof(u32);
685                 ret = handle_attach_device(dma_domain, dev, liodn, liodn_cnt);
686         } else {
687                 pr_debug("missing fsl,liodn property at %s\n",
688                          dev->of_node->full_name);
689                 ret = -EINVAL;
690         }
691
692         return ret;
693 }
694
695 static void fsl_pamu_detach_device(struct iommu_domain *domain,
696                                    struct device *dev)
697 {
698         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
699         const u32 *prop;
700         int len;
701         struct pci_dev *pdev = NULL;
702         struct pci_controller *pci_ctl;
703
704         /*
705          * Use LIODN of the PCI controller while detaching a
706          * PCI device.
707          */
708         if (dev_is_pci(dev)) {
709                 pdev = to_pci_dev(dev);
710                 pci_ctl = pci_bus_to_host(pdev->bus);
711                 /*
712                  * make dev point to pci controller device
713                  * so we can get the LIODN programmed by
714                  * u-boot.
715                  */
716                 dev = pci_ctl->parent;
717         }
718
719         prop = of_get_property(dev->of_node, "fsl,liodn", &len);
720         if (prop)
721                 detach_device(dev, dma_domain);
722         else
723                 pr_debug("missing fsl,liodn property at %s\n",
724                          dev->of_node->full_name);
725 }
726
727 static  int configure_domain_geometry(struct iommu_domain *domain, void *data)
728 {
729         struct iommu_domain_geometry *geom_attr = data;
730         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
731         dma_addr_t geom_size;
732         unsigned long flags;
733
734         geom_size = geom_attr->aperture_end - geom_attr->aperture_start + 1;
735         /*
736          * Sanity check the geometry size. Also, we do not support
737          * DMA outside of the geometry.
738          */
739         if (check_size(geom_size, geom_attr->aperture_start) ||
740             !geom_attr->force_aperture) {
741                 pr_debug("Invalid PAMU geometry attributes\n");
742                 return -EINVAL;
743         }
744
745         spin_lock_irqsave(&dma_domain->domain_lock, flags);
746         if (dma_domain->enabled) {
747                 pr_debug("Can't set geometry attributes as domain is active\n");
748                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
749                 return  -EBUSY;
750         }
751
752         /* Copy the domain geometry information */
753         memcpy(&domain->geometry, geom_attr,
754                sizeof(struct iommu_domain_geometry));
755         dma_domain->geom_size = geom_size;
756
757         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
758
759         return 0;
760 }
761
762 /* Set the domain stash attribute */
763 static int configure_domain_stash(struct fsl_dma_domain *dma_domain, void *data)
764 {
765         struct pamu_stash_attribute *stash_attr = data;
766         unsigned long flags;
767         int ret;
768
769         spin_lock_irqsave(&dma_domain->domain_lock, flags);
770
771         memcpy(&dma_domain->dma_stash, stash_attr,
772                sizeof(struct pamu_stash_attribute));
773
774         dma_domain->stash_id = get_stash_id(stash_attr->cache,
775                                             stash_attr->cpu);
776         if (dma_domain->stash_id == ~(u32)0) {
777                 pr_debug("Invalid stash attributes\n");
778                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
779                 return -EINVAL;
780         }
781
782         ret = update_domain_stash(dma_domain, dma_domain->stash_id);
783
784         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
785
786         return ret;
787 }
788
789 /* Configure domain dma state i.e. enable/disable DMA */
790 static int configure_domain_dma_state(struct fsl_dma_domain *dma_domain, bool enable)
791 {
792         struct device_domain_info *info;
793         unsigned long flags;
794         int ret;
795
796         spin_lock_irqsave(&dma_domain->domain_lock, flags);
797
798         if (enable && !dma_domain->mapped) {
799                 pr_debug("Can't enable DMA domain without valid mapping\n");
800                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
801                 return -ENODEV;
802         }
803
804         dma_domain->enabled = enable;
805         list_for_each_entry(info, &dma_domain->devices, link) {
806                 ret = (enable) ? pamu_enable_liodn(info->liodn) :
807                         pamu_disable_liodn(info->liodn);
808                 if (ret)
809                         pr_debug("Unable to set dma state for liodn %d",
810                                  info->liodn);
811         }
812         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
813
814         return 0;
815 }
816
817 static int fsl_pamu_set_domain_attr(struct iommu_domain *domain,
818                                     enum iommu_attr attr_type, void *data)
819 {
820         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
821         int ret = 0;
822
823         switch (attr_type) {
824         case DOMAIN_ATTR_GEOMETRY:
825                 ret = configure_domain_geometry(domain, data);
826                 break;
827         case DOMAIN_ATTR_FSL_PAMU_STASH:
828                 ret = configure_domain_stash(dma_domain, data);
829                 break;
830         case DOMAIN_ATTR_FSL_PAMU_ENABLE:
831                 ret = configure_domain_dma_state(dma_domain, *(int *)data);
832                 break;
833         default:
834                 pr_debug("Unsupported attribute type\n");
835                 ret = -EINVAL;
836                 break;
837         }
838
839         return ret;
840 }
841
842 static int fsl_pamu_get_domain_attr(struct iommu_domain *domain,
843                                     enum iommu_attr attr_type, void *data)
844 {
845         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
846         int ret = 0;
847
848         switch (attr_type) {
849         case DOMAIN_ATTR_FSL_PAMU_STASH:
850                 memcpy(data, &dma_domain->dma_stash,
851                        sizeof(struct pamu_stash_attribute));
852                 break;
853         case DOMAIN_ATTR_FSL_PAMU_ENABLE:
854                 *(int *)data = dma_domain->enabled;
855                 break;
856         case DOMAIN_ATTR_FSL_PAMUV1:
857                 *(int *)data = DOMAIN_ATTR_FSL_PAMUV1;
858                 break;
859         default:
860                 pr_debug("Unsupported attribute type\n");
861                 ret = -EINVAL;
862                 break;
863         }
864
865         return ret;
866 }
867
868 static struct iommu_group *get_device_iommu_group(struct device *dev)
869 {
870         struct iommu_group *group;
871
872         group = iommu_group_get(dev);
873         if (!group)
874                 group = iommu_group_alloc();
875
876         return group;
877 }
878
879 static  bool check_pci_ctl_endpt_part(struct pci_controller *pci_ctl)
880 {
881         u32 version;
882
883         /* Check the PCI controller version number by readding BRR1 register */
884         version = in_be32(pci_ctl->cfg_addr + (PCI_FSL_BRR1 >> 2));
885         version &= PCI_FSL_BRR1_VER;
886         /* If PCI controller version is >= 0x204 we can partition endpoints */
887         return version >= 0x204;
888 }
889
890 /* Get iommu group information from peer devices or devices on the parent bus */
891 static struct iommu_group *get_shared_pci_device_group(struct pci_dev *pdev)
892 {
893         struct pci_dev *tmp;
894         struct iommu_group *group;
895         struct pci_bus *bus = pdev->bus;
896
897         /*
898          * Traverese the pci bus device list to get
899          * the shared iommu group.
900          */
901         while (bus) {
902                 list_for_each_entry(tmp, &bus->devices, bus_list) {
903                         if (tmp == pdev)
904                                 continue;
905                         group = iommu_group_get(&tmp->dev);
906                         if (group)
907                                 return group;
908                 }
909
910                 bus = bus->parent;
911         }
912
913         return NULL;
914 }
915
916 static struct iommu_group *get_pci_device_group(struct pci_dev *pdev)
917 {
918         struct pci_controller *pci_ctl;
919         bool pci_endpt_partioning;
920         struct iommu_group *group = NULL;
921
922         pci_ctl = pci_bus_to_host(pdev->bus);
923         pci_endpt_partioning = check_pci_ctl_endpt_part(pci_ctl);
924         /* We can partition PCIe devices so assign device group to the device */
925         if (pci_endpt_partioning) {
926                 group = iommu_group_get_for_dev(&pdev->dev);
927
928                 /*
929                  * PCIe controller is not a paritionable entity
930                  * free the controller device iommu_group.
931                  */
932                 if (pci_ctl->parent->iommu_group)
933                         iommu_group_remove_device(pci_ctl->parent);
934         } else {
935                 /*
936                  * All devices connected to the controller will share the
937                  * PCI controllers device group. If this is the first
938                  * device to be probed for the pci controller, copy the
939                  * device group information from the PCI controller device
940                  * node and remove the PCI controller iommu group.
941                  * For subsequent devices, the iommu group information can
942                  * be obtained from sibling devices (i.e. from the bus_devices
943                  * link list).
944                  */
945                 if (pci_ctl->parent->iommu_group) {
946                         group = get_device_iommu_group(pci_ctl->parent);
947                         iommu_group_remove_device(pci_ctl->parent);
948                 } else {
949                         group = get_shared_pci_device_group(pdev);
950                 }
951         }
952
953         if (!group)
954                 group = ERR_PTR(-ENODEV);
955
956         return group;
957 }
958
959 static int fsl_pamu_add_device(struct device *dev)
960 {
961         struct iommu_group *group = ERR_PTR(-ENODEV);
962         struct pci_dev *pdev;
963         const u32 *prop;
964         int ret = 0, len;
965
966         /*
967          * For platform devices we allocate a separate group for
968          * each of the devices.
969          */
970         if (dev_is_pci(dev)) {
971                 pdev = to_pci_dev(dev);
972                 /* Don't create device groups for virtual PCI bridges */
973                 if (pdev->subordinate)
974                         return 0;
975
976                 group = get_pci_device_group(pdev);
977
978         } else {
979                 prop = of_get_property(dev->of_node, "fsl,liodn", &len);
980                 if (prop)
981                         group = get_device_iommu_group(dev);
982         }
983
984         if (IS_ERR(group))
985                 return PTR_ERR(group);
986
987         /*
988          * Check if device has already been added to an iommu group.
989          * Group could have already been created for a PCI device in
990          * the iommu_group_get_for_dev path.
991          */
992         if (!dev->iommu_group)
993                 ret = iommu_group_add_device(group, dev);
994
995         iommu_group_put(group);
996         return ret;
997 }
998
999 static void fsl_pamu_remove_device(struct device *dev)
1000 {
1001         iommu_group_remove_device(dev);
1002 }
1003
1004 static int fsl_pamu_set_windows(struct iommu_domain *domain, u32 w_count)
1005 {
1006         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
1007         unsigned long flags;
1008         int ret;
1009
1010         spin_lock_irqsave(&dma_domain->domain_lock, flags);
1011         /* Ensure domain is inactive i.e. DMA should be disabled for the domain */
1012         if (dma_domain->enabled) {
1013                 pr_debug("Can't set geometry attributes as domain is active\n");
1014                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
1015                 return  -EBUSY;
1016         }
1017
1018         /* Ensure that the geometry has been set for the domain */
1019         if (!dma_domain->geom_size) {
1020                 pr_debug("Please configure geometry before setting the number of windows\n");
1021                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
1022                 return -EINVAL;
1023         }
1024
1025         /*
1026          * Ensure we have valid window count i.e. it should be less than
1027          * maximum permissible limit and should be a power of two.
1028          */
1029         if (w_count > pamu_get_max_subwin_cnt() || !is_power_of_2(w_count)) {
1030                 pr_debug("Invalid window count\n");
1031                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
1032                 return -EINVAL;
1033         }
1034
1035         ret = pamu_set_domain_geometry(dma_domain, &domain->geometry,
1036                                        w_count > 1 ? w_count : 0);
1037         if (!ret) {
1038                 kfree(dma_domain->win_arr);
1039                 dma_domain->win_arr = kcalloc(w_count,
1040                                               sizeof(*dma_domain->win_arr),
1041                                               GFP_ATOMIC);
1042                 if (!dma_domain->win_arr) {
1043                         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
1044                         return -ENOMEM;
1045                 }
1046                 dma_domain->win_cnt = w_count;
1047         }
1048         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
1049
1050         return ret;
1051 }
1052
1053 static u32 fsl_pamu_get_windows(struct iommu_domain *domain)
1054 {
1055         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
1056
1057         return dma_domain->win_cnt;
1058 }
1059
1060 static const struct iommu_ops fsl_pamu_ops = {
1061         .capable        = fsl_pamu_capable,
1062         .domain_alloc   = fsl_pamu_domain_alloc,
1063         .domain_free    = fsl_pamu_domain_free,
1064         .attach_dev     = fsl_pamu_attach_device,
1065         .detach_dev     = fsl_pamu_detach_device,
1066         .domain_window_enable = fsl_pamu_window_enable,
1067         .domain_window_disable = fsl_pamu_window_disable,
1068         .domain_get_windows = fsl_pamu_get_windows,
1069         .domain_set_windows = fsl_pamu_set_windows,
1070         .iova_to_phys   = fsl_pamu_iova_to_phys,
1071         .domain_set_attr = fsl_pamu_set_domain_attr,
1072         .domain_get_attr = fsl_pamu_get_domain_attr,
1073         .add_device     = fsl_pamu_add_device,
1074         .remove_device  = fsl_pamu_remove_device,
1075 };
1076
1077 int __init pamu_domain_init(void)
1078 {
1079         int ret = 0;
1080
1081         ret = iommu_init_mempool();
1082         if (ret)
1083                 return ret;
1084
1085         bus_set_iommu(&platform_bus_type, &fsl_pamu_ops);
1086         bus_set_iommu(&pci_bus_type, &fsl_pamu_ops);
1087
1088         return ret;
1089 }