drivers/gpu/drm/msm/msm_gpu.c

   1 /*
   2  * Copyright (C) 2013 Red Hat
   3  * Author: Rob Clark <robdclark@gmail.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 as published by
   7  * the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 #include "msm_gpu.h"
  19 #include "msm_gem.h"
  20 #include "msm_mmu.h"
  21
  22
  23 /*
  24  * Power Management:
  25  */
  26
  27 #ifdef CONFIG_MSM_BUS_SCALING
  28 #include <mach/board.h>
  29 static void bs_init(struct msm_gpu *gpu)
  30 {
  31         if (gpu->bus_scale_table) {
  32                 gpu->bsc = msm_bus_scale_register_client(gpu->bus_scale_table);
  33                 DBG("bus scale client: %08x", gpu->bsc);
  34         }
  35 }
  36
  37 static void bs_fini(struct msm_gpu *gpu)
  38 {
  39         if (gpu->bsc) {
  40                 msm_bus_scale_unregister_client(gpu->bsc);
  41                 gpu->bsc = 0;
  42         }
  43 }
  44
  45 static void bs_set(struct msm_gpu *gpu, int idx)
  46 {
  47         if (gpu->bsc) {
  48                 DBG("set bus scaling: %d", idx);
  49                 msm_bus_scale_client_update_request(gpu->bsc, idx);
  50         }
  51 }
  52 #else
  53 static void bs_init(struct msm_gpu *gpu) {}
  54 static void bs_fini(struct msm_gpu *gpu) {}
  55 static void bs_set(struct msm_gpu *gpu, int idx) {}
  56 #endif
  57
  58 static int enable_pwrrail(struct msm_gpu *gpu)
  59 {
  60         struct drm_device *dev = gpu->dev;
  61         int ret = 0;
  62
  63         if (gpu->gpu_reg) {
  64                 ret = regulator_enable(gpu->gpu_reg);
  65                 if (ret) {
  66                         dev_err(dev->dev, "failed to enable 'gpu_reg': %d\n", ret);
  67                         return ret;
  68                 }
  69         }
  70
  71         if (gpu->gpu_cx) {
  72                 ret = regulator_enable(gpu->gpu_cx);
  73                 if (ret) {
  74                         dev_err(dev->dev, "failed to enable 'gpu_cx': %d\n", ret);
  75                         return ret;
  76                 }
  77         }
  78
  79         return 0;
  80 }
  81
  82 static int disable_pwrrail(struct msm_gpu *gpu)
  83 {
  84         if (gpu->gpu_cx)
  85                 regulator_disable(gpu->gpu_cx);
  86         if (gpu->gpu_reg)
  87                 regulator_disable(gpu->gpu_reg);
  88         return 0;
  89 }
  90
  91 static int enable_clk(struct msm_gpu *gpu)
  92 {
  93         struct clk *rate_clk = NULL;
  94         int i;
  95
  96         /* NOTE: kgsl_pwrctrl_clk() ignores grp_clks[0].. */
  97         for (i = ARRAY_SIZE(gpu->grp_clks) - 1; i > 0; i--) {
  98                 if (gpu->grp_clks[i]) {
  99                         clk_prepare(gpu->grp_clks[i]);
 100                         rate_clk = gpu->grp_clks[i];
 101                 }
 102         }
 103
 104         if (rate_clk && gpu->fast_rate)
 105                 clk_set_rate(rate_clk, gpu->fast_rate);
 106
 107         for (i = ARRAY_SIZE(gpu->grp_clks) - 1; i > 0; i--)
 108                 if (gpu->grp_clks[i])
 109                         clk_enable(gpu->grp_clks[i]);
 110
 111         return 0;
 112 }
 113
 114 static int disable_clk(struct msm_gpu *gpu)
 115 {
 116         struct clk *rate_clk = NULL;
 117         int i;
 118
 119         /* NOTE: kgsl_pwrctrl_clk() ignores grp_clks[0].. */
 120         for (i = ARRAY_SIZE(gpu->grp_clks) - 1; i > 0; i--) {
 121                 if (gpu->grp_clks[i]) {
 122                         clk_disable(gpu->grp_clks[i]);
 123                         rate_clk = gpu->grp_clks[i];
 124                 }
 125         }
 126
 127         if (rate_clk && gpu->slow_rate)
 128                 clk_set_rate(rate_clk, gpu->slow_rate);
 129
 130         for (i = ARRAY_SIZE(gpu->grp_clks) - 1; i > 0; i--)
 131                 if (gpu->grp_clks[i])
 132                         clk_unprepare(gpu->grp_clks[i]);
 133
 134         return 0;
 135 }
 136
 137 static int enable_axi(struct msm_gpu *gpu)
 138 {
 139         if (gpu->ebi1_clk)
 140                 clk_prepare_enable(gpu->ebi1_clk);
 141         if (gpu->bus_freq)
 142                 bs_set(gpu, gpu->bus_freq);
 143         return 0;
 144 }
 145
 146 static int disable_axi(struct msm_gpu *gpu)
 147 {
 148         if (gpu->ebi1_clk)
 149                 clk_disable_unprepare(gpu->ebi1_clk);
 150         if (gpu->bus_freq)
 151                 bs_set(gpu, 0);
 152         return 0;
 153 }
 154
 155 int msm_gpu_pm_resume(struct msm_gpu *gpu)
 156 {
 157         int ret;
 158
 159         DBG("%s", gpu->name);
 160
 161         ret = enable_pwrrail(gpu);
 162         if (ret)
 163                 return ret;
 164
 165         ret = enable_clk(gpu);
 166         if (ret)
 167                 return ret;
 168
 169         ret = enable_axi(gpu);
 170         if (ret)
 171                 return ret;
 172
 173         return 0;
 174 }
 175
 176 int msm_gpu_pm_suspend(struct msm_gpu *gpu)
 177 {
 178         int ret;
 179
 180         DBG("%s", gpu->name);
 181
 182         ret = disable_axi(gpu);
 183         if (ret)
 184                 return ret;
 185
 186         ret = disable_clk(gpu);
 187         if (ret)
 188                 return ret;
 189
 190         ret = disable_pwrrail(gpu);
 191         if (ret)
 192                 return ret;
 193
 194         return 0;
 195 }
 196
 197 /*
 198  * Hangcheck detection for locked gpu:
 199  */
 200
 201 static void recover_worker(struct work_struct *work)
 202 {
 203         struct msm_gpu *gpu = container_of(work, struct msm_gpu, recover_work);
 204         struct drm_device *dev = gpu->dev;
 205
 206         dev_err(dev->dev, "%s: hangcheck recover!\n", gpu->name);
 207
 208         mutex_lock(&dev->struct_mutex);
 209         gpu->funcs->recover(gpu);
 210         mutex_unlock(&dev->struct_mutex);
 211
 212         msm_gpu_retire(gpu);
 213 }
 214
 215 static void hangcheck_timer_reset(struct msm_gpu *gpu)
 216 {
 217         DBG("%s", gpu->name);
 218         mod_timer(&gpu->hangcheck_timer,
 219                         round_jiffies_up(jiffies + DRM_MSM_HANGCHECK_JIFFIES));
 220 }
 221
 222 static void hangcheck_handler(unsigned long data)
 223 {
 224         struct msm_gpu *gpu = (struct msm_gpu *)data;
 225         struct drm_device *dev = gpu->dev;
 226         struct msm_drm_private *priv = dev->dev_private;
 227         uint32_t fence = gpu->funcs->last_fence(gpu);
 228
 229         if (fence != gpu->hangcheck_fence) {
 230                 /* some progress has been made.. ya! */
 231                 gpu->hangcheck_fence = fence;
 232         } else if (fence < gpu->submitted_fence) {
 233                 /* no progress and not done.. hung! */
 234                 gpu->hangcheck_fence = fence;
 235                 dev_err(dev->dev, "%s: hangcheck detected gpu lockup!\n",
 236                                 gpu->name);
 237                 dev_err(dev->dev, "%s:     completed fence: %u\n",
 238                                 gpu->name, fence);
 239                 dev_err(dev->dev, "%s:     submitted fence: %u\n",
 240                                 gpu->name, gpu->submitted_fence);
 241                 queue_work(priv->wq, &gpu->recover_work);
 242         }
 243
 244         /* if still more pending work, reset the hangcheck timer: */
 245         if (gpu->submitted_fence > gpu->hangcheck_fence)
 246                 hangcheck_timer_reset(gpu);
 247
 248         /* workaround for missing irq: */
 249         queue_work(priv->wq, &gpu->retire_work);
 250 }
 251
 252 /*
 253  * Cmdstream submission/retirement:
 254  */
 255
 256 static void retire_worker(struct work_struct *work)
 257 {
 258         struct msm_gpu *gpu = container_of(work, struct msm_gpu, retire_work);
 259         struct drm_device *dev = gpu->dev;
 260         uint32_t fence = gpu->funcs->last_fence(gpu);
 261
 262         msm_update_fence(gpu->dev, fence);
 263
 264         mutex_lock(&dev->struct_mutex);
 265
 266         while (!list_empty(&gpu->active_list)) {
 267                 struct msm_gem_object *obj;
 268
 269                 obj = list_first_entry(&gpu->active_list,
 270                                 struct msm_gem_object, mm_list);
 271
 272                 if ((obj->read_fence <= fence) &&
 273                                 (obj->write_fence <= fence)) {
 274                         /* move to inactive: */
 275                         msm_gem_move_to_inactive(&obj->base);
 276                         msm_gem_put_iova(&obj->base, gpu->id);
 277                         drm_gem_object_unreference(&obj->base);
 278                 } else {
 279                         break;
 280                 }
 281         }
 282
 283         mutex_unlock(&dev->struct_mutex);
 284 }
 285
 286 /* call from irq handler to schedule work to retire bo's */
 287 void msm_gpu_retire(struct msm_gpu *gpu)
 288 {
 289         struct msm_drm_private *priv = gpu->dev->dev_private;
 290         queue_work(priv->wq, &gpu->retire_work);
 291 }
 292
 293 /* add bo's to gpu's ring, and kick gpu: */
 294 int msm_gpu_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
 295                 struct msm_file_private *ctx)
 296 {
 297         struct drm_device *dev = gpu->dev;
 298         struct msm_drm_private *priv = dev->dev_private;
 299         int i, ret;
 300
 301         mutex_lock(&dev->struct_mutex);
 302
 303         submit->fence = ++priv->next_fence;
 304
 305         gpu->submitted_fence = submit->fence;
 306
 307         ret = gpu->funcs->submit(gpu, submit, ctx);
 308         priv->lastctx = ctx;
 309
 310         for (i = 0; i < submit->nr_bos; i++) {
 311                 struct msm_gem_object *msm_obj = submit->bos[i].obj;
 312
 313                 /* can't happen yet.. but when we add 2d support we'll have
 314                  * to deal w/ cross-ring synchronization:
 315                  */
 316                 WARN_ON(is_active(msm_obj) && (msm_obj->gpu != gpu));
 317
 318                 if (!is_active(msm_obj)) {
 319                         uint32_t iova;
 320
 321                         /* ring takes a reference to the bo and iova: */
 322                         drm_gem_object_reference(&msm_obj->base);
 323                         msm_gem_get_iova_locked(&msm_obj->base,
 324                                         submit->gpu->id, &iova);
 325                 }
 326
 327                 if (submit->bos[i].flags & MSM_SUBMIT_BO_READ)
 328                         msm_gem_move_to_active(&msm_obj->base, gpu, false, submit->fence);
 329
 330                 if (submit->bos[i].flags & MSM_SUBMIT_BO_WRITE)
 331                         msm_gem_move_to_active(&msm_obj->base, gpu, true, submit->fence);
 332         }
 333         hangcheck_timer_reset(gpu);
 334         mutex_unlock(&dev->struct_mutex);
 335
 336         return ret;
 337 }
 338
 339 /*
 340  * Init/Cleanup:
 341  */
 342
 343 static irqreturn_t irq_handler(int irq, void *data)
 344 {
 345         struct msm_gpu *gpu = data;
 346         return gpu->funcs->irq(gpu);
 347 }
 348
 349 static const char *clk_names[] = {
 350                 "src_clk", "core_clk", "iface_clk", "mem_clk", "mem_iface_clk",
 351 };
 352
 353 int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 354                 struct msm_gpu *gpu, const struct msm_gpu_funcs *funcs,
 355                 const char *name, const char *ioname, const char *irqname, int ringsz)
 356 {
 357         struct iommu_domain *iommu;
 358         int i, ret;
 359
 360         gpu->dev = drm;
 361         gpu->funcs = funcs;
 362         gpu->name = name;
 363
 364         INIT_LIST_HEAD(&gpu->active_list);
 365         INIT_WORK(&gpu->retire_work, retire_worker);
 366         INIT_WORK(&gpu->recover_work, recover_worker);
 367
 368         setup_timer(&gpu->hangcheck_timer, hangcheck_handler,
 369                         (unsigned long)gpu);
 370
 371         BUG_ON(ARRAY_SIZE(clk_names) != ARRAY_SIZE(gpu->grp_clks));
 372
 373         /* Map registers: */
 374         gpu->mmio = msm_ioremap(pdev, ioname, name);
 375         if (IS_ERR(gpu->mmio)) {
 376                 ret = PTR_ERR(gpu->mmio);
 377                 goto fail;
 378         }
 379
 380         /* Get Interrupt: */
 381         gpu->irq = platform_get_irq_byname(pdev, irqname);
 382         if (gpu->irq < 0) {
 383                 ret = gpu->irq;
 384                 dev_err(drm->dev, "failed to get irq: %d\n", ret);
 385                 goto fail;
 386         }
 387
 388         ret = devm_request_irq(&pdev->dev, gpu->irq, irq_handler,
 389                         IRQF_TRIGGER_HIGH, gpu->name, gpu);
 390         if (ret) {
 391                 dev_err(drm->dev, "failed to request IRQ%u: %d\n", gpu->irq, ret);
 392                 goto fail;
 393         }
 394
 395         /* Acquire clocks: */
 396         for (i = 0; i < ARRAY_SIZE(clk_names); i++) {
 397                 gpu->grp_clks[i] = devm_clk_get(&pdev->dev, clk_names[i]);
 398                 DBG("grp_clks[%s]: %p", clk_names[i], gpu->grp_clks[i]);
 399                 if (IS_ERR(gpu->grp_clks[i]))
 400                         gpu->grp_clks[i] = NULL;
 401         }
 402
 403         gpu->ebi1_clk = devm_clk_get(&pdev->dev, "bus_clk");
 404         DBG("ebi1_clk: %p", gpu->ebi1_clk);
 405         if (IS_ERR(gpu->ebi1_clk))
 406                 gpu->ebi1_clk = NULL;
 407
 408         /* Acquire regulators: */
 409         gpu->gpu_reg = devm_regulator_get(&pdev->dev, "vdd");
 410         DBG("gpu_reg: %p", gpu->gpu_reg);
 411         if (IS_ERR(gpu->gpu_reg))
 412                 gpu->gpu_reg = NULL;
 413
 414         gpu->gpu_cx = devm_regulator_get(&pdev->dev, "vddcx");
 415         DBG("gpu_cx: %p", gpu->gpu_cx);
 416         if (IS_ERR(gpu->gpu_cx))
 417                 gpu->gpu_cx = NULL;
 418
 419         /* Setup IOMMU.. eventually we will (I think) do this once per context
 420          * and have separate page tables per context.  For now, to keep things
 421          * simple and to get something working, just use a single address space:
 422          */
 423         iommu = iommu_domain_alloc(&platform_bus_type);
 424         if (iommu) {
 425                 dev_info(drm->dev, "%s: using IOMMU\n", name);
 426                 gpu->mmu = msm_iommu_new(drm, iommu);
 427         } else {
 428                 dev_info(drm->dev, "%s: no IOMMU, fallback to VRAM carveout!\n", name);
 429         }
 430         gpu->id = msm_register_mmu(drm, gpu->mmu);
 431
 432         /* Create ringbuffer: */
 433         gpu->rb = msm_ringbuffer_new(gpu, ringsz);
 434         if (IS_ERR(gpu->rb)) {
 435                 ret = PTR_ERR(gpu->rb);
 436                 gpu->rb = NULL;
 437                 dev_err(drm->dev, "could not create ringbuffer: %d\n", ret);
 438                 goto fail;
 439         }
 440
 441         ret = msm_gem_get_iova_locked(gpu->rb->bo, gpu->id, &gpu->rb_iova);
 442         if (ret) {
 443                 gpu->rb_iova = 0;
 444                 dev_err(drm->dev, "could not map ringbuffer: %d\n", ret);
 445                 goto fail;
 446         }
 447
 448         bs_init(gpu);
 449
 450         return 0;
 451
 452 fail:
 453         return ret;
 454 }
 455
 456 void msm_gpu_cleanup(struct msm_gpu *gpu)
 457 {
 458         DBG("%s", gpu->name);
 459
 460         WARN_ON(!list_empty(&gpu->active_list));
 461
 462         bs_fini(gpu);
 463
 464         if (gpu->rb) {
 465                 if (gpu->rb_iova)
 466                         msm_gem_put_iova(gpu->rb->bo, gpu->id);
 467                 msm_ringbuffer_destroy(gpu->rb);
 468         }
 469
 470         if (gpu->mmu)
 471                 gpu->mmu->funcs->destroy(gpu->mmu);
 472 }