drivers/gpu/drm/amd/scheduler/gpu_scheduler.c

   1 /*
   2  * Copyright 2015 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  *
  23  */
  24 #include <linux/kthread.h>
  25 #include <linux/wait.h>
  26 #include <linux/sched.h>
  27 #include <drm/drmP.h>
  28 #include "gpu_scheduler.h"
  29
  30 #define CREATE_TRACE_POINTS
  31 #include "gpu_sched_trace.h"
  32
  33 static bool amd_sched_entity_is_ready(struct amd_sched_entity *entity);
  34 static void amd_sched_wakeup(struct amd_gpu_scheduler *sched);
  35
  36 struct kmem_cache *sched_fence_slab;
  37 atomic_t sched_fence_slab_ref = ATOMIC_INIT(0);
  38
  39 /* Initialize a given run queue struct */
  40 static void amd_sched_rq_init(struct amd_sched_rq *rq)
  41 {
  42         spin_lock_init(&rq->lock);
  43         INIT_LIST_HEAD(&rq->entities);
  44         rq->current_entity = NULL;
  45 }
  46
  47 static void amd_sched_rq_add_entity(struct amd_sched_rq *rq,
  48                                     struct amd_sched_entity *entity)
  49 {
  50         if (!list_empty(&entity->list))
  51                 return;
  52         spin_lock(&rq->lock);
  53         list_add_tail(&entity->list, &rq->entities);
  54         spin_unlock(&rq->lock);
  55 }
  56
  57 static void amd_sched_rq_remove_entity(struct amd_sched_rq *rq,
  58                                        struct amd_sched_entity *entity)
  59 {
  60         if (list_empty(&entity->list))
  61                 return;
  62         spin_lock(&rq->lock);
  63         list_del_init(&entity->list);
  64         if (rq->current_entity == entity)
  65                 rq->current_entity = NULL;
  66         spin_unlock(&rq->lock);
  67 }
  68
  69 /**
  70  * Select an entity which could provide a job to run
  71  *
  72  * @rq          The run queue to check.
  73  *
  74  * Try to find a ready entity, returns NULL if none found.
  75  */
  76 static struct amd_sched_entity *
  77 amd_sched_rq_select_entity(struct amd_sched_rq *rq)
  78 {
  79         struct amd_sched_entity *entity;
  80
  81         spin_lock(&rq->lock);
  82
  83         entity = rq->current_entity;
  84         if (entity) {
  85                 list_for_each_entry_continue(entity, &rq->entities, list) {
  86                         if (amd_sched_entity_is_ready(entity)) {
  87                                 rq->current_entity = entity;
  88                                 spin_unlock(&rq->lock);
  89                                 return entity;
  90                         }
  91                 }
  92         }
  93
  94         list_for_each_entry(entity, &rq->entities, list) {
  95
  96                 if (amd_sched_entity_is_ready(entity)) {
  97                         rq->current_entity = entity;
  98                         spin_unlock(&rq->lock);
  99                         return entity;
 100                 }
 101
 102                 if (entity == rq->current_entity)
 103                         break;
 104         }
 105
 106         spin_unlock(&rq->lock);
 107
 108         return NULL;
 109 }
 110
 111 /**
 112  * Init a context entity used by scheduler when submit to HW ring.
 113  *
 114  * @sched       The pointer to the scheduler
 115  * @entity      The pointer to a valid amd_sched_entity
 116  * @rq          The run queue this entity belongs
 117  * @kernel      If this is an entity for the kernel
 118  * @jobs        The max number of jobs in the job queue
 119  *
 120  * return 0 if succeed. negative error code on failure
 121 */
 122 int amd_sched_entity_init(struct amd_gpu_scheduler *sched,
 123                           struct amd_sched_entity *entity,
 124                           struct amd_sched_rq *rq,
 125                           uint32_t jobs)
 126 {
 127         int r;
 128
 129         if (!(sched && entity && rq))
 130                 return -EINVAL;
 131
 132         memset(entity, 0, sizeof(struct amd_sched_entity));
 133         INIT_LIST_HEAD(&entity->list);
 134         entity->rq = rq;
 135         entity->sched = sched;
 136
 137         spin_lock_init(&entity->queue_lock);
 138         r = kfifo_alloc(&entity->job_queue, jobs * sizeof(void *), GFP_KERNEL);
 139         if (r)
 140                 return r;
 141
 142         atomic_set(&entity->fence_seq, 0);
 143         entity->fence_context = fence_context_alloc(1);
 144
 145         return 0;
 146 }
 147
 148 /**
 149  * Query if entity is initialized
 150  *
 151  * @sched       Pointer to scheduler instance
 152  * @entity      The pointer to a valid scheduler entity
 153  *
 154  * return true if entity is initialized, false otherwise
 155 */
 156 static bool amd_sched_entity_is_initialized(struct amd_gpu_scheduler *sched,
 157                                             struct amd_sched_entity *entity)
 158 {
 159         return entity->sched == sched &&
 160                 entity->rq != NULL;
 161 }
 162
 163 /**
 164  * Check if entity is idle
 165  *
 166  * @entity      The pointer to a valid scheduler entity
 167  *
 168  * Return true if entity don't has any unscheduled jobs.
 169  */
 170 static bool amd_sched_entity_is_idle(struct amd_sched_entity *entity)
 171 {
 172         rmb();
 173         if (kfifo_is_empty(&entity->job_queue))
 174                 return true;
 175
 176         return false;
 177 }
 178
 179 /**
 180  * Check if entity is ready
 181  *
 182  * @entity      The pointer to a valid scheduler entity
 183  *
 184  * Return true if entity could provide a job.
 185  */
 186 static bool amd_sched_entity_is_ready(struct amd_sched_entity *entity)
 187 {
 188         if (kfifo_is_empty(&entity->job_queue))
 189                 return false;
 190
 191         if (ACCESS_ONCE(entity->dependency))
 192                 return false;
 193
 194         return true;
 195 }
 196
 197 /**
 198  * Destroy a context entity
 199  *
 200  * @sched       Pointer to scheduler instance
 201  * @entity      The pointer to a valid scheduler entity
 202  *
 203  * Cleanup and free the allocated resources.
 204  */
 205 void amd_sched_entity_fini(struct amd_gpu_scheduler *sched,
 206                            struct amd_sched_entity *entity)
 207 {
 208         struct amd_sched_rq *rq = entity->rq;
 209
 210         if (!amd_sched_entity_is_initialized(sched, entity))
 211                 return;
 212
 213         /**
 214          * The client will not queue more IBs during this fini, consume existing
 215          * queued IBs
 216         */
 217         wait_event(sched->job_scheduled, amd_sched_entity_is_idle(entity));
 218
 219         amd_sched_rq_remove_entity(rq, entity);
 220         kfifo_free(&entity->job_queue);
 221 }
 222
 223 static void amd_sched_entity_wakeup(struct fence *f, struct fence_cb *cb)
 224 {
 225         struct amd_sched_entity *entity =
 226                 container_of(cb, struct amd_sched_entity, cb);
 227         entity->dependency = NULL;
 228         fence_put(f);
 229         amd_sched_wakeup(entity->sched);
 230 }
 231
 232 static void amd_sched_entity_clear_dep(struct fence *f, struct fence_cb *cb)
 233 {
 234         struct amd_sched_entity *entity =
 235                 container_of(cb, struct amd_sched_entity, cb);
 236         entity->dependency = NULL;
 237         fence_put(f);
 238 }
 239
 240 static bool amd_sched_entity_add_dependency_cb(struct amd_sched_entity *entity)
 241 {
 242         struct amd_gpu_scheduler *sched = entity->sched;
 243         struct fence * fence = entity->dependency;
 244         struct amd_sched_fence *s_fence;
 245
 246         if (fence->context == entity->fence_context) {
 247                 /* We can ignore fences from ourself */
 248                 fence_put(entity->dependency);
 249                 return false;
 250         }
 251
 252         s_fence = to_amd_sched_fence(fence);
 253         if (s_fence && s_fence->sched == sched) {
 254                 /* Fence is from the same scheduler */
 255                 if (test_bit(AMD_SCHED_FENCE_SCHEDULED_BIT, &fence->flags)) {
 256                         /* Ignore it when it is already scheduled */
 257                         fence_put(entity->dependency);
 258                         return false;
 259                 }
 260
 261                 /* Wait for fence to be scheduled */
 262                 entity->cb.func = amd_sched_entity_clear_dep;
 263                 list_add_tail(&entity->cb.node, &s_fence->scheduled_cb);
 264                 return true;
 265         }
 266
 267         if (!fence_add_callback(entity->dependency, &entity->cb,
 268                                 amd_sched_entity_wakeup))
 269                 return true;
 270
 271         fence_put(entity->dependency);
 272         return false;
 273 }
 274
 275 static struct amd_sched_job *
 276 amd_sched_entity_pop_job(struct amd_sched_entity *entity)
 277 {
 278         struct amd_gpu_scheduler *sched = entity->sched;
 279         struct amd_sched_job *sched_job;
 280
 281         if (!kfifo_out_peek(&entity->job_queue, &sched_job, sizeof(sched_job)))
 282                 return NULL;
 283
 284         while ((entity->dependency = sched->ops->dependency(sched_job)))
 285                 if (amd_sched_entity_add_dependency_cb(entity))
 286                         return NULL;
 287
 288         return sched_job;
 289 }
 290
 291 /**
 292  * Helper to submit a job to the job queue
 293  *
 294  * @sched_job           The pointer to job required to submit
 295  *
 296  * Returns true if we could submit the job.
 297  */
 298 static bool amd_sched_entity_in(struct amd_sched_job *sched_job)
 299 {
 300         struct amd_gpu_scheduler *sched = sched_job->sched;
 301         struct amd_sched_entity *entity = sched_job->s_entity;
 302         bool added, first = false;
 303
 304         spin_lock(&entity->queue_lock);
 305         added = kfifo_in(&entity->job_queue, &sched_job,
 306                         sizeof(sched_job)) == sizeof(sched_job);
 307
 308         if (added && kfifo_len(&entity->job_queue) == sizeof(sched_job))
 309                 first = true;
 310
 311         spin_unlock(&entity->queue_lock);
 312
 313         /* first job wakes up scheduler */
 314         if (first) {
 315                 /* Add the entity to the run queue */
 316                 amd_sched_rq_add_entity(entity->rq, entity);
 317                 amd_sched_wakeup(sched);
 318         }
 319         return added;
 320 }
 321
 322 static void amd_sched_free_job(struct fence *f, struct fence_cb *cb) {
 323         struct amd_sched_job *job = container_of(cb, struct amd_sched_job, cb_free_job);
 324         schedule_work(&job->work_free_job);
 325 }
 326
 327 /* job_finish is called after hw fence signaled, and
 328  * the job had already been deleted from ring_mirror_list
 329  */
 330 void amd_sched_job_finish(struct amd_sched_job *s_job)
 331 {
 332         struct amd_sched_job *next;
 333         struct amd_gpu_scheduler *sched = s_job->sched;
 334
 335         if (sched->timeout != MAX_SCHEDULE_TIMEOUT) {
 336                 if (cancel_delayed_work(&s_job->work_tdr))
 337                         amd_sched_job_put(s_job);
 338
 339                 /* queue TDR for next job */
 340                 next = list_first_entry_or_null(&sched->ring_mirror_list,
 341                                                 struct amd_sched_job, node);
 342
 343                 if (next) {
 344                         INIT_DELAYED_WORK(&next->work_tdr, s_job->timeout_callback);
 345                         amd_sched_job_get(next);
 346                         schedule_delayed_work(&next->work_tdr, sched->timeout);
 347                 }
 348         }
 349 }
 350
 351 void amd_sched_job_begin(struct amd_sched_job *s_job)
 352 {
 353         struct amd_gpu_scheduler *sched = s_job->sched;
 354
 355         if (sched->timeout != MAX_SCHEDULE_TIMEOUT &&
 356                 list_first_entry_or_null(&sched->ring_mirror_list, struct amd_sched_job, node) == s_job)
 357         {
 358                 INIT_DELAYED_WORK(&s_job->work_tdr, s_job->timeout_callback);
 359                 amd_sched_job_get(s_job);
 360                 schedule_delayed_work(&s_job->work_tdr, sched->timeout);
 361         }
 362 }
 363
 364 /**
 365  * Submit a job to the job queue
 366  *
 367  * @sched_job           The pointer to job required to submit
 368  *
 369  * Returns 0 for success, negative error code otherwise.
 370  */
 371 void amd_sched_entity_push_job(struct amd_sched_job *sched_job)
 372 {
 373         struct amd_sched_entity *entity = sched_job->s_entity;
 374
 375         sched_job->use_sched = 1;
 376         fence_add_callback(&sched_job->s_fence->base,
 377                                         &sched_job->cb_free_job, amd_sched_free_job);
 378         trace_amd_sched_job(sched_job);
 379         wait_event(entity->sched->job_scheduled,
 380                    amd_sched_entity_in(sched_job));
 381 }
 382
 383 /* init a sched_job with basic field */
 384 int amd_sched_job_init(struct amd_sched_job *job,
 385                                                 struct amd_gpu_scheduler *sched,
 386                                                 struct amd_sched_entity *entity,
 387                                                 void (*timeout_cb)(struct work_struct *work),
 388                                                 void (*free_cb)(struct kref *refcount),
 389                                                 void *owner, struct fence **fence)
 390 {
 391         INIT_LIST_HEAD(&job->node);
 392         kref_init(&job->refcount);
 393         job->sched = sched;
 394         job->s_entity = entity;
 395         job->s_fence = amd_sched_fence_create(entity, owner);
 396         if (!job->s_fence)
 397                 return -ENOMEM;
 398
 399         job->s_fence->s_job = job;
 400         job->timeout_callback = timeout_cb;
 401         job->free_callback = free_cb;
 402
 403         if (fence)
 404                 *fence = &job->s_fence->base;
 405         return 0;
 406 }
 407
 408 /**
 409  * Return ture if we can push more jobs to the hw.
 410  */
 411 static bool amd_sched_ready(struct amd_gpu_scheduler *sched)
 412 {
 413         return atomic_read(&sched->hw_rq_count) <
 414                 sched->hw_submission_limit;
 415 }
 416
 417 /**
 418  * Wake up the scheduler when it is ready
 419  */
 420 static void amd_sched_wakeup(struct amd_gpu_scheduler *sched)
 421 {
 422         if (amd_sched_ready(sched))
 423                 wake_up_interruptible(&sched->wake_up_worker);
 424 }
 425
 426 /**
 427  * Select next entity to process
 428 */
 429 static struct amd_sched_entity *
 430 amd_sched_select_entity(struct amd_gpu_scheduler *sched)
 431 {
 432         struct amd_sched_entity *entity;
 433         int i;
 434
 435         if (!amd_sched_ready(sched))
 436                 return NULL;
 437
 438         /* Kernel run queue has higher priority than normal run queue*/
 439         for (i = 0; i < AMD_SCHED_MAX_PRIORITY; i++) {
 440                 entity = amd_sched_rq_select_entity(&sched->sched_rq[i]);
 441                 if (entity)
 442                         break;
 443         }
 444
 445         return entity;
 446 }
 447
 448 static void amd_sched_process_job(struct fence *f, struct fence_cb *cb)
 449 {
 450         struct amd_sched_fence *s_fence =
 451                 container_of(cb, struct amd_sched_fence, cb);
 452         struct amd_gpu_scheduler *sched = s_fence->sched;
 453         unsigned long flags;
 454
 455         atomic_dec(&sched->hw_rq_count);
 456
 457         /* remove job from ring_mirror_list */
 458         spin_lock_irqsave(&sched->job_list_lock, flags);
 459         list_del_init(&s_fence->s_job->node);
 460         sched->ops->finish_job(s_fence->s_job);
 461         spin_unlock_irqrestore(&sched->job_list_lock, flags);
 462
 463         amd_sched_fence_signal(s_fence);
 464
 465         trace_amd_sched_process_job(s_fence);
 466         fence_put(&s_fence->base);
 467         wake_up_interruptible(&sched->wake_up_worker);
 468 }
 469
 470 static int amd_sched_main(void *param)
 471 {
 472         struct sched_param sparam = {.sched_priority = 1};
 473         struct amd_gpu_scheduler *sched = (struct amd_gpu_scheduler *)param;
 474         int r, count;
 475
 476         sched_setscheduler(current, SCHED_FIFO, &sparam);
 477
 478         while (!kthread_should_stop()) {
 479                 struct amd_sched_entity *entity;
 480                 struct amd_sched_fence *s_fence;
 481                 struct amd_sched_job *sched_job;
 482                 struct fence *fence;
 483
 484                 wait_event_interruptible(sched->wake_up_worker,
 485                         (entity = amd_sched_select_entity(sched)) ||
 486                         kthread_should_stop());
 487
 488                 if (!entity)
 489                         continue;
 490
 491                 sched_job = amd_sched_entity_pop_job(entity);
 492                 if (!sched_job)
 493                         continue;
 494
 495                 s_fence = sched_job->s_fence;
 496
 497                 atomic_inc(&sched->hw_rq_count);
 498                 amd_sched_job_pre_schedule(sched, sched_job);
 499                 fence = sched->ops->run_job(sched_job);
 500                 amd_sched_fence_scheduled(s_fence);
 501                 if (fence) {
 502                         r = fence_add_callback(fence, &s_fence->cb,
 503                                                amd_sched_process_job);
 504                         if (r == -ENOENT)
 505                                 amd_sched_process_job(fence, &s_fence->cb);
 506                         else if (r)
 507                                 DRM_ERROR("fence add callback failed (%d)\n", r);
 508                         fence_put(fence);
 509                 } else {
 510                         DRM_ERROR("Failed to run job!\n");
 511                         amd_sched_process_job(NULL, &s_fence->cb);
 512                 }
 513
 514                 count = kfifo_out(&entity->job_queue, &sched_job,
 515                                 sizeof(sched_job));
 516                 WARN_ON(count != sizeof(sched_job));
 517                 wake_up(&sched->job_scheduled);
 518         }
 519         return 0;
 520 }
 521
 522 /**
 523  * Init a gpu scheduler instance
 524  *
 525  * @sched               The pointer to the scheduler
 526  * @ops                 The backend operations for this scheduler.
 527  * @hw_submissions      Number of hw submissions to do.
 528  * @name                Name used for debugging
 529  *
 530  * Return 0 on success, otherwise error code.
 531 */
 532 int amd_sched_init(struct amd_gpu_scheduler *sched,
 533                    const struct amd_sched_backend_ops *ops,
 534                    unsigned hw_submission, long timeout, const char *name)
 535 {
 536         int i;
 537         sched->ops = ops;
 538         sched->hw_submission_limit = hw_submission;
 539         sched->name = name;
 540         sched->timeout = timeout;
 541         for (i = 0; i < AMD_SCHED_MAX_PRIORITY; i++)
 542                 amd_sched_rq_init(&sched->sched_rq[i]);
 543
 544         init_waitqueue_head(&sched->wake_up_worker);
 545         init_waitqueue_head(&sched->job_scheduled);
 546         INIT_LIST_HEAD(&sched->ring_mirror_list);
 547         spin_lock_init(&sched->job_list_lock);
 548         atomic_set(&sched->hw_rq_count, 0);
 549         if (atomic_inc_return(&sched_fence_slab_ref) == 1) {
 550                 sched_fence_slab = kmem_cache_create(
 551                         "amd_sched_fence", sizeof(struct amd_sched_fence), 0,
 552                         SLAB_HWCACHE_ALIGN, NULL);
 553                 if (!sched_fence_slab)
 554                         return -ENOMEM;
 555         }
 556
 557         /* Each scheduler will run on a seperate kernel thread */
 558         sched->thread = kthread_run(amd_sched_main, sched, sched->name);
 559         if (IS_ERR(sched->thread)) {
 560                 DRM_ERROR("Failed to create scheduler for %s.\n", name);
 561                 return PTR_ERR(sched->thread);
 562         }
 563
 564         return 0;
 565 }
 566
 567 /**
 568  * Destroy a gpu scheduler
 569  *
 570  * @sched       The pointer to the scheduler
 571  */
 572 void amd_sched_fini(struct amd_gpu_scheduler *sched)
 573 {
 574         if (sched->thread)
 575                 kthread_stop(sched->thread);
 576         if (atomic_dec_and_test(&sched_fence_slab_ref))
 577                 kmem_cache_destroy(sched_fence_slab);
 578 }