drivers/gpu/drm/amd/scheduler/gpu_scheduler.c

   1 /*
   2  * Copyright 2015 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  *
  23  */
  24 #include <linux/kthread.h>
  25 #include <linux/wait.h>
  26 #include <linux/sched.h>
  27 #include <drm/drmP.h>
  28 #include "gpu_scheduler.h"
  29
  30 /* Initialize a given run queue struct */
  31 static void amd_sched_rq_init(struct amd_sched_rq *rq)
  32 {
  33         spin_lock_init(&rq->lock);
  34         INIT_LIST_HEAD(&rq->entities);
  35         rq->current_entity = NULL;
  36 }
  37
  38 static void amd_sched_rq_add_entity(struct amd_sched_rq *rq,
  39                                     struct amd_sched_entity *entity)
  40 {
  41         spin_lock(&rq->lock);
  42         list_add_tail(&entity->list, &rq->entities);
  43         spin_unlock(&rq->lock);
  44 }
  45
  46 static void amd_sched_rq_remove_entity(struct amd_sched_rq *rq,
  47                                        struct amd_sched_entity *entity)
  48 {
  49         spin_lock(&rq->lock);
  50         list_del_init(&entity->list);
  51         if (rq->current_entity == entity)
  52                 rq->current_entity = NULL;
  53         spin_unlock(&rq->lock);
  54 }
  55
  56 /**
  57  * Select next entity from a specified run queue with round robin policy.
  58  * It could return the same entity as current one if current is the only
  59  * available one in the queue. Return NULL if nothing available.
  60  */
  61 static struct amd_sched_entity *
  62 amd_sched_rq_select_entity(struct amd_sched_rq *rq)
  63 {
  64         struct amd_sched_entity *entity;
  65
  66         spin_lock(&rq->lock);
  67
  68         entity = rq->current_entity;
  69         if (entity) {
  70                 list_for_each_entry_continue(entity, &rq->entities, list) {
  71                         if (!kfifo_is_empty(&entity->job_queue)) {
  72                                 rq->current_entity = entity;
  73                                 spin_unlock(&rq->lock);
  74                                 return rq->current_entity;
  75                         }
  76                 }
  77         }
  78
  79         list_for_each_entry(entity, &rq->entities, list) {
  80
  81                 if (!kfifo_is_empty(&entity->job_queue)) {
  82                         rq->current_entity = entity;
  83                         spin_unlock(&rq->lock);
  84                         return rq->current_entity;
  85                 }
  86
  87                 if (entity == rq->current_entity)
  88                         break;
  89         }
  90
  91         spin_unlock(&rq->lock);
  92
  93         return NULL;
  94 }
  95
  96 /**
  97  * Note: This function should only been called inside scheduler main
  98  * function for thread safety, there is no other protection here.
  99  * return ture if scheduler has something ready to run.
 100  *
 101  * For active_hw_rq, there is only one producer(scheduler thread) and
 102  * one consumer(ISR). It should be safe to use this function in scheduler
 103  * main thread to decide whether to continue emit more IBs.
 104 */
 105 static bool is_scheduler_ready(struct amd_gpu_scheduler *sched)
 106 {
 107         unsigned long flags;
 108         bool full;
 109
 110         spin_lock_irqsave(&sched->queue_lock, flags);
 111         full = atomic64_read(&sched->hw_rq_count) <
 112                 sched->hw_submission_limit ? true : false;
 113         spin_unlock_irqrestore(&sched->queue_lock, flags);
 114
 115         return full;
 116 }
 117
 118 /**
 119  * Select next entity containing real IB submissions
 120 */
 121 static struct amd_sched_entity *
 122 select_context(struct amd_gpu_scheduler *sched)
 123 {
 124         struct amd_sched_entity *wake_entity = NULL;
 125         struct amd_sched_entity *tmp;
 126
 127         if (!is_scheduler_ready(sched))
 128                 return NULL;
 129
 130         /* Kernel run queue has higher priority than normal run queue*/
 131         tmp = amd_sched_rq_select_entity(&sched->kernel_rq);
 132         if (tmp == NULL)
 133                 tmp = amd_sched_rq_select_entity(&sched->sched_rq);
 134
 135         if (sched->current_entity && (sched->current_entity != tmp))
 136                 wake_entity = sched->current_entity;
 137         sched->current_entity = tmp;
 138         if (wake_entity && wake_entity->need_wakeup)
 139                 wake_up(&wake_entity->wait_queue);
 140         return tmp;
 141 }
 142
 143 /**
 144  * Init a context entity used by scheduler when submit to HW ring.
 145  *
 146  * @sched       The pointer to the scheduler
 147  * @entity      The pointer to a valid amd_sched_entity
 148  * @rq          The run queue this entity belongs
 149  * @kernel      If this is an entity for the kernel
 150  * @jobs        The max number of jobs in the job queue
 151  *
 152  * return 0 if succeed. negative error code on failure
 153 */
 154 int amd_sched_entity_init(struct amd_gpu_scheduler *sched,
 155                           struct amd_sched_entity *entity,
 156                           struct amd_sched_rq *rq,
 157                           uint32_t jobs)
 158 {
 159         char name[20];
 160
 161         if (!(sched && entity && rq))
 162                 return -EINVAL;
 163
 164         memset(entity, 0, sizeof(struct amd_sched_entity));
 165         spin_lock_init(&entity->lock);
 166         entity->belongto_rq = rq;
 167         entity->scheduler = sched;
 168         init_waitqueue_head(&entity->wait_queue);
 169         init_waitqueue_head(&entity->wait_emit);
 170         entity->fence_context = fence_context_alloc(1);
 171         snprintf(name, sizeof(name), "c_entity[%llu]", entity->fence_context);
 172         memcpy(entity->name, name, 20);
 173         entity->need_wakeup = false;
 174         if(kfifo_alloc(&entity->job_queue,
 175                        jobs * sizeof(void *),
 176                        GFP_KERNEL))
 177                 return -EINVAL;
 178
 179         spin_lock_init(&entity->queue_lock);
 180         atomic_set(&entity->fence_seq, 0);
 181
 182         /* Add the entity to the run queue */
 183         amd_sched_rq_add_entity(rq, entity);
 184         return 0;
 185 }
 186
 187 /**
 188  * Query if entity is initialized
 189  *
 190  * @sched       Pointer to scheduler instance
 191  * @entity      The pointer to a valid scheduler entity
 192  *
 193  * return true if entity is initialized, false otherwise
 194 */
 195 static bool is_context_entity_initialized(struct amd_gpu_scheduler *sched,
 196                                           struct amd_sched_entity *entity)
 197 {
 198         return entity->scheduler == sched &&
 199                 entity->belongto_rq != NULL;
 200 }
 201
 202 static bool is_context_entity_idle(struct amd_gpu_scheduler *sched,
 203                                    struct amd_sched_entity *entity)
 204 {
 205         /**
 206          * Idle means no pending IBs, and the entity is not
 207          * currently being used.
 208         */
 209         barrier();
 210         if ((sched->current_entity != entity) &&
 211             kfifo_is_empty(&entity->job_queue))
 212                 return true;
 213
 214         return false;
 215 }
 216
 217 /**
 218  * Destroy a context entity
 219  *
 220  * @sched       Pointer to scheduler instance
 221  * @entity      The pointer to a valid scheduler entity
 222  *
 223  * return 0 if succeed. negative error code on failure
 224  */
 225 int amd_sched_entity_fini(struct amd_gpu_scheduler *sched,
 226                             struct amd_sched_entity *entity)
 227 {
 228         int r = 0;
 229         struct amd_sched_rq *rq = entity->belongto_rq;
 230
 231         if (!is_context_entity_initialized(sched, entity))
 232                 return 0;
 233         entity->need_wakeup = true;
 234         /**
 235          * The client will not queue more IBs during this fini, consume existing
 236          * queued IBs
 237         */
 238         r = wait_event_timeout(
 239                 entity->wait_queue,
 240                 is_context_entity_idle(sched, entity),
 241                 msecs_to_jiffies(AMD_GPU_WAIT_IDLE_TIMEOUT_IN_MS)
 242                 ) ?  0 : -1;
 243
 244         if (r) {
 245                 if (entity->is_pending)
 246                         DRM_INFO("Entity %p is in waiting state during fini,\
 247                                 all pending ibs will be canceled.\n",
 248                                  entity);
 249         }
 250
 251         amd_sched_rq_remove_entity(rq, entity);
 252         kfifo_free(&entity->job_queue);
 253         return r;
 254 }
 255
 256 /**
 257  * Submit a normal job to the job queue
 258  *
 259  * @sched       The pointer to the scheduler
 260  * @c_entity    The pointer to amd_sched_entity
 261  * @job         The pointer to job required to submit
 262  * return 0 if succeed. -1 if failed.
 263  *        -2 indicate queue is full for this client, client should wait untill
 264  *           scheduler consum some queued command.
 265  *        -1 other fail.
 266 */
 267 int amd_sched_push_job(struct amd_sched_job *sched_job)
 268 {
 269         struct amd_sched_fence  *fence =
 270                 amd_sched_fence_create(sched_job->s_entity);
 271         if (!fence)
 272                 return -EINVAL;
 273         fence_get(&fence->base);
 274         sched_job->s_fence = fence;
 275         while (kfifo_in_spinlocked(&sched_job->s_entity->job_queue,
 276                                    &sched_job, sizeof(void *),
 277                                    &sched_job->s_entity->queue_lock) !=
 278                sizeof(void *)) {
 279                 /**
 280                  * Current context used up all its IB slots
 281                  * wait here, or need to check whether GPU is hung
 282                 */
 283                 schedule();
 284         }
 285         /* first job wake up scheduler */
 286         if ((kfifo_len(&sched_job->s_entity->job_queue) / sizeof(void *)) == 1)
 287                 wake_up_interruptible(&sched_job->sched->wait_queue);
 288         return 0;
 289 }
 290
 291 static void amd_sched_process_job(struct fence *f, struct fence_cb *cb)
 292 {
 293         struct amd_sched_job *sched_job =
 294                 container_of(cb, struct amd_sched_job, cb);
 295         struct amd_gpu_scheduler *sched;
 296         unsigned long flags;
 297
 298         sched = sched_job->sched;
 299         amd_sched_fence_signal(sched_job->s_fence);
 300         spin_lock_irqsave(&sched->queue_lock, flags);
 301         list_del(&sched_job->list);
 302         atomic64_dec(&sched->hw_rq_count);
 303         spin_unlock_irqrestore(&sched->queue_lock, flags);
 304         fence_put(&sched_job->s_fence->base);
 305         sched->ops->process_job(sched, sched_job);
 306         wake_up_interruptible(&sched->wait_queue);
 307 }
 308
 309 static int amd_sched_main(void *param)
 310 {
 311         int r;
 312         struct amd_sched_job *job;
 313         struct sched_param sparam = {.sched_priority = 1};
 314         struct amd_sched_entity *c_entity = NULL;
 315         struct amd_gpu_scheduler *sched = (struct amd_gpu_scheduler *)param;
 316
 317         sched_setscheduler(current, SCHED_FIFO, &sparam);
 318
 319         while (!kthread_should_stop()) {
 320                 struct fence *fence;
 321
 322                 wait_event_interruptible(sched->wait_queue,
 323                                          is_scheduler_ready(sched) &&
 324                                          (c_entity = select_context(sched)));
 325                 r = kfifo_out(&c_entity->job_queue, &job, sizeof(void *));
 326                 if (r != sizeof(void *))
 327                         continue;
 328                 r = 0;
 329                 if (sched->ops->prepare_job)
 330                         r = sched->ops->prepare_job(sched, c_entity, job);
 331                 if (!r) {
 332                         unsigned long flags;
 333                         spin_lock_irqsave(&sched->queue_lock, flags);
 334                         list_add_tail(&job->list, &sched->active_hw_rq);
 335                         atomic64_inc(&sched->hw_rq_count);
 336                         spin_unlock_irqrestore(&sched->queue_lock, flags);
 337                 }
 338                 mutex_lock(&sched->sched_lock);
 339                 fence = sched->ops->run_job(sched, c_entity, job);
 340                 if (fence) {
 341                         r = fence_add_callback(fence, &job->cb,
 342                                                amd_sched_process_job);
 343                         if (r == -ENOENT)
 344                                 amd_sched_process_job(fence, &job->cb);
 345                         else if (r)
 346                                 DRM_ERROR("fence add callback failed (%d)\n", r);
 347                         fence_put(fence);
 348                 }
 349                 mutex_unlock(&sched->sched_lock);
 350         }
 351         return 0;
 352 }
 353
 354 /**
 355  * Create a gpu scheduler
 356  *
 357  * @device      The device context for this scheduler
 358  * @ops         The backend operations for this scheduler.
 359  * @id          The scheduler is per ring, here is ring id.
 360  * @granularity The minumum ms unit the scheduler will scheduled.
 361  * @preemption  Indicate whether this ring support preemption, 0 is no.
 362  *
 363  * return the pointer to scheduler for success, otherwise return NULL
 364 */
 365 struct amd_gpu_scheduler *amd_sched_create(void *device,
 366                                            struct amd_sched_backend_ops *ops,
 367                                            unsigned ring,
 368                                            unsigned granularity,
 369                                            unsigned preemption,
 370                                            unsigned hw_submission)
 371 {
 372         struct amd_gpu_scheduler *sched;
 373         char name[20];
 374
 375         sched = kzalloc(sizeof(struct amd_gpu_scheduler), GFP_KERNEL);
 376         if (!sched)
 377                 return NULL;
 378
 379         sched->device = device;
 380         sched->ops = ops;
 381         sched->granularity = granularity;
 382         sched->ring_id = ring;
 383         sched->preemption = preemption;
 384         sched->hw_submission_limit = hw_submission;
 385         snprintf(name, sizeof(name), "gpu_sched[%d]", ring);
 386         mutex_init(&sched->sched_lock);
 387         spin_lock_init(&sched->queue_lock);
 388         amd_sched_rq_init(&sched->sched_rq);
 389         amd_sched_rq_init(&sched->kernel_rq);
 390
 391         init_waitqueue_head(&sched->wait_queue);
 392         INIT_LIST_HEAD(&sched->active_hw_rq);
 393         atomic64_set(&sched->hw_rq_count, 0);
 394         /* Each scheduler will run on a seperate kernel thread */
 395         sched->thread = kthread_create(amd_sched_main, sched, name);
 396         if (sched->thread) {
 397                 wake_up_process(sched->thread);
 398                 return sched;
 399         }
 400
 401         DRM_ERROR("Failed to create scheduler for id %d.\n", ring);
 402         kfree(sched);
 403         return NULL;
 404 }
 405
 406 /**
 407  * Destroy a gpu scheduler
 408  *
 409  * @sched       The pointer to the scheduler
 410  *
 411  * return 0 if succeed. -1 if failed.
 412  */
 413 int amd_sched_destroy(struct amd_gpu_scheduler *sched)
 414 {
 415         kthread_stop(sched->thread);
 416         kfree(sched);
 417         return  0;
 418 }