drivers/gpu/drm/amd/scheduler/gpu_scheduler.c

   1 /*
   2  * Copyright 2015 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  *
  23  */
  24 #include <linux/kthread.h>
  25 #include <linux/wait.h>
  26 #include <linux/sched.h>
  27 #include <drm/drmP.h>
  28 #include "gpu_scheduler.h"
  29
  30 static struct amd_sched_job *
  31 amd_sched_entity_pop_job(struct amd_sched_entity *entity);
  32 static void amd_sched_wakeup(struct amd_gpu_scheduler *sched);
  33
  34 /* Initialize a given run queue struct */
  35 static void amd_sched_rq_init(struct amd_sched_rq *rq)
  36 {
  37         spin_lock_init(&rq->lock);
  38         INIT_LIST_HEAD(&rq->entities);
  39         rq->current_entity = NULL;
  40 }
  41
  42 static void amd_sched_rq_add_entity(struct amd_sched_rq *rq,
  43                                     struct amd_sched_entity *entity)
  44 {
  45         spin_lock(&rq->lock);
  46         list_add_tail(&entity->list, &rq->entities);
  47         spin_unlock(&rq->lock);
  48 }
  49
  50 static void amd_sched_rq_remove_entity(struct amd_sched_rq *rq,
  51                                        struct amd_sched_entity *entity)
  52 {
  53         spin_lock(&rq->lock);
  54         list_del_init(&entity->list);
  55         if (rq->current_entity == entity)
  56                 rq->current_entity = NULL;
  57         spin_unlock(&rq->lock);
  58 }
  59
  60 /**
  61  * Select next job from a specified run queue with round robin policy.
  62  * Return NULL if nothing available.
  63  */
  64 static struct amd_sched_job *
  65 amd_sched_rq_select_job(struct amd_sched_rq *rq)
  66 {
  67         struct amd_sched_entity *entity;
  68         struct amd_sched_job *job;
  69
  70         spin_lock(&rq->lock);
  71
  72         entity = rq->current_entity;
  73         if (entity) {
  74                 list_for_each_entry_continue(entity, &rq->entities, list) {
  75                         job = amd_sched_entity_pop_job(entity);
  76                         if (job) {
  77                                 rq->current_entity = entity;
  78                                 spin_unlock(&rq->lock);
  79                                 return job;
  80                         }
  81                 }
  82         }
  83
  84         list_for_each_entry(entity, &rq->entities, list) {
  85
  86                 job = amd_sched_entity_pop_job(entity);
  87                 if (job) {
  88                         rq->current_entity = entity;
  89                         spin_unlock(&rq->lock);
  90                         return job;
  91                 }
  92
  93                 if (entity == rq->current_entity)
  94                         break;
  95         }
  96
  97         spin_unlock(&rq->lock);
  98
  99         return NULL;
 100 }
 101
 102 /**
 103  * Init a context entity used by scheduler when submit to HW ring.
 104  *
 105  * @sched       The pointer to the scheduler
 106  * @entity      The pointer to a valid amd_sched_entity
 107  * @rq          The run queue this entity belongs
 108  * @kernel      If this is an entity for the kernel
 109  * @jobs        The max number of jobs in the job queue
 110  *
 111  * return 0 if succeed. negative error code on failure
 112 */
 113 int amd_sched_entity_init(struct amd_gpu_scheduler *sched,
 114                           struct amd_sched_entity *entity,
 115                           struct amd_sched_rq *rq,
 116                           uint32_t jobs)
 117 {
 118         if (!(sched && entity && rq))
 119                 return -EINVAL;
 120
 121         memset(entity, 0, sizeof(struct amd_sched_entity));
 122         entity->belongto_rq = rq;
 123         entity->scheduler = sched;
 124         entity->fence_context = fence_context_alloc(1);
 125         if(kfifo_alloc(&entity->job_queue,
 126                        jobs * sizeof(void *),
 127                        GFP_KERNEL))
 128                 return -EINVAL;
 129
 130         spin_lock_init(&entity->queue_lock);
 131         atomic_set(&entity->fence_seq, 0);
 132
 133         /* Add the entity to the run queue */
 134         amd_sched_rq_add_entity(rq, entity);
 135         return 0;
 136 }
 137
 138 /**
 139  * Query if entity is initialized
 140  *
 141  * @sched       Pointer to scheduler instance
 142  * @entity      The pointer to a valid scheduler entity
 143  *
 144  * return true if entity is initialized, false otherwise
 145 */
 146 static bool amd_sched_entity_is_initialized(struct amd_gpu_scheduler *sched,
 147                                             struct amd_sched_entity *entity)
 148 {
 149         return entity->scheduler == sched &&
 150                 entity->belongto_rq != NULL;
 151 }
 152
 153 /**
 154  * Check if entity is idle
 155  *
 156  * @entity      The pointer to a valid scheduler entity
 157  *
 158  * Return true if entity don't has any unscheduled jobs.
 159  */
 160 static bool amd_sched_entity_is_idle(struct amd_sched_entity *entity)
 161 {
 162         rmb();
 163         if (kfifo_is_empty(&entity->job_queue))
 164                 return true;
 165
 166         return false;
 167 }
 168
 169 /**
 170  * Destroy a context entity
 171  *
 172  * @sched       Pointer to scheduler instance
 173  * @entity      The pointer to a valid scheduler entity
 174  *
 175  * Cleanup and free the allocated resources.
 176  */
 177 void amd_sched_entity_fini(struct amd_gpu_scheduler *sched,
 178                            struct amd_sched_entity *entity)
 179 {
 180         struct amd_sched_rq *rq = entity->belongto_rq;
 181
 182         if (!amd_sched_entity_is_initialized(sched, entity))
 183                 return;
 184
 185         /**
 186          * The client will not queue more IBs during this fini, consume existing
 187          * queued IBs
 188         */
 189         wait_event(sched->job_scheduled, amd_sched_entity_is_idle(entity));
 190
 191         amd_sched_rq_remove_entity(rq, entity);
 192         kfifo_free(&entity->job_queue);
 193 }
 194
 195 static void amd_sched_entity_wakeup(struct fence *f, struct fence_cb *cb)
 196 {
 197         struct amd_sched_entity *entity =
 198                 container_of(cb, struct amd_sched_entity, cb);
 199         entity->dependency = NULL;
 200         fence_put(f);
 201         amd_sched_wakeup(entity->scheduler);
 202 }
 203
 204 static struct amd_sched_job *
 205 amd_sched_entity_pop_job(struct amd_sched_entity *entity)
 206 {
 207         struct amd_gpu_scheduler *sched = entity->scheduler;
 208         struct amd_sched_job *job;
 209
 210         if (ACCESS_ONCE(entity->dependency))
 211                 return NULL;
 212
 213         if (!kfifo_out_peek(&entity->job_queue, &job, sizeof(job)))
 214                 return NULL;
 215
 216         while ((entity->dependency = sched->ops->dependency(job))) {
 217
 218                 if (fence_add_callback(entity->dependency, &entity->cb,
 219                                        amd_sched_entity_wakeup))
 220                         fence_put(entity->dependency);
 221                 else
 222                         return NULL;
 223         }
 224
 225         return job;
 226 }
 227
 228 /**
 229  * Helper to submit a job to the job queue
 230  *
 231  * @job         The pointer to job required to submit
 232  *
 233  * Returns true if we could submit the job.
 234  */
 235 static bool amd_sched_entity_in(struct amd_sched_job *job)
 236 {
 237         struct amd_sched_entity *entity = job->s_entity;
 238         bool added, first = false;
 239
 240         spin_lock(&entity->queue_lock);
 241         added = kfifo_in(&entity->job_queue, &job, sizeof(job)) == sizeof(job);
 242
 243         if (added && kfifo_len(&entity->job_queue) == sizeof(job))
 244                 first = true;
 245
 246         spin_unlock(&entity->queue_lock);
 247
 248         /* first job wakes up scheduler */
 249         if (first)
 250                 amd_sched_wakeup(job->sched);
 251
 252         return added;
 253 }
 254
 255 /**
 256  * Submit a job to the job queue
 257  *
 258  * @job         The pointer to job required to submit
 259  *
 260  * Returns 0 for success, negative error code otherwise.
 261  */
 262 int amd_sched_entity_push_job(struct amd_sched_job *sched_job)
 263 {
 264         struct amd_sched_entity *entity = sched_job->s_entity;
 265         struct amd_sched_fence *fence = amd_sched_fence_create(
 266                 entity, sched_job->owner);
 267
 268         if (!fence)
 269                 return -ENOMEM;
 270
 271         fence_get(&fence->base);
 272         sched_job->s_fence = fence;
 273
 274         wait_event(entity->scheduler->job_scheduled,
 275                    amd_sched_entity_in(sched_job));
 276
 277         return 0;
 278 }
 279
 280 /**
 281  * Return ture if we can push more jobs to the hw.
 282  */
 283 static bool amd_sched_ready(struct amd_gpu_scheduler *sched)
 284 {
 285         return atomic_read(&sched->hw_rq_count) <
 286                 sched->hw_submission_limit;
 287 }
 288
 289 /**
 290  * Wake up the scheduler when it is ready
 291  */
 292 static void amd_sched_wakeup(struct amd_gpu_scheduler *sched)
 293 {
 294         if (amd_sched_ready(sched))
 295                 wake_up_interruptible(&sched->wake_up_worker);
 296 }
 297
 298 /**
 299  * Select next to run
 300 */
 301 static struct amd_sched_job *
 302 amd_sched_select_job(struct amd_gpu_scheduler *sched)
 303 {
 304         struct amd_sched_job *job;
 305
 306         if (!amd_sched_ready(sched))
 307                 return NULL;
 308
 309         /* Kernel run queue has higher priority than normal run queue*/
 310         job = amd_sched_rq_select_job(&sched->kernel_rq);
 311         if (job == NULL)
 312                 job = amd_sched_rq_select_job(&sched->sched_rq);
 313
 314         return job;
 315 }
 316
 317 static void amd_sched_process_job(struct fence *f, struct fence_cb *cb)
 318 {
 319         struct amd_sched_job *sched_job =
 320                 container_of(cb, struct amd_sched_job, cb);
 321         struct amd_gpu_scheduler *sched;
 322
 323         sched = sched_job->sched;
 324         amd_sched_fence_signal(sched_job->s_fence);
 325         atomic_dec(&sched->hw_rq_count);
 326         fence_put(&sched_job->s_fence->base);
 327         sched->ops->process_job(sched_job);
 328         wake_up_interruptible(&sched->wake_up_worker);
 329 }
 330
 331 static int amd_sched_main(void *param)
 332 {
 333         struct sched_param sparam = {.sched_priority = 1};
 334         struct amd_gpu_scheduler *sched = (struct amd_gpu_scheduler *)param;
 335         int r, count;
 336
 337         sched_setscheduler(current, SCHED_FIFO, &sparam);
 338
 339         while (!kthread_should_stop()) {
 340                 struct amd_sched_entity *entity;
 341                 struct amd_sched_job *job;
 342                 struct fence *fence;
 343
 344                 wait_event_interruptible(sched->wake_up_worker,
 345                         kthread_should_stop() ||
 346                         (job = amd_sched_select_job(sched)));
 347
 348                 if (!job)
 349                         continue;
 350
 351                 entity = job->s_entity;
 352                 atomic_inc(&sched->hw_rq_count);
 353                 fence = sched->ops->run_job(job);
 354                 if (fence) {
 355                         r = fence_add_callback(fence, &job->cb,
 356                                                amd_sched_process_job);
 357                         if (r == -ENOENT)
 358                                 amd_sched_process_job(fence, &job->cb);
 359                         else if (r)
 360                                 DRM_ERROR("fence add callback failed (%d)\n", r);
 361                         fence_put(fence);
 362                 }
 363
 364                 count = kfifo_out(&entity->job_queue, &job, sizeof(job));
 365                 WARN_ON(count != sizeof(job));
 366                 wake_up(&sched->job_scheduled);
 367         }
 368         return 0;
 369 }
 370
 371 /**
 372  * Create a gpu scheduler
 373  *
 374  * @ops                 The backend operations for this scheduler.
 375  * @ring                The the ring id for the scheduler.
 376  * @hw_submissions      Number of hw submissions to do.
 377  *
 378  * Return the pointer to scheduler for success, otherwise return NULL
 379 */
 380 struct amd_gpu_scheduler *amd_sched_create(struct amd_sched_backend_ops *ops,
 381                                            unsigned ring, unsigned hw_submission,
 382                                            void *priv)
 383 {
 384         struct amd_gpu_scheduler *sched;
 385
 386         sched = kzalloc(sizeof(struct amd_gpu_scheduler), GFP_KERNEL);
 387         if (!sched)
 388                 return NULL;
 389
 390         sched->ops = ops;
 391         sched->ring_id = ring;
 392         sched->hw_submission_limit = hw_submission;
 393         sched->priv = priv;
 394         snprintf(sched->name, sizeof(sched->name), "amdgpu[%d]", ring);
 395         amd_sched_rq_init(&sched->sched_rq);
 396         amd_sched_rq_init(&sched->kernel_rq);
 397
 398         init_waitqueue_head(&sched->wake_up_worker);
 399         init_waitqueue_head(&sched->job_scheduled);
 400         atomic_set(&sched->hw_rq_count, 0);
 401         /* Each scheduler will run on a seperate kernel thread */
 402         sched->thread = kthread_run(amd_sched_main, sched, sched->name);
 403         if (IS_ERR(sched->thread)) {
 404                 DRM_ERROR("Failed to create scheduler for id %d.\n", ring);
 405                 kfree(sched);
 406                 return NULL;
 407         }
 408
 409         return sched;
 410 }
 411
 412 /**
 413  * Destroy a gpu scheduler
 414  *
 415  * @sched       The pointer to the scheduler
 416  *
 417  * return 0 if succeed. -1 if failed.
 418  */
 419 int amd_sched_destroy(struct amd_gpu_scheduler *sched)
 420 {
 421         kthread_stop(sched->thread);
 422         kfree(sched);
 423         return  0;
 424 }