drivers/gpu/drm/i915/i915_sw_fence.c

   1 /*
   2  * (C) Copyright 2016 Intel Corporation
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public License
   6  * as published by the Free Software Foundation; version 2
   7  * of the License.
   8  */
   9
  10 #include <linux/slab.h>
  11 #include <linux/dma-fence.h>
  12 #include <linux/reservation.h>
  13
  14 #include "i915_sw_fence.h"
  15
  16 #define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */
  17
  18 static DEFINE_SPINLOCK(i915_sw_fence_lock);
  19
  20 enum {
  21         DEBUG_FENCE_IDLE = 0,
  22         DEBUG_FENCE_NOTIFY,
  23 };
  24
  25 #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
  26
  27 static void *i915_sw_fence_debug_hint(void *addr)
  28 {
  29         return (void *)(((struct i915_sw_fence *)addr)->flags & I915_SW_FENCE_MASK);
  30 }
  31
  32 static struct debug_obj_descr i915_sw_fence_debug_descr = {
  33         .name = "i915_sw_fence",
  34         .debug_hint = i915_sw_fence_debug_hint,
  35 };
  36
  37 static inline void debug_fence_init(struct i915_sw_fence *fence)
  38 {
  39         debug_object_init(fence, &i915_sw_fence_debug_descr);
  40 }
  41
  42 static inline void debug_fence_activate(struct i915_sw_fence *fence)
  43 {
  44         debug_object_activate(fence, &i915_sw_fence_debug_descr);
  45 }
  46
  47 static inline void debug_fence_set_state(struct i915_sw_fence *fence,
  48                                          int old, int new)
  49 {
  50         debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new);
  51 }
  52
  53 static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
  54 {
  55         debug_object_deactivate(fence, &i915_sw_fence_debug_descr);
  56 }
  57
  58 static inline void debug_fence_destroy(struct i915_sw_fence *fence)
  59 {
  60         debug_object_destroy(fence, &i915_sw_fence_debug_descr);
  61 }
  62
  63 static inline void debug_fence_free(struct i915_sw_fence *fence)
  64 {
  65         debug_object_free(fence, &i915_sw_fence_debug_descr);
  66         smp_wmb(); /* flush the change in state before reallocation */
  67 }
  68
  69 static inline void debug_fence_assert(struct i915_sw_fence *fence)
  70 {
  71         debug_object_assert_init(fence, &i915_sw_fence_debug_descr);
  72 }
  73
  74 #else
  75
  76 static inline void debug_fence_init(struct i915_sw_fence *fence)
  77 {
  78 }
  79
  80 static inline void debug_fence_activate(struct i915_sw_fence *fence)
  81 {
  82 }
  83
  84 static inline void debug_fence_set_state(struct i915_sw_fence *fence,
  85                                          int old, int new)
  86 {
  87 }
  88
  89 static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
  90 {
  91 }
  92
  93 static inline void debug_fence_destroy(struct i915_sw_fence *fence)
  94 {
  95 }
  96
  97 static inline void debug_fence_free(struct i915_sw_fence *fence)
  98 {
  99 }
 100
 101 static inline void debug_fence_assert(struct i915_sw_fence *fence)
 102 {
 103 }
 104
 105 #endif
 106
 107 static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
 108                                   enum i915_sw_fence_notify state)
 109 {
 110         i915_sw_fence_notify_t fn;
 111
 112         fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK);
 113         return fn(fence, state);
 114 }
 115
 116 #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
 117 void i915_sw_fence_fini(struct i915_sw_fence *fence)
 118 {
 119         debug_fence_free(fence);
 120 }
 121 #endif
 122
 123 static void i915_sw_fence_release(struct kref *kref)
 124 {
 125         struct i915_sw_fence *fence = container_of(kref, typeof(*fence), kref);
 126
 127         WARN_ON(atomic_read(&fence->pending) > 0);
 128         debug_fence_destroy(fence);
 129
 130         if (fence->flags & I915_SW_FENCE_MASK) {
 131                 __i915_sw_fence_notify(fence, FENCE_FREE);
 132         } else {
 133                 i915_sw_fence_fini(fence);
 134                 kfree(fence);
 135         }
 136 }
 137
 138 static void i915_sw_fence_put(struct i915_sw_fence *fence)
 139 {
 140         debug_fence_assert(fence);
 141         kref_put(&fence->kref, i915_sw_fence_release);
 142 }
 143
 144 static struct i915_sw_fence *i915_sw_fence_get(struct i915_sw_fence *fence)
 145 {
 146         debug_fence_assert(fence);
 147         kref_get(&fence->kref);
 148         return fence;
 149 }
 150
 151 static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,
 152                                         struct list_head *continuation)
 153 {
 154         wait_queue_head_t *x = &fence->wait;
 155         wait_queue_entry_t *pos, *next;
 156         unsigned long flags;
 157
 158         debug_fence_deactivate(fence);
 159         atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */
 160
 161         /*
 162          * To prevent unbounded recursion as we traverse the graph of
 163          * i915_sw_fences, we move the entry list from this, the next ready
 164          * fence, to the tail of the original fence's entry list
 165          * (and so added to the list to be woken).
 166          */
 167
 168         spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation);
 169         if (continuation) {
 170                 list_for_each_entry_safe(pos, next, &x->head, entry) {
 171                         if (pos->func == autoremove_wake_function)
 172                                 pos->func(pos, TASK_NORMAL, 0, continuation);
 173                         else
 174                                 list_move_tail(&pos->entry, continuation);
 175                 }
 176         } else {
 177                 LIST_HEAD(extra);
 178
 179                 do {
 180                         list_for_each_entry_safe(pos, next, &x->head, entry)
 181                                 pos->func(pos, TASK_NORMAL, 0, &extra);
 182
 183                         if (list_empty(&extra))
 184                                 break;
 185
 186                         list_splice_tail_init(&extra, &x->head);
 187                 } while (1);
 188         }
 189         spin_unlock_irqrestore(&x->lock, flags);
 190
 191         debug_fence_assert(fence);
 192 }
 193
 194 static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
 195                                      struct list_head *continuation)
 196 {
 197         debug_fence_assert(fence);
 198
 199         if (!atomic_dec_and_test(&fence->pending))
 200                 return;
 201
 202         debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY);
 203
 204         if (fence->flags & I915_SW_FENCE_MASK &&
 205             __i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)
 206                 return;
 207
 208         debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE);
 209
 210         __i915_sw_fence_wake_up_all(fence, continuation);
 211 }
 212
 213 static void i915_sw_fence_complete(struct i915_sw_fence *fence)
 214 {
 215         debug_fence_assert(fence);
 216
 217         if (WARN_ON(i915_sw_fence_done(fence)))
 218                 return;
 219
 220         __i915_sw_fence_complete(fence, NULL);
 221 }
 222
 223 static void i915_sw_fence_await(struct i915_sw_fence *fence)
 224 {
 225         debug_fence_assert(fence);
 226         WARN_ON(atomic_inc_return(&fence->pending) <= 1);
 227 }
 228
 229 void __i915_sw_fence_init(struct i915_sw_fence *fence,
 230                           i915_sw_fence_notify_t fn,
 231                           const char *name,
 232                           struct lock_class_key *key)
 233 {
 234         BUG_ON((unsigned long)fn & ~I915_SW_FENCE_MASK);
 235
 236         debug_fence_init(fence);
 237
 238         __init_waitqueue_head(&fence->wait, name, key);
 239         kref_init(&fence->kref);
 240         atomic_set(&fence->pending, 1);
 241         fence->flags = (unsigned long)fn;
 242 }
 243
 244 static void __i915_sw_fence_commit(struct i915_sw_fence *fence)
 245 {
 246         i915_sw_fence_complete(fence);
 247         i915_sw_fence_put(fence);
 248 }
 249
 250 void i915_sw_fence_commit(struct i915_sw_fence *fence)
 251 {
 252         debug_fence_activate(fence);
 253         __i915_sw_fence_commit(fence);
 254 }
 255
 256 static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key)
 257 {
 258         list_del(&wq->entry);
 259         __i915_sw_fence_complete(wq->private, key);
 260         i915_sw_fence_put(wq->private);
 261         if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
 262                 kfree(wq);
 263         return 0;
 264 }
 265
 266 static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
 267                                     const struct i915_sw_fence * const signaler)
 268 {
 269         wait_queue_entry_t *wq;
 270
 271         if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
 272                 return false;
 273
 274         if (fence == signaler)
 275                 return true;
 276
 277         list_for_each_entry(wq, &fence->wait.head, entry) {
 278                 if (wq->func != i915_sw_fence_wake)
 279                         continue;
 280
 281                 if (__i915_sw_fence_check_if_after(wq->private, signaler))
 282                         return true;
 283         }
 284
 285         return false;
 286 }
 287
 288 static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)
 289 {
 290         wait_queue_entry_t *wq;
 291
 292         if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
 293                 return;
 294
 295         list_for_each_entry(wq, &fence->wait.head, entry) {
 296                 if (wq->func != i915_sw_fence_wake)
 297                         continue;
 298
 299                 __i915_sw_fence_clear_checked_bit(wq->private);
 300         }
 301 }
 302
 303 static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
 304                                   const struct i915_sw_fence * const signaler)
 305 {
 306         unsigned long flags;
 307         bool err;
 308
 309         if (!IS_ENABLED(CONFIG_I915_SW_FENCE_CHECK_DAG))
 310                 return false;
 311
 312         spin_lock_irqsave(&i915_sw_fence_lock, flags);
 313         err = __i915_sw_fence_check_if_after(fence, signaler);
 314         __i915_sw_fence_clear_checked_bit(fence);
 315         spin_unlock_irqrestore(&i915_sw_fence_lock, flags);
 316
 317         return err;
 318 }
 319
 320 static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
 321                                           struct i915_sw_fence *signaler,
 322                                           wait_queue_entry_t *wq, gfp_t gfp)
 323 {
 324         unsigned long flags;
 325         int pending;
 326
 327         debug_fence_assert(fence);
 328
 329         if (i915_sw_fence_done(signaler))
 330                 return 0;
 331
 332         debug_fence_assert(signaler);
 333
 334         /* The dependency graph must be acyclic. */
 335         if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))
 336                 return -EINVAL;
 337
 338         pending = 0;
 339         if (!wq) {
 340                 wq = kmalloc(sizeof(*wq), gfp);
 341                 if (!wq) {
 342                         if (!gfpflags_allow_blocking(gfp))
 343                                 return -ENOMEM;
 344
 345                         i915_sw_fence_wait(signaler);
 346                         return 0;
 347                 }
 348
 349                 pending |= I915_SW_FENCE_FLAG_ALLOC;
 350         }
 351
 352         INIT_LIST_HEAD(&wq->entry);
 353         wq->flags = pending;
 354         wq->func = i915_sw_fence_wake;
 355         wq->private = i915_sw_fence_get(fence);
 356
 357         i915_sw_fence_await(fence);
 358
 359         spin_lock_irqsave(&signaler->wait.lock, flags);
 360         if (likely(!i915_sw_fence_done(signaler))) {
 361                 __add_wait_queue_entry_tail(&signaler->wait, wq);
 362                 pending = 1;
 363         } else {
 364                 i915_sw_fence_wake(wq, 0, 0, NULL);
 365                 pending = 0;
 366         }
 367         spin_unlock_irqrestore(&signaler->wait.lock, flags);
 368
 369         return pending;
 370 }
 371
 372 int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
 373                                  struct i915_sw_fence *signaler,
 374                                  wait_queue_entry_t *wq)
 375 {
 376         return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);
 377 }
 378
 379 int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
 380                                      struct i915_sw_fence *signaler,
 381                                      gfp_t gfp)
 382 {
 383         return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
 384 }
 385
 386 struct i915_sw_dma_fence_cb {
 387         struct dma_fence_cb base;
 388         struct i915_sw_fence *fence;
 389         struct dma_fence *dma;
 390         struct timer_list timer;
 391 };
 392
 393 static void timer_i915_sw_fence_wake(unsigned long data)
 394 {
 395         struct i915_sw_dma_fence_cb *cb = (struct i915_sw_dma_fence_cb *)data;
 396
 397         pr_warn("asynchronous wait on fence %s:%s:%x timed out\n",
 398                 cb->dma->ops->get_driver_name(cb->dma),
 399                 cb->dma->ops->get_timeline_name(cb->dma),
 400                 cb->dma->seqno);
 401         dma_fence_put(cb->dma);
 402         cb->dma = NULL;
 403
 404         __i915_sw_fence_commit(cb->fence);
 405         cb->timer.function = NULL;
 406 }
 407
 408 static void dma_i915_sw_fence_wake(struct dma_fence *dma,
 409                                    struct dma_fence_cb *data)
 410 {
 411         struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base);
 412
 413         del_timer_sync(&cb->timer);
 414         if (cb->timer.function)
 415                 __i915_sw_fence_commit(cb->fence);
 416         dma_fence_put(cb->dma);
 417
 418         kfree(cb);
 419 }
 420
 421 int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
 422                                   struct dma_fence *dma,
 423                                   unsigned long timeout,
 424                                   gfp_t gfp)
 425 {
 426         struct i915_sw_dma_fence_cb *cb;
 427         int ret;
 428
 429         debug_fence_assert(fence);
 430
 431         if (dma_fence_is_signaled(dma))
 432                 return 0;
 433
 434         cb = kmalloc(sizeof(*cb), gfp);
 435         if (!cb) {
 436                 if (!gfpflags_allow_blocking(gfp))
 437                         return -ENOMEM;
 438
 439                 return dma_fence_wait(dma, false);
 440         }
 441
 442         cb->fence = i915_sw_fence_get(fence);
 443         i915_sw_fence_await(fence);
 444
 445         cb->dma = NULL;
 446         __setup_timer(&cb->timer,
 447                       timer_i915_sw_fence_wake, (unsigned long)cb,
 448                       TIMER_IRQSAFE);
 449         if (timeout) {
 450                 cb->dma = dma_fence_get(dma);
 451                 mod_timer(&cb->timer, round_jiffies_up(jiffies + timeout));
 452         }
 453
 454         ret = dma_fence_add_callback(dma, &cb->base, dma_i915_sw_fence_wake);
 455         if (ret == 0) {
 456                 ret = 1;
 457         } else {
 458                 dma_i915_sw_fence_wake(dma, &cb->base);
 459                 if (ret == -ENOENT) /* fence already signaled */
 460                         ret = 0;
 461         }
 462
 463         return ret;
 464 }
 465
 466 int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
 467                                     struct reservation_object *resv,
 468                                     const struct dma_fence_ops *exclude,
 469                                     bool write,
 470                                     unsigned long timeout,
 471                                     gfp_t gfp)
 472 {
 473         struct dma_fence *excl;
 474         int ret = 0, pending;
 475
 476         debug_fence_assert(fence);
 477
 478         if (write) {
 479                 struct dma_fence **shared;
 480                 unsigned int count, i;
 481
 482                 ret = reservation_object_get_fences_rcu(resv,
 483                                                         &excl, &count, &shared);
 484                 if (ret)
 485                         return ret;
 486
 487                 for (i = 0; i < count; i++) {
 488                         if (shared[i]->ops == exclude)
 489                                 continue;
 490
 491                         pending = i915_sw_fence_await_dma_fence(fence,
 492                                                                 shared[i],
 493                                                                 timeout,
 494                                                                 gfp);
 495                         if (pending < 0) {
 496                                 ret = pending;
 497                                 break;
 498                         }
 499
 500                         ret |= pending;
 501                 }
 502
 503                 for (i = 0; i < count; i++)
 504                         dma_fence_put(shared[i]);
 505                 kfree(shared);
 506         } else {
 507                 excl = reservation_object_get_excl_rcu(resv);
 508         }
 509
 510         if (ret >= 0 && excl && excl->ops != exclude) {
 511                 pending = i915_sw_fence_await_dma_fence(fence,
 512                                                         excl,
 513                                                         timeout,
 514                                                         gfp);
 515                 if (pending < 0)
 516                         ret = pending;
 517                 else
 518                         ret |= pending;
 519         }
 520
 521         dma_fence_put(excl);
 522
 523         return ret;
 524 }