4 * Copyright (C) 2012 Google, Inc.
6 * This software is licensed under the terms of the GNU General Public
7 * License version 2, as published by the Free Software Foundation, and
8 * may be copied, distributed, and modified under those terms.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
17 #include <linux/debugfs.h>
18 #include <linux/export.h>
19 #include <linux/file.h>
21 #include <linux/kernel.h>
22 #include <linux/poll.h>
23 #include <linux/sched.h>
24 #include <linux/seq_file.h>
25 #include <linux/slab.h>
26 #include <linux/uaccess.h>
27 #include <linux/anon_inodes.h>
31 #define CREATE_TRACE_POINTS
32 #include "trace/sync.h"
34 static void sync_fence_signal_pt(struct sync_pt *pt);
35 static int _sync_pt_has_signaled(struct sync_pt *pt);
36 static void sync_fence_free(struct kref *kref);
37 static void sync_dump(void);
39 static LIST_HEAD(sync_timeline_list_head);
40 static DEFINE_SPINLOCK(sync_timeline_list_lock);
42 static LIST_HEAD(sync_fence_list_head);
43 static DEFINE_SPINLOCK(sync_fence_list_lock);
45 struct sync_timeline *sync_timeline_create(const struct sync_timeline_ops *ops,
46 int size, const char *name)
48 struct sync_timeline *obj;
51 if (size < sizeof(struct sync_timeline))
54 obj = kzalloc(size, GFP_KERNEL);
58 kref_init(&obj->kref);
60 strlcpy(obj->name, name, sizeof(obj->name));
62 INIT_LIST_HEAD(&obj->child_list_head);
63 spin_lock_init(&obj->child_list_lock);
65 INIT_LIST_HEAD(&obj->active_list_head);
66 spin_lock_init(&obj->active_list_lock);
68 spin_lock_irqsave(&sync_timeline_list_lock, flags);
69 list_add_tail(&obj->sync_timeline_list, &sync_timeline_list_head);
70 spin_unlock_irqrestore(&sync_timeline_list_lock, flags);
74 EXPORT_SYMBOL(sync_timeline_create);
76 static void sync_timeline_free(struct kref *kref)
78 struct sync_timeline *obj =
79 container_of(kref, struct sync_timeline, kref);
82 spin_lock_irqsave(&sync_timeline_list_lock, flags);
83 list_del(&obj->sync_timeline_list);
84 spin_unlock_irqrestore(&sync_timeline_list_lock, flags);
86 if (obj->ops->release_obj)
87 obj->ops->release_obj(obj);
92 void sync_timeline_destroy(struct sync_timeline *obj)
94 obj->destroyed = true;
98 * signal any children that their parent is going away.
100 sync_timeline_signal(obj);
102 kref_put(&obj->kref, sync_timeline_free);
104 EXPORT_SYMBOL(sync_timeline_destroy);
106 static void sync_timeline_add_pt(struct sync_timeline *obj, struct sync_pt *pt)
112 spin_lock_irqsave(&obj->child_list_lock, flags);
113 list_add_tail(&pt->child_list, &obj->child_list_head);
114 spin_unlock_irqrestore(&obj->child_list_lock, flags);
117 static void sync_timeline_remove_pt(struct sync_pt *pt)
119 struct sync_timeline *obj = pt->parent;
122 spin_lock_irqsave(&obj->active_list_lock, flags);
123 if (!list_empty(&pt->active_list))
124 list_del_init(&pt->active_list);
125 spin_unlock_irqrestore(&obj->active_list_lock, flags);
127 spin_lock_irqsave(&obj->child_list_lock, flags);
128 if (!list_empty(&pt->child_list))
129 list_del_init(&pt->child_list);
131 spin_unlock_irqrestore(&obj->child_list_lock, flags);
134 void sync_timeline_signal(struct sync_timeline *obj)
137 LIST_HEAD(signaled_pts);
138 struct list_head *pos, *n;
140 trace_sync_timeline(obj);
142 spin_lock_irqsave(&obj->active_list_lock, flags);
144 list_for_each_safe(pos, n, &obj->active_list_head) {
146 container_of(pos, struct sync_pt, active_list);
148 if (_sync_pt_has_signaled(pt)) {
150 list_add(&pt->signaled_list, &signaled_pts);
151 kref_get(&pt->fence->kref);
155 spin_unlock_irqrestore(&obj->active_list_lock, flags);
157 list_for_each_safe(pos, n, &signaled_pts) {
159 container_of(pos, struct sync_pt, signaled_list);
162 sync_fence_signal_pt(pt);
163 kref_put(&pt->fence->kref, sync_fence_free);
166 EXPORT_SYMBOL(sync_timeline_signal);
168 struct sync_pt *sync_pt_create(struct sync_timeline *parent, int size)
172 if (size < sizeof(struct sync_pt))
175 pt = kzalloc(size, GFP_KERNEL);
179 INIT_LIST_HEAD(&pt->active_list);
180 kref_get(&parent->kref);
181 sync_timeline_add_pt(parent, pt);
185 EXPORT_SYMBOL(sync_pt_create);
187 void sync_pt_free(struct sync_pt *pt)
189 if (pt->parent->ops->free_pt)
190 pt->parent->ops->free_pt(pt);
192 sync_timeline_remove_pt(pt);
194 kref_put(&pt->parent->kref, sync_timeline_free);
198 EXPORT_SYMBOL(sync_pt_free);
200 /* call with pt->parent->active_list_lock held */
201 static int _sync_pt_has_signaled(struct sync_pt *pt)
203 int old_status = pt->status;
206 pt->status = pt->parent->ops->has_signaled(pt);
208 if (!pt->status && pt->parent->destroyed)
209 pt->status = -ENOENT;
211 if (pt->status != old_status)
212 pt->timestamp = ktime_get();
217 static struct sync_pt *sync_pt_dup(struct sync_pt *pt)
219 return pt->parent->ops->dup(pt);
222 /* Adds a sync pt to the active queue. Called when added to a fence */
223 static void sync_pt_activate(struct sync_pt *pt)
225 struct sync_timeline *obj = pt->parent;
229 spin_lock_irqsave(&obj->active_list_lock, flags);
231 err = _sync_pt_has_signaled(pt);
235 list_add_tail(&pt->active_list, &obj->active_list_head);
238 spin_unlock_irqrestore(&obj->active_list_lock, flags);
241 static int sync_fence_release(struct inode *inode, struct file *file);
242 static unsigned int sync_fence_poll(struct file *file, poll_table *wait);
243 static long sync_fence_ioctl(struct file *file, unsigned int cmd,
247 static const struct file_operations sync_fence_fops = {
248 .release = sync_fence_release,
249 .poll = sync_fence_poll,
250 .unlocked_ioctl = sync_fence_ioctl,
251 .compat_ioctl = sync_fence_ioctl,
254 static struct sync_fence *sync_fence_alloc(const char *name)
256 struct sync_fence *fence;
259 fence = kzalloc(sizeof(struct sync_fence), GFP_KERNEL);
263 fence->file = anon_inode_getfile("sync_fence", &sync_fence_fops,
265 if (IS_ERR(fence->file))
268 kref_init(&fence->kref);
269 strlcpy(fence->name, name, sizeof(fence->name));
271 INIT_LIST_HEAD(&fence->pt_list_head);
272 INIT_LIST_HEAD(&fence->waiter_list_head);
273 spin_lock_init(&fence->waiter_list_lock);
275 init_waitqueue_head(&fence->wq);
277 spin_lock_irqsave(&sync_fence_list_lock, flags);
278 list_add_tail(&fence->sync_fence_list, &sync_fence_list_head);
279 spin_unlock_irqrestore(&sync_fence_list_lock, flags);
288 /* TODO: implement a create which takes more that one sync_pt */
289 struct sync_fence *sync_fence_create(const char *name, struct sync_pt *pt)
291 struct sync_fence *fence;
296 fence = sync_fence_alloc(name);
301 list_add(&pt->pt_list, &fence->pt_list_head);
302 sync_pt_activate(pt);
305 * signal the fence in case pt was activated before
306 * sync_pt_activate(pt) was called
308 sync_fence_signal_pt(pt);
312 EXPORT_SYMBOL(sync_fence_create);
314 static int sync_fence_copy_pts(struct sync_fence *dst, struct sync_fence *src)
316 struct list_head *pos;
318 list_for_each(pos, &src->pt_list_head) {
319 struct sync_pt *orig_pt =
320 container_of(pos, struct sync_pt, pt_list);
321 struct sync_pt *new_pt = sync_pt_dup(orig_pt);
327 list_add(&new_pt->pt_list, &dst->pt_list_head);
333 static int sync_fence_merge_pts(struct sync_fence *dst, struct sync_fence *src)
335 struct list_head *src_pos, *dst_pos, *n;
337 list_for_each(src_pos, &src->pt_list_head) {
338 struct sync_pt *src_pt =
339 container_of(src_pos, struct sync_pt, pt_list);
340 bool collapsed = false;
342 list_for_each_safe(dst_pos, n, &dst->pt_list_head) {
343 struct sync_pt *dst_pt =
344 container_of(dst_pos, struct sync_pt, pt_list);
345 /* collapse two sync_pts on the same timeline
346 * to a single sync_pt that will signal at
347 * the later of the two
349 if (dst_pt->parent == src_pt->parent) {
350 if (dst_pt->parent->ops->compare(dst_pt, src_pt)
352 struct sync_pt *new_pt =
358 list_replace(&dst_pt->pt_list,
360 sync_pt_free(dst_pt);
368 struct sync_pt *new_pt = sync_pt_dup(src_pt);
374 list_add(&new_pt->pt_list, &dst->pt_list_head);
381 static void sync_fence_detach_pts(struct sync_fence *fence)
383 struct list_head *pos, *n;
385 list_for_each_safe(pos, n, &fence->pt_list_head) {
386 struct sync_pt *pt = container_of(pos, struct sync_pt, pt_list);
388 sync_timeline_remove_pt(pt);
392 static void sync_fence_free_pts(struct sync_fence *fence)
394 struct list_head *pos, *n;
396 list_for_each_safe(pos, n, &fence->pt_list_head) {
397 struct sync_pt *pt = container_of(pos, struct sync_pt, pt_list);
403 struct sync_fence *sync_fence_fdget(int fd)
405 struct file *file = fget(fd);
410 if (file->f_op != &sync_fence_fops)
413 return file->private_data;
419 EXPORT_SYMBOL(sync_fence_fdget);
421 void sync_fence_put(struct sync_fence *fence)
425 EXPORT_SYMBOL(sync_fence_put);
427 void sync_fence_install(struct sync_fence *fence, int fd)
429 fd_install(fd, fence->file);
431 EXPORT_SYMBOL(sync_fence_install);
433 static int sync_fence_get_status(struct sync_fence *fence)
435 struct list_head *pos;
438 list_for_each(pos, &fence->pt_list_head) {
439 struct sync_pt *pt = container_of(pos, struct sync_pt, pt_list);
440 int pt_status = pt->status;
445 } else if (status == 1) {
453 struct sync_fence *sync_fence_merge(const char *name,
454 struct sync_fence *a, struct sync_fence *b)
456 struct sync_fence *fence;
457 struct list_head *pos;
460 fence = sync_fence_alloc(name);
464 err = sync_fence_copy_pts(fence, a);
468 err = sync_fence_merge_pts(fence, b);
472 list_for_each(pos, &fence->pt_list_head) {
474 container_of(pos, struct sync_pt, pt_list);
475 sync_pt_activate(pt);
479 * signal the fence in case one of it's pts were activated before
480 * they were activated
482 sync_fence_signal_pt(list_first_entry(&fence->pt_list_head,
488 sync_fence_free_pts(fence);
492 EXPORT_SYMBOL(sync_fence_merge);
494 static void sync_fence_signal_pt(struct sync_pt *pt)
496 LIST_HEAD(signaled_waiters);
497 struct sync_fence *fence = pt->fence;
498 struct list_head *pos;
503 status = sync_fence_get_status(fence);
505 spin_lock_irqsave(&fence->waiter_list_lock, flags);
507 * this should protect against two threads racing on the signaled
508 * false -> true transition
510 if (status && !fence->status) {
511 list_for_each_safe(pos, n, &fence->waiter_list_head)
512 list_move(pos, &signaled_waiters);
514 fence->status = status;
518 spin_unlock_irqrestore(&fence->waiter_list_lock, flags);
521 list_for_each_safe(pos, n, &signaled_waiters) {
522 struct sync_fence_waiter *waiter =
523 container_of(pos, struct sync_fence_waiter,
527 waiter->callback(fence, waiter);
533 int sync_fence_wait_async(struct sync_fence *fence,
534 struct sync_fence_waiter *waiter)
539 spin_lock_irqsave(&fence->waiter_list_lock, flags);
546 list_add_tail(&waiter->waiter_list, &fence->waiter_list_head);
548 spin_unlock_irqrestore(&fence->waiter_list_lock, flags);
552 EXPORT_SYMBOL(sync_fence_wait_async);
554 int sync_fence_cancel_async(struct sync_fence *fence,
555 struct sync_fence_waiter *waiter)
557 struct list_head *pos;
562 spin_lock_irqsave(&fence->waiter_list_lock, flags);
564 * Make sure waiter is still in waiter_list because it is possible for
565 * the waiter to be removed from the list while the callback is still
568 list_for_each_safe(pos, n, &fence->waiter_list_head) {
569 struct sync_fence_waiter *list_waiter =
570 container_of(pos, struct sync_fence_waiter,
572 if (list_waiter == waiter) {
578 spin_unlock_irqrestore(&fence->waiter_list_lock, flags);
581 EXPORT_SYMBOL(sync_fence_cancel_async);
583 static bool sync_fence_check(struct sync_fence *fence)
586 * Make sure that reads to fence->status are ordered with the
587 * wait queue event triggering
590 return fence->status != 0;
593 int sync_fence_wait(struct sync_fence *fence, long timeout)
598 trace_sync_wait(fence, 1);
599 list_for_each_entry(pt, &fence->pt_list_head, pt_list)
603 timeout = msecs_to_jiffies(timeout);
604 err = wait_event_interruptible_timeout(fence->wq,
605 sync_fence_check(fence),
607 } else if (timeout < 0) {
608 err = wait_event_interruptible(fence->wq,
609 sync_fence_check(fence));
611 trace_sync_wait(fence, 0);
616 if (fence->status < 0) {
617 pr_info("fence error %d on [%p]\n", fence->status, fence);
619 return fence->status;
622 if (fence->status == 0) {
624 pr_info("fence timeout on [%p] after %dms\n", fence,
625 jiffies_to_msecs(timeout));
633 EXPORT_SYMBOL(sync_fence_wait);
635 static void sync_fence_free(struct kref *kref)
637 struct sync_fence *fence = container_of(kref, struct sync_fence, kref);
639 sync_fence_free_pts(fence);
644 static int sync_fence_release(struct inode *inode, struct file *file)
646 struct sync_fence *fence = file->private_data;
650 * We need to remove all ways to access this fence before droping
653 * start with its membership in the global fence list
655 spin_lock_irqsave(&sync_fence_list_lock, flags);
656 list_del(&fence->sync_fence_list);
657 spin_unlock_irqrestore(&sync_fence_list_lock, flags);
660 * remove its pts from their parents so that sync_timeline_signal()
661 * can't reference the fence.
663 sync_fence_detach_pts(fence);
665 kref_put(&fence->kref, sync_fence_free);
670 static unsigned int sync_fence_poll(struct file *file, poll_table *wait)
672 struct sync_fence *fence = file->private_data;
674 poll_wait(file, &fence->wq, wait);
677 * Make sure that reads to fence->status are ordered with the
678 * wait queue event triggering
682 if (fence->status == 1)
684 else if (fence->status < 0)
690 static long sync_fence_ioctl_wait(struct sync_fence *fence, unsigned long arg)
694 if (copy_from_user(&value, (void __user *)arg, sizeof(value)))
697 return sync_fence_wait(fence, value);
700 static long sync_fence_ioctl_merge(struct sync_fence *fence, unsigned long arg)
702 int fd = get_unused_fd_flags(O_CLOEXEC);
704 struct sync_fence *fence2, *fence3;
705 struct sync_merge_data data;
710 if (copy_from_user(&data, (void __user *)arg, sizeof(data))) {
715 fence2 = sync_fence_fdget(data.fd2);
716 if (fence2 == NULL) {
721 data.name[sizeof(data.name) - 1] = '\0';
722 fence3 = sync_fence_merge(data.name, fence, fence2);
723 if (fence3 == NULL) {
729 if (copy_to_user((void __user *)arg, &data, sizeof(data))) {
734 sync_fence_install(fence3, fd);
735 sync_fence_put(fence2);
739 sync_fence_put(fence3);
742 sync_fence_put(fence2);
749 static int sync_fill_pt_info(struct sync_pt *pt, void *data, int size)
751 struct sync_pt_info *info = data;
754 if (size < sizeof(struct sync_pt_info))
757 info->len = sizeof(struct sync_pt_info);
759 if (pt->parent->ops->fill_driver_data) {
760 ret = pt->parent->ops->fill_driver_data(pt, info->driver_data,
761 size - sizeof(*info));
768 strlcpy(info->obj_name, pt->parent->name, sizeof(info->obj_name));
769 strlcpy(info->driver_name, pt->parent->ops->driver_name,
770 sizeof(info->driver_name));
771 info->status = pt->status;
772 info->timestamp_ns = ktime_to_ns(pt->timestamp);
777 static long sync_fence_ioctl_fence_info(struct sync_fence *fence,
780 struct sync_fence_info_data *data;
781 struct list_head *pos;
786 if (copy_from_user(&size, (void __user *)arg, sizeof(size)))
789 if (size < sizeof(struct sync_fence_info_data))
795 data = kzalloc(size, GFP_KERNEL);
799 strlcpy(data->name, fence->name, sizeof(data->name));
800 data->status = fence->status;
801 len = sizeof(struct sync_fence_info_data);
803 list_for_each(pos, &fence->pt_list_head) {
805 container_of(pos, struct sync_pt, pt_list);
807 ret = sync_fill_pt_info(pt, (u8 *)data + len, size - len);
817 if (copy_to_user((void __user *)arg, data, len))
828 static long sync_fence_ioctl(struct file *file, unsigned int cmd,
831 struct sync_fence *fence = file->private_data;
835 return sync_fence_ioctl_wait(fence, arg);
838 return sync_fence_ioctl_merge(fence, arg);
840 case SYNC_IOC_FENCE_INFO:
841 return sync_fence_ioctl_fence_info(fence, arg);
848 #ifdef CONFIG_DEBUG_FS
849 static const char *sync_status_str(int status)
853 else if (status == 0)
859 static void sync_print_pt(struct seq_file *s, struct sync_pt *pt, bool fence)
861 int status = pt->status;
863 seq_printf(s, " %s%spt %s",
864 fence ? pt->parent->name : "",
866 sync_status_str(status));
868 struct timeval tv = ktime_to_timeval(pt->timestamp);
870 seq_printf(s, "@%ld.%06ld", tv.tv_sec, tv.tv_usec);
873 if (pt->parent->ops->timeline_value_str &&
874 pt->parent->ops->pt_value_str) {
877 pt->parent->ops->pt_value_str(pt, value, sizeof(value));
878 seq_printf(s, ": %s", value);
880 pt->parent->ops->timeline_value_str(pt->parent, value,
882 seq_printf(s, " / %s", value);
884 } else if (pt->parent->ops->print_pt) {
886 pt->parent->ops->print_pt(s, pt);
892 static void sync_print_obj(struct seq_file *s, struct sync_timeline *obj)
894 struct list_head *pos;
897 seq_printf(s, "%s %s", obj->name, obj->ops->driver_name);
899 if (obj->ops->timeline_value_str) {
902 obj->ops->timeline_value_str(obj, value, sizeof(value));
903 seq_printf(s, ": %s", value);
904 } else if (obj->ops->print_obj) {
906 obj->ops->print_obj(s, obj);
911 spin_lock_irqsave(&obj->child_list_lock, flags);
912 list_for_each(pos, &obj->child_list_head) {
914 container_of(pos, struct sync_pt, child_list);
915 sync_print_pt(s, pt, false);
917 spin_unlock_irqrestore(&obj->child_list_lock, flags);
920 static void sync_print_fence(struct seq_file *s, struct sync_fence *fence)
922 struct list_head *pos;
925 seq_printf(s, "[%p] %s: %s\n", fence, fence->name,
926 sync_status_str(fence->status));
928 list_for_each(pos, &fence->pt_list_head) {
930 container_of(pos, struct sync_pt, pt_list);
931 sync_print_pt(s, pt, true);
934 spin_lock_irqsave(&fence->waiter_list_lock, flags);
935 list_for_each(pos, &fence->waiter_list_head) {
936 struct sync_fence_waiter *waiter =
937 container_of(pos, struct sync_fence_waiter,
940 seq_printf(s, "waiter %pF\n", waiter->callback);
942 spin_unlock_irqrestore(&fence->waiter_list_lock, flags);
945 static int sync_debugfs_show(struct seq_file *s, void *unused)
948 struct list_head *pos;
950 seq_puts(s, "objs:\n--------------\n");
952 spin_lock_irqsave(&sync_timeline_list_lock, flags);
953 list_for_each(pos, &sync_timeline_list_head) {
954 struct sync_timeline *obj =
955 container_of(pos, struct sync_timeline,
958 sync_print_obj(s, obj);
961 spin_unlock_irqrestore(&sync_timeline_list_lock, flags);
963 seq_puts(s, "fences:\n--------------\n");
965 spin_lock_irqsave(&sync_fence_list_lock, flags);
966 list_for_each(pos, &sync_fence_list_head) {
967 struct sync_fence *fence =
968 container_of(pos, struct sync_fence, sync_fence_list);
970 sync_print_fence(s, fence);
973 spin_unlock_irqrestore(&sync_fence_list_lock, flags);
977 static int sync_debugfs_open(struct inode *inode, struct file *file)
979 return single_open(file, sync_debugfs_show, inode->i_private);
982 static const struct file_operations sync_debugfs_fops = {
983 .open = sync_debugfs_open,
986 .release = single_release,
989 static __init int sync_debugfs_init(void)
991 debugfs_create_file("sync", S_IRUGO, NULL, NULL, &sync_debugfs_fops);
994 late_initcall(sync_debugfs_init);
996 #define DUMP_CHUNK 256
997 static char sync_dump_buf[64 * 1024];
998 static void sync_dump(void)
1000 struct seq_file s = {
1001 .buf = sync_dump_buf,
1002 .size = sizeof(sync_dump_buf) - 1,
1006 sync_debugfs_show(&s, NULL);
1008 for (i = 0; i < s.count; i += DUMP_CHUNK) {
1009 if ((s.count - i) > DUMP_CHUNK) {
1010 char c = s.buf[i + DUMP_CHUNK];
1012 s.buf[i + DUMP_CHUNK] = 0;
1013 pr_cont("%s", s.buf + i);
1014 s.buf[i + DUMP_CHUNK] = c;
1017 pr_cont("%s", s.buf + i);
1022 static void sync_dump(void)