4 * Copyright (C) 2007 Davide Libenzi <davidel@xmailserver.org>
8 #include <linux/file.h>
9 #include <linux/poll.h>
10 #include <linux/init.h>
12 #include <linux/sched.h>
13 #include <linux/kernel.h>
14 #include <linux/slab.h>
15 #include <linux/list.h>
16 #include <linux/spinlock.h>
17 #include <linux/anon_inodes.h>
18 #include <linux/syscalls.h>
19 #include <linux/export.h>
20 #include <linux/kref.h>
21 #include <linux/eventfd.h>
22 #include <linux/proc_fs.h>
23 #include <linux/seq_file.h>
27 wait_queue_head_t wqh;
29 * Every time that a write(2) is performed on an eventfd, the
30 * value of the __u64 being written is added to "count" and a
31 * wakeup is performed on "wqh". A read(2) will return the "count"
32 * value to userspace, and will reset "count" to zero. The kernel
33 * side eventfd_signal() also, adds to the "count" counter and
41 * eventfd_signal - Adds @n to the eventfd counter.
42 * @ctx: [in] Pointer to the eventfd context.
43 * @n: [in] Value of the counter to be added to the eventfd internal counter.
44 * The value cannot be negative.
46 * This function is supposed to be called by the kernel in paths that do not
47 * allow sleeping. In this function we allow the counter to reach the ULLONG_MAX
48 * value, and we signal this as overflow condition by returining a POLLERR
51 * Returns the amount by which the counter was incrememnted. This will be less
52 * than @n if the counter has overflowed.
54 __u64 eventfd_signal(struct eventfd_ctx *ctx, __u64 n)
58 spin_lock_irqsave(&ctx->wqh.lock, flags);
59 if (ULLONG_MAX - ctx->count < n)
60 n = ULLONG_MAX - ctx->count;
62 if (waitqueue_active(&ctx->wqh))
63 wake_up_locked_poll(&ctx->wqh, POLLIN);
64 spin_unlock_irqrestore(&ctx->wqh.lock, flags);
68 EXPORT_SYMBOL_GPL(eventfd_signal);
70 static void eventfd_free_ctx(struct eventfd_ctx *ctx)
75 static void eventfd_free(struct kref *kref)
77 struct eventfd_ctx *ctx = container_of(kref, struct eventfd_ctx, kref);
79 eventfd_free_ctx(ctx);
83 * eventfd_ctx_get - Acquires a reference to the internal eventfd context.
84 * @ctx: [in] Pointer to the eventfd context.
86 * Returns: In case of success, returns a pointer to the eventfd context.
88 struct eventfd_ctx *eventfd_ctx_get(struct eventfd_ctx *ctx)
93 EXPORT_SYMBOL_GPL(eventfd_ctx_get);
96 * eventfd_ctx_put - Releases a reference to the internal eventfd context.
97 * @ctx: [in] Pointer to eventfd context.
99 * The eventfd context reference must have been previously acquired either
100 * with eventfd_ctx_get() or eventfd_ctx_fdget().
102 void eventfd_ctx_put(struct eventfd_ctx *ctx)
104 kref_put(&ctx->kref, eventfd_free);
106 EXPORT_SYMBOL_GPL(eventfd_ctx_put);
108 static int eventfd_release(struct inode *inode, struct file *file)
110 struct eventfd_ctx *ctx = file->private_data;
112 wake_up_poll(&ctx->wqh, POLLHUP);
113 eventfd_ctx_put(ctx);
117 static unsigned int eventfd_poll(struct file *file, poll_table *wait)
119 struct eventfd_ctx *ctx = file->private_data;
120 unsigned int events = 0;
123 poll_wait(file, &ctx->wqh, wait);
125 spin_lock_irqsave(&ctx->wqh.lock, flags);
128 if (ctx->count == ULLONG_MAX)
130 if (ULLONG_MAX - 1 > ctx->count)
132 spin_unlock_irqrestore(&ctx->wqh.lock, flags);
137 static void eventfd_ctx_do_read(struct eventfd_ctx *ctx, __u64 *cnt)
139 *cnt = (ctx->flags & EFD_SEMAPHORE) ? 1 : ctx->count;
144 * eventfd_ctx_remove_wait_queue - Read the current counter and removes wait queue.
145 * @ctx: [in] Pointer to eventfd context.
146 * @wait: [in] Wait queue to be removed.
147 * @cnt: [out] Pointer to the 64-bit counter value.
149 * Returns %0 if successful, or the following error codes:
151 * -EAGAIN : The operation would have blocked.
153 * This is used to atomically remove a wait queue entry from the eventfd wait
154 * queue head, and read/reset the counter value.
156 int eventfd_ctx_remove_wait_queue(struct eventfd_ctx *ctx, wait_queue_t *wait,
161 spin_lock_irqsave(&ctx->wqh.lock, flags);
162 eventfd_ctx_do_read(ctx, cnt);
163 __remove_wait_queue(&ctx->wqh, wait);
164 if (*cnt != 0 && waitqueue_active(&ctx->wqh))
165 wake_up_locked_poll(&ctx->wqh, POLLOUT);
166 spin_unlock_irqrestore(&ctx->wqh.lock, flags);
168 return *cnt != 0 ? 0 : -EAGAIN;
170 EXPORT_SYMBOL_GPL(eventfd_ctx_remove_wait_queue);
173 * eventfd_ctx_read - Reads the eventfd counter or wait if it is zero.
174 * @ctx: [in] Pointer to eventfd context.
175 * @no_wait: [in] Different from zero if the operation should not block.
176 * @cnt: [out] Pointer to the 64-bit counter value.
178 * Returns %0 if successful, or the following error codes:
180 * -EAGAIN : The operation would have blocked but @no_wait was non-zero.
181 * -ERESTARTSYS : A signal interrupted the wait operation.
183 * If @no_wait is zero, the function might sleep until the eventfd internal
184 * counter becomes greater than zero.
186 ssize_t eventfd_ctx_read(struct eventfd_ctx *ctx, int no_wait, __u64 *cnt)
189 DECLARE_WAITQUEUE(wait, current);
191 spin_lock_irq(&ctx->wqh.lock);
197 __add_wait_queue(&ctx->wqh, &wait);
199 set_current_state(TASK_INTERRUPTIBLE);
200 if (ctx->count > 0) {
204 if (signal_pending(current)) {
208 spin_unlock_irq(&ctx->wqh.lock);
210 spin_lock_irq(&ctx->wqh.lock);
212 __remove_wait_queue(&ctx->wqh, &wait);
213 __set_current_state(TASK_RUNNING);
215 if (likely(res == 0)) {
216 eventfd_ctx_do_read(ctx, cnt);
217 if (waitqueue_active(&ctx->wqh))
218 wake_up_locked_poll(&ctx->wqh, POLLOUT);
220 spin_unlock_irq(&ctx->wqh.lock);
224 EXPORT_SYMBOL_GPL(eventfd_ctx_read);
226 static ssize_t eventfd_read(struct file *file, char __user *buf, size_t count,
229 struct eventfd_ctx *ctx = file->private_data;
233 if (count < sizeof(cnt))
235 res = eventfd_ctx_read(ctx, file->f_flags & O_NONBLOCK, &cnt);
239 return put_user(cnt, (__u64 __user *) buf) ? -EFAULT : sizeof(cnt);
242 static ssize_t eventfd_write(struct file *file, const char __user *buf, size_t count,
245 struct eventfd_ctx *ctx = file->private_data;
248 DECLARE_WAITQUEUE(wait, current);
250 if (count < sizeof(ucnt))
252 if (copy_from_user(&ucnt, buf, sizeof(ucnt)))
254 if (ucnt == ULLONG_MAX)
256 spin_lock_irq(&ctx->wqh.lock);
258 if (ULLONG_MAX - ctx->count > ucnt)
260 else if (!(file->f_flags & O_NONBLOCK)) {
261 __add_wait_queue(&ctx->wqh, &wait);
263 set_current_state(TASK_INTERRUPTIBLE);
264 if (ULLONG_MAX - ctx->count > ucnt) {
268 if (signal_pending(current)) {
272 spin_unlock_irq(&ctx->wqh.lock);
274 spin_lock_irq(&ctx->wqh.lock);
276 __remove_wait_queue(&ctx->wqh, &wait);
277 __set_current_state(TASK_RUNNING);
279 if (likely(res > 0)) {
281 if (waitqueue_active(&ctx->wqh))
282 wake_up_locked_poll(&ctx->wqh, POLLIN);
284 spin_unlock_irq(&ctx->wqh.lock);
289 #ifdef CONFIG_PROC_FS
290 static int eventfd_show_fdinfo(struct seq_file *m, struct file *f)
292 struct eventfd_ctx *ctx = f->private_data;
295 spin_lock_irq(&ctx->wqh.lock);
296 ret = seq_printf(m, "eventfd-count: %16llx\n",
297 (unsigned long long)ctx->count);
298 spin_unlock_irq(&ctx->wqh.lock);
304 static const struct file_operations eventfd_fops = {
305 #ifdef CONFIG_PROC_FS
306 .show_fdinfo = eventfd_show_fdinfo,
308 .release = eventfd_release,
309 .poll = eventfd_poll,
310 .read = eventfd_read,
311 .write = eventfd_write,
312 .llseek = noop_llseek,
316 * eventfd_fget - Acquire a reference of an eventfd file descriptor.
317 * @fd: [in] Eventfd file descriptor.
319 * Returns a pointer to the eventfd file structure in case of success, or the
320 * following error pointer:
322 * -EBADF : Invalid @fd file descriptor.
323 * -EINVAL : The @fd file descriptor is not an eventfd file.
325 struct file *eventfd_fget(int fd)
331 return ERR_PTR(-EBADF);
332 if (file->f_op != &eventfd_fops) {
334 return ERR_PTR(-EINVAL);
339 EXPORT_SYMBOL_GPL(eventfd_fget);
342 * eventfd_ctx_fdget - Acquires a reference to the internal eventfd context.
343 * @fd: [in] Eventfd file descriptor.
345 * Returns a pointer to the internal eventfd context, otherwise the error
346 * pointers returned by the following functions:
350 struct eventfd_ctx *eventfd_ctx_fdget(int fd)
353 struct eventfd_ctx *ctx;
355 file = eventfd_fget(fd);
357 return (struct eventfd_ctx *) file;
358 ctx = eventfd_ctx_get(file->private_data);
363 EXPORT_SYMBOL_GPL(eventfd_ctx_fdget);
366 * eventfd_ctx_fileget - Acquires a reference to the internal eventfd context.
367 * @file: [in] Eventfd file pointer.
369 * Returns a pointer to the internal eventfd context, otherwise the error
372 * -EINVAL : The @fd file descriptor is not an eventfd file.
374 struct eventfd_ctx *eventfd_ctx_fileget(struct file *file)
376 if (file->f_op != &eventfd_fops)
377 return ERR_PTR(-EINVAL);
379 return eventfd_ctx_get(file->private_data);
381 EXPORT_SYMBOL_GPL(eventfd_ctx_fileget);
384 * eventfd_file_create - Creates an eventfd file pointer.
385 * @count: Initial eventfd counter value.
386 * @flags: Flags for the eventfd file.
388 * This function creates an eventfd file pointer, w/out installing it into
389 * the fd table. This is useful when the eventfd file is used during the
390 * initialization of data structures that require extra setup after the eventfd
391 * creation. So the eventfd creation is split into the file pointer creation
392 * phase, and the file descriptor installation phase.
393 * In this way races with userspace closing the newly installed file descriptor
395 * Returns an eventfd file pointer, or a proper error pointer.
397 struct file *eventfd_file_create(unsigned int count, int flags)
400 struct eventfd_ctx *ctx;
402 /* Check the EFD_* constants for consistency. */
403 BUILD_BUG_ON(EFD_CLOEXEC != O_CLOEXEC);
404 BUILD_BUG_ON(EFD_NONBLOCK != O_NONBLOCK);
406 if (flags & ~EFD_FLAGS_SET)
407 return ERR_PTR(-EINVAL);
409 ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
411 return ERR_PTR(-ENOMEM);
413 kref_init(&ctx->kref);
414 init_waitqueue_head(&ctx->wqh);
418 file = anon_inode_getfile("[eventfd]", &eventfd_fops, ctx,
419 O_RDWR | (flags & EFD_SHARED_FCNTL_FLAGS));
421 eventfd_free_ctx(ctx);
426 SYSCALL_DEFINE2(eventfd2, unsigned int, count, int, flags)
431 error = get_unused_fd_flags(flags & EFD_SHARED_FCNTL_FLAGS);
436 file = eventfd_file_create(count, flags);
438 error = PTR_ERR(file);
439 goto err_put_unused_fd;
441 fd_install(fd, file);
451 SYSCALL_DEFINE1(eventfd, unsigned int, count)
453 return sys_eventfd2(count, 0);