2 * Software multibuffer async crypto daemon.
4 * Copyright (c) 2014 Tim Chen <tim.c.chen@linux.intel.com>
6 * Adapted from crypto daemon.
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the Free
10 * Software Foundation; either version 2 of the License, or (at your option)
15 #include <crypto/algapi.h>
16 #include <crypto/internal/hash.h>
17 #include <crypto/internal/aead.h>
18 #include <crypto/mcryptd.h>
19 #include <crypto/crypto_wq.h>
20 #include <linux/err.h>
21 #include <linux/init.h>
22 #include <linux/kernel.h>
23 #include <linux/list.h>
24 #include <linux/module.h>
25 #include <linux/scatterlist.h>
26 #include <linux/sched.h>
27 #include <linux/slab.h>
28 #include <linux/hardirq.h>
30 #define MCRYPTD_MAX_CPU_QLEN 100
31 #define MCRYPTD_BATCH 9
33 static void *mcryptd_alloc_instance(struct crypto_alg *alg, unsigned int head,
36 struct mcryptd_flush_list {
37 struct list_head list;
41 static struct mcryptd_flush_list __percpu *mcryptd_flist;
43 struct hashd_instance_ctx {
44 struct crypto_shash_spawn spawn;
45 struct mcryptd_queue *queue;
48 static void mcryptd_queue_worker(struct work_struct *work);
50 void mcryptd_arm_flusher(struct mcryptd_alg_cstate *cstate, unsigned long delay)
52 struct mcryptd_flush_list *flist;
54 if (!cstate->flusher_engaged) {
55 /* put the flusher on the flush list */
56 flist = per_cpu_ptr(mcryptd_flist, smp_processor_id());
57 mutex_lock(&flist->lock);
58 list_add_tail(&cstate->flush_list, &flist->list);
59 cstate->flusher_engaged = true;
60 cstate->next_flush = jiffies + delay;
61 queue_delayed_work_on(smp_processor_id(), kcrypto_wq,
62 &cstate->flush, delay);
63 mutex_unlock(&flist->lock);
66 EXPORT_SYMBOL(mcryptd_arm_flusher);
68 static int mcryptd_init_queue(struct mcryptd_queue *queue,
69 unsigned int max_cpu_qlen)
72 struct mcryptd_cpu_queue *cpu_queue;
74 queue->cpu_queue = alloc_percpu(struct mcryptd_cpu_queue);
75 pr_debug("mqueue:%p mcryptd_cpu_queue %p\n", queue, queue->cpu_queue);
76 if (!queue->cpu_queue)
78 for_each_possible_cpu(cpu) {
79 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
80 pr_debug("cpu_queue #%d %p\n", cpu, queue->cpu_queue);
81 crypto_init_queue(&cpu_queue->queue, max_cpu_qlen);
82 INIT_WORK(&cpu_queue->work, mcryptd_queue_worker);
87 static void mcryptd_fini_queue(struct mcryptd_queue *queue)
90 struct mcryptd_cpu_queue *cpu_queue;
92 for_each_possible_cpu(cpu) {
93 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
94 BUG_ON(cpu_queue->queue.qlen);
96 free_percpu(queue->cpu_queue);
99 static int mcryptd_enqueue_request(struct mcryptd_queue *queue,
100 struct crypto_async_request *request,
101 struct mcryptd_hash_request_ctx *rctx)
104 struct mcryptd_cpu_queue *cpu_queue;
107 cpu_queue = this_cpu_ptr(queue->cpu_queue);
110 err = crypto_enqueue_request(&cpu_queue->queue, request);
111 pr_debug("enqueue request: cpu %d cpu_queue %p request %p\n",
112 cpu, cpu_queue, request);
113 queue_work_on(cpu, kcrypto_wq, &cpu_queue->work);
120 * Try to opportunisticlly flush the partially completed jobs if
121 * crypto daemon is the only task running.
123 static void mcryptd_opportunistic_flush(void)
125 struct mcryptd_flush_list *flist;
126 struct mcryptd_alg_cstate *cstate;
128 flist = per_cpu_ptr(mcryptd_flist, smp_processor_id());
129 while (single_task_running()) {
130 mutex_lock(&flist->lock);
131 if (list_empty(&flist->list)) {
132 mutex_unlock(&flist->lock);
135 cstate = list_entry(flist->list.next,
136 struct mcryptd_alg_cstate, flush_list);
137 if (!cstate->flusher_engaged) {
138 mutex_unlock(&flist->lock);
141 list_del(&cstate->flush_list);
142 cstate->flusher_engaged = false;
143 mutex_unlock(&flist->lock);
144 cstate->alg_state->flusher(cstate);
149 * Called in workqueue context, do one real cryption work (via
150 * req->complete) and reschedule itself if there are more work to
153 static void mcryptd_queue_worker(struct work_struct *work)
155 struct mcryptd_cpu_queue *cpu_queue;
156 struct crypto_async_request *req, *backlog;
160 * Need to loop through more than once for multi-buffer to
164 cpu_queue = container_of(work, struct mcryptd_cpu_queue, work);
166 while (i < MCRYPTD_BATCH || single_task_running()) {
168 * preempt_disable/enable is used to prevent
169 * being preempted by mcryptd_enqueue_request()
173 backlog = crypto_get_backlog(&cpu_queue->queue);
174 req = crypto_dequeue_request(&cpu_queue->queue);
179 mcryptd_opportunistic_flush();
184 backlog->complete(backlog, -EINPROGRESS);
185 req->complete(req, 0);
186 if (!cpu_queue->queue.qlen)
190 if (cpu_queue->queue.qlen)
191 queue_work(kcrypto_wq, &cpu_queue->work);
194 void mcryptd_flusher(struct work_struct *__work)
196 struct mcryptd_alg_cstate *alg_cpu_state;
197 struct mcryptd_alg_state *alg_state;
198 struct mcryptd_flush_list *flist;
201 cpu = smp_processor_id();
202 alg_cpu_state = container_of(to_delayed_work(__work),
203 struct mcryptd_alg_cstate, flush);
204 alg_state = alg_cpu_state->alg_state;
205 if (alg_cpu_state->cpu != cpu)
206 pr_debug("mcryptd error: work on cpu %d, should be cpu %d\n",
207 cpu, alg_cpu_state->cpu);
209 if (alg_cpu_state->flusher_engaged) {
210 flist = per_cpu_ptr(mcryptd_flist, cpu);
211 mutex_lock(&flist->lock);
212 list_del(&alg_cpu_state->flush_list);
213 alg_cpu_state->flusher_engaged = false;
214 mutex_unlock(&flist->lock);
215 alg_state->flusher(alg_cpu_state);
218 EXPORT_SYMBOL_GPL(mcryptd_flusher);
220 static inline struct mcryptd_queue *mcryptd_get_queue(struct crypto_tfm *tfm)
222 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
223 struct mcryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
228 static void *mcryptd_alloc_instance(struct crypto_alg *alg, unsigned int head,
232 struct crypto_instance *inst;
235 p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL);
237 return ERR_PTR(-ENOMEM);
239 inst = (void *)(p + head);
242 if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
243 "mcryptd(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
246 memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
248 inst->alg.cra_priority = alg->cra_priority + 50;
249 inst->alg.cra_blocksize = alg->cra_blocksize;
250 inst->alg.cra_alignmask = alg->cra_alignmask;
261 static inline void mcryptd_check_internal(struct rtattr **tb, u32 *type,
264 struct crypto_attr_type *algt;
266 algt = crypto_get_attr_type(tb);
269 if ((algt->type & CRYPTO_ALG_INTERNAL))
270 *type |= CRYPTO_ALG_INTERNAL;
271 if ((algt->mask & CRYPTO_ALG_INTERNAL))
272 *mask |= CRYPTO_ALG_INTERNAL;
275 static int mcryptd_hash_init_tfm(struct crypto_tfm *tfm)
277 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
278 struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst);
279 struct crypto_shash_spawn *spawn = &ictx->spawn;
280 struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
281 struct crypto_shash *hash;
283 hash = crypto_spawn_shash(spawn);
285 return PTR_ERR(hash);
288 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
289 sizeof(struct mcryptd_hash_request_ctx) +
290 crypto_shash_descsize(hash));
294 static void mcryptd_hash_exit_tfm(struct crypto_tfm *tfm)
296 struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
298 crypto_free_shash(ctx->child);
301 static int mcryptd_hash_setkey(struct crypto_ahash *parent,
302 const u8 *key, unsigned int keylen)
304 struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(parent);
305 struct crypto_shash *child = ctx->child;
308 crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
309 crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) &
310 CRYPTO_TFM_REQ_MASK);
311 err = crypto_shash_setkey(child, key, keylen);
312 crypto_ahash_set_flags(parent, crypto_shash_get_flags(child) &
313 CRYPTO_TFM_RES_MASK);
317 static int mcryptd_hash_enqueue(struct ahash_request *req,
318 crypto_completion_t complete)
322 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
323 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
324 struct mcryptd_queue *queue =
325 mcryptd_get_queue(crypto_ahash_tfm(tfm));
327 rctx->complete = req->base.complete;
328 req->base.complete = complete;
330 ret = mcryptd_enqueue_request(queue, &req->base, rctx);
335 static void mcryptd_hash_init(struct crypto_async_request *req_async, int err)
337 struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
338 struct crypto_shash *child = ctx->child;
339 struct ahash_request *req = ahash_request_cast(req_async);
340 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
341 struct shash_desc *desc = &rctx->desc;
343 if (unlikely(err == -EINPROGRESS))
347 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
349 err = crypto_shash_init(desc);
351 req->base.complete = rctx->complete;
355 rctx->complete(&req->base, err);
359 static int mcryptd_hash_init_enqueue(struct ahash_request *req)
361 return mcryptd_hash_enqueue(req, mcryptd_hash_init);
364 static void mcryptd_hash_update(struct crypto_async_request *req_async, int err)
366 struct ahash_request *req = ahash_request_cast(req_async);
367 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
369 if (unlikely(err == -EINPROGRESS))
372 err = shash_ahash_mcryptd_update(req, &rctx->desc);
374 req->base.complete = rctx->complete;
381 rctx->complete(&req->base, err);
385 static int mcryptd_hash_update_enqueue(struct ahash_request *req)
387 return mcryptd_hash_enqueue(req, mcryptd_hash_update);
390 static void mcryptd_hash_final(struct crypto_async_request *req_async, int err)
392 struct ahash_request *req = ahash_request_cast(req_async);
393 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
395 if (unlikely(err == -EINPROGRESS))
398 err = shash_ahash_mcryptd_final(req, &rctx->desc);
400 req->base.complete = rctx->complete;
407 rctx->complete(&req->base, err);
411 static int mcryptd_hash_final_enqueue(struct ahash_request *req)
413 return mcryptd_hash_enqueue(req, mcryptd_hash_final);
416 static void mcryptd_hash_finup(struct crypto_async_request *req_async, int err)
418 struct ahash_request *req = ahash_request_cast(req_async);
419 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
421 if (unlikely(err == -EINPROGRESS))
424 err = shash_ahash_mcryptd_finup(req, &rctx->desc);
427 req->base.complete = rctx->complete;
434 rctx->complete(&req->base, err);
438 static int mcryptd_hash_finup_enqueue(struct ahash_request *req)
440 return mcryptd_hash_enqueue(req, mcryptd_hash_finup);
443 static void mcryptd_hash_digest(struct crypto_async_request *req_async, int err)
445 struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
446 struct crypto_shash *child = ctx->child;
447 struct ahash_request *req = ahash_request_cast(req_async);
448 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
449 struct shash_desc *desc = &rctx->desc;
451 if (unlikely(err == -EINPROGRESS))
455 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; /* check this again */
457 err = shash_ahash_mcryptd_digest(req, desc);
460 req->base.complete = rctx->complete;
467 rctx->complete(&req->base, err);
471 static int mcryptd_hash_digest_enqueue(struct ahash_request *req)
473 return mcryptd_hash_enqueue(req, mcryptd_hash_digest);
476 static int mcryptd_hash_export(struct ahash_request *req, void *out)
478 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
480 return crypto_shash_export(&rctx->desc, out);
483 static int mcryptd_hash_import(struct ahash_request *req, const void *in)
485 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
487 return crypto_shash_import(&rctx->desc, in);
490 static int mcryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb,
491 struct mcryptd_queue *queue)
493 struct hashd_instance_ctx *ctx;
494 struct ahash_instance *inst;
495 struct shash_alg *salg;
496 struct crypto_alg *alg;
501 mcryptd_check_internal(tb, &type, &mask);
503 salg = shash_attr_alg(tb[1], type, mask);
505 return PTR_ERR(salg);
508 pr_debug("crypto: mcryptd hash alg: %s\n", alg->cra_name);
509 inst = mcryptd_alloc_instance(alg, ahash_instance_headroom(),
515 ctx = ahash_instance_ctx(inst);
518 err = crypto_init_shash_spawn(&ctx->spawn, salg,
519 ahash_crypto_instance(inst));
523 type = CRYPTO_ALG_ASYNC;
524 if (alg->cra_flags & CRYPTO_ALG_INTERNAL)
525 type |= CRYPTO_ALG_INTERNAL;
526 inst->alg.halg.base.cra_flags = type;
528 inst->alg.halg.digestsize = salg->digestsize;
529 inst->alg.halg.base.cra_ctxsize = sizeof(struct mcryptd_hash_ctx);
531 inst->alg.halg.base.cra_init = mcryptd_hash_init_tfm;
532 inst->alg.halg.base.cra_exit = mcryptd_hash_exit_tfm;
534 inst->alg.init = mcryptd_hash_init_enqueue;
535 inst->alg.update = mcryptd_hash_update_enqueue;
536 inst->alg.final = mcryptd_hash_final_enqueue;
537 inst->alg.finup = mcryptd_hash_finup_enqueue;
538 inst->alg.export = mcryptd_hash_export;
539 inst->alg.import = mcryptd_hash_import;
540 inst->alg.setkey = mcryptd_hash_setkey;
541 inst->alg.digest = mcryptd_hash_digest_enqueue;
543 err = ahash_register_instance(tmpl, inst);
545 crypto_drop_shash(&ctx->spawn);
555 static struct mcryptd_queue mqueue;
557 static int mcryptd_create(struct crypto_template *tmpl, struct rtattr **tb)
559 struct crypto_attr_type *algt;
561 algt = crypto_get_attr_type(tb);
563 return PTR_ERR(algt);
565 switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
566 case CRYPTO_ALG_TYPE_DIGEST:
567 return mcryptd_create_hash(tmpl, tb, &mqueue);
574 static void mcryptd_free(struct crypto_instance *inst)
576 struct mcryptd_instance_ctx *ctx = crypto_instance_ctx(inst);
577 struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst);
579 switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) {
580 case CRYPTO_ALG_TYPE_AHASH:
581 crypto_drop_shash(&hctx->spawn);
582 kfree(ahash_instance(inst));
585 crypto_drop_spawn(&ctx->spawn);
590 static struct crypto_template mcryptd_tmpl = {
592 .create = mcryptd_create,
593 .free = mcryptd_free,
594 .module = THIS_MODULE,
597 struct mcryptd_ahash *mcryptd_alloc_ahash(const char *alg_name,
600 char mcryptd_alg_name[CRYPTO_MAX_ALG_NAME];
601 struct crypto_ahash *tfm;
603 if (snprintf(mcryptd_alg_name, CRYPTO_MAX_ALG_NAME,
604 "mcryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
605 return ERR_PTR(-EINVAL);
606 tfm = crypto_alloc_ahash(mcryptd_alg_name, type, mask);
608 return ERR_CAST(tfm);
609 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
610 crypto_free_ahash(tfm);
611 return ERR_PTR(-EINVAL);
614 return __mcryptd_ahash_cast(tfm);
616 EXPORT_SYMBOL_GPL(mcryptd_alloc_ahash);
618 int shash_ahash_mcryptd_digest(struct ahash_request *req,
619 struct shash_desc *desc)
623 err = crypto_shash_init(desc) ?:
624 shash_ahash_mcryptd_finup(req, desc);
628 EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_digest);
630 int shash_ahash_mcryptd_update(struct ahash_request *req,
631 struct shash_desc *desc)
633 struct crypto_shash *tfm = desc->tfm;
634 struct shash_alg *shash = crypto_shash_alg(tfm);
636 /* alignment is to be done by multi-buffer crypto algorithm if needed */
638 return shash->update(desc, NULL, 0);
640 EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_update);
642 int shash_ahash_mcryptd_finup(struct ahash_request *req,
643 struct shash_desc *desc)
645 struct crypto_shash *tfm = desc->tfm;
646 struct shash_alg *shash = crypto_shash_alg(tfm);
648 /* alignment is to be done by multi-buffer crypto algorithm if needed */
650 return shash->finup(desc, NULL, 0, req->result);
652 EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_finup);
654 int shash_ahash_mcryptd_final(struct ahash_request *req,
655 struct shash_desc *desc)
657 struct crypto_shash *tfm = desc->tfm;
658 struct shash_alg *shash = crypto_shash_alg(tfm);
660 /* alignment is to be done by multi-buffer crypto algorithm if needed */
662 return shash->final(desc, req->result);
664 EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_final);
666 struct crypto_shash *mcryptd_ahash_child(struct mcryptd_ahash *tfm)
668 struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
672 EXPORT_SYMBOL_GPL(mcryptd_ahash_child);
674 struct shash_desc *mcryptd_shash_desc(struct ahash_request *req)
676 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
679 EXPORT_SYMBOL_GPL(mcryptd_shash_desc);
681 void mcryptd_free_ahash(struct mcryptd_ahash *tfm)
683 crypto_free_ahash(&tfm->base);
685 EXPORT_SYMBOL_GPL(mcryptd_free_ahash);
688 static int __init mcryptd_init(void)
691 struct mcryptd_flush_list *flist;
693 mcryptd_flist = alloc_percpu(struct mcryptd_flush_list);
694 for_each_possible_cpu(cpu) {
695 flist = per_cpu_ptr(mcryptd_flist, cpu);
696 INIT_LIST_HEAD(&flist->list);
697 mutex_init(&flist->lock);
700 err = mcryptd_init_queue(&mqueue, MCRYPTD_MAX_CPU_QLEN);
702 free_percpu(mcryptd_flist);
706 err = crypto_register_template(&mcryptd_tmpl);
708 mcryptd_fini_queue(&mqueue);
709 free_percpu(mcryptd_flist);
715 static void __exit mcryptd_exit(void)
717 mcryptd_fini_queue(&mqueue);
718 crypto_unregister_template(&mcryptd_tmpl);
719 free_percpu(mcryptd_flist);
722 subsys_initcall(mcryptd_init);
723 module_exit(mcryptd_exit);
725 MODULE_LICENSE("GPL");
726 MODULE_DESCRIPTION("Software async multibuffer crypto daemon");
727 MODULE_ALIAS_CRYPTO("mcryptd");