From aa0bfc9cee13677f110cfb870f7b5d9cd60cf5a7 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 17 Jul 2017 15:32:30 +0800 Subject: crypto: authencesn - Fix digest_null crash When authencesn is used together with digest_null a crash will occur on the decrypt path. This is because normally we perform a special setup to preserve the ESN, but this is skipped if there is no authentication. However, on the post-authentication path it always expects the preservation to be in place, thus causing a crash when digest_null is used. This patch fixes this by also skipping the post-processing when there is no authentication. Fixes: 3c481edeec50 ("crypto: authencesn - Convert to new AEAD...") Cc: Reported-by: Jan Tluka Signed-off-by: Herbert Xu --- crypto/authencesn.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crypto/authencesn.c b/crypto/authencesn.c index 6f8f6b86..0cf5fefd 100644 --- a/crypto/authencesn.c +++ b/crypto/authencesn.c @@ -248,6 +248,9 @@ static int crypto_authenc_esn_decrypt_tail(struct aead_request *req, u8 *ihash = ohash + crypto_ahash_digestsize(auth); u32 tmp[2]; + if (!authsize) + goto decrypt; + /* Move high-order bits of sequence number back. */ scatterwalk_map_and_copy(tmp, dst, 4, 4, 0); scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen, 4, 0); @@ -256,6 +259,8 @@ static int crypto_authenc_esn_decrypt_tail(struct aead_request *req, if (crypto_memneq(ihash, ohash, authsize)) return -EBADMSG; +decrypt: + sg_init_table(areq_ctx->dst, 2); dst = scatterwalk_ffwd(areq_ctx->dst, dst, assoclen); -- cgit v1.2.3 From 8a15a4bf19477d91df0ffc20ef66dfa9f793c150 Mon Sep 17 00:00:00 2001 From: Stephan Mueller Date: Sun, 25 Jun 2017 17:12:39 +0200 Subject: crypto: algif_skcipher - overhaul memory management The updated memory management is described in the top part of the code. As one benefit of the changed memory management, the AIO and synchronous operation is now implemented in one common function. The AF_ALG operation uses the async kernel crypto API interface for each cipher operation. Thus, the only difference between the AIO and sync operation types visible from user space is: 1. the callback function to be invoked when the asynchronous operation is completed 2. whether to wait for the completion of the kernel crypto API operation or not In addition, the code structure is adjusted to match the structure of algif_aead for easier code assessment. The user space interface changed slightly as follows: the old AIO operation returned zero upon success and < 0 in case of an error to user space. As all other AF_ALG interfaces (including the sync skcipher interface) returned the number of processed bytes upon success and < 0 in case of an error, the new skcipher interface (regardless of AIO or sync) returns the number of processed bytes in case of success. Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/algif_skcipher.c | 561 ++++++++++++++++++++++++------------------------ 1 file changed, 279 insertions(+), 282 deletions(-) diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c index 43839b00..968d094f 100644 --- a/crypto/algif_skcipher.c +++ b/crypto/algif_skcipher.c @@ -10,6 +10,21 @@ * Software Foundation; either version 2 of the License, or (at your option) * any later version. * + * The following concept of the memory management is used: + * + * The kernel maintains two SGLs, the TX SGL and the RX SGL. The TX SGL is + * filled by user space with the data submitted via sendpage/sendmsg. Filling + * up the TX SGL does not cause a crypto operation -- the data will only be + * tracked by the kernel. Upon receipt of one recvmsg call, the caller must + * provide a buffer which is tracked with the RX SGL. + * + * During the processing of the recvmsg operation, the cipher request is + * allocated and prepared. As part of the recvmsg operation, the processed + * TX buffers are extracted from the TX SGL into a separate SGL. + * + * After the completion of the crypto operation, the RX SGL and the cipher + * request is released. The extracted TX SGL parts are released together with + * the RX SGL release. */ #include @@ -24,109 +39,94 @@ #include #include -struct skcipher_sg_list { +struct skcipher_tsgl { struct list_head list; - int cur; - struct scatterlist sg[0]; }; +struct skcipher_rsgl { + struct af_alg_sgl sgl; + struct list_head list; + size_t sg_num_bytes; +}; + +struct skcipher_async_req { + struct kiocb *iocb; + struct sock *sk; + + struct skcipher_rsgl first_sgl; + struct list_head rsgl_list; + + struct scatterlist *tsgl; + unsigned int tsgl_entries; + + unsigned int areqlen; + struct skcipher_request req; +}; + struct skcipher_tfm { struct crypto_skcipher *skcipher; bool has_key; }; struct skcipher_ctx { - struct list_head tsgl; - struct af_alg_sgl rsgl; + struct list_head tsgl_list; void *iv; struct af_alg_completion completion; - atomic_t inflight; size_t used; + size_t rcvused; - unsigned int len; bool more; bool merge; bool enc; - struct skcipher_request req; -}; - -struct skcipher_async_rsgl { - struct af_alg_sgl sgl; - struct list_head list; -}; - -struct skcipher_async_req { - struct kiocb *iocb; - struct skcipher_async_rsgl first_sgl; - struct list_head list; - struct scatterlist *tsg; - atomic_t *inflight; - struct skcipher_request req; + unsigned int len; }; -#define MAX_SGL_ENTS ((4096 - sizeof(struct skcipher_sg_list)) / \ +#define MAX_SGL_ENTS ((4096 - sizeof(struct skcipher_tsgl)) / \ sizeof(struct scatterlist) - 1) -static void skcipher_free_async_sgls(struct skcipher_async_req *sreq) +static inline int skcipher_sndbuf(struct sock *sk) { - struct skcipher_async_rsgl *rsgl, *tmp; - struct scatterlist *sgl; - struct scatterlist *sg; - int i, n; - - list_for_each_entry_safe(rsgl, tmp, &sreq->list, list) { - af_alg_free_sg(&rsgl->sgl); - if (rsgl != &sreq->first_sgl) - kfree(rsgl); - } - sgl = sreq->tsg; - n = sg_nents(sgl); - for_each_sg(sgl, sg, n, i) - put_page(sg_page(sg)); + struct alg_sock *ask = alg_sk(sk); + struct skcipher_ctx *ctx = ask->private; - kfree(sreq->tsg); + return max_t(int, max_t(int, sk->sk_sndbuf & PAGE_MASK, PAGE_SIZE) - + ctx->used, 0); } -static void skcipher_async_cb(struct crypto_async_request *req, int err) +static inline bool skcipher_writable(struct sock *sk) { - struct skcipher_async_req *sreq = req->data; - struct kiocb *iocb = sreq->iocb; - - atomic_dec(sreq->inflight); - skcipher_free_async_sgls(sreq); - kzfree(sreq); - iocb->ki_complete(iocb, err, err); + return PAGE_SIZE <= skcipher_sndbuf(sk); } -static inline int skcipher_sndbuf(struct sock *sk) +static inline int skcipher_rcvbuf(struct sock *sk) { struct alg_sock *ask = alg_sk(sk); struct skcipher_ctx *ctx = ask->private; - return max_t(int, max_t(int, sk->sk_sndbuf & PAGE_MASK, PAGE_SIZE) - - ctx->used, 0); + return max_t(int, max_t(int, sk->sk_rcvbuf & PAGE_MASK, PAGE_SIZE) - + ctx->rcvused, 0); } -static inline bool skcipher_writable(struct sock *sk) +static inline bool skcipher_readable(struct sock *sk) { - return PAGE_SIZE <= skcipher_sndbuf(sk); + return PAGE_SIZE <= skcipher_rcvbuf(sk); } -static int skcipher_alloc_sgl(struct sock *sk) +static int skcipher_alloc_tsgl(struct sock *sk) { struct alg_sock *ask = alg_sk(sk); struct skcipher_ctx *ctx = ask->private; - struct skcipher_sg_list *sgl; + struct skcipher_tsgl *sgl; struct scatterlist *sg = NULL; - sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list); - if (!list_empty(&ctx->tsgl)) + sgl = list_entry(ctx->tsgl_list.prev, struct skcipher_tsgl, list); + if (!list_empty(&ctx->tsgl_list)) sg = sgl->sg; if (!sg || sgl->cur >= MAX_SGL_ENTS) { @@ -142,31 +142,66 @@ static int skcipher_alloc_sgl(struct sock *sk) if (sg) sg_chain(sg, MAX_SGL_ENTS + 1, sgl->sg); - list_add_tail(&sgl->list, &ctx->tsgl); + list_add_tail(&sgl->list, &ctx->tsgl_list); } return 0; } -static void skcipher_pull_sgl(struct sock *sk, size_t used, int put) +static unsigned int skcipher_count_tsgl(struct sock *sk, size_t bytes) +{ + struct alg_sock *ask = alg_sk(sk); + struct skcipher_ctx *ctx = ask->private; + struct skcipher_tsgl *sgl, *tmp; + unsigned int i; + unsigned int sgl_count = 0; + + if (!bytes) + return 0; + + list_for_each_entry_safe(sgl, tmp, &ctx->tsgl_list, list) { + struct scatterlist *sg = sgl->sg; + + for (i = 0; i < sgl->cur; i++) { + sgl_count++; + if (sg[i].length >= bytes) + return sgl_count; + + bytes -= sg[i].length; + } + } + + return sgl_count; +} + +static void skcipher_pull_tsgl(struct sock *sk, size_t used, + struct scatterlist *dst) { struct alg_sock *ask = alg_sk(sk); struct skcipher_ctx *ctx = ask->private; - struct skcipher_sg_list *sgl; + struct skcipher_tsgl *sgl; struct scatterlist *sg; - int i; + unsigned int i; - while (!list_empty(&ctx->tsgl)) { - sgl = list_first_entry(&ctx->tsgl, struct skcipher_sg_list, + while (!list_empty(&ctx->tsgl_list)) { + sgl = list_first_entry(&ctx->tsgl_list, struct skcipher_tsgl, list); sg = sgl->sg; for (i = 0; i < sgl->cur; i++) { size_t plen = min_t(size_t, used, sg[i].length); + struct page *page = sg_page(sg + i); - if (!sg_page(sg + i)) + if (!page) continue; + /* + * Assumption: caller created skcipher_count_tsgl(len) + * SG entries in dst. + */ + if (dst) + sg_set_page(dst + i, page, plen, sg[i].offset); + sg[i].length -= plen; sg[i].offset += plen; @@ -175,27 +210,48 @@ static void skcipher_pull_sgl(struct sock *sk, size_t used, int put) if (sg[i].length) return; - if (put) - put_page(sg_page(sg + i)); + + if (!dst) + put_page(page); sg_assign_page(sg + i, NULL); } list_del(&sgl->list); - sock_kfree_s(sk, sgl, - sizeof(*sgl) + sizeof(sgl->sg[0]) * - (MAX_SGL_ENTS + 1)); + sock_kfree_s(sk, sgl, sizeof(*sgl) + sizeof(sgl->sg[0]) * + (MAX_SGL_ENTS + 1)); } if (!ctx->used) ctx->merge = 0; } -static void skcipher_free_sgl(struct sock *sk) +static void skcipher_free_areq_sgls(struct skcipher_async_req *areq) { + struct sock *sk = areq->sk; struct alg_sock *ask = alg_sk(sk); struct skcipher_ctx *ctx = ask->private; + struct skcipher_rsgl *rsgl, *tmp; + struct scatterlist *tsgl; + struct scatterlist *sg; + unsigned int i; + + list_for_each_entry_safe(rsgl, tmp, &areq->rsgl_list, list) { + ctx->rcvused -= rsgl->sg_num_bytes; + af_alg_free_sg(&rsgl->sgl); + list_del(&rsgl->list); + if (rsgl != &areq->first_sgl) + sock_kfree_s(sk, rsgl, sizeof(*rsgl)); + } + + tsgl = areq->tsgl; + for_each_sg(tsgl, sg, areq->tsgl_entries, i) { + if (!sg_page(sg)) + continue; + put_page(sg_page(sg)); + } - skcipher_pull_sgl(sk, ctx->used, 1); + if (areq->tsgl && areq->tsgl_entries) + sock_kfree_s(sk, tsgl, areq->tsgl_entries * sizeof(*tsgl)); } static int skcipher_wait_for_wmem(struct sock *sk, unsigned flags) @@ -302,7 +358,7 @@ static int skcipher_sendmsg(struct socket *sock, struct msghdr *msg, struct skcipher_tfm *skc = pask->private; struct crypto_skcipher *tfm = skc->skcipher; unsigned ivsize = crypto_skcipher_ivsize(tfm); - struct skcipher_sg_list *sgl; + struct skcipher_tsgl *sgl; struct af_alg_control con = {}; long copied = 0; bool enc = 0; @@ -349,8 +405,8 @@ static int skcipher_sendmsg(struct socket *sock, struct msghdr *msg, size_t plen; if (ctx->merge) { - sgl = list_entry(ctx->tsgl.prev, - struct skcipher_sg_list, list); + sgl = list_entry(ctx->tsgl_list.prev, + struct skcipher_tsgl, list); sg = sgl->sg + sgl->cur - 1; len = min_t(unsigned long, len, PAGE_SIZE - sg->offset - sg->length); @@ -379,11 +435,12 @@ static int skcipher_sendmsg(struct socket *sock, struct msghdr *msg, len = min_t(unsigned long, len, skcipher_sndbuf(sk)); - err = skcipher_alloc_sgl(sk); + err = skcipher_alloc_tsgl(sk); if (err) goto unlock; - sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list); + sgl = list_entry(ctx->tsgl_list.prev, struct skcipher_tsgl, + list); sg = sgl->sg; if (sgl->cur) sg_unmark_end(sg + sgl->cur - 1); @@ -435,7 +492,7 @@ static ssize_t skcipher_sendpage(struct socket *sock, struct page *page, struct sock *sk = sock->sk; struct alg_sock *ask = alg_sk(sk); struct skcipher_ctx *ctx = ask->private; - struct skcipher_sg_list *sgl; + struct skcipher_tsgl *sgl; int err = -EINVAL; if (flags & MSG_SENDPAGE_NOTLAST) @@ -454,12 +511,12 @@ static ssize_t skcipher_sendpage(struct socket *sock, struct page *page, goto unlock; } - err = skcipher_alloc_sgl(sk); + err = skcipher_alloc_tsgl(sk); if (err) goto unlock; ctx->merge = 0; - sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list); + sgl = list_entry(ctx->tsgl_list.prev, struct skcipher_tsgl, list); if (sgl->cur) sg_unmark_end(sgl->sg + sgl->cur - 1); @@ -480,25 +537,29 @@ unlock: return err ?: size; } -static int skcipher_all_sg_nents(struct skcipher_ctx *ctx) +static void skcipher_async_cb(struct crypto_async_request *req, int err) { - struct skcipher_sg_list *sgl; - struct scatterlist *sg; - int nents = 0; + struct skcipher_async_req *areq = req->data; + struct sock *sk = areq->sk; + struct kiocb *iocb = areq->iocb; + unsigned int resultlen; - list_for_each_entry(sgl, &ctx->tsgl, list) { - sg = sgl->sg; + lock_sock(sk); - while (!sg->length) - sg++; + /* Buffer size written by crypto operation. */ + resultlen = areq->req.cryptlen; - nents += sg_nents(sg); - } - return nents; + skcipher_free_areq_sgls(areq); + sock_kfree_s(sk, areq, areq->areqlen); + __sock_put(sk); + + iocb->ki_complete(iocb, err ? err : resultlen, 0); + + release_sock(sk); } -static int skcipher_recvmsg_async(struct socket *sock, struct msghdr *msg, - int flags) +static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg, + size_t ignored, int flags) { struct sock *sk = sock->sk; struct alg_sock *ask = alg_sk(sk); @@ -507,215 +568,166 @@ static int skcipher_recvmsg_async(struct socket *sock, struct msghdr *msg, struct skcipher_ctx *ctx = ask->private; struct skcipher_tfm *skc = pask->private; struct crypto_skcipher *tfm = skc->skcipher; - struct skcipher_sg_list *sgl; - struct scatterlist *sg; - struct skcipher_async_req *sreq; - struct skcipher_request *req; - struct skcipher_async_rsgl *last_rsgl = NULL; - unsigned int txbufs = 0, len = 0, tx_nents; - unsigned int reqsize = crypto_skcipher_reqsize(tfm); - unsigned int ivsize = crypto_skcipher_ivsize(tfm); - int err = -ENOMEM; - bool mark = false; - char *iv; - - sreq = kzalloc(sizeof(*sreq) + reqsize + ivsize, GFP_KERNEL); - if (unlikely(!sreq)) - goto out; - - req = &sreq->req; - iv = (char *)(req + 1) + reqsize; - sreq->iocb = msg->msg_iocb; - INIT_LIST_HEAD(&sreq->list); - sreq->inflight = &ctx->inflight; + unsigned int bs = crypto_skcipher_blocksize(tfm); + unsigned int areqlen = sizeof(struct skcipher_async_req) + + crypto_skcipher_reqsize(tfm); + struct skcipher_async_req *areq; + struct skcipher_rsgl *last_rsgl = NULL; + int err = 0; + size_t len = 0; - lock_sock(sk); - tx_nents = skcipher_all_sg_nents(ctx); - sreq->tsg = kcalloc(tx_nents, sizeof(*sg), GFP_KERNEL); - if (unlikely(!sreq->tsg)) - goto unlock; - sg_init_table(sreq->tsg, tx_nents); - memcpy(iv, ctx->iv, ivsize); - skcipher_request_set_tfm(req, tfm); - skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, - skcipher_async_cb, sreq); + /* Allocate cipher request for current operation. */ + areq = sock_kmalloc(sk, areqlen, GFP_KERNEL); + if (unlikely(!areq)) + return -ENOMEM; + areq->areqlen = areqlen; + areq->sk = sk; + INIT_LIST_HEAD(&areq->rsgl_list); + areq->tsgl = NULL; + areq->tsgl_entries = 0; - while (iov_iter_count(&msg->msg_iter)) { - struct skcipher_async_rsgl *rsgl; - int used; + /* convert iovecs of output buffers into RX SGL */ + while (msg_data_left(msg)) { + struct skcipher_rsgl *rsgl; + size_t seglen; + + /* limit the amount of readable buffers */ + if (!skcipher_readable(sk)) + break; if (!ctx->used) { err = skcipher_wait_for_data(sk, flags); if (err) goto free; } - sgl = list_first_entry(&ctx->tsgl, - struct skcipher_sg_list, list); - sg = sgl->sg; - while (!sg->length) - sg++; - - used = min_t(unsigned long, ctx->used, - iov_iter_count(&msg->msg_iter)); - used = min_t(unsigned long, used, sg->length); - - if (txbufs == tx_nents) { - struct scatterlist *tmp; - int x; - /* Ran out of tx slots in async request - * need to expand */ - tmp = kcalloc(tx_nents * 2, sizeof(*tmp), - GFP_KERNEL); - if (!tmp) { - err = -ENOMEM; - goto free; - } + seglen = min_t(size_t, ctx->used, msg_data_left(msg)); - sg_init_table(tmp, tx_nents * 2); - for (x = 0; x < tx_nents; x++) - sg_set_page(&tmp[x], sg_page(&sreq->tsg[x]), - sreq->tsg[x].length, - sreq->tsg[x].offset); - kfree(sreq->tsg); - sreq->tsg = tmp; - tx_nents *= 2; - mark = true; - } - /* Need to take over the tx sgl from ctx - * to the asynch req - these sgls will be freed later */ - sg_set_page(sreq->tsg + txbufs++, sg_page(sg), sg->length, - sg->offset); - - if (list_empty(&sreq->list)) { - rsgl = &sreq->first_sgl; - list_add_tail(&rsgl->list, &sreq->list); + if (list_empty(&areq->rsgl_list)) { + rsgl = &areq->first_sgl; } else { - rsgl = kmalloc(sizeof(*rsgl), GFP_KERNEL); + rsgl = sock_kmalloc(sk, sizeof(*rsgl), GFP_KERNEL); if (!rsgl) { err = -ENOMEM; goto free; } - list_add_tail(&rsgl->list, &sreq->list); } - used = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, used); - err = used; - if (used < 0) + rsgl->sgl.npages = 0; + list_add_tail(&rsgl->list, &areq->rsgl_list); + + /* make one iovec available as scatterlist */ + err = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, seglen); + if (err < 0) goto free; + + /* chain the new scatterlist with previous one */ if (last_rsgl) af_alg_link_sg(&last_rsgl->sgl, &rsgl->sgl); last_rsgl = rsgl; - len += used; - skcipher_pull_sgl(sk, used, 0); - iov_iter_advance(&msg->msg_iter, used); + len += err; + ctx->rcvused += err; + rsgl->sg_num_bytes = err; + iov_iter_advance(&msg->msg_iter, err); } - if (mark) - sg_mark_end(sreq->tsg + txbufs - 1); + /* Process only as much RX buffers for which we have TX data */ + if (len > ctx->used) + len = ctx->used; + + /* + * If more buffers are to be expected to be processed, process only + * full block size buffers. + */ + if (ctx->more || len < ctx->used) + len -= len % bs; + + /* + * Create a per request TX SGL for this request which tracks the + * SG entries from the global TX SGL. + */ + areq->tsgl_entries = skcipher_count_tsgl(sk, len); + if (!areq->tsgl_entries) + areq->tsgl_entries = 1; + areq->tsgl = sock_kmalloc(sk, sizeof(*areq->tsgl) * areq->tsgl_entries, + GFP_KERNEL); + if (!areq->tsgl) { + err = -ENOMEM; + goto free; + } + sg_init_table(areq->tsgl, areq->tsgl_entries); + skcipher_pull_tsgl(sk, len, areq->tsgl); + + /* Initialize the crypto operation */ + skcipher_request_set_tfm(&areq->req, tfm); + skcipher_request_set_crypt(&areq->req, areq->tsgl, + areq->first_sgl.sgl.sg, len, ctx->iv); + + if (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) { + /* AIO operation */ + areq->iocb = msg->msg_iocb; + skcipher_request_set_callback(&areq->req, + CRYPTO_TFM_REQ_MAY_SLEEP, + skcipher_async_cb, areq); + err = ctx->enc ? crypto_skcipher_encrypt(&areq->req) : + crypto_skcipher_decrypt(&areq->req); + } else { + /* Synchronous operation */ + skcipher_request_set_callback(&areq->req, + CRYPTO_TFM_REQ_MAY_SLEEP | + CRYPTO_TFM_REQ_MAY_BACKLOG, + af_alg_complete, + &ctx->completion); + err = af_alg_wait_for_completion(ctx->enc ? + crypto_skcipher_encrypt(&areq->req) : + crypto_skcipher_decrypt(&areq->req), + &ctx->completion); + } - skcipher_request_set_crypt(req, sreq->tsg, sreq->first_sgl.sgl.sg, - len, iv); - err = ctx->enc ? crypto_skcipher_encrypt(req) : - crypto_skcipher_decrypt(req); + /* AIO operation in progress */ if (err == -EINPROGRESS) { - atomic_inc(&ctx->inflight); - err = -EIOCBQUEUED; - sreq = NULL; - goto unlock; + sock_hold(sk); + return -EIOCBQUEUED; } + free: - skcipher_free_async_sgls(sreq); -unlock: - skcipher_wmem_wakeup(sk); - release_sock(sk); - kzfree(sreq); -out: - return err; + skcipher_free_areq_sgls(areq); + if (areq) + sock_kfree_s(sk, areq, areqlen); + + return err ? err : len; } -static int skcipher_recvmsg_sync(struct socket *sock, struct msghdr *msg, - int flags) +static int skcipher_recvmsg(struct socket *sock, struct msghdr *msg, + size_t ignored, int flags) { struct sock *sk = sock->sk; - struct alg_sock *ask = alg_sk(sk); - struct sock *psk = ask->parent; - struct alg_sock *pask = alg_sk(psk); - struct skcipher_ctx *ctx = ask->private; - struct skcipher_tfm *skc = pask->private; - struct crypto_skcipher *tfm = skc->skcipher; - unsigned bs = crypto_skcipher_blocksize(tfm); - struct skcipher_sg_list *sgl; - struct scatterlist *sg; - int err = -EAGAIN; - int used; - long copied = 0; + int ret = 0; lock_sock(sk); while (msg_data_left(msg)) { - if (!ctx->used) { - err = skcipher_wait_for_data(sk, flags); - if (err) - goto unlock; + int err = _skcipher_recvmsg(sock, msg, ignored, flags); + + /* + * This error covers -EIOCBQUEUED which implies that we can + * only handle one AIO request. If the caller wants to have + * multiple AIO requests in parallel, he must make multiple + * separate AIO calls. + */ + if (err <= 0) { + if (err == -EIOCBQUEUED) + ret = err; + goto out; } - used = min_t(unsigned long, ctx->used, msg_data_left(msg)); - - used = af_alg_make_sg(&ctx->rsgl, &msg->msg_iter, used); - err = used; - if (err < 0) - goto unlock; - - if (ctx->more || used < ctx->used) - used -= used % bs; - - err = -EINVAL; - if (!used) - goto free; - - sgl = list_first_entry(&ctx->tsgl, - struct skcipher_sg_list, list); - sg = sgl->sg; - - while (!sg->length) - sg++; - - skcipher_request_set_crypt(&ctx->req, sg, ctx->rsgl.sg, used, - ctx->iv); - - err = af_alg_wait_for_completion( - ctx->enc ? - crypto_skcipher_encrypt(&ctx->req) : - crypto_skcipher_decrypt(&ctx->req), - &ctx->completion); - -free: - af_alg_free_sg(&ctx->rsgl); - - if (err) - goto unlock; - - copied += used; - skcipher_pull_sgl(sk, used, 1); - iov_iter_advance(&msg->msg_iter, used); + ret += err; } - err = 0; - -unlock: +out: skcipher_wmem_wakeup(sk); release_sock(sk); - - return copied ?: err; -} - -static int skcipher_recvmsg(struct socket *sock, struct msghdr *msg, - size_t ignored, int flags) -{ - return (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) ? - skcipher_recvmsg_async(sock, msg, flags) : - skcipher_recvmsg_sync(sock, msg, flags); + return ret; } static unsigned int skcipher_poll(struct file *file, struct socket *sock, @@ -895,26 +907,16 @@ static int skcipher_setkey(void *private, const u8 *key, unsigned int keylen) return err; } -static void skcipher_wait(struct sock *sk) -{ - struct alg_sock *ask = alg_sk(sk); - struct skcipher_ctx *ctx = ask->private; - int ctr = 0; - - while (atomic_read(&ctx->inflight) && ctr++ < 100) - msleep(100); -} - static void skcipher_sock_destruct(struct sock *sk) { struct alg_sock *ask = alg_sk(sk); struct skcipher_ctx *ctx = ask->private; - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(&ctx->req); - - if (atomic_read(&ctx->inflight)) - skcipher_wait(sk); + struct sock *psk = ask->parent; + struct alg_sock *pask = alg_sk(psk); + struct skcipher_tfm *skc = pask->private; + struct crypto_skcipher *tfm = skc->skcipher; - skcipher_free_sgl(sk); + skcipher_pull_tsgl(sk, ctx->used, NULL); sock_kzfree_s(sk, ctx->iv, crypto_skcipher_ivsize(tfm)); sock_kfree_s(sk, ctx, ctx->len); af_alg_release_parent(sk); @@ -926,7 +928,7 @@ static int skcipher_accept_parent_nokey(void *private, struct sock *sk) struct alg_sock *ask = alg_sk(sk); struct skcipher_tfm *tfm = private; struct crypto_skcipher *skcipher = tfm->skcipher; - unsigned int len = sizeof(*ctx) + crypto_skcipher_reqsize(skcipher); + unsigned int len = sizeof(*ctx); ctx = sock_kmalloc(sk, len, GFP_KERNEL); if (!ctx) @@ -941,22 +943,17 @@ static int skcipher_accept_parent_nokey(void *private, struct sock *sk) memset(ctx->iv, 0, crypto_skcipher_ivsize(skcipher)); - INIT_LIST_HEAD(&ctx->tsgl); + INIT_LIST_HEAD(&ctx->tsgl_list); ctx->len = len; ctx->used = 0; + ctx->rcvused = 0; ctx->more = 0; ctx->merge = 0; ctx->enc = 0; - atomic_set(&ctx->inflight, 0); af_alg_init_completion(&ctx->completion); ask->private = ctx; - skcipher_request_set_tfm(&ctx->req, skcipher); - skcipher_request_set_callback(&ctx->req, CRYPTO_TFM_REQ_MAY_SLEEP | - CRYPTO_TFM_REQ_MAY_BACKLOG, - af_alg_complete, &ctx->completion); - sk->sk_destruct = skcipher_sock_destruct; return 0; -- cgit v1.2.3 From 0c2a2d0aa6e4c7dcc72800db02f2a7b87b783905 Mon Sep 17 00:00:00 2001 From: Stephan Mueller Date: Sun, 25 Jun 2017 17:12:59 +0200 Subject: crypto: algif_aead - overhaul memory management The updated memory management is described in the top part of the code. As one benefit of the changed memory management, the AIO and synchronous operation is now implemented in one common function. The AF_ALG operation uses the async kernel crypto API interface for each cipher operation. Thus, the only difference between the AIO and sync operation types visible from user space is: 1. the callback function to be invoked when the asynchronous operation is completed 2. whether to wait for the completion of the kernel crypto API operation or not The change includes the overhaul of the TX and RX SGL handling. The TX SGL holding the data sent from user space to the kernel is now dynamic similar to algif_skcipher. This dynamic nature allows a continuous operation of a thread sending data and a second thread receiving the data. These threads do not need to synchronize as the kernel processes as much data from the TX SGL to fill the RX SGL. The caller reading the data from the kernel defines the amount of data to be processed. Considering that the interface covers AEAD authenticating ciphers, the reader must provide the buffer in the correct size. Thus the reader defines the encryption size. Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/algif_aead.c | 766 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 442 insertions(+), 324 deletions(-) diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c index be117495..9755aac0 100644 --- a/crypto/algif_aead.c +++ b/crypto/algif_aead.c @@ -5,12 +5,26 @@ * * This file provides the user-space API for AEAD ciphers. * - * This file is derived from algif_skcipher.c. - * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation; either version 2 of the License, or (at your option) * any later version. + * + * The following concept of the memory management is used: + * + * The kernel maintains two SGLs, the TX SGL and the RX SGL. The TX SGL is + * filled by user space with the data submitted via sendpage/sendmsg. Filling + * up the TX SGL does not cause a crypto operation -- the data will only be + * tracked by the kernel. Upon receipt of one recvmsg call, the caller must + * provide a buffer which is tracked with the RX SGL. + * + * During the processing of the recvmsg operation, the cipher request is + * allocated and prepared. As part of the recvmsg operation, the processed + * TX buffers are extracted from the TX SGL into a separate SGL. + * + * After the completion of the crypto operation, the RX SGL and the cipher + * request is released. The extracted TX SGL parts are released together with + * the RX SGL release. */ #include @@ -25,24 +39,32 @@ #include #include -struct aead_sg_list { - unsigned int cur; - struct scatterlist sg[ALG_MAX_PAGES]; +struct aead_tsgl { + struct list_head list; + unsigned int cur; /* Last processed SG entry */ + struct scatterlist sg[0]; /* Array of SGs forming the SGL */ }; -struct aead_async_rsgl { +struct aead_rsgl { struct af_alg_sgl sgl; struct list_head list; + size_t sg_num_bytes; /* Bytes of data in that SGL */ }; struct aead_async_req { - struct scatterlist *tsgl; - struct aead_async_rsgl first_rsgl; - struct list_head list; struct kiocb *iocb; struct sock *sk; - unsigned int tsgls; - char iv[]; + + struct aead_rsgl first_rsgl; /* First RX SG */ + struct list_head rsgl_list; /* Track RX SGs */ + + struct scatterlist *tsgl; /* priv. TX SGL of buffers to process */ + unsigned int tsgl_entries; /* number of entries in priv. TX SGL */ + + unsigned int outlen; /* Filled output buf length */ + + unsigned int areqlen; /* Length of this data struct */ + struct aead_request aead_req; /* req ctx trails this struct */ }; struct aead_tfm { @@ -51,25 +73,26 @@ struct aead_tfm { }; struct aead_ctx { - struct aead_sg_list tsgl; - struct aead_async_rsgl first_rsgl; - struct list_head list; + struct list_head tsgl_list; /* Link to TX SGL */ void *iv; + size_t aead_assoclen; - struct af_alg_completion completion; + struct af_alg_completion completion; /* sync work queue */ - unsigned long used; + size_t used; /* TX bytes sent to kernel */ + size_t rcvused; /* total RX bytes to be processed by kernel */ - unsigned int len; - bool more; - bool merge; - bool enc; + bool more; /* More data to be expected? */ + bool merge; /* Merge new data into existing SG */ + bool enc; /* Crypto operation: enc, dec */ - size_t aead_assoclen; - struct aead_request aead_req; + unsigned int len; /* Length of allocated memory for this struct */ }; +#define MAX_SGL_ENTS ((4096 - sizeof(struct aead_tsgl)) / \ + sizeof(struct scatterlist) - 1) + static inline int aead_sndbuf(struct sock *sk) { struct alg_sock *ask = alg_sk(sk); @@ -84,9 +107,29 @@ static inline bool aead_writable(struct sock *sk) return PAGE_SIZE <= aead_sndbuf(sk); } -static inline bool aead_sufficient_data(struct aead_ctx *ctx) +static inline int aead_rcvbuf(struct sock *sk) { - unsigned as = crypto_aead_authsize(crypto_aead_reqtfm(&ctx->aead_req)); + struct alg_sock *ask = alg_sk(sk); + struct aead_ctx *ctx = ask->private; + + return max_t(int, max_t(int, sk->sk_rcvbuf & PAGE_MASK, PAGE_SIZE) - + ctx->rcvused, 0); +} + +static inline bool aead_readable(struct sock *sk) +{ + return PAGE_SIZE <= aead_rcvbuf(sk); +} + +static inline bool aead_sufficient_data(struct sock *sk) +{ + struct alg_sock *ask = alg_sk(sk); + struct sock *psk = ask->parent; + struct alg_sock *pask = alg_sk(psk); + struct aead_ctx *ctx = ask->private; + struct aead_tfm *aeadc = pask->private; + struct crypto_aead *tfm = aeadc->aead; + unsigned int as = crypto_aead_authsize(tfm); /* * The minimum amount of memory needed for an AEAD cipher is @@ -95,33 +138,166 @@ static inline bool aead_sufficient_data(struct aead_ctx *ctx) return ctx->used >= ctx->aead_assoclen + (ctx->enc ? 0 : as); } -static void aead_reset_ctx(struct aead_ctx *ctx) +static int aead_alloc_tsgl(struct sock *sk) { - struct aead_sg_list *sgl = &ctx->tsgl; + struct alg_sock *ask = alg_sk(sk); + struct aead_ctx *ctx = ask->private; + struct aead_tsgl *sgl; + struct scatterlist *sg = NULL; - sg_init_table(sgl->sg, ALG_MAX_PAGES); - sgl->cur = 0; - ctx->used = 0; - ctx->more = 0; - ctx->merge = 0; + sgl = list_entry(ctx->tsgl_list.prev, struct aead_tsgl, list); + if (!list_empty(&ctx->tsgl_list)) + sg = sgl->sg; + + if (!sg || sgl->cur >= MAX_SGL_ENTS) { + sgl = sock_kmalloc(sk, sizeof(*sgl) + + sizeof(sgl->sg[0]) * (MAX_SGL_ENTS + 1), + GFP_KERNEL); + if (!sgl) + return -ENOMEM; + + sg_init_table(sgl->sg, MAX_SGL_ENTS + 1); + sgl->cur = 0; + + if (sg) + sg_chain(sg, MAX_SGL_ENTS + 1, sgl->sg); + + list_add_tail(&sgl->list, &ctx->tsgl_list); + } + + return 0; +} + +static unsigned int aead_count_tsgl(struct sock *sk, size_t bytes) +{ + struct alg_sock *ask = alg_sk(sk); + struct aead_ctx *ctx = ask->private; + struct aead_tsgl *sgl, *tmp; + unsigned int i; + unsigned int sgl_count = 0; + + if (!bytes) + return 0; + + list_for_each_entry_safe(sgl, tmp, &ctx->tsgl_list, list) { + struct scatterlist *sg = sgl->sg; + + for (i = 0; i < sgl->cur; i++) { + sgl_count++; + if (sg[i].length >= bytes) + return sgl_count; + + bytes -= sg[i].length; + } + } + + return sgl_count; } -static void aead_put_sgl(struct sock *sk) +static void aead_pull_tsgl(struct sock *sk, size_t used, + struct scatterlist *dst) { struct alg_sock *ask = alg_sk(sk); struct aead_ctx *ctx = ask->private; - struct aead_sg_list *sgl = &ctx->tsgl; - struct scatterlist *sg = sgl->sg; + struct aead_tsgl *sgl; + struct scatterlist *sg; unsigned int i; - for (i = 0; i < sgl->cur; i++) { - if (!sg_page(sg + i)) + while (!list_empty(&ctx->tsgl_list)) { + sgl = list_first_entry(&ctx->tsgl_list, struct aead_tsgl, + list); + sg = sgl->sg; + + for (i = 0; i < sgl->cur; i++) { + size_t plen = min_t(size_t, used, sg[i].length); + struct page *page = sg_page(sg + i); + + if (!page) + continue; + + /* + * Assumption: caller created aead_count_tsgl(len) + * SG entries in dst. + */ + if (dst) + sg_set_page(dst + i, page, plen, sg[i].offset); + + sg[i].length -= plen; + sg[i].offset += plen; + + used -= plen; + ctx->used -= plen; + + if (sg[i].length) + return; + + if (!dst) + put_page(page); + sg_assign_page(sg + i, NULL); + } + + list_del(&sgl->list); + sock_kfree_s(sk, sgl, sizeof(*sgl) + sizeof(sgl->sg[0]) * + (MAX_SGL_ENTS + 1)); + } + + if (!ctx->used) + ctx->merge = 0; +} + +static void aead_free_areq_sgls(struct aead_async_req *areq) +{ + struct sock *sk = areq->sk; + struct alg_sock *ask = alg_sk(sk); + struct aead_ctx *ctx = ask->private; + struct aead_rsgl *rsgl, *tmp; + struct scatterlist *tsgl; + struct scatterlist *sg; + unsigned int i; + + list_for_each_entry_safe(rsgl, tmp, &areq->rsgl_list, list) { + ctx->rcvused -= rsgl->sg_num_bytes; + af_alg_free_sg(&rsgl->sgl); + list_del(&rsgl->list); + if (rsgl != &areq->first_rsgl) + sock_kfree_s(sk, rsgl, sizeof(*rsgl)); + } + + tsgl = areq->tsgl; + for_each_sg(tsgl, sg, areq->tsgl_entries, i) { + if (!sg_page(sg)) continue; + put_page(sg_page(sg)); + } + + if (areq->tsgl && areq->tsgl_entries) + sock_kfree_s(sk, tsgl, areq->tsgl_entries * sizeof(*tsgl)); +} + +static int aead_wait_for_wmem(struct sock *sk, unsigned int flags) +{ + DEFINE_WAIT_FUNC(wait, woken_wake_function); + int err = -ERESTARTSYS; + long timeout; + + if (flags & MSG_DONTWAIT) + return -EAGAIN; - put_page(sg_page(sg + i)); - sg_assign_page(sg + i, NULL); + sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); + + add_wait_queue(sk_sleep(sk), &wait); + for (;;) { + if (signal_pending(current)) + break; + timeout = MAX_SCHEDULE_TIMEOUT; + if (sk_wait_event(sk, &timeout, aead_writable(sk), &wait)) { + err = 0; + break; + } } - aead_reset_ctx(ctx); + remove_wait_queue(sk_sleep(sk), &wait); + + return err; } static void aead_wmem_wakeup(struct sock *sk) @@ -153,6 +329,7 @@ static int aead_wait_for_data(struct sock *sk, unsigned flags) return -EAGAIN; sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); + add_wait_queue(sk_sleep(sk), &wait); for (;;) { if (signal_pending(current)) @@ -176,8 +353,6 @@ static void aead_data_wakeup(struct sock *sk) struct aead_ctx *ctx = ask->private; struct socket_wq *wq; - if (ctx->more) - return; if (!ctx->used) return; @@ -195,15 +370,18 @@ static int aead_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) { struct sock *sk = sock->sk; struct alg_sock *ask = alg_sk(sk); + struct sock *psk = ask->parent; + struct alg_sock *pask = alg_sk(psk); struct aead_ctx *ctx = ask->private; - unsigned ivsize = - crypto_aead_ivsize(crypto_aead_reqtfm(&ctx->aead_req)); - struct aead_sg_list *sgl = &ctx->tsgl; + struct aead_tfm *aeadc = pask->private; + struct crypto_aead *tfm = aeadc->aead; + unsigned int ivsize = crypto_aead_ivsize(tfm); + struct aead_tsgl *sgl; struct af_alg_control con = {}; long copied = 0; bool enc = 0; bool init = 0; - int err = -EINVAL; + int err = 0; if (msg->msg_controllen) { err = af_alg_cmsg_send(msg, &con); @@ -227,8 +405,10 @@ static int aead_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) } lock_sock(sk); - if (!ctx->more && ctx->used) + if (!ctx->more && ctx->used) { + err = -EINVAL; goto unlock; + } if (init) { ctx->enc = enc; @@ -239,11 +419,14 @@ static int aead_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) } while (size) { + struct scatterlist *sg; size_t len = size; - struct scatterlist *sg = NULL; + size_t plen; /* use the existing memory in an allocated page */ if (ctx->merge) { + sgl = list_entry(ctx->tsgl_list.prev, + struct aead_tsgl, list); sg = sgl->sg + sgl->cur - 1; len = min_t(unsigned long, len, PAGE_SIZE - sg->offset - sg->length); @@ -264,57 +447,60 @@ static int aead_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) } if (!aead_writable(sk)) { - /* user space sent too much data */ - aead_put_sgl(sk); - err = -EMSGSIZE; - goto unlock; + err = aead_wait_for_wmem(sk, msg->msg_flags); + if (err) + goto unlock; } /* allocate a new page */ len = min_t(unsigned long, size, aead_sndbuf(sk)); - while (len) { - size_t plen = 0; - if (sgl->cur >= ALG_MAX_PAGES) { - aead_put_sgl(sk); - err = -E2BIG; - goto unlock; - } + err = aead_alloc_tsgl(sk); + if (err) + goto unlock; + + sgl = list_entry(ctx->tsgl_list.prev, struct aead_tsgl, + list); + sg = sgl->sg; + if (sgl->cur) + sg_unmark_end(sg + sgl->cur - 1); + + do { + unsigned int i = sgl->cur; - sg = sgl->sg + sgl->cur; plen = min_t(size_t, len, PAGE_SIZE); - sg_assign_page(sg, alloc_page(GFP_KERNEL)); - err = -ENOMEM; - if (!sg_page(sg)) + sg_assign_page(sg + i, alloc_page(GFP_KERNEL)); + if (!sg_page(sg + i)) { + err = -ENOMEM; goto unlock; + } - err = memcpy_from_msg(page_address(sg_page(sg)), + err = memcpy_from_msg(page_address(sg_page(sg + i)), msg, plen); if (err) { - __free_page(sg_page(sg)); - sg_assign_page(sg, NULL); + __free_page(sg_page(sg + i)); + sg_assign_page(sg + i, NULL); goto unlock; } - sg->offset = 0; - sg->length = plen; + sg[i].length = plen; len -= plen; ctx->used += plen; copied += plen; - sgl->cur++; size -= plen; - ctx->merge = plen & (PAGE_SIZE - 1); - } + sgl->cur++; + } while (len && sgl->cur < MAX_SGL_ENTS); + + if (!size) + sg_mark_end(sg + sgl->cur - 1); + + ctx->merge = plen & (PAGE_SIZE - 1); } err = 0; ctx->more = msg->msg_flags & MSG_MORE; - if (!ctx->more && !aead_sufficient_data(ctx)) { - aead_put_sgl(sk); - err = -EMSGSIZE; - } unlock: aead_data_wakeup(sk); @@ -329,15 +515,12 @@ static ssize_t aead_sendpage(struct socket *sock, struct page *page, struct sock *sk = sock->sk; struct alg_sock *ask = alg_sk(sk); struct aead_ctx *ctx = ask->private; - struct aead_sg_list *sgl = &ctx->tsgl; + struct aead_tsgl *sgl; int err = -EINVAL; if (flags & MSG_SENDPAGE_NOTLAST) flags |= MSG_MORE; - if (sgl->cur >= ALG_MAX_PAGES) - return -E2BIG; - lock_sock(sk); if (!ctx->more && ctx->used) goto unlock; @@ -346,13 +529,22 @@ static ssize_t aead_sendpage(struct socket *sock, struct page *page, goto done; if (!aead_writable(sk)) { - /* user space sent too much data */ - aead_put_sgl(sk); - err = -EMSGSIZE; - goto unlock; + err = aead_wait_for_wmem(sk, flags); + if (err) + goto unlock; } + err = aead_alloc_tsgl(sk); + if (err) + goto unlock; + ctx->merge = 0; + sgl = list_entry(ctx->tsgl_list.prev, struct aead_tsgl, list); + + if (sgl->cur) + sg_unmark_end(sgl->sg + sgl->cur - 1); + + sg_mark_end(sgl->sg + sgl->cur); get_page(page); sg_set_page(sgl->sg + sgl->cur, page, size, offset); @@ -363,11 +555,6 @@ static ssize_t aead_sendpage(struct socket *sock, struct page *page, done: ctx->more = flags & MSG_MORE; - if (!ctx->more && !aead_sufficient_data(ctx)) { - aead_put_sgl(sk); - err = -EMSGSIZE; - } - unlock: aead_data_wakeup(sk); release_sock(sk); @@ -375,204 +562,52 @@ unlock: return err ?: size; } -#define GET_ASYM_REQ(req, tfm) (struct aead_async_req *) \ - ((char *)req + sizeof(struct aead_request) + \ - crypto_aead_reqsize(tfm)) - - #define GET_REQ_SIZE(tfm) sizeof(struct aead_async_req) + \ - crypto_aead_reqsize(tfm) + crypto_aead_ivsize(tfm) + \ - sizeof(struct aead_request) - static void aead_async_cb(struct crypto_async_request *_req, int err) { - struct aead_request *req = _req->data; - struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct aead_async_req *areq = GET_ASYM_REQ(req, tfm); + struct aead_async_req *areq = _req->data; struct sock *sk = areq->sk; - struct scatterlist *sg = areq->tsgl; - struct aead_async_rsgl *rsgl; struct kiocb *iocb = areq->iocb; - unsigned int i, reqlen = GET_REQ_SIZE(tfm); - - list_for_each_entry(rsgl, &areq->list, list) { - af_alg_free_sg(&rsgl->sgl); - if (rsgl != &areq->first_rsgl) - sock_kfree_s(sk, rsgl, sizeof(*rsgl)); - } - - for (i = 0; i < areq->tsgls; i++) - put_page(sg_page(sg + i)); - - sock_kfree_s(sk, areq->tsgl, sizeof(*areq->tsgl) * areq->tsgls); - sock_kfree_s(sk, req, reqlen); - __sock_put(sk); - iocb->ki_complete(iocb, err, err); -} - -static int aead_recvmsg_async(struct socket *sock, struct msghdr *msg, - int flags) -{ - struct sock *sk = sock->sk; - struct alg_sock *ask = alg_sk(sk); - struct aead_ctx *ctx = ask->private; - struct crypto_aead *tfm = crypto_aead_reqtfm(&ctx->aead_req); - struct aead_async_req *areq; - struct aead_request *req = NULL; - struct aead_sg_list *sgl = &ctx->tsgl; - struct aead_async_rsgl *last_rsgl = NULL, *rsgl; - unsigned int as = crypto_aead_authsize(tfm); - unsigned int i, reqlen = GET_REQ_SIZE(tfm); - int err = -ENOMEM; - unsigned long used; - size_t outlen = 0; - size_t usedpages = 0; + unsigned int resultlen; lock_sock(sk); - if (ctx->more) { - err = aead_wait_for_data(sk, flags); - if (err) - goto unlock; - } - - if (!aead_sufficient_data(ctx)) - goto unlock; - - used = ctx->used; - if (ctx->enc) - outlen = used + as; - else - outlen = used - as; - - req = sock_kmalloc(sk, reqlen, GFP_KERNEL); - if (unlikely(!req)) - goto unlock; - - areq = GET_ASYM_REQ(req, tfm); - memset(&areq->first_rsgl, '\0', sizeof(areq->first_rsgl)); - INIT_LIST_HEAD(&areq->list); - areq->iocb = msg->msg_iocb; - areq->sk = sk; - memcpy(areq->iv, ctx->iv, crypto_aead_ivsize(tfm)); - aead_request_set_tfm(req, tfm); - aead_request_set_ad(req, ctx->aead_assoclen); - aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - aead_async_cb, req); - used -= ctx->aead_assoclen; - - /* take over all tx sgls from ctx */ - areq->tsgl = sock_kmalloc(sk, - sizeof(*areq->tsgl) * max_t(u32, sgl->cur, 1), - GFP_KERNEL); - if (unlikely(!areq->tsgl)) - goto free; - - sg_init_table(areq->tsgl, max_t(u32, sgl->cur, 1)); - for (i = 0; i < sgl->cur; i++) - sg_set_page(&areq->tsgl[i], sg_page(&sgl->sg[i]), - sgl->sg[i].length, sgl->sg[i].offset); - - areq->tsgls = sgl->cur; - - /* create rx sgls */ - while (outlen > usedpages && iov_iter_count(&msg->msg_iter)) { - size_t seglen = min_t(size_t, iov_iter_count(&msg->msg_iter), - (outlen - usedpages)); - - if (list_empty(&areq->list)) { - rsgl = &areq->first_rsgl; - - } else { - rsgl = sock_kmalloc(sk, sizeof(*rsgl), GFP_KERNEL); - if (unlikely(!rsgl)) { - err = -ENOMEM; - goto free; - } - } - rsgl->sgl.npages = 0; - list_add_tail(&rsgl->list, &areq->list); - - /* make one iovec available as scatterlist */ - err = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, seglen); - if (err < 0) - goto free; - - usedpages += err; - - /* chain the new scatterlist with previous one */ - if (last_rsgl) - af_alg_link_sg(&last_rsgl->sgl, &rsgl->sgl); - - last_rsgl = rsgl; - iov_iter_advance(&msg->msg_iter, err); - } + /* Buffer size written by crypto operation. */ + resultlen = areq->outlen; - /* ensure output buffer is sufficiently large */ - if (usedpages < outlen) { - err = -EINVAL; - goto unlock; - } + aead_free_areq_sgls(areq); + sock_kfree_s(sk, areq, areq->areqlen); + __sock_put(sk); - aead_request_set_crypt(req, areq->tsgl, areq->first_rsgl.sgl.sg, used, - areq->iv); - err = ctx->enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req); - if (err) { - if (err == -EINPROGRESS) { - sock_hold(sk); - err = -EIOCBQUEUED; - aead_reset_ctx(ctx); - goto unlock; - } else if (err == -EBADMSG) { - aead_put_sgl(sk); - } - goto free; - } - aead_put_sgl(sk); + iocb->ki_complete(iocb, err ? err : resultlen, 0); -free: - list_for_each_entry(rsgl, &areq->list, list) { - af_alg_free_sg(&rsgl->sgl); - if (rsgl != &areq->first_rsgl) - sock_kfree_s(sk, rsgl, sizeof(*rsgl)); - } - if (areq->tsgl) - sock_kfree_s(sk, areq->tsgl, sizeof(*areq->tsgl) * areq->tsgls); - if (req) - sock_kfree_s(sk, req, reqlen); -unlock: - aead_wmem_wakeup(sk); release_sock(sk); - return err ? err : outlen; } -static int aead_recvmsg_sync(struct socket *sock, struct msghdr *msg, int flags) +static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, + size_t ignored, int flags) { struct sock *sk = sock->sk; struct alg_sock *ask = alg_sk(sk); + struct sock *psk = ask->parent; + struct alg_sock *pask = alg_sk(psk); struct aead_ctx *ctx = ask->private; - unsigned as = crypto_aead_authsize(crypto_aead_reqtfm(&ctx->aead_req)); - struct aead_sg_list *sgl = &ctx->tsgl; - struct aead_async_rsgl *last_rsgl = NULL; - struct aead_async_rsgl *rsgl, *tmp; - int err = -EINVAL; - unsigned long used = 0; - size_t outlen = 0; - size_t usedpages = 0; - - lock_sock(sk); + struct aead_tfm *aeadc = pask->private; + struct crypto_aead *tfm = aeadc->aead; + unsigned int as = crypto_aead_authsize(tfm); + unsigned int areqlen = + sizeof(struct aead_async_req) + crypto_aead_reqsize(tfm); + struct aead_async_req *areq; + struct aead_rsgl *last_rsgl = NULL; + int err = 0; + size_t used = 0; /* [in] TX bufs to be en/decrypted */ + size_t outlen = 0; /* [out] RX bufs produced by kernel */ + size_t usedpages = 0; /* [in] RX bufs to be used from user */ + size_t processed = 0; /* [in] TX bufs to be consumed */ /* - * Please see documentation of aead_request_set_crypt for the - * description of the AEAD memory structure expected from the caller. + * Data length provided by caller via sendmsg/sendpage that has not + * yet been processed. */ - - if (ctx->more) { - err = aead_wait_for_data(sk, flags); - if (err) - goto unlock; - } - - /* data length provided by caller via sendmsg/sendpage */ used = ctx->used; /* @@ -584,8 +619,8 @@ static int aead_recvmsg_sync(struct socket *sock, struct msghdr *msg, int flags) * the error message in sendmsg/sendpage and still call recvmsg. This * check here protects the kernel integrity. */ - if (!aead_sufficient_data(ctx)) - goto unlock; + if (!aead_sufficient_data(sk)) + return -EINVAL; /* * Calculate the minimum output buffer size holding the result of the @@ -606,84 +641,170 @@ static int aead_recvmsg_sync(struct socket *sock, struct msghdr *msg, int flags) */ used -= ctx->aead_assoclen; - /* convert iovecs of output buffers into scatterlists */ - while (outlen > usedpages && iov_iter_count(&msg->msg_iter)) { - size_t seglen = min_t(size_t, iov_iter_count(&msg->msg_iter), - (outlen - usedpages)); + /* Allocate cipher request for current operation. */ + areq = sock_kmalloc(sk, areqlen, GFP_KERNEL); + if (unlikely(!areq)) + return -ENOMEM; + areq->areqlen = areqlen; + areq->sk = sk; + INIT_LIST_HEAD(&areq->rsgl_list); + areq->tsgl = NULL; + areq->tsgl_entries = 0; + + /* convert iovecs of output buffers into RX SGL */ + while (outlen > usedpages && msg_data_left(msg)) { + struct aead_rsgl *rsgl; + size_t seglen; + + /* limit the amount of readable buffers */ + if (!aead_readable(sk)) + break; - if (list_empty(&ctx->list)) { - rsgl = &ctx->first_rsgl; + if (!ctx->used) { + err = aead_wait_for_data(sk, flags); + if (err) + goto free; + } + + seglen = min_t(size_t, (outlen - usedpages), + msg_data_left(msg)); + + if (list_empty(&areq->rsgl_list)) { + rsgl = &areq->first_rsgl; } else { rsgl = sock_kmalloc(sk, sizeof(*rsgl), GFP_KERNEL); if (unlikely(!rsgl)) { err = -ENOMEM; - goto unlock; + goto free; } } + rsgl->sgl.npages = 0; - list_add_tail(&rsgl->list, &ctx->list); + list_add_tail(&rsgl->list, &areq->rsgl_list); /* make one iovec available as scatterlist */ err = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, seglen); if (err < 0) - goto unlock; - usedpages += err; + goto free; + /* chain the new scatterlist with previous one */ if (last_rsgl) af_alg_link_sg(&last_rsgl->sgl, &rsgl->sgl); last_rsgl = rsgl; - + usedpages += err; + ctx->rcvused += err; + rsgl->sg_num_bytes = err; iov_iter_advance(&msg->msg_iter, err); } - /* ensure output buffer is sufficiently large */ + /* + * Ensure output buffer is sufficiently large. If the caller provides + * less buffer space, only use the relative required input size. This + * allows AIO operation where the caller sent all data to be processed + * and the AIO operation performs the operation on the different chunks + * of the input data. + */ if (usedpages < outlen) { - err = -EINVAL; - goto unlock; - } + size_t less = outlen - usedpages; - sg_mark_end(sgl->sg + sgl->cur - 1); - aead_request_set_crypt(&ctx->aead_req, sgl->sg, ctx->first_rsgl.sgl.sg, - used, ctx->iv); - aead_request_set_ad(&ctx->aead_req, ctx->aead_assoclen); + if (used < less) { + err = -EINVAL; + goto free; + } + used -= less; + outlen -= less; + } - err = af_alg_wait_for_completion(ctx->enc ? - crypto_aead_encrypt(&ctx->aead_req) : - crypto_aead_decrypt(&ctx->aead_req), + /* + * Create a per request TX SGL for this request which tracks the + * SG entries from the global TX SGL. + */ + processed = used + ctx->aead_assoclen; + areq->tsgl_entries = aead_count_tsgl(sk, processed); + if (!areq->tsgl_entries) + areq->tsgl_entries = 1; + areq->tsgl = sock_kmalloc(sk, sizeof(*areq->tsgl) * areq->tsgl_entries, + GFP_KERNEL); + if (!areq->tsgl) { + err = -ENOMEM; + goto free; + } + sg_init_table(areq->tsgl, areq->tsgl_entries); + aead_pull_tsgl(sk, processed, areq->tsgl); + + /* Initialize the crypto operation */ + aead_request_set_crypt(&areq->aead_req, areq->tsgl, + areq->first_rsgl.sgl.sg, used, ctx->iv); + aead_request_set_ad(&areq->aead_req, ctx->aead_assoclen); + aead_request_set_tfm(&areq->aead_req, tfm); + + if (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) { + /* AIO operation */ + areq->iocb = msg->msg_iocb; + aead_request_set_callback(&areq->aead_req, + CRYPTO_TFM_REQ_MAY_BACKLOG, + aead_async_cb, areq); + err = ctx->enc ? crypto_aead_encrypt(&areq->aead_req) : + crypto_aead_decrypt(&areq->aead_req); + } else { + /* Synchronous operation */ + aead_request_set_callback(&areq->aead_req, + CRYPTO_TFM_REQ_MAY_BACKLOG, + af_alg_complete, &ctx->completion); + err = af_alg_wait_for_completion(ctx->enc ? + crypto_aead_encrypt(&areq->aead_req) : + crypto_aead_decrypt(&areq->aead_req), &ctx->completion); - - if (err) { - /* EBADMSG implies a valid cipher operation took place */ - if (err == -EBADMSG) - aead_put_sgl(sk); - - goto unlock; } - aead_put_sgl(sk); - err = 0; + /* AIO operation in progress */ + if (err == -EINPROGRESS) { + sock_hold(sk); -unlock: - list_for_each_entry_safe(rsgl, tmp, &ctx->list, list) { - af_alg_free_sg(&rsgl->sgl); - list_del(&rsgl->list); - if (rsgl != &ctx->first_rsgl) - sock_kfree_s(sk, rsgl, sizeof(*rsgl)); + /* Remember output size that will be generated. */ + areq->outlen = outlen; + + return -EIOCBQUEUED; } - INIT_LIST_HEAD(&ctx->list); - aead_wmem_wakeup(sk); - release_sock(sk); + +free: + aead_free_areq_sgls(areq); + if (areq) + sock_kfree_s(sk, areq, areqlen); return err ? err : outlen; } -static int aead_recvmsg(struct socket *sock, struct msghdr *msg, size_t ignored, - int flags) +static int aead_recvmsg(struct socket *sock, struct msghdr *msg, + size_t ignored, int flags) { - return (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) ? - aead_recvmsg_async(sock, msg, flags) : - aead_recvmsg_sync(sock, msg, flags); + struct sock *sk = sock->sk; + int ret = 0; + + lock_sock(sk); + while (msg_data_left(msg)) { + int err = _aead_recvmsg(sock, msg, ignored, flags); + + /* + * This error covers -EIOCBQUEUED which implies that we can + * only handle one AIO request. If the caller wants to have + * multiple AIO requests in parallel, he must make multiple + * separate AIO calls. + */ + if (err <= 0) { + if (err == -EIOCBQUEUED || err == -EBADMSG) + ret = err; + goto out; + } + + ret += err; + } + +out: + aead_wmem_wakeup(sk); + release_sock(sk); + return ret; } static unsigned int aead_poll(struct file *file, struct socket *sock, @@ -874,11 +995,13 @@ static void aead_sock_destruct(struct sock *sk) { struct alg_sock *ask = alg_sk(sk); struct aead_ctx *ctx = ask->private; - unsigned int ivlen = crypto_aead_ivsize( - crypto_aead_reqtfm(&ctx->aead_req)); + struct sock *psk = ask->parent; + struct alg_sock *pask = alg_sk(psk); + struct aead_tfm *aeadc = pask->private; + struct crypto_aead *tfm = aeadc->aead; + unsigned int ivlen = crypto_aead_ivsize(tfm); - WARN_ON(refcount_read(&sk->sk_refcnt) != 0); - aead_put_sgl(sk); + aead_pull_tsgl(sk, ctx->used, NULL); sock_kzfree_s(sk, ctx->iv, ivlen); sock_kfree_s(sk, ctx, ctx->len); af_alg_release_parent(sk); @@ -890,7 +1013,7 @@ static int aead_accept_parent_nokey(void *private, struct sock *sk) struct alg_sock *ask = alg_sk(sk); struct aead_tfm *tfm = private; struct crypto_aead *aead = tfm->aead; - unsigned int len = sizeof(*ctx) + crypto_aead_reqsize(aead); + unsigned int len = sizeof(*ctx); unsigned int ivlen = crypto_aead_ivsize(aead); ctx = sock_kmalloc(sk, len, GFP_KERNEL); @@ -905,23 +1028,18 @@ static int aead_accept_parent_nokey(void *private, struct sock *sk) } memset(ctx->iv, 0, ivlen); + INIT_LIST_HEAD(&ctx->tsgl_list); ctx->len = len; ctx->used = 0; + ctx->rcvused = 0; ctx->more = 0; ctx->merge = 0; ctx->enc = 0; - ctx->tsgl.cur = 0; ctx->aead_assoclen = 0; af_alg_init_completion(&ctx->completion); - sg_init_table(ctx->tsgl.sg, ALG_MAX_PAGES); - INIT_LIST_HEAD(&ctx->list); ask->private = ctx; - aead_request_set_tfm(&ctx->aead_req, aead); - aead_request_set_callback(&ctx->aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG, - af_alg_complete, &ctx->completion); - sk->sk_destruct = aead_sock_destruct; return 0; -- cgit v1.2.3 From 218f1f10b8682a183083d49e0f5bfd26af609138 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Sun, 16 Jul 2017 19:22:06 +0200 Subject: crypto: rng - ensure that the RNG is ready before using Otherwise, we might be seeding the RNG using bad randomness, which is dangerous. The one use of this function from within the kernel -- not from userspace -- is being removed (keys/big_key), so that call site isn't relevant in assessing this. Cc: Herbert Xu Signed-off-by: Jason A. Donenfeld Signed-off-by: Herbert Xu --- crypto/rng.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crypto/rng.c b/crypto/rng.c index 5e846924..b4a61866 100644 --- a/crypto/rng.c +++ b/crypto/rng.c @@ -43,12 +43,14 @@ int crypto_rng_reset(struct crypto_rng *tfm, const u8 *seed, unsigned int slen) if (!buf) return -ENOMEM; - get_random_bytes(buf, slen); + err = get_random_bytes_wait(buf, slen); + if (err) + goto out; seed = buf; } err = crypto_rng_alg(tfm)->seed(tfm, seed, slen); - +out: kzfree(buf); return err; } -- cgit v1.2.3 From 60fe5b1ecc77cd9eeb009b3814822db865fdfb82 Mon Sep 17 00:00:00 2001 From: Horia Geantă Date: Wed, 19 Jul 2017 19:40:32 +0300 Subject: crypto: tcrypt - remove AES-XTS-192 speed tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove xts(aes) speed tests with 2 x 192-bit keys, since implementations adhering strictly to IEEE 1619-2007 standard cannot cope with key sizes other than 2 x 128, 2 x 256 bits - i.e. AES-XTS-{128,256}: [...] tcrypt: test 5 (384 bit key, 16 byte blocks): caam_jr 8020000.jr: key size mismatch tcrypt: setkey() failed flags=200000 [...] Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- crypto/tcrypt.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 0dd6a432..0022a18d 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -1404,9 +1404,9 @@ static int do_test(const char *alg, u32 type, u32 mask, int m) test_cipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0, speed_template_32_40_48); test_cipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0, - speed_template_32_48_64); + speed_template_32_64); test_cipher_speed("xts(aes)", DECRYPT, sec, NULL, 0, - speed_template_32_48_64); + speed_template_32_64); test_cipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0, speed_template_16_24_32); test_cipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0, @@ -1837,9 +1837,9 @@ static int do_test(const char *alg, u32 type, u32 mask, int m) test_acipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0, speed_template_32_40_48); test_acipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0, - speed_template_32_48_64); + speed_template_32_64); test_acipher_speed("xts(aes)", DECRYPT, sec, NULL, 0, - speed_template_32_48_64); + speed_template_32_64); test_acipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0, speed_template_16_24_32); test_acipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0, -- cgit v1.2.3 From 284f9e00cf2494bf36f2fd36d10510a229a66c41 Mon Sep 17 00:00:00 2001 From: Tudor-Dan Ambarus Date: Thu, 20 Jul 2017 10:37:39 +0300 Subject: crypto: ecdh - fix concurrency on shared secret and pubkey ecdh_ctx contained static allocated data for the shared secret and public key. The shared secret and the public key were doomed to concurrency issues because they could be shared by multiple crypto requests. The concurrency is fixed by replacing per-tfm shared secret and public key with per-request dynamically allocated shared secret and public key. Signed-off-by: Tudor Ambarus Signed-off-by: Herbert Xu --- crypto/ecdh.c | 51 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/crypto/ecdh.c b/crypto/ecdh.c index 61c77089..4271fc77 100644 --- a/crypto/ecdh.c +++ b/crypto/ecdh.c @@ -20,8 +20,6 @@ struct ecdh_ctx { unsigned int curve_id; unsigned int ndigits; u64 private_key[ECC_MAX_DIGITS]; - u64 public_key[2 * ECC_MAX_DIGITS]; - u64 shared_secret[ECC_MAX_DIGITS]; }; static inline struct ecdh_ctx *ecdh_get_ctx(struct crypto_kpp *tfm) @@ -70,41 +68,58 @@ static int ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, static int ecdh_compute_value(struct kpp_request *req) { - int ret = 0; struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); struct ecdh_ctx *ctx = ecdh_get_ctx(tfm); - size_t copied, nbytes; + u64 *public_key; + u64 *shared_secret = NULL; void *buf; + size_t copied, nbytes, public_key_sz; + int ret = -ENOMEM; nbytes = ctx->ndigits << ECC_DIGITS_TO_BYTES_SHIFT; + /* Public part is a point thus it has both coordinates */ + public_key_sz = 2 * nbytes; + + public_key = kmalloc(public_key_sz, GFP_KERNEL); + if (!public_key) + return -ENOMEM; if (req->src) { - copied = sg_copy_to_buffer(req->src, 1, ctx->public_key, - 2 * nbytes); - if (copied != 2 * nbytes) - return -EINVAL; + shared_secret = kmalloc(nbytes, GFP_KERNEL); + if (!shared_secret) + goto free_pubkey; + + copied = sg_copy_to_buffer(req->src, 1, public_key, + public_key_sz); + if (copied != public_key_sz) { + ret = -EINVAL; + goto free_all; + } ret = crypto_ecdh_shared_secret(ctx->curve_id, ctx->ndigits, - ctx->private_key, - ctx->public_key, - ctx->shared_secret); + ctx->private_key, public_key, + shared_secret); - buf = ctx->shared_secret; + buf = shared_secret; } else { ret = ecc_make_pub_key(ctx->curve_id, ctx->ndigits, - ctx->private_key, ctx->public_key); - buf = ctx->public_key; - /* Public part is a point thus it has both coordinates */ - nbytes *= 2; + ctx->private_key, public_key); + buf = public_key; + nbytes = public_key_sz; } if (ret < 0) - return ret; + goto free_all; copied = sg_copy_from_buffer(req->dst, 1, buf, nbytes); if (copied != nbytes) - return -EINVAL; + ret = -EINVAL; + /* fall through */ +free_all: + kzfree(shared_secret); +free_pubkey: + kfree(public_key); return ret; } -- cgit v1.2.3 From 3014649037efac090a2c0d806d7f8d0a28a95cb3 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 21 Jul 2017 16:42:36 +0100 Subject: crypto: scompress - don't sleep with preemption disabled Due to the use of per-CPU buffers, scomp_acomp_comp_decomp() executes with preemption disabled, and so whether the CRYPTO_TFM_REQ_MAY_SLEEP flag is set is irrelevant, since we cannot sleep anyway. So disregard the flag, and use GFP_ATOMIC unconditionally. Cc: # v4.10+ Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/scompress.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crypto/scompress.c b/crypto/scompress.c index ae1d3cf2..0b40d991 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -211,9 +211,7 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) scratch_dst, &req->dlen, *ctx); if (!ret) { if (!req->dst) { - req->dst = crypto_scomp_sg_alloc(req->dlen, - req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? - GFP_KERNEL : GFP_ATOMIC); + req->dst = crypto_scomp_sg_alloc(req->dlen, GFP_ATOMIC); if (!req->dst) goto out; } -- cgit v1.2.3 From 4757f1715ee63e749594ef98c0631125f3607671 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 21 Jul 2017 16:42:37 +0100 Subject: crypto: scompress - free partially allocated scratch buffers on failure When allocating the per-CPU scratch buffers, we allocate the source and destination buffers separately, but bail immediately if the second allocation fails, without freeing the first one. Fix that. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/scompress.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crypto/scompress.c b/crypto/scompress.c index 0b40d991..2c076483 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -125,8 +125,11 @@ static int crypto_scomp_alloc_all_scratches(void) if (!scomp_src_scratches) return -ENOMEM; scomp_dst_scratches = crypto_scomp_alloc_scratches(); - if (!scomp_dst_scratches) + if (!scomp_dst_scratches) { + crypto_scomp_free_scratches(scomp_src_scratches); + scomp_src_scratches = NULL; return -ENOMEM; + } } return 0; } -- cgit v1.2.3 From 58b3573d7326a2e7ff31d82e9946602eb8fb6549 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 21 Jul 2017 16:42:38 +0100 Subject: crypto: scompress - defer allocation of scratch buffer to first use The scompress code allocates 2 x 128 KB of scratch buffers for each CPU, so that clients of the async API can use synchronous implementations even from atomic context. However, on systems such as Cavium Thunderx (which has 96 cores), this adds up to a non-negligible 24 MB. Also, 32-bit systems may prefer to use their precious vmalloc space for other things,especially since there don't appear to be any clients for the async compression API yet. So let's defer allocation of the scratch buffers until the first time we allocate an acompress cipher based on an scompress implementation. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/scompress.c | 46 +++++++++++++++++----------------------------- 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/crypto/scompress.c b/crypto/scompress.c index 2c076483..2075e2c4 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -65,11 +65,6 @@ static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg) seq_puts(m, "type : scomp\n"); } -static int crypto_scomp_init_tfm(struct crypto_tfm *tfm) -{ - return 0; -} - static void crypto_scomp_free_scratches(void * __percpu *scratches) { int i; @@ -134,6 +129,17 @@ static int crypto_scomp_alloc_all_scratches(void) return 0; } +static int crypto_scomp_init_tfm(struct crypto_tfm *tfm) +{ + int ret; + + mutex_lock(&scomp_lock); + ret = crypto_scomp_alloc_all_scratches(); + mutex_unlock(&scomp_lock); + + return ret; +} + static void crypto_scomp_sg_free(struct scatterlist *sgl) { int i, n; @@ -241,6 +247,10 @@ static void crypto_exit_scomp_ops_async(struct crypto_tfm *tfm) struct crypto_scomp **ctx = crypto_tfm_ctx(tfm); crypto_free_scomp(*ctx); + + mutex_lock(&scomp_lock); + crypto_scomp_free_all_scratches(); + mutex_unlock(&scomp_lock); } int crypto_init_scomp_ops_async(struct crypto_tfm *tfm) @@ -317,40 +327,18 @@ static const struct crypto_type crypto_scomp_type = { int crypto_register_scomp(struct scomp_alg *alg) { struct crypto_alg *base = &alg->base; - int ret = -ENOMEM; - - mutex_lock(&scomp_lock); - if (crypto_scomp_alloc_all_scratches()) - goto error; base->cra_type = &crypto_scomp_type; base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK; base->cra_flags |= CRYPTO_ALG_TYPE_SCOMPRESS; - ret = crypto_register_alg(base); - if (ret) - goto error; - - mutex_unlock(&scomp_lock); - return ret; - -error: - crypto_scomp_free_all_scratches(); - mutex_unlock(&scomp_lock); - return ret; + return crypto_register_alg(base); } EXPORT_SYMBOL_GPL(crypto_register_scomp); int crypto_unregister_scomp(struct scomp_alg *alg) { - int ret; - - mutex_lock(&scomp_lock); - ret = crypto_unregister_alg(&alg->base); - crypto_scomp_free_all_scratches(); - mutex_unlock(&scomp_lock); - - return ret; + return crypto_unregister_alg(&alg->base); } EXPORT_SYMBOL_GPL(crypto_unregister_scomp); -- cgit v1.2.3 From 52c242295c6afe64158da42e3ca3e04ea768f9c7 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 24 Jul 2017 11:28:03 +0100 Subject: crypto: algapi - use separate dst and src operands for __crypto_xor() In preparation of introducing crypto_xor_cpy(), which will use separate operands for input and output, modify the __crypto_xor() implementation, which it will share with the existing crypto_xor(), which provides the actual functionality when not using the inline version. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/algapi.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/crypto/algapi.c b/crypto/algapi.c index e4cc7615..aa699ff6 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -975,13 +975,15 @@ void crypto_inc(u8 *a, unsigned int size) } EXPORT_SYMBOL_GPL(crypto_inc); -void __crypto_xor(u8 *dst, const u8 *src, unsigned int len) +void __crypto_xor(u8 *dst, const u8 *src1, const u8 *src2, unsigned int len) { int relalign = 0; if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) { int size = sizeof(unsigned long); - int d = ((unsigned long)dst ^ (unsigned long)src) & (size - 1); + int d = (((unsigned long)dst ^ (unsigned long)src1) | + ((unsigned long)dst ^ (unsigned long)src2)) & + (size - 1); relalign = d ? 1 << __ffs(d) : size; @@ -992,34 +994,37 @@ void __crypto_xor(u8 *dst, const u8 *src, unsigned int len) * process the remainder of the input using optimal strides. */ while (((unsigned long)dst & (relalign - 1)) && len > 0) { - *dst++ ^= *src++; + *dst++ = *src1++ ^ *src2++; len--; } } while (IS_ENABLED(CONFIG_64BIT) && len >= 8 && !(relalign & 7)) { - *(u64 *)dst ^= *(u64 *)src; + *(u64 *)dst = *(u64 *)src1 ^ *(u64 *)src2; dst += 8; - src += 8; + src1 += 8; + src2 += 8; len -= 8; } while (len >= 4 && !(relalign & 3)) { - *(u32 *)dst ^= *(u32 *)src; + *(u32 *)dst = *(u32 *)src1 ^ *(u32 *)src2; dst += 4; - src += 4; + src1 += 4; + src2 += 4; len -= 4; } while (len >= 2 && !(relalign & 1)) { - *(u16 *)dst ^= *(u16 *)src; + *(u16 *)dst = *(u16 *)src1 ^ *(u16 *)src2; dst += 2; - src += 2; + src1 += 2; + src2 += 2; len -= 2; } while (len--) - *dst++ ^= *src++; + *dst++ = *src1++ ^ *src2++; } EXPORT_SYMBOL_GPL(__crypto_xor); -- cgit v1.2.3 From 96692cfc16dbe9963fc4129527fbf2f16c68f5ba Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 24 Jul 2017 11:28:04 +0100 Subject: crypto: algapi - make crypto_xor() take separate dst and src arguments There are quite a number of occurrences in the kernel of the pattern if (dst != src) memcpy(dst, src, walk.total % AES_BLOCK_SIZE); crypto_xor(dst, final, walk.total % AES_BLOCK_SIZE); or crypto_xor(keystream, src, nbytes); memcpy(dst, keystream, nbytes); where crypto_xor() is preceded or followed by a memcpy() invocation that is only there because crypto_xor() uses its output parameter as one of the inputs. To avoid having to add new instances of this pattern in the arm64 code, which will be refactored to implement non-SIMD fallbacks, add an alternative implementation called crypto_xor_cpy(), taking separate input and output arguments. This removes the need for the separate memcpy(). Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/ctr.c | 3 +-- crypto/pcbc.c | 12 ++++-------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/crypto/ctr.c b/crypto/ctr.c index 477d9226..854d924f 100644 --- a/crypto/ctr.c +++ b/crypto/ctr.c @@ -65,8 +65,7 @@ static void crypto_ctr_crypt_final(struct blkcipher_walk *walk, unsigned int nbytes = walk->nbytes; crypto_cipher_encrypt_one(tfm, keystream, ctrblk); - crypto_xor(keystream, src, nbytes); - memcpy(dst, keystream, nbytes); + crypto_xor_cpy(dst, keystream, src, nbytes); crypto_inc(ctrblk, bsize); } diff --git a/crypto/pcbc.c b/crypto/pcbc.c index 29dd2b4a..d9e45a95 100644 --- a/crypto/pcbc.c +++ b/crypto/pcbc.c @@ -55,8 +55,7 @@ static int crypto_pcbc_encrypt_segment(struct skcipher_request *req, do { crypto_xor(iv, src, bsize); crypto_cipher_encrypt_one(tfm, dst, iv); - memcpy(iv, dst, bsize); - crypto_xor(iv, src, bsize); + crypto_xor_cpy(iv, dst, src, bsize); src += bsize; dst += bsize; @@ -79,8 +78,7 @@ static int crypto_pcbc_encrypt_inplace(struct skcipher_request *req, memcpy(tmpbuf, src, bsize); crypto_xor(iv, src, bsize); crypto_cipher_encrypt_one(tfm, src, iv); - memcpy(iv, tmpbuf, bsize); - crypto_xor(iv, src, bsize); + crypto_xor_cpy(iv, tmpbuf, src, bsize); src += bsize; } while ((nbytes -= bsize) >= bsize); @@ -127,8 +125,7 @@ static int crypto_pcbc_decrypt_segment(struct skcipher_request *req, do { crypto_cipher_decrypt_one(tfm, dst, src); crypto_xor(dst, iv, bsize); - memcpy(iv, src, bsize); - crypto_xor(iv, dst, bsize); + crypto_xor_cpy(iv, dst, src, bsize); src += bsize; dst += bsize; @@ -153,8 +150,7 @@ static int crypto_pcbc_decrypt_inplace(struct skcipher_request *req, memcpy(tmpbuf, src, bsize); crypto_cipher_decrypt_one(tfm, src, src); crypto_xor(src, iv, bsize); - memcpy(iv, tmpbuf, bsize); - crypto_xor(iv, src, bsize); + crypto_xor_cpy(iv, src, tmpbuf, bsize); src += bsize; } while ((nbytes -= bsize) >= bsize); -- cgit v1.2.3 From d57f826edc9eb90ad0883370453c8dfb078a4f59 Mon Sep 17 00:00:00 2001 From: Stephan Mueller Date: Sun, 30 Jul 2017 14:31:18 +0200 Subject: crypto: algif - return error code when no data was processed If no data has been processed during recvmsg, return the error code. This covers all errors received during non-AIO operations. If any error occurs during a synchronous operation in addition to -EIOCBQUEUED or -EBADMSG (like -ENOMEM), it should be relayed to the caller. Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/algif_aead.c | 4 +++- crypto/algif_skcipher.c | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c index 9755aac0..2de056c3 100644 --- a/crypto/algif_aead.c +++ b/crypto/algif_aead.c @@ -791,9 +791,11 @@ static int aead_recvmsg(struct socket *sock, struct msghdr *msg, * only handle one AIO request. If the caller wants to have * multiple AIO requests in parallel, he must make multiple * separate AIO calls. + * + * Also return the error if no data has been processed so far. */ if (err <= 0) { - if (err == -EIOCBQUEUED || err == -EBADMSG) + if (err == -EIOCBQUEUED || err == -EBADMSG || !ret) ret = err; goto out; } diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c index 968d094f..ce3b5fba 100644 --- a/crypto/algif_skcipher.c +++ b/crypto/algif_skcipher.c @@ -714,9 +714,11 @@ static int skcipher_recvmsg(struct socket *sock, struct msghdr *msg, * only handle one AIO request. If the caller wants to have * multiple AIO requests in parallel, he must make multiple * separate AIO calls. + * + * Also return the error if no data has been processed so far. */ if (err <= 0) { - if (err == -EIOCBQUEUED) + if (err == -EIOCBQUEUED || !ret) ret = err; goto out; } -- cgit v1.2.3 From 416bea458750840116600bfb8c6a8a22e90c6781 Mon Sep 17 00:00:00 2001 From: Stephan Mueller Date: Sun, 30 Jul 2017 14:32:58 +0200 Subject: crypto: algif_aead - copy AAD from src to dst Use the NULL cipher to copy the AAD and PT/CT from the TX SGL to the RX SGL. This allows an in-place crypto operation on the RX SGL for encryption, because the TX data is always smaller or equal to the RX data (the RX data will hold the tag). For decryption, a per-request TX SGL is created which will only hold the tag value. As the RX SGL will have no space for the tag value and an in-place operation will not write the tag buffer, the TX SGL with the tag value is chained to the RX SGL. This now allows an in-place crypto operation. For example: * without the patch: kcapi -x 2 -e -c "gcm(aes)" -p 89154d0d4129d322e4487bafaa4f6b46 -k c0ece3e63198af382b5603331cc23fa8 -i 7e489b83622e7228314d878d -a afcd7202d621e06ca53b70c2bdff7fb2 -l 16 -u -s 00000000000000000000000000000000f4a3eacfbdadd3b1a17117b1d67ffc1f1e21efbbc6d83724a8c296e3bb8cda0c * with the patch: kcapi -x 2 -e -c "gcm(aes)" -p 89154d0d4129d322e4487bafaa4f6b46 -k c0ece3e63198af382b5603331cc23fa8 -i 7e489b83622e7228314d878d -a afcd7202d621e06ca53b70c2bdff7fb2 -l 16 -u -s afcd7202d621e06ca53b70c2bdff7fb2f4a3eacfbdadd3b1a17117b1d67ffc1f1e21efbbc6d83724a8c296e3bb8cda0c Tests covering this functionality have been added to libkcapi. Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/Kconfig | 2 + crypto/algif_aead.c | 183 +++++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 162 insertions(+), 23 deletions(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index caa770e5..0a121f9d 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1753,6 +1753,8 @@ config CRYPTO_USER_API_AEAD tristate "User-space interface for AEAD cipher algorithms" depends on NET select CRYPTO_AEAD + select CRYPTO_BLKCIPHER + select CRYPTO_NULL select CRYPTO_USER_API help This option enables the user-spaces interface for AEAD diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c index 2de056c3..1f0696dd 100644 --- a/crypto/algif_aead.c +++ b/crypto/algif_aead.c @@ -30,6 +30,8 @@ #include #include #include +#include +#include #include #include #include @@ -70,6 +72,7 @@ struct aead_async_req { struct aead_tfm { struct crypto_aead *aead; bool has_key; + struct crypto_skcipher *null_tfm; }; struct aead_ctx { @@ -168,7 +171,12 @@ static int aead_alloc_tsgl(struct sock *sk) return 0; } -static unsigned int aead_count_tsgl(struct sock *sk, size_t bytes) +/** + * Count number of SG entries from the beginning of the SGL to @bytes. If + * an offset is provided, the counting of the SG entries starts at the offset. + */ +static unsigned int aead_count_tsgl(struct sock *sk, size_t bytes, + size_t offset) { struct alg_sock *ask = alg_sk(sk); struct aead_ctx *ctx = ask->private; @@ -183,32 +191,55 @@ static unsigned int aead_count_tsgl(struct sock *sk, size_t bytes) struct scatterlist *sg = sgl->sg; for (i = 0; i < sgl->cur; i++) { + size_t bytes_count; + + /* Skip offset */ + if (offset >= sg[i].length) { + offset -= sg[i].length; + bytes -= sg[i].length; + continue; + } + + bytes_count = sg[i].length - offset; + + offset = 0; sgl_count++; - if (sg[i].length >= bytes) + + /* If we have seen requested number of bytes, stop */ + if (bytes_count >= bytes) return sgl_count; - bytes -= sg[i].length; + bytes -= bytes_count; } } return sgl_count; } +/** + * Release the specified buffers from TX SGL pointed to by ctx->tsgl_list for + * @used bytes. + * + * If @dst is non-null, reassign the pages to dst. The caller must release + * the pages. If @dst_offset is given only reassign the pages to @dst starting + * at the @dst_offset (byte). The caller must ensure that @dst is large + * enough (e.g. by using aead_count_tsgl with the same offset). + */ static void aead_pull_tsgl(struct sock *sk, size_t used, - struct scatterlist *dst) + struct scatterlist *dst, size_t dst_offset) { struct alg_sock *ask = alg_sk(sk); struct aead_ctx *ctx = ask->private; struct aead_tsgl *sgl; struct scatterlist *sg; - unsigned int i; + unsigned int i, j; while (!list_empty(&ctx->tsgl_list)) { sgl = list_first_entry(&ctx->tsgl_list, struct aead_tsgl, list); sg = sgl->sg; - for (i = 0; i < sgl->cur; i++) { + for (i = 0, j = 0; i < sgl->cur; i++) { size_t plen = min_t(size_t, used, sg[i].length); struct page *page = sg_page(sg + i); @@ -219,8 +250,20 @@ static void aead_pull_tsgl(struct sock *sk, size_t used, * Assumption: caller created aead_count_tsgl(len) * SG entries in dst. */ - if (dst) - sg_set_page(dst + i, page, plen, sg[i].offset); + if (dst) { + if (dst_offset >= plen) { + /* discard page before offset */ + dst_offset -= plen; + put_page(page); + } else { + /* reassign page to dst after offset */ + sg_set_page(dst + j, page, + plen - dst_offset, + sg[i].offset + dst_offset); + dst_offset = 0; + j++; + } + } sg[i].length -= plen; sg[i].offset += plen; @@ -233,6 +276,7 @@ static void aead_pull_tsgl(struct sock *sk, size_t used, if (!dst) put_page(page); + sg_assign_page(sg + i, NULL); } @@ -583,6 +627,20 @@ static void aead_async_cb(struct crypto_async_request *_req, int err) release_sock(sk); } +static int crypto_aead_copy_sgl(struct crypto_skcipher *null_tfm, + struct scatterlist *src, + struct scatterlist *dst, unsigned int len) +{ + SKCIPHER_REQUEST_ON_STACK(skreq, null_tfm); + + skcipher_request_set_tfm(skreq, null_tfm); + skcipher_request_set_callback(skreq, CRYPTO_TFM_REQ_MAY_BACKLOG, + NULL, NULL); + skcipher_request_set_crypt(skreq, src, dst, len, NULL); + + return crypto_skcipher_encrypt(skreq); +} + static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, size_t ignored, int flags) { @@ -593,11 +651,14 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, struct aead_ctx *ctx = ask->private; struct aead_tfm *aeadc = pask->private; struct crypto_aead *tfm = aeadc->aead; + struct crypto_skcipher *null_tfm = aeadc->null_tfm; unsigned int as = crypto_aead_authsize(tfm); unsigned int areqlen = sizeof(struct aead_async_req) + crypto_aead_reqsize(tfm); struct aead_async_req *areq; struct aead_rsgl *last_rsgl = NULL; + struct aead_tsgl *tsgl; + struct scatterlist *src; int err = 0; size_t used = 0; /* [in] TX bufs to be en/decrypted */ size_t outlen = 0; /* [out] RX bufs produced by kernel */ @@ -716,25 +777,91 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, outlen -= less; } + processed = used + ctx->aead_assoclen; + tsgl = list_first_entry(&ctx->tsgl_list, struct aead_tsgl, list); + /* - * Create a per request TX SGL for this request which tracks the - * SG entries from the global TX SGL. + * Copy of AAD from source to destination + * + * The AAD is copied to the destination buffer without change. Even + * when user space uses an in-place cipher operation, the kernel + * will copy the data as it does not see whether such in-place operation + * is initiated. + * + * To ensure efficiency, the following implementation ensure that the + * ciphers are invoked to perform a crypto operation in-place. This + * is achieved by memory management specified as follows. */ - processed = used + ctx->aead_assoclen; - areq->tsgl_entries = aead_count_tsgl(sk, processed); - if (!areq->tsgl_entries) - areq->tsgl_entries = 1; - areq->tsgl = sock_kmalloc(sk, sizeof(*areq->tsgl) * areq->tsgl_entries, - GFP_KERNEL); - if (!areq->tsgl) { - err = -ENOMEM; - goto free; + + /* Use the RX SGL as source (and destination) for crypto op. */ + src = areq->first_rsgl.sgl.sg; + + if (ctx->enc) { + /* + * Encryption operation - The in-place cipher operation is + * achieved by the following operation: + * + * TX SGL: AAD || PT || Tag + * | | + * | copy | + * v v + * RX SGL: AAD || PT + */ + err = crypto_aead_copy_sgl(null_tfm, tsgl->sg, + areq->first_rsgl.sgl.sg, processed); + if (err) + goto free; + aead_pull_tsgl(sk, processed, NULL, 0); + } else { + /* + * Decryption operation - To achieve an in-place cipher + * operation, the following SGL structure is used: + * + * TX SGL: AAD || CT || Tag + * | | ^ + * | copy | | Create SGL link. + * v v | + * RX SGL: AAD || CT ----+ + */ + + /* Copy AAD || CT to RX SGL buffer for in-place operation. */ + err = crypto_aead_copy_sgl(null_tfm, tsgl->sg, + areq->first_rsgl.sgl.sg, outlen); + if (err) + goto free; + + /* Create TX SGL for tag and chain it to RX SGL. */ + areq->tsgl_entries = aead_count_tsgl(sk, processed, + processed - as); + if (!areq->tsgl_entries) + areq->tsgl_entries = 1; + areq->tsgl = sock_kmalloc(sk, sizeof(*areq->tsgl) * + areq->tsgl_entries, + GFP_KERNEL); + if (!areq->tsgl) { + err = -ENOMEM; + goto free; + } + sg_init_table(areq->tsgl, areq->tsgl_entries); + + /* Release TX SGL, except for tag data and reassign tag data. */ + aead_pull_tsgl(sk, processed, areq->tsgl, processed - as); + + /* chain the areq TX SGL holding the tag with RX SGL */ + if (last_rsgl) { + /* RX SGL present */ + struct af_alg_sgl *sgl_prev = &last_rsgl->sgl; + + sg_unmark_end(sgl_prev->sg + sgl_prev->npages - 1); + sg_chain(sgl_prev->sg, sgl_prev->npages + 1, + areq->tsgl); + } else + /* no RX SGL present (e.g. authentication only) */ + src = areq->tsgl; } - sg_init_table(areq->tsgl, areq->tsgl_entries); - aead_pull_tsgl(sk, processed, areq->tsgl); /* Initialize the crypto operation */ - aead_request_set_crypt(&areq->aead_req, areq->tsgl, + aead_request_set_crypt(&areq->aead_req, src, areq->first_rsgl.sgl.sg, used, ctx->iv); aead_request_set_ad(&areq->aead_req, ctx->aead_assoclen); aead_request_set_tfm(&areq->aead_req, tfm); @@ -951,6 +1078,7 @@ static void *aead_bind(const char *name, u32 type, u32 mask) { struct aead_tfm *tfm; struct crypto_aead *aead; + struct crypto_skcipher *null_tfm; tfm = kzalloc(sizeof(*tfm), GFP_KERNEL); if (!tfm) @@ -962,7 +1090,15 @@ static void *aead_bind(const char *name, u32 type, u32 mask) return ERR_CAST(aead); } + null_tfm = crypto_get_default_null_skcipher2(); + if (IS_ERR(null_tfm)) { + crypto_free_aead(aead); + kfree(tfm); + return ERR_CAST(null_tfm); + } + tfm->aead = aead; + tfm->null_tfm = null_tfm; return tfm; } @@ -1003,7 +1139,8 @@ static void aead_sock_destruct(struct sock *sk) struct crypto_aead *tfm = aeadc->aead; unsigned int ivlen = crypto_aead_ivsize(tfm); - aead_pull_tsgl(sk, ctx->used, NULL); + aead_pull_tsgl(sk, ctx->used, NULL, 0); + crypto_put_default_null_skcipher2(); sock_kzfree_s(sk, ctx->iv, ivlen); sock_kfree_s(sk, ctx, ctx->len); af_alg_release_parent(sk); -- cgit v1.2.3 From 6eea87b94a9254c279a909be51da7ccc043bbf49 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 31 Jul 2017 22:43:55 +0200 Subject: crypto: serpent - improve __serpent_setkey with UBSAN When UBSAN is enabled, we get a very large stack frame for __serpent_setkey, when the register allocator ends up using more registers than it has, and has to spill temporary values to the stack. The code was originally optimized for in-order x86-32 CPU implementations using older compilers, but it now runs into a highly suboptimal case on all CPU architectures, as seen by this warning: crypto/serpent_generic.c: In function '__serpent_setkey': crypto/serpent_generic.c:436:1: error: the frame size of 2720 bytes is larger than 2048 bytes [-Werror=frame-larger-than=] Disabling -fsanitize=alignment would avoid that warning, presumably the option turns off a optimization step that is required for getting the register allocation right, but there is no easy way to do that on gcc-7 (gcc-8 introduces a function attribute for this). I tried to figure out a way to modify the source code instead, and noticed that the two stages of the setkey() function (keyiter and sbox) each are fine by themselves, but not when combined into one function. Splitting out the entire sbox into a separate function also happens to work fine with all compilers I tried (arm, arm64 and x86). The setkey function uses a strange way to handle offsets into the key array, using both negative and positive index values, as well as adjusting the array pointer back and forth. I have checked that this actually makes no difference to modern compilers, but I left that untouched to make the patch easier to review and to keep the code closer to the reference implementation. Link: https://patchwork.kernel.org/patch/9189575/ Signed-off-by: Arnd Bergmann Signed-off-by: Herbert Xu --- crypto/serpent_generic.c | 77 ++++++++++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 36 deletions(-) diff --git a/crypto/serpent_generic.c b/crypto/serpent_generic.c index 94970a79..7c3382fa 100644 --- a/crypto/serpent_generic.c +++ b/crypto/serpent_generic.c @@ -229,6 +229,46 @@ x4 ^= x2; \ }) +static void __serpent_setkey_sbox(u32 r0, u32 r1, u32 r2, u32 r3, u32 r4, u32 *k) +{ + k += 100; + S3(r3, r4, r0, r1, r2); store_and_load_keys(r1, r2, r4, r3, 28, 24); + S4(r1, r2, r4, r3, r0); store_and_load_keys(r2, r4, r3, r0, 24, 20); + S5(r2, r4, r3, r0, r1); store_and_load_keys(r1, r2, r4, r0, 20, 16); + S6(r1, r2, r4, r0, r3); store_and_load_keys(r4, r3, r2, r0, 16, 12); + S7(r4, r3, r2, r0, r1); store_and_load_keys(r1, r2, r0, r4, 12, 8); + S0(r1, r2, r0, r4, r3); store_and_load_keys(r0, r2, r4, r1, 8, 4); + S1(r0, r2, r4, r1, r3); store_and_load_keys(r3, r4, r1, r0, 4, 0); + S2(r3, r4, r1, r0, r2); store_and_load_keys(r2, r4, r3, r0, 0, -4); + S3(r2, r4, r3, r0, r1); store_and_load_keys(r0, r1, r4, r2, -4, -8); + S4(r0, r1, r4, r2, r3); store_and_load_keys(r1, r4, r2, r3, -8, -12); + S5(r1, r4, r2, r3, r0); store_and_load_keys(r0, r1, r4, r3, -12, -16); + S6(r0, r1, r4, r3, r2); store_and_load_keys(r4, r2, r1, r3, -16, -20); + S7(r4, r2, r1, r3, r0); store_and_load_keys(r0, r1, r3, r4, -20, -24); + S0(r0, r1, r3, r4, r2); store_and_load_keys(r3, r1, r4, r0, -24, -28); + k -= 50; + S1(r3, r1, r4, r0, r2); store_and_load_keys(r2, r4, r0, r3, 22, 18); + S2(r2, r4, r0, r3, r1); store_and_load_keys(r1, r4, r2, r3, 18, 14); + S3(r1, r4, r2, r3, r0); store_and_load_keys(r3, r0, r4, r1, 14, 10); + S4(r3, r0, r4, r1, r2); store_and_load_keys(r0, r4, r1, r2, 10, 6); + S5(r0, r4, r1, r2, r3); store_and_load_keys(r3, r0, r4, r2, 6, 2); + S6(r3, r0, r4, r2, r1); store_and_load_keys(r4, r1, r0, r2, 2, -2); + S7(r4, r1, r0, r2, r3); store_and_load_keys(r3, r0, r2, r4, -2, -6); + S0(r3, r0, r2, r4, r1); store_and_load_keys(r2, r0, r4, r3, -6, -10); + S1(r2, r0, r4, r3, r1); store_and_load_keys(r1, r4, r3, r2, -10, -14); + S2(r1, r4, r3, r2, r0); store_and_load_keys(r0, r4, r1, r2, -14, -18); + S3(r0, r4, r1, r2, r3); store_and_load_keys(r2, r3, r4, r0, -18, -22); + k -= 50; + S4(r2, r3, r4, r0, r1); store_and_load_keys(r3, r4, r0, r1, 28, 24); + S5(r3, r4, r0, r1, r2); store_and_load_keys(r2, r3, r4, r1, 24, 20); + S6(r2, r3, r4, r1, r0); store_and_load_keys(r4, r0, r3, r1, 20, 16); + S7(r4, r0, r3, r1, r2); store_and_load_keys(r2, r3, r1, r4, 16, 12); + S0(r2, r3, r1, r4, r0); store_and_load_keys(r1, r3, r4, r2, 12, 8); + S1(r1, r3, r4, r2, r0); store_and_load_keys(r0, r4, r2, r1, 8, 4); + S2(r0, r4, r2, r1, r3); store_and_load_keys(r3, r4, r0, r1, 4, 0); + S3(r3, r4, r0, r1, r2); storekeys(r1, r2, r4, r3, 0); +} + int __serpent_setkey(struct serpent_ctx *ctx, const u8 *key, unsigned int keylen) { @@ -395,42 +435,7 @@ int __serpent_setkey(struct serpent_ctx *ctx, const u8 *key, keyiter(k[23], r1, r0, r3, 131, 31); /* Apply S-boxes */ - - S3(r3, r4, r0, r1, r2); store_and_load_keys(r1, r2, r4, r3, 28, 24); - S4(r1, r2, r4, r3, r0); store_and_load_keys(r2, r4, r3, r0, 24, 20); - S5(r2, r4, r3, r0, r1); store_and_load_keys(r1, r2, r4, r0, 20, 16); - S6(r1, r2, r4, r0, r3); store_and_load_keys(r4, r3, r2, r0, 16, 12); - S7(r4, r3, r2, r0, r1); store_and_load_keys(r1, r2, r0, r4, 12, 8); - S0(r1, r2, r0, r4, r3); store_and_load_keys(r0, r2, r4, r1, 8, 4); - S1(r0, r2, r4, r1, r3); store_and_load_keys(r3, r4, r1, r0, 4, 0); - S2(r3, r4, r1, r0, r2); store_and_load_keys(r2, r4, r3, r0, 0, -4); - S3(r2, r4, r3, r0, r1); store_and_load_keys(r0, r1, r4, r2, -4, -8); - S4(r0, r1, r4, r2, r3); store_and_load_keys(r1, r4, r2, r3, -8, -12); - S5(r1, r4, r2, r3, r0); store_and_load_keys(r0, r1, r4, r3, -12, -16); - S6(r0, r1, r4, r3, r2); store_and_load_keys(r4, r2, r1, r3, -16, -20); - S7(r4, r2, r1, r3, r0); store_and_load_keys(r0, r1, r3, r4, -20, -24); - S0(r0, r1, r3, r4, r2); store_and_load_keys(r3, r1, r4, r0, -24, -28); - k -= 50; - S1(r3, r1, r4, r0, r2); store_and_load_keys(r2, r4, r0, r3, 22, 18); - S2(r2, r4, r0, r3, r1); store_and_load_keys(r1, r4, r2, r3, 18, 14); - S3(r1, r4, r2, r3, r0); store_and_load_keys(r3, r0, r4, r1, 14, 10); - S4(r3, r0, r4, r1, r2); store_and_load_keys(r0, r4, r1, r2, 10, 6); - S5(r0, r4, r1, r2, r3); store_and_load_keys(r3, r0, r4, r2, 6, 2); - S6(r3, r0, r4, r2, r1); store_and_load_keys(r4, r1, r0, r2, 2, -2); - S7(r4, r1, r0, r2, r3); store_and_load_keys(r3, r0, r2, r4, -2, -6); - S0(r3, r0, r2, r4, r1); store_and_load_keys(r2, r0, r4, r3, -6, -10); - S1(r2, r0, r4, r3, r1); store_and_load_keys(r1, r4, r3, r2, -10, -14); - S2(r1, r4, r3, r2, r0); store_and_load_keys(r0, r4, r1, r2, -14, -18); - S3(r0, r4, r1, r2, r3); store_and_load_keys(r2, r3, r4, r0, -18, -22); - k -= 50; - S4(r2, r3, r4, r0, r1); store_and_load_keys(r3, r4, r0, r1, 28, 24); - S5(r3, r4, r0, r1, r2); store_and_load_keys(r2, r3, r4, r1, 24, 20); - S6(r2, r3, r4, r1, r0); store_and_load_keys(r4, r0, r3, r1, 20, 16); - S7(r4, r0, r3, r1, r2); store_and_load_keys(r2, r3, r1, r4, 16, 12); - S0(r2, r3, r1, r4, r0); store_and_load_keys(r1, r3, r4, r2, 12, 8); - S1(r1, r3, r4, r2, r0); store_and_load_keys(r0, r4, r2, r1, 8, 4); - S2(r0, r4, r2, r1, r3); store_and_load_keys(r3, r4, r0, r1, 4, 0); - S3(r3, r4, r0, r1, r2); storekeys(r1, r2, r4, r3, 0); + __serpent_setkey_sbox(r0, r1, r2, r3, r4, ctx->expkey); return 0; } -- cgit v1.2.3 From c1b1fa586429a5445ce480ddc75ed76568caef8f Mon Sep 17 00:00:00 2001 From: Stephan Mueller Date: Wed, 2 Aug 2017 07:56:19 +0200 Subject: crypto: af_alg - consolidation of duplicate code Consolidate following data structures: skcipher_async_req, aead_async_req -> af_alg_async_req skcipher_rsgl, aead_rsql -> af_alg_rsgl skcipher_tsgl, aead_tsql -> af_alg_tsgl skcipher_ctx, aead_ctx -> af_alg_ctx Consolidate following functions: skcipher_sndbuf, aead_sndbuf -> af_alg_sndbuf skcipher_writable, aead_writable -> af_alg_writable skcipher_rcvbuf, aead_rcvbuf -> af_alg_rcvbuf skcipher_readable, aead_readable -> af_alg_readable aead_alloc_tsgl, skcipher_alloc_tsgl -> af_alg_alloc_tsgl aead_count_tsgl, skcipher_count_tsgl -> af_alg_count_tsgl aead_pull_tsgl, skcipher_pull_tsgl -> af_alg_pull_tsgl aead_free_areq_sgls, skcipher_free_areq_sgls -> af_alg_free_areq_sgls aead_wait_for_wmem, skcipher_wait_for_wmem -> af_alg_wait_for_wmem aead_wmem_wakeup, skcipher_wmem_wakeup -> af_alg_wmem_wakeup aead_wait_for_data, skcipher_wait_for_data -> af_alg_wait_for_data aead_data_wakeup, skcipher_data_wakeup -> af_alg_data_wakeup aead_sendmsg, skcipher_sendmsg -> af_alg_sendmsg aead_sendpage, skcipher_sendpage -> af_alg_sendpage aead_async_cb, skcipher_async_cb -> af_alg_async_cb aead_poll, skcipher_poll -> af_alg_poll Split out the following common code from recvmsg: af_alg_alloc_areq: allocation of the request data structure for the cipher operation af_alg_get_rsgl: creation of the RX SGL anchored in the request data structure The following changes to the implementation without affecting the functionality have been applied to synchronize slightly different code bases in algif_skcipher and algif_aead: The wakeup in af_alg_wait_for_data is triggered when either more data is received or the indicator that more data is to be expected is released. The first is triggered by user space, the second is triggered by the kernel upon finishing the processing of data (i.e. the kernel is ready for more). af_alg_sendmsg uses size_t in min_t calculation for obtaining len. Return code determination is consistent with algif_skcipher. The scope of the variable i is reduced to match algif_aead. The type of the variable i is switched from int to unsigned int to match algif_aead. af_alg_sendpage does not contain the superfluous err = 0 from aead_sendpage. af_alg_async_cb requires to store the number of output bytes in areq->outlen before the AIO callback is triggered. The POLLIN / POLLRDNORM is now set when either not more data is given or the kernel is supplied with data. This is consistent to the wakeup from sleep when the kernel waits for data. The request data structure is extended by the field last_rsgl which points to the last RX SGL list entry. This shall help recvmsg implementation to chain the RX SGL to other SG(L)s if needed. It is currently used by algif_aead which chains the tag SGL to the RX SGL during decryption. Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/af_alg.c | 693 +++++++++++++++++++++++++++++++++++++++++++++++ crypto/algif_aead.c | 701 +++--------------------------------------------- crypto/algif_skcipher.c | 638 +++---------------------------------------- 3 files changed, 770 insertions(+), 1262 deletions(-) diff --git a/crypto/af_alg.c b/crypto/af_alg.c index 92a3d540..d6936c0e 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -21,6 +21,7 @@ #include #include #include +#include #include struct alg_type_list { @@ -507,6 +508,698 @@ void af_alg_complete(struct crypto_async_request *req, int err) } EXPORT_SYMBOL_GPL(af_alg_complete); +/** + * af_alg_alloc_tsgl - allocate the TX SGL + * + * @sk socket of connection to user space + * @return: 0 upon success, < 0 upon error + */ +int af_alg_alloc_tsgl(struct sock *sk) +{ + struct alg_sock *ask = alg_sk(sk); + struct af_alg_ctx *ctx = ask->private; + struct af_alg_tsgl *sgl; + struct scatterlist *sg = NULL; + + sgl = list_entry(ctx->tsgl_list.prev, struct af_alg_tsgl, list); + if (!list_empty(&ctx->tsgl_list)) + sg = sgl->sg; + + if (!sg || sgl->cur >= MAX_SGL_ENTS) { + sgl = sock_kmalloc(sk, sizeof(*sgl) + + sizeof(sgl->sg[0]) * (MAX_SGL_ENTS + 1), + GFP_KERNEL); + if (!sgl) + return -ENOMEM; + + sg_init_table(sgl->sg, MAX_SGL_ENTS + 1); + sgl->cur = 0; + + if (sg) + sg_chain(sg, MAX_SGL_ENTS + 1, sgl->sg); + + list_add_tail(&sgl->list, &ctx->tsgl_list); + } + + return 0; +} +EXPORT_SYMBOL_GPL(af_alg_alloc_tsgl); + +/** + * aead_count_tsgl - Count number of TX SG entries + * + * The counting starts from the beginning of the SGL to @bytes. If + * an offset is provided, the counting of the SG entries starts at the offset. + * + * @sk socket of connection to user space + * @bytes Count the number of SG entries holding given number of bytes. + * @offset Start the counting of SG entries from the given offset. + * @return Number of TX SG entries found given the constraints + */ +unsigned int af_alg_count_tsgl(struct sock *sk, size_t bytes, size_t offset) +{ + struct alg_sock *ask = alg_sk(sk); + struct af_alg_ctx *ctx = ask->private; + struct af_alg_tsgl *sgl, *tmp; + unsigned int i; + unsigned int sgl_count = 0; + + if (!bytes) + return 0; + + list_for_each_entry_safe(sgl, tmp, &ctx->tsgl_list, list) { + struct scatterlist *sg = sgl->sg; + + for (i = 0; i < sgl->cur; i++) { + size_t bytes_count; + + /* Skip offset */ + if (offset >= sg[i].length) { + offset -= sg[i].length; + bytes -= sg[i].length; + continue; + } + + bytes_count = sg[i].length - offset; + + offset = 0; + sgl_count++; + + /* If we have seen requested number of bytes, stop */ + if (bytes_count >= bytes) + return sgl_count; + + bytes -= bytes_count; + } + } + + return sgl_count; +} +EXPORT_SYMBOL_GPL(af_alg_count_tsgl); + +/** + * aead_pull_tsgl - Release the specified buffers from TX SGL + * + * If @dst is non-null, reassign the pages to dst. The caller must release + * the pages. If @dst_offset is given only reassign the pages to @dst starting + * at the @dst_offset (byte). The caller must ensure that @dst is large + * enough (e.g. by using af_alg_count_tsgl with the same offset). + * + * @sk socket of connection to user space + * @used Number of bytes to pull from TX SGL + * @dst If non-NULL, buffer is reassigned to dst SGL instead of releasing. The + * caller must release the buffers in dst. + * @dst_offset Reassign the TX SGL from given offset. All buffers before + * reaching the offset is released. + */ +void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst, + size_t dst_offset) +{ + struct alg_sock *ask = alg_sk(sk); + struct af_alg_ctx *ctx = ask->private; + struct af_alg_tsgl *sgl; + struct scatterlist *sg; + unsigned int i, j; + + while (!list_empty(&ctx->tsgl_list)) { + sgl = list_first_entry(&ctx->tsgl_list, struct af_alg_tsgl, + list); + sg = sgl->sg; + + for (i = 0, j = 0; i < sgl->cur; i++) { + size_t plen = min_t(size_t, used, sg[i].length); + struct page *page = sg_page(sg + i); + + if (!page) + continue; + + /* + * Assumption: caller created af_alg_count_tsgl(len) + * SG entries in dst. + */ + if (dst) { + if (dst_offset >= plen) { + /* discard page before offset */ + dst_offset -= plen; + put_page(page); + } else { + /* reassign page to dst after offset */ + sg_set_page(dst + j, page, + plen - dst_offset, + sg[i].offset + dst_offset); + dst_offset = 0; + j++; + } + } + + sg[i].length -= plen; + sg[i].offset += plen; + + used -= plen; + ctx->used -= plen; + + if (sg[i].length) + return; + + if (!dst) + put_page(page); + + sg_assign_page(sg + i, NULL); + } + + list_del(&sgl->list); + sock_kfree_s(sk, sgl, sizeof(*sgl) + sizeof(sgl->sg[0]) * + (MAX_SGL_ENTS + 1)); + } + + if (!ctx->used) + ctx->merge = 0; +} +EXPORT_SYMBOL_GPL(af_alg_pull_tsgl); + +/** + * af_alg_free_areq_sgls - Release TX and RX SGLs of the request + * + * @areq Request holding the TX and RX SGL + */ +void af_alg_free_areq_sgls(struct af_alg_async_req *areq) +{ + struct sock *sk = areq->sk; + struct alg_sock *ask = alg_sk(sk); + struct af_alg_ctx *ctx = ask->private; + struct af_alg_rsgl *rsgl, *tmp; + struct scatterlist *tsgl; + struct scatterlist *sg; + unsigned int i; + + list_for_each_entry_safe(rsgl, tmp, &areq->rsgl_list, list) { + ctx->rcvused -= rsgl->sg_num_bytes; + af_alg_free_sg(&rsgl->sgl); + list_del(&rsgl->list); + if (rsgl != &areq->first_rsgl) + sock_kfree_s(sk, rsgl, sizeof(*rsgl)); + } + + tsgl = areq->tsgl; + for_each_sg(tsgl, sg, areq->tsgl_entries, i) { + if (!sg_page(sg)) + continue; + put_page(sg_page(sg)); + } + + if (areq->tsgl && areq->tsgl_entries) + sock_kfree_s(sk, tsgl, areq->tsgl_entries * sizeof(*tsgl)); +} +EXPORT_SYMBOL_GPL(af_alg_free_areq_sgls); + +/** + * af_alg_wait_for_wmem - wait for availability of writable memory + * + * @sk socket of connection to user space + * @flags If MSG_DONTWAIT is set, then only report if function would sleep + * @return 0 when writable memory is available, < 0 upon error + */ +int af_alg_wait_for_wmem(struct sock *sk, unsigned int flags) +{ + DEFINE_WAIT_FUNC(wait, woken_wake_function); + int err = -ERESTARTSYS; + long timeout; + + if (flags & MSG_DONTWAIT) + return -EAGAIN; + + sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); + + add_wait_queue(sk_sleep(sk), &wait); + for (;;) { + if (signal_pending(current)) + break; + timeout = MAX_SCHEDULE_TIMEOUT; + if (sk_wait_event(sk, &timeout, af_alg_writable(sk), &wait)) { + err = 0; + break; + } + } + remove_wait_queue(sk_sleep(sk), &wait); + + return err; +} +EXPORT_SYMBOL_GPL(af_alg_wait_for_wmem); + +/** + * af_alg_wmem_wakeup - wakeup caller when writable memory is available + * + * @sk socket of connection to user space + */ +void af_alg_wmem_wakeup(struct sock *sk) +{ + struct socket_wq *wq; + + if (!af_alg_writable(sk)) + return; + + rcu_read_lock(); + wq = rcu_dereference(sk->sk_wq); + if (skwq_has_sleeper(wq)) + wake_up_interruptible_sync_poll(&wq->wait, POLLIN | + POLLRDNORM | + POLLRDBAND); + sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); + rcu_read_unlock(); +} +EXPORT_SYMBOL_GPL(af_alg_wmem_wakeup); + +/** + * af_alg_wait_for_data - wait for availability of TX data + * + * @sk socket of connection to user space + * @flags If MSG_DONTWAIT is set, then only report if function would sleep + * @return 0 when writable memory is available, < 0 upon error + */ +int af_alg_wait_for_data(struct sock *sk, unsigned flags) +{ + DEFINE_WAIT_FUNC(wait, woken_wake_function); + struct alg_sock *ask = alg_sk(sk); + struct af_alg_ctx *ctx = ask->private; + long timeout; + int err = -ERESTARTSYS; + + if (flags & MSG_DONTWAIT) + return -EAGAIN; + + sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); + + add_wait_queue(sk_sleep(sk), &wait); + for (;;) { + if (signal_pending(current)) + break; + timeout = MAX_SCHEDULE_TIMEOUT; + if (sk_wait_event(sk, &timeout, (ctx->used || !ctx->more), + &wait)) { + err = 0; + break; + } + } + remove_wait_queue(sk_sleep(sk), &wait); + + sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); + + return err; +} +EXPORT_SYMBOL_GPL(af_alg_wait_for_data); + +/** + * af_alg_data_wakeup - wakeup caller when new data can be sent to kernel + * + * @sk socket of connection to user space + */ + +void af_alg_data_wakeup(struct sock *sk) +{ + struct alg_sock *ask = alg_sk(sk); + struct af_alg_ctx *ctx = ask->private; + struct socket_wq *wq; + + if (!ctx->used) + return; + + rcu_read_lock(); + wq = rcu_dereference(sk->sk_wq); + if (skwq_has_sleeper(wq)) + wake_up_interruptible_sync_poll(&wq->wait, POLLOUT | + POLLRDNORM | + POLLRDBAND); + sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT); + rcu_read_unlock(); +} +EXPORT_SYMBOL_GPL(af_alg_data_wakeup); + +/** + * af_alg_sendmsg - implementation of sendmsg system call handler + * + * The sendmsg system call handler obtains the user data and stores it + * in ctx->tsgl_list. This implies allocation of the required numbers of + * struct af_alg_tsgl. + * + * In addition, the ctx is filled with the information sent via CMSG. + * + * @sock socket of connection to user space + * @msg message from user space + * @size size of message from user space + * @ivsize the size of the IV for the cipher operation to verify that the + * user-space-provided IV has the right size + * @return the number of copied data upon success, < 0 upon error + */ +int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size, + unsigned int ivsize) +{ + struct sock *sk = sock->sk; + struct alg_sock *ask = alg_sk(sk); + struct af_alg_ctx *ctx = ask->private; + struct af_alg_tsgl *sgl; + struct af_alg_control con = {}; + long copied = 0; + bool enc = 0; + bool init = 0; + int err = 0; + + if (msg->msg_controllen) { + err = af_alg_cmsg_send(msg, &con); + if (err) + return err; + + init = 1; + switch (con.op) { + case ALG_OP_ENCRYPT: + enc = 1; + break; + case ALG_OP_DECRYPT: + enc = 0; + break; + default: + return -EINVAL; + } + + if (con.iv && con.iv->ivlen != ivsize) + return -EINVAL; + } + + lock_sock(sk); + if (!ctx->more && ctx->used) { + err = -EINVAL; + goto unlock; + } + + if (init) { + ctx->enc = enc; + if (con.iv) + memcpy(ctx->iv, con.iv->iv, ivsize); + + ctx->aead_assoclen = con.aead_assoclen; + } + + while (size) { + struct scatterlist *sg; + size_t len = size; + size_t plen; + + /* use the existing memory in an allocated page */ + if (ctx->merge) { + sgl = list_entry(ctx->tsgl_list.prev, + struct af_alg_tsgl, list); + sg = sgl->sg + sgl->cur - 1; + len = min_t(size_t, len, + PAGE_SIZE - sg->offset - sg->length); + + err = memcpy_from_msg(page_address(sg_page(sg)) + + sg->offset + sg->length, + msg, len); + if (err) + goto unlock; + + sg->length += len; + ctx->merge = (sg->offset + sg->length) & + (PAGE_SIZE - 1); + + ctx->used += len; + copied += len; + size -= len; + continue; + } + + if (!af_alg_writable(sk)) { + err = af_alg_wait_for_wmem(sk, msg->msg_flags); + if (err) + goto unlock; + } + + /* allocate a new page */ + len = min_t(unsigned long, len, af_alg_sndbuf(sk)); + + err = af_alg_alloc_tsgl(sk); + if (err) + goto unlock; + + sgl = list_entry(ctx->tsgl_list.prev, struct af_alg_tsgl, + list); + sg = sgl->sg; + if (sgl->cur) + sg_unmark_end(sg + sgl->cur - 1); + + do { + unsigned int i = sgl->cur; + + plen = min_t(size_t, len, PAGE_SIZE); + + sg_assign_page(sg + i, alloc_page(GFP_KERNEL)); + if (!sg_page(sg + i)) { + err = -ENOMEM; + goto unlock; + } + + err = memcpy_from_msg(page_address(sg_page(sg + i)), + msg, plen); + if (err) { + __free_page(sg_page(sg + i)); + sg_assign_page(sg + i, NULL); + goto unlock; + } + + sg[i].length = plen; + len -= plen; + ctx->used += plen; + copied += plen; + size -= plen; + sgl->cur++; + } while (len && sgl->cur < MAX_SGL_ENTS); + + if (!size) + sg_mark_end(sg + sgl->cur - 1); + + ctx->merge = plen & (PAGE_SIZE - 1); + } + + err = 0; + + ctx->more = msg->msg_flags & MSG_MORE; + +unlock: + af_alg_data_wakeup(sk); + release_sock(sk); + + return copied ?: err; +} +EXPORT_SYMBOL_GPL(af_alg_sendmsg); + +/** + * af_alg_sendpage - sendpage system call handler + * + * This is a generic implementation of sendpage to fill ctx->tsgl_list. + */ +ssize_t af_alg_sendpage(struct socket *sock, struct page *page, + int offset, size_t size, int flags) +{ + struct sock *sk = sock->sk; + struct alg_sock *ask = alg_sk(sk); + struct af_alg_ctx *ctx = ask->private; + struct af_alg_tsgl *sgl; + int err = -EINVAL; + + if (flags & MSG_SENDPAGE_NOTLAST) + flags |= MSG_MORE; + + lock_sock(sk); + if (!ctx->more && ctx->used) + goto unlock; + + if (!size) + goto done; + + if (!af_alg_writable(sk)) { + err = af_alg_wait_for_wmem(sk, flags); + if (err) + goto unlock; + } + + err = af_alg_alloc_tsgl(sk); + if (err) + goto unlock; + + ctx->merge = 0; + sgl = list_entry(ctx->tsgl_list.prev, struct af_alg_tsgl, list); + + if (sgl->cur) + sg_unmark_end(sgl->sg + sgl->cur - 1); + + sg_mark_end(sgl->sg + sgl->cur); + + get_page(page); + sg_set_page(sgl->sg + sgl->cur, page, size, offset); + sgl->cur++; + ctx->used += size; + +done: + ctx->more = flags & MSG_MORE; + +unlock: + af_alg_data_wakeup(sk); + release_sock(sk); + + return err ?: size; +} +EXPORT_SYMBOL_GPL(af_alg_sendpage); + +/** + * af_alg_async_cb - AIO callback handler + * + * This handler cleans up the struct af_alg_async_req upon completion of the + * AIO operation. + * + * The number of bytes to be generated with the AIO operation must be set + * in areq->outlen before the AIO callback handler is invoked. + */ +void af_alg_async_cb(struct crypto_async_request *_req, int err) +{ + struct af_alg_async_req *areq = _req->data; + struct sock *sk = areq->sk; + struct kiocb *iocb = areq->iocb; + unsigned int resultlen; + + lock_sock(sk); + + /* Buffer size written by crypto operation. */ + resultlen = areq->outlen; + + af_alg_free_areq_sgls(areq); + sock_kfree_s(sk, areq, areq->areqlen); + __sock_put(sk); + + iocb->ki_complete(iocb, err ? err : resultlen, 0); + + release_sock(sk); +} +EXPORT_SYMBOL_GPL(af_alg_async_cb); + +/** + * af_alg_poll - poll system call handler + */ +unsigned int af_alg_poll(struct file *file, struct socket *sock, + poll_table *wait) +{ + struct sock *sk = sock->sk; + struct alg_sock *ask = alg_sk(sk); + struct af_alg_ctx *ctx = ask->private; + unsigned int mask; + + sock_poll_wait(file, sk_sleep(sk), wait); + mask = 0; + + if (!ctx->more || ctx->used) + mask |= POLLIN | POLLRDNORM; + + if (af_alg_writable(sk)) + mask |= POLLOUT | POLLWRNORM | POLLWRBAND; + + return mask; +} +EXPORT_SYMBOL_GPL(af_alg_poll); + +/** + * af_alg_alloc_areq - allocate struct af_alg_async_req + * + * @sk socket of connection to user space + * @areqlen size of struct af_alg_async_req + crypto_*_reqsize + * @return allocated data structure or ERR_PTR upon error + */ +struct af_alg_async_req *af_alg_alloc_areq(struct sock *sk, + unsigned int areqlen) +{ + struct af_alg_async_req *areq = sock_kmalloc(sk, areqlen, GFP_KERNEL); + + if (unlikely(!areq)) + return ERR_PTR(-ENOMEM); + + areq->areqlen = areqlen; + areq->sk = sk; + areq->last_rsgl = NULL; + INIT_LIST_HEAD(&areq->rsgl_list); + areq->tsgl = NULL; + areq->tsgl_entries = 0; + + return areq; +} +EXPORT_SYMBOL_GPL(af_alg_alloc_areq); + +/** + * af_alg_get_rsgl - create the RX SGL for the output data from the crypto + * operation + * + * @sk socket of connection to user space + * @msg user space message + * @flags flags used to invoke recvmsg with + * @areq instance of the cryptographic request that will hold the RX SGL + * @maxsize maximum number of bytes to be pulled from user space + * @outlen number of bytes in the RX SGL + * @return 0 on success, < 0 upon error + */ +int af_alg_get_rsgl(struct sock *sk, struct msghdr *msg, int flags, + struct af_alg_async_req *areq, size_t maxsize, + size_t *outlen) +{ + struct alg_sock *ask = alg_sk(sk); + struct af_alg_ctx *ctx = ask->private; + size_t len = 0; + + while (maxsize > len && msg_data_left(msg)) { + struct af_alg_rsgl *rsgl; + size_t seglen; + int err; + + /* limit the amount of readable buffers */ + if (!af_alg_readable(sk)) + break; + + if (!ctx->used) { + err = af_alg_wait_for_data(sk, flags); + if (err) + return err; + } + + seglen = min_t(size_t, (maxsize - len), + msg_data_left(msg)); + + if (list_empty(&areq->rsgl_list)) { + rsgl = &areq->first_rsgl; + } else { + rsgl = sock_kmalloc(sk, sizeof(*rsgl), GFP_KERNEL); + if (unlikely(!rsgl)) + return -ENOMEM; + } + + rsgl->sgl.npages = 0; + list_add_tail(&rsgl->list, &areq->rsgl_list); + + /* make one iovec available as scatterlist */ + err = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, seglen); + if (err < 0) + return err; + + /* chain the new scatterlist with previous one */ + if (areq->last_rsgl) + af_alg_link_sg(&areq->last_rsgl->sgl, &rsgl->sgl); + + areq->last_rsgl = rsgl; + len += err; + ctx->rcvused += err; + rsgl->sg_num_bytes = err; + iov_iter_advance(&msg->msg_iter, err); + } + + *outlen = len; + return 0; +} +EXPORT_SYMBOL_GPL(af_alg_get_rsgl); + static int __init af_alg_init(void) { int err = proto_register(&alg_proto, 0); diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c index 1f0696dd..48d46e74 100644 --- a/crypto/algif_aead.c +++ b/crypto/algif_aead.c @@ -35,101 +35,23 @@ #include #include #include -#include #include #include #include #include -struct aead_tsgl { - struct list_head list; - unsigned int cur; /* Last processed SG entry */ - struct scatterlist sg[0]; /* Array of SGs forming the SGL */ -}; - -struct aead_rsgl { - struct af_alg_sgl sgl; - struct list_head list; - size_t sg_num_bytes; /* Bytes of data in that SGL */ -}; - -struct aead_async_req { - struct kiocb *iocb; - struct sock *sk; - - struct aead_rsgl first_rsgl; /* First RX SG */ - struct list_head rsgl_list; /* Track RX SGs */ - - struct scatterlist *tsgl; /* priv. TX SGL of buffers to process */ - unsigned int tsgl_entries; /* number of entries in priv. TX SGL */ - - unsigned int outlen; /* Filled output buf length */ - - unsigned int areqlen; /* Length of this data struct */ - struct aead_request aead_req; /* req ctx trails this struct */ -}; - struct aead_tfm { struct crypto_aead *aead; bool has_key; struct crypto_skcipher *null_tfm; }; -struct aead_ctx { - struct list_head tsgl_list; /* Link to TX SGL */ - - void *iv; - size_t aead_assoclen; - - struct af_alg_completion completion; /* sync work queue */ - - size_t used; /* TX bytes sent to kernel */ - size_t rcvused; /* total RX bytes to be processed by kernel */ - - bool more; /* More data to be expected? */ - bool merge; /* Merge new data into existing SG */ - bool enc; /* Crypto operation: enc, dec */ - - unsigned int len; /* Length of allocated memory for this struct */ -}; - -#define MAX_SGL_ENTS ((4096 - sizeof(struct aead_tsgl)) / \ - sizeof(struct scatterlist) - 1) - -static inline int aead_sndbuf(struct sock *sk) -{ - struct alg_sock *ask = alg_sk(sk); - struct aead_ctx *ctx = ask->private; - - return max_t(int, max_t(int, sk->sk_sndbuf & PAGE_MASK, PAGE_SIZE) - - ctx->used, 0); -} - -static inline bool aead_writable(struct sock *sk) -{ - return PAGE_SIZE <= aead_sndbuf(sk); -} - -static inline int aead_rcvbuf(struct sock *sk) -{ - struct alg_sock *ask = alg_sk(sk); - struct aead_ctx *ctx = ask->private; - - return max_t(int, max_t(int, sk->sk_rcvbuf & PAGE_MASK, PAGE_SIZE) - - ctx->rcvused, 0); -} - -static inline bool aead_readable(struct sock *sk) -{ - return PAGE_SIZE <= aead_rcvbuf(sk); -} - static inline bool aead_sufficient_data(struct sock *sk) { struct alg_sock *ask = alg_sk(sk); struct sock *psk = ask->parent; struct alg_sock *pask = alg_sk(psk); - struct aead_ctx *ctx = ask->private; + struct af_alg_ctx *ctx = ask->private; struct aead_tfm *aeadc = pask->private; struct crypto_aead *tfm = aeadc->aead; unsigned int as = crypto_aead_authsize(tfm); @@ -141,490 +63,17 @@ static inline bool aead_sufficient_data(struct sock *sk) return ctx->used >= ctx->aead_assoclen + (ctx->enc ? 0 : as); } -static int aead_alloc_tsgl(struct sock *sk) -{ - struct alg_sock *ask = alg_sk(sk); - struct aead_ctx *ctx = ask->private; - struct aead_tsgl *sgl; - struct scatterlist *sg = NULL; - - sgl = list_entry(ctx->tsgl_list.prev, struct aead_tsgl, list); - if (!list_empty(&ctx->tsgl_list)) - sg = sgl->sg; - - if (!sg || sgl->cur >= MAX_SGL_ENTS) { - sgl = sock_kmalloc(sk, sizeof(*sgl) + - sizeof(sgl->sg[0]) * (MAX_SGL_ENTS + 1), - GFP_KERNEL); - if (!sgl) - return -ENOMEM; - - sg_init_table(sgl->sg, MAX_SGL_ENTS + 1); - sgl->cur = 0; - - if (sg) - sg_chain(sg, MAX_SGL_ENTS + 1, sgl->sg); - - list_add_tail(&sgl->list, &ctx->tsgl_list); - } - - return 0; -} - -/** - * Count number of SG entries from the beginning of the SGL to @bytes. If - * an offset is provided, the counting of the SG entries starts at the offset. - */ -static unsigned int aead_count_tsgl(struct sock *sk, size_t bytes, - size_t offset) -{ - struct alg_sock *ask = alg_sk(sk); - struct aead_ctx *ctx = ask->private; - struct aead_tsgl *sgl, *tmp; - unsigned int i; - unsigned int sgl_count = 0; - - if (!bytes) - return 0; - - list_for_each_entry_safe(sgl, tmp, &ctx->tsgl_list, list) { - struct scatterlist *sg = sgl->sg; - - for (i = 0; i < sgl->cur; i++) { - size_t bytes_count; - - /* Skip offset */ - if (offset >= sg[i].length) { - offset -= sg[i].length; - bytes -= sg[i].length; - continue; - } - - bytes_count = sg[i].length - offset; - - offset = 0; - sgl_count++; - - /* If we have seen requested number of bytes, stop */ - if (bytes_count >= bytes) - return sgl_count; - - bytes -= bytes_count; - } - } - - return sgl_count; -} - -/** - * Release the specified buffers from TX SGL pointed to by ctx->tsgl_list for - * @used bytes. - * - * If @dst is non-null, reassign the pages to dst. The caller must release - * the pages. If @dst_offset is given only reassign the pages to @dst starting - * at the @dst_offset (byte). The caller must ensure that @dst is large - * enough (e.g. by using aead_count_tsgl with the same offset). - */ -static void aead_pull_tsgl(struct sock *sk, size_t used, - struct scatterlist *dst, size_t dst_offset) -{ - struct alg_sock *ask = alg_sk(sk); - struct aead_ctx *ctx = ask->private; - struct aead_tsgl *sgl; - struct scatterlist *sg; - unsigned int i, j; - - while (!list_empty(&ctx->tsgl_list)) { - sgl = list_first_entry(&ctx->tsgl_list, struct aead_tsgl, - list); - sg = sgl->sg; - - for (i = 0, j = 0; i < sgl->cur; i++) { - size_t plen = min_t(size_t, used, sg[i].length); - struct page *page = sg_page(sg + i); - - if (!page) - continue; - - /* - * Assumption: caller created aead_count_tsgl(len) - * SG entries in dst. - */ - if (dst) { - if (dst_offset >= plen) { - /* discard page before offset */ - dst_offset -= plen; - put_page(page); - } else { - /* reassign page to dst after offset */ - sg_set_page(dst + j, page, - plen - dst_offset, - sg[i].offset + dst_offset); - dst_offset = 0; - j++; - } - } - - sg[i].length -= plen; - sg[i].offset += plen; - - used -= plen; - ctx->used -= plen; - - if (sg[i].length) - return; - - if (!dst) - put_page(page); - - sg_assign_page(sg + i, NULL); - } - - list_del(&sgl->list); - sock_kfree_s(sk, sgl, sizeof(*sgl) + sizeof(sgl->sg[0]) * - (MAX_SGL_ENTS + 1)); - } - - if (!ctx->used) - ctx->merge = 0; -} - -static void aead_free_areq_sgls(struct aead_async_req *areq) -{ - struct sock *sk = areq->sk; - struct alg_sock *ask = alg_sk(sk); - struct aead_ctx *ctx = ask->private; - struct aead_rsgl *rsgl, *tmp; - struct scatterlist *tsgl; - struct scatterlist *sg; - unsigned int i; - - list_for_each_entry_safe(rsgl, tmp, &areq->rsgl_list, list) { - ctx->rcvused -= rsgl->sg_num_bytes; - af_alg_free_sg(&rsgl->sgl); - list_del(&rsgl->list); - if (rsgl != &areq->first_rsgl) - sock_kfree_s(sk, rsgl, sizeof(*rsgl)); - } - - tsgl = areq->tsgl; - for_each_sg(tsgl, sg, areq->tsgl_entries, i) { - if (!sg_page(sg)) - continue; - put_page(sg_page(sg)); - } - - if (areq->tsgl && areq->tsgl_entries) - sock_kfree_s(sk, tsgl, areq->tsgl_entries * sizeof(*tsgl)); -} - -static int aead_wait_for_wmem(struct sock *sk, unsigned int flags) -{ - DEFINE_WAIT_FUNC(wait, woken_wake_function); - int err = -ERESTARTSYS; - long timeout; - - if (flags & MSG_DONTWAIT) - return -EAGAIN; - - sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); - - add_wait_queue(sk_sleep(sk), &wait); - for (;;) { - if (signal_pending(current)) - break; - timeout = MAX_SCHEDULE_TIMEOUT; - if (sk_wait_event(sk, &timeout, aead_writable(sk), &wait)) { - err = 0; - break; - } - } - remove_wait_queue(sk_sleep(sk), &wait); - - return err; -} - -static void aead_wmem_wakeup(struct sock *sk) -{ - struct socket_wq *wq; - - if (!aead_writable(sk)) - return; - - rcu_read_lock(); - wq = rcu_dereference(sk->sk_wq); - if (skwq_has_sleeper(wq)) - wake_up_interruptible_sync_poll(&wq->wait, POLLIN | - POLLRDNORM | - POLLRDBAND); - sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); - rcu_read_unlock(); -} - -static int aead_wait_for_data(struct sock *sk, unsigned flags) -{ - DEFINE_WAIT_FUNC(wait, woken_wake_function); - struct alg_sock *ask = alg_sk(sk); - struct aead_ctx *ctx = ask->private; - long timeout; - int err = -ERESTARTSYS; - - if (flags & MSG_DONTWAIT) - return -EAGAIN; - - sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); - - add_wait_queue(sk_sleep(sk), &wait); - for (;;) { - if (signal_pending(current)) - break; - timeout = MAX_SCHEDULE_TIMEOUT; - if (sk_wait_event(sk, &timeout, !ctx->more, &wait)) { - err = 0; - break; - } - } - remove_wait_queue(sk_sleep(sk), &wait); - - sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); - - return err; -} - -static void aead_data_wakeup(struct sock *sk) -{ - struct alg_sock *ask = alg_sk(sk); - struct aead_ctx *ctx = ask->private; - struct socket_wq *wq; - - if (!ctx->used) - return; - - rcu_read_lock(); - wq = rcu_dereference(sk->sk_wq); - if (skwq_has_sleeper(wq)) - wake_up_interruptible_sync_poll(&wq->wait, POLLOUT | - POLLRDNORM | - POLLRDBAND); - sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT); - rcu_read_unlock(); -} - static int aead_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) { struct sock *sk = sock->sk; struct alg_sock *ask = alg_sk(sk); struct sock *psk = ask->parent; struct alg_sock *pask = alg_sk(psk); - struct aead_ctx *ctx = ask->private; struct aead_tfm *aeadc = pask->private; struct crypto_aead *tfm = aeadc->aead; unsigned int ivsize = crypto_aead_ivsize(tfm); - struct aead_tsgl *sgl; - struct af_alg_control con = {}; - long copied = 0; - bool enc = 0; - bool init = 0; - int err = 0; - - if (msg->msg_controllen) { - err = af_alg_cmsg_send(msg, &con); - if (err) - return err; - - init = 1; - switch (con.op) { - case ALG_OP_ENCRYPT: - enc = 1; - break; - case ALG_OP_DECRYPT: - enc = 0; - break; - default: - return -EINVAL; - } - - if (con.iv && con.iv->ivlen != ivsize) - return -EINVAL; - } - - lock_sock(sk); - if (!ctx->more && ctx->used) { - err = -EINVAL; - goto unlock; - } - - if (init) { - ctx->enc = enc; - if (con.iv) - memcpy(ctx->iv, con.iv->iv, ivsize); - - ctx->aead_assoclen = con.aead_assoclen; - } - - while (size) { - struct scatterlist *sg; - size_t len = size; - size_t plen; - - /* use the existing memory in an allocated page */ - if (ctx->merge) { - sgl = list_entry(ctx->tsgl_list.prev, - struct aead_tsgl, list); - sg = sgl->sg + sgl->cur - 1; - len = min_t(unsigned long, len, - PAGE_SIZE - sg->offset - sg->length); - err = memcpy_from_msg(page_address(sg_page(sg)) + - sg->offset + sg->length, - msg, len); - if (err) - goto unlock; - - sg->length += len; - ctx->merge = (sg->offset + sg->length) & - (PAGE_SIZE - 1); - - ctx->used += len; - copied += len; - size -= len; - continue; - } - - if (!aead_writable(sk)) { - err = aead_wait_for_wmem(sk, msg->msg_flags); - if (err) - goto unlock; - } - - /* allocate a new page */ - len = min_t(unsigned long, size, aead_sndbuf(sk)); - - err = aead_alloc_tsgl(sk); - if (err) - goto unlock; - - sgl = list_entry(ctx->tsgl_list.prev, struct aead_tsgl, - list); - sg = sgl->sg; - if (sgl->cur) - sg_unmark_end(sg + sgl->cur - 1); - - do { - unsigned int i = sgl->cur; - - plen = min_t(size_t, len, PAGE_SIZE); - - sg_assign_page(sg + i, alloc_page(GFP_KERNEL)); - if (!sg_page(sg + i)) { - err = -ENOMEM; - goto unlock; - } - - err = memcpy_from_msg(page_address(sg_page(sg + i)), - msg, plen); - if (err) { - __free_page(sg_page(sg + i)); - sg_assign_page(sg + i, NULL); - goto unlock; - } - - sg[i].length = plen; - len -= plen; - ctx->used += plen; - copied += plen; - size -= plen; - sgl->cur++; - } while (len && sgl->cur < MAX_SGL_ENTS); - - if (!size) - sg_mark_end(sg + sgl->cur - 1); - - ctx->merge = plen & (PAGE_SIZE - 1); - } - - err = 0; - - ctx->more = msg->msg_flags & MSG_MORE; - -unlock: - aead_data_wakeup(sk); - release_sock(sk); - - return err ?: copied; -} - -static ssize_t aead_sendpage(struct socket *sock, struct page *page, - int offset, size_t size, int flags) -{ - struct sock *sk = sock->sk; - struct alg_sock *ask = alg_sk(sk); - struct aead_ctx *ctx = ask->private; - struct aead_tsgl *sgl; - int err = -EINVAL; - - if (flags & MSG_SENDPAGE_NOTLAST) - flags |= MSG_MORE; - - lock_sock(sk); - if (!ctx->more && ctx->used) - goto unlock; - - if (!size) - goto done; - - if (!aead_writable(sk)) { - err = aead_wait_for_wmem(sk, flags); - if (err) - goto unlock; - } - - err = aead_alloc_tsgl(sk); - if (err) - goto unlock; - - ctx->merge = 0; - sgl = list_entry(ctx->tsgl_list.prev, struct aead_tsgl, list); - - if (sgl->cur) - sg_unmark_end(sgl->sg + sgl->cur - 1); - - sg_mark_end(sgl->sg + sgl->cur); - - get_page(page); - sg_set_page(sgl->sg + sgl->cur, page, size, offset); - sgl->cur++; - ctx->used += size; - - err = 0; -done: - ctx->more = flags & MSG_MORE; -unlock: - aead_data_wakeup(sk); - release_sock(sk); - - return err ?: size; -} - -static void aead_async_cb(struct crypto_async_request *_req, int err) -{ - struct aead_async_req *areq = _req->data; - struct sock *sk = areq->sk; - struct kiocb *iocb = areq->iocb; - unsigned int resultlen; - - lock_sock(sk); - - /* Buffer size written by crypto operation. */ - resultlen = areq->outlen; - - aead_free_areq_sgls(areq); - sock_kfree_s(sk, areq, areq->areqlen); - __sock_put(sk); - - iocb->ki_complete(iocb, err ? err : resultlen, 0); - - release_sock(sk); + return af_alg_sendmsg(sock, msg, size, ivsize); } static int crypto_aead_copy_sgl(struct crypto_skcipher *null_tfm, @@ -648,16 +97,13 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, struct alg_sock *ask = alg_sk(sk); struct sock *psk = ask->parent; struct alg_sock *pask = alg_sk(psk); - struct aead_ctx *ctx = ask->private; + struct af_alg_ctx *ctx = ask->private; struct aead_tfm *aeadc = pask->private; struct crypto_aead *tfm = aeadc->aead; struct crypto_skcipher *null_tfm = aeadc->null_tfm; unsigned int as = crypto_aead_authsize(tfm); - unsigned int areqlen = - sizeof(struct aead_async_req) + crypto_aead_reqsize(tfm); - struct aead_async_req *areq; - struct aead_rsgl *last_rsgl = NULL; - struct aead_tsgl *tsgl; + struct af_alg_async_req *areq; + struct af_alg_tsgl *tsgl; struct scatterlist *src; int err = 0; size_t used = 0; /* [in] TX bufs to be en/decrypted */ @@ -703,61 +149,15 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, used -= ctx->aead_assoclen; /* Allocate cipher request for current operation. */ - areq = sock_kmalloc(sk, areqlen, GFP_KERNEL); - if (unlikely(!areq)) - return -ENOMEM; - areq->areqlen = areqlen; - areq->sk = sk; - INIT_LIST_HEAD(&areq->rsgl_list); - areq->tsgl = NULL; - areq->tsgl_entries = 0; + areq = af_alg_alloc_areq(sk, sizeof(struct af_alg_async_req) + + crypto_aead_reqsize(tfm)); + if (IS_ERR(areq)) + return PTR_ERR(areq); /* convert iovecs of output buffers into RX SGL */ - while (outlen > usedpages && msg_data_left(msg)) { - struct aead_rsgl *rsgl; - size_t seglen; - - /* limit the amount of readable buffers */ - if (!aead_readable(sk)) - break; - - if (!ctx->used) { - err = aead_wait_for_data(sk, flags); - if (err) - goto free; - } - - seglen = min_t(size_t, (outlen - usedpages), - msg_data_left(msg)); - - if (list_empty(&areq->rsgl_list)) { - rsgl = &areq->first_rsgl; - } else { - rsgl = sock_kmalloc(sk, sizeof(*rsgl), GFP_KERNEL); - if (unlikely(!rsgl)) { - err = -ENOMEM; - goto free; - } - } - - rsgl->sgl.npages = 0; - list_add_tail(&rsgl->list, &areq->rsgl_list); - - /* make one iovec available as scatterlist */ - err = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, seglen); - if (err < 0) - goto free; - - /* chain the new scatterlist with previous one */ - if (last_rsgl) - af_alg_link_sg(&last_rsgl->sgl, &rsgl->sgl); - - last_rsgl = rsgl; - usedpages += err; - ctx->rcvused += err; - rsgl->sg_num_bytes = err; - iov_iter_advance(&msg->msg_iter, err); - } + err = af_alg_get_rsgl(sk, msg, flags, areq, outlen, &usedpages); + if (err) + goto free; /* * Ensure output buffer is sufficiently large. If the caller provides @@ -778,7 +178,7 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, } processed = used + ctx->aead_assoclen; - tsgl = list_first_entry(&ctx->tsgl_list, struct aead_tsgl, list); + tsgl = list_first_entry(&ctx->tsgl_list, struct af_alg_tsgl, list); /* * Copy of AAD from source to destination @@ -811,7 +211,7 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, areq->first_rsgl.sgl.sg, processed); if (err) goto free; - aead_pull_tsgl(sk, processed, NULL, 0); + af_alg_pull_tsgl(sk, processed, NULL, 0); } else { /* * Decryption operation - To achieve an in-place cipher @@ -831,8 +231,8 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, goto free; /* Create TX SGL for tag and chain it to RX SGL. */ - areq->tsgl_entries = aead_count_tsgl(sk, processed, - processed - as); + areq->tsgl_entries = af_alg_count_tsgl(sk, processed, + processed - as); if (!areq->tsgl_entries) areq->tsgl_entries = 1; areq->tsgl = sock_kmalloc(sk, sizeof(*areq->tsgl) * @@ -845,12 +245,12 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, sg_init_table(areq->tsgl, areq->tsgl_entries); /* Release TX SGL, except for tag data and reassign tag data. */ - aead_pull_tsgl(sk, processed, areq->tsgl, processed - as); + af_alg_pull_tsgl(sk, processed, areq->tsgl, processed - as); /* chain the areq TX SGL holding the tag with RX SGL */ - if (last_rsgl) { + if (usedpages) { /* RX SGL present */ - struct af_alg_sgl *sgl_prev = &last_rsgl->sgl; + struct af_alg_sgl *sgl_prev = &areq->last_rsgl->sgl; sg_unmark_end(sgl_prev->sg + sgl_prev->npages - 1); sg_chain(sgl_prev->sg, sgl_prev->npages + 1, @@ -861,28 +261,28 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, } /* Initialize the crypto operation */ - aead_request_set_crypt(&areq->aead_req, src, + aead_request_set_crypt(&areq->cra_u.aead_req, src, areq->first_rsgl.sgl.sg, used, ctx->iv); - aead_request_set_ad(&areq->aead_req, ctx->aead_assoclen); - aead_request_set_tfm(&areq->aead_req, tfm); + aead_request_set_ad(&areq->cra_u.aead_req, ctx->aead_assoclen); + aead_request_set_tfm(&areq->cra_u.aead_req, tfm); if (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) { /* AIO operation */ areq->iocb = msg->msg_iocb; - aead_request_set_callback(&areq->aead_req, + aead_request_set_callback(&areq->cra_u.aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG, - aead_async_cb, areq); - err = ctx->enc ? crypto_aead_encrypt(&areq->aead_req) : - crypto_aead_decrypt(&areq->aead_req); + af_alg_async_cb, areq); + err = ctx->enc ? crypto_aead_encrypt(&areq->cra_u.aead_req) : + crypto_aead_decrypt(&areq->cra_u.aead_req); } else { /* Synchronous operation */ - aead_request_set_callback(&areq->aead_req, + aead_request_set_callback(&areq->cra_u.aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG, af_alg_complete, &ctx->completion); err = af_alg_wait_for_completion(ctx->enc ? - crypto_aead_encrypt(&areq->aead_req) : - crypto_aead_decrypt(&areq->aead_req), - &ctx->completion); + crypto_aead_encrypt(&areq->cra_u.aead_req) : + crypto_aead_decrypt(&areq->cra_u.aead_req), + &ctx->completion); } /* AIO operation in progress */ @@ -896,9 +296,8 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, } free: - aead_free_areq_sgls(areq); - if (areq) - sock_kfree_s(sk, areq, areqlen); + af_alg_free_areq_sgls(areq); + sock_kfree_s(sk, areq, areq->areqlen); return err ? err : outlen; } @@ -931,31 +330,11 @@ static int aead_recvmsg(struct socket *sock, struct msghdr *msg, } out: - aead_wmem_wakeup(sk); + af_alg_wmem_wakeup(sk); release_sock(sk); return ret; } -static unsigned int aead_poll(struct file *file, struct socket *sock, - poll_table *wait) -{ - struct sock *sk = sock->sk; - struct alg_sock *ask = alg_sk(sk); - struct aead_ctx *ctx = ask->private; - unsigned int mask; - - sock_poll_wait(file, sk_sleep(sk), wait); - mask = 0; - - if (!ctx->more) - mask |= POLLIN | POLLRDNORM; - - if (aead_writable(sk)) - mask |= POLLOUT | POLLWRNORM | POLLWRBAND; - - return mask; -} - static struct proto_ops algif_aead_ops = { .family = PF_ALG, @@ -973,9 +352,9 @@ static struct proto_ops algif_aead_ops = { .release = af_alg_release, .sendmsg = aead_sendmsg, - .sendpage = aead_sendpage, + .sendpage = af_alg_sendpage, .recvmsg = aead_recvmsg, - .poll = aead_poll, + .poll = af_alg_poll, }; static int aead_check_key(struct socket *sock) @@ -1037,7 +416,7 @@ static ssize_t aead_sendpage_nokey(struct socket *sock, struct page *page, if (err) return err; - return aead_sendpage(sock, page, offset, size, flags); + return af_alg_sendpage(sock, page, offset, size, flags); } static int aead_recvmsg_nokey(struct socket *sock, struct msghdr *msg, @@ -1071,7 +450,7 @@ static struct proto_ops algif_aead_ops_nokey = { .sendmsg = aead_sendmsg_nokey, .sendpage = aead_sendpage_nokey, .recvmsg = aead_recvmsg_nokey, - .poll = aead_poll, + .poll = af_alg_poll, }; static void *aead_bind(const char *name, u32 type, u32 mask) @@ -1132,14 +511,14 @@ static int aead_setkey(void *private, const u8 *key, unsigned int keylen) static void aead_sock_destruct(struct sock *sk) { struct alg_sock *ask = alg_sk(sk); - struct aead_ctx *ctx = ask->private; + struct af_alg_ctx *ctx = ask->private; struct sock *psk = ask->parent; struct alg_sock *pask = alg_sk(psk); struct aead_tfm *aeadc = pask->private; struct crypto_aead *tfm = aeadc->aead; unsigned int ivlen = crypto_aead_ivsize(tfm); - aead_pull_tsgl(sk, ctx->used, NULL, 0); + af_alg_pull_tsgl(sk, ctx->used, NULL, 0); crypto_put_default_null_skcipher2(); sock_kzfree_s(sk, ctx->iv, ivlen); sock_kfree_s(sk, ctx, ctx->len); @@ -1148,7 +527,7 @@ static void aead_sock_destruct(struct sock *sk) static int aead_accept_parent_nokey(void *private, struct sock *sk) { - struct aead_ctx *ctx; + struct af_alg_ctx *ctx; struct alg_sock *ask = alg_sk(sk); struct aead_tfm *tfm = private; struct crypto_aead *aead = tfm->aead; diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c index ce3b5fba..8ae4170a 100644 --- a/crypto/algif_skcipher.c +++ b/crypto/algif_skcipher.c @@ -33,320 +33,16 @@ #include #include #include -#include #include #include #include #include -struct skcipher_tsgl { - struct list_head list; - int cur; - struct scatterlist sg[0]; -}; - -struct skcipher_rsgl { - struct af_alg_sgl sgl; - struct list_head list; - size_t sg_num_bytes; -}; - -struct skcipher_async_req { - struct kiocb *iocb; - struct sock *sk; - - struct skcipher_rsgl first_sgl; - struct list_head rsgl_list; - - struct scatterlist *tsgl; - unsigned int tsgl_entries; - - unsigned int areqlen; - struct skcipher_request req; -}; - struct skcipher_tfm { struct crypto_skcipher *skcipher; bool has_key; }; -struct skcipher_ctx { - struct list_head tsgl_list; - - void *iv; - - struct af_alg_completion completion; - - size_t used; - size_t rcvused; - - bool more; - bool merge; - bool enc; - - unsigned int len; -}; - -#define MAX_SGL_ENTS ((4096 - sizeof(struct skcipher_tsgl)) / \ - sizeof(struct scatterlist) - 1) - -static inline int skcipher_sndbuf(struct sock *sk) -{ - struct alg_sock *ask = alg_sk(sk); - struct skcipher_ctx *ctx = ask->private; - - return max_t(int, max_t(int, sk->sk_sndbuf & PAGE_MASK, PAGE_SIZE) - - ctx->used, 0); -} - -static inline bool skcipher_writable(struct sock *sk) -{ - return PAGE_SIZE <= skcipher_sndbuf(sk); -} - -static inline int skcipher_rcvbuf(struct sock *sk) -{ - struct alg_sock *ask = alg_sk(sk); - struct skcipher_ctx *ctx = ask->private; - - return max_t(int, max_t(int, sk->sk_rcvbuf & PAGE_MASK, PAGE_SIZE) - - ctx->rcvused, 0); -} - -static inline bool skcipher_readable(struct sock *sk) -{ - return PAGE_SIZE <= skcipher_rcvbuf(sk); -} - -static int skcipher_alloc_tsgl(struct sock *sk) -{ - struct alg_sock *ask = alg_sk(sk); - struct skcipher_ctx *ctx = ask->private; - struct skcipher_tsgl *sgl; - struct scatterlist *sg = NULL; - - sgl = list_entry(ctx->tsgl_list.prev, struct skcipher_tsgl, list); - if (!list_empty(&ctx->tsgl_list)) - sg = sgl->sg; - - if (!sg || sgl->cur >= MAX_SGL_ENTS) { - sgl = sock_kmalloc(sk, sizeof(*sgl) + - sizeof(sgl->sg[0]) * (MAX_SGL_ENTS + 1), - GFP_KERNEL); - if (!sgl) - return -ENOMEM; - - sg_init_table(sgl->sg, MAX_SGL_ENTS + 1); - sgl->cur = 0; - - if (sg) - sg_chain(sg, MAX_SGL_ENTS + 1, sgl->sg); - - list_add_tail(&sgl->list, &ctx->tsgl_list); - } - - return 0; -} - -static unsigned int skcipher_count_tsgl(struct sock *sk, size_t bytes) -{ - struct alg_sock *ask = alg_sk(sk); - struct skcipher_ctx *ctx = ask->private; - struct skcipher_tsgl *sgl, *tmp; - unsigned int i; - unsigned int sgl_count = 0; - - if (!bytes) - return 0; - - list_for_each_entry_safe(sgl, tmp, &ctx->tsgl_list, list) { - struct scatterlist *sg = sgl->sg; - - for (i = 0; i < sgl->cur; i++) { - sgl_count++; - if (sg[i].length >= bytes) - return sgl_count; - - bytes -= sg[i].length; - } - } - - return sgl_count; -} - -static void skcipher_pull_tsgl(struct sock *sk, size_t used, - struct scatterlist *dst) -{ - struct alg_sock *ask = alg_sk(sk); - struct skcipher_ctx *ctx = ask->private; - struct skcipher_tsgl *sgl; - struct scatterlist *sg; - unsigned int i; - - while (!list_empty(&ctx->tsgl_list)) { - sgl = list_first_entry(&ctx->tsgl_list, struct skcipher_tsgl, - list); - sg = sgl->sg; - - for (i = 0; i < sgl->cur; i++) { - size_t plen = min_t(size_t, used, sg[i].length); - struct page *page = sg_page(sg + i); - - if (!page) - continue; - - /* - * Assumption: caller created skcipher_count_tsgl(len) - * SG entries in dst. - */ - if (dst) - sg_set_page(dst + i, page, plen, sg[i].offset); - - sg[i].length -= plen; - sg[i].offset += plen; - - used -= plen; - ctx->used -= plen; - - if (sg[i].length) - return; - - if (!dst) - put_page(page); - sg_assign_page(sg + i, NULL); - } - - list_del(&sgl->list); - sock_kfree_s(sk, sgl, sizeof(*sgl) + sizeof(sgl->sg[0]) * - (MAX_SGL_ENTS + 1)); - } - - if (!ctx->used) - ctx->merge = 0; -} - -static void skcipher_free_areq_sgls(struct skcipher_async_req *areq) -{ - struct sock *sk = areq->sk; - struct alg_sock *ask = alg_sk(sk); - struct skcipher_ctx *ctx = ask->private; - struct skcipher_rsgl *rsgl, *tmp; - struct scatterlist *tsgl; - struct scatterlist *sg; - unsigned int i; - - list_for_each_entry_safe(rsgl, tmp, &areq->rsgl_list, list) { - ctx->rcvused -= rsgl->sg_num_bytes; - af_alg_free_sg(&rsgl->sgl); - list_del(&rsgl->list); - if (rsgl != &areq->first_sgl) - sock_kfree_s(sk, rsgl, sizeof(*rsgl)); - } - - tsgl = areq->tsgl; - for_each_sg(tsgl, sg, areq->tsgl_entries, i) { - if (!sg_page(sg)) - continue; - put_page(sg_page(sg)); - } - - if (areq->tsgl && areq->tsgl_entries) - sock_kfree_s(sk, tsgl, areq->tsgl_entries * sizeof(*tsgl)); -} - -static int skcipher_wait_for_wmem(struct sock *sk, unsigned flags) -{ - DEFINE_WAIT_FUNC(wait, woken_wake_function); - int err = -ERESTARTSYS; - long timeout; - - if (flags & MSG_DONTWAIT) - return -EAGAIN; - - sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); - - add_wait_queue(sk_sleep(sk), &wait); - for (;;) { - if (signal_pending(current)) - break; - timeout = MAX_SCHEDULE_TIMEOUT; - if (sk_wait_event(sk, &timeout, skcipher_writable(sk), &wait)) { - err = 0; - break; - } - } - remove_wait_queue(sk_sleep(sk), &wait); - - return err; -} - -static void skcipher_wmem_wakeup(struct sock *sk) -{ - struct socket_wq *wq; - - if (!skcipher_writable(sk)) - return; - - rcu_read_lock(); - wq = rcu_dereference(sk->sk_wq); - if (skwq_has_sleeper(wq)) - wake_up_interruptible_sync_poll(&wq->wait, POLLIN | - POLLRDNORM | - POLLRDBAND); - sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); - rcu_read_unlock(); -} - -static int skcipher_wait_for_data(struct sock *sk, unsigned flags) -{ - DEFINE_WAIT_FUNC(wait, woken_wake_function); - struct alg_sock *ask = alg_sk(sk); - struct skcipher_ctx *ctx = ask->private; - long timeout; - int err = -ERESTARTSYS; - - if (flags & MSG_DONTWAIT) { - return -EAGAIN; - } - - sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); - - add_wait_queue(sk_sleep(sk), &wait); - for (;;) { - if (signal_pending(current)) - break; - timeout = MAX_SCHEDULE_TIMEOUT; - if (sk_wait_event(sk, &timeout, ctx->used, &wait)) { - err = 0; - break; - } - } - remove_wait_queue(sk_sleep(sk), &wait); - - sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); - - return err; -} - -static void skcipher_data_wakeup(struct sock *sk) -{ - struct alg_sock *ask = alg_sk(sk); - struct skcipher_ctx *ctx = ask->private; - struct socket_wq *wq; - - if (!ctx->used) - return; - - rcu_read_lock(); - wq = rcu_dereference(sk->sk_wq); - if (skwq_has_sleeper(wq)) - wake_up_interruptible_sync_poll(&wq->wait, POLLOUT | - POLLRDNORM | - POLLRDBAND); - sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT); - rcu_read_unlock(); -} - static int skcipher_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) { @@ -354,208 +50,11 @@ static int skcipher_sendmsg(struct socket *sock, struct msghdr *msg, struct alg_sock *ask = alg_sk(sk); struct sock *psk = ask->parent; struct alg_sock *pask = alg_sk(psk); - struct skcipher_ctx *ctx = ask->private; struct skcipher_tfm *skc = pask->private; struct crypto_skcipher *tfm = skc->skcipher; unsigned ivsize = crypto_skcipher_ivsize(tfm); - struct skcipher_tsgl *sgl; - struct af_alg_control con = {}; - long copied = 0; - bool enc = 0; - bool init = 0; - int err; - int i; - - if (msg->msg_controllen) { - err = af_alg_cmsg_send(msg, &con); - if (err) - return err; - - init = 1; - switch (con.op) { - case ALG_OP_ENCRYPT: - enc = 1; - break; - case ALG_OP_DECRYPT: - enc = 0; - break; - default: - return -EINVAL; - } - - if (con.iv && con.iv->ivlen != ivsize) - return -EINVAL; - } - - err = -EINVAL; - - lock_sock(sk); - if (!ctx->more && ctx->used) - goto unlock; - - if (init) { - ctx->enc = enc; - if (con.iv) - memcpy(ctx->iv, con.iv->iv, ivsize); - } - - while (size) { - struct scatterlist *sg; - unsigned long len = size; - size_t plen; - - if (ctx->merge) { - sgl = list_entry(ctx->tsgl_list.prev, - struct skcipher_tsgl, list); - sg = sgl->sg + sgl->cur - 1; - len = min_t(unsigned long, len, - PAGE_SIZE - sg->offset - sg->length); - - err = memcpy_from_msg(page_address(sg_page(sg)) + - sg->offset + sg->length, - msg, len); - if (err) - goto unlock; - - sg->length += len; - ctx->merge = (sg->offset + sg->length) & - (PAGE_SIZE - 1); - - ctx->used += len; - copied += len; - size -= len; - continue; - } - - if (!skcipher_writable(sk)) { - err = skcipher_wait_for_wmem(sk, msg->msg_flags); - if (err) - goto unlock; - } - - len = min_t(unsigned long, len, skcipher_sndbuf(sk)); - - err = skcipher_alloc_tsgl(sk); - if (err) - goto unlock; - - sgl = list_entry(ctx->tsgl_list.prev, struct skcipher_tsgl, - list); - sg = sgl->sg; - if (sgl->cur) - sg_unmark_end(sg + sgl->cur - 1); - do { - i = sgl->cur; - plen = min_t(size_t, len, PAGE_SIZE); - - sg_assign_page(sg + i, alloc_page(GFP_KERNEL)); - err = -ENOMEM; - if (!sg_page(sg + i)) - goto unlock; - - err = memcpy_from_msg(page_address(sg_page(sg + i)), - msg, plen); - if (err) { - __free_page(sg_page(sg + i)); - sg_assign_page(sg + i, NULL); - goto unlock; - } - - sg[i].length = plen; - len -= plen; - ctx->used += plen; - copied += plen; - size -= plen; - sgl->cur++; - } while (len && sgl->cur < MAX_SGL_ENTS); - - if (!size) - sg_mark_end(sg + sgl->cur - 1); - - ctx->merge = plen & (PAGE_SIZE - 1); - } - - err = 0; - - ctx->more = msg->msg_flags & MSG_MORE; - -unlock: - skcipher_data_wakeup(sk); - release_sock(sk); - return copied ?: err; -} - -static ssize_t skcipher_sendpage(struct socket *sock, struct page *page, - int offset, size_t size, int flags) -{ - struct sock *sk = sock->sk; - struct alg_sock *ask = alg_sk(sk); - struct skcipher_ctx *ctx = ask->private; - struct skcipher_tsgl *sgl; - int err = -EINVAL; - - if (flags & MSG_SENDPAGE_NOTLAST) - flags |= MSG_MORE; - - lock_sock(sk); - if (!ctx->more && ctx->used) - goto unlock; - - if (!size) - goto done; - - if (!skcipher_writable(sk)) { - err = skcipher_wait_for_wmem(sk, flags); - if (err) - goto unlock; - } - - err = skcipher_alloc_tsgl(sk); - if (err) - goto unlock; - - ctx->merge = 0; - sgl = list_entry(ctx->tsgl_list.prev, struct skcipher_tsgl, list); - - if (sgl->cur) - sg_unmark_end(sgl->sg + sgl->cur - 1); - - sg_mark_end(sgl->sg + sgl->cur); - get_page(page); - sg_set_page(sgl->sg + sgl->cur, page, size, offset); - sgl->cur++; - ctx->used += size; - -done: - ctx->more = flags & MSG_MORE; - -unlock: - skcipher_data_wakeup(sk); - release_sock(sk); - - return err ?: size; -} - -static void skcipher_async_cb(struct crypto_async_request *req, int err) -{ - struct skcipher_async_req *areq = req->data; - struct sock *sk = areq->sk; - struct kiocb *iocb = areq->iocb; - unsigned int resultlen; - - lock_sock(sk); - - /* Buffer size written by crypto operation. */ - resultlen = areq->req.cryptlen; - - skcipher_free_areq_sgls(areq); - sock_kfree_s(sk, areq, areq->areqlen); - __sock_put(sk); - - iocb->ki_complete(iocb, err ? err : resultlen, 0); - - release_sock(sk); + return af_alg_sendmsg(sock, msg, size, ivsize); } static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg, @@ -565,72 +64,24 @@ static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg, struct alg_sock *ask = alg_sk(sk); struct sock *psk = ask->parent; struct alg_sock *pask = alg_sk(psk); - struct skcipher_ctx *ctx = ask->private; + struct af_alg_ctx *ctx = ask->private; struct skcipher_tfm *skc = pask->private; struct crypto_skcipher *tfm = skc->skcipher; unsigned int bs = crypto_skcipher_blocksize(tfm); - unsigned int areqlen = sizeof(struct skcipher_async_req) + - crypto_skcipher_reqsize(tfm); - struct skcipher_async_req *areq; - struct skcipher_rsgl *last_rsgl = NULL; + struct af_alg_async_req *areq; int err = 0; size_t len = 0; /* Allocate cipher request for current operation. */ - areq = sock_kmalloc(sk, areqlen, GFP_KERNEL); - if (unlikely(!areq)) - return -ENOMEM; - areq->areqlen = areqlen; - areq->sk = sk; - INIT_LIST_HEAD(&areq->rsgl_list); - areq->tsgl = NULL; - areq->tsgl_entries = 0; + areq = af_alg_alloc_areq(sk, sizeof(struct af_alg_async_req) + + crypto_skcipher_reqsize(tfm)); + if (IS_ERR(areq)) + return PTR_ERR(areq); /* convert iovecs of output buffers into RX SGL */ - while (msg_data_left(msg)) { - struct skcipher_rsgl *rsgl; - size_t seglen; - - /* limit the amount of readable buffers */ - if (!skcipher_readable(sk)) - break; - - if (!ctx->used) { - err = skcipher_wait_for_data(sk, flags); - if (err) - goto free; - } - - seglen = min_t(size_t, ctx->used, msg_data_left(msg)); - - if (list_empty(&areq->rsgl_list)) { - rsgl = &areq->first_sgl; - } else { - rsgl = sock_kmalloc(sk, sizeof(*rsgl), GFP_KERNEL); - if (!rsgl) { - err = -ENOMEM; - goto free; - } - } - - rsgl->sgl.npages = 0; - list_add_tail(&rsgl->list, &areq->rsgl_list); - - /* make one iovec available as scatterlist */ - err = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, seglen); - if (err < 0) - goto free; - - /* chain the new scatterlist with previous one */ - if (last_rsgl) - af_alg_link_sg(&last_rsgl->sgl, &rsgl->sgl); - - last_rsgl = rsgl; - len += err; - ctx->rcvused += err; - rsgl->sg_num_bytes = err; - iov_iter_advance(&msg->msg_iter, err); - } + err = af_alg_get_rsgl(sk, msg, flags, areq, -1, &len); + if (err) + goto free; /* Process only as much RX buffers for which we have TX data */ if (len > ctx->used) @@ -647,7 +98,7 @@ static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg, * Create a per request TX SGL for this request which tracks the * SG entries from the global TX SGL. */ - areq->tsgl_entries = skcipher_count_tsgl(sk, len); + areq->tsgl_entries = af_alg_count_tsgl(sk, len, 0); if (!areq->tsgl_entries) areq->tsgl_entries = 1; areq->tsgl = sock_kmalloc(sk, sizeof(*areq->tsgl) * areq->tsgl_entries, @@ -657,44 +108,48 @@ static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg, goto free; } sg_init_table(areq->tsgl, areq->tsgl_entries); - skcipher_pull_tsgl(sk, len, areq->tsgl); + af_alg_pull_tsgl(sk, len, areq->tsgl, 0); /* Initialize the crypto operation */ - skcipher_request_set_tfm(&areq->req, tfm); - skcipher_request_set_crypt(&areq->req, areq->tsgl, - areq->first_sgl.sgl.sg, len, ctx->iv); + skcipher_request_set_tfm(&areq->cra_u.skcipher_req, tfm); + skcipher_request_set_crypt(&areq->cra_u.skcipher_req, areq->tsgl, + areq->first_rsgl.sgl.sg, len, ctx->iv); if (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) { /* AIO operation */ areq->iocb = msg->msg_iocb; - skcipher_request_set_callback(&areq->req, + skcipher_request_set_callback(&areq->cra_u.skcipher_req, CRYPTO_TFM_REQ_MAY_SLEEP, - skcipher_async_cb, areq); - err = ctx->enc ? crypto_skcipher_encrypt(&areq->req) : - crypto_skcipher_decrypt(&areq->req); + af_alg_async_cb, areq); + err = ctx->enc ? + crypto_skcipher_encrypt(&areq->cra_u.skcipher_req) : + crypto_skcipher_decrypt(&areq->cra_u.skcipher_req); } else { /* Synchronous operation */ - skcipher_request_set_callback(&areq->req, + skcipher_request_set_callback(&areq->cra_u.skcipher_req, CRYPTO_TFM_REQ_MAY_SLEEP | CRYPTO_TFM_REQ_MAY_BACKLOG, af_alg_complete, &ctx->completion); err = af_alg_wait_for_completion(ctx->enc ? - crypto_skcipher_encrypt(&areq->req) : - crypto_skcipher_decrypt(&areq->req), + crypto_skcipher_encrypt(&areq->cra_u.skcipher_req) : + crypto_skcipher_decrypt(&areq->cra_u.skcipher_req), &ctx->completion); } /* AIO operation in progress */ if (err == -EINPROGRESS) { sock_hold(sk); + + /* Remember output size that will be generated. */ + areq->outlen = len; + return -EIOCBQUEUED; } free: - skcipher_free_areq_sgls(areq); - if (areq) - sock_kfree_s(sk, areq, areqlen); + af_alg_free_areq_sgls(areq); + sock_kfree_s(sk, areq, areq->areqlen); return err ? err : len; } @@ -727,30 +182,11 @@ static int skcipher_recvmsg(struct socket *sock, struct msghdr *msg, } out: - skcipher_wmem_wakeup(sk); + af_alg_wmem_wakeup(sk); release_sock(sk); return ret; } -static unsigned int skcipher_poll(struct file *file, struct socket *sock, - poll_table *wait) -{ - struct sock *sk = sock->sk; - struct alg_sock *ask = alg_sk(sk); - struct skcipher_ctx *ctx = ask->private; - unsigned int mask; - - sock_poll_wait(file, sk_sleep(sk), wait); - mask = 0; - - if (ctx->used) - mask |= POLLIN | POLLRDNORM; - - if (skcipher_writable(sk)) - mask |= POLLOUT | POLLWRNORM | POLLWRBAND; - - return mask; -} static struct proto_ops algif_skcipher_ops = { .family = PF_ALG, @@ -769,9 +205,9 @@ static struct proto_ops algif_skcipher_ops = { .release = af_alg_release, .sendmsg = skcipher_sendmsg, - .sendpage = skcipher_sendpage, + .sendpage = af_alg_sendpage, .recvmsg = skcipher_recvmsg, - .poll = skcipher_poll, + .poll = af_alg_poll, }; static int skcipher_check_key(struct socket *sock) @@ -833,7 +269,7 @@ static ssize_t skcipher_sendpage_nokey(struct socket *sock, struct page *page, if (err) return err; - return skcipher_sendpage(sock, page, offset, size, flags); + return af_alg_sendpage(sock, page, offset, size, flags); } static int skcipher_recvmsg_nokey(struct socket *sock, struct msghdr *msg, @@ -867,7 +303,7 @@ static struct proto_ops algif_skcipher_ops_nokey = { .sendmsg = skcipher_sendmsg_nokey, .sendpage = skcipher_sendpage_nokey, .recvmsg = skcipher_recvmsg_nokey, - .poll = skcipher_poll, + .poll = af_alg_poll, }; static void *skcipher_bind(const char *name, u32 type, u32 mask) @@ -912,13 +348,13 @@ static int skcipher_setkey(void *private, const u8 *key, unsigned int keylen) static void skcipher_sock_destruct(struct sock *sk) { struct alg_sock *ask = alg_sk(sk); - struct skcipher_ctx *ctx = ask->private; + struct af_alg_ctx *ctx = ask->private; struct sock *psk = ask->parent; struct alg_sock *pask = alg_sk(psk); struct skcipher_tfm *skc = pask->private; struct crypto_skcipher *tfm = skc->skcipher; - skcipher_pull_tsgl(sk, ctx->used, NULL); + af_alg_pull_tsgl(sk, ctx->used, NULL, 0); sock_kzfree_s(sk, ctx->iv, crypto_skcipher_ivsize(tfm)); sock_kfree_s(sk, ctx, ctx->len); af_alg_release_parent(sk); @@ -926,7 +362,7 @@ static void skcipher_sock_destruct(struct sock *sk) static int skcipher_accept_parent_nokey(void *private, struct sock *sk) { - struct skcipher_ctx *ctx; + struct af_alg_ctx *ctx; struct alg_sock *ask = alg_sk(sk); struct skcipher_tfm *tfm = private; struct crypto_skcipher *skcipher = tfm->skcipher; -- cgit v1.2.3 From 21d8dbe613a078662dd6f0fb834bb29673b237b5 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 14 Aug 2017 14:28:14 +0100 Subject: crypto: chacha20 - fix handling of chunked input Commit baef71ce346b ("crypto: chacha20 - convert generic and x86 versions to skcipher") ported the existing chacha20 code to use the new skcipher API, and introduced a bug along the way. Unfortunately, the tcrypt tests did not catch the error, and it was only found recently by Tobias. Stefan kindly diagnosed the error, and proposed a fix which is similar to the one below, with the exception that 'walk.stride' is used rather than the hardcoded block size. This does not actually matter in this case, but it's a better example of how to use the skcipher walk API. Fixes: baef71ce346b ("crypto: chacha20 - convert generic and x86 ...") Cc: # v4.11+ Cc: Steffen Klassert Reported-by: Tobias Brunner Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/chacha20_generic.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crypto/chacha20_generic.c b/crypto/chacha20_generic.c index 8b3c04d6..4a45fa48 100644 --- a/crypto/chacha20_generic.c +++ b/crypto/chacha20_generic.c @@ -91,9 +91,14 @@ int crypto_chacha20_crypt(struct skcipher_request *req) crypto_chacha20_init(state, ctx, walk.iv); while (walk.nbytes > 0) { + unsigned int nbytes = walk.nbytes; + + if (nbytes < walk.total) + nbytes = round_down(nbytes, walk.stride); + chacha20_docrypt(state, walk.dst.virt.addr, walk.src.virt.addr, - walk.nbytes); - err = skcipher_walk_done(&walk, 0); + nbytes); + err = skcipher_walk_done(&walk, walk.nbytes - nbytes); } return err; -- cgit v1.2.3 From 9f05533bf0bb1efd537387be541e8c617827ba1e Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 14 Aug 2017 14:28:15 +0100 Subject: crypto: testmgr - add chunked test cases for chacha20 We failed to catch a bug in the chacha20 code after porting it to the skcipher API. We would have caught it if any chunked tests had been defined, so define some now so we will catch future regressions. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/testmgr.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 6ceb0e27..d54971d2 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -32675,6 +32675,10 @@ static const struct cipher_testvec chacha20_enc_tv_template[] = { "\x5b\x86\x2f\x37\x30\xe3\x7c\xfd" "\xc4\xfd\x80\x6c\x22\xf2\x21", .rlen = 375, + .also_non_np = 1, + .np = 3, + .tap = { 375 - 20, 4, 16 }, + }, { /* RFC7539 A.2. Test Vector #3 */ .key = "\x1c\x92\x40\xa5\xeb\x55\xd3\x8a" "\xf3\x33\x88\x86\x04\xf6\xb5\xf0" @@ -33049,6 +33053,9 @@ static const struct cipher_testvec chacha20_enc_tv_template[] = { "\xa1\xed\xad\xd5\x76\xfa\x24\x8f" "\x98", .rlen = 1281, + .also_non_np = 1, + .np = 3, + .tap = { 1200, 1, 80 }, }, }; -- cgit v1.2.3 From 4f673910a79f5cb0e7dd87bf807eebb24834cea3 Mon Sep 17 00:00:00 2001 From: Stephan Mueller Date: Wed, 16 Aug 2017 11:56:24 +0200 Subject: crypto: algif_skcipher - only call put_page on referenced and used pages For asynchronous operation, SGs are allocated without a page mapped to them or with a page that is not used (ref-counted). If the SGL is freed, the code must only call put_page for an SG if there was a page assigned and ref-counted in the first place. This fixes a kernel crash when using io_submit with more than one iocb using the sendmsg and sendpage (vmsplice/splice) interface. Cc: Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/algif_skcipher.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c index 43839b00..903605db 100644 --- a/crypto/algif_skcipher.c +++ b/crypto/algif_skcipher.c @@ -87,8 +87,13 @@ static void skcipher_free_async_sgls(struct skcipher_async_req *sreq) } sgl = sreq->tsg; n = sg_nents(sgl); - for_each_sg(sgl, sg, n, i) - put_page(sg_page(sg)); + for_each_sg(sgl, sg, n, i) { + struct page *page = sg_page(sg); + + /* some SGs may not have a page mapped */ + if (page && page_ref_count(page)) + put_page(page); + } kfree(sreq->tsg); } -- cgit v1.2.3 From 33463ce004e82a1acac0b8cffd5c7f1d247cebdd Mon Sep 17 00:00:00 2001 From: Stephan Mueller Date: Wed, 9 Aug 2017 16:20:00 +0200 Subject: crypto: algif_aead - fix comment regarding memory layout Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/algif_aead.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c index 48d46e74..516b38c3 100644 --- a/crypto/algif_aead.c +++ b/crypto/algif_aead.c @@ -201,11 +201,11 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, * Encryption operation - The in-place cipher operation is * achieved by the following operation: * - * TX SGL: AAD || PT || Tag + * TX SGL: AAD || PT * | | * | copy | * v v - * RX SGL: AAD || PT + * RX SGL: AAD || PT || Tag */ err = crypto_aead_copy_sgl(null_tfm, tsgl->sg, areq->first_rsgl.sgl.sg, processed); -- cgit v1.2.3 From 1bb32d90974ab4f5f20d6a636cef18a3bc2c97f1 Mon Sep 17 00:00:00 2001 From: Rabin Vincent Date: Thu, 10 Aug 2017 14:53:52 +0200 Subject: crypto: hash - add crypto_(un)register_ahashes() There are already helpers to (un)register multiple normal and AEAD algos. Add one for ahashes too. Signed-off-by: Lars Persson Signed-off-by: Rabin Vincent Signed-off-by: Herbert Xu --- crypto/ahash.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/crypto/ahash.c b/crypto/ahash.c index 826cd7ab..5e8666e6 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -588,6 +588,35 @@ int crypto_unregister_ahash(struct ahash_alg *alg) } EXPORT_SYMBOL_GPL(crypto_unregister_ahash); +int crypto_register_ahashes(struct ahash_alg *algs, int count) +{ + int i, ret; + + for (i = 0; i < count; i++) { + ret = crypto_register_ahash(&algs[i]); + if (ret) + goto err; + } + + return 0; + +err: + for (--i; i >= 0; --i) + crypto_unregister_ahash(&algs[i]); + + return ret; +} +EXPORT_SYMBOL_GPL(crypto_register_ahashes); + +void crypto_unregister_ahashes(struct ahash_alg *algs, int count) +{ + int i; + + for (i = count - 1; i >= 0; --i) + crypto_unregister_ahash(&algs[i]); +} +EXPORT_SYMBOL_GPL(crypto_unregister_ahashes); + int ahash_register_instance(struct crypto_template *tmpl, struct ahash_instance *inst) { -- cgit v1.2.3 From fc628d5460223588ea5aff6e038655f2f910e2dd Mon Sep 17 00:00:00 2001 From: Stephan Mueller Date: Thu, 10 Aug 2017 16:40:03 +0200 Subject: crypto: af_alg - get_page upon reassignment to TX SGL When a page is assigned to a TX SGL, call get_page to increment the reference counter. It is possible that one page is referenced in multiple SGLs: - in the global TX SGL in case a previous af_alg_pull_tsgl only reassigned parts of a page to a per-request TX SGL - in the per-request TX SGL as assigned by af_alg_pull_tsgl Note, multiple requests can be active at the same time whose TX SGLs all point to different parts of the same page. Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/af_alg.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crypto/af_alg.c b/crypto/af_alg.c index d6936c0e..ffa9f4cc 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -641,9 +641,9 @@ void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst, if (dst_offset >= plen) { /* discard page before offset */ dst_offset -= plen; - put_page(page); } else { /* reassign page to dst after offset */ + get_page(page); sg_set_page(dst + j, page, plen - dst_offset, sg[i].offset + dst_offset); @@ -661,9 +661,7 @@ void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst, if (sg[i].length) return; - if (!dst) - put_page(page); - + put_page(page); sg_assign_page(sg + i, NULL); } -- cgit v1.2.3 From dcec1f1c58841a6dfdd8898674bd211ea4525331 Mon Sep 17 00:00:00 2001 From: Stephan Mueller Date: Thu, 14 Sep 2017 17:10:28 +0200 Subject: crypto: drbg - fix freeing of resources During the change to use aligned buffers, the deallocation code path was not updated correctly. The current code tries to free the aligned buffer pointer and not the original buffer pointer as it is supposed to. Thus, the code is updated to free the original buffer pointer and set the aligned buffer pointer that is used throughout the code to NULL. Fixes: faba028f3f7b4 ("crypto: drbg - use aligned buffers") CC: CC: Herbert Xu Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/drbg.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crypto/drbg.c b/crypto/drbg.c index 633a88e9..70018397 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -1133,10 +1133,10 @@ static inline void drbg_dealloc_state(struct drbg_state *drbg) { if (!drbg) return; - kzfree(drbg->V); - drbg->Vbuf = NULL; - kzfree(drbg->C); - drbg->Cbuf = NULL; + kzfree(drbg->Vbuf); + drbg->V = NULL; + kzfree(drbg->Cbuf); + drbg->C = NULL; kzfree(drbg->scratchpadbuf); drbg->scratchpadbuf = NULL; drbg->reseed_ctr = 0; -- cgit v1.2.3 From 508f8d23a372beae647185ce896a0c1109406b83 Mon Sep 17 00:00:00 2001 From: Stephan Mueller Date: Wed, 30 Aug 2017 09:17:39 +0200 Subject: crypto: af_alg - update correct dst SGL entry When two adjacent TX SGL are processed and parts of both TX SGLs are pulled into the per-request TX SGL, the wrong per-request TX SGL entries were updated. This fixes a NULL pointer dereference when a cipher implementation walks the TX SGL where some of the SGL entries were NULL. Fixes: 8a15a4bf1947 ("crypto: algif_skcipher - overhaul memory...") Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/af_alg.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/af_alg.c b/crypto/af_alg.c index ffa9f4cc..337cf382 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -619,14 +619,14 @@ void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst, struct af_alg_ctx *ctx = ask->private; struct af_alg_tsgl *sgl; struct scatterlist *sg; - unsigned int i, j; + unsigned int i, j = 0; while (!list_empty(&ctx->tsgl_list)) { sgl = list_first_entry(&ctx->tsgl_list, struct af_alg_tsgl, list); sg = sgl->sg; - for (i = 0, j = 0; i < sgl->cur; i++) { + for (i = 0; i < sgl->cur; i++) { size_t plen = min_t(size_t, used, sg[i].length); struct page *page = sg_page(sg + i); -- cgit v1.2.3