diff options
47 files changed, 777 insertions, 656 deletions
diff --git a/crypto/Kconfig b/crypto/Kconfig index ac5fb37e..f7911963 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -860,6 +860,17 @@ config CRYPTO_SHA3 References: http://keccak.noekeon.org/ +config CRYPTO_SM3 + tristate "SM3 digest algorithm" + select CRYPTO_HASH + help + SM3 secure hash function as defined by OSCCA GM/T 0004-2012 SM3). + It is part of the Chinese Commercial Cryptography suite. + + References: + http://www.oscca.gov.cn/UpFile/20101222141857786.pdf + https://datatracker.ietf.org/doc/html/draft-shen-sm3-hash + config CRYPTO_TGR192 tristate "Tiger digest algorithms" select CRYPTO_HASH diff --git a/crypto/Makefile b/crypto/Makefile index da190be6..d674884b 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -71,6 +71,7 @@ obj-$(CONFIG_CRYPTO_SHA1) += sha1_generic.o obj-$(CONFIG_CRYPTO_SHA256) += sha256_generic.o obj-$(CONFIG_CRYPTO_SHA512) += sha512_generic.o obj-$(CONFIG_CRYPTO_SHA3) += sha3_generic.o +obj-$(CONFIG_CRYPTO_SM3) += sm3_generic.o obj-$(CONFIG_CRYPTO_WP512) += wp512.o CFLAGS_wp512.o := $(call cc-option,-fno-schedule-insns) # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=79149 obj-$(CONFIG_CRYPTO_TGR192) += tgr192.o diff --git a/crypto/af_alg.c b/crypto/af_alg.c index 337cf382..35d4dcea 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -481,33 +481,6 @@ int af_alg_cmsg_send(struct msghdr *msg, struct af_alg_control *con) } EXPORT_SYMBOL_GPL(af_alg_cmsg_send); -int af_alg_wait_for_completion(int err, struct af_alg_completion *completion) -{ - switch (err) { - case -EINPROGRESS: - case -EBUSY: - wait_for_completion(&completion->completion); - reinit_completion(&completion->completion); - err = completion->err; - break; - }; - - return err; -} -EXPORT_SYMBOL_GPL(af_alg_wait_for_completion); - -void af_alg_complete(struct crypto_async_request *req, int err) -{ - struct af_alg_completion *completion = req->data; - - if (err == -EINPROGRESS) - return; - - completion->err = err; - complete(&completion->completion); -} -EXPORT_SYMBOL_GPL(af_alg_complete); - /** * af_alg_alloc_tsgl - allocate the TX SGL * @@ -691,7 +664,7 @@ void af_alg_free_areq_sgls(struct af_alg_async_req *areq) unsigned int i; list_for_each_entry_safe(rsgl, tmp, &areq->rsgl_list, list) { - ctx->rcvused -= rsgl->sg_num_bytes; + atomic_sub(rsgl->sg_num_bytes, &ctx->rcvused); af_alg_free_sg(&rsgl->sgl); list_del(&rsgl->list); if (rsgl != &areq->first_rsgl) @@ -699,14 +672,15 @@ void af_alg_free_areq_sgls(struct af_alg_async_req *areq) } tsgl = areq->tsgl; - for_each_sg(tsgl, sg, areq->tsgl_entries, i) { - if (!sg_page(sg)) - continue; - put_page(sg_page(sg)); - } + if (tsgl) { + for_each_sg(tsgl, sg, areq->tsgl_entries, i) { + if (!sg_page(sg)) + continue; + put_page(sg_page(sg)); + } - if (areq->tsgl && areq->tsgl_entries) sock_kfree_s(sk, tsgl, areq->tsgl_entries * sizeof(*tsgl)); + } } EXPORT_SYMBOL_GPL(af_alg_free_areq_sgls); @@ -1048,6 +1022,18 @@ unlock: EXPORT_SYMBOL_GPL(af_alg_sendpage); /** + * af_alg_free_resources - release resources required for crypto request + */ +void af_alg_free_resources(struct af_alg_async_req *areq) +{ + struct sock *sk = areq->sk; + + af_alg_free_areq_sgls(areq); + sock_kfree_s(sk, areq, areq->areqlen); +} +EXPORT_SYMBOL_GPL(af_alg_free_resources); + +/** * af_alg_async_cb - AIO callback handler * * This handler cleans up the struct af_alg_async_req upon completion of the @@ -1063,18 +1049,13 @@ void af_alg_async_cb(struct crypto_async_request *_req, int err) struct kiocb *iocb = areq->iocb; unsigned int resultlen; - lock_sock(sk); - /* Buffer size written by crypto operation. */ resultlen = areq->outlen; - af_alg_free_areq_sgls(areq); - sock_kfree_s(sk, areq, areq->areqlen); - __sock_put(sk); + af_alg_free_resources(areq); + sock_put(sk); iocb->ki_complete(iocb, err ? err : resultlen, 0); - - release_sock(sk); } EXPORT_SYMBOL_GPL(af_alg_async_cb); @@ -1157,12 +1138,6 @@ int af_alg_get_rsgl(struct sock *sk, struct msghdr *msg, int flags, if (!af_alg_readable(sk)) break; - if (!ctx->used) { - err = af_alg_wait_for_data(sk, flags); - if (err) - return err; - } - seglen = min_t(size_t, (maxsize - len), msg_data_left(msg)); @@ -1188,7 +1163,7 @@ int af_alg_get_rsgl(struct sock *sk, struct msghdr *msg, int flags, areq->last_rsgl = rsgl; len += err; - ctx->rcvused += err; + atomic_add(err, &ctx->rcvused); rsgl->sg_num_bytes = err; iov_iter_advance(&msg->msg_iter, err); } diff --git a/crypto/ahash.c b/crypto/ahash.c index 5e8666e6..3a35d67d 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -334,9 +334,7 @@ static int ahash_op_unaligned(struct ahash_request *req, return err; err = op(req); - if (err == -EINPROGRESS || - (err == -EBUSY && (ahash_request_flags(req) & - CRYPTO_TFM_REQ_MAY_BACKLOG))) + if (err == -EINPROGRESS || err == -EBUSY) return err; ahash_restore_req(req, err); @@ -394,9 +392,7 @@ static int ahash_def_finup_finish1(struct ahash_request *req, int err) req->base.complete = ahash_def_finup_done2; err = crypto_ahash_reqtfm(req)->final(req); - if (err == -EINPROGRESS || - (err == -EBUSY && (ahash_request_flags(req) & - CRYPTO_TFM_REQ_MAY_BACKLOG))) + if (err == -EINPROGRESS || err == -EBUSY) return err; out: @@ -432,9 +428,7 @@ static int ahash_def_finup(struct ahash_request *req) return err; err = tfm->update(req); - if (err == -EINPROGRESS || - (err == -EBUSY && (ahash_request_flags(req) & - CRYPTO_TFM_REQ_MAY_BACKLOG))) + if (err == -EINPROGRESS || err == -EBUSY) return err; return ahash_def_finup_finish1(req, err); diff --git a/crypto/algapi.c b/crypto/algapi.c index aa699ff6..9a636f96 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -167,6 +167,18 @@ void crypto_remove_spawns(struct crypto_alg *alg, struct list_head *list, spawn->alg = NULL; spawns = &inst->alg.cra_users; + + /* + * We may encounter an unregistered instance here, since + * an instance's spawns are set up prior to the instance + * being registered. An unregistered instance will have + * NULL ->cra_users.next, since ->cra_users isn't + * properly initialized until registration. But an + * unregistered instance cannot have any users, so treat + * it the same as ->cra_users being empty. + */ + if (spawns->next == NULL) + break; } } while ((spawns = crypto_more_spawns(alg, &stack, &top, &secondary_spawns))); @@ -897,9 +909,11 @@ int crypto_enqueue_request(struct crypto_queue *queue, int err = -EINPROGRESS; if (unlikely(queue->qlen >= queue->max_qlen)) { - err = -EBUSY; - if (!(request->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + if (!(request->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { + err = -ENOSPC; goto out; + } + err = -EBUSY; if (queue->backlog == &queue->list) queue->backlog = &request->list; } diff --git a/crypto/algboss.c b/crypto/algboss.c index 960d8548..5e6df2a0 100644 --- a/crypto/algboss.c +++ b/crypto/algboss.c @@ -122,7 +122,6 @@ static int cryptomgr_schedule_probe(struct crypto_larval *larval) int notnum = 0; name = ++p; - len = 0; for (; isalnum(*p) || *p == '-' || *p == '_'; p++) notnum |= !isdigit(*p); diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c index 516b38c3..e9885a35 100644 --- a/crypto/algif_aead.c +++ b/crypto/algif_aead.c @@ -101,16 +101,22 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, struct aead_tfm *aeadc = pask->private; struct crypto_aead *tfm = aeadc->aead; struct crypto_skcipher *null_tfm = aeadc->null_tfm; - unsigned int as = crypto_aead_authsize(tfm); + unsigned int i, as = crypto_aead_authsize(tfm); struct af_alg_async_req *areq; - struct af_alg_tsgl *tsgl; - struct scatterlist *src; + struct af_alg_tsgl *tsgl, *tmp; + struct scatterlist *rsgl_src, *tsgl_src = NULL; int err = 0; size_t used = 0; /* [in] TX bufs to be en/decrypted */ size_t outlen = 0; /* [out] RX bufs produced by kernel */ size_t usedpages = 0; /* [in] RX bufs to be used from user */ size_t processed = 0; /* [in] TX bufs to be consumed */ + if (!ctx->used) { + err = af_alg_wait_for_data(sk, flags); + if (err) + return err; + } + /* * Data length provided by caller via sendmsg/sendpage that has not * yet been processed. @@ -178,7 +184,22 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, } processed = used + ctx->aead_assoclen; - tsgl = list_first_entry(&ctx->tsgl_list, struct af_alg_tsgl, list); + list_for_each_entry_safe(tsgl, tmp, &ctx->tsgl_list, list) { + for (i = 0; i < tsgl->cur; i++) { + struct scatterlist *process_sg = tsgl->sg + i; + + if (!(process_sg->length) || !sg_page(process_sg)) + continue; + tsgl_src = process_sg; + break; + } + if (tsgl_src) + break; + } + if (processed && !tsgl_src) { + err = -EFAULT; + goto free; + } /* * Copy of AAD from source to destination @@ -194,7 +215,7 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, */ /* Use the RX SGL as source (and destination) for crypto op. */ - src = areq->first_rsgl.sgl.sg; + rsgl_src = areq->first_rsgl.sgl.sg; if (ctx->enc) { /* @@ -207,7 +228,7 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, * v v * RX SGL: AAD || PT || Tag */ - err = crypto_aead_copy_sgl(null_tfm, tsgl->sg, + err = crypto_aead_copy_sgl(null_tfm, tsgl_src, areq->first_rsgl.sgl.sg, processed); if (err) goto free; @@ -225,7 +246,7 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, */ /* Copy AAD || CT to RX SGL buffer for in-place operation. */ - err = crypto_aead_copy_sgl(null_tfm, tsgl->sg, + err = crypto_aead_copy_sgl(null_tfm, tsgl_src, areq->first_rsgl.sgl.sg, outlen); if (err) goto free; @@ -257,47 +278,48 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, areq->tsgl); } else /* no RX SGL present (e.g. authentication only) */ - src = areq->tsgl; + rsgl_src = areq->tsgl; } /* Initialize the crypto operation */ - aead_request_set_crypt(&areq->cra_u.aead_req, src, + aead_request_set_crypt(&areq->cra_u.aead_req, rsgl_src, areq->first_rsgl.sgl.sg, used, ctx->iv); aead_request_set_ad(&areq->cra_u.aead_req, ctx->aead_assoclen); aead_request_set_tfm(&areq->cra_u.aead_req, tfm); if (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) { /* AIO operation */ + sock_hold(sk); areq->iocb = msg->msg_iocb; + + /* Remember output size that will be generated. */ + areq->outlen = outlen; + aead_request_set_callback(&areq->cra_u.aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG, af_alg_async_cb, areq); err = ctx->enc ? crypto_aead_encrypt(&areq->cra_u.aead_req) : crypto_aead_decrypt(&areq->cra_u.aead_req); + + /* AIO operation in progress */ + if (err == -EINPROGRESS || err == -EBUSY) + return -EIOCBQUEUED; + + sock_put(sk); } else { /* Synchronous operation */ aead_request_set_callback(&areq->cra_u.aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG, - af_alg_complete, &ctx->completion); - err = af_alg_wait_for_completion(ctx->enc ? + crypto_req_done, &ctx->wait); + err = crypto_wait_req(ctx->enc ? crypto_aead_encrypt(&areq->cra_u.aead_req) : crypto_aead_decrypt(&areq->cra_u.aead_req), - &ctx->completion); + &ctx->wait); } - /* AIO operation in progress */ - if (err == -EINPROGRESS) { - sock_hold(sk); - - /* Remember output size that will be generated. */ - areq->outlen = outlen; - - return -EIOCBQUEUED; - } free: - af_alg_free_areq_sgls(areq); - sock_kfree_s(sk, areq, areq->areqlen); + af_alg_free_resources(areq); return err ? err : outlen; } @@ -487,6 +509,7 @@ static void aead_release(void *private) struct aead_tfm *tfm = private; crypto_free_aead(tfm->aead); + crypto_put_default_null_skcipher2(); kfree(tfm); } @@ -519,7 +542,6 @@ static void aead_sock_destruct(struct sock *sk) unsigned int ivlen = crypto_aead_ivsize(tfm); af_alg_pull_tsgl(sk, ctx->used, NULL, 0); - crypto_put_default_null_skcipher2(); sock_kzfree_s(sk, ctx->iv, ivlen); sock_kfree_s(sk, ctx, ctx->len); af_alg_release_parent(sk); @@ -549,12 +571,12 @@ static int aead_accept_parent_nokey(void *private, struct sock *sk) INIT_LIST_HEAD(&ctx->tsgl_list); ctx->len = len; ctx->used = 0; - ctx->rcvused = 0; + atomic_set(&ctx->rcvused, 0); ctx->more = 0; ctx->merge = 0; ctx->enc = 0; ctx->aead_assoclen = 0; - af_alg_init_completion(&ctx->completion); + crypto_init_wait(&ctx->wait); ask->private = ctx; diff --git a/crypto/algif_hash.c b/crypto/algif_hash.c index 5e92bd27..76d2e716 100644 --- a/crypto/algif_hash.c +++ b/crypto/algif_hash.c @@ -26,7 +26,7 @@ struct hash_ctx { u8 *result; - struct af_alg_completion completion; + struct crypto_wait wait; unsigned int len; bool more; @@ -88,8 +88,7 @@ static int hash_sendmsg(struct socket *sock, struct msghdr *msg, if ((msg->msg_flags & MSG_MORE)) hash_free_result(sk, ctx); - err = af_alg_wait_for_completion(crypto_ahash_init(&ctx->req), - &ctx->completion); + err = crypto_wait_req(crypto_ahash_init(&ctx->req), &ctx->wait); if (err) goto unlock; } @@ -110,8 +109,8 @@ static int hash_sendmsg(struct socket *sock, struct msghdr *msg, ahash_request_set_crypt(&ctx->req, ctx->sgl.sg, NULL, len); - err = af_alg_wait_for_completion(crypto_ahash_update(&ctx->req), - &ctx->completion); + err = crypto_wait_req(crypto_ahash_update(&ctx->req), + &ctx->wait); af_alg_free_sg(&ctx->sgl); if (err) goto unlock; @@ -129,8 +128,8 @@ static int hash_sendmsg(struct socket *sock, struct msghdr *msg, goto unlock; ahash_request_set_crypt(&ctx->req, NULL, ctx->result, 0); - err = af_alg_wait_for_completion(crypto_ahash_final(&ctx->req), - &ctx->completion); + err = crypto_wait_req(crypto_ahash_final(&ctx->req), + &ctx->wait); } unlock: @@ -171,7 +170,7 @@ static ssize_t hash_sendpage(struct socket *sock, struct page *page, } else { if (!ctx->more) { err = crypto_ahash_init(&ctx->req); - err = af_alg_wait_for_completion(err, &ctx->completion); + err = crypto_wait_req(err, &ctx->wait); if (err) goto unlock; } @@ -179,7 +178,7 @@ static ssize_t hash_sendpage(struct socket *sock, struct page *page, err = crypto_ahash_update(&ctx->req); } - err = af_alg_wait_for_completion(err, &ctx->completion); + err = crypto_wait_req(err, &ctx->wait); if (err) goto unlock; @@ -215,17 +214,16 @@ static int hash_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, ahash_request_set_crypt(&ctx->req, NULL, ctx->result, 0); if (!result && !ctx->more) { - err = af_alg_wait_for_completion( - crypto_ahash_init(&ctx->req), - &ctx->completion); + err = crypto_wait_req(crypto_ahash_init(&ctx->req), + &ctx->wait); if (err) goto unlock; } if (!result || ctx->more) { ctx->more = 0; - err = af_alg_wait_for_completion(crypto_ahash_final(&ctx->req), - &ctx->completion); + err = crypto_wait_req(crypto_ahash_final(&ctx->req), + &ctx->wait); if (err) goto unlock; } @@ -476,13 +474,13 @@ static int hash_accept_parent_nokey(void *private, struct sock *sk) ctx->result = NULL; ctx->len = len; ctx->more = 0; - af_alg_init_completion(&ctx->completion); + crypto_init_wait(&ctx->wait); ask->private = ctx; ahash_request_set_tfm(&ctx->req, hash); ahash_request_set_callback(&ctx->req, CRYPTO_TFM_REQ_MAY_BACKLOG, - af_alg_complete, &ctx->completion); + crypto_req_done, &ctx->wait); sk->sk_destruct = hash_sock_destruct; diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c index 8ae4170a..c5c47b68 100644 --- a/crypto/algif_skcipher.c +++ b/crypto/algif_skcipher.c @@ -72,6 +72,12 @@ static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg, int err = 0; size_t len = 0; + if (!ctx->used) { + err = af_alg_wait_for_data(sk, flags); + if (err) + return err; + } + /* Allocate cipher request for current operation. */ areq = af_alg_alloc_areq(sk, sizeof(struct af_alg_async_req) + crypto_skcipher_reqsize(tfm)); @@ -117,39 +123,39 @@ static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg, if (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) { /* AIO operation */ + sock_hold(sk); areq->iocb = msg->msg_iocb; + + /* Remember output size that will be generated. */ + areq->outlen = len; + skcipher_request_set_callback(&areq->cra_u.skcipher_req, CRYPTO_TFM_REQ_MAY_SLEEP, af_alg_async_cb, areq); err = ctx->enc ? crypto_skcipher_encrypt(&areq->cra_u.skcipher_req) : crypto_skcipher_decrypt(&areq->cra_u.skcipher_req); + + /* AIO operation in progress */ + if (err == -EINPROGRESS || err == -EBUSY) + return -EIOCBQUEUED; + + sock_put(sk); } else { /* Synchronous operation */ skcipher_request_set_callback(&areq->cra_u.skcipher_req, CRYPTO_TFM_REQ_MAY_SLEEP | CRYPTO_TFM_REQ_MAY_BACKLOG, - af_alg_complete, - &ctx->completion); - err = af_alg_wait_for_completion(ctx->enc ? + crypto_req_done, &ctx->wait); + err = crypto_wait_req(ctx->enc ? crypto_skcipher_encrypt(&areq->cra_u.skcipher_req) : crypto_skcipher_decrypt(&areq->cra_u.skcipher_req), - &ctx->completion); + &ctx->wait); } - /* AIO operation in progress */ - if (err == -EINPROGRESS) { - sock_hold(sk); - - /* Remember output size that will be generated. */ - areq->outlen = len; - - return -EIOCBQUEUED; - } free: - af_alg_free_areq_sgls(areq); - sock_kfree_s(sk, areq, areq->areqlen); + af_alg_free_resources(areq); return err ? err : len; } @@ -384,11 +390,11 @@ static int skcipher_accept_parent_nokey(void *private, struct sock *sk) INIT_LIST_HEAD(&ctx->tsgl_list); ctx->len = len; ctx->used = 0; - ctx->rcvused = 0; + atomic_set(&ctx->rcvused, 0); ctx->more = 0; ctx->merge = 0; ctx->enc = 0; - af_alg_init_completion(&ctx->completion); + crypto_init_wait(&ctx->wait); ask->private = ctx; diff --git a/crypto/api.c b/crypto/api.c index 941cd4c6..2a2479d1 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -24,6 +24,7 @@ #include <linux/sched/signal.h> #include <linux/slab.h> #include <linux/string.h> +#include <linux/completion.h> #include "internal.h" LIST_HEAD(crypto_alg_list); @@ -595,5 +596,17 @@ int crypto_has_alg(const char *name, u32 type, u32 mask) } EXPORT_SYMBOL_GPL(crypto_has_alg); +void crypto_req_done(struct crypto_async_request *req, int err) +{ + struct crypto_wait *wait = req->data; + + if (err == -EINPROGRESS) + return; + + wait->err = err; + complete(&wait->completion); +} +EXPORT_SYMBOL_GPL(crypto_req_done); + MODULE_DESCRIPTION("Cryptographic core API"); MODULE_LICENSE("GPL"); diff --git a/crypto/asymmetric_keys/pkcs7_key_type.c b/crypto/asymmetric_keys/pkcs7_key_type.c index 1063b644..e284d9cb 100644 --- a/crypto/asymmetric_keys/pkcs7_key_type.c +++ b/crypto/asymmetric_keys/pkcs7_key_type.c @@ -19,6 +19,7 @@ MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("PKCS#7 testing key type"); +MODULE_AUTHOR("Red Hat, Inc."); static unsigned pkcs7_usage; module_param_named(usage, pkcs7_usage, uint, S_IWUSR | S_IRUGO); diff --git a/crypto/asymmetric_keys/pkcs7_parser.c b/crypto/asymmetric_keys/pkcs7_parser.c index d140d8bb..a6dcaa65 100644 --- a/crypto/asymmetric_keys/pkcs7_parser.c +++ b/crypto/asymmetric_keys/pkcs7_parser.c @@ -11,6 +11,7 @@ #define pr_fmt(fmt) "PKCS7: "fmt #include <linux/kernel.h> +#include <linux/module.h> #include <linux/export.h> #include <linux/slab.h> #include <linux/err.h> @@ -19,6 +20,10 @@ #include "pkcs7_parser.h" #include "pkcs7-asn1.h" +MODULE_DESCRIPTION("PKCS#7 parser"); +MODULE_AUTHOR("Red Hat, Inc."); +MODULE_LICENSE("GPL"); + struct pkcs7_parse_context { struct pkcs7_message *msg; /* Message being constructed */ struct pkcs7_signed_info *sinfo; /* SignedInfo being constructed */ @@ -143,8 +148,10 @@ struct pkcs7_message *pkcs7_parse_message(const void *data, size_t datalen) } ret = pkcs7_check_authattrs(ctx->msg); - if (ret < 0) + if (ret < 0) { + msg = ERR_PTR(ret); goto out; + } msg = ctx->msg; ctx->msg = NULL; diff --git a/crypto/asymmetric_keys/pkcs7_trust.c b/crypto/asymmetric_keys/pkcs7_trust.c index f6a009d8..1f4e25f1 100644 --- a/crypto/asymmetric_keys/pkcs7_trust.c +++ b/crypto/asymmetric_keys/pkcs7_trust.c @@ -69,7 +69,7 @@ static int pkcs7_validate_trust_one(struct pkcs7_message *pkcs7, /* Self-signed certificates form roots of their own, and if we * don't know them, then we can't accept them. */ - if (x509->next == x509) { + if (x509->signer == x509) { kleave(" = -ENOKEY [unknown self-signed]"); return -ENOKEY; } diff --git a/crypto/asymmetric_keys/pkcs7_verify.c b/crypto/asymmetric_keys/pkcs7_verify.c index 2d93d9ec..39e6de0c 100644 --- a/crypto/asymmetric_keys/pkcs7_verify.c +++ b/crypto/asymmetric_keys/pkcs7_verify.c @@ -59,11 +59,8 @@ static int pkcs7_digest(struct pkcs7_message *pkcs7, desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; /* Digest the message [RFC2315 9.3] */ - ret = crypto_shash_init(desc); - if (ret < 0) - goto error; - ret = crypto_shash_finup(desc, pkcs7->data, pkcs7->data_len, - sig->digest); + ret = crypto_shash_digest(desc, pkcs7->data, pkcs7->data_len, + sig->digest); if (ret < 0) goto error; pr_devel("MsgDigest = [%*ph]\n", 8, sig->digest); @@ -150,7 +147,7 @@ static int pkcs7_find_key(struct pkcs7_message *pkcs7, pr_devel("Sig %u: Found cert serial match X.509[%u]\n", sinfo->index, certix); - if (x509->pub->pkey_algo != sinfo->sig->pkey_algo) { + if (strcmp(x509->pub->pkey_algo, sinfo->sig->pkey_algo) != 0) { pr_warn("Sig %u: X.509 algo and PKCS#7 sig algo don't match\n", sinfo->index); continue; diff --git a/crypto/asymmetric_keys/public_key.c b/crypto/asymmetric_keys/public_key.c index 3cd6e12c..de996586 100644 --- a/crypto/asymmetric_keys/public_key.c +++ b/crypto/asymmetric_keys/public_key.c @@ -22,6 +22,8 @@ #include <crypto/public_key.h> #include <crypto/akcipher.h> +MODULE_DESCRIPTION("In-software asymmetric public-key subtype"); +MODULE_AUTHOR("Red Hat, Inc."); MODULE_LICENSE("GPL"); /* @@ -57,29 +59,13 @@ static void public_key_destroy(void *payload0, void *payload3) public_key_signature_free(payload3); } -struct public_key_completion { - struct completion completion; - int err; -}; - -static void public_key_verify_done(struct crypto_async_request *req, int err) -{ - struct public_key_completion *compl = req->data; - - if (err == -EINPROGRESS) - return; - - compl->err = err; - complete(&compl->completion); -} - /* * Verify a signature using a public key. */ int public_key_verify_signature(const struct public_key *pkey, const struct public_key_signature *sig) { - struct public_key_completion compl; + struct crypto_wait cwait; struct crypto_akcipher *tfm; struct akcipher_request *req; struct scatterlist sig_sg, digest_sg; @@ -87,7 +73,7 @@ int public_key_verify_signature(const struct public_key *pkey, char alg_name_buf[CRYPTO_MAX_ALG_NAME]; void *output; unsigned int outlen; - int ret = -ENOMEM; + int ret; pr_devel("==>%s()\n", __func__); @@ -113,6 +99,7 @@ int public_key_verify_signature(const struct public_key *pkey, if (IS_ERR(tfm)) return PTR_ERR(tfm); + ret = -ENOMEM; req = akcipher_request_alloc(tfm, GFP_KERNEL); if (!req) goto error_free_tfm; @@ -131,21 +118,17 @@ int public_key_verify_signature(const struct public_key *pkey, sg_init_one(&digest_sg, output, outlen); akcipher_request_set_crypt(req, &sig_sg, &digest_sg, sig->s_size, outlen); - init_completion(&compl.completion); + crypto_init_wait(&cwait); akcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, - public_key_verify_done, &compl); + crypto_req_done, &cwait); /* Perform the verification calculation. This doesn't actually do the * verification, but rather calculates the hash expected by the * signature and returns that to us. */ - ret = crypto_akcipher_verify(req); - if ((ret == -EINPROGRESS) || (ret == -EBUSY)) { - wait_for_completion(&compl.completion); - ret = compl.err; - } - if (ret < 0) + ret = crypto_wait_req(crypto_akcipher_verify(req), &cwait); + if (ret) goto out_free_output; /* Do the actual verification step. */ @@ -160,6 +143,8 @@ error_free_req: error_free_tfm: crypto_free_akcipher(tfm); pr_devel("<==%s() = %d\n", __func__, ret); + if (WARN_ON_ONCE(ret > 0)) + ret = -EINVAL; return ret; } EXPORT_SYMBOL_GPL(public_key_verify_signature); diff --git a/crypto/asymmetric_keys/x509_cert_parser.c b/crypto/asymmetric_keys/x509_cert_parser.c index dd03fead..ce2df8c9 100644 --- a/crypto/asymmetric_keys/x509_cert_parser.c +++ b/crypto/asymmetric_keys/x509_cert_parser.c @@ -409,6 +409,8 @@ int x509_extract_key_data(void *context, size_t hdrlen, ctx->cert->pub->pkey_algo = "rsa"; /* Discard the BIT STRING metadata */ + if (vlen < 1 || *(const u8 *)value != 0) + return -EBADMSG; ctx->key = value + 1; ctx->key_size = vlen - 1; return 0; diff --git a/crypto/asymmetric_keys/x509_public_key.c b/crypto/asymmetric_keys/x509_public_key.c index eea71dc9..9338b455 100644 --- a/crypto/asymmetric_keys/x509_public_key.c +++ b/crypto/asymmetric_keys/x509_public_key.c @@ -79,11 +79,7 @@ int x509_get_sig_params(struct x509_certificate *cert) desc->tfm = tfm; desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; - ret = crypto_shash_init(desc); - if (ret < 0) - goto error_2; - might_sleep(); - ret = crypto_shash_finup(desc, cert->tbs, cert->tbs_size, sig->digest); + ret = crypto_shash_digest(desc, cert->tbs, cert->tbs_size, sig->digest); if (ret < 0) goto error_2; @@ -135,7 +131,7 @@ int x509_check_for_self_signed(struct x509_certificate *cert) } ret = -EKEYREJECTED; - if (cert->pub->pkey_algo != cert->sig->pkey_algo) + if (strcmp(cert->pub->pkey_algo, cert->sig->pkey_algo) != 0) goto out; ret = public_key_verify_signature(cert->pub, cert->sig); @@ -275,4 +271,5 @@ module_init(x509_key_init); module_exit(x509_key_exit); MODULE_DESCRIPTION("X.509 certificate parser"); +MODULE_AUTHOR("Red Hat, Inc."); MODULE_LICENSE("GPL"); diff --git a/crypto/chacha20poly1305.c b/crypto/chacha20poly1305.c index db1bc314..600afa99 100644 --- a/crypto/chacha20poly1305.c +++ b/crypto/chacha20poly1305.c @@ -610,6 +610,11 @@ static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb, algt->mask)); if (IS_ERR(poly)) return PTR_ERR(poly); + poly_hash = __crypto_hash_alg_common(poly); + + err = -EINVAL; + if (poly_hash->digestsize != POLY1305_DIGEST_SIZE) + goto out_put_poly; err = -ENOMEM; inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); @@ -618,7 +623,6 @@ static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb, ctx = aead_instance_ctx(inst); ctx->saltlen = CHACHAPOLY_IV_SIZE - ivsize; - poly_hash = __crypto_hash_alg_common(poly); err = crypto_init_ahash_spawn(&ctx->poly, poly_hash, aead_crypto_instance(inst)); if (err) diff --git a/crypto/cryptd.c b/crypto/cryptd.c index 0508c48a..bd43cf5b 100644 --- a/crypto/cryptd.c +++ b/crypto/cryptd.c @@ -137,16 +137,14 @@ static int cryptd_enqueue_request(struct cryptd_queue *queue, int cpu, err; struct cryptd_cpu_queue *cpu_queue; atomic_t *refcnt; - bool may_backlog; cpu = get_cpu(); cpu_queue = this_cpu_ptr(queue->cpu_queue); err = crypto_enqueue_request(&cpu_queue->queue, request); refcnt = crypto_tfm_ctx(request->tfm); - may_backlog = request->flags & CRYPTO_TFM_REQ_MAY_BACKLOG; - if (err == -EBUSY && !may_backlog) + if (err == -ENOSPC) goto out_put_cpu; queue_work_on(cpu, kcrypto_wq, &cpu_queue->work); diff --git a/crypto/cts.c b/crypto/cts.c index 243f591d..4773c188 100644 --- a/crypto/cts.c +++ b/crypto/cts.c @@ -136,8 +136,7 @@ static void crypto_cts_encrypt_done(struct crypto_async_request *areq, int err) goto out; err = cts_cbc_encrypt(req); - if (err == -EINPROGRESS || - (err == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + if (err == -EINPROGRESS || err == -EBUSY) return; out: @@ -229,8 +228,7 @@ static void crypto_cts_decrypt_done(struct crypto_async_request *areq, int err) goto out; err = cts_cbc_decrypt(req); - if (err == -EINPROGRESS || - (err == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + if (err == -EINPROGRESS || err == -EBUSY) return; out: diff --git a/crypto/dh.c b/crypto/dh.c index b1032a5c..5659fe7f 100644 --- a/crypto/dh.c +++ b/crypto/dh.c @@ -21,19 +21,12 @@ struct dh_ctx { MPI xa; }; -static inline void dh_clear_params(struct dh_ctx *ctx) +static void dh_clear_ctx(struct dh_ctx *ctx) { mpi_free(ctx->p); mpi_free(ctx->g); - ctx->p = NULL; - ctx->g = NULL; -} - -static void dh_free_ctx(struct dh_ctx *ctx) -{ - dh_clear_params(ctx); mpi_free(ctx->xa); - ctx->xa = NULL; + memset(ctx, 0, sizeof(*ctx)); } /* @@ -60,9 +53,6 @@ static int dh_check_params_length(unsigned int p_len) static int dh_set_params(struct dh_ctx *ctx, struct dh *params) { - if (unlikely(!params->p || !params->g)) - return -EINVAL; - if (dh_check_params_length(params->p_size << 3)) return -EINVAL; @@ -71,10 +61,8 @@ static int dh_set_params(struct dh_ctx *ctx, struct dh *params) return -EINVAL; ctx->g = mpi_read_raw_data(params->g, params->g_size); - if (!ctx->g) { - mpi_free(ctx->p); + if (!ctx->g) return -EINVAL; - } return 0; } @@ -86,21 +74,23 @@ static int dh_set_secret(struct crypto_kpp *tfm, const void *buf, struct dh params; /* Free the old MPI key if any */ - dh_free_ctx(ctx); + dh_clear_ctx(ctx); if (crypto_dh_decode_key(buf, len, ¶ms) < 0) - return -EINVAL; + goto err_clear_ctx; if (dh_set_params(ctx, ¶ms) < 0) - return -EINVAL; + goto err_clear_ctx; ctx->xa = mpi_read_raw_data(params.key, params.key_size); - if (!ctx->xa) { - dh_clear_params(ctx); - return -EINVAL; - } + if (!ctx->xa) + goto err_clear_ctx; return 0; + +err_clear_ctx: + dh_clear_ctx(ctx); + return -EINVAL; } static int dh_compute_value(struct kpp_request *req) @@ -158,7 +148,7 @@ static void dh_exit_tfm(struct crypto_kpp *tfm) { struct dh_ctx *ctx = dh_get_ctx(tfm); - dh_free_ctx(ctx); + dh_clear_ctx(ctx); } static struct kpp_alg dh = { diff --git a/crypto/dh_helper.c b/crypto/dh_helper.c index 8ba8a3f8..24fdb2ec 100644 --- a/crypto/dh_helper.c +++ b/crypto/dh_helper.c @@ -28,12 +28,12 @@ static inline const u8 *dh_unpack_data(void *dst, const void *src, size_t size) return src + size; } -static inline int dh_data_size(const struct dh *p) +static inline unsigned int dh_data_size(const struct dh *p) { return p->key_size + p->p_size + p->g_size; } -int crypto_dh_key_len(const struct dh *p) +unsigned int crypto_dh_key_len(const struct dh *p) { return DH_KPP_SECRET_MIN_SIZE + dh_data_size(p); } @@ -83,6 +83,14 @@ int crypto_dh_decode_key(const char *buf, unsigned int len, struct dh *params) if (secret.len != crypto_dh_key_len(params)) return -EINVAL; + /* + * Don't permit the buffer for 'key' or 'g' to be larger than 'p', since + * some drivers assume otherwise. + */ + if (params->key_size > params->p_size || + params->g_size > params->p_size) + return -EINVAL; + /* Don't allocate memory. Set pointers to data within * the given buffer */ @@ -90,6 +98,14 @@ int crypto_dh_decode_key(const char *buf, unsigned int len, struct dh *params) params->p = (void *)(ptr + params->key_size); params->g = (void *)(ptr + params->key_size + params->p_size); + /* + * Don't permit 'p' to be 0. It's not a prime number, and it's subject + * to corner cases such as 'mod 0' being undefined or + * crypto_kpp_maxsize() returning 0. + */ + if (memchr_inv(params->p, 0, params->p_size) == NULL) + return -EINVAL; + return 0; } EXPORT_SYMBOL_GPL(crypto_dh_decode_key); diff --git a/crypto/drbg.c b/crypto/drbg.c index 70018397..4faa2781 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -1651,16 +1651,6 @@ static int drbg_fini_sym_kernel(struct drbg_state *drbg) return 0; } -static void drbg_skcipher_cb(struct crypto_async_request *req, int error) -{ - struct drbg_state *drbg = req->data; - - if (error == -EINPROGRESS) - return; - drbg->ctr_async_err = error; - complete(&drbg->ctr_completion); -} - static int drbg_init_sym_kernel(struct drbg_state *drbg) { struct crypto_cipher *tfm; @@ -1691,7 +1681,7 @@ static int drbg_init_sym_kernel(struct drbg_state *drbg) return PTR_ERR(sk_tfm); } drbg->ctr_handle = sk_tfm; - init_completion(&drbg->ctr_completion); + crypto_init_wait(&drbg->ctr_wait); req = skcipher_request_alloc(sk_tfm, GFP_KERNEL); if (!req) { @@ -1700,8 +1690,9 @@ static int drbg_init_sym_kernel(struct drbg_state *drbg) return -ENOMEM; } drbg->ctr_req = req; - skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - drbg_skcipher_cb, drbg); + skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP, + crypto_req_done, &drbg->ctr_wait); alignmask = crypto_skcipher_alignmask(sk_tfm); drbg->ctr_null_value_buf = kzalloc(DRBG_CTR_NULL_LEN + alignmask, @@ -1762,21 +1753,12 @@ static int drbg_kcapi_sym_ctr(struct drbg_state *drbg, /* Output buffer may not be valid for SGL, use scratchpad */ skcipher_request_set_crypt(drbg->ctr_req, &sg_in, &sg_out, cryptlen, drbg->V); - ret = crypto_skcipher_encrypt(drbg->ctr_req); - switch (ret) { - case 0: - break; - case -EINPROGRESS: - case -EBUSY: - wait_for_completion(&drbg->ctr_completion); - if (!drbg->ctr_async_err) { - reinit_completion(&drbg->ctr_completion); - break; - } - default: + ret = crypto_wait_req(crypto_skcipher_encrypt(drbg->ctr_req), + &drbg->ctr_wait); + if (ret) goto out; - } - init_completion(&drbg->ctr_completion); + + crypto_init_wait(&drbg->ctr_wait); memcpy(outbuf, drbg->outscratchpad, cryptlen); diff --git a/crypto/ecdh.c b/crypto/ecdh.c index 4271fc77..3aca0933 100644 --- a/crypto/ecdh.c +++ b/crypto/ecdh.c @@ -131,17 +131,11 @@ static unsigned int ecdh_max_size(struct crypto_kpp *tfm) return ctx->ndigits << (ECC_DIGITS_TO_BYTES_SHIFT + 1); } -static void no_exit_tfm(struct crypto_kpp *tfm) -{ - return; -} - static struct kpp_alg ecdh = { .set_secret = ecdh_set_secret, .generate_public_key = ecdh_compute_value, .compute_shared_secret = ecdh_compute_value, .max_size = ecdh_max_size, - .exit = no_exit_tfm, .base = { .cra_name = "ecdh", .cra_driver_name = "ecdh-generic", diff --git a/crypto/ecdh_helper.c b/crypto/ecdh_helper.c index f05bea5f..d3af8e8b 100644 --- a/crypto/ecdh_helper.c +++ b/crypto/ecdh_helper.c @@ -28,7 +28,7 @@ static inline const u8 *ecdh_unpack_data(void *dst, const void *src, size_t sz) return src + sz; } -int crypto_ecdh_key_len(const struct ecdh *params) +unsigned int crypto_ecdh_key_len(const struct ecdh *params) { return ECDH_KPP_SECRET_MIN_SIZE + params->key_size; } diff --git a/crypto/gcm.c b/crypto/gcm.c index 3841b5ea..8589681f 100644 --- a/crypto/gcm.c +++ b/crypto/gcm.c @@ -14,9 +14,9 @@ #include <crypto/internal/hash.h> #include <crypto/null.h> #include <crypto/scatterwalk.h> +#include <crypto/gcm.h> #include <crypto/hash.h> #include "internal.h" -#include <linux/completion.h> #include <linux/err.h> #include <linux/init.h> #include <linux/kernel.h> @@ -78,11 +78,6 @@ struct crypto_gcm_req_priv_ctx { } u; }; -struct crypto_gcm_setkey_result { - int err; - struct completion completion; -}; - static struct { u8 buf[16]; struct scatterlist sg; @@ -98,17 +93,6 @@ static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx( return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1); } -static void crypto_gcm_setkey_done(struct crypto_async_request *req, int err) -{ - struct crypto_gcm_setkey_result *result = req->data; - - if (err == -EINPROGRESS) - return; - - result->err = err; - complete(&result->completion); -} - static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { @@ -119,7 +103,7 @@ static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, be128 hash; u8 iv[16]; - struct crypto_gcm_setkey_result result; + struct crypto_wait wait; struct scatterlist sg[1]; struct skcipher_request req; @@ -140,21 +124,18 @@ static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, if (!data) return -ENOMEM; - init_completion(&data->result.completion); + crypto_init_wait(&data->wait); sg_init_one(data->sg, &data->hash, sizeof(data->hash)); skcipher_request_set_tfm(&data->req, ctr); skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP | CRYPTO_TFM_REQ_MAY_BACKLOG, - crypto_gcm_setkey_done, - &data->result); + crypto_req_done, + &data->wait); skcipher_request_set_crypt(&data->req, data->sg, data->sg, sizeof(data->hash), data->iv); - err = crypto_skcipher_encrypt(&data->req); - if (err == -EINPROGRESS || err == -EBUSY) { - wait_for_completion(&data->result.completion); - err = data->result.err; - } + err = crypto_wait_req(crypto_skcipher_encrypt(&data->req), + &data->wait); if (err) goto out; @@ -197,8 +178,8 @@ static void crypto_gcm_init_common(struct aead_request *req) struct scatterlist *sg; memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag)); - memcpy(pctx->iv, req->iv, 12); - memcpy(pctx->iv + 12, &counter, 4); + memcpy(pctx->iv, req->iv, GCM_AES_IV_SIZE); + memcpy(pctx->iv + GCM_AES_IV_SIZE, &counter, 4); sg_init_table(pctx->src, 3); sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag)); @@ -695,7 +676,7 @@ static int crypto_gcm_create_common(struct crypto_template *tmpl, inst->alg.base.cra_alignmask = ghash->base.cra_alignmask | ctr->base.cra_alignmask; inst->alg.base.cra_ctxsize = sizeof(struct crypto_gcm_ctx); - inst->alg.ivsize = 12; + inst->alg.ivsize = GCM_AES_IV_SIZE; inst->alg.chunksize = crypto_skcipher_alg_chunksize(ctr); inst->alg.maxauthsize = 16; inst->alg.init = crypto_gcm_init_tfm; @@ -832,20 +813,20 @@ static struct aead_request *crypto_rfc4106_crypt(struct aead_request *req) u8 *iv = PTR_ALIGN((u8 *)(subreq + 1) + crypto_aead_reqsize(child), crypto_aead_alignmask(child) + 1); - scatterwalk_map_and_copy(iv + 12, req->src, 0, req->assoclen - 8, 0); + scatterwalk_map_and_copy(iv + GCM_AES_IV_SIZE, req->src, 0, req->assoclen - 8, 0); memcpy(iv, ctx->nonce, 4); memcpy(iv + 4, req->iv, 8); sg_init_table(rctx->src, 3); - sg_set_buf(rctx->src, iv + 12, req->assoclen - 8); + sg_set_buf(rctx->src, iv + GCM_AES_IV_SIZE, req->assoclen - 8); sg = scatterwalk_ffwd(rctx->src + 1, req->src, req->assoclen); if (sg != rctx->src + 1) sg_chain(rctx->src, 2, sg); if (req->src != req->dst) { sg_init_table(rctx->dst, 3); - sg_set_buf(rctx->dst, iv + 12, req->assoclen - 8); + sg_set_buf(rctx->dst, iv + GCM_AES_IV_SIZE, req->assoclen - 8); sg = scatterwalk_ffwd(rctx->dst + 1, req->dst, req->assoclen); if (sg != rctx->dst + 1) sg_chain(rctx->dst, 2, sg); @@ -957,7 +938,7 @@ static int crypto_rfc4106_create(struct crypto_template *tmpl, err = -EINVAL; /* Underlying IV size must be 12. */ - if (crypto_aead_alg_ivsize(alg) != 12) + if (crypto_aead_alg_ivsize(alg) != GCM_AES_IV_SIZE) goto out_drop_alg; /* Not a stream cipher? */ @@ -980,7 +961,7 @@ static int crypto_rfc4106_create(struct crypto_template *tmpl, inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4106_ctx); - inst->alg.ivsize = 8; + inst->alg.ivsize = GCM_RFC4106_IV_SIZE; inst->alg.chunksize = crypto_aead_alg_chunksize(alg); inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); @@ -1134,7 +1115,7 @@ static int crypto_rfc4543_init_tfm(struct crypto_aead *tfm) tfm, sizeof(struct crypto_rfc4543_req_ctx) + ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) + - align + 12); + align + GCM_AES_IV_SIZE); return 0; @@ -1199,7 +1180,7 @@ static int crypto_rfc4543_create(struct crypto_template *tmpl, err = -EINVAL; /* Underlying IV size must be 12. */ - if (crypto_aead_alg_ivsize(alg) != 12) + if (crypto_aead_alg_ivsize(alg) != GCM_AES_IV_SIZE) goto out_drop_alg; /* Not a stream cipher? */ @@ -1222,7 +1203,7 @@ static int crypto_rfc4543_create(struct crypto_template *tmpl, inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4543_ctx); - inst->alg.ivsize = 8; + inst->alg.ivsize = GCM_RFC4543_IV_SIZE; inst->alg.chunksize = crypto_aead_alg_chunksize(alg); inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); diff --git a/crypto/gf128mul.c b/crypto/gf128mul.c index dc012129..24e60195 100644 --- a/crypto/gf128mul.c +++ b/crypto/gf128mul.c @@ -156,6 +156,19 @@ static void gf128mul_x8_bbe(be128 *x) x->b = cpu_to_be64((b << 8) ^ _tt); } +void gf128mul_x8_ble(le128 *r, const le128 *x) +{ + u64 a = le64_to_cpu(x->a); + u64 b = le64_to_cpu(x->b); + + /* equivalent to gf128mul_table_be[b >> 63] (see crypto/gf128mul.c): */ + u64 _tt = gf128mul_table_be[a >> 56]; + + r->a = cpu_to_le64((a << 8) | (b >> 56)); + r->b = cpu_to_le64((b << 8) ^ _tt); +} +EXPORT_SYMBOL(gf128mul_x8_ble); + void gf128mul_lle(be128 *r, const be128 *b) { be128 p[8]; diff --git a/crypto/hmac.c b/crypto/hmac.c index 92871dc2..e7473022 100644 --- a/crypto/hmac.c +++ b/crypto/hmac.c @@ -195,11 +195,15 @@ static int hmac_create(struct crypto_template *tmpl, struct rtattr **tb) salg = shash_attr_alg(tb[1], 0, 0); if (IS_ERR(salg)) return PTR_ERR(salg); + alg = &salg->base; + /* The underlying hash algorithm must be unkeyed */ err = -EINVAL; + if (crypto_shash_alg_has_setkey(salg)) + goto out_put_alg; + ds = salg->digestsize; ss = salg->statesize; - alg = &salg->base; if (ds > alg->cra_blocksize || ss < alg->cra_blocksize) goto out_put_alg; diff --git a/crypto/keywrap.c b/crypto/keywrap.c index 72014f96..744e3513 100644 --- a/crypto/keywrap.c +++ b/crypto/keywrap.c @@ -93,18 +93,10 @@ struct crypto_kw_ctx { struct crypto_kw_block { #define SEMIBSIZE 8 - u8 A[SEMIBSIZE]; - u8 R[SEMIBSIZE]; + __be64 A; + __be64 R; }; -/* convert 64 bit integer into its string representation */ -static inline void crypto_kw_cpu_to_be64(u64 val, u8 *buf) -{ - __be64 *a = (__be64 *)buf; - - *a = cpu_to_be64(val); -} - /* * Fast forward the SGL to the "end" length minus SEMIBSIZE. * The start in the SGL defined by the fast-forward is returned with @@ -139,17 +131,10 @@ static int crypto_kw_decrypt(struct blkcipher_desc *desc, struct crypto_blkcipher *tfm = desc->tfm; struct crypto_kw_ctx *ctx = crypto_blkcipher_ctx(tfm); struct crypto_cipher *child = ctx->child; - - unsigned long alignmask = max_t(unsigned long, SEMIBSIZE, - crypto_cipher_alignmask(child)); - unsigned int i; - - u8 blockbuf[sizeof(struct crypto_kw_block) + alignmask]; - struct crypto_kw_block *block = (struct crypto_kw_block *) - PTR_ALIGN(blockbuf + 0, alignmask + 1); - - u64 t = 6 * ((nbytes) >> 3); + struct crypto_kw_block block; struct scatterlist *lsrc, *ldst; + u64 t = 6 * ((nbytes) >> 3); + unsigned int i; int ret = 0; /* @@ -160,7 +145,7 @@ static int crypto_kw_decrypt(struct blkcipher_desc *desc, return -EINVAL; /* Place the IV into block A */ - memcpy(block->A, desc->info, SEMIBSIZE); + memcpy(&block.A, desc->info, SEMIBSIZE); /* * src scatterlist is read-only. dst scatterlist is r/w. During the @@ -171,32 +156,27 @@ static int crypto_kw_decrypt(struct blkcipher_desc *desc, ldst = dst; for (i = 0; i < 6; i++) { - u8 tbe_buffer[SEMIBSIZE + alignmask]; - /* alignment for the crypto_xor and the _to_be64 operation */ - u8 *tbe = PTR_ALIGN(tbe_buffer + 0, alignmask + 1); - unsigned int tmp_nbytes = nbytes; struct scatter_walk src_walk, dst_walk; + unsigned int tmp_nbytes = nbytes; while (tmp_nbytes) { /* move pointer by tmp_nbytes in the SGL */ crypto_kw_scatterlist_ff(&src_walk, lsrc, tmp_nbytes); /* get the source block */ - scatterwalk_copychunks(block->R, &src_walk, SEMIBSIZE, + scatterwalk_copychunks(&block.R, &src_walk, SEMIBSIZE, false); - /* perform KW operation: get counter as byte string */ - crypto_kw_cpu_to_be64(t, tbe); /* perform KW operation: modify IV with counter */ - crypto_xor(block->A, tbe, SEMIBSIZE); + block.A ^= cpu_to_be64(t); t--; /* perform KW operation: decrypt block */ - crypto_cipher_decrypt_one(child, (u8*)block, - (u8*)block); + crypto_cipher_decrypt_one(child, (u8*)&block, + (u8*)&block); /* move pointer by tmp_nbytes in the SGL */ crypto_kw_scatterlist_ff(&dst_walk, ldst, tmp_nbytes); /* Copy block->R into place */ - scatterwalk_copychunks(block->R, &dst_walk, SEMIBSIZE, + scatterwalk_copychunks(&block.R, &dst_walk, SEMIBSIZE, true); tmp_nbytes -= SEMIBSIZE; @@ -208,11 +188,10 @@ static int crypto_kw_decrypt(struct blkcipher_desc *desc, } /* Perform authentication check */ - if (crypto_memneq("\xA6\xA6\xA6\xA6\xA6\xA6\xA6\xA6", block->A, - SEMIBSIZE)) + if (block.A != cpu_to_be64(0xa6a6a6a6a6a6a6a6)) ret = -EBADMSG; - memzero_explicit(block, sizeof(struct crypto_kw_block)); + memzero_explicit(&block, sizeof(struct crypto_kw_block)); return ret; } @@ -224,17 +203,10 @@ static int crypto_kw_encrypt(struct blkcipher_desc *desc, struct crypto_blkcipher *tfm = desc->tfm; struct crypto_kw_ctx *ctx = crypto_blkcipher_ctx(tfm); struct crypto_cipher *child = ctx->child; - - unsigned long alignmask = max_t(unsigned long, SEMIBSIZE, - crypto_cipher_alignmask(child)); - unsigned int i; - - u8 blockbuf[sizeof(struct crypto_kw_block) + alignmask]; - struct crypto_kw_block *block = (struct crypto_kw_block *) - PTR_ALIGN(blockbuf + 0, alignmask + 1); - - u64 t = 1; + struct crypto_kw_block block; struct scatterlist *lsrc, *ldst; + u64 t = 1; + unsigned int i; /* * Require at least 2 semiblocks (note, the 3rd semiblock that is @@ -249,7 +221,7 @@ static int crypto_kw_encrypt(struct blkcipher_desc *desc, * Place the predefined IV into block A -- for encrypt, the caller * does not need to provide an IV, but he needs to fetch the final IV. */ - memcpy(block->A, "\xA6\xA6\xA6\xA6\xA6\xA6\xA6\xA6", SEMIBSIZE); + block.A = cpu_to_be64(0xa6a6a6a6a6a6a6a6); /* * src scatterlist is read-only. dst scatterlist is r/w. During the @@ -260,30 +232,26 @@ static int crypto_kw_encrypt(struct blkcipher_desc *desc, ldst = dst; for (i = 0; i < 6; i++) { - u8 tbe_buffer[SEMIBSIZE + alignmask]; - u8 *tbe = PTR_ALIGN(tbe_buffer + 0, alignmask + 1); - unsigned int tmp_nbytes = nbytes; struct scatter_walk src_walk, dst_walk; + unsigned int tmp_nbytes = nbytes; scatterwalk_start(&src_walk, lsrc); scatterwalk_start(&dst_walk, ldst); while (tmp_nbytes) { /* get the source block */ - scatterwalk_copychunks(block->R, &src_walk, SEMIBSIZE, + scatterwalk_copychunks(&block.R, &src_walk, SEMIBSIZE, false); /* perform KW operation: encrypt block */ - crypto_cipher_encrypt_one(child, (u8 *)block, - (u8 *)block); - /* perform KW operation: get counter as byte string */ - crypto_kw_cpu_to_be64(t, tbe); + crypto_cipher_encrypt_one(child, (u8 *)&block, + (u8 *)&block); /* perform KW operation: modify IV with counter */ - crypto_xor(block->A, tbe, SEMIBSIZE); + block.A ^= cpu_to_be64(t); t++; /* Copy block->R into place */ - scatterwalk_copychunks(block->R, &dst_walk, SEMIBSIZE, + scatterwalk_copychunks(&block.R, &dst_walk, SEMIBSIZE, true); tmp_nbytes -= SEMIBSIZE; @@ -295,9 +263,9 @@ static int crypto_kw_encrypt(struct blkcipher_desc *desc, } /* establish the IV for the caller to pick up */ - memcpy(desc->info, block->A, SEMIBSIZE); + memcpy(desc->info, &block.A, SEMIBSIZE); - memzero_explicit(block, sizeof(struct crypto_kw_block)); + memzero_explicit(&block, sizeof(struct crypto_kw_block)); return 0; } diff --git a/crypto/lrw.c b/crypto/lrw.c index a8bfae44..cbbd7c50 100644 --- a/crypto/lrw.c +++ b/crypto/lrw.c @@ -328,9 +328,7 @@ static int do_encrypt(struct skcipher_request *req, int err) crypto_skcipher_encrypt(subreq) ?: post_crypt(req); - if (err == -EINPROGRESS || - (err == -EBUSY && - req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + if (err == -EINPROGRESS || err == -EBUSY) return err; } @@ -380,9 +378,7 @@ static int do_decrypt(struct skcipher_request *req, int err) crypto_skcipher_decrypt(subreq) ?: post_crypt(req); - if (err == -EINPROGRESS || - (err == -EBUSY && - req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + if (err == -EINPROGRESS || err == -EBUSY) return err; } @@ -610,9 +606,12 @@ static int create(struct crypto_template *tmpl, struct rtattr **tb) ecb_name[len - 1] = 0; if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, - "lrw(%s)", ecb_name) >= CRYPTO_MAX_ALG_NAME) - return -ENAMETOOLONG; - } + "lrw(%s)", ecb_name) >= CRYPTO_MAX_ALG_NAME) { + err = -ENAMETOOLONG; + goto err_drop_spawn; + } + } else + goto err_drop_spawn; inst->alg.base.cra_flags = alg->base.cra_flags & CRYPTO_ALG_ASYNC; inst->alg.base.cra_priority = alg->base.cra_priority; diff --git a/crypto/mcryptd.c b/crypto/mcryptd.c index 4e647265..eca04d37 100644 --- a/crypto/mcryptd.c +++ b/crypto/mcryptd.c @@ -81,6 +81,7 @@ static int mcryptd_init_queue(struct mcryptd_queue *queue, pr_debug("cpu_queue #%d %p\n", cpu, queue->cpu_queue); crypto_init_queue(&cpu_queue->queue, max_cpu_qlen); INIT_WORK(&cpu_queue->work, mcryptd_queue_worker); + spin_lock_init(&cpu_queue->q_lock); } return 0; } @@ -104,15 +105,16 @@ static int mcryptd_enqueue_request(struct mcryptd_queue *queue, int cpu, err; struct mcryptd_cpu_queue *cpu_queue; - cpu = get_cpu(); - cpu_queue = this_cpu_ptr(queue->cpu_queue); - rctx->tag.cpu = cpu; + cpu_queue = raw_cpu_ptr(queue->cpu_queue); + spin_lock(&cpu_queue->q_lock); + cpu = smp_processor_id(); + rctx->tag.cpu = smp_processor_id(); err = crypto_enqueue_request(&cpu_queue->queue, request); pr_debug("enqueue request: cpu %d cpu_queue %p request %p\n", cpu, cpu_queue, request); + spin_unlock(&cpu_queue->q_lock); queue_work_on(cpu, kcrypto_wq, &cpu_queue->work); - put_cpu(); return err; } @@ -161,16 +163,11 @@ static void mcryptd_queue_worker(struct work_struct *work) cpu_queue = container_of(work, struct mcryptd_cpu_queue, work); i = 0; while (i < MCRYPTD_BATCH || single_task_running()) { - /* - * preempt_disable/enable is used to prevent - * being preempted by mcryptd_enqueue_request() - */ - local_bh_disable(); - preempt_disable(); + + spin_lock_bh(&cpu_queue->q_lock); backlog = crypto_get_backlog(&cpu_queue->queue); req = crypto_dequeue_request(&cpu_queue->queue); - preempt_enable(); - local_bh_enable(); + spin_unlock_bh(&cpu_queue->q_lock); if (!req) { mcryptd_opportunistic_flush(); @@ -185,7 +182,7 @@ static void mcryptd_queue_worker(struct work_struct *work) ++i; } if (cpu_queue->queue.qlen) - queue_work(kcrypto_wq, &cpu_queue->work); + queue_work_on(smp_processor_id(), kcrypto_wq, &cpu_queue->work); } void mcryptd_flusher(struct work_struct *__work) diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c index ee9cfb99..f8ec3d4b 100644 --- a/crypto/pcrypt.c +++ b/crypto/pcrypt.c @@ -254,6 +254,14 @@ static void pcrypt_aead_exit_tfm(struct crypto_aead *tfm) crypto_free_aead(ctx->child); } +static void pcrypt_free(struct aead_instance *inst) +{ + struct pcrypt_instance_ctx *ctx = aead_instance_ctx(inst); + + crypto_drop_aead(&ctx->spawn); + kfree(inst); +} + static int pcrypt_init_instance(struct crypto_instance *inst, struct crypto_alg *alg) { @@ -319,6 +327,8 @@ static int pcrypt_create_aead(struct crypto_template *tmpl, struct rtattr **tb, inst->alg.encrypt = pcrypt_aead_encrypt; inst->alg.decrypt = pcrypt_aead_decrypt; + inst->free = pcrypt_free; + err = aead_register_instance(tmpl, inst); if (err) goto out_drop_aead; @@ -349,14 +359,6 @@ static int pcrypt_create(struct crypto_template *tmpl, struct rtattr **tb) return -EINVAL; } -static void pcrypt_free(struct crypto_instance *inst) -{ - struct pcrypt_instance_ctx *ctx = crypto_instance_ctx(inst); - - crypto_drop_aead(&ctx->spawn); - kfree(inst); -} - static int pcrypt_cpumask_change_notify(struct notifier_block *self, unsigned long val, void *data) { @@ -469,7 +471,6 @@ static void pcrypt_fini_padata(struct padata_pcrypt *pcrypt) static struct crypto_template pcrypt_tmpl = { .name = "pcrypt", .create = pcrypt_create, - .free = pcrypt_free, .module = THIS_MODULE, }; diff --git a/crypto/rmd128.c b/crypto/rmd128.c index 049486ed..40e053b9 100644 --- a/crypto/rmd128.c +++ b/crypto/rmd128.c @@ -213,8 +213,6 @@ static void rmd128_transform(u32 *state, const __le32 *in) state[2] = state[3] + aa + bbb; state[3] = state[0] + bb + ccc; state[0] = ddd; - - return; } static int rmd128_init(struct shash_desc *desc) diff --git a/crypto/rmd160.c b/crypto/rmd160.c index de585e51..5f3e6ea3 100644 --- a/crypto/rmd160.c +++ b/crypto/rmd160.c @@ -256,8 +256,6 @@ static void rmd160_transform(u32 *state, const __le32 *in) state[3] = state[4] + aa + bbb; state[4] = state[0] + bb + ccc; state[0] = ddd; - - return; } static int rmd160_init(struct shash_desc *desc) diff --git a/crypto/rmd256.c b/crypto/rmd256.c index 4ec02a75..f50c025c 100644 --- a/crypto/rmd256.c +++ b/crypto/rmd256.c @@ -228,8 +228,6 @@ static void rmd256_transform(u32 *state, const __le32 *in) state[5] += bbb; state[6] += ccc; state[7] += ddd; - - return; } static int rmd256_init(struct shash_desc *desc) diff --git a/crypto/rmd320.c b/crypto/rmd320.c index 770f2cb3..e1315e48 100644 --- a/crypto/rmd320.c +++ b/crypto/rmd320.c @@ -275,8 +275,6 @@ static void rmd320_transform(u32 *state, const __le32 *in) state[7] += ccc; state[8] += ddd; state[9] += eee; - - return; } static int rmd320_init(struct shash_desc *desc) diff --git a/crypto/rsa-pkcs1pad.c b/crypto/rsa-pkcs1pad.c index 407c64bd..2908f93c 100644 --- a/crypto/rsa-pkcs1pad.c +++ b/crypto/rsa-pkcs1pad.c @@ -279,9 +279,7 @@ static int pkcs1pad_encrypt(struct akcipher_request *req) req->dst, ctx->key_size - 1, req->dst_len); err = crypto_akcipher_encrypt(&req_ctx->child_req); - if (err != -EINPROGRESS && - (err != -EBUSY || - !(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))) + if (err != -EINPROGRESS && err != -EBUSY) return pkcs1pad_encrypt_sign_complete(req, err); return err; @@ -383,9 +381,7 @@ static int pkcs1pad_decrypt(struct akcipher_request *req) ctx->key_size); err = crypto_akcipher_decrypt(&req_ctx->child_req); - if (err != -EINPROGRESS && - (err != -EBUSY || - !(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))) + if (err != -EINPROGRESS && err != -EBUSY) return pkcs1pad_decrypt_complete(req, err); return err; @@ -440,9 +436,7 @@ static int pkcs1pad_sign(struct akcipher_request *req) req->dst, ctx->key_size - 1, req->dst_len); err = crypto_akcipher_sign(&req_ctx->child_req); - if (err != -EINPROGRESS && - (err != -EBUSY || - !(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))) + if (err != -EINPROGRESS && err != -EBUSY) return pkcs1pad_encrypt_sign_complete(req, err); return err; @@ -561,9 +555,7 @@ static int pkcs1pad_verify(struct akcipher_request *req) ctx->key_size); err = crypto_akcipher_verify(&req_ctx->child_req); - if (err != -EINPROGRESS && - (err != -EBUSY || - !(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))) + if (err != -EINPROGRESS && err != -EBUSY) return pkcs1pad_verify_complete(req, err); return err; diff --git a/crypto/rsa_helper.c b/crypto/rsa_helper.c index 0b66dc82..cad395d7 100644 --- a/crypto/rsa_helper.c +++ b/crypto/rsa_helper.c @@ -30,7 +30,7 @@ int rsa_get_n(void *context, size_t hdrlen, unsigned char tag, return -EINVAL; if (fips_enabled) { - while (!*ptr && n_sz) { + while (n_sz && !*ptr) { ptr++; n_sz--; } diff --git a/crypto/salsa20_generic.c b/crypto/salsa20_generic.c index f550b5d9..d7da0eea 100644 --- a/crypto/salsa20_generic.c +++ b/crypto/salsa20_generic.c @@ -188,13 +188,6 @@ static int encrypt(struct blkcipher_desc *desc, salsa20_ivsetup(ctx, walk.iv); - if (likely(walk.nbytes == nbytes)) - { - salsa20_encrypt_bytes(ctx, walk.dst.virt.addr, - walk.src.virt.addr, nbytes); - return blkcipher_walk_done(desc, &walk, 0); - } - while (walk.nbytes >= 64) { salsa20_encrypt_bytes(ctx, walk.dst.virt.addr, walk.src.virt.addr, diff --git a/crypto/shash.c b/crypto/shash.c index 325a14da..e849d3ee 100644 --- a/crypto/shash.c +++ b/crypto/shash.c @@ -25,11 +25,12 @@ static const struct crypto_type crypto_shash_type; -static int shash_no_setkey(struct crypto_shash *tfm, const u8 *key, - unsigned int keylen) +int shash_no_setkey(struct crypto_shash *tfm, const u8 *key, + unsigned int keylen) { return -ENOSYS; } +EXPORT_SYMBOL_GPL(shash_no_setkey); static int shash_setkey_unaligned(struct crypto_shash *tfm, const u8 *key, unsigned int keylen) diff --git a/crypto/skcipher.c b/crypto/skcipher.c index d5692e35..11af5fd6 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -449,6 +449,8 @@ static int skcipher_walk_skcipher(struct skcipher_walk *walk, walk->total = req->cryptlen; walk->nbytes = 0; + walk->iv = req->iv; + walk->oiv = req->iv; if (unlikely(!walk->total)) return 0; @@ -456,9 +458,6 @@ static int skcipher_walk_skcipher(struct skcipher_walk *walk, scatterwalk_start(&walk->in, req->src); scatterwalk_start(&walk->out, req->dst); - walk->iv = req->iv; - walk->oiv = req->iv; - walk->flags &= ~SKCIPHER_WALK_SLEEP; walk->flags |= req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? SKCIPHER_WALK_SLEEP : 0; @@ -510,6 +509,8 @@ static int skcipher_walk_aead_common(struct skcipher_walk *walk, int err; walk->nbytes = 0; + walk->iv = req->iv; + walk->oiv = req->iv; if (unlikely(!walk->total)) return 0; @@ -522,8 +523,8 @@ static int skcipher_walk_aead_common(struct skcipher_walk *walk, scatterwalk_copychunks(NULL, &walk->in, req->assoclen, 2); scatterwalk_copychunks(NULL, &walk->out, req->assoclen, 2); - walk->iv = req->iv; - walk->oiv = req->iv; + scatterwalk_done(&walk->in, 0, walk->total); + scatterwalk_done(&walk->out, 0, walk->total); if (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) walk->flags |= SKCIPHER_WALK_SLEEP; diff --git a/crypto/sm3_generic.c b/crypto/sm3_generic.c new file mode 100644 index 00000000..9e823d99 --- /dev/null +++ b/crypto/sm3_generic.c @@ -0,0 +1,210 @@ +/* + * SM3 secure hash, as specified by OSCCA GM/T 0004-2012 SM3 and + * described at https://tools.ietf.org/html/draft-shen-sm3-hash-01 + * + * Copyright (C) 2017 ARM Limited or its affiliates. + * Written by Gilad Ben-Yossef <gilad@benyossef.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see <http://www.gnu.org/licenses/>. + */ + +#include <crypto/internal/hash.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/mm.h> +#include <linux/types.h> +#include <crypto/sm3.h> +#include <crypto/sm3_base.h> +#include <linux/bitops.h> +#include <asm/byteorder.h> +#include <asm/unaligned.h> + +const u8 sm3_zero_message_hash[SM3_DIGEST_SIZE] = { + 0x1A, 0xB2, 0x1D, 0x83, 0x55, 0xCF, 0xA1, 0x7F, + 0x8e, 0x61, 0x19, 0x48, 0x31, 0xE8, 0x1A, 0x8F, + 0x22, 0xBE, 0xC8, 0xC7, 0x28, 0xFE, 0xFB, 0x74, + 0x7E, 0xD0, 0x35, 0xEB, 0x50, 0x82, 0xAA, 0x2B +}; +EXPORT_SYMBOL_GPL(sm3_zero_message_hash); + +static inline u32 p0(u32 x) +{ + return x ^ rol32(x, 9) ^ rol32(x, 17); +} + +static inline u32 p1(u32 x) +{ + return x ^ rol32(x, 15) ^ rol32(x, 23); +} + +static inline u32 ff(unsigned int n, u32 a, u32 b, u32 c) +{ + return (n < 16) ? (a ^ b ^ c) : ((a & b) | (a & c) | (b & c)); +} + +static inline u32 gg(unsigned int n, u32 e, u32 f, u32 g) +{ + return (n < 16) ? (e ^ f ^ g) : ((e & f) | ((~e) & g)); +} + +static inline u32 t(unsigned int n) +{ + return (n < 16) ? SM3_T1 : SM3_T2; +} + +static void sm3_expand(u32 *t, u32 *w, u32 *wt) +{ + int i; + unsigned int tmp; + + /* load the input */ + for (i = 0; i <= 15; i++) + w[i] = get_unaligned_be32((__u32 *)t + i); + + for (i = 16; i <= 67; i++) { + tmp = w[i - 16] ^ w[i - 9] ^ rol32(w[i - 3], 15); + w[i] = p1(tmp) ^ (rol32(w[i - 13], 7)) ^ w[i - 6]; + } + + for (i = 0; i <= 63; i++) + wt[i] = w[i] ^ w[i + 4]; +} + +static void sm3_compress(u32 *w, u32 *wt, u32 *m) +{ + u32 ss1; + u32 ss2; + u32 tt1; + u32 tt2; + u32 a, b, c, d, e, f, g, h; + int i; + + a = m[0]; + b = m[1]; + c = m[2]; + d = m[3]; + e = m[4]; + f = m[5]; + g = m[6]; + h = m[7]; + + for (i = 0; i <= 63; i++) { + + ss1 = rol32((rol32(a, 12) + e + rol32(t(i), i)), 7); + + ss2 = ss1 ^ rol32(a, 12); + + tt1 = ff(i, a, b, c) + d + ss2 + *wt; + wt++; + + tt2 = gg(i, e, f, g) + h + ss1 + *w; + w++; + + d = c; + c = rol32(b, 9); + b = a; + a = tt1; + h = g; + g = rol32(f, 19); + f = e; + e = p0(tt2); + } + + m[0] = a ^ m[0]; + m[1] = b ^ m[1]; + m[2] = c ^ m[2]; + m[3] = d ^ m[3]; + m[4] = e ^ m[4]; + m[5] = f ^ m[5]; + m[6] = g ^ m[6]; + m[7] = h ^ m[7]; + + a = b = c = d = e = f = g = h = ss1 = ss2 = tt1 = tt2 = 0; +} + +static void sm3_transform(struct sm3_state *sst, u8 const *src) +{ + unsigned int w[68]; + unsigned int wt[64]; + + sm3_expand((u32 *)src, w, wt); + sm3_compress(w, wt, sst->state); + + memzero_explicit(w, sizeof(w)); + memzero_explicit(wt, sizeof(wt)); +} + +static void sm3_generic_block_fn(struct sm3_state *sst, u8 const *src, + int blocks) +{ + while (blocks--) { + sm3_transform(sst, src); + src += SM3_BLOCK_SIZE; + } +} + +int crypto_sm3_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + return sm3_base_do_update(desc, data, len, sm3_generic_block_fn); +} +EXPORT_SYMBOL(crypto_sm3_update); + +static int sm3_final(struct shash_desc *desc, u8 *out) +{ + sm3_base_do_finalize(desc, sm3_generic_block_fn); + return sm3_base_finish(desc, out); +} + +int crypto_sm3_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *hash) +{ + sm3_base_do_update(desc, data, len, sm3_generic_block_fn); + return sm3_final(desc, hash); +} +EXPORT_SYMBOL(crypto_sm3_finup); + +static struct shash_alg sm3_alg = { + .digestsize = SM3_DIGEST_SIZE, + .init = sm3_base_init, + .update = crypto_sm3_update, + .final = sm3_final, + .finup = crypto_sm3_finup, + .descsize = sizeof(struct sm3_state), + .base = { + .cra_name = "sm3", + .cra_driver_name = "sm3-generic", + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SM3_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +static int __init sm3_generic_mod_init(void) +{ + return crypto_register_shash(&sm3_alg); +} + +static void __exit sm3_generic_mod_fini(void) +{ + crypto_unregister_shash(&sm3_alg); +} + +module_init(sm3_generic_mod_init); +module_exit(sm3_generic_mod_fini); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("SM3 Secure Hash Algorithm"); + +MODULE_ALIAS_CRYPTO("sm3"); +MODULE_ALIAS_CRYPTO("sm3-generic"); diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 0022a18d..9267cbdb 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -70,7 +70,7 @@ static int mode; static char *tvmem[TVMEMSIZE]; static char *check[] = { - "des", "md5", "des3_ede", "rot13", "sha1", "sha224", "sha256", + "des", "md5", "des3_ede", "rot13", "sha1", "sha224", "sha256", "sm3", "blowfish", "twofish", "serpent", "sha384", "sha512", "md4", "aes", "cast6", "arc4", "michael_mic", "deflate", "crc32c", "tea", "xtea", "khazad", "wp512", "wp384", "wp256", "tnepres", "xeta", "fcrypt", @@ -79,34 +79,11 @@ static char *check[] = { NULL }; -struct tcrypt_result { - struct completion completion; - int err; -}; - -static void tcrypt_complete(struct crypto_async_request *req, int err) -{ - struct tcrypt_result *res = req->data; - - if (err == -EINPROGRESS) - return; - - res->err = err; - complete(&res->completion); -} - static inline int do_one_aead_op(struct aead_request *req, int ret) { - if (ret == -EINPROGRESS || ret == -EBUSY) { - struct tcrypt_result *tr = req->base.data; + struct crypto_wait *wait = req->base.data; - ret = wait_for_completion_interruptible(&tr->completion); - if (!ret) - ret = tr->err; - reinit_completion(&tr->completion); - } - - return ret; + return crypto_wait_req(ret, wait); } static int test_aead_jiffies(struct aead_request *req, int enc, @@ -248,7 +225,7 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, char *axbuf[XBUFSIZE]; unsigned int *b_size; unsigned int iv_len; - struct tcrypt_result result; + struct crypto_wait wait; iv = kzalloc(MAX_IVLEN, GFP_KERNEL); if (!iv) @@ -284,7 +261,7 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, goto out_notfm; } - init_completion(&result.completion); + crypto_init_wait(&wait); printk(KERN_INFO "\ntesting speed of %s (%s) %s\n", algo, get_driver_name(crypto_aead, tfm), e); @@ -296,7 +273,7 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, } aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - tcrypt_complete, &result); + crypto_req_done, &wait); i = 0; do { @@ -340,7 +317,7 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, } sg_init_aead(sg, xbuf, - *b_size + (enc ? authsize : 0)); + *b_size + (enc ? 0 : authsize)); sg_init_aead(sgout, xoutbuf, *b_size + (enc ? authsize : 0)); @@ -348,7 +325,9 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, sg_set_buf(&sg[0], assoc, aad_size); sg_set_buf(&sgout[0], assoc, aad_size); - aead_request_set_crypt(req, sg, sgout, *b_size, iv); + aead_request_set_crypt(req, sg, sgout, + *b_size + (enc ? 0 : authsize), + iv); aead_request_set_ad(req, aad_size); if (secs) @@ -381,7 +360,6 @@ out_noaxbuf: testmgr_free_buf(xbuf); out_noxbuf: kfree(iv); - return; } static void test_hash_sg_init(struct scatterlist *sg) @@ -397,21 +375,16 @@ static void test_hash_sg_init(struct scatterlist *sg) static inline int do_one_ahash_op(struct ahash_request *req, int ret) { - if (ret == -EINPROGRESS || ret == -EBUSY) { - struct tcrypt_result *tr = req->base.data; + struct crypto_wait *wait = req->base.data; - wait_for_completion(&tr->completion); - reinit_completion(&tr->completion); - ret = tr->err; - } - return ret; + return crypto_wait_req(ret, wait); } struct test_mb_ahash_data { struct scatterlist sg[TVMEMSIZE]; char result[64]; struct ahash_request *req; - struct tcrypt_result tresult; + struct crypto_wait wait; char *xbuf[XBUFSIZE]; }; @@ -440,7 +413,7 @@ static void test_mb_ahash_speed(const char *algo, unsigned int sec, if (testmgr_alloc_buf(data[i].xbuf)) goto out; - init_completion(&data[i].tresult.completion); + crypto_init_wait(&data[i].wait); data[i].req = ahash_request_alloc(tfm, GFP_KERNEL); if (!data[i].req) { @@ -449,8 +422,8 @@ static void test_mb_ahash_speed(const char *algo, unsigned int sec, goto out; } - ahash_request_set_callback(data[i].req, 0, - tcrypt_complete, &data[i].tresult); + ahash_request_set_callback(data[i].req, 0, crypto_req_done, + &data[i].wait); test_hash_sg_init(data[i].sg); } @@ -492,16 +465,16 @@ static void test_mb_ahash_speed(const char *algo, unsigned int sec, if (ret) break; - complete(&data[k].tresult.completion); - data[k].tresult.err = 0; + crypto_req_done(&data[k].req->base, 0); } for (j = 0; j < k; j++) { - struct tcrypt_result *tr = &data[j].tresult; + struct crypto_wait *wait = &data[j].wait; + int wait_ret; - wait_for_completion(&tr->completion); - if (tr->err) - ret = tr->err; + wait_ret = crypto_wait_req(-EINPROGRESS, wait); + if (wait_ret) + ret = wait_ret; } end = get_cycles(); @@ -679,7 +652,7 @@ static void test_ahash_speed_common(const char *algo, unsigned int secs, struct hash_speed *speed, unsigned mask) { struct scatterlist sg[TVMEMSIZE]; - struct tcrypt_result tresult; + struct crypto_wait wait; struct ahash_request *req; struct crypto_ahash *tfm; char *output; @@ -708,9 +681,9 @@ static void test_ahash_speed_common(const char *algo, unsigned int secs, goto out; } - init_completion(&tresult.completion); + crypto_init_wait(&wait); ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - tcrypt_complete, &tresult); + crypto_req_done, &wait); output = kmalloc(MAX_DIGEST_SIZE, GFP_KERNEL); if (!output) @@ -765,15 +738,9 @@ static void test_hash_speed(const char *algo, unsigned int secs, static inline int do_one_acipher_op(struct skcipher_request *req, int ret) { - if (ret == -EINPROGRESS || ret == -EBUSY) { - struct tcrypt_result *tr = req->base.data; - - wait_for_completion(&tr->completion); - reinit_completion(&tr->completion); - ret = tr->err; - } + struct crypto_wait *wait = req->base.data; - return ret; + return crypto_wait_req(ret, wait); } static int test_acipher_jiffies(struct skcipher_request *req, int enc, @@ -853,7 +820,7 @@ static void test_skcipher_speed(const char *algo, int enc, unsigned int secs, unsigned int tcount, u8 *keysize, bool async) { unsigned int ret, i, j, k, iv_len; - struct tcrypt_result tresult; + struct crypto_wait wait; const char *key; char iv[128]; struct skcipher_request *req; @@ -866,7 +833,7 @@ static void test_skcipher_speed(const char *algo, int enc, unsigned int secs, else e = "decryption"; - init_completion(&tresult.completion); + crypto_init_wait(&wait); tfm = crypto_alloc_skcipher(algo, 0, async ? 0 : CRYPTO_ALG_ASYNC); @@ -887,7 +854,7 @@ static void test_skcipher_speed(const char *algo, int enc, unsigned int secs, } skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - tcrypt_complete, &tresult); + crypto_req_done, &wait); i = 0; do { @@ -1269,6 +1236,10 @@ static int do_test(const char *alg, u32 type, u32 mask, int m) ret += tcrypt_test("sha3-512"); break; + case 52: + ret += tcrypt_test("sm3"); + break; + case 100: ret += tcrypt_test("hmac(md5)"); break; @@ -1603,115 +1574,116 @@ static int do_test(const char *alg, u32 type, u32 mask, int m) speed_template_32); break; - case 300: if (alg) { test_hash_speed(alg, sec, generic_hash_speed_template); break; } - /* fall through */ - case 301: test_hash_speed("md4", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; - + /* fall through */ case 302: test_hash_speed("md5", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; - + /* fall through */ case 303: test_hash_speed("sha1", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; - + /* fall through */ case 304: test_hash_speed("sha256", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; - + /* fall through */ case 305: test_hash_speed("sha384", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; - + /* fall through */ case 306: test_hash_speed("sha512", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; - + /* fall through */ case 307: test_hash_speed("wp256", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; - + /* fall through */ case 308: test_hash_speed("wp384", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; - + /* fall through */ case 309: test_hash_speed("wp512", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; - + /* fall through */ case 310: test_hash_speed("tgr128", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; - + /* fall through */ case 311: test_hash_speed("tgr160", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; - + /* fall through */ case 312: test_hash_speed("tgr192", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; - + /* fall through */ case 313: test_hash_speed("sha224", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; - + /* fall through */ case 314: test_hash_speed("rmd128", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; - + /* fall through */ case 315: test_hash_speed("rmd160", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; - + /* fall through */ case 316: test_hash_speed("rmd256", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; - + /* fall through */ case 317: test_hash_speed("rmd320", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; - + /* fall through */ case 318: test_hash_speed("ghash-generic", sec, hash_speed_template_16); if (mode > 300 && mode < 400) break; - + /* fall through */ case 319: test_hash_speed("crc32c", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; - + /* fall through */ case 320: test_hash_speed("crct10dif", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; - + /* fall through */ case 321: test_hash_speed("poly1305", sec, poly1305_speed_template); if (mode > 300 && mode < 400) break; - + /* fall through */ case 322: test_hash_speed("sha3-224", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; - + /* fall through */ case 323: test_hash_speed("sha3-256", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; - + /* fall through */ case 324: test_hash_speed("sha3-384", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; - + /* fall through */ case 325: test_hash_speed("sha3-512", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; - + /* fall through */ + case 326: + test_hash_speed("sm3", sec, generic_hash_speed_template); + if (mode > 300 && mode < 400) break; + /* fall through */ case 399: break; @@ -1720,106 +1692,107 @@ static int do_test(const char *alg, u32 type, u32 mask, int m) test_ahash_speed(alg, sec, generic_hash_speed_template); break; } - /* fall through */ - case 401: test_ahash_speed("md4", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; - + /* fall through */ case 402: test_ahash_speed("md5", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; - + /* fall through */ case 403: test_ahash_speed("sha1", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; - + /* fall through */ case 404: test_ahash_speed("sha256", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; - + /* fall through */ case 405: test_ahash_speed("sha384", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; - + /* fall through */ case 406: test_ahash_speed("sha512", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; - + /* fall through */ case 407: test_ahash_speed("wp256", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; - + /* fall through */ case 408: test_ahash_speed("wp384", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; - + /* fall through */ case 409: test_ahash_speed("wp512", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; - + /* fall through */ case 410: test_ahash_speed("tgr128", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; - + /* fall through */ case 411: test_ahash_speed("tgr160", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; - + /* fall through */ case 412: test_ahash_speed("tgr192", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; - + /* fall through */ case 413: test_ahash_speed("sha224", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; - + /* fall through */ case 414: test_ahash_speed("rmd128", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; - + /* fall through */ case 415: test_ahash_speed("rmd160", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; - + /* fall through */ case 416: test_ahash_speed("rmd256", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; - + /* fall through */ case 417: test_ahash_speed("rmd320", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; - + /* fall through */ case 418: test_ahash_speed("sha3-224", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; - + /* fall through */ case 419: test_ahash_speed("sha3-256", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; - + /* fall through */ case 420: test_ahash_speed("sha3-384", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; - - + /* fall through */ case 421: test_ahash_speed("sha3-512", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; - + /* fall through */ case 422: test_mb_ahash_speed("sha1", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; - + /* fall through */ case 423: test_mb_ahash_speed("sha256", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; - + /* fall through */ case 424: test_mb_ahash_speed("sha512", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; - + /* fall through */ + case 425: + test_mb_ahash_speed("sm3", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + /* fall through */ case 499: break; diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 7125ba38..29d7020b 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -76,11 +76,6 @@ int alg_test(const char *driver, const char *alg, u32 type, u32 mask) #define ENCRYPT 1 #define DECRYPT 0 -struct tcrypt_result { - struct completion completion; - int err; -}; - struct aead_test_suite { struct { const struct aead_testvec *vecs; @@ -155,17 +150,6 @@ static void hexdump(unsigned char *buf, unsigned int len) buf, len, false); } -static void tcrypt_complete(struct crypto_async_request *req, int err) -{ - struct tcrypt_result *res = req->data; - - if (err == -EINPROGRESS) - return; - - res->err = err; - complete(&res->completion); -} - static int testmgr_alloc_buf(char *buf[XBUFSIZE]) { int i; @@ -193,20 +177,10 @@ static void testmgr_free_buf(char *buf[XBUFSIZE]) free_page((unsigned long)buf[i]); } -static int wait_async_op(struct tcrypt_result *tr, int ret) -{ - if (ret == -EINPROGRESS || ret == -EBUSY) { - wait_for_completion(&tr->completion); - reinit_completion(&tr->completion); - ret = tr->err; - } - return ret; -} - static int ahash_partial_update(struct ahash_request **preq, struct crypto_ahash *tfm, const struct hash_testvec *template, void *hash_buff, int k, int temp, struct scatterlist *sg, - const char *algo, char *result, struct tcrypt_result *tresult) + const char *algo, char *result, struct crypto_wait *wait) { char *state; struct ahash_request *req; @@ -236,7 +210,7 @@ static int ahash_partial_update(struct ahash_request **preq, } ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - tcrypt_complete, tresult); + crypto_req_done, wait); memcpy(hash_buff, template->plaintext + temp, template->tap[k]); @@ -247,7 +221,7 @@ static int ahash_partial_update(struct ahash_request **preq, pr_err("alg: hash: Failed to import() for %s\n", algo); goto out; } - ret = wait_async_op(tresult, crypto_ahash_update(req)); + ret = crypto_wait_req(crypto_ahash_update(req), wait); if (ret) goto out; *preq = req; @@ -272,7 +246,7 @@ static int __test_hash(struct crypto_ahash *tfm, char *result; char *key; struct ahash_request *req; - struct tcrypt_result tresult; + struct crypto_wait wait; void *hash_buff; char *xbuf[XBUFSIZE]; int ret = -ENOMEM; @@ -286,7 +260,7 @@ static int __test_hash(struct crypto_ahash *tfm, if (testmgr_alloc_buf(xbuf)) goto out_nobuf; - init_completion(&tresult.completion); + crypto_init_wait(&wait); req = ahash_request_alloc(tfm, GFP_KERNEL); if (!req) { @@ -295,7 +269,7 @@ static int __test_hash(struct crypto_ahash *tfm, goto out_noreq; } ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - tcrypt_complete, &tresult); + crypto_req_done, &wait); j = 0; for (i = 0; i < tcount; i++) { @@ -335,26 +309,26 @@ static int __test_hash(struct crypto_ahash *tfm, ahash_request_set_crypt(req, sg, result, template[i].psize); if (use_digest) { - ret = wait_async_op(&tresult, crypto_ahash_digest(req)); + ret = crypto_wait_req(crypto_ahash_digest(req), &wait); if (ret) { pr_err("alg: hash: digest failed on test %d " "for %s: ret=%d\n", j, algo, -ret); goto out; } } else { - ret = wait_async_op(&tresult, crypto_ahash_init(req)); + ret = crypto_wait_req(crypto_ahash_init(req), &wait); if (ret) { pr_err("alg: hash: init failed on test %d " "for %s: ret=%d\n", j, algo, -ret); goto out; } - ret = wait_async_op(&tresult, crypto_ahash_update(req)); + ret = crypto_wait_req(crypto_ahash_update(req), &wait); if (ret) { pr_err("alg: hash: update failed on test %d " "for %s: ret=%d\n", j, algo, -ret); goto out; } - ret = wait_async_op(&tresult, crypto_ahash_final(req)); + ret = crypto_wait_req(crypto_ahash_final(req), &wait); if (ret) { pr_err("alg: hash: final failed on test %d " "for %s: ret=%d\n", j, algo, -ret); @@ -420,22 +394,10 @@ static int __test_hash(struct crypto_ahash *tfm, } ahash_request_set_crypt(req, sg, result, template[i].psize); - ret = crypto_ahash_digest(req); - switch (ret) { - case 0: - break; - case -EINPROGRESS: - case -EBUSY: - wait_for_completion(&tresult.completion); - reinit_completion(&tresult.completion); - ret = tresult.err; - if (!ret) - break; - /* fall through */ - default: - printk(KERN_ERR "alg: hash: digest failed " - "on chunking test %d for %s: " - "ret=%d\n", j, algo, -ret); + ret = crypto_wait_req(crypto_ahash_digest(req), &wait); + if (ret) { + pr_err("alg: hash: digest failed on chunking test %d for %s: ret=%d\n", + j, algo, -ret); goto out; } @@ -486,13 +448,13 @@ static int __test_hash(struct crypto_ahash *tfm, } ahash_request_set_crypt(req, sg, result, template[i].tap[0]); - ret = wait_async_op(&tresult, crypto_ahash_init(req)); + ret = crypto_wait_req(crypto_ahash_init(req), &wait); if (ret) { pr_err("alg: hash: init failed on test %d for %s: ret=%d\n", j, algo, -ret); goto out; } - ret = wait_async_op(&tresult, crypto_ahash_update(req)); + ret = crypto_wait_req(crypto_ahash_update(req), &wait); if (ret) { pr_err("alg: hash: update failed on test %d for %s: ret=%d\n", j, algo, -ret); @@ -503,7 +465,7 @@ static int __test_hash(struct crypto_ahash *tfm, for (k = 1; k < template[i].np; k++) { ret = ahash_partial_update(&req, tfm, &template[i], hash_buff, k, temp, &sg[0], algo, result, - &tresult); + &wait); if (ret) { pr_err("alg: hash: partial update failed on test %d for %s: ret=%d\n", j, algo, -ret); @@ -511,7 +473,7 @@ static int __test_hash(struct crypto_ahash *tfm, } temp += template[i].tap[k]; } - ret = wait_async_op(&tresult, crypto_ahash_final(req)); + ret = crypto_wait_req(crypto_ahash_final(req), &wait); if (ret) { pr_err("alg: hash: final failed on test %d for %s: ret=%d\n", j, algo, -ret); @@ -580,7 +542,7 @@ static int __test_aead(struct crypto_aead *tfm, int enc, struct scatterlist *sg; struct scatterlist *sgout; const char *e, *d; - struct tcrypt_result result; + struct crypto_wait wait; unsigned int authsize, iv_len; void *input; void *output; @@ -619,7 +581,7 @@ static int __test_aead(struct crypto_aead *tfm, int enc, else e = "decryption"; - init_completion(&result.completion); + crypto_init_wait(&wait); req = aead_request_alloc(tfm, GFP_KERNEL); if (!req) { @@ -629,7 +591,7 @@ static int __test_aead(struct crypto_aead *tfm, int enc, } aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - tcrypt_complete, &result); + crypto_req_done, &wait); iv_len = crypto_aead_ivsize(tfm); @@ -709,7 +671,8 @@ static int __test_aead(struct crypto_aead *tfm, int enc, aead_request_set_ad(req, template[i].alen); - ret = enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req); + ret = crypto_wait_req(enc ? crypto_aead_encrypt(req) + : crypto_aead_decrypt(req), &wait); switch (ret) { case 0: @@ -722,13 +685,6 @@ static int __test_aead(struct crypto_aead *tfm, int enc, goto out; } break; - case -EINPROGRESS: - case -EBUSY: - wait_for_completion(&result.completion); - reinit_completion(&result.completion); - ret = result.err; - if (!ret) - break; case -EBADMSG: if (template[i].novrfy) /* verification failure was expected */ @@ -866,7 +822,8 @@ static int __test_aead(struct crypto_aead *tfm, int enc, aead_request_set_ad(req, template[i].alen); - ret = enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req); + ret = crypto_wait_req(enc ? crypto_aead_encrypt(req) + : crypto_aead_decrypt(req), &wait); switch (ret) { case 0: @@ -879,13 +836,6 @@ static int __test_aead(struct crypto_aead *tfm, int enc, goto out; } break; - case -EINPROGRESS: - case -EBUSY: - wait_for_completion(&result.completion); - reinit_completion(&result.completion); - ret = result.err; - if (!ret) - break; case -EBADMSG: if (template[i].novrfy) /* verification failure was expected */ @@ -1083,7 +1033,7 @@ static int __test_skcipher(struct crypto_skcipher *tfm, int enc, struct scatterlist sg[8]; struct scatterlist sgout[8]; const char *e, *d; - struct tcrypt_result result; + struct crypto_wait wait; void *data; char iv[MAX_IVLEN]; char *xbuf[XBUFSIZE]; @@ -1107,7 +1057,7 @@ static int __test_skcipher(struct crypto_skcipher *tfm, int enc, else e = "decryption"; - init_completion(&result.completion); + crypto_init_wait(&wait); req = skcipher_request_alloc(tfm, GFP_KERNEL); if (!req) { @@ -1117,7 +1067,7 @@ static int __test_skcipher(struct crypto_skcipher *tfm, int enc, } skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - tcrypt_complete, &result); + crypto_req_done, &wait); j = 0; for (i = 0; i < tcount; i++) { @@ -1164,21 +1114,10 @@ static int __test_skcipher(struct crypto_skcipher *tfm, int enc, skcipher_request_set_crypt(req, sg, (diff_dst) ? sgout : sg, template[i].ilen, iv); - ret = enc ? crypto_skcipher_encrypt(req) : - crypto_skcipher_decrypt(req); + ret = crypto_wait_req(enc ? crypto_skcipher_encrypt(req) : + crypto_skcipher_decrypt(req), &wait); - switch (ret) { - case 0: - break; - case -EINPROGRESS: - case -EBUSY: - wait_for_completion(&result.completion); - reinit_completion(&result.completion); - ret = result.err; - if (!ret) - break; - /* fall through */ - default: + if (ret) { pr_err("alg: skcipher%s: %s failed on test %d for %s: ret=%d\n", d, e, j, algo, -ret); goto out; @@ -1272,21 +1211,10 @@ static int __test_skcipher(struct crypto_skcipher *tfm, int enc, skcipher_request_set_crypt(req, sg, (diff_dst) ? sgout : sg, template[i].ilen, iv); - ret = enc ? crypto_skcipher_encrypt(req) : - crypto_skcipher_decrypt(req); + ret = crypto_wait_req(enc ? crypto_skcipher_encrypt(req) : + crypto_skcipher_decrypt(req), &wait); - switch (ret) { - case 0: - break; - case -EINPROGRESS: - case -EBUSY: - wait_for_completion(&result.completion); - reinit_completion(&result.completion); - ret = result.err; - if (!ret) - break; - /* fall through */ - default: + if (ret) { pr_err("alg: skcipher%s: %s failed on chunk test %d for %s: ret=%d\n", d, e, j, algo, -ret); goto out; @@ -1462,7 +1390,7 @@ static int test_acomp(struct crypto_acomp *tfm, int ret; struct scatterlist src, dst; struct acomp_req *req; - struct tcrypt_result result; + struct crypto_wait wait; output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL); if (!output) @@ -1486,7 +1414,7 @@ static int test_acomp(struct crypto_acomp *tfm, } memset(output, 0, dlen); - init_completion(&result.completion); + crypto_init_wait(&wait); sg_init_one(&src, input_vec, ilen); sg_init_one(&dst, output, dlen); @@ -1501,9 +1429,9 @@ static int test_acomp(struct crypto_acomp *tfm, acomp_request_set_params(req, &src, &dst, ilen, dlen); acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - tcrypt_complete, &result); + crypto_req_done, &wait); - ret = wait_async_op(&result, crypto_acomp_compress(req)); + ret = crypto_wait_req(crypto_acomp_compress(req), &wait); if (ret) { pr_err("alg: acomp: compression failed on test %d for %s: ret=%d\n", i + 1, algo, -ret); @@ -1516,10 +1444,10 @@ static int test_acomp(struct crypto_acomp *tfm, dlen = COMP_BUF_SIZE; sg_init_one(&src, output, ilen); sg_init_one(&dst, decomp_out, dlen); - init_completion(&result.completion); + crypto_init_wait(&wait); acomp_request_set_params(req, &src, &dst, ilen, dlen); - ret = wait_async_op(&result, crypto_acomp_decompress(req)); + ret = crypto_wait_req(crypto_acomp_decompress(req), &wait); if (ret) { pr_err("alg: acomp: compression failed on test %d for %s: ret=%d\n", i + 1, algo, -ret); @@ -1563,7 +1491,7 @@ static int test_acomp(struct crypto_acomp *tfm, } memset(output, 0, dlen); - init_completion(&result.completion); + crypto_init_wait(&wait); sg_init_one(&src, input_vec, ilen); sg_init_one(&dst, output, dlen); @@ -1578,9 +1506,9 @@ static int test_acomp(struct crypto_acomp *tfm, acomp_request_set_params(req, &src, &dst, ilen, dlen); acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - tcrypt_complete, &result); + crypto_req_done, &wait); - ret = wait_async_op(&result, crypto_acomp_decompress(req)); + ret = crypto_wait_req(crypto_acomp_decompress(req), &wait); if (ret) { pr_err("alg: acomp: decompression failed on test %d for %s: ret=%d\n", i + 1, algo, -ret); @@ -2000,7 +1928,7 @@ static int do_test_kpp(struct crypto_kpp *tfm, const struct kpp_testvec *vec, void *a_public = NULL; void *a_ss = NULL; void *shared_secret = NULL; - struct tcrypt_result result; + struct crypto_wait wait; unsigned int out_len_max; int err = -ENOMEM; struct scatterlist src, dst; @@ -2009,7 +1937,7 @@ static int do_test_kpp(struct crypto_kpp *tfm, const struct kpp_testvec *vec, if (!req) return err; - init_completion(&result.completion); + crypto_init_wait(&wait); err = crypto_kpp_set_secret(tfm, vec->secret, vec->secret_size); if (err < 0) @@ -2027,10 +1955,10 @@ static int do_test_kpp(struct crypto_kpp *tfm, const struct kpp_testvec *vec, sg_init_one(&dst, output_buf, out_len_max); kpp_request_set_output(req, &dst, out_len_max); kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - tcrypt_complete, &result); + crypto_req_done, &wait); /* Compute party A's public key */ - err = wait_async_op(&result, crypto_kpp_generate_public_key(req)); + err = crypto_wait_req(crypto_kpp_generate_public_key(req), &wait); if (err) { pr_err("alg: %s: Party A: generate public key test failed. err %d\n", alg, err); @@ -2069,8 +1997,8 @@ static int do_test_kpp(struct crypto_kpp *tfm, const struct kpp_testvec *vec, kpp_request_set_input(req, &src, vec->b_public_size); kpp_request_set_output(req, &dst, out_len_max); kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - tcrypt_complete, &result); - err = wait_async_op(&result, crypto_kpp_compute_shared_secret(req)); + crypto_req_done, &wait); + err = crypto_wait_req(crypto_kpp_compute_shared_secret(req), &wait); if (err) { pr_err("alg: %s: Party A: compute shared secret test failed. err %d\n", alg, err); @@ -2100,9 +2028,9 @@ static int do_test_kpp(struct crypto_kpp *tfm, const struct kpp_testvec *vec, kpp_request_set_input(req, &src, vec->expected_a_public_size); kpp_request_set_output(req, &dst, out_len_max); kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - tcrypt_complete, &result); - err = wait_async_op(&result, - crypto_kpp_compute_shared_secret(req)); + crypto_req_done, &wait); + err = crypto_wait_req(crypto_kpp_compute_shared_secret(req), + &wait); if (err) { pr_err("alg: %s: Party B: compute shared secret failed. err %d\n", alg, err); @@ -2179,7 +2107,7 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, struct akcipher_request *req; void *outbuf_enc = NULL; void *outbuf_dec = NULL; - struct tcrypt_result result; + struct crypto_wait wait; unsigned int out_len_max, out_len = 0; int err = -ENOMEM; struct scatterlist src, dst, src_tab[2]; @@ -2191,7 +2119,7 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, if (!req) goto free_xbuf; - init_completion(&result.completion); + crypto_init_wait(&wait); if (vecs->public_key_vec) err = crypto_akcipher_set_pub_key(tfm, vecs->key, @@ -2220,13 +2148,13 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, akcipher_request_set_crypt(req, src_tab, &dst, vecs->m_size, out_len_max); akcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - tcrypt_complete, &result); + crypto_req_done, &wait); - err = wait_async_op(&result, vecs->siggen_sigver_test ? - /* Run asymmetric signature generation */ - crypto_akcipher_sign(req) : - /* Run asymmetric encrypt */ - crypto_akcipher_encrypt(req)); + err = crypto_wait_req(vecs->siggen_sigver_test ? + /* Run asymmetric signature generation */ + crypto_akcipher_sign(req) : + /* Run asymmetric encrypt */ + crypto_akcipher_encrypt(req), &wait); if (err) { pr_err("alg: akcipher: encrypt test failed. err %d\n", err); goto free_all; @@ -2261,14 +2189,14 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, sg_init_one(&src, xbuf[0], vecs->c_size); sg_init_one(&dst, outbuf_dec, out_len_max); - init_completion(&result.completion); + crypto_init_wait(&wait); akcipher_request_set_crypt(req, &src, &dst, vecs->c_size, out_len_max); - err = wait_async_op(&result, vecs->siggen_sigver_test ? - /* Run asymmetric signature verification */ - crypto_akcipher_verify(req) : - /* Run asymmetric decrypt */ - crypto_akcipher_decrypt(req)); + err = crypto_wait_req(vecs->siggen_sigver_test ? + /* Run asymmetric signature verification */ + crypto_akcipher_verify(req) : + /* Run asymmetric decrypt */ + crypto_akcipher_decrypt(req), &wait); if (err) { pr_err("alg: akcipher: decrypt test failed. err %d\n", err); goto free_all; @@ -3500,6 +3428,12 @@ static const struct alg_test_desc alg_test_descs[] = { .hash = __VECS(sha512_tv_template) } }, { + .alg = "sm3", + .test = alg_test_hash, + .suite = { + .hash = __VECS(sm3_tv_template) + } + }, { .alg = "tgr128", .test = alg_test_hash, .suite = { diff --git a/crypto/testmgr.h b/crypto/testmgr.h index d54971d2..a714b629 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -1497,6 +1497,73 @@ static const struct hash_testvec crct10dif_tv_template[] = { } }; +/* Example vectors below taken from + * http://www.oscca.gov.cn/UpFile/20101222141857786.pdf + * + * The rest taken from + * https://github.com/adamws/oscca-sm3 + */ +static const struct hash_testvec sm3_tv_template[] = { + { + .plaintext = "", + .psize = 0, + .digest = (u8 *)(u8 []) { + 0x1A, 0xB2, 0x1D, 0x83, 0x55, 0xCF, 0xA1, 0x7F, + 0x8e, 0x61, 0x19, 0x48, 0x31, 0xE8, 0x1A, 0x8F, + 0x22, 0xBE, 0xC8, 0xC7, 0x28, 0xFE, 0xFB, 0x74, + 0x7E, 0xD0, 0x35, 0xEB, 0x50, 0x82, 0xAA, 0x2B } + }, { + .plaintext = "a", + .psize = 1, + .digest = (u8 *)(u8 []) { + 0x62, 0x34, 0x76, 0xAC, 0x18, 0xF6, 0x5A, 0x29, + 0x09, 0xE4, 0x3C, 0x7F, 0xEC, 0x61, 0xB4, 0x9C, + 0x7E, 0x76, 0x4A, 0x91, 0xA1, 0x8C, 0xCB, 0x82, + 0xF1, 0x91, 0x7A, 0x29, 0xC8, 0x6C, 0x5E, 0x88 } + }, { + /* A.1. Example 1 */ + .plaintext = "abc", + .psize = 3, + .digest = (u8 *)(u8 []) { + 0x66, 0xC7, 0xF0, 0xF4, 0x62, 0xEE, 0xED, 0xD9, + 0xD1, 0xF2, 0xD4, 0x6B, 0xDC, 0x10, 0xE4, 0xE2, + 0x41, 0x67, 0xC4, 0x87, 0x5C, 0xF2, 0xF7, 0xA2, + 0x29, 0x7D, 0xA0, 0x2B, 0x8F, 0x4B, 0xA8, 0xE0 } + }, { + .plaintext = "abcdefghijklmnopqrstuvwxyz", + .psize = 26, + .digest = (u8 *)(u8 []) { + 0xB8, 0x0F, 0xE9, 0x7A, 0x4D, 0xA2, 0x4A, 0xFC, + 0x27, 0x75, 0x64, 0xF6, 0x6A, 0x35, 0x9E, 0xF4, + 0x40, 0x46, 0x2A, 0xD2, 0x8D, 0xCC, 0x6D, 0x63, + 0xAD, 0xB2, 0x4D, 0x5C, 0x20, 0xA6, 0x15, 0x95 } + }, { + /* A.1. Example 2 */ + .plaintext = "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdab" + "cdabcdabcdabcdabcd", + .psize = 64, + .digest = (u8 *)(u8 []) { + 0xDE, 0xBE, 0x9F, 0xF9, 0x22, 0x75, 0xB8, 0xA1, + 0x38, 0x60, 0x48, 0x89, 0xC1, 0x8E, 0x5A, 0x4D, + 0x6F, 0xDB, 0x70, 0xE5, 0x38, 0x7E, 0x57, 0x65, + 0x29, 0x3D, 0xCB, 0xA3, 0x9C, 0x0C, 0x57, 0x32 } + }, { + .plaintext = "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd" + "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd" + "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd" + "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd" + "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd" + "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd" + "abcdabcdabcdabcdabcdabcdabcdabcd", + .psize = 256, + .digest = (u8 *)(u8 []) { + 0xB9, 0x65, 0x76, 0x4C, 0x8B, 0xEB, 0xB0, 0x91, + 0xC7, 0x60, 0x2B, 0x74, 0xAF, 0xD3, 0x4E, 0xEF, + 0xB5, 0x31, 0xDC, 0xCB, 0x4E, 0x00, 0x76, 0xD9, + 0xB7, 0xCD, 0x81, 0x31, 0x99, 0xB4, 0x59, 0x71 } + } +}; + /* * SHA1 test vectors from from FIPS PUB 180-1 * Long vector from CAVS 5.0 diff --git a/crypto/xor.c b/crypto/xor.c index 263af9fb..bce9fe7a 100644 --- a/crypto/xor.c +++ b/crypto/xor.c @@ -122,12 +122,7 @@ calibrate_xor_blocks(void) goto out; } - /* - * Note: Since the memory is not actually used for _anything_ but to - * test the XOR speed, we don't really want kmemcheck to warn about - * reading uninitialized bytes here. - */ - b1 = (void *) __get_free_pages(GFP_KERNEL | __GFP_NOTRACK, 2); + b1 = (void *) __get_free_pages(GFP_KERNEL, 2); if (!b1) { printk(KERN_WARNING "xor: Yikes! No memory available.\n"); return -ENOMEM; diff --git a/crypto/xts.c b/crypto/xts.c index e31828ed..f317c48b 100644 --- a/crypto/xts.c +++ b/crypto/xts.c @@ -269,9 +269,7 @@ static int do_encrypt(struct skcipher_request *req, int err) crypto_skcipher_encrypt(subreq) ?: post_crypt(req); - if (err == -EINPROGRESS || - (err == -EBUSY && - req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + if (err == -EINPROGRESS || err == -EBUSY) return err; } @@ -321,9 +319,7 @@ static int do_decrypt(struct skcipher_request *req, int err) crypto_skcipher_decrypt(subreq) ?: post_crypt(req); - if (err == -EINPROGRESS || - (err == -EBUSY && - req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + if (err == -EINPROGRESS || err == -EBUSY) return err; } |