diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2019-09-18 12:11:14 -0700 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2019-09-18 12:11:14 -0700 |
commit | d95e2b63e317a3532c0d20695a5dd8b156acd707 (patch) | |
tree | 07cf0a14fd81808c897f37190ce762af6c678d94 /crypto/xts.c | |
parent | c37dd209adcf9a17dccab32db8a0f9fa068bf81b (diff) | |
parent | 24a9f0e4cb52070c48602e9e6d31470dd98ccdcf (diff) | |
download | linux-crypto-d95e2b63e317a3532c0d20695a5dd8b156acd707.tar.gz linux-crypto-d95e2b63e317a3532c0d20695a5dd8b156acd707.zip |
Merge branch 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6
Pull crypto updates from Herbert Xu:
"API:
- Add the ability to abort a skcipher walk.
Algorithms:
- Fix XTS to actually do the stealing.
- Add library helpers for AES and DES for single-block users.
- Add library helpers for SHA256.
- Add new DES key verification helper.
- Add surrounding bits for ESSIV generator.
- Add accelerations for aegis128.
- Add test vectors for lzo-rle.
Drivers:
- Add i.MX8MQ support to caam.
- Add gcm/ccm/cfb/ofb aes support in inside-secure.
- Add ofb/cfb aes support in media-tek.
- Add HiSilicon ZIP accelerator support.
Others:
- Fix potential race condition in padata.
- Use unbound workqueues in padata"
* 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (311 commits)
crypto: caam - Cast to long first before pointer conversion
crypto: ccree - enable CTS support in AES-XTS
crypto: inside-secure - Probe transform record cache RAM sizes
crypto: inside-secure - Base RD fetchcount on actual RD FIFO size
crypto: inside-secure - Base CD fetchcount on actual CD FIFO size
crypto: inside-secure - Enable extended algorithms on newer HW
crypto: inside-secure: Corrected configuration of EIP96_TOKEN_CTRL
crypto: inside-secure - Add EIP97/EIP197 and endianness detection
padata: remove cpu_index from the parallel_queue
padata: unbind parallel jobs from specific CPUs
padata: use separate workqueues for parallel and serial work
padata, pcrypt: take CPU hotplug lock internally in padata_alloc_possible
crypto: pcrypt - remove padata cpumask notifier
padata: make padata_do_parallel find alternate callback CPU
workqueue: require CPU hotplug read exclusion for apply_workqueue_attrs
workqueue: unconfine alloc/apply/free_workqueue_attrs()
padata: allocate workqueue internally
arm64: dts: imx8mq: Add CAAM node
random: Use wait_event_freezable() in add_hwgenerator_randomness()
crypto: ux500 - Fix COMPILE_TEST warnings
...
Diffstat (limited to 'crypto/xts.c')
-rw-r--r-- | crypto/xts.c | 152 |
1 files changed, 132 insertions, 20 deletions
diff --git a/crypto/xts.c b/crypto/xts.c index 11211003..ab117633 100644 --- a/crypto/xts.c +++ b/crypto/xts.c @@ -1,8 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* XTS: as defined in IEEE1619/D16 * http://grouper.ieee.org/groups/1619/email/pdf00086.pdf - * (sector sizes which are not a multiple of 16 bytes are, - * however currently unsupported) * * Copyright (c) 2007 Rik Snel <rsnel@cube.dyndns.org> * @@ -34,6 +32,8 @@ struct xts_instance_ctx { struct rctx { le128 t; + struct scatterlist *tail; + struct scatterlist sg[2]; struct skcipher_request subreq; }; @@ -84,10 +84,11 @@ static int setkey(struct crypto_skcipher *parent, const u8 *key, * mutliple calls to the 'ecb(..)' instance, which usually would be slower than * just doing the gf128mul_x_ble() calls again. */ -static int xor_tweak(struct skcipher_request *req, bool second_pass) +static int xor_tweak(struct skcipher_request *req, bool second_pass, bool enc) { struct rctx *rctx = skcipher_request_ctx(req); struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const bool cts = (req->cryptlen % XTS_BLOCK_SIZE); const int bs = XTS_BLOCK_SIZE; struct skcipher_walk w; le128 t = rctx->t; @@ -109,6 +110,20 @@ static int xor_tweak(struct skcipher_request *req, bool second_pass) wdst = w.dst.virt.addr; do { + if (unlikely(cts) && + w.total - w.nbytes + avail < 2 * XTS_BLOCK_SIZE) { + if (!enc) { + if (second_pass) + rctx->t = t; + gf128mul_x_ble(&t, &t); + } + le128_xor(wdst, &t, wsrc); + if (enc && second_pass) + gf128mul_x_ble(&rctx->t, &t); + skcipher_walk_done(&w, avail - bs); + return 0; + } + le128_xor(wdst++, &t, wsrc++); gf128mul_x_ble(&t, &t); } while ((avail -= bs) >= bs); @@ -119,17 +134,71 @@ static int xor_tweak(struct skcipher_request *req, bool second_pass) return err; } -static int xor_tweak_pre(struct skcipher_request *req) +static int xor_tweak_pre(struct skcipher_request *req, bool enc) { - return xor_tweak(req, false); + return xor_tweak(req, false, enc); } -static int xor_tweak_post(struct skcipher_request *req) +static int xor_tweak_post(struct skcipher_request *req, bool enc) { - return xor_tweak(req, true); + return xor_tweak(req, true, enc); } -static void crypt_done(struct crypto_async_request *areq, int err) +static void cts_done(struct crypto_async_request *areq, int err) +{ + struct skcipher_request *req = areq->data; + le128 b; + + if (!err) { + struct rctx *rctx = skcipher_request_ctx(req); + + scatterwalk_map_and_copy(&b, rctx->tail, 0, XTS_BLOCK_SIZE, 0); + le128_xor(&b, &rctx->t, &b); + scatterwalk_map_and_copy(&b, rctx->tail, 0, XTS_BLOCK_SIZE, 1); + } + + skcipher_request_complete(req, err); +} + +static int cts_final(struct skcipher_request *req, + int (*crypt)(struct skcipher_request *req)) +{ + struct priv *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + int offset = req->cryptlen & ~(XTS_BLOCK_SIZE - 1); + struct rctx *rctx = skcipher_request_ctx(req); + struct skcipher_request *subreq = &rctx->subreq; + int tail = req->cryptlen % XTS_BLOCK_SIZE; + le128 b[2]; + int err; + + rctx->tail = scatterwalk_ffwd(rctx->sg, req->dst, + offset - XTS_BLOCK_SIZE); + + scatterwalk_map_and_copy(b, rctx->tail, 0, XTS_BLOCK_SIZE, 0); + memcpy(b + 1, b, tail); + scatterwalk_map_and_copy(b, req->src, offset, tail, 0); + + le128_xor(b, &rctx->t, b); + + scatterwalk_map_and_copy(b, rctx->tail, 0, XTS_BLOCK_SIZE + tail, 1); + + skcipher_request_set_tfm(subreq, ctx->child); + skcipher_request_set_callback(subreq, req->base.flags, cts_done, req); + skcipher_request_set_crypt(subreq, rctx->tail, rctx->tail, + XTS_BLOCK_SIZE, NULL); + + err = crypt(subreq); + if (err) + return err; + + scatterwalk_map_and_copy(b, rctx->tail, 0, XTS_BLOCK_SIZE, 0); + le128_xor(b, &rctx->t, b); + scatterwalk_map_and_copy(b, rctx->tail, 0, XTS_BLOCK_SIZE, 1); + + return 0; +} + +static void encrypt_done(struct crypto_async_request *areq, int err) { struct skcipher_request *req = areq->data; @@ -137,47 +206,90 @@ static void crypt_done(struct crypto_async_request *areq, int err) struct rctx *rctx = skcipher_request_ctx(req); rctx->subreq.base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; - err = xor_tweak_post(req); + err = xor_tweak_post(req, true); + + if (!err && unlikely(req->cryptlen % XTS_BLOCK_SIZE)) { + err = cts_final(req, crypto_skcipher_encrypt); + if (err == -EINPROGRESS) + return; + } } skcipher_request_complete(req, err); } -static void init_crypt(struct skcipher_request *req) +static void decrypt_done(struct crypto_async_request *areq, int err) +{ + struct skcipher_request *req = areq->data; + + if (!err) { + struct rctx *rctx = skcipher_request_ctx(req); + + rctx->subreq.base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + err = xor_tweak_post(req, false); + + if (!err && unlikely(req->cryptlen % XTS_BLOCK_SIZE)) { + err = cts_final(req, crypto_skcipher_decrypt); + if (err == -EINPROGRESS) + return; + } + } + + skcipher_request_complete(req, err); +} + +static int init_crypt(struct skcipher_request *req, crypto_completion_t compl) { struct priv *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); struct rctx *rctx = skcipher_request_ctx(req); struct skcipher_request *subreq = &rctx->subreq; + if (req->cryptlen < XTS_BLOCK_SIZE) + return -EINVAL; + skcipher_request_set_tfm(subreq, ctx->child); - skcipher_request_set_callback(subreq, req->base.flags, crypt_done, req); + skcipher_request_set_callback(subreq, req->base.flags, compl, req); skcipher_request_set_crypt(subreq, req->dst, req->dst, - req->cryptlen, NULL); + req->cryptlen & ~(XTS_BLOCK_SIZE - 1), NULL); /* calculate first value of T */ crypto_cipher_encrypt_one(ctx->tweak, (u8 *)&rctx->t, req->iv); + + return 0; } static int encrypt(struct skcipher_request *req) { struct rctx *rctx = skcipher_request_ctx(req); struct skcipher_request *subreq = &rctx->subreq; + int err; - init_crypt(req); - return xor_tweak_pre(req) ?: - crypto_skcipher_encrypt(subreq) ?: - xor_tweak_post(req); + err = init_crypt(req, encrypt_done) ?: + xor_tweak_pre(req, true) ?: + crypto_skcipher_encrypt(subreq) ?: + xor_tweak_post(req, true); + + if (err || likely((req->cryptlen % XTS_BLOCK_SIZE) == 0)) + return err; + + return cts_final(req, crypto_skcipher_encrypt); } static int decrypt(struct skcipher_request *req) { struct rctx *rctx = skcipher_request_ctx(req); struct skcipher_request *subreq = &rctx->subreq; + int err; + + err = init_crypt(req, decrypt_done) ?: + xor_tweak_pre(req, false) ?: + crypto_skcipher_decrypt(subreq) ?: + xor_tweak_post(req, false); + + if (err || likely((req->cryptlen % XTS_BLOCK_SIZE) == 0)) + return err; - init_crypt(req); - return xor_tweak_pre(req) ?: - crypto_skcipher_decrypt(subreq) ?: - xor_tweak_post(req); + return cts_final(req, crypto_skcipher_decrypt); } static int init_tfm(struct crypto_skcipher *tfm) |