From 0eadd1eb3fd1251972218b40696b7f18e9521a5e Mon Sep 17 00:00:00 2001 From: Iuliana Prodan Date: Tue, 28 Apr 2020 18:49:04 +0300 Subject: crypto: engine - support for parallel requests based on retry mechanism Added support for executing multiple requests, in parallel, for crypto engine based on a retry mechanism. If hardware was unable to execute a backlog request, enqueue it back in front of crypto-engine queue, to keep the order of requests. A new variable is added, retry_support (this is to keep the backward compatibility of crypto-engine) , which keeps track whether the hardware has support for retry mechanism and, also, if can run multiple requests. If do_one_request() returns: >= 0: hardware executed the request successfully; < 0: this is the old error path. If hardware has support for retry mechanism, the request is put back in front of crypto-engine queue. For backwards compatibility, if the retry support is not available, the crypto-engine will work as before. If hardware queue is full (-ENOSPC), requeue request regardless of MAY_BACKLOG flag. If hardware throws any other error code (like -EIO, -EINVAL, -ENOMEM, etc.) only MAY_BACKLOG requests are enqueued back into crypto-engine's queue, since the others can be dropped. The new crypto_engine_alloc_init_and_set function, initializes crypto-engine, sets the maximum size for crypto-engine software queue (not hardcoded anymore) and the retry_support variable is set, by default, to false. On crypto_pump_requests(), if do_one_request() returns >= 0, a new request is send to hardware, until there is no space in hardware and do_one_request() returns < 0. By default, retry_support is false and crypto-engine will work as before - will send requests to hardware, one-by-one, on crypto_pump_requests(), and complete it, on crypto_finalize_request(), and so on. To support multiple requests, in each driver, retry_support must be set on true, and if do_one_request() returns an error the request must not be freed, since it will be enqueued back into crypto-engine's queue. When all drivers, that use crypto-engine now, will be updated for retry mechanism, the retry_support variable can be removed. Signed-off-by: Iuliana Prodan Signed-off-by: Herbert Xu --- crypto/crypto_engine.c | 146 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 116 insertions(+), 30 deletions(-) (limited to 'crypto/crypto_engine.c') diff --git a/crypto/crypto_engine.c b/crypto/crypto_engine.c index eb029ff1..ee192731 100644 --- a/crypto/crypto_engine.c +++ b/crypto/crypto_engine.c @@ -22,32 +22,36 @@ * @err: error number */ static void crypto_finalize_request(struct crypto_engine *engine, - struct crypto_async_request *req, int err) + struct crypto_async_request *req, int err) { unsigned long flags; - bool finalize_cur_req = false; + bool finalize_req = false; int ret; struct crypto_engine_ctx *enginectx; - spin_lock_irqsave(&engine->queue_lock, flags); - if (engine->cur_req == req) - finalize_cur_req = true; - spin_unlock_irqrestore(&engine->queue_lock, flags); + /* + * If hardware cannot enqueue more requests + * and retry mechanism is not supported + * make sure we are completing the current request + */ + if (!engine->retry_support) { + spin_lock_irqsave(&engine->queue_lock, flags); + if (engine->cur_req == req) { + finalize_req = true; + engine->cur_req = NULL; + } + spin_unlock_irqrestore(&engine->queue_lock, flags); + } - if (finalize_cur_req) { + if (finalize_req || engine->retry_support) { enginectx = crypto_tfm_ctx(req->tfm); - if (engine->cur_req_prepared && + if (enginectx->op.prepare_request && enginectx->op.unprepare_request) { ret = enginectx->op.unprepare_request(engine, req); if (ret) dev_err(engine->dev, "failed to unprepare request\n"); } - spin_lock_irqsave(&engine->queue_lock, flags); - engine->cur_req = NULL; - engine->cur_req_prepared = false; - spin_unlock_irqrestore(&engine->queue_lock, flags); } - req->complete(req, err); kthread_queue_work(engine->kworker, &engine->pump_requests); @@ -74,7 +78,7 @@ static void crypto_pump_requests(struct crypto_engine *engine, spin_lock_irqsave(&engine->queue_lock, flags); /* Make sure we are not already running a request */ - if (engine->cur_req) + if (!engine->retry_support && engine->cur_req) goto out; /* If another context is idling then defer */ @@ -108,13 +112,21 @@ static void crypto_pump_requests(struct crypto_engine *engine, goto out; } +start_request: /* Get the fist request from the engine queue to handle */ backlog = crypto_get_backlog(&engine->queue); async_req = crypto_dequeue_request(&engine->queue); if (!async_req) goto out; - engine->cur_req = async_req; + /* + * If hardware doesn't support the retry mechanism, + * keep track of the request we are processing now. + * We'll need it on completion (crypto_finalize_request). + */ + if (!engine->retry_support) + engine->cur_req = async_req; + if (backlog) backlog->complete(backlog, -EINPROGRESS); @@ -130,7 +142,7 @@ static void crypto_pump_requests(struct crypto_engine *engine, ret = engine->prepare_crypt_hardware(engine); if (ret) { dev_err(engine->dev, "failed to prepare crypt hardware\n"); - goto req_err; + goto req_err_2; } } @@ -141,28 +153,81 @@ static void crypto_pump_requests(struct crypto_engine *engine, if (ret) { dev_err(engine->dev, "failed to prepare request: %d\n", ret); - goto req_err; + goto req_err_2; } - engine->cur_req_prepared = true; } if (!enginectx->op.do_one_request) { dev_err(engine->dev, "failed to do request\n"); ret = -EINVAL; - goto req_err; + goto req_err_1; } + ret = enginectx->op.do_one_request(engine, async_req); - if (ret) { - dev_err(engine->dev, "Failed to do one request from queue: %d\n", ret); - goto req_err; + + /* Request unsuccessfully executed by hardware */ + if (ret < 0) { + /* + * If hardware queue is full (-ENOSPC), requeue request + * regardless of backlog flag. + * If hardware throws any other error code, + * requeue only backlog requests. + * Otherwise, unprepare and complete the request. + */ + if (!engine->retry_support || + ((ret != -ENOSPC) && + !(async_req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG))) { + dev_err(engine->dev, + "Failed to do one request from queue: %d\n", + ret); + goto req_err_1; + } + /* + * If retry mechanism is supported, + * unprepare current request and + * enqueue it back into crypto-engine queue. + */ + if (enginectx->op.unprepare_request) { + ret = enginectx->op.unprepare_request(engine, + async_req); + if (ret) + dev_err(engine->dev, + "failed to unprepare request\n"); + } + spin_lock_irqsave(&engine->queue_lock, flags); + /* + * If hardware was unable to execute request, enqueue it + * back in front of crypto-engine queue, to keep the order + * of requests. + */ + crypto_enqueue_request_head(&engine->queue, async_req); + + kthread_queue_work(engine->kworker, &engine->pump_requests); + goto out; } - return; -req_err: - crypto_finalize_request(engine, async_req, ret); + goto retry; + +req_err_1: + if (enginectx->op.unprepare_request) { + ret = enginectx->op.unprepare_request(engine, async_req); + if (ret) + dev_err(engine->dev, "failed to unprepare request\n"); + } + +req_err_2: + async_req->complete(async_req, ret); + +retry: + /* If retry mechanism is supported, send new requests to engine */ + if (engine->retry_support) { + spin_lock_irqsave(&engine->queue_lock, flags); + goto start_request; + } return; out: spin_unlock_irqrestore(&engine->queue_lock, flags); + return; } static void crypto_pump_work(struct kthread_work *work) @@ -386,15 +451,20 @@ int crypto_engine_stop(struct crypto_engine *engine) EXPORT_SYMBOL_GPL(crypto_engine_stop); /** - * crypto_engine_alloc_init - allocate crypto hardware engine structure and - * initialize it. + * crypto_engine_alloc_init_and_set - allocate crypto hardware engine structure + * and initialize it by setting the maximum number of entries in the software + * crypto-engine queue. * @dev: the device attached with one hardware engine + * @retry_support: whether hardware has support for retry mechanism * @rt: whether this queue is set to run as a realtime task + * @qlen: maximum size of the crypto-engine queue * * This must be called from context that can sleep. * Return: the crypto engine structure on success, else NULL. */ -struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt) +struct crypto_engine *crypto_engine_alloc_init_and_set(struct device *dev, + bool retry_support, + bool rt, int qlen) { struct sched_param param = { .sched_priority = MAX_RT_PRIO / 2 }; struct crypto_engine *engine; @@ -411,12 +481,12 @@ struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt) engine->running = false; engine->busy = false; engine->idling = false; - engine->cur_req_prepared = false; + engine->retry_support = retry_support; engine->priv_data = dev; snprintf(engine->name, sizeof(engine->name), "%s-engine", dev_name(dev)); - crypto_init_queue(&engine->queue, CRYPTO_ENGINE_MAX_QLEN); + crypto_init_queue(&engine->queue, qlen); spin_lock_init(&engine->queue_lock); engine->kworker = kthread_create_worker(0, "%s", engine->name); @@ -433,6 +503,22 @@ struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt) return engine; } +EXPORT_SYMBOL_GPL(crypto_engine_alloc_init_and_set); + +/** + * crypto_engine_alloc_init - allocate crypto hardware engine structure and + * initialize it. + * @dev: the device attached with one hardware engine + * @rt: whether this queue is set to run as a realtime task + * + * This must be called from context that can sleep. + * Return: the crypto engine structure on success, else NULL. + */ +struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt) +{ + return crypto_engine_alloc_init_and_set(dev, false, rt, + CRYPTO_ENGINE_MAX_QLEN); +} EXPORT_SYMBOL_GPL(crypto_engine_alloc_init); /** -- cgit v1.2.3 From d87bb7584bf1fed90dc9fbb048b80a1df1bac508 Mon Sep 17 00:00:00 2001 From: Iuliana Prodan Date: Tue, 28 Apr 2020 18:49:05 +0300 Subject: crypto: engine - support for batch requests Added support for batch requests, per crypto engine. A new callback is added, do_batch_requests, which executes a batch of requests. This has the crypto_engine structure as argument (for cases when more than one crypto-engine is used). The crypto_engine_alloc_init_and_set function, initializes crypto-engine, but also, sets the do_batch_requests callback. On crypto_pump_requests, if do_batch_requests callback is implemented in a driver, this will be executed. The link between the requests will be done in driver, if possible. do_batch_requests is available only if the hardware has support for multiple request. Signed-off-by: Iuliana Prodan Signed-off-by: Herbert Xu --- crypto/crypto_engine.c | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) (limited to 'crypto/crypto_engine.c') diff --git a/crypto/crypto_engine.c b/crypto/crypto_engine.c index ee192731..412149e8 100644 --- a/crypto/crypto_engine.c +++ b/crypto/crypto_engine.c @@ -227,6 +227,18 @@ retry: out: spin_unlock_irqrestore(&engine->queue_lock, flags); + + /* + * Batch requests is possible only if + * hardware can enqueue multiple requests + */ + if (engine->do_batch_requests) { + ret = engine->do_batch_requests(engine); + if (ret) + dev_err(engine->dev, "failed to do batch requests: %d\n", + ret); + } + return; } @@ -456,6 +468,12 @@ EXPORT_SYMBOL_GPL(crypto_engine_stop); * crypto-engine queue. * @dev: the device attached with one hardware engine * @retry_support: whether hardware has support for retry mechanism + * @cbk_do_batch: pointer to a callback function to be invoked when executing a + * a batch of requests. + * This has the form: + * callback(struct crypto_engine *engine) + * where: + * @engine: the crypto engine structure. * @rt: whether this queue is set to run as a realtime task * @qlen: maximum size of the crypto-engine queue * @@ -464,6 +482,7 @@ EXPORT_SYMBOL_GPL(crypto_engine_stop); */ struct crypto_engine *crypto_engine_alloc_init_and_set(struct device *dev, bool retry_support, + int (*cbk_do_batch)(struct crypto_engine *engine), bool rt, int qlen) { struct sched_param param = { .sched_priority = MAX_RT_PRIO / 2 }; @@ -483,6 +502,12 @@ struct crypto_engine *crypto_engine_alloc_init_and_set(struct device *dev, engine->idling = false; engine->retry_support = retry_support; engine->priv_data = dev; + /* + * Batch requests is possible only if + * hardware has support for retry mechanism. + */ + engine->do_batch_requests = retry_support ? cbk_do_batch : NULL; + snprintf(engine->name, sizeof(engine->name), "%s-engine", dev_name(dev)); @@ -516,7 +541,7 @@ EXPORT_SYMBOL_GPL(crypto_engine_alloc_init_and_set); */ struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt) { - return crypto_engine_alloc_init_and_set(dev, false, rt, + return crypto_engine_alloc_init_and_set(dev, false, NULL, rt, CRYPTO_ENGINE_MAX_QLEN); } EXPORT_SYMBOL_GPL(crypto_engine_alloc_init); -- cgit v1.2.3 From ac64e90452fd70ce42917e4fe09f1addfa4c3c50 Mon Sep 17 00:00:00 2001 From: Iuliana Prodan Date: Wed, 20 May 2020 01:17:25 +0300 Subject: crypto: engine - do not requeue in case of fatal error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now, in crypto-engine, if hardware queue is full (-ENOSPC), requeue request regardless of MAY_BACKLOG flag. If hardware throws any other error code (like -EIO, -EINVAL, -ENOMEM, etc.) only MAY_BACKLOG requests are enqueued back into crypto-engine's queue, since the others can be dropped. The latter case can be fatal error, so those cannot be recovered from. For example, in CAAM driver, -EIO is returned in case the job descriptor is broken, so there is no possibility to fix the job descriptor. Therefore, these errors might be fatal error, so we shouldn’t requeue the request. This will just be pass back and forth between crypto-engine and hardware. Fixes: 0eadd1eb3fd1 ("crypto: engine - support for parallel requests based on retry mechanism") Signed-off-by: Iuliana Prodan Reported-by: Horia Geantă Reviewed-by: Horia Geantă Signed-off-by: Herbert Xu --- crypto/crypto_engine.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'crypto/crypto_engine.c') diff --git a/crypto/crypto_engine.c b/crypto/crypto_engine.c index 412149e8..3655d9d3 100644 --- a/crypto/crypto_engine.c +++ b/crypto/crypto_engine.c @@ -169,13 +169,10 @@ start_request: /* * If hardware queue is full (-ENOSPC), requeue request * regardless of backlog flag. - * If hardware throws any other error code, - * requeue only backlog requests. * Otherwise, unprepare and complete the request. */ if (!engine->retry_support || - ((ret != -ENOSPC) && - !(async_req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG))) { + (ret != -ENOSPC)) { dev_err(engine->dev, "Failed to do one request from queue: %d\n", ret); -- cgit v1.2.3