From 562a16de26bdc99de1baea432a317d7b0a51fafc Mon Sep 17 00:00:00 2001 From: Daniel Jordan Date: Thu, 5 Sep 2019 21:40:21 -0400 Subject: padata: allocate workqueue internally Move workqueue allocation inside of padata to prepare for further changes to how padata uses workqueues. Guarantees the workqueue is created with max_active=1, which padata relies on to work correctly. No functional change. Signed-off-by: Daniel Jordan Acked-by: Steffen Klassert Cc: Herbert Xu Cc: Jonathan Corbet Cc: Lai Jiangshan Cc: Peter Zijlstra Cc: Tejun Heo Cc: linux-crypto@vger.kernel.org Cc: linux-doc@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Herbert Xu --- crypto/pcrypt.c | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) (limited to 'crypto/pcrypt.c') diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c index 0edf5b54..d6729306 100644 --- a/crypto/pcrypt.c +++ b/crypto/pcrypt.c @@ -20,7 +20,6 @@ struct padata_pcrypt { struct padata_instance *pinst; - struct workqueue_struct *wq; /* * Cpumask for callback CPUs. It should be @@ -397,14 +396,9 @@ static int pcrypt_init_padata(struct padata_pcrypt *pcrypt, get_online_cpus(); - pcrypt->wq = alloc_workqueue("%s", WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, - 1, name); - if (!pcrypt->wq) - goto err; - - pcrypt->pinst = padata_alloc_possible(pcrypt->wq); + pcrypt->pinst = padata_alloc_possible(name); if (!pcrypt->pinst) - goto err_destroy_workqueue; + goto err; mask = kmalloc(sizeof(*mask), GFP_KERNEL); if (!mask) @@ -437,8 +431,6 @@ err_free_cpumask: kfree(mask); err_free_padata: padata_free(pcrypt->pinst); -err_destroy_workqueue: - destroy_workqueue(pcrypt->wq); err: put_online_cpus(); @@ -452,7 +444,6 @@ static void pcrypt_fini_padata(struct padata_pcrypt *pcrypt) padata_stop(pcrypt->pinst); padata_unregister_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock); - destroy_workqueue(pcrypt->wq); padata_free(pcrypt->pinst); } -- cgit v1.2.3 From 254ffbd9085d3349168a25aff2278d375740751c Mon Sep 17 00:00:00 2001 From: Daniel Jordan Date: Thu, 5 Sep 2019 21:40:24 -0400 Subject: padata: make padata_do_parallel find alternate callback CPU padata_do_parallel currently returns -EINVAL if the callback CPU isn't in the callback cpumask. pcrypt tries to prevent this situation by keeping its own callback cpumask in sync with padata's and checks that the callback CPU it passes to padata is valid. Make padata handle this instead. padata_do_parallel now takes a pointer to the callback CPU and updates it for the caller if an alternate CPU is used. Overall behavior in terms of which callback CPUs are chosen stays the same. Prepares for removal of the padata cpumask notifier in pcrypt, which will fix a lockdep complaint about nested acquisition of the CPU hotplug lock later in the series. Signed-off-by: Daniel Jordan Acked-by: Steffen Klassert Cc: Herbert Xu Cc: Lai Jiangshan Cc: Peter Zijlstra Cc: Tejun Heo Cc: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Herbert Xu --- crypto/pcrypt.c | 33 ++------------------------------- 1 file changed, 2 insertions(+), 31 deletions(-) (limited to 'crypto/pcrypt.c') diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c index d6729306..efca962a 100644 --- a/crypto/pcrypt.c +++ b/crypto/pcrypt.c @@ -57,35 +57,6 @@ struct pcrypt_aead_ctx { unsigned int cb_cpu; }; -static int pcrypt_do_parallel(struct padata_priv *padata, unsigned int *cb_cpu, - struct padata_pcrypt *pcrypt) -{ - unsigned int cpu_index, cpu, i; - struct pcrypt_cpumask *cpumask; - - cpu = *cb_cpu; - - rcu_read_lock_bh(); - cpumask = rcu_dereference_bh(pcrypt->cb_cpumask); - if (cpumask_test_cpu(cpu, cpumask->mask)) - goto out; - - if (!cpumask_weight(cpumask->mask)) - goto out; - - cpu_index = cpu % cpumask_weight(cpumask->mask); - - cpu = cpumask_first(cpumask->mask); - for (i = 0; i < cpu_index; i++) - cpu = cpumask_next(cpu, cpumask->mask); - - *cb_cpu = cpu; - -out: - rcu_read_unlock_bh(); - return padata_do_parallel(pcrypt->pinst, padata, cpu); -} - static int pcrypt_aead_setkey(struct crypto_aead *parent, const u8 *key, unsigned int keylen) { @@ -157,7 +128,7 @@ static int pcrypt_aead_encrypt(struct aead_request *req) req->cryptlen, req->iv); aead_request_set_ad(creq, req->assoclen); - err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pencrypt); + err = padata_do_parallel(pencrypt.pinst, padata, &ctx->cb_cpu); if (!err) return -EINPROGRESS; @@ -199,7 +170,7 @@ static int pcrypt_aead_decrypt(struct aead_request *req) req->cryptlen, req->iv); aead_request_set_ad(creq, req->assoclen); - err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pdecrypt); + err = padata_do_parallel(pdecrypt.pinst, padata, &ctx->cb_cpu); if (!err) return -EINPROGRESS; -- cgit v1.2.3 From bd386d43070c947c33b9ca0e84463deca5e81c27 Mon Sep 17 00:00:00 2001 From: Daniel Jordan Date: Thu, 5 Sep 2019 21:40:25 -0400 Subject: crypto: pcrypt - remove padata cpumask notifier Now that padata_do_parallel takes care of finding an alternate callback CPU, there's no need for pcrypt's callback cpumask, so remove it and the notifier callback that keeps it in sync. Signed-off-by: Daniel Jordan Acked-by: Steffen Klassert Cc: Herbert Xu Cc: Lai Jiangshan Cc: Peter Zijlstra Cc: Tejun Heo Cc: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Herbert Xu --- crypto/pcrypt.c | 125 ++++++++------------------------------------------------ 1 file changed, 18 insertions(+), 107 deletions(-) (limited to 'crypto/pcrypt.c') diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c index efca962a..2ec36e6a 100644 --- a/crypto/pcrypt.c +++ b/crypto/pcrypt.c @@ -18,33 +18,8 @@ #include #include -struct padata_pcrypt { - struct padata_instance *pinst; - - /* - * Cpumask for callback CPUs. It should be - * equal to serial cpumask of corresponding padata instance, - * so it is updated when padata notifies us about serial - * cpumask change. - * - * cb_cpumask is protected by RCU. This fact prevents us from - * using cpumask_var_t directly because the actual type of - * cpumsak_var_t depends on kernel configuration(particularly on - * CONFIG_CPUMASK_OFFSTACK macro). Depending on the configuration - * cpumask_var_t may be either a pointer to the struct cpumask - * or a variable allocated on the stack. Thus we can not safely use - * cpumask_var_t with RCU operations such as rcu_assign_pointer or - * rcu_dereference. So cpumask_var_t is wrapped with struct - * pcrypt_cpumask which makes possible to use it with RCU. - */ - struct pcrypt_cpumask { - cpumask_var_t mask; - } *cb_cpumask; - struct notifier_block nblock; -}; - -static struct padata_pcrypt pencrypt; -static struct padata_pcrypt pdecrypt; +static struct padata_instance *pencrypt; +static struct padata_instance *pdecrypt; static struct kset *pcrypt_kset; struct pcrypt_instance_ctx { @@ -128,7 +103,7 @@ static int pcrypt_aead_encrypt(struct aead_request *req) req->cryptlen, req->iv); aead_request_set_ad(creq, req->assoclen); - err = padata_do_parallel(pencrypt.pinst, padata, &ctx->cb_cpu); + err = padata_do_parallel(pencrypt, padata, &ctx->cb_cpu); if (!err) return -EINPROGRESS; @@ -170,7 +145,7 @@ static int pcrypt_aead_decrypt(struct aead_request *req) req->cryptlen, req->iv); aead_request_set_ad(creq, req->assoclen); - err = padata_do_parallel(pdecrypt.pinst, padata, &ctx->cb_cpu); + err = padata_do_parallel(pdecrypt, padata, &ctx->cb_cpu); if (!err) return -EINPROGRESS; @@ -317,36 +292,6 @@ static int pcrypt_create(struct crypto_template *tmpl, struct rtattr **tb) return -EINVAL; } -static int pcrypt_cpumask_change_notify(struct notifier_block *self, - unsigned long val, void *data) -{ - struct padata_pcrypt *pcrypt; - struct pcrypt_cpumask *new_mask, *old_mask; - struct padata_cpumask *cpumask = (struct padata_cpumask *)data; - - if (!(val & PADATA_CPU_SERIAL)) - return 0; - - pcrypt = container_of(self, struct padata_pcrypt, nblock); - new_mask = kmalloc(sizeof(*new_mask), GFP_KERNEL); - if (!new_mask) - return -ENOMEM; - if (!alloc_cpumask_var(&new_mask->mask, GFP_KERNEL)) { - kfree(new_mask); - return -ENOMEM; - } - - old_mask = pcrypt->cb_cpumask; - - cpumask_copy(new_mask->mask, cpumask->cbcpu); - rcu_assign_pointer(pcrypt->cb_cpumask, new_mask); - synchronize_rcu(); - - free_cpumask_var(old_mask->mask); - kfree(old_mask); - return 0; -} - static int pcrypt_sysfs_add(struct padata_instance *pinst, const char *name) { int ret; @@ -359,63 +304,29 @@ static int pcrypt_sysfs_add(struct padata_instance *pinst, const char *name) return ret; } -static int pcrypt_init_padata(struct padata_pcrypt *pcrypt, - const char *name) +static int pcrypt_init_padata(struct padata_instance **pinst, const char *name) { int ret = -ENOMEM; - struct pcrypt_cpumask *mask; get_online_cpus(); - pcrypt->pinst = padata_alloc_possible(name); - if (!pcrypt->pinst) - goto err; - - mask = kmalloc(sizeof(*mask), GFP_KERNEL); - if (!mask) - goto err_free_padata; - if (!alloc_cpumask_var(&mask->mask, GFP_KERNEL)) { - kfree(mask); - goto err_free_padata; - } - - cpumask_and(mask->mask, cpu_possible_mask, cpu_online_mask); - rcu_assign_pointer(pcrypt->cb_cpumask, mask); - - pcrypt->nblock.notifier_call = pcrypt_cpumask_change_notify; - ret = padata_register_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock); - if (ret) - goto err_free_cpumask; + *pinst = padata_alloc_possible(name); + if (!*pinst) + return ret; - ret = pcrypt_sysfs_add(pcrypt->pinst, name); + ret = pcrypt_sysfs_add(*pinst, name); if (ret) - goto err_unregister_notifier; + padata_free(*pinst); put_online_cpus(); - return ret; - -err_unregister_notifier: - padata_unregister_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock); -err_free_cpumask: - free_cpumask_var(mask->mask); - kfree(mask); -err_free_padata: - padata_free(pcrypt->pinst); -err: - put_online_cpus(); - return ret; } -static void pcrypt_fini_padata(struct padata_pcrypt *pcrypt) +static void pcrypt_fini_padata(struct padata_instance *pinst) { - free_cpumask_var(pcrypt->cb_cpumask->mask); - kfree(pcrypt->cb_cpumask); - - padata_stop(pcrypt->pinst); - padata_unregister_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock); - padata_free(pcrypt->pinst); + padata_stop(pinst); + padata_free(pinst); } static struct crypto_template pcrypt_tmpl = { @@ -440,13 +351,13 @@ static int __init pcrypt_init(void) if (err) goto err_deinit_pencrypt; - padata_start(pencrypt.pinst); - padata_start(pdecrypt.pinst); + padata_start(pencrypt); + padata_start(pdecrypt); return crypto_register_template(&pcrypt_tmpl); err_deinit_pencrypt: - pcrypt_fini_padata(&pencrypt); + pcrypt_fini_padata(pencrypt); err_unreg_kset: kset_unregister(pcrypt_kset); err: @@ -455,8 +366,8 @@ err: static void __exit pcrypt_exit(void) { - pcrypt_fini_padata(&pencrypt); - pcrypt_fini_padata(&pdecrypt); + pcrypt_fini_padata(pencrypt); + pcrypt_fini_padata(pdecrypt); kset_unregister(pcrypt_kset); crypto_unregister_template(&pcrypt_tmpl); -- cgit v1.2.3 From 24a9f0e4cb52070c48602e9e6d31470dd98ccdcf Mon Sep 17 00:00:00 2001 From: Daniel Jordan Date: Thu, 5 Sep 2019 21:40:26 -0400 Subject: padata, pcrypt: take CPU hotplug lock internally in padata_alloc_possible With pcrypt's cpumask no longer used, take the CPU hotplug lock inside padata_alloc_possible. Useful later in the series for avoiding nested acquisition of the CPU hotplug lock in padata when padata_alloc_possible is allocating an unbound workqueue. Without this patch, this nested acquisition would happen later in the series: pcrypt_init_padata get_online_cpus alloc_padata_possible alloc_padata alloc_workqueue(WQ_UNBOUND) // later in the series alloc_and_link_pwqs apply_wqattrs_lock get_online_cpus // recursive rwsem acquisition Signed-off-by: Daniel Jordan Acked-by: Steffen Klassert Cc: Herbert Xu Cc: Lai Jiangshan Cc: Peter Zijlstra Cc: Tejun Heo Cc: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Herbert Xu --- crypto/pcrypt.c | 4 ---- 1 file changed, 4 deletions(-) (limited to 'crypto/pcrypt.c') diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c index 2ec36e6a..543792e0 100644 --- a/crypto/pcrypt.c +++ b/crypto/pcrypt.c @@ -308,8 +308,6 @@ static int pcrypt_init_padata(struct padata_instance **pinst, const char *name) { int ret = -ENOMEM; - get_online_cpus(); - *pinst = padata_alloc_possible(name); if (!*pinst) return ret; @@ -318,8 +316,6 @@ static int pcrypt_init_padata(struct padata_instance **pinst, const char *name) if (ret) padata_free(*pinst); - put_online_cpus(); - return ret; } -- cgit v1.2.3