blob: 89649655bf4b5985c248fe43c684e8c425a34b33 [file] [log] [blame]
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright 2019 Google LLC
*/
/*
* Refer to Documentation/block/inline-encryption.rst for detailed explanation.
*/
#define pr_fmt(fmt) "blk-crypto: " fmt
#include <linux/blk-crypto.h>
#include <linux/keyslot-manager.h>
#include <linux/mempool.h>
#include <linux/blk-cgroup.h>
#include <linux/crypto.h>
#include <crypto/skcipher.h>
#include <crypto/algapi.h>
#include <linux/module.h>
#include <linux/sched/mm.h>
/* Represents a crypto mode supported by blk-crypto */
struct blk_crypto_mode {
const char *cipher_str; /* crypto API name (for fallback case) */
size_t keysize; /* key size in bytes */
};
static const struct blk_crypto_mode blk_crypto_modes[] = {
[BLK_ENCRYPTION_MODE_AES_256_XTS] = {
.cipher_str = "xts(aes)",
.keysize = 64,
},
};
static unsigned int num_prealloc_bounce_pg = 32;
module_param(num_prealloc_bounce_pg, uint, 0);
MODULE_PARM_DESC(num_prealloc_bounce_pg,
"Number of preallocated bounce pages for blk-crypto to use during crypto API fallback encryption");
#define BLK_CRYPTO_MAX_KEY_SIZE 64
static int blk_crypto_num_keyslots = 100;
module_param_named(num_keyslots, blk_crypto_num_keyslots, int, 0);
MODULE_PARM_DESC(num_keyslots,
"Number of keyslots for crypto API fallback in blk-crypto.");
static struct blk_crypto_keyslot {
struct crypto_skcipher *tfm;
enum blk_crypto_mode_num crypto_mode;
u8 key[BLK_CRYPTO_MAX_KEY_SIZE];
struct crypto_skcipher *tfms[ARRAY_SIZE(blk_crypto_modes)];
} *blk_crypto_keyslots;
/*
* Allocating a crypto tfm during I/O can deadlock, so we have to preallocate
* all of a mode's tfms when that mode starts being used. Since each mode may
* need all the keyslots at some point, each mode needs its own tfm for each
* keyslot; thus, a keyslot may contain tfms for multiple modes. However, to
* match the behavior of real inline encryption hardware (which only supports a
* single encryption context per keyslot), we only allow one tfm per keyslot to
* be used at a time - the rest of the unused tfms have their keys cleared.
*/
static struct mutex tfms_lock[ARRAY_SIZE(blk_crypto_modes)];
static bool tfms_inited[ARRAY_SIZE(blk_crypto_modes)];
struct work_mem {
struct work_struct crypto_work;
struct bio *bio;
};
/* The following few vars are only used during the crypto API fallback */
static struct keyslot_manager *blk_crypto_ksm;
static struct workqueue_struct *blk_crypto_wq;
static mempool_t *blk_crypto_page_pool;
static struct kmem_cache *blk_crypto_work_mem_cache;
bool bio_crypt_swhandled(struct bio *bio)
{
return bio_has_crypt_ctx(bio) &&
bio->bi_crypt_context->processing_ksm == blk_crypto_ksm;
}
static u8 blank_key[BLK_CRYPTO_MAX_KEY_SIZE];
static void evict_keyslot(unsigned int slot)
{
struct blk_crypto_keyslot *slotp = &blk_crypto_keyslots[slot];
enum blk_crypto_mode_num crypto_mode = slotp->crypto_mode;
int err;
WARN_ON(slotp->crypto_mode == BLK_ENCRYPTION_MODE_INVALID);
/* Clear the key in the skcipher */
err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], blank_key,
blk_crypto_modes[crypto_mode].keysize);
WARN_ON(err);
memzero_explicit(slotp->key, BLK_CRYPTO_MAX_KEY_SIZE);
slotp->crypto_mode = BLK_ENCRYPTION_MODE_INVALID;
}
static int blk_crypto_keyslot_program(void *priv, const u8 *key,
enum blk_crypto_mode_num crypto_mode,
unsigned int data_unit_size,
unsigned int slot)
{
struct blk_crypto_keyslot *slotp = &blk_crypto_keyslots[slot];
const struct blk_crypto_mode *mode = &blk_crypto_modes[crypto_mode];
size_t keysize = mode->keysize;
int err;
if (crypto_mode != slotp->crypto_mode &&
slotp->crypto_mode != BLK_ENCRYPTION_MODE_INVALID) {
evict_keyslot(slot);
}
if (!slotp->tfms[crypto_mode])
return -ENOMEM;
slotp->crypto_mode = crypto_mode;
err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], key, keysize);
if (err) {
evict_keyslot(slot);
return err;
}
memcpy(slotp->key, key, keysize);
return 0;
}
static int blk_crypto_keyslot_evict(void *priv, const u8 *key,
enum blk_crypto_mode_num crypto_mode,
unsigned int data_unit_size,
unsigned int slot)
{
evict_keyslot(slot);
return 0;
}
static int blk_crypto_keyslot_find(void *priv,
const u8 *key,
enum blk_crypto_mode_num crypto_mode,
unsigned int data_unit_size_bytes)
{
int slot;
const size_t keysize = blk_crypto_modes[crypto_mode].keysize;
for (slot = 0; slot < blk_crypto_num_keyslots; slot++) {
if (blk_crypto_keyslots[slot].crypto_mode == crypto_mode &&
!crypto_memneq(blk_crypto_keyslots[slot].key, key, keysize))
return slot;
}
return -ENOKEY;
}
static bool blk_crypto_mode_supported(void *priv,
enum blk_crypto_mode_num crypt_mode,
unsigned int data_unit_size)
{
/* All blk_crypto_modes are required to have a crypto API fallback. */
return true;
}
/*
* The crypto API fallback KSM ops - only used for a bio when it specifies a
* blk_crypto_mode for which we failed to get a keyslot in the device's inline
* encryption hardware (which probably means the device doesn't have inline
* encryption hardware that supports that crypto mode).
*/
static const struct keyslot_mgmt_ll_ops blk_crypto_ksm_ll_ops = {
.keyslot_program = blk_crypto_keyslot_program,
.keyslot_evict = blk_crypto_keyslot_evict,
.keyslot_find = blk_crypto_keyslot_find,
.crypto_mode_supported = blk_crypto_mode_supported,
};
static void blk_crypto_encrypt_endio(struct bio *enc_bio)
{
struct bio *src_bio = enc_bio->bi_private;
int i;
for (i = 0; i < enc_bio->bi_vcnt; i++)
mempool_free(enc_bio->bi_io_vec[i].bv_page,
blk_crypto_page_pool);
src_bio->bi_status = enc_bio->bi_status;
bio_put(enc_bio);
bio_endio(src_bio);
}
static struct bio *blk_crypto_clone_bio(struct bio *bio_src)
{
struct bvec_iter iter;
struct bio_vec bv;
struct bio *bio;
bio = bio_alloc_bioset(GFP_NOIO, bio_segments(bio_src), NULL);
if (!bio)
return NULL;
bio->bi_disk = bio_src->bi_disk;
bio->bi_opf = bio_src->bi_opf;
bio->bi_ioprio = bio_src->bi_ioprio;
bio->bi_write_hint = bio_src->bi_write_hint;
bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector;
bio->bi_iter.bi_size = bio_src->bi_iter.bi_size;
bio_for_each_segment(bv, bio_src, iter)
bio->bi_io_vec[bio->bi_vcnt++] = bv;
if (bio_integrity(bio_src) &&
bio_integrity_clone(bio, bio_src, GFP_NOIO) < 0) {
bio_put(bio);
return NULL;
}
bio_clone_blkg_association(bio, bio_src);
blkcg_bio_issue_init(bio);
return bio;
}
/* Check that all I/O segments are data unit aligned */
static int bio_crypt_check_alignment(struct bio *bio)
{
int data_unit_size = 1 << bio->bi_crypt_context->data_unit_size_bits;
struct bvec_iter iter;
struct bio_vec bv;
bio_for_each_segment(bv, bio, iter) {
if (!IS_ALIGNED(bv.bv_len | bv.bv_offset, data_unit_size))
return -EIO;
}
return 0;
}
static int blk_crypto_alloc_cipher_req(struct bio *src_bio,
struct skcipher_request **ciph_req_ptr,
struct crypto_wait *wait)
{
int slot;
struct skcipher_request *ciph_req;
struct blk_crypto_keyslot *slotp;
slot = bio_crypt_get_keyslot(src_bio);
slotp = &blk_crypto_keyslots[slot];
ciph_req = skcipher_request_alloc(slotp->tfms[slotp->crypto_mode],
GFP_NOIO);
if (!ciph_req) {
src_bio->bi_status = BLK_STS_RESOURCE;
return -ENOMEM;
}
skcipher_request_set_callback(ciph_req,
CRYPTO_TFM_REQ_MAY_BACKLOG |
CRYPTO_TFM_REQ_MAY_SLEEP,
crypto_req_done, wait);
*ciph_req_ptr = ciph_req;
return 0;
}
static int blk_crypto_split_bio_if_needed(struct bio **bio_ptr)
{
struct bio *bio = *bio_ptr;
unsigned int i = 0;
unsigned int num_sectors = 0;
struct bio_vec bv;
struct bvec_iter iter;
bio_for_each_segment(bv, bio, iter) {
num_sectors += bv.bv_len >> SECTOR_SHIFT;
if (++i == BIO_MAX_PAGES)
break;
}
if (num_sectors < bio_sectors(bio)) {
struct bio *split_bio;
split_bio = bio_split(bio, num_sectors, GFP_NOIO, NULL);
if (!split_bio) {
bio->bi_status = BLK_STS_RESOURCE;
return -ENOMEM;
}
bio_chain(split_bio, bio);
generic_make_request(bio);
*bio_ptr = split_bio;
}
return 0;
}
/*
* The crypto API fallback's encryption routine.
* Allocate a bounce bio for encryption, encrypt the input bio using
* crypto API, and replace *bio_ptr with the bounce bio. May split input
* bio if it's too large.
*/
static int blk_crypto_encrypt_bio(struct bio **bio_ptr)
{
struct bio *src_bio;
struct skcipher_request *ciph_req = NULL;
DECLARE_CRYPTO_WAIT(wait);
int err = 0;
u64 curr_dun;
union {
__le64 dun;
u8 bytes[16];
} iv;
struct scatterlist src, dst;
struct bio *enc_bio;
struct bio_vec *enc_bvec;
int i, j;
int data_unit_size;
/* Split the bio if it's too big for single page bvec */
err = blk_crypto_split_bio_if_needed(bio_ptr);
if (err)
return err;
src_bio = *bio_ptr;
data_unit_size = 1 << src_bio->bi_crypt_context->data_unit_size_bits;
/* Allocate bounce bio for encryption */
enc_bio = blk_crypto_clone_bio(src_bio);
if (!enc_bio) {
src_bio->bi_status = BLK_STS_RESOURCE;
return -ENOMEM;
}
/*
* Use the crypto API fallback keyslot manager to get a crypto_skcipher
* for the algorithm and key specified for this bio.
*/
err = bio_crypt_ctx_acquire_keyslot(src_bio, blk_crypto_ksm);
if (err) {
src_bio->bi_status = BLK_STS_IOERR;
goto out_put_enc_bio;
}
/* and then allocate an skcipher_request for it */
err = blk_crypto_alloc_cipher_req(src_bio, &ciph_req, &wait);
if (err)
goto out_release_keyslot;
curr_dun = bio_crypt_data_unit_num(src_bio);
sg_init_table(&src, 1);
sg_init_table(&dst, 1);
skcipher_request_set_crypt(ciph_req, &src, &dst,
data_unit_size, iv.bytes);
/* Encrypt each page in the bounce bio */
for (i = 0, enc_bvec = enc_bio->bi_io_vec; i < enc_bio->bi_vcnt;
enc_bvec++, i++) {
struct page *plaintext_page = enc_bvec->bv_page;
struct page *ciphertext_page =
mempool_alloc(blk_crypto_page_pool, GFP_NOIO);
enc_bvec->bv_page = ciphertext_page;
if (!ciphertext_page) {
src_bio->bi_status = BLK_STS_RESOURCE;
err = -ENOMEM;
goto out_free_bounce_pages;
}
sg_set_page(&src, plaintext_page, data_unit_size,
enc_bvec->bv_offset);
sg_set_page(&dst, ciphertext_page, data_unit_size,
enc_bvec->bv_offset);
/* Encrypt each data unit in this page */
for (j = 0; j < enc_bvec->bv_len; j += data_unit_size) {
memset(&iv, 0, sizeof(iv));
iv.dun = cpu_to_le64(curr_dun);
err = crypto_wait_req(crypto_skcipher_encrypt(ciph_req),
&wait);
if (err) {
i++;
src_bio->bi_status = BLK_STS_RESOURCE;
goto out_free_bounce_pages;
}
curr_dun++;
src.offset += data_unit_size;
dst.offset += data_unit_size;
}
}
enc_bio->bi_private = src_bio;
enc_bio->bi_end_io = blk_crypto_encrypt_endio;
*bio_ptr = enc_bio;
enc_bio = NULL;
err = 0;
goto out_free_ciph_req;
out_free_bounce_pages:
while (i > 0)
mempool_free(enc_bio->bi_io_vec[--i].bv_page,
blk_crypto_page_pool);
out_free_ciph_req:
skcipher_request_free(ciph_req);
out_release_keyslot:
bio_crypt_ctx_release_keyslot(src_bio);
out_put_enc_bio:
if (enc_bio)
bio_put(enc_bio);
return err;
}
/*
* The crypto API fallback's main decryption routine.
* Decrypts input bio in place.
*/
static void blk_crypto_decrypt_bio(struct work_struct *w)
{
struct work_mem *work_mem =
container_of(w, struct work_mem, crypto_work);
struct bio *bio = work_mem->bio;
struct skcipher_request *ciph_req = NULL;
DECLARE_CRYPTO_WAIT(wait);
struct bio_vec bv;
struct bvec_iter iter;
u64 curr_dun;
union {
__le64 dun;
u8 bytes[16];
} iv;
struct scatterlist sg;
int data_unit_size = 1 << bio->bi_crypt_context->data_unit_size_bits;
int i;
int err;
/*
* Use the crypto API fallback keyslot manager to get a crypto_skcipher
* for the algorithm and key specified for this bio.
*/
if (bio_crypt_ctx_acquire_keyslot(bio, blk_crypto_ksm)) {
bio->bi_status = BLK_STS_RESOURCE;
goto out_no_keyslot;
}
/* and then allocate an skcipher_request for it */
err = blk_crypto_alloc_cipher_req(bio, &ciph_req, &wait);
if (err)
goto out;
curr_dun = bio_crypt_sw_data_unit_num(bio);
sg_init_table(&sg, 1);
skcipher_request_set_crypt(ciph_req, &sg, &sg, data_unit_size,
iv.bytes);
/* Decrypt each segment in the bio */
__bio_for_each_segment(bv, bio, iter,
bio->bi_crypt_context->crypt_iter) {
struct page *page = bv.bv_page;
sg_set_page(&sg, page, data_unit_size, bv.bv_offset);
/* Decrypt each data unit in the segment */
for (i = 0; i < bv.bv_len; i += data_unit_size) {
memset(&iv, 0, sizeof(iv));
iv.dun = cpu_to_le64(curr_dun);
if (crypto_wait_req(crypto_skcipher_decrypt(ciph_req),
&wait)) {
bio->bi_status = BLK_STS_IOERR;
goto out;
}
curr_dun++;
sg.offset += data_unit_size;
}
}
out:
skcipher_request_free(ciph_req);
bio_crypt_ctx_release_keyslot(bio);
out_no_keyslot:
kmem_cache_free(blk_crypto_work_mem_cache, work_mem);
bio_endio(bio);
}
/* Queue bio for decryption */
static void blk_crypto_queue_decrypt_bio(struct bio *bio)
{
struct work_mem *work_mem =
kmem_cache_zalloc(blk_crypto_work_mem_cache, GFP_ATOMIC);
if (!work_mem) {
bio->bi_status = BLK_STS_RESOURCE;
bio_endio(bio);
return;
}
INIT_WORK(&work_mem->crypto_work, blk_crypto_decrypt_bio);
work_mem->bio = bio;
queue_work(blk_crypto_wq, &work_mem->crypto_work);
}
/**
* blk_crypto_submit_bio - handle submitting bio for inline encryption
*
* @bio_ptr: pointer to original bio pointer
*
* If the bio doesn't have inline encryption enabled or the submitter already
* specified a keyslot for the target device, do nothing. Else, a raw key must
* have been provided, so acquire a device keyslot for it if supported. Else,
* use the crypto API fallback.
*
* When the crypto API fallback is used for encryption, blk-crypto may choose to
* split the bio into 2 - the first one that will continue to be processed and
* the second one that will be resubmitted via generic_make_request.
* A bounce bio will be allocated to encrypt the contents of the aforementioned
* "first one", and *bio_ptr will be updated to this bounce bio.
*
* Return: 0 if bio submission should continue; nonzero if bio_endio() was
* already called so bio submission should abort.
*/
int blk_crypto_submit_bio(struct bio **bio_ptr)
{
struct bio *bio = *bio_ptr;
struct request_queue *q;
int err;
struct bio_crypt_ctx *crypt_ctx;
if (!bio_has_crypt_ctx(bio) || !bio_has_data(bio))
return 0;
/*
* When a read bio is marked for sw decryption, its bi_iter is saved
* so that when we decrypt the bio later, we know what part of it was
* marked for sw decryption (when the bio is passed down after
* blk_crypto_submit bio, it may be split or advanced so we cannot rely
* on the bi_iter while decrypting in blk_crypto_endio)
*/
if (bio_crypt_swhandled(bio))
return 0;
err = bio_crypt_check_alignment(bio);
if (err) {
bio->bi_status = BLK_STS_IOERR;
goto out;
}
crypt_ctx = bio->bi_crypt_context;
q = bio->bi_disk->queue;
if (bio_crypt_has_keyslot(bio)) {
/* Key already programmed into device? */
if (q->ksm == crypt_ctx->processing_ksm)
return 0;
/* Nope, release the existing keyslot. */
bio_crypt_ctx_release_keyslot(bio);
}
/* Get device keyslot if supported */
if (q->ksm) {
err = bio_crypt_ctx_acquire_keyslot(bio, q->ksm);
if (!err)
return 0;
pr_warn_once("Failed to acquire keyslot for %s (err=%d). Falling back to crypto API.\n",
bio->bi_disk->disk_name, err);
}
/* Fallback to crypto API */
if (!READ_ONCE(tfms_inited[bio->bi_crypt_context->crypto_mode])) {
err = -EIO;
bio->bi_status = BLK_STS_IOERR;
goto out;
}
if (bio_data_dir(bio) == WRITE) {
/* Encrypt the data now */
err = blk_crypto_encrypt_bio(bio_ptr);
if (err)
goto out;
} else {
/* Mark bio as swhandled */
bio->bi_crypt_context->processing_ksm = blk_crypto_ksm;
bio->bi_crypt_context->crypt_iter = bio->bi_iter;
bio->bi_crypt_context->sw_data_unit_num =
bio->bi_crypt_context->data_unit_num;
}
return 0;
out:
bio_endio(*bio_ptr);
return err;
}
/**
* blk_crypto_endio - clean up bio w.r.t inline encryption during bio_endio
*
* @bio - the bio to clean up
*
* If blk_crypto_submit_bio decided to fallback to crypto API for this
* bio, we queue the bio for decryption into a workqueue and return false,
* and call bio_endio(bio) at a later time (after the bio has been decrypted).
*
* If the bio is not to be decrypted by the crypto API, this function releases
* the reference to the keyslot that blk_crypto_submit_bio got.
*
* Return: true if bio_endio should continue; false otherwise (bio_endio will
* be called again when bio has been decrypted).
*/
bool blk_crypto_endio(struct bio *bio)
{
if (!bio_has_crypt_ctx(bio))
return true;
if (bio_crypt_swhandled(bio)) {
/*
* The only bios that are swhandled when they reach here
* are those with bio_data_dir(bio) == READ, since WRITE
* bios that are encrypted by the crypto API fallback are
* handled by blk_crypto_encrypt_endio.
*/
/* If there was an IO error, don't decrypt. */
if (bio->bi_status)
return true;
blk_crypto_queue_decrypt_bio(bio);
return false;
}
if (bio_crypt_has_keyslot(bio))
bio_crypt_ctx_release_keyslot(bio);
return true;
}
/**
* blk_crypto_start_using_mode() - Allocate skciphers for a
* mode_num for all keyslots
* @mode_num - the blk_crypto_mode we want to allocate ciphers for.
*
* Upper layers (filesystems) should call this function to ensure that a
* the crypto API fallback has transforms for this algorithm, if they become
* necessary.
*
* Return: 0 on success and -err on error.
*/
int blk_crypto_start_using_mode(enum blk_crypto_mode_num mode_num,
unsigned int data_unit_size,
struct request_queue *q)
{
struct blk_crypto_keyslot *slotp;
int err = 0;
int i;
/*
* Fast path
* Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num]
* for each i are visible before we try to access them.
*/
if (likely(smp_load_acquire(&tfms_inited[mode_num])))
return 0;
/*
* If the keyslot manager of the request queue supports this
* crypto mode, then we don't need to allocate this mode.
*/
if (keyslot_manager_crypto_mode_supported(q->ksm, mode_num,
data_unit_size)) {
return 0;
}
mutex_lock(&tfms_lock[mode_num]);
if (likely(tfms_inited[mode_num]))
goto out;
for (i = 0; i < blk_crypto_num_keyslots; i++) {
slotp = &blk_crypto_keyslots[i];
slotp->tfms[mode_num] = crypto_alloc_skcipher(
blk_crypto_modes[mode_num].cipher_str,
0, 0);
if (IS_ERR(slotp->tfms[mode_num])) {
err = PTR_ERR(slotp->tfms[mode_num]);
slotp->tfms[mode_num] = NULL;
goto out_free_tfms;
}
crypto_skcipher_set_flags(slotp->tfms[mode_num],
CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
}
/*
* Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num]
* for each i are visible before we set tfms_inited[mode_num].
*/
smp_store_release(&tfms_inited[mode_num], true);
goto out;
out_free_tfms:
for (i = 0; i < blk_crypto_num_keyslots; i++) {
slotp = &blk_crypto_keyslots[i];
crypto_free_skcipher(slotp->tfms[mode_num]);
slotp->tfms[mode_num] = NULL;
}
out:
mutex_unlock(&tfms_lock[mode_num]);
return err;
}
EXPORT_SYMBOL(blk_crypto_start_using_mode);
/**
* blk_crypto_evict_key() - Evict a key from any inline encryption hardware
* it may have been programmed into
* @q - The request queue who's keyslot manager this key might have been
* programmed into
* @key - The key to evict
* @mode - The blk_crypto_mode_num used with this key
* @data_unit_size - The data unit size used with this key
*
* Upper layers (filesystems) should call this function to ensure that a key
* is evicted from hardware that it might have been programmed into. This
* will call keyslot_manager_evict_key on the queue's keyslot manager, if one
* exists, and supports the crypto algorithm with the specified data unit size.
* Otherwise, it will evict the key from the blk_crypto_ksm.
*
* Return: 0 on success, -err on error.
*/
int blk_crypto_evict_key(struct request_queue *q, const u8 *key,
enum blk_crypto_mode_num mode,
unsigned int data_unit_size)
{
struct keyslot_manager *ksm = blk_crypto_ksm;
if (q && q->ksm && keyslot_manager_crypto_mode_supported(q->ksm, mode,
data_unit_size)) {
ksm = q->ksm;
}
return keyslot_manager_evict_key(ksm, key, mode, data_unit_size);
}
EXPORT_SYMBOL(blk_crypto_evict_key);
int __init blk_crypto_init(void)
{
int i;
int err = -ENOMEM;
prandom_bytes(blank_key, BLK_CRYPTO_MAX_KEY_SIZE);
blk_crypto_ksm = keyslot_manager_create(blk_crypto_num_keyslots,
&blk_crypto_ksm_ll_ops,
NULL);
if (!blk_crypto_ksm)
goto out;
blk_crypto_wq = alloc_workqueue("blk_crypto_wq",
WQ_UNBOUND | WQ_HIGHPRI |
WQ_MEM_RECLAIM,
num_online_cpus());
if (!blk_crypto_wq)
goto out_free_ksm;
blk_crypto_keyslots = kcalloc(blk_crypto_num_keyslots,
sizeof(*blk_crypto_keyslots),
GFP_KERNEL);
if (!blk_crypto_keyslots)
goto out_free_workqueue;
for (i = 0; i < blk_crypto_num_keyslots; i++) {
blk_crypto_keyslots[i].crypto_mode =
BLK_ENCRYPTION_MODE_INVALID;
}
for (i = 0; i < ARRAY_SIZE(blk_crypto_modes); i++)
mutex_init(&tfms_lock[i]);
blk_crypto_page_pool =
mempool_create_page_pool(num_prealloc_bounce_pg, 0);
if (!blk_crypto_page_pool)
goto out_free_keyslots;
blk_crypto_work_mem_cache = KMEM_CACHE(work_mem, SLAB_RECLAIM_ACCOUNT);
if (!blk_crypto_work_mem_cache)
goto out_free_page_pool;
return 0;
out_free_page_pool:
mempool_destroy(blk_crypto_page_pool);
blk_crypto_page_pool = NULL;
out_free_keyslots:
kzfree(blk_crypto_keyslots);
blk_crypto_keyslots = NULL;
out_free_workqueue:
destroy_workqueue(blk_crypto_wq);
blk_crypto_wq = NULL;
out_free_ksm:
keyslot_manager_destroy(blk_crypto_ksm);
blk_crypto_ksm = NULL;
out:
pr_warn("No memory for blk-crypto crypto API fallback.");
return err;
}