From 5c6ac1d4f8fbdbed65dbeb8cf149d736409d16a1 Mon Sep 17 00:00:00 2001 From: Robert Baronescu Date: Tue, 10 Oct 2017 13:21:59 +0300 Subject: [PATCH 001/166] crypto: tcrypt - fix S/G table for test_aead_speed() In case buffer length is a multiple of PAGE_SIZE, the S/G table is incorrectly generated. Fix this by handling buflen = k * PAGE_SIZE separately. Signed-off-by: Robert Baronescu Signed-off-by: Herbert Xu --- crypto/tcrypt.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 9267cbdb14d2..3ced1ba1fd11 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -198,11 +198,13 @@ static void sg_init_aead(struct scatterlist *sg, char *xbuf[XBUFSIZE], } sg_init_table(sg, np + 1); - np--; + if (rem) + np--; for (k = 0; k < np; k++) sg_set_buf(&sg[k + 1], xbuf[k], PAGE_SIZE); - sg_set_buf(&sg[k + 1], xbuf[k], rem); + if (rem) + sg_set_buf(&sg[k + 1], xbuf[k], rem); } static void test_aead_speed(const char *algo, int enc, unsigned int secs, From f82f59cf665562aa1ce6097827d958382724dce4 Mon Sep 17 00:00:00 2001 From: Fabien DESSENNE Date: Thu, 19 Oct 2017 17:10:29 +0200 Subject: [PATCH 002/166] dt-bindings: Document STM32 CRYP bindings Document device tree bindings for the STM32 CRYP. Signed-off-by: Fabien Dessenne Acked-by: Rob Herring Signed-off-by: Herbert Xu --- .../bindings/crypto/st,stm32-cryp.txt | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 Documentation/devicetree/bindings/crypto/st,stm32-cryp.txt diff --git a/Documentation/devicetree/bindings/crypto/st,stm32-cryp.txt b/Documentation/devicetree/bindings/crypto/st,stm32-cryp.txt new file mode 100644 index 000000000000..970487fa40b8 --- /dev/null +++ b/Documentation/devicetree/bindings/crypto/st,stm32-cryp.txt @@ -0,0 +1,19 @@ +* STMicroelectronics STM32 CRYP + +Required properties: +- compatible: Should be "st,stm32f756-cryp". +- reg: The address and length of the peripheral registers space +- clocks: The input clock of the CRYP instance +- interrupts: The CRYP interrupt + +Optional properties: +- resets: The input reset of the CRYP instance + +Example: +crypto@50060000 { + compatible = "st,stm32f756-cryp"; + reg = <0x50060000 0x400>; + interrupts = <79>; + clocks = <&rcc 0 STM32F7_AHB2_CLOCK(CRYP)>; + resets = <&rcc STM32F7_AHB2_RESET(CRYP)>; +}; From 9e054ec21ef8344345b28603fb272fe999f735db Mon Sep 17 00:00:00 2001 From: Fabien DESSENNE Date: Thu, 19 Oct 2017 17:10:30 +0200 Subject: [PATCH 003/166] crypto: stm32 - Support for STM32 CRYP crypto module This module registers block cipher algorithms that make use of the STMicroelectronics STM32 crypto "CRYP1" hardware. The following algorithms are supported: - aes: ecb, cbc, ctr - des: ecb, cbc - tdes: ecb, cbc Signed-off-by: Fabien Dessenne Signed-off-by: Herbert Xu --- drivers/crypto/stm32/Kconfig | 9 + drivers/crypto/stm32/Makefile | 3 +- drivers/crypto/stm32/stm32-cryp.c | 1172 +++++++++++++++++++++++++++++ 3 files changed, 1183 insertions(+), 1 deletion(-) create mode 100644 drivers/crypto/stm32/stm32-cryp.c diff --git a/drivers/crypto/stm32/Kconfig b/drivers/crypto/stm32/Kconfig index 602332e02729..61ef00b6bf45 100644 --- a/drivers/crypto/stm32/Kconfig +++ b/drivers/crypto/stm32/Kconfig @@ -18,3 +18,12 @@ config HASH_DEV_STM32 help This enables support for the HASH hw accelerator which can be found on STMicroelectronics STM32 SOC. + +config CRYP_DEV_STM32 + tristate "Support for STM32 cryp accelerators" + depends on ARCH_STM32 + select CRYPTO_HASH + select CRYPTO_ENGINE + help + This enables support for the CRYP (AES/DES/TDES) hw accelerator which + can be found on STMicroelectronics STM32 SOC. diff --git a/drivers/crypto/stm32/Makefile b/drivers/crypto/stm32/Makefile index 73cd56cad0cc..2c19fc155bfd 100644 --- a/drivers/crypto/stm32/Makefile +++ b/drivers/crypto/stm32/Makefile @@ -1,2 +1,3 @@ obj-$(CONFIG_CRC_DEV_STM32) += stm32_crc32.o -obj-$(CONFIG_HASH_DEV_STM32) += stm32-hash.o \ No newline at end of file +obj-$(CONFIG_HASH_DEV_STM32) += stm32-hash.o +obj-$(CONFIG_CRYP_DEV_STM32) += stm32-cryp.o diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c new file mode 100644 index 000000000000..459755940632 --- /dev/null +++ b/drivers/crypto/stm32/stm32-cryp.c @@ -0,0 +1,1172 @@ +/* + * Copyright (C) STMicroelectronics SA 2017 + * Author: Fabien Dessenne + * License terms: GNU General Public License (GPL), version 2 + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#define DRIVER_NAME "stm32-cryp" + +/* Bit [0] encrypt / decrypt */ +#define FLG_ENCRYPT BIT(0) +/* Bit [8..1] algo & operation mode */ +#define FLG_AES BIT(1) +#define FLG_DES BIT(2) +#define FLG_TDES BIT(3) +#define FLG_ECB BIT(4) +#define FLG_CBC BIT(5) +#define FLG_CTR BIT(6) +/* Mode mask = bits [15..0] */ +#define FLG_MODE_MASK GENMASK(15, 0) + +/* Registers */ +#define CRYP_CR 0x00000000 +#define CRYP_SR 0x00000004 +#define CRYP_DIN 0x00000008 +#define CRYP_DOUT 0x0000000C +#define CRYP_DMACR 0x00000010 +#define CRYP_IMSCR 0x00000014 +#define CRYP_RISR 0x00000018 +#define CRYP_MISR 0x0000001C +#define CRYP_K0LR 0x00000020 +#define CRYP_K0RR 0x00000024 +#define CRYP_K1LR 0x00000028 +#define CRYP_K1RR 0x0000002C +#define CRYP_K2LR 0x00000030 +#define CRYP_K2RR 0x00000034 +#define CRYP_K3LR 0x00000038 +#define CRYP_K3RR 0x0000003C +#define CRYP_IV0LR 0x00000040 +#define CRYP_IV0RR 0x00000044 +#define CRYP_IV1LR 0x00000048 +#define CRYP_IV1RR 0x0000004C + +/* Registers values */ +#define CR_DEC_NOT_ENC 0x00000004 +#define CR_TDES_ECB 0x00000000 +#define CR_TDES_CBC 0x00000008 +#define CR_DES_ECB 0x00000010 +#define CR_DES_CBC 0x00000018 +#define CR_AES_ECB 0x00000020 +#define CR_AES_CBC 0x00000028 +#define CR_AES_CTR 0x00000030 +#define CR_AES_KP 0x00000038 +#define CR_AES_UNKNOWN 0xFFFFFFFF +#define CR_ALGO_MASK 0x00080038 +#define CR_DATA32 0x00000000 +#define CR_DATA16 0x00000040 +#define CR_DATA8 0x00000080 +#define CR_DATA1 0x000000C0 +#define CR_KEY128 0x00000000 +#define CR_KEY192 0x00000100 +#define CR_KEY256 0x00000200 +#define CR_FFLUSH 0x00004000 +#define CR_CRYPEN 0x00008000 + +#define SR_BUSY 0x00000010 +#define SR_OFNE 0x00000004 + +#define IMSCR_IN BIT(0) +#define IMSCR_OUT BIT(1) + +#define MISR_IN BIT(0) +#define MISR_OUT BIT(1) + +/* Misc */ +#define AES_BLOCK_32 (AES_BLOCK_SIZE / sizeof(u32)) +#define _walked_in (cryp->in_walk.offset - cryp->in_sg->offset) +#define _walked_out (cryp->out_walk.offset - cryp->out_sg->offset) + +struct stm32_cryp_ctx { + struct stm32_cryp *cryp; + int keylen; + u32 key[AES_KEYSIZE_256 / sizeof(u32)]; + unsigned long flags; +}; + +struct stm32_cryp_reqctx { + unsigned long mode; +}; + +struct stm32_cryp { + struct list_head list; + struct device *dev; + void __iomem *regs; + struct clk *clk; + unsigned long flags; + u32 irq_status; + struct stm32_cryp_ctx *ctx; + + struct crypto_engine *engine; + + struct mutex lock; /* protects req */ + struct ablkcipher_request *req; + + size_t hw_blocksize; + + size_t total_in; + size_t total_in_save; + size_t total_out; + size_t total_out_save; + + struct scatterlist *in_sg; + struct scatterlist *out_sg; + struct scatterlist *out_sg_save; + + struct scatterlist in_sgl; + struct scatterlist out_sgl; + bool sgs_copied; + + int in_sg_len; + int out_sg_len; + + struct scatter_walk in_walk; + struct scatter_walk out_walk; + + u32 last_ctr[4]; +}; + +struct stm32_cryp_list { + struct list_head dev_list; + spinlock_t lock; /* protect dev_list */ +}; + +static struct stm32_cryp_list cryp_list = { + .dev_list = LIST_HEAD_INIT(cryp_list.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(cryp_list.lock), +}; + +static inline bool is_aes(struct stm32_cryp *cryp) +{ + return cryp->flags & FLG_AES; +} + +static inline bool is_des(struct stm32_cryp *cryp) +{ + return cryp->flags & FLG_DES; +} + +static inline bool is_tdes(struct stm32_cryp *cryp) +{ + return cryp->flags & FLG_TDES; +} + +static inline bool is_ecb(struct stm32_cryp *cryp) +{ + return cryp->flags & FLG_ECB; +} + +static inline bool is_cbc(struct stm32_cryp *cryp) +{ + return cryp->flags & FLG_CBC; +} + +static inline bool is_ctr(struct stm32_cryp *cryp) +{ + return cryp->flags & FLG_CTR; +} + +static inline bool is_encrypt(struct stm32_cryp *cryp) +{ + return cryp->flags & FLG_ENCRYPT; +} + +static inline bool is_decrypt(struct stm32_cryp *cryp) +{ + return !is_encrypt(cryp); +} + +static inline u32 stm32_cryp_read(struct stm32_cryp *cryp, u32 ofst) +{ + return readl_relaxed(cryp->regs + ofst); +} + +static inline void stm32_cryp_write(struct stm32_cryp *cryp, u32 ofst, u32 val) +{ + writel_relaxed(val, cryp->regs + ofst); +} + +static inline int stm32_cryp_wait_busy(struct stm32_cryp *cryp) +{ + u32 status; + + return readl_relaxed_poll_timeout(cryp->regs + CRYP_SR, status, + !(status & SR_BUSY), 10, 100000); +} + +static struct stm32_cryp *stm32_cryp_find_dev(struct stm32_cryp_ctx *ctx) +{ + struct stm32_cryp *tmp, *cryp = NULL; + + spin_lock_bh(&cryp_list.lock); + if (!ctx->cryp) { + list_for_each_entry(tmp, &cryp_list.dev_list, list) { + cryp = tmp; + break; + } + ctx->cryp = cryp; + } else { + cryp = ctx->cryp; + } + + spin_unlock_bh(&cryp_list.lock); + + return cryp; +} + +static int stm32_cryp_check_aligned(struct scatterlist *sg, size_t total, + size_t align) +{ + int len = 0; + + if (!total) + return 0; + + if (!IS_ALIGNED(total, align)) + return -EINVAL; + + while (sg) { + if (!IS_ALIGNED(sg->offset, sizeof(u32))) + return -EINVAL; + + if (!IS_ALIGNED(sg->length, align)) + return -EINVAL; + + len += sg->length; + sg = sg_next(sg); + } + + if (len != total) + return -EINVAL; + + return 0; +} + +static int stm32_cryp_check_io_aligned(struct stm32_cryp *cryp) +{ + int ret; + + ret = stm32_cryp_check_aligned(cryp->in_sg, cryp->total_in, + cryp->hw_blocksize); + if (ret) + return ret; + + ret = stm32_cryp_check_aligned(cryp->out_sg, cryp->total_out, + cryp->hw_blocksize); + + return ret; +} + +static void sg_copy_buf(void *buf, struct scatterlist *sg, + unsigned int start, unsigned int nbytes, int out) +{ + struct scatter_walk walk; + + if (!nbytes) + return; + + scatterwalk_start(&walk, sg); + scatterwalk_advance(&walk, start); + scatterwalk_copychunks(buf, &walk, nbytes, out); + scatterwalk_done(&walk, out, 0); +} + +static int stm32_cryp_copy_sgs(struct stm32_cryp *cryp) +{ + void *buf_in, *buf_out; + int pages, total_in, total_out; + + if (!stm32_cryp_check_io_aligned(cryp)) { + cryp->sgs_copied = 0; + return 0; + } + + total_in = ALIGN(cryp->total_in, cryp->hw_blocksize); + pages = total_in ? get_order(total_in) : 1; + buf_in = (void *)__get_free_pages(GFP_ATOMIC, pages); + + total_out = ALIGN(cryp->total_out, cryp->hw_blocksize); + pages = total_out ? get_order(total_out) : 1; + buf_out = (void *)__get_free_pages(GFP_ATOMIC, pages); + + if (!buf_in || !buf_out) { + dev_err(cryp->dev, "Can't allocate pages when unaligned\n"); + cryp->sgs_copied = 0; + return -EFAULT; + } + + sg_copy_buf(buf_in, cryp->in_sg, 0, cryp->total_in, 0); + + sg_init_one(&cryp->in_sgl, buf_in, total_in); + cryp->in_sg = &cryp->in_sgl; + cryp->in_sg_len = 1; + + sg_init_one(&cryp->out_sgl, buf_out, total_out); + cryp->out_sg_save = cryp->out_sg; + cryp->out_sg = &cryp->out_sgl; + cryp->out_sg_len = 1; + + cryp->sgs_copied = 1; + + return 0; +} + +static void stm32_cryp_hw_write_iv(struct stm32_cryp *cryp, u32 *iv) +{ + if (!iv) + return; + + stm32_cryp_write(cryp, CRYP_IV0LR, cpu_to_be32(*iv++)); + stm32_cryp_write(cryp, CRYP_IV0RR, cpu_to_be32(*iv++)); + + if (is_aes(cryp)) { + stm32_cryp_write(cryp, CRYP_IV1LR, cpu_to_be32(*iv++)); + stm32_cryp_write(cryp, CRYP_IV1RR, cpu_to_be32(*iv++)); + } +} + +static void stm32_cryp_hw_write_key(struct stm32_cryp *c) +{ + unsigned int i; + int r_id; + + if (is_des(c)) { + stm32_cryp_write(c, CRYP_K1LR, cpu_to_be32(c->ctx->key[0])); + stm32_cryp_write(c, CRYP_K1RR, cpu_to_be32(c->ctx->key[1])); + } else { + r_id = CRYP_K3RR; + for (i = c->ctx->keylen / sizeof(u32); i > 0; i--, r_id -= 4) + stm32_cryp_write(c, r_id, + cpu_to_be32(c->ctx->key[i - 1])); + } +} + +static u32 stm32_cryp_get_hw_mode(struct stm32_cryp *cryp) +{ + if (is_aes(cryp) && is_ecb(cryp)) + return CR_AES_ECB; + + if (is_aes(cryp) && is_cbc(cryp)) + return CR_AES_CBC; + + if (is_aes(cryp) && is_ctr(cryp)) + return CR_AES_CTR; + + if (is_des(cryp) && is_ecb(cryp)) + return CR_DES_ECB; + + if (is_des(cryp) && is_cbc(cryp)) + return CR_DES_CBC; + + if (is_tdes(cryp) && is_ecb(cryp)) + return CR_TDES_ECB; + + if (is_tdes(cryp) && is_cbc(cryp)) + return CR_TDES_CBC; + + dev_err(cryp->dev, "Unknown mode\n"); + return CR_AES_UNKNOWN; +} + +static int stm32_cryp_hw_init(struct stm32_cryp *cryp) +{ + int ret; + u32 cfg, hw_mode; + + /* Disable interrupt */ + stm32_cryp_write(cryp, CRYP_IMSCR, 0); + + /* Set key */ + stm32_cryp_hw_write_key(cryp); + + /* Set configuration */ + cfg = CR_DATA8 | CR_FFLUSH; + + switch (cryp->ctx->keylen) { + case AES_KEYSIZE_128: + cfg |= CR_KEY128; + break; + + case AES_KEYSIZE_192: + cfg |= CR_KEY192; + break; + + default: + case AES_KEYSIZE_256: + cfg |= CR_KEY256; + break; + } + + hw_mode = stm32_cryp_get_hw_mode(cryp); + if (hw_mode == CR_AES_UNKNOWN) + return -EINVAL; + + /* AES ECB/CBC decrypt: run key preparation first */ + if (is_decrypt(cryp) && + ((hw_mode == CR_AES_ECB) || (hw_mode == CR_AES_CBC))) { + stm32_cryp_write(cryp, CRYP_CR, cfg | CR_AES_KP | CR_CRYPEN); + + /* Wait for end of processing */ + ret = stm32_cryp_wait_busy(cryp); + if (ret) { + dev_err(cryp->dev, "Timeout (key preparation)\n"); + return ret; + } + } + + cfg |= hw_mode; + + if (is_decrypt(cryp)) + cfg |= CR_DEC_NOT_ENC; + + /* Apply config and flush (valid when CRYPEN = 0) */ + stm32_cryp_write(cryp, CRYP_CR, cfg); + + switch (hw_mode) { + case CR_DES_CBC: + case CR_TDES_CBC: + case CR_AES_CBC: + case CR_AES_CTR: + stm32_cryp_hw_write_iv(cryp, (u32 *)cryp->req->info); + break; + + default: + break; + } + + /* Enable now */ + cfg |= CR_CRYPEN; + + stm32_cryp_write(cryp, CRYP_CR, cfg); + + return 0; +} + +static void stm32_cryp_finish_req(struct stm32_cryp *cryp) +{ + int err = 0; + + if (cryp->sgs_copied) { + void *buf_in, *buf_out; + int pages, len; + + buf_in = sg_virt(&cryp->in_sgl); + buf_out = sg_virt(&cryp->out_sgl); + + sg_copy_buf(buf_out, cryp->out_sg_save, 0, + cryp->total_out_save, 1); + + len = ALIGN(cryp->total_in_save, cryp->hw_blocksize); + pages = len ? get_order(len) : 1; + free_pages((unsigned long)buf_in, pages); + + len = ALIGN(cryp->total_out_save, cryp->hw_blocksize); + pages = len ? get_order(len) : 1; + free_pages((unsigned long)buf_out, pages); + } + + crypto_finalize_cipher_request(cryp->engine, cryp->req, err); + cryp->req = NULL; + + memset(cryp->ctx->key, 0, cryp->ctx->keylen); + + mutex_unlock(&cryp->lock); +} + +static int stm32_cryp_cpu_start(struct stm32_cryp *cryp) +{ + /* Enable interrupt and let the IRQ handler do everything */ + stm32_cryp_write(cryp, CRYP_IMSCR, IMSCR_IN | IMSCR_OUT); + + return 0; +} + +static int stm32_cryp_cra_init(struct crypto_tfm *tfm) +{ + tfm->crt_ablkcipher.reqsize = sizeof(struct stm32_cryp_reqctx); + + return 0; +} + +static int stm32_cryp_crypt(struct ablkcipher_request *req, unsigned long mode) +{ + struct stm32_cryp_ctx *ctx = crypto_ablkcipher_ctx( + crypto_ablkcipher_reqtfm(req)); + struct stm32_cryp_reqctx *rctx = ablkcipher_request_ctx(req); + struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx); + + if (!cryp) + return -ENODEV; + + rctx->mode = mode; + + return crypto_transfer_cipher_request_to_engine(cryp->engine, req); +} + +static int stm32_cryp_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct stm32_cryp_ctx *ctx = crypto_ablkcipher_ctx(tfm); + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + + return 0; +} + +static int stm32_cryp_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int keylen) +{ + if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_256) + return -EINVAL; + else + return stm32_cryp_setkey(tfm, key, keylen); +} + +static int stm32_cryp_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int keylen) +{ + if (keylen != DES_KEY_SIZE) + return -EINVAL; + else + return stm32_cryp_setkey(tfm, key, keylen); +} + +static int stm32_cryp_tdes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int keylen) +{ + if (keylen != (3 * DES_KEY_SIZE)) + return -EINVAL; + else + return stm32_cryp_setkey(tfm, key, keylen); +} + +static int stm32_cryp_aes_ecb_encrypt(struct ablkcipher_request *req) +{ + return stm32_cryp_crypt(req, FLG_AES | FLG_ECB | FLG_ENCRYPT); +} + +static int stm32_cryp_aes_ecb_decrypt(struct ablkcipher_request *req) +{ + return stm32_cryp_crypt(req, FLG_AES | FLG_ECB); +} + +static int stm32_cryp_aes_cbc_encrypt(struct ablkcipher_request *req) +{ + return stm32_cryp_crypt(req, FLG_AES | FLG_CBC | FLG_ENCRYPT); +} + +static int stm32_cryp_aes_cbc_decrypt(struct ablkcipher_request *req) +{ + return stm32_cryp_crypt(req, FLG_AES | FLG_CBC); +} + +static int stm32_cryp_aes_ctr_encrypt(struct ablkcipher_request *req) +{ + return stm32_cryp_crypt(req, FLG_AES | FLG_CTR | FLG_ENCRYPT); +} + +static int stm32_cryp_aes_ctr_decrypt(struct ablkcipher_request *req) +{ + return stm32_cryp_crypt(req, FLG_AES | FLG_CTR); +} + +static int stm32_cryp_des_ecb_encrypt(struct ablkcipher_request *req) +{ + return stm32_cryp_crypt(req, FLG_DES | FLG_ECB | FLG_ENCRYPT); +} + +static int stm32_cryp_des_ecb_decrypt(struct ablkcipher_request *req) +{ + return stm32_cryp_crypt(req, FLG_DES | FLG_ECB); +} + +static int stm32_cryp_des_cbc_encrypt(struct ablkcipher_request *req) +{ + return stm32_cryp_crypt(req, FLG_DES | FLG_CBC | FLG_ENCRYPT); +} + +static int stm32_cryp_des_cbc_decrypt(struct ablkcipher_request *req) +{ + return stm32_cryp_crypt(req, FLG_DES | FLG_CBC); +} + +static int stm32_cryp_tdes_ecb_encrypt(struct ablkcipher_request *req) +{ + return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB | FLG_ENCRYPT); +} + +static int stm32_cryp_tdes_ecb_decrypt(struct ablkcipher_request *req) +{ + return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB); +} + +static int stm32_cryp_tdes_cbc_encrypt(struct ablkcipher_request *req) +{ + return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC | FLG_ENCRYPT); +} + +static int stm32_cryp_tdes_cbc_decrypt(struct ablkcipher_request *req) +{ + return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC); +} + +static int stm32_cryp_prepare_req(struct crypto_engine *engine, + struct ablkcipher_request *req) +{ + struct stm32_cryp_ctx *ctx; + struct stm32_cryp *cryp; + struct stm32_cryp_reqctx *rctx; + int ret; + + if (!req) + return -EINVAL; + + ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req)); + + cryp = ctx->cryp; + + if (!cryp) + return -ENODEV; + + mutex_lock(&cryp->lock); + + rctx = ablkcipher_request_ctx(req); + rctx->mode &= FLG_MODE_MASK; + + ctx->cryp = cryp; + + cryp->flags = (cryp->flags & ~FLG_MODE_MASK) | rctx->mode; + cryp->hw_blocksize = is_aes(cryp) ? AES_BLOCK_SIZE : DES_BLOCK_SIZE; + cryp->ctx = ctx; + + cryp->req = req; + cryp->total_in = req->nbytes; + cryp->total_out = cryp->total_in; + + cryp->total_in_save = cryp->total_in; + cryp->total_out_save = cryp->total_out; + + cryp->in_sg = req->src; + cryp->out_sg = req->dst; + cryp->out_sg_save = cryp->out_sg; + + cryp->in_sg_len = sg_nents_for_len(cryp->in_sg, cryp->total_in); + if (cryp->in_sg_len < 0) { + dev_err(cryp->dev, "Cannot get in_sg_len\n"); + ret = cryp->in_sg_len; + goto out; + } + + cryp->out_sg_len = sg_nents_for_len(cryp->out_sg, cryp->total_out); + if (cryp->out_sg_len < 0) { + dev_err(cryp->dev, "Cannot get out_sg_len\n"); + ret = cryp->out_sg_len; + goto out; + } + + ret = stm32_cryp_copy_sgs(cryp); + if (ret) + goto out; + + scatterwalk_start(&cryp->in_walk, cryp->in_sg); + scatterwalk_start(&cryp->out_walk, cryp->out_sg); + + ret = stm32_cryp_hw_init(cryp); +out: + if (ret) + mutex_unlock(&cryp->lock); + + return ret; +} + +static int stm32_cryp_prepare_cipher_req(struct crypto_engine *engine, + struct ablkcipher_request *req) +{ + return stm32_cryp_prepare_req(engine, req); +} + +static int stm32_cryp_cipher_one_req(struct crypto_engine *engine, + struct ablkcipher_request *req) +{ + struct stm32_cryp_ctx *ctx = crypto_ablkcipher_ctx( + crypto_ablkcipher_reqtfm(req)); + struct stm32_cryp *cryp = ctx->cryp; + + if (!cryp) + return -ENODEV; + + return stm32_cryp_cpu_start(cryp); +} + +static u32 *stm32_cryp_next_out(struct stm32_cryp *cryp, u32 *dst, + unsigned int n) +{ + scatterwalk_advance(&cryp->out_walk, n); + + if (unlikely(cryp->out_sg->length == _walked_out)) { + cryp->out_sg = sg_next(cryp->out_sg); + if (cryp->out_sg) { + scatterwalk_start(&cryp->out_walk, cryp->out_sg); + return (sg_virt(cryp->out_sg) + _walked_out); + } + } + + return (u32 *)((u8 *)dst + n); +} + +static u32 *stm32_cryp_next_in(struct stm32_cryp *cryp, u32 *src, + unsigned int n) +{ + scatterwalk_advance(&cryp->in_walk, n); + + if (unlikely(cryp->in_sg->length == _walked_in)) { + cryp->in_sg = sg_next(cryp->in_sg); + if (cryp->in_sg) { + scatterwalk_start(&cryp->in_walk, cryp->in_sg); + return (sg_virt(cryp->in_sg) + _walked_in); + } + } + + return (u32 *)((u8 *)src + n); +} + +static void stm32_cryp_check_ctr_counter(struct stm32_cryp *cryp) +{ + u32 cr; + + if (unlikely(cryp->last_ctr[3] == 0xFFFFFFFF)) { + cryp->last_ctr[3] = 0; + cryp->last_ctr[2]++; + if (!cryp->last_ctr[2]) { + cryp->last_ctr[1]++; + if (!cryp->last_ctr[1]) + cryp->last_ctr[0]++; + } + + cr = stm32_cryp_read(cryp, CRYP_CR); + stm32_cryp_write(cryp, CRYP_CR, cr & ~CR_CRYPEN); + + stm32_cryp_hw_write_iv(cryp, (u32 *)cryp->last_ctr); + + stm32_cryp_write(cryp, CRYP_CR, cr); + } + + cryp->last_ctr[0] = stm32_cryp_read(cryp, CRYP_IV0LR); + cryp->last_ctr[1] = stm32_cryp_read(cryp, CRYP_IV0RR); + cryp->last_ctr[2] = stm32_cryp_read(cryp, CRYP_IV1LR); + cryp->last_ctr[3] = stm32_cryp_read(cryp, CRYP_IV1RR); +} + +static bool stm32_cryp_irq_read_data(struct stm32_cryp *cryp) +{ + unsigned int i, j; + u32 d32, *dst; + u8 *d8; + + dst = sg_virt(cryp->out_sg) + _walked_out; + + for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++) { + if (likely(cryp->total_out >= sizeof(u32))) { + /* Read a full u32 */ + *dst = stm32_cryp_read(cryp, CRYP_DOUT); + + dst = stm32_cryp_next_out(cryp, dst, sizeof(u32)); + cryp->total_out -= sizeof(u32); + } else if (!cryp->total_out) { + /* Empty fifo out (data from input padding) */ + d32 = stm32_cryp_read(cryp, CRYP_DOUT); + } else { + /* Read less than an u32 */ + d32 = stm32_cryp_read(cryp, CRYP_DOUT); + d8 = (u8 *)&d32; + + for (j = 0; j < cryp->total_out; j++) { + *((u8 *)dst) = *(d8++); + dst = stm32_cryp_next_out(cryp, dst, 1); + } + cryp->total_out = 0; + } + } + + return !cryp->total_out || !cryp->total_in; +} + +static void stm32_cryp_irq_write_block(struct stm32_cryp *cryp) +{ + unsigned int i, j; + u32 *src; + u8 d8[4]; + + src = sg_virt(cryp->in_sg) + _walked_in; + + for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++) { + if (likely(cryp->total_in >= sizeof(u32))) { + /* Write a full u32 */ + stm32_cryp_write(cryp, CRYP_DIN, *src); + + src = stm32_cryp_next_in(cryp, src, sizeof(u32)); + cryp->total_in -= sizeof(u32); + } else if (!cryp->total_in) { + /* Write padding data */ + stm32_cryp_write(cryp, CRYP_DIN, 0); + } else { + /* Write less than an u32 */ + memset(d8, 0, sizeof(u32)); + for (j = 0; j < cryp->total_in; j++) { + d8[j] = *((u8 *)src); + src = stm32_cryp_next_in(cryp, src, 1); + } + + stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8); + cryp->total_in = 0; + } + } +} + +static void stm32_cryp_irq_write_data(struct stm32_cryp *cryp) +{ + if (unlikely(!cryp->total_in)) { + dev_warn(cryp->dev, "No more data to process\n"); + return; + } + + if (is_aes(cryp) && is_ctr(cryp)) + stm32_cryp_check_ctr_counter(cryp); + + stm32_cryp_irq_write_block(cryp); +} + +static irqreturn_t stm32_cryp_irq_thread(int irq, void *arg) +{ + struct stm32_cryp *cryp = arg; + + if (cryp->irq_status & MISR_OUT) + /* Output FIFO IRQ: read data */ + if (unlikely(stm32_cryp_irq_read_data(cryp))) { + /* All bytes processed, finish */ + stm32_cryp_write(cryp, CRYP_IMSCR, 0); + stm32_cryp_finish_req(cryp); + return IRQ_HANDLED; + } + + if (cryp->irq_status & MISR_IN) { + /* Input FIFO IRQ: write data */ + stm32_cryp_irq_write_data(cryp); + } + + return IRQ_HANDLED; +} + +static irqreturn_t stm32_cryp_irq(int irq, void *arg) +{ + struct stm32_cryp *cryp = arg; + + cryp->irq_status = stm32_cryp_read(cryp, CRYP_MISR); + + return IRQ_WAKE_THREAD; +} + +static struct crypto_alg crypto_algs[] = { +{ + .cra_name = "ecb(aes)", + .cra_driver_name = "stm32-ecb-aes", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_cryp_ctx), + .cra_alignmask = 0xf, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = stm32_cryp_cra_init, + .cra_ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = stm32_cryp_aes_setkey, + .encrypt = stm32_cryp_aes_ecb_encrypt, + .decrypt = stm32_cryp_aes_ecb_decrypt, + } +}, +{ + .cra_name = "cbc(aes)", + .cra_driver_name = "stm32-cbc-aes", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_cryp_ctx), + .cra_alignmask = 0xf, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = stm32_cryp_cra_init, + .cra_ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = stm32_cryp_aes_setkey, + .encrypt = stm32_cryp_aes_cbc_encrypt, + .decrypt = stm32_cryp_aes_cbc_decrypt, + } +}, +{ + .cra_name = "ctr(aes)", + .cra_driver_name = "stm32-ctr-aes", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct stm32_cryp_ctx), + .cra_alignmask = 0xf, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = stm32_cryp_cra_init, + .cra_ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = stm32_cryp_aes_setkey, + .encrypt = stm32_cryp_aes_ctr_encrypt, + .decrypt = stm32_cryp_aes_ctr_decrypt, + } +}, +{ + .cra_name = "ecb(des)", + .cra_driver_name = "stm32-ecb-des", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_cryp_ctx), + .cra_alignmask = 0xf, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = stm32_cryp_cra_init, + .cra_ablkcipher = { + .min_keysize = DES_BLOCK_SIZE, + .max_keysize = DES_BLOCK_SIZE, + .setkey = stm32_cryp_des_setkey, + .encrypt = stm32_cryp_des_ecb_encrypt, + .decrypt = stm32_cryp_des_ecb_decrypt, + } +}, +{ + .cra_name = "cbc(des)", + .cra_driver_name = "stm32-cbc-des", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_cryp_ctx), + .cra_alignmask = 0xf, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = stm32_cryp_cra_init, + .cra_ablkcipher = { + .min_keysize = DES_BLOCK_SIZE, + .max_keysize = DES_BLOCK_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = stm32_cryp_des_setkey, + .encrypt = stm32_cryp_des_cbc_encrypt, + .decrypt = stm32_cryp_des_cbc_decrypt, + } +}, +{ + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "stm32-ecb-des3", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_cryp_ctx), + .cra_alignmask = 0xf, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = stm32_cryp_cra_init, + .cra_ablkcipher = { + .min_keysize = 3 * DES_BLOCK_SIZE, + .max_keysize = 3 * DES_BLOCK_SIZE, + .setkey = stm32_cryp_tdes_setkey, + .encrypt = stm32_cryp_tdes_ecb_encrypt, + .decrypt = stm32_cryp_tdes_ecb_decrypt, + } +}, +{ + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "stm32-cbc-des3", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_cryp_ctx), + .cra_alignmask = 0xf, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = stm32_cryp_cra_init, + .cra_ablkcipher = { + .min_keysize = 3 * DES_BLOCK_SIZE, + .max_keysize = 3 * DES_BLOCK_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = stm32_cryp_tdes_setkey, + .encrypt = stm32_cryp_tdes_cbc_encrypt, + .decrypt = stm32_cryp_tdes_cbc_decrypt, + } +}, +}; + +static const struct of_device_id stm32_dt_ids[] = { + { .compatible = "st,stm32f756-cryp", }, + {}, +}; +MODULE_DEVICE_TABLE(of, sti_dt_ids); + +static int stm32_cryp_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct stm32_cryp *cryp; + struct resource *res; + struct reset_control *rst; + int irq, ret; + + cryp = devm_kzalloc(dev, sizeof(*cryp), GFP_KERNEL); + if (!cryp) + return -ENOMEM; + + cryp->dev = dev; + + mutex_init(&cryp->lock); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + cryp->regs = devm_ioremap_resource(dev, res); + if (IS_ERR(cryp->regs)) { + dev_err(dev, "Cannot map CRYP IO\n"); + return PTR_ERR(cryp->regs); + } + + irq = platform_get_irq(pdev, 0); + if (irq < 0) { + dev_err(dev, "Cannot get IRQ resource\n"); + return irq; + } + + ret = devm_request_threaded_irq(dev, irq, stm32_cryp_irq, + stm32_cryp_irq_thread, IRQF_ONESHOT, + dev_name(dev), cryp); + if (ret) { + dev_err(dev, "Cannot grab IRQ\n"); + return ret; + } + + cryp->clk = devm_clk_get(dev, NULL); + if (IS_ERR(cryp->clk)) { + dev_err(dev, "Could not get clock\n"); + return PTR_ERR(cryp->clk); + } + + ret = clk_prepare_enable(cryp->clk); + if (ret) { + dev_err(cryp->dev, "Failed to enable clock\n"); + return ret; + } + + rst = devm_reset_control_get(dev, NULL); + if (!IS_ERR(rst)) { + reset_control_assert(rst); + udelay(2); + reset_control_deassert(rst); + } + + platform_set_drvdata(pdev, cryp); + + spin_lock(&cryp_list.lock); + list_add(&cryp->list, &cryp_list.dev_list); + spin_unlock(&cryp_list.lock); + + /* Initialize crypto engine */ + cryp->engine = crypto_engine_alloc_init(dev, 1); + if (!cryp->engine) { + dev_err(dev, "Could not init crypto engine\n"); + ret = -ENOMEM; + goto err_engine1; + } + + cryp->engine->prepare_cipher_request = stm32_cryp_prepare_cipher_req; + cryp->engine->cipher_one_request = stm32_cryp_cipher_one_req; + + ret = crypto_engine_start(cryp->engine); + if (ret) { + dev_err(dev, "Could not start crypto engine\n"); + goto err_engine2; + } + + ret = crypto_register_algs(crypto_algs, ARRAY_SIZE(crypto_algs)); + if (ret) { + dev_err(dev, "Could not register algs\n"); + goto err_algs; + } + + dev_info(dev, "Initialized\n"); + + return 0; + +err_algs: +err_engine2: + crypto_engine_exit(cryp->engine); +err_engine1: + spin_lock(&cryp_list.lock); + list_del(&cryp->list); + spin_unlock(&cryp_list.lock); + + clk_disable_unprepare(cryp->clk); + + return ret; +} + +static int stm32_cryp_remove(struct platform_device *pdev) +{ + struct stm32_cryp *cryp = platform_get_drvdata(pdev); + + if (!cryp) + return -ENODEV; + + crypto_unregister_algs(crypto_algs, ARRAY_SIZE(crypto_algs)); + + crypto_engine_exit(cryp->engine); + + spin_lock(&cryp_list.lock); + list_del(&cryp->list); + spin_unlock(&cryp_list.lock); + + clk_disable_unprepare(cryp->clk); + + return 0; +} + +static struct platform_driver stm32_cryp_driver = { + .probe = stm32_cryp_probe, + .remove = stm32_cryp_remove, + .driver = { + .name = DRIVER_NAME, + .of_match_table = stm32_dt_ids, + }, +}; + +module_platform_driver(stm32_cryp_driver); + +MODULE_AUTHOR("Fabien Dessenne "); +MODULE_DESCRIPTION("STMicrolectronics STM32 CRYP hardware driver"); +MODULE_LICENSE("GPL"); From 21bb0ef43c11184961792790919d6daf9823e57a Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Tue, 7 Nov 2017 16:44:38 -0800 Subject: [PATCH 004/166] hwrng: bcm2835 - Obtain base register via resource In preparation for consolidating bcm63xx-rng into bcm2835-rng, make sure that we obtain the base register via platform_get_resource() since we need to support the non-DT enabled MIPS-based BCM63xx DSL SoCs. Signed-off-by: Florian Fainelli Signed-off-by: Herbert Xu --- drivers/char/hw_random/bcm2835-rng.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/char/hw_random/bcm2835-rng.c b/drivers/char/hw_random/bcm2835-rng.c index 574211a49549..a818418a7e4c 100644 --- a/drivers/char/hw_random/bcm2835-rng.c +++ b/drivers/char/hw_random/bcm2835-rng.c @@ -81,21 +81,23 @@ static int bcm2835_rng_probe(struct platform_device *pdev) void (*rng_setup)(void __iomem *base); const struct of_device_id *rng_id; void __iomem *rng_base; + struct resource *r; int err; + r = platform_get_resource(pdev, IORESOURCE_MEM, 0); + /* map peripheral */ - rng_base = of_iomap(np, 0); - if (!rng_base) { + rng_base = devm_ioremap_resource(dev, r); + if (IS_ERR(rng_base)) { dev_err(dev, "failed to remap rng regs"); - return -ENODEV; + return PTR_ERR(rng_base); } bcm2835_rng_ops.priv = (unsigned long)rng_base; rng_id = of_match_node(bcm2835_rng_of_match, np); - if (!rng_id) { - iounmap(rng_base); + if (!rng_id) return -EINVAL; - } + /* Check for rng init function, execute it */ rng_setup = rng_id->data; if (rng_setup) @@ -107,10 +109,9 @@ static int bcm2835_rng_probe(struct platform_device *pdev) /* register driver */ err = hwrng_register(&bcm2835_rng_ops); - if (err) { + if (err) dev_err(dev, "hwrng registration failed\n"); - iounmap(rng_base); - } else + else dev_info(dev, "hwrng registered\n"); return err; @@ -125,7 +126,6 @@ static int bcm2835_rng_remove(struct platform_device *pdev) /* unregister driver */ hwrng_unregister(&bcm2835_rng_ops); - iounmap(rng_base); return 0; } From b788479f68263fbb191daa5b2007de45f0548a72 Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Tue, 7 Nov 2017 16:44:39 -0800 Subject: [PATCH 005/166] hwrng: bcm2835 - Define a driver private context Instead of making hwrng::priv host the base register address, define a driver private context, make it per platform device instance and pass it down the different functions. Signed-off-by: Florian Fainelli Reviewed-by: Eric Anholt Signed-off-by: Herbert Xu --- drivers/char/hw_random/bcm2835-rng.c | 55 +++++++++++++++++----------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/drivers/char/hw_random/bcm2835-rng.c b/drivers/char/hw_random/bcm2835-rng.c index a818418a7e4c..0d72147ab45b 100644 --- a/drivers/char/hw_random/bcm2835-rng.c +++ b/drivers/char/hw_random/bcm2835-rng.c @@ -29,6 +29,11 @@ #define RNG_INT_OFF 0x1 +struct bcm2835_rng_priv { + struct hwrng rng; + void __iomem *base; +}; + static void __init nsp_rng_init(void __iomem *base) { u32 val; @@ -39,34 +44,34 @@ static void __init nsp_rng_init(void __iomem *base) writel(val, base + RNG_INT_MASK); } +static inline struct bcm2835_rng_priv *to_rng_priv(struct hwrng *rng) +{ + return container_of(rng, struct bcm2835_rng_priv, rng); +} + static int bcm2835_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) { - void __iomem *rng_base = (void __iomem *)rng->priv; + struct bcm2835_rng_priv *priv = to_rng_priv(rng); u32 max_words = max / sizeof(u32); u32 num_words, count; - while ((__raw_readl(rng_base + RNG_STATUS) >> 24) == 0) { + while ((__raw_readl(priv->base + RNG_STATUS) >> 24) == 0) { if (!wait) return 0; cpu_relax(); } - num_words = readl(rng_base + RNG_STATUS) >> 24; + num_words = readl(priv->base + RNG_STATUS) >> 24; if (num_words > max_words) num_words = max_words; for (count = 0; count < num_words; count++) - ((u32 *)buf)[count] = readl(rng_base + RNG_DATA); + ((u32 *)buf)[count] = readl(priv->base + RNG_DATA); return num_words * sizeof(u32); } -static struct hwrng bcm2835_rng_ops = { - .name = "bcm2835", - .read = bcm2835_rng_read, -}; - static const struct of_device_id bcm2835_rng_of_match[] = { { .compatible = "brcm,bcm2835-rng"}, { .compatible = "brcm,bcm-nsp-rng", .data = nsp_rng_init}, @@ -80,19 +85,27 @@ static int bcm2835_rng_probe(struct platform_device *pdev) struct device_node *np = dev->of_node; void (*rng_setup)(void __iomem *base); const struct of_device_id *rng_id; - void __iomem *rng_base; + struct bcm2835_rng_priv *priv; struct resource *r; int err; + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + platform_set_drvdata(pdev, priv); + r = platform_get_resource(pdev, IORESOURCE_MEM, 0); /* map peripheral */ - rng_base = devm_ioremap_resource(dev, r); - if (IS_ERR(rng_base)) { + priv->base = devm_ioremap_resource(dev, r); + if (IS_ERR(priv->base)) { dev_err(dev, "failed to remap rng regs"); - return PTR_ERR(rng_base); + return PTR_ERR(priv->base); } - bcm2835_rng_ops.priv = (unsigned long)rng_base; + + priv->rng.name = "bcm2835-rng"; + priv->rng.read = bcm2835_rng_read; rng_id = of_match_node(bcm2835_rng_of_match, np); if (!rng_id) @@ -101,14 +114,14 @@ static int bcm2835_rng_probe(struct platform_device *pdev) /* Check for rng init function, execute it */ rng_setup = rng_id->data; if (rng_setup) - rng_setup(rng_base); + rng_setup(priv->base); /* set warm-up count & enable */ - __raw_writel(RNG_WARMUP_COUNT, rng_base + RNG_STATUS); - __raw_writel(RNG_RBGEN, rng_base + RNG_CTRL); + __raw_writel(RNG_WARMUP_COUNT, priv->base + RNG_STATUS); + __raw_writel(RNG_RBGEN, priv->base + RNG_CTRL); /* register driver */ - err = hwrng_register(&bcm2835_rng_ops); + err = hwrng_register(&priv->rng); if (err) dev_err(dev, "hwrng registration failed\n"); else @@ -119,13 +132,13 @@ static int bcm2835_rng_probe(struct platform_device *pdev) static int bcm2835_rng_remove(struct platform_device *pdev) { - void __iomem *rng_base = (void __iomem *)bcm2835_rng_ops.priv; + struct bcm2835_rng_priv *priv = platform_get_drvdata(pdev); /* disable rng hardware */ - __raw_writel(0, rng_base + RNG_CTRL); + __raw_writel(0, priv->base + RNG_CTRL); /* unregister driver */ - hwrng_unregister(&bcm2835_rng_ops); + hwrng_unregister(&priv->rng); return 0; } From a815777553826079929e0910fcf3fda9a1e8438b Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Tue, 7 Nov 2017 16:44:40 -0800 Subject: [PATCH 006/166] hwrng: bcm2835 - Move enabling to hwrng::init We should be moving the enabling of the HWRNG into a hwrng::init callback since we can be disabled and enabled every time a different hwrng is selected in the system. Signed-off-by: Florian Fainelli Signed-off-by: Herbert Xu --- drivers/char/hw_random/bcm2835-rng.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/drivers/char/hw_random/bcm2835-rng.c b/drivers/char/hw_random/bcm2835-rng.c index 0d72147ab45b..82000a637504 100644 --- a/drivers/char/hw_random/bcm2835-rng.c +++ b/drivers/char/hw_random/bcm2835-rng.c @@ -72,6 +72,17 @@ static int bcm2835_rng_read(struct hwrng *rng, void *buf, size_t max, return num_words * sizeof(u32); } +static int bcm2835_rng_init(struct hwrng *rng) +{ + struct bcm2835_rng_priv *priv = to_rng_priv(rng); + + /* set warm-up count & enable */ + __raw_writel(RNG_WARMUP_COUNT, priv->base + RNG_STATUS); + __raw_writel(RNG_RBGEN, priv->base + RNG_CTRL); + + return 0; +} + static const struct of_device_id bcm2835_rng_of_match[] = { { .compatible = "brcm,bcm2835-rng"}, { .compatible = "brcm,bcm-nsp-rng", .data = nsp_rng_init}, @@ -105,6 +116,7 @@ static int bcm2835_rng_probe(struct platform_device *pdev) } priv->rng.name = "bcm2835-rng"; + priv->rng.init = bcm2835_rng_init; priv->rng.read = bcm2835_rng_read; rng_id = of_match_node(bcm2835_rng_of_match, np); @@ -116,10 +128,6 @@ static int bcm2835_rng_probe(struct platform_device *pdev) if (rng_setup) rng_setup(priv->base); - /* set warm-up count & enable */ - __raw_writel(RNG_WARMUP_COUNT, priv->base + RNG_STATUS); - __raw_writel(RNG_RBGEN, priv->base + RNG_CTRL); - /* register driver */ err = hwrng_register(&priv->rng); if (err) From ec94bca7be6b29c1aebf2e1e95855ee6ca0d5d62 Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Tue, 7 Nov 2017 16:44:41 -0800 Subject: [PATCH 007/166] hwrng: bcm2835 - Implementation cleanup callback We should be disabling the RNG in a hwrng::cleanup callback if we are not longer the system selected RNG, not wait until the device driver is removed. Signed-off-by: Florian Fainelli Signed-off-by: Herbert Xu --- drivers/char/hw_random/bcm2835-rng.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/drivers/char/hw_random/bcm2835-rng.c b/drivers/char/hw_random/bcm2835-rng.c index 82000a637504..4d0356110b1b 100644 --- a/drivers/char/hw_random/bcm2835-rng.c +++ b/drivers/char/hw_random/bcm2835-rng.c @@ -83,6 +83,14 @@ static int bcm2835_rng_init(struct hwrng *rng) return 0; } +static void bcm2835_rng_cleanup(struct hwrng *rng) +{ + struct bcm2835_rng_priv *priv = to_rng_priv(rng); + + /* disable rng hardware */ + __raw_writel(0, priv->base + RNG_CTRL); +} + static const struct of_device_id bcm2835_rng_of_match[] = { { .compatible = "brcm,bcm2835-rng"}, { .compatible = "brcm,bcm-nsp-rng", .data = nsp_rng_init}, @@ -118,6 +126,7 @@ static int bcm2835_rng_probe(struct platform_device *pdev) priv->rng.name = "bcm2835-rng"; priv->rng.init = bcm2835_rng_init; priv->rng.read = bcm2835_rng_read; + priv->rng.cleanup = bcm2835_rng_cleanup; rng_id = of_match_node(bcm2835_rng_of_match, np); if (!rng_id) @@ -142,9 +151,6 @@ static int bcm2835_rng_remove(struct platform_device *pdev) { struct bcm2835_rng_priv *priv = platform_get_drvdata(pdev); - /* disable rng hardware */ - __raw_writel(0, priv->base + RNG_CTRL); - /* unregister driver */ hwrng_unregister(&priv->rng); From 16a4c04b3729910f864512a1150fee5ad19efb4a Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Tue, 7 Nov 2017 16:44:42 -0800 Subject: [PATCH 008/166] hwrng: bcm2835 - Use device managed helpers Now that we have moved the RNG disabling into a hwrng::cleanup callback, we can use the device managed registration operation and remove our remove callback since it won't do anything necessary. Signed-off-by: Florian Fainelli Reviewed-by: Eric Anholt Signed-off-by: Herbert Xu --- drivers/char/hw_random/bcm2835-rng.c | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/drivers/char/hw_random/bcm2835-rng.c b/drivers/char/hw_random/bcm2835-rng.c index 4d0356110b1b..67b9bd3be28d 100644 --- a/drivers/char/hw_random/bcm2835-rng.c +++ b/drivers/char/hw_random/bcm2835-rng.c @@ -138,7 +138,7 @@ static int bcm2835_rng_probe(struct platform_device *pdev) rng_setup(priv->base); /* register driver */ - err = hwrng_register(&priv->rng); + err = devm_hwrng_register(dev, &priv->rng); if (err) dev_err(dev, "hwrng registration failed\n"); else @@ -147,16 +147,6 @@ static int bcm2835_rng_probe(struct platform_device *pdev) return err; } -static int bcm2835_rng_remove(struct platform_device *pdev) -{ - struct bcm2835_rng_priv *priv = platform_get_drvdata(pdev); - - /* unregister driver */ - hwrng_unregister(&priv->rng); - - return 0; -} - MODULE_DEVICE_TABLE(of, bcm2835_rng_of_match); static struct platform_driver bcm2835_rng_driver = { @@ -165,7 +155,6 @@ static struct platform_driver bcm2835_rng_driver = { .of_match_table = bcm2835_rng_of_match, }, .probe = bcm2835_rng_probe, - .remove = bcm2835_rng_remove, }; module_platform_driver(bcm2835_rng_driver); From 04b154fa86afb6ccfd38ef9b563c098b13f5984d Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Tue, 7 Nov 2017 16:44:43 -0800 Subject: [PATCH 009/166] hwrng: bcm2835 - Rework interrupt masking The interrupt masking done for Northstart Plus and Northstar (BCM5301X) is moved from being a function pointer mapped to of_device_id::data into a proper part of the hwrng::init callback. While at it, we also make the of_data be a proper structure indicating the platform specifics, since the day we need to add a second type of platform information, we would have to do that anyway. Signed-off-by: Florian Fainelli Reviewed-by: Eric Anholt Signed-off-by: Herbert Xu --- drivers/char/hw_random/bcm2835-rng.c | 39 ++++++++++++++++------------ 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/drivers/char/hw_random/bcm2835-rng.c b/drivers/char/hw_random/bcm2835-rng.c index 67b9bd3be28d..ed20e0b6b7ae 100644 --- a/drivers/char/hw_random/bcm2835-rng.c +++ b/drivers/char/hw_random/bcm2835-rng.c @@ -32,18 +32,9 @@ struct bcm2835_rng_priv { struct hwrng rng; void __iomem *base; + bool mask_interrupts; }; -static void __init nsp_rng_init(void __iomem *base) -{ - u32 val; - - /* mask the interrupt */ - val = readl(base + RNG_INT_MASK); - val |= RNG_INT_OFF; - writel(val, base + RNG_INT_MASK); -} - static inline struct bcm2835_rng_priv *to_rng_priv(struct hwrng *rng) { return container_of(rng, struct bcm2835_rng_priv, rng); @@ -75,6 +66,14 @@ static int bcm2835_rng_read(struct hwrng *rng, void *buf, size_t max, static int bcm2835_rng_init(struct hwrng *rng) { struct bcm2835_rng_priv *priv = to_rng_priv(rng); + u32 val; + + if (priv->mask_interrupts) { + /* mask the interrupt */ + val = readl(priv->base + RNG_INT_MASK); + val |= RNG_INT_OFF; + writel(val, priv->base + RNG_INT_MASK); + } /* set warm-up count & enable */ __raw_writel(RNG_WARMUP_COUNT, priv->base + RNG_STATUS); @@ -91,18 +90,26 @@ static void bcm2835_rng_cleanup(struct hwrng *rng) __raw_writel(0, priv->base + RNG_CTRL); } +struct bcm2835_rng_of_data { + bool mask_interrupts; +}; + +static const struct bcm2835_rng_of_data nsp_rng_of_data = { + .mask_interrupts = true, +}; + static const struct of_device_id bcm2835_rng_of_match[] = { { .compatible = "brcm,bcm2835-rng"}, - { .compatible = "brcm,bcm-nsp-rng", .data = nsp_rng_init}, - { .compatible = "brcm,bcm5301x-rng", .data = nsp_rng_init}, + { .compatible = "brcm,bcm-nsp-rng", .data = &nsp_rng_of_data }, + { .compatible = "brcm,bcm5301x-rng", .data = &nsp_rng_of_data }, {}, }; static int bcm2835_rng_probe(struct platform_device *pdev) { + const struct bcm2835_rng_of_data *of_data; struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; - void (*rng_setup)(void __iomem *base); const struct of_device_id *rng_id; struct bcm2835_rng_priv *priv; struct resource *r; @@ -133,9 +140,9 @@ static int bcm2835_rng_probe(struct platform_device *pdev) return -EINVAL; /* Check for rng init function, execute it */ - rng_setup = rng_id->data; - if (rng_setup) - rng_setup(priv->base); + of_data = rng_id->data; + if (of_data) + priv->mask_interrupts = of_data->mask_interrupts; /* register driver */ err = devm_hwrng_register(dev, &priv->rng); From 791af4f4907a8bee879713cb19fa90e6369f8386 Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Tue, 7 Nov 2017 16:44:44 -0800 Subject: [PATCH 010/166] hwrng: bcm2835 - Manage an optional clock One of the last steps before bcm63xx-rng can be eliminated is to manage a clock during hwrng::init and hwrng::cleanup, so fetch it in the probe function, and manage it during these two steps when valid. Signed-off-by: Florian Fainelli Reviewed-by: Eric Anholt Signed-off-by: Herbert Xu --- drivers/char/hw_random/bcm2835-rng.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/drivers/char/hw_random/bcm2835-rng.c b/drivers/char/hw_random/bcm2835-rng.c index ed20e0b6b7ae..99b56fd5482c 100644 --- a/drivers/char/hw_random/bcm2835-rng.c +++ b/drivers/char/hw_random/bcm2835-rng.c @@ -15,6 +15,7 @@ #include #include #include +#include #define RNG_CTRL 0x0 #define RNG_STATUS 0x4 @@ -33,6 +34,7 @@ struct bcm2835_rng_priv { struct hwrng rng; void __iomem *base; bool mask_interrupts; + struct clk *clk; }; static inline struct bcm2835_rng_priv *to_rng_priv(struct hwrng *rng) @@ -66,8 +68,15 @@ static int bcm2835_rng_read(struct hwrng *rng, void *buf, size_t max, static int bcm2835_rng_init(struct hwrng *rng) { struct bcm2835_rng_priv *priv = to_rng_priv(rng); + int ret = 0; u32 val; + if (!IS_ERR(priv->clk)) { + ret = clk_prepare_enable(priv->clk); + if (ret) + return ret; + } + if (priv->mask_interrupts) { /* mask the interrupt */ val = readl(priv->base + RNG_INT_MASK); @@ -79,7 +88,7 @@ static int bcm2835_rng_init(struct hwrng *rng) __raw_writel(RNG_WARMUP_COUNT, priv->base + RNG_STATUS); __raw_writel(RNG_RBGEN, priv->base + RNG_CTRL); - return 0; + return ret; } static void bcm2835_rng_cleanup(struct hwrng *rng) @@ -88,6 +97,9 @@ static void bcm2835_rng_cleanup(struct hwrng *rng) /* disable rng hardware */ __raw_writel(0, priv->base + RNG_CTRL); + + if (!IS_ERR(priv->clk)) + clk_disable_unprepare(priv->clk); } struct bcm2835_rng_of_data { @@ -130,6 +142,9 @@ static int bcm2835_rng_probe(struct platform_device *pdev) return PTR_ERR(priv->base); } + /* Clock is optional on most platforms */ + priv->clk = devm_clk_get(dev, NULL); + priv->rng.name = "bcm2835-rng"; priv->rng.init = bcm2835_rng_init; priv->rng.read = bcm2835_rng_read; From abd42026eab99691d8ab9e1cd417553cfadf9b76 Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Tue, 7 Nov 2017 16:44:45 -0800 Subject: [PATCH 011/166] hwrng: bcm2835 - Abstract I/O accessors In preparation for allowing BCM63xx to use this driver, we abstract I/O accessors such that we can easily change those later on. Signed-off-by: Florian Fainelli Reviewed-by: Eric Anholt Signed-off-by: Herbert Xu --- drivers/char/hw_random/bcm2835-rng.c | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/drivers/char/hw_random/bcm2835-rng.c b/drivers/char/hw_random/bcm2835-rng.c index 99b56fd5482c..3a607472687d 100644 --- a/drivers/char/hw_random/bcm2835-rng.c +++ b/drivers/char/hw_random/bcm2835-rng.c @@ -42,6 +42,17 @@ static inline struct bcm2835_rng_priv *to_rng_priv(struct hwrng *rng) return container_of(rng, struct bcm2835_rng_priv, rng); } +static inline u32 rng_readl(struct bcm2835_rng_priv *priv, u32 offset) +{ + return readl(priv->base + offset); +} + +static inline void rng_writel(struct bcm2835_rng_priv *priv, u32 val, + u32 offset) +{ + writel(val, priv->base + offset); +} + static int bcm2835_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) { @@ -49,18 +60,18 @@ static int bcm2835_rng_read(struct hwrng *rng, void *buf, size_t max, u32 max_words = max / sizeof(u32); u32 num_words, count; - while ((__raw_readl(priv->base + RNG_STATUS) >> 24) == 0) { + while ((rng_readl(priv, RNG_STATUS) >> 24) == 0) { if (!wait) return 0; cpu_relax(); } - num_words = readl(priv->base + RNG_STATUS) >> 24; + num_words = rng_readl(priv, RNG_STATUS) >> 24; if (num_words > max_words) num_words = max_words; for (count = 0; count < num_words; count++) - ((u32 *)buf)[count] = readl(priv->base + RNG_DATA); + ((u32 *)buf)[count] = rng_readl(priv, RNG_DATA); return num_words * sizeof(u32); } @@ -79,14 +90,14 @@ static int bcm2835_rng_init(struct hwrng *rng) if (priv->mask_interrupts) { /* mask the interrupt */ - val = readl(priv->base + RNG_INT_MASK); + val = rng_readl(priv, RNG_INT_MASK); val |= RNG_INT_OFF; - writel(val, priv->base + RNG_INT_MASK); + rng_writel(priv, val, RNG_INT_MASK); } /* set warm-up count & enable */ - __raw_writel(RNG_WARMUP_COUNT, priv->base + RNG_STATUS); - __raw_writel(RNG_RBGEN, priv->base + RNG_CTRL); + rng_writel(priv, RNG_WARMUP_COUNT, RNG_STATUS); + rng_writel(priv, RNG_RBGEN, RNG_CTRL); return ret; } @@ -96,7 +107,7 @@ static void bcm2835_rng_cleanup(struct hwrng *rng) struct bcm2835_rng_priv *priv = to_rng_priv(rng); /* disable rng hardware */ - __raw_writel(0, priv->base + RNG_CTRL); + rng_writel(priv, 0, RNG_CTRL); if (!IS_ERR(priv->clk)) clk_disable_unprepare(priv->clk); From 6f09359a6810d1c903c97231803ef4518a3f7558 Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Tue, 7 Nov 2017 16:44:46 -0800 Subject: [PATCH 012/166] hwrng: bcm2835 - Add Broadcom MIPS I/O accessors Broadcom MIPS HW is always strapped to match the system-wide endian such that all I/O access to this RNG block is done with the native CPU endian, account for that. Signed-off-by: Florian Fainelli Signed-off-by: Herbert Xu --- drivers/char/hw_random/bcm2835-rng.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/drivers/char/hw_random/bcm2835-rng.c b/drivers/char/hw_random/bcm2835-rng.c index 3a607472687d..6dd8f48701b5 100644 --- a/drivers/char/hw_random/bcm2835-rng.c +++ b/drivers/char/hw_random/bcm2835-rng.c @@ -44,13 +44,22 @@ static inline struct bcm2835_rng_priv *to_rng_priv(struct hwrng *rng) static inline u32 rng_readl(struct bcm2835_rng_priv *priv, u32 offset) { - return readl(priv->base + offset); + /* MIPS chips strapped for BE will automagically configure the + * peripheral registers for CPU-native byte order. + */ + if (IS_ENABLED(CONFIG_MIPS) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) + return __raw_readl(priv->base + offset); + else + return readl(priv->base + offset); } static inline void rng_writel(struct bcm2835_rng_priv *priv, u32 val, u32 offset) { - writel(val, priv->base + offset); + if (IS_ENABLED(CONFIG_MIPS) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) + __raw_writel(val, priv->base + offset); + else + writel(val, priv->base + offset); } static int bcm2835_rng_read(struct hwrng *rng, void *buf, size_t max, From 67fe2fa8cfb2deed0ec5f072a163083e36eaa0b1 Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Tue, 7 Nov 2017 16:44:47 -0800 Subject: [PATCH 013/166] dt-bindings: rng: Incorporate brcm,bcm6368.txt binding Since the same block is used on BCM2835 and BCM6368, merge the bindings and remove the brcm,bcm6368.txt binding document. Acked-by: Rob Herring Signed-off-by: Florian Fainelli Signed-off-by: Herbert Xu --- .../devicetree/bindings/rng/brcm,bcm2835.txt | 22 ++++++++++++++++--- .../devicetree/bindings/rng/brcm,bcm6368.txt | 17 -------------- 2 files changed, 19 insertions(+), 20 deletions(-) delete mode 100644 Documentation/devicetree/bindings/rng/brcm,bcm6368.txt diff --git a/Documentation/devicetree/bindings/rng/brcm,bcm2835.txt b/Documentation/devicetree/bindings/rng/brcm,bcm2835.txt index 26542690b578..627b29531a32 100644 --- a/Documentation/devicetree/bindings/rng/brcm,bcm2835.txt +++ b/Documentation/devicetree/bindings/rng/brcm,bcm2835.txt @@ -1,11 +1,19 @@ -BCM2835 Random number generator +BCM2835/6368 Random number generator Required properties: -- compatible : should be "brcm,bcm2835-rng" or "brcm,bcm-nsp-rng" or - "brcm,bcm5301x-rng" +- compatible : should be one of + "brcm,bcm2835-rng" + "brcm,bcm-nsp-rng" + "brcm,bcm5301x-rng" or + "brcm,bcm6368-rng" - reg : Specifies base physical address and size of the registers. +Optional properties: + +- clocks : phandle to clock-controller plus clock-specifier pair +- clock-names : "ipsec" as a clock name + Example: rng { @@ -17,3 +25,11 @@ rng@18033000 { compatible = "brcm,bcm-nsp-rng"; reg = <0x18033000 0x14>; }; + +random: rng@10004180 { + compatible = "brcm,bcm6368-rng"; + reg = <0x10004180 0x14>; + + clocks = <&periph_clk 18>; + clock-names = "ipsec"; +}; diff --git a/Documentation/devicetree/bindings/rng/brcm,bcm6368.txt b/Documentation/devicetree/bindings/rng/brcm,bcm6368.txt deleted file mode 100644 index 4b5ac600bfbd..000000000000 --- a/Documentation/devicetree/bindings/rng/brcm,bcm6368.txt +++ /dev/null @@ -1,17 +0,0 @@ -BCM6368 Random number generator - -Required properties: - -- compatible : should be "brcm,bcm6368-rng" -- reg : Specifies base physical address and size of the registers -- clocks : phandle to clock-controller plus clock-specifier pair -- clock-names : "ipsec" as a clock name - -Example: - random: rng@10004180 { - compatible = "brcm,bcm6368-rng"; - reg = <0x10004180 0x14>; - - clocks = <&periph_clk 18>; - clock-names = "ipsec"; - }; From 8705f24f7b575f8fe1fe52764ea5774db51e3cf8 Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Tue, 7 Nov 2017 16:44:48 -0800 Subject: [PATCH 014/166] hwrng: bcm2835 - Enable BCM2835 RNG to work on BCM63xx platforms We have now incorporated all necessary functionality for the BCM63xx platforms to successfully migrate over bcm2835-rng, so add the final bits: Kconfig selection and proper platform_device device type matching to keep the same platform device name for registration to work. Signed-off-by: Florian Fainelli Signed-off-by: Herbert Xu --- drivers/char/hw_random/Kconfig | 7 ++++--- drivers/char/hw_random/bcm2835-rng.c | 11 ++++++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig index f6e3e5abc117..f13482eabbb9 100644 --- a/drivers/char/hw_random/Kconfig +++ b/drivers/char/hw_random/Kconfig @@ -87,12 +87,13 @@ config HW_RANDOM_BCM63XX If unusure, say Y. config HW_RANDOM_BCM2835 - tristate "Broadcom BCM2835 Random Number Generator support" - depends on ARCH_BCM2835 || ARCH_BCM_NSP || ARCH_BCM_5301X + tristate "Broadcom BCM2835/BCM63xx Random Number Generator support" + depends on ARCH_BCM2835 || ARCH_BCM_NSP || ARCH_BCM_5301X || \ + ARCH_BCM_63XX || BCM63XX || BMIPS_GENERIC default HW_RANDOM ---help--- This driver provides kernel-side support for the Random Number - Generator hardware found on the Broadcom BCM2835 SoCs. + Generator hardware found on the Broadcom BCM2835 and BCM63xx SoCs. To compile this driver as a module, choose M here: the module will be called bcm2835-rng diff --git a/drivers/char/hw_random/bcm2835-rng.c b/drivers/char/hw_random/bcm2835-rng.c index 6dd8f48701b5..25e56311a197 100644 --- a/drivers/char/hw_random/bcm2835-rng.c +++ b/drivers/char/hw_random/bcm2835-rng.c @@ -134,6 +134,7 @@ static const struct of_device_id bcm2835_rng_of_match[] = { { .compatible = "brcm,bcm2835-rng"}, { .compatible = "brcm,bcm-nsp-rng", .data = &nsp_rng_of_data }, { .compatible = "brcm,bcm5301x-rng", .data = &nsp_rng_of_data }, + { .compatible = "brcm,bcm6368-rng"}, {}, }; @@ -165,7 +166,7 @@ static int bcm2835_rng_probe(struct platform_device *pdev) /* Clock is optional on most platforms */ priv->clk = devm_clk_get(dev, NULL); - priv->rng.name = "bcm2835-rng"; + priv->rng.name = pdev->name; priv->rng.init = bcm2835_rng_init; priv->rng.read = bcm2835_rng_read; priv->rng.cleanup = bcm2835_rng_cleanup; @@ -191,12 +192,20 @@ static int bcm2835_rng_probe(struct platform_device *pdev) MODULE_DEVICE_TABLE(of, bcm2835_rng_of_match); +static struct platform_device_id bcm2835_rng_devtype[] = { + { .name = "bcm2835-rng" }, + { .name = "bcm63xx-rng" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(platform, bcm2835_rng_devtype); + static struct platform_driver bcm2835_rng_driver = { .driver = { .name = "bcm2835-rng", .of_match_table = bcm2835_rng_of_match, }, .probe = bcm2835_rng_probe, + .id_table = bcm2835_rng_devtype, }; module_platform_driver(bcm2835_rng_driver); From 0c2616a7d9d2d3b35507258b730e0fa76c5bf09d Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Tue, 7 Nov 2017 16:44:49 -0800 Subject: [PATCH 015/166] hwrng: bcm63xx - Remove since bcm2835-rng takes over bcm2835-rng is now capable of supporting the BCM63xx hardware, so remove the driver which duplicates the same functionality. Signed-off-by: Florian Fainelli Reviewed-by: Eric Anholt Signed-off-by: Herbert Xu --- drivers/char/hw_random/Kconfig | 13 --- drivers/char/hw_random/Makefile | 1 - drivers/char/hw_random/bcm63xx-rng.c | 154 --------------------------- 3 files changed, 168 deletions(-) delete mode 100644 drivers/char/hw_random/bcm63xx-rng.c diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig index f13482eabbb9..90e4bb24819e 100644 --- a/drivers/char/hw_random/Kconfig +++ b/drivers/char/hw_random/Kconfig @@ -73,19 +73,6 @@ config HW_RANDOM_ATMEL If unsure, say Y. -config HW_RANDOM_BCM63XX - tristate "Broadcom BCM63xx Random Number Generator support" - depends on BCM63XX || BMIPS_GENERIC - default HW_RANDOM - ---help--- - This driver provides kernel-side support for the Random Number - Generator hardware found on the Broadcom BCM63xx SoCs. - - To compile this driver as a module, choose M here: the - module will be called bcm63xx-rng - - If unusure, say Y. - config HW_RANDOM_BCM2835 tristate "Broadcom BCM2835/BCM63xx Random Number Generator support" depends on ARCH_BCM2835 || ARCH_BCM_NSP || ARCH_BCM_5301X || \ diff --git a/drivers/char/hw_random/Makefile b/drivers/char/hw_random/Makefile index f3728d008fff..e7146a84d44a 100644 --- a/drivers/char/hw_random/Makefile +++ b/drivers/char/hw_random/Makefile @@ -9,7 +9,6 @@ obj-$(CONFIG_HW_RANDOM_TIMERIOMEM) += timeriomem-rng.o obj-$(CONFIG_HW_RANDOM_INTEL) += intel-rng.o obj-$(CONFIG_HW_RANDOM_AMD) += amd-rng.o obj-$(CONFIG_HW_RANDOM_ATMEL) += atmel-rng.o -obj-$(CONFIG_HW_RANDOM_BCM63XX) += bcm63xx-rng.o obj-$(CONFIG_HW_RANDOM_GEODE) += geode-rng.o obj-$(CONFIG_HW_RANDOM_N2RNG) += n2-rng.o n2-rng-y := n2-drv.o n2-asm.o diff --git a/drivers/char/hw_random/bcm63xx-rng.c b/drivers/char/hw_random/bcm63xx-rng.c deleted file mode 100644 index 5132c9cde50d..000000000000 --- a/drivers/char/hw_random/bcm63xx-rng.c +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Broadcom BCM63xx Random Number Generator support - * - * Copyright (C) 2011, Florian Fainelli - * Copyright (C) 2009, Broadcom Corporation - * - */ -#include -#include -#include -#include -#include -#include -#include -#include - -#define RNG_CTRL 0x00 -#define RNG_EN (1 << 0) - -#define RNG_STAT 0x04 -#define RNG_AVAIL_MASK (0xff000000) - -#define RNG_DATA 0x08 -#define RNG_THRES 0x0c -#define RNG_MASK 0x10 - -struct bcm63xx_rng_priv { - struct hwrng rng; - struct clk *clk; - void __iomem *regs; -}; - -#define to_rng_priv(rng) container_of(rng, struct bcm63xx_rng_priv, rng) - -static int bcm63xx_rng_init(struct hwrng *rng) -{ - struct bcm63xx_rng_priv *priv = to_rng_priv(rng); - u32 val; - int error; - - error = clk_prepare_enable(priv->clk); - if (error) - return error; - - val = __raw_readl(priv->regs + RNG_CTRL); - val |= RNG_EN; - __raw_writel(val, priv->regs + RNG_CTRL); - - return 0; -} - -static void bcm63xx_rng_cleanup(struct hwrng *rng) -{ - struct bcm63xx_rng_priv *priv = to_rng_priv(rng); - u32 val; - - val = __raw_readl(priv->regs + RNG_CTRL); - val &= ~RNG_EN; - __raw_writel(val, priv->regs + RNG_CTRL); - - clk_disable_unprepare(priv->clk); -} - -static int bcm63xx_rng_data_present(struct hwrng *rng, int wait) -{ - struct bcm63xx_rng_priv *priv = to_rng_priv(rng); - - return __raw_readl(priv->regs + RNG_STAT) & RNG_AVAIL_MASK; -} - -static int bcm63xx_rng_data_read(struct hwrng *rng, u32 *data) -{ - struct bcm63xx_rng_priv *priv = to_rng_priv(rng); - - *data = __raw_readl(priv->regs + RNG_DATA); - - return 4; -} - -static int bcm63xx_rng_probe(struct platform_device *pdev) -{ - struct resource *r; - int ret; - struct bcm63xx_rng_priv *priv; - - r = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!r) { - dev_err(&pdev->dev, "no iomem resource\n"); - return -ENXIO; - } - - priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; - - priv->rng.name = pdev->name; - priv->rng.init = bcm63xx_rng_init; - priv->rng.cleanup = bcm63xx_rng_cleanup; - priv->rng.data_present = bcm63xx_rng_data_present; - priv->rng.data_read = bcm63xx_rng_data_read; - - priv->clk = devm_clk_get(&pdev->dev, "ipsec"); - if (IS_ERR(priv->clk)) { - ret = PTR_ERR(priv->clk); - dev_err(&pdev->dev, "no clock for device: %d\n", ret); - return ret; - } - - if (!devm_request_mem_region(&pdev->dev, r->start, - resource_size(r), pdev->name)) { - dev_err(&pdev->dev, "request mem failed"); - return -EBUSY; - } - - priv->regs = devm_ioremap_nocache(&pdev->dev, r->start, - resource_size(r)); - if (!priv->regs) { - dev_err(&pdev->dev, "ioremap failed"); - return -ENOMEM; - } - - ret = devm_hwrng_register(&pdev->dev, &priv->rng); - if (ret) { - dev_err(&pdev->dev, "failed to register rng device: %d\n", - ret); - return ret; - } - - dev_info(&pdev->dev, "registered RNG driver\n"); - - return 0; -} - -#ifdef CONFIG_OF -static const struct of_device_id bcm63xx_rng_of_match[] = { - { .compatible = "brcm,bcm6368-rng", }, - {}, -}; -MODULE_DEVICE_TABLE(of, bcm63xx_rng_of_match); -#endif - -static struct platform_driver bcm63xx_rng_driver = { - .probe = bcm63xx_rng_probe, - .driver = { - .name = "bcm63xx-rng", - .of_match_table = of_match_ptr(bcm63xx_rng_of_match), - }, -}; - -module_platform_driver(bcm63xx_rng_driver); - -MODULE_AUTHOR("Florian Fainelli "); -MODULE_DESCRIPTION("Broadcom BCM63xx RNG driver"); -MODULE_LICENSE("GPL"); From d7fc6cfdf1efaafbd708c08f0be44d71bb9f7fde Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Thu, 9 Nov 2017 08:54:13 +0000 Subject: [PATCH 016/166] crypto: chelsio - remove redundant assignments to reqctx and dst_size Pointer reqctx is assigned the same value twice, once on initialization and again a few statements later, remove the second redundant assignment. Variable dst_size is assigned but it is never read, so the variable is redundant and can be removed. Cleans up clang warnings: drivers/crypto/chelsio/chcr_algo.c:156:29: warning: Value stored to 'reqctx' during its initialization is never read drivers/crypto/chelsio/chcr_algo.c:2020:2: warning: Value stored to 'dst_size' is never read Signed-off-by: Colin Ian King Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 4eed7171e2ae..300e66fe8a3c 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -160,7 +160,6 @@ static inline void chcr_handle_ahash_resp(struct ahash_request *req, if (input == NULL) goto out; - reqctx = ahash_request_ctx(req); digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req)); if (reqctx->is_sg_map) chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); @@ -2014,11 +2013,8 @@ static int chcr_aead_common_init(struct aead_request *req, struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); int error = -EINVAL; - unsigned int dst_size; unsigned int authsize = crypto_aead_authsize(tfm); - dst_size = req->assoclen + req->cryptlen + (op_type ? - -authsize : authsize); /* validate key size */ if (aeadctx->enckey_len == 0) goto err; From 7814f552ff826fefa5e1b24083c7a06a9378e9ef Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 10 Nov 2017 00:25:43 +0300 Subject: [PATCH 017/166] crypto: chelsio - Fix an error code in chcr_hash_dma_map() The dma_map_sg() function returns zero on error and positive values on success. We want to return -ENOMEM on failure here and zero on success. Fixes: 2f47d5804311 ("crypto: chelsio - Move DMA un/mapping to chcr from lld cxgb4 driver") Signed-off-by: Dan Carpenter Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 300e66fe8a3c..af08dd264ca7 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -2410,7 +2410,7 @@ static inline int chcr_hash_dma_map(struct device *dev, error = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); if (!error) - return error; + return -ENOMEM; req_ctx->is_sg_map = 1; return 0; } From f7daa71560ef384c282400e325fc0a6ae61a0bb7 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 10 Nov 2017 00:26:04 +0300 Subject: [PATCH 018/166] crypto: s5p-sss - Remove a stray tab This code seems correct, but the goto was indented too far. Signed-off-by: Dan Carpenter Reviewed-by: Krzysztof Kozlowski Signed-off-by: Herbert Xu --- drivers/crypto/s5p-sss.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c index 142c6020cec7..62830a43d959 100644 --- a/drivers/crypto/s5p-sss.c +++ b/drivers/crypto/s5p-sss.c @@ -1461,7 +1461,7 @@ static void s5p_hash_tasklet_cb(unsigned long data) &dd->hash_flags)) { /* hash or semi-hash ready */ clear_bit(HASH_FLAGS_DMA_READY, &dd->hash_flags); - goto finish; + goto finish; } } From 4c0e22c90510308433272d7ba281b1eb4eda8209 Mon Sep 17 00:00:00 2001 From: Pierre Date: Sun, 12 Nov 2017 15:24:32 +0100 Subject: [PATCH 019/166] crypto: ecc - Fix NULL pointer deref. on no default_rng If crypto_get_default_rng returns an error, the function ecc_gen_privkey should return an error. Instead, it currently tries to use the default_rng nevertheless, thus creating a kernel panic with a NULL pointer dereference. Returning the error directly, as was supposedly intended when looking at the code, fixes this. Signed-off-by: Pierre Ducroquet Reviewed-by: PrasannaKumar Muralidharan Signed-off-by: Herbert Xu --- crypto/ecc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/ecc.c b/crypto/ecc.c index 633a9bcdc574..18f32f2a5e1c 100644 --- a/crypto/ecc.c +++ b/crypto/ecc.c @@ -964,7 +964,7 @@ int ecc_gen_privkey(unsigned int curve_id, unsigned int ndigits, u64 *privkey) * DRBG with a security strength of 256. */ if (crypto_get_default_rng()) - err = -EFAULT; + return -EFAULT; err = crypto_rng_get_bytes(crypto_default_rng, (u8 *)priv, nbytes); crypto_put_default_rng(); From 1af39daaad4a868a3abd220e0ec9643473398c96 Mon Sep 17 00:00:00 2001 From: Martin Kepplinger Date: Tue, 14 Nov 2017 10:25:15 +0100 Subject: [PATCH 020/166] crypto: replace FSF address with web source in license notices A few years ago the FSF moved and "59 Temple Place" is wrong. Having this still in our source files feels old and unmaintained. Let's take the license statement serious and not confuse users. As https://www.gnu.org/licenses/gpl-howto.html suggests, we replace the postal address with "". Signed-off-by: Martin Kepplinger Signed-off-by: Herbert Xu --- crypto/ablk_helper.c | 4 +--- crypto/camellia_generic.c | 3 +-- crypto/cast5_generic.c | 3 +-- crypto/cast6_generic.c | 3 +-- crypto/simd.c | 4 +--- crypto/twofish_common.c | 5 ++--- crypto/twofish_generic.c | 5 ++--- crypto/xcbc.c | 3 +-- 8 files changed, 10 insertions(+), 20 deletions(-) diff --git a/crypto/ablk_helper.c b/crypto/ablk_helper.c index 1441f07d0a19..6e5d2f149b89 100644 --- a/crypto/ablk_helper.c +++ b/crypto/ablk_helper.c @@ -18,9 +18,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA + * along with this program. If not, see . * */ diff --git a/crypto/camellia_generic.c b/crypto/camellia_generic.c index a02286bf319e..32ddd4836ff5 100644 --- a/crypto/camellia_generic.c +++ b/crypto/camellia_generic.c @@ -13,8 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * along with this program. If not, see . */ /* diff --git a/crypto/cast5_generic.c b/crypto/cast5_generic.c index df5c72629383..66169c178314 100644 --- a/crypto/cast5_generic.c +++ b/crypto/cast5_generic.c @@ -16,8 +16,7 @@ * any later version. * * You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA +* along with this program. If not, see . */ diff --git a/crypto/cast6_generic.c b/crypto/cast6_generic.c index 058c8d755d03..c8e5ec69790e 100644 --- a/crypto/cast6_generic.c +++ b/crypto/cast6_generic.c @@ -13,8 +13,7 @@ * any later version. * * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA + * along with this program. If not, see . */ diff --git a/crypto/simd.c b/crypto/simd.c index 88203370a62f..208226d7f908 100644 --- a/crypto/simd.c +++ b/crypto/simd.c @@ -19,9 +19,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA + * along with this program. If not, see . * */ diff --git a/crypto/twofish_common.c b/crypto/twofish_common.c index 5f62c4f9f6e0..f3a0dd25f871 100644 --- a/crypto/twofish_common.c +++ b/crypto/twofish_common.c @@ -24,9 +24,8 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA + * along with this program. If not, see . + * * * This code is a "clean room" implementation, written from the paper * _Twofish: A 128-Bit Block Cipher_ by Bruce Schneier, John Kelsey, diff --git a/crypto/twofish_generic.c b/crypto/twofish_generic.c index ebf7a3efb572..07e62433fbfb 100644 --- a/crypto/twofish_generic.c +++ b/crypto/twofish_generic.c @@ -23,9 +23,8 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA + * along with this program. If not, see . + * * * This code is a "clean room" implementation, written from the paper * _Twofish: A 128-Bit Block Cipher_ by Bruce Schneier, John Kelsey, diff --git a/crypto/xcbc.c b/crypto/xcbc.c index df90b332554c..25c75af50d3f 100644 --- a/crypto/xcbc.c +++ b/crypto/xcbc.c @@ -12,8 +12,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * along with this program. If not, see . * * Author: * Kazunori Miyazawa From 6905c461e539caab1192c8b149c79fd2793abcde Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Tue, 14 Nov 2017 14:32:17 +0000 Subject: [PATCH 021/166] crypto: nx - fix spelling mistake: "availavle" -> "available" Trivial fix to spelling mistake in pr_err error message text. Also fix spelling mistake in proceeding comment. Signed-off-by: Colin Ian King Acked-by: Haren Myneni Signed-off-by: Herbert Xu --- drivers/crypto/nx/nx-842-powernv.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c index f2246a5abcf6..1e87637c412d 100644 --- a/drivers/crypto/nx/nx-842-powernv.c +++ b/drivers/crypto/nx/nx-842-powernv.c @@ -743,8 +743,8 @@ static int nx842_open_percpu_txwins(void) } if (!per_cpu(cpu_txwin, i)) { - /* shoudn't happen, Each chip will have NX engine */ - pr_err("NX engine is not availavle for CPU %d\n", i); + /* shouldn't happen, Each chip will have NX engine */ + pr_err("NX engine is not available for CPU %d\n", i); return -EINVAL; } } From 5601e014fe7229dae405c9ad72081d65ac102962 Mon Sep 17 00:00:00 2001 From: Tudor-Dan Ambarus Date: Tue, 14 Nov 2017 16:59:15 +0200 Subject: [PATCH 022/166] crypto: tcrypt - set assoc in sg_init_aead() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Results better code readability. Signed-off-by: Tudor Ambarus Reviewed-by: Horia Geantă Signed-off-by: Herbert Xu --- crypto/tcrypt.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 3ced1ba1fd11..28b4882f1944 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -185,7 +185,8 @@ static void testmgr_free_buf(char *buf[XBUFSIZE]) } static void sg_init_aead(struct scatterlist *sg, char *xbuf[XBUFSIZE], - unsigned int buflen) + unsigned int buflen, const void *assoc, + unsigned int aad_size) { int np = (buflen + PAGE_SIZE - 1)/PAGE_SIZE; int k, rem; @@ -198,6 +199,9 @@ static void sg_init_aead(struct scatterlist *sg, char *xbuf[XBUFSIZE], } sg_init_table(sg, np + 1); + + sg_set_buf(&sg[0], assoc, aad_size); + if (rem) np--; for (k = 0; k < np; k++) @@ -318,14 +322,12 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, goto out; } - sg_init_aead(sg, xbuf, - *b_size + (enc ? 0 : authsize)); + sg_init_aead(sg, xbuf, *b_size + (enc ? 0 : authsize), + assoc, aad_size); sg_init_aead(sgout, xoutbuf, - *b_size + (enc ? authsize : 0)); - - sg_set_buf(&sg[0], assoc, aad_size); - sg_set_buf(&sgout[0], assoc, aad_size); + *b_size + (enc ? authsize : 0), assoc, + aad_size); aead_request_set_crypt(req, sg, sgout, *b_size + (enc ? 0 : authsize), From c9683276dd89906ca9b65696d09104d542171421 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Wed, 15 Nov 2017 11:44:28 +0100 Subject: [PATCH 023/166] crypto: keywrap - Add missing ULL suffixes for 64-bit constants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On 32-bit (e.g. with m68k-linux-gnu-gcc-4.1): crypto/keywrap.c: In function ‘crypto_kw_decrypt’: crypto/keywrap.c:191: warning: integer constant is too large for ‘long’ type crypto/keywrap.c: In function ‘crypto_kw_encrypt’: crypto/keywrap.c:224: warning: integer constant is too large for ‘long’ type Fixes: 9e49451d7a15365d ("crypto: keywrap - simplify code") Signed-off-by: Geert Uytterhoeven Reviewed-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/keywrap.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/keywrap.c b/crypto/keywrap.c index 744e35134c45..ec5c6a087c90 100644 --- a/crypto/keywrap.c +++ b/crypto/keywrap.c @@ -188,7 +188,7 @@ static int crypto_kw_decrypt(struct blkcipher_desc *desc, } /* Perform authentication check */ - if (block.A != cpu_to_be64(0xa6a6a6a6a6a6a6a6)) + if (block.A != cpu_to_be64(0xa6a6a6a6a6a6a6a6ULL)) ret = -EBADMSG; memzero_explicit(&block, sizeof(struct crypto_kw_block)); @@ -221,7 +221,7 @@ static int crypto_kw_encrypt(struct blkcipher_desc *desc, * Place the predefined IV into block A -- for encrypt, the caller * does not need to provide an IV, but he needs to fetch the final IV. */ - block.A = cpu_to_be64(0xa6a6a6a6a6a6a6a6); + block.A = cpu_to_be64(0xa6a6a6a6a6a6a6a6ULL); /* * src scatterlist is read-only. dst scatterlist is r/w. During the From 87aae50af730a28dc1d8846d86dca5e9aa724a9f Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Wed, 15 Nov 2017 12:37:19 +0000 Subject: [PATCH 024/166] crypto: cavium - fix memory leak on info The object info is being leaked on an error return path, fix this by setting ret to -ENOMEM and exiting via the request_cleanup path that will free info. Detected by CoverityScan, CID#1408439 ("Resource Leak") Fixes: c694b233295b ("crypto: cavium - Add the Virtual Function driver for CPT") Signed-off-by: Colin Ian King Signed-off-by: Herbert Xu --- drivers/crypto/cavium/cpt/cptvf_reqmanager.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/cavium/cpt/cptvf_reqmanager.c b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c index 169e66231bcf..b0ba4331944b 100644 --- a/drivers/crypto/cavium/cpt/cptvf_reqmanager.c +++ b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c @@ -459,7 +459,8 @@ int process_request(struct cpt_vf *cptvf, struct cpt_request_info *req) info->completion_addr = kzalloc(sizeof(union cpt_res_s), GFP_KERNEL); if (unlikely(!info->completion_addr)) { dev_err(&pdev->dev, "Unable to allocate memory for completion_addr\n"); - return -ENOMEM; + ret = -ENOMEM; + goto request_cleanup; } result = (union cpt_res_s *)info->completion_addr; From 23eb61d304f9a6a6ec507ee27dd2b6c631c26c14 Mon Sep 17 00:00:00 2001 From: Gilad Ben-Yossef Date: Thu, 16 Nov 2017 06:15:28 +0000 Subject: [PATCH 025/166] dt-bindings: add device tree binding for Arm TrustZone CryptoCell crypto engine The Arm TrustZone CryptoCell is a hardware security engine. This patch adds DT bindings for its Rich Execution Environment crypto engine. A driver supporting this device is already present in the staging tree. Signed-off-by: Gilad Ben-Yossef Acked-by: Rob Herring Signed-off-by: Herbert Xu --- .../bindings/crypto/arm-cryptocell.txt | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 Documentation/devicetree/bindings/crypto/arm-cryptocell.txt diff --git a/Documentation/devicetree/bindings/crypto/arm-cryptocell.txt b/Documentation/devicetree/bindings/crypto/arm-cryptocell.txt new file mode 100644 index 000000000000..cec8d5d74e26 --- /dev/null +++ b/Documentation/devicetree/bindings/crypto/arm-cryptocell.txt @@ -0,0 +1,22 @@ +Arm TrustZone CryptoCell cryptographic engine + +Required properties: +- compatible: Should be "arm,cryptocell-712-ree". +- reg: Base physical address of the engine and length of memory mapped region. +- interrupts: Interrupt number for the device. + +Optional properties: +- interrupt-parent: The phandle for the interrupt controller that services + interrupts for this device. +- clocks: Reference to the crypto engine clock. +- dma-coherent: Present if dma operations are coherent. + +Examples: + + arm_cc712: crypto@80000000 { + compatible = "arm,cryptocell-712-ree"; + interrupt-parent = <&intc>; + interrupts = < 0 30 4 >; + reg = < 0x80000000 0x10000 >; + + }; From a6ec572bfa7d529be78afbb703f8d9381954cdff Mon Sep 17 00:00:00 2001 From: Atul Gupta Date: Thu, 16 Nov 2017 16:56:39 +0530 Subject: [PATCH 026/166] cxgb4: Add support for Inline IPSec Tx Added Tx routine for ULD - define interface for ULD Tx. Export routines used for Tx data - Routines common for data transmit are used by cxgb4 and chcr drivers. - EXPORT routines enable transmit from chcr driver. Signed-off-by: Atul Gupta Signed-off-by: Ganesh Goudar Signed-off-by: Herbert Xu --- drivers/net/ethernet/chelsio/cxgb4/cxgb4.h | 23 ++++ .../ethernet/chelsio/cxgb4/cxgb4_debugfs.c | 2 + .../net/ethernet/chelsio/cxgb4/cxgb4_main.c | 2 +- .../net/ethernet/chelsio/cxgb4/cxgb4_uld.c | 1 + .../net/ethernet/chelsio/cxgb4/cxgb4_uld.h | 3 + drivers/net/ethernet/chelsio/cxgb4/sge.c | 102 +++++++++--------- drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h | 7 ++ 7 files changed, 88 insertions(+), 52 deletions(-) diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h index 6f9fa6e3c42a..c48a7521d63e 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h @@ -58,6 +58,13 @@ extern struct list_head adapter_list; extern struct mutex uld_mutex; +/* Suspend an Ethernet Tx queue with fewer available descriptors than this. + * This is the same as calc_tx_descs() for a TSO packet with + * nr_frags == MAX_SKB_FRAGS. + */ +#define ETHTXQ_STOP_THRES \ + (1 + DIV_ROUND_UP((3 * MAX_SKB_FRAGS) / 2 + (MAX_SKB_FRAGS & 1), 8)) + enum { MAX_NPORTS = 4, /* max # of ports */ SERNUM_LEN = 24, /* Serial # length */ @@ -564,6 +571,7 @@ enum { /* adapter flags */ enum { ULP_CRYPTO_LOOKASIDE = 1 << 0, + ULP_CRYPTO_IPSEC_INLINE = 1 << 1, }; struct rx_sw_desc; @@ -968,6 +976,11 @@ enum { SCHED_CLASS_RATEMODE_ABS = 1, /* Kb/s */ }; +struct tx_sw_desc { /* SW state per Tx descriptor */ + struct sk_buff *skb; + struct ulptx_sgl *sgl; +}; + /* Support for "sched_queue" command to allow one or more NIC TX Queues * to be bound to a TX Scheduling Class. */ @@ -1700,4 +1713,14 @@ void free_rspq_fl(struct adapter *adap, struct sge_rspq *rq, struct sge_fl *fl); void free_tx_desc(struct adapter *adap, struct sge_txq *q, unsigned int n, bool unmap); void free_txq(struct adapter *adap, struct sge_txq *q); +void cxgb4_reclaim_completed_tx(struct adapter *adap, + struct sge_txq *q, bool unmap); +int cxgb4_map_skb(struct device *dev, const struct sk_buff *skb, + dma_addr_t *addr); +void cxgb4_inline_tx_skb(const struct sk_buff *skb, const struct sge_txq *q, + void *pos); +void cxgb4_write_sgl(const struct sk_buff *skb, struct sge_txq *q, + struct ulptx_sgl *sgl, u64 *end, unsigned int start, + const dma_addr_t *addr); +void cxgb4_ring_tx_db(struct adapter *adap, struct sge_txq *q, int n); #endif /* __CXGB4_H__ */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c index 917663b35603..cf471831ee71 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c @@ -3096,6 +3096,8 @@ static int chcr_show(struct seq_file *seq, void *v) atomic_read(&adap->chcr_stats.error)); seq_printf(seq, "Fallback: %10u \n", atomic_read(&adap->chcr_stats.fallback)); + seq_printf(seq, "IPSec PDU: %10u\n", + atomic_read(&adap->chcr_stats.ipsec_cnt)); return 0; } diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c index 6f900ffe25cc..05a4abfd5ec1 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c @@ -4096,7 +4096,7 @@ static int adap_init0(struct adapter *adap) } else { adap->vres.ncrypto_fc = val[0]; } - adap->params.crypto |= ULP_CRYPTO_LOOKASIDE; + adap->params.crypto = ntohs(caps_cmd.cryptocaps); adap->num_uld += 1; } #undef FW_PARAM_PFVF diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c index 71a315bc1409..6b5fea4532f3 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c @@ -637,6 +637,7 @@ static void uld_init(struct adapter *adap, struct cxgb4_lld_info *lld) lld->nchan = adap->params.nports; lld->nports = adap->params.nports; lld->wr_cred = adap->params.ofldq_wr_cred; + lld->crypto = adap->params.crypto; lld->iscsi_iolen = MAXRXDATA_G(t4_read_reg(adap, TP_PARA_REG2_A)); lld->iscsi_tagmask = t4_read_reg(adap, ULP_RX_ISCSI_TAGMASK_A); lld->iscsi_pgsz_order = t4_read_reg(adap, ULP_RX_ISCSI_PSZ_A); diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h index 08e709ab6dd4..1d37672902da 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h @@ -297,6 +297,7 @@ struct chcr_stats_debug { atomic_t complete; atomic_t error; atomic_t fallback; + atomic_t ipsec_cnt; }; #define OCQ_WIN_OFFSET(pdev, vres) \ @@ -322,6 +323,7 @@ struct cxgb4_lld_info { unsigned char wr_cred; /* WR 16-byte credits */ unsigned char adapter_type; /* type of adapter */ unsigned char fw_api_ver; /* FW API version */ + unsigned char crypto; /* crypto support */ unsigned int fw_vers; /* FW version */ unsigned int iscsi_iolen; /* iSCSI max I/O length */ unsigned int cclk_ps; /* Core clock period in psec */ @@ -370,6 +372,7 @@ struct cxgb4_uld_info { struct t4_lro_mgr *lro_mgr, struct napi_struct *napi); void (*lro_flush)(struct t4_lro_mgr *); + int (*tx_handler)(struct sk_buff *skb, struct net_device *dev); }; int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p); diff --git a/drivers/net/ethernet/chelsio/cxgb4/sge.c b/drivers/net/ethernet/chelsio/cxgb4/sge.c index 922f2f937789..6c7b0ac0b48b 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/sge.c +++ b/drivers/net/ethernet/chelsio/cxgb4/sge.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -53,6 +54,7 @@ #include "t4_msg.h" #include "t4fw_api.h" #include "cxgb4_ptp.h" +#include "cxgb4_uld.h" /* * Rx buffer size. We use largish buffers if possible but settle for single @@ -109,14 +111,6 @@ */ #define NOMEM_TMR_IDX (SGE_NTIMERS - 1) -/* - * Suspend an Ethernet Tx queue with fewer available descriptors than this. - * This is the same as calc_tx_descs() for a TSO packet with - * nr_frags == MAX_SKB_FRAGS. - */ -#define ETHTXQ_STOP_THRES \ - (1 + DIV_ROUND_UP((3 * MAX_SKB_FRAGS) / 2 + (MAX_SKB_FRAGS & 1), 8)) - /* * Suspension threshold for non-Ethernet Tx queues. We require enough room * for a full sized WR. @@ -134,11 +128,6 @@ */ #define MAX_CTRL_WR_LEN SGE_MAX_WR_LEN -struct tx_sw_desc { /* SW state per Tx descriptor */ - struct sk_buff *skb; - struct ulptx_sgl *sgl; -}; - struct rx_sw_desc { /* SW state per Rx descriptor */ struct page *page; dma_addr_t dma_addr; @@ -248,8 +237,8 @@ static inline bool fl_starving(const struct adapter *adapter, return fl->avail - fl->pend_cred <= s->fl_starve_thres; } -static int map_skb(struct device *dev, const struct sk_buff *skb, - dma_addr_t *addr) +int cxgb4_map_skb(struct device *dev, const struct sk_buff *skb, + dma_addr_t *addr) { const skb_frag_t *fp, *end; const struct skb_shared_info *si; @@ -277,6 +266,7 @@ unwind: out_err: return -ENOMEM; } +EXPORT_SYMBOL(cxgb4_map_skb); #ifdef CONFIG_NEED_DMA_MAP_STATE static void unmap_skb(struct device *dev, const struct sk_buff *skb, @@ -411,7 +401,7 @@ static inline int reclaimable(const struct sge_txq *q) } /** - * reclaim_completed_tx - reclaims completed Tx descriptors + * cxgb4_reclaim_completed_tx - reclaims completed Tx descriptors * @adap: the adapter * @q: the Tx queue to reclaim completed descriptors from * @unmap: whether the buffers should be unmapped for DMA @@ -420,7 +410,7 @@ static inline int reclaimable(const struct sge_txq *q) * and frees the associated buffers if possible. Called with the Tx * queue locked. */ -static inline void reclaim_completed_tx(struct adapter *adap, struct sge_txq *q, +inline void cxgb4_reclaim_completed_tx(struct adapter *adap, struct sge_txq *q, bool unmap) { int avail = reclaimable(q); @@ -437,6 +427,7 @@ static inline void reclaim_completed_tx(struct adapter *adap, struct sge_txq *q, q->in_use -= avail; } } +EXPORT_SYMBOL(cxgb4_reclaim_completed_tx); static inline int get_buf_size(struct adapter *adapter, const struct rx_sw_desc *d) @@ -833,7 +824,7 @@ static inline unsigned int calc_tx_descs(const struct sk_buff *skb) } /** - * write_sgl - populate a scatter/gather list for a packet + * cxgb4_write_sgl - populate a scatter/gather list for a packet * @skb: the packet * @q: the Tx queue we are writing into * @sgl: starting location for writing the SGL @@ -849,9 +840,9 @@ static inline unsigned int calc_tx_descs(const struct sk_buff *skb) * right after the end of the SGL but does not account for any potential * wrap around, i.e., @end > @sgl. */ -static void write_sgl(const struct sk_buff *skb, struct sge_txq *q, - struct ulptx_sgl *sgl, u64 *end, unsigned int start, - const dma_addr_t *addr) +void cxgb4_write_sgl(const struct sk_buff *skb, struct sge_txq *q, + struct ulptx_sgl *sgl, u64 *end, unsigned int start, + const dma_addr_t *addr) { unsigned int i, len; struct ulptx_sge_pair *to; @@ -903,6 +894,7 @@ static void write_sgl(const struct sk_buff *skb, struct sge_txq *q, if ((uintptr_t)end & 8) /* 0-pad to multiple of 16 */ *end = 0; } +EXPORT_SYMBOL(cxgb4_write_sgl); /* This function copies 64 byte coalesced work request to * memory mapped BAR2 space. For coalesced WR SGE fetches @@ -921,14 +913,14 @@ static void cxgb_pio_copy(u64 __iomem *dst, u64 *src) } /** - * ring_tx_db - check and potentially ring a Tx queue's doorbell + * cxgb4_ring_tx_db - check and potentially ring a Tx queue's doorbell * @adap: the adapter * @q: the Tx queue * @n: number of new descriptors to give to HW * * Ring the doorbel for a Tx queue. */ -static inline void ring_tx_db(struct adapter *adap, struct sge_txq *q, int n) +inline void cxgb4_ring_tx_db(struct adapter *adap, struct sge_txq *q, int n) { /* Make sure that all writes to the TX Descriptors are committed * before we tell the hardware about them. @@ -995,9 +987,10 @@ static inline void ring_tx_db(struct adapter *adap, struct sge_txq *q, int n) wmb(); } } +EXPORT_SYMBOL(cxgb4_ring_tx_db); /** - * inline_tx_skb - inline a packet's data into Tx descriptors + * cxgb4_inline_tx_skb - inline a packet's data into Tx descriptors * @skb: the packet * @q: the Tx queue where the packet will be inlined * @pos: starting position in the Tx queue where to inline the packet @@ -1007,8 +1000,8 @@ static inline void ring_tx_db(struct adapter *adap, struct sge_txq *q, int n) * Most of the complexity of this operation is dealing with wrap arounds * in the middle of the packet we want to inline. */ -static void inline_tx_skb(const struct sk_buff *skb, const struct sge_txq *q, - void *pos) +void cxgb4_inline_tx_skb(const struct sk_buff *skb, + const struct sge_txq *q, void *pos) { u64 *p; int left = (void *)q->stat - pos; @@ -1030,6 +1023,7 @@ static void inline_tx_skb(const struct sk_buff *skb, const struct sge_txq *q, if ((uintptr_t)p & 8) *p = 0; } +EXPORT_SYMBOL(cxgb4_inline_tx_skb); static void *inline_tx_skb_header(const struct sk_buff *skb, const struct sge_txq *q, void *pos, @@ -1199,6 +1193,12 @@ out_free: dev_kfree_skb_any(skb); pi = netdev_priv(dev); adap = pi->adapter; + ssi = skb_shinfo(skb); +#ifdef CONFIG_CHELSIO_IPSEC_INLINE + if (xfrm_offload(skb) && !ssi->gso_size) + return adap->uld[CXGB4_ULD_CRYPTO].tx_handler(skb, dev); +#endif /* CHELSIO_IPSEC_INLINE */ + qidx = skb_get_queue_mapping(skb); if (ptp_enabled) { spin_lock(&adap->ptp_lock); @@ -1215,7 +1215,7 @@ out_free: dev_kfree_skb_any(skb); } skb_tx_timestamp(skb); - reclaim_completed_tx(adap, &q->q, true); + cxgb4_reclaim_completed_tx(adap, &q->q, true); cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F; #ifdef CONFIG_CHELSIO_T4_FCOE @@ -1245,7 +1245,7 @@ out_free: dev_kfree_skb_any(skb); immediate = true; if (!immediate && - unlikely(map_skb(adap->pdev_dev, skb, addr) < 0)) { + unlikely(cxgb4_map_skb(adap->pdev_dev, skb, addr) < 0)) { q->mapping_err++; if (ptp_enabled) spin_unlock(&adap->ptp_lock); @@ -1264,7 +1264,6 @@ out_free: dev_kfree_skb_any(skb); end = (u64 *)wr + flits; len = immediate ? skb->len : 0; - ssi = skb_shinfo(skb); if (ssi->gso_size) { struct cpl_tx_pkt_lso *lso = (void *)wr; bool v6 = (ssi->gso_type & SKB_GSO_TCPV6) != 0; @@ -1341,13 +1340,13 @@ out_free: dev_kfree_skb_any(skb); cpl->ctrl1 = cpu_to_be64(cntrl); if (immediate) { - inline_tx_skb(skb, &q->q, cpl + 1); + cxgb4_inline_tx_skb(skb, &q->q, cpl + 1); dev_consume_skb_any(skb); } else { int last_desc; - write_sgl(skb, &q->q, (struct ulptx_sgl *)(cpl + 1), end, 0, - addr); + cxgb4_write_sgl(skb, &q->q, (struct ulptx_sgl *)(cpl + 1), + end, 0, addr); skb_orphan(skb); last_desc = q->q.pidx + ndesc - 1; @@ -1359,7 +1358,7 @@ out_free: dev_kfree_skb_any(skb); txq_advance(&q->q, ndesc); - ring_tx_db(adap, &q->q, ndesc); + cxgb4_ring_tx_db(adap, &q->q, ndesc); if (ptp_enabled) spin_unlock(&adap->ptp_lock); return NETDEV_TX_OK; @@ -1369,9 +1368,9 @@ out_free: dev_kfree_skb_any(skb); * reclaim_completed_tx_imm - reclaim completed control-queue Tx descs * @q: the SGE control Tx queue * - * This is a variant of reclaim_completed_tx() that is used for Tx queues - * that send only immediate data (presently just the control queues) and - * thus do not have any sk_buffs to release. + * This is a variant of cxgb4_reclaim_completed_tx() that is used + * for Tx queues that send only immediate data (presently just + * the control queues) and thus do not have any sk_buffs to release. */ static inline void reclaim_completed_tx_imm(struct sge_txq *q) { @@ -1446,13 +1445,13 @@ static int ctrl_xmit(struct sge_ctrl_txq *q, struct sk_buff *skb) } wr = (struct fw_wr_hdr *)&q->q.desc[q->q.pidx]; - inline_tx_skb(skb, &q->q, wr); + cxgb4_inline_tx_skb(skb, &q->q, wr); txq_advance(&q->q, ndesc); if (unlikely(txq_avail(&q->q) < TXQ_STOP_THRES)) ctrlq_check_stop(q, wr); - ring_tx_db(q->adap, &q->q, ndesc); + cxgb4_ring_tx_db(q->adap, &q->q, ndesc); spin_unlock(&q->sendq.lock); kfree_skb(skb); @@ -1487,7 +1486,7 @@ static void restart_ctrlq(unsigned long data) txq_advance(&q->q, ndesc); spin_unlock(&q->sendq.lock); - inline_tx_skb(skb, &q->q, wr); + cxgb4_inline_tx_skb(skb, &q->q, wr); kfree_skb(skb); if (unlikely(txq_avail(&q->q) < TXQ_STOP_THRES)) { @@ -1500,14 +1499,15 @@ static void restart_ctrlq(unsigned long data) } } if (written > 16) { - ring_tx_db(q->adap, &q->q, written); + cxgb4_ring_tx_db(q->adap, &q->q, written); written = 0; } spin_lock(&q->sendq.lock); } q->full = 0; -ringdb: if (written) - ring_tx_db(q->adap, &q->q, written); +ringdb: + if (written) + cxgb4_ring_tx_db(q->adap, &q->q, written); spin_unlock(&q->sendq.lock); } @@ -1650,7 +1650,7 @@ static void service_ofldq(struct sge_uld_txq *q) */ spin_unlock(&q->sendq.lock); - reclaim_completed_tx(q->adap, &q->q, false); + cxgb4_reclaim_completed_tx(q->adap, &q->q, false); flits = skb->priority; /* previously saved */ ndesc = flits_to_desc(flits); @@ -1661,9 +1661,9 @@ static void service_ofldq(struct sge_uld_txq *q) pos = (u64 *)&q->q.desc[q->q.pidx]; if (is_ofld_imm(skb)) - inline_tx_skb(skb, &q->q, pos); - else if (map_skb(q->adap->pdev_dev, skb, - (dma_addr_t *)skb->head)) { + cxgb4_inline_tx_skb(skb, &q->q, pos); + else if (cxgb4_map_skb(q->adap->pdev_dev, skb, + (dma_addr_t *)skb->head)) { txq_stop_maperr(q); spin_lock(&q->sendq.lock); break; @@ -1694,9 +1694,9 @@ static void service_ofldq(struct sge_uld_txq *q) pos = (void *)txq->desc; } - write_sgl(skb, &q->q, (void *)pos, - end, hdr_len, - (dma_addr_t *)skb->head); + cxgb4_write_sgl(skb, &q->q, (void *)pos, + end, hdr_len, + (dma_addr_t *)skb->head); #ifdef CONFIG_NEED_DMA_MAP_STATE skb->dev = q->adap->port[0]; skb->destructor = deferred_unmap_destructor; @@ -1710,7 +1710,7 @@ static void service_ofldq(struct sge_uld_txq *q) txq_advance(&q->q, ndesc); written += ndesc; if (unlikely(written > 32)) { - ring_tx_db(q->adap, &q->q, written); + cxgb4_ring_tx_db(q->adap, &q->q, written); written = 0; } @@ -1725,7 +1725,7 @@ static void service_ofldq(struct sge_uld_txq *q) kfree_skb(skb); } if (likely(written)) - ring_tx_db(q->adap, &q->q, written); + cxgb4_ring_tx_db(q->adap, &q->q, written); /*Indicate that no thread is processing the Pending Send Queue * currently. diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h b/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h index 57eb4ad3485d..be3658301832 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h +++ b/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h @@ -513,6 +513,13 @@ struct fw_ulptx_wr { u64 cookie; }; +#define FW_ULPTX_WR_DATA_S 28 +#define FW_ULPTX_WR_DATA_M 0x1 +#define FW_ULPTX_WR_DATA_V(x) ((x) << FW_ULPTX_WR_DATA_S) +#define FW_ULPTX_WR_DATA_G(x) \ + (((x) >> FW_ULPTX_WR_DATA_S) & FW_ULPTX_WR_DATA_M) +#define FW_ULPTX_WR_DATA_F FW_ULPTX_WR_DATA_V(1U) + struct fw_tp_wr { __be32 op_to_immdlen; __be32 flowid_len16; From 6dad4e8ab3ec65c3b948ad79e83751cf0f04cbdf Mon Sep 17 00:00:00 2001 From: Atul Gupta Date: Thu, 16 Nov 2017 16:57:08 +0530 Subject: [PATCH 027/166] chcr: Add support for Inline IPSec register xfrmdev_ops callbacks, Send IPsec tunneled data to HW for inline processing. The driver use hardware crypto accelerator to encrypt and generate ICV for the transmitted packet in Inline mode. Signed-off-by: Atul Gupta Signed-off-by: Harsh Jain Signed-off-by: Ganesh Goudar Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/Kconfig | 10 + drivers/crypto/chelsio/Makefile | 1 + drivers/crypto/chelsio/chcr_algo.c | 221 ++++----- drivers/crypto/chelsio/chcr_algo.h | 15 - drivers/crypto/chelsio/chcr_core.c | 14 + drivers/crypto/chelsio/chcr_core.h | 38 ++ drivers/crypto/chelsio/chcr_crypto.h | 69 +-- drivers/crypto/chelsio/chcr_ipsec.c | 660 +++++++++++++++++++++++++++ 8 files changed, 860 insertions(+), 168 deletions(-) create mode 100644 drivers/crypto/chelsio/chcr_ipsec.c diff --git a/drivers/crypto/chelsio/Kconfig b/drivers/crypto/chelsio/Kconfig index 3e104f5aa0c2..51932c7bde5d 100644 --- a/drivers/crypto/chelsio/Kconfig +++ b/drivers/crypto/chelsio/Kconfig @@ -18,3 +18,13 @@ config CRYPTO_DEV_CHELSIO To compile this driver as a module, choose M here: the module will be called chcr. + +config CHELSIO_IPSEC_INLINE + bool "Chelsio IPSec XFRM Tx crypto offload" + depends on CHELSIO_T4 + depends on CRYPTO_DEV_CHELSIO + depends on XFRM_OFFLOAD + depends on INET_ESP_OFFLOAD || INET6_ESP_OFFLOAD + default n + ---help--- + Enable support for IPSec Tx Inline. diff --git a/drivers/crypto/chelsio/Makefile b/drivers/crypto/chelsio/Makefile index bebdf06687ad..eaecaf1ebcf3 100644 --- a/drivers/crypto/chelsio/Makefile +++ b/drivers/crypto/chelsio/Makefile @@ -2,3 +2,4 @@ ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4 obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chcr.o chcr-objs := chcr_core.o chcr_algo.o +chcr-$(CONFIG_CHELSIO_IPSEC_INLINE) += chcr_ipsec.o diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index af08dd264ca7..e61ec8a46340 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -73,6 +73,25 @@ #define IV AES_BLOCK_SIZE +unsigned int sgl_ent_len[] = {0, 0, 16, 24, 40, 48, 64, 72, 88, + 96, 112, 120, 136, 144, 160, 168, 184, + 192, 208, 216, 232, 240, 256, 264, 280, + 288, 304, 312, 328, 336, 352, 360, 376}; + +unsigned int dsgl_ent_len[] = {0, 32, 32, 48, 48, 64, 64, 80, 80, + 112, 112, 128, 128, 144, 144, 160, 160, + 192, 192, 208, 208, 224, 224, 240, 240, + 272, 272, 288, 288, 304, 304, 320, 320}; + +static u32 round_constant[11] = { + 0x01000000, 0x02000000, 0x04000000, 0x08000000, + 0x10000000, 0x20000000, 0x40000000, 0x80000000, + 0x1B000000, 0x36000000, 0x6C000000 +}; + +static int chcr_handle_cipher_resp(struct ablkcipher_request *req, + unsigned char *input, int err); + static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx) { return ctx->crypto_ctx->aeadctx; @@ -108,18 +127,6 @@ static inline int is_ofld_imm(const struct sk_buff *skb) return (skb->len <= SGE_MAX_WR_LEN); } -/* - * sgl_len - calculates the size of an SGL of the given capacity - * @n: the number of SGL entries - * Calculates the number of flits needed for a scatter/gather list that - * can hold the given number of entries. - */ -static inline unsigned int sgl_len(unsigned int n) -{ - n--; - return (3 * n) / 2 + (n & 1) + 2; -} - static int sg_nents_xlen(struct scatterlist *sg, unsigned int reqlen, unsigned int entlen, unsigned int skip) @@ -182,30 +189,17 @@ static inline void chcr_handle_ahash_resp(struct ahash_request *req, } out: req->base.complete(&req->base, err); +} - } - -static inline void chcr_handle_aead_resp(struct aead_request *req, - unsigned char *input, - int err) +static inline int get_aead_subtype(struct crypto_aead *aead) { - struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); - struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm)); - - - chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op); - if (reqctx->b0_dma) - dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->b0_dma, - reqctx->b0_len, DMA_BIDIRECTIONAL); - if (reqctx->verify == VERIFY_SW) { - chcr_verify_tag(req, input, &err); - reqctx->verify = VERIFY_HW; + struct aead_alg *alg = crypto_aead_alg(aead); + struct chcr_alg_template *chcr_crypto_alg = + container_of(alg, struct chcr_alg_template, alg.aead); + return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK; } - req->base.complete(&req->base, err); -} -static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err) +void chcr_verify_tag(struct aead_request *req, u8 *input, int *err) { u8 temp[SHA512_DIGEST_SIZE]; struct crypto_aead *tfm = crypto_aead_reqtfm(req); @@ -230,6 +224,25 @@ static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err) *err = 0; } +static inline void chcr_handle_aead_resp(struct aead_request *req, + unsigned char *input, + int err) +{ + struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm)); + + chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op); + if (reqctx->b0_dma) + dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->b0_dma, + reqctx->b0_len, DMA_BIDIRECTIONAL); + if (reqctx->verify == VERIFY_SW) { + chcr_verify_tag(req, input, &err); + reqctx->verify = VERIFY_HW; + } + req->base.complete(&req->base, err); +} + /* * chcr_handle_resp - Unmap the DMA buffers associated with the request * @req: crypto request @@ -594,14 +607,6 @@ static void ulptx_walk_add_sg(struct ulptx_walk *walk, } } -static inline int get_aead_subtype(struct crypto_aead *aead) -{ - struct aead_alg *alg = crypto_aead_alg(aead); - struct chcr_alg_template *chcr_crypto_alg = - container_of(alg, struct chcr_alg_template, alg.aead); - return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK; -} - static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm) { struct crypto_alg *alg = tfm->__crt_alg; @@ -1100,7 +1105,6 @@ static int chcr_final_cipher_iv(struct ablkcipher_request *req, } - static int chcr_handle_cipher_resp(struct ablkcipher_request *req, unsigned char *input, int err) { @@ -2198,9 +2202,9 @@ err: return ERR_PTR(error); } -static int chcr_aead_dma_map(struct device *dev, - struct aead_request *req, - unsigned short op_type) +int chcr_aead_dma_map(struct device *dev, + struct aead_request *req, + unsigned short op_type) { int error; struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); @@ -2242,9 +2246,9 @@ err: return -ENOMEM; } -static void chcr_aead_dma_unmap(struct device *dev, - struct aead_request *req, - unsigned short op_type) +void chcr_aead_dma_unmap(struct device *dev, + struct aead_request *req, + unsigned short op_type) { struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); struct crypto_aead *tfm = crypto_aead_reqtfm(req); @@ -2269,10 +2273,10 @@ static void chcr_aead_dma_unmap(struct device *dev, } } -static inline void chcr_add_aead_src_ent(struct aead_request *req, - struct ulptx_sgl *ulptx, - unsigned int assoclen, - unsigned short op_type) +void chcr_add_aead_src_ent(struct aead_request *req, + struct ulptx_sgl *ulptx, + unsigned int assoclen, + unsigned short op_type) { struct ulptx_walk ulp_walk; struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); @@ -2304,11 +2308,11 @@ static inline void chcr_add_aead_src_ent(struct aead_request *req, } } -static inline void chcr_add_aead_dst_ent(struct aead_request *req, - struct cpl_rx_phys_dsgl *phys_cpl, - unsigned int assoclen, - unsigned short op_type, - unsigned short qid) +void chcr_add_aead_dst_ent(struct aead_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + unsigned int assoclen, + unsigned short op_type, + unsigned short qid) { struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); struct crypto_aead *tfm = crypto_aead_reqtfm(req); @@ -2326,9 +2330,9 @@ static inline void chcr_add_aead_dst_ent(struct aead_request *req, dsgl_walk_end(&dsgl_walk, qid); } -static inline void chcr_add_cipher_src_ent(struct ablkcipher_request *req, - struct ulptx_sgl *ulptx, - struct cipher_wr_param *wrparam) +void chcr_add_cipher_src_ent(struct ablkcipher_request *req, + struct ulptx_sgl *ulptx, + struct cipher_wr_param *wrparam) { struct ulptx_walk ulp_walk; struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); @@ -2351,10 +2355,10 @@ static inline void chcr_add_cipher_src_ent(struct ablkcipher_request *req, } } -static inline void chcr_add_cipher_dst_ent(struct ablkcipher_request *req, - struct cpl_rx_phys_dsgl *phys_cpl, - struct cipher_wr_param *wrparam, - unsigned short qid) +void chcr_add_cipher_dst_ent(struct ablkcipher_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + struct cipher_wr_param *wrparam, + unsigned short qid) { struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); struct dsgl_walk dsgl_walk; @@ -2369,9 +2373,9 @@ static inline void chcr_add_cipher_dst_ent(struct ablkcipher_request *req, dsgl_walk_end(&dsgl_walk, qid); } -static inline void chcr_add_hash_src_ent(struct ahash_request *req, - struct ulptx_sgl *ulptx, - struct hash_wr_param *param) +void chcr_add_hash_src_ent(struct ahash_request *req, + struct ulptx_sgl *ulptx, + struct hash_wr_param *param) { struct ulptx_walk ulp_walk; struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req); @@ -2398,9 +2402,8 @@ static inline void chcr_add_hash_src_ent(struct ahash_request *req, } } - -static inline int chcr_hash_dma_map(struct device *dev, - struct ahash_request *req) +int chcr_hash_dma_map(struct device *dev, + struct ahash_request *req) { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); int error = 0; @@ -2415,8 +2418,8 @@ static inline int chcr_hash_dma_map(struct device *dev, return 0; } -static inline void chcr_hash_dma_unmap(struct device *dev, - struct ahash_request *req) +void chcr_hash_dma_unmap(struct device *dev, + struct ahash_request *req) { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); @@ -2429,9 +2432,8 @@ static inline void chcr_hash_dma_unmap(struct device *dev, } - -static int chcr_cipher_dma_map(struct device *dev, - struct ablkcipher_request *req) +int chcr_cipher_dma_map(struct device *dev, + struct ablkcipher_request *req) { int error; struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); @@ -2465,8 +2467,9 @@ err: dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); return -ENOMEM; } -static void chcr_cipher_dma_unmap(struct device *dev, - struct ablkcipher_request *req) + +void chcr_cipher_dma_unmap(struct device *dev, + struct ablkcipher_request *req) { struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); @@ -3371,6 +3374,40 @@ out: aeadctx->enckey_len = 0; return -EINVAL; } + +static int chcr_aead_op(struct aead_request *req, + unsigned short op_type, + int size, + create_wr_t create_wr_fn) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct uld_ctx *u_ctx; + struct sk_buff *skb; + + if (!a_ctx(tfm)->dev) { + pr_err("chcr : %s : No crypto device.\n", __func__); + return -ENXIO; + } + u_ctx = ULD_CTX(a_ctx(tfm)); + if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], + a_ctx(tfm)->tx_qidx)) { + if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + return -EBUSY; + } + + /* Form a WR from req */ + skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[a_ctx(tfm)->rx_qidx], size, + op_type); + + if (IS_ERR(skb) || !skb) + return PTR_ERR(skb); + + skb->dev = u_ctx->lldi.ports[0]; + set_wr_txq(skb, CPL_PRIORITY_DATA, a_ctx(tfm)->tx_qidx); + chcr_send_wr(skb); + return -EINPROGRESS; +} + static int chcr_aead_encrypt(struct aead_request *req) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); @@ -3423,38 +3460,6 @@ static int chcr_aead_decrypt(struct aead_request *req) } } -static int chcr_aead_op(struct aead_request *req, - unsigned short op_type, - int size, - create_wr_t create_wr_fn) -{ - struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct uld_ctx *u_ctx; - struct sk_buff *skb; - - if (!a_ctx(tfm)->dev) { - pr_err("chcr : %s : No crypto device.\n", __func__); - return -ENXIO; - } - u_ctx = ULD_CTX(a_ctx(tfm)); - if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - a_ctx(tfm)->tx_qidx)) { - if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) - return -EBUSY; - } - - /* Form a WR from req */ - skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[a_ctx(tfm)->rx_qidx], size, - op_type); - - if (IS_ERR(skb) || !skb) - return PTR_ERR(skb); - - skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, a_ctx(tfm)->tx_qidx); - chcr_send_wr(skb); - return -EINPROGRESS; -} static struct chcr_alg_template driver_algs[] = { /* AES-CBC */ { diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h index 96c9335ee728..d1673a5d4bf5 100644 --- a/drivers/crypto/chelsio/chcr_algo.h +++ b/drivers/crypto/chelsio/chcr_algo.h @@ -226,15 +226,6 @@ #define SPACE_LEFT(len) \ ((SGE_MAX_WR_LEN - WR_MIN_LEN - (len))) -unsigned int sgl_ent_len[] = {0, 0, 16, 24, 40, 48, 64, 72, 88, - 96, 112, 120, 136, 144, 160, 168, 184, - 192, 208, 216, 232, 240, 256, 264, 280, - 288, 304, 312, 328, 336, 352, 360, 376}; -unsigned int dsgl_ent_len[] = {0, 32, 32, 48, 48, 64, 64, 80, 80, - 112, 112, 128, 128, 144, 144, 160, 160, - 192, 192, 208, 208, 224, 224, 240, 240, - 272, 272, 288, 288, 304, 304, 320, 320}; - struct algo_param { unsigned int auth_mode; unsigned int mk_size; @@ -404,10 +395,4 @@ static inline u32 aes_ks_subword(const u32 w) return *(u32 *)(&bytes[0]); } -static u32 round_constant[11] = { - 0x01000000, 0x02000000, 0x04000000, 0x08000000, - 0x10000000, 0x20000000, 0x40000000, 0x80000000, - 0x1B000000, 0x36000000, 0x6C000000 -}; - #endif /* __CHCR_ALGO_H__ */ diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c index f5a2624081dc..04f277cade7c 100644 --- a/drivers/crypto/chelsio/chcr_core.c +++ b/drivers/crypto/chelsio/chcr_core.c @@ -48,6 +48,9 @@ static struct cxgb4_uld_info chcr_uld_info = { .add = chcr_uld_add, .state_change = chcr_uld_state_change, .rx_handler = chcr_uld_rx_handler, +#ifdef CONFIG_CHELSIO_IPSEC_INLINE + .tx_handler = chcr_uld_tx_handler, +#endif /* CONFIG_CHELSIO_IPSEC_INLINE */ }; struct uld_ctx *assign_chcr_device(void) @@ -164,6 +167,10 @@ static void *chcr_uld_add(const struct cxgb4_lld_info *lld) goto out; } u_ctx->lldi = *lld; +#ifdef CONFIG_CHELSIO_IPSEC_INLINE + if (lld->crypto & ULP_CRYPTO_IPSEC_INLINE) + chcr_add_xfrmops(lld); +#endif /* CONFIG_CHELSIO_IPSEC_INLINE */ out: return u_ctx; } @@ -187,6 +194,13 @@ int chcr_uld_rx_handler(void *handle, const __be64 *rsp, return 0; } +#ifdef CONFIG_CHELSIO_IPSEC_INLINE +int chcr_uld_tx_handler(struct sk_buff *skb, struct net_device *dev) +{ + return chcr_ipsec_xmit(skb, dev); +} +#endif /* CONFIG_CHELSIO_IPSEC_INLINE */ + static int chcr_uld_state_change(void *handle, enum cxgb4_state state) { struct uld_ctx *u_ctx = handle; diff --git a/drivers/crypto/chelsio/chcr_core.h b/drivers/crypto/chelsio/chcr_core.h index 94e7412f6164..3c29ee09b8b5 100644 --- a/drivers/crypto/chelsio/chcr_core.h +++ b/drivers/crypto/chelsio/chcr_core.h @@ -39,6 +39,7 @@ #include #include "t4_hw.h" #include "cxgb4.h" +#include "t4_msg.h" #include "cxgb4_uld.h" #define DRV_MODULE_NAME "chcr" @@ -89,12 +90,49 @@ struct uld_ctx { struct chcr_dev *dev; }; +struct chcr_ipsec_req { + struct ulp_txpkt ulptx; + struct ulptx_idata sc_imm; + struct cpl_tx_sec_pdu sec_cpl; + struct _key_ctx key_ctx; +}; + +struct chcr_ipsec_wr { + struct fw_ulptx_wr wreq; + struct chcr_ipsec_req req; +}; + +struct ipsec_sa_entry { + int hmac_ctrl; + unsigned int enckey_len; + unsigned int kctx_len; + unsigned int authsize; + __be32 key_ctx_hdr; + char salt[MAX_SALT]; + char key[2 * AES_MAX_KEY_SIZE]; +}; + +/* + * sgl_len - calculates the size of an SGL of the given capacity + * @n: the number of SGL entries + * Calculates the number of flits needed for a scatter/gather list that + * can hold the given number of entries. + */ +static inline unsigned int sgl_len(unsigned int n) +{ + n--; + return (3 * n) / 2 + (n & 1) + 2; +} + struct uld_ctx *assign_chcr_device(void); int chcr_send_wr(struct sk_buff *skb); int start_crypto(void); int stop_crypto(void); int chcr_uld_rx_handler(void *handle, const __be64 *rsp, const struct pkt_gl *pgl); +int chcr_uld_tx_handler(struct sk_buff *skb, struct net_device *dev); int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, int err); +int chcr_ipsec_xmit(struct sk_buff *skb, struct net_device *dev); +void chcr_add_xfrmops(const struct cxgb4_lld_info *lld); #endif /* __CHCR_CORE_H__ */ diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h index 94a87e3ad9bc..ea2c578805e6 100644 --- a/drivers/crypto/chelsio/chcr_crypto.h +++ b/drivers/crypto/chelsio/chcr_crypto.h @@ -210,8 +210,6 @@ struct dsgl_walk { struct phys_sge_pairs *to; }; - - struct chcr_gcm_ctx { u8 ghash_h[AEAD_H_SIZE]; }; @@ -227,8 +225,6 @@ struct __aead_ctx { struct chcr_authenc_ctx authenc[0]; }; - - struct chcr_aead_ctx { __be32 key_ctx_hdr; unsigned int enckey_len; @@ -240,8 +236,6 @@ struct chcr_aead_ctx { struct __aead_ctx ctx[0]; }; - - struct hmac_ctx { struct crypto_shash *base_hash; u8 ipad[CHCR_HASH_MAX_BLOCK_SIZE_128]; @@ -307,44 +301,29 @@ typedef struct sk_buff *(*create_wr_t)(struct aead_request *req, int size, unsigned short op_type); -static int chcr_aead_op(struct aead_request *req_base, - unsigned short op_type, - int size, - create_wr_t create_wr_fn); -static inline int get_aead_subtype(struct crypto_aead *aead); -static int chcr_handle_cipher_resp(struct ablkcipher_request *req, - unsigned char *input, int err); -static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err); -static int chcr_aead_dma_map(struct device *dev, struct aead_request *req, - unsigned short op_type); -static void chcr_aead_dma_unmap(struct device *dev, struct aead_request - *req, unsigned short op_type); -static inline void chcr_add_aead_dst_ent(struct aead_request *req, - struct cpl_rx_phys_dsgl *phys_cpl, - unsigned int assoclen, - unsigned short op_type, - unsigned short qid); -static inline void chcr_add_aead_src_ent(struct aead_request *req, - struct ulptx_sgl *ulptx, - unsigned int assoclen, - unsigned short op_type); -static inline void chcr_add_cipher_src_ent(struct ablkcipher_request *req, - struct ulptx_sgl *ulptx, - struct cipher_wr_param *wrparam); -static int chcr_cipher_dma_map(struct device *dev, - struct ablkcipher_request *req); -static void chcr_cipher_dma_unmap(struct device *dev, - struct ablkcipher_request *req); -static inline void chcr_add_cipher_dst_ent(struct ablkcipher_request *req, - struct cpl_rx_phys_dsgl *phys_cpl, - struct cipher_wr_param *wrparam, - unsigned short qid); +void chcr_verify_tag(struct aead_request *req, u8 *input, int *err); +int chcr_aead_dma_map(struct device *dev, struct aead_request *req, + unsigned short op_type); +void chcr_aead_dma_unmap(struct device *dev, struct aead_request *req, + unsigned short op_type); +void chcr_add_aead_dst_ent(struct aead_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + unsigned int assoclen, unsigned short op_type, + unsigned short qid); +void chcr_add_aead_src_ent(struct aead_request *req, struct ulptx_sgl *ulptx, + unsigned int assoclen, unsigned short op_type); +void chcr_add_cipher_src_ent(struct ablkcipher_request *req, + struct ulptx_sgl *ulptx, + struct cipher_wr_param *wrparam); +int chcr_cipher_dma_map(struct device *dev, struct ablkcipher_request *req); +void chcr_cipher_dma_unmap(struct device *dev, struct ablkcipher_request *req); +void chcr_add_cipher_dst_ent(struct ablkcipher_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + struct cipher_wr_param *wrparam, + unsigned short qid); int sg_nents_len_skip(struct scatterlist *sg, u64 len, u64 skip); -static inline void chcr_add_hash_src_ent(struct ahash_request *req, - struct ulptx_sgl *ulptx, - struct hash_wr_param *param); -static inline int chcr_hash_dma_map(struct device *dev, - struct ahash_request *req); -static inline void chcr_hash_dma_unmap(struct device *dev, - struct ahash_request *req); +void chcr_add_hash_src_ent(struct ahash_request *req, struct ulptx_sgl *ulptx, + struct hash_wr_param *param); +int chcr_hash_dma_map(struct device *dev, struct ahash_request *req); +void chcr_hash_dma_unmap(struct device *dev, struct ahash_request *req); #endif /* __CHCR_CRYPTO_H__ */ diff --git a/drivers/crypto/chelsio/chcr_ipsec.c b/drivers/crypto/chelsio/chcr_ipsec.c new file mode 100644 index 000000000000..f90f991c133f --- /dev/null +++ b/drivers/crypto/chelsio/chcr_ipsec.c @@ -0,0 +1,660 @@ +/* + * This file is part of the Chelsio T6 Crypto driver for Linux. + * + * Copyright (c) 2003-2017 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Written and Maintained by: + * Atul Gupta (atul.gupta@chelsio.com) + */ + +#define pr_fmt(fmt) "chcr:" fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "chcr_core.h" +#include "chcr_algo.h" +#include "chcr_crypto.h" + +/* + * Max Tx descriptor space we allow for an Ethernet packet to be inlined + * into a WR. + */ +#define MAX_IMM_TX_PKT_LEN 256 +#define GCM_ESP_IV_SIZE 8 + +static int chcr_xfrm_add_state(struct xfrm_state *x); +static void chcr_xfrm_del_state(struct xfrm_state *x); +static void chcr_xfrm_free_state(struct xfrm_state *x); +static bool chcr_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x); + +static const struct xfrmdev_ops chcr_xfrmdev_ops = { + .xdo_dev_state_add = chcr_xfrm_add_state, + .xdo_dev_state_delete = chcr_xfrm_del_state, + .xdo_dev_state_free = chcr_xfrm_free_state, + .xdo_dev_offload_ok = chcr_ipsec_offload_ok, +}; + +/* Add offload xfrms to Chelsio Interface */ +void chcr_add_xfrmops(const struct cxgb4_lld_info *lld) +{ + struct net_device *netdev = NULL; + int i; + + for (i = 0; i < lld->nports; i++) { + netdev = lld->ports[i]; + if (!netdev) + continue; + netdev->xfrmdev_ops = &chcr_xfrmdev_ops; + netdev->hw_enc_features |= NETIF_F_HW_ESP; + netdev->features |= NETIF_F_HW_ESP; + rtnl_lock(); + netdev_change_features(netdev); + rtnl_unlock(); + } +} + +static inline int chcr_ipsec_setauthsize(struct xfrm_state *x, + struct ipsec_sa_entry *sa_entry) +{ + int hmac_ctrl; + int authsize = x->aead->alg_icv_len / 8; + + sa_entry->authsize = authsize; + + switch (authsize) { + case ICV_8: + hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; + break; + case ICV_12: + hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; + break; + case ICV_16: + hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; + break; + default: + return -EINVAL; + } + return hmac_ctrl; +} + +static inline int chcr_ipsec_setkey(struct xfrm_state *x, + struct ipsec_sa_entry *sa_entry) +{ + struct crypto_cipher *cipher; + int keylen = (x->aead->alg_key_len + 7) / 8; + unsigned char *key = x->aead->alg_key; + int ck_size, key_ctx_size = 0; + unsigned char ghash_h[AEAD_H_SIZE]; + int ret = 0; + + if (keylen > 3) { + keylen -= 4; /* nonce/salt is present in the last 4 bytes */ + memcpy(sa_entry->salt, key + keylen, 4); + } + + if (keylen == AES_KEYSIZE_128) { + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; + } else if (keylen == AES_KEYSIZE_192) { + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; + } else if (keylen == AES_KEYSIZE_256) { + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; + } else { + pr_err("GCM: Invalid key length %d\n", keylen); + ret = -EINVAL; + goto out; + } + + memcpy(sa_entry->key, key, keylen); + sa_entry->enckey_len = keylen; + key_ctx_size = sizeof(struct _key_ctx) + + ((DIV_ROUND_UP(keylen, 16)) << 4) + + AEAD_H_SIZE; + + sa_entry->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, + CHCR_KEYCTX_MAC_KEY_SIZE_128, + 0, 0, + key_ctx_size >> 4); + + /* Calculate the H = CIPH(K, 0 repeated 16 times). + * It will go in key context + */ + cipher = crypto_alloc_cipher("aes-generic", 0, 0); + if (IS_ERR(cipher)) { + sa_entry->enckey_len = 0; + ret = -ENOMEM; + goto out; + } + + ret = crypto_cipher_setkey(cipher, key, keylen); + if (ret) { + sa_entry->enckey_len = 0; + goto out1; + } + memset(ghash_h, 0, AEAD_H_SIZE); + crypto_cipher_encrypt_one(cipher, ghash_h, ghash_h); + memcpy(sa_entry->key + (DIV_ROUND_UP(sa_entry->enckey_len, 16) * + 16), ghash_h, AEAD_H_SIZE); + sa_entry->kctx_len = ((DIV_ROUND_UP(sa_entry->enckey_len, 16)) << 4) + + AEAD_H_SIZE; +out1: + crypto_free_cipher(cipher); +out: + return ret; +} + +/* + * chcr_xfrm_add_state + * returns 0 on success, negative error if failed to send message to FPGA + * positive error if FPGA returned a bad response + */ +static int chcr_xfrm_add_state(struct xfrm_state *x) +{ + struct net_device *netdev = x->xso.dev; + struct port_info *pi = netdev_priv(netdev); + struct ipsec_sa_entry *sa_entry; + struct adapter *adap; + int res = 0; + + adap = pi->adapter; + + if (x->props.aalgo != SADB_AALG_NONE) { + pr_debug("CHCR: Cannot offload authenticated xfrm states\n"); + return -EINVAL; + } + if (x->props.calgo != SADB_X_CALG_NONE) { + pr_debug("CHCR: Cannot offload compressed xfrm states\n"); + return -EINVAL; + } + if (x->props.flags & XFRM_STATE_ESN) { + pr_debug("CHCR: Cannot offload ESN xfrm states\n"); + return -EINVAL; + } + if (x->props.family != AF_INET && + x->props.family != AF_INET6) { + pr_debug("CHCR: Only IPv4/6 xfrm state offloaded\n"); + return -EINVAL; + } + if (x->props.mode != XFRM_MODE_TRANSPORT && + x->props.mode != XFRM_MODE_TUNNEL) { + pr_debug("CHCR: Only transport and tunnel xfrm offload\n"); + return -EINVAL; + } + if (x->id.proto != IPPROTO_ESP) { + pr_debug("CHCR: Only ESP xfrm state offloaded\n"); + return -EINVAL; + } + if (x->encap) { + pr_debug("CHCR: Encapsulated xfrm state not offloaded\n"); + return -EINVAL; + } + if (!x->aead) { + pr_debug("CHCR: Cannot offload xfrm states without aead\n"); + return -EINVAL; + } + if (x->aead->alg_icv_len != 128 && + x->aead->alg_icv_len != 96) { + pr_debug("CHCR: Cannot offload xfrm states with AEAD ICV length other than 96b & 128b\n"); + return -EINVAL; + } + if ((x->aead->alg_key_len != 128 + 32) && + (x->aead->alg_key_len != 256 + 32)) { + pr_debug("CHCR: Cannot offload xfrm states with AEAD key length other than 128/256 bit\n"); + return -EINVAL; + } + if (x->tfcpad) { + pr_debug("CHCR: Cannot offload xfrm states with tfc padding\n"); + return -EINVAL; + } + if (!x->geniv) { + pr_debug("CHCR: Cannot offload xfrm states without geniv\n"); + return -EINVAL; + } + if (strcmp(x->geniv, "seqiv")) { + pr_debug("CHCR: Cannot offload xfrm states with geniv other than seqiv\n"); + return -EINVAL; + } + + sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL); + if (!sa_entry) { + res = -ENOMEM; + goto out; + } + + sa_entry->hmac_ctrl = chcr_ipsec_setauthsize(x, sa_entry); + chcr_ipsec_setkey(x, sa_entry); + x->xso.offload_handle = (unsigned long)sa_entry; + try_module_get(THIS_MODULE); +out: + return res; +} + +static void chcr_xfrm_del_state(struct xfrm_state *x) +{ + /* do nothing */ + if (!x->xso.offload_handle) + return; +} + +static void chcr_xfrm_free_state(struct xfrm_state *x) +{ + struct ipsec_sa_entry *sa_entry; + + if (!x->xso.offload_handle) + return; + + sa_entry = (struct ipsec_sa_entry *)x->xso.offload_handle; + kfree(sa_entry); + module_put(THIS_MODULE); +} + +static bool chcr_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x) +{ + /* Offload with IP options is not supported yet */ + if (ip_hdr(skb)->ihl > 5) + return false; + + return true; +} + +static inline int is_eth_imm(const struct sk_buff *skb, unsigned int kctx_len) +{ + int hdrlen = sizeof(struct chcr_ipsec_req) + kctx_len; + + hdrlen += sizeof(struct cpl_tx_pkt); + if (skb->len <= MAX_IMM_TX_PKT_LEN - hdrlen) + return hdrlen; + return 0; +} + +static inline unsigned int calc_tx_sec_flits(const struct sk_buff *skb, + unsigned int kctx_len) +{ + unsigned int flits; + int hdrlen = is_eth_imm(skb, kctx_len); + + /* If the skb is small enough, we can pump it out as a work request + * with only immediate data. In that case we just have to have the + * TX Packet header plus the skb data in the Work Request. + */ + + if (hdrlen) + return DIV_ROUND_UP(skb->len + hdrlen, sizeof(__be64)); + + flits = sgl_len(skb_shinfo(skb)->nr_frags + 1); + + /* Otherwise, we're going to have to construct a Scatter gather list + * of the skb body and fragments. We also include the flits necessary + * for the TX Packet Work Request and CPL. We always have a firmware + * Write Header (incorporated as part of the cpl_tx_pkt_lso and + * cpl_tx_pkt structures), followed by either a TX Packet Write CPL + * message or, if we're doing a Large Send Offload, an LSO CPL message + * with an embedded TX Packet Write CPL message. + */ + flits += (sizeof(struct fw_ulptx_wr) + + sizeof(struct chcr_ipsec_req) + + kctx_len + + sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64); + return flits; +} + +inline void *copy_cpltx_pktxt(struct sk_buff *skb, + struct net_device *dev, + void *pos) +{ + struct adapter *adap; + struct port_info *pi; + struct sge_eth_txq *q; + struct cpl_tx_pkt_core *cpl; + u64 cntrl; + u32 ctrl0, qidx; + + pi = netdev_priv(dev); + adap = pi->adapter; + qidx = skb->queue_mapping; + q = &adap->sge.ethtxq[qidx + pi->first_qset]; + + cpl = (struct cpl_tx_pkt_core *)pos; + + if (skb->ip_summed == CHECKSUM_PARTIAL) + cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F; + ctrl0 = TXPKT_OPCODE_V(CPL_TX_PKT_XT) | TXPKT_INTF_V(pi->tx_chan) | + TXPKT_PF_V(adap->pf); + if (skb_vlan_tag_present(skb)) { + q->vlan_ins++; + cntrl |= TXPKT_VLAN_VLD_F | TXPKT_VLAN_V(skb_vlan_tag_get(skb)); + } + + cpl->ctrl0 = htonl(ctrl0); + cpl->pack = htons(0); + cpl->len = htons(skb->len); + cpl->ctrl1 = cpu_to_be64(cntrl); + + pos += sizeof(struct cpl_tx_pkt_core); + return pos; +} + +inline void *copy_key_cpltx_pktxt(struct sk_buff *skb, + struct net_device *dev, + void *pos, + struct ipsec_sa_entry *sa_entry) +{ + struct adapter *adap; + struct port_info *pi; + struct sge_eth_txq *q; + unsigned int len, qidx; + struct _key_ctx *key_ctx; + int left, eoq, key_len; + + pi = netdev_priv(dev); + adap = pi->adapter; + qidx = skb->queue_mapping; + q = &adap->sge.ethtxq[qidx + pi->first_qset]; + len = sa_entry->enckey_len + sizeof(struct cpl_tx_pkt_core); + key_len = sa_entry->kctx_len; + + /* end of queue, reset pos to start of queue */ + eoq = (void *)q->q.stat - pos; + left = eoq; + if (!eoq) { + pos = q->q.desc; + left = 64 * q->q.size; + } + + /* Copy the Key context header */ + key_ctx = (struct _key_ctx *)pos; + key_ctx->ctx_hdr = sa_entry->key_ctx_hdr; + memcpy(key_ctx->salt, sa_entry->salt, MAX_SALT); + pos += sizeof(struct _key_ctx); + left -= sizeof(struct _key_ctx); + + if (likely(len <= left)) { + memcpy(key_ctx->key, sa_entry->key, key_len); + pos += key_len; + } else { + if (key_len <= left) { + memcpy(pos, sa_entry->key, key_len); + pos += key_len; + } else { + memcpy(pos, sa_entry->key, left); + memcpy(q->q.desc, sa_entry->key + left, + key_len - left); + pos = q->q.desc + (key_len - left); + } + } + /* Copy CPL TX PKT XT */ + pos = copy_cpltx_pktxt(skb, dev, pos); + + return pos; +} + +inline void *chcr_crypto_wreq(struct sk_buff *skb, + struct net_device *dev, + void *pos, + int credits, + struct ipsec_sa_entry *sa_entry) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adap = pi->adapter; + unsigned int immdatalen = 0; + unsigned int ivsize = GCM_ESP_IV_SIZE; + struct chcr_ipsec_wr *wr; + unsigned int flits; + u32 wr_mid; + int qidx = skb_get_queue_mapping(skb); + struct sge_eth_txq *q = &adap->sge.ethtxq[qidx + pi->first_qset]; + unsigned int kctx_len = sa_entry->kctx_len; + int qid = q->q.cntxt_id; + + atomic_inc(&adap->chcr_stats.ipsec_cnt); + + flits = calc_tx_sec_flits(skb, kctx_len); + + if (is_eth_imm(skb, kctx_len)) + immdatalen = skb->len; + + /* WR Header */ + wr = (struct chcr_ipsec_wr *)pos; + wr->wreq.op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); + wr_mid = FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(flits, 2)); + + if (unlikely(credits < ETHTXQ_STOP_THRES)) { + netif_tx_stop_queue(q->txq); + q->q.stops++; + wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; + } + wr_mid |= FW_ULPTX_WR_DATA_F; + wr->wreq.flowid_len16 = htonl(wr_mid); + + /* ULPTX */ + wr->req.ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(pi->port_id, qid); + wr->req.ulptx.len = htonl(DIV_ROUND_UP(flits, 2) - 1); + + /* Sub-command */ + wr->req.sc_imm.cmd_more = FILL_CMD_MORE(immdatalen); + wr->req.sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) + + sizeof(wr->req.key_ctx) + + kctx_len + + sizeof(struct cpl_tx_pkt_core) + + immdatalen); + + /* CPL_SEC_PDU */ + wr->req.sec_cpl.op_ivinsrtofst = htonl( + CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) | + CPL_TX_SEC_PDU_CPLLEN_V(2) | + CPL_TX_SEC_PDU_PLACEHOLDER_V(1) | + CPL_TX_SEC_PDU_IVINSRTOFST_V( + (skb_transport_offset(skb) + + sizeof(struct ip_esp_hdr) + 1))); + + wr->req.sec_cpl.pldlen = htonl(skb->len); + + wr->req.sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( + (skb_transport_offset(skb) + 1), + (skb_transport_offset(skb) + + sizeof(struct ip_esp_hdr)), + (skb_transport_offset(skb) + + sizeof(struct ip_esp_hdr) + + GCM_ESP_IV_SIZE + 1), 0); + + wr->req.sec_cpl.cipherstop_lo_authinsert = + FILL_SEC_CPL_AUTHINSERT(0, skb_transport_offset(skb) + + sizeof(struct ip_esp_hdr) + + GCM_ESP_IV_SIZE + 1, + sa_entry->authsize, + sa_entry->authsize); + wr->req.sec_cpl.seqno_numivs = + FILL_SEC_CPL_SCMD0_SEQNO(CHCR_ENCRYPT_OP, 1, + CHCR_SCMD_CIPHER_MODE_AES_GCM, + CHCR_SCMD_AUTH_MODE_GHASH, + sa_entry->hmac_ctrl, + ivsize >> 1); + wr->req.sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, + 0, 0, 0); + + pos += sizeof(struct fw_ulptx_wr) + + sizeof(struct ulp_txpkt) + + sizeof(struct ulptx_idata) + + sizeof(struct cpl_tx_sec_pdu); + + pos = copy_key_cpltx_pktxt(skb, dev, pos, sa_entry); + + return pos; +} + +/** + * flits_to_desc - returns the num of Tx descriptors for the given flits + * @n: the number of flits + * + * Returns the number of Tx descriptors needed for the supplied number + * of flits. + */ +static inline unsigned int flits_to_desc(unsigned int n) +{ + WARN_ON(n > SGE_MAX_WR_LEN / 8); + return DIV_ROUND_UP(n, 8); +} + +static inline unsigned int txq_avail(const struct sge_txq *q) +{ + return q->size - 1 - q->in_use; +} + +static void eth_txq_stop(struct sge_eth_txq *q) +{ + netif_tx_stop_queue(q->txq); + q->q.stops++; +} + +static inline void txq_advance(struct sge_txq *q, unsigned int n) +{ + q->in_use += n; + q->pidx += n; + if (q->pidx >= q->size) + q->pidx -= q->size; +} + +/* + * chcr_ipsec_xmit called from ULD Tx handler + */ +int chcr_ipsec_xmit(struct sk_buff *skb, struct net_device *dev) +{ + struct xfrm_state *x = xfrm_input_state(skb); + struct ipsec_sa_entry *sa_entry; + u64 *pos, *end, *before, cntrl, *sgl; + int qidx, left, credits; + unsigned int flits = 0, ndesc, kctx_len; + struct adapter *adap; + struct sge_eth_txq *q; + struct port_info *pi; + dma_addr_t addr[MAX_SKB_FRAGS + 1]; + bool immediate = false; + + if (!x->xso.offload_handle) + return NETDEV_TX_BUSY; + + sa_entry = (struct ipsec_sa_entry *)x->xso.offload_handle; + kctx_len = sa_entry->kctx_len; + + if (skb->sp->len != 1) { +out_free: dev_kfree_skb_any(skb); + return NETDEV_TX_OK; + } + + pi = netdev_priv(dev); + adap = pi->adapter; + qidx = skb->queue_mapping; + q = &adap->sge.ethtxq[qidx + pi->first_qset]; + + cxgb4_reclaim_completed_tx(adap, &q->q, true); + cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F; + + flits = calc_tx_sec_flits(skb, sa_entry->kctx_len); + ndesc = flits_to_desc(flits); + credits = txq_avail(&q->q) - ndesc; + + if (unlikely(credits < 0)) { + eth_txq_stop(q); + dev_err(adap->pdev_dev, + "%s: Tx ring %u full while queue awake! cred:%d %d %d flits:%d\n", + dev->name, qidx, credits, ndesc, txq_avail(&q->q), + flits); + return NETDEV_TX_BUSY; + } + + if (is_eth_imm(skb, kctx_len)) + immediate = true; + + if (!immediate && + unlikely(cxgb4_map_skb(adap->pdev_dev, skb, addr) < 0)) { + q->mapping_err++; + goto out_free; + } + + pos = (u64 *)&q->q.desc[q->q.pidx]; + before = (u64 *)pos; + end = (u64 *)pos + flits; + /* Setup IPSec CPL */ + pos = (void *)chcr_crypto_wreq(skb, dev, (void *)pos, + credits, sa_entry); + if (before > (u64 *)pos) { + left = (u8 *)end - (u8 *)q->q.stat; + end = (void *)q->q.desc + left; + } + if (pos == (u64 *)q->q.stat) { + left = (u8 *)end - (u8 *)q->q.stat; + end = (void *)q->q.desc + left; + pos = (void *)q->q.desc; + } + + sgl = (void *)pos; + if (immediate) { + cxgb4_inline_tx_skb(skb, &q->q, sgl); + dev_consume_skb_any(skb); + } else { + int last_desc; + + cxgb4_write_sgl(skb, &q->q, (void *)sgl, end, + 0, addr); + skb_orphan(skb); + + last_desc = q->q.pidx + ndesc - 1; + if (last_desc >= q->q.size) + last_desc -= q->q.size; + q->q.sdesc[last_desc].skb = skb; + q->q.sdesc[last_desc].sgl = (struct ulptx_sgl *)sgl; + } + txq_advance(&q->q, ndesc); + + cxgb4_ring_tx_db(adap, &q->q, ndesc); + return NETDEV_TX_OK; +} From 79e53b2a5db63ab0a0a3b9c11612999a01efba15 Mon Sep 17 00:00:00 2001 From: Yang Shi Date: Sat, 18 Nov 2017 07:02:18 +0800 Subject: [PATCH 028/166] crypto: remove unused hardirq.h Preempt counter APIs have been split out, currently, hardirq.h just includes irq_enter/exit APIs which are not used by crypto at all. So, remove the unused hardirq.h. Signed-off-by: Yang Shi Cc: Herbert Xu Cc: "David S. Miller" Cc: linux-crypto@vger.kernel.org Signed-off-by: Herbert Xu --- crypto/ablk_helper.c | 1 - crypto/blkcipher.c | 1 - crypto/mcryptd.c | 1 - 3 files changed, 3 deletions(-) diff --git a/crypto/ablk_helper.c b/crypto/ablk_helper.c index 6e5d2f149b89..09776bb1360e 100644 --- a/crypto/ablk_helper.c +++ b/crypto/ablk_helper.c @@ -26,7 +26,6 @@ #include #include #include -#include #include #include #include diff --git a/crypto/blkcipher.c b/crypto/blkcipher.c index 6c43a0a17a55..01c0d4aa2563 100644 --- a/crypto/blkcipher.c +++ b/crypto/blkcipher.c @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include diff --git a/crypto/mcryptd.c b/crypto/mcryptd.c index 4e6472658852..9fa362c1ab44 100644 --- a/crypto/mcryptd.c +++ b/crypto/mcryptd.c @@ -26,7 +26,6 @@ #include #include #include -#include #define MCRYPTD_MAX_CPU_QLEN 100 #define MCRYPTD_BATCH 9 From 1964e333baf4e91b05defdf2572ea6d0845ffbd7 Mon Sep 17 00:00:00 2001 From: Christophe Jaillet Date: Sat, 18 Nov 2017 14:45:45 +0100 Subject: [PATCH 029/166] crypto: hifn_795x - Fix a memory leak in the error handling path of 'hifn_probe()' 'dev' is leaking in the error handling path of 'hifn_probe()'. Add a 'kfree(dev)' to match the code in 'hifn_remove()' Signed-off-by: Christophe JAILLET Reviewed-by: PrasannaKumar Muralidharan Signed-off-by: Herbert Xu --- drivers/crypto/hifn_795x.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c index e09d4055b19e..a5a36fe7bf2c 100644 --- a/drivers/crypto/hifn_795x.c +++ b/drivers/crypto/hifn_795x.c @@ -2579,6 +2579,7 @@ err_out_unmap_bars: for (i = 0; i < 3; ++i) if (dev->bar[i]) iounmap(dev->bar[i]); + kfree(dev); err_out_free_regions: pci_release_regions(pdev); From 019cd46984d04703a39924178f503a98436ac0d7 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 21 Nov 2017 13:40:17 +0000 Subject: [PATCH 030/166] crypto: arm64/aes-ce-cipher - move assembler code to .S file Most crypto drivers involving kernel mode NEON take care to put the code that actually touches the NEON register file in a separate compilation unit, to prevent the compiler from reordering code that preserves or restores the NEON context with code that may corrupt it. This is necessary because we currently have no way to express the restrictions imposed upon use of the NEON in kernel mode in a way that the compiler understands. However, in the case of aes-ce-cipher, it did not seem unreasonable to deviate from this rule, given how it does not seem possible for the compiler to reorder cross object function calls with asm blocks whose in- and output constraints reflect that it reads from and writes to memory. Now that LTO is being proposed for the arm64 kernel, it is time to revisit this. The link time optimization may replace the function calls to kernel_neon_begin() and kernel_neon_end() with instantiations of the IR that make up its implementation, allowing further reordering with the asm block. So let's clean this up, and move the asm() blocks into a separate .S file. Signed-off-by: Ard Biesheuvel Reviewed-By: Nick Desaulniers Signed-off-by: Herbert Xu --- arch/arm64/crypto/Makefile | 2 +- arch/arm64/crypto/aes-ce-core.S | 87 +++++++++++++ .../crypto/{aes-ce-cipher.c => aes-ce-glue.c} | 115 ++---------------- 3 files changed, 100 insertions(+), 104 deletions(-) create mode 100644 arch/arm64/crypto/aes-ce-core.S rename arch/arm64/crypto/{aes-ce-cipher.c => aes-ce-glue.c} (62%) diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile index b5edc5918c28..f5e8295fd756 100644 --- a/arch/arm64/crypto/Makefile +++ b/arch/arm64/crypto/Makefile @@ -24,7 +24,7 @@ obj-$(CONFIG_CRYPTO_CRC32_ARM64_CE) += crc32-ce.o crc32-ce-y:= crc32-ce-core.o crc32-ce-glue.o obj-$(CONFIG_CRYPTO_AES_ARM64_CE) += aes-ce-cipher.o -CFLAGS_aes-ce-cipher.o += -march=armv8-a+crypto +aes-ce-cipher-y := aes-ce-core.o aes-ce-glue.o obj-$(CONFIG_CRYPTO_AES_ARM64_CE_CCM) += aes-ce-ccm.o aes-ce-ccm-y := aes-ce-ccm-glue.o aes-ce-ccm-core.o diff --git a/arch/arm64/crypto/aes-ce-core.S b/arch/arm64/crypto/aes-ce-core.S new file mode 100644 index 000000000000..8efdfdade393 --- /dev/null +++ b/arch/arm64/crypto/aes-ce-core.S @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2013 - 2017 Linaro Ltd + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include + + .arch armv8-a+crypto + +ENTRY(__aes_ce_encrypt) + sub w3, w3, #2 + ld1 {v0.16b}, [x2] + ld1 {v1.4s}, [x0], #16 + cmp w3, #10 + bmi 0f + bne 3f + mov v3.16b, v1.16b + b 2f +0: mov v2.16b, v1.16b + ld1 {v3.4s}, [x0], #16 +1: aese v0.16b, v2.16b + aesmc v0.16b, v0.16b +2: ld1 {v1.4s}, [x0], #16 + aese v0.16b, v3.16b + aesmc v0.16b, v0.16b +3: ld1 {v2.4s}, [x0], #16 + subs w3, w3, #3 + aese v0.16b, v1.16b + aesmc v0.16b, v0.16b + ld1 {v3.4s}, [x0], #16 + bpl 1b + aese v0.16b, v2.16b + eor v0.16b, v0.16b, v3.16b + st1 {v0.16b}, [x1] + ret +ENDPROC(__aes_ce_encrypt) + +ENTRY(__aes_ce_decrypt) + sub w3, w3, #2 + ld1 {v0.16b}, [x2] + ld1 {v1.4s}, [x0], #16 + cmp w3, #10 + bmi 0f + bne 3f + mov v3.16b, v1.16b + b 2f +0: mov v2.16b, v1.16b + ld1 {v3.4s}, [x0], #16 +1: aesd v0.16b, v2.16b + aesimc v0.16b, v0.16b +2: ld1 {v1.4s}, [x0], #16 + aesd v0.16b, v3.16b + aesimc v0.16b, v0.16b +3: ld1 {v2.4s}, [x0], #16 + subs w3, w3, #3 + aesd v0.16b, v1.16b + aesimc v0.16b, v0.16b + ld1 {v3.4s}, [x0], #16 + bpl 1b + aesd v0.16b, v2.16b + eor v0.16b, v0.16b, v3.16b + st1 {v0.16b}, [x1] + ret +ENDPROC(__aes_ce_decrypt) + +/* + * __aes_ce_sub() - use the aese instruction to perform the AES sbox + * substitution on each byte in 'input' + */ +ENTRY(__aes_ce_sub) + dup v1.4s, w0 + movi v0.16b, #0 + aese v0.16b, v1.16b + umov w0, v0.s[0] + ret +ENDPROC(__aes_ce_sub) + +ENTRY(__aes_ce_invert) + ld1 {v0.4s}, [x1] + aesimc v1.16b, v0.16b + st1 {v1.4s}, [x0] + ret +ENDPROC(__aes_ce_invert) diff --git a/arch/arm64/crypto/aes-ce-cipher.c b/arch/arm64/crypto/aes-ce-glue.c similarity index 62% rename from arch/arm64/crypto/aes-ce-cipher.c rename to arch/arm64/crypto/aes-ce-glue.c index 6a75cd75ed11..e6b3227bbf57 100644 --- a/arch/arm64/crypto/aes-ce-cipher.c +++ b/arch/arm64/crypto/aes-ce-glue.c @@ -29,6 +29,13 @@ struct aes_block { u8 b[AES_BLOCK_SIZE]; }; +asmlinkage void __aes_ce_encrypt(u32 *rk, u8 *out, const u8 *in, int rounds); +asmlinkage void __aes_ce_decrypt(u32 *rk, u8 *out, const u8 *in, int rounds); + +asmlinkage u32 __aes_ce_sub(u32 l); +asmlinkage void __aes_ce_invert(struct aes_block *out, + const struct aes_block *in); + static int num_rounds(struct crypto_aes_ctx *ctx) { /* @@ -44,10 +51,6 @@ static int num_rounds(struct crypto_aes_ctx *ctx) static void aes_cipher_encrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[]) { struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm); - struct aes_block *out = (struct aes_block *)dst; - struct aes_block const *in = (struct aes_block *)src; - void *dummy0; - int dummy1; if (!may_use_simd()) { __aes_arm64_encrypt(ctx->key_enc, dst, src, num_rounds(ctx)); @@ -55,49 +58,13 @@ static void aes_cipher_encrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[]) } kernel_neon_begin(); - - __asm__(" ld1 {v0.16b}, %[in] ;" - " ld1 {v1.4s}, [%[key]], #16 ;" - " cmp %w[rounds], #10 ;" - " bmi 0f ;" - " bne 3f ;" - " mov v3.16b, v1.16b ;" - " b 2f ;" - "0: mov v2.16b, v1.16b ;" - " ld1 {v3.4s}, [%[key]], #16 ;" - "1: aese v0.16b, v2.16b ;" - " aesmc v0.16b, v0.16b ;" - "2: ld1 {v1.4s}, [%[key]], #16 ;" - " aese v0.16b, v3.16b ;" - " aesmc v0.16b, v0.16b ;" - "3: ld1 {v2.4s}, [%[key]], #16 ;" - " subs %w[rounds], %w[rounds], #3 ;" - " aese v0.16b, v1.16b ;" - " aesmc v0.16b, v0.16b ;" - " ld1 {v3.4s}, [%[key]], #16 ;" - " bpl 1b ;" - " aese v0.16b, v2.16b ;" - " eor v0.16b, v0.16b, v3.16b ;" - " st1 {v0.16b}, %[out] ;" - - : [out] "=Q"(*out), - [key] "=r"(dummy0), - [rounds] "=r"(dummy1) - : [in] "Q"(*in), - "1"(ctx->key_enc), - "2"(num_rounds(ctx) - 2) - : "cc"); - + __aes_ce_encrypt(ctx->key_enc, dst, src, num_rounds(ctx)); kernel_neon_end(); } static void aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[]) { struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm); - struct aes_block *out = (struct aes_block *)dst; - struct aes_block const *in = (struct aes_block *)src; - void *dummy0; - int dummy1; if (!may_use_simd()) { __aes_arm64_decrypt(ctx->key_dec, dst, src, num_rounds(ctx)); @@ -105,62 +72,10 @@ static void aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[]) } kernel_neon_begin(); - - __asm__(" ld1 {v0.16b}, %[in] ;" - " ld1 {v1.4s}, [%[key]], #16 ;" - " cmp %w[rounds], #10 ;" - " bmi 0f ;" - " bne 3f ;" - " mov v3.16b, v1.16b ;" - " b 2f ;" - "0: mov v2.16b, v1.16b ;" - " ld1 {v3.4s}, [%[key]], #16 ;" - "1: aesd v0.16b, v2.16b ;" - " aesimc v0.16b, v0.16b ;" - "2: ld1 {v1.4s}, [%[key]], #16 ;" - " aesd v0.16b, v3.16b ;" - " aesimc v0.16b, v0.16b ;" - "3: ld1 {v2.4s}, [%[key]], #16 ;" - " subs %w[rounds], %w[rounds], #3 ;" - " aesd v0.16b, v1.16b ;" - " aesimc v0.16b, v0.16b ;" - " ld1 {v3.4s}, [%[key]], #16 ;" - " bpl 1b ;" - " aesd v0.16b, v2.16b ;" - " eor v0.16b, v0.16b, v3.16b ;" - " st1 {v0.16b}, %[out] ;" - - : [out] "=Q"(*out), - [key] "=r"(dummy0), - [rounds] "=r"(dummy1) - : [in] "Q"(*in), - "1"(ctx->key_dec), - "2"(num_rounds(ctx) - 2) - : "cc"); - + __aes_ce_decrypt(ctx->key_dec, dst, src, num_rounds(ctx)); kernel_neon_end(); } -/* - * aes_sub() - use the aese instruction to perform the AES sbox substitution - * on each byte in 'input' - */ -static u32 aes_sub(u32 input) -{ - u32 ret; - - __asm__("dup v1.4s, %w[in] ;" - "movi v0.16b, #0 ;" - "aese v0.16b, v1.16b ;" - "umov %w[out], v0.4s[0] ;" - - : [out] "=r"(ret) - : [in] "r"(input) - : "v0","v1"); - - return ret; -} - int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, unsigned int key_len) { @@ -189,7 +104,7 @@ int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, u32 *rki = ctx->key_enc + (i * kwords); u32 *rko = rki + kwords; - rko[0] = ror32(aes_sub(rki[kwords - 1]), 8) ^ rcon[i] ^ rki[0]; + rko[0] = ror32(__aes_ce_sub(rki[kwords - 1]), 8) ^ rcon[i] ^ rki[0]; rko[1] = rko[0] ^ rki[1]; rko[2] = rko[1] ^ rki[2]; rko[3] = rko[2] ^ rki[3]; @@ -202,7 +117,7 @@ int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, } else if (key_len == AES_KEYSIZE_256) { if (i >= 6) break; - rko[4] = aes_sub(rko[3]) ^ rki[4]; + rko[4] = __aes_ce_sub(rko[3]) ^ rki[4]; rko[5] = rko[4] ^ rki[5]; rko[6] = rko[5] ^ rki[6]; rko[7] = rko[6] ^ rki[7]; @@ -221,13 +136,7 @@ int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, key_dec[0] = key_enc[j]; for (i = 1, j--; j > 0; i++, j--) - __asm__("ld1 {v0.4s}, %[in] ;" - "aesimc v1.16b, v0.16b ;" - "st1 {v1.4s}, %[out] ;" - - : [out] "=Q"(key_dec[i]) - : [in] "Q"(key_enc[j]) - : "v0","v1"); + __aes_ce_invert(key_dec + i, key_enc + j); key_dec[i] = key_enc[0]; kernel_neon_end(); From c3a536056374ef8412c7ffa13790303ec02d8f46 Mon Sep 17 00:00:00 2001 From: Jon Maxwell Date: Wed, 22 Nov 2017 16:08:17 +1100 Subject: [PATCH 031/166] crypto: cryptd - Add cryptd_max_cpu_qlen module parameter Make the cryptd queue length configurable. We recently had customer where this needed to be tuned to accommodate the aesni_intel module and prevent packet drop. Signed-off-by: Jon Maxwell Signed-off-by: Herbert Xu --- crypto/cryptd.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crypto/cryptd.c b/crypto/cryptd.c index bd43cf5be14c..b1eb131c0f10 100644 --- a/crypto/cryptd.c +++ b/crypto/cryptd.c @@ -32,7 +32,9 @@ #include #include -#define CRYPTD_MAX_CPU_QLEN 1000 +unsigned int cryptd_max_cpu_qlen = 1000; +module_param(cryptd_max_cpu_qlen, uint, 0); +MODULE_PARM_DESC(cryptd_max_cpu_qlen, "Set cryptd Max queue depth"); struct cryptd_cpu_queue { struct crypto_queue queue; @@ -116,6 +118,7 @@ static int cryptd_init_queue(struct cryptd_queue *queue, crypto_init_queue(&cpu_queue->queue, max_cpu_qlen); INIT_WORK(&cpu_queue->work, cryptd_queue_worker); } + pr_info("cryptd: max_cpu_qlen set to %d\n", max_cpu_qlen); return 0; } @@ -1372,7 +1375,7 @@ static int __init cryptd_init(void) { int err; - err = cryptd_init_queue(&queue, CRYPTD_MAX_CPU_QLEN); + err = cryptd_init_queue(&queue, cryptd_max_cpu_qlen); if (err) return err; From ecf3220d882ae84844909ed6323032aac47aff93 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 22 Nov 2017 11:51:35 -0800 Subject: [PATCH 032/166] crypto: chacha20 - Fix unaligned access when loading constants The four 32-bit constants for the initial state of ChaCha20 were loaded from a char array which is not guaranteed to have the needed alignment. Fix it by just assigning the constants directly instead. Signed-off-by: Eric Biggers Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/chacha20_generic.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/crypto/chacha20_generic.c b/crypto/chacha20_generic.c index 4a45fa4890c0..ec84e7837aac 100644 --- a/crypto/chacha20_generic.c +++ b/crypto/chacha20_generic.c @@ -41,12 +41,10 @@ static void chacha20_docrypt(u32 *state, u8 *dst, const u8 *src, void crypto_chacha20_init(u32 *state, struct chacha20_ctx *ctx, u8 *iv) { - static const char constant[16] = "expand 32-byte k"; - - state[0] = le32_to_cpuvp(constant + 0); - state[1] = le32_to_cpuvp(constant + 4); - state[2] = le32_to_cpuvp(constant + 8); - state[3] = le32_to_cpuvp(constant + 12); + state[0] = 0x61707865; /* "expa" */ + state[1] = 0x3320646e; /* "nd 3" */ + state[2] = 0x79622d32; /* "2-by" */ + state[3] = 0x6b206574; /* "te k" */ state[4] = ctx->key[0]; state[5] = ctx->key[1]; state[6] = ctx->key[2]; From dbd872a123fab81d0fa235a265c39e5ccdf735b3 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 22 Nov 2017 11:51:36 -0800 Subject: [PATCH 033/166] crypto: chacha20 - Use unaligned access macros when loading key and IV The generic ChaCha20 implementation has a cra_alignmask of 3, which ensures that the key passed into crypto_chacha20_setkey() and the IV passed into crypto_chacha20_init() are 4-byte aligned. However, these functions are also called from the ARM and ARM64 implementations of ChaCha20, which intentionally do not have a cra_alignmask set. This is broken because 32-bit words are being loaded from potentially-unaligned buffers without the unaligned access macros. Fix it by using the unaligned access macros when loading the key and IV. Signed-off-by: Eric Biggers Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/chacha20_generic.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/crypto/chacha20_generic.c b/crypto/chacha20_generic.c index ec84e7837aac..b5a10ebf1b82 100644 --- a/crypto/chacha20_generic.c +++ b/crypto/chacha20_generic.c @@ -9,16 +9,12 @@ * (at your option) any later version. */ +#include #include #include #include #include -static inline u32 le32_to_cpuvp(const void *p) -{ - return le32_to_cpup(p); -} - static void chacha20_docrypt(u32 *state, u8 *dst, const u8 *src, unsigned int bytes) { @@ -53,10 +49,10 @@ void crypto_chacha20_init(u32 *state, struct chacha20_ctx *ctx, u8 *iv) state[9] = ctx->key[5]; state[10] = ctx->key[6]; state[11] = ctx->key[7]; - state[12] = le32_to_cpuvp(iv + 0); - state[13] = le32_to_cpuvp(iv + 4); - state[14] = le32_to_cpuvp(iv + 8); - state[15] = le32_to_cpuvp(iv + 12); + state[12] = get_unaligned_le32(iv + 0); + state[13] = get_unaligned_le32(iv + 4); + state[14] = get_unaligned_le32(iv + 8); + state[15] = get_unaligned_le32(iv + 12); } EXPORT_SYMBOL_GPL(crypto_chacha20_init); @@ -70,7 +66,7 @@ int crypto_chacha20_setkey(struct crypto_skcipher *tfm, const u8 *key, return -EINVAL; for (i = 0; i < ARRAY_SIZE(ctx->key); i++) - ctx->key[i] = le32_to_cpuvp(key + i * sizeof(u32)); + ctx->key[i] = get_unaligned_le32(key + i * sizeof(u32)); return 0; } From a1c73383c032ea467c4e766d4471edf91cf384cb Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 22 Nov 2017 11:51:37 -0800 Subject: [PATCH 034/166] crypto: chacha20 - Remove cra_alignmask Now that crypto_chacha20_setkey() and crypto_chacha20_init() use the unaligned access macros and crypto_xor() also accepts unaligned buffers, there is no need to have a cra_alignmask set for chacha20-generic. Signed-off-by: Eric Biggers Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/chacha20_generic.c | 1 - 1 file changed, 1 deletion(-) diff --git a/crypto/chacha20_generic.c b/crypto/chacha20_generic.c index b5a10ebf1b82..bb4affbd591c 100644 --- a/crypto/chacha20_generic.c +++ b/crypto/chacha20_generic.c @@ -105,7 +105,6 @@ static struct skcipher_alg alg = { .base.cra_priority = 100, .base.cra_blocksize = 1, .base.cra_ctxsize = sizeof(struct chacha20_ctx), - .base.cra_alignmask = sizeof(u32) - 1, .base.cra_module = THIS_MODULE, .min_keysize = CHACHA20_KEY_SIZE, From 796c99fbd7e20a8d78189fc0166a524d78f635a0 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 22 Nov 2017 11:51:38 -0800 Subject: [PATCH 035/166] crypto: x86/chacha20 - Remove cra_alignmask Now that the generic ChaCha20 implementation no longer needs a cra_alignmask, the x86 one doesn't either -- given that the x86 implementation doesn't need the alignment itself. Signed-off-by: Eric Biggers Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/x86/crypto/chacha20_glue.c | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/x86/crypto/chacha20_glue.c b/arch/x86/crypto/chacha20_glue.c index 1e6af1b35f7b..dce7c5d39c2f 100644 --- a/arch/x86/crypto/chacha20_glue.c +++ b/arch/x86/crypto/chacha20_glue.c @@ -107,7 +107,6 @@ static struct skcipher_alg alg = { .base.cra_priority = 300, .base.cra_blocksize = 1, .base.cra_ctxsize = sizeof(struct chacha20_ctx), - .base.cra_alignmask = sizeof(u32) - 1, .base.cra_module = THIS_MODULE, .min_keysize = CHACHA20_KEY_SIZE, From 9f480faec58cd6197a007ea1dcac6b7c3daf1139 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 22 Nov 2017 11:51:39 -0800 Subject: [PATCH 036/166] crypto: chacha20 - Fix keystream alignment for chacha20_block() When chacha20_block() outputs the keystream block, it uses 'u32' stores directly. However, the callers (crypto/chacha20_generic.c and drivers/char/random.c) declare the keystream buffer as a 'u8' array, which is not guaranteed to have the needed alignment. Fix it by having both callers declare the keystream as a 'u32' array. For now this is preferable to switching over to the unaligned access macros because chacha20_block() is only being used in cases where we can easily control the alignment (stack buffers). Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/chacha20_generic.c | 6 +++--- drivers/char/random.c | 24 ++++++++++++------------ include/crypto/chacha20.h | 3 ++- lib/chacha20.c | 2 +- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/crypto/chacha20_generic.c b/crypto/chacha20_generic.c index bb4affbd591c..e451c3cb6a56 100644 --- a/crypto/chacha20_generic.c +++ b/crypto/chacha20_generic.c @@ -18,20 +18,20 @@ static void chacha20_docrypt(u32 *state, u8 *dst, const u8 *src, unsigned int bytes) { - u8 stream[CHACHA20_BLOCK_SIZE]; + u32 stream[CHACHA20_BLOCK_WORDS]; if (dst != src) memcpy(dst, src, bytes); while (bytes >= CHACHA20_BLOCK_SIZE) { chacha20_block(state, stream); - crypto_xor(dst, stream, CHACHA20_BLOCK_SIZE); + crypto_xor(dst, (const u8 *)stream, CHACHA20_BLOCK_SIZE); bytes -= CHACHA20_BLOCK_SIZE; dst += CHACHA20_BLOCK_SIZE; } if (bytes) { chacha20_block(state, stream); - crypto_xor(dst, stream, bytes); + crypto_xor(dst, (const u8 *)stream, bytes); } } diff --git a/drivers/char/random.c b/drivers/char/random.c index ec42c8bb9b0d..11304bbc78cc 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -431,9 +431,9 @@ static int crng_init = 0; static int crng_init_cnt = 0; #define CRNG_INIT_CNT_THRESH (2*CHACHA20_KEY_SIZE) static void _extract_crng(struct crng_state *crng, - __u8 out[CHACHA20_BLOCK_SIZE]); + __u32 out[CHACHA20_BLOCK_WORDS]); static void _crng_backtrack_protect(struct crng_state *crng, - __u8 tmp[CHACHA20_BLOCK_SIZE], int used); + __u32 tmp[CHACHA20_BLOCK_WORDS], int used); static void process_random_ready_list(void); static void _get_random_bytes(void *buf, int nbytes); @@ -817,7 +817,7 @@ static void crng_reseed(struct crng_state *crng, struct entropy_store *r) unsigned long flags; int i, num; union { - __u8 block[CHACHA20_BLOCK_SIZE]; + __u32 block[CHACHA20_BLOCK_WORDS]; __u32 key[8]; } buf; @@ -851,7 +851,7 @@ static void crng_reseed(struct crng_state *crng, struct entropy_store *r) } static void _extract_crng(struct crng_state *crng, - __u8 out[CHACHA20_BLOCK_SIZE]) + __u32 out[CHACHA20_BLOCK_WORDS]) { unsigned long v, flags; @@ -867,7 +867,7 @@ static void _extract_crng(struct crng_state *crng, spin_unlock_irqrestore(&crng->lock, flags); } -static void extract_crng(__u8 out[CHACHA20_BLOCK_SIZE]) +static void extract_crng(__u32 out[CHACHA20_BLOCK_WORDS]) { struct crng_state *crng = NULL; @@ -885,7 +885,7 @@ static void extract_crng(__u8 out[CHACHA20_BLOCK_SIZE]) * enough) to mutate the CRNG key to provide backtracking protection. */ static void _crng_backtrack_protect(struct crng_state *crng, - __u8 tmp[CHACHA20_BLOCK_SIZE], int used) + __u32 tmp[CHACHA20_BLOCK_WORDS], int used) { unsigned long flags; __u32 *s, *d; @@ -897,14 +897,14 @@ static void _crng_backtrack_protect(struct crng_state *crng, used = 0; } spin_lock_irqsave(&crng->lock, flags); - s = (__u32 *) &tmp[used]; + s = &tmp[used / sizeof(__u32)]; d = &crng->state[4]; for (i=0; i < 8; i++) *d++ ^= *s++; spin_unlock_irqrestore(&crng->lock, flags); } -static void crng_backtrack_protect(__u8 tmp[CHACHA20_BLOCK_SIZE], int used) +static void crng_backtrack_protect(__u32 tmp[CHACHA20_BLOCK_WORDS], int used) { struct crng_state *crng = NULL; @@ -920,7 +920,7 @@ static void crng_backtrack_protect(__u8 tmp[CHACHA20_BLOCK_SIZE], int used) static ssize_t extract_crng_user(void __user *buf, size_t nbytes) { ssize_t ret = 0, i = CHACHA20_BLOCK_SIZE; - __u8 tmp[CHACHA20_BLOCK_SIZE]; + __u32 tmp[CHACHA20_BLOCK_WORDS]; int large_request = (nbytes > 256); while (nbytes) { @@ -1507,7 +1507,7 @@ static void _warn_unseeded_randomness(const char *func_name, void *caller, */ static void _get_random_bytes(void *buf, int nbytes) { - __u8 tmp[CHACHA20_BLOCK_SIZE]; + __u32 tmp[CHACHA20_BLOCK_WORDS]; trace_get_random_bytes(nbytes, _RET_IP_); @@ -2114,7 +2114,7 @@ u64 get_random_u64(void) if (use_lock) read_lock_irqsave(&batched_entropy_reset_lock, flags); if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0) { - extract_crng((u8 *)batch->entropy_u64); + extract_crng((__u32 *)batch->entropy_u64); batch->position = 0; } ret = batch->entropy_u64[batch->position++]; @@ -2144,7 +2144,7 @@ u32 get_random_u32(void) if (use_lock) read_lock_irqsave(&batched_entropy_reset_lock, flags); if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0) { - extract_crng((u8 *)batch->entropy_u32); + extract_crng(batch->entropy_u32); batch->position = 0; } ret = batch->entropy_u32[batch->position++]; diff --git a/include/crypto/chacha20.h b/include/crypto/chacha20.h index caaa470389e0..b83d66073db0 100644 --- a/include/crypto/chacha20.h +++ b/include/crypto/chacha20.h @@ -13,12 +13,13 @@ #define CHACHA20_IV_SIZE 16 #define CHACHA20_KEY_SIZE 32 #define CHACHA20_BLOCK_SIZE 64 +#define CHACHA20_BLOCK_WORDS (CHACHA20_BLOCK_SIZE / sizeof(u32)) struct chacha20_ctx { u32 key[8]; }; -void chacha20_block(u32 *state, void *stream); +void chacha20_block(u32 *state, u32 *stream); void crypto_chacha20_init(u32 *state, struct chacha20_ctx *ctx, u8 *iv); int crypto_chacha20_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keysize); diff --git a/lib/chacha20.c b/lib/chacha20.c index 250ceed9ec9a..29d3801dee24 100644 --- a/lib/chacha20.c +++ b/lib/chacha20.c @@ -21,7 +21,7 @@ static inline u32 rotl32(u32 v, u8 n) return (v << n) | (v >> (sizeof(v) * 8 - n)); } -extern void chacha20_block(u32 *state, void *stream) +void chacha20_block(u32 *state, u32 *stream) { u32 x[16], *out = stream; int i; From 45223b78118cbab12744a7e193b6c7c208c15c27 Mon Sep 17 00:00:00 2001 From: Corentin Labbe Date: Fri, 24 Nov 2017 08:22:07 +0000 Subject: [PATCH 037/166] crypto: arm64/aes - do not call crypto_unregister_skcipher twice on error When a cipher fails to register in aes_init(), the error path goes thought aes_exit() then crypto_unregister_skciphers(). Since aes_exit calls also crypto_unregister_skcipher, this triggers a refcount_t: underflow; use-after-free. Signed-off-by: Corentin Labbe Signed-off-by: Herbert Xu --- arch/arm64/crypto/aes-glue.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c index 998ba519a026..2fa850e86aa8 100644 --- a/arch/arm64/crypto/aes-glue.c +++ b/arch/arm64/crypto/aes-glue.c @@ -665,6 +665,7 @@ static int __init aes_init(void) unregister_simds: aes_exit(); + return err; unregister_ciphers: crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); return err; From b5b9007730ce1d90deaf25d7f678511550744bdc Mon Sep 17 00:00:00 2001 From: Hauke Mehrtens Date: Sun, 26 Nov 2017 00:16:46 +0100 Subject: [PATCH 038/166] crypto: ecdh - fix typo in KPP dependency of CRYPTO_ECDH This fixes a typo in the CRYPTO_KPP dependency of CRYPTO_ECDH. Fixes: 3c4b23901a0c ("crypto: ecdh - Add ECDH software support") Cc: # v4.8+ Signed-off-by: Hauke Mehrtens Signed-off-by: Herbert Xu --- crypto/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index f7911963bb79..9327fbfccf5a 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -130,7 +130,7 @@ config CRYPTO_DH config CRYPTO_ECDH tristate "ECDH algorithm" - select CRYTPO_KPP + select CRYPTO_KPP select CRYPTO_RNG_DEFAULT help Generic implementation of the ECDH algorithm From 26d85e5f3b68f8526b516959a7dd4d1fbab00fd0 Mon Sep 17 00:00:00 2001 From: "Gomonovych, Vasyl" Date: Tue, 28 Nov 2017 00:06:06 +0100 Subject: [PATCH 039/166] crypto: arm/aes-neonbs - Use PTR_ERR_OR_ZERO() Fix ptr_ret.cocci warnings: arch/arm/crypto/aes-neonbs-glue.c:184:1-3: WARNING: PTR_ERR_OR_ZERO can be used arch/arm/crypto/aes-neonbs-glue.c:261:1-3: WARNING: PTR_ERR_OR_ZERO can be used Use PTR_ERR_OR_ZERO rather than if(IS_ERR(...)) + PTR_ERR Generated by: scripts/coccinelle/api/ptr_ret.cocci Signed-off-by: Vasyl Gomonovych Signed-off-by: Herbert Xu --- arch/arm/crypto/aes-neonbs-glue.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/arch/arm/crypto/aes-neonbs-glue.c b/arch/arm/crypto/aes-neonbs-glue.c index 18768f330449..07e31941dc67 100644 --- a/arch/arm/crypto/aes-neonbs-glue.c +++ b/arch/arm/crypto/aes-neonbs-glue.c @@ -181,9 +181,8 @@ static int cbc_init(struct crypto_tfm *tfm) struct aesbs_cbc_ctx *ctx = crypto_tfm_ctx(tfm); ctx->enc_tfm = crypto_alloc_cipher("aes", 0, 0); - if (IS_ERR(ctx->enc_tfm)) - return PTR_ERR(ctx->enc_tfm); - return 0; + + return PTR_ERR_OR_ZERO(ctx->enc_tfm); } static void cbc_exit(struct crypto_tfm *tfm) @@ -258,9 +257,8 @@ static int xts_init(struct crypto_tfm *tfm) struct aesbs_xts_ctx *ctx = crypto_tfm_ctx(tfm); ctx->tweak_tfm = crypto_alloc_cipher("aes", 0, 0); - if (IS_ERR(ctx->tweak_tfm)) - return PTR_ERR(ctx->tweak_tfm); - return 0; + + return PTR_ERR_OR_ZERO(ctx->tweak_tfm); } static void xts_exit(struct crypto_tfm *tfm) From 9db09e3bad65393dc23b0279beb7f3754d77065c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Tue, 28 Nov 2017 18:48:08 +0200 Subject: [PATCH 040/166] crypto: caam/qi - use correct print specifier for size_t MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix below warnings on ARMv7 by using %zu for printing size_t values: drivers/crypto/caam/caamalg_qi.c: In function aead_edesc_alloc: drivers/crypto/caam/caamalg_qi.c:417:17: warning: format %lu expects argument of type long unsigned int, but argument 4 has type unsigned int [-Wformat=] sizeof(struct qm_sg_entry)) ^ drivers/crypto/caam/caamalg_qi.c:672:16: note: in expansion of macro CAAM_QI_MAX_AEAD_SG qm_sg_ents, CAAM_QI_MAX_AEAD_SG); ^ drivers/crypto/caam/caamalg_qi.c: In function ablkcipher_edesc_alloc: drivers/crypto/caam/caamalg_qi.c:440:17: warning: format %lu expects argument of type long unsigned int, but argument 4 has type unsigned int [-Wformat=] sizeof(struct qm_sg_entry)) ^ drivers/crypto/caam/caamalg_qi.c:909:16: note: in expansion of macro CAAM_QI_MAX_ABLKCIPHER_SG qm_sg_ents, CAAM_QI_MAX_ABLKCIPHER_SG); ^ drivers/crypto/caam/caamalg_qi.c: In function ablkcipher_giv_edesc_alloc: drivers/crypto/caam/caamalg_qi.c:440:17: warning: format %lu expects argument of type long unsigned int, but argument 4 has type unsigned int [-Wformat=] sizeof(struct qm_sg_entry)) ^ drivers/crypto/caam/caamalg_qi.c:1062:16: note: in expansion of macro CAAM_QI_MAX_ABLKCIPHER_SG qm_sg_ents, CAAM_QI_MAX_ABLKCIPHER_SG); ^ Fixes: eb9ba37dc15a ("crypto: caam/qi - handle large number of S/Gs case") Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg_qi.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index f9f08fce4356..ad14b69a052e 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -668,7 +668,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, qm_sg_ents = 1 + !!ivsize + mapped_src_nents + (mapped_dst_nents > 1 ? mapped_dst_nents : 0); if (unlikely(qm_sg_ents > CAAM_QI_MAX_AEAD_SG)) { - dev_err(qidev, "Insufficient S/G entries: %d > %lu\n", + dev_err(qidev, "Insufficient S/G entries: %d > %zu\n", qm_sg_ents, CAAM_QI_MAX_AEAD_SG); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, iv_dma, ivsize, op_type, 0, 0); @@ -905,7 +905,7 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; if (unlikely(qm_sg_ents > CAAM_QI_MAX_ABLKCIPHER_SG)) { - dev_err(qidev, "Insufficient S/G entries: %d > %lu\n", + dev_err(qidev, "Insufficient S/G entries: %d > %zu\n", qm_sg_ents, CAAM_QI_MAX_ABLKCIPHER_SG); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, iv_dma, ivsize, op_type, 0, 0); @@ -1058,7 +1058,7 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc( } if (unlikely(qm_sg_ents > CAAM_QI_MAX_ABLKCIPHER_SG)) { - dev_err(qidev, "Insufficient S/G entries: %d > %lu\n", + dev_err(qidev, "Insufficient S/G entries: %d > %zu\n", qm_sg_ents, CAAM_QI_MAX_ABLKCIPHER_SG); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, iv_dma, ivsize, GIVENCRYPT, 0, 0); From 761a6982198cf117686027377184e1513cba7e45 Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Thu, 30 Nov 2017 12:04:33 +0100 Subject: [PATCH 041/166] crypto: stm32 - fix module device table name This patch fix the following build failure: CC [M] drivers/crypto/stm32/stm32-cryp.o In file included from drivers/crypto/stm32/stm32-cryp.c:11:0: drivers/crypto/stm32/stm32-cryp.c:1049:25: error: 'sti_dt_ids' undeclared here (not in a function) MODULE_DEVICE_TABLE(of, sti_dt_ids); Let's replace sti_dt_ids with stm32_dt_ids which is just declared before. Signed-off-by: Corentin Labbe Reviewed-by: Fabien Dessenne Signed-off-by: Herbert Xu --- drivers/crypto/stm32/stm32-cryp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c index 459755940632..cf1dddbeaa2c 100644 --- a/drivers/crypto/stm32/stm32-cryp.c +++ b/drivers/crypto/stm32/stm32-cryp.c @@ -1032,7 +1032,7 @@ static const struct of_device_id stm32_dt_ids[] = { { .compatible = "st,stm32f756-cryp", }, {}, }; -MODULE_DEVICE_TABLE(of, sti_dt_ids); +MODULE_DEVICE_TABLE(of, stm32_dt_ids); static int stm32_cryp_probe(struct platform_device *pdev) { From eaf356e4be27abcc3b741a3dd7e9bd530390c607 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Thu, 30 Nov 2017 11:26:14 +0000 Subject: [PATCH 042/166] crypto: cryptd - make cryptd_max_cpu_qlen module parameter static The cryptd_max_cpu_qlen module parameter is local to the source and does not need to be in global scope, so make it static. Cleans up sparse warning: crypto/cryptd.c:35:14: warning: symbol 'cryptd_max_cpu_qlen' was not declared. Should it be static? Signed-off-by: Colin Ian King Signed-off-by: Herbert Xu --- crypto/cryptd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/cryptd.c b/crypto/cryptd.c index b1eb131c0f10..552e3a86e829 100644 --- a/crypto/cryptd.c +++ b/crypto/cryptd.c @@ -32,7 +32,7 @@ #include #include -unsigned int cryptd_max_cpu_qlen = 1000; +static unsigned int cryptd_max_cpu_qlen = 1000; module_param(cryptd_max_cpu_qlen, uint, 0); MODULE_PARM_DESC(cryptd_max_cpu_qlen, "Set cryptd Max queue depth"); From 8579e0767c230f25f8abc71d4e99eba6754d3766 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Thu, 30 Nov 2017 11:32:08 +0000 Subject: [PATCH 043/166] crypto: chelsio - make arrays sgl_ent_len and dsgl_ent_len static The arrays sgl_ent_len and dsgl_ent_len are local to the source and do not need to be in global scope, so make them static. Also re-format the declarations to match the following round_constant array declaration style. Cleans up sparse warnings: drivers/crypto/chelsio/chcr_algo.c:76:14: warning: symbol 'sgl_ent_len' was not declared. Should it be static? drivers/crypto/chelsio/chcr_algo.c:81:14: warning: symbol 'dsgl_ent_len' was not declared. Should it be static? Signed-off-by: Colin Ian King Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index e61ec8a46340..b663b93b7d01 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -73,15 +73,19 @@ #define IV AES_BLOCK_SIZE -unsigned int sgl_ent_len[] = {0, 0, 16, 24, 40, 48, 64, 72, 88, - 96, 112, 120, 136, 144, 160, 168, 184, - 192, 208, 216, 232, 240, 256, 264, 280, - 288, 304, 312, 328, 336, 352, 360, 376}; +static unsigned int sgl_ent_len[] = { + 0, 0, 16, 24, 40, 48, 64, 72, 88, + 96, 112, 120, 136, 144, 160, 168, 184, + 192, 208, 216, 232, 240, 256, 264, 280, + 288, 304, 312, 328, 336, 352, 360, 376 +}; -unsigned int dsgl_ent_len[] = {0, 32, 32, 48, 48, 64, 64, 80, 80, - 112, 112, 128, 128, 144, 144, 160, 160, - 192, 192, 208, 208, 224, 224, 240, 240, - 272, 272, 288, 288, 304, 304, 320, 320}; +static unsigned int dsgl_ent_len[] = { + 0, 32, 32, 48, 48, 64, 64, 80, 80, + 112, 112, 128, 128, 144, 144, 160, 160, + 192, 192, 208, 208, 224, 224, 240, 240, + 272, 272, 288, 288, 304, 304, 320, 320 +}; static u32 round_constant[11] = { 0x01000000, 0x02000000, 0x04000000, 0x08000000, From 267469ea65fd2ef5412177b3269811adfde5a29d Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Thu, 30 Nov 2017 14:11:44 +0000 Subject: [PATCH 044/166] chcr: remove unused variables net_device, pi, adap and cntrl Variables adap, pi and cntrl are assigned but are never read, hence they are redundant and can be removed. Cleans up various clang build warnings. Signed-off-by: Colin Ian King Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_ipsec.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_ipsec.c b/drivers/crypto/chelsio/chcr_ipsec.c index f90f991c133f..a0f003ffd1f4 100644 --- a/drivers/crypto/chelsio/chcr_ipsec.c +++ b/drivers/crypto/chelsio/chcr_ipsec.c @@ -199,14 +199,9 @@ out: */ static int chcr_xfrm_add_state(struct xfrm_state *x) { - struct net_device *netdev = x->xso.dev; - struct port_info *pi = netdev_priv(netdev); struct ipsec_sa_entry *sa_entry; - struct adapter *adap; int res = 0; - adap = pi->adapter; - if (x->props.aalgo != SADB_AALG_NONE) { pr_debug("CHCR: Cannot offload authenticated xfrm states\n"); return -EINVAL; @@ -570,7 +565,7 @@ int chcr_ipsec_xmit(struct sk_buff *skb, struct net_device *dev) { struct xfrm_state *x = xfrm_input_state(skb); struct ipsec_sa_entry *sa_entry; - u64 *pos, *end, *before, cntrl, *sgl; + u64 *pos, *end, *before, *sgl; int qidx, left, credits; unsigned int flits = 0, ndesc, kctx_len; struct adapter *adap; @@ -596,7 +591,6 @@ out_free: dev_kfree_skb_any(skb); q = &adap->sge.ethtxq[qidx + pi->first_qset]; cxgb4_reclaim_completed_tx(adap, &q->q, true); - cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F; flits = calc_tx_sec_flits(skb, sa_entry->kctx_len); ndesc = flits_to_desc(flits); From df807a19957c2917fd5a4f5812e240a250066742 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Thu, 30 Nov 2017 14:23:54 +0000 Subject: [PATCH 045/166] chcr: ensure cntrl is initialized to fix bit-wise or'ing of garabage data In the case where skb->ip_summed != CHECKSUM_PARTIAL then cntrl contains garbage value and this is possibly being bit-wise or'd and stored into cpl->ctrl1. Fix this by initializing cntrl to zero. Cleans up clang warning: drivers/crypto/chelsio/chcr_ipsec.c:374:9: warning: The left expression of the compound assignment is an uninitialized value. The computed value will also be garbage Fixes: 6dad4e8ab3ec ("chcr: Add support for Inline IPSec") Signed-off-by: Colin Ian King Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_ipsec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/chelsio/chcr_ipsec.c b/drivers/crypto/chelsio/chcr_ipsec.c index a0f003ffd1f4..64a030f8fb21 100644 --- a/drivers/crypto/chelsio/chcr_ipsec.c +++ b/drivers/crypto/chelsio/chcr_ipsec.c @@ -350,7 +350,7 @@ inline void *copy_cpltx_pktxt(struct sk_buff *skb, struct port_info *pi; struct sge_eth_txq *q; struct cpl_tx_pkt_core *cpl; - u64 cntrl; + u64 cntrl = 0; u32 ctrl0, qidx; pi = netdev_priv(dev); From 65cd9588c597753ab2e567af6e5e32e3f6645e5d Mon Sep 17 00:00:00 2001 From: Kamil Konieczny Date: Fri, 1 Dec 2017 17:51:02 +0100 Subject: [PATCH 046/166] MAINTAINERS: Add myself as co-maintainer for s5p-sss.c Add myself as co-maintainer for Samsung Security SubSystem driver. I have added major functionality to the driver [hash acceleration], I have access to documentation and to hardware for testing, I can also dedicate some of my paid time for reviewing and verifying changes to the driver. Signed-off-by: Kamil Konieczny Signed-off-by: Herbert Xu --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index aa71ab52fd76..3f6cadf2e087 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -11982,6 +11982,7 @@ F: drivers/media/i2c/s5k5baf.c SAMSUNG S5P Security SubSystem (SSS) DRIVER M: Krzysztof Kozlowski M: Vladimir Zapolskiy +M: Kamil Konieczny L: linux-crypto@vger.kernel.org L: linux-samsung-soc@vger.kernel.org S: Maintained From 144f3d18a2300c6a3ab83a2e257699a10404aeb4 Mon Sep 17 00:00:00 2001 From: Pravin Shedge Date: Tue, 5 Dec 2017 07:27:22 +0530 Subject: [PATCH 047/166] crypto: drivers - remove duplicate includes These duplicate includes have been found with scripts/checkincludes.pl but they have been removed manually to avoid removing false positives. Signed-off-by: Pravin Shedge Signed-off-by: Herbert Xu --- drivers/crypto/bcm/cipher.c | 1 - drivers/crypto/cavium/nitrox/nitrox_reqmgr.c | 1 - drivers/crypto/ccp/ccp-crypto-aes-galois.c | 1 - 3 files changed, 3 deletions(-) diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c index ce70b44d0fb6..2b75f95bbe1b 100644 --- a/drivers/crypto/bcm/cipher.c +++ b/drivers/crypto/bcm/cipher.c @@ -42,7 +42,6 @@ #include #include #include -#include #include #include "util.h" diff --git a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c index 4addc238a6ef..deaefd532aaa 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c +++ b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c @@ -6,7 +6,6 @@ #include "nitrox_dev.h" #include "nitrox_req.h" #include "nitrox_csr.h" -#include "nitrox_req.h" /* SLC_STORE_INFO */ #define MIN_UDD_LEN 16 diff --git a/drivers/crypto/ccp/ccp-crypto-aes-galois.c b/drivers/crypto/ccp/ccp-crypto-aes-galois.c index ff02b713c6f6..ca1f0d780b61 100644 --- a/drivers/crypto/ccp/ccp-crypto-aes-galois.c +++ b/drivers/crypto/ccp/ccp-crypto-aes-galois.c @@ -21,7 +21,6 @@ #include #include #include -#include #include "ccp-crypto.h" From 209c14bfb3b751a537f4af302730e8d9f24cd9cd Mon Sep 17 00:00:00 2001 From: Atul Gupta Date: Tue, 5 Dec 2017 15:59:22 +0530 Subject: [PATCH 048/166] crypto: chelsio - fix a type cast error fix a type cast error for queue descriptor Reported-by: Dan Carpenter Signed-off-by: Atul Gupta Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_ipsec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/chelsio/chcr_ipsec.c b/drivers/crypto/chelsio/chcr_ipsec.c index 64a030f8fb21..db1e241104ed 100644 --- a/drivers/crypto/chelsio/chcr_ipsec.c +++ b/drivers/crypto/chelsio/chcr_ipsec.c @@ -423,7 +423,7 @@ inline void *copy_key_cpltx_pktxt(struct sk_buff *skb, memcpy(pos, sa_entry->key, left); memcpy(q->q.desc, sa_entry->key + left, key_len - left); - pos = q->q.desc + (key_len - left); + pos = (u8 *)q->q.desc + (key_len - left); } } /* Copy CPL TX PKT XT */ From 3fc1264271c151e150b16026a25f531b3c2e79d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stelmach?= Date: Tue, 5 Dec 2017 17:20:46 +0100 Subject: [PATCH 049/166] crypto: exynos - Icrease the priority of the driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit exynos-rng is one of many implementations of stdrng. With priority as low as 100 it isn't selected, if software implementations (DRBG) are available. The value 300 was selected to give the PRNG priority before software implementations, but allow them to be selected in FIPS-mode (fips=1 in the kernel command line). Signed-off-by: Łukasz Stelmach Reviewed-by: Stephan Mueller Reviewed-by: Krzysztof Kozlowski Signed-off-by: Herbert Xu --- drivers/crypto/exynos-rng.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/exynos-rng.c b/drivers/crypto/exynos-rng.c index 451620b475a0..ed6ba796ad71 100644 --- a/drivers/crypto/exynos-rng.c +++ b/drivers/crypto/exynos-rng.c @@ -259,7 +259,7 @@ static struct rng_alg exynos_rng_alg = { .base = { .cra_name = "stdrng", .cra_driver_name = "exynos_rng", - .cra_priority = 100, + .cra_priority = 300, .cra_ctxsize = sizeof(struct exynos_rng_ctx), .cra_module = THIS_MODULE, .cra_init = exynos_rng_kcapi_init, From cadc9ab50350a24140612814109e826bb0e2ae37 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 7 Dec 2017 10:55:59 -0800 Subject: [PATCH 050/166] crypto: api - Unexport crypto_larval_lookup() crypto_larval_lookup() is not used outside of crypto/api.c, so unexport it and mark it 'static'. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/api.c | 4 ++-- crypto/internal.h | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/crypto/api.c b/crypto/api.c index 2a2479d168aa..6da802d7be67 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -205,7 +205,8 @@ struct crypto_alg *crypto_alg_lookup(const char *name, u32 type, u32 mask) } EXPORT_SYMBOL_GPL(crypto_alg_lookup); -struct crypto_alg *crypto_larval_lookup(const char *name, u32 type, u32 mask) +static struct crypto_alg *crypto_larval_lookup(const char *name, u32 type, + u32 mask) { struct crypto_alg *alg; @@ -231,7 +232,6 @@ struct crypto_alg *crypto_larval_lookup(const char *name, u32 type, u32 mask) return crypto_larval_add(name, type, mask); } -EXPORT_SYMBOL_GPL(crypto_larval_lookup); int crypto_probing_notify(unsigned long val, void *v) { diff --git a/crypto/internal.h b/crypto/internal.h index f07320423191..ae65e5fcaa59 100644 --- a/crypto/internal.h +++ b/crypto/internal.h @@ -78,7 +78,6 @@ int crypto_init_compress_ops(struct crypto_tfm *tfm); struct crypto_larval *crypto_larval_alloc(const char *name, u32 type, u32 mask); void crypto_larval_kill(struct crypto_alg *alg); -struct crypto_alg *crypto_larval_lookup(const char *name, u32 type, u32 mask); void crypto_alg_tested(const char *name, int err); void crypto_remove_spawns(struct crypto_alg *alg, struct list_head *list, From 3a2d4fb51e5a96ab2a3846f9fe876750c8d14391 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 7 Dec 2017 10:56:34 -0800 Subject: [PATCH 051/166] crypto: null - Get rid of crypto_{get,put}_default_null_skcipher2() Since commit 499a66e6b689 ("crypto: null - Remove default null blkcipher"), crypto_get_default_null_skcipher2() and crypto_put_default_null_skcipher2() are the same as their non-2 equivalents. So switch callers of the "2" versions over to the original versions and remove the "2" versions. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/aead.c | 6 +++--- crypto/algif_aead.c | 4 ++-- crypto/authenc.c | 4 ++-- crypto/authencesn.c | 4 ++-- crypto/gcm.c | 4 ++-- include/crypto/null.h | 10 ---------- 6 files changed, 11 insertions(+), 21 deletions(-) diff --git a/crypto/aead.c b/crypto/aead.c index f794b30a9407..fe00cbd7243d 100644 --- a/crypto/aead.c +++ b/crypto/aead.c @@ -295,7 +295,7 @@ int aead_init_geniv(struct crypto_aead *aead) if (err) goto out; - ctx->sknull = crypto_get_default_null_skcipher2(); + ctx->sknull = crypto_get_default_null_skcipher(); err = PTR_ERR(ctx->sknull); if (IS_ERR(ctx->sknull)) goto out; @@ -315,7 +315,7 @@ out: return err; drop_null: - crypto_put_default_null_skcipher2(); + crypto_put_default_null_skcipher(); goto out; } EXPORT_SYMBOL_GPL(aead_init_geniv); @@ -325,7 +325,7 @@ void aead_exit_geniv(struct crypto_aead *tfm) struct aead_geniv_ctx *ctx = crypto_aead_ctx(tfm); crypto_free_aead(ctx->child); - crypto_put_default_null_skcipher2(); + crypto_put_default_null_skcipher(); } EXPORT_SYMBOL_GPL(aead_exit_geniv); diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c index 9d73be28cf01..87a27eb19680 100644 --- a/crypto/algif_aead.c +++ b/crypto/algif_aead.c @@ -469,7 +469,7 @@ static void *aead_bind(const char *name, u32 type, u32 mask) return ERR_CAST(aead); } - null_tfm = crypto_get_default_null_skcipher2(); + null_tfm = crypto_get_default_null_skcipher(); if (IS_ERR(null_tfm)) { crypto_free_aead(aead); kfree(tfm); @@ -487,7 +487,7 @@ static void aead_release(void *private) struct aead_tfm *tfm = private; crypto_free_aead(tfm->aead); - crypto_put_default_null_skcipher2(); + crypto_put_default_null_skcipher(); kfree(tfm); } diff --git a/crypto/authenc.c b/crypto/authenc.c index 875470b0e026..d3d6d72fe649 100644 --- a/crypto/authenc.c +++ b/crypto/authenc.c @@ -329,7 +329,7 @@ static int crypto_authenc_init_tfm(struct crypto_aead *tfm) if (IS_ERR(enc)) goto err_free_ahash; - null = crypto_get_default_null_skcipher2(); + null = crypto_get_default_null_skcipher(); err = PTR_ERR(null); if (IS_ERR(null)) goto err_free_skcipher; @@ -363,7 +363,7 @@ static void crypto_authenc_exit_tfm(struct crypto_aead *tfm) crypto_free_ahash(ctx->auth); crypto_free_skcipher(ctx->enc); - crypto_put_default_null_skcipher2(); + crypto_put_default_null_skcipher(); } static void crypto_authenc_free(struct aead_instance *inst) diff --git a/crypto/authencesn.c b/crypto/authencesn.c index 0cf5fefdb859..15f91ddd7f0e 100644 --- a/crypto/authencesn.c +++ b/crypto/authencesn.c @@ -352,7 +352,7 @@ static int crypto_authenc_esn_init_tfm(struct crypto_aead *tfm) if (IS_ERR(enc)) goto err_free_ahash; - null = crypto_get_default_null_skcipher2(); + null = crypto_get_default_null_skcipher(); err = PTR_ERR(null); if (IS_ERR(null)) goto err_free_skcipher; @@ -389,7 +389,7 @@ static void crypto_authenc_esn_exit_tfm(struct crypto_aead *tfm) crypto_free_ahash(ctx->auth); crypto_free_skcipher(ctx->enc); - crypto_put_default_null_skcipher2(); + crypto_put_default_null_skcipher(); } static void crypto_authenc_esn_free(struct aead_instance *inst) diff --git a/crypto/gcm.c b/crypto/gcm.c index 8589681fb9f6..0ad879e1f9b2 100644 --- a/crypto/gcm.c +++ b/crypto/gcm.c @@ -1101,7 +1101,7 @@ static int crypto_rfc4543_init_tfm(struct crypto_aead *tfm) if (IS_ERR(aead)) return PTR_ERR(aead); - null = crypto_get_default_null_skcipher2(); + null = crypto_get_default_null_skcipher(); err = PTR_ERR(null); if (IS_ERR(null)) goto err_free_aead; @@ -1129,7 +1129,7 @@ static void crypto_rfc4543_exit_tfm(struct crypto_aead *tfm) struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm); crypto_free_aead(ctx->child); - crypto_put_default_null_skcipher2(); + crypto_put_default_null_skcipher(); } static void crypto_rfc4543_free(struct aead_instance *inst) diff --git a/include/crypto/null.h b/include/crypto/null.h index 5757c0a4b321..15aeef6e30ef 100644 --- a/include/crypto/null.h +++ b/include/crypto/null.h @@ -12,14 +12,4 @@ struct crypto_skcipher *crypto_get_default_null_skcipher(void); void crypto_put_default_null_skcipher(void); -static inline struct crypto_skcipher *crypto_get_default_null_skcipher2(void) -{ - return crypto_get_default_null_skcipher(); -} - -static inline void crypto_put_default_null_skcipher2(void) -{ - crypto_put_default_null_skcipher(); -} - #endif From 8c9478a400b7efdeab89797cfb77e4e845fb34fb Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 11 Dec 2017 13:07:00 +0100 Subject: [PATCH 052/166] crypto: qat - reduce stack size with KASAN Passing the register value by reference here leads a large amount of stack being used when CONFIG_KASAN is enabled: drivers/crypto/qat/qat_common/qat_hal.c: In function 'qat_hal_exec_micro_inst.constprop': drivers/crypto/qat/qat_common/qat_hal.c:963:1: error: the frame size of 1792 bytes is larger than 1536 bytes [-Werror=frame-larger-than=] Changing the register-read function to return the value instead reduces the stack size to around 800 bytes, most of which is for the 'savuwords' array. The function now no longer returns an error code, but nothing ever evaluated that anyway. Signed-off-by: Arnd Bergmann Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_hal.c | 133 ++++++++++++------------ 1 file changed, 67 insertions(+), 66 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 8c4fd255a601..ff149e176f64 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -117,19 +117,19 @@ void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle, #define CSR_RETRY_TIMES 500 static int qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned int csr, - unsigned int *value) + unsigned char ae, unsigned int csr) { unsigned int iterations = CSR_RETRY_TIMES; + int value; do { - *value = GET_AE_CSR(handle, ae, csr); + value = GET_AE_CSR(handle, ae, csr); if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS)) - return 0; + return value; } while (iterations--); pr_err("QAT: Read CSR timeout\n"); - return -EFAULT; + return 0; } static int qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle *handle, @@ -154,9 +154,9 @@ static void qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle *handle, { unsigned int cur_ctx; - qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx); + cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER); qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); - qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT, events); + *events = qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT); qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); } @@ -169,13 +169,13 @@ static int qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle, int times = MAX_RETRY_TIMES; int elapsed_cycles = 0; - qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, &base_cnt); + base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT); base_cnt &= 0xffff; while ((int)cycles > elapsed_cycles && times--) { if (chk_inactive) - qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &csr); + csr = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS); - qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, &cur_cnt); + cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT); cur_cnt &= 0xffff; elapsed_cycles = cur_cnt - base_cnt; @@ -207,7 +207,7 @@ int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle, } /* Sets the accelaration engine context mode to either four or eight */ - qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr); + csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); csr = IGNORE_W1C_MASK & csr; new_csr = (mode == 4) ? SET_BIT(csr, CE_INUSE_CONTEXTS_BITPOS) : @@ -221,7 +221,7 @@ int qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle, { unsigned int csr, new_csr; - qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr); + csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); csr &= IGNORE_W1C_MASK; new_csr = (mode) ? @@ -240,7 +240,7 @@ int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle, { unsigned int csr, new_csr; - qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr); + csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); csr &= IGNORE_W1C_MASK; switch (lm_type) { case ICP_LMEM0: @@ -328,7 +328,7 @@ static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle, { unsigned int ctx, cur_ctx; - qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx); + cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER); for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { if (!(ctx_mask & (1 << ctx))) @@ -340,16 +340,18 @@ static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle, qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); } -static void qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle, +static unsigned int qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned char ctx, - unsigned int ae_csr, unsigned int *csr_val) + unsigned int ae_csr) { - unsigned int cur_ctx; + unsigned int cur_ctx, csr_val; - qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx); + cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER); qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); - qat_hal_rd_ae_csr(handle, ae, ae_csr, csr_val); + csr_val = qat_hal_rd_ae_csr(handle, ae, ae_csr); qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); + + return csr_val; } static void qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle, @@ -358,7 +360,7 @@ static void qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle, { unsigned int ctx, cur_ctx; - qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx); + cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER); for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { if (!(ctx_mask & (1 << ctx))) continue; @@ -374,7 +376,7 @@ static void qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle, { unsigned int ctx, cur_ctx; - qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx); + cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER); for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { if (!(ctx_mask & (1 << ctx))) continue; @@ -392,13 +394,11 @@ static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle) int times = MAX_RETRY_TIMES; for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { - qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, - (unsigned int *)&base_cnt); + base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT); base_cnt &= 0xffff; do { - qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, - (unsigned int *)&cur_cnt); + cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT); cur_cnt &= 0xffff; } while (times-- && (cur_cnt == base_cnt)); @@ -416,8 +416,8 @@ int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle, { unsigned int enable = 0, active = 0; - qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &enable); - qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &active); + enable = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); + active = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS); if ((enable & (0xff << CE_ENABLE_BITPOS)) || (active & (1 << ACS_ABO_BITPOS))) return 1; @@ -540,7 +540,7 @@ static void qat_hal_disable_ctx(struct icp_qat_fw_loader_handle *handle, { unsigned int ctx; - qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx); + ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); ctx &= IGNORE_W1C_MASK & (~((ctx_mask & ICP_QAT_UCLO_AE_ALL_CTX) << CE_ENABLE_BITPOS)); qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx); @@ -583,7 +583,7 @@ void qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle, unsigned int ustore_addr; unsigned int i; - qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr); + ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS); uaddr |= UA_ECS; qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); for (i = 0; i < words_num; i++) { @@ -604,7 +604,7 @@ static void qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle, { unsigned int ctx; - qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx); + ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); ctx &= IGNORE_W1C_MASK; ctx_mask &= (ctx & CE_INUSE_CONTEXTS) ? 0x55 : 0xFF; ctx |= (ctx_mask << CE_ENABLE_BITPOS); @@ -636,10 +636,10 @@ static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) int ret = 0; for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { - qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val); + csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL); csr_val &= ~(1 << MMC_SHARE_CS_BITPOS); qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val); - qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr_val); + csr_val = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); csr_val &= IGNORE_W1C_MASK; csr_val |= CE_NN_MODE; qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val); @@ -648,7 +648,7 @@ static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT, handle->hal_handle->upc_mask & INIT_PC_VALUE); - qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx); + savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS); qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 0); qat_hal_put_wakeup_event(handle, ae, ctx_mask, XCWE_VOLUNTARY); qat_hal_wr_indr_csr(handle, ae, ctx_mask, @@ -760,7 +760,7 @@ int qat_hal_init(struct adf_accel_dev *accel_dev) for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { unsigned int csr_val = 0; - qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE, &csr_val); + csr_val = qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE); csr_val |= 0x1; qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val); } @@ -826,16 +826,16 @@ static void qat_hal_get_uwords(struct icp_qat_fw_loader_handle *handle, unsigned int i, uwrd_lo, uwrd_hi; unsigned int ustore_addr, misc_control; - qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &misc_control); + misc_control = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL); qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, misc_control & 0xfffffffb); - qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr); + ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS); uaddr |= UA_ECS; for (i = 0; i < words_num; i++) { qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); uaddr++; - qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER, &uwrd_lo); - qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER, &uwrd_hi); + uwrd_lo = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER); + uwrd_hi = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER); uword[i] = uwrd_hi; uword[i] = (uword[i] << 0x20) | uwrd_lo; } @@ -849,7 +849,7 @@ void qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle, { unsigned int i, ustore_addr; - qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr); + ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS); uaddr |= UA_ECS; qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); for (i = 0; i < words_num; i++) { @@ -890,26 +890,27 @@ static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle, return -EINVAL; } /* save current context */ - qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT, &ind_lm_addr0); - qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT, &ind_lm_addr1); - qat_hal_rd_indr_csr(handle, ae, ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX, - &ind_lm_addr_byte0); - qat_hal_rd_indr_csr(handle, ae, ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX, - &ind_lm_addr_byte1); + ind_lm_addr0 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT); + ind_lm_addr1 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT); + ind_lm_addr_byte0 = qat_hal_rd_indr_csr(handle, ae, ctx, + INDIRECT_LM_ADDR_0_BYTE_INDEX); + ind_lm_addr_byte1 = qat_hal_rd_indr_csr(handle, ae, ctx, + INDIRECT_LM_ADDR_1_BYTE_INDEX); if (inst_num <= MAX_EXEC_INST) qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords); qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events); - qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT, &savpc); + savpc = qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT); savpc = (savpc & handle->hal_handle->upc_mask) >> 0; - qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); + ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); ctx_enables &= IGNORE_W1C_MASK; - qat_hal_rd_ae_csr(handle, ae, CC_ENABLE, &savcc); - qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx); - qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL, &ctxarb_ctl); - qat_hal_rd_indr_csr(handle, ae, ctx, FUTURE_COUNT_SIGNAL_INDIRECT, - &ind_cnt_sig); - qat_hal_rd_indr_csr(handle, ae, ctx, CTX_SIG_EVENTS_INDIRECT, &ind_sig); - qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, &act_sig); + savcc = qat_hal_rd_ae_csr(handle, ae, CC_ENABLE); + savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS); + ctxarb_ctl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL); + ind_cnt_sig = qat_hal_rd_indr_csr(handle, ae, ctx, + FUTURE_COUNT_SIGNAL_INDIRECT); + ind_sig = qat_hal_rd_indr_csr(handle, ae, ctx, + CTX_SIG_EVENTS_INDIRECT); + act_sig = qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE); /* execute micro codes */ qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); qat_hal_wr_uwords(handle, ae, 0, inst_num, micro_inst); @@ -927,8 +928,8 @@ static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle, if (endpc) { unsigned int ctx_status; - qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT, - &ctx_status); + ctx_status = qat_hal_rd_indr_csr(handle, ae, ctx, + CTX_STS_INDIRECT); *endpc = ctx_status & handle->hal_handle->upc_mask; } /* retore to saved context */ @@ -938,7 +939,7 @@ static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle, qat_hal_put_wakeup_event(handle, ae, (1 << ctx), wakeup_events); qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, handle->hal_handle->upc_mask & savpc); - qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val); + csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL); newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS); qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val); qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc); @@ -986,16 +987,16 @@ static int qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle, insts = (uint64_t)0xA030000000ull | ((reg_addr & 0x3ff) << 10); break; } - qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx); - qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL, &ctxarb_cntl); - qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); + savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS); + ctxarb_cntl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL); + ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); ctx_enables &= IGNORE_W1C_MASK; if (ctx != (savctx & ACS_ACNO)) qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, ctx & ACS_ACNO); qat_hal_get_uwords(handle, ae, 0, 1, &savuword); qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); - qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr); + ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS); uaddr = UA_ECS; qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); insts = qat_hal_set_uword_ecc(insts); @@ -1011,7 +1012,7 @@ static int qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle, * the instruction should have been executed * prior to clearing the ECS in putUwords */ - qat_hal_rd_ae_csr(handle, ae, ALU_OUT, data); + *data = qat_hal_rd_ae_csr(handle, ae, ALU_OUT); qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); qat_hal_wr_uwords(handle, ae, 0, 1, &savuword); if (ctx != (savctx & ACS_ACNO)) @@ -1188,7 +1189,7 @@ static int qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle, unsigned short mask; unsigned short dr_offset = 0x10; - status = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); + status = ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); if (CE_INUSE_CONTEXTS & ctx_enables) { if (ctx & 0x1) { pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx); @@ -1238,7 +1239,7 @@ static int qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle, const int num_inst = ARRAY_SIZE(micro_inst), code_off = 1; const unsigned short gprnum = 0, dly = num_inst * 0x5; - qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); + ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); if (CE_INUSE_CONTEXTS & ctx_enables) { if (ctx & 0x1) { pr_err("QAT: 4-ctx mode,ctx=0x%x\n", ctx); @@ -1282,7 +1283,7 @@ static int qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle *handle, unsigned int ctx_enables; int stat = 0; - qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); + ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); ctx_enables &= IGNORE_W1C_MASK; qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables | CE_NN_MODE); @@ -1299,7 +1300,7 @@ static int qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle { unsigned int ctx_enables; - qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); + ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); if (ctx_enables & CE_INUSE_CONTEXTS) { /* 4-ctx mode */ *relreg = absreg_num & 0x1F; From f5c421d545356e90d9c6bc544d29c700dc91a488 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 11 Dec 2017 13:58:26 -0800 Subject: [PATCH 053/166] crypto: gf128mul - remove incorrect comment The comment in gf128mul_x8_ble() was copy-and-pasted from gf128mul.h and makes no sense in the new context. Remove it. Cc: Harsh Jain Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/gf128mul.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/crypto/gf128mul.c b/crypto/gf128mul.c index 24e601954c7a..a4b1c026aaee 100644 --- a/crypto/gf128mul.c +++ b/crypto/gf128mul.c @@ -160,8 +160,6 @@ void gf128mul_x8_ble(le128 *r, const le128 *x) { u64 a = le64_to_cpu(x->a); u64 b = le64_to_cpu(x->b); - - /* equivalent to gf128mul_table_be[b >> 63] (see crypto/gf128mul.c): */ u64 _tt = gf128mul_table_be[a >> 56]; r->a = cpu_to_le64((a << 8) | (b >> 56)); From 6d6e71feb183aa588c849e20e7baa47cb162928a Mon Sep 17 00:00:00 2001 From: Lars Persson Date: Tue, 12 Dec 2017 12:40:22 +0100 Subject: [PATCH 054/166] crypto: artpec6 - set correct iv size for gcm(aes) The IV size should not include the 32 bit counter. Because we had the IV size set as 16 the transform only worked when the IV input was zero padded. Fixes: a21eb94fc4d3 ("crypto: axis - add ARTPEC-6/7 crypto accelerator driver") Signed-off-by: Lars Persson Signed-off-by: Herbert Xu --- drivers/crypto/axis/artpec6_crypto.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c index 456278440863..22df6b55e172 100644 --- a/drivers/crypto/axis/artpec6_crypto.c +++ b/drivers/crypto/axis/artpec6_crypto.c @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -1934,7 +1935,7 @@ static int artpec6_crypto_prepare_aead(struct aead_request *areq) memcpy(req_ctx->hw_ctx.J0, areq->iv, crypto_aead_ivsize(cipher)); // The HW omits the initial increment of the counter field. - crypto_inc(req_ctx->hw_ctx.J0+12, 4); + memcpy(req_ctx->hw_ctx.J0 + GCM_AES_IV_SIZE, "\x00\x00\x00\x01", 4); ret = artpec6_crypto_setup_out_descr(common, &req_ctx->hw_ctx, sizeof(struct artpec6_crypto_aead_hw_ctx), false, false); @@ -2956,7 +2957,7 @@ static struct aead_alg aead_algos[] = { .setkey = artpec6_crypto_aead_set_key, .encrypt = artpec6_crypto_aead_encrypt, .decrypt = artpec6_crypto_aead_decrypt, - .ivsize = AES_BLOCK_SIZE, + .ivsize = GCM_AES_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, .base = { From f76d38b26c114c5e6c8252f3c7dac42bbd69c2c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stelmach?= Date: Tue, 12 Dec 2017 17:36:04 +0100 Subject: [PATCH 055/166] crypto: exynos - Support Exynos5250+ SoCs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support for PRNG in Exynos5250+ SoCs. Signed-off-by: Łukasz Stelmach Reviewed-by: Krzysztof Kozlowski Signed-off-by: Herbert Xu --- .../bindings/crypto/samsung,exynos-rng4.txt | 4 ++- drivers/crypto/exynos-rng.c | 27 +++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/crypto/samsung,exynos-rng4.txt b/Documentation/devicetree/bindings/crypto/samsung,exynos-rng4.txt index 4ca8dd4d7e66..a13fbdb4bd88 100644 --- a/Documentation/devicetree/bindings/crypto/samsung,exynos-rng4.txt +++ b/Documentation/devicetree/bindings/crypto/samsung,exynos-rng4.txt @@ -2,7 +2,9 @@ Exynos Pseudo Random Number Generator Required properties: -- compatible : Should be "samsung,exynos4-rng". +- compatible : One of: + - "samsung,exynos4-rng" for Exynos4210 and Exynos4412 + - "samsung,exynos5250-prng" for Exynos5250+ - reg : Specifies base physical address and size of the registers map. - clocks : Phandle to clock-controller plus clock-specifier pair. - clock-names : "secss" as a clock name. diff --git a/drivers/crypto/exynos-rng.c b/drivers/crypto/exynos-rng.c index ed6ba796ad71..825c09619eb8 100644 --- a/drivers/crypto/exynos-rng.c +++ b/drivers/crypto/exynos-rng.c @@ -22,12 +22,17 @@ #include #include #include +#include #include #include #define EXYNOS_RNG_CONTROL 0x0 #define EXYNOS_RNG_STATUS 0x10 + +#define EXYNOS_RNG_SEED_CONF 0x14 +#define EXYNOS_RNG_GEN_PRNG BIT(1) + #define EXYNOS_RNG_SEED_BASE 0x140 #define EXYNOS_RNG_SEED(n) (EXYNOS_RNG_SEED_BASE + (n * 0x4)) #define EXYNOS_RNG_OUT_BASE 0x160 @@ -43,6 +48,12 @@ #define EXYNOS_RNG_SEED_REGS 5 #define EXYNOS_RNG_SEED_SIZE (EXYNOS_RNG_SEED_REGS * 4) +enum exynos_prng_type { + EXYNOS_PRNG_UNKNOWN = 0, + EXYNOS_PRNG_EXYNOS4, + EXYNOS_PRNG_EXYNOS5, +}; + /* * Driver re-seeds itself with generated random numbers to increase * the randomness. @@ -63,6 +74,7 @@ struct exynos_rng_ctx { /* Device associated memory */ struct exynos_rng_dev { struct device *dev; + enum exynos_prng_type type; void __iomem *mem; struct clk *clk; /* Generated numbers stored for seeding during resume */ @@ -160,8 +172,13 @@ static int exynos_rng_get_random(struct exynos_rng_dev *rng, { int retry = EXYNOS_RNG_WAIT_RETRIES; - exynos_rng_writel(rng, EXYNOS_RNG_CONTROL_START, - EXYNOS_RNG_CONTROL); + if (rng->type == EXYNOS_PRNG_EXYNOS4) { + exynos_rng_writel(rng, EXYNOS_RNG_CONTROL_START, + EXYNOS_RNG_CONTROL); + } else if (rng->type == EXYNOS_PRNG_EXYNOS5) { + exynos_rng_writel(rng, EXYNOS_RNG_GEN_PRNG, + EXYNOS_RNG_SEED_CONF); + } while (!(exynos_rng_readl(rng, EXYNOS_RNG_STATUS) & EXYNOS_RNG_STATUS_RNG_DONE) && --retry) @@ -279,6 +296,8 @@ static int exynos_rng_probe(struct platform_device *pdev) if (!rng) return -ENOMEM; + rng->type = (enum exynos_prng_type)of_device_get_match_data(&pdev->dev); + rng->dev = &pdev->dev; rng->clk = devm_clk_get(&pdev->dev, "secss"); if (IS_ERR(rng->clk)) { @@ -367,6 +386,10 @@ static SIMPLE_DEV_PM_OPS(exynos_rng_pm_ops, exynos_rng_suspend, static const struct of_device_id exynos_rng_dt_match[] = { { .compatible = "samsung,exynos4-rng", + .data = (const void *)EXYNOS_PRNG_EXYNOS4, + }, { + .compatible = "samsung,exynos5250-prng", + .data = (const void *)EXYNOS_PRNG_EXYNOS5, }, { }, }; From 3a5a5e5c07e35366f313a5301fe3568349ab8655 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stelmach?= Date: Tue, 12 Dec 2017 17:36:05 +0100 Subject: [PATCH 056/166] crypto: exynos - Improve performance of PRNG MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use memcpy_fromio() instead of custom exynos_rng_copy_random() function to retrieve generated numbers from the registers of PRNG. Signed-off-by: Łukasz Stelmach Reviewed-by: Krzysztof Kozlowski Signed-off-by: Herbert Xu --- drivers/crypto/exynos-rng.c | 31 ++----------------------------- 1 file changed, 2 insertions(+), 29 deletions(-) diff --git a/drivers/crypto/exynos-rng.c b/drivers/crypto/exynos-rng.c index 825c09619eb8..dcdd444d0b3b 100644 --- a/drivers/crypto/exynos-rng.c +++ b/drivers/crypto/exynos-rng.c @@ -130,34 +130,6 @@ static int exynos_rng_set_seed(struct exynos_rng_dev *rng, return 0; } -/* - * Read from output registers and put the data under 'dst' array, - * up to dlen bytes. - * - * Returns number of bytes actually stored in 'dst' (dlen - * or EXYNOS_RNG_SEED_SIZE). - */ -static unsigned int exynos_rng_copy_random(struct exynos_rng_dev *rng, - u8 *dst, unsigned int dlen) -{ - unsigned int cnt = 0; - int i, j; - u32 val; - - for (j = 0; j < EXYNOS_RNG_SEED_REGS; j++) { - val = exynos_rng_readl(rng, EXYNOS_RNG_OUT(j)); - - for (i = 0; i < 4; i++) { - dst[cnt] = val & 0xff; - val >>= 8; - if (++cnt >= dlen) - return cnt; - } - } - - return cnt; -} - /* * Start the engine and poll for finish. Then read from output registers * filling the 'dst' buffer up to 'dlen' bytes or up to size of generated @@ -190,7 +162,8 @@ static int exynos_rng_get_random(struct exynos_rng_dev *rng, /* Clear status bit */ exynos_rng_writel(rng, EXYNOS_RNG_STATUS_RNG_DONE, EXYNOS_RNG_STATUS); - *read = exynos_rng_copy_random(rng, dst, dlen); + *read = min_t(size_t, dlen, EXYNOS_RNG_SEED_SIZE); + memcpy_fromio(dst, rng->mem + EXYNOS_RNG_OUT_BASE, *read); return 0; } From 99c9acfe594e07085b11cc9c317c9c9961155ee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stelmach?= Date: Tue, 12 Dec 2017 17:36:06 +0100 Subject: [PATCH 057/166] crypto: exynos - Reseed PRNG after generating 2^16 random bytes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reseed PRNG after reading 65 kB of randomness. Although this may reduce performance, in most cases the loss is not noticeable. Also the time based threshold for reseeding is changed to one second. Reseeding is performed whenever either limit is exceeded. Reseeding of a PRNG does not increase entropy, but it helps preventing backtracking the internal state of the device from its output sequence, and hence, prevents potential attacker from predicting numbers to be generated. Signed-off-by: Łukasz Stelmach Reviewed-by: Stephan Mueller Reviewed-by: Krzysztof Kozlowski Signed-off-by: Herbert Xu --- drivers/crypto/exynos-rng.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/exynos-rng.c b/drivers/crypto/exynos-rng.c index dcdd444d0b3b..825ed7bfd881 100644 --- a/drivers/crypto/exynos-rng.c +++ b/drivers/crypto/exynos-rng.c @@ -55,12 +55,14 @@ enum exynos_prng_type { }; /* - * Driver re-seeds itself with generated random numbers to increase - * the randomness. + * Driver re-seeds itself with generated random numbers to hinder + * backtracking of the original seed. * * Time for next re-seed in ms. */ -#define EXYNOS_RNG_RESEED_TIME 100 +#define EXYNOS_RNG_RESEED_TIME 1000 +#define EXYNOS_RNG_RESEED_BYTES 65536 + /* * In polling mode, do not wait infinitely for the engine to finish the work. */ @@ -82,6 +84,8 @@ struct exynos_rng_dev { unsigned int seed_save_len; /* Time of last seeding in jiffies */ unsigned long last_seeding; + /* Bytes generated since last seeding */ + unsigned long bytes_seeding; }; static struct exynos_rng_dev *exynos_rng_dev; @@ -126,6 +130,7 @@ static int exynos_rng_set_seed(struct exynos_rng_dev *rng, } rng->last_seeding = jiffies; + rng->bytes_seeding = 0; return 0; } @@ -164,6 +169,7 @@ static int exynos_rng_get_random(struct exynos_rng_dev *rng, EXYNOS_RNG_STATUS); *read = min_t(size_t, dlen, EXYNOS_RNG_SEED_SIZE); memcpy_fromio(dst, rng->mem + EXYNOS_RNG_OUT_BASE, *read); + rng->bytes_seeding += *read; return 0; } @@ -177,7 +183,8 @@ static void exynos_rng_reseed(struct exynos_rng_dev *rng) unsigned int read = 0; u8 seed[EXYNOS_RNG_SEED_SIZE]; - if (time_before(now, next_seeding)) + if (time_before(now, next_seeding) && + rng->bytes_seeding < EXYNOS_RNG_RESEED_BYTES) return; if (exynos_rng_get_random(rng, seed, sizeof(seed), &read)) From 25cf7f06af2264d240c91255bb9c6082d98cc73e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stelmach?= Date: Tue, 12 Dec 2017 17:36:07 +0100 Subject: [PATCH 058/166] crypto: exynos - Introduce mutex to prevent concurrent access to hardware MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hardware operations like reading random numbers and setting a seed need to be conducted in a single thread. Therefore a mutex is required to prevent multiple threads (processes) from accessing the hardware at the same time. The sequence of mutex_lock() and mutex_unlock() in the exynos_rng_reseed() function enables switching between different threads waiting for the driver to generate random numbers for them. Signed-off-by: Łukasz Stelmach Reviewed-by: Krzysztof Kozlowski Signed-off-by: Herbert Xu --- drivers/crypto/exynos-rng.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/drivers/crypto/exynos-rng.c b/drivers/crypto/exynos-rng.c index 825ed7bfd881..4a06092074b9 100644 --- a/drivers/crypto/exynos-rng.c +++ b/drivers/crypto/exynos-rng.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -79,6 +80,7 @@ struct exynos_rng_dev { enum exynos_prng_type type; void __iomem *mem; struct clk *clk; + struct mutex lock; /* Generated numbers stored for seeding during resume */ u8 seed_save[EXYNOS_RNG_SEED_SIZE]; unsigned int seed_save_len; @@ -191,6 +193,10 @@ static void exynos_rng_reseed(struct exynos_rng_dev *rng) return; exynos_rng_set_seed(rng, seed, read); + + /* Let others do some of their job. */ + mutex_unlock(&rng->lock); + mutex_lock(&rng->lock); } static int exynos_rng_generate(struct crypto_rng *tfm, @@ -206,6 +212,7 @@ static int exynos_rng_generate(struct crypto_rng *tfm, if (ret) return ret; + mutex_lock(&rng->lock); do { ret = exynos_rng_get_random(rng, dst, dlen, &read); if (ret) @@ -216,6 +223,7 @@ static int exynos_rng_generate(struct crypto_rng *tfm, exynos_rng_reseed(rng); } while (dlen > 0); + mutex_unlock(&rng->lock); clk_disable_unprepare(rng->clk); @@ -233,7 +241,9 @@ static int exynos_rng_seed(struct crypto_rng *tfm, const u8 *seed, if (ret) return ret; + mutex_lock(&rng->lock); ret = exynos_rng_set_seed(ctx->rng, seed, slen); + mutex_unlock(&rng->lock); clk_disable_unprepare(rng->clk); @@ -278,6 +288,8 @@ static int exynos_rng_probe(struct platform_device *pdev) rng->type = (enum exynos_prng_type)of_device_get_match_data(&pdev->dev); + mutex_init(&rng->lock); + rng->dev = &pdev->dev; rng->clk = devm_clk_get(&pdev->dev, "secss"); if (IS_ERR(rng->clk)) { @@ -328,9 +340,14 @@ static int __maybe_unused exynos_rng_suspend(struct device *dev) if (ret) return ret; + mutex_lock(&rng->lock); + /* Get new random numbers and store them for seeding on resume. */ exynos_rng_get_random(rng, rng->seed_save, sizeof(rng->seed_save), &(rng->seed_save_len)); + + mutex_unlock(&rng->lock); + dev_dbg(rng->dev, "Stored %u bytes for seeding on system resume\n", rng->seed_save_len); @@ -353,8 +370,12 @@ static int __maybe_unused exynos_rng_resume(struct device *dev) if (ret) return ret; + mutex_lock(&rng->lock); + ret = exynos_rng_set_seed(rng, rng->seed_save, rng->seed_save_len); + mutex_unlock(&rng->lock); + clk_disable_unprepare(rng->clk); return ret; From 1f83f4d15d285689176a825c298e9e1c95c90a20 Mon Sep 17 00:00:00 2001 From: Corentin Labbe Date: Tue, 12 Dec 2017 19:30:13 +0000 Subject: [PATCH 059/166] crypto: echainiv - Remove unused alg/spawn variable This patch remove two unused variable and some dead "code" using it. Fixes: 66008d4230f6 ("crypto: echainiv - Remove AEAD compatibility code") Signed-off-by: Corentin Labbe Signed-off-by: Herbert Xu --- crypto/echainiv.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/crypto/echainiv.c b/crypto/echainiv.c index e3d889b122e0..45819e6015bf 100644 --- a/crypto/echainiv.c +++ b/crypto/echainiv.c @@ -118,8 +118,6 @@ static int echainiv_aead_create(struct crypto_template *tmpl, struct rtattr **tb) { struct aead_instance *inst; - struct crypto_aead_spawn *spawn; - struct aead_alg *alg; int err; inst = aead_geniv_alloc(tmpl, tb, 0, 0); @@ -127,9 +125,6 @@ static int echainiv_aead_create(struct crypto_template *tmpl, if (IS_ERR(inst)) return PTR_ERR(inst); - spawn = aead_instance_ctx(inst); - alg = crypto_spawn_aead_alg(spawn); - err = -EINVAL; if (inst->alg.ivsize & (sizeof(u64) - 1) || !inst->alg.ivsize) goto free_inst; From d94c3d65df24833f2745618bd797d3e2612d6418 Mon Sep 17 00:00:00 2001 From: Corentin Labbe Date: Tue, 12 Dec 2017 19:30:14 +0000 Subject: [PATCH 060/166] crypto: seqiv - Remove unused alg/spawn variable This patch remove two unused variable and some dead "code" using it. Fixes: 92932d03c2b3 ("crypto: seqiv - Remove AEAD compatibility code") Signed-off-by: Corentin Labbe Signed-off-by: Herbert Xu --- crypto/seqiv.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/crypto/seqiv.c b/crypto/seqiv.c index 570b7d1aa0ca..39dbf2f7e5f5 100644 --- a/crypto/seqiv.c +++ b/crypto/seqiv.c @@ -144,8 +144,6 @@ static int seqiv_aead_decrypt(struct aead_request *req) static int seqiv_aead_create(struct crypto_template *tmpl, struct rtattr **tb) { struct aead_instance *inst; - struct crypto_aead_spawn *spawn; - struct aead_alg *alg; int err; inst = aead_geniv_alloc(tmpl, tb, 0, 0); @@ -153,9 +151,6 @@ static int seqiv_aead_create(struct crypto_template *tmpl, struct rtattr **tb) if (IS_ERR(inst)) return PTR_ERR(inst); - spawn = aead_instance_ctx(inst); - alg = crypto_spawn_aead_alg(spawn); - err = -EINVAL; if (inst->alg.ivsize != sizeof(u64)) goto free_inst; From 106840c41096a01079d3a2025225029c13713802 Mon Sep 17 00:00:00 2001 From: Sabrina Dubroca Date: Wed, 13 Dec 2017 14:53:43 +0100 Subject: [PATCH 061/166] crypto: aesni - fix typo in generic_gcmaes_decrypt generic_gcmaes_decrypt needs to use generic_gcmaes_ctx, not aesni_rfc4106_gcm_ctx. This is actually harmless because the fields in struct generic_gcmaes_ctx share the layout of the same fields in aesni_rfc4106_gcm_ctx. Fixes: cce2ea8d90fe ("crypto: aesni - add generic gcm(aes)") Cc: Signed-off-by: Sabrina Dubroca Reviewed-by: Stefano Brivio Signed-off-by: Herbert Xu --- arch/x86/crypto/aesni-intel_glue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index 3bf3dcf29825..8981ed1eb7ad 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -1117,7 +1117,7 @@ static int generic_gcmaes_decrypt(struct aead_request *req) { __be32 counter = cpu_to_be32(1); struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); + struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm); void *aes_ctx = &(ctx->aes_key_expanded); u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); From fc8517bf627c9b834f80274a1bc9ecd39b27231b Mon Sep 17 00:00:00 2001 From: Sabrina Dubroca Date: Wed, 13 Dec 2017 14:54:36 +0100 Subject: [PATCH 062/166] crypto: aesni - add wrapper for generic gcm(aes) When I added generic-gcm-aes I didn't add a wrapper like the one provided for rfc4106(gcm(aes)). We need to add a cryptd wrapper to fall back on in case the FPU is not available, otherwise we might corrupt the FPU state. Fixes: cce2ea8d90fe ("crypto: aesni - add generic gcm(aes)") Cc: Reported-by: Ilya Lesokhin Signed-off-by: Sabrina Dubroca Reviewed-by: Stefano Brivio Signed-off-by: Herbert Xu --- arch/x86/crypto/aesni-intel_glue.c | 66 ++++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 12 deletions(-) diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index 8981ed1eb7ad..a5ee78d723cd 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -690,8 +690,8 @@ static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key, rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); } -static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key, - unsigned int key_len) +static int gcmaes_wrapper_set_key(struct crypto_aead *parent, const u8 *key, + unsigned int key_len) { struct cryptd_aead **ctx = crypto_aead_ctx(parent); struct cryptd_aead *cryptd_tfm = *ctx; @@ -716,8 +716,8 @@ static int common_rfc4106_set_authsize(struct crypto_aead *aead, /* This is the Integrity Check Value (aka the authentication tag length and can * be 8, 12 or 16 bytes long. */ -static int rfc4106_set_authsize(struct crypto_aead *parent, - unsigned int authsize) +static int gcmaes_wrapper_set_authsize(struct crypto_aead *parent, + unsigned int authsize) { struct cryptd_aead **ctx = crypto_aead_ctx(parent); struct cryptd_aead *cryptd_tfm = *ctx; @@ -929,7 +929,7 @@ static int helper_rfc4106_decrypt(struct aead_request *req) aes_ctx); } -static int rfc4106_encrypt(struct aead_request *req) +static int gcmaes_wrapper_encrypt(struct aead_request *req) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct cryptd_aead **ctx = crypto_aead_ctx(tfm); @@ -945,7 +945,7 @@ static int rfc4106_encrypt(struct aead_request *req) return crypto_aead_encrypt(req); } -static int rfc4106_decrypt(struct aead_request *req) +static int gcmaes_wrapper_decrypt(struct aead_request *req) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct cryptd_aead **ctx = crypto_aead_ctx(tfm); @@ -1128,6 +1128,30 @@ static int generic_gcmaes_decrypt(struct aead_request *req) aes_ctx); } +static int generic_gcmaes_init(struct crypto_aead *aead) +{ + struct cryptd_aead *cryptd_tfm; + struct cryptd_aead **ctx = crypto_aead_ctx(aead); + + cryptd_tfm = cryptd_alloc_aead("__driver-generic-gcm-aes-aesni", + CRYPTO_ALG_INTERNAL, + CRYPTO_ALG_INTERNAL); + if (IS_ERR(cryptd_tfm)) + return PTR_ERR(cryptd_tfm); + + *ctx = cryptd_tfm; + crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base)); + + return 0; +} + +static void generic_gcmaes_exit(struct crypto_aead *aead) +{ + struct cryptd_aead **ctx = crypto_aead_ctx(aead); + + cryptd_free_aead(*ctx); +} + static struct aead_alg aesni_aead_algs[] = { { .setkey = common_rfc4106_set_key, .setauthsize = common_rfc4106_set_authsize, @@ -1147,10 +1171,10 @@ static struct aead_alg aesni_aead_algs[] = { { }, { .init = rfc4106_init, .exit = rfc4106_exit, - .setkey = rfc4106_set_key, - .setauthsize = rfc4106_set_authsize, - .encrypt = rfc4106_encrypt, - .decrypt = rfc4106_decrypt, + .setkey = gcmaes_wrapper_set_key, + .setauthsize = gcmaes_wrapper_set_authsize, + .encrypt = gcmaes_wrapper_encrypt, + .decrypt = gcmaes_wrapper_decrypt, .ivsize = GCM_RFC4106_IV_SIZE, .maxauthsize = 16, .base = { @@ -1169,14 +1193,32 @@ static struct aead_alg aesni_aead_algs[] = { { .decrypt = generic_gcmaes_decrypt, .ivsize = GCM_AES_IV_SIZE, .maxauthsize = 16, + .base = { + .cra_name = "__generic-gcm-aes-aesni", + .cra_driver_name = "__driver-generic-gcm-aes-aesni", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_INTERNAL, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct generic_gcmaes_ctx), + .cra_alignmask = AESNI_ALIGN - 1, + .cra_module = THIS_MODULE, + }, +}, { + .init = generic_gcmaes_init, + .exit = generic_gcmaes_exit, + .setkey = gcmaes_wrapper_set_key, + .setauthsize = gcmaes_wrapper_set_authsize, + .encrypt = gcmaes_wrapper_encrypt, + .decrypt = gcmaes_wrapper_decrypt, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = 16, .base = { .cra_name = "gcm(aes)", .cra_driver_name = "generic-gcm-aesni", .cra_priority = 400, .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct generic_gcmaes_ctx), - .cra_alignmask = AESNI_ALIGN - 1, + .cra_ctxsize = sizeof(struct cryptd_aead *), .cra_module = THIS_MODULE, }, } }; From 62c2eddd2d574ddcb125c89c1246e7f8226ce068 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 14 Dec 2017 15:26:43 +0100 Subject: [PATCH 063/166] crypto: inside-secure - remove extra empty line Cosmetic patch removing an extra empty line between header inclusions. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel_hash.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 0c5a5820b06e..d2d4551565fa 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -14,7 +14,6 @@ #include #include - #include "safexcel.h" struct safexcel_ahash_ctx { From dfbcc08faadd696d3c4687d7df540956f2be4f7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 14 Dec 2017 15:26:44 +0100 Subject: [PATCH 064/166] crypto: inside-secure - fix typo in a comment Cosmetic patch fixing one typo in one of the driver's comments. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel_hash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index d2d4551565fa..d4afc070ff58 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -504,7 +504,7 @@ static int safexcel_ahash_cache(struct ahash_request *areq) return areq->nbytes; } - /* We could'nt cache all the data */ + /* We couldn't cache all the data */ return -E2BIG; } From 60310eae7dc12182e8901a41e0ba885aaac0b542 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 14 Dec 2017 15:26:45 +0100 Subject: [PATCH 065/166] crypto: inside-secure - remove useless memset This patch removes an useless memset in the ahash_export function, as the zeroed buffer will be entirely overridden the next line. Suggested-by: Ofer Heifetz Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel_hash.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index d4afc070ff58..ac922b91b2d7 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -624,7 +624,6 @@ static int safexcel_ahash_export(struct ahash_request *areq, void *out) export->processed = req->processed; memcpy(export->state, req->state, req->state_sz); - memset(export->cache, 0, crypto_ahash_blocksize(ahash)); memcpy(export->cache, req->cache, crypto_ahash_blocksize(ahash)); return 0; From cc75f5ce82cc52a1c152fa22ea8c172d84fb5bdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 14 Dec 2017 15:26:46 +0100 Subject: [PATCH 066/166] crypto: inside-secure - comment the caching function Cosmetic patch adding a few comments to the ahash caching function to understand easily what calculations are made in the functions; and how the function is working. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel_hash.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index ac922b91b2d7..e53e6b2331db 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -484,13 +484,23 @@ static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm) return 0; } +/* safexcel_ahash_cache: cache data until at least one request can be sent to + * the engine, aka. when there is at least 1 block size in the pipe. + */ static int safexcel_ahash_cache(struct ahash_request *areq) { struct safexcel_ahash_req *req = ahash_request_ctx(areq); struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); int queued, cache_len; + /* cache_len: everyting accepted by the driver but not sent yet, + * tot sz handled by update() - last req sz - tot sz handled by send() + */ cache_len = req->len - areq->nbytes - req->processed; + /* queued: everything accepted by the driver which will be handled by + * the next send() calls. + * tot sz handled by update() - tot sz handled by send() + */ queued = req->len - req->processed; /* From c4daf4cc9c8dc728ff4afc043a82997072bfb2fa Mon Sep 17 00:00:00 2001 From: Ofer Heifetz Date: Thu, 14 Dec 2017 15:26:47 +0100 Subject: [PATCH 067/166] crypto: inside-secure - refrain from unneeded invalidations The check to know if an invalidation is needed (i.e. when the context changes) is done even if the context does not exist yet. This happens when first setting a key for ciphers and/or hmac operations. This commits adds a check in the _setkey functions to only check if an invalidation is needed when a context exists, as there is no need to perform this check otherwise. Signed-off-by: Ofer Heifetz [Antoine: commit message and added a comment and reworked one of the checks] Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- .../crypto/inside-secure/safexcel_cipher.c | 10 ++++---- drivers/crypto/inside-secure/safexcel_hash.c | 24 ++++++++++++------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index fcc0a606d748..794849b859a7 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -78,10 +78,12 @@ static int safexcel_aes_setkey(struct crypto_skcipher *ctfm, const u8 *key, return ret; } - for (i = 0; i < len / sizeof(u32); i++) { - if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) { - ctx->base.needs_inv = true; - break; + if (ctx->base.ctxr_dma) { + for (i = 0; i < len / sizeof(u32); i++) { + if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) { + ctx->base.needs_inv = true; + break; + } } } diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index e53e6b2331db..d156637d6144 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -527,10 +527,16 @@ static int safexcel_ahash_enqueue(struct ahash_request *areq) req->needs_inv = false; - if (req->processed && ctx->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) - ctx->base.needs_inv = safexcel_ahash_needs_inv_get(areq); - if (ctx->base.ctxr) { + if (!ctx->base.needs_inv && req->processed && + ctx->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) + /* We're still setting needs_inv here, even though it is + * cleared right away, because the needs_inv flag can be + * set in other functions and we want to keep the same + * logic. + */ + ctx->base.needs_inv = safexcel_ahash_needs_inv_get(areq); + if (ctx->base.needs_inv) { ctx->base.needs_inv = false; req->needs_inv = true; @@ -928,11 +934,13 @@ static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key, if (ret) return ret; - for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++) { - if (ctx->ipad[i] != le32_to_cpu(istate.state[i]) || - ctx->opad[i] != le32_to_cpu(ostate.state[i])) { - ctx->base.needs_inv = true; - break; + if (ctx->base.ctxr) { + for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++) { + if (ctx->ipad[i] != le32_to_cpu(istate.state[i]) || + ctx->opad[i] != le32_to_cpu(ostate.state[i])) { + ctx->base.needs_inv = true; + break; + } } } From 12bf4142d88823814900267ff40c953555d80265 Mon Sep 17 00:00:00 2001 From: Ofer Heifetz Date: Thu, 14 Dec 2017 15:26:48 +0100 Subject: [PATCH 068/166] crypto: inside-secure - EBUSY is not an error on async request When initializing the IVs crypto_ahash_update() is called, which at some point will call crypto_enqueue_request(). This function can return -EBUSY when no resource is available and the request is queued. Since this is a valid case, -EBUSY shouldn't be treated as an error. Signed-off-by: Ofer Heifetz [Antoine: commit message] Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel_hash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index d156637d6144..4a4d95302a93 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -862,7 +862,7 @@ static int safexcel_hmac_init_iv(struct ahash_request *areq, req->last_req = true; ret = crypto_ahash_update(areq); - if (ret && ret != -EINPROGRESS) + if (ret && ret != -EINPROGRESS && ret != -EBUSY) return ret; wait_for_completion_interruptible(&result.completion); From 847ccfc5866950873b20c6fd980808f6043061ae Mon Sep 17 00:00:00 2001 From: Ofer Heifetz Date: Thu, 14 Dec 2017 15:26:49 +0100 Subject: [PATCH 069/166] crypto: inside-secure - move cipher crypto mode to request context The cipher direction can be different for requests within the same transformation context. This patch moves the direction flag from the context to the request scope. Signed-off-by: Ofer Heifetz [Antoine: commit message] Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel_cipher.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 794849b859a7..62382a7c4cbe 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -27,7 +27,6 @@ struct safexcel_cipher_ctx { struct safexcel_context base; struct safexcel_crypto_priv *priv; - enum safexcel_cipher_direction direction; u32 mode; __le32 key[8]; @@ -35,6 +34,7 @@ struct safexcel_cipher_ctx { }; struct safexcel_cipher_req { + enum safexcel_cipher_direction direction; bool needs_inv; }; @@ -97,12 +97,15 @@ static int safexcel_aes_setkey(struct crypto_skcipher *ctfm, const u8 *key, } static int safexcel_context_control(struct safexcel_cipher_ctx *ctx, + struct crypto_async_request *async, struct safexcel_command_desc *cdesc) { struct safexcel_crypto_priv *priv = ctx->priv; + struct skcipher_request *req = skcipher_request_cast(async); + struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); int ctrl_size; - if (ctx->direction == SAFEXCEL_ENCRYPT) + if (sreq->direction == SAFEXCEL_ENCRYPT) cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_CRYPTO_OUT; else cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_CRYPTO_IN; @@ -245,7 +248,7 @@ static int safexcel_aes_send(struct crypto_async_request *async, n_cdesc++; if (n_cdesc == 1) { - safexcel_context_control(ctx, cdesc); + safexcel_context_control(ctx, async, cdesc); safexcel_cipher_token(ctx, async, cdesc, req->cryptlen); } @@ -469,7 +472,7 @@ static int safexcel_aes(struct skcipher_request *req, int ret, ring; sreq->needs_inv = false; - ctx->direction = dir; + sreq->direction = dir; ctx->mode = mode; if (ctx->base.ctxr) { From 5290ad6e9a685154a112274325683e5db5af76b0 Mon Sep 17 00:00:00 2001 From: Ofer Heifetz Date: Thu, 14 Dec 2017 15:26:50 +0100 Subject: [PATCH 070/166] crypto: inside-secure - remove unused parameter in invalidate_cache The SafeXcel context isn't used in the cache invalidation function. This cosmetic patch removes it (as well as from the function prototype in the header file and when the function is called). Signed-off-by: Ofer Heifetz [Antoine: commit message] Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 1 - drivers/crypto/inside-secure/safexcel.h | 1 - drivers/crypto/inside-secure/safexcel_cipher.c | 2 +- drivers/crypto/inside-secure/safexcel_hash.c | 2 +- 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 4bcef78a08aa..f250c3c1ab0f 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -540,7 +540,6 @@ void safexcel_inv_complete(struct crypto_async_request *req, int error) } int safexcel_invalidate_cache(struct crypto_async_request *async, - struct safexcel_context *ctx, struct safexcel_crypto_priv *priv, dma_addr_t ctxr_dma, int ring, struct safexcel_request *request) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 304c5838c11a..d12c2b479a5e 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -539,7 +539,6 @@ void safexcel_free_context(struct safexcel_crypto_priv *priv, struct crypto_async_request *req, int result_sz); int safexcel_invalidate_cache(struct crypto_async_request *async, - struct safexcel_context *ctx, struct safexcel_crypto_priv *priv, dma_addr_t ctxr_dma, int ring, struct safexcel_request *request); diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 62382a7c4cbe..fe1d588d6a25 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -395,7 +395,7 @@ static int safexcel_cipher_send_inv(struct crypto_async_request *async, struct safexcel_crypto_priv *priv = ctx->priv; int ret; - ret = safexcel_invalidate_cache(async, &ctx->base, priv, + ret = safexcel_invalidate_cache(async, priv, ctx->base.ctxr_dma, ring, request); if (unlikely(ret)) return ret; diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 4a4d95302a93..d233f4a09712 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -417,7 +417,7 @@ static int safexcel_ahash_send_inv(struct crypto_async_request *async, struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); int ret; - ret = safexcel_invalidate_cache(async, &ctx->base, ctx->priv, + ret = safexcel_invalidate_cache(async, ctx->priv, ctx->base.ctxr_dma, ring, request); if (unlikely(ret)) return ret; From 8472e778108cc652f9e36282293d0663ba14a0ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 14 Dec 2017 15:26:51 +0100 Subject: [PATCH 071/166] crypto: inside-secure - move request dequeueing into a workqueue This patch moves the request dequeueing into a workqueue to improve the coalescing of interrupts when sending requests to the engine; as the engine is capable of having one single interrupt for n requests sent. Using a workqueue allows to send more request at once. Suggested-by: Ofer Heifetz Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 29 ++++++++++--------- drivers/crypto/inside-secure/safexcel.h | 2 +- .../crypto/inside-secure/safexcel_cipher.c | 12 ++++---- drivers/crypto/inside-secure/safexcel_hash.c | 12 ++++---- 4 files changed, 29 insertions(+), 26 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index f250c3c1ab0f..f422af3eed2f 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -429,8 +429,6 @@ void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring) struct safexcel_request *request; int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results; - priv->ring[ring].need_dequeue = false; - do { spin_lock_bh(&priv->ring[ring].queue_lock); backlog = crypto_get_backlog(&priv->ring[ring].queue); @@ -445,8 +443,6 @@ void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring) spin_lock_bh(&priv->ring[ring].queue_lock); crypto_enqueue_request(&priv->ring[ring].queue, req); spin_unlock_bh(&priv->ring[ring].queue_lock); - - priv->ring[ring].need_dequeue = true; goto finalize; } @@ -455,7 +451,6 @@ void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring) if (ret) { kfree(request); req->complete(req, ret); - priv->ring[ring].need_dequeue = true; goto finalize; } @@ -471,9 +466,7 @@ void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring) } while (nreq++ < EIP197_MAX_BATCH_SZ); finalize: - if (nreq == EIP197_MAX_BATCH_SZ) - priv->ring[ring].need_dequeue = true; - else if (!nreq) + if (!nreq) return; spin_lock_bh(&priv->ring[ring].lock); @@ -628,13 +621,18 @@ static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv static void safexcel_handle_result_work(struct work_struct *work) { struct safexcel_work_data *data = - container_of(work, struct safexcel_work_data, work); + container_of(work, struct safexcel_work_data, result_work); struct safexcel_crypto_priv *priv = data->priv; safexcel_handle_result_descriptor(priv, data->ring); +} - if (priv->ring[data->ring].need_dequeue) - safexcel_dequeue(data->priv, data->ring); +static void safexcel_dequeue_work(struct work_struct *work) +{ + struct safexcel_work_data *data = + container_of(work, struct safexcel_work_data, work); + + safexcel_dequeue(data->priv, data->ring); } struct safexcel_ring_irq_data { @@ -665,7 +663,10 @@ static irqreturn_t safexcel_irq_ring(int irq, void *data) */ dev_err(priv->dev, "RDR: fatal error."); } else if (likely(stat & EIP197_xDR_THRESH)) { - queue_work(priv->ring[ring].workqueue, &priv->ring[ring].work_data.work); + queue_work(priv->ring[ring].workqueue, + &priv->ring[ring].work_data.result_work); + queue_work(priv->ring[ring].workqueue, + &priv->ring[ring].work_data.work); } /* ACK the interrupts */ @@ -846,7 +847,9 @@ static int safexcel_probe(struct platform_device *pdev) priv->ring[i].work_data.priv = priv; priv->ring[i].work_data.ring = i; - INIT_WORK(&priv->ring[i].work_data.work, safexcel_handle_result_work); + INIT_WORK(&priv->ring[i].work_data.result_work, + safexcel_handle_result_work); + INIT_WORK(&priv->ring[i].work_data.work, safexcel_dequeue_work); snprintf(wq_name, 9, "wq_ring%d", i); priv->ring[i].workqueue = create_singlethread_workqueue(wq_name); diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index d12c2b479a5e..8e9c65183439 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -459,6 +459,7 @@ struct safexcel_config { struct safexcel_work_data { struct work_struct work; + struct work_struct result_work; struct safexcel_crypto_priv *priv; int ring; }; @@ -489,7 +490,6 @@ struct safexcel_crypto_priv { /* queue */ struct crypto_queue queue; spinlock_t queue_lock; - bool need_dequeue; } ring[EIP197_MAX_RINGS]; }; diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index fe1d588d6a25..0e5cc230e49a 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -358,8 +358,8 @@ static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, if (enq_ret != -EINPROGRESS) *ret = enq_ret; - if (!priv->ring[ring].need_dequeue) - safexcel_dequeue(priv, ring); + queue_work(priv->ring[ring].workqueue, + &priv->ring[ring].work_data.work); *should_complete = false; @@ -448,8 +448,8 @@ static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm) crypto_enqueue_request(&priv->ring[ring].queue, &req->base); spin_unlock_bh(&priv->ring[ring].queue_lock); - if (!priv->ring[ring].need_dequeue) - safexcel_dequeue(priv, ring); + queue_work(priv->ring[ring].workqueue, + &priv->ring[ring].work_data.work); wait_for_completion_interruptible(&result.completion); @@ -495,8 +495,8 @@ static int safexcel_aes(struct skcipher_request *req, ret = crypto_enqueue_request(&priv->ring[ring].queue, &req->base); spin_unlock_bh(&priv->ring[ring].queue_lock); - if (!priv->ring[ring].need_dequeue) - safexcel_dequeue(priv, ring); + queue_work(priv->ring[ring].workqueue, + &priv->ring[ring].work_data.work); return ret; } diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index d233f4a09712..2fb5bc6b6268 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -381,8 +381,8 @@ static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, if (enq_ret != -EINPROGRESS) *ret = enq_ret; - if (!priv->ring[ring].need_dequeue) - safexcel_dequeue(priv, ring); + queue_work(priv->ring[ring].workqueue, + &priv->ring[ring].work_data.work); *should_complete = false; @@ -470,8 +470,8 @@ static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm) crypto_enqueue_request(&priv->ring[ring].queue, &req->base); spin_unlock_bh(&priv->ring[ring].queue_lock); - if (!priv->ring[ring].need_dequeue) - safexcel_dequeue(priv, ring); + queue_work(priv->ring[ring].workqueue, + &priv->ring[ring].work_data.work); wait_for_completion_interruptible(&result.completion); @@ -556,8 +556,8 @@ static int safexcel_ahash_enqueue(struct ahash_request *areq) ret = crypto_enqueue_request(&priv->ring[ring].queue, &areq->base); spin_unlock_bh(&priv->ring[ring].queue_lock); - if (!priv->ring[ring].need_dequeue) - safexcel_dequeue(priv, ring); + queue_work(priv->ring[ring].workqueue, + &priv->ring[ring].work_data.work); return ret; } From 69ee4dd5ea8ce0b45d2fae03f6bda1044577705f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 14 Dec 2017 15:26:52 +0100 Subject: [PATCH 072/166] crypto: inside-secure - use threaded IRQs for result handling This patch moves the result handling from an IRQ handler to a threaded IRQ handler, to improve the number of complete requests being handled at once. Suggested-by: Ofer Heifetz Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 41 +++++++++++++------------ drivers/crypto/inside-secure/safexcel.h | 1 - 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index f422af3eed2f..aa4943cad55e 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -618,15 +618,6 @@ static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv } } -static void safexcel_handle_result_work(struct work_struct *work) -{ - struct safexcel_work_data *data = - container_of(work, struct safexcel_work_data, result_work); - struct safexcel_crypto_priv *priv = data->priv; - - safexcel_handle_result_descriptor(priv, data->ring); -} - static void safexcel_dequeue_work(struct work_struct *work) { struct safexcel_work_data *data = @@ -644,12 +635,12 @@ static irqreturn_t safexcel_irq_ring(int irq, void *data) { struct safexcel_ring_irq_data *irq_data = data; struct safexcel_crypto_priv *priv = irq_data->priv; - int ring = irq_data->ring; + int ring = irq_data->ring, rc = IRQ_NONE; u32 status, stat; status = readl(priv->base + EIP197_HIA_AIC_R_ENABLED_STAT(ring)); if (!status) - return IRQ_NONE; + return rc; /* RDR interrupts */ if (status & EIP197_RDR_IRQ(ring)) { @@ -663,10 +654,7 @@ static irqreturn_t safexcel_irq_ring(int irq, void *data) */ dev_err(priv->dev, "RDR: fatal error."); } else if (likely(stat & EIP197_xDR_THRESH)) { - queue_work(priv->ring[ring].workqueue, - &priv->ring[ring].work_data.result_work); - queue_work(priv->ring[ring].workqueue, - &priv->ring[ring].work_data.work); + rc = IRQ_WAKE_THREAD; } /* ACK the interrupts */ @@ -677,11 +665,26 @@ static irqreturn_t safexcel_irq_ring(int irq, void *data) /* ACK the interrupts */ writel(status, priv->base + EIP197_HIA_AIC_R_ACK(ring)); + return rc; +} + +static irqreturn_t safexcel_irq_ring_thread(int irq, void *data) +{ + struct safexcel_ring_irq_data *irq_data = data; + struct safexcel_crypto_priv *priv = irq_data->priv; + int ring = irq_data->ring; + + safexcel_handle_result_descriptor(priv, ring); + + queue_work(priv->ring[ring].workqueue, + &priv->ring[ring].work_data.work); + return IRQ_HANDLED; } static int safexcel_request_ring_irq(struct platform_device *pdev, const char *name, irq_handler_t handler, + irq_handler_t threaded_handler, struct safexcel_ring_irq_data *ring_irq_priv) { int ret, irq = platform_get_irq_byname(pdev, name); @@ -691,8 +694,9 @@ static int safexcel_request_ring_irq(struct platform_device *pdev, const char *n return irq; } - ret = devm_request_irq(&pdev->dev, irq, handler, 0, - dev_name(&pdev->dev), ring_irq_priv); + ret = devm_request_threaded_irq(&pdev->dev, irq, handler, + threaded_handler, IRQF_ONESHOT, + dev_name(&pdev->dev), ring_irq_priv); if (ret) { dev_err(&pdev->dev, "unable to request IRQ %d\n", irq); return ret; @@ -839,6 +843,7 @@ static int safexcel_probe(struct platform_device *pdev) snprintf(irq_name, 6, "ring%d", i); irq = safexcel_request_ring_irq(pdev, irq_name, safexcel_irq_ring, + safexcel_irq_ring_thread, ring_irq); if (irq < 0) { ret = irq; @@ -847,8 +852,6 @@ static int safexcel_probe(struct platform_device *pdev) priv->ring[i].work_data.priv = priv; priv->ring[i].work_data.ring = i; - INIT_WORK(&priv->ring[i].work_data.result_work, - safexcel_handle_result_work); INIT_WORK(&priv->ring[i].work_data.work, safexcel_dequeue_work); snprintf(wq_name, 9, "wq_ring%d", i); diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 8e9c65183439..fffddefb0d9b 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -459,7 +459,6 @@ struct safexcel_config { struct safexcel_work_data { struct work_struct work; - struct work_struct result_work; struct safexcel_crypto_priv *priv; int ring; }; From dc7e28a3286ed4c4ce79e279c6f8b99f39ab8e68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 14 Dec 2017 15:26:53 +0100 Subject: [PATCH 073/166] crypto: inside-secure - dequeue all requests at once This patch updates the dequeueing logic to dequeue all requests at once. Since we can have many requests in the queue, the interrupt coalescing is kept so that the ring interrupt fires every EIP197_MAX_BATCH_SZ at most. To allow dequeueing all requests at once while still using reasonable settings for the interrupt coalescing, the result handling function was updated to setup the threshold interrupt when needed (i.e. when more requests than EIP197_MAX_BATCH_SZ are in the queue). When using this capability the ring is marked as busy so that the dequeue function enqueue new requests without setting the threshold interrupt. Suggested-by: Ofer Heifetz Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 60 ++++++++++++++++++++----- drivers/crypto/inside-secure/safexcel.h | 8 ++++ 2 files changed, 56 insertions(+), 12 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index aa4943cad55e..db7ad9d3eeed 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -422,6 +422,23 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv) return 0; } +/* Called with ring's lock taken */ +int safexcel_try_push_requests(struct safexcel_crypto_priv *priv, int ring, + int reqs) +{ + int coal = min_t(int, reqs, EIP197_MAX_BATCH_SZ); + + if (!coal) + return 0; + + /* Configure when we want an interrupt */ + writel(EIP197_HIA_RDR_THRESH_PKT_MODE | + EIP197_HIA_RDR_THRESH_PROC_PKT(coal), + priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_THRESH); + + return coal; +} + void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring) { struct crypto_async_request *req, *backlog; @@ -429,7 +446,7 @@ void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring) struct safexcel_request *request; int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results; - do { + while (true) { spin_lock_bh(&priv->ring[ring].queue_lock); backlog = crypto_get_backlog(&priv->ring[ring].queue); req = crypto_dequeue_request(&priv->ring[ring].queue); @@ -463,18 +480,24 @@ void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring) cdesc += commands; rdesc += results; - } while (nreq++ < EIP197_MAX_BATCH_SZ); + nreq++; + } finalize: if (!nreq) return; - spin_lock_bh(&priv->ring[ring].lock); + spin_lock_bh(&priv->ring[ring].egress_lock); - /* Configure when we want an interrupt */ - writel(EIP197_HIA_RDR_THRESH_PKT_MODE | - EIP197_HIA_RDR_THRESH_PROC_PKT(nreq), - priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_THRESH); + if (!priv->ring[ring].busy) { + nreq -= safexcel_try_push_requests(priv, ring, nreq); + if (nreq) + priv->ring[ring].busy = true; + } + + priv->ring[ring].requests_left += nreq; + + spin_unlock_bh(&priv->ring[ring].egress_lock); /* let the RDR know we have pending descriptors */ writel((rdesc * priv->config.rd_offset) << 2, @@ -483,8 +506,6 @@ finalize: /* let the CDR know we have pending descriptors */ writel((cdesc * priv->config.cd_offset) << 2, priv->base + EIP197_HIA_CDR(ring) + EIP197_HIA_xDR_PREP_COUNT); - - spin_unlock_bh(&priv->ring[ring].lock); } void safexcel_free_context(struct safexcel_crypto_priv *priv, @@ -579,14 +600,14 @@ static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv { struct safexcel_request *sreq; struct safexcel_context *ctx; - int ret, i, nreq, ndesc = 0; + int ret, i, nreq, ndesc = 0, done; bool should_complete; nreq = readl(priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PROC_COUNT); nreq >>= 24; nreq &= GENMASK(6, 0); if (!nreq) - return; + goto requests_left; for (i = 0; i < nreq; i++) { spin_lock_bh(&priv->ring[ring].egress_lock); @@ -601,7 +622,7 @@ static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv if (ndesc < 0) { kfree(sreq); dev_err(priv->dev, "failed to handle result (%d)", ndesc); - return; + goto requests_left; } writel(EIP197_xDR_PROC_xD_PKT(1) | @@ -616,6 +637,18 @@ static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv kfree(sreq); } + +requests_left: + spin_lock_bh(&priv->ring[ring].egress_lock); + + done = safexcel_try_push_requests(priv, ring, + priv->ring[ring].requests_left); + + priv->ring[ring].requests_left -= done; + if (!done && !priv->ring[ring].requests_left) + priv->ring[ring].busy = false; + + spin_unlock_bh(&priv->ring[ring].egress_lock); } static void safexcel_dequeue_work(struct work_struct *work) @@ -861,6 +894,9 @@ static int safexcel_probe(struct platform_device *pdev) goto err_clk; } + priv->ring[i].requests_left = 0; + priv->ring[i].busy = false; + crypto_init_queue(&priv->ring[i].queue, EIP197_DEFAULT_RING_SIZE); diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index fffddefb0d9b..531e3e9d8384 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -489,6 +489,14 @@ struct safexcel_crypto_priv { /* queue */ struct crypto_queue queue; spinlock_t queue_lock; + + /* Number of requests in the engine that needs the threshold + * interrupt to be set up. + */ + int requests_left; + + /* The ring is currently handling at least one request */ + bool busy; } ring[EIP197_MAX_RINGS]; }; From fc8c72b2cb269b130ad7d781023cd65514736c79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 14 Dec 2017 15:26:54 +0100 Subject: [PATCH 074/166] crypto: inside-secure - increase the ring size Increase the ring size to handle more requests in parallel, while keeping the batch size (for interrupt coalescing) to its previous value. The ring size and batch size are now unlinked. Suggested-by: Ofer Heifetz Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 531e3e9d8384..2a0ab6ce716a 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -19,11 +19,11 @@ #define EIP197_HIA_VERSION_BE 0x35ca /* Static configuration */ -#define EIP197_DEFAULT_RING_SIZE 64 +#define EIP197_DEFAULT_RING_SIZE 400 #define EIP197_MAX_TOKENS 5 #define EIP197_MAX_RINGS 4 #define EIP197_FETCH_COUNT 1 -#define EIP197_MAX_BATCH_SZ EIP197_DEFAULT_RING_SIZE +#define EIP197_MAX_BATCH_SZ 64 #define EIP197_GFP_FLAGS(base) ((base).flags & CRYPTO_TFM_REQ_MAY_SLEEP ? \ GFP_KERNEL : GFP_ATOMIC) From 2313e9f7ecaa6119ed877b523be7cd51768a6ddc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 14 Dec 2017 15:26:55 +0100 Subject: [PATCH 075/166] crypto: inside-secure - acknowledge the result requests all at once This patches moves the result request acknowledgment from a per request process to acknowledging all the result requests handled at once. Suggested-by: Ofer Heifetz Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index db7ad9d3eeed..36c9a72c47b1 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -600,7 +600,7 @@ static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv { struct safexcel_request *sreq; struct safexcel_context *ctx; - int ret, i, nreq, ndesc = 0, done; + int ret, i, nreq, ndesc = 0, tot_descs = 0, done; bool should_complete; nreq = readl(priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PROC_COUNT); @@ -622,13 +622,9 @@ static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv if (ndesc < 0) { kfree(sreq); dev_err(priv->dev, "failed to handle result (%d)", ndesc); - goto requests_left; + goto acknowledge; } - writel(EIP197_xDR_PROC_xD_PKT(1) | - EIP197_xDR_PROC_xD_COUNT(ndesc * priv->config.rd_offset), - priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PROC_COUNT); - if (should_complete) { local_bh_disable(); sreq->req->complete(sreq->req, ret); @@ -636,6 +632,14 @@ static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv } kfree(sreq); + tot_descs += ndesc; + } + +acknowledge: + if (i) { + writel(EIP197_xDR_PROC_xD_PKT(i) | + EIP197_xDR_PROC_xD_COUNT(tot_descs * priv->config.rd_offset), + priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PROC_COUNT); } requests_left: From 7f77f5a4fc0875e4fc1110e48d9a522a9e639d0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 14 Dec 2017 15:26:56 +0100 Subject: [PATCH 076/166] crypto: inside-secure - handle more result requests when counter is full This patch modifies the result handling logic to continue handling results when the completed requests counter is full and not showing the actual number of requests to handle. Suggested-by: Ofer Heifetz Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 15 ++++++++++++--- drivers/crypto/inside-secure/safexcel.h | 2 ++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 36c9a72c47b1..8042922b4ed8 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -600,12 +600,15 @@ static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv { struct safexcel_request *sreq; struct safexcel_context *ctx; - int ret, i, nreq, ndesc = 0, tot_descs = 0, done; + int ret, i, nreq, ndesc, tot_descs, done; bool should_complete; +handle_results: + tot_descs = 0; + nreq = readl(priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PROC_COUNT); - nreq >>= 24; - nreq &= GENMASK(6, 0); + nreq >>= EIP197_xDR_PROC_xD_PKT_OFFSET; + nreq &= EIP197_xDR_PROC_xD_PKT_MASK; if (!nreq) goto requests_left; @@ -642,6 +645,12 @@ acknowledge: priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PROC_COUNT); } + /* If the number of requests overflowed the counter, try to proceed more + * requests. + */ + if (nreq == EIP197_xDR_PROC_xD_PKT_MASK) + goto handle_results; + requests_left: spin_lock_bh(&priv->ring[ring].egress_lock); diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 2a0ab6ce716a..0c47e792192d 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -117,6 +117,8 @@ #define EIP197_xDR_PREP_CLR_COUNT BIT(31) /* EIP197_HIA_xDR_PROC_COUNT */ +#define EIP197_xDR_PROC_xD_PKT_OFFSET 24 +#define EIP197_xDR_PROC_xD_PKT_MASK GENMASK(6, 0) #define EIP197_xDR_PROC_xD_COUNT(n) ((n) << 2) #define EIP197_xDR_PROC_xD_PKT(n) ((n) << 24) #define EIP197_xDR_PROC_CLR_COUNT BIT(31) From 8732b298fa1cd266d8962d45109ba5cfa4212f37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 14 Dec 2017 15:26:57 +0100 Subject: [PATCH 077/166] crypto: inside-secure - retry to proceed the request later on fail The dequeueing function was putting back a request in the crypto queue on failure (when not enough resources are available) which is not perfect as the request will be handled much later. This patch updates this logic by keeping a reference on the failed request to try proceeding it later when enough resources are available. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 34 +++++++++++++++++-------- drivers/crypto/inside-secure/safexcel.h | 6 +++++ 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 8042922b4ed8..4c7f205d83f0 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -446,29 +446,36 @@ void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring) struct safexcel_request *request; int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results; + /* If a request wasn't properly dequeued because of a lack of resources, + * proceeded it first, + */ + req = priv->ring[ring].req; + backlog = priv->ring[ring].backlog; + if (req) + goto handle_req; + while (true) { spin_lock_bh(&priv->ring[ring].queue_lock); backlog = crypto_get_backlog(&priv->ring[ring].queue); req = crypto_dequeue_request(&priv->ring[ring].queue); spin_unlock_bh(&priv->ring[ring].queue_lock); - if (!req) - goto finalize; - - request = kzalloc(sizeof(*request), EIP197_GFP_FLAGS(*req)); - if (!request) { - spin_lock_bh(&priv->ring[ring].queue_lock); - crypto_enqueue_request(&priv->ring[ring].queue, req); - spin_unlock_bh(&priv->ring[ring].queue_lock); + if (!req) { + priv->ring[ring].req = NULL; + priv->ring[ring].backlog = NULL; goto finalize; } +handle_req: + request = kzalloc(sizeof(*request), EIP197_GFP_FLAGS(*req)); + if (!request) + goto request_failed; + ctx = crypto_tfm_ctx(req->tfm); ret = ctx->send(req, ring, request, &commands, &results); if (ret) { kfree(request); - req->complete(req, ret); - goto finalize; + goto request_failed; } if (backlog) @@ -483,6 +490,13 @@ void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring) nreq++; } +request_failed: + /* Not enough resources to handle all the requests. Bail out and save + * the request and the backlog for the next dequeue call (per-ring). + */ + priv->ring[ring].req = req; + priv->ring[ring].backlog = backlog; + finalize: if (!nreq) return; diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 0c47e792192d..d4955abf873b 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -499,6 +499,12 @@ struct safexcel_crypto_priv { /* The ring is currently handling at least one request */ bool busy; + + /* Store for current requests when bailing out of the dequeueing + * function when no enough resources are available. + */ + struct crypto_async_request *req; + struct crypto_async_request *backlog; } ring[EIP197_MAX_RINGS]; }; From 871df319bd48acb4ce48b8985136c8ba40af3a02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 14 Dec 2017 15:26:58 +0100 Subject: [PATCH 078/166] crypto: inside-secure - EIP97 support The Inside Secure SafeXcel driver was firstly designed to support the EIP197 cryptographic engine which is an evolution (with much more feature, better performances) of the EIP97 cryptographic engine. This patch convert the Inside Secure SafeXcel driver to support both engines (EIP97 + EIP197). The main differences are the register offsets and the context invalidation process which is EIP197 specific. This patch adds an indirection on the register offsets and adds checks not to send any invalidation request when driving the EIP97. A new compatible is added as well to bind the driver from device trees. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 212 +++++++++++------- drivers/crypto/inside-secure/safexcel.h | 151 +++++++++---- .../crypto/inside-secure/safexcel_cipher.c | 20 +- drivers/crypto/inside-secure/safexcel_hash.c | 19 +- 4 files changed, 264 insertions(+), 138 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 4c7f205d83f0..0d685be75241 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -108,10 +108,10 @@ static void eip197_write_firmware(struct safexcel_crypto_priv *priv, writel(EIP197_PE_ICE_x_CTRL_SW_RESET | EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR | EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR, - priv->base + ctrl); + EIP197_PE(priv) + ctrl); /* Enable access to the program memory */ - writel(prog_en, priv->base + EIP197_PE_ICE_RAM_CTRL); + writel(prog_en, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL); /* Write the firmware */ for (i = 0; i < fw->size / sizeof(u32); i++) @@ -119,12 +119,12 @@ static void eip197_write_firmware(struct safexcel_crypto_priv *priv, priv->base + EIP197_CLASSIFICATION_RAMS + i * sizeof(u32)); /* Disable access to the program memory */ - writel(0, priv->base + EIP197_PE_ICE_RAM_CTRL); + writel(0, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL); /* Release engine from reset */ - val = readl(priv->base + ctrl); + val = readl(EIP197_PE(priv) + ctrl); val &= ~EIP197_PE_ICE_x_CTRL_SW_RESET; - writel(val, priv->base + ctrl); + writel(val, EIP197_PE(priv) + ctrl); } static int eip197_load_firmwares(struct safexcel_crypto_priv *priv) @@ -145,14 +145,14 @@ static int eip197_load_firmwares(struct safexcel_crypto_priv *priv) } /* Clear the scratchpad memory */ - val = readl(priv->base + EIP197_PE_ICE_SCRATCH_CTRL); + val = readl(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL); val |= EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER | EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN | EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS | EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS; - writel(val, priv->base + EIP197_PE_ICE_SCRATCH_CTRL); + writel(val, EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL); - memset(priv->base + EIP197_PE_ICE_SCRATCH_RAM, 0, + memset(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_RAM, 0, EIP197_NUM_OF_SCRATCH_BLOCKS * sizeof(u32)); eip197_write_firmware(priv, fw[FW_IFPP], EIP197_PE_ICE_FPP_CTRL, @@ -173,7 +173,7 @@ static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv) u32 hdw, cd_size_rnd, val; int i; - hdw = readl(priv->base + EIP197_HIA_OPTIONS); + hdw = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS); hdw &= GENMASK(27, 25); hdw >>= 25; @@ -182,26 +182,25 @@ static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv) for (i = 0; i < priv->config.rings; i++) { /* ring base address */ writel(lower_32_bits(priv->ring[i].cdr.base_dma), - priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO); + EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO); writel(upper_32_bits(priv->ring[i].cdr.base_dma), - priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); + EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.cd_offset << 16) | priv->config.cd_size, - priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_DESC_SIZE); + EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE); writel(((EIP197_FETCH_COUNT * (cd_size_rnd << hdw)) << 16) | (EIP197_FETCH_COUNT * priv->config.cd_offset), - priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_CFG); + EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG); /* Configure DMA tx control */ val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS); val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS); - writel(val, - priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_DMA_CFG); + writel(val, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DMA_CFG); /* clear any pending interrupt */ writel(GENMASK(5, 0), - priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_STAT); + EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT); } return 0; @@ -212,7 +211,7 @@ static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv) u32 hdw, rd_size_rnd, val; int i; - hdw = readl(priv->base + EIP197_HIA_OPTIONS); + hdw = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS); hdw &= GENMASK(27, 25); hdw >>= 25; @@ -221,33 +220,33 @@ static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv) for (i = 0; i < priv->config.rings; i++) { /* ring base address */ writel(lower_32_bits(priv->ring[i].rdr.base_dma), - priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO); + EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO); writel(upper_32_bits(priv->ring[i].rdr.base_dma), - priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); + EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.rd_offset << 16) | priv->config.rd_size, - priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_DESC_SIZE); + EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE); writel(((EIP197_FETCH_COUNT * (rd_size_rnd << hdw)) << 16) | (EIP197_FETCH_COUNT * priv->config.rd_offset), - priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_CFG); + EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG); /* Configure DMA tx control */ val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS); val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS); val |= EIP197_HIA_xDR_WR_RES_BUF | EIP197_HIA_xDR_WR_CTRL_BUG; writel(val, - priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_DMA_CFG); + EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DMA_CFG); /* clear any pending interrupt */ writel(GENMASK(7, 0), - priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_STAT); + EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT); /* enable ring interrupt */ - val = readl(priv->base + EIP197_HIA_AIC_R_ENABLE_CTRL(i)); + val = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i)); val |= EIP197_RDR_IRQ(i); - writel(val, priv->base + EIP197_HIA_AIC_R_ENABLE_CTRL(i)); + writel(val, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i)); } return 0; @@ -259,39 +258,40 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv) int i, ret; /* Determine endianess and configure byte swap */ - version = readl(priv->base + EIP197_HIA_VERSION); - val = readl(priv->base + EIP197_HIA_MST_CTRL); + version = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_VERSION); + val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL); if ((version & 0xffff) == EIP197_HIA_VERSION_BE) val |= EIP197_MST_CTRL_BYTE_SWAP; else if (((version >> 16) & 0xffff) == EIP197_HIA_VERSION_LE) val |= (EIP197_MST_CTRL_NO_BYTE_SWAP >> 24); - writel(val, priv->base + EIP197_HIA_MST_CTRL); - + writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL); /* Configure wr/rd cache values */ writel(EIP197_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) | EIP197_MST_CTRL_WD_CACHE(WR_CACHE_4BITS), - priv->base + EIP197_MST_CTRL); + EIP197_HIA_GEN_CFG(priv) + EIP197_MST_CTRL); /* Interrupts reset */ /* Disable all global interrupts */ - writel(0, priv->base + EIP197_HIA_AIC_G_ENABLE_CTRL); + writel(0, EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ENABLE_CTRL); /* Clear any pending interrupt */ - writel(GENMASK(31, 0), priv->base + EIP197_HIA_AIC_G_ACK); + writel(GENMASK(31, 0), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK); /* Data Fetch Engine configuration */ /* Reset all DFE threads */ writel(EIP197_DxE_THR_CTRL_RESET_PE, - priv->base + EIP197_HIA_DFE_THR_CTRL); + EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL); - /* Reset HIA input interface arbiter */ - writel(EIP197_HIA_RA_PE_CTRL_RESET, - priv->base + EIP197_HIA_RA_PE_CTRL); + if (priv->version == EIP197) { + /* Reset HIA input interface arbiter */ + writel(EIP197_HIA_RA_PE_CTRL_RESET, + EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL); + } /* DMA transfer size to use */ val = EIP197_HIA_DFE_CFG_DIS_DEBUG; @@ -299,29 +299,32 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv) val |= EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(5) | EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(7); val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS); val |= EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS); - writel(val, priv->base + EIP197_HIA_DFE_CFG); + writel(val, EIP197_HIA_DFE(priv) + EIP197_HIA_DFE_CFG); /* Leave the DFE threads reset state */ - writel(0, priv->base + EIP197_HIA_DFE_THR_CTRL); + writel(0, EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL); /* Configure the procesing engine thresholds */ writel(EIP197_PE_IN_xBUF_THRES_MIN(5) | EIP197_PE_IN_xBUF_THRES_MAX(9), - priv->base + EIP197_PE_IN_DBUF_THRES); + EIP197_PE(priv) + EIP197_PE_IN_DBUF_THRES); writel(EIP197_PE_IN_xBUF_THRES_MIN(5) | EIP197_PE_IN_xBUF_THRES_MAX(7), - priv->base + EIP197_PE_IN_TBUF_THRES); + EIP197_PE(priv) + EIP197_PE_IN_TBUF_THRES); - /* enable HIA input interface arbiter and rings */ - writel(EIP197_HIA_RA_PE_CTRL_EN | GENMASK(priv->config.rings - 1, 0), - priv->base + EIP197_HIA_RA_PE_CTRL); + if (priv->version == EIP197) { + /* enable HIA input interface arbiter and rings */ + writel(EIP197_HIA_RA_PE_CTRL_EN | + GENMASK(priv->config.rings - 1, 0), + EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL); + } /* Data Store Engine configuration */ /* Reset all DSE threads */ writel(EIP197_DxE_THR_CTRL_RESET_PE, - priv->base + EIP197_HIA_DSE_THR_CTRL); + EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL); /* Wait for all DSE threads to complete */ - while ((readl(priv->base + EIP197_HIA_DSE_THR_STAT) & + while ((readl(EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_STAT) & GENMASK(15, 12)) != GENMASK(15, 12)) ; @@ -330,15 +333,19 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv) val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(7) | EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(8); val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS); val |= EIP197_HIA_DSE_CFG_ALLWAYS_BUFFERABLE; - val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR; - writel(val, priv->base + EIP197_HIA_DSE_CFG); + /* FIXME: instability issues can occur for EIP97 but disabling it impact + * performances. + */ + if (priv->version == EIP197) + val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR; + writel(val, EIP197_HIA_DSE(priv) + EIP197_HIA_DSE_CFG); /* Leave the DSE threads reset state */ - writel(0, priv->base + EIP197_HIA_DSE_THR_CTRL); + writel(0, EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL); /* Configure the procesing engine thresholds */ writel(EIP197_PE_OUT_DBUF_THRES_MIN(7) | EIP197_PE_OUT_DBUF_THRES_MAX(8), - priv->base + EIP197_PE_OUT_DBUF_THRES); + EIP197_PE(priv) + EIP197_PE_OUT_DBUF_THRES); /* Processing Engine configuration */ @@ -348,73 +355,75 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv) val |= EIP197_ALG_AES_ECB | EIP197_ALG_AES_CBC; val |= EIP197_ALG_SHA1 | EIP197_ALG_HMAC_SHA1; val |= EIP197_ALG_SHA2; - writel(val, priv->base + EIP197_PE_EIP96_FUNCTION_EN); + writel(val, EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION_EN); /* Command Descriptor Rings prepare */ for (i = 0; i < priv->config.rings; i++) { /* Clear interrupts for this ring */ writel(GENMASK(31, 0), - priv->base + EIP197_HIA_AIC_R_ENABLE_CLR(i)); + EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CLR(i)); /* Disable external triggering */ - writel(0, priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_CFG); + writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG); /* Clear the pending prepared counter */ writel(EIP197_xDR_PREP_CLR_COUNT, - priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PREP_COUNT); + EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT); /* Clear the pending processed counter */ writel(EIP197_xDR_PROC_CLR_COUNT, - priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PROC_COUNT); + EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT); writel(0, - priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PREP_PNTR); + EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR); writel(0, - priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PROC_PNTR); + EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR); writel((EIP197_DEFAULT_RING_SIZE * priv->config.cd_offset) << 2, - priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_RING_SIZE); + EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_SIZE); } /* Result Descriptor Ring prepare */ for (i = 0; i < priv->config.rings; i++) { /* Disable external triggering*/ - writel(0, priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_CFG); + writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG); /* Clear the pending prepared counter */ writel(EIP197_xDR_PREP_CLR_COUNT, - priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PREP_COUNT); + EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT); /* Clear the pending processed counter */ writel(EIP197_xDR_PROC_CLR_COUNT, - priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PROC_COUNT); + EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT); writel(0, - priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PREP_PNTR); + EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR); writel(0, - priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PROC_PNTR); + EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR); /* Ring size */ writel((EIP197_DEFAULT_RING_SIZE * priv->config.rd_offset) << 2, - priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_RING_SIZE); + EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_SIZE); } /* Enable command descriptor rings */ writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0), - priv->base + EIP197_HIA_DFE_THR_CTRL); + EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL); /* Enable result descriptor rings */ writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0), - priv->base + EIP197_HIA_DSE_THR_CTRL); + EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL); /* Clear any HIA interrupt */ - writel(GENMASK(30, 20), priv->base + EIP197_HIA_AIC_G_ACK); + writel(GENMASK(30, 20), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK); - eip197_trc_cache_init(priv); + if (priv->version == EIP197) { + eip197_trc_cache_init(priv); - ret = eip197_load_firmwares(priv); - if (ret) - return ret; + ret = eip197_load_firmwares(priv); + if (ret) + return ret; + } safexcel_hw_setup_cdesc_rings(priv); safexcel_hw_setup_rdesc_rings(priv); @@ -434,7 +443,7 @@ int safexcel_try_push_requests(struct safexcel_crypto_priv *priv, int ring, /* Configure when we want an interrupt */ writel(EIP197_HIA_RDR_THRESH_PKT_MODE | EIP197_HIA_RDR_THRESH_PROC_PKT(coal), - priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_THRESH); + EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_THRESH); return coal; } @@ -515,11 +524,11 @@ finalize: /* let the RDR know we have pending descriptors */ writel((rdesc * priv->config.rd_offset) << 2, - priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PREP_COUNT); + EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT); /* let the CDR know we have pending descriptors */ writel((cdesc * priv->config.cd_offset) << 2, - priv->base + EIP197_HIA_CDR(ring) + EIP197_HIA_xDR_PREP_COUNT); + EIP197_HIA_CDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT); } void safexcel_free_context(struct safexcel_crypto_priv *priv, @@ -620,7 +629,7 @@ static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv handle_results: tot_descs = 0; - nreq = readl(priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PROC_COUNT); + nreq = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT); nreq >>= EIP197_xDR_PROC_xD_PKT_OFFSET; nreq &= EIP197_xDR_PROC_xD_PKT_MASK; if (!nreq) @@ -656,7 +665,7 @@ acknowledge: if (i) { writel(EIP197_xDR_PROC_xD_PKT(i) | EIP197_xDR_PROC_xD_COUNT(tot_descs * priv->config.rd_offset), - priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PROC_COUNT); + EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT); } /* If the number of requests overflowed the counter, try to proceed more @@ -698,13 +707,13 @@ static irqreturn_t safexcel_irq_ring(int irq, void *data) int ring = irq_data->ring, rc = IRQ_NONE; u32 status, stat; - status = readl(priv->base + EIP197_HIA_AIC_R_ENABLED_STAT(ring)); + status = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLED_STAT(ring)); if (!status) return rc; /* RDR interrupts */ if (status & EIP197_RDR_IRQ(ring)) { - stat = readl(priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_STAT); + stat = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT); if (unlikely(stat & EIP197_xDR_ERR)) { /* @@ -719,11 +728,11 @@ static irqreturn_t safexcel_irq_ring(int irq, void *data) /* ACK the interrupts */ writel(stat & 0xff, - priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_STAT); + EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT); } /* ACK the interrupts */ - writel(status, priv->base + EIP197_HIA_AIC_R_ACK(ring)); + writel(status, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ACK(ring)); return rc; } @@ -819,11 +828,11 @@ static void safexcel_configure(struct safexcel_crypto_priv *priv) { u32 val, mask; - val = readl(priv->base + EIP197_HIA_OPTIONS); + val = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS); val = (val & GENMASK(27, 25)) >> 25; mask = BIT(val) - 1; - val = readl(priv->base + EIP197_HIA_OPTIONS); + val = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS); priv->config.rings = min_t(u32, val & GENMASK(3, 0), max_rings); priv->config.cd_size = (sizeof(struct safexcel_command_desc) / sizeof(u32)); @@ -833,6 +842,35 @@ static void safexcel_configure(struct safexcel_crypto_priv *priv) priv->config.rd_offset = (priv->config.rd_size + mask) & ~mask; } +static void safexcel_init_register_offsets(struct safexcel_crypto_priv *priv) +{ + struct safexcel_register_offsets *offsets = &priv->offsets; + + if (priv->version == EIP197) { + offsets->hia_aic = EIP197_HIA_AIC_BASE; + offsets->hia_aic_g = EIP197_HIA_AIC_G_BASE; + offsets->hia_aic_r = EIP197_HIA_AIC_R_BASE; + offsets->hia_aic_xdr = EIP197_HIA_AIC_xDR_BASE; + offsets->hia_dfe = EIP197_HIA_DFE_BASE; + offsets->hia_dfe_thr = EIP197_HIA_DFE_THR_BASE; + offsets->hia_dse = EIP197_HIA_DSE_BASE; + offsets->hia_dse_thr = EIP197_HIA_DSE_THR_BASE; + offsets->hia_gen_cfg = EIP197_HIA_GEN_CFG_BASE; + offsets->pe = EIP197_PE_BASE; + } else { + offsets->hia_aic = EIP97_HIA_AIC_BASE; + offsets->hia_aic_g = EIP97_HIA_AIC_G_BASE; + offsets->hia_aic_r = EIP97_HIA_AIC_R_BASE; + offsets->hia_aic_xdr = EIP97_HIA_AIC_xDR_BASE; + offsets->hia_dfe = EIP97_HIA_DFE_BASE; + offsets->hia_dfe_thr = EIP97_HIA_DFE_THR_BASE; + offsets->hia_dse = EIP97_HIA_DSE_BASE; + offsets->hia_dse_thr = EIP97_HIA_DSE_THR_BASE; + offsets->hia_gen_cfg = EIP97_HIA_GEN_CFG_BASE; + offsets->pe = EIP97_PE_BASE; + } +} + static int safexcel_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -845,6 +883,9 @@ static int safexcel_probe(struct platform_device *pdev) return -ENOMEM; priv->dev = dev; + priv->version = (enum safexcel_eip_version)of_device_get_match_data(dev); + + safexcel_init_register_offsets(priv); res = platform_get_resource(pdev, IORESOURCE_MEM, 0); priv->base = devm_ioremap_resource(dev, res); @@ -971,7 +1012,14 @@ static int safexcel_remove(struct platform_device *pdev) } static const struct of_device_id safexcel_of_match_table[] = { - { .compatible = "inside-secure,safexcel-eip197" }, + { + .compatible = "inside-secure,safexcel-eip97", + .data = (void *)EIP97, + }, + { + .compatible = "inside-secure,safexcel-eip197", + .data = (void *)EIP197, + }, {}, }; diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index d4955abf873b..4e219c21608b 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -28,55 +28,94 @@ #define EIP197_GFP_FLAGS(base) ((base).flags & CRYPTO_TFM_REQ_MAY_SLEEP ? \ GFP_KERNEL : GFP_ATOMIC) +/* Register base offsets */ +#define EIP197_HIA_AIC(priv) ((priv)->base + (priv)->offsets.hia_aic) +#define EIP197_HIA_AIC_G(priv) ((priv)->base + (priv)->offsets.hia_aic_g) +#define EIP197_HIA_AIC_R(priv) ((priv)->base + (priv)->offsets.hia_aic_r) +#define EIP197_HIA_AIC_xDR(priv) ((priv)->base + (priv)->offsets.hia_aic_xdr) +#define EIP197_HIA_DFE(priv) ((priv)->base + (priv)->offsets.hia_dfe) +#define EIP197_HIA_DFE_THR(priv) ((priv)->base + (priv)->offsets.hia_dfe_thr) +#define EIP197_HIA_DSE(priv) ((priv)->base + (priv)->offsets.hia_dse) +#define EIP197_HIA_DSE_THR(priv) ((priv)->base + (priv)->offsets.hia_dse_thr) +#define EIP197_HIA_GEN_CFG(priv) ((priv)->base + (priv)->offsets.hia_gen_cfg) +#define EIP197_PE(priv) ((priv)->base + (priv)->offsets.pe) + +/* EIP197 base offsets */ +#define EIP197_HIA_AIC_BASE 0x90000 +#define EIP197_HIA_AIC_G_BASE 0x90000 +#define EIP197_HIA_AIC_R_BASE 0x90800 +#define EIP197_HIA_AIC_xDR_BASE 0x80000 +#define EIP197_HIA_DFE_BASE 0x8c000 +#define EIP197_HIA_DFE_THR_BASE 0x8c040 +#define EIP197_HIA_DSE_BASE 0x8d000 +#define EIP197_HIA_DSE_THR_BASE 0x8d040 +#define EIP197_HIA_GEN_CFG_BASE 0xf0000 +#define EIP197_PE_BASE 0xa0000 + +/* EIP97 base offsets */ +#define EIP97_HIA_AIC_BASE 0x0 +#define EIP97_HIA_AIC_G_BASE 0x0 +#define EIP97_HIA_AIC_R_BASE 0x0 +#define EIP97_HIA_AIC_xDR_BASE 0x0 +#define EIP97_HIA_DFE_BASE 0xf000 +#define EIP97_HIA_DFE_THR_BASE 0xf200 +#define EIP97_HIA_DSE_BASE 0xf400 +#define EIP97_HIA_DSE_THR_BASE 0xf600 +#define EIP97_HIA_GEN_CFG_BASE 0x10000 +#define EIP97_PE_BASE 0x10000 + /* CDR/RDR register offsets */ -#define EIP197_HIA_xDR_OFF(r) (0x80000 + (r) * 0x1000) -#define EIP197_HIA_CDR(r) (EIP197_HIA_xDR_OFF(r)) -#define EIP197_HIA_RDR(r) (EIP197_HIA_xDR_OFF(r) + 0x800) -#define EIP197_HIA_xDR_RING_BASE_ADDR_LO 0x0 -#define EIP197_HIA_xDR_RING_BASE_ADDR_HI 0x4 -#define EIP197_HIA_xDR_RING_SIZE 0x18 -#define EIP197_HIA_xDR_DESC_SIZE 0x1c -#define EIP197_HIA_xDR_CFG 0x20 -#define EIP197_HIA_xDR_DMA_CFG 0x24 -#define EIP197_HIA_xDR_THRESH 0x28 -#define EIP197_HIA_xDR_PREP_COUNT 0x2c -#define EIP197_HIA_xDR_PROC_COUNT 0x30 -#define EIP197_HIA_xDR_PREP_PNTR 0x34 -#define EIP197_HIA_xDR_PROC_PNTR 0x38 -#define EIP197_HIA_xDR_STAT 0x3c +#define EIP197_HIA_xDR_OFF(priv, r) (EIP197_HIA_AIC_xDR(priv) + (r) * 0x1000) +#define EIP197_HIA_CDR(priv, r) (EIP197_HIA_xDR_OFF(priv, r)) +#define EIP197_HIA_RDR(priv, r) (EIP197_HIA_xDR_OFF(priv, r) + 0x800) +#define EIP197_HIA_xDR_RING_BASE_ADDR_LO 0x0000 +#define EIP197_HIA_xDR_RING_BASE_ADDR_HI 0x0004 +#define EIP197_HIA_xDR_RING_SIZE 0x0018 +#define EIP197_HIA_xDR_DESC_SIZE 0x001c +#define EIP197_HIA_xDR_CFG 0x0020 +#define EIP197_HIA_xDR_DMA_CFG 0x0024 +#define EIP197_HIA_xDR_THRESH 0x0028 +#define EIP197_HIA_xDR_PREP_COUNT 0x002c +#define EIP197_HIA_xDR_PROC_COUNT 0x0030 +#define EIP197_HIA_xDR_PREP_PNTR 0x0034 +#define EIP197_HIA_xDR_PROC_PNTR 0x0038 +#define EIP197_HIA_xDR_STAT 0x003c /* register offsets */ -#define EIP197_HIA_DFE_CFG 0x8c000 -#define EIP197_HIA_DFE_THR_CTRL 0x8c040 -#define EIP197_HIA_DFE_THR_STAT 0x8c044 -#define EIP197_HIA_DSE_CFG 0x8d000 -#define EIP197_HIA_DSE_THR_CTRL 0x8d040 -#define EIP197_HIA_DSE_THR_STAT 0x8d044 -#define EIP197_HIA_RA_PE_CTRL 0x90010 -#define EIP197_HIA_RA_PE_STAT 0x90014 +#define EIP197_HIA_DFE_CFG 0x0000 +#define EIP197_HIA_DFE_THR_CTRL 0x0000 +#define EIP197_HIA_DFE_THR_STAT 0x0004 +#define EIP197_HIA_DSE_CFG 0x0000 +#define EIP197_HIA_DSE_THR_CTRL 0x0000 +#define EIP197_HIA_DSE_THR_STAT 0x0004 +#define EIP197_HIA_RA_PE_CTRL 0x0010 +#define EIP197_HIA_RA_PE_STAT 0x0014 #define EIP197_HIA_AIC_R_OFF(r) ((r) * 0x1000) -#define EIP197_HIA_AIC_R_ENABLE_CTRL(r) (0x9e808 - EIP197_HIA_AIC_R_OFF(r)) -#define EIP197_HIA_AIC_R_ENABLED_STAT(r) (0x9e810 - EIP197_HIA_AIC_R_OFF(r)) -#define EIP197_HIA_AIC_R_ACK(r) (0x9e810 - EIP197_HIA_AIC_R_OFF(r)) -#define EIP197_HIA_AIC_R_ENABLE_CLR(r) (0x9e814 - EIP197_HIA_AIC_R_OFF(r)) -#define EIP197_HIA_AIC_G_ENABLE_CTRL 0x9f808 -#define EIP197_HIA_AIC_G_ENABLED_STAT 0x9f810 -#define EIP197_HIA_AIC_G_ACK 0x9f810 -#define EIP197_HIA_MST_CTRL 0x9fff4 -#define EIP197_HIA_OPTIONS 0x9fff8 -#define EIP197_HIA_VERSION 0x9fffc -#define EIP197_PE_IN_DBUF_THRES 0xa0000 -#define EIP197_PE_IN_TBUF_THRES 0xa0100 -#define EIP197_PE_ICE_SCRATCH_RAM 0xa0800 -#define EIP197_PE_ICE_PUE_CTRL 0xa0c80 -#define EIP197_PE_ICE_SCRATCH_CTRL 0xa0d04 -#define EIP197_PE_ICE_FPP_CTRL 0xa0d80 -#define EIP197_PE_ICE_RAM_CTRL 0xa0ff0 -#define EIP197_PE_EIP96_FUNCTION_EN 0xa1004 -#define EIP197_PE_EIP96_CONTEXT_CTRL 0xa1008 -#define EIP197_PE_EIP96_CONTEXT_STAT 0xa100c -#define EIP197_PE_OUT_DBUF_THRES 0xa1c00 -#define EIP197_PE_OUT_TBUF_THRES 0xa1d00 +#define EIP197_HIA_AIC_R_ENABLE_CTRL(r) (0xe008 - EIP197_HIA_AIC_R_OFF(r)) +#define EIP197_HIA_AIC_R_ENABLED_STAT(r) (0xe010 - EIP197_HIA_AIC_R_OFF(r)) +#define EIP197_HIA_AIC_R_ACK(r) (0xe010 - EIP197_HIA_AIC_R_OFF(r)) +#define EIP197_HIA_AIC_R_ENABLE_CLR(r) (0xe014 - EIP197_HIA_AIC_R_OFF(r)) +#define EIP197_HIA_AIC_G_ENABLE_CTRL 0xf808 +#define EIP197_HIA_AIC_G_ENABLED_STAT 0xf810 +#define EIP197_HIA_AIC_G_ACK 0xf810 +#define EIP197_HIA_MST_CTRL 0xfff4 +#define EIP197_HIA_OPTIONS 0xfff8 +#define EIP197_HIA_VERSION 0xfffc +#define EIP197_PE_IN_DBUF_THRES 0x0000 +#define EIP197_PE_IN_TBUF_THRES 0x0100 +#define EIP197_PE_ICE_SCRATCH_RAM 0x0800 +#define EIP197_PE_ICE_PUE_CTRL 0x0c80 +#define EIP197_PE_ICE_SCRATCH_CTRL 0x0d04 +#define EIP197_PE_ICE_FPP_CTRL 0x0d80 +#define EIP197_PE_ICE_RAM_CTRL 0x0ff0 +#define EIP197_PE_EIP96_FUNCTION_EN 0x1004 +#define EIP197_PE_EIP96_CONTEXT_CTRL 0x1008 +#define EIP197_PE_EIP96_CONTEXT_STAT 0x100c +#define EIP197_PE_OUT_DBUF_THRES 0x1c00 +#define EIP197_PE_OUT_TBUF_THRES 0x1d00 +#define EIP197_MST_CTRL 0xfff4 + +/* EIP197-specific registers, no indirection */ #define EIP197_CLASSIFICATION_RAMS 0xe0000 #define EIP197_TRC_CTRL 0xf0800 #define EIP197_TRC_LASTRES 0xf0804 @@ -90,7 +129,6 @@ #define EIP197_TRC_ECCDATASTAT 0xf083c #define EIP197_TRC_ECCDATA 0xf0840 #define EIP197_CS_RAM_CTRL 0xf7ff0 -#define EIP197_MST_CTRL 0xffff4 /* EIP197_HIA_xDR_DESC_SIZE */ #define EIP197_xDR_DESC_MODE_64BIT BIT(31) @@ -465,12 +503,33 @@ struct safexcel_work_data { int ring; }; +enum safexcel_eip_version { + EIP97, + EIP197, +}; + +struct safexcel_register_offsets { + u32 hia_aic; + u32 hia_aic_g; + u32 hia_aic_r; + u32 hia_aic_xdr; + u32 hia_dfe; + u32 hia_dfe_thr; + u32 hia_dse; + u32 hia_dse_thr; + u32 hia_gen_cfg; + u32 pe; +}; + struct safexcel_crypto_priv { void __iomem *base; struct device *dev; struct clk *clk; struct safexcel_config config; + enum safexcel_eip_version version; + struct safexcel_register_offsets offsets; + /* context DMA pool */ struct dma_pool *context_pool; diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 0e5cc230e49a..63a8768ed2ae 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -69,6 +69,7 @@ static int safexcel_aes_setkey(struct crypto_skcipher *ctfm, const u8 *key, { struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_crypto_priv *priv = ctx->priv; struct crypto_aes_ctx aes; int ret, i; @@ -78,7 +79,7 @@ static int safexcel_aes_setkey(struct crypto_skcipher *ctfm, const u8 *key, return ret; } - if (ctx->base.ctxr_dma) { + if (priv->version == EIP197 && ctx->base.ctxr_dma) { for (i = 0; i < len / sizeof(u32); i++) { if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) { ctx->base.needs_inv = true; @@ -411,9 +412,13 @@ static int safexcel_send(struct crypto_async_request *async, int *commands, int *results) { struct skcipher_request *req = skcipher_request_cast(async); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); + struct safexcel_crypto_priv *priv = ctx->priv; int ret; + BUG_ON(priv->version == EIP97 && sreq->needs_inv); + if (sreq->needs_inv) ret = safexcel_cipher_send_inv(async, ring, request, commands, results); @@ -476,7 +481,7 @@ static int safexcel_aes(struct skcipher_request *req, ctx->mode = mode; if (ctx->base.ctxr) { - if (ctx->base.needs_inv) { + if (priv->version == EIP197 && ctx->base.needs_inv) { sreq->needs_inv = true; ctx->base.needs_inv = false; } @@ -544,9 +549,14 @@ static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm) memzero_explicit(ctx->base.ctxr->data, 8 * sizeof(u32)); - ret = safexcel_cipher_exit_inv(tfm); - if (ret) - dev_warn(priv->dev, "cipher: invalidation error %d\n", ret); + if (priv->version == EIP197) { + ret = safexcel_cipher_exit_inv(tfm); + if (ret) + dev_warn(priv->dev, "cipher: invalidation error %d\n", ret); + } else { + dma_pool_free(priv->context_pool, ctx->base.ctxr, + ctx->base.ctxr_dma); + } } struct safexcel_alg_template safexcel_alg_ecb_aes = { diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 2fb5bc6b6268..d94614afc53d 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -397,6 +397,8 @@ static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring, struct safexcel_ahash_req *req = ahash_request_ctx(areq); int err; + BUG_ON(priv->version == EIP97 && req->needs_inv); + if (req->needs_inv) { req->needs_inv = false; err = safexcel_handle_inv_result(priv, ring, async, @@ -528,7 +530,8 @@ static int safexcel_ahash_enqueue(struct ahash_request *areq) req->needs_inv = false; if (ctx->base.ctxr) { - if (!ctx->base.needs_inv && req->processed && + if (priv->version == EIP197 && + !ctx->base.needs_inv && req->processed && ctx->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) /* We're still setting needs_inv here, even though it is * cleared right away, because the needs_inv flag can be @@ -721,9 +724,14 @@ static void safexcel_ahash_cra_exit(struct crypto_tfm *tfm) if (!ctx->base.ctxr) return; - ret = safexcel_ahash_exit_inv(tfm); - if (ret) - dev_warn(priv->dev, "hash: invalidation error %d\n", ret); + if (priv->version == EIP197) { + ret = safexcel_ahash_exit_inv(tfm); + if (ret) + dev_warn(priv->dev, "hash: invalidation error %d\n", ret); + } else { + dma_pool_free(priv->context_pool, ctx->base.ctxr, + ctx->base.ctxr_dma); + } } struct safexcel_alg_template safexcel_alg_sha1 = { @@ -927,6 +935,7 @@ static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); + struct safexcel_crypto_priv *priv = ctx->priv; struct safexcel_ahash_export_state istate, ostate; int ret, i; @@ -934,7 +943,7 @@ static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key, if (ret) return ret; - if (ctx->base.ctxr) { + if (priv->version == EIP197 && ctx->base.ctxr) { for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++) { if (ctx->ipad[i] != le32_to_cpu(istate.state[i]) || ctx->opad[i] != le32_to_cpu(ostate.state[i])) { From 0e4b52942b1c76f89e0dcb829f72e123d0678f54 Mon Sep 17 00:00:00 2001 From: Gary R Hook Date: Fri, 15 Dec 2017 13:55:59 -0600 Subject: [PATCH 079/166] hwrng: core - Clean up RNG list when last hwrng is unregistered Commit 142a27f0a731 added support for a "best" RNG, and in doing so introduced a hang from rmmod/modprobe -r when the last RNG on the list was unloaded. When the hwrng list is depleted, return the global variables to their original state and decrement all references to the object. Fixes: 142a27f0a731 ("hwrng: core - Reset user selected rng by writing "" to rng_current") Signed-off-by: Gary R Hook Reviewed-by: PrasannaKumar Muralidharan Signed-off-by: Herbert Xu --- drivers/char/hw_random/core.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index 657b8770b6b9..91bb98c42a1c 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -306,6 +306,10 @@ static int enable_best_rng(void) ret = ((new_rng == current_rng) ? 0 : set_current_rng(new_rng)); if (!ret) cur_rng_set_by_user = 0; + } else { + drop_current_rng(); + cur_rng_set_by_user = 0; + ret = 0; } return ret; From 7c3f13238992884abf3782dcc8e97e0862e5c23e Mon Sep 17 00:00:00 2001 From: Gilad Ben-Yossef Date: Sun, 17 Dec 2017 08:29:00 +0000 Subject: [PATCH 080/166] crypto: tcrypt - use multi buf for ahash mb test The multi buffer ahash speed test was allocating multiple buffers for use with the multiple outstanding requests it was starting but never actually using them (except to free them), instead using a different single statically allocated buffer for all requests. Fix this by actually using the allocated buffers for the test. It is noted that it may seem tempting to instead remove the allocation and free of the multiple buffers and leave things as they are since this is a hash test where the input is read only. However, after consideration I believe that multiple buffers better reflect real life scenario with regard to data cache and TLB behaviours etc. Signed-off-by: Gilad Ben-Yossef Signed-off-by: Herbert Xu --- crypto/tcrypt.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 28b4882f1944..a0c4e0dbffa4 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -385,7 +385,7 @@ static inline int do_one_ahash_op(struct ahash_request *req, int ret) } struct test_mb_ahash_data { - struct scatterlist sg[TVMEMSIZE]; + struct scatterlist sg[XBUFSIZE]; char result[64]; struct ahash_request *req; struct crypto_wait wait; @@ -428,7 +428,12 @@ static void test_mb_ahash_speed(const char *algo, unsigned int sec, ahash_request_set_callback(data[i].req, 0, crypto_req_done, &data[i].wait); - test_hash_sg_init(data[i].sg); + + sg_init_table(data[i].sg, XBUFSIZE); + for (j = 0; j < XBUFSIZE; j++) { + sg_set_buf(data[i].sg + j, data[i].xbuf[j], PAGE_SIZE); + memset(data[i].xbuf[j], 0xff, PAGE_SIZE); + } } pr_info("\ntesting speed of multibuffer %s (%s)\n", algo, @@ -439,9 +444,9 @@ static void test_mb_ahash_speed(const char *algo, unsigned int sec, if (speed[i].blen != speed[i].plen) continue; - if (speed[i].blen > TVMEMSIZE * PAGE_SIZE) { + if (speed[i].blen > XBUFSIZE * PAGE_SIZE) { pr_err("template (%u) too big for tvmem (%lu)\n", - speed[i].blen, TVMEMSIZE * PAGE_SIZE); + speed[i].blen, XBUFSIZE * PAGE_SIZE); goto out; } From 4431bd49530c7379dffaf0963d69bdab7fbead05 Mon Sep 17 00:00:00 2001 From: Gilad Ben-Yossef Date: Sun, 17 Dec 2017 08:29:01 +0000 Subject: [PATCH 081/166] crypto: tcrypt - fix AEAD decryption speed test The AEAD speed test pretended to support decryption, however that support was broken as decryption requires a valid auth field which the test did not provide. Fix this by running the encryption path once with inout/output sgls switched to calculate the auth field prior to performing decryption speed tests. Signed-off-by: Gilad Ben-Yossef Signed-off-by: Herbert Xu --- crypto/tcrypt.c | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index a0c4e0dbffa4..69c8e6392ffc 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -329,10 +329,30 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, *b_size + (enc ? authsize : 0), assoc, aad_size); + aead_request_set_ad(req, aad_size); + + if (!enc) { + + /* + * For decryption we need a proper auth so + * we do the encryption path once with buffers + * reversed (input <-> output) to calculate it + */ + aead_request_set_crypt(req, sgout, sg, + *b_size, iv); + ret = do_one_aead_op(req, + crypto_aead_encrypt(req)); + + if (ret) { + pr_err("calculating auth failed failed (%d)\n", + ret); + break; + } + } + aead_request_set_crypt(req, sg, sgout, *b_size + (enc ? 0 : authsize), iv); - aead_request_set_ad(req, aad_size); if (secs) ret = test_aead_jiffies(req, enc, *b_size, @@ -1566,16 +1586,24 @@ static int do_test(const char *alg, u32 type, u32 mask, int m) NULL, 0, 16, 16, aead_speed_template_20); test_aead_speed("gcm(aes)", ENCRYPT, sec, NULL, 0, 16, 8, speed_template_16_24_32); + test_aead_speed("rfc4106(gcm(aes))", DECRYPT, sec, + NULL, 0, 16, 16, aead_speed_template_20); + test_aead_speed("gcm(aes)", DECRYPT, sec, + NULL, 0, 16, 8, speed_template_16_24_32); break; case 212: test_aead_speed("rfc4309(ccm(aes))", ENCRYPT, sec, NULL, 0, 16, 16, aead_speed_template_19); + test_aead_speed("rfc4309(ccm(aes))", DECRYPT, sec, + NULL, 0, 16, 16, aead_speed_template_19); break; case 213: test_aead_speed("rfc7539esp(chacha20,poly1305)", ENCRYPT, sec, NULL, 0, 16, 8, aead_speed_template_36); + test_aead_speed("rfc7539esp(chacha20,poly1305)", DECRYPT, sec, + NULL, 0, 16, 8, aead_speed_template_36); break; case 214: From 8fcdc86856b24d3f76b4d70665f2cbbe5102d8de Mon Sep 17 00:00:00 2001 From: Gilad Ben-Yossef Date: Sun, 17 Dec 2017 08:29:02 +0000 Subject: [PATCH 082/166] crypto: tcrypt - allow setting num of bufs For multiple buffers speed tests, the number of buffers, or requests, used actually sets the level of parallelism a tfm provider may utilize to hide latency. The existing number (of 8) is good for some software based providers but not enough for many HW providers with deep FIFOs. Add a module parameter that allows setting the number of multiple buffers/requests used, leaving the default at 8. Signed-off-by: Gilad Ben-Yossef Signed-off-by: Herbert Xu --- crypto/tcrypt.c | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 69c8e6392ffc..260436004a76 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -67,6 +67,7 @@ static char *alg = NULL; static u32 type; static u32 mask; static int mode; +static u32 num_mb = 8; static char *tvmem[TVMEMSIZE]; static char *check[] = { @@ -413,7 +414,7 @@ struct test_mb_ahash_data { }; static void test_mb_ahash_speed(const char *algo, unsigned int sec, - struct hash_speed *speed) + struct hash_speed *speed, u32 num_mb) { struct test_mb_ahash_data *data; struct crypto_ahash *tfm; @@ -422,7 +423,7 @@ static void test_mb_ahash_speed(const char *algo, unsigned int sec, unsigned int i, j, k; int ret; - data = kzalloc(sizeof(*data) * 8, GFP_KERNEL); + data = kcalloc(num_mb, sizeof(*data), GFP_KERNEL); if (!data) return; @@ -433,7 +434,7 @@ static void test_mb_ahash_speed(const char *algo, unsigned int sec, goto free_data; } - for (i = 0; i < 8; ++i) { + for (i = 0; i < num_mb; ++i) { if (testmgr_alloc_buf(data[i].xbuf)) goto out; @@ -473,7 +474,7 @@ static void test_mb_ahash_speed(const char *algo, unsigned int sec, if (speed[i].klen) crypto_ahash_setkey(tfm, tvmem[0], speed[i].klen); - for (k = 0; k < 8; k++) + for (k = 0; k < num_mb; k++) ahash_request_set_crypt(data[k].req, data[k].sg, data[k].result, speed[i].blen); @@ -484,7 +485,7 @@ static void test_mb_ahash_speed(const char *algo, unsigned int sec, start = get_cycles(); - for (k = 0; k < 8; k++) { + for (k = 0; k < num_mb; k++) { ret = crypto_ahash_digest(data[k].req); if (ret == -EINPROGRESS) { ret = 0; @@ -509,7 +510,7 @@ static void test_mb_ahash_speed(const char *algo, unsigned int sec, end = get_cycles(); cycles = end - start; pr_cont("%6lu cycles/operation, %4lu cycles/byte\n", - cycles, cycles / (8 * speed[i].blen)); + cycles, cycles / (num_mb * speed[i].blen)); if (ret) { pr_err("At least one hashing failed ret=%d\n", ret); @@ -518,10 +519,10 @@ static void test_mb_ahash_speed(const char *algo, unsigned int sec, } out: - for (k = 0; k < 8; ++k) + for (k = 0; k < num_mb; ++k) ahash_request_free(data[k].req); - for (k = 0; k < 8; ++k) + for (k = 0; k < num_mb; ++k) testmgr_free_buf(data[k].xbuf); crypto_free_ahash(tfm); @@ -1815,19 +1816,23 @@ static int do_test(const char *alg, u32 type, u32 mask, int m) if (mode > 400 && mode < 500) break; /* fall through */ case 422: - test_mb_ahash_speed("sha1", sec, generic_hash_speed_template); + test_mb_ahash_speed("sha1", sec, generic_hash_speed_template, + num_mb); if (mode > 400 && mode < 500) break; /* fall through */ case 423: - test_mb_ahash_speed("sha256", sec, generic_hash_speed_template); + test_mb_ahash_speed("sha256", sec, generic_hash_speed_template, + num_mb); if (mode > 400 && mode < 500) break; /* fall through */ case 424: - test_mb_ahash_speed("sha512", sec, generic_hash_speed_template); + test_mb_ahash_speed("sha512", sec, generic_hash_speed_template, + num_mb); if (mode > 400 && mode < 500) break; /* fall through */ case 425: - test_mb_ahash_speed("sm3", sec, generic_hash_speed_template); + test_mb_ahash_speed("sm3", sec, generic_hash_speed_template, + num_mb); if (mode > 400 && mode < 500) break; /* fall through */ case 499: @@ -2106,6 +2111,8 @@ module_param(mode, int, 0); module_param(sec, uint, 0); MODULE_PARM_DESC(sec, "Length in seconds of speed tests " "(defaults to zero which uses CPU cycles instead)"); +module_param(num_mb, uint, 0000); +MODULE_PARM_DESC(num_mb, "Number of concurrent requests to be used in mb speed tests (defaults to 8)"); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Quick & dirty crypto testing module"); From b34a0f67ba62027394598e3c47fd3549c5c8e294 Mon Sep 17 00:00:00 2001 From: Gilad Ben-Yossef Date: Sun, 17 Dec 2017 08:29:03 +0000 Subject: [PATCH 083/166] crypto: tcrypt - add multi buf ahash jiffies test The multi buffer concurrent requests ahash speed test only supported the cycles mode. Add support for the so called jiffies mode that test performance of bytes/sec. We only add support for digest mode at the moment. Signed-off-by: Gilad Ben-Yossef Signed-off-by: Herbert Xu --- crypto/tcrypt.c | 112 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 82 insertions(+), 30 deletions(-) diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 260436004a76..e406b00db89c 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -413,13 +413,87 @@ struct test_mb_ahash_data { char *xbuf[XBUFSIZE]; }; -static void test_mb_ahash_speed(const char *algo, unsigned int sec, +static inline int do_mult_ahash_op(struct test_mb_ahash_data *data, u32 num_mb) +{ + int i, rc[num_mb], err = 0; + + /* Fire up a bunch of concurrent requests */ + for (i = 0; i < num_mb; i++) + rc[i] = crypto_ahash_digest(data[i].req); + + /* Wait for all requests to finish */ + for (i = 0; i < num_mb; i++) { + rc[i] = crypto_wait_req(rc[i], &data[i].wait); + + if (rc[i]) { + pr_info("concurrent request %d error %d\n", i, rc[i]); + err = rc[i]; + } + } + + return err; +} + +static int test_mb_ahash_jiffies(struct test_mb_ahash_data *data, int blen, + int secs, u32 num_mb) +{ + unsigned long start, end; + int bcount; + int ret; + + for (start = jiffies, end = start + secs * HZ, bcount = 0; + time_before(jiffies, end); bcount++) { + ret = do_mult_ahash_op(data, num_mb); + if (ret) + return ret; + } + + pr_cont("%d operations in %d seconds (%ld bytes)\n", + bcount * num_mb, secs, (long)bcount * blen * num_mb); + return 0; +} + +static int test_mb_ahash_cycles(struct test_mb_ahash_data *data, int blen, + u32 num_mb) +{ + unsigned long cycles = 0; + int ret = 0; + int i; + + /* Warm-up run. */ + for (i = 0; i < 4; i++) { + ret = do_mult_ahash_op(data, num_mb); + if (ret) + goto out; + } + + /* The real thing. */ + for (i = 0; i < 8; i++) { + cycles_t start, end; + + start = get_cycles(); + ret = do_mult_ahash_op(data, num_mb); + end = get_cycles(); + + if (ret) + goto out; + + cycles += end - start; + } + +out: + if (ret == 0) + pr_cont("1 operation in %lu cycles (%d bytes)\n", + (cycles + 4) / (8 * num_mb), blen); + + return ret; +} + +static void test_mb_ahash_speed(const char *algo, unsigned int secs, struct hash_speed *speed, u32 num_mb) { struct test_mb_ahash_data *data; struct crypto_ahash *tfm; - unsigned long start, end; - unsigned long cycles; unsigned int i, j, k; int ret; @@ -483,34 +557,12 @@ static void test_mb_ahash_speed(const char *algo, unsigned int sec, i, speed[i].blen, speed[i].plen, speed[i].blen / speed[i].plen); - start = get_cycles(); + if (secs) + ret = test_mb_ahash_jiffies(data, speed[i].blen, secs, + num_mb); + else + ret = test_mb_ahash_cycles(data, speed[i].blen, num_mb); - for (k = 0; k < num_mb; k++) { - ret = crypto_ahash_digest(data[k].req); - if (ret == -EINPROGRESS) { - ret = 0; - continue; - } - - if (ret) - break; - - crypto_req_done(&data[k].req->base, 0); - } - - for (j = 0; j < k; j++) { - struct crypto_wait *wait = &data[j].wait; - int wait_ret; - - wait_ret = crypto_wait_req(-EINPROGRESS, wait); - if (wait_ret) - ret = wait_ret; - } - - end = get_cycles(); - cycles = end - start; - pr_cont("%6lu cycles/operation, %4lu cycles/byte\n", - cycles, cycles / (num_mb * speed[i].blen)); if (ret) { pr_err("At least one hashing failed ret=%d\n", ret); From e161c5930c150abab95d2ccad428d68ce1780ea1 Mon Sep 17 00:00:00 2001 From: Gilad Ben-Yossef Date: Sun, 17 Dec 2017 08:29:04 +0000 Subject: [PATCH 084/166] crypto: tcrypt - add multibuf skcipher speed test The performance of some skcipher tfm providers is affected by the amount of parallelism possible with the processing. Introduce an async skcipher concurrent multiple buffer processing speed test to be able to test performance of such tfm providers. Signed-off-by: Gilad Ben-Yossef Signed-off-by: Herbert Xu --- crypto/tcrypt.c | 460 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 460 insertions(+) diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index e406b00db89c..d617c1956533 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -818,6 +818,254 @@ static void test_hash_speed(const char *algo, unsigned int secs, return test_ahash_speed_common(algo, secs, speed, CRYPTO_ALG_ASYNC); } +struct test_mb_skcipher_data { + struct scatterlist sg[XBUFSIZE]; + struct skcipher_request *req; + struct crypto_wait wait; + char *xbuf[XBUFSIZE]; +}; + +static int do_mult_acipher_op(struct test_mb_skcipher_data *data, int enc, + u32 num_mb) +{ + int i, rc[num_mb], err = 0; + + /* Fire up a bunch of concurrent requests */ + for (i = 0; i < num_mb; i++) { + if (enc == ENCRYPT) + rc[i] = crypto_skcipher_encrypt(data[i].req); + else + rc[i] = crypto_skcipher_decrypt(data[i].req); + } + + /* Wait for all requests to finish */ + for (i = 0; i < num_mb; i++) { + rc[i] = crypto_wait_req(rc[i], &data[i].wait); + + if (rc[i]) { + pr_info("concurrent request %d error %d\n", i, rc[i]); + err = rc[i]; + } + } + + return err; +} + +static int test_mb_acipher_jiffies(struct test_mb_skcipher_data *data, int enc, + int blen, int secs, u32 num_mb) +{ + unsigned long start, end; + int bcount; + int ret; + + for (start = jiffies, end = start + secs * HZ, bcount = 0; + time_before(jiffies, end); bcount++) { + ret = do_mult_acipher_op(data, enc, num_mb); + if (ret) + return ret; + } + + pr_cont("%d operations in %d seconds (%ld bytes)\n", + bcount * num_mb, secs, (long)bcount * blen * num_mb); + return 0; +} + +static int test_mb_acipher_cycles(struct test_mb_skcipher_data *data, int enc, + int blen, u32 num_mb) +{ + unsigned long cycles = 0; + int ret = 0; + int i; + + /* Warm-up run. */ + for (i = 0; i < 4; i++) { + ret = do_mult_acipher_op(data, enc, num_mb); + if (ret) + goto out; + } + + /* The real thing. */ + for (i = 0; i < 8; i++) { + cycles_t start, end; + + start = get_cycles(); + ret = do_mult_acipher_op(data, enc, num_mb); + end = get_cycles(); + + if (ret) + goto out; + + cycles += end - start; + } + +out: + if (ret == 0) + pr_cont("1 operation in %lu cycles (%d bytes)\n", + (cycles + 4) / (8 * num_mb), blen); + + return ret; +} + +static void test_mb_skcipher_speed(const char *algo, int enc, int secs, + struct cipher_speed_template *template, + unsigned int tcount, u8 *keysize, u32 num_mb) +{ + struct test_mb_skcipher_data *data; + struct crypto_skcipher *tfm; + unsigned int i, j, iv_len; + const char *key; + const char *e; + u32 *b_size; + char iv[128]; + int ret; + + if (enc == ENCRYPT) + e = "encryption"; + else + e = "decryption"; + + data = kcalloc(num_mb, sizeof(*data), GFP_KERNEL); + if (!data) + return; + + tfm = crypto_alloc_skcipher(algo, 0, 0); + if (IS_ERR(tfm)) { + pr_err("failed to load transform for %s: %ld\n", + algo, PTR_ERR(tfm)); + goto out_free_data; + } + + for (i = 0; i < num_mb; ++i) + if (testmgr_alloc_buf(data[i].xbuf)) { + while (i--) + testmgr_free_buf(data[i].xbuf); + goto out_free_tfm; + } + + + for (i = 0; i < num_mb; ++i) + if (testmgr_alloc_buf(data[i].xbuf)) { + while (i--) + testmgr_free_buf(data[i].xbuf); + goto out_free_tfm; + } + + + for (i = 0; i < num_mb; ++i) { + data[i].req = skcipher_request_alloc(tfm, GFP_KERNEL); + if (!data[i].req) { + pr_err("alg: skcipher: Failed to allocate request for %s\n", + algo); + while (i--) + skcipher_request_free(data[i].req); + goto out_free_xbuf; + } + } + + for (i = 0; i < num_mb; ++i) { + skcipher_request_set_callback(data[i].req, + CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &data[i].wait); + crypto_init_wait(&data[i].wait); + } + + pr_info("\ntesting speed of multibuffer %s (%s) %s\n", algo, + get_driver_name(crypto_skcipher, tfm), e); + + i = 0; + do { + b_size = block_sizes; + do { + if (*b_size > XBUFSIZE * PAGE_SIZE) { + pr_err("template (%u) too big for bufufer (%lu)\n", + *b_size, XBUFSIZE * PAGE_SIZE); + goto out; + } + + pr_info("test %u (%d bit key, %d byte blocks): ", i, + *keysize * 8, *b_size); + + /* Set up tfm global state, i.e. the key */ + + memset(tvmem[0], 0xff, PAGE_SIZE); + key = tvmem[0]; + for (j = 0; j < tcount; j++) { + if (template[j].klen == *keysize) { + key = template[j].key; + break; + } + } + + crypto_skcipher_clear_flags(tfm, ~0); + + ret = crypto_skcipher_setkey(tfm, key, *keysize); + if (ret) { + pr_err("setkey() failed flags=%x\n", + crypto_skcipher_get_flags(tfm)); + goto out; + } + + iv_len = crypto_skcipher_ivsize(tfm); + if (iv_len) + memset(&iv, 0xff, iv_len); + + /* Now setup per request stuff, i.e. buffers */ + + for (j = 0; j < num_mb; ++j) { + struct test_mb_skcipher_data *cur = &data[j]; + unsigned int k = *b_size; + unsigned int pages = DIV_ROUND_UP(k, PAGE_SIZE); + unsigned int p = 0; + + sg_init_table(cur->sg, pages); + + while (k > PAGE_SIZE) { + sg_set_buf(cur->sg + p, cur->xbuf[p], + PAGE_SIZE); + memset(cur->xbuf[p], 0xff, PAGE_SIZE); + p++; + k -= PAGE_SIZE; + } + + sg_set_buf(cur->sg + p, cur->xbuf[p], k); + memset(cur->xbuf[p], 0xff, k); + + skcipher_request_set_crypt(cur->req, cur->sg, + cur->sg, *b_size, + iv); + } + + if (secs) + ret = test_mb_acipher_jiffies(data, enc, + *b_size, secs, + num_mb); + else + ret = test_mb_acipher_cycles(data, enc, + *b_size, num_mb); + + if (ret) { + pr_err("%s() failed flags=%x\n", e, + crypto_skcipher_get_flags(tfm)); + break; + } + b_size++; + i++; + } while (*b_size); + keysize++; + } while (*keysize); + +out: + for (i = 0; i < num_mb; ++i) + skcipher_request_free(data[i].req); +out_free_xbuf: + for (i = 0; i < num_mb; ++i) + testmgr_free_buf(data[i].xbuf); +out_free_tfm: + crypto_free_skcipher(tfm); +out_free_data: + kfree(data); +} + static inline int do_one_acipher_op(struct skcipher_request *req, int ret) { struct crypto_wait *wait = req->base.data; @@ -2102,6 +2350,218 @@ static int do_test(const char *alg, u32 type, u32 mask, int m) speed_template_8_32); break; + case 600: + test_mb_skcipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0, + speed_template_16_24_32, num_mb); + test_mb_skcipher_speed("ecb(aes)", DECRYPT, sec, NULL, 0, + speed_template_16_24_32, num_mb); + test_mb_skcipher_speed("cbc(aes)", ENCRYPT, sec, NULL, 0, + speed_template_16_24_32, num_mb); + test_mb_skcipher_speed("cbc(aes)", DECRYPT, sec, NULL, 0, + speed_template_16_24_32, num_mb); + test_mb_skcipher_speed("lrw(aes)", ENCRYPT, sec, NULL, 0, + speed_template_32_40_48, num_mb); + test_mb_skcipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0, + speed_template_32_40_48, num_mb); + test_mb_skcipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0, + speed_template_32_64, num_mb); + test_mb_skcipher_speed("xts(aes)", DECRYPT, sec, NULL, 0, + speed_template_32_64, num_mb); + test_mb_skcipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0, + speed_template_16_24_32, num_mb); + test_mb_skcipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0, + speed_template_16_24_32, num_mb); + test_mb_skcipher_speed("ctr(aes)", ENCRYPT, sec, NULL, 0, + speed_template_16_24_32, num_mb); + test_mb_skcipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0, + speed_template_16_24_32, num_mb); + test_mb_skcipher_speed("cfb(aes)", ENCRYPT, sec, NULL, 0, + speed_template_16_24_32, num_mb); + test_mb_skcipher_speed("cfb(aes)", DECRYPT, sec, NULL, 0, + speed_template_16_24_32, num_mb); + test_mb_skcipher_speed("ofb(aes)", ENCRYPT, sec, NULL, 0, + speed_template_16_24_32, num_mb); + test_mb_skcipher_speed("ofb(aes)", DECRYPT, sec, NULL, 0, + speed_template_16_24_32, num_mb); + test_mb_skcipher_speed("rfc3686(ctr(aes))", ENCRYPT, sec, NULL, + 0, speed_template_20_28_36, num_mb); + test_mb_skcipher_speed("rfc3686(ctr(aes))", DECRYPT, sec, NULL, + 0, speed_template_20_28_36, num_mb); + break; + + case 601: + test_mb_skcipher_speed("ecb(des3_ede)", ENCRYPT, sec, + des3_speed_template, DES3_SPEED_VECTORS, + speed_template_24, num_mb); + test_mb_skcipher_speed("ecb(des3_ede)", DECRYPT, sec, + des3_speed_template, DES3_SPEED_VECTORS, + speed_template_24, num_mb); + test_mb_skcipher_speed("cbc(des3_ede)", ENCRYPT, sec, + des3_speed_template, DES3_SPEED_VECTORS, + speed_template_24, num_mb); + test_mb_skcipher_speed("cbc(des3_ede)", DECRYPT, sec, + des3_speed_template, DES3_SPEED_VECTORS, + speed_template_24, num_mb); + test_mb_skcipher_speed("cfb(des3_ede)", ENCRYPT, sec, + des3_speed_template, DES3_SPEED_VECTORS, + speed_template_24, num_mb); + test_mb_skcipher_speed("cfb(des3_ede)", DECRYPT, sec, + des3_speed_template, DES3_SPEED_VECTORS, + speed_template_24, num_mb); + test_mb_skcipher_speed("ofb(des3_ede)", ENCRYPT, sec, + des3_speed_template, DES3_SPEED_VECTORS, + speed_template_24, num_mb); + test_mb_skcipher_speed("ofb(des3_ede)", DECRYPT, sec, + des3_speed_template, DES3_SPEED_VECTORS, + speed_template_24, num_mb); + break; + + case 602: + test_mb_skcipher_speed("ecb(des)", ENCRYPT, sec, NULL, 0, + speed_template_8, num_mb); + test_mb_skcipher_speed("ecb(des)", DECRYPT, sec, NULL, 0, + speed_template_8, num_mb); + test_mb_skcipher_speed("cbc(des)", ENCRYPT, sec, NULL, 0, + speed_template_8, num_mb); + test_mb_skcipher_speed("cbc(des)", DECRYPT, sec, NULL, 0, + speed_template_8, num_mb); + test_mb_skcipher_speed("cfb(des)", ENCRYPT, sec, NULL, 0, + speed_template_8, num_mb); + test_mb_skcipher_speed("cfb(des)", DECRYPT, sec, NULL, 0, + speed_template_8, num_mb); + test_mb_skcipher_speed("ofb(des)", ENCRYPT, sec, NULL, 0, + speed_template_8, num_mb); + test_mb_skcipher_speed("ofb(des)", DECRYPT, sec, NULL, 0, + speed_template_8, num_mb); + break; + + case 603: + test_mb_skcipher_speed("ecb(serpent)", ENCRYPT, sec, NULL, 0, + speed_template_16_32, num_mb); + test_mb_skcipher_speed("ecb(serpent)", DECRYPT, sec, NULL, 0, + speed_template_16_32, num_mb); + test_mb_skcipher_speed("cbc(serpent)", ENCRYPT, sec, NULL, 0, + speed_template_16_32, num_mb); + test_mb_skcipher_speed("cbc(serpent)", DECRYPT, sec, NULL, 0, + speed_template_16_32, num_mb); + test_mb_skcipher_speed("ctr(serpent)", ENCRYPT, sec, NULL, 0, + speed_template_16_32, num_mb); + test_mb_skcipher_speed("ctr(serpent)", DECRYPT, sec, NULL, 0, + speed_template_16_32, num_mb); + test_mb_skcipher_speed("lrw(serpent)", ENCRYPT, sec, NULL, 0, + speed_template_32_48, num_mb); + test_mb_skcipher_speed("lrw(serpent)", DECRYPT, sec, NULL, 0, + speed_template_32_48, num_mb); + test_mb_skcipher_speed("xts(serpent)", ENCRYPT, sec, NULL, 0, + speed_template_32_64, num_mb); + test_mb_skcipher_speed("xts(serpent)", DECRYPT, sec, NULL, 0, + speed_template_32_64, num_mb); + break; + + case 604: + test_mb_skcipher_speed("ecb(twofish)", ENCRYPT, sec, NULL, 0, + speed_template_16_24_32, num_mb); + test_mb_skcipher_speed("ecb(twofish)", DECRYPT, sec, NULL, 0, + speed_template_16_24_32, num_mb); + test_mb_skcipher_speed("cbc(twofish)", ENCRYPT, sec, NULL, 0, + speed_template_16_24_32, num_mb); + test_mb_skcipher_speed("cbc(twofish)", DECRYPT, sec, NULL, 0, + speed_template_16_24_32, num_mb); + test_mb_skcipher_speed("ctr(twofish)", ENCRYPT, sec, NULL, 0, + speed_template_16_24_32, num_mb); + test_mb_skcipher_speed("ctr(twofish)", DECRYPT, sec, NULL, 0, + speed_template_16_24_32, num_mb); + test_mb_skcipher_speed("lrw(twofish)", ENCRYPT, sec, NULL, 0, + speed_template_32_40_48, num_mb); + test_mb_skcipher_speed("lrw(twofish)", DECRYPT, sec, NULL, 0, + speed_template_32_40_48, num_mb); + test_mb_skcipher_speed("xts(twofish)", ENCRYPT, sec, NULL, 0, + speed_template_32_48_64, num_mb); + test_mb_skcipher_speed("xts(twofish)", DECRYPT, sec, NULL, 0, + speed_template_32_48_64, num_mb); + break; + + case 605: + test_mb_skcipher_speed("ecb(arc4)", ENCRYPT, sec, NULL, 0, + speed_template_8, num_mb); + break; + + case 606: + test_mb_skcipher_speed("ecb(cast5)", ENCRYPT, sec, NULL, 0, + speed_template_8_16, num_mb); + test_mb_skcipher_speed("ecb(cast5)", DECRYPT, sec, NULL, 0, + speed_template_8_16, num_mb); + test_mb_skcipher_speed("cbc(cast5)", ENCRYPT, sec, NULL, 0, + speed_template_8_16, num_mb); + test_mb_skcipher_speed("cbc(cast5)", DECRYPT, sec, NULL, 0, + speed_template_8_16, num_mb); + test_mb_skcipher_speed("ctr(cast5)", ENCRYPT, sec, NULL, 0, + speed_template_8_16, num_mb); + test_mb_skcipher_speed("ctr(cast5)", DECRYPT, sec, NULL, 0, + speed_template_8_16, num_mb); + break; + + case 607: + test_mb_skcipher_speed("ecb(cast6)", ENCRYPT, sec, NULL, 0, + speed_template_16_32, num_mb); + test_mb_skcipher_speed("ecb(cast6)", DECRYPT, sec, NULL, 0, + speed_template_16_32, num_mb); + test_mb_skcipher_speed("cbc(cast6)", ENCRYPT, sec, NULL, 0, + speed_template_16_32, num_mb); + test_mb_skcipher_speed("cbc(cast6)", DECRYPT, sec, NULL, 0, + speed_template_16_32, num_mb); + test_mb_skcipher_speed("ctr(cast6)", ENCRYPT, sec, NULL, 0, + speed_template_16_32, num_mb); + test_mb_skcipher_speed("ctr(cast6)", DECRYPT, sec, NULL, 0, + speed_template_16_32, num_mb); + test_mb_skcipher_speed("lrw(cast6)", ENCRYPT, sec, NULL, 0, + speed_template_32_48, num_mb); + test_mb_skcipher_speed("lrw(cast6)", DECRYPT, sec, NULL, 0, + speed_template_32_48, num_mb); + test_mb_skcipher_speed("xts(cast6)", ENCRYPT, sec, NULL, 0, + speed_template_32_64, num_mb); + test_mb_skcipher_speed("xts(cast6)", DECRYPT, sec, NULL, 0, + speed_template_32_64, num_mb); + break; + + case 608: + test_mb_skcipher_speed("ecb(camellia)", ENCRYPT, sec, NULL, 0, + speed_template_16_32, num_mb); + test_mb_skcipher_speed("ecb(camellia)", DECRYPT, sec, NULL, 0, + speed_template_16_32, num_mb); + test_mb_skcipher_speed("cbc(camellia)", ENCRYPT, sec, NULL, 0, + speed_template_16_32, num_mb); + test_mb_skcipher_speed("cbc(camellia)", DECRYPT, sec, NULL, 0, + speed_template_16_32, num_mb); + test_mb_skcipher_speed("ctr(camellia)", ENCRYPT, sec, NULL, 0, + speed_template_16_32, num_mb); + test_mb_skcipher_speed("ctr(camellia)", DECRYPT, sec, NULL, 0, + speed_template_16_32, num_mb); + test_mb_skcipher_speed("lrw(camellia)", ENCRYPT, sec, NULL, 0, + speed_template_32_48, num_mb); + test_mb_skcipher_speed("lrw(camellia)", DECRYPT, sec, NULL, 0, + speed_template_32_48, num_mb); + test_mb_skcipher_speed("xts(camellia)", ENCRYPT, sec, NULL, 0, + speed_template_32_64, num_mb); + test_mb_skcipher_speed("xts(camellia)", DECRYPT, sec, NULL, 0, + speed_template_32_64, num_mb); + break; + + case 609: + test_mb_skcipher_speed("ecb(blowfish)", ENCRYPT, sec, NULL, 0, + speed_template_8_32, num_mb); + test_mb_skcipher_speed("ecb(blowfish)", DECRYPT, sec, NULL, 0, + speed_template_8_32, num_mb); + test_mb_skcipher_speed("cbc(blowfish)", ENCRYPT, sec, NULL, 0, + speed_template_8_32, num_mb); + test_mb_skcipher_speed("cbc(blowfish)", DECRYPT, sec, NULL, 0, + speed_template_8_32, num_mb); + test_mb_skcipher_speed("ctr(blowfish)", ENCRYPT, sec, NULL, 0, + speed_template_8_32, num_mb); + test_mb_skcipher_speed("ctr(blowfish)", DECRYPT, sec, NULL, 0, + speed_template_8_32, num_mb); + break; + case 1000: test_available(); break; From 427988d981c4c20a7f0421873351ccd14a88e2b2 Mon Sep 17 00:00:00 2001 From: Gilad Ben-Yossef Date: Sun, 17 Dec 2017 08:29:05 +0000 Subject: [PATCH 085/166] crypto: tcrypt - add multibuf aead speed test The performance of some aead tfm providers is affected by the amount of parallelism possible with the processing. Introduce an async aead concurrent multiple buffer processing speed test to be able to test performance of such tfm providers. Signed-off-by: Gilad Ben-Yossef Signed-off-by: Herbert Xu --- crypto/tcrypt.c | 439 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 379 insertions(+), 60 deletions(-) diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index d617c1956533..58e3344d7169 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -80,6 +80,66 @@ static char *check[] = { NULL }; +static u32 block_sizes[] = { 16, 64, 256, 1024, 8192, 0 }; +static u32 aead_sizes[] = { 16, 64, 256, 512, 1024, 2048, 4096, 8192, 0 }; + +#define XBUFSIZE 8 +#define MAX_IVLEN 32 + +static int testmgr_alloc_buf(char *buf[XBUFSIZE]) +{ + int i; + + for (i = 0; i < XBUFSIZE; i++) { + buf[i] = (void *)__get_free_page(GFP_KERNEL); + if (!buf[i]) + goto err_free_buf; + } + + return 0; + +err_free_buf: + while (i-- > 0) + free_page((unsigned long)buf[i]); + + return -ENOMEM; +} + +static void testmgr_free_buf(char *buf[XBUFSIZE]) +{ + int i; + + for (i = 0; i < XBUFSIZE; i++) + free_page((unsigned long)buf[i]); +} + +static void sg_init_aead(struct scatterlist *sg, char *xbuf[XBUFSIZE], + unsigned int buflen, const void *assoc, + unsigned int aad_size) +{ + int np = (buflen + PAGE_SIZE - 1)/PAGE_SIZE; + int k, rem; + + if (np > XBUFSIZE) { + rem = PAGE_SIZE; + np = XBUFSIZE; + } else { + rem = buflen % PAGE_SIZE; + } + + sg_init_table(sg, np + 1); + + sg_set_buf(&sg[0], assoc, aad_size); + + if (rem) + np--; + for (k = 0; k < np; k++) + sg_set_buf(&sg[k + 1], xbuf[k], PAGE_SIZE); + + if (rem) + sg_set_buf(&sg[k + 1], xbuf[k], rem); +} + static inline int do_one_aead_op(struct aead_request *req, int ret) { struct crypto_wait *wait = req->base.data; @@ -87,6 +147,298 @@ static inline int do_one_aead_op(struct aead_request *req, int ret) return crypto_wait_req(ret, wait); } +struct test_mb_aead_data { + struct scatterlist sg[XBUFSIZE]; + struct scatterlist sgout[XBUFSIZE]; + struct aead_request *req; + struct crypto_wait wait; + char *xbuf[XBUFSIZE]; + char *xoutbuf[XBUFSIZE]; + char *axbuf[XBUFSIZE]; +}; + +static int do_mult_aead_op(struct test_mb_aead_data *data, int enc, + u32 num_mb) +{ + int i, rc[num_mb], err = 0; + + /* Fire up a bunch of concurrent requests */ + for (i = 0; i < num_mb; i++) { + if (enc == ENCRYPT) + rc[i] = crypto_aead_encrypt(data[i].req); + else + rc[i] = crypto_aead_decrypt(data[i].req); + } + + /* Wait for all requests to finish */ + for (i = 0; i < num_mb; i++) { + rc[i] = crypto_wait_req(rc[i], &data[i].wait); + + if (rc[i]) { + pr_info("concurrent request %d error %d\n", i, rc[i]); + err = rc[i]; + } + } + + return err; +} + +static int test_mb_aead_jiffies(struct test_mb_aead_data *data, int enc, + int blen, int secs, u32 num_mb) +{ + unsigned long start, end; + int bcount; + int ret; + + for (start = jiffies, end = start + secs * HZ, bcount = 0; + time_before(jiffies, end); bcount++) { + ret = do_mult_aead_op(data, enc, num_mb); + if (ret) + return ret; + } + + pr_cont("%d operations in %d seconds (%ld bytes)\n", + bcount * num_mb, secs, (long)bcount * blen * num_mb); + return 0; +} + +static int test_mb_aead_cycles(struct test_mb_aead_data *data, int enc, + int blen, u32 num_mb) +{ + unsigned long cycles = 0; + int ret = 0; + int i; + + /* Warm-up run. */ + for (i = 0; i < 4; i++) { + ret = do_mult_aead_op(data, enc, num_mb); + if (ret) + goto out; + } + + /* The real thing. */ + for (i = 0; i < 8; i++) { + cycles_t start, end; + + start = get_cycles(); + ret = do_mult_aead_op(data, enc, num_mb); + end = get_cycles(); + + if (ret) + goto out; + + cycles += end - start; + } + +out: + if (ret == 0) + pr_cont("1 operation in %lu cycles (%d bytes)\n", + (cycles + 4) / (8 * num_mb), blen); + + return ret; +} + +static void test_mb_aead_speed(const char *algo, int enc, int secs, + struct aead_speed_template *template, + unsigned int tcount, u8 authsize, + unsigned int aad_size, u8 *keysize, u32 num_mb) +{ + struct test_mb_aead_data *data; + struct crypto_aead *tfm; + unsigned int i, j, iv_len; + const char *key; + const char *e; + void *assoc; + u32 *b_size; + char *iv; + int ret; + + + if (aad_size >= PAGE_SIZE) { + pr_err("associate data length (%u) too big\n", aad_size); + return; + } + + iv = kzalloc(MAX_IVLEN, GFP_KERNEL); + if (!iv) + return; + + if (enc == ENCRYPT) + e = "encryption"; + else + e = "decryption"; + + data = kcalloc(num_mb, sizeof(*data), GFP_KERNEL); + if (!data) + goto out_free_iv; + + tfm = crypto_alloc_aead(algo, 0, 0); + if (IS_ERR(tfm)) { + pr_err("failed to load transform for %s: %ld\n", + algo, PTR_ERR(tfm)); + goto out_free_data; + } + + ret = crypto_aead_setauthsize(tfm, authsize); + + for (i = 0; i < num_mb; ++i) + if (testmgr_alloc_buf(data[i].xbuf)) { + while (i--) + testmgr_free_buf(data[i].xbuf); + goto out_free_tfm; + } + + for (i = 0; i < num_mb; ++i) + if (testmgr_alloc_buf(data[i].axbuf)) { + while (i--) + testmgr_free_buf(data[i].axbuf); + goto out_free_xbuf; + } + + for (i = 0; i < num_mb; ++i) + if (testmgr_alloc_buf(data[i].xoutbuf)) { + while (i--) + testmgr_free_buf(data[i].axbuf); + goto out_free_axbuf; + } + + for (i = 0; i < num_mb; ++i) { + data[i].req = aead_request_alloc(tfm, GFP_KERNEL); + if (!data[i].req) { + pr_err("alg: skcipher: Failed to allocate request for %s\n", + algo); + while (i--) + aead_request_free(data[i].req); + goto out_free_xoutbuf; + } + } + + for (i = 0; i < num_mb; ++i) { + crypto_init_wait(&data[i].wait); + aead_request_set_callback(data[i].req, + CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &data[i].wait); + } + + pr_info("\ntesting speed of multibuffer %s (%s) %s\n", algo, + get_driver_name(crypto_aead, tfm), e); + + i = 0; + do { + b_size = aead_sizes; + do { + if (*b_size + authsize > XBUFSIZE * PAGE_SIZE) { + pr_err("template (%u) too big for bufufer (%lu)\n", + authsize + *b_size, + XBUFSIZE * PAGE_SIZE); + goto out; + } + + pr_info("test %u (%d bit key, %d byte blocks): ", i, + *keysize * 8, *b_size); + + /* Set up tfm global state, i.e. the key */ + + memset(tvmem[0], 0xff, PAGE_SIZE); + key = tvmem[0]; + for (j = 0; j < tcount; j++) { + if (template[j].klen == *keysize) { + key = template[j].key; + break; + } + } + + crypto_aead_clear_flags(tfm, ~0); + + ret = crypto_aead_setkey(tfm, key, *keysize); + if (ret) { + pr_err("setkey() failed flags=%x\n", + crypto_aead_get_flags(tfm)); + goto out; + } + + iv_len = crypto_aead_ivsize(tfm); + if (iv_len) + memset(iv, 0xff, iv_len); + + /* Now setup per request stuff, i.e. buffers */ + + for (j = 0; j < num_mb; ++j) { + struct test_mb_aead_data *cur = &data[j]; + + assoc = cur->axbuf[0]; + memset(assoc, 0xff, aad_size); + + sg_init_aead(cur->sg, cur->xbuf, + *b_size + (enc ? 0 : authsize), + assoc, aad_size); + + sg_init_aead(cur->sgout, cur->xoutbuf, + *b_size + (enc ? authsize : 0), + assoc, aad_size); + + aead_request_set_ad(cur->req, aad_size); + + if (!enc) { + + aead_request_set_crypt(cur->req, + cur->sgout, + cur->sg, + *b_size, iv); + ret = crypto_aead_encrypt(cur->req); + ret = do_one_aead_op(cur->req, ret); + + if (ret) { + pr_err("calculating auth failed failed (%d)\n", + ret); + break; + } + } + + aead_request_set_crypt(cur->req, cur->sg, + cur->sgout, *b_size + + (enc ? 0 : authsize), + iv); + + } + + if (secs) + ret = test_mb_aead_jiffies(data, enc, *b_size, + secs, num_mb); + else + ret = test_mb_aead_cycles(data, enc, *b_size, + num_mb); + + if (ret) { + pr_err("%s() failed return code=%d\n", e, ret); + break; + } + b_size++; + i++; + } while (*b_size); + keysize++; + } while (*keysize); + +out: + for (i = 0; i < num_mb; ++i) + aead_request_free(data[i].req); +out_free_xoutbuf: + for (i = 0; i < num_mb; ++i) + testmgr_free_buf(data[i].xoutbuf); +out_free_axbuf: + for (i = 0; i < num_mb; ++i) + testmgr_free_buf(data[i].axbuf); +out_free_xbuf: + for (i = 0; i < num_mb; ++i) + testmgr_free_buf(data[i].xbuf); +out_free_tfm: + crypto_free_aead(tfm); +out_free_data: + kfree(data); +out_free_iv: + kfree(iv); +} + static int test_aead_jiffies(struct aead_request *req, int enc, int blen, int secs) { @@ -152,66 +504,6 @@ out: return ret; } -static u32 block_sizes[] = { 16, 64, 256, 1024, 8192, 0 }; -static u32 aead_sizes[] = { 16, 64, 256, 512, 1024, 2048, 4096, 8192, 0 }; - -#define XBUFSIZE 8 -#define MAX_IVLEN 32 - -static int testmgr_alloc_buf(char *buf[XBUFSIZE]) -{ - int i; - - for (i = 0; i < XBUFSIZE; i++) { - buf[i] = (void *)__get_free_page(GFP_KERNEL); - if (!buf[i]) - goto err_free_buf; - } - - return 0; - -err_free_buf: - while (i-- > 0) - free_page((unsigned long)buf[i]); - - return -ENOMEM; -} - -static void testmgr_free_buf(char *buf[XBUFSIZE]) -{ - int i; - - for (i = 0; i < XBUFSIZE; i++) - free_page((unsigned long)buf[i]); -} - -static void sg_init_aead(struct scatterlist *sg, char *xbuf[XBUFSIZE], - unsigned int buflen, const void *assoc, - unsigned int aad_size) -{ - int np = (buflen + PAGE_SIZE - 1)/PAGE_SIZE; - int k, rem; - - if (np > XBUFSIZE) { - rem = PAGE_SIZE; - np = XBUFSIZE; - } else { - rem = buflen % PAGE_SIZE; - } - - sg_init_table(sg, np + 1); - - sg_set_buf(&sg[0], assoc, aad_size); - - if (rem) - np--; - for (k = 0; k < np; k++) - sg_set_buf(&sg[k + 1], xbuf[k], PAGE_SIZE); - - if (rem) - sg_set_buf(&sg[k + 1], xbuf[k], rem); -} - static void test_aead_speed(const char *algo, int enc, unsigned int secs, struct aead_speed_template *template, unsigned int tcount, u8 authsize, @@ -1912,6 +2204,33 @@ static int do_test(const char *alg, u32 type, u32 mask, int m) speed_template_32); break; + case 215: + test_mb_aead_speed("rfc4106(gcm(aes))", ENCRYPT, sec, NULL, + 0, 16, 16, aead_speed_template_20, num_mb); + test_mb_aead_speed("gcm(aes)", ENCRYPT, sec, NULL, 0, 16, 8, + speed_template_16_24_32, num_mb); + test_mb_aead_speed("rfc4106(gcm(aes))", DECRYPT, sec, NULL, + 0, 16, 16, aead_speed_template_20, num_mb); + test_mb_aead_speed("gcm(aes)", DECRYPT, sec, NULL, 0, 16, 8, + speed_template_16_24_32, num_mb); + break; + + case 216: + test_mb_aead_speed("rfc4309(ccm(aes))", ENCRYPT, sec, NULL, 0, + 16, 16, aead_speed_template_19, num_mb); + test_mb_aead_speed("rfc4309(ccm(aes))", DECRYPT, sec, NULL, 0, + 16, 16, aead_speed_template_19, num_mb); + break; + + case 217: + test_mb_aead_speed("rfc7539esp(chacha20,poly1305)", ENCRYPT, + sec, NULL, 0, 16, 8, aead_speed_template_36, + num_mb); + test_mb_aead_speed("rfc7539esp(chacha20,poly1305)", DECRYPT, + sec, NULL, 0, 16, 8, aead_speed_template_36, + num_mb); + break; + case 300: if (alg) { test_hash_speed(alg, sec, generic_hash_speed_template); From d8c7fe9f2a486a6e5f0d5229ca43807af5ab22c6 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 18 Dec 2017 16:40:26 -0800 Subject: [PATCH 086/166] crypto: x86/twofish-3way - Fix %rbp usage Using %rbp as a temporary register breaks frame pointer convention and breaks stack traces when unwinding from an interrupt in the crypto code. In twofish-3way, we can't simply replace %rbp with another register because there are none available. Instead, we use the stack to hold the values that %rbp, %r11, and %r12 were holding previously. Each of these values represents the half of the output from the previous Feistel round that is being passed on unchanged to the following round. They are only used once per round, when they are exchanged with %rax, %rbx, and %rcx. As a result, we free up 3 registers (one per block) and can reassign them so that %rbp is not used, and additionally %r14 and %r15 are not used so they do not need to be saved/restored. There may be a small overhead caused by replacing 'xchg REG, REG' with the needed sequence 'mov MEM, REG; mov REG, MEM; mov REG, REG' once per round. But, counterintuitively, when I tested "ctr-twofish-3way" on a Haswell processor, the new version was actually about 2% faster. (Perhaps 'xchg' is not as well optimized as plain moves.) Reported-by: syzbot Signed-off-by: Eric Biggers Reviewed-by: Josh Poimboeuf Signed-off-by: Herbert Xu --- arch/x86/crypto/twofish-x86_64-asm_64-3way.S | 112 ++++++++++--------- 1 file changed, 60 insertions(+), 52 deletions(-) diff --git a/arch/x86/crypto/twofish-x86_64-asm_64-3way.S b/arch/x86/crypto/twofish-x86_64-asm_64-3way.S index 1c3b7ceb36d2..e7273a606a07 100644 --- a/arch/x86/crypto/twofish-x86_64-asm_64-3way.S +++ b/arch/x86/crypto/twofish-x86_64-asm_64-3way.S @@ -55,29 +55,31 @@ #define RAB1bl %bl #define RAB2bl %cl +#define CD0 0x0(%rsp) +#define CD1 0x8(%rsp) +#define CD2 0x10(%rsp) + +# used only before/after all rounds #define RCD0 %r8 #define RCD1 %r9 #define RCD2 %r10 -#define RCD0d %r8d -#define RCD1d %r9d -#define RCD2d %r10d +# used only during rounds +#define RX0 %r8 +#define RX1 %r9 +#define RX2 %r10 -#define RX0 %rbp -#define RX1 %r11 -#define RX2 %r12 +#define RX0d %r8d +#define RX1d %r9d +#define RX2d %r10d -#define RX0d %ebp -#define RX1d %r11d -#define RX2d %r12d +#define RY0 %r11 +#define RY1 %r12 +#define RY2 %r13 -#define RY0 %r13 -#define RY1 %r14 -#define RY2 %r15 - -#define RY0d %r13d -#define RY1d %r14d -#define RY2d %r15d +#define RY0d %r11d +#define RY1d %r12d +#define RY2d %r13d #define RT0 %rdx #define RT1 %rsi @@ -85,6 +87,8 @@ #define RT0d %edx #define RT1d %esi +#define RT1bl %sil + #define do16bit_ror(rot, op1, op2, T0, T1, tmp1, tmp2, ab, dst) \ movzbl ab ## bl, tmp2 ## d; \ movzbl ab ## bh, tmp1 ## d; \ @@ -92,6 +96,11 @@ op1##l T0(CTX, tmp2, 4), dst ## d; \ op2##l T1(CTX, tmp1, 4), dst ## d; +#define swap_ab_with_cd(ab, cd, tmp) \ + movq cd, tmp; \ + movq ab, cd; \ + movq tmp, ab; + /* * Combined G1 & G2 function. Reordered with help of rotates to have moves * at begining. @@ -110,15 +119,15 @@ /* G1,2 && G2,2 */ \ do16bit_ror(32, xor, xor, Tx2, Tx3, RT0, RT1, ab ## 0, x ## 0); \ do16bit_ror(16, xor, xor, Ty3, Ty0, RT0, RT1, ab ## 0, y ## 0); \ - xchgq cd ## 0, ab ## 0; \ + swap_ab_with_cd(ab ## 0, cd ## 0, RT0); \ \ do16bit_ror(32, xor, xor, Tx2, Tx3, RT0, RT1, ab ## 1, x ## 1); \ do16bit_ror(16, xor, xor, Ty3, Ty0, RT0, RT1, ab ## 1, y ## 1); \ - xchgq cd ## 1, ab ## 1; \ + swap_ab_with_cd(ab ## 1, cd ## 1, RT0); \ \ do16bit_ror(32, xor, xor, Tx2, Tx3, RT0, RT1, ab ## 2, x ## 2); \ do16bit_ror(16, xor, xor, Ty3, Ty0, RT0, RT1, ab ## 2, y ## 2); \ - xchgq cd ## 2, ab ## 2; + swap_ab_with_cd(ab ## 2, cd ## 2, RT0); #define enc_round_end(ab, x, y, n) \ addl y ## d, x ## d; \ @@ -168,6 +177,16 @@ decrypt_round3(ba, dc, (n*2)+1); \ decrypt_round3(ba, dc, (n*2)); +#define push_cd() \ + pushq RCD2; \ + pushq RCD1; \ + pushq RCD0; + +#define pop_cd() \ + popq RCD0; \ + popq RCD1; \ + popq RCD2; + #define inpack3(in, n, xy, m) \ movq 4*(n)(in), xy ## 0; \ xorq w+4*m(CTX), xy ## 0; \ @@ -223,11 +242,8 @@ ENTRY(__twofish_enc_blk_3way) * %rdx: src, RIO * %rcx: bool, if true: xor output */ - pushq %r15; - pushq %r14; pushq %r13; pushq %r12; - pushq %rbp; pushq %rbx; pushq %rcx; /* bool xor */ @@ -235,40 +251,36 @@ ENTRY(__twofish_enc_blk_3way) inpack_enc3(); - encrypt_cycle3(RAB, RCD, 0); - encrypt_cycle3(RAB, RCD, 1); - encrypt_cycle3(RAB, RCD, 2); - encrypt_cycle3(RAB, RCD, 3); - encrypt_cycle3(RAB, RCD, 4); - encrypt_cycle3(RAB, RCD, 5); - encrypt_cycle3(RAB, RCD, 6); - encrypt_cycle3(RAB, RCD, 7); + push_cd(); + encrypt_cycle3(RAB, CD, 0); + encrypt_cycle3(RAB, CD, 1); + encrypt_cycle3(RAB, CD, 2); + encrypt_cycle3(RAB, CD, 3); + encrypt_cycle3(RAB, CD, 4); + encrypt_cycle3(RAB, CD, 5); + encrypt_cycle3(RAB, CD, 6); + encrypt_cycle3(RAB, CD, 7); + pop_cd(); popq RIO; /* dst */ - popq %rbp; /* bool xor */ + popq RT1; /* bool xor */ - testb %bpl, %bpl; + testb RT1bl, RT1bl; jnz .L__enc_xor3; outunpack_enc3(mov); popq %rbx; - popq %rbp; popq %r12; popq %r13; - popq %r14; - popq %r15; ret; .L__enc_xor3: outunpack_enc3(xor); popq %rbx; - popq %rbp; popq %r12; popq %r13; - popq %r14; - popq %r15; ret; ENDPROC(__twofish_enc_blk_3way) @@ -278,35 +290,31 @@ ENTRY(twofish_dec_blk_3way) * %rsi: dst * %rdx: src, RIO */ - pushq %r15; - pushq %r14; pushq %r13; pushq %r12; - pushq %rbp; pushq %rbx; pushq %rsi; /* dst */ inpack_dec3(); - decrypt_cycle3(RAB, RCD, 7); - decrypt_cycle3(RAB, RCD, 6); - decrypt_cycle3(RAB, RCD, 5); - decrypt_cycle3(RAB, RCD, 4); - decrypt_cycle3(RAB, RCD, 3); - decrypt_cycle3(RAB, RCD, 2); - decrypt_cycle3(RAB, RCD, 1); - decrypt_cycle3(RAB, RCD, 0); + push_cd(); + decrypt_cycle3(RAB, CD, 7); + decrypt_cycle3(RAB, CD, 6); + decrypt_cycle3(RAB, CD, 5); + decrypt_cycle3(RAB, CD, 4); + decrypt_cycle3(RAB, CD, 3); + decrypt_cycle3(RAB, CD, 2); + decrypt_cycle3(RAB, CD, 1); + decrypt_cycle3(RAB, CD, 0); + pop_cd(); popq RIO; /* dst */ outunpack_dec3(); popq %rbx; - popq %rbp; popq %r12; popq %r13; - popq %r14; - popq %r15; ret; ENDPROC(twofish_dec_blk_3way) From 6674a4fdc6c2e690b28b7f590cd20871aeee55ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Tue, 19 Dec 2017 12:16:04 +0200 Subject: [PATCH 087/166] crypto: caam - constify key data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Key data is not modified, it is copied in the shared descriptor. Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg_desc.c | 6 +++--- drivers/crypto/caam/desc_constr.h | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/crypto/caam/caamalg_desc.c b/drivers/crypto/caam/caamalg_desc.c index 530c14ee32de..2db9e85bf81c 100644 --- a/drivers/crypto/caam/caamalg_desc.c +++ b/drivers/crypto/caam/caamalg_desc.c @@ -1075,7 +1075,7 @@ void cnstr_shdsc_ablkcipher_encap(u32 * const desc, struct alginfo *cdata, /* Load nonce into CONTEXT1 reg */ if (is_rfc3686) { - u8 *nonce = cdata->key_virt + cdata->keylen; + const u8 *nonce = cdata->key_virt + cdata->keylen; append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, LDST_CLASS_IND_CCB | @@ -1140,7 +1140,7 @@ void cnstr_shdsc_ablkcipher_decap(u32 * const desc, struct alginfo *cdata, /* Load nonce into CONTEXT1 reg */ if (is_rfc3686) { - u8 *nonce = cdata->key_virt + cdata->keylen; + const u8 *nonce = cdata->key_virt + cdata->keylen; append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, LDST_CLASS_IND_CCB | @@ -1209,7 +1209,7 @@ void cnstr_shdsc_ablkcipher_givencap(u32 * const desc, struct alginfo *cdata, /* Load Nonce into CONTEXT1 reg */ if (is_rfc3686) { - u8 *nonce = cdata->key_virt + cdata->keylen; + const u8 *nonce = cdata->key_virt + cdata->keylen; append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, LDST_CLASS_IND_CCB | diff --git a/drivers/crypto/caam/desc_constr.h b/drivers/crypto/caam/desc_constr.h index ba1ca0806f0a..5b39b7d7a47a 100644 --- a/drivers/crypto/caam/desc_constr.h +++ b/drivers/crypto/caam/desc_constr.h @@ -109,7 +109,7 @@ static inline void init_job_desc_shared(u32 * const desc, dma_addr_t ptr, append_ptr(desc, ptr); } -static inline void append_data(u32 * const desc, void *data, int len) +static inline void append_data(u32 * const desc, const void *data, int len) { u32 *offset = desc_end(desc); @@ -172,7 +172,7 @@ static inline void append_cmd_ptr_extlen(u32 * const desc, dma_addr_t ptr, append_cmd(desc, len); } -static inline void append_cmd_data(u32 * const desc, void *data, int len, +static inline void append_cmd_data(u32 * const desc, const void *data, int len, u32 command) { append_cmd(desc, command | IMMEDIATE | len); @@ -271,7 +271,7 @@ APPEND_SEQ_PTR_INTLEN(in, IN) APPEND_SEQ_PTR_INTLEN(out, OUT) #define APPEND_CMD_PTR_TO_IMM(cmd, op) \ -static inline void append_##cmd##_as_imm(u32 * const desc, void *data, \ +static inline void append_##cmd##_as_imm(u32 * const desc, const void *data, \ unsigned int len, u32 options) \ { \ PRINT_POS; \ @@ -312,7 +312,7 @@ APPEND_CMD_PTR_LEN(seq_out_ptr, SEQ_OUT_PTR, u32) * from length of immediate data provided, e.g., split keys */ #define APPEND_CMD_PTR_TO_IMM2(cmd, op) \ -static inline void append_##cmd##_as_imm(u32 * const desc, void *data, \ +static inline void append_##cmd##_as_imm(u32 * const desc, const void *data, \ unsigned int data_len, \ unsigned int len, u32 options) \ { \ @@ -452,7 +452,7 @@ struct alginfo { unsigned int keylen_pad; union { dma_addr_t key_dma; - void *key_virt; + const void *key_virt; }; bool key_inline; }; From 662f70ede5975ce68a1c3524c9fa080c44a38b44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Tue, 19 Dec 2017 12:16:05 +0200 Subject: [PATCH 088/166] crypto: caam - remove needless ablkcipher key copy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ablkcipher shared descriptors are relatively small, thus there is enough space for the key to be inlined. Accordingly, there is no need to copy the key in ctx->key. Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 8 ++------ drivers/crypto/caam/caamalg_qi.c | 8 ++------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index baa8dd52472d..f5666e50c1e7 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -625,7 +625,6 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, const bool is_rfc3686 = (ctr_mode && (strstr(alg_name, "rfc3686") != NULL)); - memcpy(ctx->key, key, keylen); #ifdef DEBUG print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); @@ -648,9 +647,8 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, keylen -= CTR_RFC3686_NONCE_SIZE; } - dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); ctx->cdata.keylen = keylen; - ctx->cdata.key_virt = ctx->key; + ctx->cdata.key_virt = key; ctx->cdata.key_inline = true; /* ablkcipher_encrypt shared descriptor */ @@ -691,10 +689,8 @@ static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, return -EINVAL; } - memcpy(ctx->key, key, keylen); - dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); ctx->cdata.keylen = keylen; - ctx->cdata.key_virt = ctx->key; + ctx->cdata.key_virt = key; ctx->cdata.key_inline = true; /* xts_ablkcipher_encrypt shared descriptor */ diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index ad14b69a052e..b45401786530 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -272,7 +272,6 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, const bool is_rfc3686 = (ctr_mode && strstr(alg_name, "rfc3686")); int ret = 0; - memcpy(ctx->key, key, keylen); #ifdef DEBUG print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); @@ -295,9 +294,8 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, keylen -= CTR_RFC3686_NONCE_SIZE; } - dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); ctx->cdata.keylen = keylen; - ctx->cdata.key_virt = ctx->key; + ctx->cdata.key_virt = key; ctx->cdata.key_inline = true; /* ablkcipher encrypt, decrypt, givencrypt shared descriptors */ @@ -356,10 +354,8 @@ static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, return -EINVAL; } - memcpy(ctx->key, key, keylen); - dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); ctx->cdata.keylen = keylen; - ctx->cdata.key_virt = ctx->key; + ctx->cdata.key_virt = key; ctx->cdata.key_inline = true; /* xts ablkcipher encrypt, decrypt shared descriptors */ From 9fe712df08ea805868789bb6800f2226f5fd5285 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Tue, 19 Dec 2017 12:16:06 +0200 Subject: [PATCH 089/166] crypto: caam - save Era in driver's private data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Save Era in driver's private data for further usage, like deciding whether an erratum applies or a feature is available based on its value. Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/ctrl.c | 4 +++- drivers/crypto/caam/intern.h | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index 027e121c6f70..75d280cb2dc0 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -611,6 +611,8 @@ static int caam_probe(struct platform_device *pdev) goto iounmap_ctrl; } + ctrlpriv->era = caam_get_era(); + ret = of_platform_populate(nprop, caam_match, NULL, dev); if (ret) { dev_err(dev, "JR platform devices creation error\n"); @@ -742,7 +744,7 @@ static int caam_probe(struct platform_device *pdev) /* Report "alive" for developer to see */ dev_info(dev, "device ID = 0x%016llx (Era %d)\n", caam_id, - caam_get_era()); + ctrlpriv->era); dev_info(dev, "job rings = %d, qi = %d, dpaa2 = %s\n", ctrlpriv->total_jobrs, ctrlpriv->qi_present, caam_dpaa2 ? "yes" : "no"); diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h index 91f1107276e5..7696a774a362 100644 --- a/drivers/crypto/caam/intern.h +++ b/drivers/crypto/caam/intern.h @@ -84,6 +84,7 @@ struct caam_drv_private { u8 qi_present; /* Nonzero if QI present in device */ int secvio_irq; /* Security violation interrupt number */ int virt_en; /* Virtualization enabled in CAAM */ + int era; /* CAAM Era (internal HW revision) */ #define RNG4_MAX_HANDLES 2 /* RNG4 block */ From 7e0880b9fbbe7d21e29347020bef0b7329cb0e9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Tue, 19 Dec 2017 12:16:07 +0200 Subject: [PATCH 090/166] crypto: caam - add Derived Key Protocol (DKP) support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Offload split key generation in CAAM engine, using DKP. DKP is supported starting with Era 6. Note that the way assoclen is transmitted from the job descriptor to the shared descriptor changes - DPOVRD register is used instead of MATH3 (where available), since DKP protocol thrashes the MATH registers. The replacement of MDHA split key generation with DKP has the side effect of the crypto engine writing the authentication key, and thus the DMA mapping direction for the buffer holding the key has to change from DMA_TO_DEVICE to DMA_BIDIRECTIONAL. There are two cases: -key is inlined in descriptor - descriptor buffer mapping changes -key is referenced - key buffer mapping changes Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 112 ++++++++++++------ drivers/crypto/caam/caamalg_desc.c | 176 ++++++++++++++++++----------- drivers/crypto/caam/caamalg_desc.h | 10 +- drivers/crypto/caam/caamalg_qi.c | 54 +++++++-- drivers/crypto/caam/caamhash.c | 73 ++++++++---- drivers/crypto/caam/desc.h | 29 +++++ drivers/crypto/caam/desc_constr.h | 41 +++++++ drivers/crypto/caam/key_gen.c | 30 ----- drivers/crypto/caam/key_gen.h | 30 +++++ 9 files changed, 384 insertions(+), 171 deletions(-) diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index f5666e50c1e7..2188235be02d 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -108,6 +108,7 @@ struct caam_ctx { dma_addr_t sh_desc_dec_dma; dma_addr_t sh_desc_givenc_dma; dma_addr_t key_dma; + enum dma_data_direction dir; struct device *jrdev; struct alginfo adata; struct alginfo cdata; @@ -118,6 +119,7 @@ static int aead_null_set_sh_desc(struct crypto_aead *aead) { struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); u32 *desc; int rem_bytes = CAAM_DESC_BYTES_MAX - AEAD_DESC_JOB_IO_LEN - ctx->adata.keylen_pad; @@ -136,9 +138,10 @@ static int aead_null_set_sh_desc(struct crypto_aead *aead) /* aead_encrypt shared descriptor */ desc = ctx->sh_desc_enc; - cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize); + cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize, + ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); /* * Job Descriptor and Shared Descriptors @@ -154,9 +157,10 @@ static int aead_null_set_sh_desc(struct crypto_aead *aead) /* aead_decrypt shared descriptor */ desc = ctx->sh_desc_dec; - cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize); + cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize, + ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); return 0; } @@ -168,6 +172,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead) unsigned int ivsize = crypto_aead_ivsize(aead); struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); u32 ctx1_iv_off = 0; u32 *desc, *nonce = NULL; u32 inl_mask; @@ -234,9 +239,9 @@ static int aead_set_sh_desc(struct crypto_aead *aead) desc = ctx->sh_desc_enc; cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ivsize, ctx->authsize, is_rfc3686, nonce, ctx1_iv_off, - false); + false, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); skip_enc: /* @@ -266,9 +271,9 @@ skip_enc: desc = ctx->sh_desc_dec; cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, ivsize, ctx->authsize, alg->caam.geniv, is_rfc3686, - nonce, ctx1_iv_off, false); + nonce, ctx1_iv_off, false, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); if (!alg->caam.geniv) goto skip_givenc; @@ -300,9 +305,9 @@ skip_enc: desc = ctx->sh_desc_enc; cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, ivsize, ctx->authsize, is_rfc3686, nonce, - ctx1_iv_off, false); + ctx1_iv_off, false, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); skip_givenc: return 0; @@ -346,7 +351,7 @@ static int gcm_set_sh_desc(struct crypto_aead *aead) desc = ctx->sh_desc_enc; cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ctx->authsize); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); /* * Job Descriptor and Shared Descriptors @@ -363,7 +368,7 @@ static int gcm_set_sh_desc(struct crypto_aead *aead) desc = ctx->sh_desc_dec; cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ctx->authsize); dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); return 0; } @@ -405,7 +410,7 @@ static int rfc4106_set_sh_desc(struct crypto_aead *aead) desc = ctx->sh_desc_enc; cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ctx->authsize); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); /* * Job Descriptor and Shared Descriptors @@ -422,7 +427,7 @@ static int rfc4106_set_sh_desc(struct crypto_aead *aead) desc = ctx->sh_desc_dec; cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ctx->authsize); dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); return 0; } @@ -465,7 +470,7 @@ static int rfc4543_set_sh_desc(struct crypto_aead *aead) desc = ctx->sh_desc_enc; cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ctx->authsize); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); /* * Job Descriptor and Shared Descriptors @@ -482,7 +487,7 @@ static int rfc4543_set_sh_desc(struct crypto_aead *aead) desc = ctx->sh_desc_dec; cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ctx->authsize); dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); return 0; } @@ -503,6 +508,7 @@ static int aead_setkey(struct crypto_aead *aead, { struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); struct crypto_authenc_keys keys; int ret = 0; @@ -517,6 +523,27 @@ static int aead_setkey(struct crypto_aead *aead, DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); #endif + /* + * If DKP is supported, use it in the shared descriptor to generate + * the split key. + */ + if (ctrlpriv->era >= 6) { + ctx->adata.keylen = keys.authkeylen; + ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & + OP_ALG_ALGSEL_MASK); + + if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) + goto badkey; + + memcpy(ctx->key, keys.authkey, keys.authkeylen); + memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, + keys.enckeylen); + dma_sync_single_for_device(jrdev, ctx->key_dma, + ctx->adata.keylen_pad + + keys.enckeylen, ctx->dir); + goto skip_split_key; + } + ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, keys.authkey, keys.authkeylen, CAAM_MAX_KEY_SIZE - keys.enckeylen); @@ -527,12 +554,14 @@ static int aead_setkey(struct crypto_aead *aead, /* postpend encryption key to auth split key */ memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + - keys.enckeylen, DMA_TO_DEVICE); + keys.enckeylen, ctx->dir); #ifdef DEBUG print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, ctx->adata.keylen_pad + keys.enckeylen, 1); #endif + +skip_split_key: ctx->cdata.keylen = keys.enckeylen; return aead_set_sh_desc(aead); badkey: @@ -552,7 +581,7 @@ static int gcm_setkey(struct crypto_aead *aead, #endif memcpy(ctx->key, key, keylen); - dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); + dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir); ctx->cdata.keylen = keylen; return gcm_set_sh_desc(aead); @@ -580,7 +609,7 @@ static int rfc4106_setkey(struct crypto_aead *aead, */ ctx->cdata.keylen = keylen - 4; dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, - DMA_TO_DEVICE); + ctx->dir); return rfc4106_set_sh_desc(aead); } @@ -606,7 +635,7 @@ static int rfc4543_setkey(struct crypto_aead *aead, */ ctx->cdata.keylen = keylen - 4; dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, - DMA_TO_DEVICE); + ctx->dir); return rfc4543_set_sh_desc(aead); } @@ -656,21 +685,21 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, cnstr_shdsc_ablkcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686, ctx1_iv_off); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); /* ablkcipher_decrypt shared descriptor */ desc = ctx->sh_desc_dec; cnstr_shdsc_ablkcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686, ctx1_iv_off); dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); /* ablkcipher_givencrypt shared descriptor */ desc = ctx->sh_desc_givenc; cnstr_shdsc_ablkcipher_givencap(desc, &ctx->cdata, ivsize, is_rfc3686, ctx1_iv_off); dma_sync_single_for_device(jrdev, ctx->sh_desc_givenc_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); return 0; } @@ -697,13 +726,13 @@ static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, desc = ctx->sh_desc_enc; cnstr_shdsc_xts_ablkcipher_encap(desc, &ctx->cdata); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); /* xts_ablkcipher_decrypt shared descriptor */ desc = ctx->sh_desc_dec; cnstr_shdsc_xts_ablkcipher_decap(desc, &ctx->cdata); dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); return 0; } @@ -975,9 +1004,6 @@ static void init_aead_job(struct aead_request *req, append_seq_out_ptr(desc, dst_dma, req->assoclen + req->cryptlen - authsize, out_options); - - /* REG3 = assoclen */ - append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); } static void init_gcm_job(struct aead_request *req, @@ -992,6 +1018,7 @@ static void init_gcm_job(struct aead_request *req, unsigned int last; init_aead_job(req, edesc, all_contig, encrypt); + append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); /* BUG This should not be specific to generic GCM. */ last = 0; @@ -1018,6 +1045,7 @@ static void init_authenc_job(struct aead_request *req, struct caam_aead_alg, aead); unsigned int ivsize = crypto_aead_ivsize(aead); struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == OP_ALG_AAI_CTR_MOD128); const bool is_rfc3686 = alg->caam.rfc3686; @@ -1041,6 +1069,15 @@ static void init_authenc_job(struct aead_request *req, init_aead_job(req, edesc, all_contig, encrypt); + /* + * {REG3, DPOVRD} = assoclen, depending on whether MATH command supports + * having DPOVRD as destination. + */ + if (ctrlpriv->era < 3) + append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); + else + append_math_add_imm_u32(desc, DPOVRD, ZERO, IMM, req->assoclen); + if (ivsize && ((is_rfc3686 && encrypt) || !alg->caam.geniv)) append_load_as_imm(desc, req->iv, ivsize, LDST_CLASS_1_CCB | @@ -3224,9 +3261,11 @@ struct caam_crypto_alg { struct caam_alg_entry caam; }; -static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam) +static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, + bool uses_dkp) { dma_addr_t dma_addr; + struct caam_drv_private *priv; ctx->jrdev = caam_jr_alloc(); if (IS_ERR(ctx->jrdev)) { @@ -3234,10 +3273,16 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam) return PTR_ERR(ctx->jrdev); } + priv = dev_get_drvdata(ctx->jrdev->parent); + if (priv->era >= 6 && uses_dkp) + ctx->dir = DMA_BIDIRECTIONAL; + else + ctx->dir = DMA_TO_DEVICE; + dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_enc, offsetof(struct caam_ctx, sh_desc_enc_dma), - DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); + ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); if (dma_mapping_error(ctx->jrdev, dma_addr)) { dev_err(ctx->jrdev, "unable to map key, shared descriptors\n"); caam_jr_free(ctx->jrdev); @@ -3265,7 +3310,7 @@ static int caam_cra_init(struct crypto_tfm *tfm) container_of(alg, struct caam_crypto_alg, crypto_alg); struct caam_ctx *ctx = crypto_tfm_ctx(tfm); - return caam_init_common(ctx, &caam_alg->caam); + return caam_init_common(ctx, &caam_alg->caam, false); } static int caam_aead_init(struct crypto_aead *tfm) @@ -3275,14 +3320,15 @@ static int caam_aead_init(struct crypto_aead *tfm) container_of(alg, struct caam_aead_alg, aead); struct caam_ctx *ctx = crypto_aead_ctx(tfm); - return caam_init_common(ctx, &caam_alg->caam); + return caam_init_common(ctx, &caam_alg->caam, + alg->setkey == aead_setkey); } static void caam_exit_common(struct caam_ctx *ctx) { dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_enc_dma, offsetof(struct caam_ctx, sh_desc_enc_dma), - DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); + ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); caam_jr_free(ctx->jrdev); } diff --git a/drivers/crypto/caam/caamalg_desc.c b/drivers/crypto/caam/caamalg_desc.c index 2db9e85bf81c..ceb93fbb76e6 100644 --- a/drivers/crypto/caam/caamalg_desc.c +++ b/drivers/crypto/caam/caamalg_desc.c @@ -45,16 +45,16 @@ static inline void append_dec_op1(u32 *desc, u32 type) * cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor * (non-protocol) with no (null) encryption. * @desc: pointer to buffer used for descriptor construction - * @adata: pointer to authentication transform definitions. Note that since a - * split key is to be used, the size of the split key itself is - * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, - * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP. + * @adata: pointer to authentication transform definitions. + * A split key is required for SEC Era < 6; the size of the split key + * is specified in this case. Valid algorithm values - one of + * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed + * with OP_ALG_AAI_HMAC_PRECOMP. * @icvsize: integrity check value (ICV) size (truncated or full) - * - * Note: Requires an MDHA split key. + * @era: SEC Era */ void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata, - unsigned int icvsize) + unsigned int icvsize, int era) { u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; @@ -63,13 +63,18 @@ void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata, /* Skip if already shared */ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_SHRD); - if (adata->key_inline) - append_key_as_imm(desc, adata->key_virt, adata->keylen_pad, - adata->keylen, CLASS_2 | KEY_DEST_MDHA_SPLIT | - KEY_ENC); - else - append_key(desc, adata->key_dma, adata->keylen, CLASS_2 | - KEY_DEST_MDHA_SPLIT | KEY_ENC); + if (era < 6) { + if (adata->key_inline) + append_key_as_imm(desc, adata->key_virt, + adata->keylen_pad, adata->keylen, + CLASS_2 | KEY_DEST_MDHA_SPLIT | + KEY_ENC); + else + append_key(desc, adata->key_dma, adata->keylen, + CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); + } else { + append_proto_dkp(desc, adata); + } set_jump_tgt_here(desc, key_jump_cmd); /* assoclen + cryptlen = seqinlen */ @@ -121,16 +126,16 @@ EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap); * cnstr_shdsc_aead_null_decap - IPSec ESP decapsulation shared descriptor * (non-protocol) with no (null) decryption. * @desc: pointer to buffer used for descriptor construction - * @adata: pointer to authentication transform definitions. Note that since a - * split key is to be used, the size of the split key itself is - * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, - * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP. + * @adata: pointer to authentication transform definitions. + * A split key is required for SEC Era < 6; the size of the split key + * is specified in this case. Valid algorithm values - one of + * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed + * with OP_ALG_AAI_HMAC_PRECOMP. * @icvsize: integrity check value (ICV) size (truncated or full) - * - * Note: Requires an MDHA split key. + * @era: SEC Era */ void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata, - unsigned int icvsize) + unsigned int icvsize, int era) { u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd; @@ -139,13 +144,18 @@ void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata, /* Skip if already shared */ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_SHRD); - if (adata->key_inline) - append_key_as_imm(desc, adata->key_virt, adata->keylen_pad, - adata->keylen, CLASS_2 | - KEY_DEST_MDHA_SPLIT | KEY_ENC); - else - append_key(desc, adata->key_dma, adata->keylen, CLASS_2 | - KEY_DEST_MDHA_SPLIT | KEY_ENC); + if (era < 6) { + if (adata->key_inline) + append_key_as_imm(desc, adata->key_virt, + adata->keylen_pad, adata->keylen, + CLASS_2 | KEY_DEST_MDHA_SPLIT | + KEY_ENC); + else + append_key(desc, adata->key_dma, adata->keylen, + CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); + } else { + append_proto_dkp(desc, adata); + } set_jump_tgt_here(desc, key_jump_cmd); /* Class 2 operation */ @@ -204,7 +214,7 @@ EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap); static void init_sh_desc_key_aead(u32 * const desc, struct alginfo * const cdata, struct alginfo * const adata, - const bool is_rfc3686, u32 *nonce) + const bool is_rfc3686, u32 *nonce, int era) { u32 *key_jump_cmd; unsigned int enckeylen = cdata->keylen; @@ -224,13 +234,18 @@ static void init_sh_desc_key_aead(u32 * const desc, if (is_rfc3686) enckeylen -= CTR_RFC3686_NONCE_SIZE; - if (adata->key_inline) - append_key_as_imm(desc, adata->key_virt, adata->keylen_pad, - adata->keylen, CLASS_2 | - KEY_DEST_MDHA_SPLIT | KEY_ENC); - else - append_key(desc, adata->key_dma, adata->keylen, CLASS_2 | - KEY_DEST_MDHA_SPLIT | KEY_ENC); + if (era < 6) { + if (adata->key_inline) + append_key_as_imm(desc, adata->key_virt, + adata->keylen_pad, adata->keylen, + CLASS_2 | KEY_DEST_MDHA_SPLIT | + KEY_ENC); + else + append_key(desc, adata->key_dma, adata->keylen, + CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); + } else { + append_proto_dkp(desc, adata); + } if (cdata->key_inline) append_key_as_imm(desc, cdata->key_virt, enckeylen, @@ -261,26 +276,27 @@ static void init_sh_desc_key_aead(u32 * const desc, * @cdata: pointer to block cipher transform definitions * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. - * @adata: pointer to authentication transform definitions. Note that since a - * split key is to be used, the size of the split key itself is - * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, - * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP. + * @adata: pointer to authentication transform definitions. + * A split key is required for SEC Era < 6; the size of the split key + * is specified in this case. Valid algorithm values - one of + * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed + * with OP_ALG_AAI_HMAC_PRECOMP. * @ivsize: initialization vector size * @icvsize: integrity check value (ICV) size (truncated or full) * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template * @nonce: pointer to rfc3686 nonce * @ctx1_iv_off: IV offset in CONTEXT1 register * @is_qi: true when called from caam/qi - * - * Note: Requires an MDHA split key. + * @era: SEC Era */ void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata, struct alginfo *adata, unsigned int ivsize, unsigned int icvsize, const bool is_rfc3686, - u32 *nonce, const u32 ctx1_iv_off, const bool is_qi) + u32 *nonce, const u32 ctx1_iv_off, const bool is_qi, + int era) { /* Note: Context registers are saved. */ - init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce); + init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era); /* Class 2 operation */ append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | @@ -306,8 +322,13 @@ void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata, } /* Read and write assoclen bytes */ - append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); - append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + if (is_qi || era < 3) { + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + } else { + append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ); + } /* Skip assoc data */ append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); @@ -350,27 +371,27 @@ EXPORT_SYMBOL(cnstr_shdsc_aead_encap); * @cdata: pointer to block cipher transform definitions * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. - * @adata: pointer to authentication transform definitions. Note that since a - * split key is to be used, the size of the split key itself is - * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, - * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP. + * @adata: pointer to authentication transform definitions. + * A split key is required for SEC Era < 6; the size of the split key + * is specified in this case. Valid algorithm values - one of + * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed + * with OP_ALG_AAI_HMAC_PRECOMP. * @ivsize: initialization vector size * @icvsize: integrity check value (ICV) size (truncated or full) * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template * @nonce: pointer to rfc3686 nonce * @ctx1_iv_off: IV offset in CONTEXT1 register * @is_qi: true when called from caam/qi - * - * Note: Requires an MDHA split key. + * @era: SEC Era */ void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata, struct alginfo *adata, unsigned int ivsize, unsigned int icvsize, const bool geniv, const bool is_rfc3686, u32 *nonce, - const u32 ctx1_iv_off, const bool is_qi) + const u32 ctx1_iv_off, const bool is_qi, int era) { /* Note: Context registers are saved. */ - init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce); + init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era); /* Class 2 operation */ append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | @@ -397,11 +418,23 @@ void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata, } /* Read and write assoclen bytes */ - append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); - if (geniv) - append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM, ivsize); - else - append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + if (is_qi || era < 3) { + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + if (geniv) + append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM, + ivsize); + else + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, + CAAM_CMD_SZ); + } else { + append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ); + if (geniv) + append_math_add_imm_u32(desc, VARSEQOUTLEN, DPOVRD, IMM, + ivsize); + else + append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, + CAAM_CMD_SZ); + } /* Skip assoc data */ append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); @@ -456,29 +489,29 @@ EXPORT_SYMBOL(cnstr_shdsc_aead_decap); * @cdata: pointer to block cipher transform definitions * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. - * @adata: pointer to authentication transform definitions. Note that since a - * split key is to be used, the size of the split key itself is - * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, - * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP. + * @adata: pointer to authentication transform definitions. + * A split key is required for SEC Era < 6; the size of the split key + * is specified in this case. Valid algorithm values - one of + * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed + * with OP_ALG_AAI_HMAC_PRECOMP. * @ivsize: initialization vector size * @icvsize: integrity check value (ICV) size (truncated or full) * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template * @nonce: pointer to rfc3686 nonce * @ctx1_iv_off: IV offset in CONTEXT1 register * @is_qi: true when called from caam/qi - * - * Note: Requires an MDHA split key. + * @era: SEC Era */ void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata, struct alginfo *adata, unsigned int ivsize, unsigned int icvsize, const bool is_rfc3686, u32 *nonce, const u32 ctx1_iv_off, - const bool is_qi) + const bool is_qi, int era) { u32 geniv, moveiv; /* Note: Context registers are saved. */ - init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce); + init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era); if (is_qi) { u32 *wait_load_cmd; @@ -528,8 +561,13 @@ copy_iv: OP_ALG_ENCRYPT); /* Read and write assoclen bytes */ - append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); - append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + if (is_qi || era < 3) { + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + } else { + append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ); + } /* Skip assoc data */ append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); diff --git a/drivers/crypto/caam/caamalg_desc.h b/drivers/crypto/caam/caamalg_desc.h index e412ec8f7005..5f9445ae2114 100644 --- a/drivers/crypto/caam/caamalg_desc.h +++ b/drivers/crypto/caam/caamalg_desc.h @@ -43,28 +43,28 @@ 15 * CAAM_CMD_SZ) void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata, - unsigned int icvsize); + unsigned int icvsize, int era); void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata, - unsigned int icvsize); + unsigned int icvsize, int era); void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata, struct alginfo *adata, unsigned int ivsize, unsigned int icvsize, const bool is_rfc3686, u32 *nonce, const u32 ctx1_iv_off, - const bool is_qi); + const bool is_qi, int era); void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata, struct alginfo *adata, unsigned int ivsize, unsigned int icvsize, const bool geniv, const bool is_rfc3686, u32 *nonce, - const u32 ctx1_iv_off, const bool is_qi); + const u32 ctx1_iv_off, const bool is_qi, int era); void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata, struct alginfo *adata, unsigned int ivsize, unsigned int icvsize, const bool is_rfc3686, u32 *nonce, const u32 ctx1_iv_off, - const bool is_qi); + const bool is_qi, int era); void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata, unsigned int icvsize); diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index b45401786530..4aecc9435f69 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -53,6 +53,7 @@ struct caam_ctx { u32 sh_desc_givenc[DESC_MAX_USED_LEN]; u8 key[CAAM_MAX_KEY_SIZE]; dma_addr_t key_dma; + enum dma_data_direction dir; struct alginfo adata; struct alginfo cdata; unsigned int authsize; @@ -74,6 +75,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead) const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == OP_ALG_AAI_CTR_MOD128); const bool is_rfc3686 = alg->caam.rfc3686; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); if (!ctx->cdata.keylen || !ctx->authsize) return 0; @@ -124,7 +126,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead) cnstr_shdsc_aead_encap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata, ivsize, ctx->authsize, is_rfc3686, nonce, - ctx1_iv_off, true); + ctx1_iv_off, true, ctrlpriv->era); skip_enc: /* aead_decrypt shared descriptor */ @@ -149,7 +151,8 @@ skip_enc: cnstr_shdsc_aead_decap(ctx->sh_desc_dec, &ctx->cdata, &ctx->adata, ivsize, ctx->authsize, alg->caam.geniv, - is_rfc3686, nonce, ctx1_iv_off, true); + is_rfc3686, nonce, ctx1_iv_off, true, + ctrlpriv->era); if (!alg->caam.geniv) goto skip_givenc; @@ -176,7 +179,7 @@ skip_enc: cnstr_shdsc_aead_givencap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata, ivsize, ctx->authsize, is_rfc3686, nonce, - ctx1_iv_off, true); + ctx1_iv_off, true, ctrlpriv->era); skip_givenc: return 0; @@ -197,6 +200,7 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key, { struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); struct crypto_authenc_keys keys; int ret = 0; @@ -211,6 +215,27 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key, DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); #endif + /* + * If DKP is supported, use it in the shared descriptor to generate + * the split key. + */ + if (ctrlpriv->era >= 6) { + ctx->adata.keylen = keys.authkeylen; + ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & + OP_ALG_ALGSEL_MASK); + + if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) + goto badkey; + + memcpy(ctx->key, keys.authkey, keys.authkeylen); + memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, + keys.enckeylen); + dma_sync_single_for_device(jrdev, ctx->key_dma, + ctx->adata.keylen_pad + + keys.enckeylen, ctx->dir); + goto skip_split_key; + } + ret = gen_split_key(jrdev, ctx->key, &ctx->adata, keys.authkey, keys.authkeylen, CAAM_MAX_KEY_SIZE - keys.enckeylen); @@ -220,13 +245,14 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key, /* postpend encryption key to auth split key */ memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + - keys.enckeylen, DMA_TO_DEVICE); + keys.enckeylen, ctx->dir); #ifdef DEBUG print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, ctx->adata.keylen_pad + keys.enckeylen, 1); #endif +skip_split_key: ctx->cdata.keylen = keys.enckeylen; ret = aead_set_sh_desc(aead); @@ -2119,7 +2145,8 @@ struct caam_crypto_alg { struct caam_alg_entry caam; }; -static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam) +static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, + bool uses_dkp) { struct caam_drv_private *priv; @@ -2133,8 +2160,14 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam) return PTR_ERR(ctx->jrdev); } + priv = dev_get_drvdata(ctx->jrdev->parent); + if (priv->era >= 6 && uses_dkp) + ctx->dir = DMA_BIDIRECTIONAL; + else + ctx->dir = DMA_TO_DEVICE; + ctx->key_dma = dma_map_single(ctx->jrdev, ctx->key, sizeof(ctx->key), - DMA_TO_DEVICE); + ctx->dir); if (dma_mapping_error(ctx->jrdev, ctx->key_dma)) { dev_err(ctx->jrdev, "unable to map key\n"); caam_jr_free(ctx->jrdev); @@ -2145,7 +2178,6 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam) ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; - priv = dev_get_drvdata(ctx->jrdev->parent); ctx->qidev = priv->qidev; spin_lock_init(&ctx->lock); @@ -2163,7 +2195,7 @@ static int caam_cra_init(struct crypto_tfm *tfm) crypto_alg); struct caam_ctx *ctx = crypto_tfm_ctx(tfm); - return caam_init_common(ctx, &caam_alg->caam); + return caam_init_common(ctx, &caam_alg->caam, false); } static int caam_aead_init(struct crypto_aead *tfm) @@ -2173,7 +2205,8 @@ static int caam_aead_init(struct crypto_aead *tfm) aead); struct caam_ctx *ctx = crypto_aead_ctx(tfm); - return caam_init_common(ctx, &caam_alg->caam); + return caam_init_common(ctx, &caam_alg->caam, + alg->setkey == aead_setkey); } static void caam_exit_common(struct caam_ctx *ctx) @@ -2182,8 +2215,7 @@ static void caam_exit_common(struct caam_ctx *ctx) caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]); caam_drv_ctx_rel(ctx->drv_ctx[GIVENCRYPT]); - dma_unmap_single(ctx->jrdev, ctx->key_dma, sizeof(ctx->key), - DMA_TO_DEVICE); + dma_unmap_single(ctx->jrdev, ctx->key_dma, sizeof(ctx->key), ctx->dir); caam_jr_free(ctx->jrdev); } diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c index 616720a04e7a..0beb28196e20 100644 --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -107,6 +107,7 @@ struct caam_hash_ctx { dma_addr_t sh_desc_update_first_dma; dma_addr_t sh_desc_fin_dma; dma_addr_t sh_desc_digest_dma; + enum dma_data_direction dir; struct device *jrdev; u8 key[CAAM_MAX_HASH_KEY_SIZE]; int ctx_len; @@ -241,7 +242,8 @@ static inline int ctx_map_to_sec4_sg(struct device *jrdev, * read and write to seqout */ static inline void ahash_gen_sh_desc(u32 *desc, u32 state, int digestsize, - struct caam_hash_ctx *ctx, bool import_ctx) + struct caam_hash_ctx *ctx, bool import_ctx, + int era) { u32 op = ctx->adata.algtype; u32 *skip_key_load; @@ -254,9 +256,12 @@ static inline void ahash_gen_sh_desc(u32 *desc, u32 state, int digestsize, skip_key_load = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_SHRD); - append_key_as_imm(desc, ctx->key, ctx->adata.keylen_pad, - ctx->adata.keylen, CLASS_2 | - KEY_DEST_MDHA_SPLIT | KEY_ENC); + if (era < 6) + append_key_as_imm(desc, ctx->key, ctx->adata.keylen_pad, + ctx->adata.keylen, CLASS_2 | + KEY_DEST_MDHA_SPLIT | KEY_ENC); + else + append_proto_dkp(desc, &ctx->adata); set_jump_tgt_here(desc, skip_key_load); @@ -289,13 +294,17 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash) struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); int digestsize = crypto_ahash_digestsize(ahash); struct device *jrdev = ctx->jrdev; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); u32 *desc; + ctx->adata.key_virt = ctx->key; + /* ahash_update shared descriptor */ desc = ctx->sh_desc_update; - ahash_gen_sh_desc(desc, OP_ALG_AS_UPDATE, ctx->ctx_len, ctx, true); + ahash_gen_sh_desc(desc, OP_ALG_AS_UPDATE, ctx->ctx_len, ctx, true, + ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); #ifdef DEBUG print_hex_dump(KERN_ERR, "ahash update shdesc@"__stringify(__LINE__)": ", @@ -304,9 +313,10 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash) /* ahash_update_first shared descriptor */ desc = ctx->sh_desc_update_first; - ahash_gen_sh_desc(desc, OP_ALG_AS_INIT, ctx->ctx_len, ctx, false); + ahash_gen_sh_desc(desc, OP_ALG_AS_INIT, ctx->ctx_len, ctx, false, + ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); #ifdef DEBUG print_hex_dump(KERN_ERR, "ahash update first shdesc@"__stringify(__LINE__)": ", @@ -315,9 +325,10 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash) /* ahash_final shared descriptor */ desc = ctx->sh_desc_fin; - ahash_gen_sh_desc(desc, OP_ALG_AS_FINALIZE, digestsize, ctx, true); + ahash_gen_sh_desc(desc, OP_ALG_AS_FINALIZE, digestsize, ctx, true, + ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); #ifdef DEBUG print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, @@ -326,9 +337,10 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash) /* ahash_digest shared descriptor */ desc = ctx->sh_desc_digest; - ahash_gen_sh_desc(desc, OP_ALG_AS_INITFINAL, digestsize, ctx, false); + ahash_gen_sh_desc(desc, OP_ALG_AS_INITFINAL, digestsize, ctx, false, + ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); #ifdef DEBUG print_hex_dump(KERN_ERR, "ahash digest shdesc@"__stringify(__LINE__)": ", @@ -421,6 +433,7 @@ static int ahash_setkey(struct crypto_ahash *ahash, struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); int blocksize = crypto_tfm_alg_blocksize(&ahash->base); int digestsize = crypto_ahash_digestsize(ahash); + struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); int ret; u8 *hashed_key = NULL; @@ -441,16 +454,26 @@ static int ahash_setkey(struct crypto_ahash *ahash, key = hashed_key; } - ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key, keylen, - CAAM_MAX_HASH_KEY_SIZE); - if (ret) - goto bad_free_key; + /* + * If DKP is supported, use it in the shared descriptor to generate + * the split key. + */ + if (ctrlpriv->era >= 6) { + ctx->adata.key_inline = true; + ctx->adata.keylen = keylen; + ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & + OP_ALG_ALGSEL_MASK); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, - ctx->adata.keylen_pad, 1); -#endif + if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE) + goto bad_free_key; + + memcpy(ctx->key, key, keylen); + } else { + ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key, + keylen, CAAM_MAX_HASH_KEY_SIZE); + if (ret) + goto bad_free_key; + } kfree(hashed_key); return ahash_set_sh_desc(ahash); @@ -1715,6 +1738,7 @@ static int caam_hash_cra_init(struct crypto_tfm *tfm) HASH_MSG_LEN + 64, HASH_MSG_LEN + SHA512_DIGEST_SIZE }; dma_addr_t dma_addr; + struct caam_drv_private *priv; /* * Get a Job ring from Job Ring driver to ensure in-order @@ -1726,10 +1750,13 @@ static int caam_hash_cra_init(struct crypto_tfm *tfm) return PTR_ERR(ctx->jrdev); } + priv = dev_get_drvdata(ctx->jrdev->parent); + ctx->dir = priv->era >= 6 ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; + dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_update, offsetof(struct caam_hash_ctx, sh_desc_update_dma), - DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); + ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); if (dma_mapping_error(ctx->jrdev, dma_addr)) { dev_err(ctx->jrdev, "unable to map shared descriptors\n"); caam_jr_free(ctx->jrdev); @@ -1764,7 +1791,7 @@ static void caam_hash_cra_exit(struct crypto_tfm *tfm) dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_update_dma, offsetof(struct caam_hash_ctx, sh_desc_update_dma), - DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); + ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); caam_jr_free(ctx->jrdev); } diff --git a/drivers/crypto/caam/desc.h b/drivers/crypto/caam/desc.h index 8142de7ba050..f76ff160a02c 100644 --- a/drivers/crypto/caam/desc.h +++ b/drivers/crypto/caam/desc.h @@ -444,6 +444,18 @@ #define OP_PCLID_DSAVERIFY (0x16 << OP_PCLID_SHIFT) #define OP_PCLID_RSAENC_PUBKEY (0x18 << OP_PCLID_SHIFT) #define OP_PCLID_RSADEC_PRVKEY (0x19 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_MD5 (0x20 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_SHA1 (0x21 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_SHA224 (0x22 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_SHA256 (0x23 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_SHA384 (0x24 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_SHA512 (0x25 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_RIF_MD5 (0x60 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_RIF_SHA1 (0x61 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_RIF_SHA224 (0x62 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_RIF_SHA256 (0x63 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_RIF_SHA384 (0x64 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_RIF_SHA512 (0x65 << OP_PCLID_SHIFT) /* Assuming OP_TYPE = OP_TYPE_DECAP_PROTOCOL/ENCAP_PROTOCOL */ #define OP_PCLID_IPSEC (0x01 << OP_PCLID_SHIFT) @@ -1093,6 +1105,22 @@ /* MacSec protinfos */ #define OP_PCL_MACSEC 0x0001 +/* Derived Key Protocol (DKP) Protinfo */ +#define OP_PCL_DKP_SRC_SHIFT 14 +#define OP_PCL_DKP_SRC_MASK (3 << OP_PCL_DKP_SRC_SHIFT) +#define OP_PCL_DKP_SRC_IMM (0 << OP_PCL_DKP_SRC_SHIFT) +#define OP_PCL_DKP_SRC_SEQ (1 << OP_PCL_DKP_SRC_SHIFT) +#define OP_PCL_DKP_SRC_PTR (2 << OP_PCL_DKP_SRC_SHIFT) +#define OP_PCL_DKP_SRC_SGF (3 << OP_PCL_DKP_SRC_SHIFT) +#define OP_PCL_DKP_DST_SHIFT 12 +#define OP_PCL_DKP_DST_MASK (3 << OP_PCL_DKP_DST_SHIFT) +#define OP_PCL_DKP_DST_IMM (0 << OP_PCL_DKP_DST_SHIFT) +#define OP_PCL_DKP_DST_SEQ (1 << OP_PCL_DKP_DST_SHIFT) +#define OP_PCL_DKP_DST_PTR (2 << OP_PCL_DKP_DST_SHIFT) +#define OP_PCL_DKP_DST_SGF (3 << OP_PCL_DKP_DST_SHIFT) +#define OP_PCL_DKP_KEY_SHIFT 0 +#define OP_PCL_DKP_KEY_MASK (0xfff << OP_PCL_DKP_KEY_SHIFT) + /* PKI unidirectional protocol protinfo bits */ #define OP_PCL_PKPROT_TEST 0x0008 #define OP_PCL_PKPROT_DECRYPT 0x0004 @@ -1452,6 +1480,7 @@ #define MATH_DEST_REG1 (0x01 << MATH_DEST_SHIFT) #define MATH_DEST_REG2 (0x02 << MATH_DEST_SHIFT) #define MATH_DEST_REG3 (0x03 << MATH_DEST_SHIFT) +#define MATH_DEST_DPOVRD (0x07 << MATH_DEST_SHIFT) #define MATH_DEST_SEQINLEN (0x08 << MATH_DEST_SHIFT) #define MATH_DEST_SEQOUTLEN (0x09 << MATH_DEST_SHIFT) #define MATH_DEST_VARSEQINLEN (0x0a << MATH_DEST_SHIFT) diff --git a/drivers/crypto/caam/desc_constr.h b/drivers/crypto/caam/desc_constr.h index 5b39b7d7a47a..d4256fa4a1d6 100644 --- a/drivers/crypto/caam/desc_constr.h +++ b/drivers/crypto/caam/desc_constr.h @@ -496,4 +496,45 @@ static inline int desc_inline_query(unsigned int sd_base_len, return (rem_bytes >= 0) ? 0 : -1; } +/** + * append_proto_dkp - Derived Key Protocol (DKP): key -> split key + * @desc: pointer to buffer used for descriptor construction + * @adata: pointer to authentication transform definitions. + * keylen should be the length of initial key, while keylen_pad + * the length of the derived (split) key. + * Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, SHA224, + * SHA256, SHA384, SHA512}. + */ +static inline void append_proto_dkp(u32 * const desc, struct alginfo *adata) +{ + u32 protid; + + /* + * Quick & dirty translation from OP_ALG_ALGSEL_{MD5, SHA*} + * to OP_PCLID_DKP_{MD5, SHA*} + */ + protid = (adata->algtype & OP_ALG_ALGSEL_SUBMASK) | + (0x20 << OP_ALG_ALGSEL_SHIFT); + + if (adata->key_inline) { + int words; + + append_operation(desc, OP_TYPE_UNI_PROTOCOL | protid | + OP_PCL_DKP_SRC_IMM | OP_PCL_DKP_DST_IMM | + adata->keylen); + append_data(desc, adata->key_virt, adata->keylen); + + /* Reserve space in descriptor buffer for the derived key */ + words = (ALIGN(adata->keylen_pad, CAAM_CMD_SZ) - + ALIGN(adata->keylen, CAAM_CMD_SZ)) / CAAM_CMD_SZ; + if (words) + (*desc) = cpu_to_caam32(caam32_to_cpu(*desc) + words); + } else { + append_operation(desc, OP_TYPE_UNI_PROTOCOL | protid | + OP_PCL_DKP_SRC_PTR | OP_PCL_DKP_DST_PTR | + adata->keylen); + append_ptr(desc, adata->key_dma); + } +} + #endif /* DESC_CONSTR_H */ diff --git a/drivers/crypto/caam/key_gen.c b/drivers/crypto/caam/key_gen.c index 8c79c3a153dc..312b5f042f31 100644 --- a/drivers/crypto/caam/key_gen.c +++ b/drivers/crypto/caam/key_gen.c @@ -11,36 +11,6 @@ #include "desc_constr.h" #include "key_gen.h" -/** - * split_key_len - Compute MDHA split key length for a given algorithm - * @hash: Hashing algorithm selection, one of OP_ALG_ALGSEL_* - MD5, SHA1, - * SHA224, SHA384, SHA512. - * - * Return: MDHA split key length - */ -static inline u32 split_key_len(u32 hash) -{ - /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */ - static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 }; - u32 idx; - - idx = (hash & OP_ALG_ALGSEL_SUBMASK) >> OP_ALG_ALGSEL_SHIFT; - - return (u32)(mdpadlen[idx] * 2); -} - -/** - * split_key_pad_len - Compute MDHA split key pad length for a given algorithm - * @hash: Hashing algorithm selection, one of OP_ALG_ALGSEL_* - MD5, SHA1, - * SHA224, SHA384, SHA512. - * - * Return: MDHA split key pad length - */ -static inline u32 split_key_pad_len(u32 hash) -{ - return ALIGN(split_key_len(hash), 16); -} - void split_key_done(struct device *dev, u32 *desc, u32 err, void *context) { diff --git a/drivers/crypto/caam/key_gen.h b/drivers/crypto/caam/key_gen.h index 5db055c25bd2..818f78f6fc1a 100644 --- a/drivers/crypto/caam/key_gen.h +++ b/drivers/crypto/caam/key_gen.h @@ -6,6 +6,36 @@ * */ +/** + * split_key_len - Compute MDHA split key length for a given algorithm + * @hash: Hashing algorithm selection, one of OP_ALG_ALGSEL_* - MD5, SHA1, + * SHA224, SHA384, SHA512. + * + * Return: MDHA split key length + */ +static inline u32 split_key_len(u32 hash) +{ + /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */ + static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 }; + u32 idx; + + idx = (hash & OP_ALG_ALGSEL_SUBMASK) >> OP_ALG_ALGSEL_SHIFT; + + return (u32)(mdpadlen[idx] * 2); +} + +/** + * split_key_pad_len - Compute MDHA split key pad length for a given algorithm + * @hash: Hashing algorithm selection, one of OP_ALG_ALGSEL_* - MD5, SHA1, + * SHA224, SHA384, SHA512. + * + * Return: MDHA split key pad length + */ +static inline u32 split_key_pad_len(u32 hash) +{ + return ALIGN(split_key_len(hash), 16); +} + struct split_key_result { struct completion completion; int err; From 14359bd7a045d36f55704d6bc8659645e529c81e Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Wed, 20 Dec 2017 16:03:12 +0800 Subject: [PATCH 091/166] crypto: scomp - delete unused comments There are no init and exit callbacks, so delete its comments. Signed-off-by: Zhou Wang Signed-off-by: Herbert Xu --- include/crypto/internal/scompress.h | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/include/crypto/internal/scompress.h b/include/crypto/internal/scompress.h index ccad9b2c9bd6..0f6ddac1acfc 100644 --- a/include/crypto/internal/scompress.h +++ b/include/crypto/internal/scompress.h @@ -28,17 +28,6 @@ struct crypto_scomp { * @free_ctx: Function frees context allocated with alloc_ctx * @compress: Function performs a compress operation * @decompress: Function performs a de-compress operation - * @init: Initialize the cryptographic transformation object. - * This function is used to initialize the cryptographic - * transformation object. This function is called only once at - * the instantiation time, right after the transformation context - * was allocated. In case the cryptographic hardware has some - * special requirements which need to be handled by software, this - * function shall check for the precise requirement of the - * transformation and put any software fallbacks in place. - * @exit: Deinitialize the cryptographic transformation object. This is a - * counterpart to @init, used to remove various changes set in - * @init. * @base: Common crypto API algorithm data structure */ struct scomp_alg { From 02d9e320c693cc14512836a54ce3953e399fd3a8 Mon Sep 17 00:00:00 2001 From: Corentin Labbe Date: Wed, 20 Dec 2017 18:19:32 +0000 Subject: [PATCH 092/166] crypto: stm32 - Use standard CONFIG name All hardware crypto devices have their CONFIG names using the following convention: CRYPTO_DEV_name_algo This patch apply this conventions on STM32 CONFIG names. Signed-off-by: Corentin Labbe Reviewed-by: Fabien Dessenne Signed-off-by: Herbert Xu --- drivers/crypto/stm32/Kconfig | 6 +++--- drivers/crypto/stm32/Makefile | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/crypto/stm32/Kconfig b/drivers/crypto/stm32/Kconfig index 61ef00b6bf45..63aa78c0b12b 100644 --- a/drivers/crypto/stm32/Kconfig +++ b/drivers/crypto/stm32/Kconfig @@ -1,4 +1,4 @@ -config CRC_DEV_STM32 +config CRYPTO_DEV_STM32_CRC tristate "Support for STM32 crc accelerators" depends on ARCH_STM32 select CRYPTO_HASH @@ -6,7 +6,7 @@ config CRC_DEV_STM32 This enables support for the CRC32 hw accelerator which can be found on STMicroelectronics STM32 SOC. -config HASH_DEV_STM32 +config CRYPTO_DEV_STM32_HASH tristate "Support for STM32 hash accelerators" depends on ARCH_STM32 depends on HAS_DMA @@ -19,7 +19,7 @@ config HASH_DEV_STM32 This enables support for the HASH hw accelerator which can be found on STMicroelectronics STM32 SOC. -config CRYP_DEV_STM32 +config CRYPTO_DEV_STM32_CRYP tristate "Support for STM32 cryp accelerators" depends on ARCH_STM32 select CRYPTO_HASH diff --git a/drivers/crypto/stm32/Makefile b/drivers/crypto/stm32/Makefile index 2c19fc155bfd..53d1bb94b221 100644 --- a/drivers/crypto/stm32/Makefile +++ b/drivers/crypto/stm32/Makefile @@ -1,3 +1,3 @@ -obj-$(CONFIG_CRC_DEV_STM32) += stm32_crc32.o -obj-$(CONFIG_HASH_DEV_STM32) += stm32-hash.o -obj-$(CONFIG_CRYP_DEV_STM32) += stm32-cryp.o +obj-$(CONFIG_CRYPTO_DEV_STM32_CRC) += stm32_crc32.o +obj-$(CONFIG_CRYPTO_DEV_STM32_HASH) += stm32-hash.o +obj-$(CONFIG_CRYPTO_DEV_STM32_CRYP) += stm32-cryp.o From b20209c91e23a9bbad9cac2f80bc16b3c259e10e Mon Sep 17 00:00:00 2001 From: Junaid Shahid Date: Wed, 20 Dec 2017 17:08:37 -0800 Subject: [PATCH 093/166] crypto: aesni - Fix out-of-bounds access of the data buffer in generic-gcm-aesni The aesni_gcm_enc/dec functions can access memory before the start of the data buffer if the length of the data buffer is less than 16 bytes. This is because they perform the read via a single 16-byte load. This can potentially result in accessing a page that is not mapped and thus causing the machine to crash. This patch fixes that by reading the partial block byte-by-byte and optionally an via 8-byte load if the block was at least 8 bytes. Fixes: 0487ccac ("crypto: aesni - make non-AVX AES-GCM work with any aadlen") Cc: Signed-off-by: Junaid Shahid Signed-off-by: Herbert Xu --- arch/x86/crypto/aesni-intel_asm.S | 87 ++++++++++++++++--------------- 1 file changed, 45 insertions(+), 42 deletions(-) diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S index 16627fec80b2..c36b850fdc81 100644 --- a/arch/x86/crypto/aesni-intel_asm.S +++ b/arch/x86/crypto/aesni-intel_asm.S @@ -256,6 +256,37 @@ aad_shift_arr: pxor \TMP1, \GH # result is in TMP1 .endm +# Reads DLEN bytes starting at DPTR and stores in XMMDst +# where 0 < DLEN < 16 +# Clobbers %rax, DLEN and XMM1 +.macro READ_PARTIAL_BLOCK DPTR DLEN XMM1 XMMDst + cmp $8, \DLEN + jl _read_lt8_\@ + mov (\DPTR), %rax + MOVQ_R64_XMM %rax, \XMMDst + sub $8, \DLEN + jz _done_read_partial_block_\@ + xor %eax, %eax +_read_next_byte_\@: + shl $8, %rax + mov 7(\DPTR, \DLEN, 1), %al + dec \DLEN + jnz _read_next_byte_\@ + MOVQ_R64_XMM %rax, \XMM1 + pslldq $8, \XMM1 + por \XMM1, \XMMDst + jmp _done_read_partial_block_\@ +_read_lt8_\@: + xor %eax, %eax +_read_next_byte_lt8_\@: + shl $8, %rax + mov -1(\DPTR, \DLEN, 1), %al + dec \DLEN + jnz _read_next_byte_lt8_\@ + MOVQ_R64_XMM %rax, \XMMDst +_done_read_partial_block_\@: +.endm + /* * if a = number of total plaintext bytes * b = floor(a/16) @@ -1385,14 +1416,6 @@ _esb_loop_\@: * * AAD Format with 64-bit Extended Sequence Number * -* aadLen: -* from the definition of the spec, aadLen can only be 8 or 12 bytes. -* The code supports 16 too but for other sizes, the code will fail. -* -* TLen: -* from the definition of the spec, TLen can only be 8, 12 or 16 bytes. -* For other sizes, the code will fail. -* * poly = x^128 + x^127 + x^126 + x^121 + 1 * *****************************************************************************/ @@ -1486,19 +1509,16 @@ _zero_cipher_left_decrypt: PSHUFB_XMM %xmm10, %xmm0 ENCRYPT_SINGLE_BLOCK %xmm0, %xmm1 # E(K, Yn) - sub $16, %r11 - add %r13, %r11 - movdqu (%arg3,%r11,1), %xmm1 # receive the last <16 byte block - lea SHIFT_MASK+16(%rip), %r12 - sub %r13, %r12 -# adjust the shuffle mask pointer to be able to shift 16-%r13 bytes -# (%r13 is the number of bytes in plaintext mod 16) - movdqu (%r12), %xmm2 # get the appropriate shuffle mask - PSHUFB_XMM %xmm2, %xmm1 # right shift 16-%r13 butes + lea (%arg3,%r11,1), %r10 + mov %r13, %r12 + READ_PARTIAL_BLOCK %r10 %r12 %xmm2 %xmm1 + + lea ALL_F+16(%rip), %r12 + sub %r13, %r12 movdqa %xmm1, %xmm2 pxor %xmm1, %xmm0 # Ciphertext XOR E(K, Yn) - movdqu ALL_F-SHIFT_MASK(%r12), %xmm1 + movdqu (%r12), %xmm1 # get the appropriate mask to mask out top 16-%r13 bytes of %xmm0 pand %xmm1, %xmm0 # mask out top 16-%r13 bytes of %xmm0 pand %xmm1, %xmm2 @@ -1507,9 +1527,6 @@ _zero_cipher_left_decrypt: pxor %xmm2, %xmm8 GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6 - # GHASH computation for the last <16 byte block - sub %r13, %r11 - add $16, %r11 # output %r13 bytes MOVQ_R64_XMM %xmm0, %rax @@ -1663,14 +1680,6 @@ ENDPROC(aesni_gcm_dec) * * AAD Format with 64-bit Extended Sequence Number * -* aadLen: -* from the definition of the spec, aadLen can only be 8 or 12 bytes. -* The code supports 16 too but for other sizes, the code will fail. -* -* TLen: -* from the definition of the spec, TLen can only be 8, 12 or 16 bytes. -* For other sizes, the code will fail. -* * poly = x^128 + x^127 + x^126 + x^121 + 1 ***************************************************************************/ ENTRY(aesni_gcm_enc) @@ -1763,19 +1772,16 @@ _zero_cipher_left_encrypt: movdqa SHUF_MASK(%rip), %xmm10 PSHUFB_XMM %xmm10, %xmm0 - ENCRYPT_SINGLE_BLOCK %xmm0, %xmm1 # Encrypt(K, Yn) - sub $16, %r11 - add %r13, %r11 - movdqu (%arg3,%r11,1), %xmm1 # receive the last <16 byte blocks - lea SHIFT_MASK+16(%rip), %r12 + + lea (%arg3,%r11,1), %r10 + mov %r13, %r12 + READ_PARTIAL_BLOCK %r10 %r12 %xmm2 %xmm1 + + lea ALL_F+16(%rip), %r12 sub %r13, %r12 - # adjust the shuffle mask pointer to be able to shift 16-r13 bytes - # (%r13 is the number of bytes in plaintext mod 16) - movdqu (%r12), %xmm2 # get the appropriate shuffle mask - PSHUFB_XMM %xmm2, %xmm1 # shift right 16-r13 byte pxor %xmm1, %xmm0 # Plaintext XOR Encrypt(K, Yn) - movdqu ALL_F-SHIFT_MASK(%r12), %xmm1 + movdqu (%r12), %xmm1 # get the appropriate mask to mask out top 16-r13 bytes of xmm0 pand %xmm1, %xmm0 # mask out top 16-r13 bytes of xmm0 movdqa SHUF_MASK(%rip), %xmm10 @@ -1784,9 +1790,6 @@ _zero_cipher_left_encrypt: pxor %xmm0, %xmm8 GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6 # GHASH computation for the last <16 byte block - sub %r13, %r11 - add $16, %r11 - movdqa SHUF_MASK(%rip), %xmm10 PSHUFB_XMM %xmm10, %xmm0 From 1ecdd37e308ca149dc378cce225068cbac54e3a6 Mon Sep 17 00:00:00 2001 From: Junaid Shahid Date: Wed, 20 Dec 2017 17:08:38 -0800 Subject: [PATCH 094/166] crypto: aesni - Fix out-of-bounds access of the AAD buffer in generic-gcm-aesni The aesni_gcm_enc/dec functions can access memory after the end of the AAD buffer if the AAD length is not a multiple of 4 bytes. It didn't matter with rfc4106-gcm-aesni as in that case the AAD was always followed by the 8 byte IV, but that is no longer the case with generic-gcm-aesni. This can potentially result in accessing a page that is not mapped and thus causing the machine to crash. This patch fixes that by reading the last <16 byte block of the AAD byte-by-byte and optionally via an 8-byte load if the block was at least 8 bytes. Fixes: 0487ccac ("crypto: aesni - make non-AVX AES-GCM work with any aadlen") Cc: Signed-off-by: Junaid Shahid Signed-off-by: Herbert Xu --- arch/x86/crypto/aesni-intel_asm.S | 112 ++++-------------------------- 1 file changed, 12 insertions(+), 100 deletions(-) diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S index c36b850fdc81..76d8cd426a31 100644 --- a/arch/x86/crypto/aesni-intel_asm.S +++ b/arch/x86/crypto/aesni-intel_asm.S @@ -89,30 +89,6 @@ SHIFT_MASK: .octa 0x0f0e0d0c0b0a09080706050403020100 ALL_F: .octa 0xffffffffffffffffffffffffffffffff .octa 0x00000000000000000000000000000000 -.section .rodata -.align 16 -.type aad_shift_arr, @object -.size aad_shift_arr, 272 -aad_shift_arr: - .octa 0xffffffffffffffffffffffffffffffff - .octa 0xffffffffffffffffffffffffffffff0C - .octa 0xffffffffffffffffffffffffffff0D0C - .octa 0xffffffffffffffffffffffffff0E0D0C - .octa 0xffffffffffffffffffffffff0F0E0D0C - .octa 0xffffffffffffffffffffff0C0B0A0908 - .octa 0xffffffffffffffffffff0D0C0B0A0908 - .octa 0xffffffffffffffffff0E0D0C0B0A0908 - .octa 0xffffffffffffffff0F0E0D0C0B0A0908 - .octa 0xffffffffffffff0C0B0A090807060504 - .octa 0xffffffffffff0D0C0B0A090807060504 - .octa 0xffffffffff0E0D0C0B0A090807060504 - .octa 0xffffffff0F0E0D0C0B0A090807060504 - .octa 0xffffff0C0B0A09080706050403020100 - .octa 0xffff0D0C0B0A09080706050403020100 - .octa 0xff0E0D0C0B0A09080706050403020100 - .octa 0x0F0E0D0C0B0A09080706050403020100 - - .text @@ -303,62 +279,30 @@ _done_read_partial_block_\@: XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation MOVADQ SHUF_MASK(%rip), %xmm14 mov arg7, %r10 # %r10 = AAD - mov arg8, %r12 # %r12 = aadLen - mov %r12, %r11 + mov arg8, %r11 # %r11 = aadLen pxor %xmm\i, %xmm\i pxor \XMM2, \XMM2 cmp $16, %r11 - jl _get_AAD_rest8\num_initial_blocks\operation + jl _get_AAD_rest\num_initial_blocks\operation _get_AAD_blocks\num_initial_blocks\operation: movdqu (%r10), %xmm\i PSHUFB_XMM %xmm14, %xmm\i # byte-reflect the AAD data pxor %xmm\i, \XMM2 GHASH_MUL \XMM2, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 add $16, %r10 - sub $16, %r12 sub $16, %r11 cmp $16, %r11 jge _get_AAD_blocks\num_initial_blocks\operation movdqu \XMM2, %xmm\i + + /* read the last <16B of AAD */ +_get_AAD_rest\num_initial_blocks\operation: cmp $0, %r11 je _get_AAD_done\num_initial_blocks\operation - pxor %xmm\i,%xmm\i - - /* read the last <16B of AAD. since we have at least 4B of - data right after the AAD (the ICV, and maybe some CT), we can - read 4B/8B blocks safely, and then get rid of the extra stuff */ -_get_AAD_rest8\num_initial_blocks\operation: - cmp $4, %r11 - jle _get_AAD_rest4\num_initial_blocks\operation - movq (%r10), \TMP1 - add $8, %r10 - sub $8, %r11 - pslldq $8, \TMP1 - psrldq $8, %xmm\i - pxor \TMP1, %xmm\i - jmp _get_AAD_rest8\num_initial_blocks\operation -_get_AAD_rest4\num_initial_blocks\operation: - cmp $0, %r11 - jle _get_AAD_rest0\num_initial_blocks\operation - mov (%r10), %eax - movq %rax, \TMP1 - add $4, %r10 - sub $4, %r10 - pslldq $12, \TMP1 - psrldq $4, %xmm\i - pxor \TMP1, %xmm\i -_get_AAD_rest0\num_initial_blocks\operation: - /* finalize: shift out the extra bytes we read, and align - left. since pslldq can only shift by an immediate, we use - vpshufb and an array of shuffle masks */ - movq %r12, %r11 - salq $4, %r11 - movdqu aad_shift_arr(%r11), \TMP1 - PSHUFB_XMM \TMP1, %xmm\i -_get_AAD_rest_final\num_initial_blocks\operation: + READ_PARTIAL_BLOCK %r10, %r11, \TMP1, %xmm\i PSHUFB_XMM %xmm14, %xmm\i # byte-reflect the AAD data pxor \XMM2, %xmm\i GHASH_MUL %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 @@ -562,62 +506,30 @@ _initial_blocks_done\num_initial_blocks\operation: XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation MOVADQ SHUF_MASK(%rip), %xmm14 mov arg7, %r10 # %r10 = AAD - mov arg8, %r12 # %r12 = aadLen - mov %r12, %r11 + mov arg8, %r11 # %r11 = aadLen pxor %xmm\i, %xmm\i pxor \XMM2, \XMM2 cmp $16, %r11 - jl _get_AAD_rest8\num_initial_blocks\operation + jl _get_AAD_rest\num_initial_blocks\operation _get_AAD_blocks\num_initial_blocks\operation: movdqu (%r10), %xmm\i PSHUFB_XMM %xmm14, %xmm\i # byte-reflect the AAD data pxor %xmm\i, \XMM2 GHASH_MUL \XMM2, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 add $16, %r10 - sub $16, %r12 sub $16, %r11 cmp $16, %r11 jge _get_AAD_blocks\num_initial_blocks\operation movdqu \XMM2, %xmm\i + + /* read the last <16B of AAD */ +_get_AAD_rest\num_initial_blocks\operation: cmp $0, %r11 je _get_AAD_done\num_initial_blocks\operation - pxor %xmm\i,%xmm\i - - /* read the last <16B of AAD. since we have at least 4B of - data right after the AAD (the ICV, and maybe some PT), we can - read 4B/8B blocks safely, and then get rid of the extra stuff */ -_get_AAD_rest8\num_initial_blocks\operation: - cmp $4, %r11 - jle _get_AAD_rest4\num_initial_blocks\operation - movq (%r10), \TMP1 - add $8, %r10 - sub $8, %r11 - pslldq $8, \TMP1 - psrldq $8, %xmm\i - pxor \TMP1, %xmm\i - jmp _get_AAD_rest8\num_initial_blocks\operation -_get_AAD_rest4\num_initial_blocks\operation: - cmp $0, %r11 - jle _get_AAD_rest0\num_initial_blocks\operation - mov (%r10), %eax - movq %rax, \TMP1 - add $4, %r10 - sub $4, %r10 - pslldq $12, \TMP1 - psrldq $4, %xmm\i - pxor \TMP1, %xmm\i -_get_AAD_rest0\num_initial_blocks\operation: - /* finalize: shift out the extra bytes we read, and align - left. since pslldq can only shift by an immediate, we use - vpshufb and an array of shuffle masks */ - movq %r12, %r11 - salq $4, %r11 - movdqu aad_shift_arr(%r11), \TMP1 - PSHUFB_XMM \TMP1, %xmm\i -_get_AAD_rest_final\num_initial_blocks\operation: + READ_PARTIAL_BLOCK %r10, %r11, \TMP1, %xmm\i PSHUFB_XMM %xmm14, %xmm\i # byte-reflect the AAD data pxor \XMM2, %xmm\i GHASH_MUL %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 From 08b21fbf4bc4befbd77cd19422a81149ba8cda00 Mon Sep 17 00:00:00 2001 From: Cheah Kok Cheong Date: Thu, 21 Dec 2017 19:35:30 +0800 Subject: [PATCH 095/166] padata: add SPDX identifier Add SPDX license identifier according to the type of license text found in the file. Cc: Philippe Ombredanne Signed-off-by: Cheah Kok Cheong Acked-by: Steffen Klassert Signed-off-by: Herbert Xu --- kernel/padata.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/padata.c b/kernel/padata.c index 57c0074d50cc..d568cc56405f 100644 --- a/kernel/padata.c +++ b/kernel/padata.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * padata.c - generic interface to process data streams in parallel * From 6cd225cc5d8a6526b2bea41955882be55d4f109e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stelmach?= Date: Fri, 22 Dec 2017 17:48:35 +0100 Subject: [PATCH 096/166] hwrng: exynos - add Samsung Exynos True RNG driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support for True Random Number Generator found in Samsung Exynos 5250+ SoCs. Signed-off-by: Łukasz Stelmach Reviewed-by: Krzysztof Kozlowski Acked-by: Philippe Ombredanne Signed-off-by: Herbert Xu --- MAINTAINERS | 7 + drivers/char/hw_random/Kconfig | 12 ++ drivers/char/hw_random/Makefile | 1 + drivers/char/hw_random/exynos-trng.c | 237 +++++++++++++++++++++++++++ 4 files changed, 257 insertions(+) create mode 100644 drivers/char/hw_random/exynos-trng.c diff --git a/MAINTAINERS b/MAINTAINERS index e6d849d0d153..1082846edb9b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -11933,6 +11933,13 @@ S: Maintained F: drivers/crypto/exynos-rng.c F: Documentation/devicetree/bindings/crypto/samsung,exynos-rng4.txt +SAMSUNG EXYNOS TRUE RANDOM NUMBER GENERATOR (TRNG) DRIVER +M: Łukasz Stelmach +L: linux-samsung-soc@vger.kernel.org +S: Maintained +F: drivers/char/hw_random/exynos-trng.c +F: Documentation/devicetree/bindings/rng/samsung,exynos5250-trng.txt + SAMSUNG FRAMEBUFFER DRIVER M: Jingoo Han L: linux-fbdev@vger.kernel.org diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig index 90e4bb24819e..32f715394904 100644 --- a/drivers/char/hw_random/Kconfig +++ b/drivers/char/hw_random/Kconfig @@ -437,6 +437,18 @@ config HW_RANDOM_S390 If unsure, say Y. +config HW_RANDOM_EXYNOS + tristate "Samsung Exynos True Random Number Generator support" + depends on ARCH_EXYNOS || COMPILE_TEST + default HW_RANDOM + ---help--- + This driver provides support for the True Random Number + Generator available in Exynos SoCs. + + To compile this driver as a module, choose M here: the module + will be called exynos-trng. + + If unsure, say Y. endif # HW_RANDOM config UML_RANDOM diff --git a/drivers/char/hw_random/Makefile b/drivers/char/hw_random/Makefile index e7146a84d44a..7c8a66fe8cf6 100644 --- a/drivers/char/hw_random/Makefile +++ b/drivers/char/hw_random/Makefile @@ -13,6 +13,7 @@ obj-$(CONFIG_HW_RANDOM_GEODE) += geode-rng.o obj-$(CONFIG_HW_RANDOM_N2RNG) += n2-rng.o n2-rng-y := n2-drv.o n2-asm.o obj-$(CONFIG_HW_RANDOM_VIA) += via-rng.o +obj-$(CONFIG_HW_RANDOM_EXYNOS) += exynos-trng.o obj-$(CONFIG_HW_RANDOM_IXP4XX) += ixp4xx-rng.o obj-$(CONFIG_HW_RANDOM_OMAP) += omap-rng.o obj-$(CONFIG_HW_RANDOM_OMAP3_ROM) += omap3-rom-rng.o diff --git a/drivers/char/hw_random/exynos-trng.c b/drivers/char/hw_random/exynos-trng.c new file mode 100644 index 000000000000..34d6f51ecbee --- /dev/null +++ b/drivers/char/hw_random/exynos-trng.c @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * RNG driver for Exynos TRNGs + * + * Author: Łukasz Stelmach + * + * Copyright 2017 (c) Samsung Electronics Software, Inc. + * + * Based on the Exynos PRNG driver drivers/crypto/exynos-rng by + * Krzysztof Kozłowski + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define EXYNOS_TRNG_CLKDIV (0x0) + +#define EXYNOS_TRNG_CTRL (0x20) +#define EXYNOS_TRNG_CTRL_RNGEN BIT(31) + +#define EXYNOS_TRNG_POST_CTRL (0x30) +#define EXYNOS_TRNG_ONLINE_CTRL (0x40) +#define EXYNOS_TRNG_ONLINE_STAT (0x44) +#define EXYNOS_TRNG_ONLINE_MAXCHI2 (0x48) +#define EXYNOS_TRNG_FIFO_CTRL (0x50) +#define EXYNOS_TRNG_FIFO_0 (0x80) +#define EXYNOS_TRNG_FIFO_1 (0x84) +#define EXYNOS_TRNG_FIFO_2 (0x88) +#define EXYNOS_TRNG_FIFO_3 (0x8c) +#define EXYNOS_TRNG_FIFO_4 (0x90) +#define EXYNOS_TRNG_FIFO_5 (0x94) +#define EXYNOS_TRNG_FIFO_6 (0x98) +#define EXYNOS_TRNG_FIFO_7 (0x9c) +#define EXYNOS_TRNG_FIFO_LEN (8) +#define EXYNOS_TRNG_CLOCK_RATE (500000) + + +struct exynos_trng_dev { + struct device *dev; + void __iomem *mem; + struct clk *clk; + struct hwrng rng; +}; + +static int exynos_trng_do_read(struct hwrng *rng, void *data, size_t max, + bool wait) +{ + struct exynos_trng_dev *trng; + u32 val; + + max = min_t(size_t, max, (EXYNOS_TRNG_FIFO_LEN * 4)); + + trng = (struct exynos_trng_dev *)rng->priv; + + writel_relaxed(max * 8, trng->mem + EXYNOS_TRNG_FIFO_CTRL); + val = readl_poll_timeout(trng->mem + EXYNOS_TRNG_FIFO_CTRL, val, + val == 0, 200, 1000000); + if (val < 0) + return val; + + memcpy_fromio(data, trng->mem + EXYNOS_TRNG_FIFO_0, max); + + return max; +} + +static int exynos_trng_init(struct hwrng *rng) +{ + struct exynos_trng_dev *trng = (struct exynos_trng_dev *)rng->priv; + unsigned long sss_rate; + u32 val; + + sss_rate = clk_get_rate(trng->clk); + + /* + * For most TRNG circuits the clock frequency of under 500 kHz + * is safe. + */ + val = sss_rate / (EXYNOS_TRNG_CLOCK_RATE * 2); + if (val > 0x7fff) { + dev_err(trng->dev, "clock divider too large: %d", val); + return -ERANGE; + } + val = val << 1; + writel_relaxed(val, trng->mem + EXYNOS_TRNG_CLKDIV); + + /* Enable the generator. */ + val = EXYNOS_TRNG_CTRL_RNGEN; + writel_relaxed(val, trng->mem + EXYNOS_TRNG_CTRL); + + /* + * Disable post-processing. /dev/hwrng is supposed to deliver + * unprocessed data. + */ + writel_relaxed(0, trng->mem + EXYNOS_TRNG_POST_CTRL); + + return 0; +} + +static int exynos_trng_probe(struct platform_device *pdev) +{ + struct exynos_trng_dev *trng; + struct resource *res; + int ret = -ENOMEM; + + trng = devm_kzalloc(&pdev->dev, sizeof(*trng), GFP_KERNEL); + if (!trng) + return ret; + + trng->rng.name = devm_kstrdup(&pdev->dev, dev_name(&pdev->dev), + GFP_KERNEL); + if (!trng->rng.name) + return ret; + + trng->rng.init = exynos_trng_init; + trng->rng.read = exynos_trng_do_read; + trng->rng.priv = (unsigned long) trng; + + platform_set_drvdata(pdev, trng); + trng->dev = &pdev->dev; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + trng->mem = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(trng->mem)) { + dev_err(&pdev->dev, "Could not map IO resources.\n"); + return PTR_ERR(trng->mem); + } + + pm_runtime_enable(&pdev->dev); + ret = pm_runtime_get_sync(&pdev->dev); + if (ret < 0) { + dev_err(&pdev->dev, "Could not get runtime PM.\n"); + goto err_pm_get; + } + + trng->clk = devm_clk_get(&pdev->dev, "secss"); + if (IS_ERR(trng->clk)) { + ret = PTR_ERR(trng->clk); + dev_err(&pdev->dev, "Could not get clock.\n"); + goto err_clock; + } + + ret = clk_prepare_enable(trng->clk); + if (ret) { + dev_err(&pdev->dev, "Could not enable the clk.\n"); + goto err_clock; + } + + ret = hwrng_register(&trng->rng); + if (ret) { + dev_err(&pdev->dev, "Could not register hwrng device.\n"); + goto err_register; + } + + dev_info(&pdev->dev, "Exynos True Random Number Generator.\n"); + + return 0; + +err_register: + clk_disable_unprepare(trng->clk); + +err_clock: + pm_runtime_put_sync(&pdev->dev); + +err_pm_get: + pm_runtime_disable(&pdev->dev); + + return ret; +} + +static int exynos_trng_remove(struct platform_device *pdev) +{ + struct exynos_trng_dev *trng = platform_get_drvdata(pdev); + + hwrng_unregister(&trng->rng); + clk_disable_unprepare(trng->clk); + + pm_runtime_put_sync(&pdev->dev); + pm_runtime_disable(&pdev->dev); + + return 0; +} + +static int __maybe_unused exynos_trng_suspend(struct device *dev) +{ + pm_runtime_put_sync(dev); + + return 0; +} + +static int __maybe_unused exynos_trng_resume(struct device *dev) +{ + int ret; + + ret = pm_runtime_get_sync(dev); + if (ret < 0) { + dev_err(dev, "Could not get runtime PM.\n"); + pm_runtime_put_noidle(dev); + return ret; + } + + return 0; +} + +static SIMPLE_DEV_PM_OPS(exynos_trng_pm_ops, exynos_trng_suspend, + exynos_trng_resume); + +static const struct of_device_id exynos_trng_dt_match[] = { + { + .compatible = "samsung,exynos5250-trng", + }, + { }, +}; +MODULE_DEVICE_TABLE(of, exynos_trng_dt_match); + +static struct platform_driver exynos_trng_driver = { + .driver = { + .name = "exynos-trng", + .pm = &exynos_trng_pm_ops, + .of_match_table = exynos_trng_dt_match, + }, + .probe = exynos_trng_probe, + .remove = exynos_trng_remove, +}; + +module_platform_driver(exynos_trng_driver); +MODULE_AUTHOR("Łukasz Stelmach"); +MODULE_DESCRIPTION("H/W TRNG driver for Exynos chips"); +MODULE_LICENSE("GPL v2"); From b0a191cebea13c38f3a427180721f92d8e259b81 Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Fri, 22 Dec 2017 21:18:35 +0100 Subject: [PATCH 097/166] crypto: crypto4xx - shuffle iomap in front of request_irq It is possible to avoid the ce_base null pointer check in the drivers' interrupt handler routine "crypto4xx_ce_interrupt_handler()" by simply doing the iomap in front of the IRQ registration. This way, the ce_base will always be valid in the handler and a branch in an critical path can be avoided. Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_core.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index c44954e274bc..50d5e64fbdbf 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -1075,9 +1075,6 @@ static irqreturn_t crypto4xx_ce_interrupt_handler(int irq, void *data) struct device *dev = (struct device *)data; struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev); - if (!core_dev->dev->ce_base) - return 0; - writel(PPC4XX_INTERRUPT_CLR, core_dev->dev->ce_base + CRYPTO4XX_INT_CLR); tasklet_schedule(&core_dev->tasklet); @@ -1325,13 +1322,6 @@ static int crypto4xx_probe(struct platform_device *ofdev) tasklet_init(&core_dev->tasklet, crypto4xx_bh_tasklet_cb, (unsigned long) dev); - /* Register for Crypto isr, Crypto Engine IRQ */ - core_dev->irq = irq_of_parse_and_map(ofdev->dev.of_node, 0); - rc = request_irq(core_dev->irq, crypto4xx_ce_interrupt_handler, 0, - core_dev->dev->name, dev); - if (rc) - goto err_request_irq; - core_dev->dev->ce_base = of_iomap(ofdev->dev.of_node, 0); if (!core_dev->dev->ce_base) { dev_err(dev, "failed to of_iomap\n"); @@ -1339,6 +1329,13 @@ static int crypto4xx_probe(struct platform_device *ofdev) goto err_iomap; } + /* Register for Crypto isr, Crypto Engine IRQ */ + core_dev->irq = irq_of_parse_and_map(ofdev->dev.of_node, 0); + rc = request_irq(core_dev->irq, crypto4xx_ce_interrupt_handler, 0, + core_dev->dev->name, dev); + if (rc) + goto err_request_irq; + /* need to setup pdr, rdr, gdr and sdr before this */ crypto4xx_hw_init(core_dev->dev); @@ -1352,11 +1349,11 @@ static int crypto4xx_probe(struct platform_device *ofdev) return 0; err_start_dev: - iounmap(core_dev->dev->ce_base); -err_iomap: free_irq(core_dev->irq, dev); err_request_irq: irq_dispose_mapping(core_dev->irq); + iounmap(core_dev->dev->ce_base); +err_iomap: tasklet_kill(&core_dev->tasklet); err_build_sdr: crypto4xx_destroy_sdr(core_dev->dev); From b66c685a482117d4e9ee987d252ca673689a5302 Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Fri, 22 Dec 2017 21:18:36 +0100 Subject: [PATCH 098/166] crypto: crypto4xx - support Revision B parts This patch adds support for the crypto4xx RevB cores found in the 460EX, 460SX and later cores (like the APM821xx). Without this patch, the crypto4xx driver will not be able to process any offloaded requests and simply hang indefinitely. Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_core.c | 48 ++++++++++++++++++++++--- drivers/crypto/amcc/crypto4xx_core.h | 1 + drivers/crypto/amcc/crypto4xx_reg_def.h | 4 ++- 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 50d5e64fbdbf..fde0136029f1 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -128,7 +128,14 @@ static void crypto4xx_hw_init(struct crypto4xx_device *dev) writel(PPC4XX_INT_DESCR_CNT, dev->ce_base + CRYPTO4XX_INT_DESCR_CNT); writel(PPC4XX_INT_DESCR_CNT, dev->ce_base + CRYPTO4XX_INT_DESCR_CNT); writel(PPC4XX_INT_CFG, dev->ce_base + CRYPTO4XX_INT_CFG); - writel(PPC4XX_PD_DONE_INT, dev->ce_base + CRYPTO4XX_INT_EN); + if (dev->is_revb) { + writel(PPC4XX_INT_TIMEOUT_CNT_REVB << 10, + dev->ce_base + CRYPTO4XX_INT_TIMEOUT_CNT); + writel(PPC4XX_PD_DONE_INT | PPC4XX_TMO_ERR_INT, + dev->ce_base + CRYPTO4XX_INT_EN); + } else { + writel(PPC4XX_PD_DONE_INT, dev->ce_base + CRYPTO4XX_INT_EN); + } } int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size) @@ -1070,18 +1077,29 @@ static void crypto4xx_bh_tasklet_cb(unsigned long data) /** * Top Half of isr. */ -static irqreturn_t crypto4xx_ce_interrupt_handler(int irq, void *data) +static inline irqreturn_t crypto4xx_interrupt_handler(int irq, void *data, + u32 clr_val) { struct device *dev = (struct device *)data; struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev); - writel(PPC4XX_INTERRUPT_CLR, - core_dev->dev->ce_base + CRYPTO4XX_INT_CLR); + writel(clr_val, core_dev->dev->ce_base + CRYPTO4XX_INT_CLR); tasklet_schedule(&core_dev->tasklet); return IRQ_HANDLED; } +static irqreturn_t crypto4xx_ce_interrupt_handler(int irq, void *data) +{ + return crypto4xx_interrupt_handler(irq, data, PPC4XX_INTERRUPT_CLR); +} + +static irqreturn_t crypto4xx_ce_interrupt_handler_revb(int irq, void *data) +{ + return crypto4xx_interrupt_handler(irq, data, PPC4XX_INTERRUPT_CLR | + PPC4XX_TMO_ERR_INT); +} + /** * Supported Crypto Algorithms */ @@ -1263,6 +1281,8 @@ static int crypto4xx_probe(struct platform_device *ofdev) struct resource res; struct device *dev = &ofdev->dev; struct crypto4xx_core_device *core_dev; + u32 pvr; + bool is_revb = true; rc = of_address_to_resource(ofdev->dev.of_node, 0, &res); if (rc) @@ -1279,6 +1299,7 @@ static int crypto4xx_probe(struct platform_device *ofdev) mfdcri(SDR0, PPC405EX_SDR0_SRST) | PPC405EX_CE_RESET); mtdcri(SDR0, PPC405EX_SDR0_SRST, mfdcri(SDR0, PPC405EX_SDR0_SRST) & ~PPC405EX_CE_RESET); + is_revb = false; } else if (of_find_compatible_node(NULL, NULL, "amcc,ppc460sx-crypto")) { mtdcri(SDR0, PPC460SX_SDR0_SRST, @@ -1301,7 +1322,22 @@ static int crypto4xx_probe(struct platform_device *ofdev) if (!core_dev->dev) goto err_alloc_dev; + /* + * Older version of 460EX/GT have a hardware bug. + * Hence they do not support H/W based security intr coalescing + */ + pvr = mfspr(SPRN_PVR); + if (is_revb && ((pvr >> 4) == 0x130218A)) { + u32 min = PVR_MIN(pvr); + + if (min < 4) { + dev_info(dev, "RevA detected - disable interrupt coalescing\n"); + is_revb = false; + } + } + core_dev->dev->core_dev = core_dev; + core_dev->dev->is_revb = is_revb; core_dev->device = dev; spin_lock_init(&core_dev->lock); INIT_LIST_HEAD(&core_dev->dev->alg_list); @@ -1331,7 +1367,9 @@ static int crypto4xx_probe(struct platform_device *ofdev) /* Register for Crypto isr, Crypto Engine IRQ */ core_dev->irq = irq_of_parse_and_map(ofdev->dev.of_node, 0); - rc = request_irq(core_dev->irq, crypto4xx_ce_interrupt_handler, 0, + rc = request_irq(core_dev->irq, is_revb ? + crypto4xx_ce_interrupt_handler_revb : + crypto4xx_ce_interrupt_handler, 0, core_dev->dev->name, dev); if (rc) goto err_request_irq; diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index 8ac3bd37203b..013d9992a44e 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -109,6 +109,7 @@ struct crypto4xx_device { struct list_head alg_list; /* List of algorithm supported by this device */ struct ratelimit_state aead_ratelimit; + bool is_revb; }; struct crypto4xx_core_device { diff --git a/drivers/crypto/amcc/crypto4xx_reg_def.h b/drivers/crypto/amcc/crypto4xx_reg_def.h index 0a22ec5d1a96..472331787e04 100644 --- a/drivers/crypto/amcc/crypto4xx_reg_def.h +++ b/drivers/crypto/amcc/crypto4xx_reg_def.h @@ -121,13 +121,15 @@ #define PPC4XX_PD_SIZE 6 #define PPC4XX_CTX_DONE_INT 0x2000 #define PPC4XX_PD_DONE_INT 0x8000 +#define PPC4XX_TMO_ERR_INT 0x40000 #define PPC4XX_BYTE_ORDER 0x22222 #define PPC4XX_INTERRUPT_CLR 0x3ffff #define PPC4XX_PRNG_CTRL_AUTO_EN 0x3 #define PPC4XX_DC_3DES_EN 1 #define PPC4XX_TRNG_EN 0x00020000 -#define PPC4XX_INT_DESCR_CNT 4 +#define PPC4XX_INT_DESCR_CNT 7 #define PPC4XX_INT_TIMEOUT_CNT 0 +#define PPC4XX_INT_TIMEOUT_CNT_REVB 0x3FF #define PPC4XX_INT_CFG 1 /** * all follow define are ad hoc From 57268aba3834ae1fde93ccc95f2e0540ba98cedd Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Fri, 22 Dec 2017 21:18:37 +0100 Subject: [PATCH 099/166] crypto: crypto4xx - fix missing irq devname crypto4xx_device's name variable is not set to anything. The common devname for request_irq seems to be the module name. This will fix the seemingly anonymous interrupt entry in /proc/interrupts for crypto4xx. Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_core.c | 2 +- drivers/crypto/amcc/crypto4xx_core.h | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index fde0136029f1..4b03318775ac 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -1370,7 +1370,7 @@ static int crypto4xx_probe(struct platform_device *ofdev) rc = request_irq(core_dev->irq, is_revb ? crypto4xx_ce_interrupt_handler_revb : crypto4xx_ce_interrupt_handler, 0, - core_dev->dev->name, dev); + KBUILD_MODNAME, dev); if (rc) goto err_request_irq; diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index 013d9992a44e..61a02a4c5794 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -82,7 +82,6 @@ struct pd_uinfo { struct crypto4xx_device { struct crypto4xx_core_device *core_dev; - char *name; void __iomem *ce_base; void __iomem *trng_base; From 333b1928566f92ab3d9a108d570d93400c07db6d Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Fri, 22 Dec 2017 21:18:38 +0100 Subject: [PATCH 100/166] crypto: crypto4xx - kill MODULE_NAME KBUILD_MODNAME provides the same value. Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_core.c | 2 +- drivers/crypto/amcc/crypto4xx_core.h | 2 -- drivers/crypto/amcc/crypto4xx_trng.c | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 4b03318775ac..f148627e925c 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -1432,7 +1432,7 @@ MODULE_DEVICE_TABLE(of, crypto4xx_match); static struct platform_driver crypto4xx_driver = { .driver = { - .name = MODULE_NAME, + .name = KBUILD_MODNAME, .of_match_table = crypto4xx_match, }, .probe = crypto4xx_probe, diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index 61a02a4c5794..23b726da6534 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -28,8 +28,6 @@ #include "crypto4xx_reg_def.h" #include "crypto4xx_sa.h" -#define MODULE_NAME "crypto4xx" - #define PPC460SX_SDR0_SRST 0x201 #define PPC405EX_SDR0_SRST 0x200 #define PPC460EX_SDR0_SRST 0x201 diff --git a/drivers/crypto/amcc/crypto4xx_trng.c b/drivers/crypto/amcc/crypto4xx_trng.c index 677ca17fd223..5e63742b0d22 100644 --- a/drivers/crypto/amcc/crypto4xx_trng.c +++ b/drivers/crypto/amcc/crypto4xx_trng.c @@ -92,7 +92,7 @@ void ppc4xx_trng_probe(struct crypto4xx_core_device *core_dev) if (!rng) goto err_out; - rng->name = MODULE_NAME; + rng->name = KBUILD_MODNAME; rng->data_present = ppc4xx_trng_data_present; rng->data_read = ppc4xx_trng_data_read; rng->priv = (unsigned long) dev; From 0b5a7f71b4c557b15ec54a1b49023bc1b21044cc Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Sat, 23 Dec 2017 19:45:46 +0100 Subject: [PATCH 101/166] crypto: crypto4xx - perform aead icv check in the driver The ccm-aes-ppc4xx now fails one of testmgr's expected failure test cases as such: |decryption failed on test 10 for ccm-aes-ppc4xx: |ret was 0, |expected -EBADMSG It doesn't look like the hardware sets the authentication failure flag. The original vendor source from which this was ported does not have any special code or notes about why this would happen or if there are any WAs. Hence, this patch converts the aead_done callback handler to perform the icv check in the driver. And this fixes the false negative and the ccm-aes-ppc4xx passes the selftests once again. |name : ccm(aes) |driver : ccm-aes-ppc4xx |module : crypto4xx |priority : 300 |refcnt : 1 |selftest : passed |internal : no |type : aead |async : yes |blocksize : 1 |ivsize : 16 |maxauthsize : 16 |geniv : Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_alg.c | 6 +--- drivers/crypto/amcc/crypto4xx_core.c | 54 ++++++++++++++-------------- 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index eeaf27859d80..ea83d0bff0e9 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -256,10 +256,6 @@ static inline bool crypto4xx_aead_need_fallback(struct aead_request *req, if (is_ccm && !(req->iv[0] == 1 || req->iv[0] == 3)) return true; - /* CCM - fix CBC MAC mismatch in special case */ - if (is_ccm && decrypt && !req->assoclen) - return true; - return false; } @@ -330,7 +326,7 @@ int crypto4xx_setkey_aes_ccm(struct crypto_aead *cipher, const u8 *key, sa = (struct dynamic_sa_ctl *) ctx->sa_in; sa->sa_contents.w = SA_AES_CCM_CONTENTS | (keylen << 2); - set_dynamic_sa_command_0(sa, SA_NOT_SAVE_HASH, SA_NOT_SAVE_IV, + set_dynamic_sa_command_0(sa, SA_SAVE_HASH, SA_NOT_SAVE_IV, SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE, SA_NO_HEADER_PROC, SA_HASH_ALG_CBC_MAC, SA_CIPHER_ALG_AES, diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index f148627e925c..ad19aa806931 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -577,15 +577,14 @@ static void crypto4xx_aead_done(struct crypto4xx_device *dev, struct pd_uinfo *pd_uinfo, struct ce_pd *pd) { - struct aead_request *aead_req; - struct crypto4xx_ctx *ctx; + struct aead_request *aead_req = container_of(pd_uinfo->async_req, + struct aead_request, base); struct scatterlist *dst = pd_uinfo->dest_va; + size_t cp_len = crypto_aead_authsize( + crypto_aead_reqtfm(aead_req)); + u32 icv[cp_len]; int err = 0; - aead_req = container_of(pd_uinfo->async_req, struct aead_request, - base); - ctx = crypto_tfm_ctx(aead_req->base.tfm); - if (pd_uinfo->using_sd) { crypto4xx_copy_pkt_to_dst(dev, pd, pd_uinfo, pd->pd_ctl_len.bf.pkt_len, @@ -597,38 +596,39 @@ static void crypto4xx_aead_done(struct crypto4xx_device *dev, if (pd_uinfo->sa_va->sa_command_0.bf.dir == DIR_OUTBOUND) { /* append icv at the end */ - size_t cp_len = crypto_aead_authsize( - crypto_aead_reqtfm(aead_req)); - u32 icv[cp_len]; - crypto4xx_memcpy_from_le32(icv, pd_uinfo->sr_va->save_digest, cp_len); scatterwalk_map_and_copy(icv, dst, aead_req->cryptlen, cp_len, 1); + } else { + /* check icv at the end */ + scatterwalk_map_and_copy(icv, aead_req->src, + aead_req->assoclen + aead_req->cryptlen - + cp_len, cp_len, 0); + + crypto4xx_memcpy_from_le32(icv, icv, cp_len); + + if (crypto_memneq(icv, pd_uinfo->sr_va->save_digest, cp_len)) + err = -EBADMSG; } crypto4xx_ret_sg_desc(dev, pd_uinfo); if (pd->pd_ctl.bf.status & 0xff) { - if (pd->pd_ctl.bf.status & 0x1) { - /* authentication error */ - err = -EBADMSG; - } else { - if (!__ratelimit(&dev->aead_ratelimit)) { - if (pd->pd_ctl.bf.status & 2) - pr_err("pad fail error\n"); - if (pd->pd_ctl.bf.status & 4) - pr_err("seqnum fail\n"); - if (pd->pd_ctl.bf.status & 8) - pr_err("error _notify\n"); - pr_err("aead return err status = 0x%02x\n", - pd->pd_ctl.bf.status & 0xff); - pr_err("pd pad_ctl = 0x%08x\n", - pd->pd_ctl.bf.pd_pad_ctl); - } - err = -EINVAL; + if (!__ratelimit(&dev->aead_ratelimit)) { + if (pd->pd_ctl.bf.status & 2) + pr_err("pad fail error\n"); + if (pd->pd_ctl.bf.status & 4) + pr_err("seqnum fail\n"); + if (pd->pd_ctl.bf.status & 8) + pr_err("error _notify\n"); + pr_err("aead return err status = 0x%02x\n", + pd->pd_ctl.bf.status & 0xff); + pr_err("pd pad_ctl = 0x%08x\n", + pd->pd_ctl.bf.pd_pad_ctl); } + err = -EINVAL; } if (pd_uinfo->state & PD_ENTRY_BUSY) From c957f8b3e2e54b29f53ef69decc87bbc858c9b58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Tue, 26 Dec 2017 17:21:16 +0100 Subject: [PATCH 102/166] crypto: inside-secure - avoid unmapping DMA memory that was not mapped This patch adds a parameter in the SafeXcel ahash request structure to keep track of the number of SG entries mapped. This allows not to call dma_unmap_sg() when dma_map_sg() wasn't called in the first place. This also removes a warning when the debugging of the DMA-API is enabled in the kernel configuration: "DMA-API: device driver tries to free DMA memory it has not allocated". Cc: stable@vger.kernel.org Fixes: 1b44c5a60c13 ("crypto: inside-secure - add SafeXcel EIP197 crypto engine driver") Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel_hash.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index d94614afc53d..fbc03d6e7db7 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -33,6 +33,8 @@ struct safexcel_ahash_req { bool hmac; bool needs_inv; + int nents; + u8 state_sz; /* expected sate size, only set once */ u32 state[SHA256_DIGEST_SIZE / sizeof(u32)] __aligned(sizeof(u32)); @@ -151,8 +153,10 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin memcpy(areq->result, sreq->state, crypto_ahash_digestsize(ahash)); - dma_unmap_sg(priv->dev, areq->src, - sg_nents_for_len(areq->src, areq->nbytes), DMA_TO_DEVICE); + if (sreq->nents) { + dma_unmap_sg(priv->dev, areq->src, sreq->nents, DMA_TO_DEVICE); + sreq->nents = 0; + } safexcel_free_context(priv, async, sreq->state_sz); @@ -177,7 +181,7 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, struct safexcel_command_desc *cdesc, *first_cdesc = NULL; struct safexcel_result_desc *rdesc; struct scatterlist *sg; - int i, nents, queued, len, cache_len, extra, n_cdesc = 0, ret = 0; + int i, queued, len, cache_len, extra, n_cdesc = 0, ret = 0; queued = len = req->len - req->processed; if (queued < crypto_ahash_blocksize(ahash)) @@ -233,15 +237,15 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, } /* Now handle the current ahash request buffer(s) */ - nents = dma_map_sg(priv->dev, areq->src, - sg_nents_for_len(areq->src, areq->nbytes), - DMA_TO_DEVICE); - if (!nents) { + req->nents = dma_map_sg(priv->dev, areq->src, + sg_nents_for_len(areq->src, areq->nbytes), + DMA_TO_DEVICE); + if (!req->nents) { ret = -ENOMEM; goto cdesc_rollback; } - for_each_sg(areq->src, sg, nents, i) { + for_each_sg(areq->src, sg, req->nents, i) { int sglen = sg_dma_len(sg); /* Do not overflow the request */ From 809778e02cd45d0625439fee67688f655627bb3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Tue, 26 Dec 2017 17:21:17 +0100 Subject: [PATCH 103/166] crypto: inside-secure - fix hash when length is a multiple of a block This patch fixes the hash support in the SafeXcel driver when the update size is a multiple of a block size, and when a final call is made just after with a size of 0. In such cases the driver should cache the last block from the update to avoid handling 0 length data on the final call (that's a hardware limitation). Cc: stable@vger.kernel.org Fixes: 1b44c5a60c13 ("crypto: inside-secure - add SafeXcel EIP197 crypto engine driver") Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel_hash.c | 34 ++++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index fbc03d6e7db7..122a2a58e98f 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -189,17 +189,31 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, else cache_len = queued - areq->nbytes; - /* - * If this is not the last request and the queued data does not fit - * into full blocks, cache it for the next send() call. - */ - extra = queued & (crypto_ahash_blocksize(ahash) - 1); - if (!req->last_req && extra) { - sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), - req->cache_next, extra, areq->nbytes - extra); + if (!req->last_req) { + /* If this is not the last request and the queued data does not + * fit into full blocks, cache it for the next send() call. + */ + extra = queued & (crypto_ahash_blocksize(ahash) - 1); + if (!extra) + /* If this is not the last request and the queued data + * is a multiple of a block, cache the last one for now. + */ + extra = queued - crypto_ahash_blocksize(ahash); - queued -= extra; - len -= extra; + if (extra) { + sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), + req->cache_next, extra, + areq->nbytes - extra); + + queued -= extra; + len -= extra; + + if (!queued) { + *commands = 0; + *results = 0; + return 0; + } + } } spin_lock_bh(&priv->ring[ring].egress_lock); From ce8614a312ef750bb60677bb13680cb27d1c284b Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 29 Dec 2017 10:00:46 -0600 Subject: [PATCH 104/166] crypto: algapi - convert cra_refcnt to refcount_t Reference counters should use refcount_t rather than atomic_t, since the refcount_t implementation can prevent overflows, reducing the exploitability of reference leak bugs. crypto_alg.cra_refcount is a reference counter with the usual semantics, so switch it over to refcount_t. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/algapi.c | 8 ++++---- crypto/api.c | 2 +- crypto/crypto_user.c | 4 ++-- crypto/internal.h | 4 ++-- crypto/proc.c | 2 +- include/linux/crypto.h | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/crypto/algapi.c b/crypto/algapi.c index 60d7366ed343..8084a76e01d8 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -62,7 +62,7 @@ static int crypto_check_alg(struct crypto_alg *alg) if (alg->cra_priority < 0) return -EINVAL; - atomic_set(&alg->cra_refcnt, 1); + refcount_set(&alg->cra_refcnt, 1); return crypto_set_driver_name(alg); } @@ -224,7 +224,7 @@ static struct crypto_larval *__crypto_register_alg(struct crypto_alg *alg) if (!larval->adult) goto free_larval; - atomic_set(&larval->alg.cra_refcnt, 1); + refcount_set(&larval->alg.cra_refcnt, 1); memcpy(larval->alg.cra_driver_name, alg->cra_driver_name, CRYPTO_MAX_ALG_NAME); larval->alg.cra_priority = alg->cra_priority; @@ -399,7 +399,7 @@ int crypto_unregister_alg(struct crypto_alg *alg) if (ret) return ret; - BUG_ON(atomic_read(&alg->cra_refcnt) != 1); + BUG_ON(refcount_read(&alg->cra_refcnt) != 1); if (alg->cra_destroy) alg->cra_destroy(alg); @@ -490,7 +490,7 @@ void crypto_unregister_template(struct crypto_template *tmpl) up_write(&crypto_alg_sem); hlist_for_each_entry_safe(inst, n, list, list) { - BUG_ON(atomic_read(&inst->alg.cra_refcnt) != 1); + BUG_ON(refcount_read(&inst->alg.cra_refcnt) != 1); crypto_free_instance(inst); } crypto_remove_final(&users); diff --git a/crypto/api.c b/crypto/api.c index 6da802d7be67..70a894e52ff3 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -137,7 +137,7 @@ static struct crypto_alg *crypto_larval_add(const char *name, u32 type, if (IS_ERR(larval)) return ERR_CAST(larval); - atomic_set(&larval->alg.cra_refcnt, 2); + refcount_set(&larval->alg.cra_refcnt, 2); down_write(&crypto_alg_sem); alg = __crypto_alg_lookup(name, type, mask); diff --git a/crypto/crypto_user.c b/crypto/crypto_user.c index 0dbe2be7f783..5c291eedaa70 100644 --- a/crypto/crypto_user.c +++ b/crypto/crypto_user.c @@ -169,7 +169,7 @@ static int crypto_report_one(struct crypto_alg *alg, ualg->cru_type = 0; ualg->cru_mask = 0; ualg->cru_flags = alg->cra_flags; - ualg->cru_refcnt = atomic_read(&alg->cra_refcnt); + ualg->cru_refcnt = refcount_read(&alg->cra_refcnt); if (nla_put_u32(skb, CRYPTOCFGA_PRIORITY_VAL, alg->cra_priority)) goto nla_put_failure; @@ -387,7 +387,7 @@ static int crypto_del_alg(struct sk_buff *skb, struct nlmsghdr *nlh, goto drop_alg; err = -EBUSY; - if (atomic_read(&alg->cra_refcnt) > 2) + if (refcount_read(&alg->cra_refcnt) > 2) goto drop_alg; err = crypto_unregister_instance((struct crypto_instance *)alg); diff --git a/crypto/internal.h b/crypto/internal.h index ae65e5fcaa59..1388af6da85a 100644 --- a/crypto/internal.h +++ b/crypto/internal.h @@ -105,13 +105,13 @@ int crypto_type_has_alg(const char *name, const struct crypto_type *frontend, static inline struct crypto_alg *crypto_alg_get(struct crypto_alg *alg) { - atomic_inc(&alg->cra_refcnt); + refcount_inc(&alg->cra_refcnt); return alg; } static inline void crypto_alg_put(struct crypto_alg *alg) { - if (atomic_dec_and_test(&alg->cra_refcnt) && alg->cra_destroy) + if (refcount_dec_and_test(&alg->cra_refcnt) && alg->cra_destroy) alg->cra_destroy(alg); } diff --git a/crypto/proc.c b/crypto/proc.c index 2cc10c96d753..822fcef6d91c 100644 --- a/crypto/proc.c +++ b/crypto/proc.c @@ -46,7 +46,7 @@ static int c_show(struct seq_file *m, void *p) seq_printf(m, "driver : %s\n", alg->cra_driver_name); seq_printf(m, "module : %s\n", module_name(alg->cra_module)); seq_printf(m, "priority : %d\n", alg->cra_priority); - seq_printf(m, "refcnt : %d\n", atomic_read(&alg->cra_refcnt)); + seq_printf(m, "refcnt : %u\n", refcount_read(&alg->cra_refcnt)); seq_printf(m, "selftest : %s\n", (alg->cra_flags & CRYPTO_ALG_TESTED) ? "passed" : "unknown"); diff --git a/include/linux/crypto.h b/include/linux/crypto.h index 78508ca4b108..231e59f90d32 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -447,7 +447,7 @@ struct crypto_alg { unsigned int cra_alignmask; int cra_priority; - atomic_t cra_refcnt; + refcount_t cra_refcnt; char cra_name[CRYPTO_MAX_ALG_NAME]; char cra_driver_name[CRYPTO_MAX_ALG_NAME]; From 8b55107c57f763c7ca393e72f7ce6f89ea1ba49a Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 29 Dec 2017 10:06:46 -0600 Subject: [PATCH 105/166] crypto: algapi - remove unused notifications There is a message posted to the crypto notifier chain when an algorithm is unregistered, and when a template is registered or unregistered. But nothing is listening for those messages; currently there are only listeners for the algorithm request and registration messages. Get rid of these unused notifications for now. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/algapi.c | 5 ----- crypto/internal.h | 3 --- 2 files changed, 8 deletions(-) diff --git a/crypto/algapi.c b/crypto/algapi.c index 8084a76e01d8..9895cafcce7e 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -123,7 +123,6 @@ static void crypto_remove_instance(struct crypto_instance *inst, if (!tmpl || !crypto_tmpl_get(tmpl)) return; - crypto_notify(CRYPTO_MSG_ALG_UNREGISTER, &inst->alg); list_move(&inst->alg.cra_list, list); hlist_del(&inst->list); inst->alg.cra_destroy = crypto_destroy_instance; @@ -380,7 +379,6 @@ static int crypto_remove_alg(struct crypto_alg *alg, struct list_head *list) alg->cra_flags |= CRYPTO_ALG_DEAD; - crypto_notify(CRYPTO_MSG_ALG_UNREGISTER, alg); list_del_init(&alg->cra_list); crypto_remove_spawns(alg, list, NULL); @@ -458,7 +456,6 @@ int crypto_register_template(struct crypto_template *tmpl) } list_add(&tmpl->list, &crypto_template_list); - crypto_notify(CRYPTO_MSG_TMPL_REGISTER, tmpl); err = 0; out: up_write(&crypto_alg_sem); @@ -485,8 +482,6 @@ void crypto_unregister_template(struct crypto_template *tmpl) BUG_ON(err); } - crypto_notify(CRYPTO_MSG_TMPL_UNREGISTER, tmpl); - up_write(&crypto_alg_sem); hlist_for_each_entry_safe(inst, n, list, list) { diff --git a/crypto/internal.h b/crypto/internal.h index 1388af6da85a..5ac27fba10e8 100644 --- a/crypto/internal.h +++ b/crypto/internal.h @@ -30,9 +30,6 @@ enum { CRYPTO_MSG_ALG_REQUEST, CRYPTO_MSG_ALG_REGISTER, - CRYPTO_MSG_ALG_UNREGISTER, - CRYPTO_MSG_TMPL_REGISTER, - CRYPTO_MSG_TMPL_UNREGISTER, }; struct crypto_instance; From fcfbeedf79adc7abaea35b0f88ec23cf546d3b77 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 29 Dec 2017 10:10:24 -0600 Subject: [PATCH 106/166] crypto: poly1305 - use unaligned access macros to output digest Currently the only part of poly1305-generic which is assuming special alignment is the part where the final digest is written. Switch this over to the unaligned access macros so that we'll be able to remove the cra_alignmask. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/poly1305_generic.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/crypto/poly1305_generic.c b/crypto/poly1305_generic.c index b1c2d57dc734..d752901ba0bc 100644 --- a/crypto/poly1305_generic.c +++ b/crypto/poly1305_generic.c @@ -210,7 +210,6 @@ EXPORT_SYMBOL_GPL(crypto_poly1305_update); int crypto_poly1305_final(struct shash_desc *desc, u8 *dst) { struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - __le32 *mac = (__le32 *)dst; u32 h0, h1, h2, h3, h4; u32 g0, g1, g2, g3, g4; u32 mask; @@ -267,10 +266,10 @@ int crypto_poly1305_final(struct shash_desc *desc, u8 *dst) h3 = (h3 >> 18) | (h4 << 8); /* mac = (h + s) % (2^128) */ - f = (f >> 32) + h0 + dctx->s[0]; mac[0] = cpu_to_le32(f); - f = (f >> 32) + h1 + dctx->s[1]; mac[1] = cpu_to_le32(f); - f = (f >> 32) + h2 + dctx->s[2]; mac[2] = cpu_to_le32(f); - f = (f >> 32) + h3 + dctx->s[3]; mac[3] = cpu_to_le32(f); + f = (f >> 32) + h0 + dctx->s[0]; put_unaligned_le32(f, dst + 0); + f = (f >> 32) + h1 + dctx->s[1]; put_unaligned_le32(f, dst + 4); + f = (f >> 32) + h2 + dctx->s[2]; put_unaligned_le32(f, dst + 8); + f = (f >> 32) + h3 + dctx->s[3]; put_unaligned_le32(f, dst + 12); return 0; } From 4c7dfbd42183184d20134a795cf152c9701ccddd Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 29 Dec 2017 10:10:25 -0600 Subject: [PATCH 107/166] crypto: poly1305 - remove cra_alignmask Now that nothing in poly1305-generic assumes any special alignment, remove the cra_alignmask so that the crypto API does not have to unnecessarily align the buffers. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/poly1305_generic.c | 1 - 1 file changed, 1 deletion(-) diff --git a/crypto/poly1305_generic.c b/crypto/poly1305_generic.c index d752901ba0bc..d92617aeb783 100644 --- a/crypto/poly1305_generic.c +++ b/crypto/poly1305_generic.c @@ -287,7 +287,6 @@ static struct shash_alg poly1305_alg = { .cra_driver_name = "poly1305-generic", .cra_priority = 100, .cra_flags = CRYPTO_ALG_TYPE_SHASH, - .cra_alignmask = sizeof(u32) - 1, .cra_blocksize = POLY1305_BLOCK_SIZE, .cra_module = THIS_MODULE, }, From b7dac3731848923399c7a92563e3754e8f3b5236 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 29 Dec 2017 10:10:26 -0600 Subject: [PATCH 108/166] crypto: x86/poly1305 - remove cra_alignmask crypto_poly1305_final() no longer requires a cra_alignmask, and nothing else in the x86 poly1305-simd implementation does either. So remove the cra_alignmask so that the crypto API does not have to unnecessarily align the buffers. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/poly1305_glue.c | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/x86/crypto/poly1305_glue.c b/arch/x86/crypto/poly1305_glue.c index e32142bc071d..f58f89b05a7f 100644 --- a/arch/x86/crypto/poly1305_glue.c +++ b/arch/x86/crypto/poly1305_glue.c @@ -171,7 +171,6 @@ static struct shash_alg alg = { .cra_driver_name = "poly1305-simd", .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_SHASH, - .cra_alignmask = sizeof(u32) - 1, .cra_blocksize = POLY1305_BLOCK_SIZE, .cra_module = THIS_MODULE, }, From 75d68369b544acc5d14c18a827654dfff248d09d Mon Sep 17 00:00:00 2001 From: Himanshu Jha Date: Sun, 31 Dec 2017 17:54:23 +0530 Subject: [PATCH 109/166] crypto: Use zeroing memory allocator instead of allocator/memset Use dma_zalloc_coherent for allocating zeroed memory and remove unnecessary memset function. Done using Coccinelle. Generated-by: scripts/coccinelle/api/alloc/kzalloc-simple.cocci 0-day tested with no failures. Signed-off-by: Himanshu Jha Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_core.c | 8 +++----- drivers/crypto/ixp4xx_crypto.c | 7 +++---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index ad19aa806931..76f459ad2821 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -282,14 +282,12 @@ static u32 crypto4xx_put_pd_to_pdr(struct crypto4xx_device *dev, u32 idx) */ static u32 crypto4xx_build_gdr(struct crypto4xx_device *dev) { - dev->gdr = dma_alloc_coherent(dev->core_dev->device, - sizeof(struct ce_gd) * PPC4XX_NUM_GD, - &dev->gdr_pa, GFP_ATOMIC); + dev->gdr = dma_zalloc_coherent(dev->core_dev->device, + sizeof(struct ce_gd) * PPC4XX_NUM_GD, + &dev->gdr_pa, GFP_ATOMIC); if (!dev->gdr) return -ENOMEM; - memset(dev->gdr, 0, sizeof(struct ce_gd) * PPC4XX_NUM_GD); - return 0; } diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c index 8705b28eb02c..717a26607bdb 100644 --- a/drivers/crypto/ixp4xx_crypto.c +++ b/drivers/crypto/ixp4xx_crypto.c @@ -260,12 +260,11 @@ static int setup_crypt_desc(void) { struct device *dev = &pdev->dev; BUILD_BUG_ON(sizeof(struct crypt_ctl) != 64); - crypt_virt = dma_alloc_coherent(dev, - NPE_QLEN * sizeof(struct crypt_ctl), - &crypt_phys, GFP_ATOMIC); + crypt_virt = dma_zalloc_coherent(dev, + NPE_QLEN * sizeof(struct crypt_ctl), + &crypt_phys, GFP_ATOMIC); if (!crypt_virt) return -ENOMEM; - memset(crypt_virt, 0, NPE_QLEN * sizeof(struct crypt_ctl)); return 0; } From 7660b1fb367eb3723b48d3980451fc4f25a05021 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sun, 31 Dec 2017 18:02:45 -0600 Subject: [PATCH 110/166] crypto: chacha20 - use rol32() macro from bitops.h For chacha20_block(), use the existing 32-bit left-rotate function instead of defining one ourselves. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- lib/chacha20.c | 69 +++++++++++++++++++++++--------------------------- 1 file changed, 32 insertions(+), 37 deletions(-) diff --git a/lib/chacha20.c b/lib/chacha20.c index 29d3801dee24..c1cc50fb68c9 100644 --- a/lib/chacha20.c +++ b/lib/chacha20.c @@ -16,11 +16,6 @@ #include #include -static inline u32 rotl32(u32 v, u8 n) -{ - return (v << n) | (v >> (sizeof(v) * 8 - n)); -} - void chacha20_block(u32 *state, u32 *stream) { u32 x[16], *out = stream; @@ -30,45 +25,45 @@ void chacha20_block(u32 *state, u32 *stream) x[i] = state[i]; for (i = 0; i < 20; i += 2) { - x[0] += x[4]; x[12] = rotl32(x[12] ^ x[0], 16); - x[1] += x[5]; x[13] = rotl32(x[13] ^ x[1], 16); - x[2] += x[6]; x[14] = rotl32(x[14] ^ x[2], 16); - x[3] += x[7]; x[15] = rotl32(x[15] ^ x[3], 16); + x[0] += x[4]; x[12] = rol32(x[12] ^ x[0], 16); + x[1] += x[5]; x[13] = rol32(x[13] ^ x[1], 16); + x[2] += x[6]; x[14] = rol32(x[14] ^ x[2], 16); + x[3] += x[7]; x[15] = rol32(x[15] ^ x[3], 16); - x[8] += x[12]; x[4] = rotl32(x[4] ^ x[8], 12); - x[9] += x[13]; x[5] = rotl32(x[5] ^ x[9], 12); - x[10] += x[14]; x[6] = rotl32(x[6] ^ x[10], 12); - x[11] += x[15]; x[7] = rotl32(x[7] ^ x[11], 12); + x[8] += x[12]; x[4] = rol32(x[4] ^ x[8], 12); + x[9] += x[13]; x[5] = rol32(x[5] ^ x[9], 12); + x[10] += x[14]; x[6] = rol32(x[6] ^ x[10], 12); + x[11] += x[15]; x[7] = rol32(x[7] ^ x[11], 12); - x[0] += x[4]; x[12] = rotl32(x[12] ^ x[0], 8); - x[1] += x[5]; x[13] = rotl32(x[13] ^ x[1], 8); - x[2] += x[6]; x[14] = rotl32(x[14] ^ x[2], 8); - x[3] += x[7]; x[15] = rotl32(x[15] ^ x[3], 8); + x[0] += x[4]; x[12] = rol32(x[12] ^ x[0], 8); + x[1] += x[5]; x[13] = rol32(x[13] ^ x[1], 8); + x[2] += x[6]; x[14] = rol32(x[14] ^ x[2], 8); + x[3] += x[7]; x[15] = rol32(x[15] ^ x[3], 8); - x[8] += x[12]; x[4] = rotl32(x[4] ^ x[8], 7); - x[9] += x[13]; x[5] = rotl32(x[5] ^ x[9], 7); - x[10] += x[14]; x[6] = rotl32(x[6] ^ x[10], 7); - x[11] += x[15]; x[7] = rotl32(x[7] ^ x[11], 7); + x[8] += x[12]; x[4] = rol32(x[4] ^ x[8], 7); + x[9] += x[13]; x[5] = rol32(x[5] ^ x[9], 7); + x[10] += x[14]; x[6] = rol32(x[6] ^ x[10], 7); + x[11] += x[15]; x[7] = rol32(x[7] ^ x[11], 7); - x[0] += x[5]; x[15] = rotl32(x[15] ^ x[0], 16); - x[1] += x[6]; x[12] = rotl32(x[12] ^ x[1], 16); - x[2] += x[7]; x[13] = rotl32(x[13] ^ x[2], 16); - x[3] += x[4]; x[14] = rotl32(x[14] ^ x[3], 16); + x[0] += x[5]; x[15] = rol32(x[15] ^ x[0], 16); + x[1] += x[6]; x[12] = rol32(x[12] ^ x[1], 16); + x[2] += x[7]; x[13] = rol32(x[13] ^ x[2], 16); + x[3] += x[4]; x[14] = rol32(x[14] ^ x[3], 16); - x[10] += x[15]; x[5] = rotl32(x[5] ^ x[10], 12); - x[11] += x[12]; x[6] = rotl32(x[6] ^ x[11], 12); - x[8] += x[13]; x[7] = rotl32(x[7] ^ x[8], 12); - x[9] += x[14]; x[4] = rotl32(x[4] ^ x[9], 12); + x[10] += x[15]; x[5] = rol32(x[5] ^ x[10], 12); + x[11] += x[12]; x[6] = rol32(x[6] ^ x[11], 12); + x[8] += x[13]; x[7] = rol32(x[7] ^ x[8], 12); + x[9] += x[14]; x[4] = rol32(x[4] ^ x[9], 12); - x[0] += x[5]; x[15] = rotl32(x[15] ^ x[0], 8); - x[1] += x[6]; x[12] = rotl32(x[12] ^ x[1], 8); - x[2] += x[7]; x[13] = rotl32(x[13] ^ x[2], 8); - x[3] += x[4]; x[14] = rotl32(x[14] ^ x[3], 8); + x[0] += x[5]; x[15] = rol32(x[15] ^ x[0], 8); + x[1] += x[6]; x[12] = rol32(x[12] ^ x[1], 8); + x[2] += x[7]; x[13] = rol32(x[13] ^ x[2], 8); + x[3] += x[4]; x[14] = rol32(x[14] ^ x[3], 8); - x[10] += x[15]; x[5] = rotl32(x[5] ^ x[10], 7); - x[11] += x[12]; x[6] = rotl32(x[6] ^ x[11], 7); - x[8] += x[13]; x[7] = rotl32(x[7] ^ x[8], 7); - x[9] += x[14]; x[4] = rotl32(x[4] ^ x[9], 7); + x[10] += x[15]; x[5] = rol32(x[5] ^ x[10], 7); + x[11] += x[12]; x[6] = rol32(x[6] ^ x[11], 7); + x[8] += x[13]; x[7] = rol32(x[7] ^ x[8], 7); + x[9] += x[14]; x[4] = rol32(x[4] ^ x[9], 7); } for (i = 0; i < ARRAY_SIZE(x); i++) From da1729ce4866be266575a114208c7d5350bf310d Mon Sep 17 00:00:00 2001 From: Joey Pabalinas Date: Mon, 1 Jan 2018 10:40:14 -1000 Subject: [PATCH 111/166] crypto: testmgr - change `guard` to unsigned char When char is signed, storing the values 0xba (186) and 0xad (173) in the `guard` array produces signed overflow. Change the type of `guard` to static unsigned char to correct undefined behavior and reduce function stack usage. Signed-off-by: Joey Pabalinas Signed-off-by: Herbert Xu --- crypto/testmgr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 29d7020b8826..44a85d4b3561 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -185,7 +185,7 @@ static int ahash_partial_update(struct ahash_request **preq, char *state; struct ahash_request *req; int statesize, ret = -EINVAL; - const char guard[] = { 0x00, 0xba, 0xad, 0x00 }; + static const unsigned char guard[] = { 0x00, 0xba, 0xad, 0x00 }; req = *preq; statesize = crypto_ahash_statesize( From bb30b8848c85e18ca7e371d0a869e94b3e383bdf Mon Sep 17 00:00:00 2001 From: Stephan Mueller Date: Tue, 2 Jan 2018 08:55:25 +0100 Subject: [PATCH 112/166] crypto: af_alg - whitelist mask and type The user space interface allows specifying the type and mask field used to allocate the cipher. Only a subset of the possible flags are intended for user space. Therefore, white-list the allowed flags. In case the user space caller uses at least one non-allowed flag, EINVAL is returned. Reported-by: syzbot Cc: Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/af_alg.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crypto/af_alg.c b/crypto/af_alg.c index 35d4dcea381f..5231f421ad00 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -150,7 +150,7 @@ EXPORT_SYMBOL_GPL(af_alg_release_parent); static int alg_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) { - const u32 forbidden = CRYPTO_ALG_INTERNAL; + const u32 allowed = CRYPTO_ALG_KERN_DRIVER_ONLY; struct sock *sk = sock->sk; struct alg_sock *ask = alg_sk(sk); struct sockaddr_alg *sa = (void *)uaddr; @@ -158,6 +158,10 @@ static int alg_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) void *private; int err; + /* If caller uses non-allowed flag, return error. */ + if ((sa->salg_feat & ~allowed) || (sa->salg_mask & ~allowed)) + return -EINVAL; + if (sock->state == SS_CONNECTED) return -EINVAL; @@ -176,9 +180,7 @@ static int alg_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) if (IS_ERR(type)) return PTR_ERR(type); - private = type->bind(sa->salg_name, - sa->salg_feat & ~forbidden, - sa->salg_mask & ~forbidden); + private = type->bind(sa->salg_name, sa->salg_feat, sa->salg_mask); if (IS_ERR(private)) { module_put(type->owner); return PTR_ERR(private); From 38dbe2d190b09389b0344d6ccb08adf02d930222 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Tue, 2 Jan 2018 09:21:06 +0000 Subject: [PATCH 113/166] crypto: tcrypt - fix spelling mistake: "bufufer"-> "buffer" Trivial fix to spelling mistakes in pr_err error message text. Signed-off-by: Colin Ian King Signed-off-by: Herbert Xu --- crypto/tcrypt.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 58e3344d7169..f61d2f40dd90 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -328,7 +328,7 @@ static void test_mb_aead_speed(const char *algo, int enc, int secs, b_size = aead_sizes; do { if (*b_size + authsize > XBUFSIZE * PAGE_SIZE) { - pr_err("template (%u) too big for bufufer (%lu)\n", + pr_err("template (%u) too big for buffer (%lu)\n", authsize + *b_size, XBUFSIZE * PAGE_SIZE); goto out; @@ -1269,7 +1269,7 @@ static void test_mb_skcipher_speed(const char *algo, int enc, int secs, b_size = block_sizes; do { if (*b_size > XBUFSIZE * PAGE_SIZE) { - pr_err("template (%u) too big for bufufer (%lu)\n", + pr_err("template (%u) too big for buffer (%lu)\n", *b_size, XBUFSIZE * PAGE_SIZE); goto out; } From c6ba4f3e68e9dd41e75777aba8ba138c49e6ea60 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Tue, 2 Jan 2018 15:43:04 +0000 Subject: [PATCH 114/166] crypto: tcrypt - free xoutbuf instead of axbuf There seems to be a cut-n-paste bug with the name of the buffer being free'd, xoutbuf should be used instead of axbuf. Detected by CoverityScan, CID#1463420 ("Copy-paste error") Fixes: 427988d981c4 ("crypto: tcrypt - add multibuf aead speed test") Signed-off-by: Colin Ian King Signed-off-by: Herbert Xu --- crypto/tcrypt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index f61d2f40dd90..14213a096fd2 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -298,7 +298,7 @@ static void test_mb_aead_speed(const char *algo, int enc, int secs, for (i = 0; i < num_mb; ++i) if (testmgr_alloc_buf(data[i].xoutbuf)) { while (i--) - testmgr_free_buf(data[i].axbuf); + testmgr_free_buf(data[i].xoutbuf); goto out_free_axbuf; } From cd6ed77ad5d223dc6299fb58f62e0f5267f7e2ba Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 3 Jan 2018 11:16:22 -0800 Subject: [PATCH 115/166] crypto: hash - introduce crypto_hash_alg_has_setkey() Templates that use an shash spawn can use crypto_shash_alg_has_setkey() to determine whether the underlying algorithm requires a key or not. But there was no corresponding function for ahash spawns. Add it. Note that the new function actually has to support both shash and ahash algorithms, since the ahash API can be used with either. Cc: stable@vger.kernel.org Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/ahash.c | 11 +++++++++++ include/crypto/internal/hash.h | 2 ++ 2 files changed, 13 insertions(+) diff --git a/crypto/ahash.c b/crypto/ahash.c index 3a35d67de7d9..d2c8895bb2fe 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -649,5 +649,16 @@ struct hash_alg_common *ahash_attr_alg(struct rtattr *rta, u32 type, u32 mask) } EXPORT_SYMBOL_GPL(ahash_attr_alg); +bool crypto_hash_alg_has_setkey(struct hash_alg_common *halg) +{ + struct crypto_alg *alg = &halg->base; + + if (alg->cra_type != &crypto_ahash_type) + return crypto_shash_alg_has_setkey(__crypto_shash_alg(alg)); + + return __crypto_ahash_alg(alg)->setkey != NULL; +} +EXPORT_SYMBOL_GPL(crypto_hash_alg_has_setkey); + MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Asynchronous cryptographic hash type"); diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index c2bae8da642c..27040a46d50a 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -90,6 +90,8 @@ static inline bool crypto_shash_alg_has_setkey(struct shash_alg *alg) return alg->setkey != shash_no_setkey; } +bool crypto_hash_alg_has_setkey(struct hash_alg_common *halg); + int crypto_init_ahash_spawn(struct crypto_ahash_spawn *spawn, struct hash_alg_common *alg, struct crypto_instance *inst); From 841a3ff329713f796a63356fef6e2f72e4a3f6a3 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 3 Jan 2018 11:16:23 -0800 Subject: [PATCH 116/166] crypto: cryptd - pass through absence of ->setkey() When the cryptd template is used to wrap an unkeyed hash algorithm, don't install a ->setkey() method to the cryptd instance. This change is necessary for cryptd to keep working with unkeyed hash algorithms once we start enforcing that ->setkey() is called when present. Cc: stable@vger.kernel.org Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/cryptd.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crypto/cryptd.c b/crypto/cryptd.c index 552e3a86e829..457ae3e66a41 100644 --- a/crypto/cryptd.c +++ b/crypto/cryptd.c @@ -914,7 +914,8 @@ static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, inst->alg.finup = cryptd_hash_finup_enqueue; inst->alg.export = cryptd_hash_export; inst->alg.import = cryptd_hash_import; - inst->alg.setkey = cryptd_hash_setkey; + if (crypto_shash_alg_has_setkey(salg)) + inst->alg.setkey = cryptd_hash_setkey; inst->alg.digest = cryptd_hash_digest_enqueue; err = ahash_register_instance(tmpl, inst); From fa59b92d299f2787e6bae1ff078ee0982e80211f Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 3 Jan 2018 11:16:24 -0800 Subject: [PATCH 117/166] crypto: mcryptd - pass through absence of ->setkey() When the mcryptd template is used to wrap an unkeyed hash algorithm, don't install a ->setkey() method to the mcryptd instance. This change is necessary for mcryptd to keep working with unkeyed hash algorithms once we start enforcing that ->setkey() is called when present. Cc: stable@vger.kernel.org Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/mcryptd.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crypto/mcryptd.c b/crypto/mcryptd.c index 2908382861ee..ace346b976b3 100644 --- a/crypto/mcryptd.c +++ b/crypto/mcryptd.c @@ -534,7 +534,8 @@ static int mcryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, inst->alg.finup = mcryptd_hash_finup_enqueue; inst->alg.export = mcryptd_hash_export; inst->alg.import = mcryptd_hash_import; - inst->alg.setkey = mcryptd_hash_setkey; + if (crypto_hash_alg_has_setkey(halg)) + inst->alg.setkey = mcryptd_hash_setkey; inst->alg.digest = mcryptd_hash_digest_enqueue; err = ahash_register_instance(tmpl, inst); From a16e772e664b9a261424107784804cffc8894977 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 3 Jan 2018 11:16:25 -0800 Subject: [PATCH 118/166] crypto: poly1305 - remove ->setkey() method Since Poly1305 requires a nonce per invocation, the Linux kernel implementations of Poly1305 don't use the crypto API's keying mechanism and instead expect the key and nonce as the first 32 bytes of the data. But ->setkey() is still defined as a stub returning an error code. This prevents Poly1305 from being used through AF_ALG and will also break it completely once we start enforcing that all crypto API users (not just AF_ALG) call ->setkey() if present. Fix it by removing crypto_poly1305_setkey(), leaving ->setkey as NULL. Cc: stable@vger.kernel.org Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/poly1305_glue.c | 1 - crypto/poly1305_generic.c | 17 +++++------------ include/crypto/poly1305.h | 2 -- 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/arch/x86/crypto/poly1305_glue.c b/arch/x86/crypto/poly1305_glue.c index f58f89b05a7f..790377797544 100644 --- a/arch/x86/crypto/poly1305_glue.c +++ b/arch/x86/crypto/poly1305_glue.c @@ -164,7 +164,6 @@ static struct shash_alg alg = { .init = poly1305_simd_init, .update = poly1305_simd_update, .final = crypto_poly1305_final, - .setkey = crypto_poly1305_setkey, .descsize = sizeof(struct poly1305_simd_desc_ctx), .base = { .cra_name = "poly1305", diff --git a/crypto/poly1305_generic.c b/crypto/poly1305_generic.c index d92617aeb783..b7a3a0613a30 100644 --- a/crypto/poly1305_generic.c +++ b/crypto/poly1305_generic.c @@ -47,17 +47,6 @@ int crypto_poly1305_init(struct shash_desc *desc) } EXPORT_SYMBOL_GPL(crypto_poly1305_init); -int crypto_poly1305_setkey(struct crypto_shash *tfm, - const u8 *key, unsigned int keylen) -{ - /* Poly1305 requires a unique key for each tag, which implies that - * we can't set it on the tfm that gets accessed by multiple users - * simultaneously. Instead we expect the key as the first 32 bytes in - * the update() call. */ - return -ENOTSUPP; -} -EXPORT_SYMBOL_GPL(crypto_poly1305_setkey); - static void poly1305_setrkey(struct poly1305_desc_ctx *dctx, const u8 *key) { /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ @@ -76,6 +65,11 @@ static void poly1305_setskey(struct poly1305_desc_ctx *dctx, const u8 *key) dctx->s[3] = get_unaligned_le32(key + 12); } +/* + * Poly1305 requires a unique key for each tag, which implies that we can't set + * it on the tfm that gets accessed by multiple users simultaneously. Instead we + * expect the key as the first 32 bytes in the update() call. + */ unsigned int crypto_poly1305_setdesckey(struct poly1305_desc_ctx *dctx, const u8 *src, unsigned int srclen) { @@ -280,7 +274,6 @@ static struct shash_alg poly1305_alg = { .init = crypto_poly1305_init, .update = crypto_poly1305_update, .final = crypto_poly1305_final, - .setkey = crypto_poly1305_setkey, .descsize = sizeof(struct poly1305_desc_ctx), .base = { .cra_name = "poly1305", diff --git a/include/crypto/poly1305.h b/include/crypto/poly1305.h index c65567d01e8e..f718a19da82f 100644 --- a/include/crypto/poly1305.h +++ b/include/crypto/poly1305.h @@ -31,8 +31,6 @@ struct poly1305_desc_ctx { }; int crypto_poly1305_init(struct shash_desc *desc); -int crypto_poly1305_setkey(struct crypto_shash *tfm, - const u8 *key, unsigned int keylen); unsigned int crypto_poly1305_setdesckey(struct poly1305_desc_ctx *dctx, const u8 *src, unsigned int srclen); int crypto_poly1305_update(struct shash_desc *desc, From a208fa8f33031b9e0aba44c7d1b7e68eb0cbd29e Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 3 Jan 2018 11:16:26 -0800 Subject: [PATCH 119/166] crypto: hash - annotate algorithms taking optional key We need to consistently enforce that keyed hashes cannot be used without setting the key. To do this we need a reliable way to determine whether a given hash algorithm is keyed or not. AF_ALG currently does this by checking for the presence of a ->setkey() method. However, this is actually slightly broken because the CRC-32 algorithms implement ->setkey() but can also be used without a key. (The CRC-32 "key" is not actually a cryptographic key but rather represents the initial state. If not overridden, then a default initial state is used.) Prepare to fix this by introducing a flag CRYPTO_ALG_OPTIONAL_KEY which indicates that the algorithm has a ->setkey() method, but it is not required to be called. Then set it on all the CRC-32 algorithms. The same also applies to the Adler-32 implementation in Lustre. Also, the cryptd and mcryptd templates have to pass through the flag from their underlying algorithm. Cc: stable@vger.kernel.org Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm/crypto/crc32-ce-glue.c | 2 ++ arch/arm64/crypto/crc32-ce-glue.c | 2 ++ arch/powerpc/crypto/crc32c-vpmsum_glue.c | 1 + arch/s390/crypto/crc32-vx.c | 3 +++ arch/sparc/crypto/crc32c_glue.c | 1 + arch/x86/crypto/crc32-pclmul_glue.c | 1 + arch/x86/crypto/crc32c-intel_glue.c | 1 + crypto/crc32_generic.c | 1 + crypto/crc32c_generic.c | 1 + crypto/cryptd.c | 7 +++---- crypto/mcryptd.c | 7 +++---- drivers/crypto/bfin_crc.c | 3 ++- drivers/crypto/stm32/stm32_crc32.c | 2 ++ .../staging/lustre/lnet/libcfs/linux/linux-crypto-adler.c | 1 + include/linux/crypto.h | 6 ++++++ 15 files changed, 30 insertions(+), 9 deletions(-) diff --git a/arch/arm/crypto/crc32-ce-glue.c b/arch/arm/crypto/crc32-ce-glue.c index 1b0e0e86ee9c..96e62ec105d0 100644 --- a/arch/arm/crypto/crc32-ce-glue.c +++ b/arch/arm/crypto/crc32-ce-glue.c @@ -188,6 +188,7 @@ static struct shash_alg crc32_pmull_algs[] = { { .base.cra_name = "crc32", .base.cra_driver_name = "crc32-arm-ce", .base.cra_priority = 200, + .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, .base.cra_blocksize = 1, .base.cra_module = THIS_MODULE, }, { @@ -203,6 +204,7 @@ static struct shash_alg crc32_pmull_algs[] = { { .base.cra_name = "crc32c", .base.cra_driver_name = "crc32c-arm-ce", .base.cra_priority = 200, + .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, .base.cra_blocksize = 1, .base.cra_module = THIS_MODULE, } }; diff --git a/arch/arm64/crypto/crc32-ce-glue.c b/arch/arm64/crypto/crc32-ce-glue.c index 624f4137918c..34b4e3d46aab 100644 --- a/arch/arm64/crypto/crc32-ce-glue.c +++ b/arch/arm64/crypto/crc32-ce-glue.c @@ -185,6 +185,7 @@ static struct shash_alg crc32_pmull_algs[] = { { .base.cra_name = "crc32", .base.cra_driver_name = "crc32-arm64-ce", .base.cra_priority = 200, + .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, .base.cra_blocksize = 1, .base.cra_module = THIS_MODULE, }, { @@ -200,6 +201,7 @@ static struct shash_alg crc32_pmull_algs[] = { { .base.cra_name = "crc32c", .base.cra_driver_name = "crc32c-arm64-ce", .base.cra_priority = 200, + .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, .base.cra_blocksize = 1, .base.cra_module = THIS_MODULE, } }; diff --git a/arch/powerpc/crypto/crc32c-vpmsum_glue.c b/arch/powerpc/crypto/crc32c-vpmsum_glue.c index f058e0c3e4d4..fd1d6c83f0c0 100644 --- a/arch/powerpc/crypto/crc32c-vpmsum_glue.c +++ b/arch/powerpc/crypto/crc32c-vpmsum_glue.c @@ -141,6 +141,7 @@ static struct shash_alg alg = { .cra_name = "crc32c", .cra_driver_name = "crc32c-vpmsum", .cra_priority = 200, + .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, .cra_blocksize = CHKSUM_BLOCK_SIZE, .cra_ctxsize = sizeof(u32), .cra_module = THIS_MODULE, diff --git a/arch/s390/crypto/crc32-vx.c b/arch/s390/crypto/crc32-vx.c index 436865926c26..423ee05887e6 100644 --- a/arch/s390/crypto/crc32-vx.c +++ b/arch/s390/crypto/crc32-vx.c @@ -239,6 +239,7 @@ static struct shash_alg crc32_vx_algs[] = { .cra_name = "crc32", .cra_driver_name = "crc32-vx", .cra_priority = 200, + .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, .cra_blocksize = CRC32_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crc_ctx), .cra_module = THIS_MODULE, @@ -259,6 +260,7 @@ static struct shash_alg crc32_vx_algs[] = { .cra_name = "crc32be", .cra_driver_name = "crc32be-vx", .cra_priority = 200, + .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, .cra_blocksize = CRC32_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crc_ctx), .cra_module = THIS_MODULE, @@ -279,6 +281,7 @@ static struct shash_alg crc32_vx_algs[] = { .cra_name = "crc32c", .cra_driver_name = "crc32c-vx", .cra_priority = 200, + .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, .cra_blocksize = CRC32_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crc_ctx), .cra_module = THIS_MODULE, diff --git a/arch/sparc/crypto/crc32c_glue.c b/arch/sparc/crypto/crc32c_glue.c index d1064e46efe8..8aa664638c3c 100644 --- a/arch/sparc/crypto/crc32c_glue.c +++ b/arch/sparc/crypto/crc32c_glue.c @@ -133,6 +133,7 @@ static struct shash_alg alg = { .cra_name = "crc32c", .cra_driver_name = "crc32c-sparc64", .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, .cra_blocksize = CHKSUM_BLOCK_SIZE, .cra_ctxsize = sizeof(u32), .cra_alignmask = 7, diff --git a/arch/x86/crypto/crc32-pclmul_glue.c b/arch/x86/crypto/crc32-pclmul_glue.c index 27226df3f7d8..c8d9cdacbf10 100644 --- a/arch/x86/crypto/crc32-pclmul_glue.c +++ b/arch/x86/crypto/crc32-pclmul_glue.c @@ -162,6 +162,7 @@ static struct shash_alg alg = { .cra_name = "crc32", .cra_driver_name = "crc32-pclmul", .cra_priority = 200, + .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, .cra_blocksize = CHKSUM_BLOCK_SIZE, .cra_ctxsize = sizeof(u32), .cra_module = THIS_MODULE, diff --git a/arch/x86/crypto/crc32c-intel_glue.c b/arch/x86/crypto/crc32c-intel_glue.c index c194d5717ae5..5773e1161072 100644 --- a/arch/x86/crypto/crc32c-intel_glue.c +++ b/arch/x86/crypto/crc32c-intel_glue.c @@ -226,6 +226,7 @@ static struct shash_alg alg = { .cra_name = "crc32c", .cra_driver_name = "crc32c-intel", .cra_priority = 200, + .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, .cra_blocksize = CHKSUM_BLOCK_SIZE, .cra_ctxsize = sizeof(u32), .cra_module = THIS_MODULE, diff --git a/crypto/crc32_generic.c b/crypto/crc32_generic.c index aa2a25fc7482..718cbce8d169 100644 --- a/crypto/crc32_generic.c +++ b/crypto/crc32_generic.c @@ -133,6 +133,7 @@ static struct shash_alg alg = { .cra_name = "crc32", .cra_driver_name = "crc32-generic", .cra_priority = 100, + .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, .cra_blocksize = CHKSUM_BLOCK_SIZE, .cra_ctxsize = sizeof(u32), .cra_module = THIS_MODULE, diff --git a/crypto/crc32c_generic.c b/crypto/crc32c_generic.c index 4c0a0e271876..372320399622 100644 --- a/crypto/crc32c_generic.c +++ b/crypto/crc32c_generic.c @@ -146,6 +146,7 @@ static struct shash_alg alg = { .cra_name = "crc32c", .cra_driver_name = "crc32c-generic", .cra_priority = 100, + .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, .cra_blocksize = CHKSUM_BLOCK_SIZE, .cra_alignmask = 3, .cra_ctxsize = sizeof(struct chksum_ctx), diff --git a/crypto/cryptd.c b/crypto/cryptd.c index 457ae3e66a41..addca7bae33f 100644 --- a/crypto/cryptd.c +++ b/crypto/cryptd.c @@ -896,10 +896,9 @@ static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, if (err) goto out_free_inst; - type = CRYPTO_ALG_ASYNC; - if (alg->cra_flags & CRYPTO_ALG_INTERNAL) - type |= CRYPTO_ALG_INTERNAL; - inst->alg.halg.base.cra_flags = type; + inst->alg.halg.base.cra_flags = CRYPTO_ALG_ASYNC | + (alg->cra_flags & (CRYPTO_ALG_INTERNAL | + CRYPTO_ALG_OPTIONAL_KEY)); inst->alg.halg.digestsize = salg->digestsize; inst->alg.halg.statesize = salg->statesize; diff --git a/crypto/mcryptd.c b/crypto/mcryptd.c index ace346b976b3..fe5129d6ff4e 100644 --- a/crypto/mcryptd.c +++ b/crypto/mcryptd.c @@ -516,10 +516,9 @@ static int mcryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, if (err) goto out_free_inst; - type = CRYPTO_ALG_ASYNC; - if (alg->cra_flags & CRYPTO_ALG_INTERNAL) - type |= CRYPTO_ALG_INTERNAL; - inst->alg.halg.base.cra_flags = type; + inst->alg.halg.base.cra_flags = CRYPTO_ALG_ASYNC | + (alg->cra_flags & (CRYPTO_ALG_INTERNAL | + CRYPTO_ALG_OPTIONAL_KEY)); inst->alg.halg.digestsize = halg->digestsize; inst->alg.halg.statesize = halg->statesize; diff --git a/drivers/crypto/bfin_crc.c b/drivers/crypto/bfin_crc.c index a118b9bed669..bfbf8bf77f03 100644 --- a/drivers/crypto/bfin_crc.c +++ b/drivers/crypto/bfin_crc.c @@ -494,7 +494,8 @@ static struct ahash_alg algs = { .cra_driver_name = DRIVER_NAME, .cra_priority = 100, .cra_flags = CRYPTO_ALG_TYPE_AHASH | - CRYPTO_ALG_ASYNC, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_OPTIONAL_KEY, .cra_blocksize = CHKSUM_BLOCK_SIZE, .cra_ctxsize = sizeof(struct bfin_crypto_crc_ctx), .cra_alignmask = 3, diff --git a/drivers/crypto/stm32/stm32_crc32.c b/drivers/crypto/stm32/stm32_crc32.c index 090582baecfe..8f09b8430893 100644 --- a/drivers/crypto/stm32/stm32_crc32.c +++ b/drivers/crypto/stm32/stm32_crc32.c @@ -208,6 +208,7 @@ static struct shash_alg algs[] = { .cra_name = "crc32", .cra_driver_name = DRIVER_NAME, .cra_priority = 200, + .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, .cra_blocksize = CHKSUM_BLOCK_SIZE, .cra_alignmask = 3, .cra_ctxsize = sizeof(struct stm32_crc_ctx), @@ -229,6 +230,7 @@ static struct shash_alg algs[] = { .cra_name = "crc32c", .cra_driver_name = DRIVER_NAME, .cra_priority = 200, + .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, .cra_blocksize = CHKSUM_BLOCK_SIZE, .cra_alignmask = 3, .cra_ctxsize = sizeof(struct stm32_crc_ctx), diff --git a/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto-adler.c b/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto-adler.c index 2e5d311d2438..db81ed527452 100644 --- a/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto-adler.c +++ b/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto-adler.c @@ -120,6 +120,7 @@ static struct shash_alg alg = { .cra_name = "adler32", .cra_driver_name = "adler32-zlib", .cra_priority = 100, + .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, .cra_blocksize = CHKSUM_BLOCK_SIZE, .cra_ctxsize = sizeof(u32), .cra_module = THIS_MODULE, diff --git a/include/linux/crypto.h b/include/linux/crypto.h index 231e59f90d32..d2e33a90825b 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -106,6 +106,12 @@ */ #define CRYPTO_ALG_INTERNAL 0x00002000 +/* + * Set if the algorithm has a ->setkey() method but can be used without + * calling it first, i.e. there is a default key. + */ +#define CRYPTO_ALG_OPTIONAL_KEY 0x00004000 + /* * Transform masks and values (for crt_flags). */ From 9fa68f620041be04720d0cbfb1bd3ddfc6310b24 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 3 Jan 2018 11:16:27 -0800 Subject: [PATCH 120/166] crypto: hash - prevent using keyed hashes without setting key Currently, almost none of the keyed hash algorithms check whether a key has been set before proceeding. Some algorithms are okay with this and will effectively just use a key of all 0's or some other bogus default. However, others will severely break, as demonstrated using "hmac(sha3-512-generic)", the unkeyed use of which causes a kernel crash via a (potentially exploitable) stack buffer overflow. A while ago, this problem was solved for AF_ALG by pairing each hash transform with a 'has_key' bool. However, there are still other places in the kernel where userspace can specify an arbitrary hash algorithm by name, and the kernel uses it as unkeyed hash without checking whether it is really unkeyed. Examples of this include: - KEYCTL_DH_COMPUTE, via the KDF extension - dm-verity - dm-crypt, via the ESSIV support - dm-integrity, via the "internal hash" mode with no key given - drbd (Distributed Replicated Block Device) This bug is especially bad for KEYCTL_DH_COMPUTE as that requires no privileges to call. Fix the bug for all users by adding a flag CRYPTO_TFM_NEED_KEY to the ->crt_flags of each hash transform that indicates whether the transform still needs to be keyed or not. Then, make the hash init, import, and digest functions return -ENOKEY if the key is still needed. The new flag also replaces the 'has_key' bool which algif_hash was previously using, thereby simplifying the algif_hash implementation. Reported-by: syzbot Cc: stable@vger.kernel.org Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/ahash.c | 22 ++++++++++++++---- crypto/algif_hash.c | 52 +++++++++--------------------------------- crypto/shash.c | 25 ++++++++++++++++---- include/crypto/hash.h | 34 +++++++++++++++++++-------- include/linux/crypto.h | 2 ++ 5 files changed, 75 insertions(+), 60 deletions(-) diff --git a/crypto/ahash.c b/crypto/ahash.c index d2c8895bb2fe..266fc1d64f61 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -193,11 +193,18 @@ int crypto_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { unsigned long alignmask = crypto_ahash_alignmask(tfm); + int err; if ((unsigned long)key & alignmask) - return ahash_setkey_unaligned(tfm, key, keylen); + err = ahash_setkey_unaligned(tfm, key, keylen); + else + err = tfm->setkey(tfm, key, keylen); - return tfm->setkey(tfm, key, keylen); + if (err) + return err; + + crypto_ahash_clear_flags(tfm, CRYPTO_TFM_NEED_KEY); + return 0; } EXPORT_SYMBOL_GPL(crypto_ahash_setkey); @@ -368,7 +375,12 @@ EXPORT_SYMBOL_GPL(crypto_ahash_finup); int crypto_ahash_digest(struct ahash_request *req) { - return crypto_ahash_op(req, crypto_ahash_reqtfm(req)->digest); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + + if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) + return -ENOKEY; + + return crypto_ahash_op(req, tfm->digest); } EXPORT_SYMBOL_GPL(crypto_ahash_digest); @@ -450,7 +462,6 @@ static int crypto_ahash_init_tfm(struct crypto_tfm *tfm) struct ahash_alg *alg = crypto_ahash_alg(hash); hash->setkey = ahash_nosetkey; - hash->has_setkey = false; hash->export = ahash_no_export; hash->import = ahash_no_import; @@ -465,7 +476,8 @@ static int crypto_ahash_init_tfm(struct crypto_tfm *tfm) if (alg->setkey) { hash->setkey = alg->setkey; - hash->has_setkey = true; + if (!(alg->halg.base.cra_flags & CRYPTO_ALG_OPTIONAL_KEY)) + crypto_ahash_set_flags(hash, CRYPTO_TFM_NEED_KEY); } if (alg->export) hash->export = alg->export; diff --git a/crypto/algif_hash.c b/crypto/algif_hash.c index 76d2e716c792..6c9b1927a520 100644 --- a/crypto/algif_hash.c +++ b/crypto/algif_hash.c @@ -34,11 +34,6 @@ struct hash_ctx { struct ahash_request req; }; -struct algif_hash_tfm { - struct crypto_ahash *hash; - bool has_key; -}; - static int hash_alloc_result(struct sock *sk, struct hash_ctx *ctx) { unsigned ds; @@ -307,7 +302,7 @@ static int hash_check_key(struct socket *sock) int err = 0; struct sock *psk; struct alg_sock *pask; - struct algif_hash_tfm *tfm; + struct crypto_ahash *tfm; struct sock *sk = sock->sk; struct alg_sock *ask = alg_sk(sk); @@ -321,7 +316,7 @@ static int hash_check_key(struct socket *sock) err = -ENOKEY; lock_sock_nested(psk, SINGLE_DEPTH_NESTING); - if (!tfm->has_key) + if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) goto unlock; if (!pask->refcnt++) @@ -412,41 +407,17 @@ static struct proto_ops algif_hash_ops_nokey = { static void *hash_bind(const char *name, u32 type, u32 mask) { - struct algif_hash_tfm *tfm; - struct crypto_ahash *hash; - - tfm = kzalloc(sizeof(*tfm), GFP_KERNEL); - if (!tfm) - return ERR_PTR(-ENOMEM); - - hash = crypto_alloc_ahash(name, type, mask); - if (IS_ERR(hash)) { - kfree(tfm); - return ERR_CAST(hash); - } - - tfm->hash = hash; - - return tfm; + return crypto_alloc_ahash(name, type, mask); } static void hash_release(void *private) { - struct algif_hash_tfm *tfm = private; - - crypto_free_ahash(tfm->hash); - kfree(tfm); + crypto_free_ahash(private); } static int hash_setkey(void *private, const u8 *key, unsigned int keylen) { - struct algif_hash_tfm *tfm = private; - int err; - - err = crypto_ahash_setkey(tfm->hash, key, keylen); - tfm->has_key = !err; - - return err; + return crypto_ahash_setkey(private, key, keylen); } static void hash_sock_destruct(struct sock *sk) @@ -461,11 +432,10 @@ static void hash_sock_destruct(struct sock *sk) static int hash_accept_parent_nokey(void *private, struct sock *sk) { - struct hash_ctx *ctx; + struct crypto_ahash *tfm = private; struct alg_sock *ask = alg_sk(sk); - struct algif_hash_tfm *tfm = private; - struct crypto_ahash *hash = tfm->hash; - unsigned len = sizeof(*ctx) + crypto_ahash_reqsize(hash); + struct hash_ctx *ctx; + unsigned int len = sizeof(*ctx) + crypto_ahash_reqsize(tfm); ctx = sock_kmalloc(sk, len, GFP_KERNEL); if (!ctx) @@ -478,7 +448,7 @@ static int hash_accept_parent_nokey(void *private, struct sock *sk) ask->private = ctx; - ahash_request_set_tfm(&ctx->req, hash); + ahash_request_set_tfm(&ctx->req, tfm); ahash_request_set_callback(&ctx->req, CRYPTO_TFM_REQ_MAY_BACKLOG, crypto_req_done, &ctx->wait); @@ -489,9 +459,9 @@ static int hash_accept_parent_nokey(void *private, struct sock *sk) static int hash_accept_parent(void *private, struct sock *sk) { - struct algif_hash_tfm *tfm = private; + struct crypto_ahash *tfm = private; - if (!tfm->has_key && crypto_ahash_has_setkey(tfm->hash)) + if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; return hash_accept_parent_nokey(private, sk); diff --git a/crypto/shash.c b/crypto/shash.c index e849d3ee2e27..5d732c6bb4b2 100644 --- a/crypto/shash.c +++ b/crypto/shash.c @@ -58,11 +58,18 @@ int crypto_shash_setkey(struct crypto_shash *tfm, const u8 *key, { struct shash_alg *shash = crypto_shash_alg(tfm); unsigned long alignmask = crypto_shash_alignmask(tfm); + int err; if ((unsigned long)key & alignmask) - return shash_setkey_unaligned(tfm, key, keylen); + err = shash_setkey_unaligned(tfm, key, keylen); + else + err = shash->setkey(tfm, key, keylen); - return shash->setkey(tfm, key, keylen); + if (err) + return err; + + crypto_shash_clear_flags(tfm, CRYPTO_TFM_NEED_KEY); + return 0; } EXPORT_SYMBOL_GPL(crypto_shash_setkey); @@ -181,6 +188,9 @@ int crypto_shash_digest(struct shash_desc *desc, const u8 *data, struct shash_alg *shash = crypto_shash_alg(tfm); unsigned long alignmask = crypto_shash_alignmask(tfm); + if (crypto_shash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) + return -ENOKEY; + if (((unsigned long)data | (unsigned long)out) & alignmask) return shash_digest_unaligned(desc, data, len, out); @@ -360,7 +370,8 @@ int crypto_init_shash_ops_async(struct crypto_tfm *tfm) crt->digest = shash_async_digest; crt->setkey = shash_async_setkey; - crt->has_setkey = alg->setkey != shash_no_setkey; + crypto_ahash_set_flags(crt, crypto_shash_get_flags(shash) & + CRYPTO_TFM_NEED_KEY); if (alg->export) crt->export = shash_async_export; @@ -375,8 +386,14 @@ int crypto_init_shash_ops_async(struct crypto_tfm *tfm) static int crypto_shash_init_tfm(struct crypto_tfm *tfm) { struct crypto_shash *hash = __crypto_shash_cast(tfm); + struct shash_alg *alg = crypto_shash_alg(hash); + + hash->descsize = alg->descsize; + + if (crypto_shash_alg_has_setkey(alg) && + !(alg->base.cra_flags & CRYPTO_ALG_OPTIONAL_KEY)) + crypto_shash_set_flags(hash, CRYPTO_TFM_NEED_KEY); - hash->descsize = crypto_shash_alg(hash)->descsize; return 0; } diff --git a/include/crypto/hash.h b/include/crypto/hash.h index 0ed31fd80242..3880793e280e 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -210,7 +210,6 @@ struct crypto_ahash { unsigned int keylen); unsigned int reqsize; - bool has_setkey; struct crypto_tfm base; }; @@ -410,11 +409,6 @@ static inline void *ahash_request_ctx(struct ahash_request *req) int crypto_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen); -static inline bool crypto_ahash_has_setkey(struct crypto_ahash *tfm) -{ - return tfm->has_setkey; -} - /** * crypto_ahash_finup() - update and finalize message digest * @req: reference to the ahash_request handle that holds all information @@ -487,7 +481,12 @@ static inline int crypto_ahash_export(struct ahash_request *req, void *out) */ static inline int crypto_ahash_import(struct ahash_request *req, const void *in) { - return crypto_ahash_reqtfm(req)->import(req, in); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + + if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) + return -ENOKEY; + + return tfm->import(req, in); } /** @@ -503,7 +502,12 @@ static inline int crypto_ahash_import(struct ahash_request *req, const void *in) */ static inline int crypto_ahash_init(struct ahash_request *req) { - return crypto_ahash_reqtfm(req)->init(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + + if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) + return -ENOKEY; + + return tfm->init(req); } /** @@ -855,7 +859,12 @@ static inline int crypto_shash_export(struct shash_desc *desc, void *out) */ static inline int crypto_shash_import(struct shash_desc *desc, const void *in) { - return crypto_shash_alg(desc->tfm)->import(desc, in); + struct crypto_shash *tfm = desc->tfm; + + if (crypto_shash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) + return -ENOKEY; + + return crypto_shash_alg(tfm)->import(desc, in); } /** @@ -871,7 +880,12 @@ static inline int crypto_shash_import(struct shash_desc *desc, const void *in) */ static inline int crypto_shash_init(struct shash_desc *desc) { - return crypto_shash_alg(desc->tfm)->init(desc); + struct crypto_shash *tfm = desc->tfm; + + if (crypto_shash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) + return -ENOKEY; + + return crypto_shash_alg(tfm)->init(desc); } /** diff --git a/include/linux/crypto.h b/include/linux/crypto.h index d2e33a90825b..7e6e84cf6383 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -115,6 +115,8 @@ /* * Transform masks and values (for crt_flags). */ +#define CRYPTO_TFM_NEED_KEY 0x00000001 + #define CRYPTO_TFM_REQ_MASK 0x000fff00 #define CRYPTO_TFM_RES_MASK 0xfff00000 From 4e1d14bcd10a33537918a9a747ab90fc5c2e6d7f Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 3 Jan 2018 11:16:28 -0800 Subject: [PATCH 121/166] crypto: ghash - remove checks for key being set Now that the crypto API prevents a keyed hash from being used without setting the key, there's no need for GHASH to do this check itself. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/ghash-generic.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/crypto/ghash-generic.c b/crypto/ghash-generic.c index 12ad3e3a84e3..1bffb3f712dd 100644 --- a/crypto/ghash-generic.c +++ b/crypto/ghash-generic.c @@ -56,9 +56,6 @@ static int ghash_update(struct shash_desc *desc, struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); u8 *dst = dctx->buffer; - if (!ctx->gf128) - return -ENOKEY; - if (dctx->bytes) { int n = min(srclen, dctx->bytes); u8 *pos = dst + (GHASH_BLOCK_SIZE - dctx->bytes); @@ -111,9 +108,6 @@ static int ghash_final(struct shash_desc *desc, u8 *dst) struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); u8 *buf = dctx->buffer; - if (!ctx->gf128) - return -ENOKEY; - ghash_flush(ctx, dctx); memcpy(dst, buf, GHASH_BLOCK_SIZE); From f8d33fac84806eebd2ba31a3136066eeca19255f Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 3 Jan 2018 11:16:29 -0800 Subject: [PATCH 122/166] crypto: skcipher - prevent using skciphers without setting key Similar to what was done for the hash API, update the skcipher API to track whether each transform has been keyed, and reject encryption/decryption if a key is needed but one hasn't been set. This isn't as important as the equivalent fix for the hash API because symmetric ciphers almost always require a key (the "null cipher" is the only exception), so are unlikely to be used without one. Still, tracking the key will prevent accidental unkeyed use. algif_skcipher also had to track the key anyway, so the new flag replaces that and simplifies the algif_skcipher implementation. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/algif_skcipher.c | 59 +++++++++------------------------------ crypto/skcipher.c | 30 +++++++++++++++++--- include/crypto/skcipher.h | 11 ++++---- 3 files changed, 45 insertions(+), 55 deletions(-) diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c index c5c47b680152..c88e5e4cd6a6 100644 --- a/crypto/algif_skcipher.c +++ b/crypto/algif_skcipher.c @@ -38,11 +38,6 @@ #include #include -struct skcipher_tfm { - struct crypto_skcipher *skcipher; - bool has_key; -}; - static int skcipher_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) { @@ -50,8 +45,7 @@ static int skcipher_sendmsg(struct socket *sock, struct msghdr *msg, struct alg_sock *ask = alg_sk(sk); struct sock *psk = ask->parent; struct alg_sock *pask = alg_sk(psk); - struct skcipher_tfm *skc = pask->private; - struct crypto_skcipher *tfm = skc->skcipher; + struct crypto_skcipher *tfm = pask->private; unsigned ivsize = crypto_skcipher_ivsize(tfm); return af_alg_sendmsg(sock, msg, size, ivsize); @@ -65,8 +59,7 @@ static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg, struct sock *psk = ask->parent; struct alg_sock *pask = alg_sk(psk); struct af_alg_ctx *ctx = ask->private; - struct skcipher_tfm *skc = pask->private; - struct crypto_skcipher *tfm = skc->skcipher; + struct crypto_skcipher *tfm = pask->private; unsigned int bs = crypto_skcipher_blocksize(tfm); struct af_alg_async_req *areq; int err = 0; @@ -221,7 +214,7 @@ static int skcipher_check_key(struct socket *sock) int err = 0; struct sock *psk; struct alg_sock *pask; - struct skcipher_tfm *tfm; + struct crypto_skcipher *tfm; struct sock *sk = sock->sk; struct alg_sock *ask = alg_sk(sk); @@ -235,7 +228,7 @@ static int skcipher_check_key(struct socket *sock) err = -ENOKEY; lock_sock_nested(psk, SINGLE_DEPTH_NESTING); - if (!tfm->has_key) + if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) goto unlock; if (!pask->refcnt++) @@ -314,41 +307,17 @@ static struct proto_ops algif_skcipher_ops_nokey = { static void *skcipher_bind(const char *name, u32 type, u32 mask) { - struct skcipher_tfm *tfm; - struct crypto_skcipher *skcipher; - - tfm = kzalloc(sizeof(*tfm), GFP_KERNEL); - if (!tfm) - return ERR_PTR(-ENOMEM); - - skcipher = crypto_alloc_skcipher(name, type, mask); - if (IS_ERR(skcipher)) { - kfree(tfm); - return ERR_CAST(skcipher); - } - - tfm->skcipher = skcipher; - - return tfm; + return crypto_alloc_skcipher(name, type, mask); } static void skcipher_release(void *private) { - struct skcipher_tfm *tfm = private; - - crypto_free_skcipher(tfm->skcipher); - kfree(tfm); + crypto_free_skcipher(private); } static int skcipher_setkey(void *private, const u8 *key, unsigned int keylen) { - struct skcipher_tfm *tfm = private; - int err; - - err = crypto_skcipher_setkey(tfm->skcipher, key, keylen); - tfm->has_key = !err; - - return err; + return crypto_skcipher_setkey(private, key, keylen); } static void skcipher_sock_destruct(struct sock *sk) @@ -357,8 +326,7 @@ static void skcipher_sock_destruct(struct sock *sk) struct af_alg_ctx *ctx = ask->private; struct sock *psk = ask->parent; struct alg_sock *pask = alg_sk(psk); - struct skcipher_tfm *skc = pask->private; - struct crypto_skcipher *tfm = skc->skcipher; + struct crypto_skcipher *tfm = pask->private; af_alg_pull_tsgl(sk, ctx->used, NULL, 0); sock_kzfree_s(sk, ctx->iv, crypto_skcipher_ivsize(tfm)); @@ -370,22 +338,21 @@ static int skcipher_accept_parent_nokey(void *private, struct sock *sk) { struct af_alg_ctx *ctx; struct alg_sock *ask = alg_sk(sk); - struct skcipher_tfm *tfm = private; - struct crypto_skcipher *skcipher = tfm->skcipher; + struct crypto_skcipher *tfm = private; unsigned int len = sizeof(*ctx); ctx = sock_kmalloc(sk, len, GFP_KERNEL); if (!ctx) return -ENOMEM; - ctx->iv = sock_kmalloc(sk, crypto_skcipher_ivsize(skcipher), + ctx->iv = sock_kmalloc(sk, crypto_skcipher_ivsize(tfm), GFP_KERNEL); if (!ctx->iv) { sock_kfree_s(sk, ctx, len); return -ENOMEM; } - memset(ctx->iv, 0, crypto_skcipher_ivsize(skcipher)); + memset(ctx->iv, 0, crypto_skcipher_ivsize(tfm)); INIT_LIST_HEAD(&ctx->tsgl_list); ctx->len = len; @@ -405,9 +372,9 @@ static int skcipher_accept_parent_nokey(void *private, struct sock *sk) static int skcipher_accept_parent(void *private, struct sock *sk) { - struct skcipher_tfm *tfm = private; + struct crypto_skcipher *tfm = private; - if (!tfm->has_key && crypto_skcipher_has_setkey(tfm->skcipher)) + if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; return skcipher_accept_parent_nokey(private, sk); diff --git a/crypto/skcipher.c b/crypto/skcipher.c index 11af5fd6a443..0fe2a2923ad0 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -598,8 +598,11 @@ static int skcipher_setkey_blkcipher(struct crypto_skcipher *tfm, err = crypto_blkcipher_setkey(blkcipher, key, keylen); crypto_skcipher_set_flags(tfm, crypto_blkcipher_get_flags(blkcipher) & CRYPTO_TFM_RES_MASK); + if (err) + return err; - return err; + crypto_skcipher_clear_flags(tfm, CRYPTO_TFM_NEED_KEY); + return 0; } static int skcipher_crypt_blkcipher(struct skcipher_request *req, @@ -674,6 +677,9 @@ static int crypto_init_skcipher_ops_blkcipher(struct crypto_tfm *tfm) skcipher->ivsize = crypto_blkcipher_ivsize(blkcipher); skcipher->keysize = calg->cra_blkcipher.max_keysize; + if (skcipher->keysize) + crypto_skcipher_set_flags(skcipher, CRYPTO_TFM_NEED_KEY); + return 0; } @@ -692,8 +698,11 @@ static int skcipher_setkey_ablkcipher(struct crypto_skcipher *tfm, crypto_skcipher_set_flags(tfm, crypto_ablkcipher_get_flags(ablkcipher) & CRYPTO_TFM_RES_MASK); + if (err) + return err; - return err; + crypto_skcipher_clear_flags(tfm, CRYPTO_TFM_NEED_KEY); + return 0; } static int skcipher_crypt_ablkcipher(struct skcipher_request *req, @@ -767,6 +776,9 @@ static int crypto_init_skcipher_ops_ablkcipher(struct crypto_tfm *tfm) sizeof(struct ablkcipher_request); skcipher->keysize = calg->cra_ablkcipher.max_keysize; + if (skcipher->keysize) + crypto_skcipher_set_flags(skcipher, CRYPTO_TFM_NEED_KEY); + return 0; } @@ -796,6 +808,7 @@ static int skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, { struct skcipher_alg *cipher = crypto_skcipher_alg(tfm); unsigned long alignmask = crypto_skcipher_alignmask(tfm); + int err; if (keylen < cipher->min_keysize || keylen > cipher->max_keysize) { crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); @@ -803,9 +816,15 @@ static int skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, } if ((unsigned long)key & alignmask) - return skcipher_setkey_unaligned(tfm, key, keylen); + err = skcipher_setkey_unaligned(tfm, key, keylen); + else + err = cipher->setkey(tfm, key, keylen); - return cipher->setkey(tfm, key, keylen); + if (err) + return err; + + crypto_skcipher_clear_flags(tfm, CRYPTO_TFM_NEED_KEY); + return 0; } static void crypto_skcipher_exit_tfm(struct crypto_tfm *tfm) @@ -834,6 +853,9 @@ static int crypto_skcipher_init_tfm(struct crypto_tfm *tfm) skcipher->ivsize = alg->ivsize; skcipher->keysize = alg->max_keysize; + if (skcipher->keysize) + crypto_skcipher_set_flags(skcipher, CRYPTO_TFM_NEED_KEY); + if (alg->exit) skcipher->base.exit = crypto_skcipher_exit_tfm; diff --git a/include/crypto/skcipher.h b/include/crypto/skcipher.h index 562001cb412b..2f327f090c3e 100644 --- a/include/crypto/skcipher.h +++ b/include/crypto/skcipher.h @@ -401,11 +401,6 @@ static inline int crypto_skcipher_setkey(struct crypto_skcipher *tfm, return tfm->setkey(tfm, key, keylen); } -static inline bool crypto_skcipher_has_setkey(struct crypto_skcipher *tfm) -{ - return tfm->keysize; -} - static inline unsigned int crypto_skcipher_default_keysize( struct crypto_skcipher *tfm) { @@ -442,6 +437,9 @@ static inline int crypto_skcipher_encrypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) + return -ENOKEY; + return tfm->encrypt(req); } @@ -460,6 +458,9 @@ static inline int crypto_skcipher_decrypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) + return -ENOKEY; + return tfm->decrypt(req); } From dc26c17f743aa8e4720a5fda577dde855f2e36f8 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 3 Jan 2018 11:16:30 -0800 Subject: [PATCH 123/166] crypto: aead - prevent using AEADs without setting key Similar to what was done for the hash API, update the AEAD API to track whether each transform has been keyed, and reject encryption/decryption if a key is needed but one hasn't been set. This isn't quite as important as the equivalent fix for the hash API because AEADs always require a key, so are unlikely to be used without one. Still, tracking the key will prevent accidental unkeyed use. algif_aead also had to track the key anyway, so the new flag replaces that and slightly simplifies the algif_aead implementation. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/aead.c | 13 +++++++++++-- crypto/algif_aead.c | 11 +++-------- include/crypto/aead.h | 10 +++++++++- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/crypto/aead.c b/crypto/aead.c index fe00cbd7243d..60b3bbe973e7 100644 --- a/crypto/aead.c +++ b/crypto/aead.c @@ -54,11 +54,18 @@ int crypto_aead_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int keylen) { unsigned long alignmask = crypto_aead_alignmask(tfm); + int err; if ((unsigned long)key & alignmask) - return setkey_unaligned(tfm, key, keylen); + err = setkey_unaligned(tfm, key, keylen); + else + err = crypto_aead_alg(tfm)->setkey(tfm, key, keylen); - return crypto_aead_alg(tfm)->setkey(tfm, key, keylen); + if (err) + return err; + + crypto_aead_clear_flags(tfm, CRYPTO_TFM_NEED_KEY); + return 0; } EXPORT_SYMBOL_GPL(crypto_aead_setkey); @@ -93,6 +100,8 @@ static int crypto_aead_init_tfm(struct crypto_tfm *tfm) struct crypto_aead *aead = __crypto_aead_cast(tfm); struct aead_alg *alg = crypto_aead_alg(aead); + crypto_aead_set_flags(aead, CRYPTO_TFM_NEED_KEY); + aead->authsize = alg->maxauthsize; if (alg->exit) diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c index d963c8cf8a55..4b07edd5a9ff 100644 --- a/crypto/algif_aead.c +++ b/crypto/algif_aead.c @@ -42,7 +42,6 @@ struct aead_tfm { struct crypto_aead *aead; - bool has_key; struct crypto_skcipher *null_tfm; }; @@ -398,7 +397,7 @@ static int aead_check_key(struct socket *sock) err = -ENOKEY; lock_sock_nested(psk, SINGLE_DEPTH_NESTING); - if (!tfm->has_key) + if (crypto_aead_get_flags(tfm->aead) & CRYPTO_TFM_NEED_KEY) goto unlock; if (!pask->refcnt++) @@ -523,12 +522,8 @@ static int aead_setauthsize(void *private, unsigned int authsize) static int aead_setkey(void *private, const u8 *key, unsigned int keylen) { struct aead_tfm *tfm = private; - int err; - err = crypto_aead_setkey(tfm->aead, key, keylen); - tfm->has_key = !err; - - return err; + return crypto_aead_setkey(tfm->aead, key, keylen); } static void aead_sock_destruct(struct sock *sk) @@ -589,7 +584,7 @@ static int aead_accept_parent(void *private, struct sock *sk) { struct aead_tfm *tfm = private; - if (!tfm->has_key) + if (crypto_aead_get_flags(tfm->aead) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; return aead_accept_parent_nokey(private, sk); diff --git a/include/crypto/aead.h b/include/crypto/aead.h index 03b97629442c..1e26f790b03f 100644 --- a/include/crypto/aead.h +++ b/include/crypto/aead.h @@ -327,7 +327,12 @@ static inline struct crypto_aead *crypto_aead_reqtfm(struct aead_request *req) */ static inline int crypto_aead_encrypt(struct aead_request *req) { - return crypto_aead_alg(crypto_aead_reqtfm(req))->encrypt(req); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + + if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY) + return -ENOKEY; + + return crypto_aead_alg(aead)->encrypt(req); } /** @@ -356,6 +361,9 @@ static inline int crypto_aead_decrypt(struct aead_request *req) { struct crypto_aead *aead = crypto_aead_reqtfm(req); + if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY) + return -ENOKEY; + if (req->cryptlen < crypto_aead_authsize(aead)) return -EINVAL; From 148b974deea927f5dbb6c468af2707b488bfa2de Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 3 Jan 2018 23:39:27 +0100 Subject: [PATCH 124/166] crypto: aes-generic - build with -Os on gcc-7+ While testing other changes, I discovered that gcc-7.2.1 produces badly optimized code for aes_encrypt/aes_decrypt. This is especially true when CONFIG_UBSAN_SANITIZE_ALL is enabled, where it leads to extremely large stack usage that in turn might cause kernel stack overflows: crypto/aes_generic.c: In function 'aes_encrypt': crypto/aes_generic.c:1371:1: warning: the frame size of 4880 bytes is larger than 2048 bytes [-Wframe-larger-than=] crypto/aes_generic.c: In function 'aes_decrypt': crypto/aes_generic.c:1441:1: warning: the frame size of 4864 bytes is larger than 2048 bytes [-Wframe-larger-than=] I verified that this problem exists on all architectures that are supported by gcc-7.2, though arm64 in particular is less affected than the others. I also found that gcc-7.1 and gcc-8 do not show the extreme stack usage but still produce worse code than earlier versions for this file, apparently because of optimization passes that generally provide a substantial improvement in object code quality but understandably fail to find any shortcuts in the AES algorithm. Possible workarounds include a) disabling -ftree-pre and -ftree-sra optimizations, this was an earlier patch I tried, which reliably fixed the stack usage, but caused a serious performance regression in some versions, as later testing found. b) disabling UBSAN on this file or all ciphers, as suggested by Ard Biesheuvel. This would lead to massively better crypto performance in UBSAN-enabled kernels and avoid the stack usage, but there is a concern over whether we should exclude arbitrary files from UBSAN at all. c) Forcing the optimization level in a different way. Similar to a), but rather than deselecting specific optimization stages, this now uses "gcc -Os" for this file, regardless of the CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE/SIZE option. This is a reliable workaround for the stack consumption on all architecture, and I've retested the performance results now on x86, cycles/byte (lower is better) for cbc(aes-generic) with 256 bit keys: -O2 -Os gcc-6.3.1 14.9 15.1 gcc-7.0.1 14.7 15.3 gcc-7.1.1 15.3 14.7 gcc-7.2.1 16.8 15.9 gcc-8.0.0 15.5 15.6 This implements the option c) by enabling forcing -Os on all compiler versions starting with gcc-7.1. As a workaround for PR83356, it would only be needed for gcc-7.2+ with UBSAN enabled, but since it also shows better performance on gcc-7.1 without UBSAN, it seems appropriate to use the faster version here as well. Side note: during testing, I also played with the AES code in libressl, which had a similar performance regression from gcc-6 to gcc-7.2, but was three times slower overall. It might be interesting to investigate that further and possibly port the Linux implementation into that. Link: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=83356 Link: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=83651 Cc: Richard Biener Cc: Jakub Jelinek Cc: Ard Biesheuvel Signed-off-by: Arnd Bergmann Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/crypto/Makefile b/crypto/Makefile index d674884b2d51..daa69360e054 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -99,6 +99,7 @@ obj-$(CONFIG_CRYPTO_TWOFISH_COMMON) += twofish_common.o obj-$(CONFIG_CRYPTO_SERPENT) += serpent_generic.o CFLAGS_serpent_generic.o := $(call cc-option,-fsched-pressure) # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=79149 obj-$(CONFIG_CRYPTO_AES) += aes_generic.o +CFLAGS_aes_generic.o := $(call cc-ifversion, -ge, 0701, -Os) # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=83356 obj-$(CONFIG_CRYPTO_AES_TI) += aes_ti.o obj-$(CONFIG_CRYPTO_CAMELLIA) += camellia_generic.o obj-$(CONFIG_CRYPTO_CAST_COMMON) += cast_common.o From b62b3db76f73c5a8cc132f78e7174bf57d582182 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 5 Jan 2018 11:09:57 -0800 Subject: [PATCH 125/166] crypto: salsa20-generic - cleanup and convert to skcipher API Convert salsa20-generic from the deprecated "blkcipher" API to the "skcipher" API, in the process fixing it up to be thread-safe (as the crypto API expects) by maintaining each request's state separately from the transform context. Also remove the unnecessary cra_alignmask and tighten validation of the key size by accepting only 16 or 32 bytes, not anything in between. These changes bring the code close to the way chacha20-generic does things, so hopefully it will be easier to maintain in the future. However, the way Salsa20 interprets the IV is still slightly different; that was not changed. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/salsa20_generic.c | 248 +++++++++++++++++---------------------- 1 file changed, 108 insertions(+), 140 deletions(-) diff --git a/crypto/salsa20_generic.c b/crypto/salsa20_generic.c index d7da0eea5622..8c77bc78a09f 100644 --- a/crypto/salsa20_generic.c +++ b/crypto/salsa20_generic.c @@ -19,49 +19,27 @@ * */ -#include +#include +#include #include -#include -#include -#include -#include -#include -#include -#define SALSA20_IV_SIZE 8U -#define SALSA20_MIN_KEY_SIZE 16U -#define SALSA20_MAX_KEY_SIZE 32U +#define SALSA20_IV_SIZE 8 +#define SALSA20_MIN_KEY_SIZE 16 +#define SALSA20_MAX_KEY_SIZE 32 +#define SALSA20_BLOCK_SIZE 64 -/* - * Start of code taken from D. J. Bernstein's reference implementation. - * With some modifications and optimizations made to suit our needs. - */ - -/* -salsa20-ref.c version 20051118 -D. J. Bernstein -Public domain. -*/ - -#define U32TO8_LITTLE(p, v) \ - { (p)[0] = (v >> 0) & 0xff; (p)[1] = (v >> 8) & 0xff; \ - (p)[2] = (v >> 16) & 0xff; (p)[3] = (v >> 24) & 0xff; } -#define U8TO32_LITTLE(p) \ - (((u32)((p)[0]) ) | ((u32)((p)[1]) << 8) | \ - ((u32)((p)[2]) << 16) | ((u32)((p)[3]) << 24) ) - -struct salsa20_ctx -{ - u32 input[16]; +struct salsa20_ctx { + u32 initial_state[16]; }; -static void salsa20_wordtobyte(u8 output[64], const u32 input[16]) +static void salsa20_block(u32 *state, __le32 *stream) { u32 x[16]; int i; - memcpy(x, input, sizeof(x)); - for (i = 20; i > 0; i -= 2) { + memcpy(x, state, sizeof(x)); + + for (i = 0; i < 20; i += 2) { x[ 4] ^= rol32((x[ 0] + x[12]), 7); x[ 8] ^= rol32((x[ 4] + x[ 0]), 9); x[12] ^= rol32((x[ 8] + x[ 4]), 13); @@ -95,145 +73,135 @@ static void salsa20_wordtobyte(u8 output[64], const u32 input[16]) x[14] ^= rol32((x[13] + x[12]), 13); x[15] ^= rol32((x[14] + x[13]), 18); } - for (i = 0; i < 16; ++i) - x[i] += input[i]; - for (i = 0; i < 16; ++i) - U32TO8_LITTLE(output + 4 * i,x[i]); + + for (i = 0; i < 16; i++) + stream[i] = cpu_to_le32(x[i] + state[i]); + + if (++state[8] == 0) + state[9]++; } -static const char sigma[16] = "expand 32-byte k"; -static const char tau[16] = "expand 16-byte k"; - -static void salsa20_keysetup(struct salsa20_ctx *ctx, const u8 *k, u32 kbytes) +static void salsa20_docrypt(u32 *state, u8 *dst, const u8 *src, + unsigned int bytes) { - const char *constants; - - ctx->input[1] = U8TO32_LITTLE(k + 0); - ctx->input[2] = U8TO32_LITTLE(k + 4); - ctx->input[3] = U8TO32_LITTLE(k + 8); - ctx->input[4] = U8TO32_LITTLE(k + 12); - if (kbytes == 32) { /* recommended */ - k += 16; - constants = sigma; - } else { /* kbytes == 16 */ - constants = tau; - } - ctx->input[11] = U8TO32_LITTLE(k + 0); - ctx->input[12] = U8TO32_LITTLE(k + 4); - ctx->input[13] = U8TO32_LITTLE(k + 8); - ctx->input[14] = U8TO32_LITTLE(k + 12); - ctx->input[0] = U8TO32_LITTLE(constants + 0); - ctx->input[5] = U8TO32_LITTLE(constants + 4); - ctx->input[10] = U8TO32_LITTLE(constants + 8); - ctx->input[15] = U8TO32_LITTLE(constants + 12); -} - -static void salsa20_ivsetup(struct salsa20_ctx *ctx, const u8 *iv) -{ - ctx->input[6] = U8TO32_LITTLE(iv + 0); - ctx->input[7] = U8TO32_LITTLE(iv + 4); - ctx->input[8] = 0; - ctx->input[9] = 0; -} - -static void salsa20_encrypt_bytes(struct salsa20_ctx *ctx, u8 *dst, - const u8 *src, unsigned int bytes) -{ - u8 buf[64]; + __le32 stream[SALSA20_BLOCK_SIZE / sizeof(__le32)]; if (dst != src) memcpy(dst, src, bytes); - while (bytes) { - salsa20_wordtobyte(buf, ctx->input); - - ctx->input[8]++; - if (!ctx->input[8]) - ctx->input[9]++; - - if (bytes <= 64) { - crypto_xor(dst, buf, bytes); - return; - } - - crypto_xor(dst, buf, 64); - bytes -= 64; - dst += 64; + while (bytes >= SALSA20_BLOCK_SIZE) { + salsa20_block(state, stream); + crypto_xor(dst, (const u8 *)stream, SALSA20_BLOCK_SIZE); + bytes -= SALSA20_BLOCK_SIZE; + dst += SALSA20_BLOCK_SIZE; + } + if (bytes) { + salsa20_block(state, stream); + crypto_xor(dst, (const u8 *)stream, bytes); } } -/* - * End of code taken from D. J. Bernstein's reference implementation. - */ - -static int setkey(struct crypto_tfm *tfm, const u8 *key, - unsigned int keysize) +static void salsa20_init(u32 *state, const struct salsa20_ctx *ctx, + const u8 *iv) { - struct salsa20_ctx *ctx = crypto_tfm_ctx(tfm); - salsa20_keysetup(ctx, key, keysize); + memcpy(state, ctx->initial_state, sizeof(ctx->initial_state)); + state[6] = get_unaligned_le32(iv + 0); + state[7] = get_unaligned_le32(iv + 4); +} + +static int salsa20_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keysize) +{ + static const char sigma[16] = "expand 32-byte k"; + static const char tau[16] = "expand 16-byte k"; + struct salsa20_ctx *ctx = crypto_skcipher_ctx(tfm); + const char *constants; + + if (keysize != SALSA20_MIN_KEY_SIZE && + keysize != SALSA20_MAX_KEY_SIZE) + return -EINVAL; + + ctx->initial_state[1] = get_unaligned_le32(key + 0); + ctx->initial_state[2] = get_unaligned_le32(key + 4); + ctx->initial_state[3] = get_unaligned_le32(key + 8); + ctx->initial_state[4] = get_unaligned_le32(key + 12); + if (keysize == 32) { /* recommended */ + key += 16; + constants = sigma; + } else { /* keysize == 16 */ + constants = tau; + } + ctx->initial_state[11] = get_unaligned_le32(key + 0); + ctx->initial_state[12] = get_unaligned_le32(key + 4); + ctx->initial_state[13] = get_unaligned_le32(key + 8); + ctx->initial_state[14] = get_unaligned_le32(key + 12); + ctx->initial_state[0] = get_unaligned_le32(constants + 0); + ctx->initial_state[5] = get_unaligned_le32(constants + 4); + ctx->initial_state[10] = get_unaligned_le32(constants + 8); + ctx->initial_state[15] = get_unaligned_le32(constants + 12); + + /* space for the nonce; it will be overridden for each request */ + ctx->initial_state[6] = 0; + ctx->initial_state[7] = 0; + + /* initial block number */ + ctx->initial_state[8] = 0; + ctx->initial_state[9] = 0; + return 0; } -static int encrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) +static int salsa20_crypt(struct skcipher_request *req) { - struct blkcipher_walk walk; - struct crypto_blkcipher *tfm = desc->tfm; - struct salsa20_ctx *ctx = crypto_blkcipher_ctx(tfm); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct salsa20_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; + u32 state[16]; int err; - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt_block(desc, &walk, 64); + err = skcipher_walk_virt(&walk, req, true); - salsa20_ivsetup(ctx, walk.iv); + salsa20_init(state, ctx, walk.iv); - while (walk.nbytes >= 64) { - salsa20_encrypt_bytes(ctx, walk.dst.virt.addr, - walk.src.virt.addr, - walk.nbytes - (walk.nbytes % 64)); - err = blkcipher_walk_done(desc, &walk, walk.nbytes % 64); - } + while (walk.nbytes > 0) { + unsigned int nbytes = walk.nbytes; - if (walk.nbytes) { - salsa20_encrypt_bytes(ctx, walk.dst.virt.addr, - walk.src.virt.addr, walk.nbytes); - err = blkcipher_walk_done(desc, &walk, 0); + if (nbytes < walk.total) + nbytes = round_down(nbytes, walk.stride); + + salsa20_docrypt(state, walk.dst.virt.addr, walk.src.virt.addr, + nbytes); + err = skcipher_walk_done(&walk, walk.nbytes - nbytes); } return err; } -static struct crypto_alg alg = { - .cra_name = "salsa20", - .cra_driver_name = "salsa20-generic", - .cra_priority = 100, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_type = &crypto_blkcipher_type, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct salsa20_ctx), - .cra_alignmask = 3, - .cra_module = THIS_MODULE, - .cra_u = { - .blkcipher = { - .setkey = setkey, - .encrypt = encrypt, - .decrypt = encrypt, - .min_keysize = SALSA20_MIN_KEY_SIZE, - .max_keysize = SALSA20_MAX_KEY_SIZE, - .ivsize = SALSA20_IV_SIZE, - } - } +static struct skcipher_alg alg = { + .base.cra_name = "salsa20", + .base.cra_driver_name = "salsa20-generic", + .base.cra_priority = 100, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct salsa20_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = SALSA20_MIN_KEY_SIZE, + .max_keysize = SALSA20_MAX_KEY_SIZE, + .ivsize = SALSA20_IV_SIZE, + .chunksize = SALSA20_BLOCK_SIZE, + .setkey = salsa20_setkey, + .encrypt = salsa20_crypt, + .decrypt = salsa20_crypt, }; static int __init salsa20_generic_mod_init(void) { - return crypto_register_alg(&alg); + return crypto_register_skcipher(&alg); } static void __exit salsa20_generic_mod_fini(void) { - crypto_unregister_alg(&alg); + crypto_unregister_skcipher(&alg); } module_init(salsa20_generic_mod_init); From eb772f37ae8163a89e28a435f6a18742ae06653b Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 5 Jan 2018 11:09:58 -0800 Subject: [PATCH 126/166] crypto: salsa20 - export generic helpers Export the Salsa20 constants, transform context, and initialization functions so that they can be reused by the x86 implementation. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/salsa20_generic.c | 20 +++++++------------- include/crypto/salsa20.h | 27 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 13 deletions(-) create mode 100644 include/crypto/salsa20.h diff --git a/crypto/salsa20_generic.c b/crypto/salsa20_generic.c index 8c77bc78a09f..5074006a56c3 100644 --- a/crypto/salsa20_generic.c +++ b/crypto/salsa20_generic.c @@ -21,17 +21,9 @@ #include #include +#include #include -#define SALSA20_IV_SIZE 8 -#define SALSA20_MIN_KEY_SIZE 16 -#define SALSA20_MAX_KEY_SIZE 32 -#define SALSA20_BLOCK_SIZE 64 - -struct salsa20_ctx { - u32 initial_state[16]; -}; - static void salsa20_block(u32 *state, __le32 *stream) { u32 x[16]; @@ -101,15 +93,16 @@ static void salsa20_docrypt(u32 *state, u8 *dst, const u8 *src, } } -static void salsa20_init(u32 *state, const struct salsa20_ctx *ctx, +void crypto_salsa20_init(u32 *state, const struct salsa20_ctx *ctx, const u8 *iv) { memcpy(state, ctx->initial_state, sizeof(ctx->initial_state)); state[6] = get_unaligned_le32(iv + 0); state[7] = get_unaligned_le32(iv + 4); } +EXPORT_SYMBOL_GPL(crypto_salsa20_init); -static int salsa20_setkey(struct crypto_skcipher *tfm, const u8 *key, +int crypto_salsa20_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keysize) { static const char sigma[16] = "expand 32-byte k"; @@ -150,6 +143,7 @@ static int salsa20_setkey(struct crypto_skcipher *tfm, const u8 *key, return 0; } +EXPORT_SYMBOL_GPL(crypto_salsa20_setkey); static int salsa20_crypt(struct skcipher_request *req) { @@ -161,7 +155,7 @@ static int salsa20_crypt(struct skcipher_request *req) err = skcipher_walk_virt(&walk, req, true); - salsa20_init(state, ctx, walk.iv); + crypto_salsa20_init(state, ctx, walk.iv); while (walk.nbytes > 0) { unsigned int nbytes = walk.nbytes; @@ -189,7 +183,7 @@ static struct skcipher_alg alg = { .max_keysize = SALSA20_MAX_KEY_SIZE, .ivsize = SALSA20_IV_SIZE, .chunksize = SALSA20_BLOCK_SIZE, - .setkey = salsa20_setkey, + .setkey = crypto_salsa20_setkey, .encrypt = salsa20_crypt, .decrypt = salsa20_crypt, }; diff --git a/include/crypto/salsa20.h b/include/crypto/salsa20.h new file mode 100644 index 000000000000..19ed48aefc86 --- /dev/null +++ b/include/crypto/salsa20.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Common values for the Salsa20 algorithm + */ + +#ifndef _CRYPTO_SALSA20_H +#define _CRYPTO_SALSA20_H + +#include + +#define SALSA20_IV_SIZE 8 +#define SALSA20_MIN_KEY_SIZE 16 +#define SALSA20_MAX_KEY_SIZE 32 +#define SALSA20_BLOCK_SIZE 64 + +struct crypto_skcipher; + +struct salsa20_ctx { + u32 initial_state[16]; +}; + +void crypto_salsa20_init(u32 *state, const struct salsa20_ctx *ctx, + const u8 *iv); +int crypto_salsa20_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keysize); + +#endif /* _CRYPTO_SALSA20_H */ From c9a3ff8f22a2df5ec6de18ba616a863392269a10 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 5 Jan 2018 11:09:59 -0800 Subject: [PATCH 127/166] crypto: x86/salsa20 - cleanup and convert to skcipher API Convert salsa20-asm from the deprecated "blkcipher" API to the "skcipher" API, in the process fixing it up to use the generic helpers. This allows removing the salsa20_keysetup() and salsa20_ivsetup() assembly functions, which aren't performance critical; the C versions do just fine. This also fixes the same bug that salsa20-generic had, where the state array was being maintained directly in the transform context rather than on the stack or in the request context. Thus, if multiple threads used the same Salsa20 transform concurrently they produced the wrong results. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/salsa20-i586-asm_32.S | 184 +----------------------- arch/x86/crypto/salsa20-x86_64-asm_64.S | 114 --------------- arch/x86/crypto/salsa20_glue.c | 105 ++++++-------- crypto/Kconfig | 2 + 4 files changed, 46 insertions(+), 359 deletions(-) diff --git a/arch/x86/crypto/salsa20-i586-asm_32.S b/arch/x86/crypto/salsa20-i586-asm_32.S index 329452b8f794..6014b7b9e52a 100644 --- a/arch/x86/crypto/salsa20-i586-asm_32.S +++ b/arch/x86/crypto/salsa20-i586-asm_32.S @@ -1,6 +1,7 @@ -# salsa20_pm.s version 20051229 -# D. J. Bernstein -# Public domain. +# Derived from: +# salsa20_pm.s version 20051229 +# D. J. Bernstein +# Public domain. #include @@ -935,180 +936,3 @@ ENTRY(salsa20_encrypt_bytes) # goto bytesatleast1 jmp ._bytesatleast1 ENDPROC(salsa20_encrypt_bytes) - -# enter salsa20_keysetup -ENTRY(salsa20_keysetup) - mov %esp,%eax - and $31,%eax - add $256,%eax - sub %eax,%esp - # eax_stack = eax - movl %eax,64(%esp) - # ebx_stack = ebx - movl %ebx,68(%esp) - # esi_stack = esi - movl %esi,72(%esp) - # edi_stack = edi - movl %edi,76(%esp) - # ebp_stack = ebp - movl %ebp,80(%esp) - # k = arg2 - movl 8(%esp,%eax),%ecx - # kbits = arg3 - movl 12(%esp,%eax),%edx - # x = arg1 - movl 4(%esp,%eax),%eax - # in1 = *(uint32 *) (k + 0) - movl 0(%ecx),%ebx - # in2 = *(uint32 *) (k + 4) - movl 4(%ecx),%esi - # in3 = *(uint32 *) (k + 8) - movl 8(%ecx),%edi - # in4 = *(uint32 *) (k + 12) - movl 12(%ecx),%ebp - # *(uint32 *) (x + 4) = in1 - movl %ebx,4(%eax) - # *(uint32 *) (x + 8) = in2 - movl %esi,8(%eax) - # *(uint32 *) (x + 12) = in3 - movl %edi,12(%eax) - # *(uint32 *) (x + 16) = in4 - movl %ebp,16(%eax) - # kbits - 256 - cmp $256,%edx - # goto kbits128 if unsigned< - jb ._kbits128 -._kbits256: - # in11 = *(uint32 *) (k + 16) - movl 16(%ecx),%edx - # in12 = *(uint32 *) (k + 20) - movl 20(%ecx),%ebx - # in13 = *(uint32 *) (k + 24) - movl 24(%ecx),%esi - # in14 = *(uint32 *) (k + 28) - movl 28(%ecx),%ecx - # *(uint32 *) (x + 44) = in11 - movl %edx,44(%eax) - # *(uint32 *) (x + 48) = in12 - movl %ebx,48(%eax) - # *(uint32 *) (x + 52) = in13 - movl %esi,52(%eax) - # *(uint32 *) (x + 56) = in14 - movl %ecx,56(%eax) - # in0 = 1634760805 - mov $1634760805,%ecx - # in5 = 857760878 - mov $857760878,%edx - # in10 = 2036477234 - mov $2036477234,%ebx - # in15 = 1797285236 - mov $1797285236,%esi - # *(uint32 *) (x + 0) = in0 - movl %ecx,0(%eax) - # *(uint32 *) (x + 20) = in5 - movl %edx,20(%eax) - # *(uint32 *) (x + 40) = in10 - movl %ebx,40(%eax) - # *(uint32 *) (x + 60) = in15 - movl %esi,60(%eax) - # goto keysetupdone - jmp ._keysetupdone -._kbits128: - # in11 = *(uint32 *) (k + 0) - movl 0(%ecx),%edx - # in12 = *(uint32 *) (k + 4) - movl 4(%ecx),%ebx - # in13 = *(uint32 *) (k + 8) - movl 8(%ecx),%esi - # in14 = *(uint32 *) (k + 12) - movl 12(%ecx),%ecx - # *(uint32 *) (x + 44) = in11 - movl %edx,44(%eax) - # *(uint32 *) (x + 48) = in12 - movl %ebx,48(%eax) - # *(uint32 *) (x + 52) = in13 - movl %esi,52(%eax) - # *(uint32 *) (x + 56) = in14 - movl %ecx,56(%eax) - # in0 = 1634760805 - mov $1634760805,%ecx - # in5 = 824206446 - mov $824206446,%edx - # in10 = 2036477238 - mov $2036477238,%ebx - # in15 = 1797285236 - mov $1797285236,%esi - # *(uint32 *) (x + 0) = in0 - movl %ecx,0(%eax) - # *(uint32 *) (x + 20) = in5 - movl %edx,20(%eax) - # *(uint32 *) (x + 40) = in10 - movl %ebx,40(%eax) - # *(uint32 *) (x + 60) = in15 - movl %esi,60(%eax) -._keysetupdone: - # eax = eax_stack - movl 64(%esp),%eax - # ebx = ebx_stack - movl 68(%esp),%ebx - # esi = esi_stack - movl 72(%esp),%esi - # edi = edi_stack - movl 76(%esp),%edi - # ebp = ebp_stack - movl 80(%esp),%ebp - # leave - add %eax,%esp - ret -ENDPROC(salsa20_keysetup) - -# enter salsa20_ivsetup -ENTRY(salsa20_ivsetup) - mov %esp,%eax - and $31,%eax - add $256,%eax - sub %eax,%esp - # eax_stack = eax - movl %eax,64(%esp) - # ebx_stack = ebx - movl %ebx,68(%esp) - # esi_stack = esi - movl %esi,72(%esp) - # edi_stack = edi - movl %edi,76(%esp) - # ebp_stack = ebp - movl %ebp,80(%esp) - # iv = arg2 - movl 8(%esp,%eax),%ecx - # x = arg1 - movl 4(%esp,%eax),%eax - # in6 = *(uint32 *) (iv + 0) - movl 0(%ecx),%edx - # in7 = *(uint32 *) (iv + 4) - movl 4(%ecx),%ecx - # in8 = 0 - mov $0,%ebx - # in9 = 0 - mov $0,%esi - # *(uint32 *) (x + 24) = in6 - movl %edx,24(%eax) - # *(uint32 *) (x + 28) = in7 - movl %ecx,28(%eax) - # *(uint32 *) (x + 32) = in8 - movl %ebx,32(%eax) - # *(uint32 *) (x + 36) = in9 - movl %esi,36(%eax) - # eax = eax_stack - movl 64(%esp),%eax - # ebx = ebx_stack - movl 68(%esp),%ebx - # esi = esi_stack - movl 72(%esp),%esi - # edi = edi_stack - movl 76(%esp),%edi - # ebp = ebp_stack - movl 80(%esp),%ebp - # leave - add %eax,%esp - ret -ENDPROC(salsa20_ivsetup) diff --git a/arch/x86/crypto/salsa20-x86_64-asm_64.S b/arch/x86/crypto/salsa20-x86_64-asm_64.S index 10db30d58006..03a4918f41ee 100644 --- a/arch/x86/crypto/salsa20-x86_64-asm_64.S +++ b/arch/x86/crypto/salsa20-x86_64-asm_64.S @@ -803,117 +803,3 @@ ENTRY(salsa20_encrypt_bytes) # goto bytesatleast1 jmp ._bytesatleast1 ENDPROC(salsa20_encrypt_bytes) - -# enter salsa20_keysetup -ENTRY(salsa20_keysetup) - mov %rsp,%r11 - and $31,%r11 - add $256,%r11 - sub %r11,%rsp - # k = arg2 - mov %rsi,%rsi - # kbits = arg3 - mov %rdx,%rdx - # x = arg1 - mov %rdi,%rdi - # in0 = *(uint64 *) (k + 0) - movq 0(%rsi),%r8 - # in2 = *(uint64 *) (k + 8) - movq 8(%rsi),%r9 - # *(uint64 *) (x + 4) = in0 - movq %r8,4(%rdi) - # *(uint64 *) (x + 12) = in2 - movq %r9,12(%rdi) - # unsigned * + * Also modified to set up the initial state using the generic C code rather + * than in assembly. + * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation; either version 2 of the License, or (at your option) @@ -18,93 +21,65 @@ * */ -#include +#include +#include +#include #include -#include -#define SALSA20_IV_SIZE 8U -#define SALSA20_MIN_KEY_SIZE 16U -#define SALSA20_MAX_KEY_SIZE 32U +asmlinkage void salsa20_encrypt_bytes(u32 state[16], const u8 *src, u8 *dst, + u32 bytes); -struct salsa20_ctx +static int salsa20_asm_crypt(struct skcipher_request *req) { - u32 input[16]; -}; - -asmlinkage void salsa20_keysetup(struct salsa20_ctx *ctx, const u8 *k, - u32 keysize, u32 ivsize); -asmlinkage void salsa20_ivsetup(struct salsa20_ctx *ctx, const u8 *iv); -asmlinkage void salsa20_encrypt_bytes(struct salsa20_ctx *ctx, - const u8 *src, u8 *dst, u32 bytes); - -static int setkey(struct crypto_tfm *tfm, const u8 *key, - unsigned int keysize) -{ - struct salsa20_ctx *ctx = crypto_tfm_ctx(tfm); - salsa20_keysetup(ctx, key, keysize*8, SALSA20_IV_SIZE*8); - return 0; -} - -static int encrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) -{ - struct blkcipher_walk walk; - struct crypto_blkcipher *tfm = desc->tfm; - struct salsa20_ctx *ctx = crypto_blkcipher_ctx(tfm); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct salsa20_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; + u32 state[16]; int err; - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt_block(desc, &walk, 64); + err = skcipher_walk_virt(&walk, req, true); - salsa20_ivsetup(ctx, walk.iv); + crypto_salsa20_init(state, ctx, walk.iv); - while (walk.nbytes >= 64) { - salsa20_encrypt_bytes(ctx, walk.src.virt.addr, - walk.dst.virt.addr, - walk.nbytes - (walk.nbytes % 64)); - err = blkcipher_walk_done(desc, &walk, walk.nbytes % 64); - } + while (walk.nbytes > 0) { + unsigned int nbytes = walk.nbytes; - if (walk.nbytes) { - salsa20_encrypt_bytes(ctx, walk.src.virt.addr, - walk.dst.virt.addr, walk.nbytes); - err = blkcipher_walk_done(desc, &walk, 0); + if (nbytes < walk.total) + nbytes = round_down(nbytes, walk.stride); + + salsa20_encrypt_bytes(state, walk.src.virt.addr, + walk.dst.virt.addr, nbytes); + err = skcipher_walk_done(&walk, walk.nbytes - nbytes); } return err; } -static struct crypto_alg alg = { - .cra_name = "salsa20", - .cra_driver_name = "salsa20-asm", - .cra_priority = 200, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_type = &crypto_blkcipher_type, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct salsa20_ctx), - .cra_alignmask = 3, - .cra_module = THIS_MODULE, - .cra_u = { - .blkcipher = { - .setkey = setkey, - .encrypt = encrypt, - .decrypt = encrypt, - .min_keysize = SALSA20_MIN_KEY_SIZE, - .max_keysize = SALSA20_MAX_KEY_SIZE, - .ivsize = SALSA20_IV_SIZE, - } - } +static struct skcipher_alg alg = { + .base.cra_name = "salsa20", + .base.cra_driver_name = "salsa20-asm", + .base.cra_priority = 200, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct salsa20_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = SALSA20_MIN_KEY_SIZE, + .max_keysize = SALSA20_MAX_KEY_SIZE, + .ivsize = SALSA20_IV_SIZE, + .chunksize = SALSA20_BLOCK_SIZE, + .setkey = crypto_salsa20_setkey, + .encrypt = salsa20_asm_crypt, + .decrypt = salsa20_asm_crypt, }; static int __init init(void) { - return crypto_register_alg(&alg); + return crypto_register_skcipher(&alg); } static void __exit fini(void) { - crypto_unregister_alg(&alg); + crypto_unregister_skcipher(&alg); } module_init(init); diff --git a/crypto/Kconfig b/crypto/Kconfig index 9327fbfccf5a..b44c0ae04eb2 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1339,6 +1339,7 @@ config CRYPTO_SALSA20_586 tristate "Salsa20 stream cipher algorithm (i586)" depends on (X86 || UML_X86) && !64BIT select CRYPTO_BLKCIPHER + select CRYPTO_SALSA20 help Salsa20 stream cipher algorithm. @@ -1352,6 +1353,7 @@ config CRYPTO_SALSA20_X86_64 tristate "Salsa20 stream cipher algorithm (x86_64)" depends on (X86 || UML_X86) && 64BIT select CRYPTO_BLKCIPHER + select CRYPTO_SALSA20 help Salsa20 stream cipher algorithm. From b40fa82cd6138350f723aa47b37e3e3e80906b40 Mon Sep 17 00:00:00 2001 From: tcharding Date: Sun, 7 Jan 2018 10:01:43 +1100 Subject: [PATCH 128/166] crypto: doc - clear htmldocs build warnings for crypto/hash SPHINX build emits multiple warnings of kind: warning: duplicate section name 'Note' (when building kernel via make target 'htmldocs') This is caused by repeated use of comments of form: * Note: soau soaeusoa uoe We can change the format without loss of clarity and clear the build warnings. Add '**[mandatory]**' or '**[optional]**' as kernel-doc field element description prefix This renders in HTML as (prefixes in bold) final [mandatory] Retrieve result from the driver. This function finalizes the transformation and retrieves the resulting hash from the driver and pushes it back to upper layers. No data processing happens at this point unless hardware requires it to finish the transformation (then the data buffered by the device driver is processed). Signed-off-by: Tobin C. Harding Signed-off-by: Herbert Xu --- include/crypto/hash.h | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/include/crypto/hash.h b/include/crypto/hash.h index 3880793e280e..2d1849dffb80 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -71,12 +71,11 @@ struct ahash_request { /** * struct ahash_alg - asynchronous message digest definition - * @init: Initialize the transformation context. Intended only to initialize the + * @init: **[mandatory]** Initialize the transformation context. Intended only to initialize the * state of the HASH transformation at the beginning. This shall fill in * the internal structures used during the entire duration of the whole * transformation. No data processing happens at this point. - * Note: mandatory. - * @update: Push a chunk of data into the driver for transformation. This + * @update: **[mandatory]** Push a chunk of data into the driver for transformation. This * function actually pushes blocks of data from upper layers into the * driver, which then passes those to the hardware as seen fit. This * function must not finalize the HASH transformation by calculating the @@ -85,20 +84,17 @@ struct ahash_request { * context, as this function may be called in parallel with the same * transformation object. Data processing can happen synchronously * [SHASH] or asynchronously [AHASH] at this point. - * Note: mandatory. - * @final: Retrieve result from the driver. This function finalizes the + * @final: **[mandatory]** Retrieve result from the driver. This function finalizes the * transformation and retrieves the resulting hash from the driver and * pushes it back to upper layers. No data processing happens at this * point unless hardware requires it to finish the transformation * (then the data buffered by the device driver is processed). - * Note: mandatory. - * @finup: Combination of @update and @final. This function is effectively a + * @finup: **[optional]** Combination of @update and @final. This function is effectively a * combination of @update and @final calls issued in sequence. As some * hardware cannot do @update and @final separately, this callback was * added to allow such hardware to be used at least by IPsec. Data * processing can happen synchronously [SHASH] or asynchronously [AHASH] * at this point. - * Note: optional. * @digest: Combination of @init and @update and @final. This function * effectively behaves as the entire chain of operations, @init, * @update and @final issued in sequence. Just like @finup, this was From 9555003a3373b0cb7ca753779115f1d6f96fcab1 Mon Sep 17 00:00:00 2001 From: Antoine Tenart Date: Thu, 14 Dec 2017 15:26:59 +0100 Subject: [PATCH 129/166] Documentation/bindings: crypto: document the SafeXcel EIP97 compatible This patch adds the SafeXcel EIP97 compatible to the Inside Secure device tree bindings documentation. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- .../devicetree/bindings/crypto/inside-secure-safexcel.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/crypto/inside-secure-safexcel.txt b/Documentation/devicetree/bindings/crypto/inside-secure-safexcel.txt index fbc07d12322f..30c3ce6b502e 100644 --- a/Documentation/devicetree/bindings/crypto/inside-secure-safexcel.txt +++ b/Documentation/devicetree/bindings/crypto/inside-secure-safexcel.txt @@ -1,7 +1,8 @@ Inside Secure SafeXcel cryptographic engine Required properties: -- compatible: Should be "inside-secure,safexcel-eip197". +- compatible: Should be "inside-secure,safexcel-eip197" or + "inside-secure,safexcel-eip97". - reg: Base physical address of the engine and length of memory mapped region. - interrupts: Interrupt numbers for the rings and engine. - interrupt-names: Should be "ring0", "ring1", "ring2", "ring3", "eip", "mem". From 9f9cf046bf90b2c5eee92a4a78c71ead976762d5 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Tue, 9 Jan 2018 18:57:35 +0100 Subject: [PATCH 130/166] crypto: exynos-rng - Add SPDX license identifier and correct module license Replace GPL license statement with SPDX GPL-2.0 license identifier and correct the module license to GPLv2. The license itself was a generic GPL because of copy-and-paste from old drivers/char/hw_random/exynos-rng.c driver (on which this was based on). However the module license indicated GPL-2.0 or later. GPL-2.0 was intended by author so fix up this mess. Signed-off-by: Krzysztof Kozlowski Signed-off-by: Herbert Xu --- drivers/crypto/exynos-rng.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/drivers/crypto/exynos-rng.c b/drivers/crypto/exynos-rng.c index 4a06092074b9..86f5f459762e 100644 --- a/drivers/crypto/exynos-rng.c +++ b/drivers/crypto/exynos-rng.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * exynos-rng.c - Random Number Generator driver for the Exynos * @@ -6,15 +7,6 @@ * Loosely based on old driver from drivers/char/hw_random/exynos-rng.c: * Copyright (C) 2012 Samsung Electronics * Jonghwa Lee - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include @@ -410,4 +402,4 @@ module_platform_driver(exynos_rng_driver); MODULE_DESCRIPTION("Exynos H/W Random Number Generator driver"); MODULE_AUTHOR("Krzysztof Kozlowski "); -MODULE_LICENSE("GPL"); +MODULE_LICENSE("GPL v2"); From 5c8d850c961aa0812dfec2457418ae479eccec73 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Tue, 9 Jan 2018 18:57:36 +0100 Subject: [PATCH 131/166] crypto: s5p-sss - Add SPDX license identifier Replace GPL license statement with SPDX GPL-2.0 license identifier. Signed-off-by: Krzysztof Kozlowski Signed-off-by: Herbert Xu --- drivers/crypto/s5p-sss.c | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c index 62830a43d959..188f44b7eb27 100644 --- a/drivers/crypto/s5p-sss.c +++ b/drivers/crypto/s5p-sss.c @@ -1,17 +1,13 @@ -/* - * Cryptographic API. - * - * Support for Samsung S5PV210 and Exynos HW acceleration. - * - * Copyright (C) 2011 NetUP Inc. All rights reserved. - * Copyright (c) 2017 Samsung Electronics Co., Ltd. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as published - * by the Free Software Foundation. - * - * Hash part based on omap-sham.c driver. - */ +// SPDX-License-Identifier: GPL-2.0 +// +// Cryptographic API. +// +// Support for Samsung S5PV210 and Exynos HW acceleration. +// +// Copyright (C) 2011 NetUP Inc. All rights reserved. +// Copyright (c) 2017 Samsung Electronics Co., Ltd. All rights reserved. +// +// Hash part based on omap-sham.c driver. #include #include From bb6c8c467bce25ccd0c485b7c3214dde6b0bcb33 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 9 Jan 2018 18:23:02 +0000 Subject: [PATCH 132/166] crypto: arm64 - implement SHA-512 using special instructions Implement the SHA-512 using the new special instructions that have been introduced as an optional extension in ARMv8.2. Signed-off-by: Ard Biesheuvel Tested-by: Steve Capper Signed-off-by: Herbert Xu --- arch/arm64/crypto/Kconfig | 6 + arch/arm64/crypto/Makefile | 3 + arch/arm64/crypto/sha512-ce-core.S | 207 +++++++++++++++++++++++++++++ arch/arm64/crypto/sha512-ce-glue.c | 119 +++++++++++++++++ 4 files changed, 335 insertions(+) create mode 100644 arch/arm64/crypto/sha512-ce-core.S create mode 100644 arch/arm64/crypto/sha512-ce-glue.c diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index 70c517aa4501..aad288f4b9de 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -29,6 +29,12 @@ config CRYPTO_SHA2_ARM64_CE select CRYPTO_HASH select CRYPTO_SHA256_ARM64 +config CRYPTO_SHA512_ARM64_CE + tristate "SHA-384/SHA-512 digest algorithm (ARMv8 Crypto Extensions)" + depends on KERNEL_MODE_NEON + select CRYPTO_HASH + select CRYPTO_SHA512_ARM64 + config CRYPTO_GHASH_ARM64_CE tristate "GHASH/AES-GCM using ARMv8 Crypto Extensions" depends on KERNEL_MODE_NEON diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile index f5e8295fd756..b438b3dc9b4c 100644 --- a/arch/arm64/crypto/Makefile +++ b/arch/arm64/crypto/Makefile @@ -14,6 +14,9 @@ sha1-ce-y := sha1-ce-glue.o sha1-ce-core.o obj-$(CONFIG_CRYPTO_SHA2_ARM64_CE) += sha2-ce.o sha2-ce-y := sha2-ce-glue.o sha2-ce-core.o +obj-$(CONFIG_CRYPTO_SHA512_ARM64_CE) += sha512-ce.o +sha512-ce-y := sha512-ce-glue.o sha512-ce-core.o + obj-$(CONFIG_CRYPTO_GHASH_ARM64_CE) += ghash-ce.o ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o diff --git a/arch/arm64/crypto/sha512-ce-core.S b/arch/arm64/crypto/sha512-ce-core.S new file mode 100644 index 000000000000..6c562f8df0b0 --- /dev/null +++ b/arch/arm64/crypto/sha512-ce-core.S @@ -0,0 +1,207 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * sha512-ce-core.S - core SHA-384/SHA-512 transform using v8 Crypto Extensions + * + * Copyright (C) 2018 Linaro Ltd + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include + + // + // Temporary - for testing only. binutils has no support for these yet + // + .irp b,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 + .set .Lq\b, \b + .set .Lv\b\().2d, \b + .endr + + .macro sha512h, rd, rn, rm + .inst 0xce608000 | .L\rd | (.L\rn << 5) | (.L\rm << 16) + .endm + + .macro sha512h2, rd, rn, rm + .inst 0xce608400 | .L\rd | (.L\rn << 5) | (.L\rm << 16) + .endm + + .macro sha512su0, rd, rn + .inst 0xcec08000 | .L\rd | (.L\rn << 5) + .endm + + .macro sha512su1, rd, rn, rm + .inst 0xce608800 | .L\rd | (.L\rn << 5) | (.L\rm << 16) + .endm + + .text + .arch armv8-a+crypto + + /* + * The SHA-512 round constants + */ + .align 4 +.Lsha512_rcon: + .quad 0x428a2f98d728ae22, 0x7137449123ef65cd + .quad 0xb5c0fbcfec4d3b2f, 0xe9b5dba58189dbbc + .quad 0x3956c25bf348b538, 0x59f111f1b605d019 + .quad 0x923f82a4af194f9b, 0xab1c5ed5da6d8118 + .quad 0xd807aa98a3030242, 0x12835b0145706fbe + .quad 0x243185be4ee4b28c, 0x550c7dc3d5ffb4e2 + .quad 0x72be5d74f27b896f, 0x80deb1fe3b1696b1 + .quad 0x9bdc06a725c71235, 0xc19bf174cf692694 + .quad 0xe49b69c19ef14ad2, 0xefbe4786384f25e3 + .quad 0x0fc19dc68b8cd5b5, 0x240ca1cc77ac9c65 + .quad 0x2de92c6f592b0275, 0x4a7484aa6ea6e483 + .quad 0x5cb0a9dcbd41fbd4, 0x76f988da831153b5 + .quad 0x983e5152ee66dfab, 0xa831c66d2db43210 + .quad 0xb00327c898fb213f, 0xbf597fc7beef0ee4 + .quad 0xc6e00bf33da88fc2, 0xd5a79147930aa725 + .quad 0x06ca6351e003826f, 0x142929670a0e6e70 + .quad 0x27b70a8546d22ffc, 0x2e1b21385c26c926 + .quad 0x4d2c6dfc5ac42aed, 0x53380d139d95b3df + .quad 0x650a73548baf63de, 0x766a0abb3c77b2a8 + .quad 0x81c2c92e47edaee6, 0x92722c851482353b + .quad 0xa2bfe8a14cf10364, 0xa81a664bbc423001 + .quad 0xc24b8b70d0f89791, 0xc76c51a30654be30 + .quad 0xd192e819d6ef5218, 0xd69906245565a910 + .quad 0xf40e35855771202a, 0x106aa07032bbd1b8 + .quad 0x19a4c116b8d2d0c8, 0x1e376c085141ab53 + .quad 0x2748774cdf8eeb99, 0x34b0bcb5e19b48a8 + .quad 0x391c0cb3c5c95a63, 0x4ed8aa4ae3418acb + .quad 0x5b9cca4f7763e373, 0x682e6ff3d6b2b8a3 + .quad 0x748f82ee5defb2fc, 0x78a5636f43172f60 + .quad 0x84c87814a1f0ab72, 0x8cc702081a6439ec + .quad 0x90befffa23631e28, 0xa4506cebde82bde9 + .quad 0xbef9a3f7b2c67915, 0xc67178f2e372532b + .quad 0xca273eceea26619c, 0xd186b8c721c0c207 + .quad 0xeada7dd6cde0eb1e, 0xf57d4f7fee6ed178 + .quad 0x06f067aa72176fba, 0x0a637dc5a2c898a6 + .quad 0x113f9804bef90dae, 0x1b710b35131c471b + .quad 0x28db77f523047d84, 0x32caab7b40c72493 + .quad 0x3c9ebe0a15c9bebc, 0x431d67c49c100d4c + .quad 0x4cc5d4becb3e42b6, 0x597f299cfc657e2a + .quad 0x5fcb6fab3ad6faec, 0x6c44198c4a475817 + + .macro dround, i0, i1, i2, i3, i4, rc0, rc1, in0, in1, in2, in3, in4 + .ifnb \rc1 + ld1 {v\rc1\().2d}, [x3], #16 + .endif + add v\rc0\().2d, v\rc0\().2d, v\in0\().2d + ext v6.16b, v\i2\().16b, v\i3\().16b, #8 + ext v\rc0\().16b, v\rc0\().16b, v\rc0\().16b, #8 + ext v7.16b, v\i1\().16b, v\i2\().16b, #8 + add v\i3\().2d, v\i3\().2d, v\rc0\().2d + .ifnb \in1 + ext v10.16b, v\in3\().16b, v\in4\().16b, #8 + sha512su0 v\in0\().2d, v\in1\().2d + .endif + sha512h q\i3, q6, v7.2d + .ifnb \in1 + sha512su1 v\in0\().2d, v\in2\().2d, v10.2d + .endif + add v\i4\().2d, v\i1\().2d, v\i3\().2d + sha512h2 q\i3, q\i1, v\i0\().2d + .endm + + /* + * void sha512_ce_transform(struct sha512_state *sst, u8 const *src, + * int blocks) + */ +ENTRY(sha512_ce_transform) + /* load state */ + ld1 {v20.2d-v23.2d}, [x0] + + /* load input */ +0: ld1 {v12.2d-v15.2d}, [x1], #64 + ld1 {v16.2d-v19.2d}, [x1], #64 + sub w2, w2, #1 + + /* load round constants */ + adr x3, .Lsha512_rcon + +CPU_LE( rev64 v12.16b, v12.16b ) +CPU_LE( rev64 v13.16b, v13.16b ) +CPU_LE( rev64 v14.16b, v14.16b ) +CPU_LE( rev64 v15.16b, v15.16b ) +CPU_LE( rev64 v16.16b, v16.16b ) +CPU_LE( rev64 v17.16b, v17.16b ) +CPU_LE( rev64 v18.16b, v18.16b ) +CPU_LE( rev64 v19.16b, v19.16b ) + + ld1 {v8.2d}, [x3], #16 + + mov v0.16b, v20.16b + mov v1.16b, v21.16b + mov v2.16b, v22.16b + mov v3.16b, v23.16b + + // v0 ab cd -- ef gh ab + // v1 cd -- ef gh ab cd + // v2 ef gh ab cd -- ef + // v3 gh ab cd -- ef gh + // v4 -- ef gh ab cd -- + + dround 0, 1, 2, 3, 4, 8, 9, 12, 13, 19, 16, 17 + dround 3, 0, 4, 2, 1, 9, 8, 13, 14, 12, 17, 18 + dround 2, 3, 1, 4, 0, 8, 9, 14, 15, 13, 18, 19 + dround 4, 2, 0, 1, 3, 9, 8, 15, 16, 14, 19, 12 + dround 1, 4, 3, 0, 2, 8, 9, 16, 17, 15, 12, 13 + + dround 0, 1, 2, 3, 4, 9, 8, 17, 18, 16, 13, 14 + dround 3, 0, 4, 2, 1, 8, 9, 18, 19, 17, 14, 15 + dround 2, 3, 1, 4, 0, 9, 8, 19, 12, 18, 15, 16 + dround 4, 2, 0, 1, 3, 8, 9, 12, 13, 19, 16, 17 + dround 1, 4, 3, 0, 2, 9, 8, 13, 14, 12, 17, 18 + + dround 0, 1, 2, 3, 4, 8, 9, 14, 15, 13, 18, 19 + dround 3, 0, 4, 2, 1, 9, 8, 15, 16, 14, 19, 12 + dround 2, 3, 1, 4, 0, 8, 9, 16, 17, 15, 12, 13 + dround 4, 2, 0, 1, 3, 9, 8, 17, 18, 16, 13, 14 + dround 1, 4, 3, 0, 2, 8, 9, 18, 19, 17, 14, 15 + + dround 0, 1, 2, 3, 4, 9, 8, 19, 12, 18, 15, 16 + dround 3, 0, 4, 2, 1, 8, 9, 12, 13, 19, 16, 17 + dround 2, 3, 1, 4, 0, 9, 8, 13, 14, 12, 17, 18 + dround 4, 2, 0, 1, 3, 8, 9, 14, 15, 13, 18, 19 + dround 1, 4, 3, 0, 2, 9, 8, 15, 16, 14, 19, 12 + + dround 0, 1, 2, 3, 4, 8, 9, 16, 17, 15, 12, 13 + dround 3, 0, 4, 2, 1, 9, 8, 17, 18, 16, 13, 14 + dround 2, 3, 1, 4, 0, 8, 9, 18, 19, 17, 14, 15 + dround 4, 2, 0, 1, 3, 9, 8, 19, 12, 18, 15, 16 + dround 1, 4, 3, 0, 2, 8, 9, 12, 13, 19, 16, 17 + + dround 0, 1, 2, 3, 4, 9, 8, 13, 14, 12, 17, 18 + dround 3, 0, 4, 2, 1, 8, 9, 14, 15, 13, 18, 19 + dround 2, 3, 1, 4, 0, 9, 8, 15, 16, 14, 19, 12 + dround 4, 2, 0, 1, 3, 8, 9, 16, 17, 15, 12, 13 + dround 1, 4, 3, 0, 2, 9, 8, 17, 18, 16, 13, 14 + + dround 0, 1, 2, 3, 4, 8, 9, 18, 19, 17, 14, 15 + dround 3, 0, 4, 2, 1, 9, 8, 19, 12, 18, 15, 16 + dround 2, 3, 1, 4, 0, 8, 9, 12 + dround 4, 2, 0, 1, 3, 9, 8, 13 + dround 1, 4, 3, 0, 2, 8, 9, 14 + + dround 0, 1, 2, 3, 4, 9, 8, 15 + dround 3, 0, 4, 2, 1, 8, 9, 16 + dround 2, 3, 1, 4, 0, 9, 8, 17 + dround 4, 2, 0, 1, 3, 8, 9, 18 + dround 1, 4, 3, 0, 2, 9, , 19 + + /* update state */ + add v20.2d, v20.2d, v0.2d + add v21.2d, v21.2d, v1.2d + add v22.2d, v22.2d, v2.2d + add v23.2d, v23.2d, v3.2d + + /* handled all input blocks? */ + cbnz w2, 0b + + /* store new state */ +3: st1 {v20.2d-v23.2d}, [x0] + ret +ENDPROC(sha512_ce_transform) diff --git a/arch/arm64/crypto/sha512-ce-glue.c b/arch/arm64/crypto/sha512-ce-glue.c new file mode 100644 index 000000000000..a77c8632a589 --- /dev/null +++ b/arch/arm64/crypto/sha512-ce-glue.c @@ -0,0 +1,119 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * sha512-ce-glue.c - SHA-384/SHA-512 using ARMv8 Crypto Extensions + * + * Copyright (C) 2018 Linaro Ltd + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +MODULE_DESCRIPTION("SHA-384/SHA-512 secure hash using ARMv8 Crypto Extensions"); +MODULE_AUTHOR("Ard Biesheuvel "); +MODULE_LICENSE("GPL v2"); + +asmlinkage void sha512_ce_transform(struct sha512_state *sst, u8 const *src, + int blocks); + +asmlinkage void sha512_block_data_order(u64 *digest, u8 const *src, int blocks); + +static int sha512_ce_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + if (!may_use_simd()) + return sha512_base_do_update(desc, data, len, + (sha512_block_fn *)sha512_block_data_order); + + kernel_neon_begin(); + sha512_base_do_update(desc, data, len, + (sha512_block_fn *)sha512_ce_transform); + kernel_neon_end(); + + return 0; +} + +static int sha512_ce_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) +{ + if (!may_use_simd()) { + if (len) + sha512_base_do_update(desc, data, len, + (sha512_block_fn *)sha512_block_data_order); + sha512_base_do_finalize(desc, + (sha512_block_fn *)sha512_block_data_order); + return sha512_base_finish(desc, out); + } + + kernel_neon_begin(); + sha512_base_do_update(desc, data, len, + (sha512_block_fn *)sha512_ce_transform); + sha512_base_do_finalize(desc, (sha512_block_fn *)sha512_ce_transform); + kernel_neon_end(); + return sha512_base_finish(desc, out); +} + +static int sha512_ce_final(struct shash_desc *desc, u8 *out) +{ + if (!may_use_simd()) { + sha512_base_do_finalize(desc, + (sha512_block_fn *)sha512_block_data_order); + return sha512_base_finish(desc, out); + } + + kernel_neon_begin(); + sha512_base_do_finalize(desc, (sha512_block_fn *)sha512_ce_transform); + kernel_neon_end(); + return sha512_base_finish(desc, out); +} + +static struct shash_alg algs[] = { { + .init = sha384_base_init, + .update = sha512_ce_update, + .final = sha512_ce_final, + .finup = sha512_ce_finup, + .descsize = sizeof(struct sha512_state), + .digestsize = SHA384_DIGEST_SIZE, + .base.cra_name = "sha384", + .base.cra_driver_name = "sha384-ce", + .base.cra_priority = 200, + .base.cra_flags = CRYPTO_ALG_TYPE_SHASH, + .base.cra_blocksize = SHA512_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, +}, { + .init = sha512_base_init, + .update = sha512_ce_update, + .final = sha512_ce_final, + .finup = sha512_ce_finup, + .descsize = sizeof(struct sha512_state), + .digestsize = SHA512_DIGEST_SIZE, + .base.cra_name = "sha512", + .base.cra_driver_name = "sha512-ce", + .base.cra_priority = 200, + .base.cra_flags = CRYPTO_ALG_TYPE_SHASH, + .base.cra_blocksize = SHA512_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, +} }; + +static int __init sha512_ce_mod_init(void) +{ + return crypto_register_shashes(algs, ARRAY_SIZE(algs)); +} + +static void __exit sha512_ce_mod_fini(void) +{ + crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); +} + +module_cpu_feature_match(SHA512, sha512_ce_mod_init); +module_exit(sha512_ce_mod_fini); From 4565da7c3995a07eea54b5e4a79d2c06209c0c85 Mon Sep 17 00:00:00 2001 From: Sean Wang Date: Wed, 10 Jan 2018 12:02:46 +0800 Subject: [PATCH 133/166] hwrng: mediatek - Setup default RNG quality When hw_random device's quality is non-zero, it will automatically fill the kernel's entropy pool at boot. For the purpose, one conservative quality value is being picked up as the default value. Signed-off-by: Sean Wang Signed-off-by: Herbert Xu --- drivers/char/hw_random/mtk-rng.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/char/hw_random/mtk-rng.c b/drivers/char/hw_random/mtk-rng.c index 8da7bcf54105..7f99cd52b40e 100644 --- a/drivers/char/hw_random/mtk-rng.c +++ b/drivers/char/hw_random/mtk-rng.c @@ -135,6 +135,7 @@ static int mtk_rng_probe(struct platform_device *pdev) #endif priv->rng.read = mtk_rng_read; priv->rng.priv = (unsigned long)&pdev->dev; + priv->rng.quality = 900; priv->clk = devm_clk_get(&pdev->dev, "rng"); if (IS_ERR(priv->clk)) { From a8bc71d4960391f732ec0307ebd69e4a25aaf4d3 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Wed, 10 Jan 2018 12:36:58 +0300 Subject: [PATCH 134/166] hwrng: exynos - Signedness bug in exynos_trng_do_read() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "val" needs to be signed for the error handling to work. Fixes: 6cd225cc5d8a ("hwrng: exynos - add Samsung Exynos True RNG driver") Signed-off-by: Dan Carpenter Reviewed-by: Krzysztof Kozlowski Acked-by: Łukasz Stelmach Signed-off-by: Herbert Xu --- drivers/char/hw_random/exynos-trng.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/char/hw_random/exynos-trng.c b/drivers/char/hw_random/exynos-trng.c index 34d6f51ecbee..f4643e3ec346 100644 --- a/drivers/char/hw_random/exynos-trng.c +++ b/drivers/char/hw_random/exynos-trng.c @@ -55,7 +55,7 @@ static int exynos_trng_do_read(struct hwrng *rng, void *data, size_t max, bool wait) { struct exynos_trng_dev *trng; - u32 val; + int val; max = min_t(size_t, max, (EXYNOS_TRNG_FIFO_LEN * 4)); From 2273f42df1a52bb3aa827163e285528afbf7616c Mon Sep 17 00:00:00 2001 From: "weiyongjun \\(A\\)" Date: Wed, 10 Jan 2018 13:30:59 +0000 Subject: [PATCH 135/166] hwrng: exynos - remove redundant dev_err call in exynos_trng_probe() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is a error message within devm_ioremap_resource already, so remove the dev_err call to avoid redundant error message. Signed-off-by: Wei Yongjun Reviewed-by: Krzysztof Kozlowski Acked-by: Łukasz Stelmach Signed-off-by: Herbert Xu --- drivers/char/hw_random/exynos-trng.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/char/hw_random/exynos-trng.c b/drivers/char/hw_random/exynos-trng.c index f4643e3ec346..1947aed7c044 100644 --- a/drivers/char/hw_random/exynos-trng.c +++ b/drivers/char/hw_random/exynos-trng.c @@ -129,10 +129,8 @@ static int exynos_trng_probe(struct platform_device *pdev) res = platform_get_resource(pdev, IORESOURCE_MEM, 0); trng->mem = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(trng->mem)) { - dev_err(&pdev->dev, "Could not map IO resources.\n"); + if (IS_ERR(trng->mem)) return PTR_ERR(trng->mem); - } pm_runtime_enable(&pdev->dev); ret = pm_runtime_get_sync(&pdev->dev); From 37d728f76c41ab819a9fd31d701de55102559484 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 10 Jan 2018 15:15:43 +0000 Subject: [PATCH 136/166] crypto: marvell/cesa - Fix DMA API misuse phys_to_dma() is an internal helper for certain DMA API implementations, and is not appropriate for drivers to use. It appears that what the CESA driver really wants to be using is dma_map_resource() - admittedly that didn't exist when the offending code was first merged, but it does now. Signed-off-by: Robin Murphy Acked-by: Boris Brezillon Signed-off-by: Herbert Xu --- drivers/crypto/marvell/cesa.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/drivers/crypto/marvell/cesa.c b/drivers/crypto/marvell/cesa.c index 293832488cc9..f81fa4a3e66b 100644 --- a/drivers/crypto/marvell/cesa.c +++ b/drivers/crypto/marvell/cesa.c @@ -15,6 +15,7 @@ */ #include +#include #include #include #include @@ -409,8 +410,11 @@ static int mv_cesa_get_sram(struct platform_device *pdev, int idx) if (IS_ERR(engine->sram)) return PTR_ERR(engine->sram); - engine->sram_dma = phys_to_dma(cesa->dev, - (phys_addr_t)res->start); + engine->sram_dma = dma_map_resource(cesa->dev, res->start, + cesa->sram_size, + DMA_BIDIRECTIONAL, 0); + if (dma_mapping_error(cesa->dev, engine->sram_dma)) + return -ENOMEM; return 0; } @@ -420,11 +424,12 @@ static void mv_cesa_put_sram(struct platform_device *pdev, int idx) struct mv_cesa_dev *cesa = platform_get_drvdata(pdev); struct mv_cesa_engine *engine = &cesa->engines[idx]; - if (!engine->pool) - return; - - gen_pool_free(engine->pool, (unsigned long)engine->sram, - cesa->sram_size); + if (engine->pool) + gen_pool_free(engine->pool, (unsigned long)engine->sram, + cesa->sram_size); + else + dma_unmap_resource(cesa->dev, engine->sram_dma, + cesa->sram_size, DMA_BIDIRECTIONAL, 0); } static int mv_cesa_probe(struct platform_device *pdev) From db6deea4899e2336de42da43c545eeac2eead4af Mon Sep 17 00:00:00 2001 From: Harsh Jain Date: Thu, 11 Jan 2018 16:45:48 +0530 Subject: [PATCH 137/166] crypto: chelsio - Fix Indentation Fix inconsistent Indenting. Reported-by: Dan Carpenter Signed-off-by: Harsh Jain Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index b663b93b7d01..f6b11610ed5b 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -683,7 +683,7 @@ static int chcr_sg_ent_in_wr(struct scatterlist *src, if (srclen <= dstlen) break; less = min_t(unsigned int, sg_dma_len(dst) - offset - - dstskip, CHCR_DST_SG_SIZE); + dstskip, CHCR_DST_SG_SIZE); dstlen += less; offset += less; if (offset == sg_dma_len(dst)) { @@ -694,7 +694,7 @@ static int chcr_sg_ent_in_wr(struct scatterlist *src, dstskip = 0; } src = sg_next(src); - srcskip = 0; + srcskip = 0; } return min(srclen, dstlen); } @@ -1142,10 +1142,10 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req, bytes = chcr_sg_ent_in_wr(reqctx->srcsg, reqctx->dstsg, 1, SPACE_LEFT(ablkctx->enckey_len), reqctx->src_ofst, reqctx->dst_ofst); - if ((bytes + reqctx->processed) >= req->nbytes) - bytes = req->nbytes - reqctx->processed; - else - bytes = ROUND_16(bytes); + if ((bytes + reqctx->processed) >= req->nbytes) + bytes = req->nbytes - reqctx->processed; + else + bytes = ROUND_16(bytes); } else { /*CTR mode counter overfloa*/ bytes = req->nbytes - reqctx->processed; @@ -1246,15 +1246,15 @@ static int process_cipher(struct ablkcipher_request *req, MIN_CIPHER_SG, SPACE_LEFT(ablkctx->enckey_len), 0, 0); - if ((bytes + reqctx->processed) >= req->nbytes) - bytes = req->nbytes - reqctx->processed; - else - bytes = ROUND_16(bytes); + if ((bytes + reqctx->processed) >= req->nbytes) + bytes = req->nbytes - reqctx->processed; + else + bytes = ROUND_16(bytes); } else { bytes = req->nbytes; } if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == - CRYPTO_ALG_SUB_TYPE_CTR) { + CRYPTO_ALG_SUB_TYPE_CTR) { bytes = adjust_ctr_overflow(req->info, bytes); } if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == @@ -2399,10 +2399,8 @@ void chcr_add_hash_src_ent(struct ahash_request *req, ulptx_walk_add_page(&ulp_walk, param->bfr_len, &reqctx->dma_addr); ulptx_walk_add_sg(&ulp_walk, req->src, param->sg_len, - 0); -// reqctx->srcsg = ulp_walk.last_sg; -// reqctx->src_ofst = ulp_walk.last_sg_len; - ulptx_walk_end(&ulp_walk); + 0); + ulptx_walk_end(&ulp_walk); } } From 8daa32b9357de54eb68d78bbe668d52a83d24e71 Mon Sep 17 00:00:00 2001 From: Harsh Jain Date: Thu, 11 Jan 2018 16:45:49 +0530 Subject: [PATCH 138/166] crypto: chelsio - check for sg null Add warning message if sg is NULL after skipping bytes. Reported-by: Dan Carpenter Signed-off-by: Harsh Jain Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index f6b11610ed5b..280377fc69e0 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -574,7 +574,8 @@ static void ulptx_walk_add_sg(struct ulptx_walk *walk, skip = 0; } } - if (walk->nents == 0) { + WARN(!sg, "SG should not be null here\n"); + if (sg && (walk->nents == 0)) { small = min_t(unsigned int, sg_dma_len(sg) - skip_len, len); sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE); walk->sgl->len0 = cpu_to_be32(sgmin); From 209897d54a7741522a696ec4ad463049157ca082 Mon Sep 17 00:00:00 2001 From: Harsh Jain Date: Thu, 11 Jan 2018 16:45:50 +0530 Subject: [PATCH 139/166] crypto: chelsio - Fix IV updated in XTS operation Skip decrypt operation on IV received from HW for last request. Signed-off-by: Harsh Jain Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) mode change 100644 => 100755 drivers/crypto/chelsio/chcr_algo.c diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c old mode 100644 new mode 100755 index 280377fc69e0..27ee92061e3d --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -1017,7 +1017,8 @@ static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes) return bytes; } -static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv) +static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv, + u32 isfinal) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); @@ -1044,7 +1045,8 @@ static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv) for (i = 0; i < (round % 8); i++) gf128mul_x_ble((le128 *)iv, (le128 *)iv); - crypto_cipher_decrypt_one(cipher, iv, iv); + if (!isfinal) + crypto_cipher_decrypt_one(cipher, iv, iv); out: return ret; } @@ -1065,7 +1067,7 @@ static int chcr_update_cipher_iv(struct ablkcipher_request *req, CTR_RFC3686_IV_SIZE) = cpu_to_be32((reqctx->processed / AES_BLOCK_SIZE) + 1); else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS) - ret = chcr_update_tweak(req, iv); + ret = chcr_update_tweak(req, iv, 0); else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) { if (reqctx->op) sg_pcopy_to_buffer(req->src, sg_nents(req->src), iv, @@ -1096,7 +1098,7 @@ static int chcr_final_cipher_iv(struct ablkcipher_request *req, ctr_add_iv(iv, req->info, (reqctx->processed / AES_BLOCK_SIZE)); else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS) - ret = chcr_update_tweak(req, iv); + ret = chcr_update_tweak(req, iv, 1); else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) { if (reqctx->op) sg_pcopy_to_buffer(req->src, sg_nents(req->src), iv, From 3d64bd670269b1391c924a04722441fc5cb3fc3a Mon Sep 17 00:00:00 2001 From: Harsh Jain Date: Thu, 11 Jan 2018 16:45:51 +0530 Subject: [PATCH 140/166] crypto: chelsio - Add authenc versions of ctr and sha Add ctr and sha combination of algo in authenc mode. Signed-off-by: Harsh Jain Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 215 ++++++++++++++++++++++++--- drivers/crypto/chelsio/chcr_crypto.h | 7 +- 2 files changed, 199 insertions(+), 23 deletions(-) mode change 100755 => 100644 drivers/crypto/chelsio/chcr_algo.c diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c old mode 100755 new mode 100644 index 27ee92061e3d..5cc84c459526 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -2090,7 +2090,7 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, struct cpl_rx_phys_dsgl *phys_cpl; struct ulptx_sgl *ulptx; unsigned int transhdr_len; - unsigned int dst_size = 0, temp; + unsigned int dst_size = 0, temp, subtype = get_aead_subtype(tfm); unsigned int kctx_len = 0, dnents; unsigned int assoclen = req->assoclen; unsigned int authsize = crypto_aead_authsize(tfm); @@ -2104,7 +2104,8 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, return NULL; reqctx->b0_dma = 0; - if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_NULL) { + if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL || + subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) { null = 1; assoclen = 0; } @@ -2169,16 +2170,23 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, temp & 0xF, null ? 0 : assoclen + IV + 1, temp, temp); + if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL || + subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA) + temp = CHCR_SCMD_CIPHER_MODE_AES_CTR; + else + temp = CHCR_SCMD_CIPHER_MODE_AES_CBC; chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type == CHCR_ENCRYPT_OP) ? 1 : 0, - CHCR_SCMD_CIPHER_MODE_AES_CBC, + temp, actx->auth_mode, aeadctx->hmac_ctrl, IV >> 1); chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0, 0, dst_size); chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; - if (op_type == CHCR_ENCRYPT_OP) + if (op_type == CHCR_ENCRYPT_OP || + subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA || + subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); else @@ -2188,7 +2196,16 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 4), actx->h_iopad, kctx_len - (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 4)); - memcpy(reqctx->iv, req->iv, IV); + if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA || + subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) { + memcpy(reqctx->iv, aeadctx->nonce, CTR_RFC3686_NONCE_SIZE); + memcpy(reqctx->iv + CTR_RFC3686_NONCE_SIZE, req->iv, + CTR_RFC3686_IV_SIZE); + *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE + + CTR_RFC3686_IV_SIZE) = cpu_to_be32(1); + } else { + memcpy(reqctx->iv, req->iv, IV); + } phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid); @@ -3216,7 +3233,7 @@ static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); /* it contains auth and cipher key both*/ struct crypto_authenc_keys keys; - unsigned int bs; + unsigned int bs, subtype; unsigned int max_authsize = crypto_aead_alg(authenc)->maxauthsize; int err = 0, i, key_ctx_len = 0; unsigned char ck_size = 0; @@ -3245,6 +3262,15 @@ static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, pr_err("chcr : Unsupported digest size\n"); goto out; } + subtype = get_aead_subtype(authenc); + if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA || + subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) { + if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE) + goto out; + memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen + - CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE); + keys.enckeylen -= CTR_RFC3686_NONCE_SIZE; + } if (keys.enckeylen == AES_KEYSIZE_128) { ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; } else if (keys.enckeylen == AES_KEYSIZE_192) { @@ -3262,9 +3288,12 @@ static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, */ memcpy(aeadctx->key, keys.enckey, keys.enckeylen); aeadctx->enckey_len = keys.enckeylen; - get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key, - aeadctx->enckey_len << 3); + if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA || + subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) { + get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key, + aeadctx->enckey_len << 3); + } base_hash = chcr_alloc_shash(max_authsize); if (IS_ERR(base_hash)) { pr_err("chcr : Base driver cannot be loaded\n"); @@ -3337,6 +3366,7 @@ static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc, struct crypto_authenc_keys keys; int err; /* it contains auth and cipher key both*/ + unsigned int subtype; int key_ctx_len = 0; unsigned char ck_size = 0; @@ -3354,6 +3384,15 @@ static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc, crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN); goto out; } + subtype = get_aead_subtype(authenc); + if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA || + subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) { + if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE) + goto out; + memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen + - CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE); + keys.enckeylen -= CTR_RFC3686_NONCE_SIZE; + } if (keys.enckeylen == AES_KEYSIZE_128) { ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; } else if (keys.enckeylen == AES_KEYSIZE_192) { @@ -3361,13 +3400,16 @@ static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc, } else if (keys.enckeylen == AES_KEYSIZE_256) { ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; } else { - pr_err("chcr : Unsupported cipher key\n"); + pr_err("chcr : Unsupported cipher key %d\n", keys.enckeylen); goto out; } memcpy(aeadctx->key, keys.enckey, keys.enckeylen); aeadctx->enckey_len = keys.enckeylen; - get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key, - aeadctx->enckey_len << 3); + if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA || + subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) { + get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key, + aeadctx->enckey_len << 3); + } key_ctx_len = sizeof(struct _key_ctx) + ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4); @@ -3421,8 +3463,10 @@ static int chcr_aead_encrypt(struct aead_request *req) reqctx->verify = VERIFY_HW; switch (get_aead_subtype(tfm)) { - case CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC: - case CRYPTO_ALG_SUB_TYPE_AEAD_NULL: + case CRYPTO_ALG_SUB_TYPE_CTR_SHA: + case CRYPTO_ALG_SUB_TYPE_CBC_SHA: + case CRYPTO_ALG_SUB_TYPE_CBC_NULL: + case CRYPTO_ALG_SUB_TYPE_CTR_NULL: return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0, create_authenc_wr); case CRYPTO_ALG_SUB_TYPE_AEAD_CCM: @@ -3451,8 +3495,10 @@ static int chcr_aead_decrypt(struct aead_request *req) } switch (get_aead_subtype(tfm)) { - case CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC: - case CRYPTO_ALG_SUB_TYPE_AEAD_NULL: + case CRYPTO_ALG_SUB_TYPE_CBC_SHA: + case CRYPTO_ALG_SUB_TYPE_CTR_SHA: + case CRYPTO_ALG_SUB_TYPE_CBC_NULL: + case CRYPTO_ALG_SUB_TYPE_CTR_NULL: return chcr_aead_op(req, CHCR_DECRYPT_OP, size, create_authenc_wr); case CRYPTO_ALG_SUB_TYPE_AEAD_CCM: @@ -3748,7 +3794,7 @@ static struct chcr_alg_template driver_algs[] = { } }, { - .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC, + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, .is_registered = 0, .alg.aead = { .base = { @@ -3769,7 +3815,7 @@ static struct chcr_alg_template driver_algs[] = { } }, { - .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC, + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, .is_registered = 0, .alg.aead = { .base = { @@ -3791,7 +3837,7 @@ static struct chcr_alg_template driver_algs[] = { } }, { - .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC, + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, .is_registered = 0, .alg.aead = { .base = { @@ -3811,7 +3857,7 @@ static struct chcr_alg_template driver_algs[] = { } }, { - .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC, + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, .is_registered = 0, .alg.aead = { .base = { @@ -3832,7 +3878,7 @@ static struct chcr_alg_template driver_algs[] = { } }, { - .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC, + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, .is_registered = 0, .alg.aead = { .base = { @@ -3853,7 +3899,7 @@ static struct chcr_alg_template driver_algs[] = { } }, { - .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_NULL, + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_NULL, .is_registered = 0, .alg.aead = { .base = { @@ -3873,6 +3919,133 @@ static struct chcr_alg_template driver_algs[] = { .setauthsize = chcr_authenc_null_setauthsize, } }, + { + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, + .is_registered = 0, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", + .cra_driver_name = + "authenc-hmac-sha1-rfc3686-ctr-aes-chcr", + .cra_blocksize = 1, + .cra_priority = CHCR_AEAD_PRIORITY, + .cra_ctxsize = sizeof(struct chcr_context) + + sizeof(struct chcr_aead_ctx) + + sizeof(struct chcr_authenc_ctx), + + }, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + .setkey = chcr_authenc_setkey, + .setauthsize = chcr_authenc_setauthsize, + } + }, + { + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, + .is_registered = 0, + .alg.aead = { + .base = { + + .cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", + .cra_driver_name = + "authenc-hmac-sha256-rfc3686-ctr-aes-chcr", + .cra_blocksize = 1, + .cra_priority = CHCR_AEAD_PRIORITY, + .cra_ctxsize = sizeof(struct chcr_context) + + sizeof(struct chcr_aead_ctx) + + sizeof(struct chcr_authenc_ctx), + + }, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + .setkey = chcr_authenc_setkey, + .setauthsize = chcr_authenc_setauthsize, + } + }, + { + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, + .is_registered = 0, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))", + .cra_driver_name = + "authenc-hmac-sha224-rfc3686-ctr-aes-chcr", + .cra_blocksize = 1, + .cra_priority = CHCR_AEAD_PRIORITY, + .cra_ctxsize = sizeof(struct chcr_context) + + sizeof(struct chcr_aead_ctx) + + sizeof(struct chcr_authenc_ctx), + }, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + .setkey = chcr_authenc_setkey, + .setauthsize = chcr_authenc_setauthsize, + } + }, + { + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, + .is_registered = 0, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))", + .cra_driver_name = + "authenc-hmac-sha384-rfc3686-ctr-aes-chcr", + .cra_blocksize = 1, + .cra_priority = CHCR_AEAD_PRIORITY, + .cra_ctxsize = sizeof(struct chcr_context) + + sizeof(struct chcr_aead_ctx) + + sizeof(struct chcr_authenc_ctx), + + }, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + .setkey = chcr_authenc_setkey, + .setauthsize = chcr_authenc_setauthsize, + } + }, + { + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, + .is_registered = 0, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))", + .cra_driver_name = + "authenc-hmac-sha512-rfc3686-ctr-aes-chcr", + .cra_blocksize = 1, + .cra_priority = CHCR_AEAD_PRIORITY, + .cra_ctxsize = sizeof(struct chcr_context) + + sizeof(struct chcr_aead_ctx) + + sizeof(struct chcr_authenc_ctx), + + }, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + .setkey = chcr_authenc_setkey, + .setauthsize = chcr_authenc_setauthsize, + } + }, + { + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_NULL, + .is_registered = 0, + .alg.aead = { + .base = { + .cra_name = "authenc(digest_null,rfc3686(ctr(aes)))", + .cra_driver_name = + "authenc-digest_null-rfc3686-ctr-aes-chcr", + .cra_blocksize = 1, + .cra_priority = CHCR_AEAD_PRIORITY, + .cra_ctxsize = sizeof(struct chcr_context) + + sizeof(struct chcr_aead_ctx) + + sizeof(struct chcr_authenc_ctx), + + }, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = 0, + .setkey = chcr_aead_digest_null_setkey, + .setauthsize = chcr_authenc_null_setauthsize, + } + }, + }; /* diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h index ea2c578805e6..7daf0a17a7d2 100644 --- a/drivers/crypto/chelsio/chcr_crypto.h +++ b/drivers/crypto/chelsio/chcr_crypto.h @@ -134,14 +134,16 @@ #define CRYPTO_ALG_SUB_TYPE_HASH_HMAC 0x01000000 #define CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 0x02000000 #define CRYPTO_ALG_SUB_TYPE_AEAD_GCM 0x03000000 -#define CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC 0x04000000 +#define CRYPTO_ALG_SUB_TYPE_CBC_SHA 0x04000000 #define CRYPTO_ALG_SUB_TYPE_AEAD_CCM 0x05000000 #define CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309 0x06000000 -#define CRYPTO_ALG_SUB_TYPE_AEAD_NULL 0x07000000 +#define CRYPTO_ALG_SUB_TYPE_CBC_NULL 0x07000000 #define CRYPTO_ALG_SUB_TYPE_CTR 0x08000000 #define CRYPTO_ALG_SUB_TYPE_CTR_RFC3686 0x09000000 #define CRYPTO_ALG_SUB_TYPE_XTS 0x0a000000 #define CRYPTO_ALG_SUB_TYPE_CBC 0x0b000000 +#define CRYPTO_ALG_SUB_TYPE_CTR_SHA 0x0c000000 +#define CRYPTO_ALG_SUB_TYPE_CTR_NULL 0x0d000000 #define CRYPTO_ALG_TYPE_HMAC (CRYPTO_ALG_TYPE_AHASH |\ CRYPTO_ALG_SUB_TYPE_HASH_HMAC) @@ -231,6 +233,7 @@ struct chcr_aead_ctx { struct crypto_aead *sw_cipher; u8 salt[MAX_SALT]; u8 key[CHCR_AES_MAX_KEY_LEN]; + u8 nonce[4]; u16 hmac_ctrl; u16 mayverify; struct __aead_ctx ctx[0]; From e1a018e607a33dc9f987c761daf1792082fb9ca7 Mon Sep 17 00:00:00 2001 From: Harsh Jain Date: Thu, 11 Jan 2018 16:45:52 +0530 Subject: [PATCH 141/166] crypto: chelsio - Remove dst sg size zero check sg_nents_xlen will take care of zero length sg list. Remove Destination sg list size zero check. Signed-off-by: Harsh Jain Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 43 +++++++++--------------------- 1 file changed, 13 insertions(+), 30 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 5cc84c459526..a9c894bf9c01 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -2109,20 +2109,14 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, null = 1; assoclen = 0; } - dst_size = assoclen + req->cryptlen + (op_type ? -authsize : - authsize); error = chcr_aead_common_init(req, op_type); if (error) return ERR_PTR(error); - if (dst_size) { dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0); dnents += sg_nents_xlen(req->dst, req->cryptlen + (op_type ? -authsize : authsize), CHCR_DST_SG_SIZE, req->assoclen); dnents += MIN_AUTH_SG; // For IV - } else { - dnents = 0; - } dst_size = get_space_for_phys_dsgl(dnents); kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4) @@ -2687,8 +2681,6 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, sub_type = get_aead_subtype(tfm); if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) assoclen -= 8; - dst_size = assoclen + req->cryptlen + (op_type ? -authsize : - authsize); error = chcr_aead_common_init(req, op_type); if (error) return ERR_PTR(error); @@ -2698,15 +2690,11 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, error = aead_ccm_validate_input(op_type, req, aeadctx, sub_type); if (error) goto err; - if (dst_size) { - dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0); - dnents += sg_nents_xlen(req->dst, req->cryptlen - + (op_type ? -authsize : authsize), - CHCR_DST_SG_SIZE, req->assoclen); - dnents += MIN_CCM_SG; // For IV and B0 - } else { - dnents = 0; - } + dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0); + dnents += sg_nents_xlen(req->dst, req->cryptlen + + (op_type ? -authsize : authsize), + CHCR_DST_SG_SIZE, req->assoclen); + dnents += MIN_CCM_SG; // For IV and B0 dst_size = get_space_for_phys_dsgl(dnents); kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) * 2; transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); @@ -2801,19 +2789,14 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, assoclen = req->assoclen - 8; reqctx->b0_dma = 0; - dst_size = assoclen + req->cryptlen + (op_type ? -authsize : authsize); error = chcr_aead_common_init(req, op_type); - if (error) - return ERR_PTR(error); - if (dst_size) { - dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0); - dnents += sg_nents_xlen(req->dst, - req->cryptlen + (op_type ? -authsize : authsize), + if (error) + return ERR_PTR(error); + dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0); + dnents += sg_nents_xlen(req->dst, req->cryptlen + + (op_type ? -authsize : authsize), CHCR_DST_SG_SIZE, req->assoclen); - dnents += MIN_GCM_SG; // For IV - } else { - dnents = 0; - } + dnents += MIN_GCM_SG; // For IV dst_size = get_space_for_phys_dsgl(dnents); kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) + AEAD_H_SIZE; @@ -2850,10 +2833,10 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( assoclen ? 1 : 0, assoclen, assoclen + IV + 1, 0); - chcr_req->sec_cpl.cipherstop_lo_authinsert = + chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0, assoclen + IV + 1, temp, temp); - chcr_req->sec_cpl.seqno_numivs = + chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type == CHCR_ENCRYPT_OP) ? 1 : 0, CHCR_SCMD_CIPHER_MODE_AES_GCM, From cd0bb67f21dadaf0a0240776f6e39a14b4f7625b Mon Sep 17 00:00:00 2001 From: Martin Kaiser Date: Thu, 11 Jan 2018 22:06:39 +0100 Subject: [PATCH 142/166] hwrng: imx-rngc - simplify the power management definitions Use the SIMPLE_DEV_PM_OPS() macro instead of populating a struct dev_pm_ops directly. The suspend and resume functions will now be used for both hibernation and suspend to ram. If power management is disabled, SIMPLE_DEV_PM_OPS() evaluates to nothing, The two functions won't be used and won't be included in the kernel. Mark them as __maybe_unused to clarify that this is intended behaviour. With these modifications in place, we don't need the #ifdefs for power management any more. Signed-off-by: Martin Kaiser Signed-off-by: Herbert Xu --- drivers/char/hw_random/imx-rngc.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/drivers/char/hw_random/imx-rngc.c b/drivers/char/hw_random/imx-rngc.c index 88db42d30760..eca87249bcff 100644 --- a/drivers/char/hw_random/imx-rngc.c +++ b/drivers/char/hw_random/imx-rngc.c @@ -282,8 +282,7 @@ static int __exit imx_rngc_remove(struct platform_device *pdev) return 0; } -#ifdef CONFIG_PM -static int imx_rngc_suspend(struct device *dev) +static int __maybe_unused imx_rngc_suspend(struct device *dev) { struct imx_rngc *rngc = dev_get_drvdata(dev); @@ -292,7 +291,7 @@ static int imx_rngc_suspend(struct device *dev) return 0; } -static int imx_rngc_resume(struct device *dev) +static int __maybe_unused imx_rngc_resume(struct device *dev) { struct imx_rngc *rngc = dev_get_drvdata(dev); @@ -301,11 +300,7 @@ static int imx_rngc_resume(struct device *dev) return 0; } -static const struct dev_pm_ops imx_rngc_pm_ops = { - .suspend = imx_rngc_suspend, - .resume = imx_rngc_resume, -}; -#endif +SIMPLE_DEV_PM_OPS(imx_rngc_pm_ops, imx_rngc_suspend, imx_rngc_resume); static const struct of_device_id imx_rngc_dt_ids[] = { { .compatible = "fsl,imx25-rngb", .data = NULL, }, @@ -316,9 +311,7 @@ MODULE_DEVICE_TABLE(of, imx_rngc_dt_ids); static struct platform_driver imx_rngc_driver = { .driver = { .name = "imx_rngc", -#ifdef CONFIG_PM .pm = &imx_rngc_pm_ops, -#endif .of_match_table = imx_rngc_dt_ids, }, .remove = __exit_p(imx_rngc_remove), From 96a74f7bdc9c1326a06b69797725a285e9282cb3 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 10 Jan 2018 12:11:37 +0000 Subject: [PATCH 143/166] crypto: arm64/aes-cipher - move S-box to .rodata section Move the AES inverse S-box to the .rodata section where it is safe from abuse by speculation. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/aes-cipher-core.S | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/arch/arm64/crypto/aes-cipher-core.S b/arch/arm64/crypto/aes-cipher-core.S index 6d2445d603cc..3a44eada2347 100644 --- a/arch/arm64/crypto/aes-cipher-core.S +++ b/arch/arm64/crypto/aes-cipher-core.S @@ -125,6 +125,16 @@ CPU_BE( rev w7, w7 ) ret .endm +ENTRY(__aes_arm64_encrypt) + do_crypt fround, crypto_ft_tab, crypto_ft_tab + 1, 2 +ENDPROC(__aes_arm64_encrypt) + + .align 5 +ENTRY(__aes_arm64_decrypt) + do_crypt iround, crypto_it_tab, __aes_arm64_inverse_sbox, 0 +ENDPROC(__aes_arm64_decrypt) + + .section ".rodata", "a" .align L1_CACHE_SHIFT .type __aes_arm64_inverse_sbox, %object __aes_arm64_inverse_sbox: @@ -161,12 +171,3 @@ __aes_arm64_inverse_sbox: .byte 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26 .byte 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d .size __aes_arm64_inverse_sbox, . - __aes_arm64_inverse_sbox - -ENTRY(__aes_arm64_encrypt) - do_crypt fround, crypto_ft_tab, crypto_ft_tab + 1, 2 -ENDPROC(__aes_arm64_encrypt) - - .align 5 -ENTRY(__aes_arm64_decrypt) - do_crypt iround, crypto_it_tab, __aes_arm64_inverse_sbox, 0 -ENDPROC(__aes_arm64_decrypt) From 62c24708761aa41fcda5986170276b7aa2669798 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 10 Jan 2018 12:11:38 +0000 Subject: [PATCH 144/166] crypto: arm64/aes-neon - move literal data to .rodata section Move the S-boxes and some other literals to the .rodata section where it is safe from being exploited by speculative execution. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/aes-neon.S | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/arm64/crypto/aes-neon.S b/arch/arm64/crypto/aes-neon.S index f1e3aa2732f9..1c7b45b7268e 100644 --- a/arch/arm64/crypto/aes-neon.S +++ b/arch/arm64/crypto/aes-neon.S @@ -32,10 +32,10 @@ /* preload the entire Sbox */ .macro prepare, sbox, shiftrows, temp - adr \temp, \sbox movi v12.16b, #0x1b - ldr q13, \shiftrows - ldr q14, .Lror32by8 + ldr_l q13, \shiftrows, \temp + ldr_l q14, .Lror32by8, \temp + adr_l \temp, \sbox ld1 {v16.16b-v19.16b}, [\temp], #64 ld1 {v20.16b-v23.16b}, [\temp], #64 ld1 {v24.16b-v27.16b}, [\temp], #64 @@ -272,7 +272,7 @@ #include "aes-modes.S" - .text + .section ".rodata", "a" .align 6 .LForward_Sbox: .byte 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5 From eec7d4593a0c922c2d12649a373825cfa65c17ea Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 10 Jan 2018 12:11:39 +0000 Subject: [PATCH 145/166] crypto: arm64/crc32 - move literal data to .rodata section Move CRC32 literal data to the .rodata section where it is safe from being exploited by speculative execution. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/crc32-ce-core.S | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/arm64/crypto/crc32-ce-core.S b/arch/arm64/crypto/crc32-ce-core.S index 18f5a8442276..16ed3c7ebd37 100644 --- a/arch/arm64/crypto/crc32-ce-core.S +++ b/arch/arm64/crypto/crc32-ce-core.S @@ -50,7 +50,7 @@ #include #include - .text + .section ".rodata", "a" .align 6 .cpu generic+crypto+crc @@ -115,12 +115,13 @@ * uint crc32_pmull_le(unsigned char const *buffer, * size_t len, uint crc32) */ + .text ENTRY(crc32_pmull_le) - adr x3, .Lcrc32_constants + adr_l x3, .Lcrc32_constants b 0f ENTRY(crc32c_pmull_le) - adr x3, .Lcrc32c_constants + adr_l x3, .Lcrc32c_constants 0: bic LEN, LEN, #15 ld1 {v1.16b-v4.16b}, [BUF], #0x40 From 325f562d8f4a2047c26705b9788ce38a099225ed Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 10 Jan 2018 12:11:40 +0000 Subject: [PATCH 146/166] crypto: arm64/crct10dif - move literal data to .rodata section Move the CRC-T10DIF literal data to the .rodata section where it is safe from being exploited by speculative execution. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/crct10dif-ce-core.S | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/arch/arm64/crypto/crct10dif-ce-core.S b/arch/arm64/crypto/crct10dif-ce-core.S index d5b5a8c038c8..f179c01bd55c 100644 --- a/arch/arm64/crypto/crct10dif-ce-core.S +++ b/arch/arm64/crypto/crct10dif-ce-core.S @@ -128,7 +128,7 @@ CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 ) // XOR the initial_crc value eor v0.16b, v0.16b, v10.16b - ldr q10, rk3 // xmm10 has rk3 and rk4 + ldr_l q10, rk3, x8 // xmm10 has rk3 and rk4 // type of pmull instruction // will determine which constant to use @@ -184,13 +184,13 @@ CPU_LE( ext v12.16b, v12.16b, v12.16b, #8 ) // fold the 8 vector registers to 1 vector register with different // constants - ldr q10, rk9 + ldr_l q10, rk9, x8 .macro fold16, reg, rk pmull v8.1q, \reg\().1d, v10.1d pmull2 \reg\().1q, \reg\().2d, v10.2d .ifnb \rk - ldr q10, \rk + ldr_l q10, \rk, x8 .endif eor v7.16b, v7.16b, v8.16b eor v7.16b, v7.16b, \reg\().16b @@ -251,7 +251,7 @@ CPU_LE( ext v1.16b, v1.16b, v1.16b, #8 ) // get rid of the extra data that was loaded before // load the shift constant - adr x4, tbl_shf_table + 16 + adr_l x4, tbl_shf_table + 16 sub x4, x4, arg3 ld1 {v0.16b}, [x4] @@ -275,7 +275,7 @@ CPU_LE( ext v1.16b, v1.16b, v1.16b, #8 ) _128_done: // compute crc of a 128-bit value - ldr q10, rk5 // rk5 and rk6 in xmm10 + ldr_l q10, rk5, x8 // rk5 and rk6 in xmm10 // 64b fold ext v0.16b, vzr.16b, v7.16b, #8 @@ -291,7 +291,7 @@ _128_done: // barrett reduction _barrett: - ldr q10, rk7 + ldr_l q10, rk7, x8 mov v0.d[0], v7.d[1] pmull v0.1q, v0.1d, v10.1d @@ -321,7 +321,7 @@ CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 ) b.eq _128_done // exactly 16 left b.lt _less_than_16_left - ldr q10, rk1 // rk1 and rk2 in xmm10 + ldr_l q10, rk1, x8 // rk1 and rk2 in xmm10 // update the counter. subtract 32 instead of 16 to save one // instruction from the loop @@ -333,7 +333,7 @@ CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 ) _less_than_16_left: // shl r9, 4 - adr x0, tbl_shf_table + 16 + adr_l x0, tbl_shf_table + 16 sub x0, x0, arg3 ld1 {v0.16b}, [x0] movi v9.16b, #0x80 @@ -345,6 +345,7 @@ ENDPROC(crc_t10dif_pmull) // precomputed constants // these constants are precomputed from the poly: // 0x8bb70000 (0x8bb7 scaled to 32 bits) + .section ".rodata", "a" .align 4 // Q = 0x18BB70000 // rk1 = 2^(32*3) mod Q << 32 From 17a0607019d56e136173600372a371c28a6c0076 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 10 Jan 2018 12:11:41 +0000 Subject: [PATCH 147/166] crypto: arm64/sha2-ce - move the round constant table to .rodata section Move the SHA2 round constant table to the .rodata section where it is safe from being exploited by speculative execution. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/sha2-ce-core.S | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/arm64/crypto/sha2-ce-core.S b/arch/arm64/crypto/sha2-ce-core.S index 679c6c002f4f..4c3c89b812ce 100644 --- a/arch/arm64/crypto/sha2-ce-core.S +++ b/arch/arm64/crypto/sha2-ce-core.S @@ -53,6 +53,7 @@ /* * The SHA-256 round constants */ + .section ".rodata", "a" .align 4 .Lsha2_rcon: .word 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5 @@ -76,9 +77,10 @@ * void sha2_ce_transform(struct sha256_ce_state *sst, u8 const *src, * int blocks) */ + .text ENTRY(sha2_ce_transform) /* load round constants */ - adr x8, .Lsha2_rcon + adr_l x8, .Lsha2_rcon ld1 { v0.4s- v3.4s}, [x8], #64 ld1 { v4.4s- v7.4s}, [x8], #64 ld1 { v8.4s-v11.4s}, [x8], #64 From 20b04c02bcb2d1e6a642bbe9e759157bb198499e Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 10 Jan 2018 12:11:42 +0000 Subject: [PATCH 148/166] crypto: arm64/sha1-ce - get rid of literal pool Load the four SHA-1 round constants using immediates rather than literal pool entries, to avoid having executable data that may be exploitable under speculation attacks. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/sha1-ce-core.S | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/arch/arm64/crypto/sha1-ce-core.S b/arch/arm64/crypto/sha1-ce-core.S index 8550408735a0..46049850727d 100644 --- a/arch/arm64/crypto/sha1-ce-core.S +++ b/arch/arm64/crypto/sha1-ce-core.S @@ -58,12 +58,11 @@ sha1su1 v\s0\().4s, v\s3\().4s .endm - /* - * The SHA1 round constants - */ - .align 4 -.Lsha1_rcon: - .word 0x5a827999, 0x6ed9eba1, 0x8f1bbcdc, 0xca62c1d6 + .macro loadrc, k, val, tmp + movz \tmp, :abs_g0_nc:\val + movk \tmp, :abs_g1:\val + dup \k, \tmp + .endm /* * void sha1_ce_transform(struct sha1_ce_state *sst, u8 const *src, @@ -71,11 +70,10 @@ */ ENTRY(sha1_ce_transform) /* load round constants */ - adr x6, .Lsha1_rcon - ld1r {k0.4s}, [x6], #4 - ld1r {k1.4s}, [x6], #4 - ld1r {k2.4s}, [x6], #4 - ld1r {k3.4s}, [x6] + loadrc k0.4s, 0x5a827999, w6 + loadrc k1.4s, 0x6ed9eba1, w6 + loadrc k2.4s, 0x8f1bbcdc, w6 + loadrc k3.4s, 0xca62c1d6, w6 /* load state */ ld1 {dgav.4s}, [x0] From 5abc8db013174ef6022bf7af81adf210c54e4549 Mon Sep 17 00:00:00 2001 From: Harsh Jain Date: Fri, 19 Jan 2018 11:09:58 +0530 Subject: [PATCH 149/166] crypto: chelsio - Fix indentation warning Fix Warning introduced in changeset e1a018e607a3 ("crypto: chelsio - Remove dst sg size zero check") Reported-by: Stephen Rothwell Signed-off-by: Harsh Jain Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index a9c894bf9c01..34a02d690548 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -2112,11 +2112,11 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, error = chcr_aead_common_init(req, op_type); if (error) return ERR_PTR(error); - dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0); - dnents += sg_nents_xlen(req->dst, req->cryptlen + - (op_type ? -authsize : authsize), CHCR_DST_SG_SIZE, - req->assoclen); - dnents += MIN_AUTH_SG; // For IV + dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0); + dnents += sg_nents_xlen(req->dst, req->cryptlen + + (op_type ? -authsize : authsize), CHCR_DST_SG_SIZE, + req->assoclen); + dnents += MIN_AUTH_SG; // For IV dst_size = get_space_for_phys_dsgl(dnents); kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4) From 6e36719fbe90213fbba9f50093fa2d4d69b0e93c Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 15 Jan 2018 17:07:22 +0100 Subject: [PATCH 150/166] crypto: aes-generic - fix aes-generic regression on powerpc My last bugfix added -Os on the command line, which unfortunately caused a build regression on powerpc in some configurations. I've done some more analysis of the original problem and found slightly different workaround that avoids this regression and also results in better performance on gcc-7.0: -fcode-hoisting is an optimization step that got added in gcc-7 and that for all gcc-7 versions causes worse performance. This disables -fcode-hoisting on all compilers that understand the option. For gcc-7.1 and 7.2 I found the same performance as my previous patch (using -Os), in gcc-7.0 it was even better. On gcc-8 I could see no change in performance from this patch. In theory, code hoisting should not be able make things better for the AES cipher, so leaving it disabled for gcc-8 only serves to simplify the Makefile change. Reported-by: kbuild test robot Link: https://www.mail-archive.com/linux-crypto@vger.kernel.org/msg30418.html Link: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=83356 Link: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=83651 Fixes: 148b974deea9 ("crypto: aes-generic - build with -Os on gcc-7+") Signed-off-by: Arnd Bergmann Signed-off-by: Herbert Xu --- crypto/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/Makefile b/crypto/Makefile index daa69360e054..cdbc03b35510 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -99,7 +99,7 @@ obj-$(CONFIG_CRYPTO_TWOFISH_COMMON) += twofish_common.o obj-$(CONFIG_CRYPTO_SERPENT) += serpent_generic.o CFLAGS_serpent_generic.o := $(call cc-option,-fsched-pressure) # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=79149 obj-$(CONFIG_CRYPTO_AES) += aes_generic.o -CFLAGS_aes_generic.o := $(call cc-ifversion, -ge, 0701, -Os) # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=83356 +CFLAGS_aes_generic.o := $(call cc-option,-fno-code-hoisting) # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=83356 obj-$(CONFIG_CRYPTO_AES_TI) += aes_ti.o obj-$(CONFIG_CRYPTO_CAMELLIA) += camellia_generic.o obj-$(CONFIG_CRYPTO_CAST_COMMON) += cast_common.o From 059bfd1171069025dc134a95f7fcce27483fba41 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Tue, 16 Jan 2018 08:41:58 +0100 Subject: [PATCH 151/166] crypto: inside-secure - make function safexcel_try_push_requests static The function safexcel_try_push_requests is local to the source and does not need to be in global scope, so make it static. Cleans up sparse warning: symbol 'safexcel_try_push_requests' was not declared. Should it be static? Signed-off-by: Colin Ian King [Antoine: fixed alignment] Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 0d685be75241..225e74a7f724 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -432,8 +432,8 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv) } /* Called with ring's lock taken */ -int safexcel_try_push_requests(struct safexcel_crypto_priv *priv, int ring, - int reqs) +static int safexcel_try_push_requests(struct safexcel_crypto_priv *priv, + int ring, int reqs) { int coal = min_t(int, reqs, EIP197_MAX_BATCH_SZ); From 466d7b9f6175e4ecd409f619ff9fbbd49467ad66 Mon Sep 17 00:00:00 2001 From: Kamil Konieczny Date: Tue, 16 Jan 2018 15:26:13 +0100 Subject: [PATCH 152/166] crypto: testmgr - test misuse of result in ahash Async hash operations can use result pointer in final/finup/digest, but not in init/update/export/import, so test it for misuse. Signed-off-by: Kamil Konieczny Signed-off-by: Herbert Xu --- crypto/testmgr.c | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 44a85d4b3561..d5e23a142a04 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -177,6 +177,18 @@ static void testmgr_free_buf(char *buf[XBUFSIZE]) free_page((unsigned long)buf[i]); } +static int ahash_guard_result(char *result, char c, int size) +{ + int i; + + for (i = 0; i < size; i++) { + if (result[i] != c) + return -EINVAL; + } + + return 0; +} + static int ahash_partial_update(struct ahash_request **preq, struct crypto_ahash *tfm, const struct hash_testvec *template, void *hash_buff, int k, int temp, struct scatterlist *sg, @@ -186,6 +198,7 @@ static int ahash_partial_update(struct ahash_request **preq, struct ahash_request *req; int statesize, ret = -EINVAL; static const unsigned char guard[] = { 0x00, 0xba, 0xad, 0x00 }; + int digestsize = crypto_ahash_digestsize(tfm); req = *preq; statesize = crypto_ahash_statesize( @@ -196,12 +209,19 @@ static int ahash_partial_update(struct ahash_request **preq, goto out_nostate; } memcpy(state + statesize, guard, sizeof(guard)); + memset(result, 1, digestsize); ret = crypto_ahash_export(req, state); WARN_ON(memcmp(state + statesize, guard, sizeof(guard))); if (ret) { pr_err("alg: hash: Failed to export() for %s\n", algo); goto out; } + ret = ahash_guard_result(result, 1, digestsize); + if (ret) { + pr_err("alg: hash: Failed, export used req->result for %s\n", + algo); + goto out; + } ahash_request_free(req); req = ahash_request_alloc(tfm, GFP_KERNEL); if (!req) { @@ -221,6 +241,12 @@ static int ahash_partial_update(struct ahash_request **preq, pr_err("alg: hash: Failed to import() for %s\n", algo); goto out; } + ret = ahash_guard_result(result, 1, digestsize); + if (ret) { + pr_err("alg: hash: Failed, import used req->result for %s\n", + algo); + goto out; + } ret = crypto_wait_req(crypto_ahash_update(req), wait); if (ret) goto out; @@ -316,18 +342,31 @@ static int __test_hash(struct crypto_ahash *tfm, goto out; } } else { + memset(result, 1, digest_size); ret = crypto_wait_req(crypto_ahash_init(req), &wait); if (ret) { pr_err("alg: hash: init failed on test %d " "for %s: ret=%d\n", j, algo, -ret); goto out; } + ret = ahash_guard_result(result, 1, digest_size); + if (ret) { + pr_err("alg: hash: init failed on test %d " + "for %s: used req->result\n", j, algo); + goto out; + } ret = crypto_wait_req(crypto_ahash_update(req), &wait); if (ret) { pr_err("alg: hash: update failed on test %d " "for %s: ret=%d\n", j, algo, -ret); goto out; } + ret = ahash_guard_result(result, 1, digest_size); + if (ret) { + pr_err("alg: hash: update failed on test %d " + "for %s: used req->result\n", j, algo); + goto out; + } ret = crypto_wait_req(crypto_ahash_final(req), &wait); if (ret) { pr_err("alg: hash: final failed on test %d " From 235699e2a99fe40c11a377942e60bad11ce4046e Mon Sep 17 00:00:00 2001 From: "weiyongjun \\(A\\)" Date: Wed, 17 Jan 2018 11:26:47 +0000 Subject: [PATCH 153/166] crypto: axis - remove unnecessary platform_get_resource() error check devm_ioremap_resource() already checks if the resource is NULL, so remove the unnecessary platform_get_resource() error check. Signed-off-by: Wei Yongjun Signed-off-by: Herbert Xu --- drivers/crypto/axis/artpec6_crypto.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c index 22df6b55e172..0fb8bbf41a8d 100644 --- a/drivers/crypto/axis/artpec6_crypto.c +++ b/drivers/crypto/axis/artpec6_crypto.c @@ -3042,9 +3042,6 @@ static int artpec6_crypto_probe(struct platform_device *pdev) variant = (enum artpec6_crypto_variant)match->data; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!res) - return -ENODEV; - base = devm_ioremap_resource(&pdev->dev, res); if (IS_ERR(base)) return PTR_ERR(base); From 60bcf2652ecab0917c6db0bdaa0de362327fa358 Mon Sep 17 00:00:00 2001 From: "weiyongjun \\(A\\)" Date: Wed, 17 Jan 2018 11:40:24 +0000 Subject: [PATCH 154/166] crypto: stm32 - remove redundant dev_err call in stm32_cryp_probe() There is a error message within devm_ioremap_resource already, so remove the dev_err call to avoid redundant error message. Signed-off-by: Wei Yongjun Reviewed-by: Fabien Dessenne Signed-off-by: Herbert Xu --- drivers/crypto/stm32/stm32-cryp.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c index cf1dddbeaa2c..4a06a7a665ee 100644 --- a/drivers/crypto/stm32/stm32-cryp.c +++ b/drivers/crypto/stm32/stm32-cryp.c @@ -1052,10 +1052,8 @@ static int stm32_cryp_probe(struct platform_device *pdev) res = platform_get_resource(pdev, IORESOURCE_MEM, 0); cryp->regs = devm_ioremap_resource(dev, res); - if (IS_ERR(cryp->regs)) { - dev_err(dev, "Cannot map CRYP IO\n"); + if (IS_ERR(cryp->regs)) return PTR_ERR(cryp->regs); - } irq = platform_get_irq(pdev, 0); if (irq < 0) { From c4fc5d6d0b3750a17e3ee032f2c47620bed4e25a Mon Sep 17 00:00:00 2001 From: "weiyongjun \\(A\\)" Date: Wed, 17 Jan 2018 11:40:34 +0000 Subject: [PATCH 155/166] hwrng: bcm2835 - Remove redundant dev_err call in bcm2835_rng_probe() There is a error message within devm_ioremap_resource already, so remove the dev_err call to avoid redundant error message. Signed-off-by: Wei Yongjun Reviewed-by: Eric Anholt Acked-by: Florian Fainelli Signed-off-by: Herbert Xu --- drivers/char/hw_random/bcm2835-rng.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/char/hw_random/bcm2835-rng.c b/drivers/char/hw_random/bcm2835-rng.c index 25e56311a197..7a84cec30c3a 100644 --- a/drivers/char/hw_random/bcm2835-rng.c +++ b/drivers/char/hw_random/bcm2835-rng.c @@ -158,10 +158,8 @@ static int bcm2835_rng_probe(struct platform_device *pdev) /* map peripheral */ priv->base = devm_ioremap_resource(dev, r); - if (IS_ERR(priv->base)) { - dev_err(dev, "failed to remap rng regs"); + if (IS_ERR(priv->base)) return PTR_ERR(priv->base); - } /* Clock is optional on most platforms */ priv->clk = devm_clk_get(dev, NULL); From 980b4c95e78e4113cb7b9f430f121dab1c814b6c Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Wed, 17 Jan 2018 19:50:56 +0100 Subject: [PATCH 156/166] crypto: artpec6 - remove select on non-existing CRYPTO_SHA384 Since CRYPTO_SHA384 does not exists, Kconfig should not select it. Anyway, all SHA384 stuff is in CRYPTO_SHA512 which is already selected. Fixes: a21eb94fc4d3i ("crypto: axis - add ARTPEC-6/7 crypto accelerator driver") Signed-off-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 47ec920d5b71..4b741b83e23f 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -723,7 +723,6 @@ config CRYPTO_DEV_ARTPEC6 select CRYPTO_HASH select CRYPTO_SHA1 select CRYPTO_SHA256 - select CRYPTO_SHA384 select CRYPTO_SHA512 help Enables the driver for the on-chip crypto accelerator From 9c674e1e2f9e24fa4392167efe343749008338e0 Mon Sep 17 00:00:00 2001 From: Stephan Mueller Date: Thu, 18 Jan 2018 20:41:09 +0100 Subject: [PATCH 157/166] crypto: aesni - handle zero length dst buffer GCM can be invoked with a zero destination buffer. This is possible if the AAD and the ciphertext have zero lengths and only the tag exists in the source buffer (i.e. a source buffer cannot be zero). In this case, the GCM cipher only performs the authentication and no decryption operation. When the destination buffer has zero length, it is possible that no page is mapped to the SG pointing to the destination. In this case, sg_page(req->dst) is an invalid access. Therefore, page accesses should only be allowed if the req->dst->length is non-zero which is the indicator that a page must exist. This fixes a crash that can be triggered by user space via AF_ALG. CC: Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- arch/x86/crypto/aesni-intel_glue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index a5ee78d723cd..34cf1c1f8c98 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -824,7 +824,7 @@ static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen, if (sg_is_last(req->src) && (!PageHighMem(sg_page(req->src)) || req->src->offset + req->src->length <= PAGE_SIZE) && - sg_is_last(req->dst) && + sg_is_last(req->dst) && req->dst->length && (!PageHighMem(sg_page(req->dst)) || req->dst->offset + req->dst->length <= PAGE_SIZE)) { one_entry_in_sg = 1; From c013cee99d5a18aec8c71fee8f5f41369cd12595 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 19 Jan 2018 12:04:33 +0000 Subject: [PATCH 158/166] crypto: sha3-generic - fixes for alignment and big endian operation Ensure that the input is byte swabbed before injecting it into the SHA3 transform. Use the get_unaligned() accessor for this so that we don't perform unaligned access inadvertently on architectures that do not support that. Cc: Fixes: 53964b9ee63b7075 ("crypto: sha3 - Add SHA-3 hash algorithm") Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/sha3_generic.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crypto/sha3_generic.c b/crypto/sha3_generic.c index 7e8ed96236ce..a68be626017c 100644 --- a/crypto/sha3_generic.c +++ b/crypto/sha3_generic.c @@ -18,6 +18,7 @@ #include #include #include +#include #define KECCAK_ROUNDS 24 @@ -149,7 +150,7 @@ static int sha3_update(struct shash_desc *desc, const u8 *data, unsigned int i; for (i = 0; i < sctx->rsizw; i++) - sctx->st[i] ^= ((u64 *) src)[i]; + sctx->st[i] ^= get_unaligned_le64(src + 8 * i); keccakf(sctx->st); done += sctx->rsiz; @@ -174,7 +175,7 @@ static int sha3_final(struct shash_desc *desc, u8 *out) sctx->buf[sctx->rsiz - 1] |= 0x80; for (i = 0; i < sctx->rsizw; i++) - sctx->st[i] ^= ((u64 *) sctx->buf)[i]; + sctx->st[i] ^= get_unaligned_le64(sctx->buf + 8 * i); keccakf(sctx->st); From 83dee2ce1ae791c3dc0c9d4d3a8d42cb109613f6 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 19 Jan 2018 12:04:34 +0000 Subject: [PATCH 159/166] crypto: sha3-generic - rewrite KECCAK transform to help the compiler optimize The way the KECCAK transform is currently coded involves many references into the state array using indexes that are calculated at runtime using simple but non-trivial arithmetic. This forces the compiler to treat the state matrix as an array in memory rather than keep it in registers, which results in poor performance. So instead, let's rephrase the algorithm using fixed array indexes only. This helps the compiler keep the state matrix in registers, resulting in the following speedup (SHA3-256 performance in cycles per byte): before after speedup Intel Core i7 @ 2.0 GHz (2.9 turbo) 100.6 35.7 2.8x Cortex-A57 @ 2.0 GHz (64-bit mode) 101.6 12.7 8.0x Cortex-A53 @ 1.0 GHz 224.4 15.8 14.2x Cortex-A57 @ 2.0 GHz (32-bit mode) 201.8 63.0 3.2x Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/sha3_generic.c | 132 ++++++++++++++++++++++++++++++------------ 1 file changed, 95 insertions(+), 37 deletions(-) diff --git a/crypto/sha3_generic.c b/crypto/sha3_generic.c index a68be626017c..5fecb609e3be 100644 --- a/crypto/sha3_generic.c +++ b/crypto/sha3_generic.c @@ -5,6 +5,7 @@ * http://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.202.pdf * * SHA-3 code by Jeff Garzik + * Ard Biesheuvel * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free @@ -22,8 +23,6 @@ #define KECCAK_ROUNDS 24 -#define ROTL64(x, y) (((x) << (y)) | ((x) >> (64 - (y)))) - static const u64 keccakf_rndc[24] = { 0x0000000000000001ULL, 0x0000000000008082ULL, 0x800000000000808aULL, 0x8000000080008000ULL, 0x000000000000808bULL, 0x0000000080000001ULL, @@ -35,53 +34,112 @@ static const u64 keccakf_rndc[24] = { 0x8000000000008080ULL, 0x0000000080000001ULL, 0x8000000080008008ULL }; -static const int keccakf_rotc[24] = { - 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 2, 14, - 27, 41, 56, 8, 25, 43, 62, 18, 39, 61, 20, 44 -}; - -static const int keccakf_piln[24] = { - 10, 7, 11, 17, 18, 3, 5, 16, 8, 21, 24, 4, - 15, 23, 19, 13, 12, 2, 20, 14, 22, 9, 6, 1 -}; - /* update the state with given number of rounds */ -static void keccakf(u64 st[25]) +static void __attribute__((__optimize__("O3"))) keccakf(u64 st[25]) { - int i, j, round; - u64 t, bc[5]; + u64 t[5], tt, bc[5]; + int round; for (round = 0; round < KECCAK_ROUNDS; round++) { /* Theta */ - for (i = 0; i < 5; i++) - bc[i] = st[i] ^ st[i + 5] ^ st[i + 10] ^ st[i + 15] - ^ st[i + 20]; + bc[0] = st[0] ^ st[5] ^ st[10] ^ st[15] ^ st[20]; + bc[1] = st[1] ^ st[6] ^ st[11] ^ st[16] ^ st[21]; + bc[2] = st[2] ^ st[7] ^ st[12] ^ st[17] ^ st[22]; + bc[3] = st[3] ^ st[8] ^ st[13] ^ st[18] ^ st[23]; + bc[4] = st[4] ^ st[9] ^ st[14] ^ st[19] ^ st[24]; - for (i = 0; i < 5; i++) { - t = bc[(i + 4) % 5] ^ ROTL64(bc[(i + 1) % 5], 1); - for (j = 0; j < 25; j += 5) - st[j + i] ^= t; - } + t[0] = bc[4] ^ rol64(bc[1], 1); + t[1] = bc[0] ^ rol64(bc[2], 1); + t[2] = bc[1] ^ rol64(bc[3], 1); + t[3] = bc[2] ^ rol64(bc[4], 1); + t[4] = bc[3] ^ rol64(bc[0], 1); + + st[0] ^= t[0]; /* Rho Pi */ - t = st[1]; - for (i = 0; i < 24; i++) { - j = keccakf_piln[i]; - bc[0] = st[j]; - st[j] = ROTL64(t, keccakf_rotc[i]); - t = bc[0]; - } + tt = st[1]; + st[ 1] = rol64(st[ 6] ^ t[1], 44); + st[ 6] = rol64(st[ 9] ^ t[4], 20); + st[ 9] = rol64(st[22] ^ t[2], 61); + st[22] = rol64(st[14] ^ t[4], 39); + st[14] = rol64(st[20] ^ t[0], 18); + st[20] = rol64(st[ 2] ^ t[2], 62); + st[ 2] = rol64(st[12] ^ t[2], 43); + st[12] = rol64(st[13] ^ t[3], 25); + st[13] = rol64(st[19] ^ t[4], 8); + st[19] = rol64(st[23] ^ t[3], 56); + st[23] = rol64(st[15] ^ t[0], 41); + st[15] = rol64(st[ 4] ^ t[4], 27); + st[ 4] = rol64(st[24] ^ t[4], 14); + st[24] = rol64(st[21] ^ t[1], 2); + st[21] = rol64(st[ 8] ^ t[3], 55); + st[ 8] = rol64(st[16] ^ t[1], 45); + st[16] = rol64(st[ 5] ^ t[0], 36); + st[ 5] = rol64(st[ 3] ^ t[3], 28); + st[ 3] = rol64(st[18] ^ t[3], 21); + st[18] = rol64(st[17] ^ t[2], 15); + st[17] = rol64(st[11] ^ t[1], 10); + st[11] = rol64(st[ 7] ^ t[2], 6); + st[ 7] = rol64(st[10] ^ t[0], 3); + st[10] = rol64( tt ^ t[1], 1); /* Chi */ - for (j = 0; j < 25; j += 5) { - for (i = 0; i < 5; i++) - bc[i] = st[j + i]; - for (i = 0; i < 5; i++) - st[j + i] ^= (~bc[(i + 1) % 5]) & - bc[(i + 2) % 5]; - } + bc[ 0] = ~st[ 1] & st[ 2]; + bc[ 1] = ~st[ 2] & st[ 3]; + bc[ 2] = ~st[ 3] & st[ 4]; + bc[ 3] = ~st[ 4] & st[ 0]; + bc[ 4] = ~st[ 0] & st[ 1]; + st[ 0] ^= bc[ 0]; + st[ 1] ^= bc[ 1]; + st[ 2] ^= bc[ 2]; + st[ 3] ^= bc[ 3]; + st[ 4] ^= bc[ 4]; + + bc[ 0] = ~st[ 6] & st[ 7]; + bc[ 1] = ~st[ 7] & st[ 8]; + bc[ 2] = ~st[ 8] & st[ 9]; + bc[ 3] = ~st[ 9] & st[ 5]; + bc[ 4] = ~st[ 5] & st[ 6]; + st[ 5] ^= bc[ 0]; + st[ 6] ^= bc[ 1]; + st[ 7] ^= bc[ 2]; + st[ 8] ^= bc[ 3]; + st[ 9] ^= bc[ 4]; + + bc[ 0] = ~st[11] & st[12]; + bc[ 1] = ~st[12] & st[13]; + bc[ 2] = ~st[13] & st[14]; + bc[ 3] = ~st[14] & st[10]; + bc[ 4] = ~st[10] & st[11]; + st[10] ^= bc[ 0]; + st[11] ^= bc[ 1]; + st[12] ^= bc[ 2]; + st[13] ^= bc[ 3]; + st[14] ^= bc[ 4]; + + bc[ 0] = ~st[16] & st[17]; + bc[ 1] = ~st[17] & st[18]; + bc[ 2] = ~st[18] & st[19]; + bc[ 3] = ~st[19] & st[15]; + bc[ 4] = ~st[15] & st[16]; + st[15] ^= bc[ 0]; + st[16] ^= bc[ 1]; + st[17] ^= bc[ 2]; + st[18] ^= bc[ 3]; + st[19] ^= bc[ 4]; + + bc[ 0] = ~st[21] & st[22]; + bc[ 1] = ~st[22] & st[23]; + bc[ 2] = ~st[23] & st[24]; + bc[ 3] = ~st[24] & st[20]; + bc[ 4] = ~st[20] & st[21]; + st[20] ^= bc[ 0]; + st[21] ^= bc[ 1]; + st[22] ^= bc[ 2]; + st[23] ^= bc[ 3]; + st[24] ^= bc[ 4]; /* Iota */ st[0] ^= keccakf_rndc[round]; From beeb504adf3d08c0e916f43259e8e2ad6bdd30ee Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 19 Jan 2018 12:04:35 +0000 Subject: [PATCH 160/166] crypto: sha3-generic - simplify code In preparation of exposing the generic SHA3 implementation to other versions as a fallback, simplify the code, and remove an inconsistency in the output handling (endian swabbing rsizw words of state before writing the output does not make sense) Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/sha3_generic.c | 186 ++++++++++++++---------------------------- include/crypto/sha3.h | 1 - 2 files changed, 60 insertions(+), 127 deletions(-) diff --git a/crypto/sha3_generic.c b/crypto/sha3_generic.c index 5fecb609e3be..c7084a24eaf9 100644 --- a/crypto/sha3_generic.c +++ b/crypto/sha3_generic.c @@ -18,7 +18,6 @@ #include #include #include -#include #include #define KECCAK_ROUNDS 24 @@ -146,43 +145,16 @@ static void __attribute__((__optimize__("O3"))) keccakf(u64 st[25]) } } -static void sha3_init(struct sha3_state *sctx, unsigned int digest_sz) +static int sha3_init(struct shash_desc *desc) { - memset(sctx, 0, sizeof(*sctx)); - sctx->md_len = digest_sz; - sctx->rsiz = 200 - 2 * digest_sz; + struct sha3_state *sctx = shash_desc_ctx(desc); + unsigned int digest_size = crypto_shash_digestsize(desc->tfm); + + sctx->rsiz = 200 - 2 * digest_size; sctx->rsizw = sctx->rsiz / 8; -} + sctx->partial = 0; -static int sha3_224_init(struct shash_desc *desc) -{ - struct sha3_state *sctx = shash_desc_ctx(desc); - - sha3_init(sctx, SHA3_224_DIGEST_SIZE); - return 0; -} - -static int sha3_256_init(struct shash_desc *desc) -{ - struct sha3_state *sctx = shash_desc_ctx(desc); - - sha3_init(sctx, SHA3_256_DIGEST_SIZE); - return 0; -} - -static int sha3_384_init(struct shash_desc *desc) -{ - struct sha3_state *sctx = shash_desc_ctx(desc); - - sha3_init(sctx, SHA3_384_DIGEST_SIZE); - return 0; -} - -static int sha3_512_init(struct shash_desc *desc) -{ - struct sha3_state *sctx = shash_desc_ctx(desc); - - sha3_init(sctx, SHA3_512_DIGEST_SIZE); + memset(sctx->st, 0, sizeof(sctx->st)); return 0; } @@ -227,6 +199,8 @@ static int sha3_final(struct shash_desc *desc, u8 *out) { struct sha3_state *sctx = shash_desc_ctx(desc); unsigned int i, inlen = sctx->partial; + unsigned int digest_size = crypto_shash_digestsize(desc->tfm); + __le64 *digest = (__le64 *)out; sctx->buf[inlen++] = 0x06; memset(sctx->buf + inlen, 0, sctx->rsiz - inlen); @@ -237,110 +211,70 @@ static int sha3_final(struct shash_desc *desc, u8 *out) keccakf(sctx->st); - for (i = 0; i < sctx->rsizw; i++) - sctx->st[i] = cpu_to_le64(sctx->st[i]); + for (i = 0; i < digest_size / 8; i++) + put_unaligned_le64(sctx->st[i], digest++); - memcpy(out, sctx->st, sctx->md_len); + if (digest_size & 4) + put_unaligned_le32(sctx->st[i], (__le32 *)digest); memset(sctx, 0, sizeof(*sctx)); return 0; } -static struct shash_alg sha3_224 = { - .digestsize = SHA3_224_DIGEST_SIZE, - .init = sha3_224_init, - .update = sha3_update, - .final = sha3_final, - .descsize = sizeof(struct sha3_state), - .base = { - .cra_name = "sha3-224", - .cra_driver_name = "sha3-224-generic", - .cra_flags = CRYPTO_ALG_TYPE_SHASH, - .cra_blocksize = SHA3_224_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static struct shash_alg sha3_256 = { - .digestsize = SHA3_256_DIGEST_SIZE, - .init = sha3_256_init, - .update = sha3_update, - .final = sha3_final, - .descsize = sizeof(struct sha3_state), - .base = { - .cra_name = "sha3-256", - .cra_driver_name = "sha3-256-generic", - .cra_flags = CRYPTO_ALG_TYPE_SHASH, - .cra_blocksize = SHA3_256_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static struct shash_alg sha3_384 = { - .digestsize = SHA3_384_DIGEST_SIZE, - .init = sha3_384_init, - .update = sha3_update, - .final = sha3_final, - .descsize = sizeof(struct sha3_state), - .base = { - .cra_name = "sha3-384", - .cra_driver_name = "sha3-384-generic", - .cra_flags = CRYPTO_ALG_TYPE_SHASH, - .cra_blocksize = SHA3_384_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static struct shash_alg sha3_512 = { - .digestsize = SHA3_512_DIGEST_SIZE, - .init = sha3_512_init, - .update = sha3_update, - .final = sha3_final, - .descsize = sizeof(struct sha3_state), - .base = { - .cra_name = "sha3-512", - .cra_driver_name = "sha3-512-generic", - .cra_flags = CRYPTO_ALG_TYPE_SHASH, - .cra_blocksize = SHA3_512_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; +static struct shash_alg algs[] = { { + .digestsize = SHA3_224_DIGEST_SIZE, + .init = sha3_init, + .update = sha3_update, + .final = sha3_final, + .descsize = sizeof(struct sha3_state), + .base.cra_name = "sha3-224", + .base.cra_driver_name = "sha3-224-generic", + .base.cra_flags = CRYPTO_ALG_TYPE_SHASH, + .base.cra_blocksize = SHA3_224_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, +}, { + .digestsize = SHA3_256_DIGEST_SIZE, + .init = sha3_init, + .update = sha3_update, + .final = sha3_final, + .descsize = sizeof(struct sha3_state), + .base.cra_name = "sha3-256", + .base.cra_driver_name = "sha3-256-generic", + .base.cra_flags = CRYPTO_ALG_TYPE_SHASH, + .base.cra_blocksize = SHA3_256_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, +}, { + .digestsize = SHA3_384_DIGEST_SIZE, + .init = sha3_init, + .update = sha3_update, + .final = sha3_final, + .descsize = sizeof(struct sha3_state), + .base.cra_name = "sha3-384", + .base.cra_driver_name = "sha3-384-generic", + .base.cra_flags = CRYPTO_ALG_TYPE_SHASH, + .base.cra_blocksize = SHA3_384_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, +}, { + .digestsize = SHA3_512_DIGEST_SIZE, + .init = sha3_init, + .update = sha3_update, + .final = sha3_final, + .descsize = sizeof(struct sha3_state), + .base.cra_name = "sha3-512", + .base.cra_driver_name = "sha3-512-generic", + .base.cra_flags = CRYPTO_ALG_TYPE_SHASH, + .base.cra_blocksize = SHA3_512_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, +} }; static int __init sha3_generic_mod_init(void) { - int ret; - - ret = crypto_register_shash(&sha3_224); - if (ret < 0) - goto err_out; - ret = crypto_register_shash(&sha3_256); - if (ret < 0) - goto err_out_224; - ret = crypto_register_shash(&sha3_384); - if (ret < 0) - goto err_out_256; - ret = crypto_register_shash(&sha3_512); - if (ret < 0) - goto err_out_384; - - return 0; - -err_out_384: - crypto_unregister_shash(&sha3_384); -err_out_256: - crypto_unregister_shash(&sha3_256); -err_out_224: - crypto_unregister_shash(&sha3_224); -err_out: - return ret; + return crypto_register_shashes(algs, ARRAY_SIZE(algs)); } static void __exit sha3_generic_mod_fini(void) { - crypto_unregister_shash(&sha3_224); - crypto_unregister_shash(&sha3_256); - crypto_unregister_shash(&sha3_384); - crypto_unregister_shash(&sha3_512); + crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); } module_init(sha3_generic_mod_init); diff --git a/include/crypto/sha3.h b/include/crypto/sha3.h index b9d9bd553b48..1339dcdbc9b2 100644 --- a/include/crypto/sha3.h +++ b/include/crypto/sha3.h @@ -19,7 +19,6 @@ struct sha3_state { u64 st[25]; - unsigned int md_len; unsigned int rsiz; unsigned int rsizw; From 6657674b23b8a8458a3222ec3da2fd376c78ae79 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 19 Jan 2018 12:04:36 +0000 Subject: [PATCH 161/166] crypto: sha3-generic - export init/update/final routines To allow accelerated implementations to fall back to the generic routines, e.g., in contexts where a SIMD based implementation is not allowed to run, expose the generic SHA3 init/update/final routines to other modules. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/sha3_generic.c | 33 ++++++++++++++++++--------------- include/crypto/sha3.h | 5 +++++ 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/crypto/sha3_generic.c b/crypto/sha3_generic.c index c7084a24eaf9..a965b9d80559 100644 --- a/crypto/sha3_generic.c +++ b/crypto/sha3_generic.c @@ -145,7 +145,7 @@ static void __attribute__((__optimize__("O3"))) keccakf(u64 st[25]) } } -static int sha3_init(struct shash_desc *desc) +int crypto_sha3_init(struct shash_desc *desc) { struct sha3_state *sctx = shash_desc_ctx(desc); unsigned int digest_size = crypto_shash_digestsize(desc->tfm); @@ -157,8 +157,9 @@ static int sha3_init(struct shash_desc *desc) memset(sctx->st, 0, sizeof(sctx->st)); return 0; } +EXPORT_SYMBOL(crypto_sha3_init); -static int sha3_update(struct shash_desc *desc, const u8 *data, +int crypto_sha3_update(struct shash_desc *desc, const u8 *data, unsigned int len) { struct sha3_state *sctx = shash_desc_ctx(desc); @@ -194,8 +195,9 @@ static int sha3_update(struct shash_desc *desc, const u8 *data, return 0; } +EXPORT_SYMBOL(crypto_sha3_update); -static int sha3_final(struct shash_desc *desc, u8 *out) +int crypto_sha3_final(struct shash_desc *desc, u8 *out) { struct sha3_state *sctx = shash_desc_ctx(desc); unsigned int i, inlen = sctx->partial; @@ -220,12 +222,13 @@ static int sha3_final(struct shash_desc *desc, u8 *out) memset(sctx, 0, sizeof(*sctx)); return 0; } +EXPORT_SYMBOL(crypto_sha3_final); static struct shash_alg algs[] = { { .digestsize = SHA3_224_DIGEST_SIZE, - .init = sha3_init, - .update = sha3_update, - .final = sha3_final, + .init = crypto_sha3_init, + .update = crypto_sha3_update, + .final = crypto_sha3_final, .descsize = sizeof(struct sha3_state), .base.cra_name = "sha3-224", .base.cra_driver_name = "sha3-224-generic", @@ -234,9 +237,9 @@ static struct shash_alg algs[] = { { .base.cra_module = THIS_MODULE, }, { .digestsize = SHA3_256_DIGEST_SIZE, - .init = sha3_init, - .update = sha3_update, - .final = sha3_final, + .init = crypto_sha3_init, + .update = crypto_sha3_update, + .final = crypto_sha3_final, .descsize = sizeof(struct sha3_state), .base.cra_name = "sha3-256", .base.cra_driver_name = "sha3-256-generic", @@ -245,9 +248,9 @@ static struct shash_alg algs[] = { { .base.cra_module = THIS_MODULE, }, { .digestsize = SHA3_384_DIGEST_SIZE, - .init = sha3_init, - .update = sha3_update, - .final = sha3_final, + .init = crypto_sha3_init, + .update = crypto_sha3_update, + .final = crypto_sha3_final, .descsize = sizeof(struct sha3_state), .base.cra_name = "sha3-384", .base.cra_driver_name = "sha3-384-generic", @@ -256,9 +259,9 @@ static struct shash_alg algs[] = { { .base.cra_module = THIS_MODULE, }, { .digestsize = SHA3_512_DIGEST_SIZE, - .init = sha3_init, - .update = sha3_update, - .final = sha3_final, + .init = crypto_sha3_init, + .update = crypto_sha3_update, + .final = crypto_sha3_final, .descsize = sizeof(struct sha3_state), .base.cra_name = "sha3-512", .base.cra_driver_name = "sha3-512-generic", diff --git a/include/crypto/sha3.h b/include/crypto/sha3.h index 1339dcdbc9b2..080f60c2e6b1 100644 --- a/include/crypto/sha3.h +++ b/include/crypto/sha3.h @@ -26,4 +26,9 @@ struct sha3_state { u8 buf[SHA3_224_BLOCK_SIZE]; }; +int crypto_sha3_init(struct shash_desc *desc); +int crypto_sha3_update(struct shash_desc *desc, const u8 *data, + unsigned int len); +int crypto_sha3_final(struct shash_desc *desc, u8 *out); + #endif From d60031dd59dbdb01bb4faca20957d1efd5b3b7b9 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 19 Jan 2018 12:04:37 +0000 Subject: [PATCH 162/166] crypto: testmgr - add new testcases for sha3 All current SHA3 test cases are smaller than the SHA3 block size, which means not all code paths are being exercised. So add a new test case to each variant, and make one of the existing test cases chunked. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/testmgr.h | 550 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 550 insertions(+) diff --git a/crypto/testmgr.h b/crypto/testmgr.h index a714b6293959..6044f6906bd6 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -1052,6 +1052,142 @@ static const struct hash_testvec sha3_224_tv_template[] = { "\xc9\xfd\x55\x74\x49\x44\x79\xba" "\x5c\x7e\x7a\xb7\x6e\xf2\x64\xea" "\xd0\xfc\xce\x33", + .np = 2, + .tap = { 28, 28 }, + }, { + .plaintext = "\x08\x9f\x13\xaa\x41\xd8\x4c\xe3" + "\x7a\x11\x85\x1c\xb3\x27\xbe\x55" + "\xec\x60\xf7\x8e\x02\x99\x30\xc7" + "\x3b\xd2\x69\x00\x74\x0b\xa2\x16" + "\xad\x44\xdb\x4f\xe6\x7d\x14\x88" + "\x1f\xb6\x2a\xc1\x58\xef\x63\xfa" + "\x91\x05\x9c\x33\xca\x3e\xd5\x6c" + "\x03\x77\x0e\xa5\x19\xb0\x47\xde" + "\x52\xe9\x80\x17\x8b\x22\xb9\x2d" + "\xc4\x5b\xf2\x66\xfd\x94\x08\x9f" + "\x36\xcd\x41\xd8\x6f\x06\x7a\x11" + "\xa8\x1c\xb3\x4a\xe1\x55\xec\x83" + "\x1a\x8e\x25\xbc\x30\xc7\x5e\xf5" + "\x69\x00\x97\x0b\xa2\x39\xd0\x44" + "\xdb\x72\x09\x7d\x14\xab\x1f\xb6" + "\x4d\xe4\x58\xef\x86\x1d\x91\x28" + "\xbf\x33\xca\x61\xf8\x6c\x03\x9a" + "\x0e\xa5\x3c\xd3\x47\xde\x75\x0c" + "\x80\x17\xae\x22\xb9\x50\xe7\x5b" + "\xf2\x89\x20\x94\x2b\xc2\x36\xcd" + "\x64\xfb\x6f\x06\x9d\x11\xa8\x3f" + "\xd6\x4a\xe1\x78\x0f\x83\x1a\xb1" + "\x25\xbc\x53\xea\x5e\xf5\x8c\x00" + "\x97\x2e\xc5\x39\xd0\x67\xfe\x72" + "\x09\xa0\x14\xab\x42\xd9\x4d\xe4" + "\x7b\x12\x86\x1d\xb4\x28\xbf\x56" + "\xed\x61\xf8\x8f\x03\x9a\x31\xc8" + "\x3c\xd3\x6a\x01\x75\x0c\xa3\x17" + "\xae\x45\xdc\x50\xe7\x7e\x15\x89" + "\x20\xb7\x2b\xc2\x59\xf0\x64\xfb" + "\x92\x06\x9d\x34\xcb\x3f\xd6\x6d" + "\x04\x78\x0f\xa6\x1a\xb1\x48\xdf" + "\x53\xea\x81\x18\x8c\x23\xba\x2e" + "\xc5\x5c\xf3\x67\xfe\x95\x09\xa0" + "\x37\xce\x42\xd9\x70\x07\x7b\x12" + "\xa9\x1d\xb4\x4b\xe2\x56\xed\x84" + "\x1b\x8f\x26\xbd\x31\xc8\x5f\xf6" + "\x6a\x01\x98\x0c\xa3\x3a\xd1\x45" + "\xdc\x73\x0a\x7e\x15\xac\x20\xb7" + "\x4e\xe5\x59\xf0\x87\x1e\x92\x29" + "\xc0\x34\xcb\x62\xf9\x6d\x04\x9b" + "\x0f\xa6\x3d\xd4\x48\xdf\x76\x0d" + "\x81\x18\xaf\x23\xba\x51\xe8\x5c" + "\xf3\x8a\x21\x95\x2c\xc3\x37\xce" + "\x65\xfc\x70\x07\x9e\x12\xa9\x40" + "\xd7\x4b\xe2\x79\x10\x84\x1b\xb2" + "\x26\xbd\x54\xeb\x5f\xf6\x8d\x01" + "\x98\x2f\xc6\x3a\xd1\x68\xff\x73" + "\x0a\xa1\x15\xac\x43\xda\x4e\xe5" + "\x7c\x13\x87\x1e\xb5\x29\xc0\x57" + "\xee\x62\xf9\x90\x04\x9b\x32\xc9" + "\x3d\xd4\x6b\x02\x76\x0d\xa4\x18" + "\xaf\x46\xdd\x51\xe8\x7f\x16\x8a" + "\x21\xb8\x2c\xc3\x5a\xf1\x65\xfc" + "\x93\x07\x9e\x35\xcc\x40\xd7\x6e" + "\x05\x79\x10\xa7\x1b\xb2\x49\xe0" + "\x54\xeb\x82\x19\x8d\x24\xbb\x2f" + "\xc6\x5d\xf4\x68\xff\x96\x0a\xa1" + "\x38\xcf\x43\xda\x71\x08\x7c\x13" + "\xaa\x1e\xb5\x4c\xe3\x57\xee\x85" + "\x1c\x90\x27\xbe\x32\xc9\x60\xf7" + "\x6b\x02\x99\x0d\xa4\x3b\xd2\x46" + "\xdd\x74\x0b\x7f\x16\xad\x21\xb8" + "\x4f\xe6\x5a\xf1\x88\x1f\x93\x2a" + "\xc1\x35\xcc\x63\xfa\x6e\x05\x9c" + "\x10\xa7\x3e\xd5\x49\xe0\x77\x0e" + "\x82\x19\xb0\x24\xbb\x52\xe9\x5d" + "\xf4\x8b\x22\x96\x2d\xc4\x38\xcf" + "\x66\xfd\x71\x08\x9f\x13\xaa\x41" + "\xd8\x4c\xe3\x7a\x11\x85\x1c\xb3" + "\x27\xbe\x55\xec\x60\xf7\x8e\x02" + "\x99\x30\xc7\x3b\xd2\x69\x00\x74" + "\x0b\xa2\x16\xad\x44\xdb\x4f\xe6" + "\x7d\x14\x88\x1f\xb6\x2a\xc1\x58" + "\xef\x63\xfa\x91\x05\x9c\x33\xca" + "\x3e\xd5\x6c\x03\x77\x0e\xa5\x19" + "\xb0\x47\xde\x52\xe9\x80\x17\x8b" + "\x22\xb9\x2d\xc4\x5b\xf2\x66\xfd" + "\x94\x08\x9f\x36\xcd\x41\xd8\x6f" + "\x06\x7a\x11\xa8\x1c\xb3\x4a\xe1" + "\x55\xec\x83\x1a\x8e\x25\xbc\x30" + "\xc7\x5e\xf5\x69\x00\x97\x0b\xa2" + "\x39\xd0\x44\xdb\x72\x09\x7d\x14" + "\xab\x1f\xb6\x4d\xe4\x58\xef\x86" + "\x1d\x91\x28\xbf\x33\xca\x61\xf8" + "\x6c\x03\x9a\x0e\xa5\x3c\xd3\x47" + "\xde\x75\x0c\x80\x17\xae\x22\xb9" + "\x50\xe7\x5b\xf2\x89\x20\x94\x2b" + "\xc2\x36\xcd\x64\xfb\x6f\x06\x9d" + "\x11\xa8\x3f\xd6\x4a\xe1\x78\x0f" + "\x83\x1a\xb1\x25\xbc\x53\xea\x5e" + "\xf5\x8c\x00\x97\x2e\xc5\x39\xd0" + "\x67\xfe\x72\x09\xa0\x14\xab\x42" + "\xd9\x4d\xe4\x7b\x12\x86\x1d\xb4" + "\x28\xbf\x56\xed\x61\xf8\x8f\x03" + "\x9a\x31\xc8\x3c\xd3\x6a\x01\x75" + "\x0c\xa3\x17\xae\x45\xdc\x50\xe7" + "\x7e\x15\x89\x20\xb7\x2b\xc2\x59" + "\xf0\x64\xfb\x92\x06\x9d\x34\xcb" + "\x3f\xd6\x6d\x04\x78\x0f\xa6\x1a" + "\xb1\x48\xdf\x53\xea\x81\x18\x8c" + "\x23\xba\x2e\xc5\x5c\xf3\x67\xfe" + "\x95\x09\xa0\x37\xce\x42\xd9\x70" + "\x07\x7b\x12\xa9\x1d\xb4\x4b\xe2" + "\x56\xed\x84\x1b\x8f\x26\xbd\x31" + "\xc8\x5f\xf6\x6a\x01\x98\x0c\xa3" + "\x3a\xd1\x45\xdc\x73\x0a\x7e\x15" + "\xac\x20\xb7\x4e\xe5\x59\xf0\x87" + "\x1e\x92\x29\xc0\x34\xcb\x62\xf9" + "\x6d\x04\x9b\x0f\xa6\x3d\xd4\x48" + "\xdf\x76\x0d\x81\x18\xaf\x23\xba" + "\x51\xe8\x5c\xf3\x8a\x21\x95\x2c" + "\xc3\x37\xce\x65\xfc\x70\x07\x9e" + "\x12\xa9\x40\xd7\x4b\xe2\x79\x10" + "\x84\x1b\xb2\x26\xbd\x54\xeb\x5f" + "\xf6\x8d\x01\x98\x2f\xc6\x3a\xd1" + "\x68\xff\x73\x0a\xa1\x15\xac\x43" + "\xda\x4e\xe5\x7c\x13\x87\x1e\xb5" + "\x29\xc0\x57\xee\x62\xf9\x90\x04" + "\x9b\x32\xc9\x3d\xd4\x6b\x02\x76" + "\x0d\xa4\x18\xaf\x46\xdd\x51\xe8" + "\x7f\x16\x8a\x21\xb8\x2c\xc3\x5a" + "\xf1\x65\xfc\x93\x07\x9e\x35\xcc" + "\x40\xd7\x6e\x05\x79\x10\xa7\x1b" + "\xb2\x49\xe0\x54\xeb\x82\x19\x8d" + "\x24\xbb\x2f\xc6\x5d\xf4\x68\xff" + "\x96\x0a\xa1\x38\xcf\x43\xda\x71" + "\x08\x7c\x13\xaa\x1e\xb5\x4c", + .psize = 1023, + .digest = "\x7d\x0f\x2f\xb7\x65\x3b\xa7\x26" + "\xc3\x88\x20\x71\x15\x06\xe8\x2d" + "\xa3\x92\x44\xab\x3e\xe7\xff\x86" + "\xb6\x79\x10\x72", }, }; @@ -1077,6 +1213,142 @@ static const struct hash_testvec sha3_256_tv_template[] = { "\x49\x10\x03\x76\xa8\x23\x5e\x2c" "\x82\xe1\xb9\x99\x8a\x99\x9e\x21" "\xdb\x32\xdd\x97\x49\x6d\x33\x76", + .np = 2, + .tap = { 28, 28 }, + }, { + .plaintext = "\x08\x9f\x13\xaa\x41\xd8\x4c\xe3" + "\x7a\x11\x85\x1c\xb3\x27\xbe\x55" + "\xec\x60\xf7\x8e\x02\x99\x30\xc7" + "\x3b\xd2\x69\x00\x74\x0b\xa2\x16" + "\xad\x44\xdb\x4f\xe6\x7d\x14\x88" + "\x1f\xb6\x2a\xc1\x58\xef\x63\xfa" + "\x91\x05\x9c\x33\xca\x3e\xd5\x6c" + "\x03\x77\x0e\xa5\x19\xb0\x47\xde" + "\x52\xe9\x80\x17\x8b\x22\xb9\x2d" + "\xc4\x5b\xf2\x66\xfd\x94\x08\x9f" + "\x36\xcd\x41\xd8\x6f\x06\x7a\x11" + "\xa8\x1c\xb3\x4a\xe1\x55\xec\x83" + "\x1a\x8e\x25\xbc\x30\xc7\x5e\xf5" + "\x69\x00\x97\x0b\xa2\x39\xd0\x44" + "\xdb\x72\x09\x7d\x14\xab\x1f\xb6" + "\x4d\xe4\x58\xef\x86\x1d\x91\x28" + "\xbf\x33\xca\x61\xf8\x6c\x03\x9a" + "\x0e\xa5\x3c\xd3\x47\xde\x75\x0c" + "\x80\x17\xae\x22\xb9\x50\xe7\x5b" + "\xf2\x89\x20\x94\x2b\xc2\x36\xcd" + "\x64\xfb\x6f\x06\x9d\x11\xa8\x3f" + "\xd6\x4a\xe1\x78\x0f\x83\x1a\xb1" + "\x25\xbc\x53\xea\x5e\xf5\x8c\x00" + "\x97\x2e\xc5\x39\xd0\x67\xfe\x72" + "\x09\xa0\x14\xab\x42\xd9\x4d\xe4" + "\x7b\x12\x86\x1d\xb4\x28\xbf\x56" + "\xed\x61\xf8\x8f\x03\x9a\x31\xc8" + "\x3c\xd3\x6a\x01\x75\x0c\xa3\x17" + "\xae\x45\xdc\x50\xe7\x7e\x15\x89" + "\x20\xb7\x2b\xc2\x59\xf0\x64\xfb" + "\x92\x06\x9d\x34\xcb\x3f\xd6\x6d" + "\x04\x78\x0f\xa6\x1a\xb1\x48\xdf" + "\x53\xea\x81\x18\x8c\x23\xba\x2e" + "\xc5\x5c\xf3\x67\xfe\x95\x09\xa0" + "\x37\xce\x42\xd9\x70\x07\x7b\x12" + "\xa9\x1d\xb4\x4b\xe2\x56\xed\x84" + "\x1b\x8f\x26\xbd\x31\xc8\x5f\xf6" + "\x6a\x01\x98\x0c\xa3\x3a\xd1\x45" + "\xdc\x73\x0a\x7e\x15\xac\x20\xb7" + "\x4e\xe5\x59\xf0\x87\x1e\x92\x29" + "\xc0\x34\xcb\x62\xf9\x6d\x04\x9b" + "\x0f\xa6\x3d\xd4\x48\xdf\x76\x0d" + "\x81\x18\xaf\x23\xba\x51\xe8\x5c" + "\xf3\x8a\x21\x95\x2c\xc3\x37\xce" + "\x65\xfc\x70\x07\x9e\x12\xa9\x40" + "\xd7\x4b\xe2\x79\x10\x84\x1b\xb2" + "\x26\xbd\x54\xeb\x5f\xf6\x8d\x01" + "\x98\x2f\xc6\x3a\xd1\x68\xff\x73" + "\x0a\xa1\x15\xac\x43\xda\x4e\xe5" + "\x7c\x13\x87\x1e\xb5\x29\xc0\x57" + "\xee\x62\xf9\x90\x04\x9b\x32\xc9" + "\x3d\xd4\x6b\x02\x76\x0d\xa4\x18" + "\xaf\x46\xdd\x51\xe8\x7f\x16\x8a" + "\x21\xb8\x2c\xc3\x5a\xf1\x65\xfc" + "\x93\x07\x9e\x35\xcc\x40\xd7\x6e" + "\x05\x79\x10\xa7\x1b\xb2\x49\xe0" + "\x54\xeb\x82\x19\x8d\x24\xbb\x2f" + "\xc6\x5d\xf4\x68\xff\x96\x0a\xa1" + "\x38\xcf\x43\xda\x71\x08\x7c\x13" + "\xaa\x1e\xb5\x4c\xe3\x57\xee\x85" + "\x1c\x90\x27\xbe\x32\xc9\x60\xf7" + "\x6b\x02\x99\x0d\xa4\x3b\xd2\x46" + "\xdd\x74\x0b\x7f\x16\xad\x21\xb8" + "\x4f\xe6\x5a\xf1\x88\x1f\x93\x2a" + "\xc1\x35\xcc\x63\xfa\x6e\x05\x9c" + "\x10\xa7\x3e\xd5\x49\xe0\x77\x0e" + "\x82\x19\xb0\x24\xbb\x52\xe9\x5d" + "\xf4\x8b\x22\x96\x2d\xc4\x38\xcf" + "\x66\xfd\x71\x08\x9f\x13\xaa\x41" + "\xd8\x4c\xe3\x7a\x11\x85\x1c\xb3" + "\x27\xbe\x55\xec\x60\xf7\x8e\x02" + "\x99\x30\xc7\x3b\xd2\x69\x00\x74" + "\x0b\xa2\x16\xad\x44\xdb\x4f\xe6" + "\x7d\x14\x88\x1f\xb6\x2a\xc1\x58" + "\xef\x63\xfa\x91\x05\x9c\x33\xca" + "\x3e\xd5\x6c\x03\x77\x0e\xa5\x19" + "\xb0\x47\xde\x52\xe9\x80\x17\x8b" + "\x22\xb9\x2d\xc4\x5b\xf2\x66\xfd" + "\x94\x08\x9f\x36\xcd\x41\xd8\x6f" + "\x06\x7a\x11\xa8\x1c\xb3\x4a\xe1" + "\x55\xec\x83\x1a\x8e\x25\xbc\x30" + "\xc7\x5e\xf5\x69\x00\x97\x0b\xa2" + "\x39\xd0\x44\xdb\x72\x09\x7d\x14" + "\xab\x1f\xb6\x4d\xe4\x58\xef\x86" + "\x1d\x91\x28\xbf\x33\xca\x61\xf8" + "\x6c\x03\x9a\x0e\xa5\x3c\xd3\x47" + "\xde\x75\x0c\x80\x17\xae\x22\xb9" + "\x50\xe7\x5b\xf2\x89\x20\x94\x2b" + "\xc2\x36\xcd\x64\xfb\x6f\x06\x9d" + "\x11\xa8\x3f\xd6\x4a\xe1\x78\x0f" + "\x83\x1a\xb1\x25\xbc\x53\xea\x5e" + "\xf5\x8c\x00\x97\x2e\xc5\x39\xd0" + "\x67\xfe\x72\x09\xa0\x14\xab\x42" + "\xd9\x4d\xe4\x7b\x12\x86\x1d\xb4" + "\x28\xbf\x56\xed\x61\xf8\x8f\x03" + "\x9a\x31\xc8\x3c\xd3\x6a\x01\x75" + "\x0c\xa3\x17\xae\x45\xdc\x50\xe7" + "\x7e\x15\x89\x20\xb7\x2b\xc2\x59" + "\xf0\x64\xfb\x92\x06\x9d\x34\xcb" + "\x3f\xd6\x6d\x04\x78\x0f\xa6\x1a" + "\xb1\x48\xdf\x53\xea\x81\x18\x8c" + "\x23\xba\x2e\xc5\x5c\xf3\x67\xfe" + "\x95\x09\xa0\x37\xce\x42\xd9\x70" + "\x07\x7b\x12\xa9\x1d\xb4\x4b\xe2" + "\x56\xed\x84\x1b\x8f\x26\xbd\x31" + "\xc8\x5f\xf6\x6a\x01\x98\x0c\xa3" + "\x3a\xd1\x45\xdc\x73\x0a\x7e\x15" + "\xac\x20\xb7\x4e\xe5\x59\xf0\x87" + "\x1e\x92\x29\xc0\x34\xcb\x62\xf9" + "\x6d\x04\x9b\x0f\xa6\x3d\xd4\x48" + "\xdf\x76\x0d\x81\x18\xaf\x23\xba" + "\x51\xe8\x5c\xf3\x8a\x21\x95\x2c" + "\xc3\x37\xce\x65\xfc\x70\x07\x9e" + "\x12\xa9\x40\xd7\x4b\xe2\x79\x10" + "\x84\x1b\xb2\x26\xbd\x54\xeb\x5f" + "\xf6\x8d\x01\x98\x2f\xc6\x3a\xd1" + "\x68\xff\x73\x0a\xa1\x15\xac\x43" + "\xda\x4e\xe5\x7c\x13\x87\x1e\xb5" + "\x29\xc0\x57\xee\x62\xf9\x90\x04" + "\x9b\x32\xc9\x3d\xd4\x6b\x02\x76" + "\x0d\xa4\x18\xaf\x46\xdd\x51\xe8" + "\x7f\x16\x8a\x21\xb8\x2c\xc3\x5a" + "\xf1\x65\xfc\x93\x07\x9e\x35\xcc" + "\x40\xd7\x6e\x05\x79\x10\xa7\x1b" + "\xb2\x49\xe0\x54\xeb\x82\x19\x8d" + "\x24\xbb\x2f\xc6\x5d\xf4\x68\xff" + "\x96\x0a\xa1\x38\xcf\x43\xda\x71" + "\x08\x7c\x13\xaa\x1e\xb5\x4c", + .psize = 1023, + .digest = "\xde\x41\x04\xbd\xda\xda\xd9\x71" + "\xf7\xfa\x80\xf5\xea\x11\x03\xb1" + "\x3b\x6a\xbc\x5f\xb9\x66\x26\xf7" + "\x8a\x97\xbb\xf2\x07\x08\x38\x30", }, }; @@ -1109,6 +1381,144 @@ static const struct hash_testvec sha3_384_tv_template[] = { "\x9b\xfd\xbc\x32\xb9\xd4\xad\x5a" "\xa0\x4a\x1f\x07\x6e\x62\xfe\xa1" "\x9e\xef\x51\xac\xd0\x65\x7c\x22", + .np = 2, + .tap = { 28, 28 }, + }, { + .plaintext = "\x08\x9f\x13\xaa\x41\xd8\x4c\xe3" + "\x7a\x11\x85\x1c\xb3\x27\xbe\x55" + "\xec\x60\xf7\x8e\x02\x99\x30\xc7" + "\x3b\xd2\x69\x00\x74\x0b\xa2\x16" + "\xad\x44\xdb\x4f\xe6\x7d\x14\x88" + "\x1f\xb6\x2a\xc1\x58\xef\x63\xfa" + "\x91\x05\x9c\x33\xca\x3e\xd5\x6c" + "\x03\x77\x0e\xa5\x19\xb0\x47\xde" + "\x52\xe9\x80\x17\x8b\x22\xb9\x2d" + "\xc4\x5b\xf2\x66\xfd\x94\x08\x9f" + "\x36\xcd\x41\xd8\x6f\x06\x7a\x11" + "\xa8\x1c\xb3\x4a\xe1\x55\xec\x83" + "\x1a\x8e\x25\xbc\x30\xc7\x5e\xf5" + "\x69\x00\x97\x0b\xa2\x39\xd0\x44" + "\xdb\x72\x09\x7d\x14\xab\x1f\xb6" + "\x4d\xe4\x58\xef\x86\x1d\x91\x28" + "\xbf\x33\xca\x61\xf8\x6c\x03\x9a" + "\x0e\xa5\x3c\xd3\x47\xde\x75\x0c" + "\x80\x17\xae\x22\xb9\x50\xe7\x5b" + "\xf2\x89\x20\x94\x2b\xc2\x36\xcd" + "\x64\xfb\x6f\x06\x9d\x11\xa8\x3f" + "\xd6\x4a\xe1\x78\x0f\x83\x1a\xb1" + "\x25\xbc\x53\xea\x5e\xf5\x8c\x00" + "\x97\x2e\xc5\x39\xd0\x67\xfe\x72" + "\x09\xa0\x14\xab\x42\xd9\x4d\xe4" + "\x7b\x12\x86\x1d\xb4\x28\xbf\x56" + "\xed\x61\xf8\x8f\x03\x9a\x31\xc8" + "\x3c\xd3\x6a\x01\x75\x0c\xa3\x17" + "\xae\x45\xdc\x50\xe7\x7e\x15\x89" + "\x20\xb7\x2b\xc2\x59\xf0\x64\xfb" + "\x92\x06\x9d\x34\xcb\x3f\xd6\x6d" + "\x04\x78\x0f\xa6\x1a\xb1\x48\xdf" + "\x53\xea\x81\x18\x8c\x23\xba\x2e" + "\xc5\x5c\xf3\x67\xfe\x95\x09\xa0" + "\x37\xce\x42\xd9\x70\x07\x7b\x12" + "\xa9\x1d\xb4\x4b\xe2\x56\xed\x84" + "\x1b\x8f\x26\xbd\x31\xc8\x5f\xf6" + "\x6a\x01\x98\x0c\xa3\x3a\xd1\x45" + "\xdc\x73\x0a\x7e\x15\xac\x20\xb7" + "\x4e\xe5\x59\xf0\x87\x1e\x92\x29" + "\xc0\x34\xcb\x62\xf9\x6d\x04\x9b" + "\x0f\xa6\x3d\xd4\x48\xdf\x76\x0d" + "\x81\x18\xaf\x23\xba\x51\xe8\x5c" + "\xf3\x8a\x21\x95\x2c\xc3\x37\xce" + "\x65\xfc\x70\x07\x9e\x12\xa9\x40" + "\xd7\x4b\xe2\x79\x10\x84\x1b\xb2" + "\x26\xbd\x54\xeb\x5f\xf6\x8d\x01" + "\x98\x2f\xc6\x3a\xd1\x68\xff\x73" + "\x0a\xa1\x15\xac\x43\xda\x4e\xe5" + "\x7c\x13\x87\x1e\xb5\x29\xc0\x57" + "\xee\x62\xf9\x90\x04\x9b\x32\xc9" + "\x3d\xd4\x6b\x02\x76\x0d\xa4\x18" + "\xaf\x46\xdd\x51\xe8\x7f\x16\x8a" + "\x21\xb8\x2c\xc3\x5a\xf1\x65\xfc" + "\x93\x07\x9e\x35\xcc\x40\xd7\x6e" + "\x05\x79\x10\xa7\x1b\xb2\x49\xe0" + "\x54\xeb\x82\x19\x8d\x24\xbb\x2f" + "\xc6\x5d\xf4\x68\xff\x96\x0a\xa1" + "\x38\xcf\x43\xda\x71\x08\x7c\x13" + "\xaa\x1e\xb5\x4c\xe3\x57\xee\x85" + "\x1c\x90\x27\xbe\x32\xc9\x60\xf7" + "\x6b\x02\x99\x0d\xa4\x3b\xd2\x46" + "\xdd\x74\x0b\x7f\x16\xad\x21\xb8" + "\x4f\xe6\x5a\xf1\x88\x1f\x93\x2a" + "\xc1\x35\xcc\x63\xfa\x6e\x05\x9c" + "\x10\xa7\x3e\xd5\x49\xe0\x77\x0e" + "\x82\x19\xb0\x24\xbb\x52\xe9\x5d" + "\xf4\x8b\x22\x96\x2d\xc4\x38\xcf" + "\x66\xfd\x71\x08\x9f\x13\xaa\x41" + "\xd8\x4c\xe3\x7a\x11\x85\x1c\xb3" + "\x27\xbe\x55\xec\x60\xf7\x8e\x02" + "\x99\x30\xc7\x3b\xd2\x69\x00\x74" + "\x0b\xa2\x16\xad\x44\xdb\x4f\xe6" + "\x7d\x14\x88\x1f\xb6\x2a\xc1\x58" + "\xef\x63\xfa\x91\x05\x9c\x33\xca" + "\x3e\xd5\x6c\x03\x77\x0e\xa5\x19" + "\xb0\x47\xde\x52\xe9\x80\x17\x8b" + "\x22\xb9\x2d\xc4\x5b\xf2\x66\xfd" + "\x94\x08\x9f\x36\xcd\x41\xd8\x6f" + "\x06\x7a\x11\xa8\x1c\xb3\x4a\xe1" + "\x55\xec\x83\x1a\x8e\x25\xbc\x30" + "\xc7\x5e\xf5\x69\x00\x97\x0b\xa2" + "\x39\xd0\x44\xdb\x72\x09\x7d\x14" + "\xab\x1f\xb6\x4d\xe4\x58\xef\x86" + "\x1d\x91\x28\xbf\x33\xca\x61\xf8" + "\x6c\x03\x9a\x0e\xa5\x3c\xd3\x47" + "\xde\x75\x0c\x80\x17\xae\x22\xb9" + "\x50\xe7\x5b\xf2\x89\x20\x94\x2b" + "\xc2\x36\xcd\x64\xfb\x6f\x06\x9d" + "\x11\xa8\x3f\xd6\x4a\xe1\x78\x0f" + "\x83\x1a\xb1\x25\xbc\x53\xea\x5e" + "\xf5\x8c\x00\x97\x2e\xc5\x39\xd0" + "\x67\xfe\x72\x09\xa0\x14\xab\x42" + "\xd9\x4d\xe4\x7b\x12\x86\x1d\xb4" + "\x28\xbf\x56\xed\x61\xf8\x8f\x03" + "\x9a\x31\xc8\x3c\xd3\x6a\x01\x75" + "\x0c\xa3\x17\xae\x45\xdc\x50\xe7" + "\x7e\x15\x89\x20\xb7\x2b\xc2\x59" + "\xf0\x64\xfb\x92\x06\x9d\x34\xcb" + "\x3f\xd6\x6d\x04\x78\x0f\xa6\x1a" + "\xb1\x48\xdf\x53\xea\x81\x18\x8c" + "\x23\xba\x2e\xc5\x5c\xf3\x67\xfe" + "\x95\x09\xa0\x37\xce\x42\xd9\x70" + "\x07\x7b\x12\xa9\x1d\xb4\x4b\xe2" + "\x56\xed\x84\x1b\x8f\x26\xbd\x31" + "\xc8\x5f\xf6\x6a\x01\x98\x0c\xa3" + "\x3a\xd1\x45\xdc\x73\x0a\x7e\x15" + "\xac\x20\xb7\x4e\xe5\x59\xf0\x87" + "\x1e\x92\x29\xc0\x34\xcb\x62\xf9" + "\x6d\x04\x9b\x0f\xa6\x3d\xd4\x48" + "\xdf\x76\x0d\x81\x18\xaf\x23\xba" + "\x51\xe8\x5c\xf3\x8a\x21\x95\x2c" + "\xc3\x37\xce\x65\xfc\x70\x07\x9e" + "\x12\xa9\x40\xd7\x4b\xe2\x79\x10" + "\x84\x1b\xb2\x26\xbd\x54\xeb\x5f" + "\xf6\x8d\x01\x98\x2f\xc6\x3a\xd1" + "\x68\xff\x73\x0a\xa1\x15\xac\x43" + "\xda\x4e\xe5\x7c\x13\x87\x1e\xb5" + "\x29\xc0\x57\xee\x62\xf9\x90\x04" + "\x9b\x32\xc9\x3d\xd4\x6b\x02\x76" + "\x0d\xa4\x18\xaf\x46\xdd\x51\xe8" + "\x7f\x16\x8a\x21\xb8\x2c\xc3\x5a" + "\xf1\x65\xfc\x93\x07\x9e\x35\xcc" + "\x40\xd7\x6e\x05\x79\x10\xa7\x1b" + "\xb2\x49\xe0\x54\xeb\x82\x19\x8d" + "\x24\xbb\x2f\xc6\x5d\xf4\x68\xff" + "\x96\x0a\xa1\x38\xcf\x43\xda\x71" + "\x08\x7c\x13\xaa\x1e\xb5\x4c", + .psize = 1023, + .digest = "\x1b\x19\x4d\x8f\xd5\x36\x87\x71" + "\xcf\xca\x30\x85\x9b\xc1\x25\xc7" + "\x00\xcb\x73\x8a\x8e\xd4\xfe\x2b" + "\x1a\xa2\xdc\x2e\x41\xfd\x52\x51" + "\xd2\x21\xae\x2d\xc7\xae\x8c\x40" + "\xb9\xe6\x56\x48\x03\xcd\x88\x6b", }, }; @@ -1147,6 +1557,146 @@ static const struct hash_testvec sha3_512_tv_template[] = { "\xba\x1b\x0d\x8d\xc7\x8c\x08\x63" "\x46\xb5\x33\xb4\x9c\x03\x0d\x99" "\xa2\x7d\xaf\x11\x39\xd6\xe7\x5e", + .np = 2, + .tap = { 28, 28 }, + }, { + .plaintext = "\x08\x9f\x13\xaa\x41\xd8\x4c\xe3" + "\x7a\x11\x85\x1c\xb3\x27\xbe\x55" + "\xec\x60\xf7\x8e\x02\x99\x30\xc7" + "\x3b\xd2\x69\x00\x74\x0b\xa2\x16" + "\xad\x44\xdb\x4f\xe6\x7d\x14\x88" + "\x1f\xb6\x2a\xc1\x58\xef\x63\xfa" + "\x91\x05\x9c\x33\xca\x3e\xd5\x6c" + "\x03\x77\x0e\xa5\x19\xb0\x47\xde" + "\x52\xe9\x80\x17\x8b\x22\xb9\x2d" + "\xc4\x5b\xf2\x66\xfd\x94\x08\x9f" + "\x36\xcd\x41\xd8\x6f\x06\x7a\x11" + "\xa8\x1c\xb3\x4a\xe1\x55\xec\x83" + "\x1a\x8e\x25\xbc\x30\xc7\x5e\xf5" + "\x69\x00\x97\x0b\xa2\x39\xd0\x44" + "\xdb\x72\x09\x7d\x14\xab\x1f\xb6" + "\x4d\xe4\x58\xef\x86\x1d\x91\x28" + "\xbf\x33\xca\x61\xf8\x6c\x03\x9a" + "\x0e\xa5\x3c\xd3\x47\xde\x75\x0c" + "\x80\x17\xae\x22\xb9\x50\xe7\x5b" + "\xf2\x89\x20\x94\x2b\xc2\x36\xcd" + "\x64\xfb\x6f\x06\x9d\x11\xa8\x3f" + "\xd6\x4a\xe1\x78\x0f\x83\x1a\xb1" + "\x25\xbc\x53\xea\x5e\xf5\x8c\x00" + "\x97\x2e\xc5\x39\xd0\x67\xfe\x72" + "\x09\xa0\x14\xab\x42\xd9\x4d\xe4" + "\x7b\x12\x86\x1d\xb4\x28\xbf\x56" + "\xed\x61\xf8\x8f\x03\x9a\x31\xc8" + "\x3c\xd3\x6a\x01\x75\x0c\xa3\x17" + "\xae\x45\xdc\x50\xe7\x7e\x15\x89" + "\x20\xb7\x2b\xc2\x59\xf0\x64\xfb" + "\x92\x06\x9d\x34\xcb\x3f\xd6\x6d" + "\x04\x78\x0f\xa6\x1a\xb1\x48\xdf" + "\x53\xea\x81\x18\x8c\x23\xba\x2e" + "\xc5\x5c\xf3\x67\xfe\x95\x09\xa0" + "\x37\xce\x42\xd9\x70\x07\x7b\x12" + "\xa9\x1d\xb4\x4b\xe2\x56\xed\x84" + "\x1b\x8f\x26\xbd\x31\xc8\x5f\xf6" + "\x6a\x01\x98\x0c\xa3\x3a\xd1\x45" + "\xdc\x73\x0a\x7e\x15\xac\x20\xb7" + "\x4e\xe5\x59\xf0\x87\x1e\x92\x29" + "\xc0\x34\xcb\x62\xf9\x6d\x04\x9b" + "\x0f\xa6\x3d\xd4\x48\xdf\x76\x0d" + "\x81\x18\xaf\x23\xba\x51\xe8\x5c" + "\xf3\x8a\x21\x95\x2c\xc3\x37\xce" + "\x65\xfc\x70\x07\x9e\x12\xa9\x40" + "\xd7\x4b\xe2\x79\x10\x84\x1b\xb2" + "\x26\xbd\x54\xeb\x5f\xf6\x8d\x01" + "\x98\x2f\xc6\x3a\xd1\x68\xff\x73" + "\x0a\xa1\x15\xac\x43\xda\x4e\xe5" + "\x7c\x13\x87\x1e\xb5\x29\xc0\x57" + "\xee\x62\xf9\x90\x04\x9b\x32\xc9" + "\x3d\xd4\x6b\x02\x76\x0d\xa4\x18" + "\xaf\x46\xdd\x51\xe8\x7f\x16\x8a" + "\x21\xb8\x2c\xc3\x5a\xf1\x65\xfc" + "\x93\x07\x9e\x35\xcc\x40\xd7\x6e" + "\x05\x79\x10\xa7\x1b\xb2\x49\xe0" + "\x54\xeb\x82\x19\x8d\x24\xbb\x2f" + "\xc6\x5d\xf4\x68\xff\x96\x0a\xa1" + "\x38\xcf\x43\xda\x71\x08\x7c\x13" + "\xaa\x1e\xb5\x4c\xe3\x57\xee\x85" + "\x1c\x90\x27\xbe\x32\xc9\x60\xf7" + "\x6b\x02\x99\x0d\xa4\x3b\xd2\x46" + "\xdd\x74\x0b\x7f\x16\xad\x21\xb8" + "\x4f\xe6\x5a\xf1\x88\x1f\x93\x2a" + "\xc1\x35\xcc\x63\xfa\x6e\x05\x9c" + "\x10\xa7\x3e\xd5\x49\xe0\x77\x0e" + "\x82\x19\xb0\x24\xbb\x52\xe9\x5d" + "\xf4\x8b\x22\x96\x2d\xc4\x38\xcf" + "\x66\xfd\x71\x08\x9f\x13\xaa\x41" + "\xd8\x4c\xe3\x7a\x11\x85\x1c\xb3" + "\x27\xbe\x55\xec\x60\xf7\x8e\x02" + "\x99\x30\xc7\x3b\xd2\x69\x00\x74" + "\x0b\xa2\x16\xad\x44\xdb\x4f\xe6" + "\x7d\x14\x88\x1f\xb6\x2a\xc1\x58" + "\xef\x63\xfa\x91\x05\x9c\x33\xca" + "\x3e\xd5\x6c\x03\x77\x0e\xa5\x19" + "\xb0\x47\xde\x52\xe9\x80\x17\x8b" + "\x22\xb9\x2d\xc4\x5b\xf2\x66\xfd" + "\x94\x08\x9f\x36\xcd\x41\xd8\x6f" + "\x06\x7a\x11\xa8\x1c\xb3\x4a\xe1" + "\x55\xec\x83\x1a\x8e\x25\xbc\x30" + "\xc7\x5e\xf5\x69\x00\x97\x0b\xa2" + "\x39\xd0\x44\xdb\x72\x09\x7d\x14" + "\xab\x1f\xb6\x4d\xe4\x58\xef\x86" + "\x1d\x91\x28\xbf\x33\xca\x61\xf8" + "\x6c\x03\x9a\x0e\xa5\x3c\xd3\x47" + "\xde\x75\x0c\x80\x17\xae\x22\xb9" + "\x50\xe7\x5b\xf2\x89\x20\x94\x2b" + "\xc2\x36\xcd\x64\xfb\x6f\x06\x9d" + "\x11\xa8\x3f\xd6\x4a\xe1\x78\x0f" + "\x83\x1a\xb1\x25\xbc\x53\xea\x5e" + "\xf5\x8c\x00\x97\x2e\xc5\x39\xd0" + "\x67\xfe\x72\x09\xa0\x14\xab\x42" + "\xd9\x4d\xe4\x7b\x12\x86\x1d\xb4" + "\x28\xbf\x56\xed\x61\xf8\x8f\x03" + "\x9a\x31\xc8\x3c\xd3\x6a\x01\x75" + "\x0c\xa3\x17\xae\x45\xdc\x50\xe7" + "\x7e\x15\x89\x20\xb7\x2b\xc2\x59" + "\xf0\x64\xfb\x92\x06\x9d\x34\xcb" + "\x3f\xd6\x6d\x04\x78\x0f\xa6\x1a" + "\xb1\x48\xdf\x53\xea\x81\x18\x8c" + "\x23\xba\x2e\xc5\x5c\xf3\x67\xfe" + "\x95\x09\xa0\x37\xce\x42\xd9\x70" + "\x07\x7b\x12\xa9\x1d\xb4\x4b\xe2" + "\x56\xed\x84\x1b\x8f\x26\xbd\x31" + "\xc8\x5f\xf6\x6a\x01\x98\x0c\xa3" + "\x3a\xd1\x45\xdc\x73\x0a\x7e\x15" + "\xac\x20\xb7\x4e\xe5\x59\xf0\x87" + "\x1e\x92\x29\xc0\x34\xcb\x62\xf9" + "\x6d\x04\x9b\x0f\xa6\x3d\xd4\x48" + "\xdf\x76\x0d\x81\x18\xaf\x23\xba" + "\x51\xe8\x5c\xf3\x8a\x21\x95\x2c" + "\xc3\x37\xce\x65\xfc\x70\x07\x9e" + "\x12\xa9\x40\xd7\x4b\xe2\x79\x10" + "\x84\x1b\xb2\x26\xbd\x54\xeb\x5f" + "\xf6\x8d\x01\x98\x2f\xc6\x3a\xd1" + "\x68\xff\x73\x0a\xa1\x15\xac\x43" + "\xda\x4e\xe5\x7c\x13\x87\x1e\xb5" + "\x29\xc0\x57\xee\x62\xf9\x90\x04" + "\x9b\x32\xc9\x3d\xd4\x6b\x02\x76" + "\x0d\xa4\x18\xaf\x46\xdd\x51\xe8" + "\x7f\x16\x8a\x21\xb8\x2c\xc3\x5a" + "\xf1\x65\xfc\x93\x07\x9e\x35\xcc" + "\x40\xd7\x6e\x05\x79\x10\xa7\x1b" + "\xb2\x49\xe0\x54\xeb\x82\x19\x8d" + "\x24\xbb\x2f\xc6\x5d\xf4\x68\xff" + "\x96\x0a\xa1\x38\xcf\x43\xda\x71" + "\x08\x7c\x13\xaa\x1e\xb5\x4c", + .psize = 1023, + .digest = "\x59\xda\x30\xe3\x90\xe4\x3d\xde" + "\xf0\xc6\x42\x17\xd7\xb2\x26\x47" + "\x90\x28\xa6\x84\xe8\x49\x7a\x86" + "\xd6\xb8\x9e\xf8\x07\x59\x21\x03" + "\xad\xd2\xed\x48\xa3\xb9\xa5\xf0" + "\xb3\xae\x02\x2b\xb8\xaf\xc3\x3b" + "\xd6\xb0\x8f\xcb\x76\x8b\xa7\x41" + "\x32\xc2\x8e\x50\x91\x86\x90\xfb", }, }; From 15d5910e92614e642e7485bb9e89d46e4d1d65d9 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 19 Jan 2018 12:04:38 +0000 Subject: [PATCH 163/166] crypto: arm64/sha3 - new v8.2 Crypto Extensions implementation Implement the various flavours of SHA3 using the new optional EOR3/RAX1/XAR/BCAX instructions introduced by ARMv8.2. Tested-by: Steve Capper Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/Kconfig | 6 + arch/arm64/crypto/Makefile | 3 + arch/arm64/crypto/sha3-ce-core.S | 210 +++++++++++++++++++++++++++++++ arch/arm64/crypto/sha3-ce-glue.c | 161 ++++++++++++++++++++++++ 4 files changed, 380 insertions(+) create mode 100644 arch/arm64/crypto/sha3-ce-core.S create mode 100644 arch/arm64/crypto/sha3-ce-glue.c diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index aad288f4b9de..3321b2c9a2b5 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -35,6 +35,12 @@ config CRYPTO_SHA512_ARM64_CE select CRYPTO_HASH select CRYPTO_SHA512_ARM64 +config CRYPTO_SHA3_ARM64 + tristate "SHA3 digest algorithm (ARMv8.2 Crypto Extensions)" + depends on KERNEL_MODE_NEON + select CRYPTO_HASH + select CRYPTO_SHA3 + config CRYPTO_GHASH_ARM64_CE tristate "GHASH/AES-GCM using ARMv8 Crypto Extensions" depends on KERNEL_MODE_NEON diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile index b438b3dc9b4c..4ca2d146e213 100644 --- a/arch/arm64/crypto/Makefile +++ b/arch/arm64/crypto/Makefile @@ -17,6 +17,9 @@ sha2-ce-y := sha2-ce-glue.o sha2-ce-core.o obj-$(CONFIG_CRYPTO_SHA512_ARM64_CE) += sha512-ce.o sha512-ce-y := sha512-ce-glue.o sha512-ce-core.o +obj-$(CONFIG_CRYPTO_SHA3_ARM64) += sha3-ce.o +sha3-ce-y := sha3-ce-glue.o sha3-ce-core.o + obj-$(CONFIG_CRYPTO_GHASH_ARM64_CE) += ghash-ce.o ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o diff --git a/arch/arm64/crypto/sha3-ce-core.S b/arch/arm64/crypto/sha3-ce-core.S new file mode 100644 index 000000000000..332ad7530690 --- /dev/null +++ b/arch/arm64/crypto/sha3-ce-core.S @@ -0,0 +1,210 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * sha3-ce-core.S - core SHA-3 transform using v8.2 Crypto Extensions + * + * Copyright (C) 2018 Linaro Ltd + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include + + .irp b,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 + .set .Lv\b\().2d, \b + .set .Lv\b\().16b, \b + .endr + + /* + * ARMv8.2 Crypto Extensions instructions + */ + .macro eor3, rd, rn, rm, ra + .inst 0xce000000 | .L\rd | (.L\rn << 5) | (.L\ra << 10) | (.L\rm << 16) + .endm + + .macro rax1, rd, rn, rm + .inst 0xce608c00 | .L\rd | (.L\rn << 5) | (.L\rm << 16) + .endm + + .macro bcax, rd, rn, rm, ra + .inst 0xce200000 | .L\rd | (.L\rn << 5) | (.L\ra << 10) | (.L\rm << 16) + .endm + + .macro xar, rd, rn, rm, imm6 + .inst 0xce800000 | .L\rd | (.L\rn << 5) | ((\imm6) << 10) | (.L\rm << 16) + .endm + + /* + * sha3_ce_transform(u64 *st, const u8 *data, int blocks, int dg_size) + */ + .text +ENTRY(sha3_ce_transform) + /* load state */ + add x8, x0, #32 + ld1 { v0.1d- v3.1d}, [x0] + ld1 { v4.1d- v7.1d}, [x8], #32 + ld1 { v8.1d-v11.1d}, [x8], #32 + ld1 {v12.1d-v15.1d}, [x8], #32 + ld1 {v16.1d-v19.1d}, [x8], #32 + ld1 {v20.1d-v23.1d}, [x8], #32 + ld1 {v24.1d}, [x8] + +0: sub w2, w2, #1 + mov w8, #24 + adr_l x9, .Lsha3_rcon + + /* load input */ + ld1 {v25.8b-v28.8b}, [x1], #32 + ld1 {v29.8b-v31.8b}, [x1], #24 + eor v0.8b, v0.8b, v25.8b + eor v1.8b, v1.8b, v26.8b + eor v2.8b, v2.8b, v27.8b + eor v3.8b, v3.8b, v28.8b + eor v4.8b, v4.8b, v29.8b + eor v5.8b, v5.8b, v30.8b + eor v6.8b, v6.8b, v31.8b + + tbnz x3, #6, 2f // SHA3-512 + + ld1 {v25.8b-v28.8b}, [x1], #32 + ld1 {v29.8b-v30.8b}, [x1], #16 + eor v7.8b, v7.8b, v25.8b + eor v8.8b, v8.8b, v26.8b + eor v9.8b, v9.8b, v27.8b + eor v10.8b, v10.8b, v28.8b + eor v11.8b, v11.8b, v29.8b + eor v12.8b, v12.8b, v30.8b + + tbnz x3, #4, 1f // SHA3-384 or SHA3-224 + + // SHA3-256 + ld1 {v25.8b-v28.8b}, [x1], #32 + eor v13.8b, v13.8b, v25.8b + eor v14.8b, v14.8b, v26.8b + eor v15.8b, v15.8b, v27.8b + eor v16.8b, v16.8b, v28.8b + b 3f + +1: tbz x3, #2, 3f // bit 2 cleared? SHA-384 + + // SHA3-224 + ld1 {v25.8b-v28.8b}, [x1], #32 + ld1 {v29.8b}, [x1], #8 + eor v13.8b, v13.8b, v25.8b + eor v14.8b, v14.8b, v26.8b + eor v15.8b, v15.8b, v27.8b + eor v16.8b, v16.8b, v28.8b + eor v17.8b, v17.8b, v29.8b + b 3f + + // SHA3-512 +2: ld1 {v25.8b-v26.8b}, [x1], #16 + eor v7.8b, v7.8b, v25.8b + eor v8.8b, v8.8b, v26.8b + +3: sub w8, w8, #1 + + eor3 v29.16b, v4.16b, v9.16b, v14.16b + eor3 v26.16b, v1.16b, v6.16b, v11.16b + eor3 v28.16b, v3.16b, v8.16b, v13.16b + eor3 v25.16b, v0.16b, v5.16b, v10.16b + eor3 v27.16b, v2.16b, v7.16b, v12.16b + eor3 v29.16b, v29.16b, v19.16b, v24.16b + eor3 v26.16b, v26.16b, v16.16b, v21.16b + eor3 v28.16b, v28.16b, v18.16b, v23.16b + eor3 v25.16b, v25.16b, v15.16b, v20.16b + eor3 v27.16b, v27.16b, v17.16b, v22.16b + + rax1 v30.2d, v29.2d, v26.2d // bc[0] + rax1 v26.2d, v26.2d, v28.2d // bc[2] + rax1 v28.2d, v28.2d, v25.2d // bc[4] + rax1 v25.2d, v25.2d, v27.2d // bc[1] + rax1 v27.2d, v27.2d, v29.2d // bc[3] + + eor v0.16b, v0.16b, v30.16b + xar v29.2d, v1.2d, v25.2d, (64 - 1) + xar v1.2d, v6.2d, v25.2d, (64 - 44) + xar v6.2d, v9.2d, v28.2d, (64 - 20) + xar v9.2d, v22.2d, v26.2d, (64 - 61) + xar v22.2d, v14.2d, v28.2d, (64 - 39) + xar v14.2d, v20.2d, v30.2d, (64 - 18) + xar v31.2d, v2.2d, v26.2d, (64 - 62) + xar v2.2d, v12.2d, v26.2d, (64 - 43) + xar v12.2d, v13.2d, v27.2d, (64 - 25) + xar v13.2d, v19.2d, v28.2d, (64 - 8) + xar v19.2d, v23.2d, v27.2d, (64 - 56) + xar v23.2d, v15.2d, v30.2d, (64 - 41) + xar v15.2d, v4.2d, v28.2d, (64 - 27) + xar v28.2d, v24.2d, v28.2d, (64 - 14) + xar v24.2d, v21.2d, v25.2d, (64 - 2) + xar v8.2d, v8.2d, v27.2d, (64 - 55) + xar v4.2d, v16.2d, v25.2d, (64 - 45) + xar v16.2d, v5.2d, v30.2d, (64 - 36) + xar v5.2d, v3.2d, v27.2d, (64 - 28) + xar v27.2d, v18.2d, v27.2d, (64 - 21) + xar v3.2d, v17.2d, v26.2d, (64 - 15) + xar v25.2d, v11.2d, v25.2d, (64 - 10) + xar v26.2d, v7.2d, v26.2d, (64 - 6) + xar v30.2d, v10.2d, v30.2d, (64 - 3) + + bcax v20.16b, v31.16b, v22.16b, v8.16b + bcax v21.16b, v8.16b, v23.16b, v22.16b + bcax v22.16b, v22.16b, v24.16b, v23.16b + bcax v23.16b, v23.16b, v31.16b, v24.16b + bcax v24.16b, v24.16b, v8.16b, v31.16b + + ld1r {v31.2d}, [x9], #8 + + bcax v17.16b, v25.16b, v19.16b, v3.16b + bcax v18.16b, v3.16b, v15.16b, v19.16b + bcax v19.16b, v19.16b, v16.16b, v15.16b + bcax v15.16b, v15.16b, v25.16b, v16.16b + bcax v16.16b, v16.16b, v3.16b, v25.16b + + bcax v10.16b, v29.16b, v12.16b, v26.16b + bcax v11.16b, v26.16b, v13.16b, v12.16b + bcax v12.16b, v12.16b, v14.16b, v13.16b + bcax v13.16b, v13.16b, v29.16b, v14.16b + bcax v14.16b, v14.16b, v26.16b, v29.16b + + bcax v7.16b, v30.16b, v9.16b, v4.16b + bcax v8.16b, v4.16b, v5.16b, v9.16b + bcax v9.16b, v9.16b, v6.16b, v5.16b + bcax v5.16b, v5.16b, v30.16b, v6.16b + bcax v6.16b, v6.16b, v4.16b, v30.16b + + bcax v3.16b, v27.16b, v0.16b, v28.16b + bcax v4.16b, v28.16b, v1.16b, v0.16b + bcax v0.16b, v0.16b, v2.16b, v1.16b + bcax v1.16b, v1.16b, v27.16b, v2.16b + bcax v2.16b, v2.16b, v28.16b, v27.16b + + eor v0.16b, v0.16b, v31.16b + + cbnz w8, 3b + cbnz w2, 0b + + /* save state */ + st1 { v0.1d- v3.1d}, [x0], #32 + st1 { v4.1d- v7.1d}, [x0], #32 + st1 { v8.1d-v11.1d}, [x0], #32 + st1 {v12.1d-v15.1d}, [x0], #32 + st1 {v16.1d-v19.1d}, [x0], #32 + st1 {v20.1d-v23.1d}, [x0], #32 + st1 {v24.1d}, [x0] + ret +ENDPROC(sha3_ce_transform) + + .section ".rodata", "a" + .align 8 +.Lsha3_rcon: + .quad 0x0000000000000001, 0x0000000000008082, 0x800000000000808a + .quad 0x8000000080008000, 0x000000000000808b, 0x0000000080000001 + .quad 0x8000000080008081, 0x8000000000008009, 0x000000000000008a + .quad 0x0000000000000088, 0x0000000080008009, 0x000000008000000a + .quad 0x000000008000808b, 0x800000000000008b, 0x8000000000008089 + .quad 0x8000000000008003, 0x8000000000008002, 0x8000000000000080 + .quad 0x000000000000800a, 0x800000008000000a, 0x8000000080008081 + .quad 0x8000000000008080, 0x0000000080000001, 0x8000000080008008 diff --git a/arch/arm64/crypto/sha3-ce-glue.c b/arch/arm64/crypto/sha3-ce-glue.c new file mode 100644 index 000000000000..da8222e528bd --- /dev/null +++ b/arch/arm64/crypto/sha3-ce-glue.c @@ -0,0 +1,161 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * sha3-ce-glue.c - core SHA-3 transform using v8.2 Crypto Extensions + * + * Copyright (C) 2018 Linaro Ltd + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +MODULE_DESCRIPTION("SHA3 secure hash using ARMv8 Crypto Extensions"); +MODULE_AUTHOR("Ard Biesheuvel "); +MODULE_LICENSE("GPL v2"); + +asmlinkage void sha3_ce_transform(u64 *st, const u8 *data, int blocks, + int md_len); + +static int sha3_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct sha3_state *sctx = shash_desc_ctx(desc); + unsigned int digest_size = crypto_shash_digestsize(desc->tfm); + + if (!may_use_simd()) + return crypto_sha3_update(desc, data, len); + + if ((sctx->partial + len) >= sctx->rsiz) { + int blocks; + + if (sctx->partial) { + int p = sctx->rsiz - sctx->partial; + + memcpy(sctx->buf + sctx->partial, data, p); + kernel_neon_begin(); + sha3_ce_transform(sctx->st, sctx->buf, 1, digest_size); + kernel_neon_end(); + + data += p; + len -= p; + sctx->partial = 0; + } + + blocks = len / sctx->rsiz; + len %= sctx->rsiz; + + if (blocks) { + kernel_neon_begin(); + sha3_ce_transform(sctx->st, data, blocks, digest_size); + kernel_neon_end(); + data += blocks * sctx->rsiz; + } + } + + if (len) { + memcpy(sctx->buf + sctx->partial, data, len); + sctx->partial += len; + } + return 0; +} + +static int sha3_final(struct shash_desc *desc, u8 *out) +{ + struct sha3_state *sctx = shash_desc_ctx(desc); + unsigned int digest_size = crypto_shash_digestsize(desc->tfm); + __le64 *digest = (__le64 *)out; + int i; + + if (!may_use_simd()) + return crypto_sha3_final(desc, out); + + sctx->buf[sctx->partial++] = 0x06; + memset(sctx->buf + sctx->partial, 0, sctx->rsiz - sctx->partial); + sctx->buf[sctx->rsiz - 1] |= 0x80; + + kernel_neon_begin(); + sha3_ce_transform(sctx->st, sctx->buf, 1, digest_size); + kernel_neon_end(); + + for (i = 0; i < digest_size / 8; i++) + put_unaligned_le64(sctx->st[i], digest++); + + if (digest_size & 4) + put_unaligned_le32(sctx->st[i], (__le32 *)digest); + + *sctx = (struct sha3_state){}; + return 0; +} + +static struct shash_alg algs[] = { { + .digestsize = SHA3_224_DIGEST_SIZE, + .init = crypto_sha3_init, + .update = sha3_update, + .final = sha3_final, + .descsize = sizeof(struct sha3_state), + .base.cra_name = "sha3-224", + .base.cra_driver_name = "sha3-224-ce", + .base.cra_flags = CRYPTO_ALG_TYPE_SHASH, + .base.cra_blocksize = SHA3_224_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, + .base.cra_priority = 200, +}, { + .digestsize = SHA3_256_DIGEST_SIZE, + .init = crypto_sha3_init, + .update = sha3_update, + .final = sha3_final, + .descsize = sizeof(struct sha3_state), + .base.cra_name = "sha3-256", + .base.cra_driver_name = "sha3-256-ce", + .base.cra_flags = CRYPTO_ALG_TYPE_SHASH, + .base.cra_blocksize = SHA3_256_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, + .base.cra_priority = 200, +}, { + .digestsize = SHA3_384_DIGEST_SIZE, + .init = crypto_sha3_init, + .update = sha3_update, + .final = sha3_final, + .descsize = sizeof(struct sha3_state), + .base.cra_name = "sha3-384", + .base.cra_driver_name = "sha3-384-ce", + .base.cra_flags = CRYPTO_ALG_TYPE_SHASH, + .base.cra_blocksize = SHA3_384_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, + .base.cra_priority = 200, +}, { + .digestsize = SHA3_512_DIGEST_SIZE, + .init = crypto_sha3_init, + .update = sha3_update, + .final = sha3_final, + .descsize = sizeof(struct sha3_state), + .base.cra_name = "sha3-512", + .base.cra_driver_name = "sha3-512-ce", + .base.cra_flags = CRYPTO_ALG_TYPE_SHASH, + .base.cra_blocksize = SHA3_512_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, + .base.cra_priority = 200, +} }; + +static int __init sha3_neon_mod_init(void) +{ + return crypto_register_shashes(algs, ARRAY_SIZE(algs)); +} + +static void __exit sha3_neon_mod_fini(void) +{ + crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); +} + +module_cpu_feature_match(SHA3, sha3_neon_mod_init); +module_exit(sha3_neon_mod_fini); From 140aa50d68ea01e7caea6900f6f51882d4dd65c5 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 19 Jan 2018 12:04:39 +0000 Subject: [PATCH 164/166] crypto: arm64/sm3 - new v8.2 Crypto Extensions implementation Implement the Chinese SM3 secure hash algorithm using the new special instructions that have been introduced as an optional extension in ARMv8.2. Tested-by: Steve Capper Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/Kconfig | 6 ++ arch/arm64/crypto/Makefile | 3 + arch/arm64/crypto/sm3-ce-core.S | 141 ++++++++++++++++++++++++++++++++ arch/arm64/crypto/sm3-ce-glue.c | 92 +++++++++++++++++++++ 4 files changed, 242 insertions(+) create mode 100644 arch/arm64/crypto/sm3-ce-core.S create mode 100644 arch/arm64/crypto/sm3-ce-glue.c diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index 3321b2c9a2b5..285c36c7b408 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -41,6 +41,12 @@ config CRYPTO_SHA3_ARM64 select CRYPTO_HASH select CRYPTO_SHA3 +config CRYPTO_SM3_ARM64_CE + tristate "SM3 digest algorithm (ARMv8.2 Crypto Extensions)" + depends on KERNEL_MODE_NEON + select CRYPTO_HASH + select CRYPTO_SM3 + config CRYPTO_GHASH_ARM64_CE tristate "GHASH/AES-GCM using ARMv8 Crypto Extensions" depends on KERNEL_MODE_NEON diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile index 4ca2d146e213..cee9b8d9830b 100644 --- a/arch/arm64/crypto/Makefile +++ b/arch/arm64/crypto/Makefile @@ -20,6 +20,9 @@ sha512-ce-y := sha512-ce-glue.o sha512-ce-core.o obj-$(CONFIG_CRYPTO_SHA3_ARM64) += sha3-ce.o sha3-ce-y := sha3-ce-glue.o sha3-ce-core.o +obj-$(CONFIG_CRYPTO_SM3_ARM64_CE) += sm3-ce.o +sm3-ce-y := sm3-ce-glue.o sm3-ce-core.o + obj-$(CONFIG_CRYPTO_GHASH_ARM64_CE) += ghash-ce.o ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o diff --git a/arch/arm64/crypto/sm3-ce-core.S b/arch/arm64/crypto/sm3-ce-core.S new file mode 100644 index 000000000000..27169fe07a68 --- /dev/null +++ b/arch/arm64/crypto/sm3-ce-core.S @@ -0,0 +1,141 @@ +/* + * sm3-ce-core.S - SM3 secure hash using ARMv8.2 Crypto Extensions + * + * Copyright (C) 2018 Linaro Ltd + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include + + .irp b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 + .set .Lv\b\().4s, \b + .endr + + .macro sm3partw1, rd, rn, rm + .inst 0xce60c000 | .L\rd | (.L\rn << 5) | (.L\rm << 16) + .endm + + .macro sm3partw2, rd, rn, rm + .inst 0xce60c400 | .L\rd | (.L\rn << 5) | (.L\rm << 16) + .endm + + .macro sm3ss1, rd, rn, rm, ra + .inst 0xce400000 | .L\rd | (.L\rn << 5) | (.L\ra << 10) | (.L\rm << 16) + .endm + + .macro sm3tt1a, rd, rn, rm, imm2 + .inst 0xce408000 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16) + .endm + + .macro sm3tt1b, rd, rn, rm, imm2 + .inst 0xce408400 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16) + .endm + + .macro sm3tt2a, rd, rn, rm, imm2 + .inst 0xce408800 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16) + .endm + + .macro sm3tt2b, rd, rn, rm, imm2 + .inst 0xce408c00 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16) + .endm + + .macro round, ab, s0, t0, t1, i + sm3ss1 v5.4s, v8.4s, \t0\().4s, v9.4s + shl \t1\().4s, \t0\().4s, #1 + sri \t1\().4s, \t0\().4s, #31 + sm3tt1\ab v8.4s, v5.4s, v10.4s, \i + sm3tt2\ab v9.4s, v5.4s, \s0\().4s, \i + .endm + + .macro qround, ab, s0, s1, s2, s3, s4 + .ifnb \s4 + ext \s4\().16b, \s1\().16b, \s2\().16b, #12 + ext v6.16b, \s0\().16b, \s1\().16b, #12 + ext v7.16b, \s2\().16b, \s3\().16b, #8 + sm3partw1 \s4\().4s, \s0\().4s, \s3\().4s + .endif + + eor v10.16b, \s0\().16b, \s1\().16b + + round \ab, \s0, v11, v12, 0 + round \ab, \s0, v12, v11, 1 + round \ab, \s0, v11, v12, 2 + round \ab, \s0, v12, v11, 3 + + .ifnb \s4 + sm3partw2 \s4\().4s, v7.4s, v6.4s + .endif + .endm + + /* + * void sm3_ce_transform(struct sm3_state *sst, u8 const *src, + * int blocks) + */ + .text +ENTRY(sm3_ce_transform) + /* load state */ + ld1 {v8.4s-v9.4s}, [x0] + rev64 v8.4s, v8.4s + rev64 v9.4s, v9.4s + ext v8.16b, v8.16b, v8.16b, #8 + ext v9.16b, v9.16b, v9.16b, #8 + + adr_l x8, .Lt + ldp s13, s14, [x8] + + /* load input */ +0: ld1 {v0.16b-v3.16b}, [x1], #64 + sub w2, w2, #1 + + mov v15.16b, v8.16b + mov v16.16b, v9.16b + +CPU_LE( rev32 v0.16b, v0.16b ) +CPU_LE( rev32 v1.16b, v1.16b ) +CPU_LE( rev32 v2.16b, v2.16b ) +CPU_LE( rev32 v3.16b, v3.16b ) + + ext v11.16b, v13.16b, v13.16b, #4 + + qround a, v0, v1, v2, v3, v4 + qround a, v1, v2, v3, v4, v0 + qround a, v2, v3, v4, v0, v1 + qround a, v3, v4, v0, v1, v2 + + ext v11.16b, v14.16b, v14.16b, #4 + + qround b, v4, v0, v1, v2, v3 + qround b, v0, v1, v2, v3, v4 + qround b, v1, v2, v3, v4, v0 + qround b, v2, v3, v4, v0, v1 + qround b, v3, v4, v0, v1, v2 + qround b, v4, v0, v1, v2, v3 + qround b, v0, v1, v2, v3, v4 + qround b, v1, v2, v3, v4, v0 + qround b, v2, v3, v4, v0, v1 + qround b, v3, v4 + qround b, v4, v0 + qround b, v0, v1 + + eor v8.16b, v8.16b, v15.16b + eor v9.16b, v9.16b, v16.16b + + /* handled all input blocks? */ + cbnz w2, 0b + + /* save state */ + rev64 v8.4s, v8.4s + rev64 v9.4s, v9.4s + ext v8.16b, v8.16b, v8.16b, #8 + ext v9.16b, v9.16b, v9.16b, #8 + st1 {v8.4s-v9.4s}, [x0] + ret +ENDPROC(sm3_ce_transform) + + .section ".rodata", "a" + .align 3 +.Lt: .word 0x79cc4519, 0x9d8a7a87 diff --git a/arch/arm64/crypto/sm3-ce-glue.c b/arch/arm64/crypto/sm3-ce-glue.c new file mode 100644 index 000000000000..3b4948f7e26f --- /dev/null +++ b/arch/arm64/crypto/sm3-ce-glue.c @@ -0,0 +1,92 @@ +/* + * sm3-ce-glue.c - SM3 secure hash using ARMv8.2 Crypto Extensions + * + * Copyright (C) 2018 Linaro Ltd + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +MODULE_DESCRIPTION("SM3 secure hash using ARMv8 Crypto Extensions"); +MODULE_AUTHOR("Ard Biesheuvel "); +MODULE_LICENSE("GPL v2"); + +asmlinkage void sm3_ce_transform(struct sm3_state *sst, u8 const *src, + int blocks); + +static int sm3_ce_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + if (!may_use_simd()) + return crypto_sm3_update(desc, data, len); + + kernel_neon_begin(); + sm3_base_do_update(desc, data, len, sm3_ce_transform); + kernel_neon_end(); + + return 0; +} + +static int sm3_ce_final(struct shash_desc *desc, u8 *out) +{ + if (!may_use_simd()) + return crypto_sm3_finup(desc, NULL, 0, out); + + kernel_neon_begin(); + sm3_base_do_finalize(desc, sm3_ce_transform); + kernel_neon_end(); + + return sm3_base_finish(desc, out); +} + +static int sm3_ce_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) +{ + if (!may_use_simd()) + return crypto_sm3_finup(desc, data, len, out); + + kernel_neon_begin(); + sm3_base_do_update(desc, data, len, sm3_ce_transform); + kernel_neon_end(); + + return sm3_ce_final(desc, out); +} + +static struct shash_alg sm3_alg = { + .digestsize = SM3_DIGEST_SIZE, + .init = sm3_base_init, + .update = sm3_ce_update, + .final = sm3_ce_final, + .finup = sm3_ce_finup, + .descsize = sizeof(struct sm3_state), + .base.cra_name = "sm3", + .base.cra_driver_name = "sm3-ce", + .base.cra_flags = CRYPTO_ALG_TYPE_SHASH, + .base.cra_blocksize = SM3_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, + .base.cra_priority = 200, +}; + +static int __init sm3_ce_mod_init(void) +{ + return crypto_register_shash(&sm3_alg); +} + +static void __exit sm3_ce_mod_fini(void) +{ + crypto_unregister_shash(&sm3_alg); +} + +module_cpu_feature_match(SM3, sm3_ce_mod_init); +module_exit(sm3_ce_mod_fini); From fb87127bcefc17efab757606e1b1e333fd614dd0 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 19 Jan 2018 12:04:40 +0000 Subject: [PATCH 165/166] crypto: arm64/sha512 - fix/improve new v8.2 Crypto Extensions code Add a missing symbol export that prevents this code to be built as a module. Also, move the round constant table to the .rodata section, and use a more optimized version of the core transform. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/sha512-ce-core.S | 131 ++++++++++++++--------------- arch/arm64/crypto/sha512-glue.c | 1 + 2 files changed, 65 insertions(+), 67 deletions(-) diff --git a/arch/arm64/crypto/sha512-ce-core.S b/arch/arm64/crypto/sha512-ce-core.S index 6c562f8df0b0..7f3bca5c59a2 100644 --- a/arch/arm64/crypto/sha512-ce-core.S +++ b/arch/arm64/crypto/sha512-ce-core.S @@ -12,10 +12,7 @@ #include #include - // - // Temporary - for testing only. binutils has no support for these yet - // - .irp b,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 + .irp b,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19 .set .Lq\b, \b .set .Lv\b\().2d, \b .endr @@ -36,12 +33,10 @@ .inst 0xce608800 | .L\rd | (.L\rn << 5) | (.L\rm << 16) .endm - .text - .arch armv8-a+crypto - /* * The SHA-512 round constants */ + .section ".rodata", "a" .align 4 .Lsha512_rcon: .quad 0x428a2f98d728ae22, 0x7137449123ef65cd @@ -87,20 +82,20 @@ .macro dround, i0, i1, i2, i3, i4, rc0, rc1, in0, in1, in2, in3, in4 .ifnb \rc1 - ld1 {v\rc1\().2d}, [x3], #16 + ld1 {v\rc1\().2d}, [x4], #16 .endif - add v\rc0\().2d, v\rc0\().2d, v\in0\().2d + add v5.2d, v\rc0\().2d, v\in0\().2d ext v6.16b, v\i2\().16b, v\i3\().16b, #8 - ext v\rc0\().16b, v\rc0\().16b, v\rc0\().16b, #8 + ext v5.16b, v5.16b, v5.16b, #8 ext v7.16b, v\i1\().16b, v\i2\().16b, #8 - add v\i3\().2d, v\i3\().2d, v\rc0\().2d + add v\i3\().2d, v\i3\().2d, v5.2d .ifnb \in1 - ext v10.16b, v\in3\().16b, v\in4\().16b, #8 + ext v5.16b, v\in3\().16b, v\in4\().16b, #8 sha512su0 v\in0\().2d, v\in1\().2d .endif sha512h q\i3, q6, v7.2d .ifnb \in1 - sha512su1 v\in0\().2d, v\in2\().2d, v10.2d + sha512su1 v\in0\().2d, v\in2\().2d, v5.2d .endif add v\i4\().2d, v\i1\().2d, v\i3\().2d sha512h2 q\i3, q\i1, v\i0\().2d @@ -110,18 +105,20 @@ * void sha512_ce_transform(struct sha512_state *sst, u8 const *src, * int blocks) */ + .text ENTRY(sha512_ce_transform) /* load state */ - ld1 {v20.2d-v23.2d}, [x0] + ld1 {v8.2d-v11.2d}, [x0] + + /* load first 4 round constants */ + adr_l x3, .Lsha512_rcon + ld1 {v20.2d-v23.2d}, [x3], #64 /* load input */ 0: ld1 {v12.2d-v15.2d}, [x1], #64 ld1 {v16.2d-v19.2d}, [x1], #64 sub w2, w2, #1 - /* load round constants */ - adr x3, .Lsha512_rcon - CPU_LE( rev64 v12.16b, v12.16b ) CPU_LE( rev64 v13.16b, v13.16b ) CPU_LE( rev64 v14.16b, v14.16b ) @@ -131,12 +128,12 @@ CPU_LE( rev64 v17.16b, v17.16b ) CPU_LE( rev64 v18.16b, v18.16b ) CPU_LE( rev64 v19.16b, v19.16b ) - ld1 {v8.2d}, [x3], #16 + mov x4, x3 // rc pointer - mov v0.16b, v20.16b - mov v1.16b, v21.16b - mov v2.16b, v22.16b - mov v3.16b, v23.16b + mov v0.16b, v8.16b + mov v1.16b, v9.16b + mov v2.16b, v10.16b + mov v3.16b, v11.16b // v0 ab cd -- ef gh ab // v1 cd -- ef gh ab cd @@ -144,64 +141,64 @@ CPU_LE( rev64 v19.16b, v19.16b ) // v3 gh ab cd -- ef gh // v4 -- ef gh ab cd -- - dround 0, 1, 2, 3, 4, 8, 9, 12, 13, 19, 16, 17 - dround 3, 0, 4, 2, 1, 9, 8, 13, 14, 12, 17, 18 - dround 2, 3, 1, 4, 0, 8, 9, 14, 15, 13, 18, 19 - dround 4, 2, 0, 1, 3, 9, 8, 15, 16, 14, 19, 12 - dround 1, 4, 3, 0, 2, 8, 9, 16, 17, 15, 12, 13 + dround 0, 1, 2, 3, 4, 20, 24, 12, 13, 19, 16, 17 + dround 3, 0, 4, 2, 1, 21, 25, 13, 14, 12, 17, 18 + dround 2, 3, 1, 4, 0, 22, 26, 14, 15, 13, 18, 19 + dround 4, 2, 0, 1, 3, 23, 27, 15, 16, 14, 19, 12 + dround 1, 4, 3, 0, 2, 24, 28, 16, 17, 15, 12, 13 - dround 0, 1, 2, 3, 4, 9, 8, 17, 18, 16, 13, 14 - dround 3, 0, 4, 2, 1, 8, 9, 18, 19, 17, 14, 15 - dround 2, 3, 1, 4, 0, 9, 8, 19, 12, 18, 15, 16 - dround 4, 2, 0, 1, 3, 8, 9, 12, 13, 19, 16, 17 - dround 1, 4, 3, 0, 2, 9, 8, 13, 14, 12, 17, 18 + dround 0, 1, 2, 3, 4, 25, 29, 17, 18, 16, 13, 14 + dround 3, 0, 4, 2, 1, 26, 30, 18, 19, 17, 14, 15 + dround 2, 3, 1, 4, 0, 27, 31, 19, 12, 18, 15, 16 + dround 4, 2, 0, 1, 3, 28, 24, 12, 13, 19, 16, 17 + dround 1, 4, 3, 0, 2, 29, 25, 13, 14, 12, 17, 18 - dround 0, 1, 2, 3, 4, 8, 9, 14, 15, 13, 18, 19 - dround 3, 0, 4, 2, 1, 9, 8, 15, 16, 14, 19, 12 - dround 2, 3, 1, 4, 0, 8, 9, 16, 17, 15, 12, 13 - dround 4, 2, 0, 1, 3, 9, 8, 17, 18, 16, 13, 14 - dround 1, 4, 3, 0, 2, 8, 9, 18, 19, 17, 14, 15 + dround 0, 1, 2, 3, 4, 30, 26, 14, 15, 13, 18, 19 + dround 3, 0, 4, 2, 1, 31, 27, 15, 16, 14, 19, 12 + dround 2, 3, 1, 4, 0, 24, 28, 16, 17, 15, 12, 13 + dround 4, 2, 0, 1, 3, 25, 29, 17, 18, 16, 13, 14 + dround 1, 4, 3, 0, 2, 26, 30, 18, 19, 17, 14, 15 - dround 0, 1, 2, 3, 4, 9, 8, 19, 12, 18, 15, 16 - dround 3, 0, 4, 2, 1, 8, 9, 12, 13, 19, 16, 17 - dround 2, 3, 1, 4, 0, 9, 8, 13, 14, 12, 17, 18 - dround 4, 2, 0, 1, 3, 8, 9, 14, 15, 13, 18, 19 - dround 1, 4, 3, 0, 2, 9, 8, 15, 16, 14, 19, 12 + dround 0, 1, 2, 3, 4, 27, 31, 19, 12, 18, 15, 16 + dround 3, 0, 4, 2, 1, 28, 24, 12, 13, 19, 16, 17 + dround 2, 3, 1, 4, 0, 29, 25, 13, 14, 12, 17, 18 + dround 4, 2, 0, 1, 3, 30, 26, 14, 15, 13, 18, 19 + dround 1, 4, 3, 0, 2, 31, 27, 15, 16, 14, 19, 12 - dround 0, 1, 2, 3, 4, 8, 9, 16, 17, 15, 12, 13 - dround 3, 0, 4, 2, 1, 9, 8, 17, 18, 16, 13, 14 - dround 2, 3, 1, 4, 0, 8, 9, 18, 19, 17, 14, 15 - dround 4, 2, 0, 1, 3, 9, 8, 19, 12, 18, 15, 16 - dround 1, 4, 3, 0, 2, 8, 9, 12, 13, 19, 16, 17 + dround 0, 1, 2, 3, 4, 24, 28, 16, 17, 15, 12, 13 + dround 3, 0, 4, 2, 1, 25, 29, 17, 18, 16, 13, 14 + dround 2, 3, 1, 4, 0, 26, 30, 18, 19, 17, 14, 15 + dround 4, 2, 0, 1, 3, 27, 31, 19, 12, 18, 15, 16 + dround 1, 4, 3, 0, 2, 28, 24, 12, 13, 19, 16, 17 - dround 0, 1, 2, 3, 4, 9, 8, 13, 14, 12, 17, 18 - dround 3, 0, 4, 2, 1, 8, 9, 14, 15, 13, 18, 19 - dround 2, 3, 1, 4, 0, 9, 8, 15, 16, 14, 19, 12 - dround 4, 2, 0, 1, 3, 8, 9, 16, 17, 15, 12, 13 - dround 1, 4, 3, 0, 2, 9, 8, 17, 18, 16, 13, 14 + dround 0, 1, 2, 3, 4, 29, 25, 13, 14, 12, 17, 18 + dround 3, 0, 4, 2, 1, 30, 26, 14, 15, 13, 18, 19 + dround 2, 3, 1, 4, 0, 31, 27, 15, 16, 14, 19, 12 + dround 4, 2, 0, 1, 3, 24, 28, 16, 17, 15, 12, 13 + dround 1, 4, 3, 0, 2, 25, 29, 17, 18, 16, 13, 14 - dround 0, 1, 2, 3, 4, 8, 9, 18, 19, 17, 14, 15 - dround 3, 0, 4, 2, 1, 9, 8, 19, 12, 18, 15, 16 - dround 2, 3, 1, 4, 0, 8, 9, 12 - dround 4, 2, 0, 1, 3, 9, 8, 13 - dround 1, 4, 3, 0, 2, 8, 9, 14 + dround 0, 1, 2, 3, 4, 26, 30, 18, 19, 17, 14, 15 + dround 3, 0, 4, 2, 1, 27, 31, 19, 12, 18, 15, 16 + dround 2, 3, 1, 4, 0, 28, 24, 12 + dround 4, 2, 0, 1, 3, 29, 25, 13 + dround 1, 4, 3, 0, 2, 30, 26, 14 - dround 0, 1, 2, 3, 4, 9, 8, 15 - dround 3, 0, 4, 2, 1, 8, 9, 16 - dround 2, 3, 1, 4, 0, 9, 8, 17 - dround 4, 2, 0, 1, 3, 8, 9, 18 - dround 1, 4, 3, 0, 2, 9, , 19 + dround 0, 1, 2, 3, 4, 31, 27, 15 + dround 3, 0, 4, 2, 1, 24, , 16 + dround 2, 3, 1, 4, 0, 25, , 17 + dround 4, 2, 0, 1, 3, 26, , 18 + dround 1, 4, 3, 0, 2, 27, , 19 /* update state */ - add v20.2d, v20.2d, v0.2d - add v21.2d, v21.2d, v1.2d - add v22.2d, v22.2d, v2.2d - add v23.2d, v23.2d, v3.2d + add v8.2d, v8.2d, v0.2d + add v9.2d, v9.2d, v1.2d + add v10.2d, v10.2d, v2.2d + add v11.2d, v11.2d, v3.2d /* handled all input blocks? */ cbnz w2, 0b /* store new state */ -3: st1 {v20.2d-v23.2d}, [x0] +3: st1 {v8.2d-v11.2d}, [x0] ret ENDPROC(sha512_ce_transform) diff --git a/arch/arm64/crypto/sha512-glue.c b/arch/arm64/crypto/sha512-glue.c index aff35c9992a4..27db4851e380 100644 --- a/arch/arm64/crypto/sha512-glue.c +++ b/arch/arm64/crypto/sha512-glue.c @@ -27,6 +27,7 @@ MODULE_ALIAS_CRYPTO("sha512"); asmlinkage void sha512_block_data_order(u32 *digest, const void *data, unsigned int num_blks); +EXPORT_SYMBOL(sha512_block_data_order); static int sha512_update(struct shash_desc *desc, const u8 *data, unsigned int len) From 2d55807b7f7bf62bb05a8b91247c5eb7cd19ac04 Mon Sep 17 00:00:00 2001 From: Alexey Khoroshilov Date: Sat, 20 Jan 2018 00:53:15 +0300 Subject: [PATCH 166/166] crypto: picoxcell - Fix error handling in spacc_probe() If clk_get() fails, device_remove_file() looks inappropriate. The error path, where all crypto_register fail, misses resource deallocations. Found by Linux Driver Verification project (linuxtesting.org). Signed-off-by: Alexey Khoroshilov Reviewed-by: Jamie Iles Signed-off-by: Herbert Xu --- drivers/crypto/picoxcell_crypto.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/drivers/crypto/picoxcell_crypto.c b/drivers/crypto/picoxcell_crypto.c index 5a6dc53b2b9d..4ef52c9d72fc 100644 --- a/drivers/crypto/picoxcell_crypto.c +++ b/drivers/crypto/picoxcell_crypto.c @@ -1618,7 +1618,7 @@ MODULE_DEVICE_TABLE(of, spacc_of_id_table); static int spacc_probe(struct platform_device *pdev) { - int i, err, ret = -EINVAL; + int i, err, ret; struct resource *mem, *irq; struct device_node *np = pdev->dev.of_node; struct spacc_engine *engine = devm_kzalloc(&pdev->dev, sizeof(*engine), @@ -1679,22 +1679,18 @@ static int spacc_probe(struct platform_device *pdev) engine->clk = clk_get(&pdev->dev, "ref"); if (IS_ERR(engine->clk)) { dev_info(&pdev->dev, "clk unavailable\n"); - device_remove_file(&pdev->dev, &dev_attr_stat_irq_thresh); return PTR_ERR(engine->clk); } if (clk_prepare_enable(engine->clk)) { dev_info(&pdev->dev, "unable to prepare/enable clk\n"); - clk_put(engine->clk); - return -EIO; + ret = -EIO; + goto err_clk_put; } - err = device_create_file(&pdev->dev, &dev_attr_stat_irq_thresh); - if (err) { - clk_disable_unprepare(engine->clk); - clk_put(engine->clk); - return err; - } + ret = device_create_file(&pdev->dev, &dev_attr_stat_irq_thresh); + if (ret) + goto err_clk_disable; /* @@ -1725,6 +1721,7 @@ static int spacc_probe(struct platform_device *pdev) platform_set_drvdata(pdev, engine); + ret = -EINVAL; INIT_LIST_HEAD(&engine->registered_algs); for (i = 0; i < engine->num_algs; ++i) { engine->algs[i].engine = engine; @@ -1759,6 +1756,16 @@ static int spacc_probe(struct platform_device *pdev) engine->aeads[i].alg.base.cra_name); } + if (!ret) + return 0; + + del_timer_sync(&engine->packet_timeout); + device_remove_file(&pdev->dev, &dev_attr_stat_irq_thresh); +err_clk_disable: + clk_disable_unprepare(engine->clk); +err_clk_put: + clk_put(engine->clk); + return ret; }