1 // SPDX-License-Identifier: GPL-2.0-or-later
3  * sun4i-ss-cipher.c - hardware cryptographic accelerator for Allwinner A20 SoC
5  * Copyright (C) 2013-2015 Corentin LABBE <[email protected]>
8  * keysize in CBC and ECB mode.
9  * Add support also for DES and 3DES in CBC and ECB mode.
13 #include "sun4i-ss.h"
18 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_opti_poll()  local
19 	struct sun4i_ss_ctx *ss = op->ss;  in sun4i_ss_opti_poll()
22 	u32 mode = ctx->mode;  in sun4i_ss_opti_poll()  local
30 	unsigned int ileft = areq->cryptlen;  in sun4i_ss_opti_poll()
31 	unsigned int oleft = areq->cryptlen;  in sun4i_ss_opti_poll()
41 	if (!areq->cryptlen)  in sun4i_ss_opti_poll()
44 	if (!areq->src || !areq->dst) {  in sun4i_ss_opti_poll()
45 		dev_err_ratelimited(ss->dev, "ERROR: Some SGs are NULL\n");  in sun4i_ss_opti_poll()
46 		return -EINVAL;  in sun4i_ss_opti_poll()
49 	if (areq->iv && ivsize > 0 && mode & SS_DECRYPTION) {  in sun4i_ss_opti_poll()
50 		scatterwalk_map_and_copy(ctx->backup_iv, areq->src,  in sun4i_ss_opti_poll()
51 					 areq->cryptlen - ivsize, ivsize, 0);  in sun4i_ss_opti_poll()
56 		algt->stat_opti++;  in sun4i_ss_opti_poll()
57 		algt->stat_bytes += areq->cryptlen;  in sun4i_ss_opti_poll()
60 	spin_lock_irqsave(&ss->slock, flags);  in sun4i_ss_opti_poll()
62 	for (i = 0; i < op->keylen / 4; i++)  in sun4i_ss_opti_poll()
63 		writesl(ss->base + SS_KEY0 + i * 4, &op->key[i], 1);  in sun4i_ss_opti_poll()
65 	if (areq->iv) {  in sun4i_ss_opti_poll()
67 			v = *(u32 *)(areq->iv + i * 4);  in sun4i_ss_opti_poll()
68 			writesl(ss->base + SS_IV0 + i * 4, &v, 1);  in sun4i_ss_opti_poll()
71 	writel(mode, ss->base + SS_CTL);  in sun4i_ss_opti_poll()
74 	ileft = areq->cryptlen / 4;  in sun4i_ss_opti_poll()
75 	oleft = areq->cryptlen / 4;  in sun4i_ss_opti_poll()
80 			sg_miter_start(&mi, areq->src, sg_nents(areq->src),  in sun4i_ss_opti_poll()
86 				dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n");  in sun4i_ss_opti_poll()
87 				err = -EINVAL;  in sun4i_ss_opti_poll()
91 			todo = min_t(size_t, todo, (mi.length - oi) / 4);  in sun4i_ss_opti_poll()
93 				ileft -= todo;  in sun4i_ss_opti_poll()
94 				writesl(ss->base + SS_RXFIFO, mi.addr + oi, todo);  in sun4i_ss_opti_poll()
104 		spaces = readl(ss->base + SS_FCSR);  in sun4i_ss_opti_poll()
108 		sg_miter_start(&mo, areq->dst, sg_nents(areq->dst),  in sun4i_ss_opti_poll()
114 			dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n");  in sun4i_ss_opti_poll()
115 			err = -EINVAL;  in sun4i_ss_opti_poll()
119 		todo = min_t(size_t, todo, (mo.length - oo) / 4);  in sun4i_ss_opti_poll()
121 			oleft -= todo;  in sun4i_ss_opti_poll()
122 			readsl(ss->base + SS_TXFIFO, mo.addr + oo, todo);  in sun4i_ss_opti_poll()
132 	if (areq->iv) {  in sun4i_ss_opti_poll()
133 		if (mode & SS_DECRYPTION) {  in sun4i_ss_opti_poll()
134 			memcpy(areq->iv, ctx->backup_iv, ivsize);  in sun4i_ss_opti_poll()
135 			memzero_explicit(ctx->backup_iv, ivsize);  in sun4i_ss_opti_poll()
137 			scatterwalk_map_and_copy(areq->iv, areq->dst, areq->cryptlen - ivsize,  in sun4i_ss_opti_poll()
143 	writel(0, ss->base + SS_CTL);  in sun4i_ss_opti_poll()
144 	spin_unlock_irqrestore(&ss->slock, flags);  in sun4i_ss_opti_poll()
151 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_cipher_poll_fallback()  local
159 		algt->stat_fb++;  in sun4i_ss_cipher_poll_fallback()
162 	skcipher_request_set_tfm(&ctx->fallback_req, op->fallback_tfm);  in sun4i_ss_cipher_poll_fallback()
163 	skcipher_request_set_callback(&ctx->fallback_req, areq->base.flags,  in sun4i_ss_cipher_poll_fallback()
164 				      areq->base.complete, areq->base.data);  in sun4i_ss_cipher_poll_fallback()
165 	skcipher_request_set_crypt(&ctx->fallback_req, areq->src, areq->dst,  in sun4i_ss_cipher_poll_fallback()
166 				   areq->cryptlen, areq->iv);  in sun4i_ss_cipher_poll_fallback()
167 	if (ctx->mode & SS_DECRYPTION)  in sun4i_ss_cipher_poll_fallback()
168 		err = crypto_skcipher_decrypt(&ctx->fallback_req);  in sun4i_ss_cipher_poll_fallback()
170 		err = crypto_skcipher_encrypt(&ctx->fallback_req);  in sun4i_ss_cipher_poll_fallback()
179 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_cipher_poll()  local
180 	struct sun4i_ss_ctx *ss = op->ss;  in sun4i_ss_cipher_poll()
182 	struct scatterlist *in_sg = areq->src;  in sun4i_ss_cipher_poll()
183 	struct scatterlist *out_sg = areq->dst;  in sun4i_ss_cipher_poll()
188 	u32 mode = ctx->mode;  in sun4i_ss_cipher_poll()  local
196 	unsigned int ileft = areq->cryptlen;  in sun4i_ss_cipher_poll()
197 	unsigned int oleft = areq->cryptlen;  in sun4i_ss_cipher_poll()
209 	if (!areq->cryptlen)  in sun4i_ss_cipher_poll()
212 	if (!areq->src || !areq->dst) {  in sun4i_ss_cipher_poll()
213 		dev_err_ratelimited(ss->dev, "ERROR: Some SGs are NULL\n");  in sun4i_ss_cipher_poll()
214 		return -EINVAL;  in sun4i_ss_cipher_poll()
218 	if (areq->cryptlen % algt->alg.crypto.base.cra_blocksize)  in sun4i_ss_cipher_poll()
226 		if ((in_sg->length | in_sg->offset) & 3u)  in sun4i_ss_cipher_poll()
231 		if ((out_sg->length | out_sg->offset) & 3u)  in sun4i_ss_cipher_poll()
242 	if (areq->iv && ivsize > 0 && mode & SS_DECRYPTION) {  in sun4i_ss_cipher_poll()
243 		scatterwalk_map_and_copy(ctx->backup_iv, areq->src,  in sun4i_ss_cipher_poll()
244 					 areq->cryptlen - ivsize, ivsize, 0);  in sun4i_ss_cipher_poll()
248 		algt->stat_req++;  in sun4i_ss_cipher_poll()
249 		algt->stat_bytes += areq->cryptlen;  in sun4i_ss_cipher_poll()
252 	spin_lock_irqsave(&ss->slock, flags);  in sun4i_ss_cipher_poll()
254 	for (i = 0; i < op->keylen / 4; i++)  in sun4i_ss_cipher_poll()
255 		writesl(ss->base + SS_KEY0 + i * 4, &op->key[i], 1);  in sun4i_ss_cipher_poll()
257 	if (areq->iv) {  in sun4i_ss_cipher_poll()
259 			v = *(u32 *)(areq->iv + i * 4);  in sun4i_ss_cipher_poll()
260 			writesl(ss->base + SS_IV0 + i * 4, &v, 1);  in sun4i_ss_cipher_poll()
263 	writel(mode, ss->base + SS_CTL);  in sun4i_ss_cipher_poll()
265 	ileft = areq->cryptlen;  in sun4i_ss_cipher_poll()
266 	oleft = areq->cryptlen;  in sun4i_ss_cipher_poll()
272 			sg_miter_start(&mi, areq->src, sg_nents(areq->src),  in sun4i_ss_cipher_poll()
278 				dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n");  in sun4i_ss_cipher_poll()
279 				err = -EINVAL;  in sun4i_ss_cipher_poll()
287 			todo = min_t(size_t, todo, (mi.length - oi) / 4);  in sun4i_ss_cipher_poll()
289 				writesl(ss->base + SS_RXFIFO, mi.addr + oi,  in sun4i_ss_cipher_poll()
291 				ileft -= todo * 4;  in sun4i_ss_cipher_poll()
301 				todo = min(rx_cnt * 4 - ob, ileft);  in sun4i_ss_cipher_poll()
302 				todo = min_t(size_t, todo, mi.length - oi);  in sun4i_ss_cipher_poll()
303 				memcpy(ss->buf + ob, mi.addr + oi, todo);  in sun4i_ss_cipher_poll()
304 				ileft -= todo;  in sun4i_ss_cipher_poll()
308 					writesl(ss->base + SS_RXFIFO, ss->buf,  in sun4i_ss_cipher_poll()
320 		spaces = readl(ss->base + SS_FCSR);  in sun4i_ss_cipher_poll()
326 		sg_miter_start(&mo, areq->dst, sg_nents(areq->dst),  in sun4i_ss_cipher_poll()
332 			dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n");  in sun4i_ss_cipher_poll()
333 			err = -EINVAL;  in sun4i_ss_cipher_poll()
338 		todo = min_t(size_t, todo, (mo.length - oo) / 4);  in sun4i_ss_cipher_poll()
341 			readsl(ss->base + SS_TXFIFO, mo.addr + oo, todo);  in sun4i_ss_cipher_poll()
342 			oleft -= todo * 4;  in sun4i_ss_cipher_poll()
353 			readsl(ss->base + SS_TXFIFO, ss->bufo, tx_cnt);  in sun4i_ss_cipher_poll()
364 					     mo.length - oo, obl - obo);  in sun4i_ss_cipher_poll()
365 				memcpy(mo.addr + oo, ss->bufo + obo, todo);  in sun4i_ss_cipher_poll()
366 				oleft -= todo;  in sun4i_ss_cipher_poll()
379 	if (areq->iv) {  in sun4i_ss_cipher_poll()
380 		if (mode & SS_DECRYPTION) {  in sun4i_ss_cipher_poll()
381 			memcpy(areq->iv, ctx->backup_iv, ivsize);  in sun4i_ss_cipher_poll()
382 			memzero_explicit(ctx->backup_iv, ivsize);  in sun4i_ss_cipher_poll()
384 			scatterwalk_map_and_copy(areq->iv, areq->dst, areq->cryptlen - ivsize,  in sun4i_ss_cipher_poll()
390 	writel(0, ss->base + SS_CTL);  in sun4i_ss_cipher_poll()
391 	spin_unlock_irqrestore(&ss->slock, flags);  in sun4i_ss_cipher_poll()
400 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_cbc_aes_encrypt()  local
403 	rctx->mode = SS_OP_AES | SS_CBC | SS_ENABLED | SS_ENCRYPTION |  in sun4i_ss_cbc_aes_encrypt()
404 		op->keymode;  in sun4i_ss_cbc_aes_encrypt()
411 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_cbc_aes_decrypt()  local
414 	rctx->mode = SS_OP_AES | SS_CBC | SS_ENABLED | SS_DECRYPTION |  in sun4i_ss_cbc_aes_decrypt()
415 		op->keymode;  in sun4i_ss_cbc_aes_decrypt()
423 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_ecb_aes_encrypt()  local
426 	rctx->mode = SS_OP_AES | SS_ECB | SS_ENABLED | SS_ENCRYPTION |  in sun4i_ss_ecb_aes_encrypt()
427 		op->keymode;  in sun4i_ss_ecb_aes_encrypt()
434 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_ecb_aes_decrypt()  local
437 	rctx->mode = SS_OP_AES | SS_ECB | SS_ENABLED | SS_DECRYPTION |  in sun4i_ss_ecb_aes_decrypt()
438 		op->keymode;  in sun4i_ss_ecb_aes_decrypt()
446 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_cbc_des_encrypt()  local
449 	rctx->mode = SS_OP_DES | SS_CBC | SS_ENABLED | SS_ENCRYPTION |  in sun4i_ss_cbc_des_encrypt()
450 		op->keymode;  in sun4i_ss_cbc_des_encrypt()
457 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_cbc_des_decrypt()  local
460 	rctx->mode = SS_OP_DES | SS_CBC | SS_ENABLED | SS_DECRYPTION |  in sun4i_ss_cbc_des_decrypt()
461 		op->keymode;  in sun4i_ss_cbc_des_decrypt()
469 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_ecb_des_encrypt()  local
472 	rctx->mode = SS_OP_DES | SS_ECB | SS_ENABLED | SS_ENCRYPTION |  in sun4i_ss_ecb_des_encrypt()
473 		op->keymode;  in sun4i_ss_ecb_des_encrypt()
480 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_ecb_des_decrypt()  local
483 	rctx->mode = SS_OP_DES | SS_ECB | SS_ENABLED | SS_DECRYPTION |  in sun4i_ss_ecb_des_decrypt()
484 		op->keymode;  in sun4i_ss_ecb_des_decrypt()
492 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_cbc_des3_encrypt()  local
495 	rctx->mode = SS_OP_3DES | SS_CBC | SS_ENABLED | SS_ENCRYPTION |  in sun4i_ss_cbc_des3_encrypt()
496 		op->keymode;  in sun4i_ss_cbc_des3_encrypt()
503 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_cbc_des3_decrypt()  local
506 	rctx->mode = SS_OP_3DES | SS_CBC | SS_ENABLED | SS_DECRYPTION |  in sun4i_ss_cbc_des3_decrypt()
507 		op->keymode;  in sun4i_ss_cbc_des3_decrypt()
515 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_ecb_des3_encrypt()  local
518 	rctx->mode = SS_OP_3DES | SS_ECB | SS_ENABLED | SS_ENCRYPTION |  in sun4i_ss_ecb_des3_encrypt()
519 		op->keymode;  in sun4i_ss_ecb_des3_encrypt()
526 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_ecb_des3_decrypt()  local
529 	rctx->mode = SS_OP_3DES | SS_ECB | SS_ENABLED | SS_DECRYPTION |  in sun4i_ss_ecb_des3_decrypt()
530 		op->keymode;  in sun4i_ss_ecb_des3_decrypt()
536 	struct sun4i_tfm_ctx *op = crypto_tfm_ctx(tfm);  in sun4i_ss_cipher_init()  local
541 	memset(op, 0, sizeof(struct sun4i_tfm_ctx));  in sun4i_ss_cipher_init()
543 	algt = container_of(tfm->__crt_alg, struct sun4i_ss_alg_template,  in sun4i_ss_cipher_init()
545 	op->ss = algt->ss;  in sun4i_ss_cipher_init()
547 	op->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);  in sun4i_ss_cipher_init()
548 	if (IS_ERR(op->fallback_tfm)) {  in sun4i_ss_cipher_init()
549 		dev_err(op->ss->dev, "ERROR: Cannot allocate fallback for %s %ld\n",  in sun4i_ss_cipher_init()
550 			name, PTR_ERR(op->fallback_tfm));  in sun4i_ss_cipher_init()
551 		return PTR_ERR(op->fallback_tfm);  in sun4i_ss_cipher_init()
556 				    crypto_skcipher_reqsize(op->fallback_tfm));  in sun4i_ss_cipher_init()
558 	err = pm_runtime_resume_and_get(op->ss->dev);  in sun4i_ss_cipher_init()
564 	crypto_free_skcipher(op->fallback_tfm);  in sun4i_ss_cipher_init()
570 	struct sun4i_tfm_ctx *op = crypto_tfm_ctx(tfm);  in sun4i_ss_cipher_exit()  local
572 	crypto_free_skcipher(op->fallback_tfm);  in sun4i_ss_cipher_exit()
573 	pm_runtime_put(op->ss->dev);  in sun4i_ss_cipher_exit()
576 /* check and set the AES key, prepare the mode to be used */
580 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_aes_setkey()  local
581 	struct sun4i_ss_ctx *ss = op->ss;  in sun4i_ss_aes_setkey()
585 		op->keymode = SS_AES_128BITS;  in sun4i_ss_aes_setkey()
588 		op->keymode = SS_AES_192BITS;  in sun4i_ss_aes_setkey()
591 		op->keymode = SS_AES_256BITS;  in sun4i_ss_aes_setkey()
594 		dev_dbg(ss->dev, "ERROR: Invalid keylen %u\n", keylen);  in sun4i_ss_aes_setkey()
595 		return -EINVAL;  in sun4i_ss_aes_setkey()
597 	op->keylen = keylen;  in sun4i_ss_aes_setkey()
598 	memcpy(op->key, key, keylen);  in sun4i_ss_aes_setkey()
600 	crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK);  in sun4i_ss_aes_setkey()
601 	crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);  in sun4i_ss_aes_setkey()
603 	return crypto_skcipher_setkey(op->fallback_tfm, key, keylen);  in sun4i_ss_aes_setkey()
606 /* check and set the DES key, prepare the mode to be used */
610 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_des_setkey()  local
617 	op->keylen = keylen;  in sun4i_ss_des_setkey()
618 	memcpy(op->key, key, keylen);  in sun4i_ss_des_setkey()
620 	crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK);  in sun4i_ss_des_setkey()
621 	crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);  in sun4i_ss_des_setkey()
623 	return crypto_skcipher_setkey(op->fallback_tfm, key, keylen);  in sun4i_ss_des_setkey()
626 /* check and set the 3DES key, prepare the mode to be used */
630 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_des3_setkey()  local
637 	op->keylen = keylen;  in sun4i_ss_des3_setkey()
638 	memcpy(op->key, key, keylen);  in sun4i_ss_des3_setkey()
640 	crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK);  in sun4i_ss_des3_setkey()
641 	crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);  in sun4i_ss_des3_setkey()
643 	return crypto_skcipher_setkey(op->fallback_tfm, key, keylen);  in sun4i_ss_des3_setkey()