aesni-intel_glue.c revision 54b6a1bd5364aca95cd6ffae00f2b64c6511122c 
1/*
2 * Support for Intel AES-NI instructions. This file contains glue
3 * code, the real AES implementation is in intel-aes_asm.S.
4 *
5 * Copyright (C) 2008, Intel Corp.
6 *    Author: Huang Ying <ying.huang@intel.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 */
13
14#include <linux/hardirq.h>
15#include <linux/types.h>
16#include <linux/crypto.h>
17#include <linux/err.h>
18#include <crypto/algapi.h>
19#include <crypto/aes.h>
20#include <crypto/cryptd.h>
21#include <asm/i387.h>
22#include <asm/aes.h>
23
24struct async_aes_ctx {
25	struct cryptd_ablkcipher *cryptd_tfm;
26};
27
28#define AESNI_ALIGN	16
29#define AES_BLOCK_MASK	(~(AES_BLOCK_SIZE-1))
30
31asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
32			     unsigned int key_len);
33asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
34			  const u8 *in);
35asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
36			  const u8 *in);
37asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
38			      const u8 *in, unsigned int len);
39asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
40			      const u8 *in, unsigned int len);
41asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
42			      const u8 *in, unsigned int len, u8 *iv);
43asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
44			      const u8 *in, unsigned int len, u8 *iv);
45
46static inline int kernel_fpu_using(void)
47{
48	if (in_interrupt() && !(read_cr0() & X86_CR0_TS))
49		return 1;
50	return 0;
51}
52
53static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
54{
55	unsigned long addr = (unsigned long)raw_ctx;
56	unsigned long align = AESNI_ALIGN;
57
58	if (align <= crypto_tfm_ctx_alignment())
59		align = 1;
60	return (struct crypto_aes_ctx *)ALIGN(addr, align);
61}
62
63static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
64			      const u8 *in_key, unsigned int key_len)
65{
66	struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
67	u32 *flags = &tfm->crt_flags;
68	int err;
69
70	if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
71	    key_len != AES_KEYSIZE_256) {
72		*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
73		return -EINVAL;
74	}
75
76	if (kernel_fpu_using())
77		err = crypto_aes_expand_key(ctx, in_key, key_len);
78	else {
79		kernel_fpu_begin();
80		err = aesni_set_key(ctx, in_key, key_len);
81		kernel_fpu_end();
82	}
83
84	return err;
85}
86
87static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
88		       unsigned int key_len)
89{
90	return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
91}
92
93static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
94{
95	struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
96
97	if (kernel_fpu_using())
98		crypto_aes_encrypt_x86(ctx, dst, src);
99	else {
100		kernel_fpu_begin();
101		aesni_enc(ctx, dst, src);
102		kernel_fpu_end();
103	}
104}
105
106static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
107{
108	struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
109
110	if (kernel_fpu_using())
111		crypto_aes_decrypt_x86(ctx, dst, src);
112	else {
113		kernel_fpu_begin();
114		aesni_dec(ctx, dst, src);
115		kernel_fpu_end();
116	}
117}
118
119static struct crypto_alg aesni_alg = {
120	.cra_name		= "aes",
121	.cra_driver_name	= "aes-aesni",
122	.cra_priority		= 300,
123	.cra_flags		= CRYPTO_ALG_TYPE_CIPHER,
124	.cra_blocksize		= AES_BLOCK_SIZE,
125	.cra_ctxsize		= sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
126	.cra_alignmask		= 0,
127	.cra_module		= THIS_MODULE,
128	.cra_list		= LIST_HEAD_INIT(aesni_alg.cra_list),
129	.cra_u	= {
130		.cipher	= {
131			.cia_min_keysize	= AES_MIN_KEY_SIZE,
132			.cia_max_keysize	= AES_MAX_KEY_SIZE,
133			.cia_setkey		= aes_set_key,
134			.cia_encrypt		= aes_encrypt,
135			.cia_decrypt		= aes_decrypt
136		}
137	}
138};
139
140static int ecb_encrypt(struct blkcipher_desc *desc,
141		       struct scatterlist *dst, struct scatterlist *src,
142		       unsigned int nbytes)
143{
144	struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
145	struct blkcipher_walk walk;
146	int err;
147
148	blkcipher_walk_init(&walk, dst, src, nbytes);
149	err = blkcipher_walk_virt(desc, &walk);
150
151	kernel_fpu_begin();
152	while ((nbytes = walk.nbytes)) {
153		aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
154			      nbytes & AES_BLOCK_MASK);
155		nbytes &= AES_BLOCK_SIZE - 1;
156		err = blkcipher_walk_done(desc, &walk, nbytes);
157	}
158	kernel_fpu_end();
159
160	return err;
161}
162
163static int ecb_decrypt(struct blkcipher_desc *desc,
164		       struct scatterlist *dst, struct scatterlist *src,
165		       unsigned int nbytes)
166{
167	struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
168	struct blkcipher_walk walk;
169	int err;
170
171	blkcipher_walk_init(&walk, dst, src, nbytes);
172	err = blkcipher_walk_virt(desc, &walk);
173
174	kernel_fpu_begin();
175	while ((nbytes = walk.nbytes)) {
176		aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
177			      nbytes & AES_BLOCK_MASK);
178		nbytes &= AES_BLOCK_SIZE - 1;
179		err = blkcipher_walk_done(desc, &walk, nbytes);
180	}
181	kernel_fpu_end();
182
183	return err;
184}
185
186static struct crypto_alg blk_ecb_alg = {
187	.cra_name		= "__ecb-aes-aesni",
188	.cra_driver_name	= "__driver-ecb-aes-aesni",
189	.cra_priority		= 0,
190	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
191	.cra_blocksize		= AES_BLOCK_SIZE,
192	.cra_ctxsize		= sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
193	.cra_alignmask		= 0,
194	.cra_type		= &crypto_blkcipher_type,
195	.cra_module		= THIS_MODULE,
196	.cra_list		= LIST_HEAD_INIT(blk_ecb_alg.cra_list),
197	.cra_u = {
198		.blkcipher = {
199			.min_keysize	= AES_MIN_KEY_SIZE,
200			.max_keysize	= AES_MAX_KEY_SIZE,
201			.setkey		= aes_set_key,
202			.encrypt	= ecb_encrypt,
203			.decrypt	= ecb_decrypt,
204		},
205	},
206};
207
208static int cbc_encrypt(struct blkcipher_desc *desc,
209		       struct scatterlist *dst, struct scatterlist *src,
210		       unsigned int nbytes)
211{
212	struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
213	struct blkcipher_walk walk;
214	int err;
215
216	blkcipher_walk_init(&walk, dst, src, nbytes);
217	err = blkcipher_walk_virt(desc, &walk);
218
219	kernel_fpu_begin();
220	while ((nbytes = walk.nbytes)) {
221		aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
222			      nbytes & AES_BLOCK_MASK, walk.iv);
223		nbytes &= AES_BLOCK_SIZE - 1;
224		err = blkcipher_walk_done(desc, &walk, nbytes);
225	}
226	kernel_fpu_end();
227
228	return err;
229}
230
231static int cbc_decrypt(struct blkcipher_desc *desc,
232		       struct scatterlist *dst, struct scatterlist *src,
233		       unsigned int nbytes)
234{
235	struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
236	struct blkcipher_walk walk;
237	int err;
238
239	blkcipher_walk_init(&walk, dst, src, nbytes);
240	err = blkcipher_walk_virt(desc, &walk);
241
242	kernel_fpu_begin();
243	while ((nbytes = walk.nbytes)) {
244		aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
245			      nbytes & AES_BLOCK_MASK, walk.iv);
246		nbytes &= AES_BLOCK_SIZE - 1;
247		err = blkcipher_walk_done(desc, &walk, nbytes);
248	}
249	kernel_fpu_end();
250
251	return err;
252}
253
254static struct crypto_alg blk_cbc_alg = {
255	.cra_name		= "__cbc-aes-aesni",
256	.cra_driver_name	= "__driver-cbc-aes-aesni",
257	.cra_priority		= 0,
258	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
259	.cra_blocksize		= AES_BLOCK_SIZE,
260	.cra_ctxsize		= sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
261	.cra_alignmask		= 0,
262	.cra_type		= &crypto_blkcipher_type,
263	.cra_module		= THIS_MODULE,
264	.cra_list		= LIST_HEAD_INIT(blk_cbc_alg.cra_list),
265	.cra_u = {
266		.blkcipher = {
267			.min_keysize	= AES_MIN_KEY_SIZE,
268			.max_keysize	= AES_MAX_KEY_SIZE,
269			.setkey		= aes_set_key,
270			.encrypt	= cbc_encrypt,
271			.decrypt	= cbc_decrypt,
272		},
273	},
274};
275
276static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
277			unsigned int key_len)
278{
279	struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
280
281	return crypto_ablkcipher_setkey(&ctx->cryptd_tfm->base, key, key_len);
282}
283
284static int ablk_encrypt(struct ablkcipher_request *req)
285{
286	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
287	struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
288
289	if (kernel_fpu_using()) {
290		struct ablkcipher_request *cryptd_req =
291			ablkcipher_request_ctx(req);
292		memcpy(cryptd_req, req, sizeof(*req));
293		ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
294		return crypto_ablkcipher_encrypt(cryptd_req);
295	} else {
296		struct blkcipher_desc desc;
297		desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
298		desc.info = req->info;
299		desc.flags = 0;
300		return crypto_blkcipher_crt(desc.tfm)->encrypt(
301			&desc, req->dst, req->src, req->nbytes);
302	}
303}
304
305static int ablk_decrypt(struct ablkcipher_request *req)
306{
307	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
308	struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
309
310	if (kernel_fpu_using()) {
311		struct ablkcipher_request *cryptd_req =
312			ablkcipher_request_ctx(req);
313		memcpy(cryptd_req, req, sizeof(*req));
314		ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
315		return crypto_ablkcipher_decrypt(cryptd_req);
316	} else {
317		struct blkcipher_desc desc;
318		desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
319		desc.info = req->info;
320		desc.flags = 0;
321		return crypto_blkcipher_crt(desc.tfm)->decrypt(
322			&desc, req->dst, req->src, req->nbytes);
323	}
324}
325
326static void ablk_exit(struct crypto_tfm *tfm)
327{
328	struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
329
330	cryptd_free_ablkcipher(ctx->cryptd_tfm);
331}
332
333static void ablk_init_common(struct crypto_tfm *tfm,
334			     struct cryptd_ablkcipher *cryptd_tfm)
335{
336	struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
337
338	ctx->cryptd_tfm = cryptd_tfm;
339	tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
340		crypto_ablkcipher_reqsize(&cryptd_tfm->base);
341}
342
343static int ablk_ecb_init(struct crypto_tfm *tfm)
344{
345	struct cryptd_ablkcipher *cryptd_tfm;
346
347	cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ecb-aes-aesni", 0, 0);
348	if (IS_ERR(cryptd_tfm))
349		return PTR_ERR(cryptd_tfm);
350	ablk_init_common(tfm, cryptd_tfm);
351	return 0;
352}
353
354static struct crypto_alg ablk_ecb_alg = {
355	.cra_name		= "ecb(aes)",
356	.cra_driver_name	= "ecb-aes-aesni",
357	.cra_priority		= 400,
358	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
359	.cra_blocksize		= AES_BLOCK_SIZE,
360	.cra_ctxsize		= sizeof(struct async_aes_ctx),
361	.cra_alignmask		= 0,
362	.cra_type		= &crypto_ablkcipher_type,
363	.cra_module		= THIS_MODULE,
364	.cra_list		= LIST_HEAD_INIT(ablk_ecb_alg.cra_list),
365	.cra_init		= ablk_ecb_init,
366	.cra_exit		= ablk_exit,
367	.cra_u = {
368		.ablkcipher = {
369			.min_keysize	= AES_MIN_KEY_SIZE,
370			.max_keysize	= AES_MAX_KEY_SIZE,
371			.setkey		= ablk_set_key,
372			.encrypt	= ablk_encrypt,
373			.decrypt	= ablk_decrypt,
374		},
375	},
376};
377
378static int ablk_cbc_init(struct crypto_tfm *tfm)
379{
380	struct cryptd_ablkcipher *cryptd_tfm;
381
382	cryptd_tfm = cryptd_alloc_ablkcipher("__driver-cbc-aes-aesni", 0, 0);
383	if (IS_ERR(cryptd_tfm))
384		return PTR_ERR(cryptd_tfm);
385	ablk_init_common(tfm, cryptd_tfm);
386	return 0;
387}
388
389static struct crypto_alg ablk_cbc_alg = {
390	.cra_name		= "cbc(aes)",
391	.cra_driver_name	= "cbc-aes-aesni",
392	.cra_priority		= 400,
393	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
394	.cra_blocksize		= AES_BLOCK_SIZE,
395	.cra_ctxsize		= sizeof(struct async_aes_ctx),
396	.cra_alignmask		= 0,
397	.cra_type		= &crypto_ablkcipher_type,
398	.cra_module		= THIS_MODULE,
399	.cra_list		= LIST_HEAD_INIT(ablk_cbc_alg.cra_list),
400	.cra_init		= ablk_cbc_init,
401	.cra_exit		= ablk_exit,
402	.cra_u = {
403		.ablkcipher = {
404			.min_keysize	= AES_MIN_KEY_SIZE,
405			.max_keysize	= AES_MAX_KEY_SIZE,
406			.ivsize		= AES_BLOCK_SIZE,
407			.setkey		= ablk_set_key,
408			.encrypt	= ablk_encrypt,
409			.decrypt	= ablk_decrypt,
410		},
411	},
412};
413
414static int __init aesni_init(void)
415{
416	int err;
417
418	if (!cpu_has_aes) {
419		printk(KERN_ERR "Intel AES-NI instructions are not detected.\n");
420		return -ENODEV;
421	}
422	if ((err = crypto_register_alg(&aesni_alg)))
423		goto aes_err;
424	if ((err = crypto_register_alg(&blk_ecb_alg)))
425		goto blk_ecb_err;
426	if ((err = crypto_register_alg(&blk_cbc_alg)))
427		goto blk_cbc_err;
428	if ((err = crypto_register_alg(&ablk_ecb_alg)))
429		goto ablk_ecb_err;
430	if ((err = crypto_register_alg(&ablk_cbc_alg)))
431		goto ablk_cbc_err;
432
433	return err;
434
435ablk_cbc_err:
436	crypto_unregister_alg(&ablk_ecb_alg);
437ablk_ecb_err:
438	crypto_unregister_alg(&blk_cbc_alg);
439blk_cbc_err:
440	crypto_unregister_alg(&blk_ecb_alg);
441blk_ecb_err:
442	crypto_unregister_alg(&aesni_alg);
443aes_err:
444	return err;
445}
446
447static void __exit aesni_exit(void)
448{
449	crypto_unregister_alg(&ablk_cbc_alg);
450	crypto_unregister_alg(&ablk_ecb_alg);
451	crypto_unregister_alg(&blk_cbc_alg);
452	crypto_unregister_alg(&blk_ecb_alg);
453	crypto_unregister_alg(&aesni_alg);
454}
455
456module_init(aesni_init);
457module_exit(aesni_exit);
458
459MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
460MODULE_LICENSE("GPL");
461MODULE_ALIAS("aes");
462