1/*
2 * Accelerated GHASH implementation with Intel PCLMULQDQ-NI
3 * instructions. This file contains glue code.
4 *
5 * Copyright (c) 2009 Intel Corp.
6 *   Author: Huang Ying <ying.huang@intel.com>
7 *
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License version 2 as published
10 * by the Free Software Foundation.
11 */
12
13#include <linux/err.h>
14#include <linux/module.h>
15#include <linux/init.h>
16#include <linux/kernel.h>
17#include <linux/crypto.h>
18#include <crypto/algapi.h>
19#include <crypto/cryptd.h>
20#include <crypto/gf128mul.h>
21#include <crypto/internal/hash.h>
22#include <asm/i387.h>
23#include <asm/cpu_device_id.h>
24
25#define GHASH_BLOCK_SIZE	16
26#define GHASH_DIGEST_SIZE	16
27
28void clmul_ghash_mul(char *dst, const be128 *shash);
29
30void clmul_ghash_update(char *dst, const char *src, unsigned int srclen,
31			const be128 *shash);
32
33void clmul_ghash_setkey(be128 *shash, const u8 *key);
34
35struct ghash_async_ctx {
36	struct cryptd_ahash *cryptd_tfm;
37};
38
39struct ghash_ctx {
40	be128 shash;
41};
42
43struct ghash_desc_ctx {
44	u8 buffer[GHASH_BLOCK_SIZE];
45	u32 bytes;
46};
47
48static int ghash_init(struct shash_desc *desc)
49{
50	struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
51
52	memset(dctx, 0, sizeof(*dctx));
53
54	return 0;
55}
56
57static int ghash_setkey(struct crypto_shash *tfm,
58			const u8 *key, unsigned int keylen)
59{
60	struct ghash_ctx *ctx = crypto_shash_ctx(tfm);
61
62	if (keylen != GHASH_BLOCK_SIZE) {
63		crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
64		return -EINVAL;
65	}
66
67	clmul_ghash_setkey(&ctx->shash, key);
68
69	return 0;
70}
71
72static int ghash_update(struct shash_desc *desc,
73			 const u8 *src, unsigned int srclen)
74{
75	struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
76	struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm);
77	u8 *dst = dctx->buffer;
78
79	kernel_fpu_begin();
80	if (dctx->bytes) {
81		int n = min(srclen, dctx->bytes);
82		u8 *pos = dst + (GHASH_BLOCK_SIZE - dctx->bytes);
83
84		dctx->bytes -= n;
85		srclen -= n;
86
87		while (n--)
88			*pos++ ^= *src++;
89
90		if (!dctx->bytes)
91			clmul_ghash_mul(dst, &ctx->shash);
92	}
93
94	clmul_ghash_update(dst, src, srclen, &ctx->shash);
95	kernel_fpu_end();
96
97	if (srclen & 0xf) {
98		src += srclen - (srclen & 0xf);
99		srclen &= 0xf;
100		dctx->bytes = GHASH_BLOCK_SIZE - srclen;
101		while (srclen--)
102			*dst++ ^= *src++;
103	}
104
105	return 0;
106}
107
108static void ghash_flush(struct ghash_ctx *ctx, struct ghash_desc_ctx *dctx)
109{
110	u8 *dst = dctx->buffer;
111
112	if (dctx->bytes) {
113		u8 *tmp = dst + (GHASH_BLOCK_SIZE - dctx->bytes);
114
115		while (dctx->bytes--)
116			*tmp++ ^= 0;
117
118		kernel_fpu_begin();
119		clmul_ghash_mul(dst, &ctx->shash);
120		kernel_fpu_end();
121	}
122
123	dctx->bytes = 0;
124}
125
126static int ghash_final(struct shash_desc *desc, u8 *dst)
127{
128	struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
129	struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm);
130	u8 *buf = dctx->buffer;
131
132	ghash_flush(ctx, dctx);
133	memcpy(dst, buf, GHASH_BLOCK_SIZE);
134
135	return 0;
136}
137
138static struct shash_alg ghash_alg = {
139	.digestsize	= GHASH_DIGEST_SIZE,
140	.init		= ghash_init,
141	.update		= ghash_update,
142	.final		= ghash_final,
143	.setkey		= ghash_setkey,
144	.descsize	= sizeof(struct ghash_desc_ctx),
145	.base		= {
146		.cra_name		= "__ghash",
147		.cra_driver_name	= "__ghash-pclmulqdqni",
148		.cra_priority		= 0,
149		.cra_flags		= CRYPTO_ALG_TYPE_SHASH,
150		.cra_blocksize		= GHASH_BLOCK_SIZE,
151		.cra_ctxsize		= sizeof(struct ghash_ctx),
152		.cra_module		= THIS_MODULE,
153		.cra_list		= LIST_HEAD_INIT(ghash_alg.base.cra_list),
154	},
155};
156
157static int ghash_async_init(struct ahash_request *req)
158{
159	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
160	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
161	struct ahash_request *cryptd_req = ahash_request_ctx(req);
162	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
163
164	if (!irq_fpu_usable()) {
165		memcpy(cryptd_req, req, sizeof(*req));
166		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
167		return crypto_ahash_init(cryptd_req);
168	} else {
169		struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
170		struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
171
172		desc->tfm = child;
173		desc->flags = req->base.flags;
174		return crypto_shash_init(desc);
175	}
176}
177
178static int ghash_async_update(struct ahash_request *req)
179{
180	struct ahash_request *cryptd_req = ahash_request_ctx(req);
181
182	if (!irq_fpu_usable()) {
183		struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
184		struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
185		struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
186
187		memcpy(cryptd_req, req, sizeof(*req));
188		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
189		return crypto_ahash_update(cryptd_req);
190	} else {
191		struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
192		return shash_ahash_update(req, desc);
193	}
194}
195
196static int ghash_async_final(struct ahash_request *req)
197{
198	struct ahash_request *cryptd_req = ahash_request_ctx(req);
199
200	if (!irq_fpu_usable()) {
201		struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
202		struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
203		struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
204
205		memcpy(cryptd_req, req, sizeof(*req));
206		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
207		return crypto_ahash_final(cryptd_req);
208	} else {
209		struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
210		return crypto_shash_final(desc, req->result);
211	}
212}
213
214static int ghash_async_digest(struct ahash_request *req)
215{
216	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
217	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
218	struct ahash_request *cryptd_req = ahash_request_ctx(req);
219	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
220
221	if (!irq_fpu_usable()) {
222		memcpy(cryptd_req, req, sizeof(*req));
223		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
224		return crypto_ahash_digest(cryptd_req);
225	} else {
226		struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
227		struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
228
229		desc->tfm = child;
230		desc->flags = req->base.flags;
231		return shash_ahash_digest(req, desc);
232	}
233}
234
235static int ghash_async_setkey(struct crypto_ahash *tfm, const u8 *key,
236			      unsigned int keylen)
237{
238	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
239	struct crypto_ahash *child = &ctx->cryptd_tfm->base;
240	int err;
241
242	crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
243	crypto_ahash_set_flags(child, crypto_ahash_get_flags(tfm)
244			       & CRYPTO_TFM_REQ_MASK);
245	err = crypto_ahash_setkey(child, key, keylen);
246	crypto_ahash_set_flags(tfm, crypto_ahash_get_flags(child)
247			       & CRYPTO_TFM_RES_MASK);
248
249	return err;
250}
251
252static int ghash_async_init_tfm(struct crypto_tfm *tfm)
253{
254	struct cryptd_ahash *cryptd_tfm;
255	struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);
256
257	cryptd_tfm = cryptd_alloc_ahash("__ghash-pclmulqdqni", 0, 0);
258	if (IS_ERR(cryptd_tfm))
259		return PTR_ERR(cryptd_tfm);
260	ctx->cryptd_tfm = cryptd_tfm;
261	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
262				 sizeof(struct ahash_request) +
263				 crypto_ahash_reqsize(&cryptd_tfm->base));
264
265	return 0;
266}
267
268static void ghash_async_exit_tfm(struct crypto_tfm *tfm)
269{
270	struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);
271
272	cryptd_free_ahash(ctx->cryptd_tfm);
273}
274
275static struct ahash_alg ghash_async_alg = {
276	.init		= ghash_async_init,
277	.update		= ghash_async_update,
278	.final		= ghash_async_final,
279	.setkey		= ghash_async_setkey,
280	.digest		= ghash_async_digest,
281	.halg = {
282		.digestsize	= GHASH_DIGEST_SIZE,
283		.base = {
284			.cra_name		= "ghash",
285			.cra_driver_name	= "ghash-clmulni",
286			.cra_priority		= 400,
287			.cra_flags		= CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
288			.cra_blocksize		= GHASH_BLOCK_SIZE,
289			.cra_type		= &crypto_ahash_type,
290			.cra_module		= THIS_MODULE,
291			.cra_list		= LIST_HEAD_INIT(ghash_async_alg.halg.base.cra_list),
292			.cra_init		= ghash_async_init_tfm,
293			.cra_exit		= ghash_async_exit_tfm,
294		},
295	},
296};
297
298static const struct x86_cpu_id pcmul_cpu_id[] = {
299	X86_FEATURE_MATCH(X86_FEATURE_PCLMULQDQ), /* Pickle-Mickle-Duck */
300	{}
301};
302MODULE_DEVICE_TABLE(x86cpu, pcmul_cpu_id);
303
304static int __init ghash_pclmulqdqni_mod_init(void)
305{
306	int err;
307
308	if (!x86_match_cpu(pcmul_cpu_id))
309		return -ENODEV;
310
311	err = crypto_register_shash(&ghash_alg);
312	if (err)
313		goto err_out;
314	err = crypto_register_ahash(&ghash_async_alg);
315	if (err)
316		goto err_shash;
317
318	return 0;
319
320err_shash:
321	crypto_unregister_shash(&ghash_alg);
322err_out:
323	return err;
324}
325
326static void __exit ghash_pclmulqdqni_mod_exit(void)
327{
328	crypto_unregister_ahash(&ghash_async_alg);
329	crypto_unregister_shash(&ghash_alg);
330}
331
332module_init(ghash_pclmulqdqni_mod_init);
333module_exit(ghash_pclmulqdqni_mod_exit);
334
335MODULE_LICENSE("GPL");
336MODULE_DESCRIPTION("GHASH Message Digest Algorithm, "
337		   "acclerated by PCLMULQDQ-NI");
338MODULE_ALIAS("ghash");
339