10e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying/* 20e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying * Accelerated GHASH implementation with Intel PCLMULQDQ-NI 30e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying * instructions. This file contains glue code. 40e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying * 50e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying * Copyright (c) 2009 Intel Corp. 60e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying * Author: Huang Ying <ying.huang@intel.com> 70e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying * 80e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying * This program is free software; you can redistribute it and/or modify it 90e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying * under the terms of the GNU General Public License version 2 as published 100e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying * by the Free Software Foundation. 110e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying */ 120e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 1352f6c5ad430e41736133acac179607b224eaaa11Randy Dunlap#include <linux/err.h> 140e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying#include <linux/module.h> 150e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying#include <linux/init.h> 160e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying#include <linux/kernel.h> 170e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying#include <linux/crypto.h> 180e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying#include <crypto/algapi.h> 190e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying#include <crypto/cryptd.h> 200e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying#include <crypto/gf128mul.h> 210e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying#include <crypto/internal/hash.h> 220e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying#include <asm/i387.h> 233bd391f056df61e928de1680ff4a3e7e07e5b399Andi Kleen#include <asm/cpu_device_id.h> 240e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 250e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying#define GHASH_BLOCK_SIZE 16 260e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying#define GHASH_DIGEST_SIZE 16 270e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 280e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Yingvoid clmul_ghash_mul(char *dst, const be128 *shash); 290e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 300e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Yingvoid clmul_ghash_update(char *dst, const char *src, unsigned int srclen, 310e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying const be128 *shash); 320e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 330e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Yingvoid clmul_ghash_setkey(be128 *shash, const u8 *key); 340e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 350e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Yingstruct ghash_async_ctx { 360e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct cryptd_ahash *cryptd_tfm; 370e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying}; 380e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 390e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Yingstruct ghash_ctx { 400e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying be128 shash; 410e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying}; 420e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 430e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Yingstruct ghash_desc_ctx { 440e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying u8 buffer[GHASH_BLOCK_SIZE]; 450e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying u32 bytes; 460e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying}; 470e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 480e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Yingstatic int ghash_init(struct shash_desc *desc) 490e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying{ 500e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); 510e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 520e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying memset(dctx, 0, sizeof(*dctx)); 530e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 540e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying return 0; 550e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying} 560e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 570e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Yingstatic int ghash_setkey(struct crypto_shash *tfm, 580e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying const u8 *key, unsigned int keylen) 590e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying{ 600e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct ghash_ctx *ctx = crypto_shash_ctx(tfm); 610e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 620e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying if (keylen != GHASH_BLOCK_SIZE) { 630e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 640e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying return -EINVAL; 650e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying } 660e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 670e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying clmul_ghash_setkey(&ctx->shash, key); 680e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 690e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying return 0; 700e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying} 710e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 720e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Yingstatic int ghash_update(struct shash_desc *desc, 730e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying const u8 *src, unsigned int srclen) 740e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying{ 750e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); 760e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); 770e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying u8 *dst = dctx->buffer; 780e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 790e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying kernel_fpu_begin(); 800e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying if (dctx->bytes) { 810e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying int n = min(srclen, dctx->bytes); 820e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying u8 *pos = dst + (GHASH_BLOCK_SIZE - dctx->bytes); 830e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 840e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying dctx->bytes -= n; 850e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying srclen -= n; 860e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 870e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying while (n--) 880e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying *pos++ ^= *src++; 890e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 900e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying if (!dctx->bytes) 910e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying clmul_ghash_mul(dst, &ctx->shash); 920e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying } 930e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 940e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying clmul_ghash_update(dst, src, srclen, &ctx->shash); 950e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying kernel_fpu_end(); 960e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 970e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying if (srclen & 0xf) { 980e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying src += srclen - (srclen & 0xf); 990e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying srclen &= 0xf; 1000e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying dctx->bytes = GHASH_BLOCK_SIZE - srclen; 1010e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying while (srclen--) 1020e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying *dst++ ^= *src++; 1030e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying } 1040e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 1050e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying return 0; 1060e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying} 1070e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 1080e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Yingstatic void ghash_flush(struct ghash_ctx *ctx, struct ghash_desc_ctx *dctx) 1090e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying{ 1100e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying u8 *dst = dctx->buffer; 1110e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 1120e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying if (dctx->bytes) { 1130e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying u8 *tmp = dst + (GHASH_BLOCK_SIZE - dctx->bytes); 1140e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 1150e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying while (dctx->bytes--) 1160e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying *tmp++ ^= 0; 1170e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 1180e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying kernel_fpu_begin(); 1190e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying clmul_ghash_mul(dst, &ctx->shash); 1200e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying kernel_fpu_end(); 1210e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying } 1220e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 1230e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying dctx->bytes = 0; 1240e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying} 1250e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 1260e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Yingstatic int ghash_final(struct shash_desc *desc, u8 *dst) 1270e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying{ 1280e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); 1290e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); 1300e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying u8 *buf = dctx->buffer; 1310e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 1320e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying ghash_flush(ctx, dctx); 1330e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying memcpy(dst, buf, GHASH_BLOCK_SIZE); 1340e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 1350e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying return 0; 1360e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying} 1370e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 1380e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Yingstatic struct shash_alg ghash_alg = { 1390e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .digestsize = GHASH_DIGEST_SIZE, 1400e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .init = ghash_init, 1410e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .update = ghash_update, 1420e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .final = ghash_final, 1430e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .setkey = ghash_setkey, 1440e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .descsize = sizeof(struct ghash_desc_ctx), 1450e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .base = { 1460e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .cra_name = "__ghash", 1470e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .cra_driver_name = "__ghash-pclmulqdqni", 1480e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .cra_priority = 0, 1490e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .cra_flags = CRYPTO_ALG_TYPE_SHASH, 1500e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .cra_blocksize = GHASH_BLOCK_SIZE, 1510e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .cra_ctxsize = sizeof(struct ghash_ctx), 1520e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .cra_module = THIS_MODULE, 1530e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .cra_list = LIST_HEAD_INIT(ghash_alg.base.cra_list), 1540e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying }, 1550e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying}; 1560e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 1570e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Yingstatic int ghash_async_init(struct ahash_request *req) 1580e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying{ 1590e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1600e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 1610e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct ahash_request *cryptd_req = ahash_request_ctx(req); 1620e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; 1630e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 16401dd95827726534230d8f03f7e6faafe24e49260Huang Ying if (!irq_fpu_usable()) { 1650e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying memcpy(cryptd_req, req, sizeof(*req)); 1660e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); 1670e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying return crypto_ahash_init(cryptd_req); 1680e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying } else { 1690e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 1700e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm); 1710e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 1720e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying desc->tfm = child; 1730e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying desc->flags = req->base.flags; 1740e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying return crypto_shash_init(desc); 1750e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying } 1760e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying} 1770e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 1780e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Yingstatic int ghash_async_update(struct ahash_request *req) 1790e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying{ 1800e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct ahash_request *cryptd_req = ahash_request_ctx(req); 1810e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 18201dd95827726534230d8f03f7e6faafe24e49260Huang Ying if (!irq_fpu_usable()) { 1830e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1840e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 1850e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; 1860e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 1870e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying memcpy(cryptd_req, req, sizeof(*req)); 1880e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); 1890e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying return crypto_ahash_update(cryptd_req); 1900e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying } else { 1910e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 1920e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying return shash_ahash_update(req, desc); 1930e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying } 1940e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying} 1950e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 1960e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Yingstatic int ghash_async_final(struct ahash_request *req) 1970e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying{ 1980e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct ahash_request *cryptd_req = ahash_request_ctx(req); 1990e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 20001dd95827726534230d8f03f7e6faafe24e49260Huang Ying if (!irq_fpu_usable()) { 2010e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2020e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 2030e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; 2040e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 2050e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying memcpy(cryptd_req, req, sizeof(*req)); 2060e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); 2070e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying return crypto_ahash_final(cryptd_req); 2080e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying } else { 2090e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 2100e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying return crypto_shash_final(desc, req->result); 2110e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying } 2120e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying} 2130e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 2140e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Yingstatic int ghash_async_digest(struct ahash_request *req) 2150e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying{ 2160e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2170e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 2180e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct ahash_request *cryptd_req = ahash_request_ctx(req); 2190e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; 2200e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 22101dd95827726534230d8f03f7e6faafe24e49260Huang Ying if (!irq_fpu_usable()) { 2220e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying memcpy(cryptd_req, req, sizeof(*req)); 2230e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); 2240e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying return crypto_ahash_digest(cryptd_req); 2250e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying } else { 2260e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 2270e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm); 2280e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 2290e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying desc->tfm = child; 2300e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying desc->flags = req->base.flags; 2310e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying return shash_ahash_digest(req, desc); 2320e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying } 2330e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying} 2340e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 2350e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Yingstatic int ghash_async_setkey(struct crypto_ahash *tfm, const u8 *key, 2360e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying unsigned int keylen) 2370e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying{ 2380e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 2390e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct crypto_ahash *child = &ctx->cryptd_tfm->base; 2400e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying int err; 2410e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 2420e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK); 2430e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying crypto_ahash_set_flags(child, crypto_ahash_get_flags(tfm) 2440e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying & CRYPTO_TFM_REQ_MASK); 2450e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying err = crypto_ahash_setkey(child, key, keylen); 2460e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying crypto_ahash_set_flags(tfm, crypto_ahash_get_flags(child) 2470e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying & CRYPTO_TFM_RES_MASK); 2480e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 249c3e73e76a90b1e790e0bb7bb36135be9232f58deGustavo F. Padovan return err; 2500e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying} 2510e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 2520e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Yingstatic int ghash_async_init_tfm(struct crypto_tfm *tfm) 2530e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying{ 2540e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct cryptd_ahash *cryptd_tfm; 2550e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm); 2560e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 2570e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying cryptd_tfm = cryptd_alloc_ahash("__ghash-pclmulqdqni", 0, 0); 2580e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying if (IS_ERR(cryptd_tfm)) 2590e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying return PTR_ERR(cryptd_tfm); 2600e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying ctx->cryptd_tfm = cryptd_tfm; 2610e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 2620e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying sizeof(struct ahash_request) + 2630e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying crypto_ahash_reqsize(&cryptd_tfm->base)); 2640e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 2650e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying return 0; 2660e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying} 2670e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 2680e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Yingstatic void ghash_async_exit_tfm(struct crypto_tfm *tfm) 2690e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying{ 2700e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm); 2710e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 2720e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying cryptd_free_ahash(ctx->cryptd_tfm); 2730e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying} 2740e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 2750e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Yingstatic struct ahash_alg ghash_async_alg = { 2760e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .init = ghash_async_init, 2770e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .update = ghash_async_update, 2780e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .final = ghash_async_final, 2790e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .setkey = ghash_async_setkey, 2800e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .digest = ghash_async_digest, 2810e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .halg = { 2820e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .digestsize = GHASH_DIGEST_SIZE, 2830e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .base = { 2840e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .cra_name = "ghash", 2850e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .cra_driver_name = "ghash-clmulni", 2860e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .cra_priority = 400, 2870e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, 2880e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .cra_blocksize = GHASH_BLOCK_SIZE, 2890e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .cra_type = &crypto_ahash_type, 2900e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .cra_module = THIS_MODULE, 2910e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .cra_list = LIST_HEAD_INIT(ghash_async_alg.halg.base.cra_list), 2920e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .cra_init = ghash_async_init_tfm, 2930e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying .cra_exit = ghash_async_exit_tfm, 2940e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying }, 2950e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying }, 2960e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying}; 2970e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 2983bd391f056df61e928de1680ff4a3e7e07e5b399Andi Kleenstatic const struct x86_cpu_id pcmul_cpu_id[] = { 2993bd391f056df61e928de1680ff4a3e7e07e5b399Andi Kleen X86_FEATURE_MATCH(X86_FEATURE_PCLMULQDQ), /* Pickle-Mickle-Duck */ 3003bd391f056df61e928de1680ff4a3e7e07e5b399Andi Kleen {} 3013bd391f056df61e928de1680ff4a3e7e07e5b399Andi Kleen}; 3023bd391f056df61e928de1680ff4a3e7e07e5b399Andi KleenMODULE_DEVICE_TABLE(x86cpu, pcmul_cpu_id); 3033bd391f056df61e928de1680ff4a3e7e07e5b399Andi Kleen 3040e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Yingstatic int __init ghash_pclmulqdqni_mod_init(void) 3050e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying{ 3060e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying int err; 3070e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 3083bd391f056df61e928de1680ff4a3e7e07e5b399Andi Kleen if (!x86_match_cpu(pcmul_cpu_id)) 3090e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying return -ENODEV; 3100e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 3110e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying err = crypto_register_shash(&ghash_alg); 3120e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying if (err) 3130e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying goto err_out; 3140e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying err = crypto_register_ahash(&ghash_async_alg); 3150e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying if (err) 3160e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying goto err_shash; 3170e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 3180e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying return 0; 3190e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 3200e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Yingerr_shash: 3210e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying crypto_unregister_shash(&ghash_alg); 3220e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Yingerr_out: 3230e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying return err; 3240e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying} 3250e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 3260e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Yingstatic void __exit ghash_pclmulqdqni_mod_exit(void) 3270e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying{ 3280e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying crypto_unregister_ahash(&ghash_async_alg); 3290e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying crypto_unregister_shash(&ghash_alg); 3300e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying} 3310e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 3320e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Yingmodule_init(ghash_pclmulqdqni_mod_init); 3330e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Yingmodule_exit(ghash_pclmulqdqni_mod_exit); 3340e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying 3350e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang YingMODULE_LICENSE("GPL"); 3360e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang YingMODULE_DESCRIPTION("GHASH Message Digest Algorithm, " 3370e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang Ying "acclerated by PCLMULQDQ-NI"); 3380e1227d356e9b2fe0500d6cc7084f752040a1e0eHuang YingMODULE_ALIAS("ghash"); 339