154b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying/* 254b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying * Support for Intel AES-NI instructions. This file contains glue 354b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying * code, the real AES implementation is in intel-aes_asm.S. 454b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying * 554b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying * Copyright (C) 2008, Intel Corp. 654b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying * Author: Huang Ying <ying.huang@intel.com> 754b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying * 80bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD 90bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * interface for 64-bit kernels. 100bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * Authors: Adrian Hoban <adrian.hoban@intel.com> 110bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * Gabriele Paoloni <gabriele.paoloni@intel.com> 120bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * Tadeusz Struk (tadeusz.struk@intel.com) 130bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * Aidan O'Mahony (aidan.o.mahony@intel.com) 140bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * Copyright (c) 2010, Intel Corporation. 150bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * 1654b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying * This program is free software; you can redistribute it and/or modify 1754b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying * it under the terms of the GNU General Public License as published by 1854b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying * the Free Software Foundation; either version 2 of the License, or 1954b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying * (at your option) any later version. 2054b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying */ 2154b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 2254b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying#include <linux/hardirq.h> 2354b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying#include <linux/types.h> 2454b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying#include <linux/crypto.h> 257c52d55170ce84ddf9c0ad4e020ef1d7a97975a7Paul Gortmaker#include <linux/module.h> 2654b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying#include <linux/err.h> 2754b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying#include <crypto/algapi.h> 2854b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying#include <crypto/aes.h> 2954b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying#include <crypto/cryptd.h> 3012387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying#include <crypto/ctr.h> 31023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna#include <crypto/b128ops.h> 32023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna#include <crypto/lrw.h> 33023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna#include <crypto/xts.h> 343bd391f056df61e928de1680ff4a3e7e07e5b399Andi Kleen#include <asm/cpu_device_id.h> 3554b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying#include <asm/i387.h> 3670ef2601feb09d40f4086d055700b7923b3c2d6fJussi Kivilinna#include <asm/crypto/aes.h> 37801201aa25646291aa87ba97250f6614639eb2eaArd Biesheuvel#include <crypto/ablk_helper.h> 380bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk#include <crypto/scatterwalk.h> 390bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk#include <crypto/internal/aead.h> 400bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk#include <linux/workqueue.h> 410bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk#include <linux/spinlock.h> 42c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna#ifdef CONFIG_X86_64 43c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna#include <asm/crypto/glue_helper.h> 44c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna#endif 4554b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 462cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying#if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE) 472cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying#define HAS_PCBC 482cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying#endif 492cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying 500bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk/* This data is stored at the end of the crypto_tfm struct. 510bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * It's a type of per "session" data storage location. 520bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * This needs to be 16 byte aligned. 530bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk */ 540bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Strukstruct aesni_rfc4106_gcm_ctx { 550bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk u8 hash_subkey[16]; 560bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct crypto_aes_ctx aes_key_expanded; 570bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk u8 nonce[4]; 580bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct cryptd_aead *cryptd_tfm; 590bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk}; 600bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 610bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Strukstruct aesni_gcm_set_hash_subkey_result { 620bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk int err; 630bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct completion completion; 640bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk}; 650bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 660bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Strukstruct aesni_hash_subkey_req_data { 670bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk u8 iv[16]; 680bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct aesni_gcm_set_hash_subkey_result result; 690bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct scatterlist sg; 700bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk}; 710bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 720bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk#define AESNI_ALIGN (16) 7354b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying#define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1)) 740bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk#define RFC4106_HASH_SUBKEY_SIZE 16 7554b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 76023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinnastruct aesni_lrw_ctx { 77023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna struct lrw_table_ctx lrw_table; 78023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna u8 raw_aes_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1]; 79023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna}; 80023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 81023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinnastruct aesni_xts_ctx { 82023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1]; 83023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1]; 84023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna}; 85023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 8654b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Yingasmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, 8754b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying unsigned int key_len); 8854b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Yingasmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out, 8954b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying const u8 *in); 9054b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Yingasmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out, 9154b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying const u8 *in); 9254b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Yingasmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out, 9354b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying const u8 *in, unsigned int len); 9454b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Yingasmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out, 9554b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying const u8 *in, unsigned int len); 9654b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Yingasmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, 9754b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying const u8 *in, unsigned int len, u8 *iv); 9854b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Yingasmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, 9954b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying const u8 *in, unsigned int len, u8 *iv); 1009bed4aca296fdf9c1b85a8f093e92018dc9864f3Randy Dunlap 1019bed4aca296fdf9c1b85a8f093e92018dc9864f3Randy Dunlapint crypto_fpu_init(void); 1029bed4aca296fdf9c1b85a8f093e92018dc9864f3Randy Dunlapvoid crypto_fpu_exit(void); 1039bed4aca296fdf9c1b85a8f093e92018dc9864f3Randy Dunlap 104d764593af924930d5c15685bc5946cb943da1a55Tim Chen#define AVX_GEN2_OPTSIZE 640 105d764593af924930d5c15685bc5946cb943da1a55Tim Chen#define AVX_GEN4_OPTSIZE 4096 106d764593af924930d5c15685bc5946cb943da1a55Tim Chen 1070d258efb6a58fe047197c3b9cff8746bb176d58aMathias Krause#ifdef CONFIG_X86_64 10822cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan 10922cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayananstatic void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out, 11022cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan const u8 *in, unsigned int len, u8 *iv); 11112387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Yingasmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out, 11212387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying const u8 *in, unsigned int len, u8 *iv); 11354b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 114c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinnaasmlinkage void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, u8 *out, 115c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna const u8 *in, bool enc, u8 *iv); 116c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna 1170bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk/* asmlinkage void aesni_gcm_enc() 1180bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * void *ctx, AES Key schedule. Starts on a 16 byte boundary. 1190bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * u8 *out, Ciphertext output. Encrypt in-place is allowed. 1200bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * const u8 *in, Plaintext input 1210bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * unsigned long plaintext_len, Length of data in bytes for encryption. 1220bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association) 1230bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * concatenated with 8 byte Initialisation Vector (from IPSec ESP 1240bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * Payload) concatenated with 0x00000001. 16-byte aligned pointer. 1250bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 1260bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * const u8 *aad, Additional Authentication Data (AAD) 1270bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this 1280bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * is going to be 8 or 12 bytes 1290bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * u8 *auth_tag, Authenticated Tag output. 1300bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * unsigned long auth_tag_len), Authenticated Tag Length in bytes. 1310bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * Valid values are 16 (most likely), 12 or 8. 1320bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk */ 1330bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Strukasmlinkage void aesni_gcm_enc(void *ctx, u8 *out, 1340bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk const u8 *in, unsigned long plaintext_len, u8 *iv, 1350bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 1360bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk u8 *auth_tag, unsigned long auth_tag_len); 1370bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 1380bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk/* asmlinkage void aesni_gcm_dec() 1390bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * void *ctx, AES Key schedule. Starts on a 16 byte boundary. 1400bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * u8 *out, Plaintext output. Decrypt in-place is allowed. 1410bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * const u8 *in, Ciphertext input 1420bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * unsigned long ciphertext_len, Length of data in bytes for decryption. 1430bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association) 1440bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * concatenated with 8 byte Initialisation Vector (from IPSec ESP 1450bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * Payload) concatenated with 0x00000001. 16-byte aligned pointer. 1460bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 1470bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * const u8 *aad, Additional Authentication Data (AAD) 1480bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going 1490bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * to be 8 or 12 bytes 1500bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * u8 *auth_tag, Authenticated Tag output. 1510bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * unsigned long auth_tag_len) Authenticated Tag Length in bytes. 1520bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * Valid values are 16 (most likely), 12 or 8. 1530bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk */ 1540bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Strukasmlinkage void aesni_gcm_dec(void *ctx, u8 *out, 1550bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk const u8 *in, unsigned long ciphertext_len, u8 *iv, 1560bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 1570bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk u8 *auth_tag, unsigned long auth_tag_len); 1580bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 159d764593af924930d5c15685bc5946cb943da1a55Tim Chen 160d764593af924930d5c15685bc5946cb943da1a55Tim Chen#ifdef CONFIG_AS_AVX 16122cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayananasmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv, 16222cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan void *keys, u8 *out, unsigned int num_bytes); 16322cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayananasmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv, 16422cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan void *keys, u8 *out, unsigned int num_bytes); 16522cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayananasmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv, 16622cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan void *keys, u8 *out, unsigned int num_bytes); 167d764593af924930d5c15685bc5946cb943da1a55Tim Chen/* 168d764593af924930d5c15685bc5946cb943da1a55Tim Chen * asmlinkage void aesni_gcm_precomp_avx_gen2() 169d764593af924930d5c15685bc5946cb943da1a55Tim Chen * gcm_data *my_ctx_data, context data 170d764593af924930d5c15685bc5946cb943da1a55Tim Chen * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 171d764593af924930d5c15685bc5946cb943da1a55Tim Chen */ 172d764593af924930d5c15685bc5946cb943da1a55Tim Chenasmlinkage void aesni_gcm_precomp_avx_gen2(void *my_ctx_data, u8 *hash_subkey); 173d764593af924930d5c15685bc5946cb943da1a55Tim Chen 174d764593af924930d5c15685bc5946cb943da1a55Tim Chenasmlinkage void aesni_gcm_enc_avx_gen2(void *ctx, u8 *out, 175d764593af924930d5c15685bc5946cb943da1a55Tim Chen const u8 *in, unsigned long plaintext_len, u8 *iv, 176d764593af924930d5c15685bc5946cb943da1a55Tim Chen const u8 *aad, unsigned long aad_len, 177d764593af924930d5c15685bc5946cb943da1a55Tim Chen u8 *auth_tag, unsigned long auth_tag_len); 178d764593af924930d5c15685bc5946cb943da1a55Tim Chen 179d764593af924930d5c15685bc5946cb943da1a55Tim Chenasmlinkage void aesni_gcm_dec_avx_gen2(void *ctx, u8 *out, 180d764593af924930d5c15685bc5946cb943da1a55Tim Chen const u8 *in, unsigned long ciphertext_len, u8 *iv, 181d764593af924930d5c15685bc5946cb943da1a55Tim Chen const u8 *aad, unsigned long aad_len, 182d764593af924930d5c15685bc5946cb943da1a55Tim Chen u8 *auth_tag, unsigned long auth_tag_len); 183d764593af924930d5c15685bc5946cb943da1a55Tim Chen 184d764593af924930d5c15685bc5946cb943da1a55Tim Chenstatic void aesni_gcm_enc_avx(void *ctx, u8 *out, 185d764593af924930d5c15685bc5946cb943da1a55Tim Chen const u8 *in, unsigned long plaintext_len, u8 *iv, 186d764593af924930d5c15685bc5946cb943da1a55Tim Chen u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 187d764593af924930d5c15685bc5946cb943da1a55Tim Chen u8 *auth_tag, unsigned long auth_tag_len) 188d764593af924930d5c15685bc5946cb943da1a55Tim Chen{ 189d764593af924930d5c15685bc5946cb943da1a55Tim Chen if (plaintext_len < AVX_GEN2_OPTSIZE) { 190d764593af924930d5c15685bc5946cb943da1a55Tim Chen aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad, 191d764593af924930d5c15685bc5946cb943da1a55Tim Chen aad_len, auth_tag, auth_tag_len); 192d764593af924930d5c15685bc5946cb943da1a55Tim Chen } else { 193d764593af924930d5c15685bc5946cb943da1a55Tim Chen aesni_gcm_precomp_avx_gen2(ctx, hash_subkey); 194d764593af924930d5c15685bc5946cb943da1a55Tim Chen aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad, 195d764593af924930d5c15685bc5946cb943da1a55Tim Chen aad_len, auth_tag, auth_tag_len); 196d764593af924930d5c15685bc5946cb943da1a55Tim Chen } 197d764593af924930d5c15685bc5946cb943da1a55Tim Chen} 198d764593af924930d5c15685bc5946cb943da1a55Tim Chen 199d764593af924930d5c15685bc5946cb943da1a55Tim Chenstatic void aesni_gcm_dec_avx(void *ctx, u8 *out, 200d764593af924930d5c15685bc5946cb943da1a55Tim Chen const u8 *in, unsigned long ciphertext_len, u8 *iv, 201d764593af924930d5c15685bc5946cb943da1a55Tim Chen u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 202d764593af924930d5c15685bc5946cb943da1a55Tim Chen u8 *auth_tag, unsigned long auth_tag_len) 203d764593af924930d5c15685bc5946cb943da1a55Tim Chen{ 204d764593af924930d5c15685bc5946cb943da1a55Tim Chen if (ciphertext_len < AVX_GEN2_OPTSIZE) { 205d764593af924930d5c15685bc5946cb943da1a55Tim Chen aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey, aad, 206d764593af924930d5c15685bc5946cb943da1a55Tim Chen aad_len, auth_tag, auth_tag_len); 207d764593af924930d5c15685bc5946cb943da1a55Tim Chen } else { 208d764593af924930d5c15685bc5946cb943da1a55Tim Chen aesni_gcm_precomp_avx_gen2(ctx, hash_subkey); 209d764593af924930d5c15685bc5946cb943da1a55Tim Chen aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad, 210d764593af924930d5c15685bc5946cb943da1a55Tim Chen aad_len, auth_tag, auth_tag_len); 211d764593af924930d5c15685bc5946cb943da1a55Tim Chen } 212d764593af924930d5c15685bc5946cb943da1a55Tim Chen} 213d764593af924930d5c15685bc5946cb943da1a55Tim Chen#endif 214d764593af924930d5c15685bc5946cb943da1a55Tim Chen 215d764593af924930d5c15685bc5946cb943da1a55Tim Chen#ifdef CONFIG_AS_AVX2 216d764593af924930d5c15685bc5946cb943da1a55Tim Chen/* 217d764593af924930d5c15685bc5946cb943da1a55Tim Chen * asmlinkage void aesni_gcm_precomp_avx_gen4() 218d764593af924930d5c15685bc5946cb943da1a55Tim Chen * gcm_data *my_ctx_data, context data 219d764593af924930d5c15685bc5946cb943da1a55Tim Chen * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 220d764593af924930d5c15685bc5946cb943da1a55Tim Chen */ 221d764593af924930d5c15685bc5946cb943da1a55Tim Chenasmlinkage void aesni_gcm_precomp_avx_gen4(void *my_ctx_data, u8 *hash_subkey); 222d764593af924930d5c15685bc5946cb943da1a55Tim Chen 223d764593af924930d5c15685bc5946cb943da1a55Tim Chenasmlinkage void aesni_gcm_enc_avx_gen4(void *ctx, u8 *out, 224d764593af924930d5c15685bc5946cb943da1a55Tim Chen const u8 *in, unsigned long plaintext_len, u8 *iv, 225d764593af924930d5c15685bc5946cb943da1a55Tim Chen const u8 *aad, unsigned long aad_len, 226d764593af924930d5c15685bc5946cb943da1a55Tim Chen u8 *auth_tag, unsigned long auth_tag_len); 227d764593af924930d5c15685bc5946cb943da1a55Tim Chen 228d764593af924930d5c15685bc5946cb943da1a55Tim Chenasmlinkage void aesni_gcm_dec_avx_gen4(void *ctx, u8 *out, 229d764593af924930d5c15685bc5946cb943da1a55Tim Chen const u8 *in, unsigned long ciphertext_len, u8 *iv, 230d764593af924930d5c15685bc5946cb943da1a55Tim Chen const u8 *aad, unsigned long aad_len, 231d764593af924930d5c15685bc5946cb943da1a55Tim Chen u8 *auth_tag, unsigned long auth_tag_len); 232d764593af924930d5c15685bc5946cb943da1a55Tim Chen 233d764593af924930d5c15685bc5946cb943da1a55Tim Chenstatic void aesni_gcm_enc_avx2(void *ctx, u8 *out, 234d764593af924930d5c15685bc5946cb943da1a55Tim Chen const u8 *in, unsigned long plaintext_len, u8 *iv, 235d764593af924930d5c15685bc5946cb943da1a55Tim Chen u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 236d764593af924930d5c15685bc5946cb943da1a55Tim Chen u8 *auth_tag, unsigned long auth_tag_len) 237d764593af924930d5c15685bc5946cb943da1a55Tim Chen{ 238d764593af924930d5c15685bc5946cb943da1a55Tim Chen if (plaintext_len < AVX_GEN2_OPTSIZE) { 239d764593af924930d5c15685bc5946cb943da1a55Tim Chen aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad, 240d764593af924930d5c15685bc5946cb943da1a55Tim Chen aad_len, auth_tag, auth_tag_len); 241d764593af924930d5c15685bc5946cb943da1a55Tim Chen } else if (plaintext_len < AVX_GEN4_OPTSIZE) { 242d764593af924930d5c15685bc5946cb943da1a55Tim Chen aesni_gcm_precomp_avx_gen2(ctx, hash_subkey); 243d764593af924930d5c15685bc5946cb943da1a55Tim Chen aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad, 244d764593af924930d5c15685bc5946cb943da1a55Tim Chen aad_len, auth_tag, auth_tag_len); 245d764593af924930d5c15685bc5946cb943da1a55Tim Chen } else { 246d764593af924930d5c15685bc5946cb943da1a55Tim Chen aesni_gcm_precomp_avx_gen4(ctx, hash_subkey); 247d764593af924930d5c15685bc5946cb943da1a55Tim Chen aesni_gcm_enc_avx_gen4(ctx, out, in, plaintext_len, iv, aad, 248d764593af924930d5c15685bc5946cb943da1a55Tim Chen aad_len, auth_tag, auth_tag_len); 249d764593af924930d5c15685bc5946cb943da1a55Tim Chen } 250d764593af924930d5c15685bc5946cb943da1a55Tim Chen} 251d764593af924930d5c15685bc5946cb943da1a55Tim Chen 252d764593af924930d5c15685bc5946cb943da1a55Tim Chenstatic void aesni_gcm_dec_avx2(void *ctx, u8 *out, 253d764593af924930d5c15685bc5946cb943da1a55Tim Chen const u8 *in, unsigned long ciphertext_len, u8 *iv, 254d764593af924930d5c15685bc5946cb943da1a55Tim Chen u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 255d764593af924930d5c15685bc5946cb943da1a55Tim Chen u8 *auth_tag, unsigned long auth_tag_len) 256d764593af924930d5c15685bc5946cb943da1a55Tim Chen{ 257d764593af924930d5c15685bc5946cb943da1a55Tim Chen if (ciphertext_len < AVX_GEN2_OPTSIZE) { 258d764593af924930d5c15685bc5946cb943da1a55Tim Chen aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey, 259d764593af924930d5c15685bc5946cb943da1a55Tim Chen aad, aad_len, auth_tag, auth_tag_len); 260d764593af924930d5c15685bc5946cb943da1a55Tim Chen } else if (ciphertext_len < AVX_GEN4_OPTSIZE) { 261d764593af924930d5c15685bc5946cb943da1a55Tim Chen aesni_gcm_precomp_avx_gen2(ctx, hash_subkey); 262d764593af924930d5c15685bc5946cb943da1a55Tim Chen aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad, 263d764593af924930d5c15685bc5946cb943da1a55Tim Chen aad_len, auth_tag, auth_tag_len); 264d764593af924930d5c15685bc5946cb943da1a55Tim Chen } else { 265d764593af924930d5c15685bc5946cb943da1a55Tim Chen aesni_gcm_precomp_avx_gen4(ctx, hash_subkey); 266d764593af924930d5c15685bc5946cb943da1a55Tim Chen aesni_gcm_dec_avx_gen4(ctx, out, in, ciphertext_len, iv, aad, 267d764593af924930d5c15685bc5946cb943da1a55Tim Chen aad_len, auth_tag, auth_tag_len); 268d764593af924930d5c15685bc5946cb943da1a55Tim Chen } 269d764593af924930d5c15685bc5946cb943da1a55Tim Chen} 270d764593af924930d5c15685bc5946cb943da1a55Tim Chen#endif 271d764593af924930d5c15685bc5946cb943da1a55Tim Chen 272d764593af924930d5c15685bc5946cb943da1a55Tim Chenstatic void (*aesni_gcm_enc_tfm)(void *ctx, u8 *out, 273d764593af924930d5c15685bc5946cb943da1a55Tim Chen const u8 *in, unsigned long plaintext_len, u8 *iv, 274d764593af924930d5c15685bc5946cb943da1a55Tim Chen u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 275d764593af924930d5c15685bc5946cb943da1a55Tim Chen u8 *auth_tag, unsigned long auth_tag_len); 276d764593af924930d5c15685bc5946cb943da1a55Tim Chen 277d764593af924930d5c15685bc5946cb943da1a55Tim Chenstatic void (*aesni_gcm_dec_tfm)(void *ctx, u8 *out, 278d764593af924930d5c15685bc5946cb943da1a55Tim Chen const u8 *in, unsigned long ciphertext_len, u8 *iv, 279d764593af924930d5c15685bc5946cb943da1a55Tim Chen u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 280d764593af924930d5c15685bc5946cb943da1a55Tim Chen u8 *auth_tag, unsigned long auth_tag_len); 281d764593af924930d5c15685bc5946cb943da1a55Tim Chen 2820bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Strukstatic inline struct 2830bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Strukaesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm) 2840bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk{ 2850bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk return 2860bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk (struct aesni_rfc4106_gcm_ctx *) 2870bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk PTR_ALIGN((u8 *) 2880bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN); 2890bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk} 290559ad0ff1368baea14dbc3207d55b02bd69bda4bMathias Krause#endif 2910bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 29254b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Yingstatic inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx) 29354b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying{ 29454b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying unsigned long addr = (unsigned long)raw_ctx; 29554b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying unsigned long align = AESNI_ALIGN; 29654b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 29754b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying if (align <= crypto_tfm_ctx_alignment()) 29854b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying align = 1; 29954b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying return (struct crypto_aes_ctx *)ALIGN(addr, align); 30054b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying} 30154b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 30254b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Yingstatic int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx, 30354b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying const u8 *in_key, unsigned int key_len) 30454b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying{ 30554b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx); 30654b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying u32 *flags = &tfm->crt_flags; 30754b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying int err; 30854b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 30954b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 && 31054b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying key_len != AES_KEYSIZE_256) { 31154b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 31254b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying return -EINVAL; 31354b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying } 31454b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 31513b79b971564ddd0f14e706592472adc8199e912Huang Ying if (!irq_fpu_usable()) 31654b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying err = crypto_aes_expand_key(ctx, in_key, key_len); 31754b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying else { 31854b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying kernel_fpu_begin(); 31954b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying err = aesni_set_key(ctx, in_key, key_len); 32054b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying kernel_fpu_end(); 32154b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying } 32254b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 32354b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying return err; 32454b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying} 32554b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 32654b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Yingstatic int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 32754b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying unsigned int key_len) 32854b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying{ 32954b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len); 33054b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying} 33154b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 33254b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Yingstatic void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 33354b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying{ 33454b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 33554b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 33613b79b971564ddd0f14e706592472adc8199e912Huang Ying if (!irq_fpu_usable()) 33754b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying crypto_aes_encrypt_x86(ctx, dst, src); 33854b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying else { 33954b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying kernel_fpu_begin(); 34054b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying aesni_enc(ctx, dst, src); 34154b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying kernel_fpu_end(); 34254b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying } 34354b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying} 34454b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 34554b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Yingstatic void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 34654b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying{ 34754b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 34854b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 34913b79b971564ddd0f14e706592472adc8199e912Huang Ying if (!irq_fpu_usable()) 35054b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying crypto_aes_decrypt_x86(ctx, dst, src); 35154b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying else { 35254b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying kernel_fpu_begin(); 35354b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying aesni_dec(ctx, dst, src); 35454b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying kernel_fpu_end(); 35554b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying } 35654b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying} 35754b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 3582cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Yingstatic void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 3592cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying{ 3602cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 3612cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying 3622cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying aesni_enc(ctx, dst, src); 3632cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying} 3642cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying 3652cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Yingstatic void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 3662cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying{ 3672cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 3682cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying 3692cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying aesni_dec(ctx, dst, src); 3702cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying} 3712cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying 37254b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Yingstatic int ecb_encrypt(struct blkcipher_desc *desc, 37354b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying struct scatterlist *dst, struct scatterlist *src, 37454b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying unsigned int nbytes) 37554b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying{ 37654b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 37754b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying struct blkcipher_walk walk; 37854b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying int err; 37954b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 38054b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying blkcipher_walk_init(&walk, dst, src, nbytes); 38154b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying err = blkcipher_walk_virt(desc, &walk); 3829251b64fb2d2326d28f0e0646a9e4fb8bbb51d8eHuang Ying desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 38354b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 38454b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying kernel_fpu_begin(); 38554b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying while ((nbytes = walk.nbytes)) { 38654b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 38754b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying nbytes & AES_BLOCK_MASK); 38854b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying nbytes &= AES_BLOCK_SIZE - 1; 38954b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying err = blkcipher_walk_done(desc, &walk, nbytes); 39054b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying } 39154b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying kernel_fpu_end(); 39254b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 39354b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying return err; 39454b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying} 39554b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 39654b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Yingstatic int ecb_decrypt(struct blkcipher_desc *desc, 39754b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying struct scatterlist *dst, struct scatterlist *src, 39854b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying unsigned int nbytes) 39954b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying{ 40054b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 40154b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying struct blkcipher_walk walk; 40254b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying int err; 40354b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 40454b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying blkcipher_walk_init(&walk, dst, src, nbytes); 40554b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying err = blkcipher_walk_virt(desc, &walk); 4069251b64fb2d2326d28f0e0646a9e4fb8bbb51d8eHuang Ying desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 40754b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 40854b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying kernel_fpu_begin(); 40954b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying while ((nbytes = walk.nbytes)) { 41054b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 41154b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying nbytes & AES_BLOCK_MASK); 41254b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying nbytes &= AES_BLOCK_SIZE - 1; 41354b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying err = blkcipher_walk_done(desc, &walk, nbytes); 41454b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying } 41554b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying kernel_fpu_end(); 41654b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 41754b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying return err; 41854b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying} 41954b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 42054b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Yingstatic int cbc_encrypt(struct blkcipher_desc *desc, 42154b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying struct scatterlist *dst, struct scatterlist *src, 42254b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying unsigned int nbytes) 42354b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying{ 42454b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 42554b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying struct blkcipher_walk walk; 42654b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying int err; 42754b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 42854b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying blkcipher_walk_init(&walk, dst, src, nbytes); 42954b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying err = blkcipher_walk_virt(desc, &walk); 4309251b64fb2d2326d28f0e0646a9e4fb8bbb51d8eHuang Ying desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 43154b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 43254b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying kernel_fpu_begin(); 43354b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying while ((nbytes = walk.nbytes)) { 43454b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 43554b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying nbytes & AES_BLOCK_MASK, walk.iv); 43654b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying nbytes &= AES_BLOCK_SIZE - 1; 43754b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying err = blkcipher_walk_done(desc, &walk, nbytes); 43854b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying } 43954b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying kernel_fpu_end(); 44054b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 44154b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying return err; 44254b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying} 44354b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 44454b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Yingstatic int cbc_decrypt(struct blkcipher_desc *desc, 44554b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying struct scatterlist *dst, struct scatterlist *src, 44654b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying unsigned int nbytes) 44754b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying{ 44854b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 44954b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying struct blkcipher_walk walk; 45054b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying int err; 45154b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 45254b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying blkcipher_walk_init(&walk, dst, src, nbytes); 45354b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying err = blkcipher_walk_virt(desc, &walk); 4549251b64fb2d2326d28f0e0646a9e4fb8bbb51d8eHuang Ying desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 45554b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 45654b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying kernel_fpu_begin(); 45754b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying while ((nbytes = walk.nbytes)) { 45854b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 45954b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying nbytes & AES_BLOCK_MASK, walk.iv); 46054b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying nbytes &= AES_BLOCK_SIZE - 1; 46154b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying err = blkcipher_walk_done(desc, &walk, nbytes); 46254b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying } 46354b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying kernel_fpu_end(); 46454b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 46554b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying return err; 46654b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying} 46754b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 4680d258efb6a58fe047197c3b9cff8746bb176d58aMathias Krause#ifdef CONFIG_X86_64 46912387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Yingstatic void ctr_crypt_final(struct crypto_aes_ctx *ctx, 47012387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying struct blkcipher_walk *walk) 47112387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying{ 47212387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying u8 *ctrblk = walk->iv; 47312387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying u8 keystream[AES_BLOCK_SIZE]; 47412387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying u8 *src = walk->src.virt.addr; 47512387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying u8 *dst = walk->dst.virt.addr; 47612387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying unsigned int nbytes = walk->nbytes; 47712387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying 47812387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying aesni_enc(ctx, keystream, ctrblk); 47912387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying crypto_xor(keystream, src, nbytes); 48012387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying memcpy(dst, keystream, nbytes); 48112387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying crypto_inc(ctrblk, AES_BLOCK_SIZE); 48212387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying} 48312387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying 4845cfed7b3351ce33b60c4a1d02ffd712cdf4f49cdMathias Krause#ifdef CONFIG_AS_AVX 48522cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayananstatic void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out, 48622cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan const u8 *in, unsigned int len, u8 *iv) 48722cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan{ 48822cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan /* 48922cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan * based on key length, override with the by8 version 49022cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan * of ctr mode encryption/decryption for improved performance 49122cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan * aes_set_key_common() ensures that key length is one of 49222cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan * {128,192,256} 49322cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan */ 49422cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan if (ctx->key_length == AES_KEYSIZE_128) 49522cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len); 49622cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan else if (ctx->key_length == AES_KEYSIZE_192) 49722cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len); 49822cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan else 49922cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len); 50022cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan} 50122cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan#endif 50222cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan 50312387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Yingstatic int ctr_crypt(struct blkcipher_desc *desc, 50412387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying struct scatterlist *dst, struct scatterlist *src, 50512387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying unsigned int nbytes) 50612387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying{ 50712387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 50812387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying struct blkcipher_walk walk; 50912387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying int err; 51012387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying 51112387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying blkcipher_walk_init(&walk, dst, src, nbytes); 51212387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE); 51312387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 51412387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying 51512387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying kernel_fpu_begin(); 51612387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { 51722cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr, 51822cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan nbytes & AES_BLOCK_MASK, walk.iv); 51912387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying nbytes &= AES_BLOCK_SIZE - 1; 52012387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying err = blkcipher_walk_done(desc, &walk, nbytes); 52112387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying } 52212387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying if (walk.nbytes) { 52312387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying ctr_crypt_final(ctx, &walk); 52412387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying err = blkcipher_walk_done(desc, &walk, 0); 52512387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying } 52612387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying kernel_fpu_end(); 52712387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying 52812387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying return err; 52912387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying} 5300d258efb6a58fe047197c3b9cff8746bb176d58aMathias Krause#endif 53112387a46bb150f5608de4aa9a90dfdddbf991e3fHuang Ying 53254b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Yingstatic int ablk_ecb_init(struct crypto_tfm *tfm) 53354b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying{ 534ef45b834319f8a18f257a40ba4bce6b829ef1708Jussi Kivilinna return ablk_init_common(tfm, "__driver-ecb-aes-aesni"); 53554b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying} 53654b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 53754b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Yingstatic int ablk_cbc_init(struct crypto_tfm *tfm) 53854b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying{ 539ef45b834319f8a18f257a40ba4bce6b829ef1708Jussi Kivilinna return ablk_init_common(tfm, "__driver-cbc-aes-aesni"); 54054b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying} 54154b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 5420d258efb6a58fe047197c3b9cff8746bb176d58aMathias Krause#ifdef CONFIG_X86_64 5432cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Yingstatic int ablk_ctr_init(struct crypto_tfm *tfm) 5442cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying{ 545ef45b834319f8a18f257a40ba4bce6b829ef1708Jussi Kivilinna return ablk_init_common(tfm, "__driver-ctr-aes-aesni"); 5462cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying} 5472cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying 5480d258efb6a58fe047197c3b9cff8746bb176d58aMathias Krause#endif 5492cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying 5502cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying#ifdef HAS_PCBC 5512cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Yingstatic int ablk_pcbc_init(struct crypto_tfm *tfm) 5522cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying{ 553ef45b834319f8a18f257a40ba4bce6b829ef1708Jussi Kivilinna return ablk_init_common(tfm, "fpu(pcbc(__driver-aes-aesni))"); 5542cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying} 5552cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying#endif 5562cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying 557023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinnastatic void lrw_xts_encrypt_callback(void *ctx, u8 *blks, unsigned int nbytes) 5582cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying{ 559023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna aesni_ecb_enc(ctx, blks, blks, nbytes); 560023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna} 561023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 562023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinnastatic void lrw_xts_decrypt_callback(void *ctx, u8 *blks, unsigned int nbytes) 563023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna{ 564023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna aesni_ecb_dec(ctx, blks, blks, nbytes); 565023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna} 566023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 567023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinnastatic int lrw_aesni_setkey(struct crypto_tfm *tfm, const u8 *key, 568023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna unsigned int keylen) 569023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna{ 570023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm); 571023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna int err; 572023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 573023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna err = aes_set_key_common(tfm, ctx->raw_aes_ctx, key, 574023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna keylen - AES_BLOCK_SIZE); 575023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna if (err) 576023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna return err; 577023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 578023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna return lrw_init_table(&ctx->lrw_table, key + keylen - AES_BLOCK_SIZE); 579023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna} 580023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 581023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinnastatic void lrw_aesni_exit_tfm(struct crypto_tfm *tfm) 582023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna{ 583023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm); 584023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 585023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna lrw_free_table(&ctx->lrw_table); 586023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna} 587023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 588023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinnastatic int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 589023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna struct scatterlist *src, unsigned int nbytes) 590023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna{ 591023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 592023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna be128 buf[8]; 593023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna struct lrw_crypt_req req = { 594023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .tbuf = buf, 595023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .tbuflen = sizeof(buf), 596023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 597023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .table_ctx = &ctx->lrw_table, 598023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .crypt_ctx = aes_ctx(ctx->raw_aes_ctx), 599023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .crypt_fn = lrw_xts_encrypt_callback, 600023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna }; 601023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna int ret; 602023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 603023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 604023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 605023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna kernel_fpu_begin(); 606023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna ret = lrw_crypt(desc, dst, src, nbytes, &req); 607023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna kernel_fpu_end(); 608023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 609023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna return ret; 610023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna} 611023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 612023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinnastatic int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 613023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna struct scatterlist *src, unsigned int nbytes) 614023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna{ 615023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 616023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna be128 buf[8]; 617023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna struct lrw_crypt_req req = { 618023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .tbuf = buf, 619023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .tbuflen = sizeof(buf), 620023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 621023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .table_ctx = &ctx->lrw_table, 622023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .crypt_ctx = aes_ctx(ctx->raw_aes_ctx), 623023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .crypt_fn = lrw_xts_decrypt_callback, 624023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna }; 625023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna int ret; 626023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 627023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 628023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 629023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna kernel_fpu_begin(); 630023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna ret = lrw_crypt(desc, dst, src, nbytes, &req); 631023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna kernel_fpu_end(); 632023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 633023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna return ret; 634023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna} 635023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 636023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinnastatic int xts_aesni_setkey(struct crypto_tfm *tfm, const u8 *key, 637023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna unsigned int keylen) 638023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna{ 639023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna struct aesni_xts_ctx *ctx = crypto_tfm_ctx(tfm); 640023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna u32 *flags = &tfm->crt_flags; 641023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna int err; 642023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 643023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna /* key consists of keys of equal size concatenated, therefore 644023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna * the length must be even 645023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna */ 646023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna if (keylen % 2) { 647023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 648023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna return -EINVAL; 649023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna } 650023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 651023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna /* first half of xts-key is for crypt */ 652023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna err = aes_set_key_common(tfm, ctx->raw_crypt_ctx, key, keylen / 2); 653023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna if (err) 654023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna return err; 655023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 656023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna /* second half of xts-key is for tweak */ 657023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna return aes_set_key_common(tfm, ctx->raw_tweak_ctx, key + keylen / 2, 658023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna keylen / 2); 659023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna} 660023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 661023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 66232bec973a8435afc0b032da22174701f836549b2Jussi Kivilinnastatic void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in) 66332bec973a8435afc0b032da22174701f836549b2Jussi Kivilinna{ 66432bec973a8435afc0b032da22174701f836549b2Jussi Kivilinna aesni_enc(ctx, out, in); 66532bec973a8435afc0b032da22174701f836549b2Jussi Kivilinna} 66632bec973a8435afc0b032da22174701f836549b2Jussi Kivilinna 667c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna#ifdef CONFIG_X86_64 668c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna 669c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinnastatic void aesni_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv) 670c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna{ 671c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_enc)); 672c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna} 673c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna 674c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinnastatic void aesni_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv) 675c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna{ 676c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_dec)); 677c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna} 678c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna 679c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinnastatic void aesni_xts_enc8(void *ctx, u128 *dst, const u128 *src, le128 *iv) 680c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna{ 681c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, true, (u8 *)iv); 682c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna} 683c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna 684c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinnastatic void aesni_xts_dec8(void *ctx, u128 *dst, const u128 *src, le128 *iv) 685c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna{ 686c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, false, (u8 *)iv); 687c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna} 688c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna 689c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinnastatic const struct common_glue_ctx aesni_enc_xts = { 690c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna .num_funcs = 2, 691c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna .fpu_blocks_limit = 1, 692c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna 693c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna .funcs = { { 694c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna .num_blocks = 8, 695c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc8) } 696c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna }, { 697c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna .num_blocks = 1, 698c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc) } 699c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna } } 700c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna}; 701c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna 702c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinnastatic const struct common_glue_ctx aesni_dec_xts = { 703c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna .num_funcs = 2, 704c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna .fpu_blocks_limit = 1, 705c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna 706c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna .funcs = { { 707c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna .num_blocks = 8, 708c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec8) } 709c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna }, { 710c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna .num_blocks = 1, 711c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec) } 712c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna } } 713c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna}; 714c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna 715c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinnastatic int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 716c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna struct scatterlist *src, unsigned int nbytes) 717c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna{ 718c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 719c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna 720c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna return glue_xts_crypt_128bit(&aesni_enc_xts, desc, dst, src, nbytes, 721c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna XTS_TWEAK_CAST(aesni_xts_tweak), 722c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna aes_ctx(ctx->raw_tweak_ctx), 723c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna aes_ctx(ctx->raw_crypt_ctx)); 724c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna} 725c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna 726c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinnastatic int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 727c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna struct scatterlist *src, unsigned int nbytes) 728c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna{ 729c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 730c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna 731c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna return glue_xts_crypt_128bit(&aesni_dec_xts, desc, dst, src, nbytes, 732c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna XTS_TWEAK_CAST(aesni_xts_tweak), 733c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna aes_ctx(ctx->raw_tweak_ctx), 734c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna aes_ctx(ctx->raw_crypt_ctx)); 735c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna} 736c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna 737c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna#else 738c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna 739023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinnastatic int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 740023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna struct scatterlist *src, unsigned int nbytes) 741023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna{ 742023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 743023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna be128 buf[8]; 744023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna struct xts_crypt_req req = { 745023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .tbuf = buf, 746023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .tbuflen = sizeof(buf), 747023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 748023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .tweak_ctx = aes_ctx(ctx->raw_tweak_ctx), 74932bec973a8435afc0b032da22174701f836549b2Jussi Kivilinna .tweak_fn = aesni_xts_tweak, 750023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .crypt_ctx = aes_ctx(ctx->raw_crypt_ctx), 751023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .crypt_fn = lrw_xts_encrypt_callback, 752023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna }; 753023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna int ret; 754023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 755023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 756023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 757023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna kernel_fpu_begin(); 758023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna ret = xts_crypt(desc, dst, src, nbytes, &req); 759023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna kernel_fpu_end(); 760023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 761023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna return ret; 762023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna} 763023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 764023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinnastatic int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 765023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna struct scatterlist *src, unsigned int nbytes) 766023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna{ 767023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 768023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna be128 buf[8]; 769023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna struct xts_crypt_req req = { 770023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .tbuf = buf, 771023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .tbuflen = sizeof(buf), 772023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 773023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .tweak_ctx = aes_ctx(ctx->raw_tweak_ctx), 77432bec973a8435afc0b032da22174701f836549b2Jussi Kivilinna .tweak_fn = aesni_xts_tweak, 775023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .crypt_ctx = aes_ctx(ctx->raw_crypt_ctx), 776023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .crypt_fn = lrw_xts_decrypt_callback, 777023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna }; 778023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna int ret; 779023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 780023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 781023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 782023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna kernel_fpu_begin(); 783023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna ret = xts_crypt(desc, dst, src, nbytes, &req); 784023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna kernel_fpu_end(); 785023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna 786023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna return ret; 7872cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying} 7882cf4ac8beb9dc50a315a6155b7b70e754d511958Huang Ying 789c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna#endif 790c456a9cd1ac4eae9147ffd7ac4fb77ca0fa980c6Jussi Kivilinna 791559ad0ff1368baea14dbc3207d55b02bd69bda4bMathias Krause#ifdef CONFIG_X86_64 7920bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Strukstatic int rfc4106_init(struct crypto_tfm *tfm) 7930bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk{ 7940bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct cryptd_aead *cryptd_tfm; 7950bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *) 7960bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN); 79760af520cf264ea26b2af3a6871bbd71850522aeaTadeusz Struk struct crypto_aead *cryptd_child; 79860af520cf264ea26b2af3a6871bbd71850522aeaTadeusz Struk struct aesni_rfc4106_gcm_ctx *child_ctx; 7990bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0); 8000bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if (IS_ERR(cryptd_tfm)) 8010bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk return PTR_ERR(cryptd_tfm); 80260af520cf264ea26b2af3a6871bbd71850522aeaTadeusz Struk 80360af520cf264ea26b2af3a6871bbd71850522aeaTadeusz Struk cryptd_child = cryptd_aead_child(cryptd_tfm); 80460af520cf264ea26b2af3a6871bbd71850522aeaTadeusz Struk child_ctx = aesni_rfc4106_gcm_ctx_get(cryptd_child); 80560af520cf264ea26b2af3a6871bbd71850522aeaTadeusz Struk memcpy(child_ctx, ctx, sizeof(*ctx)); 8060bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk ctx->cryptd_tfm = cryptd_tfm; 8070bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk tfm->crt_aead.reqsize = sizeof(struct aead_request) 8080bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk + crypto_aead_reqsize(&cryptd_tfm->base); 8090bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk return 0; 8100bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk} 8110bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 8120bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Strukstatic void rfc4106_exit(struct crypto_tfm *tfm) 8130bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk{ 8140bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct aesni_rfc4106_gcm_ctx *ctx = 8150bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk (struct aesni_rfc4106_gcm_ctx *) 8160bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN); 8170bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if (!IS_ERR(ctx->cryptd_tfm)) 8180bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk cryptd_free_aead(ctx->cryptd_tfm); 8190bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk return; 8200bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk} 8210bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 8220bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Strukstatic void 8230bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Strukrfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err) 8240bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk{ 8250bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct aesni_gcm_set_hash_subkey_result *result = req->data; 8260bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 8270bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if (err == -EINPROGRESS) 8280bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk return; 8290bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk result->err = err; 8300bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk complete(&result->completion); 8310bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk} 8320bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 8330bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Strukstatic int 8340bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Strukrfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len) 8350bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk{ 8360bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct crypto_ablkcipher *ctr_tfm; 8370bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct ablkcipher_request *req; 8380bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk int ret = -EINVAL; 8390bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct aesni_hash_subkey_req_data *req_data; 8400bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 8410bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0); 8420bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if (IS_ERR(ctr_tfm)) 8430bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk return PTR_ERR(ctr_tfm); 8440bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 8450bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk crypto_ablkcipher_clear_flags(ctr_tfm, ~0); 8460bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 8470bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len); 8487efd95f6270e210be90b94466bd3405b81e8d667Jesper Juhl if (ret) 849fc9044e2db8c13746cd886d6276028b27ed5c78eJesper Juhl goto out_free_ablkcipher; 8500bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 851fc9044e2db8c13746cd886d6276028b27ed5c78eJesper Juhl ret = -ENOMEM; 8520bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL); 853fc9044e2db8c13746cd886d6276028b27ed5c78eJesper Juhl if (!req) 8547efd95f6270e210be90b94466bd3405b81e8d667Jesper Juhl goto out_free_ablkcipher; 8550bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 8560bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk req_data = kmalloc(sizeof(*req_data), GFP_KERNEL); 857fc9044e2db8c13746cd886d6276028b27ed5c78eJesper Juhl if (!req_data) 8587efd95f6270e210be90b94466bd3405b81e8d667Jesper Juhl goto out_free_request; 859fc9044e2db8c13746cd886d6276028b27ed5c78eJesper Juhl 8600bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk memset(req_data->iv, 0, sizeof(req_data->iv)); 8610bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 8620bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk /* Clear the data in the hash sub key container to zero.*/ 8630bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk /* We want to cipher all zeros to create the hash sub key. */ 8640bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE); 8650bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 8660bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk init_completion(&req_data->result.completion); 8670bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE); 8680bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk ablkcipher_request_set_tfm(req, ctr_tfm); 8690bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP | 8700bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk CRYPTO_TFM_REQ_MAY_BACKLOG, 8710bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk rfc4106_set_hash_subkey_done, 8720bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk &req_data->result); 8730bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 8740bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk ablkcipher_request_set_crypt(req, &req_data->sg, 8750bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv); 8760bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 8770bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk ret = crypto_ablkcipher_encrypt(req); 8780bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if (ret == -EINPROGRESS || ret == -EBUSY) { 8790bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk ret = wait_for_completion_interruptible 8800bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk (&req_data->result.completion); 8810bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if (!ret) 8820bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk ret = req_data->result.err; 8830bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk } 884fc9044e2db8c13746cd886d6276028b27ed5c78eJesper Juhl kfree(req_data); 8857efd95f6270e210be90b94466bd3405b81e8d667Jesper Juhlout_free_request: 8860bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk ablkcipher_request_free(req); 8877efd95f6270e210be90b94466bd3405b81e8d667Jesper Juhlout_free_ablkcipher: 8880bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk crypto_free_ablkcipher(ctr_tfm); 8890bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk return ret; 8900bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk} 8910bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 8920bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Strukstatic int rfc4106_set_key(struct crypto_aead *parent, const u8 *key, 8930bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk unsigned int key_len) 8940bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk{ 8950bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk int ret = 0; 8960bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct crypto_tfm *tfm = crypto_aead_tfm(parent); 8970bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent); 89860af520cf264ea26b2af3a6871bbd71850522aeaTadeusz Struk struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 89960af520cf264ea26b2af3a6871bbd71850522aeaTadeusz Struk struct aesni_rfc4106_gcm_ctx *child_ctx = 90060af520cf264ea26b2af3a6871bbd71850522aeaTadeusz Struk aesni_rfc4106_gcm_ctx_get(cryptd_child); 901bf084d8f6eb4ded3f90a6ab79bb682db00ebfbd4Milan Broz u8 *new_key_align, *new_key_mem = NULL; 9020bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 9030bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if (key_len < 4) { 9040bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 9050bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk return -EINVAL; 9060bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk } 9070bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk /*Account for 4 byte nonce at the end.*/ 9080bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk key_len -= 4; 9090bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if (key_len != AES_KEYSIZE_128) { 9100bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 9110bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk return -EINVAL; 9120bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk } 9130bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 9140bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce)); 9150bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk /*This must be on a 16 byte boundary!*/ 9160bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN) 9170bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk return -EINVAL; 9180bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 9190bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if ((unsigned long)key % AESNI_ALIGN) { 9200bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk /*key is not aligned: use an auxuliar aligned pointer*/ 9210bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL); 9220bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if (!new_key_mem) 9230bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk return -ENOMEM; 9240bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 925bf084d8f6eb4ded3f90a6ab79bb682db00ebfbd4Milan Broz new_key_align = PTR_ALIGN(new_key_mem, AESNI_ALIGN); 926bf084d8f6eb4ded3f90a6ab79bb682db00ebfbd4Milan Broz memcpy(new_key_align, key, key_len); 927bf084d8f6eb4ded3f90a6ab79bb682db00ebfbd4Milan Broz key = new_key_align; 9280bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk } 9290bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 9300bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if (!irq_fpu_usable()) 9310bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk ret = crypto_aes_expand_key(&(ctx->aes_key_expanded), 9320bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk key, key_len); 9330bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk else { 9340bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk kernel_fpu_begin(); 9350bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len); 9360bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk kernel_fpu_end(); 9370bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk } 9380bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk /*This must be on a 16 byte boundary!*/ 9390bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) { 9400bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk ret = -EINVAL; 9410bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk goto exit; 9420bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk } 9430bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); 94460af520cf264ea26b2af3a6871bbd71850522aeaTadeusz Struk memcpy(child_ctx, ctx, sizeof(*ctx)); 9450bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Strukexit: 9460bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk kfree(new_key_mem); 9470bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk return ret; 9480bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk} 9490bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 9500bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk/* This is the Integrity Check Value (aka the authentication tag length and can 9510bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * be 8, 12 or 16 bytes long. */ 9520bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Strukstatic int rfc4106_set_authsize(struct crypto_aead *parent, 9530bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk unsigned int authsize) 9540bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk{ 9550bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent); 9560bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 9570bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 9580bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk switch (authsize) { 9590bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk case 8: 9600bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk case 12: 9610bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk case 16: 9620bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk break; 9630bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk default: 9640bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk return -EINVAL; 9650bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk } 9660bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk crypto_aead_crt(parent)->authsize = authsize; 9670bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk crypto_aead_crt(cryptd_child)->authsize = authsize; 9680bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk return 0; 9690bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk} 9700bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 9710bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Strukstatic int rfc4106_encrypt(struct aead_request *req) 9720bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk{ 9730bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk int ret; 9740bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct crypto_aead *tfm = crypto_aead_reqtfm(req); 9750bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 9760bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 9770bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if (!irq_fpu_usable()) { 9780bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct aead_request *cryptd_req = 9790bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk (struct aead_request *) aead_request_ctx(req); 9800bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk memcpy(cryptd_req, req, sizeof(*req)); 9810bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 9820bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk return crypto_aead_encrypt(cryptd_req); 9830bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk } else { 98460af520cf264ea26b2af3a6871bbd71850522aeaTadeusz Struk struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 9850bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk kernel_fpu_begin(); 9860bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk ret = cryptd_child->base.crt_aead.encrypt(req); 9870bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk kernel_fpu_end(); 9880bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk return ret; 9890bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk } 9900bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk} 9910bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 9920bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Strukstatic int rfc4106_decrypt(struct aead_request *req) 9930bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk{ 9940bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk int ret; 9950bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct crypto_aead *tfm = crypto_aead_reqtfm(req); 9960bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 9970bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 9980bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if (!irq_fpu_usable()) { 9990bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct aead_request *cryptd_req = 10000bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk (struct aead_request *) aead_request_ctx(req); 10010bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk memcpy(cryptd_req, req, sizeof(*req)); 10020bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 10030bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk return crypto_aead_decrypt(cryptd_req); 10040bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk } else { 100560af520cf264ea26b2af3a6871bbd71850522aeaTadeusz Struk struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 10060bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk kernel_fpu_begin(); 10070bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk ret = cryptd_child->base.crt_aead.decrypt(req); 10080bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk kernel_fpu_end(); 10090bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk return ret; 10100bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk } 10110bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk} 10120bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 10130bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Strukstatic int __driver_rfc4106_encrypt(struct aead_request *req) 10140bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk{ 10150bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk u8 one_entry_in_sg = 0; 10160bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk u8 *src, *dst, *assoc; 10170bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk __be32 counter = cpu_to_be32(1); 10180bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct crypto_aead *tfm = crypto_aead_reqtfm(req); 10190bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 10200bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk void *aes_ctx = &(ctx->aes_key_expanded); 10210bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk unsigned long auth_tag_len = crypto_aead_authsize(tfm); 10220bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk u8 iv_tab[16+AESNI_ALIGN]; 10230bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN); 10240bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct scatter_walk src_sg_walk; 10250bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct scatter_walk assoc_sg_walk; 10260bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct scatter_walk dst_sg_walk; 10270bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk unsigned int i; 10280bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 10290bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk /* Assuming we are supporting rfc4106 64-bit extended */ 10300bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk /* sequence numbers We need to have the AAD length equal */ 10310bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk /* to 8 or 12 bytes */ 10320bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if (unlikely(req->assoclen != 8 && req->assoclen != 12)) 10330bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk return -EINVAL; 10340bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk /* IV below built */ 10350bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk for (i = 0; i < 4; i++) 10360bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk *(iv+i) = ctx->nonce[i]; 10370bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk for (i = 0; i < 8; i++) 10380bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk *(iv+4+i) = req->iv[i]; 10390bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk *((__be32 *)(iv+12)) = counter; 10400bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 10410bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) { 10420bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk one_entry_in_sg = 1; 10430bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk scatterwalk_start(&src_sg_walk, req->src); 10440bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk scatterwalk_start(&assoc_sg_walk, req->assoc); 10458fd75e1216e0ba601a746177e6c102d5593b572fCong Wang src = scatterwalk_map(&src_sg_walk); 10468fd75e1216e0ba601a746177e6c102d5593b572fCong Wang assoc = scatterwalk_map(&assoc_sg_walk); 10470bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk dst = src; 10480bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if (unlikely(req->src != req->dst)) { 10490bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk scatterwalk_start(&dst_sg_walk, req->dst); 10508fd75e1216e0ba601a746177e6c102d5593b572fCong Wang dst = scatterwalk_map(&dst_sg_walk); 10510bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk } 10520bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 10530bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk } else { 10540bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk /* Allocate memory for src, dst, assoc */ 10550bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen, 10560bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk GFP_ATOMIC); 10570bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if (unlikely(!src)) 10580bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk return -ENOMEM; 10590bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk assoc = (src + req->cryptlen + auth_tag_len); 10600bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0); 10610bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk scatterwalk_map_and_copy(assoc, req->assoc, 0, 10620bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk req->assoclen, 0); 10630bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk dst = src; 10640bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk } 10650bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 1066d764593af924930d5c15685bc5946cb943da1a55Tim Chen aesni_gcm_enc_tfm(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv, 10670bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst 10680bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk + ((unsigned long)req->cryptlen), auth_tag_len); 10690bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 10700bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk /* The authTag (aka the Integrity Check Value) needs to be written 10710bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk * back to the packet. */ 10720bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if (one_entry_in_sg) { 10730bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if (unlikely(req->src != req->dst)) { 10748fd75e1216e0ba601a746177e6c102d5593b572fCong Wang scatterwalk_unmap(dst); 10750bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk scatterwalk_done(&dst_sg_walk, 0, 0); 10760bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk } 10778fd75e1216e0ba601a746177e6c102d5593b572fCong Wang scatterwalk_unmap(src); 10788fd75e1216e0ba601a746177e6c102d5593b572fCong Wang scatterwalk_unmap(assoc); 10790bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk scatterwalk_done(&src_sg_walk, 0, 0); 10800bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk scatterwalk_done(&assoc_sg_walk, 0, 0); 10810bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk } else { 10820bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk scatterwalk_map_and_copy(dst, req->dst, 0, 10830bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk req->cryptlen + auth_tag_len, 1); 10840bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk kfree(src); 10850bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk } 10860bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk return 0; 10870bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk} 10880bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 10890bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Strukstatic int __driver_rfc4106_decrypt(struct aead_request *req) 10900bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk{ 10910bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk u8 one_entry_in_sg = 0; 10920bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk u8 *src, *dst, *assoc; 10930bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk unsigned long tempCipherLen = 0; 10940bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk __be32 counter = cpu_to_be32(1); 10950bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk int retval = 0; 10960bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct crypto_aead *tfm = crypto_aead_reqtfm(req); 10970bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 10980bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk void *aes_ctx = &(ctx->aes_key_expanded); 10990bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk unsigned long auth_tag_len = crypto_aead_authsize(tfm); 11000bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk u8 iv_and_authTag[32+AESNI_ALIGN]; 11010bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN); 11020bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk u8 *authTag = iv + 16; 11030bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct scatter_walk src_sg_walk; 11040bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct scatter_walk assoc_sg_walk; 11050bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk struct scatter_walk dst_sg_walk; 11060bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk unsigned int i; 11070bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 11080bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if (unlikely((req->cryptlen < auth_tag_len) || 11090bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk (req->assoclen != 8 && req->assoclen != 12))) 11100bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk return -EINVAL; 11110bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk /* Assuming we are supporting rfc4106 64-bit extended */ 11120bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk /* sequence numbers We need to have the AAD length */ 11130bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk /* equal to 8 or 12 bytes */ 11140bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 11150bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len); 11160bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk /* IV below built */ 11170bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk for (i = 0; i < 4; i++) 11180bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk *(iv+i) = ctx->nonce[i]; 11190bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk for (i = 0; i < 8; i++) 11200bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk *(iv+4+i) = req->iv[i]; 11210bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk *((__be32 *)(iv+12)) = counter; 11220bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 11230bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) { 11240bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk one_entry_in_sg = 1; 11250bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk scatterwalk_start(&src_sg_walk, req->src); 11260bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk scatterwalk_start(&assoc_sg_walk, req->assoc); 11278fd75e1216e0ba601a746177e6c102d5593b572fCong Wang src = scatterwalk_map(&src_sg_walk); 11288fd75e1216e0ba601a746177e6c102d5593b572fCong Wang assoc = scatterwalk_map(&assoc_sg_walk); 11290bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk dst = src; 11300bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if (unlikely(req->src != req->dst)) { 11310bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk scatterwalk_start(&dst_sg_walk, req->dst); 11328fd75e1216e0ba601a746177e6c102d5593b572fCong Wang dst = scatterwalk_map(&dst_sg_walk); 11330bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk } 11340bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 11350bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk } else { 11360bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk /* Allocate memory for src, dst, assoc */ 11370bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC); 11380bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if (!src) 11390bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk return -ENOMEM; 11400bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk assoc = (src + req->cryptlen + auth_tag_len); 11410bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0); 11420bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk scatterwalk_map_and_copy(assoc, req->assoc, 0, 11430bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk req->assoclen, 0); 11440bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk dst = src; 11450bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk } 11460bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 1147d764593af924930d5c15685bc5946cb943da1a55Tim Chen aesni_gcm_dec_tfm(aes_ctx, dst, src, tempCipherLen, iv, 11480bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk ctx->hash_subkey, assoc, (unsigned long)req->assoclen, 11490bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk authTag, auth_tag_len); 11500bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 11510bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk /* Compare generated tag with passed in tag. */ 1152fed286110f4bab01f93f06c32951fbc120fb71b1Daniel Borkmann retval = crypto_memneq(src + tempCipherLen, authTag, auth_tag_len) ? 11530bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk -EBADMSG : 0; 11540bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 11550bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if (one_entry_in_sg) { 11560bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk if (unlikely(req->src != req->dst)) { 11578fd75e1216e0ba601a746177e6c102d5593b572fCong Wang scatterwalk_unmap(dst); 11580bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk scatterwalk_done(&dst_sg_walk, 0, 0); 11590bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk } 11608fd75e1216e0ba601a746177e6c102d5593b572fCong Wang scatterwalk_unmap(src); 11618fd75e1216e0ba601a746177e6c102d5593b572fCong Wang scatterwalk_unmap(assoc); 11620bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk scatterwalk_done(&src_sg_walk, 0, 0); 11630bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk scatterwalk_done(&assoc_sg_walk, 0, 0); 11640bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk } else { 11650bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1); 11660bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk kfree(src); 11670bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk } 11680bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk return retval; 11690bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk} 1170fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna#endif 11710bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 1172fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinnastatic struct crypto_alg aesni_algs[] = { { 1173fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_name = "aes", 1174fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_driver_name = "aes-aesni", 1175fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_priority = 300, 1176fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 1177fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_blocksize = AES_BLOCK_SIZE, 1178fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 1179fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna AESNI_ALIGN - 1, 1180fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_alignmask = 0, 1181fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_module = THIS_MODULE, 1182fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_u = { 1183fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cipher = { 1184fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cia_min_keysize = AES_MIN_KEY_SIZE, 1185fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cia_max_keysize = AES_MAX_KEY_SIZE, 1186fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cia_setkey = aes_set_key, 1187fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cia_encrypt = aes_encrypt, 1188fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cia_decrypt = aes_decrypt 1189fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna } 1190fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna } 1191fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna}, { 1192fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_name = "__aes-aesni", 1193fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_driver_name = "__driver-aes-aesni", 1194fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_priority = 0, 1195fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 1196fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_blocksize = AES_BLOCK_SIZE, 1197fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 1198fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna AESNI_ALIGN - 1, 1199fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_alignmask = 0, 1200fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_module = THIS_MODULE, 1201fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_u = { 1202fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cipher = { 1203fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cia_min_keysize = AES_MIN_KEY_SIZE, 1204fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cia_max_keysize = AES_MAX_KEY_SIZE, 1205fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cia_setkey = aes_set_key, 1206fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cia_encrypt = __aes_encrypt, 1207fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cia_decrypt = __aes_decrypt 1208fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna } 1209fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna } 1210fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna}, { 1211fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_name = "__ecb-aes-aesni", 1212fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_driver_name = "__driver-ecb-aes-aesni", 1213fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_priority = 0, 1214fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 1215fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_blocksize = AES_BLOCK_SIZE, 1216fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 1217fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna AESNI_ALIGN - 1, 1218fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_alignmask = 0, 1219fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_type = &crypto_blkcipher_type, 1220fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_module = THIS_MODULE, 1221fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_u = { 1222fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .blkcipher = { 1223fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .min_keysize = AES_MIN_KEY_SIZE, 1224fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .max_keysize = AES_MAX_KEY_SIZE, 1225fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .setkey = aes_set_key, 1226fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .encrypt = ecb_encrypt, 1227fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .decrypt = ecb_decrypt, 1228fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna }, 1229fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna }, 1230fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna}, { 1231fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_name = "__cbc-aes-aesni", 1232fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_driver_name = "__driver-cbc-aes-aesni", 1233fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_priority = 0, 1234fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 1235fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_blocksize = AES_BLOCK_SIZE, 1236fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 1237fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna AESNI_ALIGN - 1, 1238fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_alignmask = 0, 1239fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_type = &crypto_blkcipher_type, 1240fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_module = THIS_MODULE, 1241fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_u = { 1242fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .blkcipher = { 1243fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .min_keysize = AES_MIN_KEY_SIZE, 1244fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .max_keysize = AES_MAX_KEY_SIZE, 1245fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .setkey = aes_set_key, 1246fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .encrypt = cbc_encrypt, 1247fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .decrypt = cbc_decrypt, 1248fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna }, 1249fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna }, 1250fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna}, { 1251fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_name = "ecb(aes)", 1252fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_driver_name = "ecb-aes-aesni", 1253fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_priority = 400, 1254fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1255fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_blocksize = AES_BLOCK_SIZE, 1256a9629d7142ea22567eaa999232d8a31a7493665aJussi Kivilinna .cra_ctxsize = sizeof(struct async_helper_ctx), 1257fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_alignmask = 0, 1258fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_type = &crypto_ablkcipher_type, 1259fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_module = THIS_MODULE, 1260fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_init = ablk_ecb_init, 1261fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_exit = ablk_exit, 1262fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_u = { 1263fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .ablkcipher = { 1264fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .min_keysize = AES_MIN_KEY_SIZE, 1265fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .max_keysize = AES_MAX_KEY_SIZE, 1266fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .setkey = ablk_set_key, 1267fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .encrypt = ablk_encrypt, 1268fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .decrypt = ablk_decrypt, 1269fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna }, 1270fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna }, 1271fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna}, { 1272fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_name = "cbc(aes)", 1273fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_driver_name = "cbc-aes-aesni", 1274fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_priority = 400, 1275fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1276fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_blocksize = AES_BLOCK_SIZE, 1277a9629d7142ea22567eaa999232d8a31a7493665aJussi Kivilinna .cra_ctxsize = sizeof(struct async_helper_ctx), 1278fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_alignmask = 0, 1279fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_type = &crypto_ablkcipher_type, 1280fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_module = THIS_MODULE, 1281fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_init = ablk_cbc_init, 1282fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_exit = ablk_exit, 1283fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_u = { 1284fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .ablkcipher = { 1285fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .min_keysize = AES_MIN_KEY_SIZE, 1286fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .max_keysize = AES_MAX_KEY_SIZE, 1287fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .ivsize = AES_BLOCK_SIZE, 1288fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .setkey = ablk_set_key, 1289fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .encrypt = ablk_encrypt, 1290fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .decrypt = ablk_decrypt, 1291fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna }, 1292fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna }, 1293fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna#ifdef CONFIG_X86_64 1294fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna}, { 1295fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_name = "__ctr-aes-aesni", 1296fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_driver_name = "__driver-ctr-aes-aesni", 1297fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_priority = 0, 1298fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 1299fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_blocksize = 1, 1300fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 1301fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna AESNI_ALIGN - 1, 1302fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_alignmask = 0, 1303fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_type = &crypto_blkcipher_type, 1304fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_module = THIS_MODULE, 1305fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_u = { 1306fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .blkcipher = { 1307fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .min_keysize = AES_MIN_KEY_SIZE, 1308fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .max_keysize = AES_MAX_KEY_SIZE, 1309fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .ivsize = AES_BLOCK_SIZE, 1310fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .setkey = aes_set_key, 1311fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .encrypt = ctr_crypt, 1312fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .decrypt = ctr_crypt, 1313fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna }, 1314fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna }, 1315fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna}, { 1316fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_name = "ctr(aes)", 1317fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_driver_name = "ctr-aes-aesni", 1318fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_priority = 400, 1319fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1320fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_blocksize = 1, 1321a9629d7142ea22567eaa999232d8a31a7493665aJussi Kivilinna .cra_ctxsize = sizeof(struct async_helper_ctx), 1322fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_alignmask = 0, 1323fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_type = &crypto_ablkcipher_type, 1324fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_module = THIS_MODULE, 1325fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_init = ablk_ctr_init, 1326fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_exit = ablk_exit, 1327fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_u = { 1328fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .ablkcipher = { 1329fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .min_keysize = AES_MIN_KEY_SIZE, 1330fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .max_keysize = AES_MAX_KEY_SIZE, 1331fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .ivsize = AES_BLOCK_SIZE, 1332fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .setkey = ablk_set_key, 1333fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .encrypt = ablk_encrypt, 1334fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .decrypt = ablk_encrypt, 1335fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .geniv = "chainiv", 1336fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna }, 1337fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna }, 1338fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna}, { 1339fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_name = "__gcm-aes-aesni", 1340fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_driver_name = "__driver-gcm-aes-aesni", 13410bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk .cra_priority = 0, 13420bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk .cra_flags = CRYPTO_ALG_TYPE_AEAD, 13430bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk .cra_blocksize = 1, 1344fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + 1345fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna AESNI_ALIGN, 13460bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk .cra_alignmask = 0, 13470bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk .cra_type = &crypto_aead_type, 13480bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk .cra_module = THIS_MODULE, 13490bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk .cra_u = { 13500bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk .aead = { 13510bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk .encrypt = __driver_rfc4106_encrypt, 13520bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk .decrypt = __driver_rfc4106_decrypt, 13530bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk }, 13540bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk }, 1355fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna}, { 1356fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_name = "rfc4106(gcm(aes))", 1357fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_driver_name = "rfc4106-gcm-aesni", 1358fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_priority = 400, 1359fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, 1360fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_blocksize = 1, 1361fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + 1362fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna AESNI_ALIGN, 1363fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_alignmask = 0, 1364fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_type = &crypto_nivaead_type, 1365fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_module = THIS_MODULE, 1366fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_init = rfc4106_init, 1367fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_exit = rfc4106_exit, 1368fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_u = { 1369fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .aead = { 1370fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .setkey = rfc4106_set_key, 1371fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .setauthsize = rfc4106_set_authsize, 1372fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .encrypt = rfc4106_encrypt, 1373fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .decrypt = rfc4106_decrypt, 1374fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .geniv = "seqiv", 1375fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .ivsize = 8, 1376fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .maxauthsize = 16, 1377fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna }, 1378fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna }, 1379fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna#endif 1380023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna#ifdef HAS_PCBC 1381fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna}, { 1382023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_name = "pcbc(aes)", 1383023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_driver_name = "pcbc-aes-aesni", 1384fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_priority = 400, 1385fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1386fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_blocksize = AES_BLOCK_SIZE, 1387a9629d7142ea22567eaa999232d8a31a7493665aJussi Kivilinna .cra_ctxsize = sizeof(struct async_helper_ctx), 1388fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_alignmask = 0, 1389fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_type = &crypto_ablkcipher_type, 1390fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_module = THIS_MODULE, 1391023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_init = ablk_pcbc_init, 1392fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_exit = ablk_exit, 1393fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_u = { 1394fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .ablkcipher = { 1395023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .min_keysize = AES_MIN_KEY_SIZE, 1396023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .max_keysize = AES_MAX_KEY_SIZE, 1397fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .ivsize = AES_BLOCK_SIZE, 1398fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .setkey = ablk_set_key, 1399fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .encrypt = ablk_encrypt, 1400fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .decrypt = ablk_decrypt, 1401fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna }, 1402fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna }, 1403fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna#endif 1404fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna}, { 1405023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_name = "__lrw-aes-aesni", 1406023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_driver_name = "__driver-lrw-aes-aesni", 1407023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_priority = 0, 1408023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 1409023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_blocksize = AES_BLOCK_SIZE, 1410023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_ctxsize = sizeof(struct aesni_lrw_ctx), 1411023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_alignmask = 0, 1412023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_type = &crypto_blkcipher_type, 1413023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_module = THIS_MODULE, 1414023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_exit = lrw_aesni_exit_tfm, 1415023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_u = { 1416023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .blkcipher = { 1417023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE, 1418023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE, 1419023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .ivsize = AES_BLOCK_SIZE, 1420023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .setkey = lrw_aesni_setkey, 1421023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .encrypt = lrw_encrypt, 1422023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .decrypt = lrw_decrypt, 1423023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna }, 1424023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna }, 1425023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna}, { 1426023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_name = "__xts-aes-aesni", 1427023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_driver_name = "__driver-xts-aes-aesni", 1428023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_priority = 0, 1429023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 1430023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_blocksize = AES_BLOCK_SIZE, 1431023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_ctxsize = sizeof(struct aesni_xts_ctx), 1432023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_alignmask = 0, 1433023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_type = &crypto_blkcipher_type, 1434023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_module = THIS_MODULE, 1435023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_u = { 1436023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .blkcipher = { 1437023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .min_keysize = 2 * AES_MIN_KEY_SIZE, 1438023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .max_keysize = 2 * AES_MAX_KEY_SIZE, 1439023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .ivsize = AES_BLOCK_SIZE, 1440023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .setkey = xts_aesni_setkey, 1441023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .encrypt = xts_encrypt, 1442023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .decrypt = xts_decrypt, 1443023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna }, 1444023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna }, 1445023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna}, { 1446023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_name = "lrw(aes)", 1447023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_driver_name = "lrw-aes-aesni", 1448fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_priority = 400, 1449fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1450fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_blocksize = AES_BLOCK_SIZE, 1451a9629d7142ea22567eaa999232d8a31a7493665aJussi Kivilinna .cra_ctxsize = sizeof(struct async_helper_ctx), 1452fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_alignmask = 0, 1453fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_type = &crypto_ablkcipher_type, 1454fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_module = THIS_MODULE, 1455023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_init = ablk_init, 1456fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_exit = ablk_exit, 1457fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_u = { 1458fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .ablkcipher = { 1459023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE, 1460023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE, 1461fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .ivsize = AES_BLOCK_SIZE, 1462fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .setkey = ablk_set_key, 1463fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .encrypt = ablk_encrypt, 1464fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .decrypt = ablk_decrypt, 1465fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna }, 1466fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna }, 1467fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna}, { 1468fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_name = "xts(aes)", 1469fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_driver_name = "xts-aes-aesni", 1470fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_priority = 400, 1471fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1472fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_blocksize = AES_BLOCK_SIZE, 1473a9629d7142ea22567eaa999232d8a31a7493665aJussi Kivilinna .cra_ctxsize = sizeof(struct async_helper_ctx), 1474fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_alignmask = 0, 1475fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_type = &crypto_ablkcipher_type, 1476fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_module = THIS_MODULE, 1477023af608254add7ba037cd634cc5f2fb21ff6420Jussi Kivilinna .cra_init = ablk_init, 1478fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_exit = ablk_exit, 1479fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .cra_u = { 1480fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .ablkcipher = { 1481fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .min_keysize = 2 * AES_MIN_KEY_SIZE, 1482fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .max_keysize = 2 * AES_MAX_KEY_SIZE, 1483fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .ivsize = AES_BLOCK_SIZE, 1484fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .setkey = ablk_set_key, 1485fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .encrypt = ablk_encrypt, 1486fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna .decrypt = ablk_decrypt, 1487fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna }, 1488fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna }, 1489fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna} }; 14900bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 14913bd391f056df61e928de1680ff4a3e7e07e5b399Andi Kleen 14923bd391f056df61e928de1680ff4a3e7e07e5b399Andi Kleenstatic const struct x86_cpu_id aesni_cpu_id[] = { 14933bd391f056df61e928de1680ff4a3e7e07e5b399Andi Kleen X86_FEATURE_MATCH(X86_FEATURE_AES), 14943bd391f056df61e928de1680ff4a3e7e07e5b399Andi Kleen {} 14953bd391f056df61e928de1680ff4a3e7e07e5b399Andi Kleen}; 14963bd391f056df61e928de1680ff4a3e7e07e5b399Andi KleenMODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id); 14973bd391f056df61e928de1680ff4a3e7e07e5b399Andi Kleen 149854b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Yingstatic int __init aesni_init(void) 149954b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying{ 15007af6c2456851eb08d51d95de38ae8302994031e9Jussi Kivilinna int err; 150154b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 15023bd391f056df61e928de1680ff4a3e7e07e5b399Andi Kleen if (!x86_match_cpu(aesni_cpu_id)) 150354b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying return -ENODEV; 15048610d7bf608923157aee95cea30e162ed3417658Andy Shevchenko#ifdef CONFIG_X86_64 1505d764593af924930d5c15685bc5946cb943da1a55Tim Chen#ifdef CONFIG_AS_AVX2 1506d764593af924930d5c15685bc5946cb943da1a55Tim Chen if (boot_cpu_has(X86_FEATURE_AVX2)) { 1507d764593af924930d5c15685bc5946cb943da1a55Tim Chen pr_info("AVX2 version of gcm_enc/dec engaged.\n"); 1508d764593af924930d5c15685bc5946cb943da1a55Tim Chen aesni_gcm_enc_tfm = aesni_gcm_enc_avx2; 1509d764593af924930d5c15685bc5946cb943da1a55Tim Chen aesni_gcm_dec_tfm = aesni_gcm_dec_avx2; 1510d764593af924930d5c15685bc5946cb943da1a55Tim Chen } else 1511d764593af924930d5c15685bc5946cb943da1a55Tim Chen#endif 1512d764593af924930d5c15685bc5946cb943da1a55Tim Chen#ifdef CONFIG_AS_AVX 1513d764593af924930d5c15685bc5946cb943da1a55Tim Chen if (boot_cpu_has(X86_FEATURE_AVX)) { 1514d764593af924930d5c15685bc5946cb943da1a55Tim Chen pr_info("AVX version of gcm_enc/dec engaged.\n"); 1515d764593af924930d5c15685bc5946cb943da1a55Tim Chen aesni_gcm_enc_tfm = aesni_gcm_enc_avx; 1516d764593af924930d5c15685bc5946cb943da1a55Tim Chen aesni_gcm_dec_tfm = aesni_gcm_dec_avx; 1517d764593af924930d5c15685bc5946cb943da1a55Tim Chen } else 1518d764593af924930d5c15685bc5946cb943da1a55Tim Chen#endif 1519d764593af924930d5c15685bc5946cb943da1a55Tim Chen { 1520d764593af924930d5c15685bc5946cb943da1a55Tim Chen pr_info("SSE version of gcm_enc/dec engaged.\n"); 1521d764593af924930d5c15685bc5946cb943da1a55Tim Chen aesni_gcm_enc_tfm = aesni_gcm_enc; 1522d764593af924930d5c15685bc5946cb943da1a55Tim Chen aesni_gcm_dec_tfm = aesni_gcm_dec; 1523d764593af924930d5c15685bc5946cb943da1a55Tim Chen } 152422cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan aesni_ctr_enc_tfm = aesni_ctr_enc; 15255cfed7b3351ce33b60c4a1d02ffd712cdf4f49cdMathias Krause#ifdef CONFIG_AS_AVX 152622cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan if (cpu_has_avx) { 152722cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan /* optimize performance of ctr mode encryption transform */ 152822cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm; 152922cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan pr_info("AES CTR mode by8 optimization enabled\n"); 153022cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan } 153122cddcc7df8fd35d52646ee220658d26ef09da17chandramouli narayanan#endif 15328610d7bf608923157aee95cea30e162ed3417658Andy Shevchenko#endif 15330bd82f5f6355775fbaf7d3c664432ce1b862be1eTadeusz Struk 1534fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna err = crypto_fpu_init(); 1535fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna if (err) 1536fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna return err; 153754b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 1538fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna return crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); 153954b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying} 154054b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 154154b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Yingstatic void __exit aesni_exit(void) 154254b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying{ 1543fa46ccb8eb960c62c1e5e3237085d4007788a345Jussi Kivilinna crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); 1544b23b64516500df6b70fcafb820970f18538252cfAndy Lutomirski 1545b23b64516500df6b70fcafb820970f18538252cfAndy Lutomirski crypto_fpu_exit(); 154654b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying} 154754b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 154854b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Yingmodule_init(aesni_init); 154954b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Yingmodule_exit(aesni_exit); 155054b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang Ying 155154b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang YingMODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized"); 155254b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang YingMODULE_LICENSE("GPL"); 155354b6a1bd5364aca95cd6ffae00f2b64c6511122cHuang YingMODULE_ALIAS("aes"); 1554