1/* 2 * Glue Code for SSE2 assembler versions of Serpent Cipher 3 * 4 * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> 5 * 6 * Glue code based on aesni-intel_glue.c by: 7 * Copyright (C) 2008, Intel Corp. 8 * Author: Huang Ying <ying.huang@intel.com> 9 * 10 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: 11 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> 12 * CTR part based on code (crypto/ctr.c) by: 13 * (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com> 14 * 15 * This program is free software; you can redistribute it and/or modify 16 * it under the terms of the GNU General Public License as published by 17 * the Free Software Foundation; either version 2 of the License, or 18 * (at your option) any later version. 19 * 20 * This program is distributed in the hope that it will be useful, 21 * but WITHOUT ANY WARRANTY; without even the implied warranty of 22 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 23 * GNU General Public License for more details. 24 * 25 * You should have received a copy of the GNU General Public License 26 * along with this program; if not, write to the Free Software 27 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 28 * USA 29 * 30 */ 31 32#include <linux/module.h> 33#include <linux/hardirq.h> 34#include <linux/types.h> 35#include <linux/crypto.h> 36#include <linux/err.h> 37#include <crypto/algapi.h> 38#include <crypto/serpent.h> 39#include <crypto/cryptd.h> 40#include <crypto/b128ops.h> 41#include <crypto/ctr.h> 42#include <crypto/lrw.h> 43#include <crypto/xts.h> 44#include <asm/i387.h> 45#include <asm/serpent.h> 46#include <crypto/scatterwalk.h> 47#include <linux/workqueue.h> 48#include <linux/spinlock.h> 49 50struct async_serpent_ctx { 51 struct cryptd_ablkcipher *cryptd_tfm; 52}; 53 54static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes) 55{ 56 if (fpu_enabled) 57 return true; 58 59 /* SSE2 is only used when chunk to be processed is large enough, so 60 * do not enable FPU until it is necessary. 61 */ 62 if (nbytes < SERPENT_BLOCK_SIZE * SERPENT_PARALLEL_BLOCKS) 63 return false; 64 65 kernel_fpu_begin(); 66 return true; 67} 68 69static inline void serpent_fpu_end(bool fpu_enabled) 70{ 71 if (fpu_enabled) 72 kernel_fpu_end(); 73} 74 75static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk, 76 bool enc) 77{ 78 bool fpu_enabled = false; 79 struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 80 const unsigned int bsize = SERPENT_BLOCK_SIZE; 81 unsigned int nbytes; 82 int err; 83 84 err = blkcipher_walk_virt(desc, walk); 85 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 86 87 while ((nbytes = walk->nbytes)) { 88 u8 *wsrc = walk->src.virt.addr; 89 u8 *wdst = walk->dst.virt.addr; 90 91 fpu_enabled = serpent_fpu_begin(fpu_enabled, nbytes); 92 93 /* Process multi-block batch */ 94 if (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS) { 95 do { 96 if (enc) 97 serpent_enc_blk_xway(ctx, wdst, wsrc); 98 else 99 serpent_dec_blk_xway(ctx, wdst, wsrc); 100 101 wsrc += bsize * SERPENT_PARALLEL_BLOCKS; 102 wdst += bsize * SERPENT_PARALLEL_BLOCKS; 103 nbytes -= bsize * SERPENT_PARALLEL_BLOCKS; 104 } while (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS); 105 106 if (nbytes < bsize) 107 goto done; 108 } 109 110 /* Handle leftovers */ 111 do { 112 if (enc) 113 __serpent_encrypt(ctx, wdst, wsrc); 114 else 115 __serpent_decrypt(ctx, wdst, wsrc); 116 117 wsrc += bsize; 118 wdst += bsize; 119 nbytes -= bsize; 120 } while (nbytes >= bsize); 121 122done: 123 err = blkcipher_walk_done(desc, walk, nbytes); 124 } 125 126 serpent_fpu_end(fpu_enabled); 127 return err; 128} 129 130static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 131 struct scatterlist *src, unsigned int nbytes) 132{ 133 struct blkcipher_walk walk; 134 135 blkcipher_walk_init(&walk, dst, src, nbytes); 136 return ecb_crypt(desc, &walk, true); 137} 138 139static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 140 struct scatterlist *src, unsigned int nbytes) 141{ 142 struct blkcipher_walk walk; 143 144 blkcipher_walk_init(&walk, dst, src, nbytes); 145 return ecb_crypt(desc, &walk, false); 146} 147 148static unsigned int __cbc_encrypt(struct blkcipher_desc *desc, 149 struct blkcipher_walk *walk) 150{ 151 struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 152 const unsigned int bsize = SERPENT_BLOCK_SIZE; 153 unsigned int nbytes = walk->nbytes; 154 u128 *src = (u128 *)walk->src.virt.addr; 155 u128 *dst = (u128 *)walk->dst.virt.addr; 156 u128 *iv = (u128 *)walk->iv; 157 158 do { 159 u128_xor(dst, src, iv); 160 __serpent_encrypt(ctx, (u8 *)dst, (u8 *)dst); 161 iv = dst; 162 163 src += 1; 164 dst += 1; 165 nbytes -= bsize; 166 } while (nbytes >= bsize); 167 168 u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv); 169 return nbytes; 170} 171 172static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 173 struct scatterlist *src, unsigned int nbytes) 174{ 175 struct blkcipher_walk walk; 176 int err; 177 178 blkcipher_walk_init(&walk, dst, src, nbytes); 179 err = blkcipher_walk_virt(desc, &walk); 180 181 while ((nbytes = walk.nbytes)) { 182 nbytes = __cbc_encrypt(desc, &walk); 183 err = blkcipher_walk_done(desc, &walk, nbytes); 184 } 185 186 return err; 187} 188 189static unsigned int __cbc_decrypt(struct blkcipher_desc *desc, 190 struct blkcipher_walk *walk) 191{ 192 struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 193 const unsigned int bsize = SERPENT_BLOCK_SIZE; 194 unsigned int nbytes = walk->nbytes; 195 u128 *src = (u128 *)walk->src.virt.addr; 196 u128 *dst = (u128 *)walk->dst.virt.addr; 197 u128 ivs[SERPENT_PARALLEL_BLOCKS - 1]; 198 u128 last_iv; 199 int i; 200 201 /* Start of the last block. */ 202 src += nbytes / bsize - 1; 203 dst += nbytes / bsize - 1; 204 205 last_iv = *src; 206 207 /* Process multi-block batch */ 208 if (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS) { 209 do { 210 nbytes -= bsize * (SERPENT_PARALLEL_BLOCKS - 1); 211 src -= SERPENT_PARALLEL_BLOCKS - 1; 212 dst -= SERPENT_PARALLEL_BLOCKS - 1; 213 214 for (i = 0; i < SERPENT_PARALLEL_BLOCKS - 1; i++) 215 ivs[i] = src[i]; 216 217 serpent_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src); 218 219 for (i = 0; i < SERPENT_PARALLEL_BLOCKS - 1; i++) 220 u128_xor(dst + (i + 1), dst + (i + 1), ivs + i); 221 222 nbytes -= bsize; 223 if (nbytes < bsize) 224 goto done; 225 226 u128_xor(dst, dst, src - 1); 227 src -= 1; 228 dst -= 1; 229 } while (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS); 230 231 if (nbytes < bsize) 232 goto done; 233 } 234 235 /* Handle leftovers */ 236 for (;;) { 237 __serpent_decrypt(ctx, (u8 *)dst, (u8 *)src); 238 239 nbytes -= bsize; 240 if (nbytes < bsize) 241 break; 242 243 u128_xor(dst, dst, src - 1); 244 src -= 1; 245 dst -= 1; 246 } 247 248done: 249 u128_xor(dst, dst, (u128 *)walk->iv); 250 *(u128 *)walk->iv = last_iv; 251 252 return nbytes; 253} 254 255static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 256 struct scatterlist *src, unsigned int nbytes) 257{ 258 bool fpu_enabled = false; 259 struct blkcipher_walk walk; 260 int err; 261 262 blkcipher_walk_init(&walk, dst, src, nbytes); 263 err = blkcipher_walk_virt(desc, &walk); 264 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 265 266 while ((nbytes = walk.nbytes)) { 267 fpu_enabled = serpent_fpu_begin(fpu_enabled, nbytes); 268 nbytes = __cbc_decrypt(desc, &walk); 269 err = blkcipher_walk_done(desc, &walk, nbytes); 270 } 271 272 serpent_fpu_end(fpu_enabled); 273 return err; 274} 275 276static inline void u128_to_be128(be128 *dst, const u128 *src) 277{ 278 dst->a = cpu_to_be64(src->a); 279 dst->b = cpu_to_be64(src->b); 280} 281 282static inline void be128_to_u128(u128 *dst, const be128 *src) 283{ 284 dst->a = be64_to_cpu(src->a); 285 dst->b = be64_to_cpu(src->b); 286} 287 288static inline void u128_inc(u128 *i) 289{ 290 i->b++; 291 if (!i->b) 292 i->a++; 293} 294 295static void ctr_crypt_final(struct blkcipher_desc *desc, 296 struct blkcipher_walk *walk) 297{ 298 struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 299 u8 *ctrblk = walk->iv; 300 u8 keystream[SERPENT_BLOCK_SIZE]; 301 u8 *src = walk->src.virt.addr; 302 u8 *dst = walk->dst.virt.addr; 303 unsigned int nbytes = walk->nbytes; 304 305 __serpent_encrypt(ctx, keystream, ctrblk); 306 crypto_xor(keystream, src, nbytes); 307 memcpy(dst, keystream, nbytes); 308 309 crypto_inc(ctrblk, SERPENT_BLOCK_SIZE); 310} 311 312static unsigned int __ctr_crypt(struct blkcipher_desc *desc, 313 struct blkcipher_walk *walk) 314{ 315 struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 316 const unsigned int bsize = SERPENT_BLOCK_SIZE; 317 unsigned int nbytes = walk->nbytes; 318 u128 *src = (u128 *)walk->src.virt.addr; 319 u128 *dst = (u128 *)walk->dst.virt.addr; 320 u128 ctrblk; 321 be128 ctrblocks[SERPENT_PARALLEL_BLOCKS]; 322 int i; 323 324 be128_to_u128(&ctrblk, (be128 *)walk->iv); 325 326 /* Process multi-block batch */ 327 if (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS) { 328 do { 329 /* create ctrblks for parallel encrypt */ 330 for (i = 0; i < SERPENT_PARALLEL_BLOCKS; i++) { 331 if (dst != src) 332 dst[i] = src[i]; 333 334 u128_to_be128(&ctrblocks[i], &ctrblk); 335 u128_inc(&ctrblk); 336 } 337 338 serpent_enc_blk_xway_xor(ctx, (u8 *)dst, 339 (u8 *)ctrblocks); 340 341 src += SERPENT_PARALLEL_BLOCKS; 342 dst += SERPENT_PARALLEL_BLOCKS; 343 nbytes -= bsize * SERPENT_PARALLEL_BLOCKS; 344 } while (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS); 345 346 if (nbytes < bsize) 347 goto done; 348 } 349 350 /* Handle leftovers */ 351 do { 352 if (dst != src) 353 *dst = *src; 354 355 u128_to_be128(&ctrblocks[0], &ctrblk); 356 u128_inc(&ctrblk); 357 358 __serpent_encrypt(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks); 359 u128_xor(dst, dst, (u128 *)ctrblocks); 360 361 src += 1; 362 dst += 1; 363 nbytes -= bsize; 364 } while (nbytes >= bsize); 365 366done: 367 u128_to_be128((be128 *)walk->iv, &ctrblk); 368 return nbytes; 369} 370 371static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, 372 struct scatterlist *src, unsigned int nbytes) 373{ 374 bool fpu_enabled = false; 375 struct blkcipher_walk walk; 376 int err; 377 378 blkcipher_walk_init(&walk, dst, src, nbytes); 379 err = blkcipher_walk_virt_block(desc, &walk, SERPENT_BLOCK_SIZE); 380 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 381 382 while ((nbytes = walk.nbytes) >= SERPENT_BLOCK_SIZE) { 383 fpu_enabled = serpent_fpu_begin(fpu_enabled, nbytes); 384 nbytes = __ctr_crypt(desc, &walk); 385 err = blkcipher_walk_done(desc, &walk, nbytes); 386 } 387 388 serpent_fpu_end(fpu_enabled); 389 390 if (walk.nbytes) { 391 ctr_crypt_final(desc, &walk); 392 err = blkcipher_walk_done(desc, &walk, 0); 393 } 394 395 return err; 396} 397 398struct crypt_priv { 399 struct serpent_ctx *ctx; 400 bool fpu_enabled; 401}; 402 403static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) 404{ 405 const unsigned int bsize = SERPENT_BLOCK_SIZE; 406 struct crypt_priv *ctx = priv; 407 int i; 408 409 ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes); 410 411 if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) { 412 serpent_enc_blk_xway(ctx->ctx, srcdst, srcdst); 413 return; 414 } 415 416 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) 417 __serpent_encrypt(ctx->ctx, srcdst, srcdst); 418} 419 420static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) 421{ 422 const unsigned int bsize = SERPENT_BLOCK_SIZE; 423 struct crypt_priv *ctx = priv; 424 int i; 425 426 ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes); 427 428 if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) { 429 serpent_dec_blk_xway(ctx->ctx, srcdst, srcdst); 430 return; 431 } 432 433 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) 434 __serpent_decrypt(ctx->ctx, srcdst, srcdst); 435} 436 437struct serpent_lrw_ctx { 438 struct lrw_table_ctx lrw_table; 439 struct serpent_ctx serpent_ctx; 440}; 441 442static int lrw_serpent_setkey(struct crypto_tfm *tfm, const u8 *key, 443 unsigned int keylen) 444{ 445 struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm); 446 int err; 447 448 err = __serpent_setkey(&ctx->serpent_ctx, key, keylen - 449 SERPENT_BLOCK_SIZE); 450 if (err) 451 return err; 452 453 return lrw_init_table(&ctx->lrw_table, key + keylen - 454 SERPENT_BLOCK_SIZE); 455} 456 457static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 458 struct scatterlist *src, unsigned int nbytes) 459{ 460 struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 461 be128 buf[SERPENT_PARALLEL_BLOCKS]; 462 struct crypt_priv crypt_ctx = { 463 .ctx = &ctx->serpent_ctx, 464 .fpu_enabled = false, 465 }; 466 struct lrw_crypt_req req = { 467 .tbuf = buf, 468 .tbuflen = sizeof(buf), 469 470 .table_ctx = &ctx->lrw_table, 471 .crypt_ctx = &crypt_ctx, 472 .crypt_fn = encrypt_callback, 473 }; 474 int ret; 475 476 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 477 ret = lrw_crypt(desc, dst, src, nbytes, &req); 478 serpent_fpu_end(crypt_ctx.fpu_enabled); 479 480 return ret; 481} 482 483static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 484 struct scatterlist *src, unsigned int nbytes) 485{ 486 struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 487 be128 buf[SERPENT_PARALLEL_BLOCKS]; 488 struct crypt_priv crypt_ctx = { 489 .ctx = &ctx->serpent_ctx, 490 .fpu_enabled = false, 491 }; 492 struct lrw_crypt_req req = { 493 .tbuf = buf, 494 .tbuflen = sizeof(buf), 495 496 .table_ctx = &ctx->lrw_table, 497 .crypt_ctx = &crypt_ctx, 498 .crypt_fn = decrypt_callback, 499 }; 500 int ret; 501 502 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 503 ret = lrw_crypt(desc, dst, src, nbytes, &req); 504 serpent_fpu_end(crypt_ctx.fpu_enabled); 505 506 return ret; 507} 508 509static void lrw_exit_tfm(struct crypto_tfm *tfm) 510{ 511 struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm); 512 513 lrw_free_table(&ctx->lrw_table); 514} 515 516struct serpent_xts_ctx { 517 struct serpent_ctx tweak_ctx; 518 struct serpent_ctx crypt_ctx; 519}; 520 521static int xts_serpent_setkey(struct crypto_tfm *tfm, const u8 *key, 522 unsigned int keylen) 523{ 524 struct serpent_xts_ctx *ctx = crypto_tfm_ctx(tfm); 525 u32 *flags = &tfm->crt_flags; 526 int err; 527 528 /* key consists of keys of equal size concatenated, therefore 529 * the length must be even 530 */ 531 if (keylen % 2) { 532 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 533 return -EINVAL; 534 } 535 536 /* first half of xts-key is for crypt */ 537 err = __serpent_setkey(&ctx->crypt_ctx, key, keylen / 2); 538 if (err) 539 return err; 540 541 /* second half of xts-key is for tweak */ 542 return __serpent_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2); 543} 544 545static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 546 struct scatterlist *src, unsigned int nbytes) 547{ 548 struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 549 be128 buf[SERPENT_PARALLEL_BLOCKS]; 550 struct crypt_priv crypt_ctx = { 551 .ctx = &ctx->crypt_ctx, 552 .fpu_enabled = false, 553 }; 554 struct xts_crypt_req req = { 555 .tbuf = buf, 556 .tbuflen = sizeof(buf), 557 558 .tweak_ctx = &ctx->tweak_ctx, 559 .tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt), 560 .crypt_ctx = &crypt_ctx, 561 .crypt_fn = encrypt_callback, 562 }; 563 int ret; 564 565 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 566 ret = xts_crypt(desc, dst, src, nbytes, &req); 567 serpent_fpu_end(crypt_ctx.fpu_enabled); 568 569 return ret; 570} 571 572static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 573 struct scatterlist *src, unsigned int nbytes) 574{ 575 struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 576 be128 buf[SERPENT_PARALLEL_BLOCKS]; 577 struct crypt_priv crypt_ctx = { 578 .ctx = &ctx->crypt_ctx, 579 .fpu_enabled = false, 580 }; 581 struct xts_crypt_req req = { 582 .tbuf = buf, 583 .tbuflen = sizeof(buf), 584 585 .tweak_ctx = &ctx->tweak_ctx, 586 .tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt), 587 .crypt_ctx = &crypt_ctx, 588 .crypt_fn = decrypt_callback, 589 }; 590 int ret; 591 592 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 593 ret = xts_crypt(desc, dst, src, nbytes, &req); 594 serpent_fpu_end(crypt_ctx.fpu_enabled); 595 596 return ret; 597} 598 599static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key, 600 unsigned int key_len) 601{ 602 struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm); 603 struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base; 604 int err; 605 606 crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); 607 crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm) 608 & CRYPTO_TFM_REQ_MASK); 609 err = crypto_ablkcipher_setkey(child, key, key_len); 610 crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child) 611 & CRYPTO_TFM_RES_MASK); 612 return err; 613} 614 615static int __ablk_encrypt(struct ablkcipher_request *req) 616{ 617 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 618 struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm); 619 struct blkcipher_desc desc; 620 621 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm); 622 desc.info = req->info; 623 desc.flags = 0; 624 625 return crypto_blkcipher_crt(desc.tfm)->encrypt( 626 &desc, req->dst, req->src, req->nbytes); 627} 628 629static int ablk_encrypt(struct ablkcipher_request *req) 630{ 631 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 632 struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm); 633 634 if (!irq_fpu_usable()) { 635 struct ablkcipher_request *cryptd_req = 636 ablkcipher_request_ctx(req); 637 638 memcpy(cryptd_req, req, sizeof(*req)); 639 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 640 641 return crypto_ablkcipher_encrypt(cryptd_req); 642 } else { 643 return __ablk_encrypt(req); 644 } 645} 646 647static int ablk_decrypt(struct ablkcipher_request *req) 648{ 649 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 650 struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm); 651 652 if (!irq_fpu_usable()) { 653 struct ablkcipher_request *cryptd_req = 654 ablkcipher_request_ctx(req); 655 656 memcpy(cryptd_req, req, sizeof(*req)); 657 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 658 659 return crypto_ablkcipher_decrypt(cryptd_req); 660 } else { 661 struct blkcipher_desc desc; 662 663 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm); 664 desc.info = req->info; 665 desc.flags = 0; 666 667 return crypto_blkcipher_crt(desc.tfm)->decrypt( 668 &desc, req->dst, req->src, req->nbytes); 669 } 670} 671 672static void ablk_exit(struct crypto_tfm *tfm) 673{ 674 struct async_serpent_ctx *ctx = crypto_tfm_ctx(tfm); 675 676 cryptd_free_ablkcipher(ctx->cryptd_tfm); 677} 678 679static int ablk_init(struct crypto_tfm *tfm) 680{ 681 struct async_serpent_ctx *ctx = crypto_tfm_ctx(tfm); 682 struct cryptd_ablkcipher *cryptd_tfm; 683 char drv_name[CRYPTO_MAX_ALG_NAME]; 684 685 snprintf(drv_name, sizeof(drv_name), "__driver-%s", 686 crypto_tfm_alg_driver_name(tfm)); 687 688 cryptd_tfm = cryptd_alloc_ablkcipher(drv_name, 0, 0); 689 if (IS_ERR(cryptd_tfm)) 690 return PTR_ERR(cryptd_tfm); 691 692 ctx->cryptd_tfm = cryptd_tfm; 693 tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) + 694 crypto_ablkcipher_reqsize(&cryptd_tfm->base); 695 696 return 0; 697} 698 699static struct crypto_alg serpent_algs[10] = { { 700 .cra_name = "__ecb-serpent-sse2", 701 .cra_driver_name = "__driver-ecb-serpent-sse2", 702 .cra_priority = 0, 703 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 704 .cra_blocksize = SERPENT_BLOCK_SIZE, 705 .cra_ctxsize = sizeof(struct serpent_ctx), 706 .cra_alignmask = 0, 707 .cra_type = &crypto_blkcipher_type, 708 .cra_module = THIS_MODULE, 709 .cra_list = LIST_HEAD_INIT(serpent_algs[0].cra_list), 710 .cra_u = { 711 .blkcipher = { 712 .min_keysize = SERPENT_MIN_KEY_SIZE, 713 .max_keysize = SERPENT_MAX_KEY_SIZE, 714 .setkey = serpent_setkey, 715 .encrypt = ecb_encrypt, 716 .decrypt = ecb_decrypt, 717 }, 718 }, 719}, { 720 .cra_name = "__cbc-serpent-sse2", 721 .cra_driver_name = "__driver-cbc-serpent-sse2", 722 .cra_priority = 0, 723 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 724 .cra_blocksize = SERPENT_BLOCK_SIZE, 725 .cra_ctxsize = sizeof(struct serpent_ctx), 726 .cra_alignmask = 0, 727 .cra_type = &crypto_blkcipher_type, 728 .cra_module = THIS_MODULE, 729 .cra_list = LIST_HEAD_INIT(serpent_algs[1].cra_list), 730 .cra_u = { 731 .blkcipher = { 732 .min_keysize = SERPENT_MIN_KEY_SIZE, 733 .max_keysize = SERPENT_MAX_KEY_SIZE, 734 .setkey = serpent_setkey, 735 .encrypt = cbc_encrypt, 736 .decrypt = cbc_decrypt, 737 }, 738 }, 739}, { 740 .cra_name = "__ctr-serpent-sse2", 741 .cra_driver_name = "__driver-ctr-serpent-sse2", 742 .cra_priority = 0, 743 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 744 .cra_blocksize = 1, 745 .cra_ctxsize = sizeof(struct serpent_ctx), 746 .cra_alignmask = 0, 747 .cra_type = &crypto_blkcipher_type, 748 .cra_module = THIS_MODULE, 749 .cra_list = LIST_HEAD_INIT(serpent_algs[2].cra_list), 750 .cra_u = { 751 .blkcipher = { 752 .min_keysize = SERPENT_MIN_KEY_SIZE, 753 .max_keysize = SERPENT_MAX_KEY_SIZE, 754 .ivsize = SERPENT_BLOCK_SIZE, 755 .setkey = serpent_setkey, 756 .encrypt = ctr_crypt, 757 .decrypt = ctr_crypt, 758 }, 759 }, 760}, { 761 .cra_name = "__lrw-serpent-sse2", 762 .cra_driver_name = "__driver-lrw-serpent-sse2", 763 .cra_priority = 0, 764 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 765 .cra_blocksize = SERPENT_BLOCK_SIZE, 766 .cra_ctxsize = sizeof(struct serpent_lrw_ctx), 767 .cra_alignmask = 0, 768 .cra_type = &crypto_blkcipher_type, 769 .cra_module = THIS_MODULE, 770 .cra_list = LIST_HEAD_INIT(serpent_algs[3].cra_list), 771 .cra_exit = lrw_exit_tfm, 772 .cra_u = { 773 .blkcipher = { 774 .min_keysize = SERPENT_MIN_KEY_SIZE + 775 SERPENT_BLOCK_SIZE, 776 .max_keysize = SERPENT_MAX_KEY_SIZE + 777 SERPENT_BLOCK_SIZE, 778 .ivsize = SERPENT_BLOCK_SIZE, 779 .setkey = lrw_serpent_setkey, 780 .encrypt = lrw_encrypt, 781 .decrypt = lrw_decrypt, 782 }, 783 }, 784}, { 785 .cra_name = "__xts-serpent-sse2", 786 .cra_driver_name = "__driver-xts-serpent-sse2", 787 .cra_priority = 0, 788 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 789 .cra_blocksize = SERPENT_BLOCK_SIZE, 790 .cra_ctxsize = sizeof(struct serpent_xts_ctx), 791 .cra_alignmask = 0, 792 .cra_type = &crypto_blkcipher_type, 793 .cra_module = THIS_MODULE, 794 .cra_list = LIST_HEAD_INIT(serpent_algs[4].cra_list), 795 .cra_u = { 796 .blkcipher = { 797 .min_keysize = SERPENT_MIN_KEY_SIZE * 2, 798 .max_keysize = SERPENT_MAX_KEY_SIZE * 2, 799 .ivsize = SERPENT_BLOCK_SIZE, 800 .setkey = xts_serpent_setkey, 801 .encrypt = xts_encrypt, 802 .decrypt = xts_decrypt, 803 }, 804 }, 805}, { 806 .cra_name = "ecb(serpent)", 807 .cra_driver_name = "ecb-serpent-sse2", 808 .cra_priority = 400, 809 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 810 .cra_blocksize = SERPENT_BLOCK_SIZE, 811 .cra_ctxsize = sizeof(struct async_serpent_ctx), 812 .cra_alignmask = 0, 813 .cra_type = &crypto_ablkcipher_type, 814 .cra_module = THIS_MODULE, 815 .cra_list = LIST_HEAD_INIT(serpent_algs[5].cra_list), 816 .cra_init = ablk_init, 817 .cra_exit = ablk_exit, 818 .cra_u = { 819 .ablkcipher = { 820 .min_keysize = SERPENT_MIN_KEY_SIZE, 821 .max_keysize = SERPENT_MAX_KEY_SIZE, 822 .setkey = ablk_set_key, 823 .encrypt = ablk_encrypt, 824 .decrypt = ablk_decrypt, 825 }, 826 }, 827}, { 828 .cra_name = "cbc(serpent)", 829 .cra_driver_name = "cbc-serpent-sse2", 830 .cra_priority = 400, 831 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 832 .cra_blocksize = SERPENT_BLOCK_SIZE, 833 .cra_ctxsize = sizeof(struct async_serpent_ctx), 834 .cra_alignmask = 0, 835 .cra_type = &crypto_ablkcipher_type, 836 .cra_module = THIS_MODULE, 837 .cra_list = LIST_HEAD_INIT(serpent_algs[6].cra_list), 838 .cra_init = ablk_init, 839 .cra_exit = ablk_exit, 840 .cra_u = { 841 .ablkcipher = { 842 .min_keysize = SERPENT_MIN_KEY_SIZE, 843 .max_keysize = SERPENT_MAX_KEY_SIZE, 844 .ivsize = SERPENT_BLOCK_SIZE, 845 .setkey = ablk_set_key, 846 .encrypt = __ablk_encrypt, 847 .decrypt = ablk_decrypt, 848 }, 849 }, 850}, { 851 .cra_name = "ctr(serpent)", 852 .cra_driver_name = "ctr-serpent-sse2", 853 .cra_priority = 400, 854 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 855 .cra_blocksize = 1, 856 .cra_ctxsize = sizeof(struct async_serpent_ctx), 857 .cra_alignmask = 0, 858 .cra_type = &crypto_ablkcipher_type, 859 .cra_module = THIS_MODULE, 860 .cra_list = LIST_HEAD_INIT(serpent_algs[7].cra_list), 861 .cra_init = ablk_init, 862 .cra_exit = ablk_exit, 863 .cra_u = { 864 .ablkcipher = { 865 .min_keysize = SERPENT_MIN_KEY_SIZE, 866 .max_keysize = SERPENT_MAX_KEY_SIZE, 867 .ivsize = SERPENT_BLOCK_SIZE, 868 .setkey = ablk_set_key, 869 .encrypt = ablk_encrypt, 870 .decrypt = ablk_encrypt, 871 .geniv = "chainiv", 872 }, 873 }, 874}, { 875 .cra_name = "lrw(serpent)", 876 .cra_driver_name = "lrw-serpent-sse2", 877 .cra_priority = 400, 878 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 879 .cra_blocksize = SERPENT_BLOCK_SIZE, 880 .cra_ctxsize = sizeof(struct async_serpent_ctx), 881 .cra_alignmask = 0, 882 .cra_type = &crypto_ablkcipher_type, 883 .cra_module = THIS_MODULE, 884 .cra_list = LIST_HEAD_INIT(serpent_algs[8].cra_list), 885 .cra_init = ablk_init, 886 .cra_exit = ablk_exit, 887 .cra_u = { 888 .ablkcipher = { 889 .min_keysize = SERPENT_MIN_KEY_SIZE + 890 SERPENT_BLOCK_SIZE, 891 .max_keysize = SERPENT_MAX_KEY_SIZE + 892 SERPENT_BLOCK_SIZE, 893 .ivsize = SERPENT_BLOCK_SIZE, 894 .setkey = ablk_set_key, 895 .encrypt = ablk_encrypt, 896 .decrypt = ablk_decrypt, 897 }, 898 }, 899}, { 900 .cra_name = "xts(serpent)", 901 .cra_driver_name = "xts-serpent-sse2", 902 .cra_priority = 400, 903 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 904 .cra_blocksize = SERPENT_BLOCK_SIZE, 905 .cra_ctxsize = sizeof(struct async_serpent_ctx), 906 .cra_alignmask = 0, 907 .cra_type = &crypto_ablkcipher_type, 908 .cra_module = THIS_MODULE, 909 .cra_list = LIST_HEAD_INIT(serpent_algs[9].cra_list), 910 .cra_init = ablk_init, 911 .cra_exit = ablk_exit, 912 .cra_u = { 913 .ablkcipher = { 914 .min_keysize = SERPENT_MIN_KEY_SIZE * 2, 915 .max_keysize = SERPENT_MAX_KEY_SIZE * 2, 916 .ivsize = SERPENT_BLOCK_SIZE, 917 .setkey = ablk_set_key, 918 .encrypt = ablk_encrypt, 919 .decrypt = ablk_decrypt, 920 }, 921 }, 922} }; 923 924static int __init serpent_sse2_init(void) 925{ 926 if (!cpu_has_xmm2) { 927 printk(KERN_INFO "SSE2 instructions are not detected.\n"); 928 return -ENODEV; 929 } 930 931 return crypto_register_algs(serpent_algs, ARRAY_SIZE(serpent_algs)); 932} 933 934static void __exit serpent_sse2_exit(void) 935{ 936 crypto_unregister_algs(serpent_algs, ARRAY_SIZE(serpent_algs)); 937} 938 939module_init(serpent_sse2_init); 940module_exit(serpent_sse2_exit); 941 942MODULE_DESCRIPTION("Serpent Cipher Algorithm, SSE2 optimized"); 943MODULE_LICENSE("GPL"); 944MODULE_ALIAS("serpent"); 945