1d9e397b599b13d642138480a28c14db7a136bf0Adam Langley/* Copyright (c) 2014, Google Inc. 2d9e397b599b13d642138480a28c14db7a136bf0Adam Langley * 3d9e397b599b13d642138480a28c14db7a136bf0Adam Langley * Permission to use, copy, modify, and/or distribute this software for any 4d9e397b599b13d642138480a28c14db7a136bf0Adam Langley * purpose with or without fee is hereby granted, provided that the above 5d9e397b599b13d642138480a28c14db7a136bf0Adam Langley * copyright notice and this permission notice appear in all copies. 6d9e397b599b13d642138480a28c14db7a136bf0Adam Langley * 7d9e397b599b13d642138480a28c14db7a136bf0Adam Langley * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8d9e397b599b13d642138480a28c14db7a136bf0Adam Langley * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9d9e397b599b13d642138480a28c14db7a136bf0Adam Langley * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 10d9e397b599b13d642138480a28c14db7a136bf0Adam Langley * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11d9e397b599b13d642138480a28c14db7a136bf0Adam Langley * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION 12d9e397b599b13d642138480a28c14db7a136bf0Adam Langley * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN 13d9e397b599b13d642138480a28c14db7a136bf0Adam Langley * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ 14d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 15d9e397b599b13d642138480a28c14db7a136bf0Adam Langley/* This implementation of poly1305 is by Andrew Moon 16d9e397b599b13d642138480a28c14db7a136bf0Adam Langley * (https://github.com/floodyberry/poly1305-donna) and released as public 17d9e397b599b13d642138480a28c14db7a136bf0Adam Langley * domain. */ 18d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 19d9e397b599b13d642138480a28c14db7a136bf0Adam Langley#include <openssl/poly1305.h> 20d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 21d9e397b599b13d642138480a28c14db7a136bf0Adam Langley#include <string.h> 22d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 23d9e397b599b13d642138480a28c14db7a136bf0Adam Langley#include <openssl/cpu.h> 24d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 254969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin#include "internal.h" 2669939df2891f62f7f00ff2ac275f1cd81a67454cRobert Sloan#include "../internal.h" 274969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin 28d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 29d9e397b599b13d642138480a28c14db7a136bf0Adam Langley#if defined(OPENSSL_WINDOWS) || !defined(OPENSSL_X86_64) 30d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 31d9e397b599b13d642138480a28c14db7a136bf0Adam Langley/* We can assume little-endian. */ 32d9e397b599b13d642138480a28c14db7a136bf0Adam Langleystatic uint32_t U8TO32_LE(const uint8_t *m) { 33d9e397b599b13d642138480a28c14db7a136bf0Adam Langley uint32_t r; 3469939df2891f62f7f00ff2ac275f1cd81a67454cRobert Sloan OPENSSL_memcpy(&r, m, sizeof(r)); 35d9e397b599b13d642138480a28c14db7a136bf0Adam Langley return r; 36d9e397b599b13d642138480a28c14db7a136bf0Adam Langley} 37d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 3869939df2891f62f7f00ff2ac275f1cd81a67454cRobert Sloanstatic void U32TO8_LE(uint8_t *m, uint32_t v) { 3969939df2891f62f7f00ff2ac275f1cd81a67454cRobert Sloan OPENSSL_memcpy(m, &v, sizeof(v)); 4069939df2891f62f7f00ff2ac275f1cd81a67454cRobert Sloan} 41d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 42d9e397b599b13d642138480a28c14db7a136bf0Adam Langleystatic uint64_t mul32x32_64(uint32_t a, uint32_t b) { return (uint64_t)a * b; } 43d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 44d9e397b599b13d642138480a28c14db7a136bf0Adam Langleystruct poly1305_state_st { 45d9e397b599b13d642138480a28c14db7a136bf0Adam Langley uint32_t r0, r1, r2, r3, r4; 46d9e397b599b13d642138480a28c14db7a136bf0Adam Langley uint32_t s1, s2, s3, s4; 47d9e397b599b13d642138480a28c14db7a136bf0Adam Langley uint32_t h0, h1, h2, h3, h4; 48d9e397b599b13d642138480a28c14db7a136bf0Adam Langley uint8_t buf[16]; 49d9e397b599b13d642138480a28c14db7a136bf0Adam Langley unsigned int buf_used; 50d9e397b599b13d642138480a28c14db7a136bf0Adam Langley uint8_t key[16]; 51d9e397b599b13d642138480a28c14db7a136bf0Adam Langley}; 52d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 53909b19f027eb0af12513f4d5589efdd67e34bd91Steven Valdezstatic inline struct poly1305_state_st *poly1305_aligned_state( 54909b19f027eb0af12513f4d5589efdd67e34bd91Steven Valdez poly1305_state *state) { 55909b19f027eb0af12513f4d5589efdd67e34bd91Steven Valdez return (struct poly1305_state_st *)(((uintptr_t)state + 63) & ~63); 56909b19f027eb0af12513f4d5589efdd67e34bd91Steven Valdez} 57909b19f027eb0af12513f4d5589efdd67e34bd91Steven Valdez 58d9e397b599b13d642138480a28c14db7a136bf0Adam Langley/* poly1305_blocks updates |state| given some amount of input data. This 59d9e397b599b13d642138480a28c14db7a136bf0Adam Langley * function may only be called with a |len| that is not a multiple of 16 at the 60d9e397b599b13d642138480a28c14db7a136bf0Adam Langley * end of the data. Otherwise the input must be buffered into 16 byte blocks. */ 61d9e397b599b13d642138480a28c14db7a136bf0Adam Langleystatic void poly1305_update(struct poly1305_state_st *state, const uint8_t *in, 62d9e397b599b13d642138480a28c14db7a136bf0Adam Langley size_t len) { 63d9e397b599b13d642138480a28c14db7a136bf0Adam Langley uint32_t t0, t1, t2, t3; 64d9e397b599b13d642138480a28c14db7a136bf0Adam Langley uint64_t t[5]; 65d9e397b599b13d642138480a28c14db7a136bf0Adam Langley uint32_t b; 66d9e397b599b13d642138480a28c14db7a136bf0Adam Langley uint64_t c; 67d9e397b599b13d642138480a28c14db7a136bf0Adam Langley size_t j; 68d9e397b599b13d642138480a28c14db7a136bf0Adam Langley uint8_t mp[16]; 69d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 70d9e397b599b13d642138480a28c14db7a136bf0Adam Langley if (len < 16) { 71d9e397b599b13d642138480a28c14db7a136bf0Adam Langley goto poly1305_donna_atmost15bytes; 72d9e397b599b13d642138480a28c14db7a136bf0Adam Langley } 73d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 74d9e397b599b13d642138480a28c14db7a136bf0Adam Langleypoly1305_donna_16bytes: 75d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t0 = U8TO32_LE(in); 76d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t1 = U8TO32_LE(in + 4); 77d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t2 = U8TO32_LE(in + 8); 78d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t3 = U8TO32_LE(in + 12); 79d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 80d9e397b599b13d642138480a28c14db7a136bf0Adam Langley in += 16; 81d9e397b599b13d642138480a28c14db7a136bf0Adam Langley len -= 16; 82d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 83d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h0 += t0 & 0x3ffffff; 84d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h1 += ((((uint64_t)t1 << 32) | t0) >> 26) & 0x3ffffff; 85d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h2 += ((((uint64_t)t2 << 32) | t1) >> 20) & 0x3ffffff; 86d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h3 += ((((uint64_t)t3 << 32) | t2) >> 14) & 0x3ffffff; 87d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h4 += (t3 >> 8) | (1 << 24); 88d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 89d9e397b599b13d642138480a28c14db7a136bf0Adam Langleypoly1305_donna_mul: 90d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t[0] = mul32x32_64(state->h0, state->r0) + mul32x32_64(state->h1, state->s4) + 91d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mul32x32_64(state->h2, state->s3) + mul32x32_64(state->h3, state->s2) + 92d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mul32x32_64(state->h4, state->s1); 93d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t[1] = mul32x32_64(state->h0, state->r1) + mul32x32_64(state->h1, state->r0) + 94d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mul32x32_64(state->h2, state->s4) + mul32x32_64(state->h3, state->s3) + 95d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mul32x32_64(state->h4, state->s2); 96d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t[2] = mul32x32_64(state->h0, state->r2) + mul32x32_64(state->h1, state->r1) + 97d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mul32x32_64(state->h2, state->r0) + mul32x32_64(state->h3, state->s4) + 98d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mul32x32_64(state->h4, state->s3); 99d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t[3] = mul32x32_64(state->h0, state->r3) + mul32x32_64(state->h1, state->r2) + 100d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mul32x32_64(state->h2, state->r1) + mul32x32_64(state->h3, state->r0) + 101d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mul32x32_64(state->h4, state->s4); 102d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t[4] = mul32x32_64(state->h0, state->r4) + mul32x32_64(state->h1, state->r3) + 103d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mul32x32_64(state->h2, state->r2) + mul32x32_64(state->h3, state->r1) + 104d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mul32x32_64(state->h4, state->r0); 105d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 106d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h0 = (uint32_t)t[0] & 0x3ffffff; 107d9e397b599b13d642138480a28c14db7a136bf0Adam Langley c = (t[0] >> 26); 108d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t[1] += c; 109d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h1 = (uint32_t)t[1] & 0x3ffffff; 110d9e397b599b13d642138480a28c14db7a136bf0Adam Langley b = (uint32_t)(t[1] >> 26); 111d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t[2] += b; 112d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h2 = (uint32_t)t[2] & 0x3ffffff; 113d9e397b599b13d642138480a28c14db7a136bf0Adam Langley b = (uint32_t)(t[2] >> 26); 114d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t[3] += b; 115d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h3 = (uint32_t)t[3] & 0x3ffffff; 116d9e397b599b13d642138480a28c14db7a136bf0Adam Langley b = (uint32_t)(t[3] >> 26); 117d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t[4] += b; 118d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h4 = (uint32_t)t[4] & 0x3ffffff; 119d9e397b599b13d642138480a28c14db7a136bf0Adam Langley b = (uint32_t)(t[4] >> 26); 120d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h0 += b * 5; 121d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 122e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley if (len >= 16) { 123d9e397b599b13d642138480a28c14db7a136bf0Adam Langley goto poly1305_donna_16bytes; 124e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley } 125d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 126d9e397b599b13d642138480a28c14db7a136bf0Adam Langley/* final bytes */ 127d9e397b599b13d642138480a28c14db7a136bf0Adam Langleypoly1305_donna_atmost15bytes: 128e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley if (!len) { 129d9e397b599b13d642138480a28c14db7a136bf0Adam Langley return; 130e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley } 131d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 132e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley for (j = 0; j < len; j++) { 133d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mp[j] = in[j]; 134e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley } 135d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mp[j++] = 1; 136e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley for (; j < 16; j++) { 137d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mp[j] = 0; 138e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley } 139d9e397b599b13d642138480a28c14db7a136bf0Adam Langley len = 0; 140d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 141d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t0 = U8TO32_LE(mp + 0); 142d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t1 = U8TO32_LE(mp + 4); 143d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t2 = U8TO32_LE(mp + 8); 144d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t3 = U8TO32_LE(mp + 12); 145d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 146d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h0 += t0 & 0x3ffffff; 147d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h1 += ((((uint64_t)t1 << 32) | t0) >> 26) & 0x3ffffff; 148d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h2 += ((((uint64_t)t2 << 32) | t1) >> 20) & 0x3ffffff; 149d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h3 += ((((uint64_t)t3 << 32) | t2) >> 14) & 0x3ffffff; 150d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h4 += (t3 >> 8); 151d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 152d9e397b599b13d642138480a28c14db7a136bf0Adam Langley goto poly1305_donna_mul; 153d9e397b599b13d642138480a28c14db7a136bf0Adam Langley} 154d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 155d9e397b599b13d642138480a28c14db7a136bf0Adam Langleyvoid CRYPTO_poly1305_init(poly1305_state *statep, const uint8_t key[32]) { 156909b19f027eb0af12513f4d5589efdd67e34bd91Steven Valdez struct poly1305_state_st *state = poly1305_aligned_state(statep); 157d9e397b599b13d642138480a28c14db7a136bf0Adam Langley uint32_t t0, t1, t2, t3; 158d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 159d9e397b599b13d642138480a28c14db7a136bf0Adam Langley#if defined(OPENSSL_ARM) && !defined(OPENSSL_NO_ASM) 1604969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin if (CRYPTO_is_NEON_capable()) { 161d9e397b599b13d642138480a28c14db7a136bf0Adam Langley CRYPTO_poly1305_init_neon(statep, key); 162d9e397b599b13d642138480a28c14db7a136bf0Adam Langley return; 163d9e397b599b13d642138480a28c14db7a136bf0Adam Langley } 164d9e397b599b13d642138480a28c14db7a136bf0Adam Langley#endif 165d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 166d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t0 = U8TO32_LE(key + 0); 167d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t1 = U8TO32_LE(key + 4); 168d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t2 = U8TO32_LE(key + 8); 169d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t3 = U8TO32_LE(key + 12); 170d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 171d9e397b599b13d642138480a28c14db7a136bf0Adam Langley /* precompute multipliers */ 172d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->r0 = t0 & 0x3ffffff; 173d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t0 >>= 26; 174d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t0 |= t1 << 6; 175d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->r1 = t0 & 0x3ffff03; 176d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t1 >>= 20; 177d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t1 |= t2 << 12; 178d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->r2 = t1 & 0x3ffc0ff; 179d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t2 >>= 14; 180d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t2 |= t3 << 18; 181d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->r3 = t2 & 0x3f03fff; 182d9e397b599b13d642138480a28c14db7a136bf0Adam Langley t3 >>= 8; 183d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->r4 = t3 & 0x00fffff; 184d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 185d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->s1 = state->r1 * 5; 186d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->s2 = state->r2 * 5; 187d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->s3 = state->r3 * 5; 188d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->s4 = state->r4 * 5; 189d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 190d9e397b599b13d642138480a28c14db7a136bf0Adam Langley /* init state */ 191d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h0 = 0; 192d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h1 = 0; 193d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h2 = 0; 194d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h3 = 0; 195d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h4 = 0; 196d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 197d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->buf_used = 0; 19869939df2891f62f7f00ff2ac275f1cd81a67454cRobert Sloan OPENSSL_memcpy(state->key, key + 16, sizeof(state->key)); 199d9e397b599b13d642138480a28c14db7a136bf0Adam Langley} 200d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 201d9e397b599b13d642138480a28c14db7a136bf0Adam Langleyvoid CRYPTO_poly1305_update(poly1305_state *statep, const uint8_t *in, 202d9e397b599b13d642138480a28c14db7a136bf0Adam Langley size_t in_len) { 203d9e397b599b13d642138480a28c14db7a136bf0Adam Langley unsigned int i; 204909b19f027eb0af12513f4d5589efdd67e34bd91Steven Valdez struct poly1305_state_st *state = poly1305_aligned_state(statep); 205d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 206d9e397b599b13d642138480a28c14db7a136bf0Adam Langley#if defined(OPENSSL_ARM) && !defined(OPENSSL_NO_ASM) 2074969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin if (CRYPTO_is_NEON_capable()) { 208d9e397b599b13d642138480a28c14db7a136bf0Adam Langley CRYPTO_poly1305_update_neon(statep, in, in_len); 209d9e397b599b13d642138480a28c14db7a136bf0Adam Langley return; 210d9e397b599b13d642138480a28c14db7a136bf0Adam Langley } 211d9e397b599b13d642138480a28c14db7a136bf0Adam Langley#endif 212d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 213d9e397b599b13d642138480a28c14db7a136bf0Adam Langley if (state->buf_used) { 214c895d6b1c580258e72e1ed3fcc86d38970ded9e1David Benjamin unsigned todo = 16 - state->buf_used; 215e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley if (todo > in_len) { 216c895d6b1c580258e72e1ed3fcc86d38970ded9e1David Benjamin todo = (unsigned)in_len; 217e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley } 218e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley for (i = 0; i < todo; i++) { 219d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->buf[state->buf_used + i] = in[i]; 220e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley } 221d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->buf_used += todo; 222d9e397b599b13d642138480a28c14db7a136bf0Adam Langley in_len -= todo; 223d9e397b599b13d642138480a28c14db7a136bf0Adam Langley in += todo; 224d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 225d9e397b599b13d642138480a28c14db7a136bf0Adam Langley if (state->buf_used == 16) { 226d9e397b599b13d642138480a28c14db7a136bf0Adam Langley poly1305_update(state, state->buf, 16); 227d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->buf_used = 0; 228d9e397b599b13d642138480a28c14db7a136bf0Adam Langley } 229d9e397b599b13d642138480a28c14db7a136bf0Adam Langley } 230d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 231d9e397b599b13d642138480a28c14db7a136bf0Adam Langley if (in_len >= 16) { 232d9e397b599b13d642138480a28c14db7a136bf0Adam Langley size_t todo = in_len & ~0xf; 233d9e397b599b13d642138480a28c14db7a136bf0Adam Langley poly1305_update(state, in, todo); 234d9e397b599b13d642138480a28c14db7a136bf0Adam Langley in += todo; 235d9e397b599b13d642138480a28c14db7a136bf0Adam Langley in_len &= 0xf; 236d9e397b599b13d642138480a28c14db7a136bf0Adam Langley } 237d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 238d9e397b599b13d642138480a28c14db7a136bf0Adam Langley if (in_len) { 239e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley for (i = 0; i < in_len; i++) { 240d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->buf[i] = in[i]; 241e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley } 242c895d6b1c580258e72e1ed3fcc86d38970ded9e1David Benjamin state->buf_used = (unsigned)in_len; 243d9e397b599b13d642138480a28c14db7a136bf0Adam Langley } 244d9e397b599b13d642138480a28c14db7a136bf0Adam Langley} 245d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 246d9e397b599b13d642138480a28c14db7a136bf0Adam Langleyvoid CRYPTO_poly1305_finish(poly1305_state *statep, uint8_t mac[16]) { 247909b19f027eb0af12513f4d5589efdd67e34bd91Steven Valdez struct poly1305_state_st *state = poly1305_aligned_state(statep); 248d9e397b599b13d642138480a28c14db7a136bf0Adam Langley uint64_t f0, f1, f2, f3; 249d9e397b599b13d642138480a28c14db7a136bf0Adam Langley uint32_t g0, g1, g2, g3, g4; 250d9e397b599b13d642138480a28c14db7a136bf0Adam Langley uint32_t b, nb; 251d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 252d9e397b599b13d642138480a28c14db7a136bf0Adam Langley#if defined(OPENSSL_ARM) && !defined(OPENSSL_NO_ASM) 2534969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin if (CRYPTO_is_NEON_capable()) { 254d9e397b599b13d642138480a28c14db7a136bf0Adam Langley CRYPTO_poly1305_finish_neon(statep, mac); 255d9e397b599b13d642138480a28c14db7a136bf0Adam Langley return; 256d9e397b599b13d642138480a28c14db7a136bf0Adam Langley } 257d9e397b599b13d642138480a28c14db7a136bf0Adam Langley#endif 258d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 259e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley if (state->buf_used) { 260d9e397b599b13d642138480a28c14db7a136bf0Adam Langley poly1305_update(state, state->buf, state->buf_used); 261e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley } 262d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 263d9e397b599b13d642138480a28c14db7a136bf0Adam Langley b = state->h0 >> 26; 264d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h0 = state->h0 & 0x3ffffff; 265d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h1 += b; 266d9e397b599b13d642138480a28c14db7a136bf0Adam Langley b = state->h1 >> 26; 267d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h1 = state->h1 & 0x3ffffff; 268d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h2 += b; 269d9e397b599b13d642138480a28c14db7a136bf0Adam Langley b = state->h2 >> 26; 270d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h2 = state->h2 & 0x3ffffff; 271d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h3 += b; 272d9e397b599b13d642138480a28c14db7a136bf0Adam Langley b = state->h3 >> 26; 273d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h3 = state->h3 & 0x3ffffff; 274d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h4 += b; 275d9e397b599b13d642138480a28c14db7a136bf0Adam Langley b = state->h4 >> 26; 276d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h4 = state->h4 & 0x3ffffff; 277d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h0 += b * 5; 278d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 279d9e397b599b13d642138480a28c14db7a136bf0Adam Langley g0 = state->h0 + 5; 280d9e397b599b13d642138480a28c14db7a136bf0Adam Langley b = g0 >> 26; 281d9e397b599b13d642138480a28c14db7a136bf0Adam Langley g0 &= 0x3ffffff; 282d9e397b599b13d642138480a28c14db7a136bf0Adam Langley g1 = state->h1 + b; 283d9e397b599b13d642138480a28c14db7a136bf0Adam Langley b = g1 >> 26; 284d9e397b599b13d642138480a28c14db7a136bf0Adam Langley g1 &= 0x3ffffff; 285d9e397b599b13d642138480a28c14db7a136bf0Adam Langley g2 = state->h2 + b; 286d9e397b599b13d642138480a28c14db7a136bf0Adam Langley b = g2 >> 26; 287d9e397b599b13d642138480a28c14db7a136bf0Adam Langley g2 &= 0x3ffffff; 288d9e397b599b13d642138480a28c14db7a136bf0Adam Langley g3 = state->h3 + b; 289d9e397b599b13d642138480a28c14db7a136bf0Adam Langley b = g3 >> 26; 290d9e397b599b13d642138480a28c14db7a136bf0Adam Langley g3 &= 0x3ffffff; 291d9e397b599b13d642138480a28c14db7a136bf0Adam Langley g4 = state->h4 + b - (1 << 26); 292d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 293d9e397b599b13d642138480a28c14db7a136bf0Adam Langley b = (g4 >> 31) - 1; 294d9e397b599b13d642138480a28c14db7a136bf0Adam Langley nb = ~b; 295d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h0 = (state->h0 & nb) | (g0 & b); 296d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h1 = (state->h1 & nb) | (g1 & b); 297d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h2 = (state->h2 & nb) | (g2 & b); 298d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h3 = (state->h3 & nb) | (g3 & b); 299d9e397b599b13d642138480a28c14db7a136bf0Adam Langley state->h4 = (state->h4 & nb) | (g4 & b); 300d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 301d9e397b599b13d642138480a28c14db7a136bf0Adam Langley f0 = ((state->h0) | (state->h1 << 26)) + (uint64_t)U8TO32_LE(&state->key[0]); 302d9e397b599b13d642138480a28c14db7a136bf0Adam Langley f1 = ((state->h1 >> 6) | (state->h2 << 20)) + 303d9e397b599b13d642138480a28c14db7a136bf0Adam Langley (uint64_t)U8TO32_LE(&state->key[4]); 304d9e397b599b13d642138480a28c14db7a136bf0Adam Langley f2 = ((state->h2 >> 12) | (state->h3 << 14)) + 305d9e397b599b13d642138480a28c14db7a136bf0Adam Langley (uint64_t)U8TO32_LE(&state->key[8]); 306d9e397b599b13d642138480a28c14db7a136bf0Adam Langley f3 = ((state->h3 >> 18) | (state->h4 << 8)) + 307d9e397b599b13d642138480a28c14db7a136bf0Adam Langley (uint64_t)U8TO32_LE(&state->key[12]); 308d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 309d9e397b599b13d642138480a28c14db7a136bf0Adam Langley U32TO8_LE(&mac[0], f0); 310d9e397b599b13d642138480a28c14db7a136bf0Adam Langley f1 += (f0 >> 32); 311d9e397b599b13d642138480a28c14db7a136bf0Adam Langley U32TO8_LE(&mac[4], f1); 312d9e397b599b13d642138480a28c14db7a136bf0Adam Langley f2 += (f1 >> 32); 313d9e397b599b13d642138480a28c14db7a136bf0Adam Langley U32TO8_LE(&mac[8], f2); 314d9e397b599b13d642138480a28c14db7a136bf0Adam Langley f3 += (f2 >> 32); 315d9e397b599b13d642138480a28c14db7a136bf0Adam Langley U32TO8_LE(&mac[12], f3); 316d9e397b599b13d642138480a28c14db7a136bf0Adam Langley} 317d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 318d9e397b599b13d642138480a28c14db7a136bf0Adam Langley#endif /* OPENSSL_WINDOWS || !OPENSSL_X86_64 */ 319