1#include <inttypes.h>
2#include <string.h>
3#include <unistd.h>
4#include <stdlib.h>
5#include <signal.h>
6#include <sys/types.h>
7#include <sys/wait.h>
8#include "crc32c.h"
9
10/*
11 * Based on a posting to lkml by Austin Zhang <austin.zhang@intel.com>
12 *
13 * Using hardware provided CRC32 instruction to accelerate the CRC32 disposal.
14 * CRC32C polynomial:0x1EDC6F41(BE)/0x82F63B78(LE)
15 * CRC32 is a new instruction in Intel SSE4.2, the reference can be found at:
16 * http://www.intel.com/products/processor/manuals/
17 * Intel(R) 64 and IA-32 Architectures Software Developer's Manual
18 * Volume 2A: Instruction Set Reference, A-M
19 */
20
21int crc32c_intel_available = 0;
22
23#ifdef ARCH_HAVE_SSE4_2
24
25#if BITS_PER_LONG == 64
26#define REX_PRE "0x48, "
27#define SCALE_F 8
28#else
29#define REX_PRE
30#define SCALE_F 4
31#endif
32
33static int crc32c_probed;
34
35static uint32_t crc32c_intel_le_hw_byte(uint32_t crc, unsigned char const *data,
36					unsigned long length)
37{
38	while (length--) {
39		__asm__ __volatile__(
40			".byte 0xf2, 0xf, 0x38, 0xf0, 0xf1"
41			:"=S"(crc)
42			:"0"(crc), "c"(*data)
43		);
44		data++;
45	}
46
47	return crc;
48}
49
50/*
51 * Steps through buffer one byte at at time, calculates reflected
52 * crc using table.
53 */
54uint32_t crc32c_intel(unsigned char const *data, unsigned long length)
55{
56	unsigned int iquotient = length / SCALE_F;
57	unsigned int iremainder = length % SCALE_F;
58#if BITS_PER_LONG == 64
59	uint64_t *ptmp = (uint64_t *) data;
60#else
61	uint32_t *ptmp = (uint32_t *) data;
62#endif
63	uint32_t crc = ~0;
64
65	while (iquotient--) {
66		__asm__ __volatile__(
67			".byte 0xf2, " REX_PRE "0xf, 0x38, 0xf1, 0xf1;"
68			:"=S"(crc)
69			:"0"(crc), "c"(*ptmp)
70		);
71		ptmp++;
72	}
73
74	if (iremainder)
75		crc = crc32c_intel_le_hw_byte(crc, (unsigned char *)ptmp,
76				 iremainder);
77
78	return crc;
79}
80
81void crc32c_intel_probe(void)
82{
83	if (!crc32c_probed) {
84		unsigned int eax, ebx, ecx = 0, edx;
85
86		eax = 1;
87
88		do_cpuid(&eax, &ebx, &ecx, &edx);
89		crc32c_intel_available = (ecx & (1 << 20)) != 0;
90		crc32c_probed = 1;
91	}
92}
93
94#endif /* ARCH_HAVE_SSE */
95