1/*
2 * This is D3DES (V5.09) by Richard Outerbridge with the double and
3 * triple-length support removed for use in VNC.  Also the bytebit[] array
4 * has been reversed so that the most significant bit in each byte of the
5 * key is ignored, not the least significant.
6 *
7 * These changes are:
8 *  Copyright (C) 1999 AT&T Laboratories Cambridge.  All Rights Reserved.
9 *
10 * This software is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
13 */
14
15/* D3DES (V5.09) -
16 *
17 * A portable, public domain, version of the Data Encryption Standard.
18 *
19 * Written with Symantec's THINK (Lightspeed) C by Richard Outerbridge.
20 * Thanks to: Dan Hoey for his excellent Initial and Inverse permutation
21 * code;  Jim Gillogly & Phil Karn for the DES key schedule code; Dennis
22 * Ferguson, Eric Young and Dana How for comparing notes; and Ray Lau,
23 * for humouring me on.
24 *
25 * Copyright (c) 1988,1989,1990,1991,1992 by Richard Outerbridge.
26 * (GEnie : OUTER; CIS : [71755,204]) Graven Imagery, 1992.
27 */
28
29#include "d3des.h"
30
31static void scrunch(unsigned char *, unsigned long *);
32static void unscrun(unsigned long *, unsigned char *);
33static void desfunc(unsigned long *, unsigned long *);
34static void cookey(unsigned long *);
35
36static unsigned long KnL[32] = { 0L };
37/*
38static unsigned long KnR[32] = { 0L };
39static unsigned long Kn3[32] = { 0L };
40static unsigned char Df_Key[24] = {
41	0x01,0x23,0x45,0x67,0x89,0xab,0xcd,0xef,
42	0xfe,0xdc,0xba,0x98,0x76,0x54,0x32,0x10,
43	0x89,0xab,0xcd,0xef,0x01,0x23,0x45,0x67 };
44*/
45
46static unsigned short bytebit[8]	= {
47	01, 02, 04, 010, 020, 040, 0100, 0200 };
48
49static unsigned long bigbyte[24] = {
50	0x800000L,	0x400000L,	0x200000L,	0x100000L,
51	0x80000L,	0x40000L,	0x20000L,	0x10000L,
52	0x8000L,	0x4000L,	0x2000L,	0x1000L,
53	0x800L, 	0x400L, 	0x200L, 	0x100L,
54	0x80L,		0x40L,		0x20L,		0x10L,
55	0x8L,		0x4L,		0x2L,		0x1L	};
56
57/* Use the key schedule specified in the Standard (ANSI X3.92-1981). */
58
59static unsigned char pc1[56] = {
60	56, 48, 40, 32, 24, 16,  8,	 0, 57, 49, 41, 33, 25, 17,
61	 9,  1, 58, 50, 42, 34, 26,	18, 10,  2, 59, 51, 43, 35,
62	62, 54, 46, 38, 30, 22, 14,	 6, 61, 53, 45, 37, 29, 21,
63	13,  5, 60, 52, 44, 36, 28,	20, 12,  4, 27, 19, 11,  3 };
64
65static unsigned char totrot[16] = {
66	1,2,4,6,8,10,12,14,15,17,19,21,23,25,27,28 };
67
68static unsigned char pc2[48] = {
69	13, 16, 10, 23,  0,  4,  2, 27, 14,  5, 20,  9,
70	22, 18, 11,  3, 25,  7, 15,  6, 26, 19, 12,  1,
71	40, 51, 30, 36, 46, 54, 29, 39, 50, 44, 32, 47,
72	43, 48, 38, 55, 33, 52, 45, 41, 49, 35, 28, 31 };
73
74void rfbDesKey(unsigned char *key,
75               int edf)
76{
77	register int i, j, l, m, n;
78	unsigned char pc1m[56], pcr[56];
79	unsigned long kn[32];
80
81	for ( j = 0; j < 56; j++ ) {
82		l = pc1[j];
83		m = l & 07;
84		pc1m[j] = (key[l >> 3] & bytebit[m]) ? 1 : 0;
85		}
86	for( i = 0; i < 16; i++ ) {
87		if( edf == DE1 ) m = (15 - i) << 1;
88		else m = i << 1;
89		n = m + 1;
90		kn[m] = kn[n] = 0L;
91		for( j = 0; j < 28; j++ ) {
92			l = j + totrot[i];
93			if( l < 28 ) pcr[j] = pc1m[l];
94			else pcr[j] = pc1m[l - 28];
95			}
96		for( j = 28; j < 56; j++ ) {
97		    l = j + totrot[i];
98		    if( l < 56 ) pcr[j] = pc1m[l];
99		    else pcr[j] = pc1m[l - 28];
100		    }
101		for( j = 0; j < 24; j++ ) {
102			if( pcr[pc2[j]] ) kn[m] |= bigbyte[j];
103			if( pcr[pc2[j+24]] ) kn[n] |= bigbyte[j];
104			}
105		}
106	cookey(kn);
107	return;
108	}
109
110static void cookey(register unsigned long *raw1)
111{
112	register unsigned long *cook, *raw0;
113	unsigned long dough[32];
114	register int i;
115
116	cook = dough;
117	for( i = 0; i < 16; i++, raw1++ ) {
118		raw0 = raw1++;
119		*cook	 = (*raw0 & 0x00fc0000L) << 6;
120		*cook	|= (*raw0 & 0x00000fc0L) << 10;
121		*cook	|= (*raw1 & 0x00fc0000L) >> 10;
122		*cook++ |= (*raw1 & 0x00000fc0L) >> 6;
123		*cook	 = (*raw0 & 0x0003f000L) << 12;
124		*cook	|= (*raw0 & 0x0000003fL) << 16;
125		*cook	|= (*raw1 & 0x0003f000L) >> 4;
126		*cook++ |= (*raw1 & 0x0000003fL);
127		}
128	rfbUseKey(dough);
129	return;
130	}
131
132void rfbCPKey(register unsigned long *into)
133{
134	register unsigned long *from, *endp;
135
136	from = KnL, endp = &KnL[32];
137	while( from < endp ) *into++ = *from++;
138	return;
139	}
140
141void rfbUseKey(register unsigned long *from)
142{
143	register unsigned long *to, *endp;
144
145	to = KnL, endp = &KnL[32];
146	while( to < endp ) *to++ = *from++;
147	return;
148	}
149
150void rfbDes(unsigned char *inblock,
151            unsigned char *outblock)
152{
153	unsigned long work[2];
154
155	scrunch(inblock, work);
156	desfunc(work, KnL);
157	unscrun(work, outblock);
158	return;
159	}
160
161static void scrunch(register unsigned char *outof,
162                    register unsigned long *into)
163{
164	*into	 = (*outof++ & 0xffL) << 24;
165	*into	|= (*outof++ & 0xffL) << 16;
166	*into	|= (*outof++ & 0xffL) << 8;
167	*into++ |= (*outof++ & 0xffL);
168	*into	 = (*outof++ & 0xffL) << 24;
169	*into	|= (*outof++ & 0xffL) << 16;
170	*into	|= (*outof++ & 0xffL) << 8;
171	*into	|= (*outof   & 0xffL);
172	return;
173	}
174
175static void unscrun(register unsigned long *outof,
176                    register unsigned char *into)
177{
178	*into++ = (unsigned char)((*outof >> 24) & 0xffL);
179	*into++ = (unsigned char)((*outof >> 16) & 0xffL);
180	*into++ = (unsigned char)((*outof >>  8) & 0xffL);
181	*into++ = (unsigned char)( *outof++	 & 0xffL);
182	*into++ = (unsigned char)((*outof >> 24) & 0xffL);
183	*into++ = (unsigned char)((*outof >> 16) & 0xffL);
184	*into++ = (unsigned char)((*outof >>  8) & 0xffL);
185	*into	= (unsigned char)( *outof	 & 0xffL);
186	return;
187	}
188
189static unsigned long SP1[64] = {
190	0x01010400L, 0x00000000L, 0x00010000L, 0x01010404L,
191	0x01010004L, 0x00010404L, 0x00000004L, 0x00010000L,
192	0x00000400L, 0x01010400L, 0x01010404L, 0x00000400L,
193	0x01000404L, 0x01010004L, 0x01000000L, 0x00000004L,
194	0x00000404L, 0x01000400L, 0x01000400L, 0x00010400L,
195	0x00010400L, 0x01010000L, 0x01010000L, 0x01000404L,
196	0x00010004L, 0x01000004L, 0x01000004L, 0x00010004L,
197	0x00000000L, 0x00000404L, 0x00010404L, 0x01000000L,
198	0x00010000L, 0x01010404L, 0x00000004L, 0x01010000L,
199	0x01010400L, 0x01000000L, 0x01000000L, 0x00000400L,
200	0x01010004L, 0x00010000L, 0x00010400L, 0x01000004L,
201	0x00000400L, 0x00000004L, 0x01000404L, 0x00010404L,
202	0x01010404L, 0x00010004L, 0x01010000L, 0x01000404L,
203	0x01000004L, 0x00000404L, 0x00010404L, 0x01010400L,
204	0x00000404L, 0x01000400L, 0x01000400L, 0x00000000L,
205	0x00010004L, 0x00010400L, 0x00000000L, 0x01010004L };
206
207static unsigned long SP2[64] = {
208	0x80108020L, 0x80008000L, 0x00008000L, 0x00108020L,
209	0x00100000L, 0x00000020L, 0x80100020L, 0x80008020L,
210	0x80000020L, 0x80108020L, 0x80108000L, 0x80000000L,
211	0x80008000L, 0x00100000L, 0x00000020L, 0x80100020L,
212	0x00108000L, 0x00100020L, 0x80008020L, 0x00000000L,
213	0x80000000L, 0x00008000L, 0x00108020L, 0x80100000L,
214	0x00100020L, 0x80000020L, 0x00000000L, 0x00108000L,
215	0x00008020L, 0x80108000L, 0x80100000L, 0x00008020L,
216	0x00000000L, 0x00108020L, 0x80100020L, 0x00100000L,
217	0x80008020L, 0x80100000L, 0x80108000L, 0x00008000L,
218	0x80100000L, 0x80008000L, 0x00000020L, 0x80108020L,
219	0x00108020L, 0x00000020L, 0x00008000L, 0x80000000L,
220	0x00008020L, 0x80108000L, 0x00100000L, 0x80000020L,
221	0x00100020L, 0x80008020L, 0x80000020L, 0x00100020L,
222	0x00108000L, 0x00000000L, 0x80008000L, 0x00008020L,
223	0x80000000L, 0x80100020L, 0x80108020L, 0x00108000L };
224
225static unsigned long SP3[64] = {
226	0x00000208L, 0x08020200L, 0x00000000L, 0x08020008L,
227	0x08000200L, 0x00000000L, 0x00020208L, 0x08000200L,
228	0x00020008L, 0x08000008L, 0x08000008L, 0x00020000L,
229	0x08020208L, 0x00020008L, 0x08020000L, 0x00000208L,
230	0x08000000L, 0x00000008L, 0x08020200L, 0x00000200L,
231	0x00020200L, 0x08020000L, 0x08020008L, 0x00020208L,
232	0x08000208L, 0x00020200L, 0x00020000L, 0x08000208L,
233	0x00000008L, 0x08020208L, 0x00000200L, 0x08000000L,
234	0x08020200L, 0x08000000L, 0x00020008L, 0x00000208L,
235	0x00020000L, 0x08020200L, 0x08000200L, 0x00000000L,
236	0x00000200L, 0x00020008L, 0x08020208L, 0x08000200L,
237	0x08000008L, 0x00000200L, 0x00000000L, 0x08020008L,
238	0x08000208L, 0x00020000L, 0x08000000L, 0x08020208L,
239	0x00000008L, 0x00020208L, 0x00020200L, 0x08000008L,
240	0x08020000L, 0x08000208L, 0x00000208L, 0x08020000L,
241	0x00020208L, 0x00000008L, 0x08020008L, 0x00020200L };
242
243static unsigned long SP4[64] = {
244	0x00802001L, 0x00002081L, 0x00002081L, 0x00000080L,
245	0x00802080L, 0x00800081L, 0x00800001L, 0x00002001L,
246	0x00000000L, 0x00802000L, 0x00802000L, 0x00802081L,
247	0x00000081L, 0x00000000L, 0x00800080L, 0x00800001L,
248	0x00000001L, 0x00002000L, 0x00800000L, 0x00802001L,
249	0x00000080L, 0x00800000L, 0x00002001L, 0x00002080L,
250	0x00800081L, 0x00000001L, 0x00002080L, 0x00800080L,
251	0x00002000L, 0x00802080L, 0x00802081L, 0x00000081L,
252	0x00800080L, 0x00800001L, 0x00802000L, 0x00802081L,
253	0x00000081L, 0x00000000L, 0x00000000L, 0x00802000L,
254	0x00002080L, 0x00800080L, 0x00800081L, 0x00000001L,
255	0x00802001L, 0x00002081L, 0x00002081L, 0x00000080L,
256	0x00802081L, 0x00000081L, 0x00000001L, 0x00002000L,
257	0x00800001L, 0x00002001L, 0x00802080L, 0x00800081L,
258	0x00002001L, 0x00002080L, 0x00800000L, 0x00802001L,
259	0x00000080L, 0x00800000L, 0x00002000L, 0x00802080L };
260
261static unsigned long SP5[64] = {
262	0x00000100L, 0x02080100L, 0x02080000L, 0x42000100L,
263	0x00080000L, 0x00000100L, 0x40000000L, 0x02080000L,
264	0x40080100L, 0x00080000L, 0x02000100L, 0x40080100L,
265	0x42000100L, 0x42080000L, 0x00080100L, 0x40000000L,
266	0x02000000L, 0x40080000L, 0x40080000L, 0x00000000L,
267	0x40000100L, 0x42080100L, 0x42080100L, 0x02000100L,
268	0x42080000L, 0x40000100L, 0x00000000L, 0x42000000L,
269	0x02080100L, 0x02000000L, 0x42000000L, 0x00080100L,
270	0x00080000L, 0x42000100L, 0x00000100L, 0x02000000L,
271	0x40000000L, 0x02080000L, 0x42000100L, 0x40080100L,
272	0x02000100L, 0x40000000L, 0x42080000L, 0x02080100L,
273	0x40080100L, 0x00000100L, 0x02000000L, 0x42080000L,
274	0x42080100L, 0x00080100L, 0x42000000L, 0x42080100L,
275	0x02080000L, 0x00000000L, 0x40080000L, 0x42000000L,
276	0x00080100L, 0x02000100L, 0x40000100L, 0x00080000L,
277	0x00000000L, 0x40080000L, 0x02080100L, 0x40000100L };
278
279static unsigned long SP6[64] = {
280	0x20000010L, 0x20400000L, 0x00004000L, 0x20404010L,
281	0x20400000L, 0x00000010L, 0x20404010L, 0x00400000L,
282	0x20004000L, 0x00404010L, 0x00400000L, 0x20000010L,
283	0x00400010L, 0x20004000L, 0x20000000L, 0x00004010L,
284	0x00000000L, 0x00400010L, 0x20004010L, 0x00004000L,
285	0x00404000L, 0x20004010L, 0x00000010L, 0x20400010L,
286	0x20400010L, 0x00000000L, 0x00404010L, 0x20404000L,
287	0x00004010L, 0x00404000L, 0x20404000L, 0x20000000L,
288	0x20004000L, 0x00000010L, 0x20400010L, 0x00404000L,
289	0x20404010L, 0x00400000L, 0x00004010L, 0x20000010L,
290	0x00400000L, 0x20004000L, 0x20000000L, 0x00004010L,
291	0x20000010L, 0x20404010L, 0x00404000L, 0x20400000L,
292	0x00404010L, 0x20404000L, 0x00000000L, 0x20400010L,
293	0x00000010L, 0x00004000L, 0x20400000L, 0x00404010L,
294	0x00004000L, 0x00400010L, 0x20004010L, 0x00000000L,
295	0x20404000L, 0x20000000L, 0x00400010L, 0x20004010L };
296
297static unsigned long SP7[64] = {
298	0x00200000L, 0x04200002L, 0x04000802L, 0x00000000L,
299	0x00000800L, 0x04000802L, 0x00200802L, 0x04200800L,
300	0x04200802L, 0x00200000L, 0x00000000L, 0x04000002L,
301	0x00000002L, 0x04000000L, 0x04200002L, 0x00000802L,
302	0x04000800L, 0x00200802L, 0x00200002L, 0x04000800L,
303	0x04000002L, 0x04200000L, 0x04200800L, 0x00200002L,
304	0x04200000L, 0x00000800L, 0x00000802L, 0x04200802L,
305	0x00200800L, 0x00000002L, 0x04000000L, 0x00200800L,
306	0x04000000L, 0x00200800L, 0x00200000L, 0x04000802L,
307	0x04000802L, 0x04200002L, 0x04200002L, 0x00000002L,
308	0x00200002L, 0x04000000L, 0x04000800L, 0x00200000L,
309	0x04200800L, 0x00000802L, 0x00200802L, 0x04200800L,
310	0x00000802L, 0x04000002L, 0x04200802L, 0x04200000L,
311	0x00200800L, 0x00000000L, 0x00000002L, 0x04200802L,
312	0x00000000L, 0x00200802L, 0x04200000L, 0x00000800L,
313	0x04000002L, 0x04000800L, 0x00000800L, 0x00200002L };
314
315static unsigned long SP8[64] = {
316	0x10001040L, 0x00001000L, 0x00040000L, 0x10041040L,
317	0x10000000L, 0x10001040L, 0x00000040L, 0x10000000L,
318	0x00040040L, 0x10040000L, 0x10041040L, 0x00041000L,
319	0x10041000L, 0x00041040L, 0x00001000L, 0x00000040L,
320	0x10040000L, 0x10000040L, 0x10001000L, 0x00001040L,
321	0x00041000L, 0x00040040L, 0x10040040L, 0x10041000L,
322	0x00001040L, 0x00000000L, 0x00000000L, 0x10040040L,
323	0x10000040L, 0x10001000L, 0x00041040L, 0x00040000L,
324	0x00041040L, 0x00040000L, 0x10041000L, 0x00001000L,
325	0x00000040L, 0x10040040L, 0x00001000L, 0x00041040L,
326	0x10001000L, 0x00000040L, 0x10000040L, 0x10040000L,
327	0x10040040L, 0x10000000L, 0x00040000L, 0x10001040L,
328	0x00000000L, 0x10041040L, 0x00040040L, 0x10000040L,
329	0x10040000L, 0x10001000L, 0x10001040L, 0x00000000L,
330	0x10041040L, 0x00041000L, 0x00041000L, 0x00001040L,
331	0x00001040L, 0x00040040L, 0x10000000L, 0x10041000L };
332
333static void desfunc(register unsigned long *block,
334                    register unsigned long *keys)
335{
336	register unsigned long fval, work, right, leftt;
337	register int round;
338
339	leftt = block[0];
340	right = block[1];
341	work = ((leftt >> 4) ^ right) & 0x0f0f0f0fL;
342	right ^= work;
343	leftt ^= (work << 4);
344	work = ((leftt >> 16) ^ right) & 0x0000ffffL;
345	right ^= work;
346	leftt ^= (work << 16);
347	work = ((right >> 2) ^ leftt) & 0x33333333L;
348	leftt ^= work;
349	right ^= (work << 2);
350	work = ((right >> 8) ^ leftt) & 0x00ff00ffL;
351	leftt ^= work;
352	right ^= (work << 8);
353	right = ((right << 1) | ((right >> 31) & 1L)) & 0xffffffffL;
354	work = (leftt ^ right) & 0xaaaaaaaaL;
355	leftt ^= work;
356	right ^= work;
357	leftt = ((leftt << 1) | ((leftt >> 31) & 1L)) & 0xffffffffL;
358
359	for( round = 0; round < 8; round++ ) {
360		work  = (right << 28) | (right >> 4);
361		work ^= *keys++;
362		fval  = SP7[ work		 & 0x3fL];
363		fval |= SP5[(work >>  8) & 0x3fL];
364		fval |= SP3[(work >> 16) & 0x3fL];
365		fval |= SP1[(work >> 24) & 0x3fL];
366		work  = right ^ *keys++;
367		fval |= SP8[ work		 & 0x3fL];
368		fval |= SP6[(work >>  8) & 0x3fL];
369		fval |= SP4[(work >> 16) & 0x3fL];
370		fval |= SP2[(work >> 24) & 0x3fL];
371		leftt ^= fval;
372		work  = (leftt << 28) | (leftt >> 4);
373		work ^= *keys++;
374		fval  = SP7[ work		 & 0x3fL];
375		fval |= SP5[(work >>  8) & 0x3fL];
376		fval |= SP3[(work >> 16) & 0x3fL];
377		fval |= SP1[(work >> 24) & 0x3fL];
378		work  = leftt ^ *keys++;
379		fval |= SP8[ work		 & 0x3fL];
380		fval |= SP6[(work >>  8) & 0x3fL];
381		fval |= SP4[(work >> 16) & 0x3fL];
382		fval |= SP2[(work >> 24) & 0x3fL];
383		right ^= fval;
384		}
385
386	right = (right << 31) | (right >> 1);
387	work = (leftt ^ right) & 0xaaaaaaaaL;
388	leftt ^= work;
389	right ^= work;
390	leftt = (leftt << 31) | (leftt >> 1);
391	work = ((leftt >> 8) ^ right) & 0x00ff00ffL;
392	right ^= work;
393	leftt ^= (work << 8);
394	work = ((leftt >> 2) ^ right) & 0x33333333L;
395	right ^= work;
396	leftt ^= (work << 2);
397	work = ((right >> 16) ^ leftt) & 0x0000ffffL;
398	leftt ^= work;
399	right ^= (work << 16);
400	work = ((right >> 4) ^ leftt) & 0x0f0f0f0fL;
401	leftt ^= work;
402	right ^= (work << 4);
403	*block++ = right;
404	*block = leftt;
405	return;
406	}
407
408/* Validation sets:
409 *
410 * Single-length key, single-length plaintext -
411 * Key	  : 0123 4567 89ab cdef
412 * Plain  : 0123 4567 89ab cde7
413 * Cipher : c957 4425 6a5e d31d
414 *
415 * Double-length key, single-length plaintext -
416 * Key	  : 0123 4567 89ab cdef fedc ba98 7654 3210
417 * Plain  : 0123 4567 89ab cde7
418 * Cipher : 7f1d 0a77 826b 8aff
419 *
420 * Double-length key, double-length plaintext -
421 * Key	  : 0123 4567 89ab cdef fedc ba98 7654 3210
422 * Plain  : 0123 4567 89ab cdef 0123 4567 89ab cdff
423 * Cipher : 27a0 8440 406a df60 278f 47cf 42d6 15d7
424 *
425 * Triple-length key, single-length plaintext -
426 * Key	  : 0123 4567 89ab cdef fedc ba98 7654 3210 89ab cdef 0123 4567
427 * Plain  : 0123 4567 89ab cde7
428 * Cipher : de0b 7c06 ae5e 0ed5
429 *
430 * Triple-length key, double-length plaintext -
431 * Key	  : 0123 4567 89ab cdef fedc ba98 7654 3210 89ab cdef 0123 4567
432 * Plain  : 0123 4567 89ab cdef 0123 4567 89ab cdff
433 * Cipher : ad0d 1b30 ac17 cf07 0ed1 1c63 81e4 4de5
434 *
435 * d3des V5.0a rwo 9208.07 18:44 Graven Imagery
436 **********************************************************************/
437