1 /*
2 * This is D3DES (V5.09) by Richard Outerbridge with the double and
3 * triple-length support removed for use in VNC. Also the bytebit[] array
4 * has been reversed so that the most significant bit in each byte of the
5 * key is ignored, not the least significant.
6 *
7 * These changes are:
8 * Copyright (C) 1999 AT&T Laboratories Cambridge. All Rights Reserved.
9 *
10 * This software is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
13 */
14
15 /* D3DES (V5.09) -
16 *
17 * A portable, public domain, version of the Data Encryption Standard.
18 *
19 * Written with Symantec's THINK (Lightspeed) C by Richard Outerbridge.
20 * Thanks to: Dan Hoey for his excellent Initial and Inverse permutation
21 * code; Jim Gillogly & Phil Karn for the DES key schedule code; Dennis
22 * Ferguson, Eric Young and Dana How for comparing notes; and Ray Lau,
23 * for humouring me on.
24 *
25 * Copyright (c) 1988,1989,1990,1991,1992 by Richard Outerbridge.
26 * (GEnie : OUTER; CIS : [71755,204]) Graven Imagery, 1992.
27 */
28
29 #include "d3des.h"
30
31 static void scrunch(unsigned char *, unsigned long *);
32 static void unscrun(unsigned long *, unsigned char *);
33 static void desfunc(unsigned long *, unsigned long *);
34 static void cookey(unsigned long *);
35
36 static unsigned long KnL[32] = { 0L };
37 /*
38 static unsigned long KnR[32] = { 0L };
39 static unsigned long Kn3[32] = { 0L };
40 static unsigned char Df_Key[24] = {
41 0x01,0x23,0x45,0x67,0x89,0xab,0xcd,0xef,
42 0xfe,0xdc,0xba,0x98,0x76,0x54,0x32,0x10,
43 0x89,0xab,0xcd,0xef,0x01,0x23,0x45,0x67 };
44 */
45
46 static unsigned short bytebit[8] = {
47 01, 02, 04, 010, 020, 040, 0100, 0200 };
48
49 static unsigned long bigbyte[24] = {
50 0x800000L, 0x400000L, 0x200000L, 0x100000L,
51 0x80000L, 0x40000L, 0x20000L, 0x10000L,
52 0x8000L, 0x4000L, 0x2000L, 0x1000L,
53 0x800L, 0x400L, 0x200L, 0x100L,
54 0x80L, 0x40L, 0x20L, 0x10L,
55 0x8L, 0x4L, 0x2L, 0x1L };
56
57 /* Use the key schedule specified in the Standard (ANSI X3.92-1981). */
58
59 static unsigned char pc1[56] = {
60 56, 48, 40, 32, 24, 16, 8, 0, 57, 49, 41, 33, 25, 17,
61 9, 1, 58, 50, 42, 34, 26, 18, 10, 2, 59, 51, 43, 35,
62 62, 54, 46, 38, 30, 22, 14, 6, 61, 53, 45, 37, 29, 21,
63 13, 5, 60, 52, 44, 36, 28, 20, 12, 4, 27, 19, 11, 3 };
64
65 static unsigned char totrot[16] = {
66 1,2,4,6,8,10,12,14,15,17,19,21,23,25,27,28 };
67
68 static unsigned char pc2[48] = {
69 13, 16, 10, 23, 0, 4, 2, 27, 14, 5, 20, 9,
70 22, 18, 11, 3, 25, 7, 15, 6, 26, 19, 12, 1,
71 40, 51, 30, 36, 46, 54, 29, 39, 50, 44, 32, 47,
72 43, 48, 38, 55, 33, 52, 45, 41, 49, 35, 28, 31 };
73
rfbDesKey(unsigned char * key,int edf)74 void rfbDesKey(unsigned char *key,
75 int edf)
76 {
77 register int i, j, l, m, n;
78 unsigned char pc1m[56], pcr[56];
79 unsigned long kn[32];
80
81 for ( j = 0; j < 56; j++ ) {
82 l = pc1[j];
83 m = l & 07;
84 pc1m[j] = (key[l >> 3] & bytebit[m]) ? 1 : 0;
85 }
86 for( i = 0; i < 16; i++ ) {
87 if( edf == DE1 ) m = (15 - i) << 1;
88 else m = i << 1;
89 n = m + 1;
90 kn[m] = kn[n] = 0L;
91 for( j = 0; j < 28; j++ ) {
92 l = j + totrot[i];
93 if( l < 28 ) pcr[j] = pc1m[l];
94 else pcr[j] = pc1m[l - 28];
95 }
96 for( j = 28; j < 56; j++ ) {
97 l = j + totrot[i];
98 if( l < 56 ) pcr[j] = pc1m[l];
99 else pcr[j] = pc1m[l - 28];
100 }
101 for( j = 0; j < 24; j++ ) {
102 if( pcr[pc2[j]] ) kn[m] |= bigbyte[j];
103 if( pcr[pc2[j+24]] ) kn[n] |= bigbyte[j];
104 }
105 }
106 cookey(kn);
107 return;
108 }
109
cookey(register unsigned long * raw1)110 static void cookey(register unsigned long *raw1)
111 {
112 register unsigned long *cook, *raw0;
113 unsigned long dough[32];
114 register int i;
115
116 cook = dough;
117 for( i = 0; i < 16; i++, raw1++ ) {
118 raw0 = raw1++;
119 *cook = (*raw0 & 0x00fc0000L) << 6;
120 *cook |= (*raw0 & 0x00000fc0L) << 10;
121 *cook |= (*raw1 & 0x00fc0000L) >> 10;
122 *cook++ |= (*raw1 & 0x00000fc0L) >> 6;
123 *cook = (*raw0 & 0x0003f000L) << 12;
124 *cook |= (*raw0 & 0x0000003fL) << 16;
125 *cook |= (*raw1 & 0x0003f000L) >> 4;
126 *cook++ |= (*raw1 & 0x0000003fL);
127 }
128 rfbUseKey(dough);
129 return;
130 }
131
rfbCPKey(register unsigned long * into)132 void rfbCPKey(register unsigned long *into)
133 {
134 register unsigned long *from, *endp;
135
136 from = KnL, endp = &KnL[32];
137 while( from < endp ) *into++ = *from++;
138 return;
139 }
140
rfbUseKey(register unsigned long * from)141 void rfbUseKey(register unsigned long *from)
142 {
143 register unsigned long *to, *endp;
144
145 to = KnL, endp = &KnL[32];
146 while( to < endp ) *to++ = *from++;
147 return;
148 }
149
rfbDes(unsigned char * inblock,unsigned char * outblock)150 void rfbDes(unsigned char *inblock,
151 unsigned char *outblock)
152 {
153 unsigned long work[2];
154
155 scrunch(inblock, work);
156 desfunc(work, KnL);
157 unscrun(work, outblock);
158 return;
159 }
160
scrunch(register unsigned char * outof,register unsigned long * into)161 static void scrunch(register unsigned char *outof,
162 register unsigned long *into)
163 {
164 *into = (*outof++ & 0xffL) << 24;
165 *into |= (*outof++ & 0xffL) << 16;
166 *into |= (*outof++ & 0xffL) << 8;
167 *into++ |= (*outof++ & 0xffL);
168 *into = (*outof++ & 0xffL) << 24;
169 *into |= (*outof++ & 0xffL) << 16;
170 *into |= (*outof++ & 0xffL) << 8;
171 *into |= (*outof & 0xffL);
172 return;
173 }
174
unscrun(register unsigned long * outof,register unsigned char * into)175 static void unscrun(register unsigned long *outof,
176 register unsigned char *into)
177 {
178 *into++ = (unsigned char)((*outof >> 24) & 0xffL);
179 *into++ = (unsigned char)((*outof >> 16) & 0xffL);
180 *into++ = (unsigned char)((*outof >> 8) & 0xffL);
181 *into++ = (unsigned char)( *outof++ & 0xffL);
182 *into++ = (unsigned char)((*outof >> 24) & 0xffL);
183 *into++ = (unsigned char)((*outof >> 16) & 0xffL);
184 *into++ = (unsigned char)((*outof >> 8) & 0xffL);
185 *into = (unsigned char)( *outof & 0xffL);
186 return;
187 }
188
189 static unsigned long SP1[64] = {
190 0x01010400L, 0x00000000L, 0x00010000L, 0x01010404L,
191 0x01010004L, 0x00010404L, 0x00000004L, 0x00010000L,
192 0x00000400L, 0x01010400L, 0x01010404L, 0x00000400L,
193 0x01000404L, 0x01010004L, 0x01000000L, 0x00000004L,
194 0x00000404L, 0x01000400L, 0x01000400L, 0x00010400L,
195 0x00010400L, 0x01010000L, 0x01010000L, 0x01000404L,
196 0x00010004L, 0x01000004L, 0x01000004L, 0x00010004L,
197 0x00000000L, 0x00000404L, 0x00010404L, 0x01000000L,
198 0x00010000L, 0x01010404L, 0x00000004L, 0x01010000L,
199 0x01010400L, 0x01000000L, 0x01000000L, 0x00000400L,
200 0x01010004L, 0x00010000L, 0x00010400L, 0x01000004L,
201 0x00000400L, 0x00000004L, 0x01000404L, 0x00010404L,
202 0x01010404L, 0x00010004L, 0x01010000L, 0x01000404L,
203 0x01000004L, 0x00000404L, 0x00010404L, 0x01010400L,
204 0x00000404L, 0x01000400L, 0x01000400L, 0x00000000L,
205 0x00010004L, 0x00010400L, 0x00000000L, 0x01010004L };
206
207 static unsigned long SP2[64] = {
208 0x80108020L, 0x80008000L, 0x00008000L, 0x00108020L,
209 0x00100000L, 0x00000020L, 0x80100020L, 0x80008020L,
210 0x80000020L, 0x80108020L, 0x80108000L, 0x80000000L,
211 0x80008000L, 0x00100000L, 0x00000020L, 0x80100020L,
212 0x00108000L, 0x00100020L, 0x80008020L, 0x00000000L,
213 0x80000000L, 0x00008000L, 0x00108020L, 0x80100000L,
214 0x00100020L, 0x80000020L, 0x00000000L, 0x00108000L,
215 0x00008020L, 0x80108000L, 0x80100000L, 0x00008020L,
216 0x00000000L, 0x00108020L, 0x80100020L, 0x00100000L,
217 0x80008020L, 0x80100000L, 0x80108000L, 0x00008000L,
218 0x80100000L, 0x80008000L, 0x00000020L, 0x80108020L,
219 0x00108020L, 0x00000020L, 0x00008000L, 0x80000000L,
220 0x00008020L, 0x80108000L, 0x00100000L, 0x80000020L,
221 0x00100020L, 0x80008020L, 0x80000020L, 0x00100020L,
222 0x00108000L, 0x00000000L, 0x80008000L, 0x00008020L,
223 0x80000000L, 0x80100020L, 0x80108020L, 0x00108000L };
224
225 static unsigned long SP3[64] = {
226 0x00000208L, 0x08020200L, 0x00000000L, 0x08020008L,
227 0x08000200L, 0x00000000L, 0x00020208L, 0x08000200L,
228 0x00020008L, 0x08000008L, 0x08000008L, 0x00020000L,
229 0x08020208L, 0x00020008L, 0x08020000L, 0x00000208L,
230 0x08000000L, 0x00000008L, 0x08020200L, 0x00000200L,
231 0x00020200L, 0x08020000L, 0x08020008L, 0x00020208L,
232 0x08000208L, 0x00020200L, 0x00020000L, 0x08000208L,
233 0x00000008L, 0x08020208L, 0x00000200L, 0x08000000L,
234 0x08020200L, 0x08000000L, 0x00020008L, 0x00000208L,
235 0x00020000L, 0x08020200L, 0x08000200L, 0x00000000L,
236 0x00000200L, 0x00020008L, 0x08020208L, 0x08000200L,
237 0x08000008L, 0x00000200L, 0x00000000L, 0x08020008L,
238 0x08000208L, 0x00020000L, 0x08000000L, 0x08020208L,
239 0x00000008L, 0x00020208L, 0x00020200L, 0x08000008L,
240 0x08020000L, 0x08000208L, 0x00000208L, 0x08020000L,
241 0x00020208L, 0x00000008L, 0x08020008L, 0x00020200L };
242
243 static unsigned long SP4[64] = {
244 0x00802001L, 0x00002081L, 0x00002081L, 0x00000080L,
245 0x00802080L, 0x00800081L, 0x00800001L, 0x00002001L,
246 0x00000000L, 0x00802000L, 0x00802000L, 0x00802081L,
247 0x00000081L, 0x00000000L, 0x00800080L, 0x00800001L,
248 0x00000001L, 0x00002000L, 0x00800000L, 0x00802001L,
249 0x00000080L, 0x00800000L, 0x00002001L, 0x00002080L,
250 0x00800081L, 0x00000001L, 0x00002080L, 0x00800080L,
251 0x00002000L, 0x00802080L, 0x00802081L, 0x00000081L,
252 0x00800080L, 0x00800001L, 0x00802000L, 0x00802081L,
253 0x00000081L, 0x00000000L, 0x00000000L, 0x00802000L,
254 0x00002080L, 0x00800080L, 0x00800081L, 0x00000001L,
255 0x00802001L, 0x00002081L, 0x00002081L, 0x00000080L,
256 0x00802081L, 0x00000081L, 0x00000001L, 0x00002000L,
257 0x00800001L, 0x00002001L, 0x00802080L, 0x00800081L,
258 0x00002001L, 0x00002080L, 0x00800000L, 0x00802001L,
259 0x00000080L, 0x00800000L, 0x00002000L, 0x00802080L };
260
261 static unsigned long SP5[64] = {
262 0x00000100L, 0x02080100L, 0x02080000L, 0x42000100L,
263 0x00080000L, 0x00000100L, 0x40000000L, 0x02080000L,
264 0x40080100L, 0x00080000L, 0x02000100L, 0x40080100L,
265 0x42000100L, 0x42080000L, 0x00080100L, 0x40000000L,
266 0x02000000L, 0x40080000L, 0x40080000L, 0x00000000L,
267 0x40000100L, 0x42080100L, 0x42080100L, 0x02000100L,
268 0x42080000L, 0x40000100L, 0x00000000L, 0x42000000L,
269 0x02080100L, 0x02000000L, 0x42000000L, 0x00080100L,
270 0x00080000L, 0x42000100L, 0x00000100L, 0x02000000L,
271 0x40000000L, 0x02080000L, 0x42000100L, 0x40080100L,
272 0x02000100L, 0x40000000L, 0x42080000L, 0x02080100L,
273 0x40080100L, 0x00000100L, 0x02000000L, 0x42080000L,
274 0x42080100L, 0x00080100L, 0x42000000L, 0x42080100L,
275 0x02080000L, 0x00000000L, 0x40080000L, 0x42000000L,
276 0x00080100L, 0x02000100L, 0x40000100L, 0x00080000L,
277 0x00000000L, 0x40080000L, 0x02080100L, 0x40000100L };
278
279 static unsigned long SP6[64] = {
280 0x20000010L, 0x20400000L, 0x00004000L, 0x20404010L,
281 0x20400000L, 0x00000010L, 0x20404010L, 0x00400000L,
282 0x20004000L, 0x00404010L, 0x00400000L, 0x20000010L,
283 0x00400010L, 0x20004000L, 0x20000000L, 0x00004010L,
284 0x00000000L, 0x00400010L, 0x20004010L, 0x00004000L,
285 0x00404000L, 0x20004010L, 0x00000010L, 0x20400010L,
286 0x20400010L, 0x00000000L, 0x00404010L, 0x20404000L,
287 0x00004010L, 0x00404000L, 0x20404000L, 0x20000000L,
288 0x20004000L, 0x00000010L, 0x20400010L, 0x00404000L,
289 0x20404010L, 0x00400000L, 0x00004010L, 0x20000010L,
290 0x00400000L, 0x20004000L, 0x20000000L, 0x00004010L,
291 0x20000010L, 0x20404010L, 0x00404000L, 0x20400000L,
292 0x00404010L, 0x20404000L, 0x00000000L, 0x20400010L,
293 0x00000010L, 0x00004000L, 0x20400000L, 0x00404010L,
294 0x00004000L, 0x00400010L, 0x20004010L, 0x00000000L,
295 0x20404000L, 0x20000000L, 0x00400010L, 0x20004010L };
296
297 static unsigned long SP7[64] = {
298 0x00200000L, 0x04200002L, 0x04000802L, 0x00000000L,
299 0x00000800L, 0x04000802L, 0x00200802L, 0x04200800L,
300 0x04200802L, 0x00200000L, 0x00000000L, 0x04000002L,
301 0x00000002L, 0x04000000L, 0x04200002L, 0x00000802L,
302 0x04000800L, 0x00200802L, 0x00200002L, 0x04000800L,
303 0x04000002L, 0x04200000L, 0x04200800L, 0x00200002L,
304 0x04200000L, 0x00000800L, 0x00000802L, 0x04200802L,
305 0x00200800L, 0x00000002L, 0x04000000L, 0x00200800L,
306 0x04000000L, 0x00200800L, 0x00200000L, 0x04000802L,
307 0x04000802L, 0x04200002L, 0x04200002L, 0x00000002L,
308 0x00200002L, 0x04000000L, 0x04000800L, 0x00200000L,
309 0x04200800L, 0x00000802L, 0x00200802L, 0x04200800L,
310 0x00000802L, 0x04000002L, 0x04200802L, 0x04200000L,
311 0x00200800L, 0x00000000L, 0x00000002L, 0x04200802L,
312 0x00000000L, 0x00200802L, 0x04200000L, 0x00000800L,
313 0x04000002L, 0x04000800L, 0x00000800L, 0x00200002L };
314
315 static unsigned long SP8[64] = {
316 0x10001040L, 0x00001000L, 0x00040000L, 0x10041040L,
317 0x10000000L, 0x10001040L, 0x00000040L, 0x10000000L,
318 0x00040040L, 0x10040000L, 0x10041040L, 0x00041000L,
319 0x10041000L, 0x00041040L, 0x00001000L, 0x00000040L,
320 0x10040000L, 0x10000040L, 0x10001000L, 0x00001040L,
321 0x00041000L, 0x00040040L, 0x10040040L, 0x10041000L,
322 0x00001040L, 0x00000000L, 0x00000000L, 0x10040040L,
323 0x10000040L, 0x10001000L, 0x00041040L, 0x00040000L,
324 0x00041040L, 0x00040000L, 0x10041000L, 0x00001000L,
325 0x00000040L, 0x10040040L, 0x00001000L, 0x00041040L,
326 0x10001000L, 0x00000040L, 0x10000040L, 0x10040000L,
327 0x10040040L, 0x10000000L, 0x00040000L, 0x10001040L,
328 0x00000000L, 0x10041040L, 0x00040040L, 0x10000040L,
329 0x10040000L, 0x10001000L, 0x10001040L, 0x00000000L,
330 0x10041040L, 0x00041000L, 0x00041000L, 0x00001040L,
331 0x00001040L, 0x00040040L, 0x10000000L, 0x10041000L };
332
desfunc(register unsigned long * block,register unsigned long * keys)333 static void desfunc(register unsigned long *block,
334 register unsigned long *keys)
335 {
336 register unsigned long fval, work, right, leftt;
337 register int round;
338
339 leftt = block[0];
340 right = block[1];
341 work = ((leftt >> 4) ^ right) & 0x0f0f0f0fL;
342 right ^= work;
343 leftt ^= (work << 4);
344 work = ((leftt >> 16) ^ right) & 0x0000ffffL;
345 right ^= work;
346 leftt ^= (work << 16);
347 work = ((right >> 2) ^ leftt) & 0x33333333L;
348 leftt ^= work;
349 right ^= (work << 2);
350 work = ((right >> 8) ^ leftt) & 0x00ff00ffL;
351 leftt ^= work;
352 right ^= (work << 8);
353 right = ((right << 1) | ((right >> 31) & 1L)) & 0xffffffffL;
354 work = (leftt ^ right) & 0xaaaaaaaaL;
355 leftt ^= work;
356 right ^= work;
357 leftt = ((leftt << 1) | ((leftt >> 31) & 1L)) & 0xffffffffL;
358
359 for( round = 0; round < 8; round++ ) {
360 work = (right << 28) | (right >> 4);
361 work ^= *keys++;
362 fval = SP7[ work & 0x3fL];
363 fval |= SP5[(work >> 8) & 0x3fL];
364 fval |= SP3[(work >> 16) & 0x3fL];
365 fval |= SP1[(work >> 24) & 0x3fL];
366 work = right ^ *keys++;
367 fval |= SP8[ work & 0x3fL];
368 fval |= SP6[(work >> 8) & 0x3fL];
369 fval |= SP4[(work >> 16) & 0x3fL];
370 fval |= SP2[(work >> 24) & 0x3fL];
371 leftt ^= fval;
372 work = (leftt << 28) | (leftt >> 4);
373 work ^= *keys++;
374 fval = SP7[ work & 0x3fL];
375 fval |= SP5[(work >> 8) & 0x3fL];
376 fval |= SP3[(work >> 16) & 0x3fL];
377 fval |= SP1[(work >> 24) & 0x3fL];
378 work = leftt ^ *keys++;
379 fval |= SP8[ work & 0x3fL];
380 fval |= SP6[(work >> 8) & 0x3fL];
381 fval |= SP4[(work >> 16) & 0x3fL];
382 fval |= SP2[(work >> 24) & 0x3fL];
383 right ^= fval;
384 }
385
386 right = (right << 31) | (right >> 1);
387 work = (leftt ^ right) & 0xaaaaaaaaL;
388 leftt ^= work;
389 right ^= work;
390 leftt = (leftt << 31) | (leftt >> 1);
391 work = ((leftt >> 8) ^ right) & 0x00ff00ffL;
392 right ^= work;
393 leftt ^= (work << 8);
394 work = ((leftt >> 2) ^ right) & 0x33333333L;
395 right ^= work;
396 leftt ^= (work << 2);
397 work = ((right >> 16) ^ leftt) & 0x0000ffffL;
398 leftt ^= work;
399 right ^= (work << 16);
400 work = ((right >> 4) ^ leftt) & 0x0f0f0f0fL;
401 leftt ^= work;
402 right ^= (work << 4);
403 *block++ = right;
404 *block = leftt;
405 return;
406 }
407
408 /* Validation sets:
409 *
410 * Single-length key, single-length plaintext -
411 * Key : 0123 4567 89ab cdef
412 * Plain : 0123 4567 89ab cde7
413 * Cipher : c957 4425 6a5e d31d
414 *
415 * Double-length key, single-length plaintext -
416 * Key : 0123 4567 89ab cdef fedc ba98 7654 3210
417 * Plain : 0123 4567 89ab cde7
418 * Cipher : 7f1d 0a77 826b 8aff
419 *
420 * Double-length key, double-length plaintext -
421 * Key : 0123 4567 89ab cdef fedc ba98 7654 3210
422 * Plain : 0123 4567 89ab cdef 0123 4567 89ab cdff
423 * Cipher : 27a0 8440 406a df60 278f 47cf 42d6 15d7
424 *
425 * Triple-length key, single-length plaintext -
426 * Key : 0123 4567 89ab cdef fedc ba98 7654 3210 89ab cdef 0123 4567
427 * Plain : 0123 4567 89ab cde7
428 * Cipher : de0b 7c06 ae5e 0ed5
429 *
430 * Triple-length key, double-length plaintext -
431 * Key : 0123 4567 89ab cdef fedc ba98 7654 3210 89ab cdef 0123 4567
432 * Plain : 0123 4567 89ab cdef 0123 4567 89ab cdff
433 * Cipher : ad0d 1b30 ac17 cf07 0ed1 1c63 81e4 4de5
434 *
435 * d3des V5.0a rwo 9208.07 18:44 Graven Imagery
436 **********************************************************************/
437