wintrust: Sign-compare warnings fix.
[wine] / dlls / wintrust / crypt.c
1 /*
2  * WinTrust Cryptography functions
3  *
4  * Copyright 2006 James Hawkins
5  * Copyright 2000-2002 Stuart Caie
6  * Copyright 2002 Patrik Stridvall
7  * Copyright 2003 Greg Turner
8  * Copyright 2008 Juan Lang
9  *
10  * This library is free software; you can redistribute it and/or
11  * modify it under the terms of the GNU Lesser General Public
12  * License as published by the Free Software Foundation; either
13  * version 2.1 of the License, or (at your option) any later version.
14  *
15  * This library is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  * Lesser General Public License for more details.
19  *
20  * You should have received a copy of the GNU Lesser General Public
21  * License along with this library; if not, write to the Free Software
22  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
23  */
24
25 #include <stdarg.h>
26 #include <stdio.h>
27 #include "windef.h"
28 #include "winbase.h"
29 #include "wintrust.h"
30 #include "mscat.h"
31 #include "mssip.h"
32 #include "imagehlp.h"
33
34 #include "wine/debug.h"
35
36 WINE_DEFAULT_DEBUG_CHANNEL(wintrust);
37
38 /***********************************************************************
39  *      CryptCATAdminAcquireContext (WINTRUST.@)
40  *
41  * Get a catalog administrator context handle.
42  *
43  * PARAMS
44  *   catAdmin  [O] Pointer to the context handle.
45  *   sysSystem [I] Pointer to a GUID for the needed subsystem.
46  *   dwFlags   [I] Reserved.
47  *
48  * RETURNS
49  *   Success: TRUE. catAdmin contains the context handle.
50  *   Failure: FALSE.
51  *
52  */
53 BOOL WINAPI CryptCATAdminAcquireContext(HCATADMIN* catAdmin,
54                     const GUID *sysSystem, DWORD dwFlags )
55 {
56     FIXME("%p %s %x\n", catAdmin, debugstr_guid(sysSystem), dwFlags);
57
58     if (!catAdmin)
59     {
60         SetLastError(ERROR_INVALID_PARAMETER);
61         return FALSE;
62     }
63
64     *catAdmin = (HCATADMIN)0xdeadbeef;
65
66     return TRUE;
67 }
68
69 /***********************************************************************
70  *             CryptCATAdminAddCatalog (WINTRUST.@)
71  */
72 HCATINFO WINAPI CryptCATAdminAddCatalog(HCATADMIN catAdmin, PWSTR catalogFile,
73                                         PWSTR selectBaseName, DWORD flags)
74 {
75     FIXME("%p %s %s %d\n", catAdmin, debugstr_w(catalogFile),
76           debugstr_w(selectBaseName), flags);
77     return (HCATINFO) 0xdeeeaaadL;
78 }
79
80 /***********************************************************************
81  *             CryptCATAdminCalcHashFromFileHandle (WINTRUST.@)
82  */
83 BOOL WINAPI CryptCATAdminCalcHashFromFileHandle(HANDLE hFile, DWORD* pcbHash,
84                                                 BYTE* pbHash, DWORD dwFlags )
85 {
86     FIXME("%p %p %p %x\n", hFile, pcbHash, pbHash, dwFlags);
87
88     if (pbHash && pcbHash) memset(pbHash, 0, *pcbHash);
89     return TRUE;
90 }
91
92 /***********************************************************************
93  *             CryptCATAdminEnumCatalogFromHash (WINTRUST.@)
94  */
95 HCATINFO WINAPI CryptCATAdminEnumCatalogFromHash(HCATADMIN hCatAdmin,
96                                                  BYTE* pbHash,
97                                                  DWORD cbHash,
98                                                  DWORD dwFlags,
99                                                  HCATINFO* phPrevCatInfo )
100 {
101     FIXME("%p %p %d %d %p\n", hCatAdmin, pbHash, cbHash, dwFlags, phPrevCatInfo);
102     return NULL;
103 }
104
105 /***********************************************************************
106  *      CryptCATAdminReleaseCatalogContext (WINTRUST.@)
107  *
108  * Release a catalog context handle.
109  *
110  * PARAMS
111  *   hCatAdmin [I] Context handle.
112  *   hCatInfo  [I] Catalog handle.
113  *   dwFlags   [I] Reserved.
114  *
115  * RETURNS
116  *   Success: TRUE.
117  *   Failure: FAIL.
118  *
119  */
120 BOOL WINAPI CryptCATAdminReleaseCatalogContext(HCATADMIN hCatAdmin,
121                                                HCATINFO hCatInfo,
122                                                DWORD dwFlags)
123 {
124     FIXME("%p %p %x\n", hCatAdmin, hCatInfo, dwFlags);
125     return TRUE;
126 }
127
128 /***********************************************************************
129  *      CryptCATAdminReleaseContext (WINTRUST.@)
130  *
131  * Release a catalog administrator context handle.
132  *
133  * PARAMS
134  *   catAdmin  [I] Context handle.
135  *   dwFlags   [I] Reserved.
136  *
137  * RETURNS
138  *   Success: TRUE.
139  *   Failure: FAIL.
140  *
141  */
142 BOOL WINAPI CryptCATAdminReleaseContext(HCATADMIN hCatAdmin, DWORD dwFlags )
143 {
144     FIXME("%p %x\n", hCatAdmin, dwFlags);
145     return TRUE;
146 }
147
148 /***********************************************************************
149  *      CryptCATAdminRemoveCatalog (WINTRUST.@)
150  *
151  * Remove a catalog file.
152  *
153  * PARAMS
154  *   catAdmin         [I] Context handle.
155  *   pwszCatalogFile  [I] Catalog file.
156  *   dwFlags          [I] Reserved.
157  *
158  * RETURNS
159  *   Success: TRUE.
160  *   Failure: FALSE.
161  *
162  */
163 BOOL WINAPI CryptCATAdminRemoveCatalog(HCATADMIN hCatAdmin, LPCWSTR pwszCatalogFile, DWORD dwFlags)
164 {
165     FIXME("%p %s %x\n", hCatAdmin, debugstr_w(pwszCatalogFile), dwFlags);
166     return DeleteFileW(pwszCatalogFile);
167 }
168
169 /***********************************************************************
170  *      CryptCATClose  (WINTRUST.@)
171  */
172 BOOL WINAPI CryptCATClose(HANDLE hCatalog)
173 {
174     FIXME("(%p) stub\n", hCatalog);
175     return TRUE;
176 }
177
178 /***********************************************************************
179  *      CryptCATEnumerateMember  (WINTRUST.@)
180  */
181 CRYPTCATMEMBER *WINAPI CryptCATEnumerateMember(HANDLE hCatalog, CRYPTCATMEMBER* pPrevMember)
182 {
183     FIXME("(%p, %p) stub\n", hCatalog, pPrevMember);
184     return NULL;
185 }
186
187 /***********************************************************************
188  *      CryptCATOpen  (WINTRUST.@)
189  */
190 HANDLE WINAPI CryptCATOpen(LPWSTR pwszFileName, DWORD fdwOpenFlags, HCRYPTPROV hProv,
191                            DWORD dwPublicVersion, DWORD dwEncodingType)
192 {
193     FIXME("(%s, %d, %ld, %d, %d) stub\n", debugstr_w(pwszFileName), fdwOpenFlags,
194           hProv, dwPublicVersion, dwEncodingType);
195     return 0;
196 }
197
198 /***********************************************************************
199  *      CryptSIPCreateIndirectData  (WINTRUST.@)
200  */
201 BOOL WINAPI CryptSIPCreateIndirectData(SIP_SUBJECTINFO* pSubjectInfo, DWORD* pcbIndirectData,
202                                        SIP_INDIRECT_DATA* pIndirectData)
203 {
204     FIXME("(%p %p %p) stub\n", pSubjectInfo, pcbIndirectData, pIndirectData);
205  
206     return FALSE;
207 }
208
209 static BOOL WINTRUST_GetSignedMsgFromPEFile(SIP_SUBJECTINFO *pSubjectInfo,
210  DWORD *pdwEncodingType, DWORD dwIndex, DWORD *pcbSignedDataMsg,
211  BYTE *pbSignedDataMsg)
212 {
213     BOOL ret;
214     WIN_CERTIFICATE *pCert = NULL;
215
216     TRACE("(%p %p %d %p %p)\n", pSubjectInfo, pdwEncodingType, dwIndex,
217           pcbSignedDataMsg, pbSignedDataMsg);
218  
219     if (!pbSignedDataMsg)
220     {
221         WIN_CERTIFICATE cert;
222
223         /* app hasn't passed buffer, just get the length */
224         ret = ImageGetCertificateHeader(pSubjectInfo->hFile, dwIndex, &cert);
225         if (ret)
226             *pcbSignedDataMsg = cert.dwLength;
227     }
228     else
229     {
230         DWORD len = 0;
231
232         ret = ImageGetCertificateData(pSubjectInfo->hFile, dwIndex, NULL, &len);
233         if (GetLastError() != ERROR_INSUFFICIENT_BUFFER)
234             goto error;
235         pCert = HeapAlloc(GetProcessHeap(), 0, len);
236         if (!pCert)
237         {
238             ret = FALSE;
239             goto error;
240         }
241         ret = ImageGetCertificateData(pSubjectInfo->hFile, dwIndex, pCert,
242          &len);
243         if (!ret)
244             goto error;
245         if (*pcbSignedDataMsg < pCert->dwLength)
246         {
247             *pcbSignedDataMsg = pCert->dwLength;
248             SetLastError(ERROR_INSUFFICIENT_BUFFER);
249             ret = FALSE;
250         }
251         else
252         {
253             memcpy(pbSignedDataMsg, pCert->bCertificate, pCert->dwLength);
254             switch (pCert->wCertificateType)
255             {
256             case WIN_CERT_TYPE_X509:
257                 *pdwEncodingType = X509_ASN_ENCODING;
258                 break;
259             case WIN_CERT_TYPE_PKCS_SIGNED_DATA:
260                 *pdwEncodingType = X509_ASN_ENCODING | PKCS_7_ASN_ENCODING;
261                 break;
262             default:
263                 FIXME("don't know what to do for encoding type %d\n",
264                  pCert->wCertificateType);
265                 *pdwEncodingType = 0;
266             }
267         }
268     }
269 error:
270     HeapFree(GetProcessHeap(), 0, pCert);
271     return ret;
272 }
273
274 /* structure offsets */
275 #define cfhead_Signature         (0x00)
276 #define cfhead_CabinetSize       (0x08)
277 #define cfhead_MinorVersion      (0x18)
278 #define cfhead_MajorVersion      (0x19)
279 #define cfhead_Flags             (0x1E)
280 #define cfhead_SIZEOF            (0x24)
281 #define cfheadext_HeaderReserved (0x00)
282 #define cfheadext_SIZEOF         (0x04)
283 #define cfsigninfo_CertOffset    (0x04)
284 #define cfsigninfo_CertSize      (0x08)
285 #define cfsigninfo_SIZEOF        (0x0C)
286
287 /* flags */
288 #define cfheadRESERVE_PRESENT          (0x0004)
289
290 /* endian-neutral reading of little-endian data */
291 #define EndGetI32(a)  ((((a)[3])<<24)|(((a)[2])<<16)|(((a)[1])<<8)|((a)[0]))
292 #define EndGetI16(a)  ((((a)[1])<<8)|((a)[0]))
293
294 /* For documentation purposes only:  this is the structure in the reserved
295  * area of a signed cabinet file.  The cert offset indicates where in the
296  * cabinet file the signature resides, and the count indicates its size.
297  */
298 typedef struct _CAB_SIGNINFO
299 {
300     WORD unk0; /* always 0? */
301     WORD unk1; /* always 0x0010? */
302     DWORD dwCertOffset;
303     DWORD cbCertBlock;
304 } CAB_SIGNINFO, *PCAB_SIGNINFO;
305
306 static BOOL WINTRUST_GetSignedMsgFromCabFile(SIP_SUBJECTINFO *pSubjectInfo,
307  DWORD *pdwEncodingType, DWORD dwIndex, DWORD *pcbSignedDataMsg,
308  BYTE *pbSignedDataMsg)
309 {
310     int header_resv;
311     LONG base_offset, cabsize;
312     USHORT flags;
313     BYTE buf[64];
314     DWORD cert_offset, cert_size, dwRead;
315
316     TRACE("(%p %p %d %p %p)\n", pSubjectInfo, pdwEncodingType, dwIndex,
317           pcbSignedDataMsg, pbSignedDataMsg);
318
319     /*
320      * FIXME: I just noticed that I am memorizing the initial file pointer
321      * offset and restoring it before reading in the rest of the header
322      * information in the cabinet.  Perhaps that's correct -- that is, perhaps
323      * this API is supposed to support "streaming" cabinets which are embedded
324      * in other files, or cabinets which begin at file offsets other than zero.
325      * Otherwise, I should instead go to the absolute beginning of the file.
326      * (Either way, the semantics of wine's FDICopy require me to leave the
327      * file pointer where it is afterwards -- If Windows does not do so, we
328      * ought to duplicate the native behavior in the FDIIsCabinet API, not here.
329      *
330      * So, the answer lies in Windows; will native cabinet.dll recognize a
331      * cabinet "file" embedded in another file?  Note that cabextract.c does
332      * support this, which implies that Microsoft's might.  I haven't tried it
333      * yet so I don't know.  ATM, most of wine's FDI cabinet routines (except
334      * this one) would not work in this way.  To fix it, we could just make the
335      * various references to absolute file positions in the code relative to an
336      * initial "beginning" offset.  Because the FDICopy API doesn't take a
337      * file-handle like this one, we would therein need to search through the
338      * file for the beginning of the cabinet (as we also do in cabextract.c).
339      * Note that this limits us to a maximum of one cabinet per. file: the first.
340      *
341      * So, in summary: either the code below is wrong, or the rest of fdi.c is
342      * wrong... I cannot imagine that both are correct ;)  One of these flaws
343      * should be fixed after determining the behavior on Windows.   We ought
344      * to check both FDIIsCabinet and FDICopy for the right behavior.
345      *
346      * -gmt
347      */
348
349     /* get basic offset & size info */
350     base_offset = SetFilePointer(pSubjectInfo->hFile, 0L, NULL, SEEK_CUR);
351
352     if (SetFilePointer(pSubjectInfo->hFile, 0, NULL, SEEK_END) == INVALID_SET_FILE_POINTER)
353     {
354         TRACE("seek error\n");
355         return FALSE;
356     }
357
358     cabsize = SetFilePointer(pSubjectInfo->hFile, 0L, NULL, SEEK_CUR);
359     if ((cabsize == -1) || (base_offset == -1) ||
360      (SetFilePointer(pSubjectInfo->hFile, base_offset, NULL, SEEK_SET) == INVALID_SET_FILE_POINTER))
361     {
362         TRACE("seek error\n");
363         return FALSE;
364     }
365
366     /* read in the CFHEADER */
367     if (!ReadFile(pSubjectInfo->hFile, buf, cfhead_SIZEOF, &dwRead, NULL) ||
368      dwRead != cfhead_SIZEOF)
369     {
370         TRACE("reading header failed\n");
371         return FALSE;
372     }
373
374     /* check basic MSCF signature */
375     if (EndGetI32(buf+cfhead_Signature) != 0x4643534d)
376     {
377         WARN("cabinet signature not present\n");
378         return FALSE;
379     }
380
381     /* Ignore the number of folders and files and the set and cabinet IDs */
382
383     /* check the header revision */
384     if ((buf[cfhead_MajorVersion] > 1) ||
385         (buf[cfhead_MajorVersion] == 1 && buf[cfhead_MinorVersion] > 3))
386     {
387         WARN("cabinet format version > 1.3\n");
388         return FALSE;
389     }
390
391     /* pull the flags out */
392     flags = EndGetI16(buf+cfhead_Flags);
393
394     if (!(flags & cfheadRESERVE_PRESENT))
395     {
396         TRACE("no header present, not signed\n");
397         return FALSE;
398     }
399
400     if (!ReadFile(pSubjectInfo->hFile, buf, cfheadext_SIZEOF, &dwRead, NULL) ||
401      dwRead != cfheadext_SIZEOF)
402     {
403         ERR("bunk reserve-sizes?\n");
404         return FALSE;
405     }
406
407     header_resv = EndGetI16(buf+cfheadext_HeaderReserved);
408     if (!header_resv)
409     {
410         TRACE("no header_resv, not signed\n");
411         return FALSE;
412     }
413     else if (header_resv < cfsigninfo_SIZEOF)
414     {
415         TRACE("header_resv too small, not signed\n");
416         return FALSE;
417     }
418
419     if (header_resv > 60000)
420     {
421         WARN("WARNING; header reserved space > 60000\n");
422     }
423
424     if (!ReadFile(pSubjectInfo->hFile, buf, cfsigninfo_SIZEOF, &dwRead, NULL) ||
425      dwRead != cfsigninfo_SIZEOF)
426     {
427         ERR("couldn't read reserve\n");
428         return FALSE;
429     }
430
431     cert_offset = EndGetI32(buf+cfsigninfo_CertOffset);
432     TRACE("cert_offset: %d\n", cert_offset);
433     cert_size = EndGetI32(buf+cfsigninfo_CertSize);
434     TRACE("cert_size: %d\n", cert_size);
435
436     /* The redundant checks are to avoid wraparound */
437     if (cert_offset > cabsize || cert_size > cabsize ||
438      cert_offset + cert_size > cabsize)
439     {
440         WARN("offset beyond file, not attempting to read\n");
441         return FALSE;
442     }
443
444     SetFilePointer(pSubjectInfo->hFile, base_offset, NULL, SEEK_SET);
445     if (!pbSignedDataMsg)
446     {
447         *pcbSignedDataMsg = cert_size;
448         return TRUE;
449     }
450     if (*pcbSignedDataMsg < cert_size)
451     {
452         *pcbSignedDataMsg = cert_size;
453         SetLastError(ERROR_INSUFFICIENT_BUFFER);
454         return FALSE;
455     }
456     if (SetFilePointer(pSubjectInfo->hFile, cert_offset, NULL, SEEK_SET) == INVALID_SET_FILE_POINTER)
457     {
458         ERR("couldn't seek to cert location\n");
459         return FALSE;
460     }
461     if (!ReadFile(pSubjectInfo->hFile, pbSignedDataMsg, cert_size, &dwRead,
462      NULL) || dwRead != cert_size)
463     {
464         ERR("couldn't read cert\n");
465         return FALSE;
466     }
467     /* The encoding of the files I've seen appears to be in ASN.1
468      * format, and there isn't a field indicating the type, so assume it
469      * always is.
470      */
471     *pdwEncodingType = X509_ASN_ENCODING | PKCS_7_ASN_ENCODING;
472     return TRUE;
473 }
474
475 static BOOL WINTRUST_GetSignedMsgFromCatFile(SIP_SUBJECTINFO *pSubjectInfo,
476  DWORD *pdwEncodingType, DWORD dwIndex, DWORD *pcbSignedDataMsg,
477  BYTE *pbSignedDataMsg)
478 {
479     BOOL ret;
480
481     TRACE("(%p %p %d %p %p)\n", pSubjectInfo, pdwEncodingType, dwIndex,
482           pcbSignedDataMsg, pbSignedDataMsg);
483
484     if (!pbSignedDataMsg)
485     {
486         *pcbSignedDataMsg = GetFileSize(pSubjectInfo->hFile, NULL);
487          ret = TRUE;
488     }
489     else
490     {
491         DWORD len = GetFileSize(pSubjectInfo->hFile, NULL);
492
493         if (*pcbSignedDataMsg < len)
494         {
495             *pcbSignedDataMsg = len;
496             SetLastError(ERROR_INSUFFICIENT_BUFFER);
497             ret = FALSE;
498         }
499         else
500         {
501             ret = ReadFile(pSubjectInfo->hFile, pbSignedDataMsg, len,
502              pcbSignedDataMsg, NULL);
503             if (ret)
504                 *pdwEncodingType = X509_ASN_ENCODING | PKCS_7_ASN_ENCODING;
505         }
506     }
507     return ret;
508 }
509
510 /***********************************************************************
511  *      CryptSIPGetSignedDataMsg  (WINTRUST.@)
512  */
513 BOOL WINAPI CryptSIPGetSignedDataMsg(SIP_SUBJECTINFO* pSubjectInfo, DWORD* pdwEncodingType,
514                                        DWORD dwIndex, DWORD* pcbSignedDataMsg, BYTE* pbSignedDataMsg)
515 {
516     static const GUID unknown = { 0xC689AAB8, 0x8E78, 0x11D0, { 0x8C,0x47,
517      0x00,0xC0,0x4F,0xC2,0x95,0xEE } };
518     static const GUID cabGUID = { 0xC689AABA, 0x8E78, 0x11D0, { 0x8C,0x47,
519      0x00,0xC0,0x4F,0xC2,0x95,0xEE } };
520     static const GUID catGUID = { 0xDE351A43, 0x8E59, 0x11D0, { 0x8C,0x47,
521      0x00,0xC0,0x4F,0xC2,0x95,0xEE }};
522     BOOL ret;
523
524     TRACE("(%p %p %d %p %p)\n", pSubjectInfo, pdwEncodingType, dwIndex,
525           pcbSignedDataMsg, pbSignedDataMsg);
526
527     if (!memcmp(pSubjectInfo->pgSubjectType, &unknown, sizeof(unknown)))
528         ret = WINTRUST_GetSignedMsgFromPEFile(pSubjectInfo, pdwEncodingType,
529          dwIndex, pcbSignedDataMsg, pbSignedDataMsg);
530     else if (!memcmp(pSubjectInfo->pgSubjectType, &cabGUID, sizeof(cabGUID)))
531         ret = WINTRUST_GetSignedMsgFromCabFile(pSubjectInfo, pdwEncodingType,
532          dwIndex, pcbSignedDataMsg, pbSignedDataMsg);
533     else if (!memcmp(pSubjectInfo->pgSubjectType, &catGUID, sizeof(catGUID)))
534         ret = WINTRUST_GetSignedMsgFromCatFile(pSubjectInfo, pdwEncodingType,
535          dwIndex, pcbSignedDataMsg, pbSignedDataMsg);
536     else
537     {
538         FIXME("unimplemented for subject type %s\n",
539          debugstr_guid(pSubjectInfo->pgSubjectType));
540         ret = FALSE;
541     }
542
543     TRACE("returning %d\n", ret);
544     return ret;
545 }
546
547 /***********************************************************************
548  *      CryptSIPPutSignedDataMsg  (WINTRUST.@)
549  */
550 BOOL WINAPI CryptSIPPutSignedDataMsg(SIP_SUBJECTINFO* pSubjectInfo, DWORD pdwEncodingType,
551                                        DWORD* pdwIndex, DWORD cbSignedDataMsg, BYTE* pbSignedDataMsg)
552 {
553     FIXME("(%p %d %p %d %p) stub\n", pSubjectInfo, pdwEncodingType, pdwIndex,
554           cbSignedDataMsg, pbSignedDataMsg);
555  
556     return FALSE;
557 }
558
559 /***********************************************************************
560  *      CryptSIPRemoveSignedDataMsg  (WINTRUST.@)
561  */
562 BOOL WINAPI CryptSIPRemoveSignedDataMsg(SIP_SUBJECTINFO* pSubjectInfo,
563                                        DWORD dwIndex)
564 {
565     FIXME("(%p %d) stub\n", pSubjectInfo, dwIndex);
566  
567     return FALSE;
568 }
569
570 /***********************************************************************
571  *      CryptSIPVerifyIndirectData  (WINTRUST.@)
572  */
573 BOOL WINAPI CryptSIPVerifyIndirectData(SIP_SUBJECTINFO* pSubjectInfo,
574                                        SIP_INDIRECT_DATA* pIndirectData)
575 {
576     FIXME("(%p %p) stub\n", pSubjectInfo, pIndirectData);
577  
578     return FALSE;
579 }