Merge branch 'jk/misc-fixes-maint'
[git] / compat / winansi.c
1 /*
2  * Copyright 2008 Peter Harris <git@peter.is-a-geek.org>
3  */
4
5 #undef NOGDI
6 #include "../git-compat-util.h"
7 #include <wingdi.h>
8 #include <winreg.h>
9
10 /*
11  ANSI codes used by git: m, K
12
13  This file is git-specific. Therefore, this file does not attempt
14  to implement any codes that are not used by git.
15 */
16
17 static HANDLE console;
18 static WORD plain_attr;
19 static WORD attr;
20 static int negative;
21 static int non_ascii_used = 0;
22 static HANDLE hthread, hread, hwrite;
23 static HANDLE hconsole1, hconsole2;
24
25 #ifdef __MINGW32__
26 typedef struct _CONSOLE_FONT_INFOEX {
27         ULONG cbSize;
28         DWORD nFont;
29         COORD dwFontSize;
30         UINT FontFamily;
31         UINT FontWeight;
32         WCHAR FaceName[LF_FACESIZE];
33 } CONSOLE_FONT_INFOEX, *PCONSOLE_FONT_INFOEX;
34 #endif
35
36 typedef BOOL (WINAPI *PGETCURRENTCONSOLEFONTEX)(HANDLE, BOOL,
37                 PCONSOLE_FONT_INFOEX);
38
39 static void warn_if_raster_font(void)
40 {
41         DWORD fontFamily = 0;
42         PGETCURRENTCONSOLEFONTEX pGetCurrentConsoleFontEx;
43
44         /* don't bother if output was ascii only */
45         if (!non_ascii_used)
46                 return;
47
48         /* GetCurrentConsoleFontEx is available since Vista */
49         pGetCurrentConsoleFontEx = (PGETCURRENTCONSOLEFONTEX) GetProcAddress(
50                         GetModuleHandle("kernel32.dll"),
51                         "GetCurrentConsoleFontEx");
52         if (pGetCurrentConsoleFontEx) {
53                 CONSOLE_FONT_INFOEX cfi;
54                 cfi.cbSize = sizeof(cfi);
55                 if (pGetCurrentConsoleFontEx(console, 0, &cfi))
56                         fontFamily = cfi.FontFamily;
57         } else {
58                 /* pre-Vista: check default console font in registry */
59                 HKEY hkey;
60                 if (ERROR_SUCCESS == RegOpenKeyExA(HKEY_CURRENT_USER, "Console",
61                                 0, KEY_READ, &hkey)) {
62                         DWORD size = sizeof(fontFamily);
63                         RegQueryValueExA(hkey, "FontFamily", NULL, NULL,
64                                         (LPVOID) &fontFamily, &size);
65                         RegCloseKey(hkey);
66                 }
67         }
68
69         if (!(fontFamily & TMPF_TRUETYPE)) {
70                 const wchar_t *msg = L"\nWarning: Your console font probably "
71                         L"doesn\'t support Unicode. If you experience strange "
72                         L"characters in the output, consider switching to a "
73                         L"TrueType font such as Consolas!\n";
74                 DWORD dummy;
75                 WriteConsoleW(console, msg, wcslen(msg), &dummy, NULL);
76         }
77 }
78
79 static int is_console(int fd)
80 {
81         CONSOLE_SCREEN_BUFFER_INFO sbi;
82         HANDLE hcon;
83
84         static int initialized = 0;
85
86         /* get OS handle of the file descriptor */
87         hcon = (HANDLE) _get_osfhandle(fd);
88         if (hcon == INVALID_HANDLE_VALUE)
89                 return 0;
90
91         /* check if its a device (i.e. console, printer, serial port) */
92         if (GetFileType(hcon) != FILE_TYPE_CHAR)
93                 return 0;
94
95         /* check if its a handle to a console output screen buffer */
96         if (!GetConsoleScreenBufferInfo(hcon, &sbi))
97                 return 0;
98
99         /* initialize attributes */
100         if (!initialized) {
101                 console = hcon;
102                 attr = plain_attr = sbi.wAttributes;
103                 negative = 0;
104                 initialized = 1;
105         }
106
107         return 1;
108 }
109
110 #define BUFFER_SIZE 4096
111 #define MAX_PARAMS 16
112
113 static void write_console(unsigned char *str, size_t len)
114 {
115         /* only called from console_thread, so a static buffer will do */
116         static wchar_t wbuf[2 * BUFFER_SIZE + 1];
117         DWORD dummy;
118
119         /* convert utf-8 to utf-16 */
120         int wlen = xutftowcsn(wbuf, (char*) str, ARRAY_SIZE(wbuf), len);
121
122         /* write directly to console */
123         WriteConsoleW(console, wbuf, wlen, &dummy, NULL);
124
125         /* remember if non-ascii characters are printed */
126         if (wlen != len)
127                 non_ascii_used = 1;
128 }
129
130 #define FOREGROUND_ALL (FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE)
131 #define BACKGROUND_ALL (BACKGROUND_RED | BACKGROUND_GREEN | BACKGROUND_BLUE)
132
133 static void set_console_attr(void)
134 {
135         WORD attributes = attr;
136         if (negative) {
137                 attributes &= ~FOREGROUND_ALL;
138                 attributes &= ~BACKGROUND_ALL;
139
140                 /* This could probably use a bitmask
141                    instead of a series of ifs */
142                 if (attr & FOREGROUND_RED)
143                         attributes |= BACKGROUND_RED;
144                 if (attr & FOREGROUND_GREEN)
145                         attributes |= BACKGROUND_GREEN;
146                 if (attr & FOREGROUND_BLUE)
147                         attributes |= BACKGROUND_BLUE;
148
149                 if (attr & BACKGROUND_RED)
150                         attributes |= FOREGROUND_RED;
151                 if (attr & BACKGROUND_GREEN)
152                         attributes |= FOREGROUND_GREEN;
153                 if (attr & BACKGROUND_BLUE)
154                         attributes |= FOREGROUND_BLUE;
155         }
156         SetConsoleTextAttribute(console, attributes);
157 }
158
159 static void erase_in_line(void)
160 {
161         CONSOLE_SCREEN_BUFFER_INFO sbi;
162         DWORD dummy; /* Needed for Windows 7 (or Vista) regression */
163
164         if (!console)
165                 return;
166
167         GetConsoleScreenBufferInfo(console, &sbi);
168         FillConsoleOutputCharacterA(console, ' ',
169                 sbi.dwSize.X - sbi.dwCursorPosition.X, sbi.dwCursorPosition,
170                 &dummy);
171 }
172
173 static void set_attr(char func, const int *params, int paramlen)
174 {
175         int i;
176         switch (func) {
177         case 'm':
178                 for (i = 0; i < paramlen; i++) {
179                         switch (params[i]) {
180                         case 0: /* reset */
181                                 attr = plain_attr;
182                                 negative = 0;
183                                 break;
184                         case 1: /* bold */
185                                 attr |= FOREGROUND_INTENSITY;
186                                 break;
187                         case 2:  /* faint */
188                         case 22: /* normal */
189                                 attr &= ~FOREGROUND_INTENSITY;
190                                 break;
191                         case 3:  /* italic */
192                                 /* Unsupported */
193                                 break;
194                         case 4:  /* underline */
195                         case 21: /* double underline */
196                                 /* Wikipedia says this flag does nothing */
197                                 /* Furthermore, mingw doesn't define this flag
198                                 attr |= COMMON_LVB_UNDERSCORE; */
199                                 break;
200                         case 24: /* no underline */
201                                 /* attr &= ~COMMON_LVB_UNDERSCORE; */
202                                 break;
203                         case 5:  /* slow blink */
204                         case 6:  /* fast blink */
205                                 /* We don't have blink, but we do have
206                                    background intensity */
207                                 attr |= BACKGROUND_INTENSITY;
208                                 break;
209                         case 25: /* no blink */
210                                 attr &= ~BACKGROUND_INTENSITY;
211                                 break;
212                         case 7:  /* negative */
213                                 negative = 1;
214                                 break;
215                         case 27: /* positive */
216                                 negative = 0;
217                                 break;
218                         case 8:  /* conceal */
219                         case 28: /* reveal */
220                                 /* Unsupported */
221                                 break;
222                         case 30: /* Black */
223                                 attr &= ~FOREGROUND_ALL;
224                                 break;
225                         case 31: /* Red */
226                                 attr &= ~FOREGROUND_ALL;
227                                 attr |= FOREGROUND_RED;
228                                 break;
229                         case 32: /* Green */
230                                 attr &= ~FOREGROUND_ALL;
231                                 attr |= FOREGROUND_GREEN;
232                                 break;
233                         case 33: /* Yellow */
234                                 attr &= ~FOREGROUND_ALL;
235                                 attr |= FOREGROUND_RED | FOREGROUND_GREEN;
236                                 break;
237                         case 34: /* Blue */
238                                 attr &= ~FOREGROUND_ALL;
239                                 attr |= FOREGROUND_BLUE;
240                                 break;
241                         case 35: /* Magenta */
242                                 attr &= ~FOREGROUND_ALL;
243                                 attr |= FOREGROUND_RED | FOREGROUND_BLUE;
244                                 break;
245                         case 36: /* Cyan */
246                                 attr &= ~FOREGROUND_ALL;
247                                 attr |= FOREGROUND_GREEN | FOREGROUND_BLUE;
248                                 break;
249                         case 37: /* White */
250                                 attr |= FOREGROUND_RED |
251                                         FOREGROUND_GREEN |
252                                         FOREGROUND_BLUE;
253                                 break;
254                         case 38: /* Unknown */
255                                 break;
256                         case 39: /* reset */
257                                 attr &= ~FOREGROUND_ALL;
258                                 attr |= (plain_attr & FOREGROUND_ALL);
259                                 break;
260                         case 40: /* Black */
261                                 attr &= ~BACKGROUND_ALL;
262                                 break;
263                         case 41: /* Red */
264                                 attr &= ~BACKGROUND_ALL;
265                                 attr |= BACKGROUND_RED;
266                                 break;
267                         case 42: /* Green */
268                                 attr &= ~BACKGROUND_ALL;
269                                 attr |= BACKGROUND_GREEN;
270                                 break;
271                         case 43: /* Yellow */
272                                 attr &= ~BACKGROUND_ALL;
273                                 attr |= BACKGROUND_RED | BACKGROUND_GREEN;
274                                 break;
275                         case 44: /* Blue */
276                                 attr &= ~BACKGROUND_ALL;
277                                 attr |= BACKGROUND_BLUE;
278                                 break;
279                         case 45: /* Magenta */
280                                 attr &= ~BACKGROUND_ALL;
281                                 attr |= BACKGROUND_RED | BACKGROUND_BLUE;
282                                 break;
283                         case 46: /* Cyan */
284                                 attr &= ~BACKGROUND_ALL;
285                                 attr |= BACKGROUND_GREEN | BACKGROUND_BLUE;
286                                 break;
287                         case 47: /* White */
288                                 attr |= BACKGROUND_RED |
289                                         BACKGROUND_GREEN |
290                                         BACKGROUND_BLUE;
291                                 break;
292                         case 48: /* Unknown */
293                                 break;
294                         case 49: /* reset */
295                                 attr &= ~BACKGROUND_ALL;
296                                 attr |= (plain_attr & BACKGROUND_ALL);
297                                 break;
298                         default:
299                                 /* Unsupported code */
300                                 break;
301                         }
302                 }
303                 set_console_attr();
304                 break;
305         case 'K':
306                 erase_in_line();
307                 break;
308         default:
309                 /* Unsupported code */
310                 break;
311         }
312 }
313
314 enum {
315         TEXT = 0, ESCAPE = 033, BRACKET = '['
316 };
317
318 static DWORD WINAPI console_thread(LPVOID unused)
319 {
320         unsigned char buffer[BUFFER_SIZE];
321         DWORD bytes;
322         int start, end = 0, c, parampos = 0, state = TEXT;
323         int params[MAX_PARAMS];
324
325         while (1) {
326                 /* read next chunk of bytes from the pipe */
327                 if (!ReadFile(hread, buffer + end, BUFFER_SIZE - end, &bytes,
328                                 NULL)) {
329                         /* exit if pipe has been closed or disconnected */
330                         if (GetLastError() == ERROR_PIPE_NOT_CONNECTED ||
331                                         GetLastError() == ERROR_BROKEN_PIPE)
332                                 break;
333                         /* ignore other errors */
334                         continue;
335                 }
336
337                 /* scan the bytes and handle ANSI control codes */
338                 bytes += end;
339                 start = end = 0;
340                 while (end < bytes) {
341                         c = buffer[end++];
342                         switch (state) {
343                         case TEXT:
344                                 if (c == ESCAPE) {
345                                         /* print text seen so far */
346                                         if (end - 1 > start)
347                                                 write_console(buffer + start,
348                                                         end - 1 - start);
349
350                                         /* then start parsing escape sequence */
351                                         start = end - 1;
352                                         memset(params, 0, sizeof(params));
353                                         parampos = 0;
354                                         state = ESCAPE;
355                                 }
356                                 break;
357
358                         case ESCAPE:
359                                 /* continue if "\033[", otherwise bail out */
360                                 state = (c == BRACKET) ? BRACKET : TEXT;
361                                 break;
362
363                         case BRACKET:
364                                 /* parse [0-9;]* into array of parameters */
365                                 if (c >= '0' && c <= '9') {
366                                         params[parampos] *= 10;
367                                         params[parampos] += c - '0';
368                                 } else if (c == ';') {
369                                         /*
370                                          * next parameter, bail out if out of
371                                          * bounds
372                                          */
373                                         parampos++;
374                                         if (parampos >= MAX_PARAMS)
375                                                 state = TEXT;
376                                 } else {
377                                         /*
378                                          * end of escape sequence, change
379                                          * console attributes
380                                          */
381                                         set_attr(c, params, parampos + 1);
382                                         start = end;
383                                         state = TEXT;
384                                 }
385                                 break;
386                         }
387                 }
388
389                 /* print remaining text unless parsing an escape sequence */
390                 if (state == TEXT && end > start) {
391                         /* check for incomplete UTF-8 sequences and fix end */
392                         if (buffer[end - 1] >= 0x80) {
393                                 if (buffer[end -1] >= 0xc0)
394                                         end--;
395                                 else if (end - 1 > start &&
396                                                 buffer[end - 2] >= 0xe0)
397                                         end -= 2;
398                                 else if (end - 2 > start &&
399                                                 buffer[end - 3] >= 0xf0)
400                                         end -= 3;
401                         }
402
403                         /* print remaining complete UTF-8 sequences */
404                         if (end > start)
405                                 write_console(buffer + start, end - start);
406
407                         /* move remaining bytes to the front */
408                         if (end < bytes)
409                                 memmove(buffer, buffer + end, bytes - end);
410                         end = bytes - end;
411                 } else {
412                         /* all data has been consumed, mark buffer empty */
413                         end = 0;
414                 }
415         }
416
417         /* check if the console font supports unicode */
418         warn_if_raster_font();
419
420         CloseHandle(hread);
421         return 0;
422 }
423
424 static void winansi_exit(void)
425 {
426         /* flush all streams */
427         _flushall();
428
429         /* signal console thread to exit */
430         FlushFileBuffers(hwrite);
431         DisconnectNamedPipe(hwrite);
432
433         /* wait for console thread to copy remaining data */
434         WaitForSingleObject(hthread, INFINITE);
435
436         /* cleanup handles... */
437         CloseHandle(hwrite);
438         CloseHandle(hthread);
439 }
440
441 static void die_lasterr(const char *fmt, ...)
442 {
443         va_list params;
444         va_start(params, fmt);
445         errno = err_win_to_posix(GetLastError());
446         die_errno(fmt, params);
447         va_end(params);
448 }
449
450 static HANDLE duplicate_handle(HANDLE hnd)
451 {
452         HANDLE hresult, hproc = GetCurrentProcess();
453         if (!DuplicateHandle(hproc, hnd, hproc, &hresult, 0, TRUE,
454                         DUPLICATE_SAME_ACCESS))
455                 die_lasterr("DuplicateHandle(%li) failed", (long) hnd);
456         return hresult;
457 }
458
459
460 /*
461  * Make MSVCRT's internal file descriptor control structure accessible
462  * so that we can tweak OS handles and flags directly (we need MSVCRT
463  * to treat our pipe handle as if it were a console).
464  *
465  * We assume that the ioinfo structure (exposed by MSVCRT.dll via
466  * __pioinfo) starts with the OS handle and the flags. The exact size
467  * varies between MSVCRT versions, so we try different sizes until
468  * toggling the FDEV bit of _pioinfo(1)->osflags is reflected in
469  * isatty(1).
470  */
471 typedef struct {
472         HANDLE osfhnd;
473         char osflags;
474 } ioinfo;
475
476 extern __declspec(dllimport) ioinfo *__pioinfo[];
477
478 static size_t sizeof_ioinfo = 0;
479
480 #define IOINFO_L2E 5
481 #define IOINFO_ARRAY_ELTS (1 << IOINFO_L2E)
482
483 #define FDEV  0x40
484
485 static inline ioinfo* _pioinfo(int fd)
486 {
487         return (ioinfo*)((char*)__pioinfo[fd >> IOINFO_L2E] +
488                         (fd & (IOINFO_ARRAY_ELTS - 1)) * sizeof_ioinfo);
489 }
490
491 static int init_sizeof_ioinfo()
492 {
493         int istty, wastty;
494         /* don't init twice */
495         if (sizeof_ioinfo)
496                 return sizeof_ioinfo >= 256;
497
498         sizeof_ioinfo = sizeof(ioinfo);
499         wastty = isatty(1);
500         while (sizeof_ioinfo < 256) {
501                 /* toggle FDEV flag, check isatty, then toggle back */
502                 _pioinfo(1)->osflags ^= FDEV;
503                 istty = isatty(1);
504                 _pioinfo(1)->osflags ^= FDEV;
505                 /* return if we found the correct size */
506                 if (istty != wastty)
507                         return 0;
508                 sizeof_ioinfo += sizeof(void*);
509         }
510         error("Tweaking file descriptors doesn't work with this MSVCRT.dll");
511         return 1;
512 }
513
514 static HANDLE swap_osfhnd(int fd, HANDLE new_handle)
515 {
516         ioinfo *pioinfo;
517         HANDLE old_handle;
518
519         /* init ioinfo size if we haven't done so */
520         if (init_sizeof_ioinfo())
521                 return INVALID_HANDLE_VALUE;
522
523         /* get ioinfo pointer and change the handles */
524         pioinfo = _pioinfo(fd);
525         old_handle = pioinfo->osfhnd;
526         pioinfo->osfhnd = new_handle;
527         return old_handle;
528 }
529
530 void winansi_init(void)
531 {
532         int con1, con2;
533         char name[32];
534
535         /* check if either stdout or stderr is a console output screen buffer */
536         con1 = is_console(1);
537         con2 = is_console(2);
538         if (!con1 && !con2)
539                 return;
540
541         /* create a named pipe to communicate with the console thread */
542         sprintf(name, "\\\\.\\pipe\\winansi%lu", GetCurrentProcessId());
543         hwrite = CreateNamedPipe(name, PIPE_ACCESS_OUTBOUND,
544                 PIPE_TYPE_BYTE | PIPE_WAIT, 1, BUFFER_SIZE, 0, 0, NULL);
545         if (hwrite == INVALID_HANDLE_VALUE)
546                 die_lasterr("CreateNamedPipe failed");
547
548         hread = CreateFile(name, GENERIC_READ, 0, NULL, OPEN_EXISTING, 0, NULL);
549         if (hread == INVALID_HANDLE_VALUE)
550                 die_lasterr("CreateFile for named pipe failed");
551
552         /* start console spool thread on the pipe's read end */
553         hthread = CreateThread(NULL, 0, console_thread, NULL, 0, NULL);
554         if (hthread == INVALID_HANDLE_VALUE)
555                 die_lasterr("CreateThread(console_thread) failed");
556
557         /* schedule cleanup routine */
558         if (atexit(winansi_exit))
559                 die_errno("atexit(winansi_exit) failed");
560
561         /* redirect stdout / stderr to the pipe */
562         if (con1)
563                 hconsole1 = swap_osfhnd(1, duplicate_handle(hwrite));
564         if (con2)
565                 hconsole2 = swap_osfhnd(2, duplicate_handle(hwrite));
566 }
567
568 /*
569  * Returns the real console handle if stdout / stderr is a pipe redirecting
570  * to the console. Allows spawn / exec to pass the console to the next process.
571  */
572 HANDLE winansi_get_osfhandle(int fd)
573 {
574         HANDLE hnd = (HANDLE) _get_osfhandle(fd);
575         if ((fd == 1 || fd == 2) && isatty(fd)
576             && GetFileType(hnd) == FILE_TYPE_PIPE)
577                 return (fd == 1) ? hconsole1 : hconsole2;
578         return hnd;
579 }