mergetools/meld: improve compatibiilty with Meld on macOS X
[git] / compat / winansi.c
1 /*
2  * Copyright 2008 Peter Harris <git@peter.is-a-geek.org>
3  */
4
5 #undef NOGDI
6 #include "../git-compat-util.h"
7 #include <wingdi.h>
8 #include <winreg.h>
9 #include "win32.h"
10
11 static int fd_is_interactive[3] = { 0, 0, 0 };
12 #define FD_CONSOLE 0x1
13 #define FD_SWAPPED 0x2
14 #define FD_MSYS    0x4
15
16 /*
17  ANSI codes used by git: m, K
18
19  This file is git-specific. Therefore, this file does not attempt
20  to implement any codes that are not used by git.
21 */
22
23 static HANDLE console;
24 static WORD plain_attr;
25 static WORD attr;
26 static int negative;
27 static int non_ascii_used = 0;
28 static HANDLE hthread, hread, hwrite;
29 static HANDLE hconsole1, hconsole2;
30
31 #ifdef __MINGW32__
32 #if !defined(__MINGW64_VERSION_MAJOR) || __MINGW64_VERSION_MAJOR < 5
33 typedef struct _CONSOLE_FONT_INFOEX {
34         ULONG cbSize;
35         DWORD nFont;
36         COORD dwFontSize;
37         UINT FontFamily;
38         UINT FontWeight;
39         WCHAR FaceName[LF_FACESIZE];
40 } CONSOLE_FONT_INFOEX, *PCONSOLE_FONT_INFOEX;
41 #endif
42 #endif
43
44 typedef BOOL (WINAPI *PGETCURRENTCONSOLEFONTEX)(HANDLE, BOOL,
45                 PCONSOLE_FONT_INFOEX);
46
47 static void warn_if_raster_font(void)
48 {
49         DWORD fontFamily = 0;
50         PGETCURRENTCONSOLEFONTEX pGetCurrentConsoleFontEx;
51
52         /* don't bother if output was ascii only */
53         if (!non_ascii_used)
54                 return;
55
56         /* GetCurrentConsoleFontEx is available since Vista */
57         pGetCurrentConsoleFontEx = (PGETCURRENTCONSOLEFONTEX) GetProcAddress(
58                         GetModuleHandle("kernel32.dll"),
59                         "GetCurrentConsoleFontEx");
60         if (pGetCurrentConsoleFontEx) {
61                 CONSOLE_FONT_INFOEX cfi;
62                 cfi.cbSize = sizeof(cfi);
63                 if (pGetCurrentConsoleFontEx(console, 0, &cfi))
64                         fontFamily = cfi.FontFamily;
65         } else {
66                 /* pre-Vista: check default console font in registry */
67                 HKEY hkey;
68                 if (ERROR_SUCCESS == RegOpenKeyExA(HKEY_CURRENT_USER, "Console",
69                                 0, KEY_READ, &hkey)) {
70                         DWORD size = sizeof(fontFamily);
71                         RegQueryValueExA(hkey, "FontFamily", NULL, NULL,
72                                         (LPVOID) &fontFamily, &size);
73                         RegCloseKey(hkey);
74                 }
75         }
76
77         if (!(fontFamily & TMPF_TRUETYPE)) {
78                 const wchar_t *msg = L"\nWarning: Your console font probably "
79                         L"doesn\'t support Unicode. If you experience strange "
80                         L"characters in the output, consider switching to a "
81                         L"TrueType font such as Consolas!\n";
82                 DWORD dummy;
83                 WriteConsoleW(console, msg, wcslen(msg), &dummy, NULL);
84         }
85 }
86
87 static int is_console(int fd)
88 {
89         CONSOLE_SCREEN_BUFFER_INFO sbi;
90         DWORD mode;
91         HANDLE hcon;
92
93         static int initialized = 0;
94
95         /* get OS handle of the file descriptor */
96         hcon = (HANDLE) _get_osfhandle(fd);
97         if (hcon == INVALID_HANDLE_VALUE)
98                 return 0;
99
100         /* check if its a device (i.e. console, printer, serial port) */
101         if (GetFileType(hcon) != FILE_TYPE_CHAR)
102                 return 0;
103
104         /* check if its a handle to a console output screen buffer */
105         if (!fd) {
106                 if (!GetConsoleMode(hcon, &mode))
107                         return 0;
108                 /*
109                  * This code path is only reached if there is no console
110                  * attached to stdout/stderr, i.e. we will not need to output
111                  * any text to any console, therefore we might just as well
112                  * use black as foreground color.
113                  */
114                 sbi.wAttributes = 0;
115         } else if (!GetConsoleScreenBufferInfo(hcon, &sbi))
116                 return 0;
117
118         if (fd >= 0 && fd <= 2)
119                 fd_is_interactive[fd] |= FD_CONSOLE;
120
121         /* initialize attributes */
122         if (!initialized) {
123                 console = hcon;
124                 attr = plain_attr = sbi.wAttributes;
125                 negative = 0;
126                 initialized = 1;
127         }
128
129         return 1;
130 }
131
132 #define BUFFER_SIZE 4096
133 #define MAX_PARAMS 16
134
135 static void write_console(unsigned char *str, size_t len)
136 {
137         /* only called from console_thread, so a static buffer will do */
138         static wchar_t wbuf[2 * BUFFER_SIZE + 1];
139         DWORD dummy;
140
141         /* convert utf-8 to utf-16 */
142         int wlen = xutftowcsn(wbuf, (char*) str, ARRAY_SIZE(wbuf), len);
143         if (wlen < 0) {
144                 wchar_t *err = L"[invalid]";
145                 WriteConsoleW(console, err, wcslen(err), &dummy, NULL);
146                 return;
147         }
148
149         /* write directly to console */
150         WriteConsoleW(console, wbuf, wlen, &dummy, NULL);
151
152         /* remember if non-ascii characters are printed */
153         if (wlen != len)
154                 non_ascii_used = 1;
155 }
156
157 #define FOREGROUND_ALL (FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE)
158 #define BACKGROUND_ALL (BACKGROUND_RED | BACKGROUND_GREEN | BACKGROUND_BLUE)
159
160 static void set_console_attr(void)
161 {
162         WORD attributes = attr;
163         if (negative) {
164                 attributes &= ~FOREGROUND_ALL;
165                 attributes &= ~BACKGROUND_ALL;
166
167                 /* This could probably use a bitmask
168                    instead of a series of ifs */
169                 if (attr & FOREGROUND_RED)
170                         attributes |= BACKGROUND_RED;
171                 if (attr & FOREGROUND_GREEN)
172                         attributes |= BACKGROUND_GREEN;
173                 if (attr & FOREGROUND_BLUE)
174                         attributes |= BACKGROUND_BLUE;
175
176                 if (attr & BACKGROUND_RED)
177                         attributes |= FOREGROUND_RED;
178                 if (attr & BACKGROUND_GREEN)
179                         attributes |= FOREGROUND_GREEN;
180                 if (attr & BACKGROUND_BLUE)
181                         attributes |= FOREGROUND_BLUE;
182         }
183         SetConsoleTextAttribute(console, attributes);
184 }
185
186 static void erase_in_line(void)
187 {
188         CONSOLE_SCREEN_BUFFER_INFO sbi;
189         DWORD dummy; /* Needed for Windows 7 (or Vista) regression */
190
191         if (!console)
192                 return;
193
194         GetConsoleScreenBufferInfo(console, &sbi);
195         FillConsoleOutputCharacterA(console, ' ',
196                 sbi.dwSize.X - sbi.dwCursorPosition.X, sbi.dwCursorPosition,
197                 &dummy);
198 }
199
200 static void set_attr(char func, const int *params, int paramlen)
201 {
202         int i;
203         switch (func) {
204         case 'm':
205                 for (i = 0; i < paramlen; i++) {
206                         switch (params[i]) {
207                         case 0: /* reset */
208                                 attr = plain_attr;
209                                 negative = 0;
210                                 break;
211                         case 1: /* bold */
212                                 attr |= FOREGROUND_INTENSITY;
213                                 break;
214                         case 2:  /* faint */
215                         case 22: /* normal */
216                                 attr &= ~FOREGROUND_INTENSITY;
217                                 break;
218                         case 3:  /* italic */
219                                 /* Unsupported */
220                                 break;
221                         case 4:  /* underline */
222                         case 21: /* double underline */
223                                 /* Wikipedia says this flag does nothing */
224                                 /* Furthermore, mingw doesn't define this flag
225                                 attr |= COMMON_LVB_UNDERSCORE; */
226                                 break;
227                         case 24: /* no underline */
228                                 /* attr &= ~COMMON_LVB_UNDERSCORE; */
229                                 break;
230                         case 5:  /* slow blink */
231                         case 6:  /* fast blink */
232                                 /* We don't have blink, but we do have
233                                    background intensity */
234                                 attr |= BACKGROUND_INTENSITY;
235                                 break;
236                         case 25: /* no blink */
237                                 attr &= ~BACKGROUND_INTENSITY;
238                                 break;
239                         case 7:  /* negative */
240                                 negative = 1;
241                                 break;
242                         case 27: /* positive */
243                                 negative = 0;
244                                 break;
245                         case 8:  /* conceal */
246                         case 28: /* reveal */
247                                 /* Unsupported */
248                                 break;
249                         case 30: /* Black */
250                                 attr &= ~FOREGROUND_ALL;
251                                 break;
252                         case 31: /* Red */
253                                 attr &= ~FOREGROUND_ALL;
254                                 attr |= FOREGROUND_RED;
255                                 break;
256                         case 32: /* Green */
257                                 attr &= ~FOREGROUND_ALL;
258                                 attr |= FOREGROUND_GREEN;
259                                 break;
260                         case 33: /* Yellow */
261                                 attr &= ~FOREGROUND_ALL;
262                                 attr |= FOREGROUND_RED | FOREGROUND_GREEN;
263                                 break;
264                         case 34: /* Blue */
265                                 attr &= ~FOREGROUND_ALL;
266                                 attr |= FOREGROUND_BLUE;
267                                 break;
268                         case 35: /* Magenta */
269                                 attr &= ~FOREGROUND_ALL;
270                                 attr |= FOREGROUND_RED | FOREGROUND_BLUE;
271                                 break;
272                         case 36: /* Cyan */
273                                 attr &= ~FOREGROUND_ALL;
274                                 attr |= FOREGROUND_GREEN | FOREGROUND_BLUE;
275                                 break;
276                         case 37: /* White */
277                                 attr |= FOREGROUND_RED |
278                                         FOREGROUND_GREEN |
279                                         FOREGROUND_BLUE;
280                                 break;
281                         case 38: /* Unknown */
282                                 break;
283                         case 39: /* reset */
284                                 attr &= ~FOREGROUND_ALL;
285                                 attr |= (plain_attr & FOREGROUND_ALL);
286                                 break;
287                         case 40: /* Black */
288                                 attr &= ~BACKGROUND_ALL;
289                                 break;
290                         case 41: /* Red */
291                                 attr &= ~BACKGROUND_ALL;
292                                 attr |= BACKGROUND_RED;
293                                 break;
294                         case 42: /* Green */
295                                 attr &= ~BACKGROUND_ALL;
296                                 attr |= BACKGROUND_GREEN;
297                                 break;
298                         case 43: /* Yellow */
299                                 attr &= ~BACKGROUND_ALL;
300                                 attr |= BACKGROUND_RED | BACKGROUND_GREEN;
301                                 break;
302                         case 44: /* Blue */
303                                 attr &= ~BACKGROUND_ALL;
304                                 attr |= BACKGROUND_BLUE;
305                                 break;
306                         case 45: /* Magenta */
307                                 attr &= ~BACKGROUND_ALL;
308                                 attr |= BACKGROUND_RED | BACKGROUND_BLUE;
309                                 break;
310                         case 46: /* Cyan */
311                                 attr &= ~BACKGROUND_ALL;
312                                 attr |= BACKGROUND_GREEN | BACKGROUND_BLUE;
313                                 break;
314                         case 47: /* White */
315                                 attr |= BACKGROUND_RED |
316                                         BACKGROUND_GREEN |
317                                         BACKGROUND_BLUE;
318                                 break;
319                         case 48: /* Unknown */
320                                 break;
321                         case 49: /* reset */
322                                 attr &= ~BACKGROUND_ALL;
323                                 attr |= (plain_attr & BACKGROUND_ALL);
324                                 break;
325                         default:
326                                 /* Unsupported code */
327                                 break;
328                         }
329                 }
330                 set_console_attr();
331                 break;
332         case 'K':
333                 erase_in_line();
334                 break;
335         default:
336                 /* Unsupported code */
337                 break;
338         }
339 }
340
341 enum {
342         TEXT = 0, ESCAPE = 033, BRACKET = '['
343 };
344
345 static DWORD WINAPI console_thread(LPVOID unused)
346 {
347         unsigned char buffer[BUFFER_SIZE];
348         DWORD bytes;
349         int start, end = 0, c, parampos = 0, state = TEXT;
350         int params[MAX_PARAMS];
351
352         while (1) {
353                 /* read next chunk of bytes from the pipe */
354                 if (!ReadFile(hread, buffer + end, BUFFER_SIZE - end, &bytes,
355                                 NULL)) {
356                         /* exit if pipe has been closed or disconnected */
357                         if (GetLastError() == ERROR_PIPE_NOT_CONNECTED ||
358                                         GetLastError() == ERROR_BROKEN_PIPE)
359                                 break;
360                         /* ignore other errors */
361                         continue;
362                 }
363
364                 /* scan the bytes and handle ANSI control codes */
365                 bytes += end;
366                 start = end = 0;
367                 while (end < bytes) {
368                         c = buffer[end++];
369                         switch (state) {
370                         case TEXT:
371                                 if (c == ESCAPE) {
372                                         /* print text seen so far */
373                                         if (end - 1 > start)
374                                                 write_console(buffer + start,
375                                                         end - 1 - start);
376
377                                         /* then start parsing escape sequence */
378                                         start = end - 1;
379                                         memset(params, 0, sizeof(params));
380                                         parampos = 0;
381                                         state = ESCAPE;
382                                 }
383                                 break;
384
385                         case ESCAPE:
386                                 /* continue if "\033[", otherwise bail out */
387                                 state = (c == BRACKET) ? BRACKET : TEXT;
388                                 break;
389
390                         case BRACKET:
391                                 /* parse [0-9;]* into array of parameters */
392                                 if (c >= '0' && c <= '9') {
393                                         params[parampos] *= 10;
394                                         params[parampos] += c - '0';
395                                 } else if (c == ';') {
396                                         /*
397                                          * next parameter, bail out if out of
398                                          * bounds
399                                          */
400                                         parampos++;
401                                         if (parampos >= MAX_PARAMS)
402                                                 state = TEXT;
403                                 } else {
404                                         /*
405                                          * end of escape sequence, change
406                                          * console attributes
407                                          */
408                                         set_attr(c, params, parampos + 1);
409                                         start = end;
410                                         state = TEXT;
411                                 }
412                                 break;
413                         }
414                 }
415
416                 /* print remaining text unless parsing an escape sequence */
417                 if (state == TEXT && end > start) {
418                         /* check for incomplete UTF-8 sequences and fix end */
419                         if (buffer[end - 1] >= 0x80) {
420                                 if (buffer[end -1] >= 0xc0)
421                                         end--;
422                                 else if (end - 1 > start &&
423                                                 buffer[end - 2] >= 0xe0)
424                                         end -= 2;
425                                 else if (end - 2 > start &&
426                                                 buffer[end - 3] >= 0xf0)
427                                         end -= 3;
428                         }
429
430                         /* print remaining complete UTF-8 sequences */
431                         if (end > start)
432                                 write_console(buffer + start, end - start);
433
434                         /* move remaining bytes to the front */
435                         if (end < bytes)
436                                 memmove(buffer, buffer + end, bytes - end);
437                         end = bytes - end;
438                 } else {
439                         /* all data has been consumed, mark buffer empty */
440                         end = 0;
441                 }
442         }
443
444         /* check if the console font supports unicode */
445         warn_if_raster_font();
446
447         CloseHandle(hread);
448         return 0;
449 }
450
451 static void winansi_exit(void)
452 {
453         /* flush all streams */
454         _flushall();
455
456         /* signal console thread to exit */
457         FlushFileBuffers(hwrite);
458         DisconnectNamedPipe(hwrite);
459
460         /* wait for console thread to copy remaining data */
461         WaitForSingleObject(hthread, INFINITE);
462
463         /* cleanup handles... */
464         CloseHandle(hwrite);
465         CloseHandle(hthread);
466 }
467
468 static void die_lasterr(const char *fmt, ...)
469 {
470         va_list params;
471         va_start(params, fmt);
472         errno = err_win_to_posix(GetLastError());
473         die_errno(fmt, params);
474         va_end(params);
475 }
476
477 static HANDLE duplicate_handle(HANDLE hnd)
478 {
479         HANDLE hresult, hproc = GetCurrentProcess();
480         if (!DuplicateHandle(hproc, hnd, hproc, &hresult, 0, TRUE,
481                         DUPLICATE_SAME_ACCESS))
482                 die_lasterr("DuplicateHandle(%li) failed",
483                         (long) (intptr_t) hnd);
484         return hresult;
485 }
486
487 static HANDLE swap_osfhnd(int fd, HANDLE new_handle)
488 {
489         /*
490          * Create a copy of the original handle associated with fd
491          * because the original will get closed when we dup2().
492          */
493         HANDLE handle = (HANDLE)_get_osfhandle(fd);
494         HANDLE duplicate = duplicate_handle(handle);
495
496         /* Create a temp fd associated with the already open "new_handle". */
497         int new_fd = _open_osfhandle((intptr_t)new_handle, O_BINARY);
498
499         assert((fd == 1) || (fd == 2));
500
501         /*
502          * Use stock dup2() to re-bind fd to the new handle.  Note that
503          * this will implicitly close(1) and close both fd=1 and the
504          * originally associated handle.  It will open a new fd=1 and
505          * call DuplicateHandle() on the handle associated with new_fd.
506          * It is because of this implicit close() that we created the
507          * copy of the original.
508          *
509          * Note that we need to update the cached console handle to the
510          * duplicated one because the dup2() call will implicitly close
511          * the original one.
512          *
513          * Note that dup2() when given target := {0,1,2} will also
514          * call SetStdHandle(), so we don't need to worry about that.
515          */
516         if (console == handle)
517                 console = duplicate;
518         dup2(new_fd, fd);
519
520         /* Close the temp fd.  This explicitly closes "new_handle"
521          * (because it has been associated with it).
522          */
523         close(new_fd);
524
525         if (fd == 2)
526                 setvbuf(stderr, NULL, _IONBF, BUFSIZ);
527         fd_is_interactive[fd] |= FD_SWAPPED;
528
529         return duplicate;
530 }
531
532 #ifdef DETECT_MSYS_TTY
533
534 #include <winternl.h>
535 #include <ntstatus.h>
536
537 static void detect_msys_tty(int fd)
538 {
539         ULONG result;
540         BYTE buffer[1024];
541         POBJECT_NAME_INFORMATION nameinfo = (POBJECT_NAME_INFORMATION) buffer;
542         PWSTR name;
543
544         /* check if fd is a pipe */
545         HANDLE h = (HANDLE) _get_osfhandle(fd);
546         if (GetFileType(h) != FILE_TYPE_PIPE)
547                 return;
548
549         /* get pipe name */
550         if (!NT_SUCCESS(NtQueryObject(h, ObjectNameInformation,
551                         buffer, sizeof(buffer) - 2, &result)))
552                 return;
553         name = nameinfo->Name.Buffer;
554         name[nameinfo->Name.Length / sizeof(*name)] = 0;
555
556         /*
557          * Check if this could be a MSYS2 pty pipe ('msys-XXXX-ptyN-XX')
558          * or a cygwin pty pipe ('cygwin-XXXX-ptyN-XX')
559          */
560         if ((!wcsstr(name, L"msys-") && !wcsstr(name, L"cygwin-")) ||
561                         !wcsstr(name, L"-pty"))
562                 return;
563
564         if (fd == 2)
565                 setvbuf(stderr, NULL, _IONBF, BUFSIZ);
566         fd_is_interactive[fd] |= FD_MSYS;
567 }
568
569 #endif
570
571 /*
572  * Wrapper for isatty().  Most calls in the main git code
573  * call isatty(1 or 2) to see if the instance is interactive
574  * and should: be colored, show progress, paginate output.
575  * We lie and give results for what the descriptor WAS at
576  * startup (and ignore any pipe redirection we internally
577  * do).
578  */
579 #undef isatty
580 int winansi_isatty(int fd)
581 {
582         if (fd >= 0 && fd <= 2)
583                 return fd_is_interactive[fd] != 0;
584         return isatty(fd);
585 }
586
587 void winansi_init(void)
588 {
589         int con1, con2;
590         char name[32];
591
592         /* check if either stdout or stderr is a console output screen buffer */
593         con1 = is_console(1);
594         con2 = is_console(2);
595
596         /* Also compute console bit for fd 0 even though we don't need the result here. */
597         is_console(0);
598
599         if (!con1 && !con2) {
600 #ifdef DETECT_MSYS_TTY
601                 /* check if stdin / stdout / stderr are MSYS2 pty pipes */
602                 detect_msys_tty(0);
603                 detect_msys_tty(1);
604                 detect_msys_tty(2);
605 #endif
606                 return;
607         }
608
609         /* create a named pipe to communicate with the console thread */
610         xsnprintf(name, sizeof(name), "\\\\.\\pipe\\winansi%lu", GetCurrentProcessId());
611         hwrite = CreateNamedPipe(name, PIPE_ACCESS_OUTBOUND,
612                 PIPE_TYPE_BYTE | PIPE_WAIT, 1, BUFFER_SIZE, 0, 0, NULL);
613         if (hwrite == INVALID_HANDLE_VALUE)
614                 die_lasterr("CreateNamedPipe failed");
615
616         hread = CreateFile(name, GENERIC_READ, 0, NULL, OPEN_EXISTING, 0, NULL);
617         if (hread == INVALID_HANDLE_VALUE)
618                 die_lasterr("CreateFile for named pipe failed");
619
620         /* start console spool thread on the pipe's read end */
621         hthread = CreateThread(NULL, 0, console_thread, NULL, 0, NULL);
622         if (hthread == INVALID_HANDLE_VALUE)
623                 die_lasterr("CreateThread(console_thread) failed");
624
625         /* schedule cleanup routine */
626         if (atexit(winansi_exit))
627                 die_errno("atexit(winansi_exit) failed");
628
629         /* redirect stdout / stderr to the pipe */
630         if (con1)
631                 hconsole1 = swap_osfhnd(1, duplicate_handle(hwrite));
632         if (con2)
633                 hconsole2 = swap_osfhnd(2, duplicate_handle(hwrite));
634 }
635
636 /*
637  * Returns the real console handle if stdout / stderr is a pipe redirecting
638  * to the console. Allows spawn / exec to pass the console to the next process.
639  */
640 HANDLE winansi_get_osfhandle(int fd)
641 {
642         if (fd == 1 && (fd_is_interactive[1] & FD_SWAPPED))
643                 return hconsole1;
644         if (fd == 2 && (fd_is_interactive[2] & FD_SWAPPED))
645                 return hconsole2;
646
647         return (HANDLE)_get_osfhandle(fd);
648 }