git.oblomov.eu Git - git/blob - fsmonitor.c

   1 #include "cache.h"
   2 #include "config.h"
   3 #include "dir.h"
   4 #include "ewah/ewok.h"
   5 #include "fsmonitor.h"
   6 #include "run-command.h"
   7 #include "strbuf.h"
   8
   9 #define INDEX_EXTENSION_VERSION1        (1)
  10 #define INDEX_EXTENSION_VERSION2        (2)
  11 #define HOOK_INTERFACE_VERSION1         (1)
  12 #define HOOK_INTERFACE_VERSION2         (2)
  13
  14 struct trace_key trace_fsmonitor = TRACE_KEY_INIT(FSMONITOR);
  15
  16 static void assert_index_minimum(struct index_state *istate, size_t pos)
  17 {
  18         if (pos > istate->cache_nr)
  19                 BUG("fsmonitor_dirty has more entries than the index (%"PRIuMAX" > %u)",
  20                     (uintmax_t)pos, istate->cache_nr);
  21 }
  22
  23 static void fsmonitor_ewah_callback(size_t pos, void *is)
  24 {
  25         struct index_state *istate = (struct index_state *)is;
  26         struct cache_entry *ce;
  27
  28         assert_index_minimum(istate, pos + 1);
  29
  30         ce = istate->cache[pos];
  31         ce->ce_flags &= ~CE_FSMONITOR_VALID;
  32 }
  33
  34 static int fsmonitor_hook_version(void)
  35 {
  36         int hook_version;
  37
  38         if (git_config_get_int("core.fsmonitorhookversion", &hook_version))
  39                 return -1;
  40
  41         if (hook_version == HOOK_INTERFACE_VERSION1 ||
  42             hook_version == HOOK_INTERFACE_VERSION2)
  43                 return hook_version;
  44
  45         warning("Invalid hook version '%i' in core.fsmonitorhookversion. "
  46                 "Must be 1 or 2.", hook_version);
  47         return -1;
  48 }
  49
  50 int read_fsmonitor_extension(struct index_state *istate, const void *data,
  51         unsigned long sz)
  52 {
  53         const char *index = data;
  54         uint32_t hdr_version;
  55         uint32_t ewah_size;
  56         struct ewah_bitmap *fsmonitor_dirty;
  57         int ret;
  58         uint64_t timestamp;
  59         struct strbuf last_update = STRBUF_INIT;
  60
  61         if (sz < sizeof(uint32_t) + 1 + sizeof(uint32_t))
  62                 return error("corrupt fsmonitor extension (too short)");
  63
  64         hdr_version = get_be32(index);
  65         index += sizeof(uint32_t);
  66         if (hdr_version == INDEX_EXTENSION_VERSION1) {
  67                 timestamp = get_be64(index);
  68                 strbuf_addf(&last_update, "%"PRIu64"", timestamp);
  69                 index += sizeof(uint64_t);
  70         } else if (hdr_version == INDEX_EXTENSION_VERSION2) {
  71                 strbuf_addstr(&last_update, index);
  72                 index += last_update.len + 1;
  73         } else {
  74                 return error("bad fsmonitor version %d", hdr_version);
  75         }
  76
  77         istate->fsmonitor_last_update = strbuf_detach(&last_update, NULL);
  78
  79         ewah_size = get_be32(index);
  80         index += sizeof(uint32_t);
  81
  82         fsmonitor_dirty = ewah_new();
  83         ret = ewah_read_mmap(fsmonitor_dirty, index, ewah_size);
  84         if (ret != ewah_size) {
  85                 ewah_free(fsmonitor_dirty);
  86                 return error("failed to parse ewah bitmap reading fsmonitor index extension");
  87         }
  88         istate->fsmonitor_dirty = fsmonitor_dirty;
  89
  90         if (!istate->split_index)
  91                 assert_index_minimum(istate, istate->fsmonitor_dirty->bit_size);
  92
  93         trace2_data_string("index", NULL, "extension/fsmn/read/token",
  94                            istate->fsmonitor_last_update);
  95         trace_printf_key(&trace_fsmonitor,
  96                          "read fsmonitor extension successful '%s'",
  97                          istate->fsmonitor_last_update);
  98         return 0;
  99 }
 100
 101 void fill_fsmonitor_bitmap(struct index_state *istate)
 102 {
 103         unsigned int i, skipped = 0;
 104         istate->fsmonitor_dirty = ewah_new();
 105         for (i = 0; i < istate->cache_nr; i++) {
 106                 if (istate->cache[i]->ce_flags & CE_REMOVE)
 107                         skipped++;
 108                 else if (!(istate->cache[i]->ce_flags & CE_FSMONITOR_VALID))
 109                         ewah_set(istate->fsmonitor_dirty, i - skipped);
 110         }
 111 }
 112
 113 void write_fsmonitor_extension(struct strbuf *sb, struct index_state *istate)
 114 {
 115         uint32_t hdr_version;
 116         uint32_t ewah_start;
 117         uint32_t ewah_size = 0;
 118         int fixup = 0;
 119
 120         if (!istate->split_index)
 121                 assert_index_minimum(istate, istate->fsmonitor_dirty->bit_size);
 122
 123         put_be32(&hdr_version, INDEX_EXTENSION_VERSION2);
 124         strbuf_add(sb, &hdr_version, sizeof(uint32_t));
 125
 126         strbuf_addstr(sb, istate->fsmonitor_last_update);
 127         strbuf_addch(sb, 0); /* Want to keep a NUL */
 128
 129         fixup = sb->len;
 130         strbuf_add(sb, &ewah_size, sizeof(uint32_t)); /* we'll fix this up later */
 131
 132         ewah_start = sb->len;
 133         ewah_serialize_strbuf(istate->fsmonitor_dirty, sb);
 134         ewah_free(istate->fsmonitor_dirty);
 135         istate->fsmonitor_dirty = NULL;
 136
 137         /* fix up size field */
 138         put_be32(&ewah_size, sb->len - ewah_start);
 139         memcpy(sb->buf + fixup, &ewah_size, sizeof(uint32_t));
 140
 141         trace2_data_string("index", NULL, "extension/fsmn/write/token",
 142                            istate->fsmonitor_last_update);
 143         trace_printf_key(&trace_fsmonitor,
 144                          "write fsmonitor extension successful '%s'",
 145                          istate->fsmonitor_last_update);
 146 }
 147
 148 /*
 149  * Call the query-fsmonitor hook passing the last update token of the saved results.
 150  */
 151 static int query_fsmonitor(int version, const char *last_update, struct strbuf *query_result)
 152 {
 153         struct child_process cp = CHILD_PROCESS_INIT;
 154         int result;
 155
 156         if (!core_fsmonitor)
 157                 return -1;
 158
 159         strvec_push(&cp.args, core_fsmonitor);
 160         strvec_pushf(&cp.args, "%d", version);
 161         strvec_pushf(&cp.args, "%s", last_update);
 162         cp.use_shell = 1;
 163         cp.dir = get_git_work_tree();
 164
 165         trace2_region_enter("fsm_hook", "query", NULL);
 166
 167         result = capture_command(&cp, query_result, 1024);
 168
 169         if (result)
 170                 trace2_data_intmax("fsm_hook", NULL, "query/failed", result);
 171         else {
 172                 trace2_data_intmax("fsm_hook", NULL, "query/response-length",
 173                                    query_result->len);
 174
 175                 if (fsmonitor_is_trivial_response(query_result))
 176                         trace2_data_intmax("fsm_hook", NULL,
 177                                            "query/trivial-response", 1);
 178         }
 179
 180         trace2_region_leave("fsm_hook", "query", NULL);
 181
 182         return result;
 183 }
 184
 185 int fsmonitor_is_trivial_response(const struct strbuf *query_result)
 186 {
 187         static char trivial_response[3] = { '\0', '/', '\0' };
 188
 189         return query_result->len >= 3 &&
 190                 !memcmp(trivial_response,
 191                         &query_result->buf[query_result->len - 3], 3);
 192 }
 193
 194 static void fsmonitor_refresh_callback(struct index_state *istate, char *name)
 195 {
 196         int i, len = strlen(name);
 197         if (name[len - 1] == '/') {
 198
 199                 /*
 200                  * TODO We should binary search to find the first path with
 201                  * TODO this directory prefix.  Then linearly update entries
 202                  * TODO while the prefix matches.  Taking care to search without
 203                  * TODO the trailing slash -- because '/' sorts after a few
 204                  * TODO interesting special chars, like '.' and ' '.
 205                  */
 206
 207                 /* Mark all entries for the folder invalid */
 208                 for (i = 0; i < istate->cache_nr; i++) {
 209                         if (istate->cache[i]->ce_flags & CE_FSMONITOR_VALID &&
 210                             starts_with(istate->cache[i]->name, name))
 211                                 istate->cache[i]->ce_flags &= ~CE_FSMONITOR_VALID;
 212                 }
 213                 /* Need to remove the / from the path for the untracked cache */
 214                 name[len - 1] = '\0';
 215         } else {
 216                 int pos = index_name_pos(istate, name, strlen(name));
 217
 218                 if (pos >= 0) {
 219                         struct cache_entry *ce = istate->cache[pos];
 220                         ce->ce_flags &= ~CE_FSMONITOR_VALID;
 221                 }
 222         }
 223
 224         /*
 225          * Mark the untracked cache dirty even if it wasn't found in the index
 226          * as it could be a new untracked file.
 227          */
 228         trace_printf_key(&trace_fsmonitor, "fsmonitor_refresh_callback '%s'", name);
 229         untracked_cache_invalidate_path(istate, name, 0);
 230 }
 231
 232 void refresh_fsmonitor(struct index_state *istate)
 233 {
 234         struct strbuf query_result = STRBUF_INIT;
 235         int query_success = 0, hook_version = -1;
 236         size_t bol = 0; /* beginning of line */
 237         uint64_t last_update;
 238         struct strbuf last_update_token = STRBUF_INIT;
 239         char *buf;
 240         unsigned int i;
 241
 242         if (!core_fsmonitor || istate->fsmonitor_has_run_once)
 243                 return;
 244
 245         hook_version = fsmonitor_hook_version();
 246
 247         istate->fsmonitor_has_run_once = 1;
 248
 249         trace_printf_key(&trace_fsmonitor, "refresh fsmonitor");
 250         /*
 251          * This could be racy so save the date/time now and query_fsmonitor
 252          * should be inclusive to ensure we don't miss potential changes.
 253          */
 254         last_update = getnanotime();
 255         if (hook_version == HOOK_INTERFACE_VERSION1)
 256                 strbuf_addf(&last_update_token, "%"PRIu64"", last_update);
 257
 258         /*
 259          * If we have a last update token, call query_fsmonitor for the set of
 260          * changes since that token, else assume everything is possibly dirty
 261          * and check it all.
 262          */
 263         if (istate->fsmonitor_last_update) {
 264                 if (hook_version == -1 || hook_version == HOOK_INTERFACE_VERSION2) {
 265                         query_success = !query_fsmonitor(HOOK_INTERFACE_VERSION2,
 266                                 istate->fsmonitor_last_update, &query_result);
 267
 268                         if (query_success) {
 269                                 if (hook_version < 0)
 270                                         hook_version = HOOK_INTERFACE_VERSION2;
 271
 272                                 /*
 273                                  * First entry will be the last update token
 274                                  * Need to use a char * variable because static
 275                                  * analysis was suggesting to use strbuf_addbuf
 276                                  * but we don't want to copy the entire strbuf
 277                                  * only the chars up to the first NUL
 278                                  */
 279                                 buf = query_result.buf;
 280                                 strbuf_addstr(&last_update_token, buf);
 281                                 if (!last_update_token.len) {
 282                                         warning("Empty last update token.");
 283                                         query_success = 0;
 284                                 } else {
 285                                         bol = last_update_token.len + 1;
 286                                 }
 287                         } else if (hook_version < 0) {
 288                                 hook_version = HOOK_INTERFACE_VERSION1;
 289                                 if (!last_update_token.len)
 290                                         strbuf_addf(&last_update_token, "%"PRIu64"", last_update);
 291                         }
 292                 }
 293
 294                 if (hook_version == HOOK_INTERFACE_VERSION1) {
 295                         query_success = !query_fsmonitor(HOOK_INTERFACE_VERSION1,
 296                                 istate->fsmonitor_last_update, &query_result);
 297                 }
 298
 299                 trace_performance_since(last_update, "fsmonitor process '%s'", core_fsmonitor);
 300                 trace_printf_key(&trace_fsmonitor, "fsmonitor process '%s' returned %s",
 301                         core_fsmonitor, query_success ? "success" : "failure");
 302         }
 303
 304         /* a fsmonitor process can return '/' to indicate all entries are invalid */
 305         if (query_success && query_result.buf[bol] != '/') {
 306                 /* Mark all entries returned by the monitor as dirty */
 307                 buf = query_result.buf;
 308                 for (i = bol; i < query_result.len; i++) {
 309                         if (buf[i] != '\0')
 310                                 continue;
 311                         fsmonitor_refresh_callback(istate, buf + bol);
 312                         bol = i + 1;
 313                 }
 314                 if (bol < query_result.len)
 315                         fsmonitor_refresh_callback(istate, buf + bol);
 316
 317                 /* Now mark the untracked cache for fsmonitor usage */
 318                 if (istate->untracked)
 319                         istate->untracked->use_fsmonitor = 1;
 320         } else {
 321
 322                 /* We only want to run the post index changed hook if we've actually changed entries, so keep track
 323                  * if we actually changed entries or not */
 324                 int is_cache_changed = 0;
 325                 /* Mark all entries invalid */
 326                 for (i = 0; i < istate->cache_nr; i++) {
 327                         if (istate->cache[i]->ce_flags & CE_FSMONITOR_VALID) {
 328                                 is_cache_changed = 1;
 329                                 istate->cache[i]->ce_flags &= ~CE_FSMONITOR_VALID;
 330                         }
 331                 }
 332
 333                 /* If we're going to check every file, ensure we save the results */
 334                 if (is_cache_changed)
 335                         istate->cache_changed |= FSMONITOR_CHANGED;
 336
 337                 if (istate->untracked)
 338                         istate->untracked->use_fsmonitor = 0;
 339         }
 340         strbuf_release(&query_result);
 341
 342         /* Now that we've updated istate, save the last_update_token */
 343         FREE_AND_NULL(istate->fsmonitor_last_update);
 344         istate->fsmonitor_last_update = strbuf_detach(&last_update_token, NULL);
 345 }
 346
 347 /*
 348  * The caller wants to turn on FSMonitor.  And when the caller writes
 349  * the index to disk, a FSMonitor extension should be included.  This
 350  * requires that `istate->fsmonitor_last_update` not be NULL.  But we
 351  * have not actually talked to a FSMonitor process yet, so we don't
 352  * have an initial value for this field.
 353  *
 354  * For a protocol V1 FSMonitor process, this field is a formatted
 355  * "nanoseconds since epoch" field.  However, for a protocol V2
 356  * FSMonitor process, this field is an opaque token.
 357  *
 358  * Historically, `add_fsmonitor()` has initialized this field to the
 359  * current time for protocol V1 processes.  There are lots of race
 360  * conditions here, but that code has shipped...
 361  *
 362  * The only true solution is to use a V2 FSMonitor and get a current
 363  * or default token value (that it understands), but we cannot do that
 364  * until we have actually talked to an instance of the FSMonitor process
 365  * (but the protocol requires that we send a token first...).
 366  *
 367  * For simplicity, just initialize like we have a V1 process and require
 368  * that V2 processes adapt.
 369  */
 370 static void initialize_fsmonitor_last_update(struct index_state *istate)
 371 {
 372         struct strbuf last_update = STRBUF_INIT;
 373
 374         strbuf_addf(&last_update, "%"PRIu64"", getnanotime());
 375         istate->fsmonitor_last_update = strbuf_detach(&last_update, NULL);
 376 }
 377
 378 void add_fsmonitor(struct index_state *istate)
 379 {
 380         unsigned int i;
 381
 382         if (!istate->fsmonitor_last_update) {
 383                 trace_printf_key(&trace_fsmonitor, "add fsmonitor");
 384                 istate->cache_changed |= FSMONITOR_CHANGED;
 385                 initialize_fsmonitor_last_update(istate);
 386
 387                 /* reset the fsmonitor state */
 388                 for (i = 0; i < istate->cache_nr; i++)
 389                         istate->cache[i]->ce_flags &= ~CE_FSMONITOR_VALID;
 390
 391                 /* reset the untracked cache */
 392                 if (istate->untracked) {
 393                         add_untracked_cache(istate);
 394                         istate->untracked->use_fsmonitor = 1;
 395                 }
 396
 397                 /* Update the fsmonitor state */
 398                 refresh_fsmonitor(istate);
 399         }
 400 }
 401
 402 void remove_fsmonitor(struct index_state *istate)
 403 {
 404         if (istate->fsmonitor_last_update) {
 405                 trace_printf_key(&trace_fsmonitor, "remove fsmonitor");
 406                 istate->cache_changed |= FSMONITOR_CHANGED;
 407                 FREE_AND_NULL(istate->fsmonitor_last_update);
 408         }
 409 }
 410
 411 void tweak_fsmonitor(struct index_state *istate)
 412 {
 413         unsigned int i;
 414         int fsmonitor_enabled = git_config_get_fsmonitor();
 415
 416         if (istate->fsmonitor_dirty) {
 417                 if (fsmonitor_enabled) {
 418                         /* Mark all entries valid */
 419                         for (i = 0; i < istate->cache_nr; i++) {
 420                                 istate->cache[i]->ce_flags |= CE_FSMONITOR_VALID;
 421                         }
 422
 423                         /* Mark all previously saved entries as dirty */
 424                         assert_index_minimum(istate, istate->fsmonitor_dirty->bit_size);
 425                         ewah_each_bit(istate->fsmonitor_dirty, fsmonitor_ewah_callback, istate);
 426
 427                         refresh_fsmonitor(istate);
 428                 }
 429
 430                 ewah_free(istate->fsmonitor_dirty);
 431                 istate->fsmonitor_dirty = NULL;
 432         }
 433
 434         switch (fsmonitor_enabled) {
 435         case -1: /* keep: do nothing */
 436                 break;
 437         case 0: /* false */
 438                 remove_fsmonitor(istate);
 439                 break;
 440         case 1: /* true */
 441                 add_fsmonitor(istate);
 442                 break;
 443         default: /* unknown value: do nothing */
 444                 break;
 445         }
 446 }