From 7204e7f9ea3067bda7e5658a06e91b67c736f8ab Mon Sep 17 00:00:00 2001

From: Lennart Poettering <lennart@poettering.net>

Date: Mon, 12 Feb 2018 16:14:58 +0100

Subject: [PATCH] sd-journal: properly handle inotify queue overflow

This adds proper handling of IN_Q_OVERFLOW: when the inotify queue runs

over we'll reiterate all directories we are looking at. At the same time

we'll mark all files and directories we encounter that way with a

generation counter we first increased. All files and directories not

marked like this are then unloaded.

With this logic we do the best when the inotify queue overflows: we

synchronize our in-memory state again with what's on disk.  This

contains some refactoring of the directory logic, to share more code

between uuid directories and "root" directories and generally make

things a bit more readable by splitting things up into smaller bits.

See: #7998 #8032

(cherry-picked from commit 858749f7312bd0adb5433075a92e1c35a2fb56ac)

Resolves: #1540538

---

 src/journal/journal-file.h     |   2 +

 src/journal/journal-internal.h |   2 +

 src/journal/sd-journal.c       | 237 ++++++++++++++++++++++++++-------

 src/shared/path-util.c         |  14 ++

 src/shared/path-util.h         |   2 +

 5 files changed, 206 insertions(+), 51 deletions(-)

diff --git a/src/journal/journal-file.h b/src/journal/journal-file.h

index c74ad5fc58..dd8ef52d2a 100644

--- a/src/journal/journal-file.h

+++ b/src/journal/journal-file.h

@@ -121,6 +121,8 @@ typedef struct JournalFile {

         void *fsprg_seed;

         size_t fsprg_seed_size;

+

+        unsigned last_seen_generation;

 #endif

 } JournalFile;

diff --git a/src/journal/journal-internal.h b/src/journal/journal-internal.h

index eb23ac28ad..999e9d8cb6 100644

--- a/src/journal/journal-internal.h

+++ b/src/journal/journal-internal.h

@@ -81,6 +81,7 @@ struct Directory {

         char *path;

         int wd;

         bool is_root;

+        unsigned last_seen_generation;

 };

 struct sd_journal {

@@ -102,6 +103,7 @@ struct sd_journal {

         int inotify_fd;

         unsigned current_invalidate_counter, last_invalidate_counter;

         usec_t last_process_usec;

+        unsigned generation;

         char *unique_field;

         JournalFile *unique_file;

diff --git a/src/journal/sd-journal.c b/src/journal/sd-journal.c

index 14b65cfedd..9186f5188e 100644

--- a/src/journal/sd-journal.c

+++ b/src/journal/sd-journal.c

@@ -1229,8 +1229,16 @@ static int add_any_file(sd_journal *j, const char *path) {

         assert(j);

         assert(path);

-        if (ordered_hashmap_get(j->files, path))

-                return 0;

+        if (path) {

+                f = ordered_hashmap_get(j->files, path);

+                if (f) {

+                        /* Mark this file as seen in this generation. This is used to GC old files in

+                         * process_q_overflow() to detect journal files that are still and discern them from those who

+                         * are gone. */

+                        f->last_seen_generation = j->generation;

+                        return 0;

+                }

+        }

         if (ordered_hashmap_size(j->files) >= JOURNAL_FILES_MAX) {

                 log_debug("Too many open journal files, not adding %s.", path);

@@ -1252,6 +1260,8 @@ static int add_any_file(sd_journal *j, const char *path) {

                 goto fail;

         }

+        f->last_seen_generation = j->generation;

+

         log_debug("File %s added.", f->path);

         check_network(j, f->fd);

@@ -1346,10 +1356,96 @@ static int dirname_is_machine_id(const char *fn) {

         return sd_id128_equal(id, machine);

 }

+static bool dirent_is_journal_file(const struct dirent *de) {

+        assert(de);

+

+        if (!IN_SET(de->d_type, DT_REG, DT_LNK, DT_UNKNOWN))

+                return false;

+

+        return endswith(de->d_name, ".journal") ||

+                endswith(de->d_name, ".journal~");

+}

+

+static bool dirent_is_id128_subdir(const struct dirent *de) {

+        assert(de);

+

+        if (!IN_SET(de->d_type, DT_DIR, DT_LNK, DT_UNKNOWN))

+                return false;

+

+        return id128_is_valid(de->d_name);

+}

+

+static int directory_open(sd_journal *j, const char *path, DIR **ret) {

+        DIR *d;

+

+        assert(j);

+        assert(path);

+        assert(ret);

+

+        d = opendir(path);

+        if (!d)

+                return -errno;

+

+        *ret = d;

+        return 0;

+}

+

+static int add_directory(sd_journal *j, const char *prefix, const char *dirname);

+

+static void directory_enumerate(sd_journal *j, Directory *m, DIR *d) {

+        struct dirent *de;

+

+        assert(j);

+        assert(m);

+        assert(d);

+

+        FOREACH_DIRENT_ALL(de, d, goto fail) {

+                if (dirent_is_journal_file(de))

+                        (void) add_file(j, m->path, de->d_name);

+

+                if (m->is_root && dirent_is_id128_subdir(de))

+                        (void) add_directory(j, m->path, de->d_name);

+        }

+

+        return;

+

+fail:

+        log_debug_errno(errno, "Failed to enumerate directory %s, ignoring: %m", m->path);

+}

+

+static void directory_watch(sd_journal *j, Directory *m, int fd, uint32_t mask) {

+        int r;

+

+        assert(j);

+        assert(m);

+        assert(fd >= 0);

+

+        /* Watch this directory if that's enabled and if it not being watched yet. */

+

+        if (m->wd > 0) /* Already have a watch? */

+                return;

+        if (j->inotify_fd < 0) /* Not watching at all? */

+                return;

+

+        m->wd = inotify_add_watch_fd(j->inotify_fd, fd, mask);

+        if (m->wd < 0) {

+                log_debug_errno(errno, "Failed to watch journal directory '%s', ignoring: %m", m->path);

+                return;

+        }

+

+        r = hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m);

+        if (r == -EEXIST)

+                log_debug_errno(r, "Directory '%s' already being watched under a different path, ignoring: %m", m->path);

+        if (r < 0) {

+                log_debug_errno(r, "Failed to add watch for journal directory '%s' to hashmap, ignoring: %m", m->path);

+                (void) inotify_rm_watch(j->inotify_fd, m->wd);

+                m->wd = -1;

+        }

+}

+

 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {

         _cleanup_free_ char *path = NULL;

         _cleanup_closedir_ DIR *d = NULL;

-        struct dirent *de = NULL;

         Directory *m;

         int r, k;

@@ -1357,7 +1453,7 @@ static int add_directory(sd_journal *j, const char *prefix, const char *dirname)

         assert(prefix);

         assert(dirname);

-        log_debug("Considering %s/%s.", prefix, dirname);

+        log_debug("Considering '%s/%s'.", prefix, dirname);

         if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&

             !(dirname_is_machine_id(dirname) > 0 || path_startswith(prefix, "/run")))

@@ -1369,9 +1465,9 @@ static int add_directory(sd_journal *j, const char *prefix, const char *dirname)

                 goto fail;

         }

-        d = opendir(path);

-        if (!d) {

-                r = log_debug_errno(errno, "Failed to open directory %s: %m", path);

+        r = directory_open(j, path, &d);

+        if (r < 0) {

+                r = log_debug_errno(errno, "Failed to open directory '%s': %m", path);

                 goto fail;

         }

@@ -1398,25 +1494,17 @@ static int add_directory(sd_journal *j, const char *prefix, const char *dirname)

                 log_debug("Directory %s added.", m->path);

         } else if (m->is_root)

-                return 0;

-

-        if (m->wd <= 0 && j->inotify_fd >= 0) {

-

-                m->wd = inotify_add_watch(j->inotify_fd, m->path,

-                                          IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|

-                                          IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|

-                                          IN_ONLYDIR);

+                return 0; /* Don't 'downgrade' from root directory */

-                if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)

-                        inotify_rm_watch(j->inotify_fd, m->wd);

-        }

+        m->last_seen_generation = j->generation;

-        FOREACH_DIRENT_ALL(de, d, return log_debug_errno(errno, "Failed to read directory %s: %m", m->path)) {

+        directory_watch(j, m, dirfd(d),

+                        IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|

+                        IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|

+                        IN_ONLYDIR);

-                if (dirent_is_file_with_suffix(de, ".journal") ||

-                    dirent_is_file_with_suffix(de, ".journal~"))

-                        (void) add_file(j, m->path, de->d_name);

-        }

+        if (!j->no_new_files)

+                directory_enumerate(j, m, d);

         check_network(j, dirfd(d));

@@ -1432,13 +1520,14 @@ fail:

 static int add_root_directory(sd_journal *j, const char *p, bool missing_ok) {

         _cleanup_closedir_ DIR *d = NULL;

-        struct dirent *de;

         Directory *m;

         int r, k;

         assert(j);

         assert(p);

+        log_debug("Considering root directory '%s'.", p);

+

         if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&

             !path_startswith(p, "/run"))

                 return -EINVAL;

@@ -1446,12 +1535,11 @@ static int add_root_directory(sd_journal *j, const char *p, bool missing_ok) {

         if (j->prefix)

                 p = strjoina(j->prefix, p);

-        d = opendir(p);

-        if (!d) {

-                if (errno == ENOENT && missing_ok)

-                        return 0;

-

-                r = log_debug_errno(errno, "Failed to open root directory %s: %m", p);

+        r = directory_open(j, p, &d);

+        if (r == -ENOENT && missing_ok)

+                return 0;

+        if (r < 0) {

+                log_debug_errno(r, "Failed to open root directory %s: %m", p);

                 goto fail;

         }

@@ -1495,19 +1583,12 @@ static int add_root_directory(sd_journal *j, const char *p, bool missing_ok) {

                         inotify_rm_watch(j->inotify_fd, m->wd);

         }

-        if (j->no_new_files)

-                return 0;

-

-        FOREACH_DIRENT_ALL(de, d, return log_debug_errno(errno, "Failed to read directory %s: %m", m->path)) {

-                sd_id128_t id;

+        directory_watch(j, m, dirfd(d),

+                        IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|

+                        IN_ONLYDIR);

-                if (dirent_is_file_with_suffix(de, ".journal") ||

-                    dirent_is_file_with_suffix(de, ".journal~"))

-                        (void) add_file(j, m->path, de->d_name);

-                else if (IN_SET(de->d_type, DT_DIR, DT_LNK, DT_UNKNOWN) &&

-                         sd_id128_from_string(de->d_name, &id) >= 0)

-                        (void) add_directory(j, m->path, de->d_name);

-        }

+        if (!j->no_new_files)

+                directory_enumerate(j, m, d);

         check_network(j, dirfd(d));

@@ -2068,6 +2149,18 @@ _public_ void sd_journal_restart_data(sd_journal *j) {

         j->current_field = 0;

 }

+static int reiterate_all_paths(sd_journal *j) {

+        assert(j);

+

+        if (j->no_new_files)

+                return add_current_paths(j);

+

+        if (j->path)

+                return add_root_directory(j, j->path, true);

+

+        return add_search_paths(j);

+}

+

 _public_ int sd_journal_get_fd(sd_journal *j) {

         int r;

@@ -2081,15 +2174,11 @@ _public_ int sd_journal_get_fd(sd_journal *j) {

         if (r < 0)

                 return r;

-        /* Iterate through all dirs again, to add them to the

-         * inotify */

-        if (j->no_new_files)

-                r = add_current_paths(j);

-        else if (j->path)

-                r = add_root_directory(j, j->path, true);

-        else

-                r = add_search_paths(j);

-        if (r < 0)

+         log_debug("Reiterating files to get inotify watches established.");

+

+        /* Iterate through all dirs again, to add them to the inotify */

+        r = reiterate_all_paths(j);

+         if (r < 0)

                 return r;

         return j->inotify_fd;

@@ -2131,12 +2220,58 @@ _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) {

         return 1;

 }

+static void process_q_overflow(sd_journal *j) {

+        JournalFile *f;

+        Directory *m;

+        Iterator i;

+

+        assert(j);

+

+        /* When the inotify queue overruns we need to enumerate and re-validate all journal files to bring our list

+         * back in sync with what's on disk. For this we pick a new generation counter value. It'll be assigned to all

+         * journal files we encounter. All journal files and all directories that don't carry it after reenumeration

+         * are subject for unloading. */

+

+        log_debug("Inotify queue overrun, reiterating everything.");

+

+        j->generation++;

+        (void) reiterate_all_paths(j);

+

+        ORDERED_HASHMAP_FOREACH(f, j->files, i) {

+

+                if (f->last_seen_generation == j->generation)

+                        continue;

+

+                log_debug("File '%s' hasn't been seen in this enumeration, removing.", f->path);

+                remove_file_real(j, f);

+        }

+

+        HASHMAP_FOREACH(m, j->directories_by_path, i) {

+

+                if (m->last_seen_generation == j->generation)

+                        continue;

+

+                if (m->is_root) /* Never GC root directories */

+                        continue;

+

+                log_debug("Directory '%s' hasn't been seen in this enumeration, removing.", f->path);

+                remove_directory(j, m);

+        }

+

+        log_debug("Reiteration complete.");

+}

+

 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {

         Directory *d;

         assert(j);

         assert(e);

+        if (e->mask & IN_Q_OVERFLOW) {

+                process_q_overflow(j);

+                return;

+        }

+

         /* Is this a subdirectory we watch? */

         d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));

         if (d) {

diff --git a/src/shared/path-util.c b/src/shared/path-util.c

index 5d4de9ec4d..fcc591686f 100644

--- a/src/shared/path-util.c

+++ b/src/shared/path-util.c

@@ -861,3 +861,17 @@ char *prefix_root(const char *root, const char *path) {

         strcpy(p, path);

         return n;

 }

+

+int inotify_add_watch_fd(int fd, int what, uint32_t mask) {

+        char path[strlen("/proc/self/fd/") + DECIMAL_STR_MAX(int) + 1];

+        int r;

+

+        /* This is like inotify_add_watch(), except that the file to watch is not referenced by a path, but by an fd */

+        xsprintf(path, "/proc/self/fd/%i", what);

+

+        r = inotify_add_watch(fd, path, mask);

+        if (r < 0)

+                return -errno;

+

+        return r;

+}

diff --git a/src/shared/path-util.h b/src/shared/path-util.h

index 34c016229c..96490e12b1 100644

--- a/src/shared/path-util.h

+++ b/src/shared/path-util.h

@@ -66,6 +66,8 @@ int fsck_exists(const char *fstype);

 char *prefix_root(const char *root, const char *path);

+int inotify_add_watch_fd(int fd, int what, uint32_t mask);

+

 /* Similar to prefix_root(), but returns an alloca() buffer, or

  * possibly a const pointer into the path parameter */

 #define prefix_roota(root, path)                                        \