From 196c36bb13de94c977a3e360dc5fa529efe2c5ca Mon Sep 17 00:00:00 2001 From: Tavian Barnes Date: Mon, 17 Jul 2023 19:05:23 -0400 Subject: bftw: Reserve space in the cache before opening files This fixes a storm of EMFILE retries observed with -j1 on a very large directory tree. Fixes: 7888fbababd22190e9f919fc272957426a27969e --- src/bftw.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) (limited to 'src/bftw.c') diff --git a/src/bftw.c b/src/bftw.c index 2262591..9d85966 100644 --- a/src/bftw.c +++ b/src/bftw.c @@ -572,7 +572,13 @@ static int bftw_cache_reserve(struct bftw_state *state) { } } - return bftw_cache_pop(cache); + if (bftw_cache_pop(cache) != 0) { + errno = EMFILE; + return -1; + } + + bfs_assert(cache->capacity > 0); + return 0; } /** Open a bftw_file relative to another one. */ @@ -587,8 +593,13 @@ static int bftw_file_openat(struct bftw_state *state, struct bftw_file *file, st at_fd = base->fd; } + int fd = -1; + if (bftw_cache_reserve(state) != 0) { + goto unpin; + } + int flags = O_RDONLY | O_CLOEXEC | O_DIRECTORY; - int fd = openat(at_fd, at_path, flags); + fd = openat(at_fd, at_path, flags); if (fd < 0 && errno == EMFILE) { if (bftw_cache_pop(cache) == 0) { @@ -597,6 +608,7 @@ static int bftw_file_openat(struct bftw_state *state, struct bftw_file *file, st cache->capacity = 1; } +unpin: if (base) { bftw_cache_unpin(cache, base); } @@ -606,7 +618,7 @@ static int bftw_file_openat(struct bftw_state *state, struct bftw_file *file, st bftw_cache_add(cache, file); } - return file->fd; + return fd; } /** Open a bftw_file. */ -- cgit v1.2.3