From 2d1d6137e66857ebd0ad97cb48d2e1bccc3c1a82 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Jul 21 2022 18:14:09 +0000 Subject: journal: Add compact mode This adds a new flag in preparation for incompatible journal changes which will be gated behind this flag. The max file size of journal files in compact mode is limited to 4 GiB. --- diff --git a/docs/JOURNAL_FILE_FORMAT.md b/docs/JOURNAL_FILE_FORMAT.md index 61a27a9..d40688c 100644 --- a/docs/JOURNAL_FILE_FORMAT.md +++ b/docs/JOURNAL_FILE_FORMAT.md @@ -259,6 +259,7 @@ enum { HEADER_INCOMPATIBLE_COMPRESSED_LZ4 = 1 << 1, HEADER_INCOMPATIBLE_KEYED_HASH = 1 << 2, HEADER_INCOMPATIBLE_COMPRESSED_ZSTD = 1 << 3, + HEADER_INCOMPATIBLE_COMPACT = 1 << 4, }; enum { @@ -276,6 +277,9 @@ HEADER_INCOMPATIBLE_KEYED_HASH indicates that instead of the unkeyed Jenkins hash function the keyed siphash24 hash function is used for the two hash tables, see below. +HEADER_INCOMPATIBLE_COMPACT indicates that the journal file uses the new binary +format that uses less space on disk compared to the original format. + HEADER_COMPATIBLE_SEALED indicates that the file includes TAG objects required for Forward Secure Sealing. diff --git a/src/libsystemd/sd-journal/journal-def.h b/src/libsystemd/sd-journal/journal-def.h index ffd1af0..e0919a7 100644 --- a/src/libsystemd/sd-journal/journal-def.h +++ b/src/libsystemd/sd-journal/journal-def.h @@ -152,19 +152,22 @@ enum { HEADER_INCOMPATIBLE_COMPRESSED_LZ4 = 1 << 1, HEADER_INCOMPATIBLE_KEYED_HASH = 1 << 2, HEADER_INCOMPATIBLE_COMPRESSED_ZSTD = 1 << 3, + HEADER_INCOMPATIBLE_COMPACT = 1 << 4, }; #define HEADER_INCOMPATIBLE_ANY \ (HEADER_INCOMPATIBLE_COMPRESSED_XZ | \ HEADER_INCOMPATIBLE_COMPRESSED_LZ4 | \ HEADER_INCOMPATIBLE_KEYED_HASH | \ - HEADER_INCOMPATIBLE_COMPRESSED_ZSTD) + HEADER_INCOMPATIBLE_COMPRESSED_ZSTD | \ + HEADER_INCOMPATIBLE_COMPACT) #define HEADER_INCOMPATIBLE_SUPPORTED \ ((HAVE_XZ ? HEADER_INCOMPATIBLE_COMPRESSED_XZ : 0) | \ (HAVE_LZ4 ? HEADER_INCOMPATIBLE_COMPRESSED_LZ4 : 0) | \ (HAVE_ZSTD ? HEADER_INCOMPATIBLE_COMPRESSED_ZSTD : 0) | \ - HEADER_INCOMPATIBLE_KEYED_HASH) + HEADER_INCOMPATIBLE_KEYED_HASH | \ + HEADER_INCOMPATIBLE_COMPACT) enum { HEADER_COMPATIBLE_SEALED = 1 << 0, diff --git a/src/libsystemd/sd-journal/journal-file.c b/src/libsystemd/sd-journal/journal-file.c index c19f0ce..a43257d 100644 --- a/src/libsystemd/sd-journal/journal-file.c +++ b/src/libsystemd/sd-journal/journal-file.c @@ -44,6 +44,7 @@ /* This is the minimum journal file size */ #define JOURNAL_FILE_SIZE_MIN (512 * 1024ULL) /* 512 KiB */ +#define JOURNAL_COMPACT_SIZE_MAX UINT32_MAX /* 4 GiB */ /* These are the lower and upper bounds if we deduce the max_use value * from the file system size */ @@ -378,7 +379,7 @@ static bool warn_wrong_flags(const JournalFile *f, bool compatible) { f->path, type, flags & ~any); flags = (flags & any) & ~supported; if (flags) { - const char* strv[5]; + const char* strv[6]; size_t n = 0; _cleanup_free_ char *t = NULL; @@ -394,6 +395,8 @@ static bool warn_wrong_flags(const JournalFile *f, bool compatible) { strv[n++] = "zstd-compressed"; if (flags & HEADER_INCOMPATIBLE_KEYED_HASH) strv[n++] = "keyed-hash"; + if (flags & HEADER_INCOMPATIBLE_COMPACT) + strv[n++] = "compact"; } strv[n] = NULL; assert(n < ELEMENTSOF(strv)); @@ -3234,7 +3237,7 @@ void journal_file_print_header(JournalFile *f) { "Sequential number ID: %s\n" "State: %s\n" "Compatible flags:%s%s\n" - "Incompatible flags:%s%s%s%s%s\n" + "Incompatible flags:%s%s%s%s%s%s\n" "Header size: %"PRIu64"\n" "Arena size: %"PRIu64"\n" "Data hash table size: %"PRIu64"\n" @@ -3261,6 +3264,7 @@ void journal_file_print_header(JournalFile *f) { JOURNAL_HEADER_COMPRESSED_LZ4(f->header) ? " COMPRESSED-LZ4" : "", JOURNAL_HEADER_COMPRESSED_ZSTD(f->header) ? " COMPRESSED-ZSTD" : "", JOURNAL_HEADER_KEYED_HASH(f->header) ? " KEYED-HASH" : "", + JOURNAL_HEADER_COMPACT(f->header) ? " COMPACT" : "", (le32toh(f->header->incompatible_flags) & ~HEADER_INCOMPATIBLE_ANY) ? " ???" : "", le64toh(f->header->header_size), le64toh(f->header->arena_size), @@ -3341,7 +3345,7 @@ static int journal_file_warn_btrfs(JournalFile *f) { return 1; } -static void journal_default_metrics(JournalMetrics *m, int fd) { +static void journal_default_metrics(JournalMetrics *m, int fd, bool compact) { struct statvfs ss; uint64_t fs_size = 0; @@ -3384,6 +3388,9 @@ static void journal_default_metrics(JournalMetrics *m, int fd) { else m->max_size = PAGE_ALIGN(m->max_size); + if (compact && m->max_size > JOURNAL_COMPACT_SIZE_MAX) + m->max_size = JOURNAL_COMPACT_SIZE_MAX; + if (m->max_size != 0) { if (m->max_size < JOURNAL_FILE_SIZE_MIN) m->max_size = JOURNAL_FILE_SIZE_MIN; @@ -3575,7 +3582,7 @@ int journal_file_open( if (journal_file_writable(f)) { if (metrics) { - journal_default_metrics(metrics, f->fd); + journal_default_metrics(metrics, f->fd, JOURNAL_HEADER_COMPACT(f->header)); f->metrics = *metrics; } else if (template) f->metrics = template->metrics; diff --git a/src/libsystemd/sd-journal/journal-file.h b/src/libsystemd/sd-journal/journal-file.h index be02b26..01942ec 100644 --- a/src/libsystemd/sd-journal/journal-file.h +++ b/src/libsystemd/sd-journal/journal-file.h @@ -184,6 +184,9 @@ static inline bool VALID_EPOCH(uint64_t u) { #define JOURNAL_HEADER_KEYED_HASH(h) \ FLAGS_SET(le32toh((h)->incompatible_flags), HEADER_INCOMPATIBLE_KEYED_HASH) +#define JOURNAL_HEADER_COMPACT(h) \ + FLAGS_SET(le32toh((h)->incompatible_flags), HEADER_INCOMPATIBLE_COMPACT) + int journal_file_move_to_object(JournalFile *f, ObjectType type, uint64_t offset, Object **ret); int journal_file_read_object_header(JournalFile *f, ObjectType type, uint64_t offset, Object *ret);