From ae2607f01609745e3137fa7494c09af10f41afc0 Mon Sep 17 00:00:00 2001 From: Neal Gompa Date: Jan 15 2022 18:26:58 +0000 Subject: [PATCH 1/3] Update to 5.14.2 --- diff --git a/SOURCES/1001-balance-mkfs-Disable-raid56-modes.patch b/SOURCES/1001-balance-mkfs-Disable-raid56-modes.patch index eb86c2b..fd11f19 100644 --- a/SOURCES/1001-balance-mkfs-Disable-raid56-modes.patch +++ b/SOURCES/1001-balance-mkfs-Disable-raid56-modes.patch @@ -1,4 +1,4 @@ -From 9a260cc0355661947dbba6929412662ed6d71d31 Mon Sep 17 00:00:00 2001 +From 349ea14a32ab63555cff47440f178ed01870e3f6 Mon Sep 17 00:00:00 2001 From: Neal Gompa Date: Wed, 1 Sep 2021 19:25:55 -0400 Subject: [PATCH] balance, mkfs: Disable raid56 modes @@ -8,15 +8,15 @@ so it is disabled. Signed-off-by: Neal Gompa --- - cmds/balance.c | 37 +++++-------------------------------- + cmds/balance.c | 35 ++++------------------------------- mkfs/main.c | 4 ++-- - 2 files changed, 7 insertions(+), 34 deletions(-) + 2 files changed, 6 insertions(+), 33 deletions(-) diff --git a/cmds/balance.c b/cmds/balance.c -index 33cbb91c..43ce19e8 100644 +index 7abc69d9..ab86ff36 100644 --- a/cmds/balance.c +++ b/cmds/balance.c -@@ -503,8 +503,7 @@ static const char * const cmd_balance_start_usage[] = { +@@ -376,8 +376,7 @@ static const char * const cmd_balance_start_usage[] = { "-d[filters] act on data chunks", "-m[filters] act on metadata chunks", "-s[filters] act on system chunks (only under -f)", @@ -26,7 +26,7 @@ index 33cbb91c..43ce19e8 100644 "--full-balance do not print warning and do not delay start", "--background|--bg", " run the balance as a background process", -@@ -527,7 +526,6 @@ static int cmd_balance_start(const struct cmd_struct *cmd, +@@ -400,7 +399,6 @@ static int cmd_balance_start(const struct cmd_struct *cmd, int background = 0; bool enqueue = false; unsigned start_flags = 0; @@ -34,7 +34,7 @@ index 33cbb91c..43ce19e8 100644 int i; memset(&args, 0, sizeof(args)); -@@ -634,8 +632,6 @@ static int cmd_balance_start(const struct cmd_struct *cmd, +@@ -507,8 +505,6 @@ static int cmd_balance_start(const struct cmd_struct *cmd, /* soft makes sense only when convert for corresponding type is set */ for (i = 0; ptrs[i]; i++) { @@ -43,12 +43,11 @@ index 33cbb91c..43ce19e8 100644 if ((ptrs[i]->flags & BTRFS_BALANCE_ARGS_SOFT) && !(ptrs[i]->flags & BTRFS_BALANCE_ARGS_CONVERT)) { error("'soft' option can be used only when converting profiles"); -@@ -645,34 +641,11 @@ static int cmd_balance_start(const struct cmd_struct *cmd, +@@ -518,33 +514,10 @@ static int cmd_balance_start(const struct cmd_struct *cmd, if (!(ptrs[i]->flags & BTRFS_BALANCE_ARGS_CONVERT)) continue; -- if (!(ptrs[i]->target & (BTRFS_BLOCK_GROUP_RAID6 | -- BTRFS_BLOCK_GROUP_RAID5))) +- if (!(ptrs[i]->target & BTRFS_BLOCK_GROUP_RAID56_MASK)) - continue; - - if (raid56_warned) @@ -73,8 +72,7 @@ index 33cbb91c..43ce19e8 100644 - printf("%2d", delay--); - fflush(stdout); - sleep(1); -+ if (ptrs[i]->target & (BTRFS_BLOCK_GROUP_RAID6 | -+ BTRFS_BLOCK_GROUP_RAID5)) { ++ if (ptrs[i]->target & BTRFS_BLOCK_GROUP_RAID56_MASK) { + error("RAID5/6 support has known problems and is disabled."); + return 1; } @@ -83,12 +81,12 @@ index 33cbb91c..43ce19e8 100644 if (!(start_flags & BALANCE_START_FILTERS) && !(start_flags & BALANCE_START_NOWARN)) { diff --git a/mkfs/main.c b/mkfs/main.c -index eab93eb3..72cb8dc6 100644 +index 4d50bd25..5f66e6ae 100644 --- a/mkfs/main.c +++ b/mkfs/main.c -@@ -1190,8 +1190,8 @@ int BOX_MAIN(mkfs)(int argc, char **argv) - if ((data_profile | metadata_profile) & - (BTRFS_BLOCK_GROUP_RAID5 | BTRFS_BLOCK_GROUP_RAID6)) { +@@ -1176,8 +1176,8 @@ int BOX_MAIN(mkfs)(int argc, char **argv) + + if ((data_profile | metadata_profile) & BTRFS_BLOCK_GROUP_RAID56_MASK) { features |= BTRFS_FEATURE_INCOMPAT_RAID56; - warning("RAID5/6 support has known problems is strongly discouraged\n" - "\t to be used besides testing or evaluation.\n"); diff --git a/SOURCES/1101-Require-clowntown-for-quota-enable.patch b/SOURCES/1101-Require-clowntown-for-quota-enable.patch index 09ab8fa..6a0a329 100644 --- a/SOURCES/1101-Require-clowntown-for-quota-enable.patch +++ b/SOURCES/1101-Require-clowntown-for-quota-enable.patch @@ -1,5 +1,4 @@ -From 72dcd3588774126e8769034847dd676a28a33dce Mon Sep 17 00:00:00 2001 -Message-Id: <72dcd3588774126e8769034847dd676a28a33dce.1628630682.git.osandov@fb.com> +From 3affbd9749792375b50ad50a335e053917ab1d04 Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Tue, 10 Aug 2021 23:24:38 +0200 Subject: [PATCH] Require --clowntown for quota enable diff --git a/SPECS/btrfs-progs.spec b/SPECS/btrfs-progs.spec index 8a66076..ddd8f79 100644 --- a/SPECS/btrfs-progs.spec +++ b/SPECS/btrfs-progs.spec @@ -1,6 +1,6 @@ Name: btrfs-progs -Version: 5.13.1 -Release: 2%{?dist} +Version: 5.14.2 +Release: 1.1%{?dist} Summary: Userspace programs for btrfs License: GPLv2 @@ -26,6 +26,7 @@ BuildRequires: libacl-devel, lzo-devel BuildRequires: pkgconfig(blkid) BuildRequires: pkgconfig(uuid) BuildRequires: pkgconfig(zlib) +BuildRequires: pkgconfig(libudev) BuildRequires: pkgconfig(libzstd) >= 1.0.0 BuildRequires: asciidoc, xmlto BuildRequires: systemd @@ -39,6 +40,8 @@ check, modify and correct any inconsistencies in the btrfs filesystem. %package -n libbtrfs Summary: btrfs filesystem-specific runtime libraries License: GPLv2 +# Upstream deprecated this library +Provides: deprecated() # This was not properly split out before Conflicts: %{name} < 4.20.2 @@ -148,6 +151,22 @@ popd %{python3_sitearch}/btrfsutil-*.egg-info/ %changelog +* Sat Jan 15 2022 Neal Gompa - 5.14.2-1.1 +- Adapt for CentOS Hyperscale +- Refresh patch for disabling raid56 mode + +* Sat Oct 09 2021 Neal Gompa - 5.14.2-1 +- Update to 5.14.2 + +* Mon Sep 20 2021 Neal Gompa - 5.14.1-1 +- Update to 5.14.1 + +* Fri Sep 10 2021 Neal Gompa - 5.14-2 +- Mark libbtrfs as deprecated, per upstream release notes + +* Fri Sep 10 2021 Neal Gompa - 5.14-1 +- Update to 5.14 + * Wed Sep 01 2021 Neal Gompa - 5.13.1-2 - Refresh patch for disabling raid56 mode with fixes from Omar Sandoval From a8554b53727273e0dd7f97242b7e1c2d0c1a59fd Mon Sep 17 00:00:00 2001 From: Neal Gompa Date: Jan 15 2022 18:32:57 +0000 Subject: [PATCH 2/3] Fix btrfs-progs metadata for update --- diff --git a/.btrfs-progs.metadata b/.btrfs-progs.metadata index 6897934..6bab114 100644 --- a/.btrfs-progs.metadata +++ b/.btrfs-progs.metadata @@ -1,3 +1,3 @@ -255894e83bdc86857c729b06ae0f97a4a55dbc37 SOURCES/btrfs-progs-v5.13.1.tar.sign -dd2cb5b51704b2a2b146ca1f96a380d70e46595f SOURCES/btrfs-progs-v5.13.1.tar.xz +63f23f89e28a9148ed97d2a6edb6c6e7dc91964a SOURCES/btrfs-progs-v5.14.2.tar.sign +e1a90fdf7e1549b661983c5bbd4aec74e31ebef1 SOURCES/btrfs-progs-v5.14.2.tar.xz 6b3b5fc33c002d029e83ae4f678089e0d16e6148 SOURCES/gpgkey-F2B41200C54EFB30380C1756C565D5F9D76D583B.gpg From dbfe2d1af0e71848e1dd2deba163b5dcc433817d Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Apr 06 2022 21:28:38 +0000 Subject: [PATCH 3/3] Update to 5.16.2 and add compressed send/receive patches for Facebook The compressed send/receive patches are v14: https://lore.kernel.org/linux-btrfs/cover.1649092662.git.osandov@fb.com/ --- diff --git a/.btrfs-progs.metadata b/.btrfs-progs.metadata index 6bab114..905d984 100644 --- a/.btrfs-progs.metadata +++ b/.btrfs-progs.metadata @@ -1,3 +1,3 @@ -63f23f89e28a9148ed97d2a6edb6c6e7dc91964a SOURCES/btrfs-progs-v5.14.2.tar.sign -e1a90fdf7e1549b661983c5bbd4aec74e31ebef1 SOURCES/btrfs-progs-v5.14.2.tar.xz +c14de7398dc741c462a0e06aaa1c2b0ba9a0d787 SOURCES/btrfs-progs-v5.16.2.tar.sign +3d5f94b2aeded999fb4cbbab04b91305864955d6 SOURCES/btrfs-progs-v5.16.2.tar.xz 6b3b5fc33c002d029e83ae4f678089e0d16e6148 SOURCES/gpgkey-F2B41200C54EFB30380C1756C565D5F9D76D583B.gpg diff --git a/SOURCES/1001-balance-mkfs-Disable-raid56-modes.patch b/SOURCES/1001-balance-mkfs-Disable-raid56-modes.patch index fd11f19..be2d411 100644 --- a/SOURCES/1001-balance-mkfs-Disable-raid56-modes.patch +++ b/SOURCES/1001-balance-mkfs-Disable-raid56-modes.patch @@ -1,4 +1,4 @@ -From 349ea14a32ab63555cff47440f178ed01870e3f6 Mon Sep 17 00:00:00 2001 +From 1338ce1136aea6cc83f7ee6d78c5ca6a07bb38d6 Mon Sep 17 00:00:00 2001 From: Neal Gompa Date: Wed, 1 Sep 2021 19:25:55 -0400 Subject: [PATCH] balance, mkfs: Disable raid56 modes @@ -65,26 +65,26 @@ index 7abc69d9..ab86ff36 100644 - if (force) { - printf("\tSafety timeout skipped due to --force\n\n"); - continue; -- } ++ if (ptrs[i]->target & BTRFS_BLOCK_GROUP_RAID56_MASK) { ++ error("RAID5/6 support has known problems and is disabled."); ++ return 1; + } - printf("\tThe operation will continue in %d seconds.\n", delay); - printf("\tUse Ctrl-C to stop.\n"); - while (delay) { - printf("%2d", delay--); - fflush(stdout); - sleep(1); -+ if (ptrs[i]->target & BTRFS_BLOCK_GROUP_RAID56_MASK) { -+ error("RAID5/6 support has known problems and is disabled."); -+ return 1; - } +- } - printf("\nStarting conversion to RAID5/6.\n"); } if (!(start_flags & BALANCE_START_FILTERS) && !(start_flags & BALANCE_START_NOWARN)) { diff --git a/mkfs/main.c b/mkfs/main.c -index 4d50bd25..5f66e6ae 100644 +index f9e8be74..11c0fc26 100644 --- a/mkfs/main.c +++ b/mkfs/main.c -@@ -1176,8 +1176,8 @@ int BOX_MAIN(mkfs)(int argc, char **argv) +@@ -1214,8 +1214,8 @@ int BOX_MAIN(mkfs)(int argc, char **argv) if ((data_profile | metadata_profile) & BTRFS_BLOCK_GROUP_RAID56_MASK) { features |= BTRFS_FEATURE_INCOMPAT_RAID56; @@ -96,5 +96,5 @@ index 4d50bd25..5f66e6ae 100644 if ((data_profile | metadata_profile) & -- -2.31.1 +2.35.1 diff --git a/SOURCES/1101-Require-clowntown-for-quota-enable.patch b/SOURCES/1101-Require-clowntown-for-quota-enable.patch index 6a0a329..d3ad946 100644 --- a/SOURCES/1101-Require-clowntown-for-quota-enable.patch +++ b/SOURCES/1101-Require-clowntown-for-quota-enable.patch @@ -1,4 +1,4 @@ -From 3affbd9749792375b50ad50a335e053917ab1d04 Mon Sep 17 00:00:00 2001 +From 5a5f05ff7d8b3dec2da3495530b1653d25b6a175 Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Tue, 10 Aug 2021 23:24:38 +0200 Subject: [PATCH] Require --clowntown for quota enable @@ -81,5 +81,5 @@ index 246dd277..07380726 100644 if (ret < 0) usage(cmd); -- -2.31.1 +2.35.1 diff --git a/SOURCES/1102-btrfs-progs-receive-support-v2-send-stream-larger-tl.patch b/SOURCES/1102-btrfs-progs-receive-support-v2-send-stream-larger-tl.patch new file mode 100644 index 0000000..ce6a2b9 --- /dev/null +++ b/SOURCES/1102-btrfs-progs-receive-support-v2-send-stream-larger-tl.patch @@ -0,0 +1,113 @@ +From 7e5cd32fbb38103b11a8a87ca5af943d0fb2804a Mon Sep 17 00:00:00 2001 +From: Boris Burkov +Date: Fri, 21 Aug 2020 00:40:00 -0700 +Subject: [PATCH] btrfs-progs: receive: support v2 send stream larger tlv_len + +An encoded extent can be up to 128K in length, which exceeds the largest +value expressible by the current send stream format's 16 bit tlv_len +field. Since encoded writes cannot be split into multiple writes by +btrfs send, the send stream format must change to accommodate encoded +writes. + +Supporting this changed format requires retooling how we store the +commands we have processed. We currently store pointers to the struct +btrfs_tlv_headers in the command buffer. This is not sufficient to +represent the new BTRFS_SEND_A_DATA format. Instead, parse the attribute +headers and store them in a new struct btrfs_send_attribute which has a +32-bit length field. This is transparent to users of the various TLV_GET +macros. + +Reviewed-by: Nikolay Borisov +Signed-off-by: Boris Burkov +--- + common/send-stream.c | 34 +++++++++++++++++++++++++--------- + 1 file changed, 25 insertions(+), 9 deletions(-) + +diff --git a/common/send-stream.c b/common/send-stream.c +index e9be922b..7d182238 100644 +--- a/common/send-stream.c ++++ b/common/send-stream.c +@@ -24,13 +24,23 @@ + #include "crypto/crc32c.h" + #include "common/utils.h" + ++struct btrfs_send_attribute { ++ u16 tlv_type; ++ /* ++ * Note: in btrfs_tlv_header, this is __le16, but we need 32 bits for ++ * attributes with file data as of version 2 of the send stream format ++ */ ++ u32 tlv_len; ++ char *data; ++}; ++ + struct btrfs_send_stream { + char read_buf[BTRFS_SEND_BUF_SIZE]; + int fd; + + int cmd; + struct btrfs_cmd_header *cmd_hdr; +- struct btrfs_tlv_header *cmd_attrs[BTRFS_SEND_A_MAX + 1]; ++ struct btrfs_send_attribute cmd_attrs[BTRFS_SEND_A_MAX + 1]; + u32 version; + + /* +@@ -152,6 +162,7 @@ static int read_cmd(struct btrfs_send_stream *sctx) + struct btrfs_tlv_header *tlv_hdr; + u16 tlv_type; + u16 tlv_len; ++ struct btrfs_send_attribute *send_attr; + + tlv_hdr = (struct btrfs_tlv_header *)data; + tlv_type = le16_to_cpu(tlv_hdr->tlv_type); +@@ -164,10 +175,15 @@ static int read_cmd(struct btrfs_send_stream *sctx) + goto out; + } + +- sctx->cmd_attrs[tlv_type] = tlv_hdr; ++ send_attr = &sctx->cmd_attrs[tlv_type]; ++ send_attr->tlv_type = tlv_type; ++ send_attr->tlv_len = tlv_len; ++ pos += sizeof(*tlv_hdr); ++ data += sizeof(*tlv_hdr); + +- data += sizeof(*tlv_hdr) + tlv_len; +- pos += sizeof(*tlv_hdr) + tlv_len; ++ send_attr->data = data; ++ pos += send_attr->tlv_len; ++ data += send_attr->tlv_len; + } + + sctx->cmd = cmd; +@@ -180,7 +196,7 @@ out: + static int tlv_get(struct btrfs_send_stream *sctx, int attr, void **data, int *len) + { + int ret; +- struct btrfs_tlv_header *hdr; ++ struct btrfs_send_attribute *send_attr; + + if (attr <= 0 || attr > BTRFS_SEND_A_MAX) { + error("invalid attribute requested, attr = %d", attr); +@@ -188,15 +204,15 @@ static int tlv_get(struct btrfs_send_stream *sctx, int attr, void **data, int *l + goto out; + } + +- hdr = sctx->cmd_attrs[attr]; +- if (!hdr) { ++ send_attr = &sctx->cmd_attrs[attr]; ++ if (!send_attr->data) { + error("attribute %d requested but not present", attr); + ret = -ENOENT; + goto out; + } + +- *len = le16_to_cpu(hdr->tlv_len); +- *data = hdr + 1; ++ *len = send_attr->tlv_len; ++ *data = send_attr->data; + + ret = 0; + +-- +2.35.1 + diff --git a/SOURCES/1103-btrfs-progs-receive-dynamically-allocate-sctx-read_b.patch b/SOURCES/1103-btrfs-progs-receive-dynamically-allocate-sctx-read_b.patch new file mode 100644 index 0000000..e120fd9 --- /dev/null +++ b/SOURCES/1103-btrfs-progs-receive-dynamically-allocate-sctx-read_b.patch @@ -0,0 +1,171 @@ +From e274c8de61c02f4a65812371bc6f1c2a6ac4d30c Mon Sep 17 00:00:00 2001 +From: Boris Burkov +Date: Fri, 21 Aug 2020 00:40:01 -0700 +Subject: [PATCH] btrfs-progs: receive: dynamically allocate sctx->read_buf + +In send stream v2, write commands can now be an arbitrary size. For that +reason, we can no longer allocate a fixed array in sctx for read_cmd. +Instead, read_cmd dynamically allocates sctx->read_buf. To avoid +needless reallocations, we reuse read_buf between read_cmd calls by also +keeping track of the size of the allocated buffer in sctx->read_buf_sz. + +We do the first allocation of the old default size at the start of +processing the stream, and we only reallocate if we encounter a command +that needs a larger buffer. + +Signed-off-by: Boris Burkov +--- + common/send-stream.c | 56 ++++++++++++++++++++++++++++-------------- + kernel-shared/send.h | 6 ++++- + libbtrfs/send-stream.c | 2 +- + 3 files changed, 43 insertions(+), 21 deletions(-) + +diff --git a/common/send-stream.c b/common/send-stream.c +index 7d182238..421cd1bb 100644 +--- a/common/send-stream.c ++++ b/common/send-stream.c +@@ -35,11 +35,11 @@ struct btrfs_send_attribute { + }; + + struct btrfs_send_stream { +- char read_buf[BTRFS_SEND_BUF_SIZE]; ++ char *read_buf; ++ size_t read_buf_sz; + int fd; + + int cmd; +- struct btrfs_cmd_header *cmd_hdr; + struct btrfs_send_attribute cmd_attrs[BTRFS_SEND_A_MAX + 1]; + u32 version; + +@@ -111,11 +111,12 @@ static int read_cmd(struct btrfs_send_stream *sctx) + u32 pos; + u32 crc; + u32 crc2; ++ struct btrfs_cmd_header *cmd_hdr; ++ size_t buf_len; + + memset(sctx->cmd_attrs, 0, sizeof(sctx->cmd_attrs)); + +- ASSERT(sizeof(*sctx->cmd_hdr) <= sizeof(sctx->read_buf)); +- ret = read_buf(sctx, sctx->read_buf, sizeof(*sctx->cmd_hdr)); ++ ret = read_buf(sctx, sctx->read_buf, sizeof(*cmd_hdr)); + if (ret < 0) + goto out; + if (ret) { +@@ -124,18 +125,25 @@ static int read_cmd(struct btrfs_send_stream *sctx) + goto out; + } + +- sctx->cmd_hdr = (struct btrfs_cmd_header *)sctx->read_buf; +- cmd = le16_to_cpu(sctx->cmd_hdr->cmd); +- cmd_len = le32_to_cpu(sctx->cmd_hdr->len); ++ cmd_hdr = (struct btrfs_cmd_header *)sctx->read_buf; ++ cmd_len = le32_to_cpu(cmd_hdr->len); ++ cmd = le16_to_cpu(cmd_hdr->cmd); ++ buf_len = sizeof(*cmd_hdr) + cmd_len; ++ if (sctx->read_buf_sz < buf_len) { ++ void *new_read_buf; + +- if (cmd_len + sizeof(*sctx->cmd_hdr) >= sizeof(sctx->read_buf)) { +- ret = -EINVAL; +- error("command length %u too big for buffer %zu", +- cmd_len, sizeof(sctx->read_buf)); +- goto out; ++ new_read_buf = realloc(sctx->read_buf, buf_len); ++ if (!new_read_buf) { ++ ret = -ENOMEM; ++ error("failed to reallocate read buffer for cmd"); ++ goto out; ++ } ++ sctx->read_buf = new_read_buf; ++ sctx->read_buf_sz = buf_len; ++ /* We need to reset cmd_hdr after realloc of sctx->read_buf */ ++ cmd_hdr = (struct btrfs_cmd_header *)sctx->read_buf; + } +- +- data = sctx->read_buf + sizeof(*sctx->cmd_hdr); ++ data = sctx->read_buf + sizeof(*cmd_hdr); + ret = read_buf(sctx, data, cmd_len); + if (ret < 0) + goto out; +@@ -145,11 +153,12 @@ static int read_cmd(struct btrfs_send_stream *sctx) + goto out; + } + +- crc = le32_to_cpu(sctx->cmd_hdr->crc); +- sctx->cmd_hdr->crc = 0; ++ crc = le32_to_cpu(cmd_hdr->crc); ++ /* in send, crc is computed with header crc = 0, replicate that */ ++ cmd_hdr->crc = 0; + + crc2 = crc32c(0, (unsigned char*)sctx->read_buf, +- sizeof(*sctx->cmd_hdr) + cmd_len); ++ sizeof(*cmd_hdr) + cmd_len); + + if (crc != crc2) { + ret = -EINVAL; +@@ -537,19 +546,28 @@ int btrfs_read_and_process_send_stream(int fd, + goto out; + } + ++ sctx.read_buf = malloc(BTRFS_SEND_BUF_SIZE_V1); ++ if (!sctx.read_buf) { ++ ret = -ENOMEM; ++ error("unable to allocate send stream read buffer"); ++ goto out; ++ } ++ sctx.read_buf_sz = BTRFS_SEND_BUF_SIZE_V1; ++ + while (1) { + ret = read_and_process_cmd(&sctx); + if (ret < 0) { + last_err = ret; + errors++; + if (max_errors > 0 && errors >= max_errors) +- goto out; ++ break; + } else if (ret > 0) { + if (!honor_end_cmd) + ret = 0; +- goto out; ++ break; + } + } ++ free(sctx.read_buf); + + out: + if (last_err && !ret) +diff --git a/kernel-shared/send.h b/kernel-shared/send.h +index e73f09df..e986b6c8 100644 +--- a/kernel-shared/send.h ++++ b/kernel-shared/send.h +@@ -33,7 +33,11 @@ extern "C" { + #define BTRFS_SEND_STREAM_MAGIC "btrfs-stream" + #define BTRFS_SEND_STREAM_VERSION 1 + +-#define BTRFS_SEND_BUF_SIZE (64 * 1024) ++/* ++ * In send stream v1, no command is larger than 64k. In send stream v2, no limit ++ * should be assumed. ++ */ ++#define BTRFS_SEND_BUF_SIZE_V1 (64 * 1024) + #define BTRFS_SEND_READ_SIZE (1024 * 48) + + enum btrfs_tlv_type { +diff --git a/libbtrfs/send-stream.c b/libbtrfs/send-stream.c +index 2b21d846..39cbb3ed 100644 +--- a/libbtrfs/send-stream.c ++++ b/libbtrfs/send-stream.c +@@ -22,7 +22,7 @@ + #include "crypto/crc32c.h" + + struct btrfs_send_stream { +- char read_buf[BTRFS_SEND_BUF_SIZE]; ++ char read_buf[BTRFS_SEND_BUF_SIZE_V1]; + int fd; + + int cmd; +-- +2.35.1 + diff --git a/SOURCES/1104-btrfs-progs-receive-support-v2-send-stream-DATA-tlv-.patch b/SOURCES/1104-btrfs-progs-receive-support-v2-send-stream-DATA-tlv-.patch new file mode 100644 index 0000000..ebc3ab3 --- /dev/null +++ b/SOURCES/1104-btrfs-progs-receive-support-v2-send-stream-DATA-tlv-.patch @@ -0,0 +1,85 @@ +From 734b41ff166263c364e89518eb46e52310eb070b Mon Sep 17 00:00:00 2001 +From: Boris Burkov +Date: Fri, 21 Aug 2020 00:40:02 -0700 +Subject: [PATCH] btrfs-progs: receive: support v2 send stream DATA tlv format + +The new format privileges the BTRFS_SEND_A_DATA attribute by +guaranteeing it will always be the last attribute in any command that +needs it, and by implicitly encoding the data length as the difference +between the total command length in the command header and the sizes of +the rest of the attributes (and of course the tlv_type identifying the +DATA attribute). To parse the new stream, we must read the tlv_type and +if it is not DATA, we proceed normally, but if it is DATA, we don't +parse a tlv_len but simply compute the length. + +In addition, we add some bounds checking when parsing each chunk of +data, as well as for the tlv_len itself. + +Reviewed-by: Nikolay Borisov +Signed-off-by: Boris Burkov +--- + common/send-stream.c | 36 ++++++++++++++++++++++++++---------- + 1 file changed, 26 insertions(+), 10 deletions(-) + +diff --git a/common/send-stream.c b/common/send-stream.c +index 421cd1bb..81a830d9 100644 +--- a/common/send-stream.c ++++ b/common/send-stream.c +@@ -168,28 +168,44 @@ static int read_cmd(struct btrfs_send_stream *sctx) + + pos = 0; + while (pos < cmd_len) { +- struct btrfs_tlv_header *tlv_hdr; + u16 tlv_type; +- u16 tlv_len; + struct btrfs_send_attribute *send_attr; + +- tlv_hdr = (struct btrfs_tlv_header *)data; +- tlv_type = le16_to_cpu(tlv_hdr->tlv_type); +- tlv_len = le16_to_cpu(tlv_hdr->tlv_len); ++ if (cmd_len - pos < sizeof(__le16)) { ++ error("send stream is truncated"); ++ ret = -EINVAL; ++ goto out; ++ } ++ tlv_type = le16_to_cpu(*(__le16 *)data); + + if (tlv_type == 0 || tlv_type > BTRFS_SEND_A_MAX) { +- error("invalid tlv in cmd tlv_type = %hu, tlv_len = %hu", +- tlv_type, tlv_len); ++ error("invalid tlv in cmd tlv_type = %hu", tlv_type); + ret = -EINVAL; + goto out; + } + + send_attr = &sctx->cmd_attrs[tlv_type]; + send_attr->tlv_type = tlv_type; +- send_attr->tlv_len = tlv_len; +- pos += sizeof(*tlv_hdr); +- data += sizeof(*tlv_hdr); + ++ pos += sizeof(tlv_type); ++ data += sizeof(tlv_type); ++ if (sctx->version >= 2 && tlv_type == BTRFS_SEND_A_DATA) { ++ send_attr->tlv_len = cmd_len - pos; ++ } else { ++ if (cmd_len - pos < sizeof(__le16)) { ++ error("send stream is truncated"); ++ ret = -EINVAL; ++ goto out; ++ } ++ send_attr->tlv_len = le16_to_cpu(*(__le16 *)data); ++ pos += sizeof(__le16); ++ data += sizeof(__le16); ++ } ++ if (cmd_len - pos < send_attr->tlv_len) { ++ error("send stream is truncated"); ++ ret = -EINVAL; ++ goto out; ++ } + send_attr->data = data; + pos += send_attr->tlv_len; + data += send_attr->tlv_len; +-- +2.35.1 + diff --git a/SOURCES/1105-btrfs-progs-receive-add-send-stream-v2-cmds-and-attr.patch b/SOURCES/1105-btrfs-progs-receive-add-send-stream-v2-cmds-and-attr.patch new file mode 100644 index 0000000..02fbc01 --- /dev/null +++ b/SOURCES/1105-btrfs-progs-receive-add-send-stream-v2-cmds-and-attr.patch @@ -0,0 +1,194 @@ +From 8ff6d9258385e4a0ae462e0cea8409e4fae05dc1 Mon Sep 17 00:00:00 2001 +From: Omar Sandoval +Date: Tue, 8 Feb 2022 11:02:02 -0800 +Subject: [PATCH] btrfs-progs: receive: add send stream v2 cmds and attrs to + send.h + +Update our copy of send.h from the kernel. This adds the new commands +and attributes for v2 as well as explicit enum numbering. + +Reviewed-by: Nikolay Borisov +Signed-off-by: Boris Burkov +Signed-off-by: Omar Sandoval +--- + kernel-shared/send.h | 138 ++++++++++++++++++++++++++----------------- + 1 file changed, 85 insertions(+), 53 deletions(-) + +diff --git a/kernel-shared/send.h b/kernel-shared/send.h +index e986b6c8..b902d054 100644 +--- a/kernel-shared/send.h ++++ b/kernel-shared/send.h +@@ -38,7 +38,6 @@ extern "C" { + * should be assumed. + */ + #define BTRFS_SEND_BUF_SIZE_V1 (64 * 1024) +-#define BTRFS_SEND_READ_SIZE (1024 * 48) + + enum btrfs_tlv_type { + BTRFS_TLV_U8, +@@ -72,77 +71,110 @@ struct btrfs_tlv_header { + + /* commands */ + enum btrfs_send_cmd { +- BTRFS_SEND_C_UNSPEC, ++ BTRFS_SEND_C_UNSPEC = 0, + +- BTRFS_SEND_C_SUBVOL, +- BTRFS_SEND_C_SNAPSHOT, ++ /* Version 1 */ ++ BTRFS_SEND_C_SUBVOL = 1, ++ BTRFS_SEND_C_SNAPSHOT = 2, + +- BTRFS_SEND_C_MKFILE, +- BTRFS_SEND_C_MKDIR, +- BTRFS_SEND_C_MKNOD, +- BTRFS_SEND_C_MKFIFO, +- BTRFS_SEND_C_MKSOCK, +- BTRFS_SEND_C_SYMLINK, ++ BTRFS_SEND_C_MKFILE = 3, ++ BTRFS_SEND_C_MKDIR = 4, ++ BTRFS_SEND_C_MKNOD = 5, ++ BTRFS_SEND_C_MKFIFO = 6, ++ BTRFS_SEND_C_MKSOCK = 7, ++ BTRFS_SEND_C_SYMLINK = 8, + +- BTRFS_SEND_C_RENAME, +- BTRFS_SEND_C_LINK, +- BTRFS_SEND_C_UNLINK, +- BTRFS_SEND_C_RMDIR, ++ BTRFS_SEND_C_RENAME = 9, ++ BTRFS_SEND_C_LINK = 10, ++ BTRFS_SEND_C_UNLINK = 11, ++ BTRFS_SEND_C_RMDIR = 12, + +- BTRFS_SEND_C_SET_XATTR, +- BTRFS_SEND_C_REMOVE_XATTR, ++ BTRFS_SEND_C_SET_XATTR = 13, ++ BTRFS_SEND_C_REMOVE_XATTR = 14, + +- BTRFS_SEND_C_WRITE, +- BTRFS_SEND_C_CLONE, ++ BTRFS_SEND_C_WRITE = 15, ++ BTRFS_SEND_C_CLONE = 16, + +- BTRFS_SEND_C_TRUNCATE, +- BTRFS_SEND_C_CHMOD, +- BTRFS_SEND_C_CHOWN, +- BTRFS_SEND_C_UTIMES, ++ BTRFS_SEND_C_TRUNCATE = 17, ++ BTRFS_SEND_C_CHMOD = 18, ++ BTRFS_SEND_C_CHOWN = 19, ++ BTRFS_SEND_C_UTIMES = 20, + +- BTRFS_SEND_C_END, +- BTRFS_SEND_C_UPDATE_EXTENT, +- __BTRFS_SEND_C_MAX, ++ BTRFS_SEND_C_END = 21, ++ BTRFS_SEND_C_UPDATE_EXTENT = 22, ++ BTRFS_SEND_C_MAX_V1 = 22, ++ ++ /* Version 2 */ ++ BTRFS_SEND_C_FALLOCATE = 23, ++ BTRFS_SEND_C_SETFLAGS = 24, ++ BTRFS_SEND_C_ENCODED_WRITE = 25, ++ BTRFS_SEND_C_MAX_V2 = 25, ++ ++ /* End */ ++ BTRFS_SEND_C_MAX = 25, + }; +-#define BTRFS_SEND_C_MAX (__BTRFS_SEND_C_MAX - 1) + + /* attributes in send stream */ + enum { +- BTRFS_SEND_A_UNSPEC, ++ BTRFS_SEND_A_UNSPEC = 0, + +- BTRFS_SEND_A_UUID, +- BTRFS_SEND_A_CTRANSID, ++ /* Version 1 */ ++ BTRFS_SEND_A_UUID = 1, ++ BTRFS_SEND_A_CTRANSID = 2, + +- BTRFS_SEND_A_INO, +- BTRFS_SEND_A_SIZE, +- BTRFS_SEND_A_MODE, +- BTRFS_SEND_A_UID, +- BTRFS_SEND_A_GID, +- BTRFS_SEND_A_RDEV, +- BTRFS_SEND_A_CTIME, +- BTRFS_SEND_A_MTIME, +- BTRFS_SEND_A_ATIME, +- BTRFS_SEND_A_OTIME, ++ BTRFS_SEND_A_INO = 3, ++ BTRFS_SEND_A_SIZE = 4, ++ BTRFS_SEND_A_MODE = 5, ++ BTRFS_SEND_A_UID = 6, ++ BTRFS_SEND_A_GID = 7, ++ BTRFS_SEND_A_RDEV = 8, ++ BTRFS_SEND_A_CTIME = 9, ++ BTRFS_SEND_A_MTIME = 10, ++ BTRFS_SEND_A_ATIME = 11, ++ BTRFS_SEND_A_OTIME = 12, + +- BTRFS_SEND_A_XATTR_NAME, +- BTRFS_SEND_A_XATTR_DATA, ++ BTRFS_SEND_A_XATTR_NAME = 13, ++ BTRFS_SEND_A_XATTR_DATA = 14, + +- BTRFS_SEND_A_PATH, +- BTRFS_SEND_A_PATH_TO, +- BTRFS_SEND_A_PATH_LINK, ++ BTRFS_SEND_A_PATH = 15, ++ BTRFS_SEND_A_PATH_TO = 16, ++ BTRFS_SEND_A_PATH_LINK = 17, + +- BTRFS_SEND_A_FILE_OFFSET, +- BTRFS_SEND_A_DATA, ++ BTRFS_SEND_A_FILE_OFFSET = 18, ++ /* ++ * As of send stream v2, this attribute is special: it must be the last ++ * attribute in a command, its header contains only the type, and its ++ * length is implicitly the remaining length of the command. ++ */ ++ BTRFS_SEND_A_DATA = 19, + +- BTRFS_SEND_A_CLONE_UUID, +- BTRFS_SEND_A_CLONE_CTRANSID, +- BTRFS_SEND_A_CLONE_PATH, +- BTRFS_SEND_A_CLONE_OFFSET, +- BTRFS_SEND_A_CLONE_LEN, ++ BTRFS_SEND_A_CLONE_UUID = 20, ++ BTRFS_SEND_A_CLONE_CTRANSID = 21, ++ BTRFS_SEND_A_CLONE_PATH = 22, ++ BTRFS_SEND_A_CLONE_OFFSET = 23, ++ BTRFS_SEND_A_CLONE_LEN = 24, + +- __BTRFS_SEND_A_MAX, ++ BTRFS_SEND_A_MAX_V1 = 24, ++ ++ /* Version 2 */ ++ BTRFS_SEND_A_FALLOCATE_MODE = 25, ++ ++ BTRFS_SEND_A_SETFLAGS_FLAGS = 26, ++ ++ BTRFS_SEND_A_UNENCODED_FILE_LEN = 27, ++ BTRFS_SEND_A_UNENCODED_LEN = 28, ++ BTRFS_SEND_A_UNENCODED_OFFSET = 29, ++ /* ++ * COMPRESSION and ENCRYPTION default to NONE (0) if omitted from ++ * BTRFS_SEND_C_ENCODED_WRITE. ++ */ ++ BTRFS_SEND_A_COMPRESSION = 30, ++ BTRFS_SEND_A_ENCRYPTION = 31, ++ BTRFS_SEND_A_MAX_V2 = 31, ++ ++ /* End */ ++ BTRFS_SEND_A_MAX = 31, + }; +-#define BTRFS_SEND_A_MAX (__BTRFS_SEND_A_MAX - 1) + + #ifdef __cplusplus + } +-- +2.35.1 + diff --git a/SOURCES/1106-btrfs-progs-receive-process-encoded_write-commands.patch b/SOURCES/1106-btrfs-progs-receive-process-encoded_write-commands.patch new file mode 100644 index 0000000..072fa48 --- /dev/null +++ b/SOURCES/1106-btrfs-progs-receive-process-encoded_write-commands.patch @@ -0,0 +1,361 @@ +From 3523dd761db148a1214269f6d3dd357231dcbcff Mon Sep 17 00:00:00 2001 +From: Boris Burkov +Date: Fri, 21 Aug 2020 00:40:05 -0700 +Subject: [PATCH] btrfs-progs: receive: process encoded_write commands + +Add a new btrfs_send_op and support for both dumping and proper receive +processing which does actual encoded writes. + +Encoded writes are only allowed on a file descriptor opened with an +extra flag that allows encoded writes, so we also add support for this +flag when opening or reusing a file for writing. + +Signed-off-by: Boris Burkov +--- + cmds/receive-dump.c | 16 +++++- + cmds/receive.c | 48 ++++++++++++++++ + common/send-stream.c | 29 ++++++++++ + common/send-stream.h | 4 ++ + ioctl.h | 132 +++++++++++++++++++++++++++++++++++++++++++ + 5 files changed, 228 insertions(+), 1 deletion(-) + +diff --git a/cmds/receive-dump.c b/cmds/receive-dump.c +index 00ad4fd1..83701b62 100644 +--- a/cmds/receive-dump.c ++++ b/cmds/receive-dump.c +@@ -318,6 +318,19 @@ static int print_update_extent(const char *path, u64 offset, u64 len, + offset, len); + } + ++static int print_encoded_write(const char *path, const void *data, u64 offset, ++ u64 len, u64 unencoded_file_len, ++ u64 unencoded_len, u64 unencoded_offset, ++ u32 compression, u32 encryption, void *user) ++{ ++ return PRINT_DUMP(user, path, "encoded_write", ++ "offset=%llu len=%llu, unencoded_file_len=%llu, " ++ "unencoded_len=%llu, unencoded_offset=%llu, " ++ "compression=%u, encryption=%u", ++ offset, len, unencoded_file_len, unencoded_len, ++ unencoded_offset, compression, encryption); ++} ++ + struct btrfs_send_ops btrfs_print_send_ops = { + .subvol = print_subvol, + .snapshot = print_snapshot, +@@ -339,5 +352,6 @@ struct btrfs_send_ops btrfs_print_send_ops = { + .chmod = print_chmod, + .chown = print_chown, + .utimes = print_utimes, +- .update_extent = print_update_extent ++ .update_extent = print_update_extent, ++ .encoded_write = print_encoded_write, + }; +diff --git a/cmds/receive.c b/cmds/receive.c +index d106e554..8226ca32 100644 +--- a/cmds/receive.c ++++ b/cmds/receive.c +@@ -29,12 +29,14 @@ + #include + #include + #include ++#include + + #include + #include + #include + #include + #include ++#include + #include + #include + +@@ -49,6 +51,7 @@ + #include "cmds/receive-dump.h" + #include "common/help.h" + #include "common/path-utils.h" ++#include "stubs.h" + + struct btrfs_receive + { +@@ -982,6 +985,50 @@ static int process_update_extent(const char *path, u64 offset, u64 len, + return 0; + } + ++static int process_encoded_write(const char *path, const void *data, u64 offset, ++ u64 len, u64 unencoded_file_len, ++ u64 unencoded_len, u64 unencoded_offset, ++ u32 compression, u32 encryption, void *user) ++{ ++ int ret; ++ struct btrfs_receive *rctx = user; ++ char full_path[PATH_MAX]; ++ struct iovec iov = { (char *)data, len }; ++ struct btrfs_ioctl_encoded_io_args encoded = { ++ .iov = &iov, ++ .iovcnt = 1, ++ .offset = offset, ++ .len = unencoded_file_len, ++ .unencoded_len = unencoded_len, ++ .unencoded_offset = unencoded_offset, ++ .compression = compression, ++ .encryption = encryption, ++ }; ++ ++ if (encryption) { ++ error("encoded_write: encryption not supported"); ++ return -EOPNOTSUPP; ++ } ++ ++ ret = path_cat_out(full_path, rctx->full_subvol_path, path); ++ if (ret < 0) { ++ error("encoded_write: path invalid: %s", path); ++ return ret; ++ } ++ ++ ret = open_inode_for_write(rctx, full_path); ++ if (ret < 0) ++ return ret; ++ ++ ret = ioctl(rctx->write_fd, BTRFS_IOC_ENCODED_WRITE, &encoded); ++ if (ret < 0) { ++ ret = -errno; ++ error("encoded_write: writing to %s failed: %m", path); ++ return ret; ++ } ++ return 0; ++} ++ + static struct btrfs_send_ops send_ops = { + .subvol = process_subvol, + .snapshot = process_snapshot, +@@ -1004,6 +1051,7 @@ static struct btrfs_send_ops send_ops = { + .chown = process_chown, + .utimes = process_utimes, + .update_extent = process_update_extent, ++ .encoded_write = process_encoded_write, + }; + + static int do_receive(struct btrfs_receive *rctx, const char *tomnt, +diff --git a/common/send-stream.c b/common/send-stream.c +index 81a830d9..ce7c40f5 100644 +--- a/common/send-stream.c ++++ b/common/send-stream.c +@@ -357,6 +357,8 @@ static int read_and_process_cmd(struct btrfs_send_stream *sctx) + struct timespec mt; + u8 uuid[BTRFS_UUID_SIZE]; + u8 clone_uuid[BTRFS_UUID_SIZE]; ++ u32 compression; ++ u32 encryption; + u64 tmp; + u64 tmp2; + u64 ctransid; +@@ -366,6 +368,9 @@ static int read_and_process_cmd(struct btrfs_send_stream *sctx) + u64 clone_offset; + u64 offset; + u64 ino; ++ u64 unencoded_file_len; ++ u64 unencoded_len; ++ u64 unencoded_offset; + int len; + int xattr_len; + +@@ -452,6 +457,30 @@ static int read_and_process_cmd(struct btrfs_send_stream *sctx) + TLV_GET(sctx, BTRFS_SEND_A_DATA, &data, &len); + ret = sctx->ops->write(path, data, offset, len, sctx->user); + break; ++ case BTRFS_SEND_C_ENCODED_WRITE: ++ TLV_GET_STRING(sctx, BTRFS_SEND_A_PATH, &path); ++ TLV_GET_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, &offset); ++ TLV_GET_U64(sctx, BTRFS_SEND_A_UNENCODED_FILE_LEN, ++ &unencoded_file_len); ++ TLV_GET_U64(sctx, BTRFS_SEND_A_UNENCODED_LEN, &unencoded_len); ++ TLV_GET_U64(sctx, BTRFS_SEND_A_UNENCODED_OFFSET, ++ &unencoded_offset); ++ /* Compression and encryption default to none if omitted. */ ++ if (sctx->cmd_attrs[BTRFS_SEND_A_COMPRESSION].data) ++ TLV_GET_U32(sctx, BTRFS_SEND_A_COMPRESSION, &compression); ++ else ++ compression = BTRFS_ENCODED_IO_COMPRESSION_NONE; ++ if (sctx->cmd_attrs[BTRFS_SEND_A_ENCRYPTION].data) ++ TLV_GET_U32(sctx, BTRFS_SEND_A_ENCRYPTION, &encryption); ++ else ++ encryption = BTRFS_ENCODED_IO_ENCRYPTION_NONE; ++ TLV_GET(sctx, BTRFS_SEND_A_DATA, &data, &len); ++ ret = sctx->ops->encoded_write(path, data, offset, len, ++ unencoded_file_len, ++ unencoded_len, unencoded_offset, ++ compression, encryption, ++ sctx->user); ++ break; + case BTRFS_SEND_C_CLONE: + TLV_GET_STRING(sctx, BTRFS_SEND_A_PATH, &path); + TLV_GET_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, &offset); +diff --git a/common/send-stream.h b/common/send-stream.h +index 2de51eac..44abbc9d 100644 +--- a/common/send-stream.h ++++ b/common/send-stream.h +@@ -53,6 +53,10 @@ struct btrfs_send_ops { + struct timespec *mt, struct timespec *ct, + void *user); + int (*update_extent)(const char *path, u64 offset, u64 len, void *user); ++ int (*encoded_write)(const char *path, const void *data, u64 offset, ++ u64 len, u64 unencoded_file_len, u64 unencoded_len, ++ u64 unencoded_offset, u32 compression, ++ u32 encryption, void *user); + }; + + int btrfs_read_and_process_send_stream(int fd, +diff --git a/ioctl.h b/ioctl.h +index 368a87b2..8adf63c2 100644 +--- a/ioctl.h ++++ b/ioctl.h +@@ -777,6 +777,134 @@ struct btrfs_ioctl_get_subvol_rootref_args { + }; + BUILD_ASSERT(sizeof(struct btrfs_ioctl_get_subvol_rootref_args) == 4096); + ++/* ++ * Data and metadata for an encoded read or write. ++ * ++ * Encoded I/O bypasses any encoding automatically done by the filesystem (e.g., ++ * compression). This can be used to read the compressed contents of a file or ++ * write pre-compressed data directly to a file. ++ * ++ * BTRFS_IOC_ENCODED_READ and BTRFS_IOC_ENCODED_WRITE are essentially ++ * preadv/pwritev with additional metadata about how the data is encoded and the ++ * size of the unencoded data. ++ * ++ * BTRFS_IOC_ENCODED_READ fills the given iovecs with the encoded data, fills ++ * the metadata fields, and returns the size of the encoded data. It reads one ++ * extent per call. It can also read data which is not encoded. ++ * ++ * BTRFS_IOC_ENCODED_WRITE uses the metadata fields, writes the encoded data ++ * from the iovecs, and returns the size of the encoded data. Note that the ++ * encoded data is not validated when it is written; if it is not valid (e.g., ++ * it cannot be decompressed), then a subsequent read may return an error. ++ * ++ * Since the filesystem page cache contains decoded data, encoded I/O bypasses ++ * the page cache. Encoded I/O requires CAP_SYS_ADMIN. ++ */ ++struct btrfs_ioctl_encoded_io_args { ++ /* Input parameters for both reads and writes. */ ++ ++ /* ++ * iovecs containing encoded data. ++ * ++ * For reads, if the size of the encoded data is larger than the sum of ++ * iov[n].iov_len for 0 <= n < iovcnt, then the ioctl fails with ++ * ENOBUFS. ++ * ++ * For writes, the size of the encoded data is the sum of iov[n].iov_len ++ * for 0 <= n < iovcnt. This must be less than 128 KiB (this limit may ++ * increase in the future). This must also be less than or equal to ++ * unencoded_len. ++ */ ++ const struct iovec __user *iov; ++ /* Number of iovecs. */ ++ unsigned long iovcnt; ++ /* ++ * Offset in file. ++ * ++ * For writes, must be aligned to the sector size of the filesystem. ++ */ ++ __s64 offset; ++ /* Currently must be zero. */ ++ __u64 flags; ++ ++ /* ++ * For reads, the following members are output parameters that will ++ * contain the returned metadata for the encoded data. ++ * For writes, the following members must be set to the metadata for the ++ * encoded data. ++ */ ++ ++ /* ++ * Length of the data in the file. ++ * ++ * Must be less than or equal to unencoded_len - unencoded_offset. For ++ * writes, must be aligned to the sector size of the filesystem unless ++ * the data ends at or beyond the current end of the file. ++ */ ++ __u64 len; ++ /* ++ * Length of the unencoded (i.e., decrypted and decompressed) data. ++ * ++ * For writes, must be no more than 128 KiB (this limit may increase in ++ * the future). If the unencoded data is actually longer than ++ * unencoded_len, then it is truncated; if it is shorter, then it is ++ * extended with zeroes. ++ */ ++ __u64 unencoded_len; ++ /* ++ * Offset from the first byte of the unencoded data to the first byte of ++ * logical data in the file. ++ * ++ * Must be less than unencoded_len. ++ */ ++ __u64 unencoded_offset; ++ /* ++ * BTRFS_ENCODED_IO_COMPRESSION_* type. ++ * ++ * For writes, must not be BTRFS_ENCODED_IO_COMPRESSION_NONE. ++ */ ++ __u32 compression; ++ /* Currently always BTRFS_ENCODED_IO_ENCRYPTION_NONE. */ ++ __u32 encryption; ++ /* ++ * Reserved for future expansion. ++ * ++ * For reads, always returned as zero. Users should check for non-zero ++ * bytes. If there are any, then the kernel has a newer version of this ++ * structure with additional information that the user definition is ++ * missing. ++ * ++ * For writes, must be zeroed. ++ */ ++ __u8 reserved[64]; ++}; ++ ++/* Data is not compressed. */ ++#define BTRFS_ENCODED_IO_COMPRESSION_NONE 0 ++/* Data is compressed as a single zlib stream. */ ++#define BTRFS_ENCODED_IO_COMPRESSION_ZLIB 1 ++/* ++ * Data is compressed as a single zstd frame with the windowLog compression ++ * parameter set to no more than 17. ++ */ ++#define BTRFS_ENCODED_IO_COMPRESSION_ZSTD 2 ++/* ++ * Data is compressed sector by sector (using the sector size indicated by the ++ * name of the constant) with LZO1X and wrapped in the format documented in ++ * fs/btrfs/lzo.c. For writes, the compression sector size must match the ++ * filesystem sector size. ++ */ ++#define BTRFS_ENCODED_IO_COMPRESSION_LZO_4K 3 ++#define BTRFS_ENCODED_IO_COMPRESSION_LZO_8K 4 ++#define BTRFS_ENCODED_IO_COMPRESSION_LZO_16K 5 ++#define BTRFS_ENCODED_IO_COMPRESSION_LZO_32K 6 ++#define BTRFS_ENCODED_IO_COMPRESSION_LZO_64K 7 ++#define BTRFS_ENCODED_IO_COMPRESSION_TYPES 8 ++ ++/* Data is not encrypted. */ ++#define BTRFS_ENCODED_IO_ENCRYPTION_NONE 0 ++#define BTRFS_ENCODED_IO_ENCRYPTION_TYPES 1 ++ + /* Error codes as returned by the kernel */ + enum btrfs_err_code { + notused, +@@ -951,6 +1079,10 @@ static inline char *btrfs_err_str(enum btrfs_err_code err_code) + struct btrfs_ioctl_ino_lookup_user_args) + #define BTRFS_IOC_SNAP_DESTROY_V2 _IOW(BTRFS_IOCTL_MAGIC, 63, \ + struct btrfs_ioctl_vol_args_v2) ++#define BTRFS_IOC_ENCODED_READ _IOR(BTRFS_IOCTL_MAGIC, 64, \ ++ struct btrfs_ioctl_encoded_io_args) ++#define BTRFS_IOC_ENCODED_WRITE _IOW(BTRFS_IOCTL_MAGIC, 64, \ ++ struct btrfs_ioctl_encoded_io_args) + + #ifdef __cplusplus + } +-- +2.35.1 + diff --git a/SOURCES/1107-btrfs-progs-receive-encoded_write-fallback-to-explic.patch b/SOURCES/1107-btrfs-progs-receive-encoded_write-fallback-to-explic.patch new file mode 100644 index 0000000..874279e --- /dev/null +++ b/SOURCES/1107-btrfs-progs-receive-encoded_write-fallback-to-explic.patch @@ -0,0 +1,373 @@ +From 8f5d9e7c4a6b9e923a02c1677854e2792822574a Mon Sep 17 00:00:00 2001 +From: Boris Burkov +Date: Fri, 21 Aug 2020 00:40:06 -0700 +Subject: [PATCH] btrfs-progs: receive: encoded_write fallback to explicit + decode and write + +An encoded_write can fail if the file system it is being applied to does +not support encoded writes or if it can't find enough contiguous space +to accommodate the encoded extent. In those cases, we can likely still +process an encoded_write by explicitly decoding the data and doing a +normal write. + +Add the necessary fallback path for decoding data compressed with zlib, +lzo, or zstd. zlib and zstd have reusable decoding context data +structures which we cache in the receive context so that we don't have +to recreate them on every encoded_write. + +Finally, add a command line flag for force-decompress which causes +receive to always use the fallback path rather than first attempting the +encoded write. + +Signed-off-by: Boris Burkov +--- + Documentation/btrfs-receive.rst | 5 + + cmds/receive.c | 261 +++++++++++++++++++++++++++++++- + 2 files changed, 259 insertions(+), 7 deletions(-) + +diff --git a/Documentation/btrfs-receive.rst b/Documentation/btrfs-receive.rst +index 86ffdcc6..b9a3cad6 100644 +--- a/Documentation/btrfs-receive.rst ++++ b/Documentation/btrfs-receive.rst +@@ -57,6 +57,11 @@ A subvolume is made read-only after the receiving process finishes successfully + If */proc* is not accessible, eg. in a chroot environment, use this option to + tell us where this filesystem is mounted. + ++--force-decompress ++ if the stream contains compressed data (see *--compressed-data* in ++ ``btrfs-send(8)``), always decompress it instead of writing it with ++ encoded I/O ++ + --dump + dump the stream metadata, one line per operation + +diff --git a/cmds/receive.c b/cmds/receive.c +index 8226ca32..5fd939ce 100644 +--- a/cmds/receive.c ++++ b/cmds/receive.c +@@ -40,6 +40,10 @@ + #include + #include + ++#include ++#include ++#include ++ + #include "kernel-shared/ctree.h" + #include "ioctl.h" + #include "cmds/commands.h" +@@ -75,6 +79,12 @@ struct btrfs_receive + char cur_subvol_path[PATH_MAX]; + + int honor_end_cmd; ++ ++ bool force_decompress; ++ ++ /* Reuse stream objects for encoded_write decompression fallback */ ++ ZSTD_DStream *zstd_dstream; ++ z_stream *zlib_stream; + }; + + static int finish_subvol(struct btrfs_receive *rctx) +@@ -985,6 +995,219 @@ static int process_update_extent(const char *path, u64 offset, u64 len, + return 0; + } + ++static int decompress_zlib(struct btrfs_receive *rctx, const char *encoded_data, ++ u64 encoded_len, char *unencoded_data, ++ u64 unencoded_len) ++{ ++ bool init = false; ++ int ret; ++ ++ if (!rctx->zlib_stream) { ++ init = true; ++ rctx->zlib_stream = malloc(sizeof(z_stream)); ++ if (!rctx->zlib_stream) { ++ error("failed to allocate zlib stream %m"); ++ return -ENOMEM; ++ } ++ } ++ rctx->zlib_stream->next_in = (void *)encoded_data; ++ rctx->zlib_stream->avail_in = encoded_len; ++ rctx->zlib_stream->next_out = (void *)unencoded_data; ++ rctx->zlib_stream->avail_out = unencoded_len; ++ ++ if (init) { ++ rctx->zlib_stream->zalloc = Z_NULL; ++ rctx->zlib_stream->zfree = Z_NULL; ++ rctx->zlib_stream->opaque = Z_NULL; ++ ret = inflateInit(rctx->zlib_stream); ++ } else { ++ ret = inflateReset(rctx->zlib_stream); ++ } ++ if (ret != Z_OK) { ++ error("zlib inflate init failed: %d", ret); ++ return -EIO; ++ } ++ ++ while (rctx->zlib_stream->avail_in > 0 && ++ rctx->zlib_stream->avail_out > 0) { ++ ret = inflate(rctx->zlib_stream, Z_FINISH); ++ if (ret == Z_STREAM_END) { ++ break; ++ } else if (ret != Z_OK) { ++ error("zlib inflate failed: %d", ret); ++ return -EIO; ++ } ++ } ++ return 0; ++} ++ ++static int decompress_zstd(struct btrfs_receive *rctx, const char *encoded_buf, ++ u64 encoded_len, char *unencoded_buf, ++ u64 unencoded_len) ++{ ++ ZSTD_inBuffer in_buf = { ++ .src = encoded_buf, ++ .size = encoded_len ++ }; ++ ZSTD_outBuffer out_buf = { ++ .dst = unencoded_buf, ++ .size = unencoded_len ++ }; ++ size_t ret; ++ ++ if (!rctx->zstd_dstream) { ++ rctx->zstd_dstream = ZSTD_createDStream(); ++ if (!rctx->zstd_dstream) { ++ error("failed to create zstd dstream"); ++ return -ENOMEM; ++ } ++ } ++ ret = ZSTD_initDStream(rctx->zstd_dstream); ++ if (ZSTD_isError(ret)) { ++ error("failed to init zstd stream: %s", ZSTD_getErrorName(ret)); ++ return -EIO; ++ } ++ while (in_buf.pos < in_buf.size && out_buf.pos < out_buf.size) { ++ ret = ZSTD_decompressStream(rctx->zstd_dstream, &out_buf, &in_buf); ++ if (ret == 0) { ++ break; ++ } else if (ZSTD_isError(ret)) { ++ error("failed to decompress zstd stream: %s", ++ ZSTD_getErrorName(ret)); ++ return -EIO; ++ } ++ } ++ return 0; ++} ++ ++static int decompress_lzo(const char *encoded_data, u64 encoded_len, ++ char *unencoded_data, u64 unencoded_len, ++ unsigned int sector_size) ++{ ++ uint32_t total_len; ++ size_t in_pos, out_pos; ++ ++ if (encoded_len < 4) { ++ error("lzo header is truncated"); ++ return -EIO; ++ } ++ memcpy(&total_len, encoded_data, 4); ++ total_len = le32toh(total_len); ++ if (total_len > encoded_len) { ++ error("lzo header is invalid"); ++ return -EIO; ++ } ++ ++ in_pos = 4; ++ out_pos = 0; ++ while (in_pos < total_len && out_pos < unencoded_len) { ++ size_t sector_remaining; ++ uint32_t src_len; ++ lzo_uint dst_len; ++ int ret; ++ ++ sector_remaining = -in_pos % sector_size; ++ if (sector_remaining < 4) { ++ if (total_len - in_pos <= sector_remaining) ++ break; ++ in_pos += sector_remaining; ++ } ++ ++ if (total_len - in_pos < 4) { ++ error("lzo segment header is truncated"); ++ return -EIO; ++ } ++ ++ memcpy(&src_len, encoded_data + in_pos, 4); ++ src_len = le32toh(src_len); ++ in_pos += 4; ++ if (src_len > total_len - in_pos) { ++ error("lzo segment header is invalid"); ++ return -EIO; ++ } ++ ++ dst_len = sector_size; ++ ret = lzo1x_decompress_safe((void *)(encoded_data + in_pos), ++ src_len, ++ (void *)(unencoded_data + out_pos), ++ &dst_len, NULL); ++ if (ret != LZO_E_OK) { ++ error("lzo1x_decompress_safe failed: %d", ret); ++ return -EIO; ++ } ++ ++ in_pos += src_len; ++ out_pos += dst_len; ++ } ++ return 0; ++} ++ ++static int decompress_and_write(struct btrfs_receive *rctx, ++ const char *encoded_data, u64 offset, ++ u64 encoded_len, u64 unencoded_file_len, ++ u64 unencoded_len, u64 unencoded_offset, ++ u32 compression) ++{ ++ int ret = 0; ++ size_t pos; ++ ssize_t w; ++ char *unencoded_data; ++ int sector_shift; ++ ++ unencoded_data = calloc(unencoded_len, 1); ++ if (!unencoded_data) { ++ error("allocating space for unencoded data failed: %m"); ++ return -errno; ++ } ++ ++ switch (compression) { ++ case BTRFS_ENCODED_IO_COMPRESSION_ZLIB: ++ ret = decompress_zlib(rctx, encoded_data, encoded_len, ++ unencoded_data, unencoded_len); ++ if (ret) ++ goto out; ++ break; ++ case BTRFS_ENCODED_IO_COMPRESSION_ZSTD: ++ ret = decompress_zstd(rctx, encoded_data, encoded_len, ++ unencoded_data, unencoded_len); ++ if (ret) ++ goto out; ++ break; ++ case BTRFS_ENCODED_IO_COMPRESSION_LZO_4K: ++ case BTRFS_ENCODED_IO_COMPRESSION_LZO_8K: ++ case BTRFS_ENCODED_IO_COMPRESSION_LZO_16K: ++ case BTRFS_ENCODED_IO_COMPRESSION_LZO_32K: ++ case BTRFS_ENCODED_IO_COMPRESSION_LZO_64K: ++ sector_shift = ++ compression - BTRFS_ENCODED_IO_COMPRESSION_LZO_4K + 12; ++ ret = decompress_lzo(encoded_data, encoded_len, unencoded_data, ++ unencoded_len, 1U << sector_shift); ++ if (ret) ++ goto out; ++ break; ++ default: ++ error("unknown compression: %d", compression); ++ ret = -EOPNOTSUPP; ++ goto out; ++ } ++ ++ pos = unencoded_offset; ++ while (pos < unencoded_file_len) { ++ w = pwrite(rctx->write_fd, unencoded_data + pos, ++ unencoded_file_len - pos, offset); ++ if (w < 0) { ++ ret = -errno; ++ error("writing unencoded data failed: %m"); ++ goto out; ++ } ++ pos += w; ++ offset += w; ++ } ++out: ++ free(unencoded_data); ++ return ret; ++} ++ + static int process_encoded_write(const char *path, const void *data, u64 offset, + u64 len, u64 unencoded_file_len, + u64 unencoded_len, u64 unencoded_offset, +@@ -1020,13 +1243,21 @@ static int process_encoded_write(const char *path, const void *data, u64 offset, + if (ret < 0) + return ret; + +- ret = ioctl(rctx->write_fd, BTRFS_IOC_ENCODED_WRITE, &encoded); +- if (ret < 0) { +- ret = -errno; +- error("encoded_write: writing to %s failed: %m", path); +- return ret; ++ if (!rctx->force_decompress) { ++ ret = ioctl(rctx->write_fd, BTRFS_IOC_ENCODED_WRITE, &encoded); ++ if (ret >= 0) ++ return 0; ++ /* Fall back for these errors, fail hard for anything else. */ ++ if (errno != ENOSPC && errno != ENOTTY && errno != EINVAL) { ++ ret = -errno; ++ error("encoded_write: writing to %s failed: %m", path); ++ return ret; ++ } + } +- return 0; ++ ++ return decompress_and_write(rctx, data, offset, len, unencoded_file_len, ++ unencoded_len, unencoded_offset, ++ compression); + } + + static struct btrfs_send_ops send_ops = { +@@ -1204,6 +1435,12 @@ out: + close(rctx->dest_dir_fd); + rctx->dest_dir_fd = -1; + } ++ if (rctx->zstd_dstream) ++ ZSTD_freeDStream(rctx->zstd_dstream); ++ if (rctx->zlib_stream) { ++ inflateEnd(rctx->zlib_stream); ++ free(rctx->zlib_stream); ++ } + + return ret; + } +@@ -1234,6 +1471,9 @@ static const char * const cmd_receive_usage[] = { + "-m ROOTMOUNT the root mount point of the destination filesystem.", + " If /proc is not accessible, use this to tell us where", + " this file system is mounted.", ++ "--force-decompress", ++ " if the stream contains compressed data, always", ++ " decompress it instead of writing it with encoded I/O", + "--dump dump stream metadata, one line per operation,", + " does not require the MOUNT parameter", + "-v deprecated, alias for global -v option", +@@ -1277,12 +1517,16 @@ static int cmd_receive(const struct cmd_struct *cmd, int argc, char **argv) + optind = 0; + while (1) { + int c; +- enum { GETOPT_VAL_DUMP = 257 }; ++ enum { ++ GETOPT_VAL_DUMP = 257, ++ GETOPT_VAL_FORCE_DECOMPRESS, ++ }; + static const struct option long_opts[] = { + { "max-errors", required_argument, NULL, 'E' }, + { "chroot", no_argument, NULL, 'C' }, + { "dump", no_argument, NULL, GETOPT_VAL_DUMP }, + { "quiet", no_argument, NULL, 'q' }, ++ { "force-decompress", no_argument, NULL, GETOPT_VAL_FORCE_DECOMPRESS }, + { NULL, 0, NULL, 0 } + }; + +@@ -1325,6 +1569,9 @@ static int cmd_receive(const struct cmd_struct *cmd, int argc, char **argv) + case GETOPT_VAL_DUMP: + dump = 1; + break; ++ case GETOPT_VAL_FORCE_DECOMPRESS: ++ rctx.force_decompress = true; ++ break; + default: + usage_unknown_option(cmd, argv); + } +-- +2.35.1 + diff --git a/SOURCES/1108-btrfs-progs-receive-process-fallocate-commands.patch b/SOURCES/1108-btrfs-progs-receive-process-fallocate-commands.patch new file mode 100644 index 0000000..f9d2582 --- /dev/null +++ b/SOURCES/1108-btrfs-progs-receive-process-fallocate-commands.patch @@ -0,0 +1,130 @@ +From 3e92b1e0b97146516f4287f5e42428a3b7dc4a3d Mon Sep 17 00:00:00 2001 +From: Boris Burkov +Date: Fri, 21 Aug 2020 00:40:07 -0700 +Subject: [PATCH] btrfs-progs: receive: process fallocate commands + +Send stream v2 can emit fallocate commands, so receive must support them +as well. The implementation simply passes along the arguments to the +syscall. Note that mode is encoded as a u32 in send stream but fallocate +takes an int, so there is a unsigned->signed conversion there. + +Reviewed-by: Nikolay Borisov +Signed-off-by: Boris Burkov +--- + cmds/receive-dump.c | 9 +++++++++ + cmds/receive.c | 25 +++++++++++++++++++++++++ + common/send-stream.c | 9 +++++++++ + common/send-stream.h | 2 ++ + 4 files changed, 45 insertions(+) + +diff --git a/cmds/receive-dump.c b/cmds/receive-dump.c +index 83701b62..fa397bcf 100644 +--- a/cmds/receive-dump.c ++++ b/cmds/receive-dump.c +@@ -331,6 +331,14 @@ static int print_encoded_write(const char *path, const void *data, u64 offset, + unencoded_offset, compression, encryption); + } + ++static int print_fallocate(const char *path, int mode, u64 offset, u64 len, ++ void *user) ++{ ++ return PRINT_DUMP(user, path, "fallocate", ++ "mode=%d offset=%llu len=%llu", ++ mode, offset, len); ++} ++ + struct btrfs_send_ops btrfs_print_send_ops = { + .subvol = print_subvol, + .snapshot = print_snapshot, +@@ -354,4 +362,5 @@ struct btrfs_send_ops btrfs_print_send_ops = { + .utimes = print_utimes, + .update_extent = print_update_extent, + .encoded_write = print_encoded_write, ++ .fallocate = print_fallocate, + }; +diff --git a/cmds/receive.c b/cmds/receive.c +index 5fd939ce..4893d693 100644 +--- a/cmds/receive.c ++++ b/cmds/receive.c +@@ -1260,6 +1260,30 @@ static int process_encoded_write(const char *path, const void *data, u64 offset, + compression); + } + ++static int process_fallocate(const char *path, int mode, u64 offset, u64 len, ++ void *user) ++{ ++ int ret; ++ struct btrfs_receive *rctx = user; ++ char full_path[PATH_MAX]; ++ ++ ret = path_cat_out(full_path, rctx->full_subvol_path, path); ++ if (ret < 0) { ++ error("fallocate: path invalid: %s", path); ++ return ret; ++ } ++ ret = open_inode_for_write(rctx, full_path); ++ if (ret < 0) ++ return ret; ++ ret = fallocate(rctx->write_fd, mode, offset, len); ++ if (ret < 0) { ++ ret = -errno; ++ error("fallocate: fallocate on %s failed: %m", path); ++ return ret; ++ } ++ return 0; ++} ++ + static struct btrfs_send_ops send_ops = { + .subvol = process_subvol, + .snapshot = process_snapshot, +@@ -1283,6 +1307,7 @@ static struct btrfs_send_ops send_ops = { + .utimes = process_utimes, + .update_extent = process_update_extent, + .encoded_write = process_encoded_write, ++ .fallocate = process_fallocate, + }; + + static int do_receive(struct btrfs_receive *rctx, const char *tomnt, +diff --git a/common/send-stream.c b/common/send-stream.c +index ce7c40f5..2d0aa624 100644 +--- a/common/send-stream.c ++++ b/common/send-stream.c +@@ -373,6 +373,7 @@ static int read_and_process_cmd(struct btrfs_send_stream *sctx) + u64 unencoded_offset; + int len; + int xattr_len; ++ int fallocate_mode; + + ret = read_cmd(sctx); + if (ret) +@@ -537,6 +538,14 @@ static int read_and_process_cmd(struct btrfs_send_stream *sctx) + case BTRFS_SEND_C_END: + ret = 1; + break; ++ case BTRFS_SEND_C_FALLOCATE: ++ TLV_GET_STRING(sctx, BTRFS_SEND_A_PATH, &path); ++ TLV_GET_U32(sctx, BTRFS_SEND_A_FALLOCATE_MODE, &fallocate_mode); ++ TLV_GET_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, &offset); ++ TLV_GET_U64(sctx, BTRFS_SEND_A_SIZE, &tmp); ++ ret = sctx->ops->fallocate(path, fallocate_mode, offset, tmp, ++ sctx->user); ++ break; + } + + tlv_get_failed: +diff --git a/common/send-stream.h b/common/send-stream.h +index 44abbc9d..61a88d3d 100644 +--- a/common/send-stream.h ++++ b/common/send-stream.h +@@ -57,6 +57,8 @@ struct btrfs_send_ops { + u64 len, u64 unencoded_file_len, u64 unencoded_len, + u64 unencoded_offset, u32 compression, + u32 encryption, void *user); ++ int (*fallocate)(const char *path, int mode, u64 offset, u64 len, ++ void *user); + }; + + int btrfs_read_and_process_send_stream(int fd, +-- +2.35.1 + diff --git a/SOURCES/1109-btrfs-progs-receive-process-setflags-ioctl-commands.patch b/SOURCES/1109-btrfs-progs-receive-process-setflags-ioctl-commands.patch new file mode 100644 index 0000000..85b0d42 --- /dev/null +++ b/SOURCES/1109-btrfs-progs-receive-process-setflags-ioctl-commands.patch @@ -0,0 +1,132 @@ +From 6e8d60a8c9ab22c489c0707ffd854bf1db9852f4 Mon Sep 17 00:00:00 2001 +From: Boris Burkov +Date: Fri, 21 Aug 2020 00:40:08 -0700 +Subject: [PATCH] btrfs-progs: receive: process setflags ioctl commands + +In send stream v2, send can emit a command for setting inode flags via +the setflags ioctl. Pass the flags attribute through to the ioctl call +in receive. + +Reviewed-by: Nikolay Borisov +Signed-off-by: Boris Burkov +--- + cmds/receive-dump.c | 6 ++++++ + cmds/receive.c | 25 +++++++++++++++++++++++++ + common/send-stream.c | 7 +++++++ + common/send-stream.h | 1 + + 4 files changed, 39 insertions(+) + +diff --git a/cmds/receive-dump.c b/cmds/receive-dump.c +index fa397bcf..df5991e1 100644 +--- a/cmds/receive-dump.c ++++ b/cmds/receive-dump.c +@@ -339,6 +339,11 @@ static int print_fallocate(const char *path, int mode, u64 offset, u64 len, + mode, offset, len); + } + ++static int print_setflags(const char *path, int flags, void *user) ++{ ++ return PRINT_DUMP(user, path, "setflags", "flags=%d", flags); ++} ++ + struct btrfs_send_ops btrfs_print_send_ops = { + .subvol = print_subvol, + .snapshot = print_snapshot, +@@ -363,4 +368,5 @@ struct btrfs_send_ops btrfs_print_send_ops = { + .update_extent = print_update_extent, + .encoded_write = print_encoded_write, + .fallocate = print_fallocate, ++ .setflags = print_setflags, + }; +diff --git a/cmds/receive.c b/cmds/receive.c +index 4893d693..7f76a04f 100644 +--- a/cmds/receive.c ++++ b/cmds/receive.c +@@ -38,6 +38,7 @@ + #include + #include + #include ++#include + #include + + #include +@@ -1284,6 +1285,29 @@ static int process_fallocate(const char *path, int mode, u64 offset, u64 len, + return 0; + } + ++static int process_setflags(const char *path, int flags, void *user) ++{ ++ int ret; ++ struct btrfs_receive *rctx = user; ++ char full_path[PATH_MAX]; ++ ++ ret = path_cat_out(full_path, rctx->full_subvol_path, path); ++ if (ret < 0) { ++ error("setflags: path invalid: %s", path); ++ return ret; ++ } ++ ret = open_inode_for_write(rctx, full_path); ++ if (ret < 0) ++ return ret; ++ ret = ioctl(rctx->write_fd, FS_IOC_SETFLAGS, &flags); ++ if (ret < 0) { ++ ret = -errno; ++ error("setflags: setflags ioctl on %s failed: %m", path); ++ return ret; ++ } ++ return 0; ++} ++ + static struct btrfs_send_ops send_ops = { + .subvol = process_subvol, + .snapshot = process_snapshot, +@@ -1308,6 +1332,7 @@ static struct btrfs_send_ops send_ops = { + .update_extent = process_update_extent, + .encoded_write = process_encoded_write, + .fallocate = process_fallocate, ++ .setflags = process_setflags, + }; + + static int do_receive(struct btrfs_receive *rctx, const char *tomnt, +diff --git a/common/send-stream.c b/common/send-stream.c +index 2d0aa624..21295cbb 100644 +--- a/common/send-stream.c ++++ b/common/send-stream.c +@@ -374,6 +374,7 @@ static int read_and_process_cmd(struct btrfs_send_stream *sctx) + int len; + int xattr_len; + int fallocate_mode; ++ int setflags_flags; + + ret = read_cmd(sctx); + if (ret) +@@ -546,8 +547,14 @@ static int read_and_process_cmd(struct btrfs_send_stream *sctx) + ret = sctx->ops->fallocate(path, fallocate_mode, offset, tmp, + sctx->user); + break; ++ case BTRFS_SEND_C_SETFLAGS: ++ TLV_GET_STRING(sctx, BTRFS_SEND_A_PATH, &path); ++ TLV_GET_U32(sctx, BTRFS_SEND_A_SETFLAGS_FLAGS, &setflags_flags); ++ ret = sctx->ops->setflags(path, setflags_flags, sctx->user); ++ break; + } + ++ + tlv_get_failed: + out: + free(path); +diff --git a/common/send-stream.h b/common/send-stream.h +index 61a88d3d..3189f889 100644 +--- a/common/send-stream.h ++++ b/common/send-stream.h +@@ -59,6 +59,7 @@ struct btrfs_send_ops { + u32 encryption, void *user); + int (*fallocate)(const char *path, int mode, u64 offset, u64 len, + void *user); ++ int (*setflags)(const char *path, int flags, void *user); + }; + + int btrfs_read_and_process_send_stream(int fd, +-- +2.35.1 + diff --git a/SOURCES/1110-btrfs-progs-send-stream-v2-ioctl-flags.patch b/SOURCES/1110-btrfs-progs-send-stream-v2-ioctl-flags.patch new file mode 100644 index 0000000..c804c8a --- /dev/null +++ b/SOURCES/1110-btrfs-progs-send-stream-v2-ioctl-flags.patch @@ -0,0 +1,297 @@ +From caaa44c482a3aae46d8a883cc60b6caf2d47bc75 Mon Sep 17 00:00:00 2001 +From: Omar Sandoval +Date: Wed, 9 Feb 2022 17:10:59 -0800 +Subject: [PATCH] btrfs-progs: send: stream v2 ioctl flags + +First, add a --proto option to allow specifying the desired send +protocol version. It defaults to one, the original version. In a couple +of releases once people are aware that protocol revisions are happening, +we can change it to default to zero, which means the latest version +supported by the kernel. This is based on Dave Sterba's patch. + +Also add a --compressed-data flag to instruct the kernel to use +encoded_write commands for compressed extents. This requires an explicit +opt in separate from the protocol version because: + +1. The user may not want compression on the receiving side, or may want + a different compression algorithm/level on the receiving side. +2. It has a soft requirement for kernel support on the receiving side + (btrfs-progs can fall back to decompressing and writing if the kernel + doesn't support BTRFS_IOC_ENCODED_WRITE, but the user may not be + prepared to pay that CPU cost). Going forward, since it's easier to + update progs than the kernel, I think we'll want to make new send + features that require kernel support opt-in, whereas anything that + only requires a progs update can happen automatically. + +Signed-off-by: Boris Burkov +Signed-off-by: Omar Sandoval +--- + Documentation/btrfs-send.rst | 22 ++++++++ + cmds/send.c | 100 ++++++++++++++++++++++++++++++++++- + ioctl.h | 19 ++++++- + kernel-shared/send.h | 2 +- + 4 files changed, 138 insertions(+), 5 deletions(-) + +diff --git a/Documentation/btrfs-send.rst b/Documentation/btrfs-send.rst +index 4526532e..291c537e 100644 +--- a/Documentation/btrfs-send.rst ++++ b/Documentation/btrfs-send.rst +@@ -60,6 +60,28 @@ please see section *SUBVOLUME FLAGS* in ``btrfs-subvolume(8)``. + used to transfer changes. This mode is faster and is useful to show the + differences in metadata. + ++--proto ++ use send protocol version N ++ ++ The default is 1, which was the original protocol version. Version 2 ++ encodes file data slightly more efficiently; it is also required for ++ sending compressed data directly (see *--compressed-data*). Version 2 ++ requires at least btrfs-progs 5.18 on both the sender and receiver and ++ at least Linux 5.18 on the sender. Passing 0 means to use the highest ++ version supported by the running kernel. ++ ++--compressed-data ++ send data that is compressed on the filesystem directly without ++ decompressing it ++ ++ If the receiver supports the *BTRFS_IOC_ENCODED_WRITE* ioctl (added in ++ Linux 5.18), it can also write it directly without decompressing it. ++ Otherwise, the receiver will fall back to decompressing it and writing ++ it normally. ++ ++ This requires protocol version 2 or higher. If *--proto* was not used, ++ then *--compressed-data* implies *--proto 2*. ++ + -q|--quiet + (deprecated) alias for global *-q* option + +diff --git a/cmds/send.c b/cmds/send.c +index 087af05c..b1adfeca 100644 +--- a/cmds/send.c ++++ b/cmds/send.c +@@ -57,6 +57,8 @@ struct btrfs_send { + u64 clone_sources_count; + + char *root_path; ++ u32 proto; ++ u32 proto_supported; + }; + + static int get_root_id(struct btrfs_send *sctx, const char *path, u64 *root_id) +@@ -259,6 +261,16 @@ static int do_send(struct btrfs_send *send, u64 parent_root_id, + memset(&io_send, 0, sizeof(io_send)); + io_send.send_fd = pipefd[1]; + send->send_fd = pipefd[0]; ++ io_send.flags = flags; ++ ++ if (send->proto_supported > 1) { ++ /* ++ * Versioned stream supported, requesting default or specific ++ * number. ++ */ ++ io_send.version = send->proto; ++ io_send.flags |= BTRFS_SEND_FLAG_VERSION; ++ } + + if (!ret) + ret = pthread_create(&t_read, NULL, read_sent_data, send); +@@ -269,7 +281,6 @@ static int do_send(struct btrfs_send *send, u64 parent_root_id, + goto out; + } + +- io_send.flags = flags; + io_send.clone_sources = (__u64*)send->clone_sources; + io_send.clone_sources_count = send->clone_sources_count; + io_send.parent_root = parent_root_id; +@@ -421,6 +432,36 @@ static void free_send_info(struct btrfs_send *sctx) + sctx->root_path = NULL; + } + ++static u32 get_sysfs_proto_supported(void) ++{ ++ int fd; ++ int ret; ++ char buf[32] = {}; ++ char *end = NULL; ++ u64 version; ++ ++ fd = sysfs_open_file("features/send_stream_version"); ++ if (fd < 0) { ++ /* ++ * No file is either no version support or old kernel with just ++ * v1. ++ */ ++ return 1; ++ } ++ ret = sysfs_read_file(fd, buf, sizeof(buf)); ++ close(fd); ++ if (ret <= 0) ++ return 1; ++ version = strtoull(buf, &end, 10); ++ if (version == ULLONG_MAX && errno == ERANGE) ++ return 1; ++ if (version > U32_MAX) { ++ warning("sysfs/send_stream_version too big: %llu", version); ++ version = 1; ++ } ++ return version; ++} ++ + static const char * const cmd_send_usage[] = { + "btrfs send [-ve] [-p ] [-c ] [-f ] [...]", + "Send the subvolume(s) to stdout.", +@@ -449,6 +490,11 @@ static const char * const cmd_send_usage[] = { + " does not contain any file data and thus cannot be used", + " to transfer changes. This mode is faster and useful to", + " show the differences in metadata.", ++ "--proto N use protocol version N, or 0 to use the highest version", ++ " supported by the sending kernel (default: 1)", ++ "--compressed-data", ++ " send data that is compressed on the filesystem directly", ++ " without decompressing it", + "-v|--verbose deprecated, alias for global -v option", + "-q|--quiet deprecated, alias for global -q option", + HELPINFO_INSERT_GLOBALS, +@@ -471,9 +517,11 @@ static int cmd_send(const struct cmd_struct *cmd, int argc, char **argv) + int full_send = 1; + int new_end_cmd_semantic = 0; + u64 send_flags = 0; ++ u64 proto = 0; + + memset(&send, 0, sizeof(send)); + send.dump_fd = fileno(stdout); ++ send.proto = 1; + outname[0] = 0; + + /* +@@ -489,11 +537,17 @@ static int cmd_send(const struct cmd_struct *cmd, int argc, char **argv) + + optind = 0; + while (1) { +- enum { GETOPT_VAL_SEND_NO_DATA = 256 }; ++ enum { ++ GETOPT_VAL_SEND_NO_DATA = 256, ++ GETOPT_VAL_PROTO, ++ GETOPT_VAL_COMPRESSED_DATA, ++ }; + static const struct option long_options[] = { + { "verbose", no_argument, NULL, 'v' }, + { "quiet", no_argument, NULL, 'q' }, + { "no-data", no_argument, NULL, GETOPT_VAL_SEND_NO_DATA }, ++ { "proto", required_argument, NULL, GETOPT_VAL_PROTO }, ++ { "compressed-data", no_argument, NULL, GETOPT_VAL_COMPRESSED_DATA }, + { NULL, 0, NULL, 0 } + }; + int c = getopt_long(argc, argv, "vqec:f:i:p:", long_options, NULL); +@@ -582,6 +636,18 @@ static int cmd_send(const struct cmd_struct *cmd, int argc, char **argv) + case GETOPT_VAL_SEND_NO_DATA: + send_flags |= BTRFS_SEND_FLAG_NO_FILE_DATA; + break; ++ case GETOPT_VAL_PROTO: ++ proto = arg_strtou64(optarg); ++ if (proto > U32_MAX) { ++ error("protocol version number too big %llu", proto); ++ ret = 1; ++ goto out; ++ } ++ send.proto = proto; ++ break; ++ case GETOPT_VAL_COMPRESSED_DATA: ++ send_flags |= BTRFS_SEND_FLAG_COMPRESSED; ++ break; + default: + usage_unknown_option(cmd, argv); + } +@@ -689,6 +755,36 @@ static int cmd_send(const struct cmd_struct *cmd, int argc, char **argv) + if ((send_flags & BTRFS_SEND_FLAG_NO_FILE_DATA) && bconf.verbose > 1) + if (bconf.verbose > 1) + fprintf(stderr, "Mode NO_FILE_DATA enabled\n"); ++ send.proto_supported = get_sysfs_proto_supported(); ++ if (send.proto_supported == 1) { ++ if (send.proto > send.proto_supported) { ++ error("requested version %u but kernel supports only %u", ++ send.proto, send.proto_supported); ++ ret = -EPROTO; ++ goto out; ++ } ++ } ++ if (send_flags & BTRFS_SEND_FLAG_COMPRESSED) { ++ /* ++ * If no protocol version was explicitly requested, then ++ * --compressed-data implies --proto 2. ++ */ ++ if (send.proto == 1 && !proto) ++ send.proto = 2; ++ ++ if (send.proto == 1) { ++ error("--compressed-data requires protocol version >= 2 (requested 1)"); ++ ret = -EINVAL; ++ goto out; ++ } else if (send.proto == 0 && send.proto_supported < 2) { ++ error("kernel does not support --compressed-data"); ++ ret = -EINVAL; ++ goto out; ++ } ++ } ++ if (bconf.verbose > 1) ++ fprintf(stderr, "Protocol version requested: %u (supported %u)\n", ++ send.proto, send.proto_supported); + + for (i = optind; i < argc; i++) { + int is_first_subvol; +diff --git a/ioctl.h b/ioctl.h +index 8adf63c2..f19695e3 100644 +--- a/ioctl.h ++++ b/ioctl.h +@@ -655,10 +655,24 @@ BUILD_ASSERT(sizeof(struct btrfs_ioctl_received_subvol_args_32) == 192); + */ + #define BTRFS_SEND_FLAG_OMIT_END_CMD 0x4 + ++/* ++ * Read the protocol version in the structure ++ */ ++#define BTRFS_SEND_FLAG_VERSION 0x8 ++ ++/* ++ * Send compressed data using the ENCODED_WRITE command instead of decompressing ++ * the data and sending it with the WRITE command. This requires protocol ++ * version >= 2. ++ */ ++#define BTRFS_SEND_FLAG_COMPRESSED 0x10 ++ + #define BTRFS_SEND_FLAG_MASK \ + (BTRFS_SEND_FLAG_NO_FILE_DATA | \ + BTRFS_SEND_FLAG_OMIT_STREAM_HEADER | \ +- BTRFS_SEND_FLAG_OMIT_END_CMD) ++ BTRFS_SEND_FLAG_OMIT_END_CMD | \ ++ BTRFS_SEND_FLAG_VERSION | \ ++ BTRFS_SEND_FLAG_COMPRESSED) + + struct btrfs_ioctl_send_args { + __s64 send_fd; /* in */ +@@ -666,7 +680,8 @@ struct btrfs_ioctl_send_args { + __u64 __user *clone_sources; /* in */ + __u64 parent_root; /* in */ + __u64 flags; /* in */ +- __u64 reserved[4]; /* in */ ++ __u32 version; /* in */ ++ __u8 reserved[28]; /* in */ + }; + /* + * Size of structure depends on pointer width, was not caught in the early +diff --git a/kernel-shared/send.h b/kernel-shared/send.h +index b902d054..1f20d01a 100644 +--- a/kernel-shared/send.h ++++ b/kernel-shared/send.h +@@ -31,7 +31,7 @@ extern "C" { + #endif + + #define BTRFS_SEND_STREAM_MAGIC "btrfs-stream" +-#define BTRFS_SEND_STREAM_VERSION 1 ++#define BTRFS_SEND_STREAM_VERSION 2 + + /* + * In send stream v1, no command is larger than 64k. In send stream v2, no limit +-- +2.35.1 + diff --git a/SOURCES/1111-btrfs-progs-receive-add-tests-for-basic-encoded_writ.patch b/SOURCES/1111-btrfs-progs-receive-add-tests-for-basic-encoded_writ.patch new file mode 100644 index 0000000..cfca478 --- /dev/null +++ b/SOURCES/1111-btrfs-progs-receive-add-tests-for-basic-encoded_writ.patch @@ -0,0 +1,141 @@ +From 6b974c6b3235cb83eb3e8c30a47706529f88d197 Mon Sep 17 00:00:00 2001 +From: Boris Burkov +Date: Fri, 21 Aug 2020 00:40:10 -0700 +Subject: [PATCH] btrfs-progs: receive: add tests for basic encoded_write + send/receive + +Adapt the existing send/receive tests by passing '-o compress-force' to +the mount commands in a new test. After writing a few files in the +various compression formats, send/receive them with and without +--force-decompress to test both the encoded_write path and the fallback +to decode+write. + +Signed-off-by: Boris Burkov +--- + .../052-receive-write-encoded/test.sh | 114 ++++++++++++++++++ + 1 file changed, 114 insertions(+) + create mode 100755 tests/misc-tests/052-receive-write-encoded/test.sh + +diff --git a/tests/misc-tests/052-receive-write-encoded/test.sh b/tests/misc-tests/052-receive-write-encoded/test.sh +new file mode 100755 +index 00000000..47330281 +--- /dev/null ++++ b/tests/misc-tests/052-receive-write-encoded/test.sh +@@ -0,0 +1,114 @@ ++#!/bin/bash ++# ++# test that we can send and receive encoded writes for three modes of ++# transparent compression: zlib, lzo, and zstd. ++ ++source "$TEST_TOP/common" ++ ++check_prereq mkfs.btrfs ++check_prereq btrfs ++ ++setup_root_helper ++prepare_test_dev ++ ++here=`pwd` ++ ++# assumes the filesystem exists, and does mount, write, snapshot, send, unmount ++# for the specified encoding option ++send_one() { ++ local str ++ local subv ++ local snap ++ ++ algorithm="$1" ++ shift ++ str="$1" ++ shift ++ ++ subv="subv-$algorithm" ++ snap="snap-$algorithm" ++ ++ run_check_mount_test_dev "-o" "compress-force=$algorithm" ++ cd "$TEST_MNT" || _fail "cannot chdir to TEST_MNT" ++ ++ run_check $SUDO_HELPER "$TOP/btrfs" subvolume create "$subv" ++ run_check $SUDO_HELPER dd if=/dev/zero of="$subv/file1" bs=1M count=1 ++ run_check $SUDO_HELPER dd if=/dev/zero of="$subv/file2" bs=500K count=1 ++ run_check $SUDO_HELPER "$TOP/btrfs" subvolume snapshot -r "$subv" "$snap" ++ run_check $SUDO_HELPER "$TOP/btrfs" send -f "$str" "$snap" "$@" ++ ++ cd "$here" || _fail "cannot chdir back to test directory" ++ run_check_umount_test_dev ++} ++ ++receive_one() { ++ local str ++ str="$1" ++ shift ++ ++ run_check_mkfs_test_dev ++ run_check_mount_test_dev ++ run_check $SUDO_HELPER "$TOP/btrfs" receive "$@" -v -f "$str" "$TEST_MNT" ++ run_check_umount_test_dev ++ run_check rm -f -- "$str" ++} ++ ++test_one_write_encoded() { ++ local str ++ local algorithm ++ algorithm="$1" ++ shift ++ str="$here/stream-$algorithm.stream" ++ ++ run_check_mkfs_test_dev ++ send_one "$algorithm" "$str" --compressed-data ++ receive_one "$str" "$@" ++} ++ ++test_one_stream_v1() { ++ local str ++ local algorithm ++ algorithm="$1" ++ shift ++ str="$here/stream-$algorithm.stream" ++ ++ run_check_mkfs_test_dev ++ send_one "$algorithm" "$str" --proto 1 ++ receive_one "$str" "$@" ++} ++ ++test_mix_write_encoded() { ++ local strzlib ++ local strlzo ++ local strzstd ++ strzlib="$here/stream-zlib.stream" ++ strlzo="$here/stream-lzo.stream" ++ strzstd="$here/stream-zstd.stream" ++ ++ run_check_mkfs_test_dev ++ ++ send_one "zlib" "$strzlib" --compressed-data ++ send_one "lzo" "$strlzo" --compressed-data ++ send_one "zstd" "$strzstd" --compressed-data ++ ++ receive_one "$strzlib" ++ receive_one "$strlzo" ++ receive_one "$strzstd" ++} ++ ++test_one_write_encoded "zlib" ++test_one_write_encoded "lzo" ++test_one_write_encoded "zstd" ++ ++# with decompression forced ++test_one_write_encoded "zlib" "--force-decompress" ++test_one_write_encoded "lzo" "--force-decompress" ++test_one_write_encoded "zstd" "--force-decompress" ++ ++# send stream v1 ++test_one_stream_v1 "zlib" ++test_one_stream_v1 "lzo" ++test_one_stream_v1 "zstd" ++ ++# files use a mix of compression algorithms ++test_mix_write_encoded +-- +2.35.1 + diff --git a/SPECS/btrfs-progs.spec b/SPECS/btrfs-progs.spec index ddd8f79..fb94cb8 100644 --- a/SPECS/btrfs-progs.spec +++ b/SPECS/btrfs-progs.spec @@ -1,5 +1,5 @@ Name: btrfs-progs -Version: 5.14.2 +Version: 5.16.2 Release: 1.1%{?dist} Summary: Userspace programs for btrfs @@ -17,6 +17,16 @@ Source2: gpgkey-F2B41200C54EFB30380C1756C565D5F9D76D583B.gpg Patch1001: 1001-balance-mkfs-Disable-raid56-modes.patch %if 0%{?facebook} Patch1101: 1101-Require-clowntown-for-quota-enable.patch +Patch1102: 1102-btrfs-progs-receive-support-v2-send-stream-larger-tl.patch +Patch1103: 1103-btrfs-progs-receive-dynamically-allocate-sctx-read_b.patch +Patch1104: 1104-btrfs-progs-receive-support-v2-send-stream-DATA-tlv-.patch +Patch1105: 1105-btrfs-progs-receive-add-send-stream-v2-cmds-and-attr.patch +Patch1106: 1106-btrfs-progs-receive-process-encoded_write-commands.patch +Patch1107: 1107-btrfs-progs-receive-encoded_write-fallback-to-explic.patch +Patch1108: 1108-btrfs-progs-receive-process-fallocate-commands.patch +Patch1109: 1109-btrfs-progs-receive-process-setflags-ioctl-commands.patch +Patch1110: 1110-btrfs-progs-send-stream-v2-ioctl-flags.patch +Patch1111: 1111-btrfs-progs-receive-add-tests-for-basic-encoded_writ.patch %endif BuildRequires: gnupg2 @@ -129,6 +139,7 @@ popd %{_mandir}/man5/* %{_mandir}/man8/* %{_udevrulesdir}/64-btrfs-dm.rules +%{_udevrulesdir}/64-btrfs-zoned.rules %{_datadir}/bash-completion/completions/btrfs %files -n libbtrfs @@ -151,10 +162,35 @@ popd %{python3_sitearch}/btrfsutil-*.egg-info/ %changelog +* Mon Apr 04 2022 Omar Sandoval - 5.16.2-1.1 +- Adapt for CentOS Hyperscale +- Add compressed send/receive patches to Facebook build + +* Wed Feb 16 2022 Neal Gompa - 5.16.2-1 +- Update to 5.16.2 + +* Sat Feb 05 2022 Igor Raits - 5.16.1-1 +- Update to 5.16.1 + +* Wed Jan 19 2022 Fedora Release Engineering - 5.16-2 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_36_Mass_Rebuild + +* Mon Jan 17 2022 Neal Gompa - 5.16-1 +- Update to 5.16 + * Sat Jan 15 2022 Neal Gompa - 5.14.2-1.1 - Adapt for CentOS Hyperscale - Refresh patch for disabling raid56 mode +* Mon Nov 22 2021 Neal Gompa - 5.15.1-1 +- Update to 5.15.1 + +* Fri Nov 05 2021 Neal Gompa - 5.15-1 +- Update to 5.15 + +* Sat Oct 30 2021 Neal Gompa - 5.14.91-1 +- Update to 5.14.91 (5.15~rc1) + * Sat Oct 09 2021 Neal Gompa - 5.14.2-1 - Update to 5.14.2