From 311e6ca7b609e578c0ec132440a4ad981db70a9e Mon Sep 17 00:00:00 2001 From: xunchang Date: Fri, 22 Mar 2019 08:54:35 -0700 Subject: Implement FuseBlockDataProvider Adds a fuse data provider that parses the metadata from a block map, reads the data from the given ranges of the block device; and provides the data to the fuse. Bug: 127071893 Test: unit tests pass, install a package from block map Change-Id: Ie9925ee9144e98642505b3f5e1a4a186d2b21ed0 --- fuse_sideload/Android.bp | 4 ++ fuse_sideload/fuse_provider.cpp | 79 ++++++++++++++++++++++++++ fuse_sideload/include/fuse_provider.h | 28 +++++++++ minadbd/Android.bp | 5 ++ otautil/include/otautil/rangeset.h | 7 +++ otautil/rangeset.cpp | 52 +++++++++++++++++ otautil/sysutil.cpp | 5 ++ tests/Android.bp | 1 + tests/unit/fuse_provider_test.cpp | 103 ++++++++++++++++++++++++++++++++++ tests/unit/rangeset_test.cpp | 24 ++++++++ tests/unit/sysutil_test.cpp | 2 +- 11 files changed, 309 insertions(+), 1 deletion(-) create mode 100644 tests/unit/fuse_provider_test.cpp diff --git a/fuse_sideload/Android.bp b/fuse_sideload/Android.bp index 8548548d2..9bf19eb85 100644 --- a/fuse_sideload/Android.bp +++ b/fuse_sideload/Android.bp @@ -34,6 +34,10 @@ cc_library { "include", ], + static_libs: [ + "libotautil", + ], + shared_libs: [ "libbase", "libcrypto", diff --git a/fuse_sideload/fuse_provider.cpp b/fuse_sideload/fuse_provider.cpp index 58786f5f3..5ee6e247f 100644 --- a/fuse_sideload/fuse_provider.cpp +++ b/fuse_sideload/fuse_provider.cpp @@ -27,8 +27,11 @@ #include #include +#include +#include #include "fuse_sideload.h" +#include "otautil/sysutil.h" FuseFileDataProvider::FuseFileDataProvider(const std::string& path, uint32_t block_size) { struct stat sb; @@ -69,3 +72,79 @@ bool FuseFileDataProvider::ReadBlockAlignedData(uint8_t* buffer, uint32_t fetch_ void FuseFileDataProvider::Close() { fd_.reset(); } + +FuseBlockDataProvider::FuseBlockDataProvider(uint64_t file_size, uint32_t fuse_block_size, + android::base::unique_fd&& fd, + uint32_t source_block_size, RangeSet ranges) + : FuseDataProvider(file_size, fuse_block_size), + fd_(std::move(fd)), + source_block_size_(source_block_size), + ranges_(std::move(ranges)) { + // Make sure the offset is also aligned with the blocks on the block device when we call + // ReadBlockAlignedData(). + CHECK_EQ(0, fuse_block_size_ % source_block_size_); +} + +bool FuseBlockDataProvider::ReadBlockAlignedData(uint8_t* buffer, uint32_t fetch_size, + uint32_t start_block) const { + uint64_t offset = static_cast(start_block) * fuse_block_size_; + if (fetch_size > file_size_ || offset > file_size_ - fetch_size) { + LOG(ERROR) << "Out of bound read, offset: " << offset << ", fetch size: " << fetch_size + << ", file size " << file_size_; + return false; + } + + auto read_ranges = + ranges_.GetSubRanges(offset / source_block_size_, fetch_size / source_block_size_); + if (!read_ranges) { + return false; + } + + uint8_t* next_out = buffer; + for (const auto& [range_start, range_end] : read_ranges.value()) { + uint64_t bytes_start = static_cast(range_start) * source_block_size_; + uint64_t bytes_to_read = static_cast(range_end - range_start) * source_block_size_; + if (!android::base::ReadFullyAtOffset(fd_, next_out, bytes_to_read, bytes_start)) { + PLOG(ERROR) << "Failed to read " << bytes_to_read << " bytes at offset " << bytes_start; + return false; + } + + next_out += bytes_to_read; + } + + if (uint64_t tailing_bytes = fetch_size % source_block_size_; tailing_bytes != 0) { + // Calculate the offset to last partial block. + uint64_t tailing_offset = + read_ranges.value() + ? static_cast((read_ranges->cend() - 1)->second) * source_block_size_ + : static_cast(start_block) * source_block_size_; + if (!android::base::ReadFullyAtOffset(fd_, next_out, tailing_bytes, tailing_offset)) { + PLOG(ERROR) << "Failed to read tailing " << tailing_bytes << " bytes at offset " + << tailing_offset; + return false; + } + } + return true; +} + +std::unique_ptr FuseBlockDataProvider::CreateFromBlockMap( + const std::string& block_map_path, uint32_t fuse_block_size) { + auto block_map = BlockMapData::ParseBlockMapFile(block_map_path); + if (!block_map) { + return nullptr; + } + + android::base::unique_fd fd(TEMP_FAILURE_RETRY(open(block_map.path().c_str(), O_RDONLY))); + if (fd == -1) { + PLOG(ERROR) << "Failed to open " << block_map.path(); + return nullptr; + } + + return std::unique_ptr( + new FuseBlockDataProvider(block_map.file_size(), fuse_block_size, std::move(fd), + block_map.block_size(), block_map.block_ranges())); +} + +void FuseBlockDataProvider::Close() { + fd_.reset(); +} diff --git a/fuse_sideload/include/fuse_provider.h b/fuse_sideload/include/fuse_provider.h index 59059cf9b..8d4ea4073 100644 --- a/fuse_sideload/include/fuse_provider.h +++ b/fuse_sideload/include/fuse_provider.h @@ -18,10 +18,13 @@ #include +#include #include #include +#include "otautil/rangeset.h" + // This is the base class to read data from source and provide the data to FUSE. class FuseDataProvider { public: @@ -70,3 +73,28 @@ class FuseFileDataProvider : public FuseDataProvider { // The underlying source to read data from. android::base::unique_fd fd_; }; + +// This class parses a block map and reads data from the underlying block device. +class FuseBlockDataProvider : public FuseDataProvider { + public: + // Constructs the fuse provider from the block map. + static std::unique_ptr CreateFromBlockMap( + const std::string& block_map_path, uint32_t fuse_block_size); + + RangeSet ranges() const { + return ranges_; + } + bool ReadBlockAlignedData(uint8_t* buffer, uint32_t fetch_size, + uint32_t start_block) const override; + void Close() override; + + private: + FuseBlockDataProvider(uint64_t file_size, uint32_t fuse_block_size, android::base::unique_fd&& fd, + uint32_t source_block_size, RangeSet ranges); + // The underlying block device to read data from. + android::base::unique_fd fd_; + // The block size of the source block device. + uint32_t source_block_size_; + // The block ranges from the source block device that consist of the file + RangeSet ranges_; +}; diff --git a/minadbd/Android.bp b/minadbd/Android.bp index 007e5057b..afd57ad2d 100644 --- a/minadbd/Android.bp +++ b/minadbd/Android.bp @@ -43,6 +43,10 @@ cc_library { "minadbd_services.cpp", ], + static_libs: [ + "libotautil", + ], + shared_libs: [ "libadbd", "libbase", @@ -96,6 +100,7 @@ cc_test { static_libs: [ "libminadbd_services", "libfusesideload", + "libotautil", "libadbd", "libcrypto", ], diff --git a/otautil/include/otautil/rangeset.h b/otautil/include/otautil/rangeset.h index e91d02ca6..a18c30e29 100644 --- a/otautil/include/otautil/rangeset.h +++ b/otautil/include/otautil/rangeset.h @@ -18,6 +18,7 @@ #include +#include #include #include #include @@ -49,6 +50,12 @@ class RangeSet { // bounds. For example, "3,5" contains blocks 3 and 4. So "3,5" and "5,7" are not overlapped. bool Overlaps(const RangeSet& other) const; + // Returns a subset of ranges starting from |start_index| with respect to the original range. The + // output range will have |num_of_blocks| blocks in size. Returns std::nullopt if the input is + // invalid. e.g. RangeSet({{0, 5}, {10, 15}}).GetSubRanges(1, 5) returns + // RangeSet({{1, 5}, {10, 11}}). + std::optional GetSubRanges(size_t start_index, size_t num_of_blocks) const; + // Returns a vector of RangeSets that contain the same set of blocks represented by the current // RangeSet. The RangeSets in the vector contain similar number of blocks, with a maximum delta // of 1-block between any two of them. For example, 14 blocks would be split into 4 + 4 + 3 + 3, diff --git a/otautil/rangeset.cpp b/otautil/rangeset.cpp index 5ab8e08fe..8ee99dd7a 100644 --- a/otautil/rangeset.cpp +++ b/otautil/rangeset.cpp @@ -184,6 +184,58 @@ bool RangeSet::Overlaps(const RangeSet& other) const { return false; } +std::optional RangeSet::GetSubRanges(size_t start_index, size_t num_of_blocks) const { + size_t end_index = start_index + num_of_blocks; // The index of final block to read plus one + if (start_index > end_index || end_index > blocks_) { + LOG(ERROR) << "Failed to get the sub ranges for start_index " << start_index + << " num_of_blocks " << num_of_blocks + << " total number of blocks the range contains is " << blocks_; + return std::nullopt; + } + + if (num_of_blocks == 0) { + LOG(WARNING) << "num_of_blocks is zero when calling GetSubRanges()"; + return RangeSet(); + } + + RangeSet result; + size_t current_index = 0; + for (const auto& [range_start, range_end] : ranges_) { + CHECK_LT(range_start, range_end); + size_t blocks_in_range = range_end - range_start; + // Linear search to skip the ranges until we reach start_block. + if (current_index + blocks_in_range <= start_index) { + current_index += blocks_in_range; + continue; + } + + size_t trimmed_range_start = range_start; + // We have found the first block range to read, trim the heading blocks. + if (current_index < start_index) { + trimmed_range_start += start_index - current_index; + } + // Trim the trailing blocks if the last range has more blocks than desired; also return the + // result. + if (current_index + blocks_in_range >= end_index) { + size_t trimmed_range_end = range_end - (current_index + blocks_in_range - end_index); + if (!result.PushBack({ trimmed_range_start, trimmed_range_end })) { + return std::nullopt; + } + + return result; + } + + if (!result.PushBack({ trimmed_range_start, range_end })) { + return std::nullopt; + } + current_index += blocks_in_range; + } + + LOG(ERROR) << "Failed to construct byte ranges to read, start_block: " << start_index + << ", num_of_blocks: " << num_of_blocks << " total number of blocks: " << blocks_; + return std::nullopt; +} + // Ranges in the the set should be mutually exclusive; and they're sorted by the start block. SortedRangeSet::SortedRangeSet(std::vector&& pairs) : RangeSet(std::move(pairs)) { std::sort(ranges_.begin(), ranges_.end()); diff --git a/otautil/sysutil.cpp b/otautil/sysutil.cpp index 2b4861809..420db4cac 100644 --- a/otautil/sysutil.cpp +++ b/otautil/sysutil.cpp @@ -94,6 +94,11 @@ BlockMapData BlockMapData::ParseBlockMapFile(const std::string& block_map_path) remaining_blocks -= range_blocks; } + if (remaining_blocks != 0) { + LOG(ERROR) << "Invalid ranges: remaining blocks " << remaining_blocks; + return {}; + } + return BlockMapData(block_dev, file_size, blksize, std::move(ranges)); } diff --git a/tests/Android.bp b/tests/Android.bp index ec2124a54..67a65ae9e 100644 --- a/tests/Android.bp +++ b/tests/Android.bp @@ -118,6 +118,7 @@ cc_test { static_libs: libapplypatch_static_libs + librecovery_static_libs + [ "librecovery_ui", + "libfusesideload", "libminui", "libotautil", "libupdater", diff --git a/tests/unit/fuse_provider_test.cpp b/tests/unit/fuse_provider_test.cpp new file mode 100644 index 000000000..c5995dd7d --- /dev/null +++ b/tests/unit/fuse_provider_test.cpp @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "fuse_provider.h" +#include "fuse_sideload.h" +#include "install/install.h" + +TEST(FuseBlockMapTest, CreateFromBlockMap_smoke) { + TemporaryFile fake_block_device; + std::vector lines = { + fake_block_device.path, "10000 4096", "3", "10 11", "20 21", "22 23", + }; + + TemporaryFile temp_file; + android::base::WriteStringToFile(android::base::Join(lines, '\n'), temp_file.path); + auto block_map_data = FuseBlockDataProvider::CreateFromBlockMap(temp_file.path, 4096); + + ASSERT_TRUE(block_map_data); + ASSERT_EQ(10000, block_map_data->file_size()); + ASSERT_EQ(4096, block_map_data->fuse_block_size()); + ASSERT_EQ(RangeSet({ { 10, 11 }, { 20, 21 }, { 22, 23 } }), block_map_data->ranges()); +} + +TEST(FuseBlockMapTest, ReadBlockAlignedData_smoke) { + std::string content; + content.reserve(40960); + for (char c = 0; c < 10; c++) { + content += std::string(4096, c); + } + TemporaryFile fake_block_device; + ASSERT_TRUE(android::base::WriteStringToFile(content, fake_block_device.path)); + + std::vector lines = { + fake_block_device.path, + "20000 4096", + "1", + "0 5", + }; + TemporaryFile temp_file; + android::base::WriteStringToFile(android::base::Join(lines, '\n'), temp_file.path); + auto block_map_data = FuseBlockDataProvider::CreateFromBlockMap(temp_file.path, 4096); + + std::vector result(2000); + ASSERT_TRUE(block_map_data->ReadBlockAlignedData(result.data(), 2000, 1)); + ASSERT_EQ(std::vector(content.begin() + 4096, content.begin() + 6096), result); + + result.resize(20000); + ASSERT_TRUE(block_map_data->ReadBlockAlignedData(result.data(), 20000, 0)); + ASSERT_EQ(std::vector(content.begin(), content.begin() + 20000), result); +} + +TEST(FuseBlockMapTest, ReadBlockAlignedData_large_fuse_block) { + std::string content; + for (char c = 0; c < 10; c++) { + content += std::string(4096, c); + } + + TemporaryFile temp_file; + ASSERT_TRUE(android::base::WriteStringToFile(content, temp_file.path)); + + std::vector lines = { + temp_file.path, "36384 4096", "2", "0 5", "6 10", + }; + TemporaryFile block_map; + ASSERT_TRUE(android::base::WriteStringToFile(android::base::Join(lines, '\n'), block_map.path)); + + auto block_map_data = FuseBlockDataProvider::CreateFromBlockMap(block_map.path, 16384); + ASSERT_TRUE(block_map_data); + + std::vector result(20000); + // Out of bound read + ASSERT_FALSE(block_map_data->ReadBlockAlignedData(result.data(), 20000, 2)); + ASSERT_TRUE(block_map_data->ReadBlockAlignedData(result.data(), 20000, 1)); + // expected source block contains: 4, 6-9 + std::string expected = content.substr(16384, 4096) + content.substr(24576, 15904); + ASSERT_EQ(std::vector(expected.begin(), expected.end()), result); +} diff --git a/tests/unit/rangeset_test.cpp b/tests/unit/rangeset_test.cpp index fc72f2f6d..699f933a0 100644 --- a/tests/unit/rangeset_test.cpp +++ b/tests/unit/rangeset_test.cpp @@ -18,6 +18,7 @@ #include #include +#include #include #include @@ -248,6 +249,29 @@ TEST(RangeSetTest, ToString) { ASSERT_EQ("6,1,3,4,6,15,22", RangeSet::Parse("6,1,3,4,6,15,22").ToString()); } +TEST(RangeSetTest, GetSubRanges_invalid) { + RangeSet range0({ { 1, 11 }, { 20, 30 } }); + ASSERT_FALSE(range0.GetSubRanges(0, 21)); // too many blocks + ASSERT_FALSE(range0.GetSubRanges(21, 1)); // start block OOB +} + +TEST(RangeSetTest, GetSubRanges_empty) { + RangeSet range0({ { 1, 11 }, { 20, 30 } }); + ASSERT_EQ(RangeSet{}, range0.GetSubRanges(1, 0)); // empty num_of_blocks +} + +TEST(RangeSetTest, GetSubRanges_smoke) { + RangeSet range0({ { 10, 11 } }); + ASSERT_EQ(RangeSet({ { 10, 11 } }), range0.GetSubRanges(0, 1)); + + RangeSet range1({ { 10, 11 }, { 20, 21 }, { 30, 31 } }); + ASSERT_EQ(range1, range1.GetSubRanges(0, 3)); + ASSERT_EQ(RangeSet({ { 20, 21 } }), range1.GetSubRanges(1, 1)); + + RangeSet range2({ { 1, 11 }, { 20, 25 }, { 30, 35 } }); + ASSERT_EQ(RangeSet({ { 10, 11 }, { 20, 25 }, { 30, 31 } }), range2.GetSubRanges(9, 7)); +} + TEST(SortedRangeSetTest, Insert) { SortedRangeSet rs({ { 2, 3 }, { 4, 6 }, { 8, 14 } }); rs.Insert({ 1, 2 }); diff --git a/tests/unit/sysutil_test.cpp b/tests/unit/sysutil_test.cpp index 3466e8eec..64b8956f7 100644 --- a/tests/unit/sysutil_test.cpp +++ b/tests/unit/sysutil_test.cpp @@ -67,7 +67,7 @@ TEST(SysUtilTest, ParseBlockMapFile_invalid_size) { "/dev/abc", "42949672950 4294967295", "1", - "0 9", + "0 10", }; TemporaryFile temp_file; -- cgit v1.2.3