From d5fbcc1ba9f5ff81ca67e80b552ee02d285ef536 Mon Sep 17 00:00:00 2001 From: Tianjie Xu Date: Wed, 11 Apr 2018 14:38:09 -0700 Subject: Remove the old log files if cache space is insufficient for OTA We set the limit of the max stash size to 80% of cache size. But the cache space can still be insufficient for the update if the log files occupy a large chunk of /cache. So remove the old logs for now to make room for the update. Bug: 77528881 Test: unit tests pass Change-Id: Ia8bcb0ace11f8164ad9290bfb360e08e31d282cb --- applypatch/applypatch.cpp | 14 +-- applypatch/freecache.cpp | 168 ++++++++++++++++++++--------- applypatch/include/applypatch/applypatch.h | 6 +- otautil/cache_location.cpp | 4 +- otautil/include/otautil/cache_location.h | 10 ++ tests/component/applypatch_test.cpp | 129 ++++++++++++++++++++++ 6 files changed, 271 insertions(+), 60 deletions(-) diff --git a/applypatch/applypatch.cpp b/applypatch/applypatch.cpp index 7645a4005..6f6c187be 100644 --- a/applypatch/applypatch.cpp +++ b/applypatch/applypatch.cpp @@ -436,13 +436,13 @@ static size_t FileSink(const unsigned char* data, size_t len, int fd) { // Return the amount of free space (in bytes) on the filesystem // containing filename. filename must exist. Return -1 on error. -size_t FreeSpaceForFile(const char* filename) { - struct statfs sf; - if (statfs(filename, &sf) != 0) { - printf("failed to statfs %s: %s\n", filename, strerror(errno)); - return -1; - } - return sf.f_bsize * sf.f_bavail; +size_t FreeSpaceForFile(const std::string& filename) { + struct statfs sf; + if (statfs(filename.c_str(), &sf) != 0) { + printf("failed to statfs %s: %s\n", filename.c_str(), strerror(errno)); + return -1; + } + return sf.f_bsize * sf.f_bavail; } int CacheSizeCheck(size_t bytes) { diff --git a/applypatch/freecache.cpp b/applypatch/freecache.cpp index ea364d8e6..cfab0f6db 100644 --- a/applypatch/freecache.cpp +++ b/applypatch/freecache.cpp @@ -14,7 +14,10 @@ * limitations under the License. */ +#include +#include #include +#include #include #include #include @@ -22,20 +25,22 @@ #include #include #include -#include -#include +#include +#include #include #include #include +#include #include #include +#include #include "applypatch/applypatch.h" #include "otautil/cache_location.h" -static int EliminateOpenFiles(std::set* files) { +static int EliminateOpenFiles(const std::string& dirname, std::set* files) { std::unique_ptr d(opendir("/proc"), closedir); if (!d) { printf("error opening /proc: %s\n", strerror(errno)); @@ -62,7 +67,7 @@ static int EliminateOpenFiles(std::set* files) { int count = readlink(fd_path.c_str(), link, sizeof(link)-1); if (count >= 0) { link[count] = '\0'; - if (strncmp(link, "/cache/", 7) == 0) { + if (android::base::StartsWith(link, dirname)) { if (files->erase(link) > 0) { printf("%s is open by %s\n", link, de->d_name); } @@ -73,77 +78,138 @@ static int EliminateOpenFiles(std::set* files) { return 0; } -static std::set FindExpendableFiles() { +static std::vector FindExpendableFiles( + const std::string& dirname, const std::function& name_filter) { std::set files; - // We're allowed to delete unopened regular files in any of these - // directories. - const char* dirs[2] = {"/cache", "/cache/recovery/otatest"}; - - for (size_t i = 0; i < sizeof(dirs)/sizeof(dirs[0]); ++i) { - std::unique_ptr d(opendir(dirs[i]), closedir); - if (!d) { - printf("error opening %s: %s\n", dirs[i], strerror(errno)); + + std::unique_ptr d(opendir(dirname.c_str()), closedir); + if (!d) { + printf("error opening %s: %s\n", dirname.c_str(), strerror(errno)); + return {}; + } + + // Look for regular files in the directory (not in any subdirectories). + struct dirent* de; + while ((de = readdir(d.get())) != 0) { + std::string path = dirname + "/" + de->d_name; + + // We can't delete cache_temp_source; if it's there we might have restarted during + // installation and could be depending on it to be there. + if (path == CacheLocation::location().cache_temp_source()) { continue; } - // Look for regular files in the directory (not in any subdirectories). - struct dirent* de; - while ((de = readdir(d.get())) != 0) { - std::string path = std::string(dirs[i]) + "/" + de->d_name; - - // We can't delete cache_temp_source; if it's there we might have restarted during - // installation and could be depending on it to be there. - if (path == CacheLocation::location().cache_temp_source()) { - continue; - } + // Do not delete the file if it doesn't have the expected format. + if (name_filter != nullptr && !name_filter(de->d_name)) { + continue; + } - struct stat st; - if (stat(path.c_str(), &st) == 0 && S_ISREG(st.st_mode)) { - files.insert(path); - } + struct stat st; + if (stat(path.c_str(), &st) == 0 && S_ISREG(st.st_mode)) { + files.insert(path); } } - printf("%zu regular files in deletable directories\n", files.size()); - if (EliminateOpenFiles(&files) < 0) { - return std::set(); + printf("%zu regular files in deletable directory\n", files.size()); + if (EliminateOpenFiles(dirname, &files) < 0) { + return {}; + } + + return std::vector(files.begin(), files.end()); +} + +// Parses the index of given log file, e.g. 3 for last_log.3; returns max number if the log name +// doesn't have the expected format so that we'll delete these ones first. +static unsigned int GetLogIndex(const std::string& log_name) { + if (log_name == "last_log" || log_name == "last_kmsg") { + return 0; + } + + unsigned int index; + if (sscanf(log_name.c_str(), "last_log.%u", &index) == 1 || + sscanf(log_name.c_str(), "last_kmsg.%u", &index) == 1) { + return index; } - return files; + + return std::numeric_limits::max(); } int MakeFreeSpaceOnCache(size_t bytes_needed) { #ifndef __ANDROID__ // TODO (xunchang) implement a heuristic cache size check during host simulation. - printf("Skip making (%zu) bytes free space on cache; program is running on host\n", bytes_needed); + printf("Skip making (%zu) bytes free space on /cache; program is running on host\n", + bytes_needed); return 0; #endif - size_t free_now = FreeSpaceForFile("/cache"); - printf("%zu bytes free on /cache (%zu needed)\n", free_now, bytes_needed); + std::vector dirs = { "/cache", CacheLocation::location().cache_log_directory() }; + for (const auto& dirname : dirs) { + if (RemoveFilesInDirectory(bytes_needed, dirname, FreeSpaceForFile)) { + return 0; + } + } - if (free_now >= bytes_needed) { - return 0; + return -1; +} + +bool RemoveFilesInDirectory(size_t bytes_needed, const std::string& dirname, + const std::function& space_checker) { + struct stat st; + if (stat(dirname.c_str(), &st) != 0) { + error(0, errno, "Unable to free space on %s", dirname.c_str()); + return false; } - std::set files = FindExpendableFiles(); - if (files.empty()) { - // nothing we can delete to free up space! - printf("no files can be deleted to free space on /cache\n"); - return -1; + if (!S_ISDIR(st.st_mode)) { + printf("%s is not a directory\n", dirname.c_str()); + return false; + } + + size_t free_now = space_checker(dirname); + printf("%zu bytes free on %s (%zu needed)\n", free_now, dirname.c_str(), bytes_needed); + + if (free_now >= bytes_needed) { + return true; } - // We could try to be smarter about which files to delete: the - // biggest ones? the smallest ones that will free up enough space? - // the oldest? the newest? - // - // Instead, we'll be dumb. + std::vector files; + if (dirname == CacheLocation::location().cache_log_directory()) { + // Deletes the log files only. + auto log_filter = [](const std::string& file_name) { + return android::base::StartsWith(file_name, "last_log") || + android::base::StartsWith(file_name, "last_kmsg"); + }; + + files = FindExpendableFiles(dirname, log_filter); + + // Older logs will come to the top of the queue. + auto comparator = [](const std::string& name1, const std::string& name2) -> bool { + unsigned int index1 = GetLogIndex(android::base::Basename(name1)); + unsigned int index2 = GetLogIndex(android::base::Basename(name2)); + if (index1 == index2) { + return name1 < name2; + } + + return index1 > index2; + }; + + std::sort(files.begin(), files.end(), comparator); + } else { + // We're allowed to delete unopened regular files in the directory. + files = FindExpendableFiles(dirname, nullptr); + } for (const auto& file : files) { - unlink(file.c_str()); - free_now = FreeSpaceForFile("/cache"); + if (unlink(file.c_str()) == -1) { + error(0, errno, "Failed to delete %s", file.c_str()); + continue; + } + + free_now = space_checker(dirname); printf("deleted %s; now %zu bytes free\n", file.c_str(), free_now); - if (free_now < bytes_needed) { - break; + if (free_now >= bytes_needed) { + return true; } } - return (free_now >= bytes_needed) ? 0 : -1; + + return false; } diff --git a/applypatch/include/applypatch/applypatch.h b/applypatch/include/applypatch/applypatch.h index 912ead1fa..021a28d05 100644 --- a/applypatch/include/applypatch/applypatch.h +++ b/applypatch/include/applypatch/applypatch.h @@ -39,7 +39,7 @@ using SinkFn = std::function; // applypatch.cpp int ShowLicenses(); -size_t FreeSpaceForFile(const char* filename); +size_t FreeSpaceForFile(const std::string& filename); int CacheSizeCheck(size_t bytes); int ParseSha1(const char* str, uint8_t* digest); @@ -79,5 +79,9 @@ int ApplyImagePatch(const unsigned char* old_data, size_t old_size, const Value& // freecache.cpp int MakeFreeSpaceOnCache(size_t bytes_needed); +// Removes the files in |dirname| until we have at least |bytes_needed| bytes of free space on +// the partition. The size of the free space is returned by calling |space_checker|. +bool RemoveFilesInDirectory(size_t bytes_needed, const std::string& dirname, + const std::function& space_checker); #endif diff --git a/otautil/cache_location.cpp b/otautil/cache_location.cpp index 8ddefec5e..6139bf17b 100644 --- a/otautil/cache_location.cpp +++ b/otautil/cache_location.cpp @@ -19,6 +19,7 @@ constexpr const char kDefaultCacheTempSource[] = "/cache/saved.file"; constexpr const char kDefaultLastCommandFile[] = "/cache/recovery/last_command"; constexpr const char kDefaultStashDirectoryBase[] = "/cache/recovery"; +constexpr const char kDefaultCacheLogDirectory[] = "/cache/recovery"; CacheLocation& CacheLocation::location() { static CacheLocation cache_location; @@ -28,4 +29,5 @@ CacheLocation& CacheLocation::location() { CacheLocation::CacheLocation() : cache_temp_source_(kDefaultCacheTempSource), last_command_file_(kDefaultLastCommandFile), - stash_directory_base_(kDefaultStashDirectoryBase) {} + stash_directory_base_(kDefaultStashDirectoryBase), + cache_log_directory_(kDefaultCacheLogDirectory) {} diff --git a/otautil/include/otautil/cache_location.h b/otautil/include/otautil/cache_location.h index f2f663816..005395e5f 100644 --- a/otautil/include/otautil/cache_location.h +++ b/otautil/include/otautil/cache_location.h @@ -49,6 +49,13 @@ class CacheLocation { stash_directory_base_ = base; } + std::string cache_log_directory() const { + return cache_log_directory_; + } + void set_cache_log_directory(const std::string& log_dir) { + cache_log_directory_ = log_dir; + } + private: CacheLocation(); DISALLOW_COPY_AND_ASSIGN(CacheLocation); @@ -64,6 +71,9 @@ class CacheLocation { // The base directory to write stashes during update. std::string stash_directory_base_; + + // The location of last_log & last_kmsg. + std::string cache_log_directory_; }; #endif // _OTAUTIL_OTAUTIL_CACHE_LOCATION_H_ diff --git a/tests/component/applypatch_test.cpp b/tests/component/applypatch_test.cpp index 916028594..aa0959bf5 100644 --- a/tests/component/applypatch_test.cpp +++ b/tests/component/applypatch_test.cpp @@ -14,8 +14,10 @@ * limitations under the License. */ +#include #include #include +#include #include #include #include @@ -23,6 +25,7 @@ #include #include +#include #include #include #include @@ -30,6 +33,7 @@ #include #include #include +#include #include #include @@ -110,6 +114,52 @@ class ApplyPatchModesTest : public ::testing::Test { TemporaryFile cache_source; }; +class FreeCacheTest : public ::testing::Test { + protected: + static constexpr size_t PARTITION_SIZE = 4096 * 10; + + // Returns a sorted list of files in |dirname|. + static std::vector FindFilesInDir(const std::string& dirname) { + std::vector file_list; + + std::unique_ptr d(opendir(dirname.c_str()), closedir); + struct dirent* de; + while ((de = readdir(d.get())) != 0) { + std::string path = dirname + "/" + de->d_name; + + struct stat st; + if (stat(path.c_str(), &st) == 0 && S_ISREG(st.st_mode)) { + file_list.emplace_back(de->d_name); + } + } + + std::sort(file_list.begin(), file_list.end()); + return file_list; + } + + static void AddFilesToDir(const std::string& dir, const std::vector& files) { + std::string zeros(4096, 0); + for (const auto& file : files) { + std::string path = dir + "/" + file; + ASSERT_TRUE(android::base::WriteStringToFile(zeros, path)); + } + } + + void SetUp() override { + CacheLocation::location().set_cache_log_directory(mock_log_dir.path); + } + + // A mock method to calculate the free space. It assumes the partition has a total size of 40960 + // bytes and all files are 4096 bytes in size. + size_t MockFreeSpaceChecker(const std::string& dirname) { + std::vector files = FindFilesInDir(dirname); + return PARTITION_SIZE - 4096 * files.size(); + } + + TemporaryDir mock_cache; + TemporaryDir mock_log_dir; +}; + TEST_F(ApplyPatchTest, CheckModeSkip) { std::vector sha1s; ASSERT_EQ(0, applypatch_check(&old_file[0], sha1s)); @@ -414,3 +464,82 @@ TEST_F(ApplyPatchModesTest, CheckModeInvalidArgs) { TEST_F(ApplyPatchModesTest, ShowLicenses) { ASSERT_EQ(0, applypatch_modes(2, (const char* []){ "applypatch", "-l" })); } + +TEST_F(FreeCacheTest, FreeCacheSmoke) { + std::vector files = { "file1", "file2", "file3" }; + AddFilesToDir(mock_cache.path, files); + ASSERT_EQ(files, FindFilesInDir(mock_cache.path)); + ASSERT_EQ(4096 * 7, MockFreeSpaceChecker(mock_cache.path)); + + ASSERT_TRUE(RemoveFilesInDirectory(4096 * 9, mock_cache.path, [&](const std::string& dir) { + return this->MockFreeSpaceChecker(dir); + })); + + ASSERT_EQ(std::vector{ "file3" }, FindFilesInDir(mock_cache.path)); + ASSERT_EQ(4096 * 9, MockFreeSpaceChecker(mock_cache.path)); +} + +TEST_F(FreeCacheTest, FreeCacheOpenFile) { + std::vector files = { "file1", "file2" }; + AddFilesToDir(mock_cache.path, files); + ASSERT_EQ(files, FindFilesInDir(mock_cache.path)); + ASSERT_EQ(4096 * 8, MockFreeSpaceChecker(mock_cache.path)); + + std::string file1_path = mock_cache.path + "/file1"s; + android::base::unique_fd fd(open(file1_path.c_str(), O_RDONLY)); + + // file1 can't be deleted as it's opened by us. + ASSERT_FALSE(RemoveFilesInDirectory(4096 * 10, mock_cache.path, [&](const std::string& dir) { + return this->MockFreeSpaceChecker(dir); + })); + + ASSERT_EQ(std::vector{ "file1" }, FindFilesInDir(mock_cache.path)); +} + +TEST_F(FreeCacheTest, FreeCacheLogsSmoke) { + std::vector log_files = { "last_log", "last_log.1", "last_kmsg.2", "last_log.5", + "last_log.10" }; + AddFilesToDir(mock_log_dir.path, log_files); + ASSERT_EQ(4096 * 5, MockFreeSpaceChecker(mock_log_dir.path)); + + ASSERT_TRUE(RemoveFilesInDirectory(4096 * 8, mock_log_dir.path, [&](const std::string& dir) { + return this->MockFreeSpaceChecker(dir); + })); + + // Logs with a higher index will be deleted first + std::vector expected = { "last_log", "last_log.1" }; + ASSERT_EQ(expected, FindFilesInDir(mock_log_dir.path)); + ASSERT_EQ(4096 * 8, MockFreeSpaceChecker(mock_log_dir.path)); +} + +TEST_F(FreeCacheTest, FreeCacheLogsStringComparison) { + std::vector log_files = { "last_log.1", "last_kmsg.1", "last_log.not_number", + "last_kmsgrandom" }; + AddFilesToDir(mock_log_dir.path, log_files); + ASSERT_EQ(4096 * 6, MockFreeSpaceChecker(mock_log_dir.path)); + + ASSERT_TRUE(RemoveFilesInDirectory(4096 * 9, mock_log_dir.path, [&](const std::string& dir) { + return this->MockFreeSpaceChecker(dir); + })); + + // Logs with incorrect format will be deleted first; and the last_kmsg with the same index is + // deleted before last_log. + std::vector expected = { "last_log.1" }; + ASSERT_EQ(expected, FindFilesInDir(mock_log_dir.path)); + ASSERT_EQ(4096 * 9, MockFreeSpaceChecker(mock_log_dir.path)); +} + +TEST_F(FreeCacheTest, FreeCacheLogsOtherFiles) { + std::vector log_files = { "last_install", "command", "block.map", "last_log", + "last_kmsg.1" }; + AddFilesToDir(mock_log_dir.path, log_files); + ASSERT_EQ(4096 * 5, MockFreeSpaceChecker(mock_log_dir.path)); + + ASSERT_FALSE(RemoveFilesInDirectory(4096 * 8, mock_log_dir.path, [&](const std::string& dir) { + return this->MockFreeSpaceChecker(dir); + })); + + // Non log files in /cache/recovery won't be deleted. + std::vector expected = { "block.map", "command", "last_install" }; + ASSERT_EQ(expected, FindFilesInDir(mock_log_dir.path)); +} -- cgit v1.2.3