From 83bd483f3db1df4b69c1612030e1a1be05b303ad Mon Sep 17 00:00:00 2001 From: Dees_Troy Date: Sat, 4 May 2013 12:39:56 +0000 Subject: Support encrypted backup files Also includes features merged from: Update twrpTar by kokotas Revised function entryExists(). Added function to get archive's uncompressed size. Added option to exclude item(s) from the archive. Revised forks() http://review.teamw.in/#/c/590/ Change-Id: I01fa2c81643161984eff2625247af75990684bd9 --- twrpTar.cpp | 1157 +++++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 936 insertions(+), 221 deletions(-) (limited to 'twrpTar.cpp') diff --git a/twrpTar.cpp b/twrpTar.cpp index 2dbfd1562..bc7f37055 100644 --- a/twrpTar.cpp +++ b/twrpTar.cpp @@ -32,6 +32,7 @@ extern "C" { #include #include #include +#include #include #include #include "twrpTar.hpp" @@ -42,6 +43,20 @@ extern "C" { using namespace std; +twrpTar::twrpTar(void) { + use_encryption = 0; + userdata_encryption = 0; + use_compression = 0; + split_archives = 0; + has_data_media = 0; + pigz_pid = 0; + oaes_pid = 0; +} + +twrpTar::~twrpTar(void) { + // Do nothing +} + void twrpTar::setfn(string fn) { tarfn = fn; } @@ -50,139 +65,428 @@ void twrpTar::setdir(string dir) { tardir = dir; } -int twrpTar::createTarGZFork() { - int status; - pid_t pid; - if ((pid = fork()) == -1) { - LOGINFO("create tar failed to fork.\n"); - return -1; - } - if (pid == 0) { - if (createTGZ() != 0) - exit(-1); - else - exit(0); - } - else { - if ((pid = wait(&status)) == -1) { - LOGINFO("Tar creation failed\n"); - return -1; - } - else { - if (WIFSIGNALED(status) != 0) { - LOGINFO("Child process ended with signal: %d\n", WTERMSIG(status)); - return -1; - } - else if (WIFEXITED(status) != 0) - LOGINFO("Tar creation successful\n"); - else { - LOGINFO("Tar creation failed\n"); - return -1; - } - } - } - return 0; +void twrpTar::setexcl(string exclude) { + tarexclude = exclude; } int twrpTar::createTarFork() { - int status; - pid_t pid; + int status = 0; + pid_t pid, rc_pid; if ((pid = fork()) == -1) { LOGINFO("create tar failed to fork.\n"); return -1; } if (pid == 0) { - if (create() != 0) - exit(-1); - else - exit(0); - } - else { - if ((pid = wait(&status)) == -1) { - LOGINFO("Tar creation failed\n"); - return -1; - } - else { - if (WIFSIGNALED(status) != 0) { - LOGINFO("Child process ended with signal: %d\n", WTERMSIG(status)); - return -1; + // Child process + if (use_encryption || userdata_encryption) { + LOGINFO("Using encryption\n"); + DIR* d; + struct dirent* de; + unsigned long long regular_size = 0, encrypt_size = 0, target_size = 0, core_count = 1; + unsigned enc_thread_id = 1, regular_thread_id = 0, i, start_thread_id = 1; + int item_len, ret, thread_error = 0; + std::vector RegularList; + std::vector EncryptList; + string FileName; + struct TarListStruct TarItem; + twrpTar reg, enc[9]; + struct stat st; + pthread_t enc_thread[9]; + pthread_attr_t tattr; + void *thread_return; + + core_count = sysconf(_SC_NPROCESSORS_CONF); + if (core_count > 8) + core_count = 8; + LOGINFO(" Core Count : %llu\n", core_count); + Archive_Current_Size = 0; + + d = opendir(tardir.c_str()); + if (d == NULL) { + LOGERR("error opening '%s'\n", tardir.c_str()); + _exit(-1); } - else if (WEXITSTATUS(status) == 0) - LOGINFO("Tar creation successful\n"); - else { - LOGINFO("Tar creation failed\n"); - return -1; + // Figure out the size of all data to be encrypted and create a list of unencrypted files + while ((de = readdir(d)) != NULL) { + FileName = tardir + "/"; + FileName += de->d_name; + if (has_data_media == 1 && FileName.size() >= 11 && strncmp(FileName.c_str(), "/data/media", 11) == 0) + continue; // Skip /data/media + if (de->d_type == DT_BLK || de->d_type == DT_CHR) + continue; + if (de->d_type == DT_DIR && strcmp(de->d_name, ".") != 0 && strcmp(de->d_name, "..") != 0) { + item_len = strlen(de->d_name); + if (userdata_encryption && ((item_len >= 3 && strncmp(de->d_name, "app", 3) == 0) || (item_len >= 6 && strncmp(de->d_name, "dalvik", 6) == 0))) { + if (Generate_TarList(FileName, &RegularList, &target_size, ®ular_thread_id) < 0) { + LOGERR("Error in Generate_TarList with regular list!\n"); + closedir(d); + _exit(-1); + } + regular_size += TWFunc::Get_Folder_Size(FileName, false); + } else { + encrypt_size += TWFunc::Get_Folder_Size(FileName, false); + } + } else if (de->d_type == DT_REG) { + stat(FileName.c_str(), &st); + encrypt_size += (unsigned long long)(st.st_size); + } + } + closedir(d); + + target_size = encrypt_size / core_count; + target_size++; + LOGINFO(" Unencrypted size: %llu\n", regular_size); + LOGINFO(" Encrypted size : %llu\n", encrypt_size); + LOGINFO(" Target size : %llu\n", target_size); + if (!userdata_encryption) { + enc_thread_id = 0; + start_thread_id = 0; + core_count--; + } + Archive_Current_Size = 0; + + d = opendir(tardir.c_str()); + if (d == NULL) { + LOGERR("error opening '%s'\n", tardir.c_str()); + _exit(-1); + } + // Divide up the encrypted file list for threading + while ((de = readdir(d)) != NULL) { + FileName = tardir + "/"; + FileName += de->d_name; + if (has_data_media == 1 && FileName.size() >= 11 && strncmp(FileName.c_str(), "/data/media", 11) == 0) + continue; // Skip /data/media + if (de->d_type == DT_BLK || de->d_type == DT_CHR) + continue; + if (de->d_type == DT_DIR && strcmp(de->d_name, ".") != 0 && strcmp(de->d_name, "..") != 0) { + item_len = strlen(de->d_name); + if (userdata_encryption && ((item_len >= 3 && strncmp(de->d_name, "app", 3) == 0) || (item_len >= 6 && strncmp(de->d_name, "dalvik", 6) == 0))) { + // Do nothing, we added these to RegularList earlier + } else { + FileName = tardir + "/"; + FileName += de->d_name; + if (Generate_TarList(FileName, &EncryptList, &target_size, &enc_thread_id) < 0) { + LOGERR("Error in Generate_TarList with encrypted list!\n"); + closedir(d); + _exit(-1); + } + } + } else if (de->d_type == DT_REG || de->d_type == DT_LNK) { + stat(FileName.c_str(), &st); + if (de->d_type == DT_REG) + Archive_Current_Size += (unsigned long long)(st.st_size); + TarItem.fn = FileName; + TarItem.thread_id = enc_thread_id; + EncryptList.push_back(TarItem); + } + } + closedir(d); + if (enc_thread_id != core_count) { + LOGERR("Error dividing up threads for encryption, %i threads for %i cores!\n", enc_thread_id, core_count); + if (enc_thread_id > core_count) + _exit(-1); + else + LOGERR("Continuining anyway."); + } + + if (userdata_encryption) { + // Create a backup of unencrypted data + reg.setfn(tarfn); + reg.ItemList = &RegularList; + reg.thread_id = 0; + reg.use_encryption = 0; + reg.use_compression = use_compression; + LOGINFO("Creating unencrypted backup...\n"); + if (createList((void*)®) != 0) { + LOGERR("Error creating unencrypted backup.\n"); + _exit(-1); + } + } + + if (pthread_attr_init(&tattr)) { + LOGERR("Unable to pthread_attr_init\n"); + _exit(-1); + } + if (pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_JOINABLE)) { + LOGERR("Error setting pthread_attr_setdetachstate\n"); + _exit(-1); + } + if (pthread_attr_setscope(&tattr, PTHREAD_SCOPE_SYSTEM)) { + LOGERR("Error setting pthread_attr_setscope\n"); + _exit(-1); + } + /*if (pthread_attr_setstacksize(&tattr, 524288)) { + LOGERR("Error setting pthread_attr_setstacksize\n"); + _exit(-1); + }*/ + + // Create threads for the divided up encryption lists + for (i = start_thread_id; i <= core_count; i++) { + enc[i].setdir(tardir); + enc[i].setfn(tarfn); + enc[i].ItemList = &EncryptList; + enc[i].thread_id = i; + enc[i].use_encryption = use_encryption; + enc[i].use_compression = use_compression; + LOGINFO("Start encryption thread %i\n", i); + ret = pthread_create(&enc_thread[i], &tattr, createList, (void*)&enc[i]); + if (ret) { + LOGINFO("Unable to create %i thread for encryption! %i\nContinuing in same thread (backup will be slower).", i, ret); + if (createList((void*)&enc[i]) != 0) { + LOGERR("Error creating encrypted backup %i.\n", i); + _exit(-1); + } else { + enc[i].thread_id = i + 1; + } + } + usleep(100000); // Need a short delay before starting the next thread or the threads will never finish for some reason. + } + if (pthread_attr_destroy(&tattr)) { + LOGERR("Failed to pthread_attr_destroy\n"); + } + for (i = start_thread_id; i <= core_count; i++) { + if (enc[i].thread_id == i) { + if (pthread_join(enc_thread[i], &thread_return)) { + LOGERR("Error joining thread %i\n", i); + _exit(-1); + } else { + LOGINFO("Joined thread %i.\n", i); + ret = (int)thread_return; + if (ret != 0) { + thread_error = 1; + LOGERR("Thread %i returned an error %i.\n", i, ret); + _exit(-1); + } + } + } else { + LOGINFO("Skipping joining thread %i because of pthread failure.\n", i); + } + } + if (thread_error) { + LOGERR("Error returned by one or more threads.\n"); + _exit(-1); } + LOGINFO("Finished encrypted backup.\n"); + _exit(0); + } else { + if (create() != 0) + _exit(-1); + else + _exit(0); } + } else { + if (TWFunc::Wait_For_Child(pid, &status, "createTarFork()") != 0) + return -1; } return 0; } int twrpTar::extractTarFork() { - int status; - pid_t pid; - if ((pid = fork()) == -1) { + int status = 0; + pid_t pid, rc_pid; + + pid = fork(); + if (pid >= 0) // fork was successful + { + if (pid == 0) // child process + { + if (TWFunc::Path_Exists(tarfn)) { + LOGINFO("Single archive\n"); + if (extract() != 0) + _exit(-1); + else + _exit(0); + } else { + LOGINFO("Multiple archives\n"); + string temp; + char actual_filename[255]; + twrpTar tars[9]; + pthread_t tar_thread[9]; + pthread_attr_t tattr; + int thread_count = 0, i, start_thread_id = 1, ret, thread_error = 0; + void *thread_return; + + basefn = tarfn; + temp = basefn + "%i%02i"; + tarfn += "000"; + if (!TWFunc::Path_Exists(tarfn)) { + LOGERR("Unable to locate '%s' or '%s'\n", basefn.c_str(), tarfn.c_str()); + _exit(-1); + } + if (TWFunc::Get_File_Type(tarfn) != 2) { + LOGINFO("First tar file '%s' not encrypted\n", tarfn.c_str()); + tars[0].basefn = basefn; + tars[0].thread_id = 0; + if (extractMulti((void*)&tars[0]) != 0) { + LOGERR("Error extracting split archive.\n"); + _exit(-1); + } + } else { + start_thread_id = 0; + } + // Start threading encrypted restores + if (pthread_attr_init(&tattr)) { + LOGERR("Unable to pthread_attr_init\n"); + _exit(-1); + } + if (pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_JOINABLE)) { + LOGERR("Error setting pthread_attr_setdetachstate\n"); + _exit(-1); + } + if (pthread_attr_setscope(&tattr, PTHREAD_SCOPE_SYSTEM)) { + LOGERR("Error setting pthread_attr_setscope\n"); + _exit(-1); + } + /*if (pthread_attr_setstacksize(&tattr, 524288)) { + LOGERR("Error setting pthread_attr_setstacksize\n"); + _exit(-1); + }*/ + for (i = start_thread_id; i < 9; i++) { + sprintf(actual_filename, temp.c_str(), i, 0); + if (TWFunc::Path_Exists(actual_filename)) { + thread_count++; + tars[i].basefn = basefn; + tars[i].thread_id = i; + LOGINFO("Creating extract thread ID %i\n", i); + ret = pthread_create(&tar_thread[i], &tattr, extractMulti, (void*)&tars[i]); + if (ret) { + LOGINFO("Unable to create %i thread for extraction! %i\nContinuing in same thread (restore will be slower).", i, ret); + if (extractMulti((void*)&tars[i]) != 0) { + LOGERR("Error extracting backup in thread %i.\n", i); + _exit(-1); + } else { + tars[i].thread_id = i + 1; + } + } + usleep(100000); // Need a short delay before starting the next thread or the threads will never finish for some reason. + } else { + break; + } + } + for (i = start_thread_id; i < thread_count + start_thread_id; i++) { + if (tars[i].thread_id == i) { + if (pthread_join(tar_thread[i], &thread_return)) { + LOGERR("Error joining thread %i\n", i); + _exit(-1); + } else { + LOGINFO("Joined thread %i.\n", i); + ret = (int)thread_return; + if (ret != 0) { + thread_error = 1; + LOGERR("Thread %i returned an error %i.\n", i, ret); + _exit(-1); + } + } + } else { + LOGINFO("Skipping joining thread %i because of pthread failure.\n", i); + } + } + if (thread_error) { + LOGERR("Error returned by one or more threads.\n"); + _exit(-1); + } + LOGINFO("Finished encrypted backup.\n"); + _exit(0); + } + } + else // parent process + { + if (TWFunc::Wait_For_Child(pid, &status, "extractTarFork()") != 0) + return -1; + } + } + else // fork has failed + { LOGINFO("extract tar failed to fork.\n"); return -1; } - if (pid == 0) { - if (extract() != 0) - exit(-1); - else - exit(0); - } - else { - if ((pid = wait(&status)) == -1) { - LOGINFO("Tar extraction failed\n"); - return -1; + return 0; +} + +int twrpTar::splitArchiveFork() { + int status = 0; + pid_t pid, rc_pid; + + pid = fork(); + if (pid >= 0) // fork was successful + { + if (pid == 0) // child process + { + if (Split_Archive() <= 0) + _exit(-1); + else + _exit(0); } - else { - if (WIFSIGNALED(status) != 0) { - LOGINFO("Child process ended with signal: %d\n", WTERMSIG(status)); - return -1; - } - else if (WEXITSTATUS(status) == 0) - LOGINFO("Tar extraction successful\n"); - else { - LOGINFO("Tar extraction failed\n"); + else // parent process + { + if (TWFunc::Wait_For_Child(pid, &status, "splitArchiveFork()") != 0) return -1; - } } } + else // fork has failed + { + LOGINFO("split archive failed to fork.\n"); + return -1; + } return 0; } -int twrpTar::splitArchiveFork() { - int status; - pid_t pid; - if ((pid = fork()) == -1) { - LOGINFO("create tar failed to fork.\n"); +int twrpTar::Generate_TarList(string Path, std::vector *TarList, unsigned long long *Target_Size, unsigned *thread_id) { + DIR* d; + struct dirent* de; + struct stat st; + string FileName; + struct TarListStruct TarItem; + string::size_type i; + bool skip; + + if (has_data_media == 1 && Path.size() >= 11 && strncmp(Path.c_str(), "/data/media", 11) == 0) + return 0; // Skip /data/media + + d = opendir(Path.c_str()); + if (d == NULL) { + LOGERR("error opening '%s' -- error: %s\n", Path.c_str(), strerror(errno)); + closedir(d); return -1; } - if (pid == 0) { - if (Split_Archive() != 0) - exit(-1); - else - exit(0); - } - else { - if ((pid = wait(&status)) == -1) { - LOGINFO("Tar creation failed\n"); - return -1; - } - else { - if (WIFSIGNALED(status) != 0) { - LOGINFO("Child process ended with signal: %d\n", WTERMSIG(status)); - return -1; + while ((de = readdir(d)) != NULL) { + // Skip excluded stuff + if (split.size() > 0) { + skip = false; + for (i = 0; i < split.size(); i++) { + if (strcmp(de->d_name, split[i].c_str()) == 0) { + LOGINFO("excluding %s\n", de->d_name); + skip = true; + break; + } } - else if (WIFEXITED(status) != 0) - LOGINFO("Tar creation successful\n"); - else { - LOGINFO("Tar creation failed\n"); + if (skip) + continue; + } + FileName = Path + "/"; + FileName += de->d_name; + if (has_data_media == 1 && FileName.size() >= 11 && strncmp(FileName.c_str(), "/data/media", 11) == 0) + continue; // Skip /data/media + if (de->d_type == DT_BLK || de->d_type == DT_CHR) + continue; + TarItem.fn = FileName; + TarItem.thread_id = *thread_id; + if (de->d_type == DT_DIR && strcmp(de->d_name, ".") != 0 && strcmp(de->d_name, "..") != 0) { + TarList->push_back(TarItem); + if (Generate_TarList(FileName, TarList, Target_Size, thread_id) < 0) return -1; + } else if (de->d_type == DT_REG || de->d_type == DT_LNK) { + stat(FileName.c_str(), &st); + TarList->push_back(TarItem); + if (de->d_type == DT_REG) + Archive_Current_Size += st.st_size; + if (Archive_Current_Size != 0 && *Target_Size != 0 && Archive_Current_Size > *Target_Size) { + *thread_id = *thread_id + 1; + Archive_Current_Size = 0; } } } + closedir(d); return 0; } @@ -193,6 +497,9 @@ int twrpTar::Generate_Multiple_Archives(string Path) { string FileName; char actual_filename[255]; + string::size_type i; + bool skip; + if (has_data_media == 1 && Path.size() >= 11 && strncmp(Path.c_str(), "/data/media", 11) == 0) return 0; // Skip /data/media LOGINFO("Path: '%s', archive filename: '%s'\n", Path.c_str(), tarfn.c_str()); @@ -204,8 +511,20 @@ int twrpTar::Generate_Multiple_Archives(string Path) { closedir(d); return -1; } - while ((de = readdir(d)) != NULL) - { + while ((de = readdir(d)) != NULL) { + // Skip excluded stuff + if (split.size() > 0) { + skip = false; + for (i = 0; i < split.size(); i++) { + if (strcmp(de->d_name, split[i].c_str()) == 0) { + LOGINFO("excluding %s\n", de->d_name); + skip = true; + break; + } + } + if (skip) + continue; + } FileName = Path + "/"; FileName += de->d_name; if (has_data_media == 1 && FileName.size() >= 11 && strncmp(FileName.c_str(), "/data/media", 11) == 0) @@ -234,8 +553,7 @@ int twrpTar::Generate_Multiple_Archives(string Path) { if (Archive_Current_Size != 0 && Archive_Current_Size + st.st_size > MAX_ARCHIVE_SIZE) { LOGINFO("Closing tar '%s', ", tarfn.c_str()); - closeTar(false); - reinit_libtar_buffer(); + closeTar(); if (TWFunc::Get_File_Size(tarfn) == 0) { LOGERR("Backup file size for '%s' is 0 bytes.\n", tarfn.c_str()); return -1; @@ -277,25 +595,23 @@ int twrpTar::Split_Archive() Archive_Current_Size = 0; sprintf(actual_filename, temp.c_str(), Archive_File_Count); tarfn = actual_filename; - init_libtar_buffer(0); + if (!tarexclude.empty()) + split = TWFunc::split_string(tarexclude, ' ', true); createTar(); DataManager::GetValue(TW_HAS_DATA_MEDIA, has_data_media); gui_print("Creating archive 1...\n"); if (Generate_Multiple_Archives(tardir) < 0) { LOGERR("Error generating multiple archives\n"); - free_libtar_buffer(); return -1; } - closeTar(false); - free_libtar_buffer(); - LOGINFO("Done, created %i archives.\n", (Archive_File_Count++)); + closeTar(); + LOGINFO("Done, created %i archives.\n", (++Archive_File_Count)); return (Archive_File_Count); } int twrpTar::extractTar() { char* charRootDir = (char*) tardir.c_str(); - bool gzip = false; - if (openTar(gzip) == -1) + if (openTar() == -1) return -1; if (tar_extract_all(t, charRootDir) != 0) { LOGERR("Unable to extract tar archive '%s'\n", tarfn.c_str()); @@ -308,37 +624,33 @@ int twrpTar::extractTar() { return 0; } -int twrpTar::getArchiveType() { - int type = 0; - string::size_type i = 0; - int firstbyte = 0, secondbyte = 0; - char header[3]; - - ifstream f; - f.open(tarfn.c_str(), ios::in | ios::binary); - f.get(header, 3); - f.close(); - firstbyte = header[i] & 0xff; - secondbyte = header[++i] & 0xff; - - if (firstbyte == 0x1f && secondbyte == 0x8b) - type = 1; // Compressed - else - type = 0; // Uncompressed - - return type; -} - int twrpTar::extract() { - int Archive_Current_Type = getArchiveType(); + Archive_Current_Type = TWFunc::Get_File_Type(tarfn); if (Archive_Current_Type == 1) { //if you return the extractTGZ function directly, stack crashes happen LOGINFO("Extracting gzipped tar\n"); - int ret = extractTGZ(); + int ret = extractTar(); return ret; - } - else { + } else if (Archive_Current_Type == 2) { + string Password; + DataManager::GetValue("tw_restore_password", Password); + int ret = TWFunc::Try_Decrypting_File(tarfn, Password); + if (ret < 1) { + LOGERR("Failed to decrypt tar file '%s'\n", tarfn.c_str()); + return -1; + } + if (ret == 1) { + LOGERR("Decrypted file is not in tar format.\n"); + return -1; + } + if (ret == 3) { + LOGINFO("Extracting encrypted and compressed tar.\n"); + Archive_Current_Type = 3; + } else + LOGINFO("Extracting encrypted tar.\n"); + return extractTar(); + } else { LOGINFO("Extracting uncompressed tar\n"); return extractTar(); } @@ -348,8 +660,17 @@ int twrpTar::tarDirs(bool include_root) { DIR* d; string mainfolder = tardir + "/", subfolder; char buf[PATH_MAX]; + + char excl[1024]; + string::size_type i; + bool skip; + d = opendir(tardir.c_str()); - if (d != NULL) { + if (d != NULL) { + if (!tarexclude.empty()) { + strcpy(excl, tarexclude.c_str()); + split = TWFunc::split_string(tarexclude, ' ', true); + } struct dirent* de; while ((de = readdir(d)) != NULL) { #ifdef RECOVERY_SDCARD_ON_DATA @@ -357,6 +678,21 @@ int twrpTar::tarDirs(bool include_root) { #endif if (de->d_type == DT_BLK || de->d_type == DT_CHR || strcmp(de->d_name, "..") == 0) continue; + + // Skip excluded stuff + if (split.size() > 0) { + skip = false; + for (i = 0; i < split.size(); i++) { + if (strcmp(de->d_name, split[i].c_str()) == 0) { + LOGINFO("excluding %s\n", de->d_name); + skip = true; + break; + } + } + if (skip) + continue; + } + subfolder = mainfolder; if (strcmp(de->d_name, ".") != 0) { subfolder += de->d_name; @@ -376,7 +712,7 @@ int twrpTar::tarDirs(bool include_root) { string temp = Strip_Root_Dir(buf); charTarPath = (char*) temp.c_str(); } - if (tar_append_tree(t, buf, charTarPath) != 0) { + if (tar_append_tree(t, buf, charTarPath, excl) != 0) { LOGERR("Error appending '%s' to tar archive '%s'\n", buf, tarfn.c_str()); return -1; } @@ -391,103 +727,476 @@ int twrpTar::tarDirs(bool include_root) { return 0; } -int twrpTar::createTGZ() { - bool gzip = true; +int twrpTar::tarList(bool include_root, std::vector *TarList, unsigned thread_id) { + struct stat st; + char buf[PATH_MAX]; + int list_size = TarList->size(), i = 0, archive_count = 0; + string temp; + char actual_filename[PATH_MAX]; + + basefn = tarfn; + temp = basefn + "%i%02i"; + sprintf(actual_filename, temp.c_str(), thread_id, archive_count); + tarfn = actual_filename; + if (createTar() != 0) { + LOGERR("Error creating tar '%s' for thread %i\n", tarfn.c_str(), thread_id); + return -2; + } + Archive_Current_Size = 0; - init_libtar_buffer(0); - if (createTar() == -1) - return -1; - if (tarDirs(false) == -1) - return -1; - if (closeTar(gzip) == -1) - return -1; - free_libtar_buffer(); + while (i < list_size) { + if (TarList->at(i).thread_id == thread_id) { + strcpy(buf, TarList->at(i).fn.c_str()); + stat(buf, &st); + if (st.st_mode & S_IFREG) { // item is a regular file + if (Archive_Current_Size + (unsigned long long)(st.st_size) > MAX_ARCHIVE_SIZE) { + if (closeTar() != 0) { + LOGERR("Error closing '%s' on thread %i\n", tarfn.c_str(), thread_id); + return -3; + } + archive_count++; + LOGINFO("Splitting thread ID %i into archive %i\n", thread_id, archive_count); + if (archive_count > 99) { + LOGINFO("BLAH!\n"); + LOGERR("Too many archives for thread %i\n", thread_id); + return -4; + } + sprintf(actual_filename, temp.c_str(), thread_id, archive_count); + tarfn = actual_filename; + if (createTar() != 0) { + LOGERR("Error creating tar '%s' for thread %i\n", tarfn.c_str(), thread_id); + return -2; + } + Archive_Current_Size = 0; + } + Archive_Current_Size += (unsigned long long)(st.st_size); + } + if (addFile(buf, include_root) != 0) { + LOGERR("Error adding file '%s' to '%s'\n", buf, tarfn.c_str()); + return -1; + } + } + i++; + } + if (closeTar() != 0) { + LOGERR("Error closing '%s' on thread %i\n", tarfn.c_str(), thread_id); + return -3; + } + LOGINFO("Thread id %i tarList done, %i archives.\n", thread_id, archive_count, i, list_size); return 0; } int twrpTar::create() { - bool gzip = false; init_libtar_buffer(0); if (createTar() == -1) return -1; if (tarDirs(false) == -1) return -1; - if (closeTar(gzip) == -1) + if (closeTar() == -1) return -1; free_libtar_buffer(); return 0; } +void* twrpTar::createList(void *cookie) { + + twrpTar* threadTar = (twrpTar*) cookie; + if (threadTar->tarList(true, threadTar->ItemList, threadTar->thread_id) == -1) { + LOGINFO("ERROR tarList for thread ID %i\n", threadTar->thread_id); + return (void*)-2; + } + LOGINFO("Thread ID %i finished successfully.\n", threadTar->thread_id); + return (void*)0; +} + +void* twrpTar::extractMulti(void *cookie) { + + twrpTar* threadTar = (twrpTar*) cookie; + int archive_count = 0; + string temp = threadTar->basefn + "%i%02i"; + char actual_filename[255]; + sprintf(actual_filename, temp.c_str(), threadTar->thread_id, archive_count); + while (TWFunc::Path_Exists(actual_filename)) { + threadTar->tarfn = actual_filename; + if (threadTar->extract() != 0) { + LOGINFO("Error extracting '%s' in thread ID %i\n", actual_filename, threadTar->thread_id); + return (void*)-2; + } + archive_count++; + if (archive_count > 99) + break; + sprintf(actual_filename, temp.c_str(), threadTar->thread_id, archive_count); + } + LOGINFO("Thread ID %i finished successfully.\n", threadTar->thread_id); + return (void*)0; +} + int twrpTar::addFilesToExistingTar(vector files, string fn) { char* charTarFile = (char*) fn.c_str(); - static tartype_t type = { open, close, read, write_tar }; - init_libtar_buffer(0); - if (tar_open(&t, charTarFile, &type, O_RDONLY | O_LARGEFILE, 0644, TAR_GNU) == -1) + if (tar_open(&t, charTarFile, NULL, O_RDONLY | O_LARGEFILE, 0644, TAR_GNU) == -1) return -1; removeEOT(charTarFile); - if (tar_open(&t, charTarFile, &type, O_WRONLY | O_APPEND | O_LARGEFILE, 0644, TAR_GNU) == -1) + if (tar_open(&t, charTarFile, NULL, O_WRONLY | O_APPEND | O_LARGEFILE, 0644, TAR_GNU) == -1) return -1; for (unsigned int i = 0; i < files.size(); ++i) { char* file = (char*) files.at(i).c_str(); if (tar_append_file(t, file, file) == -1) return -1; } - flush_libtar_buffer(t->fd); if (tar_append_eof(t) == -1) return -1; if (tar_close(t) == -1) return -1; - free_libtar_buffer(); return 0; } int twrpTar::createTar() { char* charTarFile = (char*) tarfn.c_str(); char* charRootDir = (char*) tardir.c_str(); - int use_compression = 0; static tartype_t type = { open, close, read, write_tar }; + string Password; + + if (use_encryption && use_compression) { + // Compressed and encrypted + Archive_Current_Type = 3; + LOGINFO("Using encryption and compression...\n"); + DataManager::GetValue("tw_backup_password", Password); + int i, pipes[4]; - DataManager::GetValue(TW_USE_COMPRESSION_VAR, use_compression); - if (use_compression) { - string cmd = "pigz - > '" + tarfn + "'"; - p = popen(cmd.c_str(), "w"); - fd = fileno(p); - if (!p) return -1; - if(tar_fdopen(&t, fd, charRootDir, &type, O_RDONLY | O_LARGEFILE, 0644, TAR_GNU) != 0) { - pclose(p); + if (pipe(pipes) < 0) { + LOGERR("Error creating first pipe\n"); return -1; } - } - else { - if (tar_open(&t, charTarFile, &type, O_WRONLY | O_CREAT | O_LARGEFILE, 0644, TAR_GNU) == -1) + if (pipe(pipes + 2) < 0) { + LOGERR("Error creating second pipe\n"); + return -1; + } + pigz_pid = fork(); + + if (pigz_pid < 0) { + LOGERR("pigz fork() failed\n"); + for (i = 0; i < 4; i++) + close(pipes[i]); // close all + return -1; + } else if (pigz_pid == 0) { + // pigz Child + close(pipes[1]); + close(pipes[2]); + close(0); + dup2(pipes[0], 0); + close(1); + dup2(pipes[3], 1); + if (execlp("pigz", "pigz", "-", NULL) < 0) { + LOGERR("execlp pigz ERROR!\n"); + close(pipes[0]); + close(pipes[3]); + _exit(-1); + } + } else { + // Parent + oaes_pid = fork(); + + if (oaes_pid < 0) { + LOGERR("openaes fork() failed\n"); + for (i = 0; i < 4; i++) + close(pipes[i]); // close all + return -1; + } else if (oaes_pid == 0) { + // openaes Child + int output_fd = open(tarfn.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); + if (output_fd < 0) { + LOGERR("Failed to open '%s'\n", tarfn.c_str()); + for (i = 0; i < 4; i++) + close(pipes[i]); // close all + return -1; + } + close(pipes[0]); + close(pipes[1]); + close(pipes[3]); + close(0); + dup2(pipes[2], 0); + close(1); + dup2(output_fd, 1); + if (execlp("openaes", "openaes", "enc", "--key", Password.c_str(), NULL) < 0) { + LOGERR("execlp openaes ERROR!\n"); + close(pipes[2]); + close(output_fd); + _exit(-1); + } + } else { + // Parent + close(pipes[0]); + close(pipes[2]); + close(pipes[3]); + fd = pipes[1]; + if(tar_fdopen(&t, fd, charRootDir, NULL, O_WRONLY | O_CREAT | O_EXCL | O_LARGEFILE, 0644, TAR_GNU) != 0) { + close(fd); + LOGERR("tar_fdopen failed\n"); + return -1; + } + return 0; + } + } + } else if (use_compression) { + // Compressed + Archive_Current_Type = 1; + LOGINFO("Using compression...\n"); + int pigzfd[2]; + + if (pipe(pigzfd) < 0) { + LOGERR("Error creating pipe\n"); + return -1; + } + pigz_pid = fork(); + + if (pigz_pid < 0) { + LOGERR("fork() failed\n"); + close(pigzfd[0]); + close(pigzfd[1]); + return -1; + } else if (pigz_pid == 0) { + // Child + close(pigzfd[1]); // close unused output pipe + int output_fd = open(tarfn.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); + if (output_fd < 0) { + LOGERR("Failed to open '%s'\n", tarfn.c_str()); + close(pigzfd[0]); + _exit(-1); + } + dup2(pigzfd[0], 0); // remap stdin + dup2(output_fd, 1); // remap stdout to output file + if (execlp("pigz", "pigz", "-", NULL) < 0) { + LOGERR("execlp pigz ERROR!\n"); + close(output_fd); + close(pigzfd[0]); + _exit(-1); + } + } else { + // Parent + close(pigzfd[0]); // close parent input + fd = pigzfd[1]; // copy parent output + if(tar_fdopen(&t, fd, charRootDir, NULL, O_WRONLY | O_CREAT | O_EXCL | O_LARGEFILE, 0644, TAR_GNU) != 0) { + close(fd); + LOGERR("tar_fdopen failed\n"); + return -1; + } + } + } else if (use_encryption) { + // Encrypted + Archive_Current_Type = 2; + LOGINFO("Using encryption...\n"); + DataManager::GetValue("tw_backup_password", Password); + int oaesfd[2]; + pipe(oaesfd); + oaes_pid = fork(); + + if (oaes_pid < 0) { + LOGERR("fork() failed\n"); + close(oaesfd[0]); + close(oaesfd[1]); return -1; + } else if (oaes_pid == 0) { + // Child + close(oaesfd[1]); // close unused + int output_fd = open(tarfn.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); + if (output_fd < 0) { + LOGERR("Failed to open '%s'\n", tarfn.c_str()); + _exit(-1); + } + dup2(oaesfd[0], 0); // remap stdin + dup2(output_fd, 1); // remap stdout to output file + if (execlp("openaes", "openaes", "enc", "--key", Password.c_str(), NULL) < 0) { + LOGERR("execlp openaes ERROR!\n"); + close(output_fd); + close(oaesfd[0]); + _exit(-1); + } + } else { + // Parent + close(oaesfd[0]); // close parent input + fd = oaesfd[1]; // copy parent output + if(tar_fdopen(&t, fd, charRootDir, NULL, O_WRONLY | O_CREAT | O_EXCL | O_LARGEFILE, 0644, TAR_GNU) != 0) { + close(fd); + LOGERR("tar_fdopen failed\n"); + return -1; + } + return 0; + } + } else { + // Not compressed or encrypted + init_libtar_buffer(0); + if (tar_open(&t, charTarFile, &type, O_WRONLY | O_CREAT | O_LARGEFILE, 0644, TAR_GNU) == -1) { + LOGERR("tar_open error opening '%s'\n", tarfn.c_str()); + return -1; + } } return 0; } -int twrpTar::openTar(bool gzip) { +int twrpTar::openTar() { char* charRootDir = (char*) tardir.c_str(); char* charTarFile = (char*) tarfn.c_str(); + string Password; - if (gzip) { - LOGINFO("Opening as a gzip\n"); - string cmd = "pigz -d -c '" + tarfn + "'"; - FILE* pipe = popen(cmd.c_str(), "r"); - int fd = fileno(pipe); - if (!pipe) return -1; - if(tar_fdopen(&t, fd, charRootDir, NULL, O_RDONLY | O_LARGEFILE, 0644, TAR_GNU) != 0) { - LOGINFO("tar_fdopen returned error\n"); - __pclose(pipe); + if (Archive_Current_Type == 3) { + LOGINFO("Opening encrypted and compressed backup...\n"); + DataManager::GetValue("tw_restore_password", Password); + int i, pipes[4]; + + if (pipe(pipes) < 0) { + LOGERR("Error creating first pipe\n"); return -1; } - } - else { - if (tar_open(&t, charTarFile, NULL, O_RDONLY | O_LARGEFILE, 0644, TAR_GNU) != 0) { - LOGERR("Unable to open tar archive '%s'\n", charTarFile); + if (pipe(pipes + 2) < 0) { + LOGERR("Error creating second pipe\n"); + return -1; + } + oaes_pid = fork(); + + if (oaes_pid < 0) { + LOGERR("pigz fork() failed\n"); + for (i = 0; i < 4; i++) + close(pipes[i]); // close all + return -1; + } else if (oaes_pid == 0) { + // openaes Child + close(pipes[0]); // Close pipes that are not used by this child + close(pipes[2]); + close(pipes[3]); + int input_fd = open(tarfn.c_str(), O_RDONLY | O_LARGEFILE); + if (input_fd < 0) { + LOGERR("Failed to open '%s'\n", tarfn.c_str()); + close(pipes[1]); + _exit(-1); + } + close(0); + dup2(input_fd, 0); + close(1); + dup2(pipes[1], 1); + if (execlp("openaes", "openaes", "dec", "--key", Password.c_str(), NULL) < 0) { + LOGERR("execlp openaes ERROR!\n"); + close(input_fd); + close(pipes[1]); + _exit(-1); + } + } else { + // Parent + pigz_pid = fork(); + + if (pigz_pid < 0) { + LOGERR("openaes fork() failed\n"); + for (i = 0; i < 4; i++) + close(pipes[i]); // close all + return -1; + } else if (pigz_pid == 0) { + // pigz Child + close(pipes[1]); // Close pipes not used by this child + close(pipes[2]); + close(0); + dup2(pipes[0], 0); + close(1); + dup2(pipes[3], 1); + if (execlp("pigz", "pigz", "-d", "-c", NULL) < 0) { + LOGERR("execlp pigz ERROR!\n"); + close(pipes[0]); + close(pipes[3]); + _exit(-1); + } + } else { + // Parent + close(pipes[0]); // Close pipes not used by parent + close(pipes[1]); + close(pipes[3]); + fd = pipes[2]; + if(tar_fdopen(&t, fd, charRootDir, NULL, O_RDONLY | O_LARGEFILE, 0644, TAR_GNU) != 0) { + close(fd); + LOGERR("tar_fdopen failed\n"); + return -1; + } + } + } + } else if (Archive_Current_Type == 2) { + LOGINFO("Opening encrypted backup...\n"); + DataManager::GetValue("tw_restore_password", Password); + int oaesfd[2]; + + pipe(oaesfd); + oaes_pid = fork(); + if (oaes_pid < 0) { + LOGERR("fork() failed\n"); + close(oaesfd[0]); + close(oaesfd[1]); + return -1; + } else if (oaes_pid == 0) { + // Child + close(oaesfd[0]); // Close unused pipe + int input_fd = open(tarfn.c_str(), O_RDONLY | O_LARGEFILE); + if (input_fd < 0) { + LOGERR("Failed to open '%s'\n", tarfn.c_str()); + close(oaesfd[1]); + _exit(-1); + } + close(0); // close stdin + dup2(oaesfd[1], 1); // remap stdout + dup2(input_fd, 0); // remap input fd to stdin + if (execlp("openaes", "openaes", "dec", "--key", Password.c_str(), NULL) < 0) { + LOGERR("execlp openaes ERROR!\n"); + close(input_fd); + close(oaesfd[1]); + _exit(-1); + } + } else { + // Parent + close(oaesfd[1]); // close parent output + fd = oaesfd[0]; // copy parent input + if(tar_fdopen(&t, fd, charRootDir, NULL, O_RDONLY | O_LARGEFILE, 0644, TAR_GNU) != 0) { + close(fd); + LOGERR("tar_fdopen failed\n"); + return -1; + } + } + } else if (Archive_Current_Type == 1) { + LOGINFO("Opening as a gzip...\n"); + int pigzfd[2]; + pipe(pigzfd); + + pigz_pid = fork(); + if (pigz_pid < 0) { + LOGERR("fork() failed\n"); + close(pigzfd[0]); + close(pigzfd[1]); return -1; + } else if (pigz_pid == 0) { + // Child + close(pigzfd[0]); + int input_fd = open(tarfn.c_str(), O_RDONLY | O_LARGEFILE); + if (input_fd < 0) { + LOGERR("Failed to open '%s'\n", tarfn.c_str()); + _exit(-1); + } + dup2(input_fd, 0); // remap input fd to stdin + dup2(pigzfd[1], 1); // remap stdout + if (execlp("pigz", "pigz", "-d", "-c", NULL) < 0) { + close(pigzfd[1]); + close(input_fd); + LOGERR("execlp openaes ERROR!\n"); + _exit(-1); + } + } else { + // Parent + close(pigzfd[1]); // close parent output + fd = pigzfd[0]; // copy parent input + if(tar_fdopen(&t, fd, charRootDir, NULL, O_RDONLY | O_LARGEFILE, 0644, TAR_GNU) != 0) { + close(fd); + LOGERR("tar_fdopen failed\n"); + return -1; + } } + } else if (tar_open(&t, charTarFile, NULL, O_RDONLY | O_LARGEFILE, 0644, TAR_GNU) != 0) { + LOGERR("Unable to open tar archive '%s'\n", charTarFile); + return -1; } return 0; } @@ -526,10 +1235,7 @@ int twrpTar::addFile(string fn, bool include_root) { return 0; } -int twrpTar::closeTar(bool gzip) { - int use_compression; - DataManager::GetValue(TW_USE_COMPRESSION_VAR, use_compression); - +int twrpTar::closeTar() { flush_libtar_buffer(t->fd); if (tar_append_eof(t) != 0) { LOGERR("tar_append_eof(): %s\n", strerror(errno)); @@ -540,11 +1246,16 @@ int twrpTar::closeTar(bool gzip) { LOGERR("Unable to close tar archive: '%s'\n", tarfn.c_str()); return -1; } - if (use_compression || gzip) { - LOGINFO("Closing popen and fd\n"); - pclose(p); + if (Archive_Current_Type > 1) { close(fd); + int status; + if (pigz_pid > 0 && TWFunc::Wait_For_Child(pigz_pid, &status, "pigz") != 0) + return -1; + if (oaes_pid > 0 && TWFunc::Wait_For_Child(oaes_pid, &status, "openaes") != 0) + return -1; } + free_libtar_buffer(); + TWFunc::tw_chmod(tarfn, "644"); return 0; } @@ -563,51 +1274,55 @@ int twrpTar::removeEOT(string tarFile) { return 0; } -int twrpTar::compress(string fn) { - string cmd = "pigz " + fn; - p = popen(cmd.c_str(), "r"); - if (!p) return -1; - char buffer[128]; - string result = ""; - while(!feof(p)) { - if(fgets(buffer, 128, p) != NULL) - result += buffer; - } - __pclose(p); - return 0; -} - -int twrpTar::extractTGZ() { - string splatrootdir(tardir); - bool gzip = true; - char* splatCharRootDir = (char*) splatrootdir.c_str(); - if (openTar(gzip) == -1) - return -1; - int ret = tar_extract_all(t, splatCharRootDir); - if (tar_close(t) != 0) { - LOGERR("Unable to close tar file\n"); - return -1; - } - return 0; -} - int twrpTar::entryExists(string entry) { char* searchstr = (char*)entry.c_str(); int ret; - int Archive_Current_Type = getArchiveType(); + Archive_Current_Type = TWFunc::Get_File_Type(tarfn); - if (openTar(Archive_Current_Type) == -1) + if (openTar() == -1) ret = 0; else ret = tar_find(t, searchstr); - if (tar_close(t) != 0) - LOGINFO("Unable to close tar file after searching for entry '%s'.\n", entry.c_str()); + if (closeTar() != 0) + LOGINFO("Unable to close tar after searching for entry.\n"); return ret; } +unsigned long long twrpTar::uncompressedSize() { + int type = 0; + unsigned long long total_size = 0; + string Tar, Command, result; + vector split; + + Tar = TWFunc::Get_Filename(tarfn); + type = TWFunc::Get_File_Type(tarfn); + if (type == 0) + total_size = TWFunc::Get_File_Size(tarfn); + else { + Command = "pigz -l " + tarfn; + /* if we set Command = "pigz -l " + tarfn + " | sed '1d' | cut -f5 -d' '"; + we get the uncompressed size at once. */ + TWFunc::Exec_Cmd(Command, result); + if (!result.empty()) { + /* Expected output: + compressed original reduced name + 95855838 179403776 -1.3% data.yaffs2.win + ^ + split[5] + */ + split = TWFunc::split_string(result, ' ', true); + if (split.size() > 4) + total_size = atoi(split[5].c_str()); + } + } + LOGINFO("%s's uncompressed size: %llu bytes\n", Tar.c_str(), total_size); + + return total_size; +} + extern "C" ssize_t write_tar(int fd, const void *buffer, size_t size) { return (ssize_t) write_libtar_buffer(fd, buffer, size); } -- cgit v1.2.3