summaryrefslogtreecommitdiffstats
path: root/Tools/MCADefrag
diff options
context:
space:
mode:
Diffstat (limited to 'Tools/MCADefrag')
-rw-r--r--Tools/MCADefrag/.gitignore1
-rw-r--r--Tools/MCADefrag/CMakeLists.txt144
-rw-r--r--Tools/MCADefrag/Globals.cpp10
-rw-r--r--Tools/MCADefrag/Globals.h229
-rw-r--r--Tools/MCADefrag/MCADefrag.cpp421
-rw-r--r--Tools/MCADefrag/MCADefrag.h144
6 files changed, 949 insertions, 0 deletions
diff --git a/Tools/MCADefrag/.gitignore b/Tools/MCADefrag/.gitignore
new file mode 100644
index 000000000..44a3e1f48
--- /dev/null
+++ b/Tools/MCADefrag/.gitignore
@@ -0,0 +1 @@
+*.mca
diff --git a/Tools/MCADefrag/CMakeLists.txt b/Tools/MCADefrag/CMakeLists.txt
new file mode 100644
index 000000000..7296b8ddc
--- /dev/null
+++ b/Tools/MCADefrag/CMakeLists.txt
@@ -0,0 +1,144 @@
+
+cmake_minimum_required (VERSION 2.6)
+
+project (MCADefrag)
+
+
+
+macro(add_flags_cxx FLAGS)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FLAGS}")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FLAGS}")
+ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${FLAGS}")
+ set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${FLAGS}")
+ set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${FLAGS}")
+ set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${FLAGS}")
+endmacro()
+
+
+
+
+# Add the preprocessor macros used for distinguishing between debug and release builds (CMake does this automatically for MSVC):
+if (NOT MSVC)
+ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_DEBUG")
+ set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -D_DEBUG")
+ set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DNDEBUG")
+ set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -DNDEBUG")
+endif()
+
+
+
+if(MSVC)
+ # Make build use multiple threads under MSVC:
+ add_flags_cxx("/MP")
+
+ # Make release builds use link-time code generation:
+ set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /GL")
+ set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} /GL")
+ set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /LTCG")
+ set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /LTCG")
+ set(CMAKE_MODULE_LINKER_FLAGS_RELEASE "${CMAKE_MODULE_LINKER_FLAGS_RELEASE} /LTCG")
+elseif(APPLE)
+ #on os x clang adds pthread for us but we need to add it for gcc
+ if (NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+ add_flags_cxx("-pthread")
+ endif()
+else()
+ # Let gcc / clang know that we're compiling a multi-threaded app:
+ add_flags_cxx("-pthread")
+endif()
+
+
+
+
+# Use static CRT in MSVC builds:
+if (MSVC)
+ string(REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
+ string(REPLACE "/MD" "/MT" CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
+ string(REPLACE "/MDd" "/MTd" CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}")
+ string(REPLACE "/MDd" "/MTd" CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}")
+endif()
+
+
+
+
+# Set include paths to the used libraries:
+include_directories("../../lib")
+include_directories("../../src")
+
+
+
+function(flatten_files arg1)
+ set(res "")
+ foreach(f ${${arg1}})
+ get_filename_component(f ${f} ABSOLUTE)
+ list(APPEND res ${f})
+ endforeach()
+ set(${arg1} "${res}" PARENT_SCOPE)
+endfunction()
+
+
+# Include the libraries:
+file(GLOB ZLIB_SRC "../../lib/zlib/*.c")
+file(GLOB ZLIB_HDR "../../lib/zlib/*.h")
+flatten_files(ZLIB_SRC)
+flatten_files(ZLIB_HDR)
+source_group("ZLib" FILES ${ZLIB_SRC} ${ZLIB_HDR})
+
+
+# Include the shared files:
+set(SHARED_SRC
+ ../../src/StringCompression.cpp
+ ../../src/StringUtils.cpp
+ ../../src/Log.cpp
+ ../../src/MCLogger.cpp
+)
+set(SHARED_HDR
+ ../../src/ByteBuffer.h
+ ../../src/StringUtils.h
+ ../../src/Log.h
+ ../../src/MCLogger.h
+)
+set(SHARED_OSS_SRC
+ ../../src/OSSupport/CriticalSection.cpp
+ ../../src/OSSupport/File.cpp
+ ../../src/OSSupport/IsThread.cpp
+ ../../src/OSSupport/Timer.cpp
+)
+set(SHARED_OSS_HDR
+ ../../src/OSSupport/CriticalSection.h
+ ../../src/OSSupport/File.h
+ ../../src/OSSupport/IsThread.h
+ ../../src/OSSupport/Timer.h
+)
+flatten_files(SHARED_SRC)
+flatten_files(SHARED_HDR)
+flatten_files(SHARED_OSS_SRC)
+flatten_files(SHARED_OSS_HDR)
+source_group("Shared" FILES ${SHARED_SRC} ${SHARED_HDR})
+source_group("Shared\\OSSupport" FILES ${SHARED_OSS_SRC} ${SHARED_OSS_HDR})
+
+
+
+# Include the main source files:
+set(SOURCES
+ MCADefrag.cpp
+ Globals.cpp
+)
+set(HEADERS
+ MCADefrag.h
+ Globals.h
+)
+
+source_group("" FILES ${SOURCES} ${HEADERS})
+
+add_executable(MCADefrag
+ ${SOURCES}
+ ${HEADERS}
+ ${SHARED_SRC}
+ ${SHARED_HDR}
+ ${SHARED_OSS_SRC}
+ ${SHARED_OSS_HDR}
+ ${ZLIB_SRC}
+ ${ZLIB_HDR}
+)
+
diff --git a/Tools/MCADefrag/Globals.cpp b/Tools/MCADefrag/Globals.cpp
new file mode 100644
index 000000000..13c6ae709
--- /dev/null
+++ b/Tools/MCADefrag/Globals.cpp
@@ -0,0 +1,10 @@
+
+// Globals.cpp
+
+// This file is used for precompiled header generation in MSVC environments
+
+#include "Globals.h"
+
+
+
+
diff --git a/Tools/MCADefrag/Globals.h b/Tools/MCADefrag/Globals.h
new file mode 100644
index 000000000..6f4bbdc76
--- /dev/null
+++ b/Tools/MCADefrag/Globals.h
@@ -0,0 +1,229 @@
+
+// Globals.h
+
+// This file gets included from every module in the project, so that global symbols may be introduced easily
+// Also used for precompiled header generation in MSVC environments
+
+
+
+
+
+// Compiler-dependent stuff:
+#if defined(_MSC_VER)
+ // MSVC produces warning C4481 on the override keyword usage, so disable the warning altogether
+ #pragma warning(disable:4481)
+
+ // Disable some warnings that we don't care about:
+ #pragma warning(disable:4100)
+
+ #define OBSOLETE __declspec(deprecated)
+
+ // No alignment needed in MSVC
+ #define ALIGN_8
+ #define ALIGN_16
+
+#elif defined(__GNUC__)
+
+ // TODO: Can GCC explicitly mark classes as abstract (no instances can be created)?
+ #define abstract
+
+ // TODO: Can GCC mark virtual methods as overriding (forcing them to have a virtual function of the same signature in the base class)
+ #define override
+
+ #define OBSOLETE __attribute__((deprecated))
+
+ #define ALIGN_8 __attribute__((aligned(8)))
+ #define ALIGN_16 __attribute__((aligned(16)))
+
+ // Some portability macros :)
+ #define stricmp strcasecmp
+
+#else
+
+ #error "You are using an unsupported compiler, you might need to #define some stuff here for your compiler"
+
+ /*
+ // Copy and uncomment this into another #elif section based on your compiler identification
+
+ // Explicitly mark classes as abstract (no instances can be created)
+ #define abstract
+
+ // Mark virtual methods as overriding (forcing them to have a virtual function of the same signature in the base class)
+ #define override
+
+ // Mark functions as obsolete, so that their usage results in a compile-time warning
+ #define OBSOLETE
+
+ // Mark types / variables for alignment. Do the platforms need it?
+ #define ALIGN_8
+ #define ALIGN_16
+ */
+
+#endif
+
+
+
+
+
+// Integral types with predefined sizes:
+typedef long long Int64;
+typedef int Int32;
+typedef short Int16;
+
+typedef unsigned long long UInt64;
+typedef unsigned int UInt32;
+typedef unsigned short UInt16;
+
+typedef unsigned char Byte;
+
+
+
+
+
+// A macro to disallow the copy constructor and operator= functions
+// This should be used in the private: declarations for any class that shouldn't allow copying itself
+#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
+ TypeName(const TypeName &); \
+ void operator=(const TypeName &)
+
+// A macro that is used to mark unused function parameters, to avoid pedantic warnings in gcc
+#define UNUSED(X) (void)(X)
+
+
+
+
+// OS-dependent stuff:
+#ifdef _WIN32
+ #define WIN32_LEAN_AND_MEAN
+ #include <Windows.h>
+ #include <winsock2.h>
+ #include <ws2tcpip.h>
+
+ // Windows SDK defines min and max macros, messing up with our std::min and std::max usage
+ #undef min
+ #undef max
+
+ // Windows SDK defines GetFreeSpace as a constant, probably a Win16 API remnant
+ #ifdef GetFreeSpace
+ #undef GetFreeSpace
+ #endif // GetFreeSpace
+
+ #define SocketError WSAGetLastError()
+#else
+ #include <sys/types.h>
+ #include <sys/stat.h> // for mkdir
+ #include <sys/time.h>
+ #include <sys/socket.h>
+ #include <netinet/in.h>
+ #include <arpa/inet.h>
+ #include <netdb.h>
+ #include <time.h>
+ #include <dirent.h>
+ #include <errno.h>
+ #include <iostream>
+ #include <unistd.h>
+
+ #include <cstdio>
+ #include <cstring>
+ #include <pthread.h>
+ #include <semaphore.h>
+ #include <errno.h>
+ #include <fcntl.h>
+
+ typedef int SOCKET;
+ enum
+ {
+ INVALID_SOCKET = -1,
+ };
+ #define closesocket close
+ #define SocketError errno
+#if !defined(ANDROID_NDK)
+ #include <tr1/memory>
+#endif
+#endif
+
+#if !defined(ANDROID_NDK)
+ #define USE_SQUIRREL
+#endif
+
+#if defined(ANDROID_NDK)
+ #define FILE_IO_PREFIX "/sdcard/mcserver/"
+#else
+ #define FILE_IO_PREFIX ""
+#endif
+
+
+
+
+
+// CRT stuff:
+#include <assert.h>
+#include <stdio.h>
+#include <math.h>
+#include <stdarg.h>
+#include <time.h>
+
+
+
+
+
+// STL stuff:
+#include <vector>
+#include <list>
+#include <deque>
+#include <string>
+#include <map>
+#include <algorithm>
+#include <memory>
+
+
+
+
+
+// Common headers (without macros):
+#include "StringUtils.h"
+#include "OSSupport/CriticalSection.h"
+#include "OSSupport/IsThread.h"
+#include "OSSupport/File.h"
+
+
+
+
+
+// Common definitions:
+
+/// Evaluates to the number of elements in an array (compile-time!)
+#define ARRAYCOUNT(X) (sizeof(X) / sizeof(*(X)))
+
+/// Allows arithmetic expressions like "32 KiB" (but consider using parenthesis around it, "(32 KiB)" )
+#define KiB * 1024
+#define MiB * 1024 * 1024
+
+/// Faster than (int)floorf((float)x / (float)div)
+#define FAST_FLOOR_DIV( x, div ) ( (x) < 0 ? (((int)x / div) - 1) : ((int)x / div) )
+
+// Own version of assert() that writes failed assertions to the log for review
+#ifdef NDEBUG
+ #define ASSERT(x) ((void)0)
+#else
+ #define ASSERT assert
+#endif
+
+// Pretty much the same as ASSERT() but stays in Release builds
+#define VERIFY( x ) ( !!(x) || ( LOGERROR("Verification failed: %s, file %s, line %i", #x, __FILE__, __LINE__ ), exit(1), 0 ) )
+
+
+
+
+
+/// A generic interface used mainly in ForEach() functions
+template <typename Type> class cItemCallback
+{
+public:
+ /// Called for each item in the internal list; return true to stop the loop, or false to continue enumerating
+ virtual bool Item(Type * a_Type) = 0;
+} ;
+
+
+
+
diff --git a/Tools/MCADefrag/MCADefrag.cpp b/Tools/MCADefrag/MCADefrag.cpp
new file mode 100644
index 000000000..a2de7f957
--- /dev/null
+++ b/Tools/MCADefrag/MCADefrag.cpp
@@ -0,0 +1,421 @@
+
+// MCADefrag.cpp
+
+// Implements the main app entrypoint and the cMCADefrag class representing the entire app
+
+#include "Globals.h"
+#include "MCADefrag.h"
+#include "MCLogger.h"
+#include "zlib/zlib.h"
+
+
+
+
+
+// An array of 4096 zero bytes, used for writing the padding
+static const Byte g_Zeroes[4096] = {0};
+
+
+
+
+
+int main(int argc, char ** argv)
+{
+ new cMCLogger(Printf("Defrag_%08x.log", time(NULL)));
+ cMCADefrag Defrag;
+ if (!Defrag.Init(argc, argv))
+ {
+ return 1;
+ }
+
+ Defrag.Run();
+
+ return 0;
+}
+
+
+
+
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// cMCADefrag:
+
+cMCADefrag::cMCADefrag(void) :
+ m_NumThreads(4),
+ m_ShouldRecompress(true)
+{
+}
+
+
+
+
+
+bool cMCADefrag::Init(int argc, char ** argv)
+{
+ // Nothing needed yet
+ return true;
+}
+
+
+
+
+
+void cMCADefrag::Run(void)
+{
+ // Fill the queue with MCA files
+ m_Queue = cFile::GetFolderContents(".");
+
+ // Start the processing threads:
+ for (int i = 0; i < m_NumThreads; i++)
+ {
+ StartThread();
+ }
+
+ // Wait for all the threads to finish:
+ while (!m_Threads.empty())
+ {
+ m_Threads.front()->Wait();
+ delete m_Threads.front();
+ m_Threads.pop_front();
+ }
+}
+
+
+
+
+void cMCADefrag::StartThread(void)
+{
+ cThread * Thread = new cThread(*this);
+ m_Threads.push_back(Thread);
+ Thread->Start();
+}
+
+
+
+
+
+AString cMCADefrag::GetNextFileName(void)
+{
+ cCSLock Lock(m_CS);
+ if (m_Queue.empty())
+ {
+ return AString();
+ }
+ AString res = m_Queue.back();
+ m_Queue.pop_back();
+ return res;
+}
+
+
+
+
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// cMCADefrag::cThread:
+
+cMCADefrag::cThread::cThread(cMCADefrag & a_Parent) :
+ super("MCADefrag thread"),
+ m_Parent(a_Parent),
+ m_IsChunkUncompressed(false)
+{
+}
+
+
+
+
+
+void cMCADefrag::cThread::Execute(void)
+{
+ for (;;)
+ {
+ AString FileName = m_Parent.GetNextFileName();
+ if (FileName.empty())
+ {
+ return;
+ }
+ ProcessFile(FileName);
+ }
+}
+
+
+
+
+
+void cMCADefrag::cThread::ProcessFile(const AString & a_FileName)
+{
+ // Filter out non-MCA files:
+ if ((a_FileName.length() < 4) || (a_FileName.substr(a_FileName.length() - 4, 4) != ".mca"))
+ {
+ return;
+ }
+ LOGINFO("%s", a_FileName.c_str());
+
+ // Open input and output files:
+ AString OutFileName = a_FileName + ".new";
+ cFile In, Out;
+ if (!In.Open(a_FileName, cFile::fmRead))
+ {
+ LOGWARNING("Cannot open file %s for reading, skipping file.", a_FileName.c_str());
+ return;
+ }
+ if (!Out.Open(OutFileName.c_str(), cFile::fmWrite))
+ {
+ LOGWARNING("Cannot open file %s for writing, skipping file.", OutFileName.c_str());
+ return;
+ }
+
+ // Read the Locations and Timestamps from the input file:
+ Byte Locations[4096];
+ UInt32 Timestamps[1024];
+ if (In.Read(Locations, sizeof(Locations)) != sizeof(Locations))
+ {
+ LOGWARNING("Cannot read Locations in file %s, skipping file.", a_FileName.c_str());
+ return;
+ }
+ if (In.Read(Timestamps, sizeof(Timestamps)) != sizeof(Timestamps))
+ {
+ LOGWARNING("Cannot read Timestamps in file %s, skipping file.", a_FileName.c_str());
+ return;
+ }
+
+ // Write dummy Locations to the Out file (will be overwritten once the correct ones are known)
+ if (Out.Write(Locations, sizeof(Locations)) != sizeof(Locations))
+ {
+ LOGWARNING("Cannot write Locations to file %s, skipping file.", OutFileName.c_str());
+ return;
+ }
+ m_CurrentSectorOut = 2;
+
+ // Write a copy of the Timestamps into the Out file:
+ if (Out.Write(Timestamps, sizeof(Timestamps)) != sizeof(Timestamps))
+ {
+ LOGWARNING("Cannot write Timestamps to file %s, skipping file.", OutFileName.c_str());
+ return;
+ }
+
+ // Process each chunk:
+ for (size_t i = 0; i < 1024; i++)
+ {
+ size_t idx = i * 4;
+ if (
+ (Locations[idx] == 0) &&
+ (Locations[idx + 1] == 0) &&
+ (Locations[idx + 2] == 0) &&
+ (Locations[idx + 3] == 0)
+ )
+ {
+ // Chunk not present
+ continue;
+ }
+ m_IsChunkUncompressed = false;
+ if (!ReadChunk(In, Locations + idx))
+ {
+ LOGWARNING("Cannot read chunk #%d from file %s. Skipping file.", i, a_FileName.c_str());
+ return;
+ }
+ if (!WriteChunk(Out, Locations + idx))
+ {
+ LOGWARNING("Cannot write chunk #%d to file %s. Skipping file.", i, OutFileName.c_str());
+ return;
+ }
+ }
+
+ // Write the new Locations into the MCA header:
+ Out.Seek(0);
+ if (Out.Write(Locations, sizeof(Locations)) != sizeof(Locations))
+ {
+ LOGWARNING("Cannot write updated Locations to file %s, skipping file.", OutFileName.c_str());
+ return;
+ }
+
+ // Close the files, delete orig, rename new:
+ In.Close();
+ Out.Close();
+ cFile::Delete(a_FileName);
+ cFile::Rename(OutFileName, a_FileName);
+}
+
+
+
+
+
+bool cMCADefrag::cThread::ReadChunk(cFile & a_File, const Byte * a_LocationRaw)
+{
+ int SectorNum = (a_LocationRaw[0] << 16) | (a_LocationRaw[1] << 8) | a_LocationRaw[2];
+ int SizeInSectors = a_LocationRaw[3] * (4 KiB);
+ if (a_File.Seek(SectorNum * (4 KiB)) < 0)
+ {
+ LOGWARNING("Failed to seek to chunk data - file pos %llu (%d KiB, %.02f MiB)!", (Int64)SectorNum * (4 KiB), SectorNum * 4, ((double)SectorNum) / 256);
+ return false;
+ }
+
+ // Read the exact size:
+ Byte Buf[4];
+ if (a_File.Read(Buf, 4) != 4)
+ {
+ LOGWARNING("Failed to read chunk data length");
+ return false;
+ }
+ m_CompressedChunkDataSize = (Buf[0] << 24) | (Buf[1] << 16) | (Buf[2] << 8) | Buf[3];
+ if (m_CompressedChunkDataSize > SizeInSectors)
+ {
+ LOGWARNING("Invalid chunk data - SizeInSectors (%d) smaller that RealSize (%d)", SizeInSectors, m_CompressedChunkDataSize);
+ return false;
+ }
+
+ // Read the data:
+ if (a_File.Read(m_CompressedChunkData, m_CompressedChunkDataSize) != m_CompressedChunkDataSize)
+ {
+ LOGWARNING("Failed to read chunk data!");
+ return false;
+ }
+
+ // Uncompress the data if recompression is active
+ if (m_Parent.m_ShouldRecompress)
+ {
+ m_IsChunkUncompressed = UncompressChunk();
+ if (!m_IsChunkUncompressed)
+ {
+ LOGINFO("Chunk failed to uncompress, will be copied verbatim instead.");
+ }
+ }
+
+ return true;
+}
+
+
+
+
+
+bool cMCADefrag::cThread::WriteChunk(cFile & a_File, Byte * a_LocationRaw)
+{
+ // Recompress the data if recompression is active:
+ if (m_Parent.m_ShouldRecompress)
+ {
+ if (!CompressChunk())
+ {
+ LOGINFO("Chunk failed to recompress, will be coped verbatim instead.");
+ }
+ }
+
+ // Update the Location:
+ a_LocationRaw[0] = m_CurrentSectorOut >> 16;
+ a_LocationRaw[1] = (m_CurrentSectorOut >> 8) & 0xff;
+ a_LocationRaw[2] = m_CurrentSectorOut & 0xff;
+ a_LocationRaw[3] = (m_CompressedChunkDataSize + (4 KiB) + 3) / (4 KiB); // +3 because the m_CompressedChunkDataSize doesn't include the exact-length
+ m_CurrentSectorOut += a_LocationRaw[3];
+
+ // Write the data length:
+ Byte Buf[4];
+ Buf[0] = m_CompressedChunkDataSize >> 24;
+ Buf[1] = (m_CompressedChunkDataSize >> 16) & 0xff;
+ Buf[2] = (m_CompressedChunkDataSize >> 8) & 0xff;
+ Buf[3] = m_CompressedChunkDataSize & 0xff;
+ if (a_File.Write(Buf, 4) != 4)
+ {
+ LOGWARNING("Failed to write chunk length!");
+ return false;
+ }
+
+ // Write the data:
+ if (a_File.Write(m_CompressedChunkData, m_CompressedChunkDataSize) != m_CompressedChunkDataSize)
+ {
+ LOGWARNING("Failed to write chunk data!");
+ return false;
+ }
+
+ // Pad onto the next sector:
+ int NumPadding = a_LocationRaw[3] * 4096 - (m_CompressedChunkDataSize + 4);
+ ASSERT(NumPadding >= 0);
+ if ((NumPadding > 0) && (a_File.Write(g_Zeroes, NumPadding) != NumPadding))
+ {
+ LOGWARNING("Failed to write padding");
+ return false;
+ }
+
+ return true;
+}
+
+
+
+
+
+bool cMCADefrag::cThread::UncompressChunk(void)
+{
+ switch (m_CompressedChunkData[0])
+ {
+ case COMPRESSION_GZIP: return UncompressChunkGzip();
+ case COMPRESSION_ZLIB: return UncompressChunkZlib();
+ }
+ LOGINFO("Chunk is compressed with in an unknown algorithm");
+ return false;
+}
+
+
+
+
+
+bool cMCADefrag::cThread::UncompressChunkGzip(void)
+{
+ // TODO
+ // This format is not used in practice
+ return false;
+}
+
+
+
+
+
+bool cMCADefrag::cThread::UncompressChunkZlib(void)
+{
+ // Uncompress the data:
+ z_stream strm;
+ strm.zalloc = (alloc_func)NULL;
+ strm.zfree = (free_func)NULL;
+ strm.opaque = NULL;
+ inflateInit(&strm);
+ strm.next_out = m_RawChunkData;
+ strm.avail_out = sizeof(m_RawChunkData);
+ strm.next_in = m_CompressedChunkData + 1; // The first byte is the compression method, skip it
+ strm.avail_in = m_CompressedChunkDataSize;
+ int res = inflate(&strm, Z_FINISH);
+ inflateEnd(&strm);
+ if (res != Z_STREAM_END)
+ {
+ LOGWARNING("Failed to uncompress chunk data: %s", strm.msg);
+ return false;
+ }
+ m_RawChunkDataSize = strm.total_out;
+
+ return true;
+}
+
+
+
+
+
+bool cMCADefrag::cThread::CompressChunk(void)
+{
+ // Check that the compressed data can fit:
+ uLongf CompressedSize = compressBound(m_RawChunkDataSize);
+ if (CompressedSize > sizeof(m_CompressedChunkData))
+ {
+ LOGINFO("Too much data for the internal compression buffer!");
+ return false;
+ }
+
+ // Compress the data using the highest compression factor:
+ int errorcode = compress2(m_CompressedChunkData + 1, &CompressedSize, m_RawChunkData, m_RawChunkDataSize, Z_BEST_COMPRESSION);
+ if (errorcode != Z_OK)
+ {
+ LOGINFO("Recompression failed: %d", errorcode);
+ return false;
+ }
+ m_CompressedChunkData[0] = COMPRESSION_ZLIB;
+ m_CompressedChunkDataSize = CompressedSize + 1;
+ return true;
+}
+
+
+
+
diff --git a/Tools/MCADefrag/MCADefrag.h b/Tools/MCADefrag/MCADefrag.h
new file mode 100644
index 000000000..d7fa1fc6e
--- /dev/null
+++ b/Tools/MCADefrag/MCADefrag.h
@@ -0,0 +1,144 @@
+
+// MCADefrag.h
+
+// Interfaces to the cMCADefrag class encapsulating the entire app
+
+
+
+
+
+#pragma once
+
+
+
+
+
+
+class cMCADefrag
+{
+public:
+ enum
+ {
+ MAX_COMPRESSED_CHUNK_DATA_SIZE = (1 MiB),
+ MAX_RAW_CHUNK_DATA_SIZE = (100 MiB),
+ } ;
+
+ cMCADefrag(void);
+
+ /** Reads the cmdline params and initializes the app.
+ Returns true if the app should continue, false if not. */
+ bool Init(int argc, char ** argv);
+
+ /** Runs the entire app. */
+ void Run(void);
+
+protected:
+ /** A single thread processing MCA files from the queue */
+ class cThread :
+ public cIsThread
+ {
+ typedef cIsThread super;
+
+ public:
+ cThread(cMCADefrag & a_Parent);
+
+ protected:
+ /** The compression methods, as specified by the MCA compression method byte. */
+ enum
+ {
+ COMPRESSION_GZIP = 1,
+ COMPRESSION_ZLIB = 2,
+ } ;
+
+
+ cMCADefrag & m_Parent;
+
+ /** The current compressed chunk data. Valid after a successful ReadChunk().
+ This contains only the compression method byte and the compressed data,
+ but not the exact-length preceding the data in the MCA file. */
+ unsigned char m_CompressedChunkData[MAX_COMPRESSED_CHUNK_DATA_SIZE];
+
+ /** Size of the actual current compressed chunk data, excluding the 4 exact-length bytes.
+ This is the amount of bytes in m_CompressedChunkData[] that are valid. */
+ int m_CompressedChunkDataSize;
+
+ /** The current raw chunk data. Valid after a successful ReadChunk(), if recompression is active. */
+ unsigned char m_RawChunkData[MAX_RAW_CHUNK_DATA_SIZE];
+
+ /** Size of the actual current raw chunk data. */
+ int m_RawChunkDataSize;
+
+ /** Number of the sector where the next chunk will be written by WriteChunk(). */
+ int m_CurrentSectorOut;
+
+ /** Set to true when the chunk has been successfully uncompressed. Only used if recompression is active.
+ WriteChunk() tests this flag to decide whether to call Compress(). */
+ bool m_IsChunkUncompressed;
+
+
+ /** Processes the specified file. */
+ void ProcessFile(const AString & a_FileName);
+
+ /** Reads the chunk data into m_CompressedChunkData.
+ Calls DecompressChunkData() if recompression is active.
+ a_LocationRaw is the pointer to the first byte of the Location data in the MCA header.
+ Returns true if successful. */
+ bool ReadChunk(cFile & a_File, const Byte * a_LocationRaw);
+
+ /** Writes the chunk data from m_CompressedData or m_RawChunkData (performing recompression) into file.
+ Calls CompressChunkData() for the actual compression, if recompression is active.
+ a_LocationRaw is the pointer to the first byte of the Location data to be put into the MCA header,
+ the chunk's location is stored in that memory area. Updates m_CurrentSectorOut.
+ Returns true if successful. */
+ bool WriteChunk(cFile & a_File, Byte * a_LocationRaw);
+
+ /** Uncompresses the chunk data from m_CompressedChunkData into m_RawChunkData.
+ Returns true if successful, false on failure. */
+ bool UncompressChunk(void);
+
+ /** Uncompresses the chunk data from m_CompressedChunkData into m_RawChunkData, using Gzip.
+ Returns true if successful, false on failure. */
+ bool UncompressChunkGzip(void);
+
+ /** Uncompresses the chunk data from m_CompressedChunkData into m_RawChunkData, using Zlib.
+ Returns true if successful, false on failure. */
+ bool UncompressChunkZlib(void);
+
+ /** Compresses the chunk data from m_RawChunkData into m_CompressedChunkData.
+ Returns true if successful, false on failure. */
+ bool CompressChunk(void);
+
+ // cIsThread overrides:
+ virtual void Execute(void) override;
+ } ;
+
+ typedef std::list<cThread *> cThreads;
+
+
+ /** The mutex protecting m_Files agains multithreaded access. */
+ cCriticalSection m_CS;
+
+ /** The queue of MCA files to be processed by the threads. Protected by m_CS. */
+ AStringVector m_Queue;
+
+ /** List of threads that the server has running. */
+ cThreads m_Threads;
+
+ /** The number of threads that should be started. Configurable on the command line. */
+ int m_NumThreads;
+
+ /** If set to true, the chunk data is recompressed while saving each MCA file. */
+ bool m_ShouldRecompress;
+
+
+ /** Starts a new processing thread and adds it to cThreads. */
+ void StartThread(void);
+
+ /** Retrieves one file from the queue (and removes it from the queue).
+ Returns an empty string when queue empty. */
+ AString GetNextFileName(void);
+} ;
+
+
+
+