From 50e88c8e84918c91ac77bdae5384971d5a7f3f2b Mon Sep 17 00:00:00 2001 From: localhorst Date: Tue, 28 Apr 2026 21:54:49 +0200 Subject: [PATCH 1/9] Best throughput tracker with chunksize --- .clang-format | 92 +++++++++++++++++ include/shred.h | 52 +++++++++- include/tui.h | 40 ++++---- src/shred.cpp | 263 ++++++++++++++++++++++++++++++++++++++++++------ 4 files changed, 395 insertions(+), 52 deletions(-) create mode 100644 .clang-format diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..4776e34 --- /dev/null +++ b/.clang-format @@ -0,0 +1,92 @@ +# reHDD Code Style - clang-format configuration +# Based on analyzed codebase formatting + +BasedOnStyle: LLVM + +# Indentation +IndentWidth: 4 +TabWidth: 4 +UseTab: Never +ContinuationIndentWidth: 4 + +# Braces +BreakBeforeBraces: Allman +# Allman style: +# if (condition) +# { +# statement; +# } + +# Spacing +SpaceAfterCStyleCast: false +SpaceBeforeParens: ControlStatements +SpaceInEmptyParentheses: false +SpacesInContainerLiterals: false +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false + +# Alignment +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Left +AlignOperands: true +AlignTrailingComments: false + +# Line breaks +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: None +AllowShortIfStatementsOnASingleLine: Never +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: Yes + +# Column limit +ColumnLimit: 0 +# 0 = no limit (observed in the code) + +# Breaking +BinPackArguments: true +BinPackParameters: true +BreakBeforeBinaryOperators: None +BreakBeforeTernaryOperators: true +BreakConstructorInitializers: BeforeColon +BreakInheritanceList: BeforeColon +BreakStringLiterals: false + +# Pointers and references +PointerAlignment: Left +# Type *variable (not Type* variable) + +# Include sorting +SortIncludes: false +# Keep includes in original order + +# Comments +ReflowComments: false +# Don't reformat comments + +# Access modifiers +AccessModifierOffset: -4 +# public: is outdented + +# Constructor initializers +ConstructorInitializerIndentWidth: 6 +# Observed 6 spaces for initializer lists + +# Other +Cpp11BracedListStyle: true +FixNamespaceComments: true +IncludeBlocks: Preserve +IndentCaseLabels: false +IndentPPDirectives: None +KeepEmptyLinesAtTheStartOfBlocks: false +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +PenaltyBreakBeforeFirstCallParameter: 100 +PenaltyBreakComment: 300 +PenaltyBreakString: 1000 +PenaltyExcessCharacter: 1000000 diff --git a/include/shred.h b/include/shred.h index 635f265..5949e1e 100644 --- a/include/shred.h +++ b/include/shred.h @@ -16,9 +16,28 @@ #include #include #include +#include -#define CHUNK_SIZE 1024 * 1024 * 32 // amount of bytes that are overwritten at once --> 32MB -#define TFNG_DATA_SIZE CHUNK_SIZE // amount of bytes used by tfng +// Adaptive chunk size optimization - uncomment to enable +#define ADAPTIVE_CHUNK_SIZE + +// Chunk size configuration +#define CHUNK_SIZE_START 1024 * 1024 * 32 // Starting chunk size: 32MB +#define CHUNK_SIZE_MIN 1024 * 1024 * 4 // Minimum chunk size: 4MB +#define CHUNK_SIZE_MAX 1024 * 1024 * 128 // Maximum chunk size: 128MB +#define CHUNK_SIZE_STEP_UP 1024 * 1024 * 2 // Increase step: 2MB +#define CHUNK_SIZE_STEP_DOWN 1024 * 1024 * 4 // Decrease step: 4MB +#define CHUNK_MEASURE_INTERVAL 64 // Measure performance every 64 chunks + +#ifdef ADAPTIVE_CHUNK_SIZE +// Use max buffer size when adaptive mode is enabled +#define CHUNK_SIZE CHUNK_SIZE_MAX +#define TFNG_DATA_SIZE CHUNK_SIZE_MAX +#else +// Use fixed chunk size when adaptive mode is disabled +#define CHUNK_SIZE CHUNK_SIZE_START +#define TFNG_DATA_SIZE CHUNK_SIZE +#endif // #define DEMO_DRIVE_SIZE 1024*1024*256L // 256MB // #define DEMO_DRIVE_SIZE 1024*1024*1024L // 1GB @@ -33,22 +52,47 @@ protected: public: Shred(); ~Shred(); - int shredDrive(Drive *drive, int *ipSignalFd); + int shredDrive(Drive* drive, int* ipSignalFd); private: fileDescriptor randomSrcFileDiscr; fileDescriptor driveFileDiscr; + +#ifdef ADAPTIVE_CHUNK_SIZE + unsigned char* caTfngData; // Dynamic buffer allocation for adaptive mode + unsigned char* caReadBuffer; // Dynamic buffer allocation for adaptive mode +#else unsigned char caTfngData[TFNG_DATA_SIZE]; unsigned char caReadBuffer[CHUNK_SIZE]; +#endif + unsigned long ulDriveByteSize; unsigned long ulDriveByteOverallCount = 0; // all bytes shredded in all iterations + checking -> used for progress calculation double d32Percent = 0.0; double d32TmpPercent = 0.0; +#ifdef ADAPTIVE_CHUNK_SIZE + // Adaptive chunk size optimization members + size_t currentChunkSize; + size_t bestChunkSize; + unsigned int chunkCounter; + std::chrono::high_resolution_clock::time_point measurementStartTime; + double bestThroughputMBps; + double lastThroughputMBps; + unsigned long bytesWrittenInMeasurement; + bool throughputIncreasing; + + // Adaptive methods + void startMeasurement(); + void evaluateThroughput(Drive* drive); + void adjustChunkSize(Drive* drive); + size_t getCurrentChunkSize() const; +#endif + inline double calcProgress(); int iRewindDrive(fileDescriptor file); long getDriveSizeInBytes(fileDescriptor file); - unsigned int uiCalcChecksum(fileDescriptor file, Drive *drive, int *ipSignalFd); + unsigned int uiCalcChecksum(fileDescriptor file, Drive* drive, int* ipSignalFd); void cleanup(); }; diff --git a/include/tui.h b/include/tui.h index 3826650..09f5304 100644 --- a/include/tui.h +++ b/include/tui.h @@ -49,7 +49,7 @@ public: static void initTUI(); - void updateTUI(std::list *plistDrives, uint8_t u8SelectedEntry); + void updateTUI(std::list* plistDrives, uint8_t u8SelectedEntry); static enum UserInput readUserInput(); @@ -60,28 +60,28 @@ private: static std::string sRamUsage; static std::string sLocalTime; - WINDOW *overview; - WINDOW *systemview; - WINDOW *detailview; - WINDOW *menuview; - WINDOW *dialog; - WINDOW *smartWarning; + WINDOW* overview; + WINDOW* systemview; + WINDOW* detailview; + WINDOW* menuview; + WINDOW* dialog; + WINDOW* smartWarning; - static void centerTitle(WINDOW *pwin, const char *title); - static WINDOW *createOverViewWindow(int iXSize, int iYSize); - static WINDOW *createDetailViewWindow(int iXSize, int iYSize, int iXStart, Drive &drive); - static WINDOW *overwriteDetailViewWindow(int iXSize, int iYSize, int iXStart); - static WINDOW *createEntryWindow(int iXSize, int iYSize, int iXStart, int iYStart, int iListIndex, std::string sModelFamily, std::string sSerial, std::string sCapacity, std::string sState, std::string sTime, std::string sSpeed, std::string sTemp, std::string sConnection, bool bSelected); - static WINDOW *createSystemStats(int iXSize, int iYSize, int iXStart, int iYStart); - static WINDOW *createMenuView(int iXSize, int iYSize, int iXStart, int iYStart, struct MenuState menustate); - static WINDOW *createDialog(int iXSize, int iYSize, int iXStart, int iYStart, std::string selectedTask, std::string optionA, std::string optionB); - static WINDOW *createFrozenWarning(int iXSize, int iYSize, int iXStart, int iYStart, std::string sPath, std::string sModelFamily, std::string sModelName, std::string sSerial, std::string sProgress); - static WINDOW *createSmartWarning(int iXSize, int iYSize, int iXStart, int iYStart, std::string sPath, uint32_t u32PowerOnHours, uint32_t u32PowerCycles, uint32_t u32ErrorCount, uint32_t u32Temperature); - static WINDOW *createZeroChecksumWarning(int iXSize, int iYSize, int iXStart, int iYStart, std::string sPath, std::string sModelFamily, std::string sModelName, std::string sSerial, uint32_t u32Checksum); + static void centerTitle(WINDOW* pwin, const char* title); + static WINDOW* createOverViewWindow(int iXSize, int iYSize); + static WINDOW* createDetailViewWindow(int iXSize, int iYSize, int iXStart, Drive& drive); + static WINDOW* overwriteDetailViewWindow(int iXSize, int iYSize, int iXStart); + static WINDOW* createEntryWindow(int iXSize, int iYSize, int iXStart, int iYStart, int iListIndex, std::string sModelFamily, std::string sSerial, std::string sCapacity, std::string sState, std::string sTime, std::string sSpeed, std::string sTemp, std::string sConnection, bool bSelected); + static WINDOW* createSystemStats(int iXSize, int iYSize, int iXStart, int iYStart); + static WINDOW* createMenuView(int iXSize, int iYSize, int iXStart, int iYStart, struct MenuState menustate); + static WINDOW* createDialog(int iXSize, int iYSize, int iXStart, int iYStart, std::string selectedTask, std::string optionA, std::string optionB); + static WINDOW* createFrozenWarning(int iXSize, int iYSize, int iXStart, int iYStart, std::string sPath, std::string sModelFamily, std::string sModelName, std::string sSerial, std::string sProgress); + static WINDOW* createSmartWarning(int iXSize, int iYSize, int iXStart, int iYStart, std::string sPath, uint32_t u32PowerOnHours, uint32_t u32PowerCycles, uint32_t u32ErrorCount, uint32_t u32Temperature); + static WINDOW* createZeroChecksumWarning(int iXSize, int iYSize, int iXStart, int iYStart, std::string sPath, std::string sModelFamily, std::string sModelName, std::string sSerial, uint32_t u32Checksum); - void displaySelectedDrive(Drive &drive, int stdscrX, int stdscrY); + void displaySelectedDrive(Drive& drive, int stdscrX, int stdscrY); std::string formatTimeDuration(time_t u32Duration); std::string formatSpeed(time_t u32ShredTimeDelta, unsigned long ulWrittenBytes); - static void vTruncateText(std::string *psText, uint16_t u16MaxLenght); + static void vTruncateText(std::string* psText, uint16_t u16MaxLenght); }; #endif // TUI_H_ diff --git a/src/shred.cpp b/src/shred.cpp index 6cddd6c..eef0436 100644 --- a/src/shred.cpp +++ b/src/shred.cpp @@ -17,27 +17,184 @@ extern "C" } #endif -const static char *randomsrc = (char *)"/dev/urandom"; +const static char* randomsrc = (char*)"/dev/urandom"; Shred::Shred() { +#ifdef ADAPTIVE_CHUNK_SIZE + // Allocate aligned buffers for maximum chunk size + if (posix_memalign((void**)&caTfngData, 4096, CHUNK_SIZE_MAX) != 0) + { + Logger::logThis()->error("Failed to allocate aligned buffer for tfng data"); + caTfngData = nullptr; + } + + if (posix_memalign((void**)&caReadBuffer, 4096, CHUNK_SIZE_MAX) != 0) + { + Logger::logThis()->error("Failed to allocate aligned buffer for read buffer"); + caReadBuffer = nullptr; + } + + // Initialize adaptive tracking variables + currentChunkSize = CHUNK_SIZE_START; + bestChunkSize = CHUNK_SIZE_START; + chunkCounter = 0; + bestThroughputMBps = 0.0; + lastThroughputMBps = 0.0; + bytesWrittenInMeasurement = 0; + throughputIncreasing = true; + + Logger::logThis()->info("Adaptive chunk size optimization ENABLED - Starting with " + + to_string(currentChunkSize / (1024 * 1024)) + " MB chunks"); +#endif } Shred::~Shred() { +#ifdef ADAPTIVE_CHUNK_SIZE + if (caTfngData != nullptr) + { + free(caTfngData); + caTfngData = nullptr; + } + if (caReadBuffer != nullptr) + { + free(caReadBuffer); + caReadBuffer = nullptr; + } +#endif +} + +#ifdef ADAPTIVE_CHUNK_SIZE +/** + * \brief Start performance measurement interval + * \return void + */ +void Shred::startMeasurement() +{ + measurementStartTime = std::chrono::high_resolution_clock::now(); + bytesWrittenInMeasurement = 0; + chunkCounter = 0; } /** - * \brief shred drive with shred - * \param pointer of Drive instance + * \brief Evaluate throughput after measurement interval and adjust chunk size + * \param pointer to Drive instance * \return void */ -int Shred::shredDrive(Drive *drive, int *ipSignalFd) +void Shred::evaluateThroughput(Drive* drive) +{ + auto measurementEndTime = std::chrono::high_resolution_clock::now(); + std::chrono::duration elapsed = measurementEndTime - measurementStartTime; + double elapsedSeconds = elapsed.count(); + + if (elapsedSeconds > 0.0) + { + double throughputMBps = (bytesWrittenInMeasurement / (1024.0 * 1024.0)) / elapsedSeconds; + lastThroughputMBps = throughputMBps; + + Logger::logThis()->info("Throughput measurement - ChunkSize: " + + to_string(currentChunkSize / (1024 * 1024)) + " MB, " + + "Throughput: " + to_string((int)throughputMBps) + " MB/s, " + + "Best: " + to_string((int)bestThroughputMBps) + " MB/s" + + " - Drive: " + drive->getSerial()); + + // Check if this is better than our best + if (throughputMBps > bestThroughputMBps) + { + bestThroughputMBps = throughputMBps; + bestChunkSize = currentChunkSize; + throughputIncreasing = true; + + Logger::logThis()->info("NEW BEST throughput: " + to_string((int)bestThroughputMBps) + + " MB/s with " + to_string(currentChunkSize / (1024 * 1024)) + + " MB chunks - Drive: " + drive->getSerial()); + } + else + { + throughputIncreasing = false; + } + } + + // Adjust chunk size for next measurement interval + adjustChunkSize(drive); + + // Start new measurement + startMeasurement(); +} + +/** + * \brief Adjust chunk size based on throughput trend + * \param pointer to Drive instance + * \return void + */ +void Shred::adjustChunkSize(Drive* drive) +{ + size_t oldChunkSize = currentChunkSize; + + if (throughputIncreasing) + { + // Throughput is improving - increase chunk size + currentChunkSize += CHUNK_SIZE_STEP_UP; + + // Clamp to maximum + if (currentChunkSize > CHUNK_SIZE_MAX) + { + currentChunkSize = CHUNK_SIZE_MAX; + Logger::logThis()->info("Reached maximum chunk size: " + + to_string(currentChunkSize / (1024 * 1024)) + " MB" + + " - Drive: " + drive->getSerial()); + } + } + else + { + // Throughput decreased - decrease chunk size to find sweet spot + if (currentChunkSize > CHUNK_SIZE_STEP_DOWN) + { + currentChunkSize -= CHUNK_SIZE_STEP_DOWN; + } + + // Clamp to minimum + if (currentChunkSize < CHUNK_SIZE_MIN) + { + currentChunkSize = CHUNK_SIZE_MIN; + Logger::logThis()->info("Reached minimum chunk size: " + + to_string(currentChunkSize / (1024 * 1024)) + " MB" + + " - Drive: " + drive->getSerial()); + } + } + + if (oldChunkSize != currentChunkSize) + { + Logger::logThis()->info("Adjusted chunk size: " + + to_string(oldChunkSize / (1024 * 1024)) + " MB -> " + + to_string(currentChunkSize / (1024 * 1024)) + " MB" + + " - Drive: " + drive->getSerial()); + } +} + +/** + * \brief Get current chunk size for adaptive mode + * \return current chunk size in bytes + */ +size_t Shred::getCurrentChunkSize() const +{ + return currentChunkSize; +} +#endif + +/** + * \brief shred drive with shred + * \param pointer of Drive instance + * \param file descriptor for signaling + * \return 0 on success, -1 on error + */ +int Shred::shredDrive(Drive* drive, int* ipSignalFd) { ostringstream address; - address << (void const *)&(*drive); + address << (void const*)&(*drive); Logger::logThis()->info("Shred-Task started - Drive: " + drive->getModelName() + "-" + drive->getSerial() + " @" + address.str()); - drive->bWasShredStarted = true; // Mark drive as partly shredded + drive->bWasShredStarted = true; drive->bWasShredded = false; drive->setTaskPercentage(0.0); drive->u32DriveChecksumAfterShredding = UINT32_MAX; @@ -54,9 +211,18 @@ int Shred::shredDrive(Drive *drive, int *ipSignalFd) #endif #ifndef DRYRUN - const char *cpDrivePath = drive->getPath().c_str(); + const char* cpDrivePath = drive->getPath().c_str(); unsigned char ucKey[TFNG_KEY_SIZE]; +#ifdef ADAPTIVE_CHUNK_SIZE + // Validate buffers were allocated + if (caTfngData == nullptr || caReadBuffer == nullptr) + { + Logger::logThis()->error("Shred-Task: Aligned buffers not allocated! - Drive: " + drive->getSerial()); + return -1; + } +#endif + // open random source randomSrcFileDiscr = open(randomsrc, O_RDONLY | O_LARGEFILE); if (randomSrcFileDiscr == -1) @@ -94,41 +260,59 @@ int Shred::shredDrive(Drive *drive, int *ipSignalFd) this->ulDriveByteSize = getDriveSizeInBytes(driveFileDiscr); Drive::ShredSpeed shredSpeed = drive->sShredSpeed.load(); - shredSpeed.chronoShredTimestamp = std::chrono::system_clock::now(); // set inital timestamp for speed metric - shredSpeed.ulSpeedMetricBytesWritten = 0U; // uses to calculate speed metric + shredSpeed.chronoShredTimestamp = std::chrono::system_clock::now(); + shredSpeed.ulSpeedMetricBytesWritten = 0U; drive->sShredSpeed.store(shredSpeed); #ifdef LOG_LEVEL_HIGH Logger::logThis()->info("Shred-Task: Bytes-Size of Drive: " + to_string(this->ulDriveByteSize) + " - Drive: " + drive->getSerial()); #endif +#ifdef ADAPTIVE_CHUNK_SIZE + // Start first measurement interval + startMeasurement(); +#endif + for (unsigned int uiShredIterationCounter = 0U; uiShredIterationCounter < SHRED_ITERATIONS; uiShredIterationCounter++) { - unsigned long ulDriveByteCounter = 0U; // used for one shred-iteration to keep track of the current drive position + unsigned long ulDriveByteCounter = 0U; if (uiShredIterationCounter == (SHRED_ITERATIONS - 1)) { // last shred iteration --> overwrite (just the write chunk) bytes with zeros instead with random data +#ifdef ADAPTIVE_CHUNK_SIZE + memset(caTfngData, 0U, CHUNK_SIZE_MAX); +#else memset(caTfngData, 0U, CHUNK_SIZE); +#endif } while (ulDriveByteCounter < ulDriveByteSize) { - int iBytesToShred = 0; // Bytes that will be overwritten in this chunk-iteration +#ifdef ADAPTIVE_CHUNK_SIZE + size_t activeChunkSize = getCurrentChunkSize(); +#else + size_t activeChunkSize = CHUNK_SIZE; +#endif + + int iBytesToShred = 0; if (uiShredIterationCounter != (SHRED_ITERATIONS - 1)) { - // NOT last shred iteration --> generate new random data +#ifdef ADAPTIVE_CHUNK_SIZE + tfng_prng_genrandom(caTfngData, activeChunkSize); +#else tfng_prng_genrandom(caTfngData, TFNG_DATA_SIZE); +#endif } - if ((ulDriveByteSize - ulDriveByteCounter) < CHUNK_SIZE) + if ((ulDriveByteSize - ulDriveByteCounter) < activeChunkSize) { iBytesToShred = (ulDriveByteSize - ulDriveByteCounter); } else { - iBytesToShred = CHUNK_SIZE; + iBytesToShred = activeChunkSize; } int iByteShredded = write(driveFileDiscr, caTfngData, iBytesToShred); @@ -148,17 +332,28 @@ int Shred::shredDrive(Drive *drive, int *ipSignalFd) ulDriveByteCounter += iByteShredded; ulDriveByteOverallCount += iByteShredded; + +#ifdef ADAPTIVE_CHUNK_SIZE + bytesWrittenInMeasurement += iByteShredded; + chunkCounter++; + + // Evaluate throughput after measurement interval + if (chunkCounter >= CHUNK_MEASURE_INTERVAL) + { + evaluateThroughput(drive); + } +#endif + d32Percent = this->calcProgress(); + #ifdef LOG_LEVEL_HIGH Logger::logThis()->info("Shred-Task: ByteCount: " + to_string(ulDriveByteCounter) + " - iteration: " + to_string((uiShredIterationCounter + 1)) + " - progress: " + to_string(d32Percent) + " - Drive: " + drive->getSerial()); #endif if ((d32Percent - d32TmpPercent) >= 0.01) { - // set shred percantage drive->setTaskPercentage(d32TmpPercent); d32TmpPercent = d32Percent; - // signal process in shreding write(*ipSignalFd, "A", 1); } @@ -168,26 +363,32 @@ int Shred::shredDrive(Drive *drive, int *ipSignalFd) d32Percent = 0.00; d32TmpPercent = 0.00; ulDriveByteCounter = 0U; - Logger::logThis()->info("Aborted shred for: " + drive->getModelName() + "-" + drive->getSerial()); + Logger::logThis()->info("Aborted shred for: " + drive->getModelName() + "-" + drive->getSerial()); cleanup(); return -1; } - // end one chunk write } + if (0 != iRewindDrive(driveFileDiscr)) { Logger::logThis()->error("Shred-Task: Unable to rewind drive! - Drive: " + drive->getSerial()); cleanup(); return -1; } - // end one shred iteration } - // end of all shred iteratio - tfng_prng_seedkey(NULL); // reset random generator +#ifdef ADAPTIVE_CHUNK_SIZE + Logger::logThis()->info("Shred completed - Optimal chunk size: " + + to_string(bestChunkSize / (1024 * 1024)) + " MB, " + + "Best throughput: " + to_string((int)bestThroughputMBps) + " MB/s" + + " - Drive: " + drive->getSerial()); +#endif + + tfng_prng_seedkey(NULL); drive->bWasShredded = true; Logger::logThis()->info("Shred-Task finished - Drive: " + drive->getModelName() + "-" + drive->getSerial() + " @" + address.str()); + #ifdef ZERO_CHECK drive->state = Drive::TaskState::CHECK_ACTIVE; Logger::logThis()->info("Check-Task started - Drive: " + drive->getModelName() + "-" + drive->getSerial() + " @" + address.str()); @@ -219,10 +420,9 @@ int Shred::shredDrive(Drive *drive, int *ipSignalFd) } return 0; } + /** * \brief calc shredding progress in % - * \param current byte index of the drive - * \param current shred iteration * \return double percentage */ double Shred::calcProgress() @@ -230,7 +430,7 @@ double Shred::calcProgress() unsigned int uiMaxShredIteration = SHRED_ITERATIONS; #ifdef ZERO_CHECK - uiMaxShredIteration++; // increment because we will check after SHRED_ITERATIONS the drive for non-zero bytes + uiMaxShredIteration++; #endif if (this->ulDriveByteSize == 0) return 0.0; @@ -273,20 +473,27 @@ long Shred::getDriveSizeInBytes(fileDescriptor file) return liDriveSizeTmp; } -unsigned int Shred::uiCalcChecksum(fileDescriptor file, Drive *drive, int *ipSignalFd) +unsigned int Shred::uiCalcChecksum(fileDescriptor file, Drive* drive, int* ipSignalFd) { unsigned int uiChecksum = 0; unsigned long ulDriveByteCounter = 0U; + +#ifdef ADAPTIVE_CHUNK_SIZE + size_t checkChunkSize = CHUNK_SIZE_MAX; +#else + size_t checkChunkSize = CHUNK_SIZE; +#endif + while (ulDriveByteCounter < ulDriveByteSize) { int iBytesToCheck = 0; - if ((ulDriveByteSize - ulDriveByteCounter) < CHUNK_SIZE) + if ((ulDriveByteSize - ulDriveByteCounter) < checkChunkSize) { iBytesToCheck = (ulDriveByteSize - ulDriveByteCounter); } else { - iBytesToCheck = CHUNK_SIZE; + iBytesToCheck = checkChunkSize; } int iReadBytes = read(file, caReadBuffer, iBytesToCheck); for (int iReadBytesCounter = 0U; iReadBytesCounter < iReadBytes; iReadBytesCounter++) @@ -322,4 +529,4 @@ void Shred::cleanup() { close(driveFileDiscr); close(randomSrcFileDiscr); -} \ No newline at end of file +} -- 2.54.0 From 42a1567b32db5acdb6ca0a8c0334200e7d91aa02 Mon Sep 17 00:00:00 2001 From: localhorst Date: Fri, 1 May 2026 13:12:39 +0200 Subject: [PATCH 2/9] Fix error handling if shred failes (#96) fixes https://git.mosad.xyz/localhorst/reHDD/issues/95 Reviewed-on: https://git.mosad.xyz/localhorst/reHDD/pulls/96 Co-authored-by: localhorst Co-committed-by: localhorst --- .clang-format | 92 --------------------------------------------------- include/tui.h | 18 +++++----- 2 files changed, 9 insertions(+), 101 deletions(-) delete mode 100644 .clang-format diff --git a/.clang-format b/.clang-format deleted file mode 100644 index 4776e34..0000000 --- a/.clang-format +++ /dev/null @@ -1,92 +0,0 @@ -# reHDD Code Style - clang-format configuration -# Based on analyzed codebase formatting - -BasedOnStyle: LLVM - -# Indentation -IndentWidth: 4 -TabWidth: 4 -UseTab: Never -ContinuationIndentWidth: 4 - -# Braces -BreakBeforeBraces: Allman -# Allman style: -# if (condition) -# { -# statement; -# } - -# Spacing -SpaceAfterCStyleCast: false -SpaceBeforeParens: ControlStatements -SpaceInEmptyParentheses: false -SpacesInContainerLiterals: false -SpacesInCStyleCastParentheses: false -SpacesInParentheses: false -SpacesInSquareBrackets: false - -# Alignment -AlignAfterOpenBracket: Align -AlignConsecutiveAssignments: false -AlignConsecutiveDeclarations: false -AlignEscapedNewlines: Left -AlignOperands: true -AlignTrailingComments: false - -# Line breaks -AllowShortBlocksOnASingleLine: false -AllowShortCaseLabelsOnASingleLine: false -AllowShortFunctionsOnASingleLine: None -AllowShortIfStatementsOnASingleLine: Never -AllowShortLoopsOnASingleLine: false -AlwaysBreakAfterReturnType: None -AlwaysBreakBeforeMultilineStrings: false -AlwaysBreakTemplateDeclarations: Yes - -# Column limit -ColumnLimit: 0 -# 0 = no limit (observed in the code) - -# Breaking -BinPackArguments: true -BinPackParameters: true -BreakBeforeBinaryOperators: None -BreakBeforeTernaryOperators: true -BreakConstructorInitializers: BeforeColon -BreakInheritanceList: BeforeColon -BreakStringLiterals: false - -# Pointers and references -PointerAlignment: Left -# Type *variable (not Type* variable) - -# Include sorting -SortIncludes: false -# Keep includes in original order - -# Comments -ReflowComments: false -# Don't reformat comments - -# Access modifiers -AccessModifierOffset: -4 -# public: is outdented - -# Constructor initializers -ConstructorInitializerIndentWidth: 6 -# Observed 6 spaces for initializer lists - -# Other -Cpp11BracedListStyle: true -FixNamespaceComments: true -IncludeBlocks: Preserve -IndentCaseLabels: false -IndentPPDirectives: None -KeepEmptyLinesAtTheStartOfBlocks: false -MaxEmptyLinesToKeep: 1 -NamespaceIndentation: None -PenaltyBreakBeforeFirstCallParameter: 100 -PenaltyBreakComment: 300 -PenaltyBreakString: 1000 -PenaltyExcessCharacter: 1000000 diff --git a/include/tui.h b/include/tui.h index 489504a..d4a201b 100644 --- a/include/tui.h +++ b/include/tui.h @@ -49,7 +49,7 @@ public: static void initTUI(); - void updateTUI(std::list* plistDrives, uint8_t u8SelectedEntry); + void updateTUI(std::list *plistDrives, uint8_t u8SelectedEntry); static enum UserInput readUserInput(); @@ -60,12 +60,12 @@ private: static std::string sRamUsage; static std::string sLocalTime; - WINDOW* overview; - WINDOW* systemview; - WINDOW* detailview; - WINDOW* menuview; - WINDOW* dialog; - WINDOW* smartWarning; + WINDOW *overview; + WINDOW *systemview; + WINDOW *detailview; + WINDOW *menuview; + WINDOW *dialog; + WINDOW *smartWarning; static void centerTitle(WINDOW *pwin, const char *title); static WINDOW *createOverViewWindow(int iXSize, int iYSize); @@ -79,9 +79,9 @@ private: static WINDOW *createSmartWarning(int iXSize, int iYSize, int iXStart, int iYStart, std::string sPath, uint32_t u32PowerOnHours, uint32_t u32PowerCycles, uint32_t u32ErrorCount, uint32_t u32Temperature, uint32_t u32ReallocatedSectors, uint32_t u32PendingSectors, uint32_t u32UncorrectableSectors); static WINDOW *createZeroChecksumWarning(int iXSize, int iYSize, int iXStart, int iYStart, std::string sPath, std::string sModelFamily, std::string sModelName, std::string sSerial, uint32_t u32Checksum); - void displaySelectedDrive(Drive& drive, int stdscrX, int stdscrY); + void displaySelectedDrive(Drive &drive, int stdscrX, int stdscrY); std::string formatTimeDuration(time_t u32Duration); std::string formatSpeed(time_t u32ShredTimeDelta, unsigned long ulWrittenBytes); - static void vTruncateText(std::string* psText, uint16_t u16MaxLenght); + static void vTruncateText(std::string *psText, uint16_t u16MaxLenght); }; #endif // TUI_H_ -- 2.54.0 From e017aeca0b6342a198ff35c9774a2efd7a9db904 Mon Sep 17 00:00:00 2001 From: localhorst Date: Fri, 1 May 2026 15:22:35 +0200 Subject: [PATCH 3/9] fix pointer (again) --- src/shred.cpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/shred.cpp b/src/shred.cpp index 3c98379..283f3cb 100644 --- a/src/shred.cpp +++ b/src/shred.cpp @@ -17,19 +17,19 @@ extern "C" } #endif -const static char* randomsrc = (char*)"/dev/urandom"; +const static char *randomsrc = (char *)"/dev/urandom"; Shred::Shred() { #ifdef ADAPTIVE_CHUNK_SIZE // Allocate aligned buffers for maximum chunk size - if (posix_memalign((void**)&caTfngData, 4096, CHUNK_SIZE_MAX) != 0) + if (posix_memalign((void **)&caTfngData, 4096, CHUNK_SIZE_MAX) != 0) { Logger::logThis()->error("Failed to allocate aligned buffer for tfng data"); caTfngData = nullptr; } - if (posix_memalign((void**)&caReadBuffer, 4096, CHUNK_SIZE_MAX) != 0) + if (posix_memalign((void **)&caReadBuffer, 4096, CHUNK_SIZE_MAX) != 0) { Logger::logThis()->error("Failed to allocate aligned buffer for read buffer"); caReadBuffer = nullptr; @@ -83,7 +83,7 @@ void Shred::startMeasurement() * \param file descriptor for signaling * \return 0 on success, -1 on error */ -void Shred::evaluateThroughput(Drive* drive) +void Shred::evaluateThroughput(Drive *drive) { auto measurementEndTime = std::chrono::high_resolution_clock::now(); std::chrono::duration elapsed = measurementEndTime - measurementStartTime; @@ -129,7 +129,7 @@ void Shred::evaluateThroughput(Drive* drive) * \param pointer to Drive instance * \return void */ -void Shred::adjustChunkSize(Drive* drive) +void Shred::adjustChunkSize(Drive *drive) { size_t oldChunkSize = currentChunkSize; @@ -190,10 +190,10 @@ size_t Shred::getCurrentChunkSize() const * \param file descriptor for signaling * \return 0 on success, -1 on error */ -int Shred::shredDrive(Drive* drive, int* ipSignalFd) +int Shred::shredDrive(Drive *drive, int *ipSignalFd) { ostringstream address; - address << (void const*)&(*drive); + address << (void const *)&(*drive); Logger::logThis()->info("Shred-Task started - Drive: " + drive->getModelName() + "-" + drive->getSerial() + " @" + address.str()); // Mark as started but NOT shredded yet @@ -229,7 +229,8 @@ int Shred::shredDrive(Drive* drive, int* ipSignalFd) #endif #ifndef DRYRUN - const char* cpDrivePath = drive->getPath().c_str(); + string sDrivePath = drive->getPath(); + const char *cpDrivePath = sDrivePath.c_str(); unsigned char ucKey[TFNG_KEY_SIZE]; #ifdef ADAPTIVE_CHUNK_SIZE @@ -635,7 +636,7 @@ long Shred::getDriveSizeInBytes(fileDescriptor file) * \param signal file descriptor * \return checksum value (0 = all zeros) */ -unsigned int Shred::uiCalcChecksum(fileDescriptor file, Drive* drive, int* ipSignalFd) +unsigned int Shred::uiCalcChecksum(fileDescriptor file, Drive *drive, int *ipSignalFd) { unsigned int uiChecksum = 0; unsigned long ulDriveByteCounter = 0U; -- 2.54.0 From 716ab5614f57c14d58e2c5e512244037dec1915d Mon Sep 17 00:00:00 2001 From: localhorst Date: Fri, 1 May 2026 15:28:02 +0200 Subject: [PATCH 4/9] add comments --- src/shred.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/shred.cpp b/src/shred.cpp index 283f3cb..faac86a 100644 --- a/src/shred.cpp +++ b/src/shred.cpp @@ -455,8 +455,10 @@ int Shred::shredDrive(Drive *drive, int *ipSignalFd) if ((d32Percent - d32TmpPercent) >= 0.01) { + // set shred percentage drive->setTaskPercentage(d32TmpPercent); d32TmpPercent = d32Percent; + // signal process in shredding write(*ipSignalFd, "A", 1); } @@ -563,7 +565,7 @@ double Shred::calcProgress() unsigned int uiMaxShredIteration = SHRED_ITERATIONS; #ifdef ZERO_CHECK - uiMaxShredIteration++; + uiMaxShredIteration++; // increment because we will check after SHRED_ITERATIONS the drive for non-zero bytes #endif if (this->ulDriveByteSize == 0) @@ -641,6 +643,8 @@ unsigned int Shred::uiCalcChecksum(fileDescriptor file, Drive *drive, int *ipSig unsigned int uiChecksum = 0; unsigned long ulDriveByteCounter = 0U; + Logger::logThis()->info("Check-Task: Starting checksum verification - Drive: " + drive->getSerial()); + #ifdef ADAPTIVE_CHUNK_SIZE size_t checkChunkSize = CHUNK_SIZE_MAX; #else -- 2.54.0 From 05a3750b03200e3244e2e34280a49f7f7ddb76d8 Mon Sep 17 00:00:00 2001 From: localhorst Date: Sat, 2 May 2026 12:25:21 +0200 Subject: [PATCH 5/9] cleanup and shred multi-armed bandit --- include/reHDD.h | 2 +- include/shred.h | 56 +++++++-------- scripts/reHDDLogUploader.bash | 6 -- scripts/reHDDLogUploader.service | 18 ----- scripts/reHDDLogUploader.timer | 11 --- src/shred.cpp | 119 ++++++++++++++++++++++--------- 6 files changed, 113 insertions(+), 99 deletions(-) delete mode 100644 scripts/reHDDLogUploader.bash delete mode 100644 scripts/reHDDLogUploader.service delete mode 100644 scripts/reHDDLogUploader.timer diff --git a/include/reHDD.h b/include/reHDD.h index d63f7f0..f4e0af4 100644 --- a/include/reHDD.h +++ b/include/reHDD.h @@ -8,7 +8,7 @@ #ifndef REHDD_H_ #define REHDD_H_ -#define REHDD_VERSION "V1.4.0-dev" +#define REHDD_VERSION "V1.4.0" // Drive handling Settings #define WORSE_HOURS 19200 // mark drive if at this limit or beyond diff --git a/include/shred.h b/include/shred.h index 5949e1e..ecf5dcc 100644 --- a/include/shred.h +++ b/include/shred.h @@ -18,26 +18,22 @@ #include #include -// Adaptive chunk size optimization - uncomment to enable -#define ADAPTIVE_CHUNK_SIZE - +// Adaptive chunk size optimization with multi-armed bandit - always enabled // Chunk size configuration -#define CHUNK_SIZE_START 1024 * 1024 * 32 // Starting chunk size: 32MB -#define CHUNK_SIZE_MIN 1024 * 1024 * 4 // Minimum chunk size: 4MB -#define CHUNK_SIZE_MAX 1024 * 1024 * 128 // Maximum chunk size: 128MB -#define CHUNK_SIZE_STEP_UP 1024 * 1024 * 2 // Increase step: 2MB -#define CHUNK_SIZE_STEP_DOWN 1024 * 1024 * 4 // Decrease step: 4MB -#define CHUNK_MEASURE_INTERVAL 64 // Measure performance every 64 chunks +#define CHUNK_SIZE_START 1024 * 1024 * 32 // Starting chunk size: 32MB +#define CHUNK_SIZE_MIN 1024 * 1024 * 16 // Minimum chunk size: 16MB (increased from 4MB to prevent premature convergence) +#define CHUNK_SIZE_MAX 1024 * 1024 * 128 // Maximum chunk size: 128MB +#define CHUNK_SIZE_STEP_UP 1024 * 1024 * 4 // Increase step: 4MB (symmetric with step down) +#define CHUNK_SIZE_STEP_DOWN 1024 * 1024 * 4 // Decrease step: 4MB (symmetric exploration) +#define CHUNK_MEASURE_INTERVAL 64 // Measure performance every 64 chunks -#ifdef ADAPTIVE_CHUNK_SIZE -// Use max buffer size when adaptive mode is enabled +// Multi-armed bandit exploration parameters +#define EXPLORATION_EPSILON 0.10 // 10% exploration rate (epsilon-greedy) +#define REEXPLORATION_INTERVAL 500 // Force re-exploration every 500 chunks + +// Buffer sizes - always use maximum for adaptive mode #define CHUNK_SIZE CHUNK_SIZE_MAX #define TFNG_DATA_SIZE CHUNK_SIZE_MAX -#else -// Use fixed chunk size when adaptive mode is disabled -#define CHUNK_SIZE CHUNK_SIZE_START -#define TFNG_DATA_SIZE CHUNK_SIZE -#endif // #define DEMO_DRIVE_SIZE 1024*1024*256L // 256MB // #define DEMO_DRIVE_SIZE 1024*1024*1024L // 1GB @@ -52,47 +48,49 @@ protected: public: Shred(); ~Shred(); - int shredDrive(Drive* drive, int* ipSignalFd); + int shredDrive(Drive *drive, int *ipSignalFd); private: fileDescriptor randomSrcFileDiscr; fileDescriptor driveFileDiscr; -#ifdef ADAPTIVE_CHUNK_SIZE - unsigned char* caTfngData; // Dynamic buffer allocation for adaptive mode - unsigned char* caReadBuffer; // Dynamic buffer allocation for adaptive mode -#else - unsigned char caTfngData[TFNG_DATA_SIZE]; - unsigned char caReadBuffer[CHUNK_SIZE]; -#endif + unsigned char *caTfngData; + unsigned char *caReadBuffer; unsigned long ulDriveByteSize; unsigned long ulDriveByteOverallCount = 0; // all bytes shredded in all iterations + checking -> used for progress calculation double d32Percent = 0.0; double d32TmpPercent = 0.0; -#ifdef ADAPTIVE_CHUNK_SIZE // Adaptive chunk size optimization members size_t currentChunkSize; size_t bestChunkSize; unsigned int chunkCounter; + unsigned int totalChunkCounter; // Total chunks written (for periodic re-exploration) std::chrono::high_resolution_clock::time_point measurementStartTime; double bestThroughputMBps; double lastThroughputMBps; unsigned long bytesWrittenInMeasurement; bool throughputIncreasing; + // Multi-armed bandit exploration state + bool explorationMode; // Currently in exploration mode? + size_t explorationChunkSize; // Chunk size being tested during exploration + // Adaptive methods void startMeasurement(); - void evaluateThroughput(Drive* drive); - void adjustChunkSize(Drive* drive); + void evaluateThroughput(Drive *drive); + void adjustChunkSize(Drive *drive); size_t getCurrentChunkSize() const; -#endif + + // Multi-armed bandit methods + bool shouldExplore(); // Decide: explore or exploit? + void performExploration(Drive *drive); // Execute exploration phase inline double calcProgress(); int iRewindDrive(fileDescriptor file); long getDriveSizeInBytes(fileDescriptor file); - unsigned int uiCalcChecksum(fileDescriptor file, Drive* drive, int* ipSignalFd); + unsigned int uiCalcChecksum(fileDescriptor file, Drive *drive, int *ipSignalFd); void cleanup(); }; diff --git a/scripts/reHDDLogUploader.bash b/scripts/reHDDLogUploader.bash deleted file mode 100644 index b384aa2..0000000 --- a/scripts/reHDDLogUploader.bash +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# remove comment for the following to activate log telemetie -curl -k -T /root/reHDD/reHDD.log -u "__Place_your_token_here__:" -H 'X-Requested-With: XMLHttpRequest' https://schuttercloud.com/public.php/webdav/`echo $(date '+%Y-%m-%d_%H-%M')`_reHDD.log -rm -f /root/reHDD/reHDD.log - - diff --git a/scripts/reHDDLogUploader.service b/scripts/reHDDLogUploader.service deleted file mode 100644 index 724addd..0000000 --- a/scripts/reHDDLogUploader.service +++ /dev/null @@ -1,18 +0,0 @@ -[Unit] -Description=reHDD log uploader -After=syslog.target -After=network.target -After=network-online.target -Wants=network-online.target - -[Service] -Type=oneshot -User=root -Group=root -RemainAfterExit=yes -ExecStart=/usr/bin/bash /root/reHDD/scripts/reHDDLogUploader.bash - -[Install] -WantedBy=multi-user.target - - diff --git a/scripts/reHDDLogUploader.timer b/scripts/reHDDLogUploader.timer deleted file mode 100644 index 86cc3f6..0000000 --- a/scripts/reHDDLogUploader.timer +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=reHDD log uploader timer - -[Timer] -OnActiveSec=30s -OnBootSec=10min -OnUnitActiveSec=12h - -[Install] -WantedBy=basic.target - diff --git a/src/shred.cpp b/src/shred.cpp index faac86a..7cdbacd 100644 --- a/src/shred.cpp +++ b/src/shred.cpp @@ -6,6 +6,8 @@ */ #include "../include/reHDD.h" +#include // For rand(), srand() +#include // For time() to seed random number generator using namespace std; #ifdef __cplusplus @@ -21,7 +23,9 @@ const static char *randomsrc = (char *)"/dev/urandom"; Shred::Shred() { -#ifdef ADAPTIVE_CHUNK_SIZE + // Seed random number generator for epsilon-greedy exploration + srand(static_cast(time(nullptr))); + // Allocate aligned buffers for maximum chunk size if (posix_memalign((void **)&caTfngData, 4096, CHUNK_SIZE_MAX) != 0) { @@ -39,19 +43,25 @@ Shred::Shred() currentChunkSize = CHUNK_SIZE_START; bestChunkSize = CHUNK_SIZE_START; chunkCounter = 0; + totalChunkCounter = 0; // Track total chunks for periodic re-exploration bestThroughputMBps = 0.0; lastThroughputMBps = 0.0; bytesWrittenInMeasurement = 0; throughputIncreasing = true; - Logger::logThis()->info("Adaptive chunk size optimization ENABLED - Starting with " + + // Initialize multi-armed bandit exploration state + explorationMode = false; + explorationChunkSize = CHUNK_SIZE_START; + + Logger::logThis()->info("Adaptive chunk size optimization ENABLED (Multi-Armed Bandit) - Starting with " + to_string(currentChunkSize / (1024 * 1024)) + " MB chunks"); -#endif + Logger::logThis()->info("Exploration strategy: " + to_string((int)(EXPLORATION_EPSILON * 100)) + + "% epsilon-greedy + periodic re-exploration every " + + to_string(REEXPLORATION_INTERVAL) + " chunks"); } Shred::~Shred() { -#ifdef ADAPTIVE_CHUNK_SIZE if (caTfngData != nullptr) { free(caTfngData); @@ -62,10 +72,8 @@ Shred::~Shred() free(caReadBuffer); caReadBuffer = nullptr; } -#endif } -#ifdef ADAPTIVE_CHUNK_SIZE /** * \brief Start performance measurement interval * \return void @@ -125,7 +133,55 @@ void Shred::evaluateThroughput(Drive *drive) } /** - * \brief Adjust chunk size based on throughput trend + * \brief Determine if we should explore (epsilon-greedy + periodic re-exploration) + * \return true if should explore, false if should exploit + */ +bool Shred::shouldExplore() +{ + // Periodic re-exploration: every REEXPLORATION_INTERVAL chunks + if (totalChunkCounter > 0 && (totalChunkCounter % REEXPLORATION_INTERVAL) == 0) + { + return true; + } + + // Epsilon-greedy: random exploration with probability EXPLORATION_EPSILON + double randomValue = static_cast(rand()) / RAND_MAX; + return (randomValue < EXPLORATION_EPSILON); +} + +/** + * \brief Perform exploration - try a random chunk size + * \param pointer to Drive instance + * \return void + */ +void Shred::performExploration(Drive *drive) +{ + size_t savedChunkSize = currentChunkSize; + + // Generate random chunk size between MIN and MAX (aligned to 4MB boundaries) + size_t numSteps = (CHUNK_SIZE_MAX - CHUNK_SIZE_MIN) / CHUNK_SIZE_STEP_UP; + size_t randomStep = rand() % (numSteps + 1); + explorationChunkSize = CHUNK_SIZE_MIN + (randomStep * CHUNK_SIZE_STEP_UP); + + // Clamp to valid range + if (explorationChunkSize < CHUNK_SIZE_MIN) + explorationChunkSize = CHUNK_SIZE_MIN; + if (explorationChunkSize > CHUNK_SIZE_MAX) + explorationChunkSize = CHUNK_SIZE_MAX; + + // Enter exploration mode + explorationMode = true; + currentChunkSize = explorationChunkSize; + + Logger::logThis()->info("EXPLORATION MODE: Testing " + + to_string(explorationChunkSize / (1024 * 1024)) + " MB chunks " + + "(was " + to_string(savedChunkSize / (1024 * 1024)) + " MB, best: " + + to_string(bestChunkSize / (1024 * 1024)) + " MB)" + + " - Drive: " + drive->getSerial()); +} + +/** + * \brief Adjust chunk size based on throughput trend (Multi-Armed Bandit) * \param pointer to Drive instance * \return void */ @@ -133,9 +189,28 @@ void Shred::adjustChunkSize(Drive *drive) { size_t oldChunkSize = currentChunkSize; + // Check if we should explore instead of exploit + if (shouldExplore()) + { + performExploration(drive); + return; + } + + // Exit exploration mode if we were in it + if (explorationMode) + { + explorationMode = false; + currentChunkSize = bestChunkSize; // Return to best known chunk size + Logger::logThis()->info("EXPLORATION ENDED - Returning to best known: " + + to_string(bestChunkSize / (1024 * 1024)) + " MB" + + " - Drive: " + drive->getSerial()); + return; + } + + // Normal exploitation mode: hill-climbing with symmetric steps if (throughputIncreasing) { - // Throughput is improving - increase chunk size + // Throughput is improving - increase chunk size (symmetric step) currentChunkSize += CHUNK_SIZE_STEP_UP; // Clamp to maximum @@ -149,7 +224,7 @@ void Shred::adjustChunkSize(Drive *drive) } else { - // Throughput decreased - decrease chunk size to find sweet spot + // Throughput decreased - decrease chunk size (symmetric step) if (currentChunkSize > CHUNK_SIZE_STEP_DOWN) { currentChunkSize -= CHUNK_SIZE_STEP_DOWN; @@ -182,7 +257,6 @@ size_t Shred::getCurrentChunkSize() const { return currentChunkSize; } -#endif /** * \brief shred drive with shred @@ -233,14 +307,12 @@ int Shred::shredDrive(Drive *drive, int *ipSignalFd) const char *cpDrivePath = sDrivePath.c_str(); unsigned char ucKey[TFNG_KEY_SIZE]; -#ifdef ADAPTIVE_CHUNK_SIZE // Validate buffers were allocated if (caTfngData == nullptr || caReadBuffer == nullptr) { Logger::logThis()->error("Shred-Task: Aligned buffers not allocated! - Drive: " + drive->getSerial()); return -1; } -#endif // Open random source Logger::logThis()->info("Shred-Task: Opening random source: " + string(randomsrc) + " - Drive: " + drive->getSerial()); @@ -356,10 +428,8 @@ int Shred::shredDrive(Drive *drive, int *ipSignalFd) Logger::logThis()->info("Shred-Task: Bytes-Size of Drive: " + to_string(this->ulDriveByteSize) + " - Drive: " + drive->getSerial()); #endif -#ifdef ADAPTIVE_CHUNK_SIZE // Start first measurement interval startMeasurement(); -#endif // Main shredding loop for (unsigned int uiShredIterationCounter = 0U; uiShredIterationCounter < SHRED_ITERATIONS; uiShredIterationCounter++) { @@ -370,30 +440,18 @@ int Shred::shredDrive(Drive *drive, int *ipSignalFd) if (uiShredIterationCounter == (SHRED_ITERATIONS - 1)) { // last shred iteration --> overwrite (just the write chunk) bytes with zeros instead with random data -#ifdef ADAPTIVE_CHUNK_SIZE memset(caTfngData, 0U, CHUNK_SIZE_MAX); -#else - memset(caTfngData, 0U, CHUNK_SIZE); -#endif } while (ulDriveByteCounter < ulDriveByteSize) { -#ifdef ADAPTIVE_CHUNK_SIZE size_t activeChunkSize = getCurrentChunkSize(); -#else - size_t activeChunkSize = CHUNK_SIZE; -#endif int iBytesToShred = 0; if (uiShredIterationCounter != (SHRED_ITERATIONS - 1)) { -#ifdef ADAPTIVE_CHUNK_SIZE tfng_prng_genrandom(caTfngData, activeChunkSize); -#else - tfng_prng_genrandom(caTfngData, TFNG_DATA_SIZE); -#endif } if ((ulDriveByteSize - ulDriveByteCounter) < activeChunkSize) @@ -433,16 +491,15 @@ int Shred::shredDrive(Drive *drive, int *ipSignalFd) ulDriveByteCounter += iByteShredded; ulDriveByteOverallCount += iByteShredded; -#ifdef ADAPTIVE_CHUNK_SIZE bytesWrittenInMeasurement += iByteShredded; chunkCounter++; + totalChunkCounter++; // Track total chunks for periodic re-exploration // Evaluate throughput after measurement interval if (chunkCounter >= CHUNK_MEASURE_INTERVAL) { evaluateThroughput(drive); } -#endif d32Percent = this->calcProgress(); @@ -493,12 +550,10 @@ int Shred::shredDrive(Drive *drive, int *ipSignalFd) } } -#ifdef ADAPTIVE_CHUNK_SIZE Logger::logThis()->info("Shred completed - Optimal chunk size: " + to_string(bestChunkSize / (1024 * 1024)) + " MB, " + "Best throughput: " + to_string((int)bestThroughputMBps) + " MB/s" + " - Drive: " + drive->getSerial()); -#endif // All shred iterations completed successfully tfng_prng_seedkey(NULL); @@ -645,11 +700,7 @@ unsigned int Shred::uiCalcChecksum(fileDescriptor file, Drive *drive, int *ipSig Logger::logThis()->info("Check-Task: Starting checksum verification - Drive: " + drive->getSerial()); -#ifdef ADAPTIVE_CHUNK_SIZE size_t checkChunkSize = CHUNK_SIZE_MAX; -#else - size_t checkChunkSize = CHUNK_SIZE; -#endif while (ulDriveByteCounter < ulDriveByteSize) { -- 2.54.0 From 4ae0a89ccd6a7aba2e1f5f950c209b4eeab7b44c Mon Sep 17 00:00:00 2001 From: localhorst Date: Sun, 3 May 2026 08:41:34 +0200 Subject: [PATCH 6/9] add cold start --- include/shred.h | 2 ++ src/shred.cpp | 93 ++++++++++++++++++++++++++++++++++--------------- 2 files changed, 67 insertions(+), 28 deletions(-) diff --git a/include/shred.h b/include/shred.h index ecf5dcc..e89738a 100644 --- a/include/shred.h +++ b/include/shred.h @@ -26,6 +26,7 @@ #define CHUNK_SIZE_STEP_UP 1024 * 1024 * 4 // Increase step: 4MB (symmetric with step down) #define CHUNK_SIZE_STEP_DOWN 1024 * 1024 * 4 // Decrease step: 4MB (symmetric exploration) #define CHUNK_MEASURE_INTERVAL 64 // Measure performance every 64 chunks +#define WARMUP_MEASUREMENTS 16 // Skip first 16 measurements (cache writes) // Multi-armed bandit exploration parameters #define EXPLORATION_EPSILON 0.10 // 10% exploration rate (epsilon-greedy) @@ -67,6 +68,7 @@ private: size_t bestChunkSize; unsigned int chunkCounter; unsigned int totalChunkCounter; // Total chunks written (for periodic re-exploration) + unsigned int warmupCounter; // Count warm-up measurements std::chrono::high_resolution_clock::time_point measurementStartTime; double bestThroughputMBps; double lastThroughputMBps; diff --git a/src/shred.cpp b/src/shred.cpp index 7cdbacd..22335d4 100644 --- a/src/shred.cpp +++ b/src/shred.cpp @@ -44,6 +44,7 @@ Shred::Shred() bestChunkSize = CHUNK_SIZE_START; chunkCounter = 0; totalChunkCounter = 0; // Track total chunks for periodic re-exploration + warmupCounter = 0; // Track warm-up measurements bestThroughputMBps = 0.0; lastThroughputMBps = 0.0; bytesWrittenInMeasurement = 0; @@ -55,9 +56,12 @@ Shred::Shred() Logger::logThis()->info("Adaptive chunk size optimization ENABLED (Multi-Armed Bandit) - Starting with " + to_string(currentChunkSize / (1024 * 1024)) + " MB chunks"); - Logger::logThis()->info("Exploration strategy: " + to_string((int)(EXPLORATION_EPSILON * 100)) + - "% epsilon-greedy + periodic re-exploration every " + - to_string(REEXPLORATION_INTERVAL) + " chunks"); + Logger::logThis()->info("Configuration: min=" + to_string(CHUNK_SIZE_MIN / (1024 * 1024)) + + "MB, max=" + to_string(CHUNK_SIZE_MAX / (1024 * 1024)) + + "MB, step=" + to_string(CHUNK_SIZE_STEP_UP / (1024 * 1024)) + "MB"); + Logger::logThis()->info("Exploration: " + to_string((int)(EXPLORATION_EPSILON * 100)) + + "% epsilon-greedy + periodic every " + to_string(REEXPLORATION_INTERVAL) + " chunks"); + Logger::logThis()->info("Warm-up: First " + to_string(WARMUP_MEASUREMENTS) + " measurements ignored (cold start protection)"); } Shred::~Shred() @@ -102,31 +106,48 @@ void Shred::evaluateThroughput(Drive *drive) double throughputMBps = (bytesWrittenInMeasurement / (1024.0 * 1024.0)) / elapsedSeconds; lastThroughputMBps = throughputMBps; - Logger::logThis()->info("Throughput measurement - ChunkSize: " + - to_string(currentChunkSize / (1024 * 1024)) + " MB, " + - "Throughput: " + to_string((int)throughputMBps) + " MB/s, " + - "Best: " + to_string((int)bestThroughputMBps) + " MB/s" + - " - Drive: " + drive->getSerial()); + // Warm-up period - ignore first measurements + warmupCounter++; + bool isWarmup = (warmupCounter <= WARMUP_MEASUREMENTS); - // Check if this is better than our best - if (throughputMBps > bestThroughputMBps) + if (isWarmup) { - bestThroughputMBps = throughputMBps; - bestChunkSize = currentChunkSize; - throughputIncreasing = true; - - Logger::logThis()->info("NEW BEST throughput: " + to_string((int)bestThroughputMBps) + - " MB/s with " + to_string(currentChunkSize / (1024 * 1024)) + - " MB chunks - Drive: " + drive->getSerial()); + Logger::logThis()->info("WARM-UP #" + to_string(warmupCounter) + "/" + to_string(WARMUP_MEASUREMENTS) + + " - ChunkSize: " + to_string(currentChunkSize / (1024 * 1024)) + " MB, " + + "Throughput: " + to_string((int)throughputMBps) + " MB/s (not used for optimization)" + + " - Drive: " + drive->getSerial()); } else { - throughputIncreasing = false; + Logger::logThis()->info("Throughput measurement - ChunkSize: " + + to_string(currentChunkSize / (1024 * 1024)) + " MB, " + + "Throughput: " + to_string((int)throughputMBps) + " MB/s, " + + "Best: " + to_string((int)bestThroughputMBps) + " MB/s" + + " - Drive: " + drive->getSerial()); + + // Check if this is better than our best (only after warm-up) + if (throughputMBps > bestThroughputMBps) + { + bestThroughputMBps = throughputMBps; + bestChunkSize = currentChunkSize; + throughputIncreasing = true; + + Logger::logThis()->info("NEW BEST throughput: " + to_string((int)bestThroughputMBps) + + " MB/s with " + to_string(currentChunkSize / (1024 * 1024)) + + " MB chunks - Drive: " + drive->getSerial()); + } + else + { + throughputIncreasing = false; + } } } - // Adjust chunk size for next measurement interval - adjustChunkSize(drive); + // Adjust chunk size for next measurement interval (skip during warm-up) + if (warmupCounter > WARMUP_MEASUREMENTS) + { + adjustChunkSize(drive); + } // Start new measurement startMeasurement(); @@ -158,12 +179,12 @@ void Shred::performExploration(Drive *drive) { size_t savedChunkSize = currentChunkSize; - // Generate random chunk size between MIN and MAX (aligned to 4MB boundaries) + // Generate random chunk size between MIN and MAX (aligned to STEP boundaries) size_t numSteps = (CHUNK_SIZE_MAX - CHUNK_SIZE_MIN) / CHUNK_SIZE_STEP_UP; size_t randomStep = rand() % (numSteps + 1); explorationChunkSize = CHUNK_SIZE_MIN + (randomStep * CHUNK_SIZE_STEP_UP); - // Clamp to valid range + // Clamp to valid range (safety check) if (explorationChunkSize < CHUNK_SIZE_MIN) explorationChunkSize = CHUNK_SIZE_MIN; if (explorationChunkSize > CHUNK_SIZE_MAX) @@ -173,10 +194,12 @@ void Shred::performExploration(Drive *drive) explorationMode = true; currentChunkSize = explorationChunkSize; + // Enhanced logging with debug info Logger::logThis()->info("EXPLORATION MODE: Testing " + to_string(explorationChunkSize / (1024 * 1024)) + " MB chunks " + - "(was " + to_string(savedChunkSize / (1024 * 1024)) + " MB, best: " + - to_string(bestChunkSize / (1024 * 1024)) + " MB)" + + "(randomStep=" + to_string(randomStep) + "/" + to_string(numSteps) + ", " + + "was " + to_string(savedChunkSize / (1024 * 1024)) + " MB, " + + "best: " + to_string(bestChunkSize / (1024 * 1024)) + " MB)" + " - Drive: " + drive->getSerial()); } @@ -200,10 +223,22 @@ void Shred::adjustChunkSize(Drive *drive) if (explorationMode) { explorationMode = false; - currentChunkSize = bestChunkSize; // Return to best known chunk size - Logger::logThis()->info("EXPLORATION ENDED - Returning to best known: " + - to_string(bestChunkSize / (1024 * 1024)) + " MB" + - " - Drive: " + drive->getSerial()); + + // CRITICAL: Return to best known chunk size, not current + if (currentChunkSize != bestChunkSize) + { + currentChunkSize = bestChunkSize; + Logger::logThis()->info("EXPLORATION ENDED - Returning to best known: " + + to_string(bestChunkSize / (1024 * 1024)) + " MB" + + " (exploration tested " + to_string(oldChunkSize / (1024 * 1024)) + " MB)" + + " - Drive: " + drive->getSerial()); + } + else + { + Logger::logThis()->info("EXPLORATION ENDED - Staying at current best: " + + to_string(bestChunkSize / (1024 * 1024)) + " MB" + + " - Drive: " + drive->getSerial()); + } return; } @@ -236,6 +271,7 @@ void Shred::adjustChunkSize(Drive *drive) currentChunkSize = CHUNK_SIZE_MIN; Logger::logThis()->info("Reached minimum chunk size: " + to_string(currentChunkSize / (1024 * 1024)) + " MB" + + " (best remains: " + to_string(bestChunkSize / (1024 * 1024)) + " MB)" + " - Drive: " + drive->getSerial()); } } @@ -245,6 +281,7 @@ void Shred::adjustChunkSize(Drive *drive) Logger::logThis()->info("Adjusted chunk size: " + to_string(oldChunkSize / (1024 * 1024)) + " MB -> " + to_string(currentChunkSize / (1024 * 1024)) + " MB" + + " (best: " + to_string(bestChunkSize / (1024 * 1024)) + " MB)" + " - Drive: " + drive->getSerial()); } } -- 2.54.0 From 8856fc4b9c84691cdf0127b498e5d98915c23dfd Mon Sep 17 00:00:00 2001 From: localhorst Date: Sun, 3 May 2026 08:44:37 +0200 Subject: [PATCH 7/9] fix update of best chunk size --- src/shred.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/shred.cpp b/src/shred.cpp index 22335d4..bc5ba06 100644 --- a/src/shred.cpp +++ b/src/shred.cpp @@ -136,6 +136,22 @@ void Shred::evaluateThroughput(Drive *drive) " MB/s with " + to_string(currentChunkSize / (1024 * 1024)) + " MB chunks - Drive: " + drive->getSerial()); } + else if (currentChunkSize == bestChunkSize) + { + // Update best throughput when measuring at best chunk size + // This ensures bestThroughputMBps reflects CURRENT performance, not old burst + if (throughputMBps < bestThroughputMBps) + { + Logger::logThis()->info("Updating best throughput: " + + to_string((int)bestThroughputMBps) + " MB/s -> " + + to_string((int)throughputMBps) + " MB/s " + + "(sustained performance at best chunk size: " + + to_string(bestChunkSize / (1024 * 1024)) + " MB)" + + " - Drive: " + drive->getSerial()); + bestThroughputMBps = throughputMBps; + } + throughputIncreasing = false; + } else { throughputIncreasing = false; -- 2.54.0 From 47ef3851ac28b786dba960584df656807cb54de6 Mon Sep 17 00:00:00 2001 From: localhorst Date: Sun, 3 May 2026 08:56:12 +0200 Subject: [PATCH 8/9] readme spelling --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a72e989..ec454b5 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ * process multiple drives at once ## Download USB Image ## -See reHDD-Bootable how the live image created: https://git.mosad.xyz/localhorst/reHDD-Bootable +See reHDD-Bootable how the live image is created: https://git.mosad.xyz/localhorst/reHDD-Bootable Use [Etcher](https://www.balena.io/etcher/#download) or `dd` to create an bootable USB drive . -- 2.54.0 From d449d09786feb76e0d8ae703d5e7e5f3d14a6e5c Mon Sep 17 00:00:00 2001 From: localhorst Date: Sun, 3 May 2026 10:04:34 +0200 Subject: [PATCH 9/9] fix rng bug --- src/shred.cpp | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/shred.cpp b/src/shred.cpp index bc5ba06..2550e06 100644 --- a/src/shred.cpp +++ b/src/shred.cpp @@ -196,9 +196,22 @@ void Shred::performExploration(Drive *drive) size_t savedChunkSize = currentChunkSize; // Generate random chunk size between MIN and MAX (aligned to STEP boundaries) - size_t numSteps = (CHUNK_SIZE_MAX - CHUNK_SIZE_MIN) / CHUNK_SIZE_STEP_UP; - size_t randomStep = rand() % (numSteps + 1); - explorationChunkSize = CHUNK_SIZE_MIN + (randomStep * CHUNK_SIZE_STEP_UP); + // Calculate in MB to avoid overflow + size_t minMB = CHUNK_SIZE_MIN / (1024 * 1024); + size_t maxMB = CHUNK_SIZE_MAX / (1024 * 1024); + size_t stepMB = CHUNK_SIZE_STEP_UP / (1024 * 1024); + + // Number of possible steps: (max - min) / step + size_t numSteps = (maxMB - minMB) / stepMB; + + // Generate random step: 0 to numSteps (inclusive) + // Using proper modulo to ensure range [0, numSteps] + int randVal = rand(); + size_t randomStep = static_cast(randVal) % (numSteps + 1); + + // Calculate exploration chunk size in MB, then convert to bytes + size_t explorationMB = minMB + (randomStep * stepMB); + explorationChunkSize = explorationMB * 1024 * 1024; // Clamp to valid range (safety check) if (explorationChunkSize < CHUNK_SIZE_MIN) @@ -213,7 +226,8 @@ void Shred::performExploration(Drive *drive) // Enhanced logging with debug info Logger::logThis()->info("EXPLORATION MODE: Testing " + to_string(explorationChunkSize / (1024 * 1024)) + " MB chunks " + - "(randomStep=" + to_string(randomStep) + "/" + to_string(numSteps) + ", " + + "(randomStep=" + to_string(randomStep) + "/" + to_string(numSteps) + + ", rand=" + to_string(randVal) + ", " + "was " + to_string(savedChunkSize / (1024 * 1024)) + " MB, " + "best: " + to_string(bestChunkSize / (1024 * 1024)) + " MB)" + " - Drive: " + drive->getSerial()); -- 2.54.0