Compare commits
2 Commits
1.40
..
95f5037529
| Author | SHA1 | Date | |
|---|---|---|---|
| 95f5037529 | |||
| 50e88c8e84 |
@@ -0,0 +1,92 @@
|
|||||||
|
# reHDD Code Style - clang-format configuration
|
||||||
|
# Based on analyzed codebase formatting
|
||||||
|
|
||||||
|
BasedOnStyle: LLVM
|
||||||
|
|
||||||
|
# Indentation
|
||||||
|
IndentWidth: 4
|
||||||
|
TabWidth: 4
|
||||||
|
UseTab: Never
|
||||||
|
ContinuationIndentWidth: 4
|
||||||
|
|
||||||
|
# Braces
|
||||||
|
BreakBeforeBraces: Allman
|
||||||
|
# Allman style:
|
||||||
|
# if (condition)
|
||||||
|
# {
|
||||||
|
# statement;
|
||||||
|
# }
|
||||||
|
|
||||||
|
# Spacing
|
||||||
|
SpaceAfterCStyleCast: false
|
||||||
|
SpaceBeforeParens: ControlStatements
|
||||||
|
SpaceInEmptyParentheses: false
|
||||||
|
SpacesInContainerLiterals: false
|
||||||
|
SpacesInCStyleCastParentheses: false
|
||||||
|
SpacesInParentheses: false
|
||||||
|
SpacesInSquareBrackets: false
|
||||||
|
|
||||||
|
# Alignment
|
||||||
|
AlignAfterOpenBracket: Align
|
||||||
|
AlignConsecutiveAssignments: false
|
||||||
|
AlignConsecutiveDeclarations: false
|
||||||
|
AlignEscapedNewlines: Left
|
||||||
|
AlignOperands: true
|
||||||
|
AlignTrailingComments: false
|
||||||
|
|
||||||
|
# Line breaks
|
||||||
|
AllowShortBlocksOnASingleLine: false
|
||||||
|
AllowShortCaseLabelsOnASingleLine: false
|
||||||
|
AllowShortFunctionsOnASingleLine: None
|
||||||
|
AllowShortIfStatementsOnASingleLine: Never
|
||||||
|
AllowShortLoopsOnASingleLine: false
|
||||||
|
AlwaysBreakAfterReturnType: None
|
||||||
|
AlwaysBreakBeforeMultilineStrings: false
|
||||||
|
AlwaysBreakTemplateDeclarations: Yes
|
||||||
|
|
||||||
|
# Column limit
|
||||||
|
ColumnLimit: 0
|
||||||
|
# 0 = no limit (observed in the code)
|
||||||
|
|
||||||
|
# Breaking
|
||||||
|
BinPackArguments: true
|
||||||
|
BinPackParameters: true
|
||||||
|
BreakBeforeBinaryOperators: None
|
||||||
|
BreakBeforeTernaryOperators: true
|
||||||
|
BreakConstructorInitializers: BeforeColon
|
||||||
|
BreakInheritanceList: BeforeColon
|
||||||
|
BreakStringLiterals: false
|
||||||
|
|
||||||
|
# Pointers and references
|
||||||
|
PointerAlignment: Left
|
||||||
|
# Type *variable (not Type* variable)
|
||||||
|
|
||||||
|
# Include sorting
|
||||||
|
SortIncludes: false
|
||||||
|
# Keep includes in original order
|
||||||
|
|
||||||
|
# Comments
|
||||||
|
ReflowComments: false
|
||||||
|
# Don't reformat comments
|
||||||
|
|
||||||
|
# Access modifiers
|
||||||
|
AccessModifierOffset: -4
|
||||||
|
# public: is outdented
|
||||||
|
|
||||||
|
# Constructor initializers
|
||||||
|
ConstructorInitializerIndentWidth: 6
|
||||||
|
# Observed 6 spaces for initializer lists
|
||||||
|
|
||||||
|
# Other
|
||||||
|
Cpp11BracedListStyle: true
|
||||||
|
FixNamespaceComments: true
|
||||||
|
IncludeBlocks: Preserve
|
||||||
|
IndentCaseLabels: false
|
||||||
|
IndentPPDirectives: None
|
||||||
|
KeepEmptyLinesAtTheStartOfBlocks: false
|
||||||
|
MaxEmptyLinesToKeep: 1
|
||||||
|
NamespaceIndentation: None
|
||||||
|
PenaltyBreakBeforeFirstCallParameter: 100
|
||||||
|
PenaltyBreakComment: 300
|
||||||
|
PenaltyBreakString: 1000
|
||||||
|
PenaltyExcessCharacter: 1000000
|
||||||
@@ -9,7 +9,7 @@
|
|||||||
* process multiple drives at once
|
* process multiple drives at once
|
||||||
|
|
||||||
## Download USB Image ##
|
## Download USB Image ##
|
||||||
See reHDD-Bootable how the live image is created: https://git.mosad.xyz/localhorst/reHDD-Bootable
|
See reHDD-Bootable how the live image created: https://git.mosad.xyz/localhorst/reHDD-Bootable
|
||||||
|
|
||||||
Use [Etcher](https://www.balena.io/etcher/#download) or `dd` to create an bootable USB drive .
|
Use [Etcher](https://www.balena.io/etcher/#download) or `dd` to create an bootable USB drive .
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -8,7 +8,7 @@
|
|||||||
#ifndef REHDD_H_
|
#ifndef REHDD_H_
|
||||||
#define REHDD_H_
|
#define REHDD_H_
|
||||||
|
|
||||||
#define REHDD_VERSION "V1.4.0"
|
#define REHDD_VERSION "V1.4.0-dev"
|
||||||
|
|
||||||
// Drive handling Settings
|
// Drive handling Settings
|
||||||
#define WORSE_HOURS 19200 // mark drive if at this limit or beyond
|
#define WORSE_HOURS 19200 // mark drive if at this limit or beyond
|
||||||
|
|||||||
+22
-22
@@ -18,23 +18,26 @@
|
|||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <chrono>
|
#include <chrono>
|
||||||
|
|
||||||
// Adaptive chunk size optimization with multi-armed bandit - always enabled
|
// Adaptive chunk size optimization - uncomment to enable
|
||||||
|
#define ADAPTIVE_CHUNK_SIZE
|
||||||
|
|
||||||
// Chunk size configuration
|
// Chunk size configuration
|
||||||
#define CHUNK_SIZE_START 1024 * 1024 * 32 // Starting chunk size: 32MB
|
#define CHUNK_SIZE_START 1024 * 1024 * 32 // Starting chunk size: 32MB
|
||||||
#define CHUNK_SIZE_MIN 1024 * 1024 * 16 // Minimum chunk size: 16MB (increased from 4MB to prevent premature convergence)
|
#define CHUNK_SIZE_MIN 1024 * 1024 * 4 // Minimum chunk size: 4MB
|
||||||
#define CHUNK_SIZE_MAX 1024 * 1024 * 128 // Maximum chunk size: 128MB
|
#define CHUNK_SIZE_MAX 1024 * 1024 * 128 // Maximum chunk size: 128MB
|
||||||
#define CHUNK_SIZE_STEP_UP 1024 * 1024 * 4 // Increase step: 4MB (symmetric with step down)
|
#define CHUNK_SIZE_STEP_UP 1024 * 1024 * 2 // Increase step: 2MB
|
||||||
#define CHUNK_SIZE_STEP_DOWN 1024 * 1024 * 4 // Decrease step: 4MB (symmetric exploration)
|
#define CHUNK_SIZE_STEP_DOWN 1024 * 1024 * 4 // Decrease step: 4MB
|
||||||
#define CHUNK_MEASURE_INTERVAL 64 // Measure performance every 64 chunks
|
#define CHUNK_MEASURE_INTERVAL 64 // Measure performance every 64 chunks
|
||||||
#define WARMUP_MEASUREMENTS 16 // Skip first 16 measurements (cache writes)
|
|
||||||
|
|
||||||
// Multi-armed bandit exploration parameters
|
#ifdef ADAPTIVE_CHUNK_SIZE
|
||||||
#define EXPLORATION_EPSILON 0.10 // 10% exploration rate (epsilon-greedy)
|
// Use max buffer size when adaptive mode is enabled
|
||||||
#define REEXPLORATION_INTERVAL 500 // Force re-exploration every 500 chunks
|
|
||||||
|
|
||||||
// Buffer sizes - always use maximum for adaptive mode
|
|
||||||
#define CHUNK_SIZE CHUNK_SIZE_MAX
|
#define CHUNK_SIZE CHUNK_SIZE_MAX
|
||||||
#define TFNG_DATA_SIZE CHUNK_SIZE_MAX
|
#define TFNG_DATA_SIZE CHUNK_SIZE_MAX
|
||||||
|
#else
|
||||||
|
// Use fixed chunk size when adaptive mode is disabled
|
||||||
|
#define CHUNK_SIZE CHUNK_SIZE_START
|
||||||
|
#define TFNG_DATA_SIZE CHUNK_SIZE
|
||||||
|
#endif
|
||||||
|
|
||||||
// #define DEMO_DRIVE_SIZE 1024*1024*256L // 256MB
|
// #define DEMO_DRIVE_SIZE 1024*1024*256L // 256MB
|
||||||
// #define DEMO_DRIVE_SIZE 1024*1024*1024L // 1GB
|
// #define DEMO_DRIVE_SIZE 1024*1024*1024L // 1GB
|
||||||
@@ -55,39 +58,36 @@ private:
|
|||||||
fileDescriptor randomSrcFileDiscr;
|
fileDescriptor randomSrcFileDiscr;
|
||||||
fileDescriptor driveFileDiscr;
|
fileDescriptor driveFileDiscr;
|
||||||
|
|
||||||
unsigned char *caTfngData;
|
#ifdef ADAPTIVE_CHUNK_SIZE
|
||||||
unsigned char *caReadBuffer;
|
unsigned char* caTfngData; // Dynamic buffer allocation for adaptive mode
|
||||||
|
unsigned char* caReadBuffer; // Dynamic buffer allocation for adaptive mode
|
||||||
|
#else
|
||||||
|
unsigned char caTfngData[TFNG_DATA_SIZE];
|
||||||
|
unsigned char caReadBuffer[CHUNK_SIZE];
|
||||||
|
#endif
|
||||||
|
|
||||||
unsigned long ulDriveByteSize;
|
unsigned long ulDriveByteSize;
|
||||||
unsigned long ulDriveByteOverallCount = 0; // all bytes shredded in all iterations + checking -> used for progress calculation
|
unsigned long ulDriveByteOverallCount = 0; // all bytes shredded in all iterations + checking -> used for progress calculation
|
||||||
double d32Percent = 0.0;
|
double d32Percent = 0.0;
|
||||||
double d32TmpPercent = 0.0;
|
double d32TmpPercent = 0.0;
|
||||||
|
|
||||||
|
#ifdef ADAPTIVE_CHUNK_SIZE
|
||||||
// Adaptive chunk size optimization members
|
// Adaptive chunk size optimization members
|
||||||
size_t currentChunkSize;
|
size_t currentChunkSize;
|
||||||
size_t bestChunkSize;
|
size_t bestChunkSize;
|
||||||
unsigned int chunkCounter;
|
unsigned int chunkCounter;
|
||||||
unsigned int totalChunkCounter; // Total chunks written (for periodic re-exploration)
|
|
||||||
unsigned int warmupCounter; // Count warm-up measurements
|
|
||||||
std::chrono::high_resolution_clock::time_point measurementStartTime;
|
std::chrono::high_resolution_clock::time_point measurementStartTime;
|
||||||
double bestThroughputMBps;
|
double bestThroughputMBps;
|
||||||
double lastThroughputMBps;
|
double lastThroughputMBps;
|
||||||
unsigned long bytesWrittenInMeasurement;
|
unsigned long bytesWrittenInMeasurement;
|
||||||
bool throughputIncreasing;
|
bool throughputIncreasing;
|
||||||
|
|
||||||
// Multi-armed bandit exploration state
|
|
||||||
bool explorationMode; // Currently in exploration mode?
|
|
||||||
size_t explorationChunkSize; // Chunk size being tested during exploration
|
|
||||||
|
|
||||||
// Adaptive methods
|
// Adaptive methods
|
||||||
void startMeasurement();
|
void startMeasurement();
|
||||||
void evaluateThroughput(Drive* drive);
|
void evaluateThroughput(Drive* drive);
|
||||||
void adjustChunkSize(Drive* drive);
|
void adjustChunkSize(Drive* drive);
|
||||||
size_t getCurrentChunkSize() const;
|
size_t getCurrentChunkSize() const;
|
||||||
|
#endif
|
||||||
// Multi-armed bandit methods
|
|
||||||
bool shouldExplore(); // Decide: explore or exploit?
|
|
||||||
void performExploration(Drive *drive); // Execute exploration phase
|
|
||||||
|
|
||||||
inline double calcProgress();
|
inline double calcProgress();
|
||||||
int iRewindDrive(fileDescriptor file);
|
int iRewindDrive(fileDescriptor file);
|
||||||
|
|||||||
@@ -0,0 +1,6 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# remove comment for the following to activate log telemetie
|
||||||
|
curl -k -T /root/reHDD/reHDD.log -u "__Place_your_token_here__:" -H 'X-Requested-With: XMLHttpRequest' https://schuttercloud.com/public.php/webdav/`echo $(date '+%Y-%m-%d_%H-%M')`_reHDD.log
|
||||||
|
rm -f /root/reHDD/reHDD.log
|
||||||
|
|
||||||
|
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=reHDD log uploader
|
||||||
|
After=syslog.target
|
||||||
|
After=network.target
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
User=root
|
||||||
|
Group=root
|
||||||
|
RemainAfterExit=yes
|
||||||
|
ExecStart=/usr/bin/bash /root/reHDD/scripts/reHDDLogUploader.bash
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
|
||||||
|
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=reHDD log uploader timer
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnActiveSec=30s
|
||||||
|
OnBootSec=10min
|
||||||
|
OnUnitActiveSec=12h
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=basic.target
|
||||||
|
|
||||||
+38
-161
@@ -6,8 +6,6 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
#include "../include/reHDD.h"
|
#include "../include/reHDD.h"
|
||||||
#include <cstdlib> // For rand(), srand()
|
|
||||||
#include <ctime> // For time() to seed random number generator
|
|
||||||
using namespace std;
|
using namespace std;
|
||||||
|
|
||||||
#ifdef __cplusplus
|
#ifdef __cplusplus
|
||||||
@@ -23,9 +21,7 @@ const static char *randomsrc = (char *)"/dev/urandom";
|
|||||||
|
|
||||||
Shred::Shred()
|
Shred::Shred()
|
||||||
{
|
{
|
||||||
// Seed random number generator for epsilon-greedy exploration
|
#ifdef ADAPTIVE_CHUNK_SIZE
|
||||||
srand(static_cast<unsigned int>(time(nullptr)));
|
|
||||||
|
|
||||||
// Allocate aligned buffers for maximum chunk size
|
// Allocate aligned buffers for maximum chunk size
|
||||||
if (posix_memalign((void**)&caTfngData, 4096, CHUNK_SIZE_MAX) != 0)
|
if (posix_memalign((void**)&caTfngData, 4096, CHUNK_SIZE_MAX) != 0)
|
||||||
{
|
{
|
||||||
@@ -43,29 +39,19 @@ Shred::Shred()
|
|||||||
currentChunkSize = CHUNK_SIZE_START;
|
currentChunkSize = CHUNK_SIZE_START;
|
||||||
bestChunkSize = CHUNK_SIZE_START;
|
bestChunkSize = CHUNK_SIZE_START;
|
||||||
chunkCounter = 0;
|
chunkCounter = 0;
|
||||||
totalChunkCounter = 0; // Track total chunks for periodic re-exploration
|
|
||||||
warmupCounter = 0; // Track warm-up measurements
|
|
||||||
bestThroughputMBps = 0.0;
|
bestThroughputMBps = 0.0;
|
||||||
lastThroughputMBps = 0.0;
|
lastThroughputMBps = 0.0;
|
||||||
bytesWrittenInMeasurement = 0;
|
bytesWrittenInMeasurement = 0;
|
||||||
throughputIncreasing = true;
|
throughputIncreasing = true;
|
||||||
|
|
||||||
// Initialize multi-armed bandit exploration state
|
Logger::logThis()->info("Adaptive chunk size optimization ENABLED - Starting with " +
|
||||||
explorationMode = false;
|
|
||||||
explorationChunkSize = CHUNK_SIZE_START;
|
|
||||||
|
|
||||||
Logger::logThis()->info("Adaptive chunk size optimization ENABLED (Multi-Armed Bandit) - Starting with " +
|
|
||||||
to_string(currentChunkSize / (1024 * 1024)) + " MB chunks");
|
to_string(currentChunkSize / (1024 * 1024)) + " MB chunks");
|
||||||
Logger::logThis()->info("Configuration: min=" + to_string(CHUNK_SIZE_MIN / (1024 * 1024)) +
|
#endif
|
||||||
"MB, max=" + to_string(CHUNK_SIZE_MAX / (1024 * 1024)) +
|
|
||||||
"MB, step=" + to_string(CHUNK_SIZE_STEP_UP / (1024 * 1024)) + "MB");
|
|
||||||
Logger::logThis()->info("Exploration: " + to_string((int)(EXPLORATION_EPSILON * 100)) +
|
|
||||||
"% epsilon-greedy + periodic every " + to_string(REEXPLORATION_INTERVAL) + " chunks");
|
|
||||||
Logger::logThis()->info("Warm-up: First " + to_string(WARMUP_MEASUREMENTS) + " measurements ignored (cold start protection)");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Shred::~Shred()
|
Shred::~Shred()
|
||||||
{
|
{
|
||||||
|
#ifdef ADAPTIVE_CHUNK_SIZE
|
||||||
if (caTfngData != nullptr)
|
if (caTfngData != nullptr)
|
||||||
{
|
{
|
||||||
free(caTfngData);
|
free(caTfngData);
|
||||||
@@ -76,8 +62,10 @@ Shred::~Shred()
|
|||||||
free(caReadBuffer);
|
free(caReadBuffer);
|
||||||
caReadBuffer = nullptr;
|
caReadBuffer = nullptr;
|
||||||
}
|
}
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#ifdef ADAPTIVE_CHUNK_SIZE
|
||||||
/**
|
/**
|
||||||
* \brief Start performance measurement interval
|
* \brief Start performance measurement interval
|
||||||
* \return void
|
* \return void
|
||||||
@@ -106,26 +94,13 @@ void Shred::evaluateThroughput(Drive *drive)
|
|||||||
double throughputMBps = (bytesWrittenInMeasurement / (1024.0 * 1024.0)) / elapsedSeconds;
|
double throughputMBps = (bytesWrittenInMeasurement / (1024.0 * 1024.0)) / elapsedSeconds;
|
||||||
lastThroughputMBps = throughputMBps;
|
lastThroughputMBps = throughputMBps;
|
||||||
|
|
||||||
// Warm-up period - ignore first measurements
|
|
||||||
warmupCounter++;
|
|
||||||
bool isWarmup = (warmupCounter <= WARMUP_MEASUREMENTS);
|
|
||||||
|
|
||||||
if (isWarmup)
|
|
||||||
{
|
|
||||||
Logger::logThis()->info("WARM-UP #" + to_string(warmupCounter) + "/" + to_string(WARMUP_MEASUREMENTS) +
|
|
||||||
" - ChunkSize: " + to_string(currentChunkSize / (1024 * 1024)) + " MB, " +
|
|
||||||
"Throughput: " + to_string((int)throughputMBps) + " MB/s (not used for optimization)" +
|
|
||||||
" - Drive: " + drive->getSerial());
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
Logger::logThis()->info("Throughput measurement - ChunkSize: " +
|
Logger::logThis()->info("Throughput measurement - ChunkSize: " +
|
||||||
to_string(currentChunkSize / (1024 * 1024)) + " MB, " +
|
to_string(currentChunkSize / (1024 * 1024)) + " MB, " +
|
||||||
"Throughput: " + to_string((int)throughputMBps) + " MB/s, " +
|
"Throughput: " + to_string((int)throughputMBps) + " MB/s, " +
|
||||||
"Best: " + to_string((int)bestThroughputMBps) + " MB/s" +
|
"Best: " + to_string((int)bestThroughputMBps) + " MB/s" +
|
||||||
" - Drive: " + drive->getSerial());
|
" - Drive: " + drive->getSerial());
|
||||||
|
|
||||||
// Check if this is better than our best (only after warm-up)
|
// Check if this is better than our best
|
||||||
if (throughputMBps > bestThroughputMBps)
|
if (throughputMBps > bestThroughputMBps)
|
||||||
{
|
{
|
||||||
bestThroughputMBps = throughputMBps;
|
bestThroughputMBps = throughputMBps;
|
||||||
@@ -136,105 +111,21 @@ void Shred::evaluateThroughput(Drive *drive)
|
|||||||
" MB/s with " + to_string(currentChunkSize / (1024 * 1024)) +
|
" MB/s with " + to_string(currentChunkSize / (1024 * 1024)) +
|
||||||
" MB chunks - Drive: " + drive->getSerial());
|
" MB chunks - Drive: " + drive->getSerial());
|
||||||
}
|
}
|
||||||
else if (currentChunkSize == bestChunkSize)
|
|
||||||
{
|
|
||||||
// Update best throughput when measuring at best chunk size
|
|
||||||
// This ensures bestThroughputMBps reflects CURRENT performance, not old burst
|
|
||||||
if (throughputMBps < bestThroughputMBps)
|
|
||||||
{
|
|
||||||
Logger::logThis()->info("Updating best throughput: " +
|
|
||||||
to_string((int)bestThroughputMBps) + " MB/s -> " +
|
|
||||||
to_string((int)throughputMBps) + " MB/s " +
|
|
||||||
"(sustained performance at best chunk size: " +
|
|
||||||
to_string(bestChunkSize / (1024 * 1024)) + " MB)" +
|
|
||||||
" - Drive: " + drive->getSerial());
|
|
||||||
bestThroughputMBps = throughputMBps;
|
|
||||||
}
|
|
||||||
throughputIncreasing = false;
|
|
||||||
}
|
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
throughputIncreasing = false;
|
throughputIncreasing = false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Adjust chunk size for next measurement interval (skip during warm-up)
|
// Adjust chunk size for next measurement interval
|
||||||
if (warmupCounter > WARMUP_MEASUREMENTS)
|
|
||||||
{
|
|
||||||
adjustChunkSize(drive);
|
adjustChunkSize(drive);
|
||||||
}
|
|
||||||
|
|
||||||
// Start new measurement
|
// Start new measurement
|
||||||
startMeasurement();
|
startMeasurement();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* \brief Determine if we should explore (epsilon-greedy + periodic re-exploration)
|
* \brief Adjust chunk size based on throughput trend
|
||||||
* \return true if should explore, false if should exploit
|
|
||||||
*/
|
|
||||||
bool Shred::shouldExplore()
|
|
||||||
{
|
|
||||||
// Periodic re-exploration: every REEXPLORATION_INTERVAL chunks
|
|
||||||
if (totalChunkCounter > 0 && (totalChunkCounter % REEXPLORATION_INTERVAL) == 0)
|
|
||||||
{
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Epsilon-greedy: random exploration with probability EXPLORATION_EPSILON
|
|
||||||
double randomValue = static_cast<double>(rand()) / RAND_MAX;
|
|
||||||
return (randomValue < EXPLORATION_EPSILON);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* \brief Perform exploration - try a random chunk size
|
|
||||||
* \param pointer to Drive instance
|
|
||||||
* \return void
|
|
||||||
*/
|
|
||||||
void Shred::performExploration(Drive *drive)
|
|
||||||
{
|
|
||||||
size_t savedChunkSize = currentChunkSize;
|
|
||||||
|
|
||||||
// Generate random chunk size between MIN and MAX (aligned to STEP boundaries)
|
|
||||||
// Calculate in MB to avoid overflow
|
|
||||||
size_t minMB = CHUNK_SIZE_MIN / (1024 * 1024);
|
|
||||||
size_t maxMB = CHUNK_SIZE_MAX / (1024 * 1024);
|
|
||||||
size_t stepMB = CHUNK_SIZE_STEP_UP / (1024 * 1024);
|
|
||||||
|
|
||||||
// Number of possible steps: (max - min) / step
|
|
||||||
size_t numSteps = (maxMB - minMB) / stepMB;
|
|
||||||
|
|
||||||
// Generate random step: 0 to numSteps (inclusive)
|
|
||||||
// Using proper modulo to ensure range [0, numSteps]
|
|
||||||
int randVal = rand();
|
|
||||||
size_t randomStep = static_cast<size_t>(randVal) % (numSteps + 1);
|
|
||||||
|
|
||||||
// Calculate exploration chunk size in MB, then convert to bytes
|
|
||||||
size_t explorationMB = minMB + (randomStep * stepMB);
|
|
||||||
explorationChunkSize = explorationMB * 1024 * 1024;
|
|
||||||
|
|
||||||
// Clamp to valid range (safety check)
|
|
||||||
if (explorationChunkSize < CHUNK_SIZE_MIN)
|
|
||||||
explorationChunkSize = CHUNK_SIZE_MIN;
|
|
||||||
if (explorationChunkSize > CHUNK_SIZE_MAX)
|
|
||||||
explorationChunkSize = CHUNK_SIZE_MAX;
|
|
||||||
|
|
||||||
// Enter exploration mode
|
|
||||||
explorationMode = true;
|
|
||||||
currentChunkSize = explorationChunkSize;
|
|
||||||
|
|
||||||
// Enhanced logging with debug info
|
|
||||||
Logger::logThis()->info("EXPLORATION MODE: Testing " +
|
|
||||||
to_string(explorationChunkSize / (1024 * 1024)) + " MB chunks " +
|
|
||||||
"(randomStep=" + to_string(randomStep) + "/" + to_string(numSteps) +
|
|
||||||
", rand=" + to_string(randVal) + ", " +
|
|
||||||
"was " + to_string(savedChunkSize / (1024 * 1024)) + " MB, " +
|
|
||||||
"best: " + to_string(bestChunkSize / (1024 * 1024)) + " MB)" +
|
|
||||||
" - Drive: " + drive->getSerial());
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* \brief Adjust chunk size based on throughput trend (Multi-Armed Bandit)
|
|
||||||
* \param pointer to Drive instance
|
* \param pointer to Drive instance
|
||||||
* \return void
|
* \return void
|
||||||
*/
|
*/
|
||||||
@@ -242,40 +133,9 @@ void Shred::adjustChunkSize(Drive *drive)
|
|||||||
{
|
{
|
||||||
size_t oldChunkSize = currentChunkSize;
|
size_t oldChunkSize = currentChunkSize;
|
||||||
|
|
||||||
// Check if we should explore instead of exploit
|
|
||||||
if (shouldExplore())
|
|
||||||
{
|
|
||||||
performExploration(drive);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Exit exploration mode if we were in it
|
|
||||||
if (explorationMode)
|
|
||||||
{
|
|
||||||
explorationMode = false;
|
|
||||||
|
|
||||||
// CRITICAL: Return to best known chunk size, not current
|
|
||||||
if (currentChunkSize != bestChunkSize)
|
|
||||||
{
|
|
||||||
currentChunkSize = bestChunkSize;
|
|
||||||
Logger::logThis()->info("EXPLORATION ENDED - Returning to best known: " +
|
|
||||||
to_string(bestChunkSize / (1024 * 1024)) + " MB" +
|
|
||||||
" (exploration tested " + to_string(oldChunkSize / (1024 * 1024)) + " MB)" +
|
|
||||||
" - Drive: " + drive->getSerial());
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
Logger::logThis()->info("EXPLORATION ENDED - Staying at current best: " +
|
|
||||||
to_string(bestChunkSize / (1024 * 1024)) + " MB" +
|
|
||||||
" - Drive: " + drive->getSerial());
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Normal exploitation mode: hill-climbing with symmetric steps
|
|
||||||
if (throughputIncreasing)
|
if (throughputIncreasing)
|
||||||
{
|
{
|
||||||
// Throughput is improving - increase chunk size (symmetric step)
|
// Throughput is improving - increase chunk size
|
||||||
currentChunkSize += CHUNK_SIZE_STEP_UP;
|
currentChunkSize += CHUNK_SIZE_STEP_UP;
|
||||||
|
|
||||||
// Clamp to maximum
|
// Clamp to maximum
|
||||||
@@ -289,7 +149,7 @@ void Shred::adjustChunkSize(Drive *drive)
|
|||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
// Throughput decreased - decrease chunk size (symmetric step)
|
// Throughput decreased - decrease chunk size to find sweet spot
|
||||||
if (currentChunkSize > CHUNK_SIZE_STEP_DOWN)
|
if (currentChunkSize > CHUNK_SIZE_STEP_DOWN)
|
||||||
{
|
{
|
||||||
currentChunkSize -= CHUNK_SIZE_STEP_DOWN;
|
currentChunkSize -= CHUNK_SIZE_STEP_DOWN;
|
||||||
@@ -301,7 +161,6 @@ void Shred::adjustChunkSize(Drive *drive)
|
|||||||
currentChunkSize = CHUNK_SIZE_MIN;
|
currentChunkSize = CHUNK_SIZE_MIN;
|
||||||
Logger::logThis()->info("Reached minimum chunk size: " +
|
Logger::logThis()->info("Reached minimum chunk size: " +
|
||||||
to_string(currentChunkSize / (1024 * 1024)) + " MB" +
|
to_string(currentChunkSize / (1024 * 1024)) + " MB" +
|
||||||
" (best remains: " + to_string(bestChunkSize / (1024 * 1024)) + " MB)" +
|
|
||||||
" - Drive: " + drive->getSerial());
|
" - Drive: " + drive->getSerial());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -311,7 +170,6 @@ void Shred::adjustChunkSize(Drive *drive)
|
|||||||
Logger::logThis()->info("Adjusted chunk size: " +
|
Logger::logThis()->info("Adjusted chunk size: " +
|
||||||
to_string(oldChunkSize / (1024 * 1024)) + " MB -> " +
|
to_string(oldChunkSize / (1024 * 1024)) + " MB -> " +
|
||||||
to_string(currentChunkSize / (1024 * 1024)) + " MB" +
|
to_string(currentChunkSize / (1024 * 1024)) + " MB" +
|
||||||
" (best: " + to_string(bestChunkSize / (1024 * 1024)) + " MB)" +
|
|
||||||
" - Drive: " + drive->getSerial());
|
" - Drive: " + drive->getSerial());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -324,6 +182,7 @@ size_t Shred::getCurrentChunkSize() const
|
|||||||
{
|
{
|
||||||
return currentChunkSize;
|
return currentChunkSize;
|
||||||
}
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* \brief shred drive with shred
|
* \brief shred drive with shred
|
||||||
@@ -370,16 +229,17 @@ int Shred::shredDrive(Drive *drive, int *ipSignalFd)
|
|||||||
#endif
|
#endif
|
||||||
|
|
||||||
#ifndef DRYRUN
|
#ifndef DRYRUN
|
||||||
string sDrivePath = drive->getPath();
|
const char* cpDrivePath = drive->getPath().c_str();
|
||||||
const char *cpDrivePath = sDrivePath.c_str();
|
|
||||||
unsigned char ucKey[TFNG_KEY_SIZE];
|
unsigned char ucKey[TFNG_KEY_SIZE];
|
||||||
|
|
||||||
|
#ifdef ADAPTIVE_CHUNK_SIZE
|
||||||
// Validate buffers were allocated
|
// Validate buffers were allocated
|
||||||
if (caTfngData == nullptr || caReadBuffer == nullptr)
|
if (caTfngData == nullptr || caReadBuffer == nullptr)
|
||||||
{
|
{
|
||||||
Logger::logThis()->error("Shred-Task: Aligned buffers not allocated! - Drive: " + drive->getSerial());
|
Logger::logThis()->error("Shred-Task: Aligned buffers not allocated! - Drive: " + drive->getSerial());
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
// Open random source
|
// Open random source
|
||||||
Logger::logThis()->info("Shred-Task: Opening random source: " + string(randomsrc) + " - Drive: " + drive->getSerial());
|
Logger::logThis()->info("Shred-Task: Opening random source: " + string(randomsrc) + " - Drive: " + drive->getSerial());
|
||||||
@@ -495,8 +355,10 @@ int Shred::shredDrive(Drive *drive, int *ipSignalFd)
|
|||||||
Logger::logThis()->info("Shred-Task: Bytes-Size of Drive: " + to_string(this->ulDriveByteSize) + " - Drive: " + drive->getSerial());
|
Logger::logThis()->info("Shred-Task: Bytes-Size of Drive: " + to_string(this->ulDriveByteSize) + " - Drive: " + drive->getSerial());
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#ifdef ADAPTIVE_CHUNK_SIZE
|
||||||
// Start first measurement interval
|
// Start first measurement interval
|
||||||
startMeasurement();
|
startMeasurement();
|
||||||
|
#endif
|
||||||
// Main shredding loop
|
// Main shredding loop
|
||||||
for (unsigned int uiShredIterationCounter = 0U; uiShredIterationCounter < SHRED_ITERATIONS; uiShredIterationCounter++)
|
for (unsigned int uiShredIterationCounter = 0U; uiShredIterationCounter < SHRED_ITERATIONS; uiShredIterationCounter++)
|
||||||
{
|
{
|
||||||
@@ -507,18 +369,30 @@ int Shred::shredDrive(Drive *drive, int *ipSignalFd)
|
|||||||
if (uiShredIterationCounter == (SHRED_ITERATIONS - 1))
|
if (uiShredIterationCounter == (SHRED_ITERATIONS - 1))
|
||||||
{
|
{
|
||||||
// last shred iteration --> overwrite (just the write chunk) bytes with zeros instead with random data
|
// last shred iteration --> overwrite (just the write chunk) bytes with zeros instead with random data
|
||||||
|
#ifdef ADAPTIVE_CHUNK_SIZE
|
||||||
memset(caTfngData, 0U, CHUNK_SIZE_MAX);
|
memset(caTfngData, 0U, CHUNK_SIZE_MAX);
|
||||||
|
#else
|
||||||
|
memset(caTfngData, 0U, CHUNK_SIZE);
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
while (ulDriveByteCounter < ulDriveByteSize)
|
while (ulDriveByteCounter < ulDriveByteSize)
|
||||||
{
|
{
|
||||||
|
#ifdef ADAPTIVE_CHUNK_SIZE
|
||||||
size_t activeChunkSize = getCurrentChunkSize();
|
size_t activeChunkSize = getCurrentChunkSize();
|
||||||
|
#else
|
||||||
|
size_t activeChunkSize = CHUNK_SIZE;
|
||||||
|
#endif
|
||||||
|
|
||||||
int iBytesToShred = 0;
|
int iBytesToShred = 0;
|
||||||
|
|
||||||
if (uiShredIterationCounter != (SHRED_ITERATIONS - 1))
|
if (uiShredIterationCounter != (SHRED_ITERATIONS - 1))
|
||||||
{
|
{
|
||||||
|
#ifdef ADAPTIVE_CHUNK_SIZE
|
||||||
tfng_prng_genrandom(caTfngData, activeChunkSize);
|
tfng_prng_genrandom(caTfngData, activeChunkSize);
|
||||||
|
#else
|
||||||
|
tfng_prng_genrandom(caTfngData, TFNG_DATA_SIZE);
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
if ((ulDriveByteSize - ulDriveByteCounter) < activeChunkSize)
|
if ((ulDriveByteSize - ulDriveByteCounter) < activeChunkSize)
|
||||||
@@ -558,15 +432,16 @@ int Shred::shredDrive(Drive *drive, int *ipSignalFd)
|
|||||||
ulDriveByteCounter += iByteShredded;
|
ulDriveByteCounter += iByteShredded;
|
||||||
ulDriveByteOverallCount += iByteShredded;
|
ulDriveByteOverallCount += iByteShredded;
|
||||||
|
|
||||||
|
#ifdef ADAPTIVE_CHUNK_SIZE
|
||||||
bytesWrittenInMeasurement += iByteShredded;
|
bytesWrittenInMeasurement += iByteShredded;
|
||||||
chunkCounter++;
|
chunkCounter++;
|
||||||
totalChunkCounter++; // Track total chunks for periodic re-exploration
|
|
||||||
|
|
||||||
// Evaluate throughput after measurement interval
|
// Evaluate throughput after measurement interval
|
||||||
if (chunkCounter >= CHUNK_MEASURE_INTERVAL)
|
if (chunkCounter >= CHUNK_MEASURE_INTERVAL)
|
||||||
{
|
{
|
||||||
evaluateThroughput(drive);
|
evaluateThroughput(drive);
|
||||||
}
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
d32Percent = this->calcProgress();
|
d32Percent = this->calcProgress();
|
||||||
|
|
||||||
@@ -579,10 +454,8 @@ int Shred::shredDrive(Drive *drive, int *ipSignalFd)
|
|||||||
|
|
||||||
if ((d32Percent - d32TmpPercent) >= 0.01)
|
if ((d32Percent - d32TmpPercent) >= 0.01)
|
||||||
{
|
{
|
||||||
// set shred percentage
|
|
||||||
drive->setTaskPercentage(d32TmpPercent);
|
drive->setTaskPercentage(d32TmpPercent);
|
||||||
d32TmpPercent = d32Percent;
|
d32TmpPercent = d32Percent;
|
||||||
// signal process in shredding
|
|
||||||
write(*ipSignalFd, "A", 1);
|
write(*ipSignalFd, "A", 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -617,10 +490,12 @@ int Shred::shredDrive(Drive *drive, int *ipSignalFd)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#ifdef ADAPTIVE_CHUNK_SIZE
|
||||||
Logger::logThis()->info("Shred completed - Optimal chunk size: " +
|
Logger::logThis()->info("Shred completed - Optimal chunk size: " +
|
||||||
to_string(bestChunkSize / (1024 * 1024)) + " MB, " +
|
to_string(bestChunkSize / (1024 * 1024)) + " MB, " +
|
||||||
"Best throughput: " + to_string((int)bestThroughputMBps) + " MB/s" +
|
"Best throughput: " + to_string((int)bestThroughputMBps) + " MB/s" +
|
||||||
" - Drive: " + drive->getSerial());
|
" - Drive: " + drive->getSerial());
|
||||||
|
#endif
|
||||||
|
|
||||||
// All shred iterations completed successfully
|
// All shred iterations completed successfully
|
||||||
tfng_prng_seedkey(NULL);
|
tfng_prng_seedkey(NULL);
|
||||||
@@ -687,7 +562,7 @@ double Shred::calcProgress()
|
|||||||
unsigned int uiMaxShredIteration = SHRED_ITERATIONS;
|
unsigned int uiMaxShredIteration = SHRED_ITERATIONS;
|
||||||
|
|
||||||
#ifdef ZERO_CHECK
|
#ifdef ZERO_CHECK
|
||||||
uiMaxShredIteration++; // increment because we will check after SHRED_ITERATIONS the drive for non-zero bytes
|
uiMaxShredIteration++;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
if (this->ulDriveByteSize == 0)
|
if (this->ulDriveByteSize == 0)
|
||||||
@@ -765,9 +640,11 @@ unsigned int Shred::uiCalcChecksum(fileDescriptor file, Drive *drive, int *ipSig
|
|||||||
unsigned int uiChecksum = 0;
|
unsigned int uiChecksum = 0;
|
||||||
unsigned long ulDriveByteCounter = 0U;
|
unsigned long ulDriveByteCounter = 0U;
|
||||||
|
|
||||||
Logger::logThis()->info("Check-Task: Starting checksum verification - Drive: " + drive->getSerial());
|
#ifdef ADAPTIVE_CHUNK_SIZE
|
||||||
|
|
||||||
size_t checkChunkSize = CHUNK_SIZE_MAX;
|
size_t checkChunkSize = CHUNK_SIZE_MAX;
|
||||||
|
#else
|
||||||
|
size_t checkChunkSize = CHUNK_SIZE;
|
||||||
|
#endif
|
||||||
|
|
||||||
while (ulDriveByteCounter < ulDriveByteSize)
|
while (ulDriveByteCounter < ulDriveByteSize)
|
||||||
{
|
{
|
||||||
|
|||||||
Reference in New Issue
Block a user