script/nextcloud
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

delete_files.sh: neues, generisches Script zum gezielten Löschen von Dateien

Browse Source
This commit is contained in:
chris
2026-09-15 00:26:19 +02:00
parent 93e0576777
commit a8887aeeea
2 changed files with 1122 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
delete_files.sh
Executable
+991
View File
@@ -0,0 +1,991 @@
#!/usr/bin/env bash
CUR_IFS=$IFS
script_name="$(basename $(realpath $0))"
script_dir="$(dirname $(realpath $0))"
conf_dir="${script_dir}/conf"
snippet_dir="${script_dir}/snippets"
report_dir="${script_dir}/reports"
# - Before this script deletes a file from Nextcloud's live storage, it
# - keeps a raw, byte-for-byte copy of the CURRENT on-disk ciphertext
# - (the file exactly as it is right now) in a separate directory - a
# - plain filesystem copy, independent of Nextcloud/its Trash entirely.
# - This is an extra safety net on top of (not a replacement for)
# - Nextcloud's own Trash, which is where a normal delete through the
# - Files API already sends the file if the 'files_trashbin' app is
# - enabled.
# -
DEFAULT_DELETE_BACKUP_BASE_DIR="/var/nc-delete-backup"
declare -a unsorted_website_arr
declare -a website_arr
declare -a unsorted_account_arr
declare -a account_arr
LOCK_DIR="/tmp/${script_name%%.*}.LOCK"
log_file="${LOCK_DIR}/${script_name%%.*}.log"
run_date=$(date +%Y-%m-%d-%H%M)
# =============
# --- Some functions
# =============
usage() {
[[ -n "$1" ]] && error "$1"
[[ $terminal ]] && echo -e "
\033[1mUsage:\033[m
$(basename $0) -s <website> -f <pathlist-file>
\033[1mDescription\033[m
Generic, reusable companion to restore_bad_signature.sh: deletes an
explicit, hand-picked list of files from a Nextcloud instance's live
storage, through Nextcloud's normal Files API - the same delete path
the web interface, WebDAV, or a sync client uses. If the
'files_trashbin' app is enabled (the default), the file ends up in
that account's Trash, not gone outright.
This is a plain, general-purpose tool - NOT specific to any one
account, site, or incident. It is meant for the second half of the
'bad signature' recovery workflow: after recover_bad_signature.sh
and restore_bad_signature.sh have taken care of every file that
validates cleanly or at least plausibly, a handful of files
sometimes remain that a human should look at and decide on one by
one (system/index files that don't belong in the cloud at all,
fragments from a browser 'save page' action, or anything else you
simply don't want kept). Once you've decided, list their paths in a
plain text file and point this script at it.
IMPORTANT:
- The path list file (-f) is plain text, one Nextcloud path per
line, in the SAME format used throughout this toolkit's own
reports (the 'path' column, e.g.
'/inge/files/Some/Folder/file.ext' - '/<uid>/files/...'). Blank
lines and lines starting with '#' are ignored. The account
(uid) is taken from each path's own first segment, so one list
can freely mix files from several accounts on the same site -
no separate account selection is needed.
- Nothing is guessed or auto-selected: every path this script
acts on is one YOU explicitly listed. There is no scanning, no
heuristic, no 'delete everything that looks like junk' mode
here - that judgment call happens before this script runs, not
inside it.
- Before each file is deleted (unless dry-run), the CURRENT
on-disk ciphertext is copied byte-for-byte to a separate backup
directory (default ${DEFAULT_DELETE_BACKUP_BASE_DIR}/<website>,
override with DELETE_BACKUP_BASE_DIR in the conf file) - a
plain filesystem copy that bypasses Nextcloud/encryption and
Trash entirely. This only works for local/default storage; it
is best-effort and a missing source is reported but does not
abort the run.
- The delete itself goes through Nextcloud's normal Files API
(\$node->delete()), so all of Nextcloud's own rules still
apply: if 'files_trashbin' is enabled the file is moved to that
account's Trash (recoverable there, subject to the instance's
own Trash retention settings); if it is disabled, the delete is
immediate and permanent.
- A path already absent (no matching file found) is reported as
NOT_FOUND, not as an error - nothing to do there.
Unless '-n' is given on the command line, you will be asked
interactively whether to run a dry-run (nothing is touched, just
reports what would happen) or the real thing.
\033[1mOptions\033[m
-s <website>
The site of the nextcloud instance.
-f <pathlist-file>
Plain text file with one Nextcloud path to delete per line (see
above). Required - if omitted, you will be asked for it.
-n
Dry-run: goes through selection and reporting, but does NOT back
up or delete anything. Passing it here skips the interactive mode
question mentioned above.
\033[1mExample:\033[m
Runs this script on system 'cloud-01.oopen.de'
$(basename $0) -s cloud-01.oopen.de -f /root/to_delete.txt
Dry-run only, nothing is deleted:
$(basename $0) -n -s cloud-01.oopen.de -f /root/to_delete.txt
"
clean_up 1
}
clean_up() {
# Perform program exit housekeeping
[[ -n "$delete_php_file" ]] && rm -f "$delete_php_file" 2> /dev/null
rm -rf "$LOCK_DIR"
blank_line
exit $1
}
echononl(){
if $terminal ; then
echo X\\c > /tmp/shprompt$$
if [ `wc -c /tmp/shprompt$$ | awk '{print $1}'` -eq 1 ]; then
echo -e -n "$*\\c" 1>&2
else
echo -e -n "$*" 1>&2
fi
rm /tmp/shprompt$$
fi
}
echo_done() {
if $terminal ; then
echo -e "\033[75G[ \033[32mdone\033[m ]"
fi
}
echo_ok() {
if $terminal ; then
echo -e "\033[75G[ \033[32mok\033[m ]"
fi
}
echo_warning() {
if $terminal ; then
echo -e "\033[75G[ \033[33m\033[1mwarn\033[m ]"
fi
}
echo_failed(){
if $terminal ; then
echo -e "\033[75G[ \033[1;31mfailed\033[m ]"
fi
}
echo_skipped() {
if $terminal ; then
echo -e "\033[75G[ \033[37mskipped\033[m ]"
fi
}
fatal (){
echo ""
echo ""
if $terminal ; then
echo -e " [ \033[31m\033[1mFatal\033[m ]: \033[37m\033[1m$*\033[m"
echo ""
echo -e " \033[31m\033[1mScript will be interrupted..\033[m!"
else
echo " [ Fatal ]: $*"
echo ""
echo " Script was terminated...."
fi
clean_up 1
}
error(){
echo ""
if $terminal ; then
echo -e " [ \033[31m\033[1mError\033[m ]: $*"
else
echo " [ Error ]: $*"
fi
echo ""
}
warn (){
if $terminal ; then
echo ""
echo -e " [ \033[33m\033[1mWarning\033[m ]: $*"
echo ""
fi
}
info (){
if $terminal ; then
echo ""
echo -e " [ \033[32m\033[1mInfo\033[m ]: $*"
echo ""
fi
}
# - Remove leading/trailling whitespaces
# -
trim() {
local var="$*"
var="${var#"${var%%[![:space:]]*}"}" # remove leading whitespace characters
var="${var%"${var##*[![:space:]]}"}" # remove trailing whitespace characters
echo -n "$var"
}
## - Percentage (1 decimal) of parameter 1 (part) against parameter 2 (total)
## - Returns "0.0" if total is empty, zero or non-numeric.
## -
calc_percent() {
local _part="$1"
local _total="$2"
if [[ -z "$_total" ]] || ! [[ "$_total" =~ ^[0-9]+$ ]] || [[ "$_total" -eq 0 ]] ; then
echo "0.0"
else
awk -v b="$_part" -v t="$_total" 'BEGIN { printf "%.1f", (b/t)*100 }'
fi
}
blank_line() {
if $terminal ; then
echo ""
fi
}
# - Running in a terminal?
# -
if [[ -t 1 ]] ; then
terminal=true
else
terminal=false
fi
# - This script needs root privileges (reads the Nextcloud data
# - directory for the raw ciphertext backup, 'su' into the webserver
# - user to perform the actual delete).
# -
if [[ "$(id -u)" -ne 0 ]] ; then
fatal "This script must be run as root (it needs to read the Nextcloud data directory for backups and 'su' into the webserver user). Please re-run as root, e.g. via sudo."
fi
# ----------
# - Jobhandling
# ----------
if pgrep -f "$(basename $0)" | grep -q -v $$ ; then
msg="A previos instance of script \"`basename $0`\" seems already be running."
echo ""
if $terminal ; then
echo -e "[ \033[31m\033[1mFatal\033[m ]: $msg"
echo ""
echo -e " \033[31m\033[1mScript was interupted\033[m!"
else
echo " [ Fatal ]: $msg"
echo ""
echo " Script was interupted!"
fi
echo
exit 1
else
if [[ -d "$LOCK_DIR" ]] ; then
rm -rf "$LOCK_DIR" 2> /dev/null
fi
fi
# - If job already runs, stop execution..
# -
if mkdir "$LOCK_DIR" 2> /dev/null ; then
# - Remove lockdir when the script finishes, or when it receives a signal
# -
trap clean_up SIGHUP SIGINT SIGTERM
else
msg="A previos instance of script \"`basename $0`\" seems already be running."
echo ""
if $terminal ; then
echo -e "[ \033[31m\033[1mFatal\033[m ]: $msg"
echo ""
echo -e " \033[31m\033[1mScript was interupted\033[m!"
else
echo " [ Fatal ]: $msg"
echo ""
echo " Script was interupted!"
fi
echo
exit 1
fi
# -------------
# - Read in Commandline arguments
# -------------
DRY_RUN=false
_dry_run_explicit=false
PATHLIST_FILE=""
while getopts hns:f: opt ; do
case $opt in
h) usage ;;
n) DRY_RUN=true; _dry_run_explicit=true ;;
s) WEBSITE=$OPTARG ;;
f) PATHLIST_FILE=$OPTARG ;;
\?) usage
esac
done
# =============
# --- Ask which mode to run in, unless '-n' was already given on the
# --- command line. The SAFE choice (dry-run) is the default here on
# --- purpose - this script deletes from live Nextcloud storage.
# =============
if ! $_dry_run_explicit && $terminal ; then
blank_line
echo -e "\033[37m\033[1mWhich mode should this run use?\033[m"
echo ""
echo -e " \033[1m[1] Dry-run\033[m - go through the list and report what would happen, but touch NOTHING"
echo ""
echo " [2] Real delete - actually back up and delete each listed file in Nextcloud"
info "Just press Return to use the default: [1] Dry-run."
echo -n " Select mode by number [1]: "
read _mode_choice
case "$(trim "$_mode_choice")" in
2) DRY_RUN=false ;;
""|1) DRY_RUN=true ;;
*) fatal "Invalid selection '$_mode_choice'." ;;
esac
fi
if [[ -z "$PATHLIST_FILE" ]] ; then
blank_line
echo -n " Path to the file listing the Nextcloud paths to delete (-f): "
read PATHLIST_FILE
fi
PATHLIST_FILE="$(trim "$PATHLIST_FILE")"
if [[ -z "$PATHLIST_FILE" ]] ; then
fatal "No path list file given."
fi
if [[ ! -f "$PATHLIST_FILE" ]] ; then
fatal "Path list file '$PATHLIST_FILE' not found."
fi
if [[ -z "$WEBSITE" ]] ; then
while IFS='' read -r -d '' _conf_file ; do
source $_conf_file
if [[ -n "$WEBSITE" ]] ; then
unsorted_website_arr+=("${WEBSITE}:$_conf_file")
fi
WEBSITE=""
done < <(find "${conf_dir}" -maxdepth 1 -type f -name "*.conf" -print0)
# - Sort array
# -
IFS=$'\n' website_arr=($(sort <<<"${unsorted_website_arr[*]}"))
# Which cloud instance (website) would you like to update
#
source ${snippet_dir}/get-cloud-instance-to-update.sh
else
while IFS='' read -r -d '' _conf_file ; do
if $(grep -E -q "WEBSITE=\"?${WEBSITE}\"?" ${_conf_file} 2> /dev/null) ; then
conf_file="${_conf_file}"
break
fi
done < <(find "${conf_dir}" -maxdepth 1 -type f -name "*.conf" -print0)
fi
# - Reset IFS
# -
IFS=$CUR_IFS
DEFAULT_SRC_BASE_DIR="/usr/local/src/nextcloud"
DEFAULT_HTTP_USER="www-data"
DEFAULT_HTTP_GROUP="www-data"
DEFAULT_PHP_ENGINE='FPM'
blank_line
echononl " Include Configuration file '$(basename "${conf_file}")'.."
if [[ ! -f $conf_file ]]; then
echo_skipped
fatal "Missing configuration file '$conf_file'."
else
source $conf_file
echo_ok
fi
DEFAULT_WEB_BASE_DIR="/var/www/${WEBSITE}"
[[ -n "$WEB_BASE_DIR" ]] || WEB_BASE_DIR=$DEFAULT_WEB_BASE_DIR
if [[ ! -d ${WEB_BASE_DIR} ]] ; then
fatal "Web base directory '$WEB_BASE_DIR' not found!"
fi
DATA_DIR="$(realpath ${WEB_BASE_DIR}/data)"
[[ -n "$PHP_ENGINE" ]] || PHP_ENGINE=$DEFAULT_PHP_ENGINE
INSTALL_DIR="$(realpath ${WEB_BASE_DIR}/nextcloud)"
CURRENT_VERSION="$(basename $INSTALL_DIR | cut -d"-" -f2)"
[[ -n "$DELETE_BACKUP_BASE_DIR" ]] || DELETE_BACKUP_BASE_DIR=$DEFAULT_DELETE_BACKUP_BASE_DIR
backup_dir="${DELETE_BACKUP_BASE_DIR}/${WEBSITE}"
# =============
# --- Some
# =============
# - Support systemd ?
# -
SYSTEMD_EXISTS=false
systemd=$(which systemd)
systemctl=$(which systemctl)
if [[ -n "$systemd" ]] || [[ -n "$systemctl" ]] ; then
SYSTEMD_EXISTS=true
fi
if $terminal ; then
echo ""
echo -e "\033[32m-----\033[m"
echo -e "Delete an explicit, hand-picked list of files from system \033[1m${WEB_BASE_DIR}\033[m"
echo -e "\033[1m
This deletes from Nextcloud's live storage (goes to that account's
Trash if 'files_trashbin' is enabled). Only the exact paths listed
in '${PATHLIST_FILE}' are ever touched - nothing is scanned,
guessed or auto-selected. The current on-disk ciphertext is backed
up first, independent of Nextcloud/Trash entirely.\033[m"
echo -e "\033[32m-----\033[m"
fi
# =============
# --- Some checks
# =============
DEFAULT_HTTP_USER="www-data"
DEFAULT_HTTP_GROUP="www-data"
NGINX_IS_ENABLED=false
APACHE2_IS_ENABLED=false
# Get Webservice environment as IS_HTTPD_RUNNING, HTTP_USER, HTTP_GROUP..
#
source ${snippet_dir}/get-webservice-environment.sh
# Check PHP Version
#
source ${snippet_dir}/get-php-major-version.sh
# Get full qualified PHP command
#
source ${snippet_dir}/get-path-of-php-command.sh
if [[ ! -x "$PHP_BIN" ]]; then
fatal "No PHP binary found!"
fi
# =============
# --- Read and parse the path list file: one Nextcloud path per line,
# --- blank lines and '#' comments ignored. The account (uid) for each
# --- path is taken from the path's own first segment
# --- ('/<uid>/files/...'), so one list can span several accounts.
# =============
parsed_entries_file="${LOCK_DIR}/parsed_entries.tsv"
declare -i total_malformed=0
declare -a malformed_lines=()
> "$parsed_entries_file"
while IFS= read -r _line || [[ -n "$_line" ]] ; do
_line="$(trim "$_line")"
[[ -z "$_line" ]] && continue
[[ "$_line" == \#* ]] && continue
if [[ "$_line" =~ ^/([^/]+)/files/ ]] ; then
_uid="${BASH_REMATCH[1]}"
printf '%s\t%s\n' "$_uid" "$_line" >> "$parsed_entries_file"
else
(( total_malformed++ ))
malformed_lines+=("$_line")
fi
done < "$PATHLIST_FILE"
if [[ ! -s "$parsed_entries_file" ]] ; then
fatal "No usable paths found in '$PATHLIST_FILE' (expected one '/<uid>/files/...' path per line)."
fi
unsorted_account_arr=($(cut -f1 "$parsed_entries_file" | sort -u))
IFS=$'\n' account_arr=($(sort <<<"${unsorted_account_arr[*]}"))
IFS=$CUR_IFS
blank_line
if $terminal ; then
echo -e "\033[37m\033[1mAccounts found in '$(basename "$PATHLIST_FILE")'\033[m"
echo ""
for _a in "${account_arr[@]}" ; do
_a_count=$(awk -F'\t' -v u="$_a" '$1==u' "$parsed_entries_file" | wc -l)
echo " - $_a (${_a_count})"
done
echo ""
if [[ $total_malformed -gt 0 ]] ; then
warn "${total_malformed} line(s) in '$PATHLIST_FILE' did not look like a '/<uid>/files/...' path and were IGNORED:"
for _l in "${malformed_lines[@]}" ; do
echo " $_l"
done
fi
fi
# =============
# --- Confirmation
# =============
mkdir -p "$report_dir" 2> /dev/null
delete_report_file="${report_dir}/delete_${WEBSITE}_${run_date}.tsv"
if $terminal ; then
echo ""
if $DRY_RUN ; then
echo -e "\033[1;32mStarting DRY-RUN delete for \033[1;37m${WEBSITE}\033[m"
else
echo -e "\033[1;31m\033[1mStarting REAL delete (removes from live Nextcloud storage) for \033[1;37m${WEBSITE}\033[m"
fi
echo ""
echo -e " Cloud instance..........................: $WEBSITE"
echo -e " Path list file...........................: $PATHLIST_FILE"
echo -e " Accounts affected........................: \033[33m${account_arr[*]}\033[m"
echo ""
if $DRY_RUN ; then
echo -e " Nothing will be touched - dry-run only."
else
echo -e " Ciphertext backup (before delete)........: $backup_dir"
echo -e " Files deleted at their ORIGINAL path in Nextcloud (-> Trash, if enabled)."
fi
echo -e " Delete report.............................: reports/$(basename "${delete_report_file}")"
echo ""
if $DRY_RUN ; then
info "Dry-run: nothing is backed up or deleted."
else
warn "This DELETES from Nextcloud's live storage (through the normal Files API - goes to Trash if 'files_trashbin' is enabled). Only the exact paths listed in '${PATHLIST_FILE}' are touched. The current on-disk ciphertext is copied byte-for-byte to '${backup_dir}' first, independent of Nextcloud/Trash entirely."
fi
echo ""
echo -n " Type upper case 'YES' to continue executing with this parameters: "
read OK
if [[ "$OK" = "YES" ]] ; then
echo ""
echo ""
echo -e "\033[1;32mGoing to delete the listed files for each affected account on \033[1;37m$WEBSITE \033[m"
else
fatal "Abort by user request - Answer as not 'YES'"
fi
fi
{
echo "=================================================================="
echo " Delete-Versuch (Liste) ${WEBSITE} ${run_date}"
echo "=================================================================="
echo ""
echo " Pfadliste: $(basename "$PATHLIST_FILE")"
echo " Nur die dort explizit aufgefuehrten Pfade werden geloescht."
$DRY_RUN && echo " DRY-RUN: es wurde NICHTS gesichert oder geloescht."
! $DRY_RUN && echo " Chiffretext-Sicherung (vor dem Loeschen) liegt unter: ${backup_dir}"
[[ $total_malformed -gt 0 ]] && echo " ${total_malformed} Zeile(n) aus der Pfadliste wurden ignoriert (kein '/<uid>/files/...'-Pfad)."
echo ""
echo -e "path\tstatus\tdetail"
} > "$delete_report_file"
# =============
# --- Write the embedded PHP delete script
# =============
delete_php_file="${INSTALL_DIR}/.delete_files_$$.php"
cat > "$delete_php_file" <<'PHP_DELETE_EOF'
<?php
/**
* delete_files.php
*
* Deletes an explicit list of files from Nextcloud's own storage, at
* their ORIGINAL path, through the normal Files API ($node->delete())
* - the same delete path the web interface, WebDAV, or a sync client
* uses. If the 'files_trashbin' app is enabled, Nextcloud moves the
* file to that account's Trash itself; this script does not bypass
* that.
*
* Usage: php delete_files.php <uid> <listFile> [--dry-run]
* listFile: one Nextcloud path per line (no header, no other columns)
*/
error_reporting(E_ALL);
ini_set('display_errors', '1');
register_shutdown_function(function () {
$err = error_get_last();
if ($err !== null && in_array($err['type'], [E_ERROR, E_PARSE, E_CORE_ERROR, E_COMPILE_ERROR], true)) {
fwrite(STDERR, "DEBUG: FATAL bei Shutdown: [{$err['type']}] {$err['message']} in {$err['file']}:{$err['line']}\n");
}
});
define('OC_CONSOLE', 1);
$oldWorkingDir = getcwd();
if ($oldWorkingDir === false) {
fwrite(STDERR, "Konnte aktuelles Arbeitsverzeichnis nicht ermitteln. Bitte mit absolutem Pfad aufrufen.\n");
exit(1);
}
chdir(__DIR__);
require_once __DIR__ . '/lib/base.php';
chdir($oldWorkingDir);
if (function_exists('posix_getuid') && posix_getuid() === 0) {
fwrite(STDERR, "Bitte NICHT als root ausfuehren, sondern als Webserver-User.\n");
exit(1);
}
try {
$uid = $argv[1] ?? null;
$listFile = $argv[2] ?? null;
$dryRun = in_array('--dry-run', $argv, true);
if ($uid === null || $listFile === null) {
fwrite(STDERR, "Usage: php delete_files.php <uid> <listFile> [--dry-run]\n");
exit(1);
}
if (!is_file($listFile)) {
fwrite(STDERR, "Liste nicht gefunden: $listFile\n");
exit(1);
}
$rootFolder = \OC::$server->get(\OCP\Files\IRootFolder::class);
\OC_Util::setupFS($uid);
$userFolder = $rootFolder->getUserFolder($uid);
$lines = file($listFile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
if ($lines === false) {
$lines = [];
}
$total = 0;
$deleted = 0;
$notFound = 0;
$failed = 0;
echo "path\tstatus\tdetail\n";
foreach ($lines as $path) {
$path = trim($path);
if ($path === '') {
continue;
}
$total++;
try {
$node = null;
try {
$node = $rootFolder->get($path);
} catch (\Throwable $e) {
$node = null;
}
if ($node === null) {
$notFound++;
echo "$path\tNOT_FOUND\talready absent - nothing to delete\n";
flush();
continue;
}
if ($dryRun) {
$deleted++;
$size = ($node instanceof \OCP\Files\File) ? $node->getSize() : 0;
echo "$path\tDRY_RUN\twould delete now (${size} bytes) - goes to Trash if 'files_trashbin' is enabled\n";
flush();
continue;
}
$node->delete();
// Verify: the normal read path should no longer find it
// (it may still exist in Trash - that is Nextcloud's own
// behaviour, not a failure of this delete).
clearstatcache();
$stillThere = null;
try {
$stillThere = $rootFolder->get($path);
} catch (\Throwable $e) {
$stillThere = null;
}
if ($stillThere === null) {
$deleted++;
echo "$path\tOK\tdeleted (moved to Trash, if 'files_trashbin' is enabled)\n";
} else {
$failed++;
echo "$path\tDELETE_ERROR\tdelete() did not throw, but the path is still resolvable afterwards\n";
}
flush();
} catch (\Throwable $e) {
$failed++;
$msg = str_replace(["\t", "\n", "\r"], ' ', $e->getMessage());
echo "$path\tDELETE_ERROR\t" . get_class($e) . ": $msg\n";
flush();
}
}
fwrite(STDERR, "\nFertig. Dateien verarbeitet: $total, geloescht: $deleted, nicht gefunden: $notFound, Fehler: $failed\n");
} catch (\Throwable $e) {
fwrite(STDERR, "\n!!! UNBEHANDELTE AUSNAHME !!!\n");
fwrite(STDERR, get_class($e) . ": " . $e->getMessage() . "\n");
fwrite(STDERR, "in " . $e->getFile() . ":" . $e->getLine() . "\n");
fwrite(STDERR, $e->getTraceAsString() . "\n");
exit(2);
}
PHP_DELETE_EOF
chmod 644 "$delete_php_file"
if ! $DRY_RUN ; then
mkdir -p "$backup_dir" 2> /dev/null
fi
declare -i total_selected=0
declare -i total_ok=0
declare -i total_not_found=0
declare -i total_errors=0
declare -i total_failed_users=0
for _user in "${account_arr[@]}" ; do
_sep_len=${#_user}
[[ $_sep_len -lt 9 ]] && _sep_len=9
_sep_line="$(printf '%*s' "$_sep_len" '' | tr ' ' '-')"
{
echo ""
echo ""
echo "$_sep_line"
echo "$_user"
echo "$_sep_line"
} >> "$delete_report_file"
declare -i _user_selected=0
list_file="${LOCK_DIR}/list_${_user}.tsv"
> "$list_file"
while IFS=$'\t' read -r _u _path ; do
[[ "$_u" != "$_user" ]] && continue
(( _user_selected++ ))
if ! $DRY_RUN ; then
_rel="${_path#/${_user}/files/}"
_orig_ciphertext="${DATA_DIR}/${_user}/files/${_rel}"
if [[ -f "$_orig_ciphertext" ]] ; then
_backup_target="${backup_dir}/${_user}/${_rel}"
mkdir -p "$(dirname "$_backup_target")" 2> /dev/null
cp -p "$_orig_ciphertext" "$_backup_target" 2> /dev/null
fi
fi
printf '%s\n' "$_path" >> "$list_file"
done < "$parsed_entries_file"
total_selected+=$_user_selected
user_result_tsv="${LOCK_DIR}/result_${_user}.tsv"
if $DRY_RUN ; then
echononl " Dry-run for account \033[1;37m${_user}\033[m (${_user_selected} to check).."
else
echononl " Deleting for account \033[1;37m${_user}\033[m (${_user_selected} to delete).."
fi
if [[ -s "$list_file" ]] ; then
if $DRY_RUN ; then
su -c "$PHP_BIN $delete_php_file $_user $list_file --dry-run" -s /bin/bash $HTTP_USER > "$user_result_tsv" 2> "$log_file"
else
su -c "$PHP_BIN $delete_php_file $_user $list_file" -s /bin/bash $HTTP_USER > "$user_result_tsv" 2> "$log_file"
fi
_rc=$?
else
echo -e "path\tstatus\tdetail" > "$user_result_tsv"
_rc=0
fi
if [[ $_rc -ne 0 ]]; then
echo_failed
error "$(cat "$log_file")"
echo " [ FEHLER beim Delete-Lauf - siehe Server-Log ]" >> "$delete_report_file"
(( total_failed_users++ ))
unset _user_selected
continue
fi
echo_done
$terminal && echo ""
declare -i _user_ok=0
declare -i _user_not_found=0
declare -i _user_errors=0
declare -a _user_error_lines=()
while IFS=$'\t' read -r _path _status _detail ; do
[[ "$_path" = "path" ]] && continue
[[ -z "$_path" ]] && continue
case "$_status" in
OK|DRY_RUN) (( _user_ok++ )) ;;
NOT_FOUND) (( _user_not_found++ )) ;;
*) (( _user_errors++ )); _user_error_lines+=("${_path}"$'\t'"${_status}: ${_detail}") ;;
esac
printf '%s\t%s\t%s\n' "$_path" "$_status" "$_detail" >> "$delete_report_file"
done < "$user_result_tsv"
_user_pct_ok="$(calc_percent "$_user_ok" "$_user_selected")"
if $terminal ; then
echo -e " \033[1;37mAccount ${_user}\033[m"
echo -e " Ausgewaehlt (aus der Pfadliste)............: ${_user_selected}"
if $DRY_RUN ; then
echo -e " Wuerde geloescht werden.....................: \033[1;32m${_user_ok} (${_user_pct_ok} %)\033[m"
else
if [[ $_user_ok -gt 0 ]] ; then
echo -e " Geloescht....................................: \033[1;32m${_user_ok} (${_user_pct_ok} %)\033[m"
else
echo -e " Geloescht....................................: ${_user_ok} (${_user_pct_ok} %)"
fi
fi
if [[ $_user_not_found -gt 0 ]] ; then
echo -e " Bereits nicht mehr vorhanden................: ${_user_not_found}"
fi
if [[ $_user_errors -gt 0 ]] ; then
echo -e " Fehler beim Loeschen.........................: \033[1;31m${_user_errors}\033[m"
fi
echo ""
fi
{
if [[ $_user_errors -gt 0 ]] ; then
echo ""
echo " ------------------------------------------------------------------"
echo " Fehler beim Loeschen - Account ${_user} (${_user_errors})"
echo " ------------------------------------------------------------------"
for _line in "${_user_error_lines[@]}" ; do
printf ' %s\n' "$_line"
done
fi
} >> "$delete_report_file"
{
echo ""
echo " Account ${_user}"
echo " Ausgewaehlt (aus der Pfadliste)............: ${_user_selected}"
if $DRY_RUN ; then
echo " Wuerde geloescht werden.....................: ${_user_ok} (${_user_pct_ok} %)"
else
echo " Geloescht....................................: ${_user_ok} (${_user_pct_ok} %)"
fi
[[ $_user_not_found -gt 0 ]] && echo " Bereits nicht mehr vorhanden................: ${_user_not_found}"
[[ $_user_errors -gt 0 ]] && echo " Fehler beim Loeschen.........................: ${_user_errors}"
} >> "$delete_report_file"
(( total_ok += _user_ok ))
(( total_not_found += _user_not_found ))
(( total_errors += _user_errors ))
unset _user_selected _user_ok _user_not_found _user_errors _user_error_lines
done
total_pct_ok="$(calc_percent "$total_ok" "$total_selected")"
{
echo ""
echo ""
echo "=================================================================="
echo " Gesamtergebnis"
echo "=================================================================="
echo ""
echo "Betroffene Accounts..........................: ${#account_arr[@]}"
[[ $total_failed_users -gt 0 ]] && echo "Accounts mit Delete-Fehler...................: ${total_failed_users}"
echo "Ausgewaehlt (aus der Pfadliste)..............: ${total_selected}"
if $DRY_RUN ; then
echo "Wuerde geloescht werden insgesamt............: ${total_ok} (${total_pct_ok} %)"
else
echo "Geloescht insgesamt...........................: ${total_ok} (${total_pct_ok} %)"
fi
[[ $total_not_found -gt 0 ]] && echo "Bereits nicht mehr vorhanden insgesamt.......: ${total_not_found}"
[[ $total_errors -gt 0 ]] && echo "Fehler beim Loeschen insgesamt................: ${total_errors}"
echo ""
if $DRY_RUN ; then
echo "DRY-RUN: es wurde nichts gesichert oder geloescht."
else
echo "Chiffretext-Sicherung (vor dem Loeschen) liegt unter: ${backup_dir}"
fi
} >> "$delete_report_file"
blank_line
if $terminal ; then
echo -e "\033[37m\033[1mErgebnis\033[m"
echo ""
echo -e " Betroffene Accounts..........................: ${#account_arr[@]}"
[[ $total_failed_users -gt 0 ]] && echo -e " Accounts mit Delete-Fehler...................: \033[1;31m${total_failed_users}\033[m"
echo -e " Ausgewaehlt (aus der Pfadliste)..............: ${total_selected}"
if $DRY_RUN ; then
echo -e " Wuerde geloescht werden insgesamt............: \033[1;32m${total_ok} (${total_pct_ok} %)\033[m"
else
if [[ $total_ok -gt 0 ]] ; then
echo -e " Geloescht insgesamt...........................: \033[1;32m${total_ok} (${total_pct_ok} %)\033[m"
else
echo -e " Geloescht insgesamt...........................: ${total_ok} (${total_pct_ok} %)"
fi
fi
[[ $total_not_found -gt 0 ]] && echo -e " Bereits nicht mehr vorhanden insgesamt.......: ${total_not_found}"
[[ $total_errors -gt 0 ]] && echo -e " Fehler beim Loeschen insgesamt................: \033[1;31m${total_errors}\033[m"
echo ""
echo -e " Delete-Report.................................: reports/$(basename "${delete_report_file}")"
if ! $DRY_RUN ; then
echo -e " Chiffretext-Sicherung.........................: $backup_dir"
fi
echo ""
if $DRY_RUN ; then
info "Dry-run beendet - es wurde nichts geloescht. Ohne '-n' (oder mit [2] bei der Modusabfrage) fuer den echten Delete erneut ausfuehren."
else
warn "Bitte pruefen, ob die geloeschten Dateien wie erwartet im Trash der jeweiligen Accounts liegen (sofern 'files_trashbin' aktiviert ist)."
fi
fi
clean_up 0
restore_bad_signature.sh
+130 -50
View File
@@ -59,15 +59,14 @@ usage() {
\033[1mDescription\033[m \033[1mDescription\033[m
Writes already-recovered, already-validated files (produced by a Writes already-recovered, already-validated files (produced by a
PRIOR run of recover_bad_signature.sh, validation column = VALID in PRIOR run of recover_bad_signature.sh) back into Nextcloud's own
its report) back into Nextcloud's own storage, at their ORIGINAL storage, at their ORIGINAL path - through Nextcloud's normal Files
path - through Nextcloud's normal Files API (the same write path a API (the same write path a regular upload/sync client write uses),
regular upload/sync client write uses), so the Server-Side so the Server-Side Encryption layer transparently (re-)encrypts the
Encryption layer transparently (re-)encrypts the content with a content with a fresh IV and a correct signature. Afterwards the
fresh IV and a correct signature. Afterwards the file is a file is a completely normal, healthy encrypted file again: readable
completely normal, healthy encrypted file again: readable in the in the web interface, over WebDAV, and by sync clients, with no
web interface, over WebDAV, and by sync clients, with no more 'Bad more 'Bad Signature' and no server-side flag left toggled.
Signature' and no server-side flag left toggled.
This is the ONLY script in this toolkit that writes into This is the ONLY script in this toolkit that writes into
Nextcloud's live storage - scan_bad_signature.sh and Nextcloud's live storage - scan_bad_signature.sh and
@@ -75,14 +74,35 @@ usage() {
itself. Treat it accordingly. itself. Treat it accordingly.
IMPORTANT: IMPORTANT:
- Only files whose recovery report marks them 'VALID' are ever - Two categories of entries from the recovery report are
considered - 'Datenmuell'/INVALID and 'Nicht pruefbar'/ restored, both by default, no separate flag needed:
UNVERIFIED entries are never written back, no override exists. * validation=VALID - a dedicated structural check (PDF
header, zip integrity, image signature, ...) actually
confirmed the recovered content.
* validation=UNVERIFIED ('Nicht pruefbar' in the recovery
report) - unusual/unrecognized file type, no dedicated
check exists, but nothing about it looks wrong either
(recovered/original size ratio is in the expected range,
or no size comparison applies to that check path). This
is the 'unauffaellig' case: we cannot prove it's correct,
but we also have no concrete reason to think it's not.
Entries where the size ratio actively 'looks off' (a genuine
negative signal, not just 'unchecked') are treated like
INVALID and are NEVER restored automatically, same as
'Datenmuell'/INVALID entries - no override exists for either.
Every restored UNVERIFIED file is clearly marked as such (its
own report column, its own counters) so this is always
transparent afterwards - never silently indistinguishable from
a properly verified VALID restore. If you'd rather review the
UNVERIFIED/'Nicht pruefbar' files yourself first, use
delete_files.sh to remove the ones you don't want kept, run
this script for 'VALID' coverage, then decide on the rest by
hand.
- Every file is re-validated with the CURRENT checks right - Every file is re-validated with the CURRENT checks right
before writing (not just trusted from the old report), in case before writing (not just trusted from the old report), in case
this script's validation logic has improved since that report this script's validation logic has improved since that report
was produced - files that no longer validate are skipped, not was produced - files that no longer validate (as VALID or as
forced through. plausible UNVERIFIED) are skipped, not forced through.
- Before each file is overwritten, the CURRENT on-disk ciphertext - Before each file is overwritten, the CURRENT on-disk ciphertext
(the still-broken original, exactly as it is right now) is (the still-broken original, exactly as it is right now) is
copied byte-for-byte to a separate backup directory (default copied byte-for-byte to a separate backup directory (default
@@ -113,7 +133,7 @@ usage() {
reports what would happen) or the real thing. You will then be reports what would happen) or the real thing. You will then be
asked which existing recovery report (produced by asked which existing recovery report (produced by
recover_bad_signature.sh) to use, and then which account(s) from its recover_bad_signature.sh) to use, and then which account(s) from its
VALID entries to restore for: restorable (VALID + plausible UNVERIFIED) entries to restore for:
- a blank separated list of account names - a blank separated list of account names
- or 'all' to attempt every account found in the report - or 'all' to attempt every account found in the report
@@ -792,8 +812,11 @@ if ! $_dry_run_explicit && $terminal ; then
blank_line blank_line
echo -e "\033[37m\033[1mWhich mode should this run use?\033[m" echo -e "\033[37m\033[1mWhich mode should this run use?\033[m"
echo "" echo ""
echo -e " \033[1m[1] Dry-run\033[m - go through everything (selection, re-validation, reporting), but write, back up and verify NOTHING" echo -e " \033[1m[1] Dry-run\033[m - go through everything (selection, re-validation, reporting),
but write, back up and verify NOTHING"
echo ""
echo " [2] Real restore - actually back up, overwrite and verify each file in Nextcloud" echo " [2] Real restore - actually back up, overwrite and verify each file in Nextcloud"
info "Just press Return to use the default: [1] Dry-run." info "Just press Return to use the default: [1] Dry-run."
echo -n " Select mode by number [1]: " echo -n " Select mode by number [1]: "
read _mode_choice read _mode_choice
@@ -900,6 +923,7 @@ if $terminal ; then
echo -e "Restore already-recovered \033[1mBad Signature\033[m files back into system \033[1m${WEB_BASE_DIR}\033[m" echo -e "Restore already-recovered \033[1mBad Signature\033[m files back into system \033[1m${WEB_BASE_DIR}\033[m"
echo -e "\033[1m echo -e "\033[1m
This WRITES into Nextcloud's live storage. Only files marked VALID This WRITES into Nextcloud's live storage. Only files marked VALID
or plausible/UNVERIFIED ('Nicht pruefbar', but nothing looks wrong)
by a prior recover_bad_signature.sh run are ever restored, each is by a prior recover_bad_signature.sh run are ever restored, each is
re-validated again right before writing, the current (still broken) re-validated again right before writing, the current (still broken)
original is backed up first, and every write is verified by reading original is backed up first, and every write is verified by reading
@@ -999,12 +1023,26 @@ chosen_report="${_available_reports[$((_report_choice-1))]}"
# ============= # =============
# --- Extract "user<TAB>path" pairs for every VALID entry in the chosen # --- Extract "user<TAB>path<TAB>original_size<TAB>validation" quadruples
# --- report. The report has no per-row user column - accounts are # --- for every restorable entry in the chosen report. The report has no
# --- section headers ('---------' / '<user>' / '---------'). Only # --- per-row user column - accounts are section headers ('---------' /
# --- 6-tab-field data rows belong to the following case; the header # --- '<user>' / '---------'). Only 6-tab-field data rows belong to the
# --- row, the 'Datenmuell'/'Nicht pruefbar' detail blocks (2 fields) # --- following case; the header row, the 'Datenmuell'/'Nicht pruefbar'
# --- and the account-statistics lines (no tabs) never match NF==6. # --- detail blocks (2 fields) and the account-statistics lines (no
# --- tabs) never match NF==6.
# ---
# --- Two validation values are picked up, both restored by default:
# --- VALID - a dedicated structural check confirmed the file.
# --- UNVERIFIED - no dedicated check exists ('Nicht pruefbar' in the
# --- recovery report), but nothing about it looks
# --- actively wrong either. The one sub-case that IS a
# --- concrete negative signal - the detail text says
# --- the recovered/original size ratio 'LOOKS OFF' -
# --- is deliberately excluded here, same treatment as
# --- INVALID/'Datenmuell': not restored automatically.
# --- original_size ($3) is carried through so this same distinction
# --- (plausible size ratio or not) can be re-checked with real data
# --- right before writing, not just trusted from the old report.
# ============= # =============
valid_entries_file="${LOCK_DIR}/valid_entries.tsv" valid_entries_file="${LOCK_DIR}/valid_entries.tsv"
@@ -1016,8 +1054,8 @@ awk -F'\t' '
} }
{ {
if (state == 1) { user = $0; state = 2; next } if (state == 1) { user = $0; state = 2; next }
if (NF == 6 && $1 != "path" && $5 == "VALID") { if (NF == 6 && $1 != "path" && ($5 == "VALID" || ($5 == "UNVERIFIED" && $6 !~ /LOOKS OFF/))) {
print user "\t" $1 print user "\t" $1 "\t" $3 "\t" $5
} }
} }
' "$chosen_report" > "$valid_entries_file" ' "$chosen_report" > "$valid_entries_file"
@@ -1027,21 +1065,26 @@ IFS=$'\n' account_arr=($(sort <<<"${unsorted_account_arr[*]}"))
IFS=$CUR_IFS IFS=$CUR_IFS
if [[ ${#account_arr[@]} -eq 0 ]] ; then if [[ ${#account_arr[@]} -eq 0 ]] ; then
fatal "No 'VALID' entries found in '$(basename "$chosen_report")' - nothing to restore." fatal "No restorable (VALID or plausible UNVERIFIED) entries found in '$(basename "$chosen_report")' - nothing to restore."
fi fi
blank_line blank_line
if $terminal ; then if $terminal ; then
echo -e "\033[37m\033[1mAccounts with 'VALID' entries in this report\033[m" echo -e "\033[37m\033[1mAccounts with restorable (VALID + plausible UNVERIFIED) entries in this report\033[m"
echo "" echo ""
for _a in "${account_arr[@]}" ; do for _a in "${account_arr[@]}" ; do
_a_count=$(awk -F'\t' -v u="$_a" '$1==u' "$valid_entries_file" | wc -l) _a_count=$(awk -F'\t' -v u="$_a" '$1==u' "$valid_entries_file" | wc -l)
_a_count_unverified=$(awk -F'\t' -v u="$_a" '$1==u && $4=="UNVERIFIED"' "$valid_entries_file" | wc -l)
if [[ "$_a_count_unverified" -gt 0 ]] ; then
echo " - $_a (${_a_count}, davon ${_a_count_unverified} 'Nicht pruefbar'/plausibel)"
else
echo " - $_a (${_a_count})" echo " - $_a (${_a_count})"
fi
done done
echo "" echo ""
fi fi
echo -n " Account(s) to restore VALID files for (blank separated, or 'all'): " echo -n " Account(s) to restore VALID/plausible-UNVERIFIED files for (blank separated, or 'all'): "
read _input read _input
while true ; do while true ; do
@@ -1067,7 +1110,7 @@ while true ; do
break break
fi fi
echo -n " Account(s) to restore VALID files for (blank separated, or 'all'): " echo -n " Account(s) to restore VALID/plausible-UNVERIFIED files for (blank separated, or 'all'): "
read _input read _input
done done
@@ -1106,7 +1149,7 @@ if $terminal ; then
if $DRY_RUN ; then if $DRY_RUN ; then
info "Dry-run: selection and re-validation run for real, nothing is backed up, written or verified." info "Dry-run: selection and re-validation run for real, nothing is backed up, written or verified."
else else
warn "This WRITES into Nextcloud's live storage, overwriting the still-broken original at its ORIGINAL path with the recovered content - the first script in this toolkit that does so. Only files re-validated as VALID right now are touched. The current (still broken) original is copied byte-for-byte to '${backup_dir}' first, and every write is read back and verified afterwards, but please still spot-check a few results yourself." warn "This WRITES into Nextcloud's live storage, overwriting the still-broken original at its ORIGINAL path with the recovered content - the first script in this toolkit that does so. Only files re-validated as VALID, or as plausible UNVERIFIED (no dedicated check, but nothing looks wrong - clearly marked as such in the report), right now are touched. The current (still broken) original is copied byte-for-byte to '${backup_dir}' first, and every write is read back and verified afterwards, but please still spot-check a few results yourself - especially the UNVERIFIED ones."
fi fi
echo "" echo ""
@@ -1115,7 +1158,7 @@ if $terminal ; then
if [[ "$OK" = "YES" ]] ; then if [[ "$OK" = "YES" ]] ; then
echo "" echo ""
echo "" echo ""
echo -e "\033[1;32mGoing to restore VALID files for each selected account on \033[1;37m$WEBSITE \033[m" echo -e "\033[1;32mGoing to restore VALID/plausible-UNVERIFIED files for each selected account on \033[1;37m$WEBSITE \033[m"
else else
fatal "Abort by user request - Answer as not 'YES'" fatal "Abort by user request - Answer as not 'YES'"
fi fi
@@ -1128,11 +1171,13 @@ fi
echo "==================================================================" echo "=================================================================="
echo "" echo ""
echo " Basis-Report (Recovery): $(basename "$chosen_report")" echo " Basis-Report (Recovery): $(basename "$chosen_report")"
echo " Nur Eintraege mit validation=VALID aus diesem Report werden zurueckgeschrieben." echo " Eintraege mit validation=VALID sowie validation=UNVERIFIED ('Nicht pruefbar',"
echo " aber Groessenverhaeltnis plausibel) aus diesem Report werden zurueckgeschrieben -"
echo " letztere sind in der Spalte 'origin' unten als UNVERIFIED gekennzeichnet."
$DRY_RUN && echo " DRY-RUN: es wurde NICHTS geschrieben, gesichert oder verifiziert." $DRY_RUN && echo " DRY-RUN: es wurde NICHTS geschrieben, gesichert oder verifiziert."
! $DRY_RUN && echo " Chiffretext-Sicherung (vor dem Ueberschreiben) liegt unter: ${backup_dir}" ! $DRY_RUN && echo " Chiffretext-Sicherung (vor dem Ueberschreiben) liegt unter: ${backup_dir}"
echo "" echo ""
echo -e "path\tbytes_written\tstatus\tdetail" echo -e "path\tbytes_written\tstatus\torigin\tdetail"
} > "$restore_report_file" } > "$restore_report_file"
@@ -1356,8 +1401,10 @@ if ! $DRY_RUN ; then
fi fi
declare -i total_selected=0 declare -i total_selected=0
declare -i total_selected_unverified=0
declare -i total_prevalidation_failed=0 declare -i total_prevalidation_failed=0
declare -i total_ok=0 declare -i total_ok=0
declare -i total_ok_unverified=0
declare -i total_write_errors=0 declare -i total_write_errors=0
declare -i total_failed_users=0 declare -i total_failed_users=0
@@ -1378,18 +1425,24 @@ for _user in "${selected_user_arr[@]}" ; do
user_recovery_dir="${recovery_dir}/${_user}" user_recovery_dir="${recovery_dir}/${_user}"
declare -i _user_selected=0 declare -i _user_selected=0
declare -i _user_selected_unverified=0
declare -i _user_prevalidation_failed=0 declare -i _user_prevalidation_failed=0
declare -a _user_prevalidation_failed_lines=() declare -a _user_prevalidation_failed_lines=()
declare -A _origin_of_path=()
list_file="${LOCK_DIR}/list_${_user}.tsv" list_file="${LOCK_DIR}/list_${_user}.tsv"
> "$list_file" > "$list_file"
# ============= # =============
# --- Re-validate every VALID-marked entry against the CURRENT # --- Re-validate every VALID- or plausible-UNVERIFIED-marked entry
# --- checks right before writing anything, and (unless dry-run) # --- against the CURRENT checks right before writing anything, and
# --- back up the current on-disk ciphertext byte-for-byte first. # --- (unless dry-run) back up the current on-disk ciphertext
# --- byte-for-byte first. original_size (from the recovery report,
# --- $3 of valid_entries_file) is passed through so the size-ratio
# --- part of the UNVERIFIED check can run for real here too, not
# --- just be trusted from the old report.
# ============= # =============
while IFS=$'\t' read -r _u _path ; do while IFS=$'\t' read -r _u _path _origsize _recorded_class ; do
[[ "$_u" != "$_user" ]] && continue [[ "$_u" != "$_user" ]] && continue
(( _user_selected++ )) (( _user_selected++ ))
@@ -1402,13 +1455,17 @@ for _user in "${selected_user_arr[@]}" ; do
continue continue
fi fi
_val_result="$(validate_recovered_file "$_local" "$_path" "")" _val_result="$(validate_recovered_file "$_local" "$_path" "$_origsize")"
_val_class="${_val_result%%|*}" _val_class="${_val_result%%|*}"
_val_detail="${_val_result#*|}" _val_detail="${_val_result#*|}"
if [[ "$_val_class" != "VALID" ]] ; then if [[ "$_val_class" == "VALID" ]] ; then
: # dedicated structural check confirmed it - restore as usual
elif [[ "$_val_class" == "UNVERIFIED" && "$_val_detail" != *"LOOKS OFF"* ]] ; then
(( _user_selected_unverified++ ))
else
(( _user_prevalidation_failed++ )) (( _user_prevalidation_failed++ ))
_user_prevalidation_failed_lines+=("${_path}"$'\t'"no longer validates as VALID (${_val_class}: ${_val_detail}) - skipped, not restored") _user_prevalidation_failed_lines+=("${_path}"$'\t'"no longer validates as restorable (${_val_class}: ${_val_detail}) - skipped, not restored")
continue continue
fi fi
@@ -1421,13 +1478,14 @@ for _user in "${selected_user_arr[@]}" ; do
fi fi
fi fi
_origin_of_path["$_path"]="$_val_class"
printf '%s\t%s\n' "$_path" "$_local" >> "$list_file" printf '%s\t%s\n' "$_path" "$_local" >> "$list_file"
done < "$valid_entries_file" done < "$valid_entries_file"
if [[ $_user_selected -eq 0 ]] ; then if [[ $_user_selected -eq 0 ]] ; then
warn "No 'VALID' entries for account '${_user}' found in the selected report - skipping." warn "No restorable (VALID or plausible UNVERIFIED) entries for account '${_user}' found in the selected report - skipping."
echo " [ keine VALID-Eintraege im gewaehlten Report ]" >> "$restore_report_file" echo " [ keine VALID/UNVERIFIED-Eintraege im gewaehlten Report ]" >> "$restore_report_file"
continue continue
fi fi
@@ -1463,6 +1521,7 @@ for _user in "${selected_user_arr[@]}" ; do
$terminal && echo "" $terminal && echo ""
declare -i _user_ok=0 declare -i _user_ok=0
declare -i _user_ok_unverified=0
declare -i _user_write_errors=0 declare -i _user_write_errors=0
declare -a _user_write_error_lines=() declare -a _user_write_error_lines=()
@@ -1471,8 +1530,13 @@ for _user in "${selected_user_arr[@]}" ; do
[[ "$_path" = "path" ]] && continue [[ "$_path" = "path" ]] && continue
[[ -z "$_path" ]] && continue [[ -z "$_path" ]] && continue
_origin="${_origin_of_path[$_path]:-VALID}"
case "$_status" in case "$_status" in
OK|DRY_RUN) (( _user_ok++ )) ;; OK|DRY_RUN)
(( _user_ok++ ))
[[ "$_origin" == "UNVERIFIED" ]] && (( _user_ok_unverified++ ))
;;
*) (( _user_write_errors++ )); _user_write_error_lines+=("${_path}"$'\t'"${_status}: ${_detail}") ;; *) (( _user_write_errors++ )); _user_write_error_lines+=("${_path}"$'\t'"${_status}: ${_detail}") ;;
esac esac
@@ -1481,7 +1545,7 @@ for _user in "${selected_user_arr[@]}" ; do
# reinterpret those backslashes as escape sequences and silently # reinterpret those backslashes as escape sequences and silently
# mangle/eat parts of the text. printf's %s never reinterprets its # mangle/eat parts of the text. printf's %s never reinterprets its
# argument, only the literal \t/\n in the format string itself. # argument, only the literal \t/\n in the format string itself.
printf '%s\t%s\t%s\t%s\n' "$_path" "$_bytes" "$_status" "$_detail" >> "$restore_report_file" printf '%s\t%s\t%s\t%s\t%s\n' "$_path" "$_bytes" "$_status" "$_origin" "$_detail" >> "$restore_report_file"
done < "$user_result_tsv" done < "$user_result_tsv"
@@ -1489,7 +1553,10 @@ for _user in "${selected_user_arr[@]}" ; do
if $terminal ; then if $terminal ; then
echo -e " \033[1;37mAccount ${_user}\033[m" echo -e " \033[1;37mAccount ${_user}\033[m"
echo -e " Ausgewaehlt (validation=VALID im Report)..: ${_user_selected}" echo -e " Ausgewaehlt (VALID + plausibel UNVERIFIED).: ${_user_selected}"
if [[ $_user_selected_unverified -gt 0 ]] ; then
echo -e " davon 'Nicht pruefbar'/UNVERIFIED.......: \033[33m${_user_selected_unverified}\033[m"
fi
if [[ $_user_prevalidation_failed -gt 0 ]] ; then if [[ $_user_prevalidation_failed -gt 0 ]] ; then
echo -e " Vor dem Schreiben aussortiert..............: \033[33m${_user_prevalidation_failed}\033[m" echo -e " Vor dem Schreiben aussortiert..............: \033[33m${_user_prevalidation_failed}\033[m"
fi fi
@@ -1501,6 +1568,9 @@ for _user in "${selected_user_arr[@]}" ; do
else else
echo -e " Zurueckgeschrieben & verifiziert...........: ${_user_ok} (${_user_pct_ok} %)" echo -e " Zurueckgeschrieben & verifiziert...........: ${_user_ok} (${_user_pct_ok} %)"
fi fi
if [[ $_user_ok_unverified -gt 0 ]] ; then
echo -e " davon UNVERIFIED (bitte stichprobenartig pruefen): \033[33m${_user_ok_unverified}\033[m"
fi
fi fi
if [[ $_user_write_errors -gt 0 ]] ; then if [[ $_user_write_errors -gt 0 ]] ; then
echo -e " Fehler beim Schreiben/Verifizieren.........: \033[1;31m${_user_write_errors}\033[m" echo -e " Fehler beim Schreiben/Verifizieren.........: \033[1;31m${_user_write_errors}\033[m"
@@ -1536,23 +1606,27 @@ for _user in "${selected_user_arr[@]}" ; do
{ {
echo "" echo ""
echo " Account ${_user}" echo " Account ${_user}"
echo " Ausgewaehlt (validation=VALID im Report)..: ${_user_selected}" echo " Ausgewaehlt (VALID + plausibel UNVERIFIED).: ${_user_selected}"
[[ $_user_selected_unverified -gt 0 ]] && echo " davon 'Nicht pruefbar'/UNVERIFIED.......: ${_user_selected_unverified}"
[[ $_user_prevalidation_failed -gt 0 ]] && echo " Vor dem Schreiben aussortiert..............: ${_user_prevalidation_failed}" [[ $_user_prevalidation_failed -gt 0 ]] && echo " Vor dem Schreiben aussortiert..............: ${_user_prevalidation_failed}"
if $DRY_RUN ; then if $DRY_RUN ; then
echo " Wuerde geschrieben & verifiziert werden....: ${_user_ok} (${_user_pct_ok} %)" echo " Wuerde geschrieben & verifiziert werden....: ${_user_ok} (${_user_pct_ok} %)"
else else
echo " Zurueckgeschrieben & verifiziert...........: ${_user_ok} (${_user_pct_ok} %)" echo " Zurueckgeschrieben & verifiziert...........: ${_user_ok} (${_user_pct_ok} %)"
fi fi
[[ $_user_ok_unverified -gt 0 ]] && echo " davon UNVERIFIED (bitte pruefen).........: ${_user_ok_unverified}"
[[ $_user_write_errors -gt 0 ]] && echo " Fehler beim Schreiben/Verifizieren.........: ${_user_write_errors}" [[ $_user_write_errors -gt 0 ]] && echo " Fehler beim Schreiben/Verifizieren.........: ${_user_write_errors}"
} >> "$restore_report_file" } >> "$restore_report_file"
(( total_selected += _user_selected )) (( total_selected += _user_selected ))
(( total_selected_unverified += _user_selected_unverified ))
(( total_prevalidation_failed += _user_prevalidation_failed )) (( total_prevalidation_failed += _user_prevalidation_failed ))
(( total_ok += _user_ok )) (( total_ok += _user_ok ))
(( total_ok_unverified += _user_ok_unverified ))
(( total_write_errors += _user_write_errors )) (( total_write_errors += _user_write_errors ))
unset _user_valid _user_prevalidation_failed _user_prevalidation_failed_lines unset _user_valid _user_selected_unverified _user_prevalidation_failed _user_prevalidation_failed_lines
unset _user_ok _user_write_errors _user_write_error_lines unset _user_ok _user_ok_unverified _user_write_errors _user_write_error_lines _origin_of_path
done done
@@ -1568,20 +1642,24 @@ total_pct_ok="$(calc_percent "$total_ok" "$total_selected")"
echo "" echo ""
echo "Ausgewaehlte Accounts.......................: ${#selected_user_arr[@]}" echo "Ausgewaehlte Accounts.......................: ${#selected_user_arr[@]}"
[[ $total_failed_users -gt 0 ]] && echo "Accounts mit Restore-Fehler.................: ${total_failed_users}" [[ $total_failed_users -gt 0 ]] && echo "Accounts mit Restore-Fehler.................: ${total_failed_users}"
echo "Ausgewaehlt (validation=VALID im Report)....: ${total_selected}" echo "Ausgewaehlt (VALID + plausibel UNVERIFIED)..: ${total_selected}"
[[ $total_selected_unverified -gt 0 ]] && echo " davon 'Nicht pruefbar'/UNVERIFIED.........: ${total_selected_unverified}"
[[ $total_prevalidation_failed -gt 0 ]] && echo "Vor dem Schreiben aussortiert...............: ${total_prevalidation_failed}" [[ $total_prevalidation_failed -gt 0 ]] && echo "Vor dem Schreiben aussortiert...............: ${total_prevalidation_failed}"
if $DRY_RUN ; then if $DRY_RUN ; then
echo "Wuerde geschrieben & verifiziert werden.....: ${total_ok} (${total_pct_ok} %)" echo "Wuerde geschrieben & verifiziert werden.....: ${total_ok} (${total_pct_ok} %)"
else else
echo "Zurueckgeschrieben & verifiziert insgesamt...: ${total_ok} (${total_pct_ok} %)" echo "Zurueckgeschrieben & verifiziert insgesamt...: ${total_ok} (${total_pct_ok} %)"
fi fi
[[ $total_ok_unverified -gt 0 ]] && echo " davon UNVERIFIED (bitte pruefen)..........: ${total_ok_unverified}"
[[ $total_write_errors -gt 0 ]] && echo "Fehler beim Schreiben/Verifizieren insgesamt: ${total_write_errors}" [[ $total_write_errors -gt 0 ]] && echo "Fehler beim Schreiben/Verifizieren insgesamt: ${total_write_errors}"
echo "" echo ""
if $DRY_RUN ; then if $DRY_RUN ; then
echo "DRY-RUN: es wurde nichts geschrieben, gesichert oder veraendert." echo "DRY-RUN: es wurde nichts geschrieben, gesichert oder veraendert."
else else
echo "Chiffretext-Sicherung (vor dem Ueberschreiben) liegt unter: ${backup_dir}" echo "Chiffretext-Sicherung (vor dem Ueberschreiben) liegt unter: ${backup_dir}"
echo "Bitte Ergebnisse trotz Verifikation stichprobenartig pruefen." echo "Bitte Ergebnisse trotz Verifikation stichprobenartig pruefen - das gilt"
echo "besonders fuer die oben als UNVERIFIED markierten Dateien: dort gibt es"
echo "keine dedizierte Strukturpruefung, nur ein plausibles Groessenverhaeltnis."
fi fi
} >> "$restore_report_file" } >> "$restore_report_file"
@@ -1592,7 +1670,8 @@ if $terminal ; then
echo "" echo ""
echo -e " Ausgewaehlte Accounts.......................: ${#selected_user_arr[@]}" echo -e " Ausgewaehlte Accounts.......................: ${#selected_user_arr[@]}"
[[ $total_failed_users -gt 0 ]] && echo -e " Accounts mit Restore-Fehler.................: \033[1;31m${total_failed_users}\033[m" [[ $total_failed_users -gt 0 ]] && echo -e " Accounts mit Restore-Fehler.................: \033[1;31m${total_failed_users}\033[m"
echo -e " Ausgewaehlt (validation=VALID im Report)....: ${total_selected}" echo -e " Ausgewaehlt (VALID + plausibel UNVERIFIED)..: ${total_selected}"
[[ $total_selected_unverified -gt 0 ]] && echo -e " davon 'Nicht pruefbar'/UNVERIFIED.........: \033[33m${total_selected_unverified}\033[m"
[[ $total_prevalidation_failed -gt 0 ]] && echo -e " Vor dem Schreiben aussortiert...............: \033[33m${total_prevalidation_failed}\033[m" [[ $total_prevalidation_failed -gt 0 ]] && echo -e " Vor dem Schreiben aussortiert...............: \033[33m${total_prevalidation_failed}\033[m"
if $DRY_RUN ; then if $DRY_RUN ; then
echo -e " Wuerde geschrieben & verifiziert werden.....: \033[1;32m${total_ok} (${total_pct_ok} %)\033[m" echo -e " Wuerde geschrieben & verifiziert werden.....: \033[1;32m${total_ok} (${total_pct_ok} %)\033[m"
@@ -1602,6 +1681,7 @@ if $terminal ; then
else else
echo -e " Zurueckgeschrieben & verifiziert insgesamt...: ${total_ok} (${total_pct_ok} %)" echo -e " Zurueckgeschrieben & verifiziert insgesamt...: ${total_ok} (${total_pct_ok} %)"
fi fi
[[ $total_ok_unverified -gt 0 ]] && echo -e " davon UNVERIFIED (bitte pruefen)..........: \033[33m${total_ok_unverified}\033[m"
fi fi
[[ $total_write_errors -gt 0 ]] && echo -e " Fehler beim Schreiben/Verifizieren insgesamt: \033[1;31m${total_write_errors}\033[m" [[ $total_write_errors -gt 0 ]] && echo -e " Fehler beim Schreiben/Verifizieren insgesamt: \033[1;31m${total_write_errors}\033[m"
echo "" echo ""