delete_files.sh: neues, generisches Script zum gezielten Löschen von Dateien

2026-09-15 00:26:19 +02:00
parent 93e0576777
commit a8887aeeea
2 changed files with 1122 additions and 51 deletions
@@ -59,15 +59,14 @@ usage() {
 \033[1mDescription\033[m

   Writes already-recovered, already-validated files (produced by a
-   PRIOR run of recover_bad_signature.sh, validation column = VALID in
-   its report) back into Nextcloud's own storage, at their ORIGINAL
-   path - through Nextcloud's normal Files API (the same write path a
-   regular upload/sync client write uses), so the Server-Side
-   Encryption layer transparently (re-)encrypts the content with a
-   fresh IV and a correct signature. Afterwards the file is a
-   completely normal, healthy encrypted file again: readable in the
-   web interface, over WebDAV, and by sync clients, with no more 'Bad
-   Signature' and no server-side flag left toggled.
+   PRIOR run of recover_bad_signature.sh) back into Nextcloud's own
+   storage, at their ORIGINAL path - through Nextcloud's normal Files
+   API (the same write path a regular upload/sync client write uses),
+   so the Server-Side Encryption layer transparently (re-)encrypts the
+   content with a fresh IV and a correct signature. Afterwards the
+   file is a completely normal, healthy encrypted file again: readable
+   in the web interface, over WebDAV, and by sync clients, with no
+   more 'Bad Signature' and no server-side flag left toggled.

   This is the ONLY script in this toolkit that writes into
   Nextcloud's live storage - scan_bad_signature.sh and
@@ -75,14 +74,35 @@ usage() {
   itself. Treat it accordingly.

   IMPORTANT:
-      - Only files whose recovery report marks them 'VALID' are ever
-        considered - 'Datenmuell'/INVALID and 'Nicht pruefbar'/
-        UNVERIFIED entries are never written back, no override exists.
+      - Two categories of entries from the recovery report are
+        restored, both by default, no separate flag needed:
+           * validation=VALID   - a dedicated structural check (PDF
+             header, zip integrity, image signature, ...) actually
+             confirmed the recovered content.
+           * validation=UNVERIFIED ('Nicht pruefbar' in the recovery
+             report) - unusual/unrecognized file type, no dedicated
+             check exists, but nothing about it looks wrong either
+             (recovered/original size ratio is in the expected range,
+             or no size comparison applies to that check path). This
+             is the 'unauffaellig' case: we cannot prove it's correct,
+             but we also have no concrete reason to think it's not.
+        Entries where the size ratio actively 'looks off' (a genuine
+        negative signal, not just 'unchecked') are treated like
+        INVALID and are NEVER restored automatically, same as
+        'Datenmuell'/INVALID entries - no override exists for either.
+        Every restored UNVERIFIED file is clearly marked as such (its
+        own report column, its own counters) so this is always
+        transparent afterwards - never silently indistinguishable from
+        a properly verified VALID restore. If you'd rather review the
+        UNVERIFIED/'Nicht pruefbar' files yourself first, use
+        delete_files.sh to remove the ones you don't want kept, run
+        this script for 'VALID' coverage, then decide on the rest by
+        hand.
      - Every file is re-validated with the CURRENT checks right
        before writing (not just trusted from the old report), in case
        this script's validation logic has improved since that report
-        was produced - files that no longer validate are skipped, not
-        forced through.
+        was produced - files that no longer validate (as VALID or as
+        plausible UNVERIFIED) are skipped, not forced through.
      - Before each file is overwritten, the CURRENT on-disk ciphertext
        (the still-broken original, exactly as it is right now) is
        copied byte-for-byte to a separate backup directory (default
@@ -113,7 +133,7 @@ usage() {
   reports what would happen) or the real thing. You will then be
   asked which existing recovery report (produced by
   recover_bad_signature.sh) to use, and then which account(s) from its
-   VALID entries to restore for:
+   restorable (VALID + plausible UNVERIFIED) entries to restore for:

      - a blank separated list of account names
      - or 'all' to attempt every account found in the report
@@ -792,8 +812,11 @@ if ! $_dry_run_explicit && $terminal ; then
   blank_line
   echo -e "\033[37m\033[1mWhich mode should this run use?\033[m"
   echo ""
-   echo -e "   \033[1m[1] Dry-run\033[m          - go through everything (selection, re-validation, reporting), but write, back up and verify NOTHING"
+   echo -e "   \033[1m[1] Dry-run\033[m          - go through everything (selection, re-validation, reporting), 
+                          but write, back up and verify NOTHING"
+   echo ""
   echo    "   [2] Real restore    - actually back up, overwrite and verify each file in Nextcloud"
+
   info "Just press Return to use the default: [1] Dry-run."
   echo -n "   Select mode by number [1]: "
   read _mode_choice
@@ -900,6 +923,7 @@ if $terminal ; then
   echo -e "Restore already-recovered \033[1mBad Signature\033[m files back into system \033[1m${WEB_BASE_DIR}\033[m"
   echo -e "\033[1m
   This WRITES into Nextcloud's live storage. Only files marked VALID
+   or plausible/UNVERIFIED ('Nicht pruefbar', but nothing looks wrong)
   by a prior recover_bad_signature.sh run are ever restored, each is
   re-validated again right before writing, the current (still broken)
   original is backed up first, and every write is verified by reading
@@ -999,12 +1023,26 @@ chosen_report="${_available_reports[$((_report_choice-1))]}"


 # =============
-# --- Extract "user<TAB>path" pairs for every VALID entry in the chosen
-# --- report. The report has no per-row user column - accounts are
-# --- section headers ('---------' / '<user>' / '---------'). Only
-# --- 6-tab-field data rows belong to the following case; the header
-# --- row, the 'Datenmuell'/'Nicht pruefbar' detail blocks (2 fields)
-# --- and the account-statistics lines (no tabs) never match NF==6.
+# --- Extract "user<TAB>path<TAB>original_size<TAB>validation" quadruples
+# --- for every restorable entry in the chosen report. The report has no
+# --- per-row user column - accounts are section headers ('---------' /
+# --- '<user>' / '---------'). Only 6-tab-field data rows belong to the
+# --- following case; the header row, the 'Datenmuell'/'Nicht pruefbar'
+# --- detail blocks (2 fields) and the account-statistics lines (no
+# --- tabs) never match NF==6.
+# ---
+# --- Two validation values are picked up, both restored by default:
+# ---   VALID       - a dedicated structural check confirmed the file.
+# ---   UNVERIFIED  - no dedicated check exists ('Nicht pruefbar' in the
+# ---                 recovery report), but nothing about it looks
+# ---                 actively wrong either. The one sub-case that IS a
+# ---                 concrete negative signal - the detail text says
+# ---                 the recovered/original size ratio 'LOOKS OFF' -
+# ---                 is deliberately excluded here, same treatment as
+# ---                 INVALID/'Datenmuell': not restored automatically.
+# --- original_size ($3) is carried through so this same distinction
+# --- (plausible size ratio or not) can be re-checked with real data
+# --- right before writing, not just trusted from the old report.
 # =============

 valid_entries_file="${LOCK_DIR}/valid_entries.tsv"
@@ -1016,8 +1054,8 @@ awk -F'\t' '
   }
   {
      if (state == 1) { user = $0; state = 2; next }
-      if (NF == 6 && $1 != "path" && $5 == "VALID") {
-         print user "\t" $1
+      if (NF == 6 && $1 != "path" && ($5 == "VALID" || ($5 == "UNVERIFIED" && $6 !~ /LOOKS OFF/))) {
+         print user "\t" $1 "\t" $3 "\t" $5
      }
   }
 ' "$chosen_report" > "$valid_entries_file"
@@ -1027,21 +1065,26 @@ IFS=$'\n' account_arr=($(sort <<<"${unsorted_account_arr[*]}"))
 IFS=$CUR_IFS

 if [[ ${#account_arr[@]} -eq 0 ]] ; then
-   fatal "No 'VALID' entries found in '$(basename "$chosen_report")' - nothing to restore."
+   fatal "No restorable (VALID or plausible UNVERIFIED) entries found in '$(basename "$chosen_report")' - nothing to restore."
 fi

 blank_line
 if $terminal ; then
-   echo -e "\033[37m\033[1mAccounts with 'VALID' entries in this report\033[m"
+   echo -e "\033[37m\033[1mAccounts with restorable (VALID + plausible UNVERIFIED) entries in this report\033[m"
   echo ""
   for _a in "${account_arr[@]}" ; do
      _a_count=$(awk -F'\t' -v u="$_a" '$1==u' "$valid_entries_file" | wc -l)
-      echo "   - $_a (${_a_count})"
+      _a_count_unverified=$(awk -F'\t' -v u="$_a" '$1==u && $4=="UNVERIFIED"' "$valid_entries_file" | wc -l)
+      if [[ "$_a_count_unverified" -gt 0 ]] ; then
+         echo "   - $_a (${_a_count}, davon ${_a_count_unverified} 'Nicht pruefbar'/plausibel)"
+      else
+         echo "   - $_a (${_a_count})"
+      fi
   done
   echo ""
 fi

-echo -n "   Account(s) to restore VALID files for (blank separated, or 'all'): "
+echo -n "   Account(s) to restore VALID/plausible-UNVERIFIED files for (blank separated, or 'all'): "
 read _input

 while true ; do
@@ -1067,7 +1110,7 @@ while true ; do
      break
   fi

-   echo -n "   Account(s) to restore VALID files for (blank separated, or 'all'): "
+   echo -n "   Account(s) to restore VALID/plausible-UNVERIFIED files for (blank separated, or 'all'): "
   read _input

 done
@@ -1106,7 +1149,7 @@ if $terminal ; then
   if $DRY_RUN ; then
      info "Dry-run: selection and re-validation run for real, nothing is backed up, written or verified."
   else
-      warn "This WRITES into Nextcloud's live storage, overwriting the still-broken original at its ORIGINAL path with the recovered content - the first script in this toolkit that does so. Only files re-validated as VALID right now are touched. The current (still broken) original is copied byte-for-byte to '${backup_dir}' first, and every write is read back and verified afterwards, but please still spot-check a few results yourself."
+      warn "This WRITES into Nextcloud's live storage, overwriting the still-broken original at its ORIGINAL path with the recovered content - the first script in this toolkit that does so. Only files re-validated as VALID, or as plausible UNVERIFIED (no dedicated check, but nothing looks wrong - clearly marked as such in the report), right now are touched. The current (still broken) original is copied byte-for-byte to '${backup_dir}' first, and every write is read back and verified afterwards, but please still spot-check a few results yourself - especially the UNVERIFIED ones."
   fi

   echo ""
@@ -1115,7 +1158,7 @@ if $terminal ; then
   if [[ "$OK" = "YES" ]] ; then
      echo ""
      echo ""
-      echo -e "\033[1;32mGoing to restore VALID files for each selected account on \033[1;37m$WEBSITE \033[m"
+      echo -e "\033[1;32mGoing to restore VALID/plausible-UNVERIFIED files for each selected account on \033[1;37m$WEBSITE \033[m"
   else
      fatal "Abort by user request - Answer as not 'YES'"
   fi
@@ -1128,11 +1171,13 @@ fi
   echo "=================================================================="
   echo ""
   echo " Basis-Report (Recovery): $(basename "$chosen_report")"
-   echo " Nur Eintraege mit validation=VALID aus diesem Report werden zurueckgeschrieben."
+   echo " Eintraege mit validation=VALID sowie validation=UNVERIFIED ('Nicht pruefbar',"
+   echo " aber Groessenverhaeltnis plausibel) aus diesem Report werden zurueckgeschrieben -"
+   echo " letztere sind in der Spalte 'origin' unten als UNVERIFIED gekennzeichnet."
   $DRY_RUN && echo " DRY-RUN: es wurde NICHTS geschrieben, gesichert oder verifiziert."
   ! $DRY_RUN && echo " Chiffretext-Sicherung (vor dem Ueberschreiben) liegt unter: ${backup_dir}"
   echo ""
-   echo -e "path\tbytes_written\tstatus\tdetail"
+   echo -e "path\tbytes_written\tstatus\torigin\tdetail"
 } > "$restore_report_file"


@@ -1356,8 +1401,10 @@ if ! $DRY_RUN ; then
 fi

 declare -i total_selected=0
+declare -i total_selected_unverified=0
 declare -i total_prevalidation_failed=0
 declare -i total_ok=0
+declare -i total_ok_unverified=0
 declare -i total_write_errors=0
 declare -i total_failed_users=0

@@ -1378,18 +1425,24 @@ for _user in "${selected_user_arr[@]}" ; do
   user_recovery_dir="${recovery_dir}/${_user}"

   declare -i _user_selected=0
+   declare -i _user_selected_unverified=0
   declare -i _user_prevalidation_failed=0
   declare -a _user_prevalidation_failed_lines=()
+   declare -A _origin_of_path=()

   list_file="${LOCK_DIR}/list_${_user}.tsv"
   > "$list_file"

   # =============
-   # --- Re-validate every VALID-marked entry against the CURRENT
-   # --- checks right before writing anything, and (unless dry-run)
-   # --- back up the current on-disk ciphertext byte-for-byte first.
+   # --- Re-validate every VALID- or plausible-UNVERIFIED-marked entry
+   # --- against the CURRENT checks right before writing anything, and
+   # --- (unless dry-run) back up the current on-disk ciphertext
+   # --- byte-for-byte first. original_size (from the recovery report,
+   # --- $3 of valid_entries_file) is passed through so the size-ratio
+   # --- part of the UNVERIFIED check can run for real here too, not
+   # --- just be trusted from the old report.
   # =============
-   while IFS=$'\t' read -r _u _path ; do
+   while IFS=$'\t' read -r _u _path _origsize _recorded_class ; do
      [[ "$_u" != "$_user" ]] && continue
      (( _user_selected++ ))

@@ -1402,13 +1455,17 @@ for _user in "${selected_user_arr[@]}" ; do
         continue
      fi

-      _val_result="$(validate_recovered_file "$_local" "$_path" "")"
+      _val_result="$(validate_recovered_file "$_local" "$_path" "$_origsize")"
      _val_class="${_val_result%%|*}"
      _val_detail="${_val_result#*|}"

-      if [[ "$_val_class" != "VALID" ]] ; then
+      if [[ "$_val_class" == "VALID" ]] ; then
+         : # dedicated structural check confirmed it - restore as usual
+      elif [[ "$_val_class" == "UNVERIFIED" && "$_val_detail" != *"LOOKS OFF"* ]] ; then
+         (( _user_selected_unverified++ ))
+      else
         (( _user_prevalidation_failed++ ))
-         _user_prevalidation_failed_lines+=("${_path}"$'\t'"no longer validates as VALID (${_val_class}: ${_val_detail}) - skipped, not restored")
+         _user_prevalidation_failed_lines+=("${_path}"$'\t'"no longer validates as restorable (${_val_class}: ${_val_detail}) - skipped, not restored")
         continue
      fi

@@ -1421,13 +1478,14 @@ for _user in "${selected_user_arr[@]}" ; do
         fi
      fi

+      _origin_of_path["$_path"]="$_val_class"
      printf '%s\t%s\n' "$_path" "$_local" >> "$list_file"

   done < "$valid_entries_file"

   if [[ $_user_selected -eq 0 ]] ; then
-      warn "No 'VALID' entries for account '${_user}' found in the selected report - skipping."
-      echo "   [ keine VALID-Eintraege im gewaehlten Report ]" >> "$restore_report_file"
+      warn "No restorable (VALID or plausible UNVERIFIED) entries for account '${_user}' found in the selected report - skipping."
+      echo "   [ keine VALID/UNVERIFIED-Eintraege im gewaehlten Report ]" >> "$restore_report_file"
      continue
   fi

@@ -1463,6 +1521,7 @@ for _user in "${selected_user_arr[@]}" ; do
   $terminal && echo ""

   declare -i _user_ok=0
+   declare -i _user_ok_unverified=0
   declare -i _user_write_errors=0
   declare -a _user_write_error_lines=()

@@ -1471,8 +1530,13 @@ for _user in "${selected_user_arr[@]}" ; do
      [[ "$_path" = "path" ]] && continue
      [[ -z "$_path" ]] && continue

+      _origin="${_origin_of_path[$_path]:-VALID}"
+
      case "$_status" in
-         OK|DRY_RUN) (( _user_ok++ )) ;;
+         OK|DRY_RUN)
+            (( _user_ok++ ))
+            [[ "$_origin" == "UNVERIFIED" ]] && (( _user_ok_unverified++ ))
+            ;;
         *) (( _user_write_errors++ )); _user_write_error_lines+=("${_path}"$'\t'"${_status}: ${_detail}") ;;
      esac

@@ -1481,7 +1545,7 @@ for _user in "${selected_user_arr[@]}" ; do
      # reinterpret those backslashes as escape sequences and silently
      # mangle/eat parts of the text. printf's %s never reinterprets its
      # argument, only the literal \t/\n in the format string itself.
-      printf '%s\t%s\t%s\t%s\n' "$_path" "$_bytes" "$_status" "$_detail" >> "$restore_report_file"
+      printf '%s\t%s\t%s\t%s\t%s\n' "$_path" "$_bytes" "$_status" "$_origin" "$_detail" >> "$restore_report_file"

   done < "$user_result_tsv"

@@ -1489,7 +1553,10 @@ for _user in "${selected_user_arr[@]}" ; do

   if $terminal ; then
      echo -e "      \033[1;37mAccount ${_user}\033[m"
-      echo -e   "      Ausgewaehlt (validation=VALID im Report)..: ${_user_selected}"
+      echo -e   "      Ausgewaehlt (VALID + plausibel UNVERIFIED).: ${_user_selected}"
+      if [[ $_user_selected_unverified -gt 0 ]] ; then
+         echo -e "         davon 'Nicht pruefbar'/UNVERIFIED.......: \033[33m${_user_selected_unverified}\033[m"
+      fi
      if [[ $_user_prevalidation_failed -gt 0 ]] ; then
         echo -e "      Vor dem Schreiben aussortiert..............: \033[33m${_user_prevalidation_failed}\033[m"
      fi
@@ -1501,6 +1568,9 @@ for _user in "${selected_user_arr[@]}" ; do
         else
            echo -e "      Zurueckgeschrieben & verifiziert...........: ${_user_ok} (${_user_pct_ok} %)"
         fi
+         if [[ $_user_ok_unverified -gt 0 ]] ; then
+            echo -e "         davon UNVERIFIED (bitte stichprobenartig pruefen): \033[33m${_user_ok_unverified}\033[m"
+         fi
      fi
      if [[ $_user_write_errors -gt 0 ]] ; then
         echo -e "      Fehler beim Schreiben/Verifizieren.........: \033[1;31m${_user_write_errors}\033[m"
@@ -1536,23 +1606,27 @@ for _user in "${selected_user_arr[@]}" ; do
   {
      echo ""
      echo "   Account ${_user}"
-      echo "   Ausgewaehlt (validation=VALID im Report)..: ${_user_selected}"
+      echo "   Ausgewaehlt (VALID + plausibel UNVERIFIED).: ${_user_selected}"
+      [[ $_user_selected_unverified -gt 0 ]] && echo "      davon 'Nicht pruefbar'/UNVERIFIED.......: ${_user_selected_unverified}"
      [[ $_user_prevalidation_failed -gt 0 ]] && echo "   Vor dem Schreiben aussortiert..............: ${_user_prevalidation_failed}"
      if $DRY_RUN ; then
         echo "   Wuerde geschrieben & verifiziert werden....: ${_user_ok} (${_user_pct_ok} %)"
      else
         echo "   Zurueckgeschrieben & verifiziert...........: ${_user_ok} (${_user_pct_ok} %)"
      fi
+      [[ $_user_ok_unverified -gt 0 ]] && echo "      davon UNVERIFIED (bitte pruefen).........: ${_user_ok_unverified}"
      [[ $_user_write_errors -gt 0 ]] && echo "   Fehler beim Schreiben/Verifizieren.........: ${_user_write_errors}"
   } >> "$restore_report_file"

   (( total_selected += _user_selected ))
+   (( total_selected_unverified += _user_selected_unverified ))
   (( total_prevalidation_failed += _user_prevalidation_failed ))
   (( total_ok += _user_ok ))
+   (( total_ok_unverified += _user_ok_unverified ))
   (( total_write_errors += _user_write_errors ))

-   unset _user_valid _user_prevalidation_failed _user_prevalidation_failed_lines
-   unset _user_ok _user_write_errors _user_write_error_lines
+   unset _user_valid _user_selected_unverified _user_prevalidation_failed _user_prevalidation_failed_lines
+   unset _user_ok _user_ok_unverified _user_write_errors _user_write_error_lines _origin_of_path

 done

@@ -1568,20 +1642,24 @@ total_pct_ok="$(calc_percent "$total_ok" "$total_selected")"
   echo ""
   echo "Ausgewaehlte Accounts.......................: ${#selected_user_arr[@]}"
   [[ $total_failed_users -gt 0 ]] && echo "Accounts mit Restore-Fehler.................: ${total_failed_users}"
-   echo "Ausgewaehlt (validation=VALID im Report)....: ${total_selected}"
+   echo "Ausgewaehlt (VALID + plausibel UNVERIFIED)..: ${total_selected}"
+   [[ $total_selected_unverified -gt 0 ]] && echo "   davon 'Nicht pruefbar'/UNVERIFIED.........: ${total_selected_unverified}"
   [[ $total_prevalidation_failed -gt 0 ]] && echo "Vor dem Schreiben aussortiert...............: ${total_prevalidation_failed}"
   if $DRY_RUN ; then
      echo "Wuerde geschrieben & verifiziert werden.....: ${total_ok} (${total_pct_ok} %)"
   else
      echo "Zurueckgeschrieben & verifiziert insgesamt...: ${total_ok} (${total_pct_ok} %)"
   fi
+   [[ $total_ok_unverified -gt 0 ]] && echo "   davon UNVERIFIED (bitte pruefen)..........: ${total_ok_unverified}"
   [[ $total_write_errors -gt 0 ]] && echo "Fehler beim Schreiben/Verifizieren insgesamt: ${total_write_errors}"
   echo ""
   if $DRY_RUN ; then
      echo "DRY-RUN: es wurde nichts geschrieben, gesichert oder veraendert."
   else
      echo "Chiffretext-Sicherung (vor dem Ueberschreiben) liegt unter: ${backup_dir}"
-      echo "Bitte Ergebnisse trotz Verifikation stichprobenartig pruefen."
+      echo "Bitte Ergebnisse trotz Verifikation stichprobenartig pruefen - das gilt"
+      echo "besonders fuer die oben als UNVERIFIED markierten Dateien: dort gibt es"
+      echo "keine dedizierte Strukturpruefung, nur ein plausibles Groessenverhaeltnis."
   fi
 } >> "$restore_report_file"

@@ -1592,7 +1670,8 @@ if $terminal ; then
   echo ""
   echo -e   "   Ausgewaehlte Accounts.......................: ${#selected_user_arr[@]}"
   [[ $total_failed_users -gt 0 ]] && echo -e "   Accounts mit Restore-Fehler.................: \033[1;31m${total_failed_users}\033[m"
-   echo -e   "   Ausgewaehlt (validation=VALID im Report)....: ${total_selected}"
+   echo -e   "   Ausgewaehlt (VALID + plausibel UNVERIFIED)..: ${total_selected}"
+   [[ $total_selected_unverified -gt 0 ]] && echo -e "      davon 'Nicht pruefbar'/UNVERIFIED.........: \033[33m${total_selected_unverified}\033[m"
   [[ $total_prevalidation_failed -gt 0 ]] && echo -e "   Vor dem Schreiben aussortiert...............: \033[33m${total_prevalidation_failed}\033[m"
   if $DRY_RUN ; then
      echo -e "   Wuerde geschrieben & verifiziert werden.....: \033[1;32m${total_ok} (${total_pct_ok} %)\033[m"
@@ -1602,6 +1681,7 @@ if $terminal ; then
      else
         echo -e "   Zurueckgeschrieben & verifiziert insgesamt...: ${total_ok} (${total_pct_ok} %)"
      fi
+      [[ $total_ok_unverified -gt 0 ]] && echo -e "      davon UNVERIFIED (bitte pruefen)..........: \033[33m${total_ok_unverified}\033[m"
   fi
   [[ $total_write_errors -gt 0 ]] && echo -e "   Fehler beim Schreiben/Verifizieren insgesamt: \033[1;31m${total_write_errors}\033[m"
   echo ""