From 99633664a919bf43b6ae27973dc7c40505ba1927 Mon Sep 17 00:00:00 2001 From: Christoph Date: Tue, 15 Sep 2026 18:02:38 +0200 Subject: [PATCH] upgrade-keycloak.sh: Harden Keycloak upgrade process Keep previous installation intact, build before switching the symlink, improve error handling and add service and issuer checks. --- upgrade-keycloak.sh | 309 +++++++++----------------------------------- 1 file changed, 63 insertions(+), 246 deletions(-) diff --git a/upgrade-keycloak.sh b/upgrade-keycloak.sh index 8b67711..05006e4 100755 --- a/upgrade-keycloak.sh +++ b/upgrade-keycloak.sh @@ -11,8 +11,8 @@ log_file="${LOCK_DIR}/${script_name%%.*}.log" backup_date="$(date +%Y-%m-%d-%H%M)" -crontab_backup_file="${working_dir}/crontab-root-${backup_date}" - +keycloak_service_was_active=false +symlink_switched=false # ---------- # Base Function(s) @@ -52,19 +52,18 @@ fatal(){ echo -e " Script is canceled.." echo "" fi - if [[ -f "${crontab_backup_file}" ]]; then - echononl "Reenable previously saved crontab from '$(basename "${crontab_backup_file}")'.." - crontab ${crontab_backup_file} > /dev/null 2>&1 - if [[ $? -eq 0 ]]; then + if ${keycloak_service_was_active:-false} && ! ${symlink_switched:-false} && \ + ! systemctl is-active --quiet keycloak.service ; then + echononl "Restart previous Keycloak installation.." + if systemctl start keycloak.service > /dev/null 2>&1 ; then echo_ok - rm -f ${crontab_backup_file} > /dev/null 2>&11 else echo_failed - error "$(cat $log_file)" + error "The previous Keycloak service could not be restarted automatically." fi fi - rm -rf $LOCK_DIR + rm -rf "$LOCK_DIR" exit 1 } @@ -215,7 +214,7 @@ generate_random_string() { # - Run 'clean_up' for signals SIGHUP SIGINT SIGTERM # - -trap clean_up SIGHUP SIGINT SIGTERM +trap 'fatal "Interrupted by signal."' SIGHUP SIGINT SIGTERM # - Create lock directory '$LOCK_DIR" # @@ -340,6 +339,9 @@ do echo -e "\n\t\033[33m\033[1mA Version number is required!\033[m\n" fi done +if [[ ! "${KEYCLOAK_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + fatal "Invalid Keycloak version '${KEYCLOAK_VERSION}'. Expected format: 26.7.3" +fi DOWNLOAD_ARCHIVE="keycloak-${KEYCLOAK_VERSION}.tar.gz" DOWNLOAD_URL="https://github.com/keycloak/keycloak/releases/download/${KEYCLOAK_VERSION}/${DOWNLOAD_ARCHIVE}" @@ -519,7 +521,7 @@ echo "" echo -e "\tKeycloak database type..........: ${DB_TYPE}" echo -e "\tKeycloak database name..........: ${DB_NAME}" echo -e "\tKeycloak database user..........: ${DB_USER}" -echo -e "\tKeycloak database password......: ${DB_PASS}" +echo -e "\tKeycloak database password......: ********" echo "" echo -e "\tdatabase dump file..............: ${DB_DUMP_FILE}" echo "" @@ -595,7 +597,7 @@ _var="KEYCLOAK_BASE_INSTALL_PATH" _val="${KEYCLOAK_BASE_INSTALL_PATH}" echononl "Update '${_var}' configuration file .." if ! $(grep -q -E "^\s*${_var}=\"${_val}\"" "${conf_file}" 2> /dev/null) ; then - perl -i -n -p -e "s#^s*${_var}=.*#${_var}=\"${_val}\"#" ${conf_file} > "$log_file" 2>&1 + perl -i -n -p -e "s#^\s*${_var}=.*#${_var}=\"${_val}\"#" ${conf_file} > "$log_file" 2>&1 if [[ "$?" = "0" ]]; then echo_ok else @@ -623,33 +625,13 @@ echo echo -e "\033[37m\033[1mSome pre-installation stuff..\033[m" echo -echononl "Backup crontab" -crontab -u root -l > ${crontab_backup_file} 2> $log_file -if [[ "$?" = "0" ]]; then - echo_ok -else - echo_failed - error "$(cat $log_file)" -fi - -echononl "Disable crontab for user root" -crontab -r -u root > ${log_file} 2>&1 -if [[ "$?" = "0" ]]; then - echo_ok -else - echo_failed - error "$(cat $tmp_err_msg)" -fi - - -blank_line - echononl "Stop Keycloak Service.." -if $(systemctl is-active --quiet service keycloak.service) ; then +if systemctl is-active --quiet keycloak.service ; then + keycloak_service_was_active=true systemctl stop keycloak.service > "$log_file" 2>&1 if [[ $? -ne 0 ]]; then echo_failed - error "$(cat "$log_file")" + fatal "$(cat "$log_file")" else echo_ok fi @@ -687,7 +669,7 @@ if [[ "${DB_TYPE}" = "pgsql" ]] ; then --no-password \ 2> "$log_file" \ | gzip > "${DB_DUMP_FILE}" - if [[ ${PIPESTATUS[0]} -ne 0 ]]; then + if [[ ${PIPESTATUS[0]} -ne 0 || ${PIPESTATUS[1]} -ne 0 ]]; then echo_failed error "$(cat "$log_file")" fatal "PostgreSQL backup failed! Aborting upgrade." @@ -709,7 +691,7 @@ elif [[ "${DB_TYPE}" = "mysql" ]] ; then "${DB_NAME}" \ 2> "$log_file" \ | gzip > "${DB_DUMP_FILE}" - if [[ ${PIPESTATUS[0]} -ne 0 ]]; then + if [[ ${PIPESTATUS[0]} -ne 0 || ${PIPESTATUS[1]} -ne 0 ]]; then echo_failed error "$(cat "$log_file")" fatal "MySQL backup failed! Aborting upgrade." @@ -732,7 +714,7 @@ if [[ ! -f "${working_dir}/${DOWNLOAD_ARCHIVE}" ]]; then wget -O "${working_dir}/${DOWNLOAD_ARCHIVE}" "${DOWNLOAD_URL}" > "$log_file" 2>&1 if [[ $? -ne 0 ]]; then echo_failed - error "$(cat "$log_file")" + fatal "Download failed: $(cat "$log_file")" else echo_ok fi @@ -743,36 +725,15 @@ fi blank_line -echononl "Backup Keycloak Installation directory.." -if [[ -n "${CUR_INSTALL_DIR}" ]]; then - mv "${CUR_INSTALL_DIR}" "${CUR_INSTALL_DIR}.${backup_date}" > "$log_file" 2>&1 - if [[ $? -ne 0 ]]; then - echo_failed - error "$(cat "$log_file")" - else - echo_ok - fi -else - echo_skipped +if [[ -e "${KEYCLOAK_INSTALL_DIR}" ]]; then + fatal "Target directory '${KEYCLOAK_INSTALL_DIR}' already exists. Remove or rename it manually after checking its contents." fi -echononl "Remove Symlink '${KEYCLOAK_BASE_INSTALL_PATH}/keycloak' .." -rm "${KEYCLOAK_BASE_INSTALL_PATH}/keycloak" > "$log_file" 2>&1 -if [[ $? -ne 0 ]]; then - echo_failed - error "$(cat "$log_file")" -else - echo_ok -fi - - -blank_line - echononl "Extract the Keycloak Service files.." tar -C "${KEYCLOAK_BASE_INSTALL_PATH}" -xvzf "${working_dir}/${DOWNLOAD_ARCHIVE}" > "$log_file" 2>&1 if [[ $? -ne 0 ]]; then echo_failed - error "$(cat "$log_file")" + fatal "Extracting the Keycloak archive failed: $(cat "$log_file")" else echo_ok fi @@ -787,19 +748,7 @@ if [[ -f "${KEYCLOAK_INSTALL_DIR}/conf/keycloak.conf" ]] ; then if [[ $? -ne 0 ]]; then echo_failed - error "$(cat "$log_file")" - - if $terminal ; then - - echononl "\033[1mcontinue anyway\033[m [yes/no]: " - read OK - while [[ "${OK,,}" != "yes" ]] && [[ "${OK,,}" != "no" ]] ; do - echononl "Wrong entry! - repeat [yes/nno]: " - read OK - done - [[ $OK = "yes" ]] || fatal "Abbruch durch User" - - fi + fatal "Saving the original keycloak.conf failed: $(cat "$log_file")" else echo_ok @@ -816,7 +765,7 @@ _failed=false : > "$log_file" -SRC_DIR="${CUR_INSTALL_DIR}.${backup_date}/conf" +SRC_DIR="${CUR_INSTALL_DIR}/conf" DST_DIR="${KEYCLOAK_INSTALL_DIR}/conf" for ext in conf json; do @@ -864,26 +813,26 @@ done if ${_failed} ; then echo_failed - error "$(cat "$log_file")" + fatal "Copying configuration files failed: $(cat "$log_file")" else echo_ok fi echononl "Copy 'providers/ directory from the previous installation to the new installation." -cp -a "${CUR_INSTALL_DIR}.${backup_date}/providers" "${KEYCLOAK_INSTALL_DIR}/" > "$log_file" 2>&1 +cp -a "${CUR_INSTALL_DIR}/providers/." "${KEYCLOAK_INSTALL_DIR}/providers/" > "$log_file" 2>&1 if [[ $? -ne 0 ]]; then echo_failed - error "$(cat "$log_file")" + fatal "Copying providers failed: $(cat "$log_file")" else echo_ok fi echononl "Copy 'themes/ directory from the previous installation to the new installation." -cp -a "${CUR_INSTALL_DIR}.${backup_date}/themes" "${KEYCLOAK_INSTALL_DIR}/" > "$log_file" 2>&1 +cp -a "${CUR_INSTALL_DIR}/themes/." "${KEYCLOAK_INSTALL_DIR}/themes/" > "$log_file" 2>&1 if [[ $? -ne 0 ]]; then echo_failed - error "$(cat "$log_file")" + fatal "Copying themes failed: $(cat "$log_file")" else echo_ok fi @@ -893,7 +842,7 @@ echononl "Set ownbership of installation directory '${KEYCLOAK_INSTALL_DIR}'.." chown -R ${KEYCLOAK_USER}:${KEYCLOAK_GROUP} "${KEYCLOAK_INSTALL_DIR}" > "$log_file" 2>&1 if [[ $? -ne 0 ]]; then echo_failed - error "$(cat "$log_file")" + fatal "Setting ownership failed: $(cat "$log_file")" else echo_ok fi @@ -902,155 +851,50 @@ echononl "Give write permissions to the keycloak group.." chmod -R g+w "${KEYCLOAK_INSTALL_DIR}" > "$log_file" 2>&1 if [[ $? -ne 0 ]]; then echo_failed - error "$(cat "$log_file")" + fatal "Setting group permissions failed: $(cat "$log_file")" else echo_ok fi -echononl "Symlink keycloak -> keycloak-${KEYCLOAK_VERSION} .." -ln -s "keycloak-${KEYCLOAK_VERSION}" "${KEYCLOAK_BASE_INSTALL_PATH}/keycloak" > "$log_file" 2>&1 -if [[ $? -ne 0 ]]; then - echo_failed - error "$(cat "$log_file")" -else - echo_ok -fi - - -_certs_installed=false -echononl "Run script 'check_cert_for_keycloak.sh'.." -if [[ -x "/root/bin/monitoring/check_cert_for_keycloak.sh" ]] ; then - /root/bin/monitoring/check_cert_for_keycloak.sh > "$log_file" 2>&1 - if [[ $? -ne 0 ]]; then - echo_failed - cat "$log_file" - else - echo_ok - _certs_installed=true - fi -else - warn "Certificate/Key for ${FQHN_HOSTNAME} cannot be provided." -fi - - -echononl "Add a cronjob for checking cert.." -if [[ -f "$crontab_backup_file" ]]; then - if ! grep -iq -E "/root/bin/monitoring/check_cert_for_keycloak.sh" "$crontab_backup_file" > /dev/null 2>&1; then - cat <> $crontab_backup_file - -# Check if cert for Keycloak service is up-to-date -# -51 05 * * * /root/bin/monitoring/check_cert_for_keycloak.sh -EOF - if [[ "$?" -ne 0 ]] ; then - echo_failed - else - echo_ok - fi - - else - echo_skipped - fi - -elif [[ -f "/var/spool/cron/crontabs/root" ]] ; then - - if ! grep -i -E "/root/bin/monitoring/check_cert_for_keycloak.sh" /var/spool/cron/crontabs/root > /dev/null 2>&1; then - installation_failed=false - crontab -l > /tmp/tmp_crontab 2> $log_file - if [[ "$?" -ne 0 ]] ; then - installation_failed=true - fi - - cat << EOF >> /tmp/tmp_crontab 2>> $log_file - -# Check if cert for Keycloak service is up-to-date -# -51 05 * * * /root/bin/monitoring/check_cert_for_keycloak.sh -EOF - if [[ "$?" -ne 0 ]] ; then - installation_failed=true - fi - crontab /tmp/tmp_crontab > /dev/null 2>> $log_file - if [[ "$?" -ne 0 ]] ; then - installation_failed=true - fi - rm /tmp/tmp_crontab > /dev/null 2>> $log_file - if [[ "$?" -ne 0 ]] ; then - installation_failed=true - fi - if ! $installation_failed ; then - echo_ok - else - echo_failed - error "Adding cronjob for checking cert failed!" - fi - else - echo_skipped - fi -fi - blank_line echononl "Creates a new and optimized server image.." -${KEYCLOAK_BASE_INSTALL_PATH}/keycloak/bin/kc.sh build > "$log_file" 2>&1 +"${KEYCLOAK_INSTALL_DIR}/bin/kc.sh" build > "$log_file" 2>&1 if [[ $? -ne 0 ]]; then echo_failed - error "$(cat "$log_file")" - - if $terminal ; then - - echononl "\033[1mcontinue anyway\033[m [yes/no]: " - read OK - while [[ "${OK,,}" != "yes" ]] && [[ "${OK,,}" != "no" ]] ; do - echononl "Wrong entry! - repeat [yes/nno]: " - read OK - done - [[ $OK = "yes" ]] || fatal "Abbruch durch User" - - fi - + fatal "Creating the optimized server image failed: $(cat "$log_file")" else echo_ok fi -echononl "Wait 20 seconds for the keycloak service to be fully created." -echo_wait -sleep 20 -${KEYCLOAK_BASE_INSTALL_PATH}/keycloak/bin/kc.sh show-config > "${working_dir}/current-configuration.txt" 2> "$log_file" +echononl "Write the effective configuration of the new installation.." +"${KEYCLOAK_INSTALL_DIR}/bin/kc.sh" show-config > "${working_dir}/current-configuration.txt" 2> "$log_file" if [[ $? -ne 0 ]]; then echo_failed - error "$(cat "$log_file")" + fatal "Reading the effective configuration failed: $(cat "$log_file")" else echo_ok fi blank_line +echononl "Atomically switch symlink keycloak -> keycloak-${KEYCLOAK_VERSION} .." +NEW_SYMLINK="${KEYCLOAK_BASE_INSTALL_PATH}/.keycloak.new.$$" +if ! ln -s "keycloak-${KEYCLOAK_VERSION}" "${NEW_SYMLINK}" > "$log_file" 2>&1 || \ + ! mv -Tf "${NEW_SYMLINK}" "${KEYCLOAK_BASE_INSTALL_PATH}/keycloak" >> "$log_file" 2>&1 ; then + rm -f "${NEW_SYMLINK}" + echo_failed + fatal "Switching the Keycloak symlink failed: $(cat "$log_file")" +else + echo_ok + symlink_switched=true +fi + echononl "Start Keycloak Service" systemctl start keycloak.service > "$log_file" 2>&1 if [[ $? -ne 0 ]]; then echo_failed - error "$(cat "$log_file")" - - if $terminal ; then - - echo " Perhaps it is enough to wait a moment and then start the:" - echo "" - echo " The command to start the keycloak service is:" - echo "" - echo -e " \033[33msystemctl start keycloak.service\033[m" - echo "" - - echononl "\033[1mcontinue anyway\033[m [yes/no]: " - read OK - while [[ "${OK,,}" != "yes" ]] && [[ "${OK,,}" != "no" ]] ; do - echononl "Wrong entry! - repeat [yes/nno]: " - read OK - done - [[ $OK = "yes" ]] || fatal "Abbruch durch User" - - fi - + fatal "Starting Keycloak failed. The symlink points to the new version; the old installation remains at '${CUR_INSTALL_DIR}'. Database rollback may be required before starting the old version. Details: $(cat "$log_file")" else echo_ok fi @@ -1069,62 +913,35 @@ echo echo -e "\033[37m\033[1mSome post-installation stuff..\033[m" echo -_cron_reenabled=false -echononl "Reenable previously saved crontab from '$(basename "${crontab_backup_file}")'.." -if [[ -f "${crontab_backup_file}" ]] ; then - crontab ${crontab_backup_file} > $log_file 2>&1 - if [[ $? -eq 0 ]]; then - echo_ok - _cron_reenabled=true - else - echo_failed - error "$(cat $log_file)" - fi -else - echo_skipped -fi - -blank_line - - echononl "Wait until the Keycloak service has started completely." echo_wait declare -i index=0 -declare -i _max_secs_waiting=40 +declare -i _max_secs_waiting=60 keycloak_service_started=false while true ; do # Try to establish a connection to localhost:8080 # - if $(curl -s -o /dev/null -I http://localhost:8080) ; then + if systemctl is-active --quiet keycloak.service && \ + curl --fail --silent --show-error \ + -H "Host: ${FQHN_HOSTNAME}" \ + -H "X-Forwarded-Host: ${FQHN_HOSTNAME}" \ + -H "X-Forwarded-Proto: https" \ + "http://127.0.0.1:8080/realms/master/.well-known/openid-configuration" \ + -o "${LOCK_DIR}/openid-configuration.json" 2> "$log_file" && \ + grep -Eq '"issuer"[[:space:]]*:[[:space:]]*"https://'"${FQHN_HOSTNAME//./\.}"'/realms/master"' \ + "${LOCK_DIR}/openid-configuration.json" ; then echo_ok keycloak_service_started=true break fi - if [[ ${index} -gt ${_max_secs_waiting} ]]; then + if [[ ${index} -ge ${_max_secs_waiting} ]]; then echo_failed - error "Could not connect to loacalhost on port 8080 after about 20 seconds!" - break + journalctl -u keycloak.service -n 50 --no-pager > "$log_file" 2>&1 + fatal "Keycloak did not become healthy within ${_max_secs_waiting} seconds or returned an unexpected issuer. The old installation remains at '${CUR_INSTALL_DIR}'. Do not start it against a possibly migrated database without checking compatibility or restoring '${DB_DUMP_FILE}'. Last log messages:\n$(cat "$log_file")" fi (( index++ )) sleep 1 done -blank_line - -echononl "Remove previously saved crontab file '$(basename "${crontab_backup_file}")'.." -if ${_cron_reenabled} ; then - rm "${crontab_backup_file}" > $log_file 2>&1 - if [[ $? -eq 0 ]]; then - echo_ok - else - echo_failed - error "$(cat $log_file)" - fi -else - echo_skipped -fi - - clean_up 0 -