🚨 Recovery Procedures

systemd provides two recovery targets for when normal boot fails:

# Method 1: Boot into rescue mode via GRUB
# At the GRUB menu, press 'e' to edit the boot entry
# Find the line starting with 'linux' and append:
# systemd.unit=rescue.target
# Press Ctrl-X to boot

# Method 2: Append to kernel line at GRUB
# systemd.unit=emergency.target  (even more minimal -- read-only root)

# Once in rescue/emergency mode:
systemctl list-units --failed    # see what failed
journalctl -xb                   # full boot log with explanations

# If filesystem is read-only in emergency mode, remount rw:
mount -o remount,rw /

# Fix the problem, then exit rescue mode
systemctl default    # try normal boot
# or reboot
reboot

# Boot from rescue/live media first if GRUB is completely broken
# From rescue environment, mount the system and chroot in (see Example 8)

# Once you have a working shell (rescue mode or chroot):

# Reinstall GRUB to the MBR (BIOS systems)
sudo grub2-install /dev/sda
sudo grub2-mkconfig -o /boot/grub2/grub.cfg

# UEFI systems
sudo grub2-install --target=x86_64-efi --efi-directory=/boot/efi
sudo grub2-mkconfig -o /boot/grub2/grub.cfg

# On Debian/Ubuntu
sudo grub-install /dev/sda
sudo update-grub

# Regenerate GRUB config (after kernel update or config change)
sudo grub2-mkconfig -o /boot/grub2/grub.cfg    # RHEL
sudo update-grub                                # Debian/Ubuntu

# View current GRUB entries
sudo awk -F\' '/menuentry / {print $2}' /boot/grub2/grub.cfg

# Set default boot entry
sudo grub2-set-default 0    # boot first entry by default
sudo grub2-editenv list     # view saved default

# fsck runs automatically on boot if filesystem is unclean
# To run manually the filesystem MUST be unmounted first

# Check which filesystem needs repair (from boot error or dmesg)
dmesg | grep -i "error\|corrupt\|bad block"
journalctl -b | grep -i "fsck\|ext4\|xfs"

# Unmount before fsck (NEVER run fsck on a mounted filesystem)
sudo umount /dev/sdb1

# Run fsck -- automatically fix errors
sudo fsck -y /dev/sdb1           # ext2/3/4 -- -y answers yes to all
sudo fsck.ext4 -y /dev/sdb1     # explicit ext4
sudo xfs_repair /dev/sdb1       # XFS filesystems

# If root filesystem needs repair -- boot to rescue/emergency
# then remount read-only and run fsck:
mount -o remount,ro /
fsck -y /dev/sda1

# Force fsck on next boot (for root filesystem)
sudo touch /forcefsck           # older method
sudo tune2fs -C 1 /dev/sda1     # set mount count to trigger fsck
sudo shutdown -rF now           # reboot with forced fsck

# Check filesystem health without repairing
sudo fsck -n /dev/sdb1          # dry run -- report only
sudo tune2fs -l /dev/sda1 | grep -i "state\|mount count\|check"

# Method 1: Via GRUB (physical/console access required)

# Step 1: Reboot. At GRUB menu press 'e' to edit
# Step 2: Find the line starting with 'linux'
# Step 3: At the END of that line, add:
#    rd.break
# Step 4: Press Ctrl-X to boot
# You will get a root shell before the filesystem mounts rw

# Step 5: Remount root rw and change password
mount -o remount,rw /sysroot
chroot /sysroot

# Step 6: Change root password
passwd root

# Step 7: If SELinux is enabled, relabel on next boot
touch /.autorelabel

# Step 8: Exit and reboot
exit
reboot

# Method 2: From rescue media / live boot
# Boot live media, mount the system, chroot in (see Example 8)
# Then simply: passwd root

The system is degraded because a filesystem is full. Services are failing. You need space right now:

# Find what is using space -- fast
df -h                              # which filesystem is full?
du -sh /* 2>/dev/null | sort -rh | head -10    # top directories
du -sh /var/* | sort -rh | head -10
du -sh /var/log/* | sort -rh | head -10

# Find the largest files on a filesystem
find /var -type f -size +100M -ls 2>/dev/null | sort -k7 -rn
find / -xdev -type f -size +500M -ls 2>/dev/null

# Quick wins -- safe things to remove

# Clean package cache
sudo dnf clean all          # RHEL -- can free GBs
sudo apt clean              # Debian

# Remove old kernel packages (keep current + 1)
sudo dnf remove $(dnf repoquery --installonly --latest-limit=-2 -q)

# Truncate (not delete) a log that is too large
# DO NOT delete -- the process has it open
sudo truncate -s 0 /var/log/hugefile.log

# Vacuum the systemd journal
sudo journalctl --vacuum-size=100M

# Find and remove old core dumps
sudo find /var/lib/systemd/coredump -type f -delete
sudo find / -name "core" -type f -ls 2>/dev/null

# Check for deleted files still held open (not releasing space)
sudo lsof | grep deleted | awk '{print $7, $9}' | sort -rn | head -10
# Restart the process holding the deleted file to release space

# Restore a single file from tar backup
tar -tzf /backup/etc-20260425.tar.gz | grep "nginx.conf"
tar -xzf /backup/etc-20260425.tar.gz -C / etc/nginx/nginx.conf

# Restore a directory from tar backup
tar -xzf /backup/www-20260425.tar.gz -C /restore/
# Then verify before moving to production:
diff -r /restore/var/www/html/ /var/www/html/
mv /var/www/html/ /var/www/html.broken/
mv /restore/var/www/html/ /var/www/html/

# Restore from rsync snapshot
ls /backup/snapshots/           # pick the right date
rsync -av /backup/snapshots/2026-04-25/ /var/www/html/

# Restore specific files from rsync snapshot
rsync -av /backup/snapshots/2026-04-25/config/ /var/www/html/config/

# Restore MySQL database
gunzip -c /backup/mysql-20260425.sql.gz | mysql -u root -p
# Restore single database
gunzip -c /backup/mysql-20260425.sql.gz | grep -A99999 "Current Database: \`mydb\`" | \
    grep -B99999 "Current Database:" | head -n -2 | mysql -u root -p mydb

# Restore PostgreSQL
gunzip -c /backup/postgres-20260425.sql.gz | psql -U postgres

# First check -- is it still open by a process?
sudo lsof | grep deleted | grep myfile
# If found: copy from /proc/PID/fd/FD before the process closes it
sudo cp /proc/12345/fd/3 /recovered/myfile.txt

# extundelete -- recover deleted files from ext3/ext4
sudo dnf install -y extundelete
sudo umount /dev/sdb1     # must be unmounted
sudo extundelete /dev/sdb1 --restore-all
# Recovered files appear in ./RECOVERED_FILES/

# testdisk -- recover lost partitions and files
sudo dnf install -y testdisk
sudo testdisk /dev/sda    # interactive partition recovery

# photorec -- recover files by signature (works on any filesystem)
sudo photorec /dev/sdb1

# Check if file is in a snapshot (LVM snapshot or rsync snapshot)
ls /backup/snapshots/2026-04-24/var/www/html/deletedfile.php

# Check if file is in the backup from yesterday
tar -tzf /backup/www-20260424.tar.gz | grep deletedfile

When you need to work on a broken system from outside — boot from a live USB or rescue media, then chroot into the installed system:

# Boot from RHEL/Rocky rescue media or any live Linux USB

# Step 1: Identify your disks and partitions
lsblk
fdisk -l

# Step 2: Mount the root filesystem
mount /dev/sda2 /mnt/sysroot      # adjust device as needed

# Step 3: Mount boot partition if separate
mount /dev/sda1 /mnt/sysroot/boot
mount /dev/sda1 /mnt/sysroot/boot/efi    # UEFI systems

# Step 4: Bind mount virtual filesystems
mount --bind /dev  /mnt/sysroot/dev
mount --bind /proc /mnt/sysroot/proc
mount --bind /sys  /mnt/sysroot/sys
mount --bind /run  /mnt/sysroot/run

# Step 5: Chroot into the system
chroot /mnt/sysroot /bin/bash

# Now you are inside the broken system with full access
# Run any repair commands:
passwd root                        # reset root password
grub2-install /dev/sda             # fix GRUB
grub2-mkconfig -o /boot/grub2/grub.cfg
dnf reinstall kernel               # reinstall kernel
vi /etc/fstab                      # fix bad fstab entry

# Step 6: Exit chroot and unmount
exit
umount /mnt/sysroot/dev
umount /mnt/sysroot/proc
umount /mnt/sysroot/sys
umount /mnt/sysroot/run
umount /mnt/sysroot/boot
umount /mnt/sysroot
reboot