⚙️ systemd Deep Dive

/lib/systemd/system/       # Package-installed units (don't edit)
/etc/systemd/system/       # Admin-created / override units (edit here)
/run/systemd/system/       # Runtime units (temporary, not persistent)
~/.config/systemd/user/    # Per-user units (no root needed)

.service   — daemon or one-shot process (nginx, sshd, etc.)
.timer     — schedules tasks (cron replacement)
.socket    — socket-activated services (start on first connection)
.mount     — filesystem mount point
.target    — group of units (equivalent to runlevels)
.path      — watch filesystem path, trigger service on change
.device    — udev-detected hardware device
.slice     — resource control group (cgroup)
.scope     — externally-created processes

# Start / Stop / Restart
$ sudo systemctl start nginx
$ sudo systemctl stop nginx
$ sudo systemctl restart nginx
$ sudo systemctl reload nginx         # reload config without killing process

# Enable / Disable (auto-start at boot)
$ sudo systemctl enable nginx
$ sudo systemctl disable nginx
$ sudo systemctl enable --now nginx   # enable AND start immediately

# Status and info
$ systemctl status nginx
$ systemctl is-active nginx           # active / inactive / failed
$ systemctl is-enabled nginx          # enabled / disabled / masked
$ systemctl list-units --type=service
$ systemctl list-units --type=service --state=failed

# Mask (prevent ANY start, even manually)
$ sudo systemctl mask nginx
$ sudo systemctl unmask nginx
        copy
      

[Unit]
Description=My Application Server
Documentation=https://github.com/saad/myapp
After=network.target postgresql.service
Wants=postgresql.service

[Service]
Type=simple
User=myapp
Group=myapp
WorkingDirectory=/opt/myapp
ExecStart=/opt/myapp/bin/server --port 3000
ExecReload=/bin/kill -HUP $MAINPID
Restart=on-failure
RestartSec=5
StartLimitIntervalSec=60
StartLimitBurst=3

StandardOutput=journal
StandardError=journal
SyslogIdentifier=myapp

# Security hardening:
NoNewPrivileges=yes
PrivateTmp=yes
ProtectSystem=strict
ReadWritePaths=/var/lib/myapp /var/log/myapp
ProtectHome=yes

[Install]
WantedBy=multi-user.target

# After creating/editing:
$ sudo systemctl daemon-reload
$ sudo systemctl enable --now myapp
$ systemctl status myapp
$ journalctl -u myapp -f   # tail logs
        copy
      

# Create a drop-in override:
$ sudo systemctl edit nginx
# Opens editor, creates /etc/systemd/system/nginx.service.d/override.conf

# Example override — increase restart delay:
[Service]
RestartSec=10
LimitNOFILE=65536          # increase open file limit

$ sudo systemctl daemon-reload && sudo systemctl restart nginx
        copy
      

# 1. Create the service unit: /etc/systemd/system/backup.service
[Unit]
Description=Daily Backup Job

[Service]
Type=oneshot
User=root
ExecStart=/usr/local/bin/backup.sh

# 2. Create the timer unit: /etc/systemd/system/backup.timer
[Unit]
Description=Run backup daily at 2am

[Timer]
OnCalendar=*-*-* 02:00:00          # daily at 2am
Persistent=true                     # run if missed (e.g. system was off)
RandomizedDelaySec=10min            # jitter to spread load

[Install]
WantedBy=timers.target

$ sudo systemctl daemon-reload
$ sudo systemctl enable --now backup.timer
$ systemctl list-timers --all
$ systemctl status backup.timer

# OnCalendar examples:
hourly                     # every hour
daily                      # every midnight
*-*-* 02:30:00            # every day at 2:30am
Mon..Fri *-*-* 08:00:00   # weekdays at 8am
*-*-* *:0/15:00           # every 15 minutes
        copy
      

$ journalctl                        # all logs, oldest first
$ journalctl -r                      # newest first (reverse)
$ journalctl -f                      # follow live (tail -f)
$ journalctl -n 100                 # last 100 lines

# Filter by unit:
$ journalctl -u nginx               # nginx logs
$ journalctl -u nginx -u php8.3-fpm # multiple units
$ journalctl -u nginx -f             # follow nginx logs live

# Filter by time:
$ journalctl --since "2024-01-15"
$ journalctl --since "1 hour ago"
$ journalctl -b                      # current boot
$ journalctl -b -1                  # previous boot

# Filter by priority:
$ journalctl -p err                 # errors and above
$ journalctl -p warning..err        # range: warning to error
# Priorities: emerg alert crit err warning notice info debug

# Output formats:
$ journalctl -u nginx -o json-pretty  # structured JSON
$ journalctl -u nginx -o cat          # message only
$ journalctl -u nginx -o short-iso   # ISO 8601 timestamps

# Disk usage and cleanup:
$ journalctl --disk-usage
$ sudo journalctl --vacuum-size=500M  # trim to 500 MB
$ sudo journalctl --vacuum-time=30d   # delete older than 30 days
        copy
      

poweroff.target          ← Runlevel 0 — Shutdown
rescue.target            ← Runlevel 1 — Single user / maintenance
multi-user.target        ← Runlevel 3 — CLI + networking (servers)
graphical.target         ← Runlevel 5 — GUI + networking (desktop)
reboot.target            ← Runlevel 6 — Reboot
emergency.target         ← Minimal shell, read-only root

$ systemctl get-default             # see current default target
$ sudo systemctl set-default multi-user.target   # boot to CLI
$ sudo systemctl set-default graphical.target    # boot to GUI
$ sudo systemctl isolate multi-user.target       # switch now (no reboot)
$ sudo systemctl isolate rescue.target          # maintenance mode
        copy
      

$ systemd-analyze                   # total boot time breakdown
$ systemd-analyze blame             # which services slowed boot (sorted by time)
$ systemd-analyze critical-chain    # critical path to default target
$ systemd-analyze plot > boot.svg   # generate SVG boot chart (open in browser)
$ systemd-analyze verify /etc/systemd/system/myapp.service
$ systemd-analyze security nginx    # security scoring for a service

# Speed up boot by disabling unnecessary services:
$ sudo systemctl disable bluetooth
$ sudo systemctl disable cups        # printing
$ sudo systemctl disable avahi-daemon # mDNS (not needed on servers)
$ sudo systemctl mask ModemManager   # prevent ANY start
        copy
      

# View dependency tree for a service:
$ systemctl list-dependencies nginx

# Check if a service failed:
$ systemctl is-failed nginx

# Restart a service on config change (without manual restart):
$ sudo systemctl reload nginx        # reload config
$ sudo systemctl reload-or-restart nginx  # reload if supported, else restart

# See what triggered a service start:
$ journalctl -u nginx -n 50 | grep "Started\|Stopped"

# Power commands:
$ sudo systemctl reboot
$ sudo systemctl poweroff
$ sudo systemctl suspend
$ sudo systemctl hibernate
        copy