Systemd & Services: Managing Modern Linux Systems

# Basic service commands
systemctl status nginx           # Check if service is running
systemctl start nginx            # Start service
systemctl stop nginx             # Stop service
systemctl restart nginx          # Restart service
systemctl reload nginx           # Reload configuration (without restart)
systemctl enable nginx           # Start at boot
systemctl disable nginx          # Don't start at boot

# Check service state
systemctl is-active nginx        # Check if running (returns active/inactive)
systemctl is-enabled nginx       # Check if enabled at boot
systemctl is-failed nginx        # Check if failed

# List services
systemctl list-units --type=service           # All running services
systemctl list-units --type=service --all     # All services (including stopped)
systemctl list-unit-files --type=service      # All service files

# Manage system
systemctl reboot                 # Reboot the system
systemctl poweroff               # Shutdown
systemctl suspend                # Suspend to RAM
systemctl hibernate              # Hibernate to disk

# Basic log viewing
journalctl                     # All logs (uses less pager)
journalctl -xe                 # Show last logs and explain
journalctl -f                  # Follow logs (like tail -f)
journalctl -n 50               # Last 50 lines

# Filter by service
journalctl -u nginx            # Logs for nginx service
journalctl -u nginx -u mysql   # Multiple services

# Filter by time
journalctl --since "2025-01-17"
journalctl --since "1 hour ago"
journalctl --since yesterday
journalctl --since "2025-01-17 10:00:00" --until "2025-01-17 12:00:00"

# Filter by priority
journalctl -p err               # Errors only
journalctl -p 3                 # Same (0=emerg, 1=alert, 2=crit, 3=err, 4=warning, 5=notice, 6=info, 7=debug)
journalctl -p err -p warning    # Errors and warnings

# Output formats
journalctl -o json              # JSON format
journalctl -o verbose           # Verbose with all fields
journalctl -o cat               # Just message (no metadata)

# Disk usage
journalctl --disk-usage

# Maintenance
journalctl --vacuum-size=500M   # Keep logs under 500MB
journalctl --vacuum-time=30d    # Keep logs for 30 days

# Service file location
# /etc/systemd/system/myapp.service

sudo nano /etc/systemd/system/myapp.service

[Unit]
Description=My Python Web Application
After=network.target mysql.service
Requires=mysql.service
Wants=mysql.service
Documentation=https://example.com/docs

[Service]
Type=simple
User=appuser
Group=appgroup
WorkingDirectory=/opt/myapp
Environment="APP_ENV=production"
Environment="DB_HOST=localhost"
EnvironmentFile=/etc/myapp/secrets.env

# Start command
ExecStart=/usr/bin/python3 /opt/myapp/app.py
ExecStop=/usr/bin/kill -TERM $MAINPID
ExecReload=/bin/kill -HUP $MAINPID
Restart=on-failure
RestartSec=10

# Process limits
LimitNOFILE=65536
MemoryMax=1G
CPUQuota=200%

# Security
PrivateTmp=yes
NoNewPrivileges=yes
ProtectSystem=strict
ReadWritePaths=/var/log/myapp /var/lib/myapp

[Install]
WantedBy=multi-user.target

# After creating the service file
sudo systemctl daemon-reload   # Reload systemd to read new service
sudo systemctl start myapp     # Start service
sudo systemctl enable myapp    # Enable at boot
sudo systemctl status myapp    # Check status

# View logs
journalctl -u myapp -f

# Check for errors
sudo systemctl status myapp -l  # Full status with details

# Timer file: /etc/systemd/system/backup.timer

[Unit]
Description=Run backup daily
Requires=backup.service

[Timer]
OnCalendar=daily
OnCalendar=*-*-* 02:00:00      # Run at 2 AM daily
OnCalendar=Mon..Fri 09:00:00   # Weekdays at 9 AM
Persistent=true                 # Run missed timers after boot

# Alternative time specs
# OnBootSec=15min               # 15 minutes after boot
# OnUnitActiveSec=1h            # 1 hour after last activation
# RandomizedDelaySec=30min      # Random delay between 0-30min

[Install]
WantedBy=timers.target

# Service file: /etc/systemd/system/backup.service
[Unit]
Description=Backup script

[Service]
Type=oneshot
ExecStart=/usr/local/bin/backup.sh
StandardOutput=journal

[Install]
WantedBy=multi-user.target

# Enable and start timer
sudo systemctl daemon-reload
sudo systemctl enable backup.timer
sudo systemctl start backup.timer

# Manage timers
systemctl list-timers          # List all active timers
systemctl list-timers --all    # Including inactive
systemctl status backup.timer  # Check timer status

# Test timer
systemctl start backup.service  # Run service manually

# Create simple Python web app
sudo mkdir -p /opt/sampleapp
sudo cat > /opt/sampleapp/app.py << 'EOF'
#!/usr/bin/env python3
from http.server import HTTPServer, BaseHTTPRequestHandler
import datetime

class Handler(BaseHTTPRequestHandler):
    def do_GET(self):
        self.send_response(200)
        self.send_header('Content-type', 'text/html')
        self.end_headers()
        self.wfile.write(f"""
        <html>
            <body>
                <h1>Hello from Systemd!</h1>
                <p>Time: {datetime.datetime.now()}</p>
                <p>Hostname: {self.server.server_name}</p>
            </body>
        </html>
        """.encode())

if __name__ == '__main__':
    server = HTTPServer(('0.0.0.0', 8080), Handler)
    print('Starting server on port 8080...')
    server.serve_forever()
EOF

sudo chmod +x /opt/sampleapp/app.py

# Create system user
sudo useradd -r -s /bin/false sampleapp

# Create service file
sudo cat > /etc/systemd/system/sampleapp.service << 'EOF'
[Unit]
Description=Sample Python Web App
After=network.target

[Service]
Type=simple
User=sampleapp
Group=sampleapp
WorkingDirectory=/opt/sampleapp
ExecStart=/usr/bin/python3 /opt/sampleapp/app.py
Restart=on-failure
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF

# Start the service
sudo systemctl daemon-reload
sudo systemctl start sampleapp
sudo systemctl enable sampleapp
sudo systemctl status sampleapp

# Test the app
curl http://localhost:8080

# When a service fails to start
sudo systemctl status myservice

# Common issues and fixes

# 1. Permission issues - check user/group
ls -la /opt/myapp/
# Fix: chown -R appuser:appgroup /opt/myapp/

# 2. Environment variables missing
sudo systemctl show myservice | grep Environment
# Fix: Add Environment= lines or EnvironmentFile=

# 3. Working directory issues
# Fix: Add WorkingDirectory=/path/to/app

# 4. Port already in use
sudo netstat -tulpn | grep :8080
# Fix: Change port or stop conflicting service

# 5. Syntax error in service file
sudo systemd-analyze verify /etc/systemd/system/myservice.service

# View full logs
journalctl -u myservice -n 50 --no-pager
journalctl -u myservice -f  # Follow logs in real-time