linux-server

What this skill does

# Linux Server Administration (Ubuntu / Debian)

## New Server Setup Script

Save as `/root/setup-server.sh` on the new machine, run once as root, then delete it.

```bash
#!/usr/bin/env bash
# =============================================================================
# Production Server Initial Setup — Ubuntu 22.04 / 24.04 LTS
# Run as root immediately after first login.
# =============================================================================
set -euo pipefail

# ---------------------------------------------------------------------------
# 1. Variables — edit before running
# ---------------------------------------------------------------------------
NEW_USER="deploy"
SSH_PUBLIC_KEY="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAA... user@workstation"
HOSTNAME="app-prod-01"
TIMEZONE="UTC"          # Or e.g. "America/New_York", "Europe/London"
SWAP_SIZE_GB=4          # Set to 0 to skip swap creation

# ---------------------------------------------------------------------------
# 2. Update package lists and apply security patches
# ---------------------------------------------------------------------------
apt-get update -qq
DEBIAN_FRONTEND=noninteractive apt-get upgrade -y -q

# ---------------------------------------------------------------------------
# 3. Set hostname and /etc/hosts
# ---------------------------------------------------------------------------
hostnamectl set-hostname "$HOSTNAME"
# Ensure 127.0.1.1 resolves the hostname (required by some software)
if ! grep -q "$HOSTNAME" /etc/hosts; then
    echo "127.0.1.1  $HOSTNAME" >> /etc/hosts
fi

# ---------------------------------------------------------------------------
# 4. Set timezone
# ---------------------------------------------------------------------------
timedatectl set-timezone "$TIMEZONE"

# ---------------------------------------------------------------------------
# 5. Install essential packages
# ---------------------------------------------------------------------------
DEBIAN_FRONTEND=noninteractive apt-get install -y -q \
    curl wget git vim htop iotop net-tools dnsutils \
    unzip build-essential ca-certificates gnupg lsb-release \
    fail2ban ufw chrony logrotate unattended-upgrades apt-listchanges

# ---------------------------------------------------------------------------
# 6. Configure NTP with chrony
# ---------------------------------------------------------------------------
systemctl enable --now chrony
# Verify synchronisation (should show * on the active source)
chronyc sources -v || true

# ---------------------------------------------------------------------------
# 7. Create sudo user with SSH key access
# ---------------------------------------------------------------------------
if ! id -u "$NEW_USER" &>/dev/null; then
    adduser --disabled-password --gecos "" "$NEW_USER"
fi
usermod -aG sudo "$NEW_USER"

# Set up authorized_keys
SSH_DIR="/home/${NEW_USER}/.ssh"
mkdir -p "$SSH_DIR"
echo "$SSH_PUBLIC_KEY" > "${SSH_DIR}/authorized_keys"
chmod 700 "$SSH_DIR"
chmod 600 "${SSH_DIR}/authorized_keys"
chown -R "${NEW_USER}:${NEW_USER}" "$SSH_DIR"

# ---------------------------------------------------------------------------
# 8. Harden SSH
# ---------------------------------------------------------------------------
cp /etc/ssh/sshd_config /etc/ssh/sshd_config.bak.$(date +%F)
cat > /etc/ssh/sshd_config.d/99-hardening.conf <<'SSHD_CONF'
PermitRootLogin no
PasswordAuthentication no
ChallengeResponseAuthentication no
PubkeyAuthentication yes
X11Forwarding no
AllowTcpForwarding no
MaxAuthTries 3
LoginGraceTime 30
ClientAliveInterval 300
ClientAliveCountMax 2
SSHD_CONF
systemctl reload sshd

# ---------------------------------------------------------------------------
# 9. Configure unattended security upgrades
# ---------------------------------------------------------------------------
cat > /etc/apt/apt.conf.d/20auto-upgrades <<'APT_CONF'
APT::Periodic::Update-Package-Lists "1";
APT::Periodic::Unattended-Upgrade "1";
APT::Periodic::AutocleanInterval "7";
APT_CONF'

cat > /etc/apt/apt.conf.d/50unattended-upgrades <<'APT_UU'
Unattended-Upgrade::Allowed-Origins {
    "${distro_id}:${distro_codename}-security";
    "${distro_id}ESMApps:${distro_codename}-apps-security";
    "${distro_id}ESM:${distro_codename}-infra-security";
};
Unattended-Upgrade::Mail "root";
Unattended-Upgrade::Remove-Unused-Kernel-Packages "true";
Unattended-Upgrade::Automatic-Reboot "false";
APT_UU

# ---------------------------------------------------------------------------
# 10. Create swap file
# ---------------------------------------------------------------------------
if [[ "$SWAP_SIZE_GB" -gt 0 ]] && ! swapon --show | grep -q /swapfile; then
    fallocate -l "${SWAP_SIZE_GB}G" /swapfile
    chmod 600 /swapfile
    mkswap /swapfile
    swapon /swapfile
    echo '/swapfile none swap sw 0 0' >> /etc/fstab
    # Reduce swappiness for applications (default is 60)
    echo 'vm.swappiness=10' > /etc/sysctl.d/60-swap.conf
    sysctl -p /etc/sysctl.d/60-swap.conf
fi

# ---------------------------------------------------------------------------
# 11. Apply kernel tuning (idempotent — see sysctl.d section below)
# ---------------------------------------------------------------------------
cp /dev/stdin /etc/sysctl.d/99-production.conf <<'SYSCTL'
# Network performance
net.core.somaxconn = 65535
net.core.netdev_max_backlog = 65536
net.ipv4.tcp_max_syn_backlog = 65535
net.ipv4.tcp_fin_timeout = 15
net.ipv4.tcp_keepalive_time = 300
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_keepalive_probes = 5
net.ipv4.ip_local_port_range = 1024 65535
net.ipv4.tcp_tw_reuse = 1

# File system
fs.file-max = 2097152
fs.inotify.max_user_watches = 524288

# Virtual memory
vm.swappiness = 10
vm.dirty_ratio = 15
vm.dirty_background_ratio = 5
SYSCTL
sysctl --system

echo ""
echo "=== Setup complete. Log in as '${NEW_USER}' via SSH key before closing this session. ==="
```

---

## sysctl.d Production Tuning Reference

The file above (`/etc/sysctl.d/99-production.conf`) covers the most impactful parameters. Apply changes live without rebooting:

```bash
sudo sysctl --system       # Reload all files in /etc/sysctl.d/
sudo sysctl -p /etc/sysctl.d/99-production.conf  # Reload specific file

# Verify a parameter
sudo sysctl net.core.somaxconn
```

---

## /etc/security/limits.conf — High-Concurrency App

```
# /etc/security/limits.conf
# Changes take effect on next login (not on running processes).

# Application service user (e.g., node app running as 'deploy')
deploy   soft   nofile   65535
deploy   hard   nofile   65535
deploy   soft   nproc    8192
deploy   hard   nproc    8192

# Root (required if app runs as root — avoid this)
root     soft   nofile   65535
root     hard   nofile   65535

# Wildcard fallback for all other users
*        soft   nofile   65535
*        hard   nofile   65535
```

For systemd services, `LimitNOFILE` in the unit file takes precedence over `/etc/security/limits.conf`. Set both.

Verify effective limits of a running process:

```bash
# PID of your app:
cat /proc/$(pgrep -o node)/limits
```

---

## Package Management

```bash
# Update package lists
sudo apt-get update

# Upgrade all packages (non-interactive)
sudo DEBIAN_FRONTEND=noninteractive apt-get upgrade -y

# Install specific package
sudo apt-get install -y nginx

# Remove package and its config files
sudo apt-get purge -y apache2 && sudo apt-get autoremove -y

# Hold a package at its current version (prevent unattended upgrades)
sudo apt-mark hold nginx
sudo apt-mark unhold nginx
sudo apt-mark showhold

# List installed packages
dpkg -l | grep nginx

# Show available versions
apt-cache policy nginx

# Find which package provides a file
dpkg -S /usr/sbin/nginx
apt-file search /usr/sbin/nginx  # needs apt-file package
```

---

## File Permissions

```bash
# Symbolic mode: u=user, g=group, o=others, a=all; r=4, w=2, x=1
chmod 755 /var/www/myapp          # rwxr-xr-x — directory traversable by all
chmod 644 /var/www/myapp/app.js   #