refactor(storage): remove MinIO backend, filesystem-only storage

Remove the MinIO/S3 storage backend entirely. The filesystem backend is
fully implemented, already used in production, and a migrate-storage tool
exists for any remaining MinIO deployments to migrate beforehand.

Changes:
- Delete MinIO client implementation (internal/storage/storage.go)
- Delete migrate-storage tool (cmd/migrate-storage, scripts/migrate-storage.sh)
- Remove MinIO service, volumes, and env vars from all Docker Compose files
- Simplify StorageConfig: remove Endpoint, AccessKey, SecretKey, Bucket,
  UseSSL, Region fields; add SILO_STORAGE_ROOT_DIR env override
- Change all SQL COALESCE defaults from 'minio' to 'filesystem'
- Add migration 020 to update column defaults to 'filesystem'
- Remove minio-go/v7 dependency (go mod tidy)
- Update all config examples, setup scripts, docs, and tests
This commit is contained in:
Forbes
2026-02-19 14:36:22 -06:00
parent 12ecffdabe
commit 88d1ab1f97
30 changed files with 104 additions and 849 deletions

View File

@@ -8,7 +8,6 @@
# - SSH access to the target host
# - /etc/silo/silod.env must exist on target with credentials filled in
# - PostgreSQL reachable from target (set SILO_DB_HOST to override)
# - MinIO reachable from target (set SILO_MINIO_HOST to override)
#
# Environment variables:
# SILO_DEPLOY_TARGET - target host (default: silo.example.internal)

View File

@@ -1,108 +0,0 @@
#!/bin/bash
# Migrate storage from MinIO to filesystem on a remote Silo host.
#
# Builds the migrate-storage binary locally, uploads it to the target host,
# then runs it over SSH using credentials from /etc/silo/silod.env.
#
# Usage: ./scripts/migrate-storage.sh <silo-host> <psql-host> <minio-host> [flags...]
#
# Examples:
# ./scripts/migrate-storage.sh silo.kindred.internal psql.kindred.internal minio.kindred.internal -dry-run -verbose
# ./scripts/migrate-storage.sh silo.kindred.internal psql.kindred.internal minio.kindred.internal
set -euo pipefail
if [ $# -lt 3 ]; then
echo "Usage: $0 <silo-host> <psql-host> <minio-host> [flags...]"
echo " flags are passed to migrate-storage (e.g. -dry-run -verbose)"
exit 1
fi
TARGET="$1"
DB_HOST="$2"
MINIO_HOST="$3"
shift 3
EXTRA_FLAGS="$*"
DEST_DIR="/opt/silo/data"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="${SCRIPT_DIR}/.."
echo "=== Migrate Storage: MinIO -> Filesystem ==="
echo " Target: ${TARGET}"
echo " DB host: ${DB_HOST}"
echo " MinIO: ${MINIO_HOST}"
echo " Dest: ${DEST_DIR}"
[ -n "$EXTRA_FLAGS" ] && echo " Flags: ${EXTRA_FLAGS}"
echo ""
# --- Build locally ---
echo "[1/3] Building migrate-storage binary..."
cd "$PROJECT_DIR"
GOOS=linux GOARCH=amd64 go build -o migrate-storage ./cmd/migrate-storage
echo " Built: $(du -h migrate-storage | cut -f1)"
# --- Upload ---
echo "[2/3] Uploading to ${TARGET}..."
scp migrate-storage "${TARGET}:/tmp/migrate-storage"
rm -f migrate-storage
# --- Run remotely ---
echo "[3/3] Running migration on ${TARGET}..."
ssh "$TARGET" DB_HOST="$DB_HOST" MINIO_HOST="$MINIO_HOST" DEST_DIR="$DEST_DIR" EXTRA_FLAGS="$EXTRA_FLAGS" bash -s <<'REMOTE'
set -euo pipefail
CONFIG_DIR="/etc/silo"
# Source credentials
if [ ! -f "$CONFIG_DIR/silod.env" ]; then
echo "ERROR: $CONFIG_DIR/silod.env not found on $(hostname)"
exit 1
fi
set -a
source "$CONFIG_DIR/silod.env"
set +a
# Ensure destination directory exists
sudo mkdir -p "$DEST_DIR"
sudo chown silo:silo "$DEST_DIR" 2>/dev/null || true
chmod +x /tmp/migrate-storage
# Write temporary config with the provided hosts
cat > /tmp/silo-migrate.yaml <<EOF
database:
host: "${DB_HOST}"
port: 5432
name: "silo"
user: "silo"
password: "${SILO_DB_PASSWORD}"
sslmode: "require"
max_connections: 5
storage:
endpoint: "${MINIO_HOST}:9000"
access_key: "${SILO_MINIO_ACCESS_KEY}"
secret_key: "${SILO_MINIO_SECRET_KEY}"
bucket: "silo"
use_ssl: false
region: "us-east-1"
EOF
chmod 600 /tmp/silo-migrate.yaml
echo " Config written to /tmp/silo-migrate.yaml"
echo " Starting migration..."
echo ""
# Run the migration
/tmp/migrate-storage -config /tmp/silo-migrate.yaml -dest "$DEST_DIR" $EXTRA_FLAGS
# Clean up
rm -f /tmp/silo-migrate.yaml /tmp/migrate-storage
echo ""
echo " Cleaned up temp files."
REMOTE
echo ""
echo "=== Migration complete ==="
echo " Files written to ${TARGET}:${DEST_DIR}"

View File

@@ -138,12 +138,6 @@ fi
PG_PASSWORD_DEFAULT="$(generate_secret 16)"
prompt_secret POSTGRES_PASSWORD "PostgreSQL password" "$PG_PASSWORD_DEFAULT"
# MinIO
MINIO_AK_DEFAULT="$(generate_secret 10)"
MINIO_SK_DEFAULT="$(generate_secret 16)"
prompt_secret MINIO_ACCESS_KEY "MinIO access key" "$MINIO_AK_DEFAULT"
prompt_secret MINIO_SECRET_KEY "MinIO secret key" "$MINIO_SK_DEFAULT"
# OpenLDAP
LDAP_ADMIN_PW_DEFAULT="$(generate_secret 16)"
prompt_secret LDAP_ADMIN_PASSWORD "LDAP admin password" "$LDAP_ADMIN_PW_DEFAULT"
@@ -173,10 +167,6 @@ cat > "${OUTPUT_DIR}/.env" << EOF
# PostgreSQL
POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
# MinIO
MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY}
MINIO_SECRET_KEY=${MINIO_SECRET_KEY}
# OpenLDAP
LDAP_ADMIN_PASSWORD=${LDAP_ADMIN_PASSWORD}
LDAP_USERS=${LDAP_USERS}
@@ -235,12 +225,9 @@ database:
max_connections: 10
storage:
endpoint: "minio:9000"
access_key: "${SILO_MINIO_ACCESS_KEY}"
secret_key: "${SILO_MINIO_SECRET_KEY}"
bucket: "silo-files"
use_ssl: false
region: "us-east-1"
backend: "filesystem"
filesystem:
root_dir: "/var/lib/silo/data"
schemas:
directory: "/etc/silo/schemas"
@@ -306,8 +293,6 @@ echo " deployments/config.docker.yaml - server configuration"
echo ""
echo -e "${BOLD}Credentials:${NC}"
echo " PostgreSQL: silo / ${POSTGRES_PASSWORD}"
echo " MinIO: ${MINIO_ACCESS_KEY} / ${MINIO_SECRET_KEY}"
echo " MinIO Console: http://localhost:9001"
echo " LDAP Admin: cn=admin,dc=silo,dc=local / ${LDAP_ADMIN_PASSWORD}"
echo " LDAP User: ${LDAP_USERS} / ${LDAP_PASSWORDS}"
echo " Silo Admin: ${SILO_ADMIN_USERNAME} / ${SILO_ADMIN_PASSWORD} (local fallback)"

View File

@@ -30,7 +30,6 @@ INSTALL_DIR="/opt/silo"
CONFIG_DIR="/etc/silo"
GO_VERSION="1.24.0"
DB_HOST="${SILO_DB_HOST:-psql.example.internal}"
MINIO_HOST="${SILO_MINIO_HOST:-minio.example.internal}"
log_info() { echo -e "${BLUE}[INFO]${NC} $*"; }
log_success() { echo -e "${GREEN}[OK]${NC} $*"; }
@@ -165,11 +164,6 @@ if [[ ! -f "${ENV_FILE}" ]]; then
# Database: silo, User: silo
SILO_DB_PASSWORD=
# MinIO credentials (${MINIO_HOST})
# User: silouser
SILO_MINIO_ACCESS_KEY=silouser
SILO_MINIO_SECRET_KEY=
# Authentication
# Session secret (required when auth is enabled)
SILO_SESSION_SECRET=
@@ -225,10 +219,7 @@ echo ""
echo "2. Verify database connectivity:"
echo " psql -h ${DB_HOST} -U silo -d silo -c 'SELECT 1'"
echo ""
echo "3. Verify MinIO connectivity:"
echo " curl -I http://${MINIO_HOST}:9000/minio/health/live"
echo ""
echo "4. Run the deployment:"
echo "3. Run the deployment:"
echo " sudo ${INSTALL_DIR}/src/scripts/deploy.sh"
echo ""
echo "After deployment, manage the service with:"