refactor(storage): remove MinIO backend, filesystem-only storage
Remove the MinIO/S3 storage backend entirely. The filesystem backend is fully implemented, already used in production, and a migrate-storage tool exists for any remaining MinIO deployments to migrate beforehand. Changes: - Delete MinIO client implementation (internal/storage/storage.go) - Delete migrate-storage tool (cmd/migrate-storage, scripts/migrate-storage.sh) - Remove MinIO service, volumes, and env vars from all Docker Compose files - Simplify StorageConfig: remove Endpoint, AccessKey, SecretKey, Bucket, UseSSL, Region fields; add SILO_STORAGE_ROOT_DIR env override - Change all SQL COALESCE defaults from 'minio' to 'filesystem' - Add migration 020 to update column defaults to 'filesystem' - Remove minio-go/v7 dependency (go mod tidy) - Update all config examples, setup scripts, docs, and tests
This commit is contained in:
@@ -8,7 +8,6 @@
|
||||
# - SSH access to the target host
|
||||
# - /etc/silo/silod.env must exist on target with credentials filled in
|
||||
# - PostgreSQL reachable from target (set SILO_DB_HOST to override)
|
||||
# - MinIO reachable from target (set SILO_MINIO_HOST to override)
|
||||
#
|
||||
# Environment variables:
|
||||
# SILO_DEPLOY_TARGET - target host (default: silo.example.internal)
|
||||
|
||||
@@ -1,108 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Migrate storage from MinIO to filesystem on a remote Silo host.
|
||||
#
|
||||
# Builds the migrate-storage binary locally, uploads it to the target host,
|
||||
# then runs it over SSH using credentials from /etc/silo/silod.env.
|
||||
#
|
||||
# Usage: ./scripts/migrate-storage.sh <silo-host> <psql-host> <minio-host> [flags...]
|
||||
#
|
||||
# Examples:
|
||||
# ./scripts/migrate-storage.sh silo.kindred.internal psql.kindred.internal minio.kindred.internal -dry-run -verbose
|
||||
# ./scripts/migrate-storage.sh silo.kindred.internal psql.kindred.internal minio.kindred.internal
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [ $# -lt 3 ]; then
|
||||
echo "Usage: $0 <silo-host> <psql-host> <minio-host> [flags...]"
|
||||
echo " flags are passed to migrate-storage (e.g. -dry-run -verbose)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TARGET="$1"
|
||||
DB_HOST="$2"
|
||||
MINIO_HOST="$3"
|
||||
shift 3
|
||||
EXTRA_FLAGS="$*"
|
||||
|
||||
DEST_DIR="/opt/silo/data"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="${SCRIPT_DIR}/.."
|
||||
|
||||
echo "=== Migrate Storage: MinIO -> Filesystem ==="
|
||||
echo " Target: ${TARGET}"
|
||||
echo " DB host: ${DB_HOST}"
|
||||
echo " MinIO: ${MINIO_HOST}"
|
||||
echo " Dest: ${DEST_DIR}"
|
||||
[ -n "$EXTRA_FLAGS" ] && echo " Flags: ${EXTRA_FLAGS}"
|
||||
echo ""
|
||||
|
||||
# --- Build locally ---
|
||||
echo "[1/3] Building migrate-storage binary..."
|
||||
cd "$PROJECT_DIR"
|
||||
GOOS=linux GOARCH=amd64 go build -o migrate-storage ./cmd/migrate-storage
|
||||
echo " Built: $(du -h migrate-storage | cut -f1)"
|
||||
|
||||
# --- Upload ---
|
||||
echo "[2/3] Uploading to ${TARGET}..."
|
||||
scp migrate-storage "${TARGET}:/tmp/migrate-storage"
|
||||
rm -f migrate-storage
|
||||
|
||||
# --- Run remotely ---
|
||||
echo "[3/3] Running migration on ${TARGET}..."
|
||||
ssh "$TARGET" DB_HOST="$DB_HOST" MINIO_HOST="$MINIO_HOST" DEST_DIR="$DEST_DIR" EXTRA_FLAGS="$EXTRA_FLAGS" bash -s <<'REMOTE'
|
||||
set -euo pipefail
|
||||
|
||||
CONFIG_DIR="/etc/silo"
|
||||
|
||||
# Source credentials
|
||||
if [ ! -f "$CONFIG_DIR/silod.env" ]; then
|
||||
echo "ERROR: $CONFIG_DIR/silod.env not found on $(hostname)"
|
||||
exit 1
|
||||
fi
|
||||
set -a
|
||||
source "$CONFIG_DIR/silod.env"
|
||||
set +a
|
||||
|
||||
# Ensure destination directory exists
|
||||
sudo mkdir -p "$DEST_DIR"
|
||||
sudo chown silo:silo "$DEST_DIR" 2>/dev/null || true
|
||||
|
||||
chmod +x /tmp/migrate-storage
|
||||
|
||||
# Write temporary config with the provided hosts
|
||||
cat > /tmp/silo-migrate.yaml <<EOF
|
||||
database:
|
||||
host: "${DB_HOST}"
|
||||
port: 5432
|
||||
name: "silo"
|
||||
user: "silo"
|
||||
password: "${SILO_DB_PASSWORD}"
|
||||
sslmode: "require"
|
||||
max_connections: 5
|
||||
|
||||
storage:
|
||||
endpoint: "${MINIO_HOST}:9000"
|
||||
access_key: "${SILO_MINIO_ACCESS_KEY}"
|
||||
secret_key: "${SILO_MINIO_SECRET_KEY}"
|
||||
bucket: "silo"
|
||||
use_ssl: false
|
||||
region: "us-east-1"
|
||||
EOF
|
||||
chmod 600 /tmp/silo-migrate.yaml
|
||||
|
||||
echo " Config written to /tmp/silo-migrate.yaml"
|
||||
echo " Starting migration..."
|
||||
echo ""
|
||||
|
||||
# Run the migration
|
||||
/tmp/migrate-storage -config /tmp/silo-migrate.yaml -dest "$DEST_DIR" $EXTRA_FLAGS
|
||||
|
||||
# Clean up
|
||||
rm -f /tmp/silo-migrate.yaml /tmp/migrate-storage
|
||||
echo ""
|
||||
echo " Cleaned up temp files."
|
||||
REMOTE
|
||||
|
||||
echo ""
|
||||
echo "=== Migration complete ==="
|
||||
echo " Files written to ${TARGET}:${DEST_DIR}"
|
||||
@@ -138,12 +138,6 @@ fi
|
||||
PG_PASSWORD_DEFAULT="$(generate_secret 16)"
|
||||
prompt_secret POSTGRES_PASSWORD "PostgreSQL password" "$PG_PASSWORD_DEFAULT"
|
||||
|
||||
# MinIO
|
||||
MINIO_AK_DEFAULT="$(generate_secret 10)"
|
||||
MINIO_SK_DEFAULT="$(generate_secret 16)"
|
||||
prompt_secret MINIO_ACCESS_KEY "MinIO access key" "$MINIO_AK_DEFAULT"
|
||||
prompt_secret MINIO_SECRET_KEY "MinIO secret key" "$MINIO_SK_DEFAULT"
|
||||
|
||||
# OpenLDAP
|
||||
LDAP_ADMIN_PW_DEFAULT="$(generate_secret 16)"
|
||||
prompt_secret LDAP_ADMIN_PASSWORD "LDAP admin password" "$LDAP_ADMIN_PW_DEFAULT"
|
||||
@@ -173,10 +167,6 @@ cat > "${OUTPUT_DIR}/.env" << EOF
|
||||
# PostgreSQL
|
||||
POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
||||
|
||||
# MinIO
|
||||
MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY}
|
||||
MINIO_SECRET_KEY=${MINIO_SECRET_KEY}
|
||||
|
||||
# OpenLDAP
|
||||
LDAP_ADMIN_PASSWORD=${LDAP_ADMIN_PASSWORD}
|
||||
LDAP_USERS=${LDAP_USERS}
|
||||
@@ -235,12 +225,9 @@ database:
|
||||
max_connections: 10
|
||||
|
||||
storage:
|
||||
endpoint: "minio:9000"
|
||||
access_key: "${SILO_MINIO_ACCESS_KEY}"
|
||||
secret_key: "${SILO_MINIO_SECRET_KEY}"
|
||||
bucket: "silo-files"
|
||||
use_ssl: false
|
||||
region: "us-east-1"
|
||||
backend: "filesystem"
|
||||
filesystem:
|
||||
root_dir: "/var/lib/silo/data"
|
||||
|
||||
schemas:
|
||||
directory: "/etc/silo/schemas"
|
||||
@@ -306,8 +293,6 @@ echo " deployments/config.docker.yaml - server configuration"
|
||||
echo ""
|
||||
echo -e "${BOLD}Credentials:${NC}"
|
||||
echo " PostgreSQL: silo / ${POSTGRES_PASSWORD}"
|
||||
echo " MinIO: ${MINIO_ACCESS_KEY} / ${MINIO_SECRET_KEY}"
|
||||
echo " MinIO Console: http://localhost:9001"
|
||||
echo " LDAP Admin: cn=admin,dc=silo,dc=local / ${LDAP_ADMIN_PASSWORD}"
|
||||
echo " LDAP User: ${LDAP_USERS} / ${LDAP_PASSWORDS}"
|
||||
echo " Silo Admin: ${SILO_ADMIN_USERNAME} / ${SILO_ADMIN_PASSWORD} (local fallback)"
|
||||
|
||||
@@ -30,7 +30,6 @@ INSTALL_DIR="/opt/silo"
|
||||
CONFIG_DIR="/etc/silo"
|
||||
GO_VERSION="1.24.0"
|
||||
DB_HOST="${SILO_DB_HOST:-psql.example.internal}"
|
||||
MINIO_HOST="${SILO_MINIO_HOST:-minio.example.internal}"
|
||||
|
||||
log_info() { echo -e "${BLUE}[INFO]${NC} $*"; }
|
||||
log_success() { echo -e "${GREEN}[OK]${NC} $*"; }
|
||||
@@ -165,11 +164,6 @@ if [[ ! -f "${ENV_FILE}" ]]; then
|
||||
# Database: silo, User: silo
|
||||
SILO_DB_PASSWORD=
|
||||
|
||||
# MinIO credentials (${MINIO_HOST})
|
||||
# User: silouser
|
||||
SILO_MINIO_ACCESS_KEY=silouser
|
||||
SILO_MINIO_SECRET_KEY=
|
||||
|
||||
# Authentication
|
||||
# Session secret (required when auth is enabled)
|
||||
SILO_SESSION_SECRET=
|
||||
@@ -225,10 +219,7 @@ echo ""
|
||||
echo "2. Verify database connectivity:"
|
||||
echo " psql -h ${DB_HOST} -U silo -d silo -c 'SELECT 1'"
|
||||
echo ""
|
||||
echo "3. Verify MinIO connectivity:"
|
||||
echo " curl -I http://${MINIO_HOST}:9000/minio/health/live"
|
||||
echo ""
|
||||
echo "4. Run the deployment:"
|
||||
echo "3. Run the deployment:"
|
||||
echo " sudo ${INSTALL_DIR}/src/scripts/deploy.sh"
|
||||
echo ""
|
||||
echo "After deployment, manage the service with:"
|
||||
|
||||
Reference in New Issue
Block a user