refactor(storage): remove MinIO backend, filesystem-only storage
Remove the MinIO/S3 storage backend entirely. The filesystem backend is fully implemented, already used in production, and a migrate-storage tool exists for any remaining MinIO deployments to migrate beforehand. Changes: - Delete MinIO client implementation (internal/storage/storage.go) - Delete migrate-storage tool (cmd/migrate-storage, scripts/migrate-storage.sh) - Remove MinIO service, volumes, and env vars from all Docker Compose files - Simplify StorageConfig: remove Endpoint, AccessKey, SecretKey, Bucket, UseSSL, Region fields; add SILO_STORAGE_ROOT_DIR env override - Change all SQL COALESCE defaults from 'minio' to 'filesystem' - Add migration 020 to update column defaults to 'filesystem' - Remove minio-go/v7 dependency (go mod tidy) - Update all config examples, setup scripts, docs, and tests
This commit is contained in:
@@ -5,10 +5,6 @@
|
||||
# PostgreSQL
|
||||
POSTGRES_PASSWORD=silodev
|
||||
|
||||
# MinIO
|
||||
MINIO_ACCESS_KEY=silominio
|
||||
MINIO_SECRET_KEY=silominiosecret
|
||||
|
||||
# OpenLDAP
|
||||
LDAP_ADMIN_PASSWORD=ldapadmin
|
||||
LDAP_USERS=siloadmin
|
||||
|
||||
17
Makefile
17
Makefile
@@ -1,8 +1,7 @@
|
||||
.PHONY: build run test test-integration clean migrate fmt lint \
|
||||
docker-build docker-up docker-down docker-logs docker-ps \
|
||||
docker-clean docker-rebuild \
|
||||
web-install web-dev web-build \
|
||||
migrate-storage
|
||||
web-install web-dev web-build
|
||||
|
||||
# =============================================================================
|
||||
# Local Development
|
||||
@@ -57,13 +56,6 @@ tidy:
|
||||
migrate:
|
||||
./scripts/init-db.sh
|
||||
|
||||
# Build and run MinIO → filesystem migration tool
|
||||
# Usage: make migrate-storage DEST=/opt/silo/data [ARGS="--dry-run --verbose"]
|
||||
migrate-storage:
|
||||
go build -o migrate-storage ./cmd/migrate-storage
|
||||
@echo "Built ./migrate-storage"
|
||||
@echo "Run: ./migrate-storage -config <config.yaml> -dest <dir> [-dry-run] [-verbose]"
|
||||
|
||||
# Connect to database (requires psql)
|
||||
db-shell:
|
||||
PGPASSWORD=$${SILO_DB_PASSWORD:-silodev} psql -h $${SILO_DB_HOST:-localhost} -U $${SILO_DB_USER:-silo} -d $${SILO_DB_NAME:-silo}
|
||||
@@ -76,7 +68,7 @@ db-shell:
|
||||
docker-build:
|
||||
docker build -t silo:latest -f build/package/Dockerfile .
|
||||
|
||||
# Start the full stack (postgres + minio + silo)
|
||||
# Start the full stack (postgres + silo)
|
||||
docker-up:
|
||||
docker compose -f deployments/docker-compose.yaml up -d
|
||||
|
||||
@@ -103,9 +95,6 @@ docker-logs-silo:
|
||||
docker-logs-postgres:
|
||||
docker compose -f deployments/docker-compose.yaml logs -f postgres
|
||||
|
||||
docker-logs-minio:
|
||||
docker compose -f deployments/docker-compose.yaml logs -f minio
|
||||
|
||||
# Show running containers
|
||||
docker-ps:
|
||||
docker compose -f deployments/docker-compose.yaml ps
|
||||
@@ -175,7 +164,7 @@ help:
|
||||
@echo ""
|
||||
@echo "Docker:"
|
||||
@echo " docker-build - Build Docker image"
|
||||
@echo " docker-up - Start full stack (postgres + minio + silo)"
|
||||
@echo " docker-up - Start full stack (postgres + silo)"
|
||||
@echo " docker-down - Stop the stack"
|
||||
@echo " docker-clean - Stop and remove volumes (deletes data)"
|
||||
@echo " docker-logs - View all logs"
|
||||
|
||||
@@ -34,7 +34,7 @@ silo/
|
||||
│ ├── ods/ # ODS spreadsheet library
|
||||
│ ├── partnum/ # Part number generation
|
||||
│ ├── schema/ # YAML schema parsing
|
||||
│ ├── storage/ # MinIO file storage
|
||||
│ ├── storage/ # Filesystem storage
|
||||
│ └── testutil/ # Test helpers
|
||||
├── web/ # React SPA (Vite + TypeScript)
|
||||
│ └── src/
|
||||
@@ -55,7 +55,7 @@ silo/
|
||||
|
||||
See the **[Installation Guide](docs/INSTALL.md)** for complete setup instructions.
|
||||
|
||||
**Docker Compose (quickest — includes PostgreSQL, MinIO, OpenLDAP, and Silo):**
|
||||
**Docker Compose (quickest — includes PostgreSQL, OpenLDAP, and Silo):**
|
||||
|
||||
```bash
|
||||
./scripts/setup-docker.sh
|
||||
@@ -65,7 +65,7 @@ docker compose -f deployments/docker-compose.allinone.yaml up -d
|
||||
**Development (local Go + Docker services):**
|
||||
|
||||
```bash
|
||||
make docker-up # Start PostgreSQL + MinIO in Docker
|
||||
make docker-up # Start PostgreSQL in Docker
|
||||
make run # Run silo locally with Go
|
||||
```
|
||||
|
||||
|
||||
@@ -1,288 +0,0 @@
|
||||
// Command migrate-storage downloads files from MinIO and writes them to the
|
||||
// local filesystem. It is a one-shot migration tool for moving off MinIO.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// migrate-storage -config config.yaml -dest /opt/silo/data [-dry-run] [-verbose]
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/kindredsystems/silo/internal/config"
|
||||
"github.com/kindredsystems/silo/internal/db"
|
||||
"github.com/kindredsystems/silo/internal/storage"
|
||||
"github.com/rs/zerolog"
|
||||
)
|
||||
|
||||
// fileEntry represents a single file to migrate.
|
||||
type fileEntry struct {
|
||||
key string
|
||||
versionID string // MinIO version ID; empty if not versioned
|
||||
size int64 // expected size from DB; 0 if unknown
|
||||
}
|
||||
|
||||
func main() {
|
||||
configPath := flag.String("config", "config.yaml", "Path to configuration file")
|
||||
dest := flag.String("dest", "", "Destination root directory (required)")
|
||||
dryRun := flag.Bool("dry-run", false, "Preview what would be migrated without downloading")
|
||||
verbose := flag.Bool("verbose", false, "Log every file, not just errors and summary")
|
||||
flag.Parse()
|
||||
|
||||
logger := zerolog.New(os.Stdout).With().Timestamp().Logger()
|
||||
|
||||
if *dest == "" {
|
||||
logger.Fatal().Msg("-dest is required")
|
||||
}
|
||||
|
||||
// Load config (reuses existing config for DB + MinIO credentials).
|
||||
cfg, err := config.Load(*configPath)
|
||||
if err != nil {
|
||||
logger.Fatal().Err(err).Msg("failed to load configuration")
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Connect to PostgreSQL.
|
||||
database, err := db.Connect(ctx, db.Config{
|
||||
Host: cfg.Database.Host,
|
||||
Port: cfg.Database.Port,
|
||||
Name: cfg.Database.Name,
|
||||
User: cfg.Database.User,
|
||||
Password: cfg.Database.Password,
|
||||
SSLMode: cfg.Database.SSLMode,
|
||||
MaxConnections: cfg.Database.MaxConnections,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Fatal().Err(err).Msg("failed to connect to database")
|
||||
}
|
||||
defer database.Close()
|
||||
logger.Info().Msg("connected to database")
|
||||
|
||||
// Connect to MinIO.
|
||||
store, err := storage.Connect(ctx, storage.Config{
|
||||
Endpoint: cfg.Storage.Endpoint,
|
||||
AccessKey: cfg.Storage.AccessKey,
|
||||
SecretKey: cfg.Storage.SecretKey,
|
||||
Bucket: cfg.Storage.Bucket,
|
||||
UseSSL: cfg.Storage.UseSSL,
|
||||
Region: cfg.Storage.Region,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Fatal().Err(err).Msg("failed to connect to MinIO")
|
||||
}
|
||||
logger.Info().Str("bucket", cfg.Storage.Bucket).Msg("connected to MinIO")
|
||||
|
||||
// Collect all file references from the database.
|
||||
entries, err := collectEntries(ctx, logger, database)
|
||||
if err != nil {
|
||||
logger.Fatal().Err(err).Msg("failed to collect file entries from database")
|
||||
}
|
||||
logger.Info().Int("total", len(entries)).Msg("file entries found")
|
||||
|
||||
if len(entries) == 0 {
|
||||
logger.Info().Msg("nothing to migrate")
|
||||
return
|
||||
}
|
||||
|
||||
// Migrate.
|
||||
var migrated, skipped, failed int
|
||||
start := time.Now()
|
||||
|
||||
for i, e := range entries {
|
||||
destPath := filepath.Join(*dest, e.key)
|
||||
|
||||
// Check if already migrated.
|
||||
if info, err := os.Stat(destPath); err == nil {
|
||||
if e.size > 0 && info.Size() == e.size {
|
||||
if *verbose {
|
||||
logger.Info().Str("key", e.key).Msg("skipped (already exists)")
|
||||
}
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
// Size mismatch or unknown size — re-download.
|
||||
}
|
||||
|
||||
if *dryRun {
|
||||
logger.Info().
|
||||
Str("key", e.key).
|
||||
Int64("size", e.size).
|
||||
Str("version", e.versionID).
|
||||
Msgf("[%d/%d] would migrate", i+1, len(entries))
|
||||
continue
|
||||
}
|
||||
|
||||
if err := migrateFile(ctx, store, e, destPath); err != nil {
|
||||
logger.Error().Err(err).Str("key", e.key).Msg("failed to migrate")
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
|
||||
migrated++
|
||||
if *verbose {
|
||||
logger.Info().
|
||||
Str("key", e.key).
|
||||
Int64("size", e.size).
|
||||
Msgf("[%d/%d] migrated", i+1, len(entries))
|
||||
} else if (i+1)%50 == 0 {
|
||||
logger.Info().Msgf("progress: %d/%d", i+1, len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
elapsed := time.Since(start)
|
||||
ev := logger.Info().
|
||||
Int("total", len(entries)).
|
||||
Int("migrated", migrated).
|
||||
Int("skipped", skipped).
|
||||
Int("failed", failed).
|
||||
Dur("elapsed", elapsed)
|
||||
if *dryRun {
|
||||
ev.Msg("dry run complete")
|
||||
} else {
|
||||
ev.Msg("migration complete")
|
||||
}
|
||||
|
||||
if failed > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// collectEntries queries the database for all file references across the three
|
||||
// storage domains: revision files, item file attachments, and item thumbnails.
|
||||
// It deduplicates by key.
|
||||
func collectEntries(ctx context.Context, logger zerolog.Logger, database *db.DB) ([]fileEntry, error) {
|
||||
pool := database.Pool()
|
||||
seen := make(map[string]struct{})
|
||||
var entries []fileEntry
|
||||
|
||||
add := func(key, versionID string, size int64) {
|
||||
if key == "" {
|
||||
return
|
||||
}
|
||||
if _, ok := seen[key]; ok {
|
||||
return
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
entries = append(entries, fileEntry{key: key, versionID: versionID, size: size})
|
||||
}
|
||||
|
||||
// 1. Revision files.
|
||||
rows, err := pool.Query(ctx,
|
||||
`SELECT file_key, COALESCE(file_version, ''), COALESCE(file_size, 0)
|
||||
FROM revisions WHERE file_key IS NOT NULL`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying revisions: %w", err)
|
||||
}
|
||||
for rows.Next() {
|
||||
var key, version string
|
||||
var size int64
|
||||
if err := rows.Scan(&key, &version, &size); err != nil {
|
||||
rows.Close()
|
||||
return nil, fmt.Errorf("scanning revision row: %w", err)
|
||||
}
|
||||
add(key, version, size)
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterating revisions: %w", err)
|
||||
}
|
||||
logger.Info().Int("count", len(entries)).Msg("revision files found")
|
||||
|
||||
// 2. Item file attachments.
|
||||
countBefore := len(entries)
|
||||
rows, err = pool.Query(ctx,
|
||||
`SELECT object_key, size FROM item_files`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying item_files: %w", err)
|
||||
}
|
||||
for rows.Next() {
|
||||
var key string
|
||||
var size int64
|
||||
if err := rows.Scan(&key, &size); err != nil {
|
||||
rows.Close()
|
||||
return nil, fmt.Errorf("scanning item_files row: %w", err)
|
||||
}
|
||||
add(key, "", size)
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterating item_files: %w", err)
|
||||
}
|
||||
logger.Info().Int("count", len(entries)-countBefore).Msg("item file attachments found")
|
||||
|
||||
// 3. Item thumbnails.
|
||||
countBefore = len(entries)
|
||||
rows, err = pool.Query(ctx,
|
||||
`SELECT thumbnail_key FROM items WHERE thumbnail_key IS NOT NULL`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying item thumbnails: %w", err)
|
||||
}
|
||||
for rows.Next() {
|
||||
var key string
|
||||
if err := rows.Scan(&key); err != nil {
|
||||
rows.Close()
|
||||
return nil, fmt.Errorf("scanning thumbnail row: %w", err)
|
||||
}
|
||||
add(key, "", 0)
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterating thumbnails: %w", err)
|
||||
}
|
||||
logger.Info().Int("count", len(entries)-countBefore).Msg("item thumbnails found")
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// migrateFile downloads a single file from MinIO and writes it atomically to destPath.
|
||||
func migrateFile(ctx context.Context, store *storage.Storage, e fileEntry, destPath string) error {
|
||||
// Ensure parent directory exists.
|
||||
if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil {
|
||||
return fmt.Errorf("creating directory: %w", err)
|
||||
}
|
||||
|
||||
// Download from MinIO.
|
||||
var reader io.ReadCloser
|
||||
var err error
|
||||
if e.versionID != "" {
|
||||
reader, err = store.GetVersion(ctx, e.key, e.versionID)
|
||||
} else {
|
||||
reader, err = store.Get(ctx, e.key)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("downloading from MinIO: %w", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
// Write to temp file then rename for atomicity.
|
||||
tmpPath := destPath + ".tmp"
|
||||
f, err := os.Create(tmpPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating temp file: %w", err)
|
||||
}
|
||||
|
||||
if _, err := io.Copy(f, reader); err != nil {
|
||||
f.Close()
|
||||
os.Remove(tmpPath)
|
||||
return fmt.Errorf("writing file: %w", err)
|
||||
}
|
||||
|
||||
if err := f.Close(); err != nil {
|
||||
os.Remove(tmpPath)
|
||||
return fmt.Errorf("closing temp file: %w", err)
|
||||
}
|
||||
|
||||
if err := os.Rename(tmpPath, destPath); err != nil {
|
||||
os.Remove(tmpPath)
|
||||
return fmt.Errorf("renaming temp file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -45,7 +45,6 @@ func main() {
|
||||
Str("host", cfg.Server.Host).
|
||||
Int("port", cfg.Server.Port).
|
||||
Str("database", cfg.Database.Host).
|
||||
Str("storage", cfg.Storage.Endpoint).
|
||||
Msg("starting silo server")
|
||||
|
||||
// Connect to database
|
||||
@@ -65,40 +64,17 @@ func main() {
|
||||
defer database.Close()
|
||||
logger.Info().Msg("connected to database")
|
||||
|
||||
// Connect to storage (optional - may be externally managed)
|
||||
// Connect to storage (optional — requires root_dir to be set)
|
||||
var store storage.FileStore
|
||||
switch cfg.Storage.Backend {
|
||||
case "minio", "":
|
||||
if cfg.Storage.Endpoint != "" {
|
||||
s, connErr := storage.Connect(ctx, storage.Config{
|
||||
Endpoint: cfg.Storage.Endpoint,
|
||||
AccessKey: cfg.Storage.AccessKey,
|
||||
SecretKey: cfg.Storage.SecretKey,
|
||||
Bucket: cfg.Storage.Bucket,
|
||||
UseSSL: cfg.Storage.UseSSL,
|
||||
Region: cfg.Storage.Region,
|
||||
})
|
||||
if connErr != nil {
|
||||
logger.Warn().Err(connErr).Msg("failed to connect to storage - file operations disabled")
|
||||
} else {
|
||||
store = s
|
||||
logger.Info().Msg("connected to storage")
|
||||
}
|
||||
} else {
|
||||
logger.Info().Msg("storage not configured - file operations disabled")
|
||||
}
|
||||
case "filesystem":
|
||||
if cfg.Storage.Filesystem.RootDir == "" {
|
||||
logger.Fatal().Msg("storage.filesystem.root_dir is required when backend is \"filesystem\"")
|
||||
}
|
||||
if cfg.Storage.Filesystem.RootDir != "" {
|
||||
s, fsErr := storage.NewFilesystemStore(cfg.Storage.Filesystem.RootDir)
|
||||
if fsErr != nil {
|
||||
logger.Fatal().Err(fsErr).Msg("failed to initialize filesystem storage")
|
||||
}
|
||||
store = s
|
||||
logger.Info().Str("root", cfg.Storage.Filesystem.RootDir).Msg("connected to filesystem storage")
|
||||
default:
|
||||
logger.Fatal().Str("backend", cfg.Storage.Backend).Msg("unknown storage backend")
|
||||
} else {
|
||||
logger.Info().Msg("storage not configured - file operations disabled")
|
||||
}
|
||||
|
||||
// Load schemas
|
||||
|
||||
@@ -17,17 +17,9 @@ database:
|
||||
max_connections: 10
|
||||
|
||||
storage:
|
||||
backend: "minio" # "minio" (default) or "filesystem"
|
||||
# MinIO/S3 settings (used when backend: "minio")
|
||||
endpoint: "localhost:9000" # Use "minio:9000" for Docker Compose
|
||||
access_key: "" # Use SILO_MINIO_ACCESS_KEY env var
|
||||
secret_key: "" # Use SILO_MINIO_SECRET_KEY env var
|
||||
bucket: "silo-files"
|
||||
use_ssl: true # Use false for Docker Compose (internal network)
|
||||
region: "us-east-1"
|
||||
# Filesystem settings (used when backend: "filesystem")
|
||||
# filesystem:
|
||||
# root_dir: "/var/lib/silo/objects"
|
||||
backend: "filesystem"
|
||||
filesystem:
|
||||
root_dir: "/opt/silo/data" # Override with SILO_STORAGE_ROOT_DIR env var
|
||||
|
||||
schemas:
|
||||
# Directory containing YAML schema files
|
||||
|
||||
@@ -17,12 +17,9 @@ database:
|
||||
max_connections: 10
|
||||
|
||||
storage:
|
||||
endpoint: "minio:9000"
|
||||
access_key: "${MINIO_ACCESS_KEY:-silominio}"
|
||||
secret_key: "${MINIO_SECRET_KEY:-silominiosecret}"
|
||||
bucket: "silo-files"
|
||||
use_ssl: false
|
||||
region: "us-east-1"
|
||||
backend: "filesystem"
|
||||
filesystem:
|
||||
root_dir: "/var/lib/silo/data"
|
||||
|
||||
schemas:
|
||||
directory: "/etc/silo/schemas"
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Silo All-in-One Stack
|
||||
# PostgreSQL + MinIO + OpenLDAP + Silo API + Nginx (optional)
|
||||
# PostgreSQL + OpenLDAP + Silo API + Nginx (optional)
|
||||
#
|
||||
# Quick start:
|
||||
# ./scripts/setup-docker.sh
|
||||
@@ -40,29 +40,6 @@ services:
|
||||
networks:
|
||||
- silo-net
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MinIO (S3-compatible object storage)
|
||||
# ---------------------------------------------------------------------------
|
||||
minio:
|
||||
image: minio/minio:latest
|
||||
container_name: silo-minio
|
||||
restart: unless-stopped
|
||||
command: server /data --console-address ":9001"
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ACCESS_KEY:?Run ./scripts/setup-docker.sh first}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY:?Run ./scripts/setup-docker.sh first}
|
||||
volumes:
|
||||
- minio_data:/data
|
||||
ports:
|
||||
- "9001:9001" # MinIO console (remove in hardened setups)
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
networks:
|
||||
- silo-net
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OpenLDAP (user directory for LDAP authentication)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -83,9 +60,13 @@ services:
|
||||
- openldap_data:/bitnami/openldap
|
||||
- ./ldap:/docker-entrypoint-initdb.d:ro
|
||||
ports:
|
||||
- "1389:1389" # LDAP access for debugging (remove in hardened setups)
|
||||
- "1389:1389" # LDAP access for debugging (remove in hardened setups)
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "ldapsearch -x -H ldap://localhost:1389 -b dc=silo,dc=local -D cn=admin,dc=silo,dc=local -w $${LDAP_ADMIN_PASSWORD} '(objectClass=organization)' >/dev/null 2>&1"]
|
||||
test:
|
||||
[
|
||||
"CMD-SHELL",
|
||||
"ldapsearch -x -H ldap://localhost:1389 -b dc=silo,dc=local -D cn=admin,dc=silo,dc=local -w $${LDAP_ADMIN_PASSWORD} '(objectClass=organization)' >/dev/null 2>&1",
|
||||
]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
@@ -104,8 +85,6 @@ services:
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
minio:
|
||||
condition: service_healthy
|
||||
openldap:
|
||||
condition: service_healthy
|
||||
env_file:
|
||||
@@ -117,12 +96,10 @@ services:
|
||||
SILO_DB_NAME: silo
|
||||
SILO_DB_USER: silo
|
||||
SILO_DB_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
SILO_MINIO_ENDPOINT: minio:9000
|
||||
SILO_MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
|
||||
SILO_MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
|
||||
ports:
|
||||
- "${SILO_PORT:-8080}:8080"
|
||||
volumes:
|
||||
- silo_data:/var/lib/silo/data
|
||||
- ../schemas:/etc/silo/schemas:ro
|
||||
- ./config.docker.yaml:/etc/silo/config.yaml:ro
|
||||
healthcheck:
|
||||
@@ -164,7 +141,7 @@ services:
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
minio_data:
|
||||
silo_data:
|
||||
openldap_data:
|
||||
|
||||
networks:
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
# Production Docker Compose for Silo
|
||||
# Uses external PostgreSQL (psql.example.internal) and MinIO (minio.example.internal)
|
||||
# Uses external PostgreSQL (psql.example.internal) and filesystem storage
|
||||
#
|
||||
# Usage:
|
||||
# export SILO_DB_PASSWORD=<your-password>
|
||||
# export SILO_MINIO_ACCESS_KEY=<your-access-key>
|
||||
# export SILO_MINIO_SECRET_KEY=<your-secret-key>
|
||||
# docker compose -f docker-compose.prod.yaml up -d
|
||||
|
||||
services:
|
||||
@@ -24,14 +22,6 @@ services:
|
||||
# Note: SILO_DB_PORT and SILO_DB_SSLMODE are NOT supported as direct
|
||||
# env var overrides. Set these in config.yaml instead, or use ${VAR}
|
||||
# syntax in the YAML file. See docs/CONFIGURATION.md for details.
|
||||
|
||||
# MinIO storage (minio.example.internal)
|
||||
# Supported as direct env var overrides:
|
||||
SILO_MINIO_ENDPOINT: minio.example.internal:9000
|
||||
SILO_MINIO_ACCESS_KEY: ${SILO_MINIO_ACCESS_KEY:?MinIO access key required}
|
||||
SILO_MINIO_SECRET_KEY: ${SILO_MINIO_SECRET_KEY:?MinIO secret key required}
|
||||
# Note: SILO_MINIO_BUCKET and SILO_MINIO_USE_SSL are NOT supported as
|
||||
# direct env var overrides. Set these in config.yaml instead.
|
||||
ports:
|
||||
- "8080:8080"
|
||||
volumes:
|
||||
|
||||
@@ -19,26 +19,6 @@ services:
|
||||
networks:
|
||||
- silo-network
|
||||
|
||||
minio:
|
||||
image: minio/minio:RELEASE.2023-05-04T21-44-30Z
|
||||
container_name: silo-minio
|
||||
command: server /data --console-address ":9001"
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ACCESS_KEY:-silominio}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY:-silominiosecret}
|
||||
volumes:
|
||||
- minio_data:/data
|
||||
ports:
|
||||
- "9000:9000"
|
||||
- "9001:9001"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
networks:
|
||||
- silo-network
|
||||
|
||||
silo:
|
||||
build:
|
||||
context: ..
|
||||
@@ -47,19 +27,12 @@ services:
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
minio:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
SILO_DB_HOST: postgres
|
||||
SILO_DB_PORT: 5432
|
||||
SILO_DB_NAME: silo
|
||||
SILO_DB_USER: silo
|
||||
SILO_DB_PASSWORD: ${POSTGRES_PASSWORD:-silodev}
|
||||
SILO_MINIO_ENDPOINT: minio:9000
|
||||
SILO_MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-silominio}
|
||||
SILO_MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-silominiosecret}
|
||||
SILO_MINIO_BUCKET: silo-files
|
||||
SILO_MINIO_USE_SSL: "false"
|
||||
SILO_SESSION_SECRET: ${SILO_SESSION_SECRET:-change-me-in-production}
|
||||
SILO_OIDC_CLIENT_SECRET: ${SILO_OIDC_CLIENT_SECRET:-}
|
||||
SILO_LDAP_BIND_PASSWORD: ${SILO_LDAP_BIND_PASSWORD:-}
|
||||
@@ -68,6 +41,7 @@ services:
|
||||
ports:
|
||||
- "8080:8080"
|
||||
volumes:
|
||||
- silo_data:/var/lib/silo/data
|
||||
- ../schemas:/etc/silo/schemas:ro
|
||||
- ./config.dev.yaml:/etc/silo/config.yaml:ro
|
||||
healthcheck:
|
||||
@@ -80,7 +54,7 @@ services:
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
minio_data:
|
||||
silo_data:
|
||||
|
||||
networks:
|
||||
silo-network:
|
||||
|
||||
@@ -27,6 +27,7 @@ NoNewPrivileges=yes
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
ReadWritePaths=/opt/silo/data
|
||||
ReadOnlyPaths=/etc/silo /opt/silo
|
||||
|
||||
# Resource limits
|
||||
|
||||
@@ -23,7 +23,7 @@ These cannot be disabled. They define what Silo *is*.
|
||||
|-----------|------|-------------|
|
||||
| `core` | Core PDM | Items, revisions, files, BOM, search, import/export, part number generation |
|
||||
| `schemas` | Schemas | Part numbering schema parsing, segment management, form descriptors |
|
||||
| `storage` | Storage | MinIO/S3 file storage, presigned uploads, versioning |
|
||||
| `storage` | Storage | Filesystem storage |
|
||||
|
||||
### 2.2 Optional Modules
|
||||
|
||||
@@ -470,12 +470,10 @@ Returns full config grouped by module with secrets redacted:
|
||||
"default": "kindred-rd"
|
||||
},
|
||||
"storage": {
|
||||
"endpoint": "minio:9000",
|
||||
"bucket": "silo-files",
|
||||
"access_key": "****",
|
||||
"secret_key": "****",
|
||||
"use_ssl": false,
|
||||
"region": "us-east-1",
|
||||
"backend": "filesystem",
|
||||
"filesystem": {
|
||||
"root_dir": "/var/lib/silo/data"
|
||||
},
|
||||
"status": "connected"
|
||||
},
|
||||
"database": {
|
||||
@@ -566,7 +564,7 @@ Available for modules with external connections:
|
||||
|
||||
| Module | Test Action |
|
||||
|--------|------------|
|
||||
| `storage` | Ping MinIO, verify bucket exists |
|
||||
| `storage` | Verify filesystem storage directory is accessible |
|
||||
| `auth` (ldap) | Attempt LDAP bind with configured credentials |
|
||||
| `auth` (oidc) | Fetch OIDC discovery document from issuer URL |
|
||||
| `odoo` | Attempt XML-RPC connection to Odoo |
|
||||
@@ -602,11 +600,9 @@ database:
|
||||
sslmode: disable
|
||||
|
||||
storage:
|
||||
endpoint: minio:9000
|
||||
bucket: silo-files
|
||||
access_key: silominio
|
||||
secret_key: silominiosecret
|
||||
use_ssl: false
|
||||
backend: filesystem
|
||||
filesystem:
|
||||
root_dir: /var/lib/silo/data
|
||||
|
||||
schemas:
|
||||
directory: /etc/silo/schemas
|
||||
|
||||
@@ -337,7 +337,7 @@ Supporting files:
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `web/src/components/items/CategoryPicker.tsx` | Multi-stage domain/subcategory selector |
|
||||
| `web/src/components/items/FileDropZone.tsx` | Drag-and-drop file upload with MinIO presigned URLs |
|
||||
| `web/src/components/items/FileDropZone.tsx` | Drag-and-drop file upload |
|
||||
| `web/src/components/items/TagInput.tsx` | Multi-select tag input for projects |
|
||||
| `web/src/hooks/useFormDescriptor.ts` | Fetches and caches form descriptor from `/api/schemas/{name}/form` |
|
||||
| `web/src/hooks/useFileUpload.ts` | Manages presigned URL upload flow |
|
||||
@@ -421,7 +421,7 @@ Below the picker, the selected category is shown as a breadcrumb: `Fasteners ›
|
||||
|
||||
### FileDropZone
|
||||
|
||||
Handles drag-and-drop and click-to-browse file uploads with MinIO presigned URL flow.
|
||||
Handles drag-and-drop and click-to-browse file uploads.
|
||||
|
||||
**Props**:
|
||||
|
||||
@@ -435,7 +435,7 @@ interface FileDropZoneProps {
|
||||
|
||||
interface PendingAttachment {
|
||||
file: File;
|
||||
objectKey: string; // MinIO key after upload
|
||||
objectKey: string; // storage key after upload
|
||||
uploadProgress: number; // 0-100
|
||||
uploadStatus: 'pending' | 'uploading' | 'complete' | 'error';
|
||||
error?: string;
|
||||
@@ -462,7 +462,7 @@ Clicking the zone opens a hidden `<input type="file" multiple>`.
|
||||
|
||||
1. On file selection/drop, immediately request a presigned upload URL: `POST /api/uploads/presign` with `{ filename, content_type, size }`.
|
||||
2. Backend returns `{ object_key, upload_url, expires_at }`.
|
||||
3. `PUT` the file directly to the presigned MinIO URL using `XMLHttpRequest` (for progress tracking).
|
||||
3. `PUT` the file directly to the presigned URL using `XMLHttpRequest` (for progress tracking).
|
||||
4. On completion, update `PendingAttachment.uploadStatus` to `'complete'` and store the `object_key`.
|
||||
5. The `object_key` is later sent to the item creation endpoint to associate the file.
|
||||
|
||||
@@ -589,10 +589,10 @@ Items 1-5 below are implemented. Item 4 (hierarchical categories) is resolved by
|
||||
```
|
||||
POST /api/uploads/presign
|
||||
Request: { "filename": "bracket.FCStd", "content_type": "application/octet-stream", "size": 2400000 }
|
||||
Response: { "object_key": "uploads/tmp/{uuid}/{filename}", "upload_url": "https://minio.../...", "expires_at": "2026-02-06T..." }
|
||||
Response: { "object_key": "uploads/tmp/{uuid}/{filename}", "upload_url": "https://...", "expires_at": "2026-02-06T..." }
|
||||
```
|
||||
|
||||
The Go handler generates a presigned PUT URL via the MinIO SDK. Objects are uploaded to a temporary prefix. On item creation, they're moved/linked to the item's permanent prefix.
|
||||
The Go handler generates a presigned PUT URL for direct upload. Objects are uploaded to a temporary prefix. On item creation, they're moved/linked to the item's permanent prefix.
|
||||
|
||||
### 2. File Association -- IMPLEMENTED
|
||||
|
||||
@@ -612,7 +612,7 @@ Request: { "object_key": "uploads/tmp/{uuid}/thumb.png" }
|
||||
Response: 204
|
||||
```
|
||||
|
||||
Stores the thumbnail at `items/{item_id}/thumbnail.png` in MinIO. Updates `item.thumbnail_key` column.
|
||||
Stores the thumbnail at `items/{item_id}/thumbnail.png` in storage. Updates `item.thumbnail_key` column.
|
||||
|
||||
### 4. Hierarchical Categories -- IMPLEMENTED (via Form Descriptor)
|
||||
|
||||
|
||||
13
go.mod
13
go.mod
@@ -11,7 +11,6 @@ require (
|
||||
github.com/go-ldap/ldap/v3 v3.4.12
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/jackc/pgx/v5 v5.5.4
|
||||
github.com/minio/minio-go/v7 v7.0.66
|
||||
github.com/rs/zerolog v1.32.0
|
||||
github.com/sahilm/fuzzy v0.1.1
|
||||
golang.org/x/crypto v0.47.0
|
||||
@@ -21,28 +20,16 @@ require (
|
||||
|
||||
require (
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.1 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/compress v1.17.4 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.6 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.19 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/minio/sha256-simd v1.0.1 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/rs/xid v1.5.0 // indirect
|
||||
github.com/sirupsen/logrus v1.9.3 // indirect
|
||||
golang.org/x/net v0.48.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.40.0 // indirect
|
||||
golang.org/x/text v0.33.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
)
|
||||
|
||||
27
go.sum
27
go.sum
@@ -13,8 +13,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 h1:BP4M0CvQ4S3TGls2FvczZtj5Re/2ZzkV9VwqPHH/3Bo=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
|
||||
github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s=
|
||||
@@ -26,7 +24,6 @@ github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9
|
||||
github.com/go-ldap/ldap/v3 v3.4.12 h1:1b81mv7MagXZ7+1r7cLTWmyuTqVqdwbtJSjC0DAp9s4=
|
||||
github.com/go-ldap/ldap/v3 v3.4.12/go.mod h1:+SPAGcTtOfmGsCb3h1RFiq4xpp4N636G75OEace8lNo=
|
||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
|
||||
@@ -51,13 +48,6 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
|
||||
github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
|
||||
github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
|
||||
github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
|
||||
@@ -73,31 +63,17 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
|
||||
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw=
|
||||
github.com/minio/minio-go/v7 v7.0.66/go.mod h1:DHAgmyQEGdW3Cif0UooKOyrT3Vxs82zNdV6tkKhRtbs=
|
||||
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
|
||||
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
|
||||
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||
github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0=
|
||||
github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
|
||||
github.com/sahilm/fuzzy v0.1.1 h1:ceu5RHF8DGgoi+/dR5PsECjCDH1BE3Fnmpo7aVXOdRA=
|
||||
github.com/sahilm/fuzzy v0.1.1/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y=
|
||||
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
|
||||
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
@@ -133,7 +109,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
@@ -166,8 +141,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
|
||||
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -21,7 +21,7 @@ type presignUploadRequest struct {
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
// HandlePresignUpload generates a presigned PUT URL for direct browser upload to MinIO.
|
||||
// HandlePresignUpload generates a presigned PUT URL for direct browser upload.
|
||||
func (s *Server) HandlePresignUpload(w http.ResponseWriter, r *http.Request) {
|
||||
if s.storage == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "storage_unavailable", "File storage not configured")
|
||||
@@ -317,12 +317,9 @@ func (s *Server) HandleSetItemThumbnail(w http.ResponseWriter, r *http.Request)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// storageBackend returns the configured storage backend name, defaulting to "minio".
|
||||
// storageBackend returns the configured storage backend name.
|
||||
func (s *Server) storageBackend() string {
|
||||
if s.cfg != nil && s.cfg.Storage.Backend != "" {
|
||||
return s.cfg.Storage.Backend
|
||||
}
|
||||
return "minio"
|
||||
return "filesystem"
|
||||
}
|
||||
|
||||
// HandleUploadItemFile accepts a multipart file upload and stores it as an item attachment.
|
||||
|
||||
@@ -86,7 +86,7 @@ func (ss *ServerState) ToggleReadOnly() {
|
||||
ss.SetReadOnly(!current)
|
||||
}
|
||||
|
||||
// StartStorageHealthCheck launches a periodic check of MinIO reachability.
|
||||
// StartStorageHealthCheck launches a periodic check of storage reachability.
|
||||
// Updates storageOK and broadcasts server.state on transitions.
|
||||
func (ss *ServerState) StartStorageHealthCheck() {
|
||||
if ss.storage == nil {
|
||||
|
||||
@@ -224,10 +224,8 @@ func (s *Server) buildSchemasSettings() map[string]any {
|
||||
func (s *Server) buildStorageSettings(ctx context.Context) map[string]any {
|
||||
result := map[string]any{
|
||||
"enabled": true,
|
||||
"endpoint": s.cfg.Storage.Endpoint,
|
||||
"bucket": s.cfg.Storage.Bucket,
|
||||
"use_ssl": s.cfg.Storage.UseSSL,
|
||||
"region": s.cfg.Storage.Region,
|
||||
"backend": "filesystem",
|
||||
"root_dir": s.cfg.Storage.Filesystem.RootDir,
|
||||
}
|
||||
if s.storage != nil {
|
||||
if err := s.storage.Ping(ctx); err != nil {
|
||||
|
||||
@@ -31,8 +31,8 @@ func newSettingsTestServer(t *testing.T) *Server {
|
||||
MaxConnections: 10,
|
||||
},
|
||||
Storage: config.StorageConfig{
|
||||
Endpoint: "minio:9000", Bucket: "silo", Region: "us-east-1",
|
||||
AccessKey: "minioadmin", SecretKey: "miniosecret",
|
||||
Backend: "filesystem",
|
||||
Filesystem: config.FilesystemConfig{RootDir: "/tmp/silo-test"},
|
||||
},
|
||||
Schemas: config.SchemasConfig{Directory: "/etc/silo/schemas", Default: "kindred-rd"},
|
||||
Auth: config.AuthConfig{
|
||||
|
||||
@@ -110,15 +110,9 @@ type DatabaseConfig struct {
|
||||
MaxConnections int `yaml:"max_connections"`
|
||||
}
|
||||
|
||||
// StorageConfig holds object storage settings.
|
||||
// StorageConfig holds file storage settings.
|
||||
type StorageConfig struct {
|
||||
Backend string `yaml:"backend"` // "minio" (default) or "filesystem"
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
AccessKey string `yaml:"access_key"`
|
||||
SecretKey string `yaml:"secret_key"`
|
||||
Bucket string `yaml:"bucket"`
|
||||
UseSSL bool `yaml:"use_ssl"`
|
||||
Region string `yaml:"region"`
|
||||
Backend string `yaml:"backend"` // "filesystem"
|
||||
Filesystem FilesystemConfig `yaml:"filesystem"`
|
||||
}
|
||||
|
||||
@@ -189,9 +183,6 @@ func Load(path string) (*Config, error) {
|
||||
if cfg.Database.MaxConnections == 0 {
|
||||
cfg.Database.MaxConnections = 10
|
||||
}
|
||||
if cfg.Storage.Region == "" {
|
||||
cfg.Storage.Region = "us-east-1"
|
||||
}
|
||||
if cfg.Schemas.Directory == "" {
|
||||
cfg.Schemas.Directory = "/etc/silo/schemas"
|
||||
}
|
||||
@@ -227,14 +218,8 @@ func Load(path string) (*Config, error) {
|
||||
if v := os.Getenv("SILO_DB_PASSWORD"); v != "" {
|
||||
cfg.Database.Password = v
|
||||
}
|
||||
if v := os.Getenv("SILO_MINIO_ENDPOINT"); v != "" {
|
||||
cfg.Storage.Endpoint = v
|
||||
}
|
||||
if v := os.Getenv("SILO_MINIO_ACCESS_KEY"); v != "" {
|
||||
cfg.Storage.AccessKey = v
|
||||
}
|
||||
if v := os.Getenv("SILO_MINIO_SECRET_KEY"); v != "" {
|
||||
cfg.Storage.SecretKey = v
|
||||
if v := os.Getenv("SILO_STORAGE_ROOT_DIR"); v != "" {
|
||||
cfg.Storage.Filesystem.RootDir = v
|
||||
}
|
||||
|
||||
// Auth defaults
|
||||
|
||||
@@ -14,7 +14,7 @@ type ItemFile struct {
|
||||
ContentType string
|
||||
Size int64
|
||||
ObjectKey string
|
||||
StorageBackend string // "minio" or "filesystem"
|
||||
StorageBackend string
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ func NewItemFileRepository(db *DB) *ItemFileRepository {
|
||||
// Create inserts a new item file record.
|
||||
func (r *ItemFileRepository) Create(ctx context.Context, f *ItemFile) error {
|
||||
if f.StorageBackend == "" {
|
||||
f.StorageBackend = "minio"
|
||||
f.StorageBackend = "filesystem"
|
||||
}
|
||||
err := r.db.pool.QueryRow(ctx,
|
||||
`INSERT INTO item_files (item_id, filename, content_type, size, object_key, storage_backend)
|
||||
@@ -49,7 +49,7 @@ func (r *ItemFileRepository) Create(ctx context.Context, f *ItemFile) error {
|
||||
func (r *ItemFileRepository) ListByItem(ctx context.Context, itemID string) ([]*ItemFile, error) {
|
||||
rows, err := r.db.pool.Query(ctx,
|
||||
`SELECT id, item_id, filename, content_type, size, object_key,
|
||||
COALESCE(storage_backend, 'minio'), created_at
|
||||
COALESCE(storage_backend, 'filesystem'), created_at
|
||||
FROM item_files WHERE item_id = $1 ORDER BY created_at`,
|
||||
itemID,
|
||||
)
|
||||
@@ -74,7 +74,7 @@ func (r *ItemFileRepository) Get(ctx context.Context, id string) (*ItemFile, err
|
||||
f := &ItemFile{}
|
||||
err := r.db.pool.QueryRow(ctx,
|
||||
`SELECT id, item_id, filename, content_type, size, object_key,
|
||||
COALESCE(storage_backend, 'minio'), created_at
|
||||
COALESCE(storage_backend, 'filesystem'), created_at
|
||||
FROM item_files WHERE id = $1`,
|
||||
id,
|
||||
).Scan(&f.ID, &f.ItemID, &f.Filename, &f.ContentType, &f.Size, &f.ObjectKey, &f.StorageBackend, &f.CreatedAt)
|
||||
|
||||
@@ -26,26 +26,26 @@ type Item struct {
|
||||
UpdatedBy *string
|
||||
SourcingType string // "manufactured" or "purchased"
|
||||
LongDescription *string // extended description
|
||||
ThumbnailKey *string // MinIO key for item thumbnail
|
||||
ThumbnailKey *string // storage key for item thumbnail
|
||||
}
|
||||
|
||||
// Revision represents a revision record.
|
||||
type Revision struct {
|
||||
ID string
|
||||
ItemID string
|
||||
RevisionNumber int
|
||||
Properties map[string]any
|
||||
ID string
|
||||
ItemID string
|
||||
RevisionNumber int
|
||||
Properties map[string]any
|
||||
FileKey *string
|
||||
FileVersion *string
|
||||
FileChecksum *string
|
||||
FileSize *int64
|
||||
FileStorageBackend string // "minio" or "filesystem"
|
||||
FileStorageBackend string
|
||||
ThumbnailKey *string
|
||||
CreatedAt time.Time
|
||||
CreatedBy *string
|
||||
Comment *string
|
||||
Status string // draft, review, released, obsolete
|
||||
Labels []string // arbitrary tags
|
||||
CreatedAt time.Time
|
||||
CreatedBy *string
|
||||
Comment *string
|
||||
Status string // draft, review, released, obsolete
|
||||
Labels []string // arbitrary tags
|
||||
}
|
||||
|
||||
// RevisionStatus constants
|
||||
@@ -308,7 +308,7 @@ func (r *ItemRepository) CreateRevision(ctx context.Context, rev *Revision) erro
|
||||
}
|
||||
|
||||
if rev.FileStorageBackend == "" {
|
||||
rev.FileStorageBackend = "minio"
|
||||
rev.FileStorageBackend = "filesystem"
|
||||
}
|
||||
|
||||
err = r.db.pool.QueryRow(ctx, `
|
||||
@@ -347,7 +347,7 @@ func (r *ItemRepository) GetRevisions(ctx context.Context, itemID string) ([]*Re
|
||||
if hasStatusColumn {
|
||||
rows, err = r.db.pool.Query(ctx, `
|
||||
SELECT id, item_id, revision_number, properties, file_key, file_version,
|
||||
file_checksum, file_size, COALESCE(file_storage_backend, 'minio'),
|
||||
file_checksum, file_size, COALESCE(file_storage_backend, 'filesystem'),
|
||||
thumbnail_key, created_at, created_by, comment,
|
||||
COALESCE(status, 'draft') as status, COALESCE(labels, ARRAY[]::TEXT[]) as labels
|
||||
FROM revisions
|
||||
@@ -386,7 +386,7 @@ func (r *ItemRepository) GetRevisions(ctx context.Context, itemID string) ([]*Re
|
||||
)
|
||||
rev.Status = "draft"
|
||||
rev.Labels = []string{}
|
||||
rev.FileStorageBackend = "minio"
|
||||
rev.FileStorageBackend = "filesystem"
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning revision: %w", err)
|
||||
@@ -420,7 +420,7 @@ func (r *ItemRepository) GetRevision(ctx context.Context, itemID string, revisio
|
||||
if hasStatusColumn {
|
||||
err = r.db.pool.QueryRow(ctx, `
|
||||
SELECT id, item_id, revision_number, properties, file_key, file_version,
|
||||
file_checksum, file_size, COALESCE(file_storage_backend, 'minio'),
|
||||
file_checksum, file_size, COALESCE(file_storage_backend, 'filesystem'),
|
||||
thumbnail_key, created_at, created_by, comment,
|
||||
COALESCE(status, 'draft') as status, COALESCE(labels, ARRAY[]::TEXT[]) as labels
|
||||
FROM revisions
|
||||
@@ -443,7 +443,7 @@ func (r *ItemRepository) GetRevision(ctx context.Context, itemID string, revisio
|
||||
)
|
||||
rev.Status = "draft"
|
||||
rev.Labels = []string{}
|
||||
rev.FileStorageBackend = "minio"
|
||||
rev.FileStorageBackend = "filesystem"
|
||||
}
|
||||
|
||||
if err == pgx.ErrNoRows {
|
||||
|
||||
@@ -50,7 +50,7 @@ type Registry struct {
|
||||
var builtinModules = []ModuleInfo{
|
||||
{ID: Core, Name: "Core PDM", Description: "Items, revisions, files, BOM, search, import/export", Required: true, Version: "0.2"},
|
||||
{ID: Schemas, Name: "Schemas", Description: "Part numbering schema parsing and segment management", Required: true},
|
||||
{ID: Storage, Name: "Storage", Description: "MinIO/S3 file storage, presigned uploads", Required: true},
|
||||
{ID: Storage, Name: "Storage", Description: "Filesystem storage", Required: true},
|
||||
{ID: Auth, Name: "Authentication", Description: "Local, LDAP, OIDC authentication and RBAC", DefaultEnabled: true},
|
||||
{ID: Projects, Name: "Projects", Description: "Project management and item tagging", DefaultEnabled: true},
|
||||
{ID: Audit, Name: "Audit", Description: "Audit logging, completeness scoring", DefaultEnabled: true},
|
||||
|
||||
@@ -3,6 +3,7 @@ package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"time"
|
||||
@@ -19,3 +20,21 @@ type FileStore interface {
|
||||
PresignPut(ctx context.Context, key string, expiry time.Duration) (*url.URL, error)
|
||||
Ping(ctx context.Context) error
|
||||
}
|
||||
|
||||
// PutResult contains the result of a put operation.
|
||||
type PutResult struct {
|
||||
Key string
|
||||
VersionID string
|
||||
Size int64
|
||||
Checksum string
|
||||
}
|
||||
|
||||
// FileKey generates a storage key for an item file.
|
||||
func FileKey(partNumber string, revision int) string {
|
||||
return fmt.Sprintf("items/%s/rev%d.FCStd", partNumber, revision)
|
||||
}
|
||||
|
||||
// ThumbnailKey generates a storage key for a thumbnail.
|
||||
func ThumbnailKey(partNumber string, revision int) string {
|
||||
return fmt.Sprintf("thumbnails/%s/rev%d.png", partNumber, revision)
|
||||
}
|
||||
|
||||
@@ -1,174 +0,0 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
// Config holds MinIO connection settings.
|
||||
type Config struct {
|
||||
Endpoint string
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
Bucket string
|
||||
UseSSL bool
|
||||
Region string
|
||||
}
|
||||
|
||||
// Compile-time check: *Storage implements FileStore.
|
||||
var _ FileStore = (*Storage)(nil)
|
||||
|
||||
// Storage wraps MinIO client operations.
|
||||
type Storage struct {
|
||||
client *minio.Client
|
||||
bucket string
|
||||
}
|
||||
|
||||
// Connect creates a new MinIO storage client.
|
||||
func Connect(ctx context.Context, cfg Config) (*Storage, error) {
|
||||
client, err := minio.New(cfg.Endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
|
||||
Secure: cfg.UseSSL,
|
||||
Region: cfg.Region,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating minio client: %w", err)
|
||||
}
|
||||
|
||||
// Ensure bucket exists with versioning
|
||||
exists, err := client.BucketExists(ctx, cfg.Bucket)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("checking bucket: %w", err)
|
||||
}
|
||||
if !exists {
|
||||
if err := client.MakeBucket(ctx, cfg.Bucket, minio.MakeBucketOptions{
|
||||
Region: cfg.Region,
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("creating bucket: %w", err)
|
||||
}
|
||||
// Enable versioning
|
||||
if err := client.EnableVersioning(ctx, cfg.Bucket); err != nil {
|
||||
return nil, fmt.Errorf("enabling versioning: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return &Storage{client: client, bucket: cfg.Bucket}, nil
|
||||
}
|
||||
|
||||
// PutResult contains the result of a put operation.
|
||||
type PutResult struct {
|
||||
Key string
|
||||
VersionID string
|
||||
Size int64
|
||||
Checksum string
|
||||
}
|
||||
|
||||
// Put uploads a file to storage.
|
||||
func (s *Storage) Put(ctx context.Context, key string, reader io.Reader, size int64, contentType string) (*PutResult, error) {
|
||||
info, err := s.client.PutObject(ctx, s.bucket, key, reader, size, minio.PutObjectOptions{
|
||||
ContentType: contentType,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("uploading object: %w", err)
|
||||
}
|
||||
|
||||
return &PutResult{
|
||||
Key: key,
|
||||
VersionID: info.VersionID,
|
||||
Size: info.Size,
|
||||
Checksum: info.ChecksumSHA256,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Get downloads a file from storage.
|
||||
func (s *Storage) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||
obj, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting object: %w", err)
|
||||
}
|
||||
return obj, nil
|
||||
}
|
||||
|
||||
// GetVersion downloads a specific version of a file.
|
||||
func (s *Storage) GetVersion(ctx context.Context, key, versionID string) (io.ReadCloser, error) {
|
||||
obj, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{
|
||||
VersionID: versionID,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting object version: %w", err)
|
||||
}
|
||||
return obj, nil
|
||||
}
|
||||
|
||||
// Delete removes a file from storage.
|
||||
func (s *Storage) Delete(ctx context.Context, key string) error {
|
||||
if err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}); err != nil {
|
||||
return fmt.Errorf("removing object: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Exists checks if an object exists in storage.
|
||||
func (s *Storage) Exists(ctx context.Context, key string) (bool, error) {
|
||||
_, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
resp := minio.ToErrorResponse(err)
|
||||
if resp.Code == "NoSuchKey" {
|
||||
return false, nil
|
||||
}
|
||||
return false, fmt.Errorf("checking object existence: %w", err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Ping checks if the storage backend is reachable by verifying the bucket exists.
|
||||
func (s *Storage) Ping(ctx context.Context) error {
|
||||
_, err := s.client.BucketExists(ctx, s.bucket)
|
||||
return err
|
||||
}
|
||||
|
||||
// Bucket returns the bucket name.
|
||||
func (s *Storage) Bucket() string {
|
||||
return s.bucket
|
||||
}
|
||||
|
||||
// PresignPut generates a presigned PUT URL for direct browser upload.
|
||||
func (s *Storage) PresignPut(ctx context.Context, key string, expiry time.Duration) (*url.URL, error) {
|
||||
u, err := s.client.PresignedPutObject(ctx, s.bucket, key, expiry)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("generating presigned put URL: %w", err)
|
||||
}
|
||||
return u, nil
|
||||
}
|
||||
|
||||
// Copy copies an object within the same bucket from srcKey to dstKey.
|
||||
func (s *Storage) Copy(ctx context.Context, srcKey, dstKey string) error {
|
||||
src := minio.CopySrcOptions{
|
||||
Bucket: s.bucket,
|
||||
Object: srcKey,
|
||||
}
|
||||
dst := minio.CopyDestOptions{
|
||||
Bucket: s.bucket,
|
||||
Object: dstKey,
|
||||
}
|
||||
if _, err := s.client.CopyObject(ctx, dst, src); err != nil {
|
||||
return fmt.Errorf("copying object: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// FileKey generates a storage key for an item file.
|
||||
func FileKey(partNumber string, revision int) string {
|
||||
return fmt.Sprintf("items/%s/rev%d.FCStd", partNumber, revision)
|
||||
}
|
||||
|
||||
// ThumbnailKey generates a storage key for a thumbnail.
|
||||
func ThumbnailKey(partNumber string, revision int) string {
|
||||
return fmt.Sprintf("thumbnails/%s/rev%d.png", partNumber, revision)
|
||||
}
|
||||
3
migrations/020_storage_backend_filesystem_default.sql
Normal file
3
migrations/020_storage_backend_filesystem_default.sql
Normal file
@@ -0,0 +1,3 @@
|
||||
-- Change default storage backend from 'minio' to 'filesystem'.
|
||||
ALTER TABLE item_files ALTER COLUMN storage_backend SET DEFAULT 'filesystem';
|
||||
ALTER TABLE revisions ALTER COLUMN file_storage_backend SET DEFAULT 'filesystem';
|
||||
@@ -8,7 +8,6 @@
|
||||
# - SSH access to the target host
|
||||
# - /etc/silo/silod.env must exist on target with credentials filled in
|
||||
# - PostgreSQL reachable from target (set SILO_DB_HOST to override)
|
||||
# - MinIO reachable from target (set SILO_MINIO_HOST to override)
|
||||
#
|
||||
# Environment variables:
|
||||
# SILO_DEPLOY_TARGET - target host (default: silo.example.internal)
|
||||
|
||||
@@ -1,108 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Migrate storage from MinIO to filesystem on a remote Silo host.
|
||||
#
|
||||
# Builds the migrate-storage binary locally, uploads it to the target host,
|
||||
# then runs it over SSH using credentials from /etc/silo/silod.env.
|
||||
#
|
||||
# Usage: ./scripts/migrate-storage.sh <silo-host> <psql-host> <minio-host> [flags...]
|
||||
#
|
||||
# Examples:
|
||||
# ./scripts/migrate-storage.sh silo.kindred.internal psql.kindred.internal minio.kindred.internal -dry-run -verbose
|
||||
# ./scripts/migrate-storage.sh silo.kindred.internal psql.kindred.internal minio.kindred.internal
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [ $# -lt 3 ]; then
|
||||
echo "Usage: $0 <silo-host> <psql-host> <minio-host> [flags...]"
|
||||
echo " flags are passed to migrate-storage (e.g. -dry-run -verbose)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TARGET="$1"
|
||||
DB_HOST="$2"
|
||||
MINIO_HOST="$3"
|
||||
shift 3
|
||||
EXTRA_FLAGS="$*"
|
||||
|
||||
DEST_DIR="/opt/silo/data"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="${SCRIPT_DIR}/.."
|
||||
|
||||
echo "=== Migrate Storage: MinIO -> Filesystem ==="
|
||||
echo " Target: ${TARGET}"
|
||||
echo " DB host: ${DB_HOST}"
|
||||
echo " MinIO: ${MINIO_HOST}"
|
||||
echo " Dest: ${DEST_DIR}"
|
||||
[ -n "$EXTRA_FLAGS" ] && echo " Flags: ${EXTRA_FLAGS}"
|
||||
echo ""
|
||||
|
||||
# --- Build locally ---
|
||||
echo "[1/3] Building migrate-storage binary..."
|
||||
cd "$PROJECT_DIR"
|
||||
GOOS=linux GOARCH=amd64 go build -o migrate-storage ./cmd/migrate-storage
|
||||
echo " Built: $(du -h migrate-storage | cut -f1)"
|
||||
|
||||
# --- Upload ---
|
||||
echo "[2/3] Uploading to ${TARGET}..."
|
||||
scp migrate-storage "${TARGET}:/tmp/migrate-storage"
|
||||
rm -f migrate-storage
|
||||
|
||||
# --- Run remotely ---
|
||||
echo "[3/3] Running migration on ${TARGET}..."
|
||||
ssh "$TARGET" DB_HOST="$DB_HOST" MINIO_HOST="$MINIO_HOST" DEST_DIR="$DEST_DIR" EXTRA_FLAGS="$EXTRA_FLAGS" bash -s <<'REMOTE'
|
||||
set -euo pipefail
|
||||
|
||||
CONFIG_DIR="/etc/silo"
|
||||
|
||||
# Source credentials
|
||||
if [ ! -f "$CONFIG_DIR/silod.env" ]; then
|
||||
echo "ERROR: $CONFIG_DIR/silod.env not found on $(hostname)"
|
||||
exit 1
|
||||
fi
|
||||
set -a
|
||||
source "$CONFIG_DIR/silod.env"
|
||||
set +a
|
||||
|
||||
# Ensure destination directory exists
|
||||
sudo mkdir -p "$DEST_DIR"
|
||||
sudo chown silo:silo "$DEST_DIR" 2>/dev/null || true
|
||||
|
||||
chmod +x /tmp/migrate-storage
|
||||
|
||||
# Write temporary config with the provided hosts
|
||||
cat > /tmp/silo-migrate.yaml <<EOF
|
||||
database:
|
||||
host: "${DB_HOST}"
|
||||
port: 5432
|
||||
name: "silo"
|
||||
user: "silo"
|
||||
password: "${SILO_DB_PASSWORD}"
|
||||
sslmode: "require"
|
||||
max_connections: 5
|
||||
|
||||
storage:
|
||||
endpoint: "${MINIO_HOST}:9000"
|
||||
access_key: "${SILO_MINIO_ACCESS_KEY}"
|
||||
secret_key: "${SILO_MINIO_SECRET_KEY}"
|
||||
bucket: "silo"
|
||||
use_ssl: false
|
||||
region: "us-east-1"
|
||||
EOF
|
||||
chmod 600 /tmp/silo-migrate.yaml
|
||||
|
||||
echo " Config written to /tmp/silo-migrate.yaml"
|
||||
echo " Starting migration..."
|
||||
echo ""
|
||||
|
||||
# Run the migration
|
||||
/tmp/migrate-storage -config /tmp/silo-migrate.yaml -dest "$DEST_DIR" $EXTRA_FLAGS
|
||||
|
||||
# Clean up
|
||||
rm -f /tmp/silo-migrate.yaml /tmp/migrate-storage
|
||||
echo ""
|
||||
echo " Cleaned up temp files."
|
||||
REMOTE
|
||||
|
||||
echo ""
|
||||
echo "=== Migration complete ==="
|
||||
echo " Files written to ${TARGET}:${DEST_DIR}"
|
||||
@@ -138,12 +138,6 @@ fi
|
||||
PG_PASSWORD_DEFAULT="$(generate_secret 16)"
|
||||
prompt_secret POSTGRES_PASSWORD "PostgreSQL password" "$PG_PASSWORD_DEFAULT"
|
||||
|
||||
# MinIO
|
||||
MINIO_AK_DEFAULT="$(generate_secret 10)"
|
||||
MINIO_SK_DEFAULT="$(generate_secret 16)"
|
||||
prompt_secret MINIO_ACCESS_KEY "MinIO access key" "$MINIO_AK_DEFAULT"
|
||||
prompt_secret MINIO_SECRET_KEY "MinIO secret key" "$MINIO_SK_DEFAULT"
|
||||
|
||||
# OpenLDAP
|
||||
LDAP_ADMIN_PW_DEFAULT="$(generate_secret 16)"
|
||||
prompt_secret LDAP_ADMIN_PASSWORD "LDAP admin password" "$LDAP_ADMIN_PW_DEFAULT"
|
||||
@@ -173,10 +167,6 @@ cat > "${OUTPUT_DIR}/.env" << EOF
|
||||
# PostgreSQL
|
||||
POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
||||
|
||||
# MinIO
|
||||
MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY}
|
||||
MINIO_SECRET_KEY=${MINIO_SECRET_KEY}
|
||||
|
||||
# OpenLDAP
|
||||
LDAP_ADMIN_PASSWORD=${LDAP_ADMIN_PASSWORD}
|
||||
LDAP_USERS=${LDAP_USERS}
|
||||
@@ -235,12 +225,9 @@ database:
|
||||
max_connections: 10
|
||||
|
||||
storage:
|
||||
endpoint: "minio:9000"
|
||||
access_key: "${SILO_MINIO_ACCESS_KEY}"
|
||||
secret_key: "${SILO_MINIO_SECRET_KEY}"
|
||||
bucket: "silo-files"
|
||||
use_ssl: false
|
||||
region: "us-east-1"
|
||||
backend: "filesystem"
|
||||
filesystem:
|
||||
root_dir: "/var/lib/silo/data"
|
||||
|
||||
schemas:
|
||||
directory: "/etc/silo/schemas"
|
||||
@@ -306,8 +293,6 @@ echo " deployments/config.docker.yaml - server configuration"
|
||||
echo ""
|
||||
echo -e "${BOLD}Credentials:${NC}"
|
||||
echo " PostgreSQL: silo / ${POSTGRES_PASSWORD}"
|
||||
echo " MinIO: ${MINIO_ACCESS_KEY} / ${MINIO_SECRET_KEY}"
|
||||
echo " MinIO Console: http://localhost:9001"
|
||||
echo " LDAP Admin: cn=admin,dc=silo,dc=local / ${LDAP_ADMIN_PASSWORD}"
|
||||
echo " LDAP User: ${LDAP_USERS} / ${LDAP_PASSWORDS}"
|
||||
echo " Silo Admin: ${SILO_ADMIN_USERNAME} / ${SILO_ADMIN_PASSWORD} (local fallback)"
|
||||
|
||||
@@ -30,7 +30,6 @@ INSTALL_DIR="/opt/silo"
|
||||
CONFIG_DIR="/etc/silo"
|
||||
GO_VERSION="1.24.0"
|
||||
DB_HOST="${SILO_DB_HOST:-psql.example.internal}"
|
||||
MINIO_HOST="${SILO_MINIO_HOST:-minio.example.internal}"
|
||||
|
||||
log_info() { echo -e "${BLUE}[INFO]${NC} $*"; }
|
||||
log_success() { echo -e "${GREEN}[OK]${NC} $*"; }
|
||||
@@ -165,11 +164,6 @@ if [[ ! -f "${ENV_FILE}" ]]; then
|
||||
# Database: silo, User: silo
|
||||
SILO_DB_PASSWORD=
|
||||
|
||||
# MinIO credentials (${MINIO_HOST})
|
||||
# User: silouser
|
||||
SILO_MINIO_ACCESS_KEY=silouser
|
||||
SILO_MINIO_SECRET_KEY=
|
||||
|
||||
# Authentication
|
||||
# Session secret (required when auth is enabled)
|
||||
SILO_SESSION_SECRET=
|
||||
@@ -225,10 +219,7 @@ echo ""
|
||||
echo "2. Verify database connectivity:"
|
||||
echo " psql -h ${DB_HOST} -U silo -d silo -c 'SELECT 1'"
|
||||
echo ""
|
||||
echo "3. Verify MinIO connectivity:"
|
||||
echo " curl -I http://${MINIO_HOST}:9000/minio/health/live"
|
||||
echo ""
|
||||
echo "4. Run the deployment:"
|
||||
echo "3. Run the deployment:"
|
||||
echo " sudo ${INSTALL_DIR}/src/scripts/deploy.sh"
|
||||
echo ""
|
||||
echo "After deployment, manage the service with:"
|
||||
|
||||
Reference in New Issue
Block a user