diff --git a/.env.example b/.env.example index 8f582f8..95b3c36 100644 --- a/.env.example +++ b/.env.example @@ -5,10 +5,6 @@ # PostgreSQL POSTGRES_PASSWORD=silodev -# MinIO -MINIO_ACCESS_KEY=silominio -MINIO_SECRET_KEY=silominiosecret - # OpenLDAP LDAP_ADMIN_PASSWORD=ldapadmin LDAP_USERS=siloadmin diff --git a/Makefile b/Makefile index 793ed31..ea282ed 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,7 @@ .PHONY: build run test test-integration clean migrate fmt lint \ docker-build docker-up docker-down docker-logs docker-ps \ docker-clean docker-rebuild \ - web-install web-dev web-build \ - migrate-storage + web-install web-dev web-build # ============================================================================= # Local Development @@ -57,13 +56,6 @@ tidy: migrate: ./scripts/init-db.sh -# Build and run MinIO → filesystem migration tool -# Usage: make migrate-storage DEST=/opt/silo/data [ARGS="--dry-run --verbose"] -migrate-storage: - go build -o migrate-storage ./cmd/migrate-storage - @echo "Built ./migrate-storage" - @echo "Run: ./migrate-storage -config -dest [-dry-run] [-verbose]" - # Connect to database (requires psql) db-shell: PGPASSWORD=$${SILO_DB_PASSWORD:-silodev} psql -h $${SILO_DB_HOST:-localhost} -U $${SILO_DB_USER:-silo} -d $${SILO_DB_NAME:-silo} @@ -76,7 +68,7 @@ db-shell: docker-build: docker build -t silo:latest -f build/package/Dockerfile . -# Start the full stack (postgres + minio + silo) +# Start the full stack (postgres + silo) docker-up: docker compose -f deployments/docker-compose.yaml up -d @@ -103,9 +95,6 @@ docker-logs-silo: docker-logs-postgres: docker compose -f deployments/docker-compose.yaml logs -f postgres -docker-logs-minio: - docker compose -f deployments/docker-compose.yaml logs -f minio - # Show running containers docker-ps: docker compose -f deployments/docker-compose.yaml ps @@ -175,7 +164,7 @@ help: @echo "" @echo "Docker:" @echo " docker-build - Build Docker image" - @echo " docker-up - Start full stack (postgres + minio + silo)" + @echo " docker-up - Start full stack (postgres + silo)" @echo " docker-down - Stop the stack" @echo " docker-clean - Stop and remove volumes (deletes data)" @echo " docker-logs - View all logs" diff --git a/README.md b/README.md index 3262f06..eb6fcd0 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ silo/ │ ├── ods/ # ODS spreadsheet library │ ├── partnum/ # Part number generation │ ├── schema/ # YAML schema parsing -│ ├── storage/ # MinIO file storage +│ ├── storage/ # Filesystem storage │ └── testutil/ # Test helpers ├── web/ # React SPA (Vite + TypeScript) │ └── src/ @@ -55,7 +55,7 @@ silo/ See the **[Installation Guide](docs/INSTALL.md)** for complete setup instructions. -**Docker Compose (quickest — includes PostgreSQL, MinIO, OpenLDAP, and Silo):** +**Docker Compose (quickest — includes PostgreSQL, OpenLDAP, and Silo):** ```bash ./scripts/setup-docker.sh @@ -65,7 +65,7 @@ docker compose -f deployments/docker-compose.allinone.yaml up -d **Development (local Go + Docker services):** ```bash -make docker-up # Start PostgreSQL + MinIO in Docker +make docker-up # Start PostgreSQL in Docker make run # Run silo locally with Go ``` diff --git a/cmd/migrate-storage/main.go b/cmd/migrate-storage/main.go deleted file mode 100644 index 5e64fcf..0000000 --- a/cmd/migrate-storage/main.go +++ /dev/null @@ -1,288 +0,0 @@ -// Command migrate-storage downloads files from MinIO and writes them to the -// local filesystem. It is a one-shot migration tool for moving off MinIO. -// -// Usage: -// -// migrate-storage -config config.yaml -dest /opt/silo/data [-dry-run] [-verbose] -package main - -import ( - "context" - "flag" - "fmt" - "io" - "os" - "path/filepath" - "time" - - "github.com/kindredsystems/silo/internal/config" - "github.com/kindredsystems/silo/internal/db" - "github.com/kindredsystems/silo/internal/storage" - "github.com/rs/zerolog" -) - -// fileEntry represents a single file to migrate. -type fileEntry struct { - key string - versionID string // MinIO version ID; empty if not versioned - size int64 // expected size from DB; 0 if unknown -} - -func main() { - configPath := flag.String("config", "config.yaml", "Path to configuration file") - dest := flag.String("dest", "", "Destination root directory (required)") - dryRun := flag.Bool("dry-run", false, "Preview what would be migrated without downloading") - verbose := flag.Bool("verbose", false, "Log every file, not just errors and summary") - flag.Parse() - - logger := zerolog.New(os.Stdout).With().Timestamp().Logger() - - if *dest == "" { - logger.Fatal().Msg("-dest is required") - } - - // Load config (reuses existing config for DB + MinIO credentials). - cfg, err := config.Load(*configPath) - if err != nil { - logger.Fatal().Err(err).Msg("failed to load configuration") - } - - ctx := context.Background() - - // Connect to PostgreSQL. - database, err := db.Connect(ctx, db.Config{ - Host: cfg.Database.Host, - Port: cfg.Database.Port, - Name: cfg.Database.Name, - User: cfg.Database.User, - Password: cfg.Database.Password, - SSLMode: cfg.Database.SSLMode, - MaxConnections: cfg.Database.MaxConnections, - }) - if err != nil { - logger.Fatal().Err(err).Msg("failed to connect to database") - } - defer database.Close() - logger.Info().Msg("connected to database") - - // Connect to MinIO. - store, err := storage.Connect(ctx, storage.Config{ - Endpoint: cfg.Storage.Endpoint, - AccessKey: cfg.Storage.AccessKey, - SecretKey: cfg.Storage.SecretKey, - Bucket: cfg.Storage.Bucket, - UseSSL: cfg.Storage.UseSSL, - Region: cfg.Storage.Region, - }) - if err != nil { - logger.Fatal().Err(err).Msg("failed to connect to MinIO") - } - logger.Info().Str("bucket", cfg.Storage.Bucket).Msg("connected to MinIO") - - // Collect all file references from the database. - entries, err := collectEntries(ctx, logger, database) - if err != nil { - logger.Fatal().Err(err).Msg("failed to collect file entries from database") - } - logger.Info().Int("total", len(entries)).Msg("file entries found") - - if len(entries) == 0 { - logger.Info().Msg("nothing to migrate") - return - } - - // Migrate. - var migrated, skipped, failed int - start := time.Now() - - for i, e := range entries { - destPath := filepath.Join(*dest, e.key) - - // Check if already migrated. - if info, err := os.Stat(destPath); err == nil { - if e.size > 0 && info.Size() == e.size { - if *verbose { - logger.Info().Str("key", e.key).Msg("skipped (already exists)") - } - skipped++ - continue - } - // Size mismatch or unknown size — re-download. - } - - if *dryRun { - logger.Info(). - Str("key", e.key). - Int64("size", e.size). - Str("version", e.versionID). - Msgf("[%d/%d] would migrate", i+1, len(entries)) - continue - } - - if err := migrateFile(ctx, store, e, destPath); err != nil { - logger.Error().Err(err).Str("key", e.key).Msg("failed to migrate") - failed++ - continue - } - - migrated++ - if *verbose { - logger.Info(). - Str("key", e.key). - Int64("size", e.size). - Msgf("[%d/%d] migrated", i+1, len(entries)) - } else if (i+1)%50 == 0 { - logger.Info().Msgf("progress: %d/%d", i+1, len(entries)) - } - } - - elapsed := time.Since(start) - ev := logger.Info(). - Int("total", len(entries)). - Int("migrated", migrated). - Int("skipped", skipped). - Int("failed", failed). - Dur("elapsed", elapsed) - if *dryRun { - ev.Msg("dry run complete") - } else { - ev.Msg("migration complete") - } - - if failed > 0 { - os.Exit(1) - } -} - -// collectEntries queries the database for all file references across the three -// storage domains: revision files, item file attachments, and item thumbnails. -// It deduplicates by key. -func collectEntries(ctx context.Context, logger zerolog.Logger, database *db.DB) ([]fileEntry, error) { - pool := database.Pool() - seen := make(map[string]struct{}) - var entries []fileEntry - - add := func(key, versionID string, size int64) { - if key == "" { - return - } - if _, ok := seen[key]; ok { - return - } - seen[key] = struct{}{} - entries = append(entries, fileEntry{key: key, versionID: versionID, size: size}) - } - - // 1. Revision files. - rows, err := pool.Query(ctx, - `SELECT file_key, COALESCE(file_version, ''), COALESCE(file_size, 0) - FROM revisions WHERE file_key IS NOT NULL`) - if err != nil { - return nil, fmt.Errorf("querying revisions: %w", err) - } - for rows.Next() { - var key, version string - var size int64 - if err := rows.Scan(&key, &version, &size); err != nil { - rows.Close() - return nil, fmt.Errorf("scanning revision row: %w", err) - } - add(key, version, size) - } - rows.Close() - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("iterating revisions: %w", err) - } - logger.Info().Int("count", len(entries)).Msg("revision files found") - - // 2. Item file attachments. - countBefore := len(entries) - rows, err = pool.Query(ctx, - `SELECT object_key, size FROM item_files`) - if err != nil { - return nil, fmt.Errorf("querying item_files: %w", err) - } - for rows.Next() { - var key string - var size int64 - if err := rows.Scan(&key, &size); err != nil { - rows.Close() - return nil, fmt.Errorf("scanning item_files row: %w", err) - } - add(key, "", size) - } - rows.Close() - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("iterating item_files: %w", err) - } - logger.Info().Int("count", len(entries)-countBefore).Msg("item file attachments found") - - // 3. Item thumbnails. - countBefore = len(entries) - rows, err = pool.Query(ctx, - `SELECT thumbnail_key FROM items WHERE thumbnail_key IS NOT NULL`) - if err != nil { - return nil, fmt.Errorf("querying item thumbnails: %w", err) - } - for rows.Next() { - var key string - if err := rows.Scan(&key); err != nil { - rows.Close() - return nil, fmt.Errorf("scanning thumbnail row: %w", err) - } - add(key, "", 0) - } - rows.Close() - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("iterating thumbnails: %w", err) - } - logger.Info().Int("count", len(entries)-countBefore).Msg("item thumbnails found") - - return entries, nil -} - -// migrateFile downloads a single file from MinIO and writes it atomically to destPath. -func migrateFile(ctx context.Context, store *storage.Storage, e fileEntry, destPath string) error { - // Ensure parent directory exists. - if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { - return fmt.Errorf("creating directory: %w", err) - } - - // Download from MinIO. - var reader io.ReadCloser - var err error - if e.versionID != "" { - reader, err = store.GetVersion(ctx, e.key, e.versionID) - } else { - reader, err = store.Get(ctx, e.key) - } - if err != nil { - return fmt.Errorf("downloading from MinIO: %w", err) - } - defer reader.Close() - - // Write to temp file then rename for atomicity. - tmpPath := destPath + ".tmp" - f, err := os.Create(tmpPath) - if err != nil { - return fmt.Errorf("creating temp file: %w", err) - } - - if _, err := io.Copy(f, reader); err != nil { - f.Close() - os.Remove(tmpPath) - return fmt.Errorf("writing file: %w", err) - } - - if err := f.Close(); err != nil { - os.Remove(tmpPath) - return fmt.Errorf("closing temp file: %w", err) - } - - if err := os.Rename(tmpPath, destPath); err != nil { - os.Remove(tmpPath) - return fmt.Errorf("renaming temp file: %w", err) - } - - return nil -} diff --git a/cmd/silod/main.go b/cmd/silod/main.go index 2054c56..1697ec2 100644 --- a/cmd/silod/main.go +++ b/cmd/silod/main.go @@ -45,7 +45,6 @@ func main() { Str("host", cfg.Server.Host). Int("port", cfg.Server.Port). Str("database", cfg.Database.Host). - Str("storage", cfg.Storage.Endpoint). Msg("starting silo server") // Connect to database @@ -65,40 +64,17 @@ func main() { defer database.Close() logger.Info().Msg("connected to database") - // Connect to storage (optional - may be externally managed) + // Connect to storage (optional — requires root_dir to be set) var store storage.FileStore - switch cfg.Storage.Backend { - case "minio", "": - if cfg.Storage.Endpoint != "" { - s, connErr := storage.Connect(ctx, storage.Config{ - Endpoint: cfg.Storage.Endpoint, - AccessKey: cfg.Storage.AccessKey, - SecretKey: cfg.Storage.SecretKey, - Bucket: cfg.Storage.Bucket, - UseSSL: cfg.Storage.UseSSL, - Region: cfg.Storage.Region, - }) - if connErr != nil { - logger.Warn().Err(connErr).Msg("failed to connect to storage - file operations disabled") - } else { - store = s - logger.Info().Msg("connected to storage") - } - } else { - logger.Info().Msg("storage not configured - file operations disabled") - } - case "filesystem": - if cfg.Storage.Filesystem.RootDir == "" { - logger.Fatal().Msg("storage.filesystem.root_dir is required when backend is \"filesystem\"") - } + if cfg.Storage.Filesystem.RootDir != "" { s, fsErr := storage.NewFilesystemStore(cfg.Storage.Filesystem.RootDir) if fsErr != nil { logger.Fatal().Err(fsErr).Msg("failed to initialize filesystem storage") } store = s logger.Info().Str("root", cfg.Storage.Filesystem.RootDir).Msg("connected to filesystem storage") - default: - logger.Fatal().Str("backend", cfg.Storage.Backend).Msg("unknown storage backend") + } else { + logger.Info().Msg("storage not configured - file operations disabled") } // Load schemas diff --git a/config.example.yaml b/config.example.yaml index 4a30661..8915a12 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -17,17 +17,9 @@ database: max_connections: 10 storage: - backend: "minio" # "minio" (default) or "filesystem" - # MinIO/S3 settings (used when backend: "minio") - endpoint: "localhost:9000" # Use "minio:9000" for Docker Compose - access_key: "" # Use SILO_MINIO_ACCESS_KEY env var - secret_key: "" # Use SILO_MINIO_SECRET_KEY env var - bucket: "silo-files" - use_ssl: true # Use false for Docker Compose (internal network) - region: "us-east-1" - # Filesystem settings (used when backend: "filesystem") - # filesystem: - # root_dir: "/var/lib/silo/objects" + backend: "filesystem" + filesystem: + root_dir: "/opt/silo/data" # Override with SILO_STORAGE_ROOT_DIR env var schemas: # Directory containing YAML schema files diff --git a/deployments/config.dev.yaml b/deployments/config.dev.yaml index 9f57d05..a2302dc 100644 --- a/deployments/config.dev.yaml +++ b/deployments/config.dev.yaml @@ -17,12 +17,9 @@ database: max_connections: 10 storage: - endpoint: "minio:9000" - access_key: "${MINIO_ACCESS_KEY:-silominio}" - secret_key: "${MINIO_SECRET_KEY:-silominiosecret}" - bucket: "silo-files" - use_ssl: false - region: "us-east-1" + backend: "filesystem" + filesystem: + root_dir: "/var/lib/silo/data" schemas: directory: "/etc/silo/schemas" diff --git a/deployments/docker-compose.allinone.yaml b/deployments/docker-compose.allinone.yaml index 6d73fc8..42448ff 100644 --- a/deployments/docker-compose.allinone.yaml +++ b/deployments/docker-compose.allinone.yaml @@ -1,5 +1,5 @@ # Silo All-in-One Stack -# PostgreSQL + MinIO + OpenLDAP + Silo API + Nginx (optional) +# PostgreSQL + OpenLDAP + Silo API + Nginx (optional) # # Quick start: # ./scripts/setup-docker.sh @@ -40,29 +40,6 @@ services: networks: - silo-net - # --------------------------------------------------------------------------- - # MinIO (S3-compatible object storage) - # --------------------------------------------------------------------------- - minio: - image: minio/minio:latest - container_name: silo-minio - restart: unless-stopped - command: server /data --console-address ":9001" - environment: - MINIO_ROOT_USER: ${MINIO_ACCESS_KEY:?Run ./scripts/setup-docker.sh first} - MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY:?Run ./scripts/setup-docker.sh first} - volumes: - - minio_data:/data - ports: - - "9001:9001" # MinIO console (remove in hardened setups) - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 10s - timeout: 5s - retries: 5 - networks: - - silo-net - # --------------------------------------------------------------------------- # OpenLDAP (user directory for LDAP authentication) # --------------------------------------------------------------------------- @@ -83,9 +60,13 @@ services: - openldap_data:/bitnami/openldap - ./ldap:/docker-entrypoint-initdb.d:ro ports: - - "1389:1389" # LDAP access for debugging (remove in hardened setups) + - "1389:1389" # LDAP access for debugging (remove in hardened setups) healthcheck: - test: ["CMD-SHELL", "ldapsearch -x -H ldap://localhost:1389 -b dc=silo,dc=local -D cn=admin,dc=silo,dc=local -w $${LDAP_ADMIN_PASSWORD} '(objectClass=organization)' >/dev/null 2>&1"] + test: + [ + "CMD-SHELL", + "ldapsearch -x -H ldap://localhost:1389 -b dc=silo,dc=local -D cn=admin,dc=silo,dc=local -w $${LDAP_ADMIN_PASSWORD} '(objectClass=organization)' >/dev/null 2>&1", + ] interval: 10s timeout: 5s retries: 5 @@ -104,8 +85,6 @@ services: depends_on: postgres: condition: service_healthy - minio: - condition: service_healthy openldap: condition: service_healthy env_file: @@ -117,12 +96,10 @@ services: SILO_DB_NAME: silo SILO_DB_USER: silo SILO_DB_PASSWORD: ${POSTGRES_PASSWORD} - SILO_MINIO_ENDPOINT: minio:9000 - SILO_MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} - SILO_MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} ports: - "${SILO_PORT:-8080}:8080" volumes: + - silo_data:/var/lib/silo/data - ../schemas:/etc/silo/schemas:ro - ./config.docker.yaml:/etc/silo/config.yaml:ro healthcheck: @@ -164,7 +141,7 @@ services: volumes: postgres_data: - minio_data: + silo_data: openldap_data: networks: diff --git a/deployments/docker-compose.prod.yaml b/deployments/docker-compose.prod.yaml index 88b7175..d0be4af 100644 --- a/deployments/docker-compose.prod.yaml +++ b/deployments/docker-compose.prod.yaml @@ -1,10 +1,8 @@ # Production Docker Compose for Silo -# Uses external PostgreSQL (psql.example.internal) and MinIO (minio.example.internal) +# Uses external PostgreSQL (psql.example.internal) and filesystem storage # # Usage: # export SILO_DB_PASSWORD= -# export SILO_MINIO_ACCESS_KEY= -# export SILO_MINIO_SECRET_KEY= # docker compose -f docker-compose.prod.yaml up -d services: @@ -24,14 +22,6 @@ services: # Note: SILO_DB_PORT and SILO_DB_SSLMODE are NOT supported as direct # env var overrides. Set these in config.yaml instead, or use ${VAR} # syntax in the YAML file. See docs/CONFIGURATION.md for details. - - # MinIO storage (minio.example.internal) - # Supported as direct env var overrides: - SILO_MINIO_ENDPOINT: minio.example.internal:9000 - SILO_MINIO_ACCESS_KEY: ${SILO_MINIO_ACCESS_KEY:?MinIO access key required} - SILO_MINIO_SECRET_KEY: ${SILO_MINIO_SECRET_KEY:?MinIO secret key required} - # Note: SILO_MINIO_BUCKET and SILO_MINIO_USE_SSL are NOT supported as - # direct env var overrides. Set these in config.yaml instead. ports: - "8080:8080" volumes: diff --git a/deployments/docker-compose.yaml b/deployments/docker-compose.yaml index c13425b..2105eed 100644 --- a/deployments/docker-compose.yaml +++ b/deployments/docker-compose.yaml @@ -19,26 +19,6 @@ services: networks: - silo-network - minio: - image: minio/minio:RELEASE.2023-05-04T21-44-30Z - container_name: silo-minio - command: server /data --console-address ":9001" - environment: - MINIO_ROOT_USER: ${MINIO_ACCESS_KEY:-silominio} - MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY:-silominiosecret} - volumes: - - minio_data:/data - ports: - - "9000:9000" - - "9001:9001" - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 10s - timeout: 5s - retries: 5 - networks: - - silo-network - silo: build: context: .. @@ -47,19 +27,12 @@ services: depends_on: postgres: condition: service_healthy - minio: - condition: service_healthy environment: SILO_DB_HOST: postgres SILO_DB_PORT: 5432 SILO_DB_NAME: silo SILO_DB_USER: silo SILO_DB_PASSWORD: ${POSTGRES_PASSWORD:-silodev} - SILO_MINIO_ENDPOINT: minio:9000 - SILO_MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-silominio} - SILO_MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-silominiosecret} - SILO_MINIO_BUCKET: silo-files - SILO_MINIO_USE_SSL: "false" SILO_SESSION_SECRET: ${SILO_SESSION_SECRET:-change-me-in-production} SILO_OIDC_CLIENT_SECRET: ${SILO_OIDC_CLIENT_SECRET:-} SILO_LDAP_BIND_PASSWORD: ${SILO_LDAP_BIND_PASSWORD:-} @@ -68,6 +41,7 @@ services: ports: - "8080:8080" volumes: + - silo_data:/var/lib/silo/data - ../schemas:/etc/silo/schemas:ro - ./config.dev.yaml:/etc/silo/config.yaml:ro healthcheck: @@ -80,7 +54,7 @@ services: volumes: postgres_data: - minio_data: + silo_data: networks: silo-network: diff --git a/deployments/systemd/silod.service b/deployments/systemd/silod.service index c65da90..e2bbab0 100644 --- a/deployments/systemd/silod.service +++ b/deployments/systemd/silod.service @@ -27,6 +27,7 @@ NoNewPrivileges=yes ProtectSystem=strict ProtectHome=yes PrivateTmp=yes +ReadWritePaths=/opt/silo/data ReadOnlyPaths=/etc/silo /opt/silo # Resource limits diff --git a/docs/MODULES.md b/docs/MODULES.md index d7480b5..589e0c4 100644 --- a/docs/MODULES.md +++ b/docs/MODULES.md @@ -23,7 +23,7 @@ These cannot be disabled. They define what Silo *is*. |-----------|------|-------------| | `core` | Core PDM | Items, revisions, files, BOM, search, import/export, part number generation | | `schemas` | Schemas | Part numbering schema parsing, segment management, form descriptors | -| `storage` | Storage | MinIO/S3 file storage, presigned uploads, versioning | +| `storage` | Storage | Filesystem storage | ### 2.2 Optional Modules @@ -470,12 +470,10 @@ Returns full config grouped by module with secrets redacted: "default": "kindred-rd" }, "storage": { - "endpoint": "minio:9000", - "bucket": "silo-files", - "access_key": "****", - "secret_key": "****", - "use_ssl": false, - "region": "us-east-1", + "backend": "filesystem", + "filesystem": { + "root_dir": "/var/lib/silo/data" + }, "status": "connected" }, "database": { @@ -566,7 +564,7 @@ Available for modules with external connections: | Module | Test Action | |--------|------------| -| `storage` | Ping MinIO, verify bucket exists | +| `storage` | Verify filesystem storage directory is accessible | | `auth` (ldap) | Attempt LDAP bind with configured credentials | | `auth` (oidc) | Fetch OIDC discovery document from issuer URL | | `odoo` | Attempt XML-RPC connection to Odoo | @@ -602,11 +600,9 @@ database: sslmode: disable storage: - endpoint: minio:9000 - bucket: silo-files - access_key: silominio - secret_key: silominiosecret - use_ssl: false + backend: filesystem + filesystem: + root_dir: /var/lib/silo/data schemas: directory: /etc/silo/schemas diff --git a/frontend-spec.md b/frontend-spec.md index 6eb6ab3..2214508 100644 --- a/frontend-spec.md +++ b/frontend-spec.md @@ -337,7 +337,7 @@ Supporting files: | File | Purpose | |------|---------| | `web/src/components/items/CategoryPicker.tsx` | Multi-stage domain/subcategory selector | -| `web/src/components/items/FileDropZone.tsx` | Drag-and-drop file upload with MinIO presigned URLs | +| `web/src/components/items/FileDropZone.tsx` | Drag-and-drop file upload | | `web/src/components/items/TagInput.tsx` | Multi-select tag input for projects | | `web/src/hooks/useFormDescriptor.ts` | Fetches and caches form descriptor from `/api/schemas/{name}/form` | | `web/src/hooks/useFileUpload.ts` | Manages presigned URL upload flow | @@ -421,7 +421,7 @@ Below the picker, the selected category is shown as a breadcrumb: `Fasteners › ### FileDropZone -Handles drag-and-drop and click-to-browse file uploads with MinIO presigned URL flow. +Handles drag-and-drop and click-to-browse file uploads. **Props**: @@ -435,7 +435,7 @@ interface FileDropZoneProps { interface PendingAttachment { file: File; - objectKey: string; // MinIO key after upload + objectKey: string; // storage key after upload uploadProgress: number; // 0-100 uploadStatus: 'pending' | 'uploading' | 'complete' | 'error'; error?: string; @@ -462,7 +462,7 @@ Clicking the zone opens a hidden ``. 1. On file selection/drop, immediately request a presigned upload URL: `POST /api/uploads/presign` with `{ filename, content_type, size }`. 2. Backend returns `{ object_key, upload_url, expires_at }`. -3. `PUT` the file directly to the presigned MinIO URL using `XMLHttpRequest` (for progress tracking). +3. `PUT` the file directly to the presigned URL using `XMLHttpRequest` (for progress tracking). 4. On completion, update `PendingAttachment.uploadStatus` to `'complete'` and store the `object_key`. 5. The `object_key` is later sent to the item creation endpoint to associate the file. @@ -589,10 +589,10 @@ Items 1-5 below are implemented. Item 4 (hierarchical categories) is resolved by ``` POST /api/uploads/presign Request: { "filename": "bracket.FCStd", "content_type": "application/octet-stream", "size": 2400000 } -Response: { "object_key": "uploads/tmp/{uuid}/{filename}", "upload_url": "https://minio.../...", "expires_at": "2026-02-06T..." } +Response: { "object_key": "uploads/tmp/{uuid}/{filename}", "upload_url": "https://...", "expires_at": "2026-02-06T..." } ``` -The Go handler generates a presigned PUT URL via the MinIO SDK. Objects are uploaded to a temporary prefix. On item creation, they're moved/linked to the item's permanent prefix. +The Go handler generates a presigned PUT URL for direct upload. Objects are uploaded to a temporary prefix. On item creation, they're moved/linked to the item's permanent prefix. ### 2. File Association -- IMPLEMENTED @@ -612,7 +612,7 @@ Request: { "object_key": "uploads/tmp/{uuid}/thumb.png" } Response: 204 ``` -Stores the thumbnail at `items/{item_id}/thumbnail.png` in MinIO. Updates `item.thumbnail_key` column. +Stores the thumbnail at `items/{item_id}/thumbnail.png` in storage. Updates `item.thumbnail_key` column. ### 4. Hierarchical Categories -- IMPLEMENTED (via Form Descriptor) diff --git a/go.mod b/go.mod index 6c0b886..b7570bf 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,6 @@ require ( github.com/go-ldap/ldap/v3 v3.4.12 github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.5.4 - github.com/minio/minio-go/v7 v7.0.66 github.com/rs/zerolog v1.32.0 github.com/sahilm/fuzzy v0.1.1 golang.org/x/crypto v0.47.0 @@ -21,28 +20,16 @@ require ( require ( github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect - github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect github.com/go-jose/go-jose/v4 v4.1.3 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.4 // indirect - github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect - github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/sha256-simd v1.0.1 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect - github.com/rs/xid v1.5.0 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect - golang.org/x/net v0.48.0 // indirect golang.org/x/sync v0.19.0 // indirect golang.org/x/sys v0.40.0 // indirect golang.org/x/text v0.33.0 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index da5ca9b..72116ee 100644 --- a/go.sum +++ b/go.sum @@ -13,8 +13,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 h1:BP4M0CvQ4S3TGls2FvczZtj5Re/2ZzkV9VwqPHH/3Bo= github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= @@ -26,7 +24,6 @@ github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9 github.com/go-ldap/ldap/v3 v3.4.12 h1:1b81mv7MagXZ7+1r7cLTWmyuTqVqdwbtJSjC0DAp9s4= github.com/go-ldap/ldap/v3 v3.4.12/go.mod h1:+SPAGcTtOfmGsCb3h1RFiq4xpp4N636G75OEace8lNo= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= @@ -51,13 +48,6 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= -github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= -github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= @@ -73,31 +63,17 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= -github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw= -github.com/minio/minio-go/v7 v7.0.66/go.mod h1:DHAgmyQEGdW3Cif0UooKOyrT3Vxs82zNdV6tkKhRtbs= -github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= -github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0= github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/sahilm/fuzzy v0.1.1 h1:ceu5RHF8DGgoi+/dR5PsECjCDH1BE3Fnmpo7aVXOdRA= github.com/sahilm/fuzzy v0.1.1/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -133,7 +109,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -166,8 +141,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/api/file_handlers.go b/internal/api/file_handlers.go index 9f7c07a..9606b5f 100644 --- a/internal/api/file_handlers.go +++ b/internal/api/file_handlers.go @@ -21,7 +21,7 @@ type presignUploadRequest struct { Size int64 `json:"size"` } -// HandlePresignUpload generates a presigned PUT URL for direct browser upload to MinIO. +// HandlePresignUpload generates a presigned PUT URL for direct browser upload. func (s *Server) HandlePresignUpload(w http.ResponseWriter, r *http.Request) { if s.storage == nil { writeError(w, http.StatusServiceUnavailable, "storage_unavailable", "File storage not configured") @@ -317,12 +317,9 @@ func (s *Server) HandleSetItemThumbnail(w http.ResponseWriter, r *http.Request) w.WriteHeader(http.StatusNoContent) } -// storageBackend returns the configured storage backend name, defaulting to "minio". +// storageBackend returns the configured storage backend name. func (s *Server) storageBackend() string { - if s.cfg != nil && s.cfg.Storage.Backend != "" { - return s.cfg.Storage.Backend - } - return "minio" + return "filesystem" } // HandleUploadItemFile accepts a multipart file upload and stores it as an item attachment. diff --git a/internal/api/servermode.go b/internal/api/servermode.go index a95b276..e3bf751 100644 --- a/internal/api/servermode.go +++ b/internal/api/servermode.go @@ -86,7 +86,7 @@ func (ss *ServerState) ToggleReadOnly() { ss.SetReadOnly(!current) } -// StartStorageHealthCheck launches a periodic check of MinIO reachability. +// StartStorageHealthCheck launches a periodic check of storage reachability. // Updates storageOK and broadcasts server.state on transitions. func (ss *ServerState) StartStorageHealthCheck() { if ss.storage == nil { diff --git a/internal/api/settings_handlers.go b/internal/api/settings_handlers.go index f695f41..b8d7f0b 100644 --- a/internal/api/settings_handlers.go +++ b/internal/api/settings_handlers.go @@ -224,10 +224,8 @@ func (s *Server) buildSchemasSettings() map[string]any { func (s *Server) buildStorageSettings(ctx context.Context) map[string]any { result := map[string]any{ "enabled": true, - "endpoint": s.cfg.Storage.Endpoint, - "bucket": s.cfg.Storage.Bucket, - "use_ssl": s.cfg.Storage.UseSSL, - "region": s.cfg.Storage.Region, + "backend": "filesystem", + "root_dir": s.cfg.Storage.Filesystem.RootDir, } if s.storage != nil { if err := s.storage.Ping(ctx); err != nil { diff --git a/internal/api/settings_handlers_test.go b/internal/api/settings_handlers_test.go index 716afca..d2c8a64 100644 --- a/internal/api/settings_handlers_test.go +++ b/internal/api/settings_handlers_test.go @@ -31,8 +31,8 @@ func newSettingsTestServer(t *testing.T) *Server { MaxConnections: 10, }, Storage: config.StorageConfig{ - Endpoint: "minio:9000", Bucket: "silo", Region: "us-east-1", - AccessKey: "minioadmin", SecretKey: "miniosecret", + Backend: "filesystem", + Filesystem: config.FilesystemConfig{RootDir: "/tmp/silo-test"}, }, Schemas: config.SchemasConfig{Directory: "/etc/silo/schemas", Default: "kindred-rd"}, Auth: config.AuthConfig{ diff --git a/internal/config/config.go b/internal/config/config.go index f5d1390..59ee464 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -110,15 +110,9 @@ type DatabaseConfig struct { MaxConnections int `yaml:"max_connections"` } -// StorageConfig holds object storage settings. +// StorageConfig holds file storage settings. type StorageConfig struct { - Backend string `yaml:"backend"` // "minio" (default) or "filesystem" - Endpoint string `yaml:"endpoint"` - AccessKey string `yaml:"access_key"` - SecretKey string `yaml:"secret_key"` - Bucket string `yaml:"bucket"` - UseSSL bool `yaml:"use_ssl"` - Region string `yaml:"region"` + Backend string `yaml:"backend"` // "filesystem" Filesystem FilesystemConfig `yaml:"filesystem"` } @@ -189,9 +183,6 @@ func Load(path string) (*Config, error) { if cfg.Database.MaxConnections == 0 { cfg.Database.MaxConnections = 10 } - if cfg.Storage.Region == "" { - cfg.Storage.Region = "us-east-1" - } if cfg.Schemas.Directory == "" { cfg.Schemas.Directory = "/etc/silo/schemas" } @@ -227,14 +218,8 @@ func Load(path string) (*Config, error) { if v := os.Getenv("SILO_DB_PASSWORD"); v != "" { cfg.Database.Password = v } - if v := os.Getenv("SILO_MINIO_ENDPOINT"); v != "" { - cfg.Storage.Endpoint = v - } - if v := os.Getenv("SILO_MINIO_ACCESS_KEY"); v != "" { - cfg.Storage.AccessKey = v - } - if v := os.Getenv("SILO_MINIO_SECRET_KEY"); v != "" { - cfg.Storage.SecretKey = v + if v := os.Getenv("SILO_STORAGE_ROOT_DIR"); v != "" { + cfg.Storage.Filesystem.RootDir = v } // Auth defaults diff --git a/internal/db/item_files.go b/internal/db/item_files.go index 6c6bcf4..109ead4 100644 --- a/internal/db/item_files.go +++ b/internal/db/item_files.go @@ -14,7 +14,7 @@ type ItemFile struct { ContentType string Size int64 ObjectKey string - StorageBackend string // "minio" or "filesystem" + StorageBackend string CreatedAt time.Time } @@ -31,7 +31,7 @@ func NewItemFileRepository(db *DB) *ItemFileRepository { // Create inserts a new item file record. func (r *ItemFileRepository) Create(ctx context.Context, f *ItemFile) error { if f.StorageBackend == "" { - f.StorageBackend = "minio" + f.StorageBackend = "filesystem" } err := r.db.pool.QueryRow(ctx, `INSERT INTO item_files (item_id, filename, content_type, size, object_key, storage_backend) @@ -49,7 +49,7 @@ func (r *ItemFileRepository) Create(ctx context.Context, f *ItemFile) error { func (r *ItemFileRepository) ListByItem(ctx context.Context, itemID string) ([]*ItemFile, error) { rows, err := r.db.pool.Query(ctx, `SELECT id, item_id, filename, content_type, size, object_key, - COALESCE(storage_backend, 'minio'), created_at + COALESCE(storage_backend, 'filesystem'), created_at FROM item_files WHERE item_id = $1 ORDER BY created_at`, itemID, ) @@ -74,7 +74,7 @@ func (r *ItemFileRepository) Get(ctx context.Context, id string) (*ItemFile, err f := &ItemFile{} err := r.db.pool.QueryRow(ctx, `SELECT id, item_id, filename, content_type, size, object_key, - COALESCE(storage_backend, 'minio'), created_at + COALESCE(storage_backend, 'filesystem'), created_at FROM item_files WHERE id = $1`, id, ).Scan(&f.ID, &f.ItemID, &f.Filename, &f.ContentType, &f.Size, &f.ObjectKey, &f.StorageBackend, &f.CreatedAt) diff --git a/internal/db/items.go b/internal/db/items.go index f5518b7..1750b89 100644 --- a/internal/db/items.go +++ b/internal/db/items.go @@ -26,26 +26,26 @@ type Item struct { UpdatedBy *string SourcingType string // "manufactured" or "purchased" LongDescription *string // extended description - ThumbnailKey *string // MinIO key for item thumbnail + ThumbnailKey *string // storage key for item thumbnail } // Revision represents a revision record. type Revision struct { - ID string - ItemID string - RevisionNumber int - Properties map[string]any + ID string + ItemID string + RevisionNumber int + Properties map[string]any FileKey *string FileVersion *string FileChecksum *string FileSize *int64 - FileStorageBackend string // "minio" or "filesystem" + FileStorageBackend string ThumbnailKey *string - CreatedAt time.Time - CreatedBy *string - Comment *string - Status string // draft, review, released, obsolete - Labels []string // arbitrary tags + CreatedAt time.Time + CreatedBy *string + Comment *string + Status string // draft, review, released, obsolete + Labels []string // arbitrary tags } // RevisionStatus constants @@ -308,7 +308,7 @@ func (r *ItemRepository) CreateRevision(ctx context.Context, rev *Revision) erro } if rev.FileStorageBackend == "" { - rev.FileStorageBackend = "minio" + rev.FileStorageBackend = "filesystem" } err = r.db.pool.QueryRow(ctx, ` @@ -347,7 +347,7 @@ func (r *ItemRepository) GetRevisions(ctx context.Context, itemID string) ([]*Re if hasStatusColumn { rows, err = r.db.pool.Query(ctx, ` SELECT id, item_id, revision_number, properties, file_key, file_version, - file_checksum, file_size, COALESCE(file_storage_backend, 'minio'), + file_checksum, file_size, COALESCE(file_storage_backend, 'filesystem'), thumbnail_key, created_at, created_by, comment, COALESCE(status, 'draft') as status, COALESCE(labels, ARRAY[]::TEXT[]) as labels FROM revisions @@ -386,7 +386,7 @@ func (r *ItemRepository) GetRevisions(ctx context.Context, itemID string) ([]*Re ) rev.Status = "draft" rev.Labels = []string{} - rev.FileStorageBackend = "minio" + rev.FileStorageBackend = "filesystem" } if err != nil { return nil, fmt.Errorf("scanning revision: %w", err) @@ -420,7 +420,7 @@ func (r *ItemRepository) GetRevision(ctx context.Context, itemID string, revisio if hasStatusColumn { err = r.db.pool.QueryRow(ctx, ` SELECT id, item_id, revision_number, properties, file_key, file_version, - file_checksum, file_size, COALESCE(file_storage_backend, 'minio'), + file_checksum, file_size, COALESCE(file_storage_backend, 'filesystem'), thumbnail_key, created_at, created_by, comment, COALESCE(status, 'draft') as status, COALESCE(labels, ARRAY[]::TEXT[]) as labels FROM revisions @@ -443,7 +443,7 @@ func (r *ItemRepository) GetRevision(ctx context.Context, itemID string, revisio ) rev.Status = "draft" rev.Labels = []string{} - rev.FileStorageBackend = "minio" + rev.FileStorageBackend = "filesystem" } if err == pgx.ErrNoRows { diff --git a/internal/modules/modules.go b/internal/modules/modules.go index c0fb3e0..14f2f9a 100644 --- a/internal/modules/modules.go +++ b/internal/modules/modules.go @@ -50,7 +50,7 @@ type Registry struct { var builtinModules = []ModuleInfo{ {ID: Core, Name: "Core PDM", Description: "Items, revisions, files, BOM, search, import/export", Required: true, Version: "0.2"}, {ID: Schemas, Name: "Schemas", Description: "Part numbering schema parsing and segment management", Required: true}, - {ID: Storage, Name: "Storage", Description: "MinIO/S3 file storage, presigned uploads", Required: true}, + {ID: Storage, Name: "Storage", Description: "Filesystem storage", Required: true}, {ID: Auth, Name: "Authentication", Description: "Local, LDAP, OIDC authentication and RBAC", DefaultEnabled: true}, {ID: Projects, Name: "Projects", Description: "Project management and item tagging", DefaultEnabled: true}, {ID: Audit, Name: "Audit", Description: "Audit logging, completeness scoring", DefaultEnabled: true}, diff --git a/internal/storage/interface.go b/internal/storage/interface.go index 6ff1338..e9d8079 100644 --- a/internal/storage/interface.go +++ b/internal/storage/interface.go @@ -3,6 +3,7 @@ package storage import ( "context" + "fmt" "io" "net/url" "time" @@ -19,3 +20,21 @@ type FileStore interface { PresignPut(ctx context.Context, key string, expiry time.Duration) (*url.URL, error) Ping(ctx context.Context) error } + +// PutResult contains the result of a put operation. +type PutResult struct { + Key string + VersionID string + Size int64 + Checksum string +} + +// FileKey generates a storage key for an item file. +func FileKey(partNumber string, revision int) string { + return fmt.Sprintf("items/%s/rev%d.FCStd", partNumber, revision) +} + +// ThumbnailKey generates a storage key for a thumbnail. +func ThumbnailKey(partNumber string, revision int) string { + return fmt.Sprintf("thumbnails/%s/rev%d.png", partNumber, revision) +} diff --git a/internal/storage/storage.go b/internal/storage/storage.go deleted file mode 100644 index 8e3ebaf..0000000 --- a/internal/storage/storage.go +++ /dev/null @@ -1,174 +0,0 @@ -package storage - -import ( - "context" - "fmt" - "io" - "net/url" - "time" - - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" -) - -// Config holds MinIO connection settings. -type Config struct { - Endpoint string - AccessKey string - SecretKey string - Bucket string - UseSSL bool - Region string -} - -// Compile-time check: *Storage implements FileStore. -var _ FileStore = (*Storage)(nil) - -// Storage wraps MinIO client operations. -type Storage struct { - client *minio.Client - bucket string -} - -// Connect creates a new MinIO storage client. -func Connect(ctx context.Context, cfg Config) (*Storage, error) { - client, err := minio.New(cfg.Endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""), - Secure: cfg.UseSSL, - Region: cfg.Region, - }) - if err != nil { - return nil, fmt.Errorf("creating minio client: %w", err) - } - - // Ensure bucket exists with versioning - exists, err := client.BucketExists(ctx, cfg.Bucket) - if err != nil { - return nil, fmt.Errorf("checking bucket: %w", err) - } - if !exists { - if err := client.MakeBucket(ctx, cfg.Bucket, minio.MakeBucketOptions{ - Region: cfg.Region, - }); err != nil { - return nil, fmt.Errorf("creating bucket: %w", err) - } - // Enable versioning - if err := client.EnableVersioning(ctx, cfg.Bucket); err != nil { - return nil, fmt.Errorf("enabling versioning: %w", err) - } - } - - return &Storage{client: client, bucket: cfg.Bucket}, nil -} - -// PutResult contains the result of a put operation. -type PutResult struct { - Key string - VersionID string - Size int64 - Checksum string -} - -// Put uploads a file to storage. -func (s *Storage) Put(ctx context.Context, key string, reader io.Reader, size int64, contentType string) (*PutResult, error) { - info, err := s.client.PutObject(ctx, s.bucket, key, reader, size, minio.PutObjectOptions{ - ContentType: contentType, - }) - if err != nil { - return nil, fmt.Errorf("uploading object: %w", err) - } - - return &PutResult{ - Key: key, - VersionID: info.VersionID, - Size: info.Size, - Checksum: info.ChecksumSHA256, - }, nil -} - -// Get downloads a file from storage. -func (s *Storage) Get(ctx context.Context, key string) (io.ReadCloser, error) { - obj, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{}) - if err != nil { - return nil, fmt.Errorf("getting object: %w", err) - } - return obj, nil -} - -// GetVersion downloads a specific version of a file. -func (s *Storage) GetVersion(ctx context.Context, key, versionID string) (io.ReadCloser, error) { - obj, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{ - VersionID: versionID, - }) - if err != nil { - return nil, fmt.Errorf("getting object version: %w", err) - } - return obj, nil -} - -// Delete removes a file from storage. -func (s *Storage) Delete(ctx context.Context, key string) error { - if err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}); err != nil { - return fmt.Errorf("removing object: %w", err) - } - return nil -} - -// Exists checks if an object exists in storage. -func (s *Storage) Exists(ctx context.Context, key string) (bool, error) { - _, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{}) - if err != nil { - resp := minio.ToErrorResponse(err) - if resp.Code == "NoSuchKey" { - return false, nil - } - return false, fmt.Errorf("checking object existence: %w", err) - } - return true, nil -} - -// Ping checks if the storage backend is reachable by verifying the bucket exists. -func (s *Storage) Ping(ctx context.Context) error { - _, err := s.client.BucketExists(ctx, s.bucket) - return err -} - -// Bucket returns the bucket name. -func (s *Storage) Bucket() string { - return s.bucket -} - -// PresignPut generates a presigned PUT URL for direct browser upload. -func (s *Storage) PresignPut(ctx context.Context, key string, expiry time.Duration) (*url.URL, error) { - u, err := s.client.PresignedPutObject(ctx, s.bucket, key, expiry) - if err != nil { - return nil, fmt.Errorf("generating presigned put URL: %w", err) - } - return u, nil -} - -// Copy copies an object within the same bucket from srcKey to dstKey. -func (s *Storage) Copy(ctx context.Context, srcKey, dstKey string) error { - src := minio.CopySrcOptions{ - Bucket: s.bucket, - Object: srcKey, - } - dst := minio.CopyDestOptions{ - Bucket: s.bucket, - Object: dstKey, - } - if _, err := s.client.CopyObject(ctx, dst, src); err != nil { - return fmt.Errorf("copying object: %w", err) - } - return nil -} - -// FileKey generates a storage key for an item file. -func FileKey(partNumber string, revision int) string { - return fmt.Sprintf("items/%s/rev%d.FCStd", partNumber, revision) -} - -// ThumbnailKey generates a storage key for a thumbnail. -func ThumbnailKey(partNumber string, revision int) string { - return fmt.Sprintf("thumbnails/%s/rev%d.png", partNumber, revision) -} diff --git a/migrations/020_storage_backend_filesystem_default.sql b/migrations/020_storage_backend_filesystem_default.sql new file mode 100644 index 0000000..ae0bbb9 --- /dev/null +++ b/migrations/020_storage_backend_filesystem_default.sql @@ -0,0 +1,3 @@ +-- Change default storage backend from 'minio' to 'filesystem'. +ALTER TABLE item_files ALTER COLUMN storage_backend SET DEFAULT 'filesystem'; +ALTER TABLE revisions ALTER COLUMN file_storage_backend SET DEFAULT 'filesystem'; diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 8df1bc8..a93d006 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -8,7 +8,6 @@ # - SSH access to the target host # - /etc/silo/silod.env must exist on target with credentials filled in # - PostgreSQL reachable from target (set SILO_DB_HOST to override) -# - MinIO reachable from target (set SILO_MINIO_HOST to override) # # Environment variables: # SILO_DEPLOY_TARGET - target host (default: silo.example.internal) diff --git a/scripts/migrate-storage.sh b/scripts/migrate-storage.sh deleted file mode 100755 index e1484fa..0000000 --- a/scripts/migrate-storage.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash -# Migrate storage from MinIO to filesystem on a remote Silo host. -# -# Builds the migrate-storage binary locally, uploads it to the target host, -# then runs it over SSH using credentials from /etc/silo/silod.env. -# -# Usage: ./scripts/migrate-storage.sh [flags...] -# -# Examples: -# ./scripts/migrate-storage.sh silo.kindred.internal psql.kindred.internal minio.kindred.internal -dry-run -verbose -# ./scripts/migrate-storage.sh silo.kindred.internal psql.kindred.internal minio.kindred.internal - -set -euo pipefail - -if [ $# -lt 3 ]; then - echo "Usage: $0 [flags...]" - echo " flags are passed to migrate-storage (e.g. -dry-run -verbose)" - exit 1 -fi - -TARGET="$1" -DB_HOST="$2" -MINIO_HOST="$3" -shift 3 -EXTRA_FLAGS="$*" - -DEST_DIR="/opt/silo/data" -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_DIR="${SCRIPT_DIR}/.." - -echo "=== Migrate Storage: MinIO -> Filesystem ===" -echo " Target: ${TARGET}" -echo " DB host: ${DB_HOST}" -echo " MinIO: ${MINIO_HOST}" -echo " Dest: ${DEST_DIR}" -[ -n "$EXTRA_FLAGS" ] && echo " Flags: ${EXTRA_FLAGS}" -echo "" - -# --- Build locally --- -echo "[1/3] Building migrate-storage binary..." -cd "$PROJECT_DIR" -GOOS=linux GOARCH=amd64 go build -o migrate-storage ./cmd/migrate-storage -echo " Built: $(du -h migrate-storage | cut -f1)" - -# --- Upload --- -echo "[2/3] Uploading to ${TARGET}..." -scp migrate-storage "${TARGET}:/tmp/migrate-storage" -rm -f migrate-storage - -# --- Run remotely --- -echo "[3/3] Running migration on ${TARGET}..." -ssh "$TARGET" DB_HOST="$DB_HOST" MINIO_HOST="$MINIO_HOST" DEST_DIR="$DEST_DIR" EXTRA_FLAGS="$EXTRA_FLAGS" bash -s <<'REMOTE' -set -euo pipefail - -CONFIG_DIR="/etc/silo" - -# Source credentials -if [ ! -f "$CONFIG_DIR/silod.env" ]; then - echo "ERROR: $CONFIG_DIR/silod.env not found on $(hostname)" - exit 1 -fi -set -a -source "$CONFIG_DIR/silod.env" -set +a - -# Ensure destination directory exists -sudo mkdir -p "$DEST_DIR" -sudo chown silo:silo "$DEST_DIR" 2>/dev/null || true - -chmod +x /tmp/migrate-storage - -# Write temporary config with the provided hosts -cat > /tmp/silo-migrate.yaml < "${OUTPUT_DIR}/.env" << EOF # PostgreSQL POSTGRES_PASSWORD=${POSTGRES_PASSWORD} -# MinIO -MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY} -MINIO_SECRET_KEY=${MINIO_SECRET_KEY} - # OpenLDAP LDAP_ADMIN_PASSWORD=${LDAP_ADMIN_PASSWORD} LDAP_USERS=${LDAP_USERS} @@ -235,12 +225,9 @@ database: max_connections: 10 storage: - endpoint: "minio:9000" - access_key: "${SILO_MINIO_ACCESS_KEY}" - secret_key: "${SILO_MINIO_SECRET_KEY}" - bucket: "silo-files" - use_ssl: false - region: "us-east-1" + backend: "filesystem" + filesystem: + root_dir: "/var/lib/silo/data" schemas: directory: "/etc/silo/schemas" @@ -306,8 +293,6 @@ echo " deployments/config.docker.yaml - server configuration" echo "" echo -e "${BOLD}Credentials:${NC}" echo " PostgreSQL: silo / ${POSTGRES_PASSWORD}" -echo " MinIO: ${MINIO_ACCESS_KEY} / ${MINIO_SECRET_KEY}" -echo " MinIO Console: http://localhost:9001" echo " LDAP Admin: cn=admin,dc=silo,dc=local / ${LDAP_ADMIN_PASSWORD}" echo " LDAP User: ${LDAP_USERS} / ${LDAP_PASSWORDS}" echo " Silo Admin: ${SILO_ADMIN_USERNAME} / ${SILO_ADMIN_PASSWORD} (local fallback)" diff --git a/scripts/setup-host.sh b/scripts/setup-host.sh index 6a22baa..c20c054 100755 --- a/scripts/setup-host.sh +++ b/scripts/setup-host.sh @@ -30,7 +30,6 @@ INSTALL_DIR="/opt/silo" CONFIG_DIR="/etc/silo" GO_VERSION="1.24.0" DB_HOST="${SILO_DB_HOST:-psql.example.internal}" -MINIO_HOST="${SILO_MINIO_HOST:-minio.example.internal}" log_info() { echo -e "${BLUE}[INFO]${NC} $*"; } log_success() { echo -e "${GREEN}[OK]${NC} $*"; } @@ -165,11 +164,6 @@ if [[ ! -f "${ENV_FILE}" ]]; then # Database: silo, User: silo SILO_DB_PASSWORD= -# MinIO credentials (${MINIO_HOST}) -# User: silouser -SILO_MINIO_ACCESS_KEY=silouser -SILO_MINIO_SECRET_KEY= - # Authentication # Session secret (required when auth is enabled) SILO_SESSION_SECRET= @@ -225,10 +219,7 @@ echo "" echo "2. Verify database connectivity:" echo " psql -h ${DB_HOST} -U silo -d silo -c 'SELECT 1'" echo "" -echo "3. Verify MinIO connectivity:" -echo " curl -I http://${MINIO_HOST}:9000/minio/health/live" -echo "" -echo "4. Run the deployment:" +echo "3. Run the deployment:" echo " sudo ${INSTALL_DIR}/src/scripts/deploy.sh" echo "" echo "After deployment, manage the service with:"