kindred/silo
1
1
Fork 0
Code Issues 30 Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(storage): remove MinIO backend, filesystem-only storage

Browse Source 
Remove the MinIO/S3 storage backend entirely. The filesystem backend is
fully implemented, already used in production, and a migrate-storage tool
exists for any remaining MinIO deployments to migrate beforehand.

Changes:
- Delete MinIO client implementation (internal/storage/storage.go)
- Delete migrate-storage tool (cmd/migrate-storage, scripts/migrate-storage.sh)
- Remove MinIO service, volumes, and env vars from all Docker Compose files
- Simplify StorageConfig: remove Endpoint, AccessKey, SecretKey, Bucket,
  UseSSL, Region fields; add SILO_STORAGE_ROOT_DIR env override
- Change all SQL COALESCE defaults from 'minio' to 'filesystem'
- Add migration 020 to update column defaults to 'filesystem'
- Remove minio-go/v7 dependency (go mod tidy)
- Update all config examples, setup scripts, docs, and tests
This commit is contained in:
Forbes
2026-02-19 14:36:22 -06:00
parent 12ecffdabe
commit 88d1ab1f97
30 changed files with 104 additions and 849 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
internal/api/file_handlers.go
View File

@@ -21,7 +21,7 @@ type presignUploadRequest struct {
Size int64 `json:"size"`
}
// HandlePresignUpload generates a presigned PUT URL for direct browser upload to MinIO.
// HandlePresignUpload generates a presigned PUT URL for direct browser upload.
func (s *Server) HandlePresignUpload(w http.ResponseWriter, r *http.Request) {
if s.storage == nil {
writeError(w, http.StatusServiceUnavailable, "storage_unavailable", "File storage not configured")
@@ -317,12 +317,9 @@ func (s *Server) HandleSetItemThumbnail(w http.ResponseWriter, r *http.Request)
w.WriteHeader(http.StatusNoContent)
}
// storageBackend returns the configured storage backend name, defaulting to "minio".
// storageBackend returns the configured storage backend name.
func (s *Server) storageBackend() string {
if s.cfg != nil && s.cfg.Storage.Backend != "" {
return s.cfg.Storage.Backend
}
return "minio"
return "filesystem"
}
// HandleUploadItemFile accepts a multipart file upload and stores it as an item attachment.

2
internal/api/servermode.go
View File

@@ -86,7 +86,7 @@ func (ss *ServerState) ToggleReadOnly() {
ss.SetReadOnly(!current)
}
// StartStorageHealthCheck launches a periodic check of MinIO reachability.
// StartStorageHealthCheck launches a periodic check of storage reachability.
// Updates storageOK and broadcasts server.state on transitions.
func (ss *ServerState) StartStorageHealthCheck() {
if ss.storage == nil {

6
internal/api/settings_handlers.go
View File

@@ -224,10 +224,8 @@ func (s *Server) buildSchemasSettings() map[string]any {
func (s *Server) buildStorageSettings(ctx context.Context) map[string]any {
result := map[string]any{
"enabled": true,
"endpoint": s.cfg.Storage.Endpoint,
"bucket": s.cfg.Storage.Bucket,
"use_ssl": s.cfg.Storage.UseSSL,
"region": s.cfg.Storage.Region,
"backend": "filesystem",
"root_dir": s.cfg.Storage.Filesystem.RootDir,
}
if s.storage != nil {
if err := s.storage.Ping(ctx); err != nil {

4
internal/api/settings_handlers_test.go
View File

@@ -31,8 +31,8 @@ func newSettingsTestServer(t *testing.T) *Server {
MaxConnections: 10,
},
Storage: config.StorageConfig{
Endpoint: "minio:9000", Bucket: "silo", Region: "us-east-1",
AccessKey: "minioadmin", SecretKey: "miniosecret",
Backend: "filesystem",
Filesystem: config.FilesystemConfig{RootDir: "/tmp/silo-test"},
},
Schemas: config.SchemasConfig{Directory: "/etc/silo/schemas", Default: "kindred-rd"},
Auth: config.AuthConfig{

23
internal/config/config.go
View File

@@ -110,15 +110,9 @@ type DatabaseConfig struct {
MaxConnections int `yaml:"max_connections"`
}
// StorageConfig holds object storage settings.
// StorageConfig holds file storage settings.
type StorageConfig struct {
Backend string `yaml:"backend"` // "minio" (default) or "filesystem"
Endpoint string `yaml:"endpoint"`
AccessKey string `yaml:"access_key"`
SecretKey string `yaml:"secret_key"`
Bucket string `yaml:"bucket"`
UseSSL bool `yaml:"use_ssl"`
Region string `yaml:"region"`
Backend string `yaml:"backend"` // "filesystem"
Filesystem FilesystemConfig `yaml:"filesystem"`
}
@@ -189,9 +183,6 @@ func Load(path string) (*Config, error) {
if cfg.Database.MaxConnections == 0 {
cfg.Database.MaxConnections = 10
}
if cfg.Storage.Region == "" {
cfg.Storage.Region = "us-east-1"
}
if cfg.Schemas.Directory == "" {
cfg.Schemas.Directory = "/etc/silo/schemas"
}
@@ -227,14 +218,8 @@ func Load(path string) (*Config, error) {
if v := os.Getenv("SILO_DB_PASSWORD"); v != "" {
cfg.Database.Password = v
}
if v := os.Getenv("SILO_MINIO_ENDPOINT"); v != "" {
cfg.Storage.Endpoint = v
}
if v := os.Getenv("SILO_MINIO_ACCESS_KEY"); v != "" {
cfg.Storage.AccessKey = v
}
if v := os.Getenv("SILO_MINIO_SECRET_KEY"); v != "" {
cfg.Storage.SecretKey = v
if v := os.Getenv("SILO_STORAGE_ROOT_DIR"); v != "" {
cfg.Storage.Filesystem.RootDir = v
}
// Auth defaults

8
internal/db/item_files.go
View File

@@ -14,7 +14,7 @@ type ItemFile struct {
ContentType string
Size int64
ObjectKey string
StorageBackend string // "minio" or "filesystem"
StorageBackend string
CreatedAt time.Time
}
@@ -31,7 +31,7 @@ func NewItemFileRepository(db *DB) *ItemFileRepository {
// Create inserts a new item file record.
func (r *ItemFileRepository) Create(ctx context.Context, f *ItemFile) error {
if f.StorageBackend == "" {
f.StorageBackend = "minio"
f.StorageBackend = "filesystem"
}
err := r.db.pool.QueryRow(ctx,
`INSERT INTO item_files (item_id, filename, content_type, size, object_key, storage_backend)
@@ -49,7 +49,7 @@ func (r *ItemFileRepository) Create(ctx context.Context, f *ItemFile) error {
func (r *ItemFileRepository) ListByItem(ctx context.Context, itemID string) ([]*ItemFile, error) {
rows, err := r.db.pool.Query(ctx,
`SELECT id, item_id, filename, content_type, size, object_key,
COALESCE(storage_backend, 'minio'), created_at
COALESCE(storage_backend, 'filesystem'), created_at
FROM item_files WHERE item_id = $1 ORDER BY created_at`,
itemID,
)
@@ -74,7 +74,7 @@ func (r *ItemFileRepository) Get(ctx context.Context, id string) (*ItemFile, err
f := &ItemFile{}
err := r.db.pool.QueryRow(ctx,
`SELECT id, item_id, filename, content_type, size, object_key,
COALESCE(storage_backend, 'minio'), created_at
COALESCE(storage_backend, 'filesystem'), created_at
FROM item_files WHERE id = $1`,
id,
).Scan(&f.ID, &f.ItemID, &f.Filename, &f.ContentType, &f.Size, &f.ObjectKey, &f.StorageBackend, &f.CreatedAt)

32
internal/db/items.go
View File

@@ -26,26 +26,26 @@ type Item struct {
UpdatedBy *string
SourcingType string // "manufactured" or "purchased"
LongDescription *string // extended description
ThumbnailKey *string // MinIO key for item thumbnail
ThumbnailKey *string // storage key for item thumbnail
}
// Revision represents a revision record.
type Revision struct {
ID string
ItemID string
RevisionNumber int
Properties map[string]any
ID string
ItemID string
RevisionNumber int
Properties map[string]any
FileKey *string
FileVersion *string
FileChecksum *string
FileSize *int64
FileStorageBackend string // "minio" or "filesystem"
FileStorageBackend string
ThumbnailKey *string
CreatedAt time.Time
CreatedBy *string
Comment *string
Status string // draft, review, released, obsolete
Labels []string // arbitrary tags
CreatedAt time.Time
CreatedBy *string
Comment *string
Status string // draft, review, released, obsolete
Labels []string // arbitrary tags
}
// RevisionStatus constants
@@ -308,7 +308,7 @@ func (r *ItemRepository) CreateRevision(ctx context.Context, rev *Revision) erro
}
if rev.FileStorageBackend == "" {
rev.FileStorageBackend = "minio"
rev.FileStorageBackend = "filesystem"
}
err = r.db.pool.QueryRow(ctx, `
@@ -347,7 +347,7 @@ func (r *ItemRepository) GetRevisions(ctx context.Context, itemID string) ([]*Re
if hasStatusColumn {
rows, err = r.db.pool.Query(ctx, `
SELECT id, item_id, revision_number, properties, file_key, file_version,
file_checksum, file_size, COALESCE(file_storage_backend, 'minio'),
file_checksum, file_size, COALESCE(file_storage_backend, 'filesystem'),
thumbnail_key, created_at, created_by, comment,
COALESCE(status, 'draft') as status, COALESCE(labels, ARRAY[]::TEXT[]) as labels
FROM revisions
@@ -386,7 +386,7 @@ func (r *ItemRepository) GetRevisions(ctx context.Context, itemID string) ([]*Re
)
rev.Status = "draft"
rev.Labels = []string{}
rev.FileStorageBackend = "minio"
rev.FileStorageBackend = "filesystem"
}
if err != nil {
return nil, fmt.Errorf("scanning revision: %w", err)
@@ -420,7 +420,7 @@ func (r *ItemRepository) GetRevision(ctx context.Context, itemID string, revisio
if hasStatusColumn {
err = r.db.pool.QueryRow(ctx, `
SELECT id, item_id, revision_number, properties, file_key, file_version,
file_checksum, file_size, COALESCE(file_storage_backend, 'minio'),
file_checksum, file_size, COALESCE(file_storage_backend, 'filesystem'),
thumbnail_key, created_at, created_by, comment,
COALESCE(status, 'draft') as status, COALESCE(labels, ARRAY[]::TEXT[]) as labels
FROM revisions
@@ -443,7 +443,7 @@ func (r *ItemRepository) GetRevision(ctx context.Context, itemID string, revisio
)
rev.Status = "draft"
rev.Labels = []string{}
rev.FileStorageBackend = "minio"
rev.FileStorageBackend = "filesystem"
}
if err == pgx.ErrNoRows {

2
internal/modules/modules.go
View File

@@ -50,7 +50,7 @@ type Registry struct {
var builtinModules = []ModuleInfo{
{ID: Core, Name: "Core PDM", Description: "Items, revisions, files, BOM, search, import/export", Required: true, Version: "0.2"},
{ID: Schemas, Name: "Schemas", Description: "Part numbering schema parsing and segment management", Required: true},
{ID: Storage, Name: "Storage", Description: "MinIO/S3 file storage, presigned uploads", Required: true},
{ID: Storage, Name: "Storage", Description: "Filesystem storage", Required: true},
{ID: Auth, Name: "Authentication", Description: "Local, LDAP, OIDC authentication and RBAC", DefaultEnabled: true},
{ID: Projects, Name: "Projects", Description: "Project management and item tagging", DefaultEnabled: true},
{ID: Audit, Name: "Audit", Description: "Audit logging, completeness scoring", DefaultEnabled: true},

19
internal/storage/interface.go
View File

@@ -3,6 +3,7 @@ package storage
import (
"context"
"fmt"
"io"
"net/url"
"time"
@@ -19,3 +20,21 @@ type FileStore interface {
PresignPut(ctx context.Context, key string, expiry time.Duration) (*url.URL, error)
Ping(ctx context.Context) error
}
// PutResult contains the result of a put operation.
type PutResult struct {
Key string
VersionID string
Size int64
Checksum string
}
// FileKey generates a storage key for an item file.
func FileKey(partNumber string, revision int) string {
return fmt.Sprintf("items/%s/rev%d.FCStd", partNumber, revision)
}
// ThumbnailKey generates a storage key for a thumbnail.
func ThumbnailKey(partNumber string, revision int) string {
return fmt.Sprintf("thumbnails/%s/rev%d.png", partNumber, revision)
}

174
internal/storage/storage.go
View File

@@ -1,174 +0,0 @@
package storage
import (
"context"
"fmt"
"io"
"net/url"
"time"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
// Config holds MinIO connection settings.
type Config struct {
Endpoint string
AccessKey string
SecretKey string
Bucket string
UseSSL bool
Region string
}
// Compile-time check: *Storage implements FileStore.
var _ FileStore = (*Storage)(nil)
// Storage wraps MinIO client operations.
type Storage struct {
client *minio.Client
bucket string
}
// Connect creates a new MinIO storage client.
func Connect(ctx context.Context, cfg Config) (*Storage, error) {
client, err := minio.New(cfg.Endpoint, &minio.Options{
Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
Secure: cfg.UseSSL,
Region: cfg.Region,
})
if err != nil {
return nil, fmt.Errorf("creating minio client: %w", err)
}
// Ensure bucket exists with versioning
exists, err := client.BucketExists(ctx, cfg.Bucket)
if err != nil {
return nil, fmt.Errorf("checking bucket: %w", err)
}
if !exists {
if err := client.MakeBucket(ctx, cfg.Bucket, minio.MakeBucketOptions{
Region: cfg.Region,
}); err != nil {
return nil, fmt.Errorf("creating bucket: %w", err)
}
// Enable versioning
if err := client.EnableVersioning(ctx, cfg.Bucket); err != nil {
return nil, fmt.Errorf("enabling versioning: %w", err)
}
}
return &Storage{client: client, bucket: cfg.Bucket}, nil
}
// PutResult contains the result of a put operation.
type PutResult struct {
Key string
VersionID string
Size int64
Checksum string
}
// Put uploads a file to storage.
func (s *Storage) Put(ctx context.Context, key string, reader io.Reader, size int64, contentType string) (*PutResult, error) {
info, err := s.client.PutObject(ctx, s.bucket, key, reader, size, minio.PutObjectOptions{
ContentType: contentType,
})
if err != nil {
return nil, fmt.Errorf("uploading object: %w", err)
}
return &PutResult{
Key: key,
VersionID: info.VersionID,
Size: info.Size,
Checksum: info.ChecksumSHA256,
}, nil
}
// Get downloads a file from storage.
func (s *Storage) Get(ctx context.Context, key string) (io.ReadCloser, error) {
obj, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
if err != nil {
return nil, fmt.Errorf("getting object: %w", err)
}
return obj, nil
}
// GetVersion downloads a specific version of a file.
func (s *Storage) GetVersion(ctx context.Context, key, versionID string) (io.ReadCloser, error) {
obj, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{
VersionID: versionID,
})
if err != nil {
return nil, fmt.Errorf("getting object version: %w", err)
}
return obj, nil
}
// Delete removes a file from storage.
func (s *Storage) Delete(ctx context.Context, key string) error {
if err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}); err != nil {
return fmt.Errorf("removing object: %w", err)
}
return nil
}
// Exists checks if an object exists in storage.
func (s *Storage) Exists(ctx context.Context, key string) (bool, error) {
_, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{})
if err != nil {
resp := minio.ToErrorResponse(err)
if resp.Code == "NoSuchKey" {
return false, nil
}
return false, fmt.Errorf("checking object existence: %w", err)
}
return true, nil
}
// Ping checks if the storage backend is reachable by verifying the bucket exists.
func (s *Storage) Ping(ctx context.Context) error {
_, err := s.client.BucketExists(ctx, s.bucket)
return err
}
// Bucket returns the bucket name.
func (s *Storage) Bucket() string {
return s.bucket
}
// PresignPut generates a presigned PUT URL for direct browser upload.
func (s *Storage) PresignPut(ctx context.Context, key string, expiry time.Duration) (*url.URL, error) {
u, err := s.client.PresignedPutObject(ctx, s.bucket, key, expiry)
if err != nil {
return nil, fmt.Errorf("generating presigned put URL: %w", err)
}
return u, nil
}
// Copy copies an object within the same bucket from srcKey to dstKey.
func (s *Storage) Copy(ctx context.Context, srcKey, dstKey string) error {
src := minio.CopySrcOptions{
Bucket: s.bucket,
Object: srcKey,
}
dst := minio.CopyDestOptions{
Bucket: s.bucket,
Object: dstKey,
}
if _, err := s.client.CopyObject(ctx, dst, src); err != nil {
return fmt.Errorf("copying object: %w", err)
}
return nil
}
// FileKey generates a storage key for an item file.
func FileKey(partNumber string, revision int) string {
return fmt.Sprintf("items/%s/rev%d.FCStd", partNumber, revision)
}
// ThumbnailKey generates a storage key for a thumbnail.
func ThumbnailKey(partNumber string, revision int) string {
return fmt.Sprintf("thumbnails/%s/rev%d.png", partNumber, revision)
}