feat(storage): FileStore interface abstraction + filesystem backend #134

Merged
forbes merged 2 commits from feat-storage-interface-filesystem into main 2026-02-17 17:55:10 +00:00
7 changed files with 91 additions and 28 deletions
Showing only changes of commit b531617e39 - Show all commits

View File

@@ -65,24 +65,39 @@ func main() {
logger.Info().Msg("connected to database")
// Connect to storage (optional - may be externally managed)
var store *storage.Storage
if cfg.Storage.Endpoint != "" {
store, err = storage.Connect(ctx, storage.Config{
Endpoint: cfg.Storage.Endpoint,
AccessKey: cfg.Storage.AccessKey,
SecretKey: cfg.Storage.SecretKey,
Bucket: cfg.Storage.Bucket,
UseSSL: cfg.Storage.UseSSL,
Region: cfg.Storage.Region,
})
if err != nil {
logger.Warn().Err(err).Msg("failed to connect to storage - file operations disabled")
store = nil
var store storage.FileStore
switch cfg.Storage.Backend {
case "minio", "":
if cfg.Storage.Endpoint != "" {
s, connErr := storage.Connect(ctx, storage.Config{
Endpoint: cfg.Storage.Endpoint,
AccessKey: cfg.Storage.AccessKey,
SecretKey: cfg.Storage.SecretKey,
Bucket: cfg.Storage.Bucket,
UseSSL: cfg.Storage.UseSSL,
Region: cfg.Storage.Region,
})
if connErr != nil {
logger.Warn().Err(connErr).Msg("failed to connect to storage - file operations disabled")
} else {
store = s
logger.Info().Msg("connected to storage")
}
} else {
logger.Info().Msg("connected to storage")
logger.Info().Msg("storage not configured - file operations disabled")
}
} else {
logger.Info().Msg("storage not configured - file operations disabled")
case "filesystem":
if cfg.Storage.Filesystem.RootDir == "" {
logger.Fatal().Msg("storage.filesystem.root_dir is required when backend is \"filesystem\"")
}
s, fsErr := storage.NewFilesystemStore(cfg.Storage.Filesystem.RootDir)
if fsErr != nil {
logger.Fatal().Err(fsErr).Msg("failed to initialize filesystem storage")
}
store = s
logger.Info().Str("root", cfg.Storage.Filesystem.RootDir).Msg("connected to filesystem storage")
default:
logger.Fatal().Str("backend", cfg.Storage.Backend).Msg("unknown storage backend")
}
// Load schemas

View File

@@ -17,12 +17,17 @@ database:
max_connections: 10
storage:
backend: "minio" # "minio" (default) or "filesystem"
# MinIO/S3 settings (used when backend: "minio")
endpoint: "localhost:9000" # Use "minio:9000" for Docker Compose
access_key: "" # Use SILO_MINIO_ACCESS_KEY env var
secret_key: "" # Use SILO_MINIO_SECRET_KEY env var
bucket: "silo-files"
use_ssl: true # Use false for Docker Compose (internal network)
region: "us-east-1"
# Filesystem settings (used when backend: "filesystem")
# filesystem:
# root_dir: "/var/lib/silo/objects"
schemas:
# Directory containing YAML schema files

View File

@@ -37,7 +37,7 @@ type Server struct {
schemas map[string]*schema.Schema
schemasDir string
partgen *partnum.Generator
storage *storage.Storage
storage storage.FileStore
auth *auth.Service
sessions *scs.SessionManager
oidc *auth.OIDCBackend
@@ -61,7 +61,7 @@ func NewServer(
database *db.DB,
schemas map[string]*schema.Schema,
schemasDir string,
store *storage.Storage,
store storage.FileStore,
authService *auth.Service,
sessionManager *scs.SessionManager,
oidcBackend *auth.OIDCBackend,

View File

@@ -26,13 +26,13 @@ type ServerState struct {
mu sync.RWMutex
readOnly bool
storageOK bool
storage *storage.Storage
storage storage.FileStore
broker *Broker
done chan struct{}
}
// NewServerState creates a new server state tracker.
func NewServerState(logger zerolog.Logger, store *storage.Storage, broker *Broker) *ServerState {
func NewServerState(logger zerolog.Logger, store storage.FileStore, broker *Broker) *ServerState {
return &ServerState{
logger: logger.With().Str("component", "server-state").Logger(),
storageOK: store != nil, // assume healthy if configured

View File

@@ -109,14 +109,21 @@ type DatabaseConfig struct {
MaxConnections int `yaml:"max_connections"`
}
// StorageConfig holds MinIO connection settings.
// StorageConfig holds object storage settings.
type StorageConfig struct {
Endpoint string `yaml:"endpoint"`
AccessKey string `yaml:"access_key"`
SecretKey string `yaml:"secret_key"`
Bucket string `yaml:"bucket"`
UseSSL bool `yaml:"use_ssl"`
Region string `yaml:"region"`
Backend string `yaml:"backend"` // "minio" (default) or "filesystem"
Endpoint string `yaml:"endpoint"`
AccessKey string `yaml:"access_key"`
SecretKey string `yaml:"secret_key"`
Bucket string `yaml:"bucket"`
UseSSL bool `yaml:"use_ssl"`
Region string `yaml:"region"`
Filesystem FilesystemConfig `yaml:"filesystem"`
}
// FilesystemConfig holds local filesystem storage settings.
type FilesystemConfig struct {
RootDir string `yaml:"root_dir"`
}
// SchemasConfig holds schema loading settings.

View File

@@ -0,0 +1,21 @@
// Package storage defines the FileStore interface and backend implementations.
package storage
import (
"context"
"io"
"net/url"
"time"
)
// FileStore is the interface for file storage backends.
type FileStore interface {
Put(ctx context.Context, key string, reader io.Reader, size int64, contentType string) (*PutResult, error)
Get(ctx context.Context, key string) (io.ReadCloser, error)
GetVersion(ctx context.Context, key string, versionID string) (io.ReadCloser, error)
Delete(ctx context.Context, key string) error
Exists(ctx context.Context, key string) (bool, error)
Copy(ctx context.Context, srcKey, dstKey string) error
PresignPut(ctx context.Context, key string, expiry time.Duration) (*url.URL, error)
Ping(ctx context.Context) error
}

View File

@@ -1,4 +1,3 @@
// Package storage provides MinIO file storage operations.
package storage
import (
@@ -22,6 +21,9 @@ type Config struct {
Region string
}
// Compile-time check: *Storage implements FileStore.
var _ FileStore = (*Storage)(nil)
// Storage wraps MinIO client operations.
type Storage struct {
client *minio.Client
@@ -112,6 +114,19 @@ func (s *Storage) Delete(ctx context.Context, key string) error {
return nil
}
// Exists checks if an object exists in storage.
func (s *Storage) Exists(ctx context.Context, key string) (bool, error) {
_, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{})
if err != nil {
resp := minio.ToErrorResponse(err)
if resp.Code == "NoSuchKey" {
return false, nil
}
return false, fmt.Errorf("checking object existence: %w", err)
}
return true, nil
}
// Ping checks if the storage backend is reachable by verifying the bucket exists.
func (s *Storage) Ping(ctx context.Context) error {
_, err := s.client.BucketExists(ctx, s.bucket)