From 12ecffdabe14de5bcc227157ad5e9bb036c36a90 Mon Sep 17 00:00:00 2001 From: Forbes Date: Wed, 18 Feb 2026 19:38:20 -0600 Subject: [PATCH 1/2] feat(api): approvals + ECO workflow API with YAML-configurable workflows - Add internal/workflow/ package for YAML workflow definitions (Load, LoadAll, Validate) - Add internal/db/item_approvals.go repository (Create, AddSignature, GetWithSignatures, ListByItemWithSignatures, UpdateState, UpdateSignature) - Add internal/api/approval_handlers.go with 4 endpoints: - GET /{partNumber}/approvals (list approvals with signatures) - POST /{partNumber}/approvals (create ECO with workflow + signers) - POST /{partNumber}/approvals/{id}/sign (approve or reject) - GET /workflows (list available workflow definitions) - Rule-driven state transitions: any_reject and all_required_approve - Pack approvals into silo/approvals.json on .kc checkout - Add WorkflowsConfig to config, load workflows at startup - Migration 019: add workflow_name column to item_approvals - Example workflows: engineering-change.yaml, quick-review.yaml - 7 workflow tests, all passing Closes #145 --- cmd/silod/main.go | 16 +- internal/api/approval_handlers.go | 391 ++++++++++++++++++++++ internal/api/handlers.go | 7 + internal/api/pack_handlers.go | 47 +++ internal/api/routes.go | 6 + internal/config/config.go | 27 +- internal/db/item_approvals.go | 212 ++++++++++++ internal/kc/kc.go | 21 ++ internal/kc/pack.go | 5 + internal/workflow/workflow.go | 156 +++++++++ internal/workflow/workflow_test.go | 167 +++++++++ migrations/019_approval_workflow_name.sql | 2 + workflows/engineering-change.yaml | 25 ++ workflows/quick-review.yaml | 19 ++ 14 files changed, 1091 insertions(+), 10 deletions(-) create mode 100644 internal/api/approval_handlers.go create mode 100644 internal/db/item_approvals.go create mode 100644 internal/workflow/workflow.go create mode 100644 internal/workflow/workflow_test.go create mode 100644 migrations/019_approval_workflow_name.sql create mode 100644 workflows/engineering-change.yaml create mode 100644 workflows/quick-review.yaml diff --git a/cmd/silod/main.go b/cmd/silod/main.go index db79642..2054c56 100644 --- a/cmd/silod/main.go +++ b/cmd/silod/main.go @@ -23,6 +23,7 @@ import ( "github.com/kindredsystems/silo/internal/modules" "github.com/kindredsystems/silo/internal/schema" "github.com/kindredsystems/silo/internal/storage" + "github.com/kindredsystems/silo/internal/workflow" "github.com/rs/zerolog" ) @@ -235,6 +236,19 @@ func main() { } } + // Load approval workflow definitions (optional — directory may not exist yet) + var workflows map[string]*workflow.Workflow + if _, err := os.Stat(cfg.Workflows.Directory); err == nil { + workflows, err = workflow.LoadAll(cfg.Workflows.Directory) + if err != nil { + logger.Fatal().Err(err).Str("directory", cfg.Workflows.Directory).Msg("failed to load workflow definitions") + } + logger.Info().Int("count", len(workflows)).Msg("loaded workflow definitions") + } else { + workflows = make(map[string]*workflow.Workflow) + logger.Info().Str("directory", cfg.Workflows.Directory).Msg("workflows directory not found, skipping") + } + // Initialize module registry registry := modules.NewRegistry() if err := modules.LoadState(registry, cfg, database.Pool()); err != nil { @@ -258,7 +272,7 @@ func main() { // Create API server server := api.NewServer(logger, database, schemas, cfg.Schemas.Directory, store, authService, sessionManager, oidcBackend, &cfg.Auth, broker, serverState, - jobDefs, cfg.Jobs.Directory, registry, cfg) + jobDefs, cfg.Jobs.Directory, registry, cfg, workflows) router := api.NewRouter(server, logger) // Start background sweepers for job/runner timeouts (only when jobs module enabled) diff --git a/internal/api/approval_handlers.go b/internal/api/approval_handlers.go new file mode 100644 index 0000000..d716f6b --- /dev/null +++ b/internal/api/approval_handlers.go @@ -0,0 +1,391 @@ +package api + +import ( + "encoding/json" + "net/http" + + "github.com/go-chi/chi/v5" + "github.com/kindredsystems/silo/internal/auth" + "github.com/kindredsystems/silo/internal/db" + "github.com/kindredsystems/silo/internal/workflow" +) + +// ApprovalResponse is the JSON representation for approval endpoints. +type ApprovalResponse struct { + ID string `json:"id"` + WorkflowName string `json:"workflow"` + ECONumber *string `json:"eco_number"` + State string `json:"state"` + UpdatedAt string `json:"updated_at"` + UpdatedBy *string `json:"updated_by"` + Signatures []SignatureResponse `json:"signatures"` +} + +// SignatureResponse is the JSON representation for a signature. +type SignatureResponse struct { + Username string `json:"username"` + Role string `json:"role"` + Status string `json:"status"` + SignedAt *string `json:"signed_at"` + Comment *string `json:"comment"` +} + +// CreateApprovalRequest is the JSON body for POST /approvals. +type CreateApprovalRequest struct { + Workflow string `json:"workflow"` + ECONumber string `json:"eco_number"` + Signers []SignerRequest `json:"signers"` +} + +// SignerRequest defines a signer in the create request. +type SignerRequest struct { + Username string `json:"username"` + Role string `json:"role"` +} + +// SignApprovalRequest is the JSON body for POST /approvals/{id}/sign. +type SignApprovalRequest struct { + Status string `json:"status"` + Comment *string `json:"comment"` +} + +func approvalToResponse(a *db.ItemApproval) ApprovalResponse { + sigs := make([]SignatureResponse, len(a.Signatures)) + for i, s := range a.Signatures { + var signedAt *string + if s.SignedAt != nil { + t := s.SignedAt.UTC().Format("2006-01-02T15:04:05Z") + signedAt = &t + } + sigs[i] = SignatureResponse{ + Username: s.Username, + Role: s.Role, + Status: s.Status, + SignedAt: signedAt, + Comment: s.Comment, + } + } + return ApprovalResponse{ + ID: a.ID, + WorkflowName: a.WorkflowName, + ECONumber: a.ECONumber, + State: a.State, + UpdatedAt: a.UpdatedAt.UTC().Format("2006-01-02T15:04:05Z"), + UpdatedBy: a.UpdatedBy, + Signatures: sigs, + } +} + +// HandleGetApprovals returns all approvals with signatures for an item. +// GET /api/items/{partNumber}/approvals +func (s *Server) HandleGetApprovals(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + partNumber := chi.URLParam(r, "partNumber") + + item, err := s.items.GetByPartNumber(ctx, partNumber) + if err != nil { + s.logger.Error().Err(err).Msg("failed to get item") + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get item") + return + } + if item == nil { + writeError(w, http.StatusNotFound, "not_found", "Item not found") + return + } + + approvals, err := s.approvals.ListByItemWithSignatures(ctx, item.ID) + if err != nil { + s.logger.Error().Err(err).Msg("failed to list approvals") + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to list approvals") + return + } + + resp := make([]ApprovalResponse, len(approvals)) + for i, a := range approvals { + resp[i] = approvalToResponse(a) + } + + writeJSON(w, http.StatusOK, resp) +} + +// HandleCreateApproval creates an ECO with a workflow and signers. +// POST /api/items/{partNumber}/approvals +func (s *Server) HandleCreateApproval(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + partNumber := chi.URLParam(r, "partNumber") + + item, err := s.items.GetByPartNumber(ctx, partNumber) + if err != nil { + s.logger.Error().Err(err).Msg("failed to get item") + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get item") + return + } + if item == nil { + writeError(w, http.StatusNotFound, "not_found", "Item not found") + return + } + + var req CreateApprovalRequest + if err := readJSON(r, &req); err != nil { + writeError(w, http.StatusBadRequest, "invalid_body", "Invalid JSON body") + return + } + + if len(req.Signers) == 0 { + writeError(w, http.StatusBadRequest, "invalid_body", "At least one signer is required") + return + } + + // Validate workflow exists + wf, ok := s.workflows[req.Workflow] + if !ok { + writeError(w, http.StatusBadRequest, "invalid_workflow", "Workflow '"+req.Workflow+"' not found") + return + } + + // Validate each signer's role matches a gate in the workflow + for _, signer := range req.Signers { + if !wf.HasRole(signer.Role) { + writeError(w, http.StatusBadRequest, "invalid_role", + "Role '"+signer.Role+"' is not defined in workflow '"+req.Workflow+"'") + return + } + } + + // Validate all required gates have at least one signer + signerRoles := make(map[string]bool) + for _, signer := range req.Signers { + signerRoles[signer.Role] = true + } + for _, gate := range wf.RequiredGates() { + if !signerRoles[gate.Role] { + writeError(w, http.StatusBadRequest, "missing_required_signer", + "Required role '"+gate.Role+"' ("+gate.Label+") has no assigned signer") + return + } + } + + username := "" + if user := auth.UserFromContext(ctx); user != nil { + username = user.Username + } + + var ecoNumber *string + if req.ECONumber != "" { + ecoNumber = &req.ECONumber + } + + approval := &db.ItemApproval{ + ItemID: item.ID, + WorkflowName: req.Workflow, + ECONumber: ecoNumber, + State: "pending", + UpdatedBy: &username, + } + + if err := s.approvals.Create(ctx, approval); err != nil { + s.logger.Error().Err(err).Msg("failed to create approval") + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to create approval") + return + } + + // Add signature rows for each signer + for _, signer := range req.Signers { + sig := &db.ApprovalSignature{ + ApprovalID: approval.ID, + Username: signer.Username, + Role: signer.Role, + Status: "pending", + } + if err := s.approvals.AddSignature(ctx, sig); err != nil { + s.logger.Error().Err(err).Msg("failed to add signature") + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to add signer") + return + } + } + + // Re-fetch with signatures for response + approval, err = s.approvals.GetWithSignatures(ctx, approval.ID) + if err != nil { + s.logger.Error().Err(err).Msg("failed to get approval") + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get approval") + return + } + + resp := approvalToResponse(approval) + writeJSON(w, http.StatusCreated, resp) + s.broker.Publish("approval.created", mustMarshal(map[string]any{ + "part_number": partNumber, + "approval_id": approval.ID, + "workflow": approval.WorkflowName, + "eco_number": approval.ECONumber, + })) +} + +// HandleSignApproval records an approve or reject signature. +// POST /api/items/{partNumber}/approvals/{id}/sign +func (s *Server) HandleSignApproval(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + partNumber := chi.URLParam(r, "partNumber") + approvalID := chi.URLParam(r, "id") + + item, err := s.items.GetByPartNumber(ctx, partNumber) + if err != nil { + s.logger.Error().Err(err).Msg("failed to get item") + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get item") + return + } + if item == nil { + writeError(w, http.StatusNotFound, "not_found", "Item not found") + return + } + + approval, err := s.approvals.GetWithSignatures(ctx, approvalID) + if err != nil { + s.logger.Error().Err(err).Msg("failed to get approval") + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get approval") + return + } + if approval == nil || approval.ItemID != item.ID { + writeError(w, http.StatusNotFound, "not_found", "Approval not found") + return + } + + if approval.State != "pending" { + writeError(w, http.StatusUnprocessableEntity, "invalid_state", + "Approval is in state '"+approval.State+"', signatures can only be added when 'pending'") + return + } + + var req SignApprovalRequest + if err := readJSON(r, &req); err != nil { + writeError(w, http.StatusBadRequest, "invalid_body", "Invalid JSON body") + return + } + + if req.Status != "approved" && req.Status != "rejected" { + writeError(w, http.StatusBadRequest, "invalid_status", "Status must be 'approved' or 'rejected'") + return + } + + // Get the caller's username + username := "" + if user := auth.UserFromContext(ctx); user != nil { + username = user.Username + } + + // Check that the caller has a pending signature on this approval + sig, err := s.approvals.GetSignatureForUser(ctx, approvalID, username) + if err != nil { + s.logger.Error().Err(err).Msg("failed to get signature") + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to check signature") + return + } + if sig == nil { + writeError(w, http.StatusForbidden, "not_a_signer", "You are not a signer on this approval") + return + } + if sig.Status != "pending" { + writeError(w, http.StatusConflict, "already_signed", "You have already signed this approval") + return + } + + // Update the signature + if err := s.approvals.UpdateSignature(ctx, sig.ID, req.Status, req.Comment); err != nil { + s.logger.Error().Err(err).Msg("failed to update signature") + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to update signature") + return + } + + s.broker.Publish("approval.signed", mustMarshal(map[string]any{ + "part_number": partNumber, + "approval_id": approvalID, + "username": username, + "status": req.Status, + })) + + // Evaluate auto-advance based on workflow rules + wf := s.workflows[approval.WorkflowName] + if wf != nil { + // Re-fetch signatures after update + approval, err = s.approvals.GetWithSignatures(ctx, approvalID) + if err == nil && approval != nil { + newState := evaluateApprovalState(wf, approval) + if newState != "" && newState != approval.State { + if err := s.approvals.UpdateState(ctx, approvalID, newState, username); err != nil { + s.logger.Warn().Err(err).Msg("failed to auto-advance approval state") + } else { + approval.State = newState + s.broker.Publish("approval.completed", mustMarshal(map[string]any{ + "part_number": partNumber, + "approval_id": approvalID, + "state": newState, + })) + } + } + } + } + + // Return updated approval + if approval == nil { + approval, _ = s.approvals.GetWithSignatures(ctx, approvalID) + } + if approval != nil { + writeJSON(w, http.StatusOK, approvalToResponse(approval)) + } else { + w.WriteHeader(http.StatusOK) + } +} + +// HandleListWorkflows returns all loaded workflow definitions. +// GET /api/workflows +func (s *Server) HandleListWorkflows(w http.ResponseWriter, r *http.Request) { + resp := make([]map[string]any, 0, len(s.workflows)) + for _, wf := range s.workflows { + resp = append(resp, map[string]any{ + "name": wf.Name, + "version": wf.Version, + "description": wf.Description, + "gates": wf.Gates, + }) + } + writeJSON(w, http.StatusOK, resp) +} + +// evaluateApprovalState checks workflow rules against current signatures +// and returns the new state, or "" if no transition is needed. +func evaluateApprovalState(wf *workflow.Workflow, approval *db.ItemApproval) string { + // Check for any rejection + if wf.Rules.AnyReject != "" { + for _, sig := range approval.Signatures { + if sig.Status == "rejected" { + return wf.Rules.AnyReject + } + } + } + + // Check if all required roles have approved + if wf.Rules.AllRequiredApprove != "" { + requiredRoles := make(map[string]bool) + for _, gate := range wf.RequiredGates() { + requiredRoles[gate.Role] = true + } + + // For each required role, check that all signers with that role have approved + for _, sig := range approval.Signatures { + if requiredRoles[sig.Role] && sig.Status != "approved" { + return "" // at least one required signer hasn't approved yet + } + } + + // All required signers approved + return wf.Rules.AllRequiredApprove + } + + return "" +} + +// readJSON decodes a JSON request body. +func readJSON(r *http.Request, v any) error { + return json.NewDecoder(r.Body).Decode(v) +} diff --git a/internal/api/handlers.go b/internal/api/handlers.go index 69dc0a4..c123666 100644 --- a/internal/api/handlers.go +++ b/internal/api/handlers.go @@ -25,6 +25,7 @@ import ( "github.com/kindredsystems/silo/internal/partnum" "github.com/kindredsystems/silo/internal/schema" "github.com/kindredsystems/silo/internal/storage" + "github.com/kindredsystems/silo/internal/workflow" "github.com/rs/zerolog" "gopkg.in/yaml.v3" ) @@ -58,6 +59,8 @@ type Server struct { metadata *db.ItemMetadataRepository deps *db.ItemDependencyRepository macros *db.ItemMacroRepository + approvals *db.ItemApprovalRepository + workflows map[string]*workflow.Workflow } // NewServer creates a new API server. @@ -77,6 +80,7 @@ func NewServer( jobDefsDir string, registry *modules.Registry, cfg *config.Config, + workflows map[string]*workflow.Workflow, ) *Server { items := db.NewItemRepository(database) projects := db.NewProjectRepository(database) @@ -89,6 +93,7 @@ func NewServer( metadata := db.NewItemMetadataRepository(database) itemDeps := db.NewItemDependencyRepository(database) itemMacros := db.NewItemMacroRepository(database) + itemApprovals := db.NewItemApprovalRepository(database) seqStore := &dbSequenceStore{db: database, schemas: schemas} partgen := partnum.NewGenerator(schemas, seqStore) @@ -120,6 +125,8 @@ func NewServer( metadata: metadata, deps: itemDeps, macros: itemMacros, + approvals: itemApprovals, + workflows: workflows, } } diff --git a/internal/api/pack_handlers.go b/internal/api/pack_handlers.go index 0220f11..7902d4e 100644 --- a/internal/api/pack_handlers.go +++ b/internal/api/pack_handlers.go @@ -74,11 +74,58 @@ func (s *Server) packKCFile(ctx context.Context, data []byte, item *db.Item, rev deps = []kc.Dependency{} } + // Build approvals from item_approvals table. + var approvals []kc.ApprovalEntry + dbApprovals, err := s.approvals.ListByItemWithSignatures(ctx, item.ID) + if err != nil { + s.logger.Warn().Err(err).Str("part_number", item.PartNumber).Msg("kc: failed to query approvals for packing") + } else { + approvals = make([]kc.ApprovalEntry, len(dbApprovals)) + for i, a := range dbApprovals { + sigs := make([]kc.SignatureEntry, len(a.Signatures)) + for j, sig := range a.Signatures { + var signedAt string + if sig.SignedAt != nil { + signedAt = sig.SignedAt.UTC().Format("2006-01-02T15:04:05Z") + } + var comment string + if sig.Comment != nil { + comment = *sig.Comment + } + sigs[j] = kc.SignatureEntry{ + Username: sig.Username, + Role: sig.Role, + Status: sig.Status, + SignedAt: signedAt, + Comment: comment, + } + } + var ecoNumber string + if a.ECONumber != nil { + ecoNumber = *a.ECONumber + } + var updatedBy string + if a.UpdatedBy != nil { + updatedBy = *a.UpdatedBy + } + approvals[i] = kc.ApprovalEntry{ + ID: a.ID, + WorkflowName: a.WorkflowName, + ECONumber: ecoNumber, + State: a.State, + UpdatedAt: a.UpdatedAt.UTC().Format("2006-01-02T15:04:05Z"), + UpdatedBy: updatedBy, + Signatures: sigs, + } + } + } + input := &kc.PackInput{ Manifest: manifest, Metadata: metadata, History: history, Dependencies: deps, + Approvals: approvals, } return kc.Pack(data, input) diff --git a/internal/api/routes.go b/internal/api/routes.go index 8cd96fb..677114a 100644 --- a/internal/api/routes.go +++ b/internal/api/routes.go @@ -68,6 +68,9 @@ func NewRouter(server *Server, logger zerolog.Logger) http.Handler { // SSE event stream (viewer+) r.Get("/events", server.HandleEvents) + // Workflows (viewer+) + r.Get("/workflows", server.HandleListWorkflows) + // Auth endpoints r.Get("/auth/me", server.HandleGetCurrentUser) r.Route("/auth/tokens", func(r chi.Router) { @@ -177,6 +180,7 @@ func NewRouter(server *Server, logger zerolog.Logger) http.Handler { r.Get("/dependencies/resolve", server.HandleResolveDependencies) r.Get("/macros", server.HandleGetMacros) r.Get("/macros/{filename}", server.HandleGetMacro) + r.Get("/approvals", server.HandleGetApprovals) // DAG (gated by dag module) r.Route("/dag", func(r chi.Router) { @@ -217,6 +221,8 @@ func NewRouter(server *Server, logger zerolog.Logger) http.Handler { r.Put("/metadata", server.HandleUpdateMetadata) r.Patch("/metadata/lifecycle", server.HandleUpdateLifecycle) r.Patch("/metadata/tags", server.HandleUpdateTags) + r.Post("/approvals", server.HandleCreateApproval) + r.Post("/approvals/{id}/sign", server.HandleSignApproval) }) }) }) diff --git a/internal/config/config.go b/internal/config/config.go index 1d93afd..f5d1390 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -10,15 +10,16 @@ import ( // Config holds all application configuration. type Config struct { - Server ServerConfig `yaml:"server"` - Database DatabaseConfig `yaml:"database"` - Storage StorageConfig `yaml:"storage"` - Schemas SchemasConfig `yaml:"schemas"` - FreeCAD FreeCADConfig `yaml:"freecad"` - Odoo OdooConfig `yaml:"odoo"` - Auth AuthConfig `yaml:"auth"` - Jobs JobsConfig `yaml:"jobs"` - Modules ModulesConfig `yaml:"modules"` + Server ServerConfig `yaml:"server"` + Database DatabaseConfig `yaml:"database"` + Storage StorageConfig `yaml:"storage"` + Schemas SchemasConfig `yaml:"schemas"` + FreeCAD FreeCADConfig `yaml:"freecad"` + Odoo OdooConfig `yaml:"odoo"` + Auth AuthConfig `yaml:"auth"` + Jobs JobsConfig `yaml:"jobs"` + Workflows WorkflowsConfig `yaml:"workflows"` + Modules ModulesConfig `yaml:"modules"` } // ModulesConfig holds explicit enable/disable toggles for optional modules. @@ -146,6 +147,11 @@ type JobsConfig struct { DefaultPriority int `yaml:"default_priority"` // default 100 } +// WorkflowsConfig holds approval workflow definition settings. +type WorkflowsConfig struct { + Directory string `yaml:"directory"` // default /etc/silo/workflows +} + // OdooConfig holds Odoo ERP integration settings. type OdooConfig struct { Enabled bool `yaml:"enabled"` @@ -204,6 +210,9 @@ func Load(path string) (*Config, error) { if cfg.Jobs.DefaultPriority == 0 { cfg.Jobs.DefaultPriority = 100 } + if cfg.Workflows.Directory == "" { + cfg.Workflows.Directory = "/etc/silo/workflows" + } // Override with environment variables if v := os.Getenv("SILO_DB_HOST"); v != "" { diff --git a/internal/db/item_approvals.go b/internal/db/item_approvals.go new file mode 100644 index 0000000..5923b3e --- /dev/null +++ b/internal/db/item_approvals.go @@ -0,0 +1,212 @@ +package db + +import ( + "context" + "fmt" + "time" + + "github.com/jackc/pgx/v5" +) + +// ItemApproval represents a row in the item_approvals table. +type ItemApproval struct { + ID string + ItemID string + WorkflowName string + ECONumber *string + State string // draft | pending | approved | rejected + UpdatedAt time.Time + UpdatedBy *string + Signatures []ApprovalSignature // populated by WithSignatures methods +} + +// ApprovalSignature represents a row in the approval_signatures table. +type ApprovalSignature struct { + ID string + ApprovalID string + Username string + Role string + Status string // pending | approved | rejected + SignedAt *time.Time + Comment *string +} + +// ItemApprovalRepository provides item_approvals database operations. +type ItemApprovalRepository struct { + db *DB +} + +// NewItemApprovalRepository creates a new item approval repository. +func NewItemApprovalRepository(db *DB) *ItemApprovalRepository { + return &ItemApprovalRepository{db: db} +} + +// Create inserts a new approval row. The ID is populated on return. +func (r *ItemApprovalRepository) Create(ctx context.Context, a *ItemApproval) error { + return r.db.pool.QueryRow(ctx, ` + INSERT INTO item_approvals (item_id, workflow_name, eco_number, state, updated_by) + VALUES ($1, $2, $3, $4, $5) + RETURNING id, updated_at + `, a.ItemID, a.WorkflowName, a.ECONumber, a.State, a.UpdatedBy).Scan(&a.ID, &a.UpdatedAt) +} + +// AddSignature inserts a new signature row. The ID is populated on return. +func (r *ItemApprovalRepository) AddSignature(ctx context.Context, s *ApprovalSignature) error { + return r.db.pool.QueryRow(ctx, ` + INSERT INTO approval_signatures (approval_id, username, role, status) + VALUES ($1, $2, $3, $4) + RETURNING id + `, s.ApprovalID, s.Username, s.Role, s.Status).Scan(&s.ID) +} + +// GetWithSignatures returns a single approval with its signatures. +func (r *ItemApprovalRepository) GetWithSignatures(ctx context.Context, approvalID string) (*ItemApproval, error) { + a := &ItemApproval{} + err := r.db.pool.QueryRow(ctx, ` + SELECT id, item_id, workflow_name, eco_number, state, updated_at, updated_by + FROM item_approvals + WHERE id = $1 + `, approvalID).Scan(&a.ID, &a.ItemID, &a.WorkflowName, &a.ECONumber, &a.State, &a.UpdatedAt, &a.UpdatedBy) + if err != nil { + if err == pgx.ErrNoRows { + return nil, nil + } + return nil, fmt.Errorf("getting approval: %w", err) + } + + sigs, err := r.signaturesForApproval(ctx, approvalID) + if err != nil { + return nil, err + } + a.Signatures = sigs + return a, nil +} + +// ListByItemWithSignatures returns all approvals for an item, each with signatures. +func (r *ItemApprovalRepository) ListByItemWithSignatures(ctx context.Context, itemID string) ([]*ItemApproval, error) { + rows, err := r.db.pool.Query(ctx, ` + SELECT id, item_id, workflow_name, eco_number, state, updated_at, updated_by + FROM item_approvals + WHERE item_id = $1 + ORDER BY updated_at DESC + `, itemID) + if err != nil { + return nil, fmt.Errorf("listing approvals: %w", err) + } + defer rows.Close() + + var approvals []*ItemApproval + var approvalIDs []string + for rows.Next() { + a := &ItemApproval{} + if err := rows.Scan(&a.ID, &a.ItemID, &a.WorkflowName, &a.ECONumber, &a.State, &a.UpdatedAt, &a.UpdatedBy); err != nil { + return nil, fmt.Errorf("scanning approval: %w", err) + } + approvals = append(approvals, a) + approvalIDs = append(approvalIDs, a.ID) + } + + if len(approvalIDs) == 0 { + return approvals, nil + } + + // Batch-fetch all signatures + sigRows, err := r.db.pool.Query(ctx, ` + SELECT id, approval_id, username, role, status, signed_at, comment + FROM approval_signatures + WHERE approval_id = ANY($1) + ORDER BY username + `, approvalIDs) + if err != nil { + return nil, fmt.Errorf("listing signatures: %w", err) + } + defer sigRows.Close() + + sigMap := make(map[string][]ApprovalSignature) + for sigRows.Next() { + var s ApprovalSignature + if err := sigRows.Scan(&s.ID, &s.ApprovalID, &s.Username, &s.Role, &s.Status, &s.SignedAt, &s.Comment); err != nil { + return nil, fmt.Errorf("scanning signature: %w", err) + } + sigMap[s.ApprovalID] = append(sigMap[s.ApprovalID], s) + } + + for _, a := range approvals { + a.Signatures = sigMap[a.ID] + if a.Signatures == nil { + a.Signatures = []ApprovalSignature{} + } + } + + return approvals, nil +} + +// UpdateState updates the approval state and updated_by. +func (r *ItemApprovalRepository) UpdateState(ctx context.Context, approvalID, state, updatedBy string) error { + _, err := r.db.pool.Exec(ctx, ` + UPDATE item_approvals + SET state = $2, updated_by = $3, updated_at = now() + WHERE id = $1 + `, approvalID, state, updatedBy) + if err != nil { + return fmt.Errorf("updating approval state: %w", err) + } + return nil +} + +// GetSignatureForUser returns the signature for a specific user on an approval. +func (r *ItemApprovalRepository) GetSignatureForUser(ctx context.Context, approvalID, username string) (*ApprovalSignature, error) { + s := &ApprovalSignature{} + err := r.db.pool.QueryRow(ctx, ` + SELECT id, approval_id, username, role, status, signed_at, comment + FROM approval_signatures + WHERE approval_id = $1 AND username = $2 + `, approvalID, username).Scan(&s.ID, &s.ApprovalID, &s.Username, &s.Role, &s.Status, &s.SignedAt, &s.Comment) + if err != nil { + if err == pgx.ErrNoRows { + return nil, nil + } + return nil, fmt.Errorf("getting signature: %w", err) + } + return s, nil +} + +// UpdateSignature updates a signature's status, comment, and signed_at timestamp. +func (r *ItemApprovalRepository) UpdateSignature(ctx context.Context, sigID, status string, comment *string) error { + _, err := r.db.pool.Exec(ctx, ` + UPDATE approval_signatures + SET status = $2, comment = $3, signed_at = now() + WHERE id = $1 + `, sigID, status, comment) + if err != nil { + return fmt.Errorf("updating signature: %w", err) + } + return nil +} + +// signaturesForApproval returns all signatures for a single approval. +func (r *ItemApprovalRepository) signaturesForApproval(ctx context.Context, approvalID string) ([]ApprovalSignature, error) { + rows, err := r.db.pool.Query(ctx, ` + SELECT id, approval_id, username, role, status, signed_at, comment + FROM approval_signatures + WHERE approval_id = $1 + ORDER BY username + `, approvalID) + if err != nil { + return nil, fmt.Errorf("listing signatures: %w", err) + } + defer rows.Close() + + var sigs []ApprovalSignature + for rows.Next() { + var s ApprovalSignature + if err := rows.Scan(&s.ID, &s.ApprovalID, &s.Username, &s.Role, &s.Status, &s.SignedAt, &s.Comment); err != nil { + return nil, fmt.Errorf("scanning signature: %w", err) + } + sigs = append(sigs, s) + } + if sigs == nil { + sigs = []ApprovalSignature{} + } + return sigs, nil +} diff --git a/internal/kc/kc.go b/internal/kc/kc.go index c10a38e..9954f73 100644 --- a/internal/kc/kc.go +++ b/internal/kc/kc.go @@ -64,6 +64,26 @@ type HistoryEntry struct { Labels []string `json:"labels"` } +// ApprovalEntry represents one entry in silo/approvals.json. +type ApprovalEntry struct { + ID string `json:"id"` + WorkflowName string `json:"workflow"` + ECONumber string `json:"eco_number,omitempty"` + State string `json:"state"` + UpdatedAt string `json:"updated_at"` + UpdatedBy string `json:"updated_by,omitempty"` + Signatures []SignatureEntry `json:"signatures"` +} + +// SignatureEntry represents one signer in an approval. +type SignatureEntry struct { + Username string `json:"username"` + Role string `json:"role"` + Status string `json:"status"` + SignedAt string `json:"signed_at,omitempty"` + Comment string `json:"comment,omitempty"` +} + // PackInput holds all the data needed to repack silo/ entries in a .kc file. // Each field is optional — nil/empty means the entry is omitted from the ZIP. type PackInput struct { @@ -71,6 +91,7 @@ type PackInput struct { Metadata *Metadata History []HistoryEntry Dependencies []Dependency + Approvals []ApprovalEntry } // Extract opens a ZIP archive from data and parses the silo/ directory. diff --git a/internal/kc/pack.go b/internal/kc/pack.go index 33c2ae0..7844afd 100644 --- a/internal/kc/pack.go +++ b/internal/kc/pack.go @@ -83,6 +83,11 @@ func Pack(original []byte, input *PackInput) ([]byte, error) { return nil, fmt.Errorf("kc: writing dependencies.json: %w", err) } } + if input.Approvals != nil { + if err := writeJSONEntry(zw, "silo/approvals.json", input.Approvals); err != nil { + return nil, fmt.Errorf("kc: writing approvals.json: %w", err) + } + } if err := zw.Close(); err != nil { return nil, fmt.Errorf("kc: closing zip writer: %w", err) diff --git a/internal/workflow/workflow.go b/internal/workflow/workflow.go new file mode 100644 index 0000000..971c219 --- /dev/null +++ b/internal/workflow/workflow.go @@ -0,0 +1,156 @@ +// Package workflow handles YAML approval workflow definition parsing and validation. +package workflow + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "gopkg.in/yaml.v3" +) + +// Workflow represents an approval workflow definition loaded from YAML. +type Workflow struct { + Name string `yaml:"name" json:"name"` + Version int `yaml:"version" json:"version"` + Description string `yaml:"description" json:"description"` + States []string `yaml:"states" json:"states"` + Gates []Gate `yaml:"gates" json:"gates"` + Rules Rules `yaml:"rules" json:"rules"` +} + +// Gate defines a required or optional signature role in a workflow. +type Gate struct { + Role string `yaml:"role" json:"role"` + Label string `yaml:"label" json:"label"` + Required bool `yaml:"required" json:"required"` +} + +// Rules defines how signatures determine state transitions. +type Rules struct { + AnyReject string `yaml:"any_reject" json:"any_reject"` + AllRequiredApprove string `yaml:"all_required_approve" json:"all_required_approve"` +} + +// WorkflowFile wraps a workflow for YAML parsing. +type WorkflowFile struct { + Workflow Workflow `yaml:"workflow"` +} + +var requiredStates = []string{"draft", "pending", "approved", "rejected"} + +// Load reads a workflow definition from a YAML file. +func Load(path string) (*Workflow, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading workflow file: %w", err) + } + + var wf WorkflowFile + if err := yaml.Unmarshal(data, &wf); err != nil { + return nil, fmt.Errorf("parsing workflow YAML: %w", err) + } + + w := &wf.Workflow + + if w.Version <= 0 { + w.Version = 1 + } + + if err := w.Validate(); err != nil { + return nil, fmt.Errorf("validating %s: %w", path, err) + } + + return w, nil +} + +// LoadAll reads all workflow definitions from a directory. +func LoadAll(dir string) (map[string]*Workflow, error) { + workflows := make(map[string]*Workflow) + + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("reading workflows directory: %w", err) + } + + for _, entry := range entries { + if entry.IsDir() { + continue + } + if !strings.HasSuffix(entry.Name(), ".yaml") && !strings.HasSuffix(entry.Name(), ".yml") { + continue + } + + path := filepath.Join(dir, entry.Name()) + w, err := Load(path) + if err != nil { + return nil, fmt.Errorf("loading %s: %w", entry.Name(), err) + } + workflows[w.Name] = w + } + + return workflows, nil +} + +// Validate checks that the workflow definition is well-formed. +func (w *Workflow) Validate() error { + if w.Name == "" { + return fmt.Errorf("workflow name is required") + } + + // Validate states include all required states + stateSet := make(map[string]bool, len(w.States)) + for _, s := range w.States { + stateSet[s] = true + } + for _, rs := range requiredStates { + if !stateSet[rs] { + return fmt.Errorf("workflow must include state %q", rs) + } + } + + // Validate gates + if len(w.Gates) == 0 { + return fmt.Errorf("workflow must have at least one gate") + } + for i, g := range w.Gates { + if g.Role == "" { + return fmt.Errorf("gate %d: role is required", i) + } + if g.Label == "" { + return fmt.Errorf("gate %d: label is required", i) + } + } + + // Validate rules reference valid states + if w.Rules.AnyReject != "" && !stateSet[w.Rules.AnyReject] { + return fmt.Errorf("rules.any_reject references unknown state %q", w.Rules.AnyReject) + } + if w.Rules.AllRequiredApprove != "" && !stateSet[w.Rules.AllRequiredApprove] { + return fmt.Errorf("rules.all_required_approve references unknown state %q", w.Rules.AllRequiredApprove) + } + + return nil +} + +// RequiredGates returns only the gates where Required is true. +func (w *Workflow) RequiredGates() []Gate { + var gates []Gate + for _, g := range w.Gates { + if g.Required { + gates = append(gates, g) + } + } + return gates +} + +// HasRole returns true if the workflow defines a gate with the given role. +func (w *Workflow) HasRole(role string) bool { + for _, g := range w.Gates { + if g.Role == role { + return true + } + } + return false +} diff --git a/internal/workflow/workflow_test.go b/internal/workflow/workflow_test.go new file mode 100644 index 0000000..671aeeb --- /dev/null +++ b/internal/workflow/workflow_test.go @@ -0,0 +1,167 @@ +package workflow + +import ( + "os" + "path/filepath" + "testing" +) + +func TestLoad_Valid(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.yaml") + os.WriteFile(path, []byte(` +workflow: + name: test-wf + version: 1 + description: "Test workflow" + states: [draft, pending, approved, rejected] + gates: + - role: reviewer + label: "Review" + required: true + rules: + any_reject: rejected + all_required_approve: approved +`), 0644) + + w, err := Load(path) + if err != nil { + t.Fatalf("Load() error: %v", err) + } + if w.Name != "test-wf" { + t.Errorf("Name = %q, want %q", w.Name, "test-wf") + } + if w.Version != 1 { + t.Errorf("Version = %d, want 1", w.Version) + } + if len(w.Gates) != 1 { + t.Fatalf("Gates count = %d, want 1", len(w.Gates)) + } + if w.Gates[0].Role != "reviewer" { + t.Errorf("Gates[0].Role = %q, want %q", w.Gates[0].Role, "reviewer") + } +} + +func TestLoad_MissingState(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "bad.yaml") + os.WriteFile(path, []byte(` +workflow: + name: bad + states: [draft, pending] + gates: + - role: r + label: "R" + required: true +`), 0644) + + _, err := Load(path) + if err == nil { + t.Fatal("expected error for missing required states") + } +} + +func TestLoad_NoGates(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "no-gates.yaml") + os.WriteFile(path, []byte(` +workflow: + name: no-gates + states: [draft, pending, approved, rejected] + gates: [] +`), 0644) + + _, err := Load(path) + if err == nil { + t.Fatal("expected error for no gates") + } +} + +func TestLoadAll(t *testing.T) { + dir := t.TempDir() + + os.WriteFile(filepath.Join(dir, "a.yaml"), []byte(` +workflow: + name: alpha + states: [draft, pending, approved, rejected] + gates: + - role: r + label: "R" + required: true + rules: + any_reject: rejected + all_required_approve: approved +`), 0644) + + os.WriteFile(filepath.Join(dir, "b.yml"), []byte(` +workflow: + name: beta + states: [draft, pending, approved, rejected] + gates: + - role: r + label: "R" + required: true + rules: + any_reject: rejected +`), 0644) + + // Non-yaml file should be ignored + os.WriteFile(filepath.Join(dir, "readme.txt"), []byte("ignore me"), 0644) + + wfs, err := LoadAll(dir) + if err != nil { + t.Fatalf("LoadAll() error: %v", err) + } + if len(wfs) != 2 { + t.Fatalf("LoadAll() count = %d, want 2", len(wfs)) + } + if wfs["alpha"] == nil { + t.Error("missing workflow 'alpha'") + } + if wfs["beta"] == nil { + t.Error("missing workflow 'beta'") + } +} + +func TestRequiredGates(t *testing.T) { + w := &Workflow{ + Gates: []Gate{ + {Role: "engineer", Label: "Eng", Required: true}, + {Role: "quality", Label: "QA", Required: false}, + {Role: "manager", Label: "Mgr", Required: true}, + }, + } + rg := w.RequiredGates() + if len(rg) != 2 { + t.Fatalf("RequiredGates() count = %d, want 2", len(rg)) + } + if rg[0].Role != "engineer" || rg[1].Role != "manager" { + t.Errorf("RequiredGates() roles = %v, want [engineer, manager]", rg) + } +} + +func TestHasRole(t *testing.T) { + w := &Workflow{ + Gates: []Gate{ + {Role: "engineer", Label: "Eng", Required: true}, + }, + } + if !w.HasRole("engineer") { + t.Error("HasRole(engineer) = false, want true") + } + if w.HasRole("manager") { + t.Error("HasRole(manager) = true, want false") + } +} + +func TestValidate_InvalidRuleState(t *testing.T) { + w := &Workflow{ + Name: "bad-rule", + States: []string{"draft", "pending", "approved", "rejected"}, + Gates: []Gate{{Role: "r", Label: "R", Required: true}}, + Rules: Rules{AnyReject: "nonexistent"}, + } + if err := w.Validate(); err == nil { + t.Fatal("expected error for invalid rule state reference") + } +} diff --git a/migrations/019_approval_workflow_name.sql b/migrations/019_approval_workflow_name.sql new file mode 100644 index 0000000..e7d9719 --- /dev/null +++ b/migrations/019_approval_workflow_name.sql @@ -0,0 +1,2 @@ +-- Add workflow_name column to item_approvals for YAML-configurable approval workflows. +ALTER TABLE item_approvals ADD COLUMN workflow_name TEXT NOT NULL DEFAULT 'default'; diff --git a/workflows/engineering-change.yaml b/workflows/engineering-change.yaml new file mode 100644 index 0000000..03c79c2 --- /dev/null +++ b/workflows/engineering-change.yaml @@ -0,0 +1,25 @@ +workflow: + name: engineering-change + version: 1 + description: "Standard engineering change order with peer review and manager approval" + + states: + - draft + - pending + - approved + - rejected + + gates: + - role: engineer + label: "Peer Review" + required: true + - role: manager + label: "Manager Approval" + required: true + - role: quality + label: "Quality Sign-off" + required: false + + rules: + any_reject: rejected + all_required_approve: approved diff --git a/workflows/quick-review.yaml b/workflows/quick-review.yaml new file mode 100644 index 0000000..a0974b3 --- /dev/null +++ b/workflows/quick-review.yaml @@ -0,0 +1,19 @@ +workflow: + name: quick-review + version: 1 + description: "Single reviewer approval for minor changes" + + states: + - draft + - pending + - approved + - rejected + + gates: + - role: reviewer + label: "Review" + required: true + + rules: + any_reject: rejected + all_required_approve: approved From 88d1ab1f97c1b9801df9d911a93e715c429f6f27 Mon Sep 17 00:00:00 2001 From: Forbes Date: Thu, 19 Feb 2026 14:36:22 -0600 Subject: [PATCH 2/2] refactor(storage): remove MinIO backend, filesystem-only storage Remove the MinIO/S3 storage backend entirely. The filesystem backend is fully implemented, already used in production, and a migrate-storage tool exists for any remaining MinIO deployments to migrate beforehand. Changes: - Delete MinIO client implementation (internal/storage/storage.go) - Delete migrate-storage tool (cmd/migrate-storage, scripts/migrate-storage.sh) - Remove MinIO service, volumes, and env vars from all Docker Compose files - Simplify StorageConfig: remove Endpoint, AccessKey, SecretKey, Bucket, UseSSL, Region fields; add SILO_STORAGE_ROOT_DIR env override - Change all SQL COALESCE defaults from 'minio' to 'filesystem' - Add migration 020 to update column defaults to 'filesystem' - Remove minio-go/v7 dependency (go mod tidy) - Update all config examples, setup scripts, docs, and tests --- .env.example | 4 - Makefile | 17 +- README.md | 6 +- cmd/migrate-storage/main.go | 288 ------------------ cmd/silod/main.go | 32 +- config.example.yaml | 14 +- deployments/config.dev.yaml | 9 +- deployments/docker-compose.allinone.yaml | 41 +-- deployments/docker-compose.prod.yaml | 12 +- deployments/docker-compose.yaml | 30 +- deployments/systemd/silod.service | 1 + docs/MODULES.md | 22 +- frontend-spec.md | 14 +- go.mod | 13 - go.sum | 27 -- internal/api/file_handlers.go | 9 +- internal/api/servermode.go | 2 +- internal/api/settings_handlers.go | 6 +- internal/api/settings_handlers_test.go | 4 +- internal/config/config.go | 23 +- internal/db/item_files.go | 8 +- internal/db/items.go | 32 +- internal/modules/modules.go | 2 +- internal/storage/interface.go | 19 ++ internal/storage/storage.go | 174 ----------- ...020_storage_backend_filesystem_default.sql | 3 + scripts/deploy.sh | 1 - scripts/migrate-storage.sh | 108 ------- scripts/setup-docker.sh | 21 +- scripts/setup-host.sh | 11 +- 30 files changed, 104 insertions(+), 849 deletions(-) delete mode 100644 cmd/migrate-storage/main.go delete mode 100644 internal/storage/storage.go create mode 100644 migrations/020_storage_backend_filesystem_default.sql delete mode 100755 scripts/migrate-storage.sh diff --git a/.env.example b/.env.example index 8f582f8..95b3c36 100644 --- a/.env.example +++ b/.env.example @@ -5,10 +5,6 @@ # PostgreSQL POSTGRES_PASSWORD=silodev -# MinIO -MINIO_ACCESS_KEY=silominio -MINIO_SECRET_KEY=silominiosecret - # OpenLDAP LDAP_ADMIN_PASSWORD=ldapadmin LDAP_USERS=siloadmin diff --git a/Makefile b/Makefile index 793ed31..ea282ed 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,7 @@ .PHONY: build run test test-integration clean migrate fmt lint \ docker-build docker-up docker-down docker-logs docker-ps \ docker-clean docker-rebuild \ - web-install web-dev web-build \ - migrate-storage + web-install web-dev web-build # ============================================================================= # Local Development @@ -57,13 +56,6 @@ tidy: migrate: ./scripts/init-db.sh -# Build and run MinIO → filesystem migration tool -# Usage: make migrate-storage DEST=/opt/silo/data [ARGS="--dry-run --verbose"] -migrate-storage: - go build -o migrate-storage ./cmd/migrate-storage - @echo "Built ./migrate-storage" - @echo "Run: ./migrate-storage -config -dest [-dry-run] [-verbose]" - # Connect to database (requires psql) db-shell: PGPASSWORD=$${SILO_DB_PASSWORD:-silodev} psql -h $${SILO_DB_HOST:-localhost} -U $${SILO_DB_USER:-silo} -d $${SILO_DB_NAME:-silo} @@ -76,7 +68,7 @@ db-shell: docker-build: docker build -t silo:latest -f build/package/Dockerfile . -# Start the full stack (postgres + minio + silo) +# Start the full stack (postgres + silo) docker-up: docker compose -f deployments/docker-compose.yaml up -d @@ -103,9 +95,6 @@ docker-logs-silo: docker-logs-postgres: docker compose -f deployments/docker-compose.yaml logs -f postgres -docker-logs-minio: - docker compose -f deployments/docker-compose.yaml logs -f minio - # Show running containers docker-ps: docker compose -f deployments/docker-compose.yaml ps @@ -175,7 +164,7 @@ help: @echo "" @echo "Docker:" @echo " docker-build - Build Docker image" - @echo " docker-up - Start full stack (postgres + minio + silo)" + @echo " docker-up - Start full stack (postgres + silo)" @echo " docker-down - Stop the stack" @echo " docker-clean - Stop and remove volumes (deletes data)" @echo " docker-logs - View all logs" diff --git a/README.md b/README.md index 3262f06..eb6fcd0 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ silo/ │ ├── ods/ # ODS spreadsheet library │ ├── partnum/ # Part number generation │ ├── schema/ # YAML schema parsing -│ ├── storage/ # MinIO file storage +│ ├── storage/ # Filesystem storage │ └── testutil/ # Test helpers ├── web/ # React SPA (Vite + TypeScript) │ └── src/ @@ -55,7 +55,7 @@ silo/ See the **[Installation Guide](docs/INSTALL.md)** for complete setup instructions. -**Docker Compose (quickest — includes PostgreSQL, MinIO, OpenLDAP, and Silo):** +**Docker Compose (quickest — includes PostgreSQL, OpenLDAP, and Silo):** ```bash ./scripts/setup-docker.sh @@ -65,7 +65,7 @@ docker compose -f deployments/docker-compose.allinone.yaml up -d **Development (local Go + Docker services):** ```bash -make docker-up # Start PostgreSQL + MinIO in Docker +make docker-up # Start PostgreSQL in Docker make run # Run silo locally with Go ``` diff --git a/cmd/migrate-storage/main.go b/cmd/migrate-storage/main.go deleted file mode 100644 index 5e64fcf..0000000 --- a/cmd/migrate-storage/main.go +++ /dev/null @@ -1,288 +0,0 @@ -// Command migrate-storage downloads files from MinIO and writes them to the -// local filesystem. It is a one-shot migration tool for moving off MinIO. -// -// Usage: -// -// migrate-storage -config config.yaml -dest /opt/silo/data [-dry-run] [-verbose] -package main - -import ( - "context" - "flag" - "fmt" - "io" - "os" - "path/filepath" - "time" - - "github.com/kindredsystems/silo/internal/config" - "github.com/kindredsystems/silo/internal/db" - "github.com/kindredsystems/silo/internal/storage" - "github.com/rs/zerolog" -) - -// fileEntry represents a single file to migrate. -type fileEntry struct { - key string - versionID string // MinIO version ID; empty if not versioned - size int64 // expected size from DB; 0 if unknown -} - -func main() { - configPath := flag.String("config", "config.yaml", "Path to configuration file") - dest := flag.String("dest", "", "Destination root directory (required)") - dryRun := flag.Bool("dry-run", false, "Preview what would be migrated without downloading") - verbose := flag.Bool("verbose", false, "Log every file, not just errors and summary") - flag.Parse() - - logger := zerolog.New(os.Stdout).With().Timestamp().Logger() - - if *dest == "" { - logger.Fatal().Msg("-dest is required") - } - - // Load config (reuses existing config for DB + MinIO credentials). - cfg, err := config.Load(*configPath) - if err != nil { - logger.Fatal().Err(err).Msg("failed to load configuration") - } - - ctx := context.Background() - - // Connect to PostgreSQL. - database, err := db.Connect(ctx, db.Config{ - Host: cfg.Database.Host, - Port: cfg.Database.Port, - Name: cfg.Database.Name, - User: cfg.Database.User, - Password: cfg.Database.Password, - SSLMode: cfg.Database.SSLMode, - MaxConnections: cfg.Database.MaxConnections, - }) - if err != nil { - logger.Fatal().Err(err).Msg("failed to connect to database") - } - defer database.Close() - logger.Info().Msg("connected to database") - - // Connect to MinIO. - store, err := storage.Connect(ctx, storage.Config{ - Endpoint: cfg.Storage.Endpoint, - AccessKey: cfg.Storage.AccessKey, - SecretKey: cfg.Storage.SecretKey, - Bucket: cfg.Storage.Bucket, - UseSSL: cfg.Storage.UseSSL, - Region: cfg.Storage.Region, - }) - if err != nil { - logger.Fatal().Err(err).Msg("failed to connect to MinIO") - } - logger.Info().Str("bucket", cfg.Storage.Bucket).Msg("connected to MinIO") - - // Collect all file references from the database. - entries, err := collectEntries(ctx, logger, database) - if err != nil { - logger.Fatal().Err(err).Msg("failed to collect file entries from database") - } - logger.Info().Int("total", len(entries)).Msg("file entries found") - - if len(entries) == 0 { - logger.Info().Msg("nothing to migrate") - return - } - - // Migrate. - var migrated, skipped, failed int - start := time.Now() - - for i, e := range entries { - destPath := filepath.Join(*dest, e.key) - - // Check if already migrated. - if info, err := os.Stat(destPath); err == nil { - if e.size > 0 && info.Size() == e.size { - if *verbose { - logger.Info().Str("key", e.key).Msg("skipped (already exists)") - } - skipped++ - continue - } - // Size mismatch or unknown size — re-download. - } - - if *dryRun { - logger.Info(). - Str("key", e.key). - Int64("size", e.size). - Str("version", e.versionID). - Msgf("[%d/%d] would migrate", i+1, len(entries)) - continue - } - - if err := migrateFile(ctx, store, e, destPath); err != nil { - logger.Error().Err(err).Str("key", e.key).Msg("failed to migrate") - failed++ - continue - } - - migrated++ - if *verbose { - logger.Info(). - Str("key", e.key). - Int64("size", e.size). - Msgf("[%d/%d] migrated", i+1, len(entries)) - } else if (i+1)%50 == 0 { - logger.Info().Msgf("progress: %d/%d", i+1, len(entries)) - } - } - - elapsed := time.Since(start) - ev := logger.Info(). - Int("total", len(entries)). - Int("migrated", migrated). - Int("skipped", skipped). - Int("failed", failed). - Dur("elapsed", elapsed) - if *dryRun { - ev.Msg("dry run complete") - } else { - ev.Msg("migration complete") - } - - if failed > 0 { - os.Exit(1) - } -} - -// collectEntries queries the database for all file references across the three -// storage domains: revision files, item file attachments, and item thumbnails. -// It deduplicates by key. -func collectEntries(ctx context.Context, logger zerolog.Logger, database *db.DB) ([]fileEntry, error) { - pool := database.Pool() - seen := make(map[string]struct{}) - var entries []fileEntry - - add := func(key, versionID string, size int64) { - if key == "" { - return - } - if _, ok := seen[key]; ok { - return - } - seen[key] = struct{}{} - entries = append(entries, fileEntry{key: key, versionID: versionID, size: size}) - } - - // 1. Revision files. - rows, err := pool.Query(ctx, - `SELECT file_key, COALESCE(file_version, ''), COALESCE(file_size, 0) - FROM revisions WHERE file_key IS NOT NULL`) - if err != nil { - return nil, fmt.Errorf("querying revisions: %w", err) - } - for rows.Next() { - var key, version string - var size int64 - if err := rows.Scan(&key, &version, &size); err != nil { - rows.Close() - return nil, fmt.Errorf("scanning revision row: %w", err) - } - add(key, version, size) - } - rows.Close() - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("iterating revisions: %w", err) - } - logger.Info().Int("count", len(entries)).Msg("revision files found") - - // 2. Item file attachments. - countBefore := len(entries) - rows, err = pool.Query(ctx, - `SELECT object_key, size FROM item_files`) - if err != nil { - return nil, fmt.Errorf("querying item_files: %w", err) - } - for rows.Next() { - var key string - var size int64 - if err := rows.Scan(&key, &size); err != nil { - rows.Close() - return nil, fmt.Errorf("scanning item_files row: %w", err) - } - add(key, "", size) - } - rows.Close() - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("iterating item_files: %w", err) - } - logger.Info().Int("count", len(entries)-countBefore).Msg("item file attachments found") - - // 3. Item thumbnails. - countBefore = len(entries) - rows, err = pool.Query(ctx, - `SELECT thumbnail_key FROM items WHERE thumbnail_key IS NOT NULL`) - if err != nil { - return nil, fmt.Errorf("querying item thumbnails: %w", err) - } - for rows.Next() { - var key string - if err := rows.Scan(&key); err != nil { - rows.Close() - return nil, fmt.Errorf("scanning thumbnail row: %w", err) - } - add(key, "", 0) - } - rows.Close() - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("iterating thumbnails: %w", err) - } - logger.Info().Int("count", len(entries)-countBefore).Msg("item thumbnails found") - - return entries, nil -} - -// migrateFile downloads a single file from MinIO and writes it atomically to destPath. -func migrateFile(ctx context.Context, store *storage.Storage, e fileEntry, destPath string) error { - // Ensure parent directory exists. - if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { - return fmt.Errorf("creating directory: %w", err) - } - - // Download from MinIO. - var reader io.ReadCloser - var err error - if e.versionID != "" { - reader, err = store.GetVersion(ctx, e.key, e.versionID) - } else { - reader, err = store.Get(ctx, e.key) - } - if err != nil { - return fmt.Errorf("downloading from MinIO: %w", err) - } - defer reader.Close() - - // Write to temp file then rename for atomicity. - tmpPath := destPath + ".tmp" - f, err := os.Create(tmpPath) - if err != nil { - return fmt.Errorf("creating temp file: %w", err) - } - - if _, err := io.Copy(f, reader); err != nil { - f.Close() - os.Remove(tmpPath) - return fmt.Errorf("writing file: %w", err) - } - - if err := f.Close(); err != nil { - os.Remove(tmpPath) - return fmt.Errorf("closing temp file: %w", err) - } - - if err := os.Rename(tmpPath, destPath); err != nil { - os.Remove(tmpPath) - return fmt.Errorf("renaming temp file: %w", err) - } - - return nil -} diff --git a/cmd/silod/main.go b/cmd/silod/main.go index 2054c56..1697ec2 100644 --- a/cmd/silod/main.go +++ b/cmd/silod/main.go @@ -45,7 +45,6 @@ func main() { Str("host", cfg.Server.Host). Int("port", cfg.Server.Port). Str("database", cfg.Database.Host). - Str("storage", cfg.Storage.Endpoint). Msg("starting silo server") // Connect to database @@ -65,40 +64,17 @@ func main() { defer database.Close() logger.Info().Msg("connected to database") - // Connect to storage (optional - may be externally managed) + // Connect to storage (optional — requires root_dir to be set) var store storage.FileStore - switch cfg.Storage.Backend { - case "minio", "": - if cfg.Storage.Endpoint != "" { - s, connErr := storage.Connect(ctx, storage.Config{ - Endpoint: cfg.Storage.Endpoint, - AccessKey: cfg.Storage.AccessKey, - SecretKey: cfg.Storage.SecretKey, - Bucket: cfg.Storage.Bucket, - UseSSL: cfg.Storage.UseSSL, - Region: cfg.Storage.Region, - }) - if connErr != nil { - logger.Warn().Err(connErr).Msg("failed to connect to storage - file operations disabled") - } else { - store = s - logger.Info().Msg("connected to storage") - } - } else { - logger.Info().Msg("storage not configured - file operations disabled") - } - case "filesystem": - if cfg.Storage.Filesystem.RootDir == "" { - logger.Fatal().Msg("storage.filesystem.root_dir is required when backend is \"filesystem\"") - } + if cfg.Storage.Filesystem.RootDir != "" { s, fsErr := storage.NewFilesystemStore(cfg.Storage.Filesystem.RootDir) if fsErr != nil { logger.Fatal().Err(fsErr).Msg("failed to initialize filesystem storage") } store = s logger.Info().Str("root", cfg.Storage.Filesystem.RootDir).Msg("connected to filesystem storage") - default: - logger.Fatal().Str("backend", cfg.Storage.Backend).Msg("unknown storage backend") + } else { + logger.Info().Msg("storage not configured - file operations disabled") } // Load schemas diff --git a/config.example.yaml b/config.example.yaml index 4a30661..8915a12 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -17,17 +17,9 @@ database: max_connections: 10 storage: - backend: "minio" # "minio" (default) or "filesystem" - # MinIO/S3 settings (used when backend: "minio") - endpoint: "localhost:9000" # Use "minio:9000" for Docker Compose - access_key: "" # Use SILO_MINIO_ACCESS_KEY env var - secret_key: "" # Use SILO_MINIO_SECRET_KEY env var - bucket: "silo-files" - use_ssl: true # Use false for Docker Compose (internal network) - region: "us-east-1" - # Filesystem settings (used when backend: "filesystem") - # filesystem: - # root_dir: "/var/lib/silo/objects" + backend: "filesystem" + filesystem: + root_dir: "/opt/silo/data" # Override with SILO_STORAGE_ROOT_DIR env var schemas: # Directory containing YAML schema files diff --git a/deployments/config.dev.yaml b/deployments/config.dev.yaml index 9f57d05..a2302dc 100644 --- a/deployments/config.dev.yaml +++ b/deployments/config.dev.yaml @@ -17,12 +17,9 @@ database: max_connections: 10 storage: - endpoint: "minio:9000" - access_key: "${MINIO_ACCESS_KEY:-silominio}" - secret_key: "${MINIO_SECRET_KEY:-silominiosecret}" - bucket: "silo-files" - use_ssl: false - region: "us-east-1" + backend: "filesystem" + filesystem: + root_dir: "/var/lib/silo/data" schemas: directory: "/etc/silo/schemas" diff --git a/deployments/docker-compose.allinone.yaml b/deployments/docker-compose.allinone.yaml index 6d73fc8..42448ff 100644 --- a/deployments/docker-compose.allinone.yaml +++ b/deployments/docker-compose.allinone.yaml @@ -1,5 +1,5 @@ # Silo All-in-One Stack -# PostgreSQL + MinIO + OpenLDAP + Silo API + Nginx (optional) +# PostgreSQL + OpenLDAP + Silo API + Nginx (optional) # # Quick start: # ./scripts/setup-docker.sh @@ -40,29 +40,6 @@ services: networks: - silo-net - # --------------------------------------------------------------------------- - # MinIO (S3-compatible object storage) - # --------------------------------------------------------------------------- - minio: - image: minio/minio:latest - container_name: silo-minio - restart: unless-stopped - command: server /data --console-address ":9001" - environment: - MINIO_ROOT_USER: ${MINIO_ACCESS_KEY:?Run ./scripts/setup-docker.sh first} - MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY:?Run ./scripts/setup-docker.sh first} - volumes: - - minio_data:/data - ports: - - "9001:9001" # MinIO console (remove in hardened setups) - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 10s - timeout: 5s - retries: 5 - networks: - - silo-net - # --------------------------------------------------------------------------- # OpenLDAP (user directory for LDAP authentication) # --------------------------------------------------------------------------- @@ -83,9 +60,13 @@ services: - openldap_data:/bitnami/openldap - ./ldap:/docker-entrypoint-initdb.d:ro ports: - - "1389:1389" # LDAP access for debugging (remove in hardened setups) + - "1389:1389" # LDAP access for debugging (remove in hardened setups) healthcheck: - test: ["CMD-SHELL", "ldapsearch -x -H ldap://localhost:1389 -b dc=silo,dc=local -D cn=admin,dc=silo,dc=local -w $${LDAP_ADMIN_PASSWORD} '(objectClass=organization)' >/dev/null 2>&1"] + test: + [ + "CMD-SHELL", + "ldapsearch -x -H ldap://localhost:1389 -b dc=silo,dc=local -D cn=admin,dc=silo,dc=local -w $${LDAP_ADMIN_PASSWORD} '(objectClass=organization)' >/dev/null 2>&1", + ] interval: 10s timeout: 5s retries: 5 @@ -104,8 +85,6 @@ services: depends_on: postgres: condition: service_healthy - minio: - condition: service_healthy openldap: condition: service_healthy env_file: @@ -117,12 +96,10 @@ services: SILO_DB_NAME: silo SILO_DB_USER: silo SILO_DB_PASSWORD: ${POSTGRES_PASSWORD} - SILO_MINIO_ENDPOINT: minio:9000 - SILO_MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} - SILO_MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} ports: - "${SILO_PORT:-8080}:8080" volumes: + - silo_data:/var/lib/silo/data - ../schemas:/etc/silo/schemas:ro - ./config.docker.yaml:/etc/silo/config.yaml:ro healthcheck: @@ -164,7 +141,7 @@ services: volumes: postgres_data: - minio_data: + silo_data: openldap_data: networks: diff --git a/deployments/docker-compose.prod.yaml b/deployments/docker-compose.prod.yaml index 88b7175..d0be4af 100644 --- a/deployments/docker-compose.prod.yaml +++ b/deployments/docker-compose.prod.yaml @@ -1,10 +1,8 @@ # Production Docker Compose for Silo -# Uses external PostgreSQL (psql.example.internal) and MinIO (minio.example.internal) +# Uses external PostgreSQL (psql.example.internal) and filesystem storage # # Usage: # export SILO_DB_PASSWORD= -# export SILO_MINIO_ACCESS_KEY= -# export SILO_MINIO_SECRET_KEY= # docker compose -f docker-compose.prod.yaml up -d services: @@ -24,14 +22,6 @@ services: # Note: SILO_DB_PORT and SILO_DB_SSLMODE are NOT supported as direct # env var overrides. Set these in config.yaml instead, or use ${VAR} # syntax in the YAML file. See docs/CONFIGURATION.md for details. - - # MinIO storage (minio.example.internal) - # Supported as direct env var overrides: - SILO_MINIO_ENDPOINT: minio.example.internal:9000 - SILO_MINIO_ACCESS_KEY: ${SILO_MINIO_ACCESS_KEY:?MinIO access key required} - SILO_MINIO_SECRET_KEY: ${SILO_MINIO_SECRET_KEY:?MinIO secret key required} - # Note: SILO_MINIO_BUCKET and SILO_MINIO_USE_SSL are NOT supported as - # direct env var overrides. Set these in config.yaml instead. ports: - "8080:8080" volumes: diff --git a/deployments/docker-compose.yaml b/deployments/docker-compose.yaml index c13425b..2105eed 100644 --- a/deployments/docker-compose.yaml +++ b/deployments/docker-compose.yaml @@ -19,26 +19,6 @@ services: networks: - silo-network - minio: - image: minio/minio:RELEASE.2023-05-04T21-44-30Z - container_name: silo-minio - command: server /data --console-address ":9001" - environment: - MINIO_ROOT_USER: ${MINIO_ACCESS_KEY:-silominio} - MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY:-silominiosecret} - volumes: - - minio_data:/data - ports: - - "9000:9000" - - "9001:9001" - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 10s - timeout: 5s - retries: 5 - networks: - - silo-network - silo: build: context: .. @@ -47,19 +27,12 @@ services: depends_on: postgres: condition: service_healthy - minio: - condition: service_healthy environment: SILO_DB_HOST: postgres SILO_DB_PORT: 5432 SILO_DB_NAME: silo SILO_DB_USER: silo SILO_DB_PASSWORD: ${POSTGRES_PASSWORD:-silodev} - SILO_MINIO_ENDPOINT: minio:9000 - SILO_MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-silominio} - SILO_MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-silominiosecret} - SILO_MINIO_BUCKET: silo-files - SILO_MINIO_USE_SSL: "false" SILO_SESSION_SECRET: ${SILO_SESSION_SECRET:-change-me-in-production} SILO_OIDC_CLIENT_SECRET: ${SILO_OIDC_CLIENT_SECRET:-} SILO_LDAP_BIND_PASSWORD: ${SILO_LDAP_BIND_PASSWORD:-} @@ -68,6 +41,7 @@ services: ports: - "8080:8080" volumes: + - silo_data:/var/lib/silo/data - ../schemas:/etc/silo/schemas:ro - ./config.dev.yaml:/etc/silo/config.yaml:ro healthcheck: @@ -80,7 +54,7 @@ services: volumes: postgres_data: - minio_data: + silo_data: networks: silo-network: diff --git a/deployments/systemd/silod.service b/deployments/systemd/silod.service index c65da90..e2bbab0 100644 --- a/deployments/systemd/silod.service +++ b/deployments/systemd/silod.service @@ -27,6 +27,7 @@ NoNewPrivileges=yes ProtectSystem=strict ProtectHome=yes PrivateTmp=yes +ReadWritePaths=/opt/silo/data ReadOnlyPaths=/etc/silo /opt/silo # Resource limits diff --git a/docs/MODULES.md b/docs/MODULES.md index d7480b5..589e0c4 100644 --- a/docs/MODULES.md +++ b/docs/MODULES.md @@ -23,7 +23,7 @@ These cannot be disabled. They define what Silo *is*. |-----------|------|-------------| | `core` | Core PDM | Items, revisions, files, BOM, search, import/export, part number generation | | `schemas` | Schemas | Part numbering schema parsing, segment management, form descriptors | -| `storage` | Storage | MinIO/S3 file storage, presigned uploads, versioning | +| `storage` | Storage | Filesystem storage | ### 2.2 Optional Modules @@ -470,12 +470,10 @@ Returns full config grouped by module with secrets redacted: "default": "kindred-rd" }, "storage": { - "endpoint": "minio:9000", - "bucket": "silo-files", - "access_key": "****", - "secret_key": "****", - "use_ssl": false, - "region": "us-east-1", + "backend": "filesystem", + "filesystem": { + "root_dir": "/var/lib/silo/data" + }, "status": "connected" }, "database": { @@ -566,7 +564,7 @@ Available for modules with external connections: | Module | Test Action | |--------|------------| -| `storage` | Ping MinIO, verify bucket exists | +| `storage` | Verify filesystem storage directory is accessible | | `auth` (ldap) | Attempt LDAP bind with configured credentials | | `auth` (oidc) | Fetch OIDC discovery document from issuer URL | | `odoo` | Attempt XML-RPC connection to Odoo | @@ -602,11 +600,9 @@ database: sslmode: disable storage: - endpoint: minio:9000 - bucket: silo-files - access_key: silominio - secret_key: silominiosecret - use_ssl: false + backend: filesystem + filesystem: + root_dir: /var/lib/silo/data schemas: directory: /etc/silo/schemas diff --git a/frontend-spec.md b/frontend-spec.md index 6eb6ab3..2214508 100644 --- a/frontend-spec.md +++ b/frontend-spec.md @@ -337,7 +337,7 @@ Supporting files: | File | Purpose | |------|---------| | `web/src/components/items/CategoryPicker.tsx` | Multi-stage domain/subcategory selector | -| `web/src/components/items/FileDropZone.tsx` | Drag-and-drop file upload with MinIO presigned URLs | +| `web/src/components/items/FileDropZone.tsx` | Drag-and-drop file upload | | `web/src/components/items/TagInput.tsx` | Multi-select tag input for projects | | `web/src/hooks/useFormDescriptor.ts` | Fetches and caches form descriptor from `/api/schemas/{name}/form` | | `web/src/hooks/useFileUpload.ts` | Manages presigned URL upload flow | @@ -421,7 +421,7 @@ Below the picker, the selected category is shown as a breadcrumb: `Fasteners › ### FileDropZone -Handles drag-and-drop and click-to-browse file uploads with MinIO presigned URL flow. +Handles drag-and-drop and click-to-browse file uploads. **Props**: @@ -435,7 +435,7 @@ interface FileDropZoneProps { interface PendingAttachment { file: File; - objectKey: string; // MinIO key after upload + objectKey: string; // storage key after upload uploadProgress: number; // 0-100 uploadStatus: 'pending' | 'uploading' | 'complete' | 'error'; error?: string; @@ -462,7 +462,7 @@ Clicking the zone opens a hidden ``. 1. On file selection/drop, immediately request a presigned upload URL: `POST /api/uploads/presign` with `{ filename, content_type, size }`. 2. Backend returns `{ object_key, upload_url, expires_at }`. -3. `PUT` the file directly to the presigned MinIO URL using `XMLHttpRequest` (for progress tracking). +3. `PUT` the file directly to the presigned URL using `XMLHttpRequest` (for progress tracking). 4. On completion, update `PendingAttachment.uploadStatus` to `'complete'` and store the `object_key`. 5. The `object_key` is later sent to the item creation endpoint to associate the file. @@ -589,10 +589,10 @@ Items 1-5 below are implemented. Item 4 (hierarchical categories) is resolved by ``` POST /api/uploads/presign Request: { "filename": "bracket.FCStd", "content_type": "application/octet-stream", "size": 2400000 } -Response: { "object_key": "uploads/tmp/{uuid}/{filename}", "upload_url": "https://minio.../...", "expires_at": "2026-02-06T..." } +Response: { "object_key": "uploads/tmp/{uuid}/{filename}", "upload_url": "https://...", "expires_at": "2026-02-06T..." } ``` -The Go handler generates a presigned PUT URL via the MinIO SDK. Objects are uploaded to a temporary prefix. On item creation, they're moved/linked to the item's permanent prefix. +The Go handler generates a presigned PUT URL for direct upload. Objects are uploaded to a temporary prefix. On item creation, they're moved/linked to the item's permanent prefix. ### 2. File Association -- IMPLEMENTED @@ -612,7 +612,7 @@ Request: { "object_key": "uploads/tmp/{uuid}/thumb.png" } Response: 204 ``` -Stores the thumbnail at `items/{item_id}/thumbnail.png` in MinIO. Updates `item.thumbnail_key` column. +Stores the thumbnail at `items/{item_id}/thumbnail.png` in storage. Updates `item.thumbnail_key` column. ### 4. Hierarchical Categories -- IMPLEMENTED (via Form Descriptor) diff --git a/go.mod b/go.mod index 6c0b886..b7570bf 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,6 @@ require ( github.com/go-ldap/ldap/v3 v3.4.12 github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.5.4 - github.com/minio/minio-go/v7 v7.0.66 github.com/rs/zerolog v1.32.0 github.com/sahilm/fuzzy v0.1.1 golang.org/x/crypto v0.47.0 @@ -21,28 +20,16 @@ require ( require ( github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect - github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect github.com/go-jose/go-jose/v4 v4.1.3 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.4 // indirect - github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect - github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/sha256-simd v1.0.1 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect - github.com/rs/xid v1.5.0 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect - golang.org/x/net v0.48.0 // indirect golang.org/x/sync v0.19.0 // indirect golang.org/x/sys v0.40.0 // indirect golang.org/x/text v0.33.0 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index da5ca9b..72116ee 100644 --- a/go.sum +++ b/go.sum @@ -13,8 +13,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 h1:BP4M0CvQ4S3TGls2FvczZtj5Re/2ZzkV9VwqPHH/3Bo= github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= @@ -26,7 +24,6 @@ github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9 github.com/go-ldap/ldap/v3 v3.4.12 h1:1b81mv7MagXZ7+1r7cLTWmyuTqVqdwbtJSjC0DAp9s4= github.com/go-ldap/ldap/v3 v3.4.12/go.mod h1:+SPAGcTtOfmGsCb3h1RFiq4xpp4N636G75OEace8lNo= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= @@ -51,13 +48,6 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= -github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= -github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= @@ -73,31 +63,17 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= -github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw= -github.com/minio/minio-go/v7 v7.0.66/go.mod h1:DHAgmyQEGdW3Cif0UooKOyrT3Vxs82zNdV6tkKhRtbs= -github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= -github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0= github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/sahilm/fuzzy v0.1.1 h1:ceu5RHF8DGgoi+/dR5PsECjCDH1BE3Fnmpo7aVXOdRA= github.com/sahilm/fuzzy v0.1.1/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -133,7 +109,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -166,8 +141,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/api/file_handlers.go b/internal/api/file_handlers.go index 9f7c07a..9606b5f 100644 --- a/internal/api/file_handlers.go +++ b/internal/api/file_handlers.go @@ -21,7 +21,7 @@ type presignUploadRequest struct { Size int64 `json:"size"` } -// HandlePresignUpload generates a presigned PUT URL for direct browser upload to MinIO. +// HandlePresignUpload generates a presigned PUT URL for direct browser upload. func (s *Server) HandlePresignUpload(w http.ResponseWriter, r *http.Request) { if s.storage == nil { writeError(w, http.StatusServiceUnavailable, "storage_unavailable", "File storage not configured") @@ -317,12 +317,9 @@ func (s *Server) HandleSetItemThumbnail(w http.ResponseWriter, r *http.Request) w.WriteHeader(http.StatusNoContent) } -// storageBackend returns the configured storage backend name, defaulting to "minio". +// storageBackend returns the configured storage backend name. func (s *Server) storageBackend() string { - if s.cfg != nil && s.cfg.Storage.Backend != "" { - return s.cfg.Storage.Backend - } - return "minio" + return "filesystem" } // HandleUploadItemFile accepts a multipart file upload and stores it as an item attachment. diff --git a/internal/api/servermode.go b/internal/api/servermode.go index a95b276..e3bf751 100644 --- a/internal/api/servermode.go +++ b/internal/api/servermode.go @@ -86,7 +86,7 @@ func (ss *ServerState) ToggleReadOnly() { ss.SetReadOnly(!current) } -// StartStorageHealthCheck launches a periodic check of MinIO reachability. +// StartStorageHealthCheck launches a periodic check of storage reachability. // Updates storageOK and broadcasts server.state on transitions. func (ss *ServerState) StartStorageHealthCheck() { if ss.storage == nil { diff --git a/internal/api/settings_handlers.go b/internal/api/settings_handlers.go index f695f41..b8d7f0b 100644 --- a/internal/api/settings_handlers.go +++ b/internal/api/settings_handlers.go @@ -224,10 +224,8 @@ func (s *Server) buildSchemasSettings() map[string]any { func (s *Server) buildStorageSettings(ctx context.Context) map[string]any { result := map[string]any{ "enabled": true, - "endpoint": s.cfg.Storage.Endpoint, - "bucket": s.cfg.Storage.Bucket, - "use_ssl": s.cfg.Storage.UseSSL, - "region": s.cfg.Storage.Region, + "backend": "filesystem", + "root_dir": s.cfg.Storage.Filesystem.RootDir, } if s.storage != nil { if err := s.storage.Ping(ctx); err != nil { diff --git a/internal/api/settings_handlers_test.go b/internal/api/settings_handlers_test.go index 716afca..d2c8a64 100644 --- a/internal/api/settings_handlers_test.go +++ b/internal/api/settings_handlers_test.go @@ -31,8 +31,8 @@ func newSettingsTestServer(t *testing.T) *Server { MaxConnections: 10, }, Storage: config.StorageConfig{ - Endpoint: "minio:9000", Bucket: "silo", Region: "us-east-1", - AccessKey: "minioadmin", SecretKey: "miniosecret", + Backend: "filesystem", + Filesystem: config.FilesystemConfig{RootDir: "/tmp/silo-test"}, }, Schemas: config.SchemasConfig{Directory: "/etc/silo/schemas", Default: "kindred-rd"}, Auth: config.AuthConfig{ diff --git a/internal/config/config.go b/internal/config/config.go index f5d1390..59ee464 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -110,15 +110,9 @@ type DatabaseConfig struct { MaxConnections int `yaml:"max_connections"` } -// StorageConfig holds object storage settings. +// StorageConfig holds file storage settings. type StorageConfig struct { - Backend string `yaml:"backend"` // "minio" (default) or "filesystem" - Endpoint string `yaml:"endpoint"` - AccessKey string `yaml:"access_key"` - SecretKey string `yaml:"secret_key"` - Bucket string `yaml:"bucket"` - UseSSL bool `yaml:"use_ssl"` - Region string `yaml:"region"` + Backend string `yaml:"backend"` // "filesystem" Filesystem FilesystemConfig `yaml:"filesystem"` } @@ -189,9 +183,6 @@ func Load(path string) (*Config, error) { if cfg.Database.MaxConnections == 0 { cfg.Database.MaxConnections = 10 } - if cfg.Storage.Region == "" { - cfg.Storage.Region = "us-east-1" - } if cfg.Schemas.Directory == "" { cfg.Schemas.Directory = "/etc/silo/schemas" } @@ -227,14 +218,8 @@ func Load(path string) (*Config, error) { if v := os.Getenv("SILO_DB_PASSWORD"); v != "" { cfg.Database.Password = v } - if v := os.Getenv("SILO_MINIO_ENDPOINT"); v != "" { - cfg.Storage.Endpoint = v - } - if v := os.Getenv("SILO_MINIO_ACCESS_KEY"); v != "" { - cfg.Storage.AccessKey = v - } - if v := os.Getenv("SILO_MINIO_SECRET_KEY"); v != "" { - cfg.Storage.SecretKey = v + if v := os.Getenv("SILO_STORAGE_ROOT_DIR"); v != "" { + cfg.Storage.Filesystem.RootDir = v } // Auth defaults diff --git a/internal/db/item_files.go b/internal/db/item_files.go index 6c6bcf4..109ead4 100644 --- a/internal/db/item_files.go +++ b/internal/db/item_files.go @@ -14,7 +14,7 @@ type ItemFile struct { ContentType string Size int64 ObjectKey string - StorageBackend string // "minio" or "filesystem" + StorageBackend string CreatedAt time.Time } @@ -31,7 +31,7 @@ func NewItemFileRepository(db *DB) *ItemFileRepository { // Create inserts a new item file record. func (r *ItemFileRepository) Create(ctx context.Context, f *ItemFile) error { if f.StorageBackend == "" { - f.StorageBackend = "minio" + f.StorageBackend = "filesystem" } err := r.db.pool.QueryRow(ctx, `INSERT INTO item_files (item_id, filename, content_type, size, object_key, storage_backend) @@ -49,7 +49,7 @@ func (r *ItemFileRepository) Create(ctx context.Context, f *ItemFile) error { func (r *ItemFileRepository) ListByItem(ctx context.Context, itemID string) ([]*ItemFile, error) { rows, err := r.db.pool.Query(ctx, `SELECT id, item_id, filename, content_type, size, object_key, - COALESCE(storage_backend, 'minio'), created_at + COALESCE(storage_backend, 'filesystem'), created_at FROM item_files WHERE item_id = $1 ORDER BY created_at`, itemID, ) @@ -74,7 +74,7 @@ func (r *ItemFileRepository) Get(ctx context.Context, id string) (*ItemFile, err f := &ItemFile{} err := r.db.pool.QueryRow(ctx, `SELECT id, item_id, filename, content_type, size, object_key, - COALESCE(storage_backend, 'minio'), created_at + COALESCE(storage_backend, 'filesystem'), created_at FROM item_files WHERE id = $1`, id, ).Scan(&f.ID, &f.ItemID, &f.Filename, &f.ContentType, &f.Size, &f.ObjectKey, &f.StorageBackend, &f.CreatedAt) diff --git a/internal/db/items.go b/internal/db/items.go index f5518b7..1750b89 100644 --- a/internal/db/items.go +++ b/internal/db/items.go @@ -26,26 +26,26 @@ type Item struct { UpdatedBy *string SourcingType string // "manufactured" or "purchased" LongDescription *string // extended description - ThumbnailKey *string // MinIO key for item thumbnail + ThumbnailKey *string // storage key for item thumbnail } // Revision represents a revision record. type Revision struct { - ID string - ItemID string - RevisionNumber int - Properties map[string]any + ID string + ItemID string + RevisionNumber int + Properties map[string]any FileKey *string FileVersion *string FileChecksum *string FileSize *int64 - FileStorageBackend string // "minio" or "filesystem" + FileStorageBackend string ThumbnailKey *string - CreatedAt time.Time - CreatedBy *string - Comment *string - Status string // draft, review, released, obsolete - Labels []string // arbitrary tags + CreatedAt time.Time + CreatedBy *string + Comment *string + Status string // draft, review, released, obsolete + Labels []string // arbitrary tags } // RevisionStatus constants @@ -308,7 +308,7 @@ func (r *ItemRepository) CreateRevision(ctx context.Context, rev *Revision) erro } if rev.FileStorageBackend == "" { - rev.FileStorageBackend = "minio" + rev.FileStorageBackend = "filesystem" } err = r.db.pool.QueryRow(ctx, ` @@ -347,7 +347,7 @@ func (r *ItemRepository) GetRevisions(ctx context.Context, itemID string) ([]*Re if hasStatusColumn { rows, err = r.db.pool.Query(ctx, ` SELECT id, item_id, revision_number, properties, file_key, file_version, - file_checksum, file_size, COALESCE(file_storage_backend, 'minio'), + file_checksum, file_size, COALESCE(file_storage_backend, 'filesystem'), thumbnail_key, created_at, created_by, comment, COALESCE(status, 'draft') as status, COALESCE(labels, ARRAY[]::TEXT[]) as labels FROM revisions @@ -386,7 +386,7 @@ func (r *ItemRepository) GetRevisions(ctx context.Context, itemID string) ([]*Re ) rev.Status = "draft" rev.Labels = []string{} - rev.FileStorageBackend = "minio" + rev.FileStorageBackend = "filesystem" } if err != nil { return nil, fmt.Errorf("scanning revision: %w", err) @@ -420,7 +420,7 @@ func (r *ItemRepository) GetRevision(ctx context.Context, itemID string, revisio if hasStatusColumn { err = r.db.pool.QueryRow(ctx, ` SELECT id, item_id, revision_number, properties, file_key, file_version, - file_checksum, file_size, COALESCE(file_storage_backend, 'minio'), + file_checksum, file_size, COALESCE(file_storage_backend, 'filesystem'), thumbnail_key, created_at, created_by, comment, COALESCE(status, 'draft') as status, COALESCE(labels, ARRAY[]::TEXT[]) as labels FROM revisions @@ -443,7 +443,7 @@ func (r *ItemRepository) GetRevision(ctx context.Context, itemID string, revisio ) rev.Status = "draft" rev.Labels = []string{} - rev.FileStorageBackend = "minio" + rev.FileStorageBackend = "filesystem" } if err == pgx.ErrNoRows { diff --git a/internal/modules/modules.go b/internal/modules/modules.go index c0fb3e0..14f2f9a 100644 --- a/internal/modules/modules.go +++ b/internal/modules/modules.go @@ -50,7 +50,7 @@ type Registry struct { var builtinModules = []ModuleInfo{ {ID: Core, Name: "Core PDM", Description: "Items, revisions, files, BOM, search, import/export", Required: true, Version: "0.2"}, {ID: Schemas, Name: "Schemas", Description: "Part numbering schema parsing and segment management", Required: true}, - {ID: Storage, Name: "Storage", Description: "MinIO/S3 file storage, presigned uploads", Required: true}, + {ID: Storage, Name: "Storage", Description: "Filesystem storage", Required: true}, {ID: Auth, Name: "Authentication", Description: "Local, LDAP, OIDC authentication and RBAC", DefaultEnabled: true}, {ID: Projects, Name: "Projects", Description: "Project management and item tagging", DefaultEnabled: true}, {ID: Audit, Name: "Audit", Description: "Audit logging, completeness scoring", DefaultEnabled: true}, diff --git a/internal/storage/interface.go b/internal/storage/interface.go index 6ff1338..e9d8079 100644 --- a/internal/storage/interface.go +++ b/internal/storage/interface.go @@ -3,6 +3,7 @@ package storage import ( "context" + "fmt" "io" "net/url" "time" @@ -19,3 +20,21 @@ type FileStore interface { PresignPut(ctx context.Context, key string, expiry time.Duration) (*url.URL, error) Ping(ctx context.Context) error } + +// PutResult contains the result of a put operation. +type PutResult struct { + Key string + VersionID string + Size int64 + Checksum string +} + +// FileKey generates a storage key for an item file. +func FileKey(partNumber string, revision int) string { + return fmt.Sprintf("items/%s/rev%d.FCStd", partNumber, revision) +} + +// ThumbnailKey generates a storage key for a thumbnail. +func ThumbnailKey(partNumber string, revision int) string { + return fmt.Sprintf("thumbnails/%s/rev%d.png", partNumber, revision) +} diff --git a/internal/storage/storage.go b/internal/storage/storage.go deleted file mode 100644 index 8e3ebaf..0000000 --- a/internal/storage/storage.go +++ /dev/null @@ -1,174 +0,0 @@ -package storage - -import ( - "context" - "fmt" - "io" - "net/url" - "time" - - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" -) - -// Config holds MinIO connection settings. -type Config struct { - Endpoint string - AccessKey string - SecretKey string - Bucket string - UseSSL bool - Region string -} - -// Compile-time check: *Storage implements FileStore. -var _ FileStore = (*Storage)(nil) - -// Storage wraps MinIO client operations. -type Storage struct { - client *minio.Client - bucket string -} - -// Connect creates a new MinIO storage client. -func Connect(ctx context.Context, cfg Config) (*Storage, error) { - client, err := minio.New(cfg.Endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""), - Secure: cfg.UseSSL, - Region: cfg.Region, - }) - if err != nil { - return nil, fmt.Errorf("creating minio client: %w", err) - } - - // Ensure bucket exists with versioning - exists, err := client.BucketExists(ctx, cfg.Bucket) - if err != nil { - return nil, fmt.Errorf("checking bucket: %w", err) - } - if !exists { - if err := client.MakeBucket(ctx, cfg.Bucket, minio.MakeBucketOptions{ - Region: cfg.Region, - }); err != nil { - return nil, fmt.Errorf("creating bucket: %w", err) - } - // Enable versioning - if err := client.EnableVersioning(ctx, cfg.Bucket); err != nil { - return nil, fmt.Errorf("enabling versioning: %w", err) - } - } - - return &Storage{client: client, bucket: cfg.Bucket}, nil -} - -// PutResult contains the result of a put operation. -type PutResult struct { - Key string - VersionID string - Size int64 - Checksum string -} - -// Put uploads a file to storage. -func (s *Storage) Put(ctx context.Context, key string, reader io.Reader, size int64, contentType string) (*PutResult, error) { - info, err := s.client.PutObject(ctx, s.bucket, key, reader, size, minio.PutObjectOptions{ - ContentType: contentType, - }) - if err != nil { - return nil, fmt.Errorf("uploading object: %w", err) - } - - return &PutResult{ - Key: key, - VersionID: info.VersionID, - Size: info.Size, - Checksum: info.ChecksumSHA256, - }, nil -} - -// Get downloads a file from storage. -func (s *Storage) Get(ctx context.Context, key string) (io.ReadCloser, error) { - obj, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{}) - if err != nil { - return nil, fmt.Errorf("getting object: %w", err) - } - return obj, nil -} - -// GetVersion downloads a specific version of a file. -func (s *Storage) GetVersion(ctx context.Context, key, versionID string) (io.ReadCloser, error) { - obj, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{ - VersionID: versionID, - }) - if err != nil { - return nil, fmt.Errorf("getting object version: %w", err) - } - return obj, nil -} - -// Delete removes a file from storage. -func (s *Storage) Delete(ctx context.Context, key string) error { - if err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}); err != nil { - return fmt.Errorf("removing object: %w", err) - } - return nil -} - -// Exists checks if an object exists in storage. -func (s *Storage) Exists(ctx context.Context, key string) (bool, error) { - _, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{}) - if err != nil { - resp := minio.ToErrorResponse(err) - if resp.Code == "NoSuchKey" { - return false, nil - } - return false, fmt.Errorf("checking object existence: %w", err) - } - return true, nil -} - -// Ping checks if the storage backend is reachable by verifying the bucket exists. -func (s *Storage) Ping(ctx context.Context) error { - _, err := s.client.BucketExists(ctx, s.bucket) - return err -} - -// Bucket returns the bucket name. -func (s *Storage) Bucket() string { - return s.bucket -} - -// PresignPut generates a presigned PUT URL for direct browser upload. -func (s *Storage) PresignPut(ctx context.Context, key string, expiry time.Duration) (*url.URL, error) { - u, err := s.client.PresignedPutObject(ctx, s.bucket, key, expiry) - if err != nil { - return nil, fmt.Errorf("generating presigned put URL: %w", err) - } - return u, nil -} - -// Copy copies an object within the same bucket from srcKey to dstKey. -func (s *Storage) Copy(ctx context.Context, srcKey, dstKey string) error { - src := minio.CopySrcOptions{ - Bucket: s.bucket, - Object: srcKey, - } - dst := minio.CopyDestOptions{ - Bucket: s.bucket, - Object: dstKey, - } - if _, err := s.client.CopyObject(ctx, dst, src); err != nil { - return fmt.Errorf("copying object: %w", err) - } - return nil -} - -// FileKey generates a storage key for an item file. -func FileKey(partNumber string, revision int) string { - return fmt.Sprintf("items/%s/rev%d.FCStd", partNumber, revision) -} - -// ThumbnailKey generates a storage key for a thumbnail. -func ThumbnailKey(partNumber string, revision int) string { - return fmt.Sprintf("thumbnails/%s/rev%d.png", partNumber, revision) -} diff --git a/migrations/020_storage_backend_filesystem_default.sql b/migrations/020_storage_backend_filesystem_default.sql new file mode 100644 index 0000000..ae0bbb9 --- /dev/null +++ b/migrations/020_storage_backend_filesystem_default.sql @@ -0,0 +1,3 @@ +-- Change default storage backend from 'minio' to 'filesystem'. +ALTER TABLE item_files ALTER COLUMN storage_backend SET DEFAULT 'filesystem'; +ALTER TABLE revisions ALTER COLUMN file_storage_backend SET DEFAULT 'filesystem'; diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 8df1bc8..a93d006 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -8,7 +8,6 @@ # - SSH access to the target host # - /etc/silo/silod.env must exist on target with credentials filled in # - PostgreSQL reachable from target (set SILO_DB_HOST to override) -# - MinIO reachable from target (set SILO_MINIO_HOST to override) # # Environment variables: # SILO_DEPLOY_TARGET - target host (default: silo.example.internal) diff --git a/scripts/migrate-storage.sh b/scripts/migrate-storage.sh deleted file mode 100755 index e1484fa..0000000 --- a/scripts/migrate-storage.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash -# Migrate storage from MinIO to filesystem on a remote Silo host. -# -# Builds the migrate-storage binary locally, uploads it to the target host, -# then runs it over SSH using credentials from /etc/silo/silod.env. -# -# Usage: ./scripts/migrate-storage.sh [flags...] -# -# Examples: -# ./scripts/migrate-storage.sh silo.kindred.internal psql.kindred.internal minio.kindred.internal -dry-run -verbose -# ./scripts/migrate-storage.sh silo.kindred.internal psql.kindred.internal minio.kindred.internal - -set -euo pipefail - -if [ $# -lt 3 ]; then - echo "Usage: $0 [flags...]" - echo " flags are passed to migrate-storage (e.g. -dry-run -verbose)" - exit 1 -fi - -TARGET="$1" -DB_HOST="$2" -MINIO_HOST="$3" -shift 3 -EXTRA_FLAGS="$*" - -DEST_DIR="/opt/silo/data" -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_DIR="${SCRIPT_DIR}/.." - -echo "=== Migrate Storage: MinIO -> Filesystem ===" -echo " Target: ${TARGET}" -echo " DB host: ${DB_HOST}" -echo " MinIO: ${MINIO_HOST}" -echo " Dest: ${DEST_DIR}" -[ -n "$EXTRA_FLAGS" ] && echo " Flags: ${EXTRA_FLAGS}" -echo "" - -# --- Build locally --- -echo "[1/3] Building migrate-storage binary..." -cd "$PROJECT_DIR" -GOOS=linux GOARCH=amd64 go build -o migrate-storage ./cmd/migrate-storage -echo " Built: $(du -h migrate-storage | cut -f1)" - -# --- Upload --- -echo "[2/3] Uploading to ${TARGET}..." -scp migrate-storage "${TARGET}:/tmp/migrate-storage" -rm -f migrate-storage - -# --- Run remotely --- -echo "[3/3] Running migration on ${TARGET}..." -ssh "$TARGET" DB_HOST="$DB_HOST" MINIO_HOST="$MINIO_HOST" DEST_DIR="$DEST_DIR" EXTRA_FLAGS="$EXTRA_FLAGS" bash -s <<'REMOTE' -set -euo pipefail - -CONFIG_DIR="/etc/silo" - -# Source credentials -if [ ! -f "$CONFIG_DIR/silod.env" ]; then - echo "ERROR: $CONFIG_DIR/silod.env not found on $(hostname)" - exit 1 -fi -set -a -source "$CONFIG_DIR/silod.env" -set +a - -# Ensure destination directory exists -sudo mkdir -p "$DEST_DIR" -sudo chown silo:silo "$DEST_DIR" 2>/dev/null || true - -chmod +x /tmp/migrate-storage - -# Write temporary config with the provided hosts -cat > /tmp/silo-migrate.yaml < "${OUTPUT_DIR}/.env" << EOF # PostgreSQL POSTGRES_PASSWORD=${POSTGRES_PASSWORD} -# MinIO -MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY} -MINIO_SECRET_KEY=${MINIO_SECRET_KEY} - # OpenLDAP LDAP_ADMIN_PASSWORD=${LDAP_ADMIN_PASSWORD} LDAP_USERS=${LDAP_USERS} @@ -235,12 +225,9 @@ database: max_connections: 10 storage: - endpoint: "minio:9000" - access_key: "${SILO_MINIO_ACCESS_KEY}" - secret_key: "${SILO_MINIO_SECRET_KEY}" - bucket: "silo-files" - use_ssl: false - region: "us-east-1" + backend: "filesystem" + filesystem: + root_dir: "/var/lib/silo/data" schemas: directory: "/etc/silo/schemas" @@ -306,8 +293,6 @@ echo " deployments/config.docker.yaml - server configuration" echo "" echo -e "${BOLD}Credentials:${NC}" echo " PostgreSQL: silo / ${POSTGRES_PASSWORD}" -echo " MinIO: ${MINIO_ACCESS_KEY} / ${MINIO_SECRET_KEY}" -echo " MinIO Console: http://localhost:9001" echo " LDAP Admin: cn=admin,dc=silo,dc=local / ${LDAP_ADMIN_PASSWORD}" echo " LDAP User: ${LDAP_USERS} / ${LDAP_PASSWORDS}" echo " Silo Admin: ${SILO_ADMIN_USERNAME} / ${SILO_ADMIN_PASSWORD} (local fallback)" diff --git a/scripts/setup-host.sh b/scripts/setup-host.sh index 6a22baa..c20c054 100755 --- a/scripts/setup-host.sh +++ b/scripts/setup-host.sh @@ -30,7 +30,6 @@ INSTALL_DIR="/opt/silo" CONFIG_DIR="/etc/silo" GO_VERSION="1.24.0" DB_HOST="${SILO_DB_HOST:-psql.example.internal}" -MINIO_HOST="${SILO_MINIO_HOST:-minio.example.internal}" log_info() { echo -e "${BLUE}[INFO]${NC} $*"; } log_success() { echo -e "${GREEN}[OK]${NC} $*"; } @@ -165,11 +164,6 @@ if [[ ! -f "${ENV_FILE}" ]]; then # Database: silo, User: silo SILO_DB_PASSWORD= -# MinIO credentials (${MINIO_HOST}) -# User: silouser -SILO_MINIO_ACCESS_KEY=silouser -SILO_MINIO_SECRET_KEY= - # Authentication # Session secret (required when auth is enabled) SILO_SESSION_SECRET= @@ -225,10 +219,7 @@ echo "" echo "2. Verify database connectivity:" echo " psql -h ${DB_HOST} -U silo -d silo -c 'SELECT 1'" echo "" -echo "3. Verify MinIO connectivity:" -echo " curl -I http://${MINIO_HOST}:9000/minio/health/live" -echo "" -echo "4. Run the deployment:" +echo "3. Run the deployment:" echo " sudo ${INSTALL_DIR}/src/scripts/deploy.sh" echo "" echo "After deployment, manage the service with:"