Files
silo/internal/api/file_handlers.go
Forbes 88d1ab1f97 refactor(storage): remove MinIO backend, filesystem-only storage
Remove the MinIO/S3 storage backend entirely. The filesystem backend is
fully implemented, already used in production, and a migrate-storage tool
exists for any remaining MinIO deployments to migrate beforehand.

Changes:
- Delete MinIO client implementation (internal/storage/storage.go)
- Delete migrate-storage tool (cmd/migrate-storage, scripts/migrate-storage.sh)
- Remove MinIO service, volumes, and env vars from all Docker Compose files
- Simplify StorageConfig: remove Endpoint, AccessKey, SecretKey, Bucket,
  UseSSL, Region fields; add SILO_STORAGE_ROOT_DIR env override
- Change all SQL COALESCE defaults from 'minio' to 'filesystem'
- Add migration 020 to update column defaults to 'filesystem'
- Remove minio-go/v7 dependency (go mod tidy)
- Update all config examples, setup scripts, docs, and tests
2026-02-19 14:36:22 -06:00

501 lines
15 KiB
Go

package api
import (
"encoding/json"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"time"
"github.com/go-chi/chi/v5"
"github.com/google/uuid"
"github.com/kindredsystems/silo/internal/db"
)
// presignUploadRequest is the request body for generating a presigned upload URL.
type presignUploadRequest struct {
Filename string `json:"filename"`
ContentType string `json:"content_type"`
Size int64 `json:"size"`
}
// HandlePresignUpload generates a presigned PUT URL for direct browser upload.
func (s *Server) HandlePresignUpload(w http.ResponseWriter, r *http.Request) {
if s.storage == nil {
writeError(w, http.StatusServiceUnavailable, "storage_unavailable", "File storage not configured")
return
}
var req presignUploadRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid_json", err.Error())
return
}
if req.Filename == "" {
writeError(w, http.StatusBadRequest, "invalid_request", "Filename is required")
return
}
if req.Size > 500*1024*1024 {
writeError(w, http.StatusBadRequest, "invalid_request", "File size exceeds 500MB limit")
return
}
if req.ContentType == "" {
req.ContentType = "application/octet-stream"
}
// Generate a unique temp key
id := uuid.New().String()
objectKey := fmt.Sprintf("uploads/tmp/%s/%s", id, req.Filename)
expiry := 15 * time.Minute
u, err := s.storage.PresignPut(r.Context(), objectKey, expiry)
if err != nil {
s.logger.Error().Err(err).Msg("failed to generate presigned URL")
writeError(w, http.StatusInternalServerError, "presign_failed", "Failed to generate upload URL")
return
}
writeJSON(w, http.StatusOK, map[string]any{
"object_key": objectKey,
"upload_url": u.String(),
"expires_at": time.Now().Add(expiry).Format(time.RFC3339),
})
}
// itemFileResponse is the JSON representation of an item file attachment.
type itemFileResponse struct {
ID string `json:"id"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
Size int64 `json:"size"`
ObjectKey string `json:"object_key"`
CreatedAt string `json:"created_at"`
}
func itemFileToResponse(f *db.ItemFile) itemFileResponse {
return itemFileResponse{
ID: f.ID,
Filename: f.Filename,
ContentType: f.ContentType,
Size: f.Size,
ObjectKey: f.ObjectKey,
CreatedAt: f.CreatedAt.Format(time.RFC3339),
}
}
// HandleListItemFiles lists file attachments for an item.
func (s *Server) HandleListItemFiles(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
partNumber := chi.URLParam(r, "partNumber")
item, err := s.items.GetByPartNumber(ctx, partNumber)
if err != nil {
s.logger.Error().Err(err).Msg("failed to get item")
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get item")
return
}
if item == nil {
writeError(w, http.StatusNotFound, "not_found", "Item not found")
return
}
files, err := s.itemFiles.ListByItem(ctx, item.ID)
if err != nil {
s.logger.Error().Err(err).Msg("failed to list item files")
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to list files")
return
}
result := make([]itemFileResponse, 0, len(files))
for _, f := range files {
result = append(result, itemFileToResponse(f))
}
writeJSON(w, http.StatusOK, result)
}
// associateFileRequest is the request body for associating an uploaded file with an item.
type associateFileRequest struct {
ObjectKey string `json:"object_key"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
Size int64 `json:"size"`
}
// HandleAssociateItemFile moves a temp upload to permanent storage and creates a DB record.
func (s *Server) HandleAssociateItemFile(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
partNumber := chi.URLParam(r, "partNumber")
if s.storage == nil {
writeError(w, http.StatusServiceUnavailable, "storage_unavailable", "File storage not configured")
return
}
item, err := s.items.GetByPartNumber(ctx, partNumber)
if err != nil {
s.logger.Error().Err(err).Msg("failed to get item")
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get item")
return
}
if item == nil {
writeError(w, http.StatusNotFound, "not_found", "Item not found")
return
}
var req associateFileRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid_json", err.Error())
return
}
if req.ObjectKey == "" || req.Filename == "" {
writeError(w, http.StatusBadRequest, "invalid_request", "object_key and filename are required")
return
}
// Security: only allow associating files from the temp upload prefix
if !strings.HasPrefix(req.ObjectKey, "uploads/tmp/") {
writeError(w, http.StatusBadRequest, "invalid_request", "object_key must be a temp upload")
return
}
if req.ContentType == "" {
req.ContentType = "application/octet-stream"
}
// Create DB record first to get the file ID
itemFile := &db.ItemFile{
ItemID: item.ID,
Filename: req.Filename,
ContentType: req.ContentType,
Size: req.Size,
ObjectKey: "", // will be set after copy
}
// Generate permanent key
fileID := uuid.New().String()
permanentKey := fmt.Sprintf("items/%s/files/%s/%s", item.ID, fileID, req.Filename)
// Copy from temp to permanent location
if err := s.storage.Copy(ctx, req.ObjectKey, permanentKey); err != nil {
s.logger.Error().Err(err).Str("src", req.ObjectKey).Str("dst", permanentKey).Msg("failed to copy file")
writeError(w, http.StatusInternalServerError, "copy_failed", "Failed to move file to permanent storage")
return
}
// Delete the temp object (best-effort)
_ = s.storage.Delete(ctx, req.ObjectKey)
// Save DB record with permanent key
itemFile.ObjectKey = permanentKey
if err := s.itemFiles.Create(ctx, itemFile); err != nil {
s.logger.Error().Err(err).Msg("failed to create item file record")
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to save file record")
return
}
s.logger.Info().
Str("part_number", partNumber).
Str("file_id", itemFile.ID).
Str("filename", req.Filename).
Int64("size", req.Size).
Msg("file associated with item")
writeJSON(w, http.StatusCreated, itemFileToResponse(itemFile))
}
// HandleDeleteItemFile deletes a file attachment and its storage object.
func (s *Server) HandleDeleteItemFile(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
partNumber := chi.URLParam(r, "partNumber")
fileID := chi.URLParam(r, "fileId")
item, err := s.items.GetByPartNumber(ctx, partNumber)
if err != nil {
s.logger.Error().Err(err).Msg("failed to get item")
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get item")
return
}
if item == nil {
writeError(w, http.StatusNotFound, "not_found", "Item not found")
return
}
// Get the file record to find the storage key
file, err := s.itemFiles.Get(ctx, fileID)
if err != nil {
writeError(w, http.StatusNotFound, "not_found", "File not found")
return
}
// Verify the file belongs to this item
if file.ItemID != item.ID {
writeError(w, http.StatusNotFound, "not_found", "File not found")
return
}
// Delete from storage (best-effort)
if s.storage != nil {
_ = s.storage.Delete(ctx, file.ObjectKey)
}
// Delete DB record
if err := s.itemFiles.Delete(ctx, fileID); err != nil {
s.logger.Error().Err(err).Msg("failed to delete item file record")
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to delete file")
return
}
w.WriteHeader(http.StatusNoContent)
}
// setThumbnailRequest is the request body for setting an item thumbnail.
type setThumbnailRequest struct {
ObjectKey string `json:"object_key"`
}
// HandleSetItemThumbnail copies a temp upload to the item thumbnail location.
func (s *Server) HandleSetItemThumbnail(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
partNumber := chi.URLParam(r, "partNumber")
if s.storage == nil {
writeError(w, http.StatusServiceUnavailable, "storage_unavailable", "File storage not configured")
return
}
item, err := s.items.GetByPartNumber(ctx, partNumber)
if err != nil {
s.logger.Error().Err(err).Msg("failed to get item")
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get item")
return
}
if item == nil {
writeError(w, http.StatusNotFound, "not_found", "Item not found")
return
}
var req setThumbnailRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid_json", err.Error())
return
}
if req.ObjectKey == "" {
writeError(w, http.StatusBadRequest, "invalid_request", "object_key is required")
return
}
// Security: only allow from temp upload prefix
if !strings.HasPrefix(req.ObjectKey, "uploads/tmp/") {
writeError(w, http.StatusBadRequest, "invalid_request", "object_key must be a temp upload")
return
}
// Copy to permanent thumbnail location
thumbnailKey := fmt.Sprintf("items/%s/thumbnail.png", item.ID)
if err := s.storage.Copy(ctx, req.ObjectKey, thumbnailKey); err != nil {
s.logger.Error().Err(err).Msg("failed to copy thumbnail")
writeError(w, http.StatusInternalServerError, "copy_failed", "Failed to set thumbnail")
return
}
// Delete temp object (best-effort)
_ = s.storage.Delete(ctx, req.ObjectKey)
// Update DB
if err := s.items.SetThumbnailKey(ctx, item.ID, thumbnailKey); err != nil {
s.logger.Error().Err(err).Msg("failed to update thumbnail key")
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to save thumbnail")
return
}
w.WriteHeader(http.StatusNoContent)
}
// storageBackend returns the configured storage backend name.
func (s *Server) storageBackend() string {
return "filesystem"
}
// HandleUploadItemFile accepts a multipart file upload and stores it as an item attachment.
func (s *Server) HandleUploadItemFile(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
partNumber := chi.URLParam(r, "partNumber")
if s.storage == nil {
writeError(w, http.StatusServiceUnavailable, "storage_unavailable", "File storage not configured")
return
}
item, err := s.items.GetByPartNumber(ctx, partNumber)
if err != nil {
s.logger.Error().Err(err).Msg("failed to get item")
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get item")
return
}
if item == nil {
writeError(w, http.StatusNotFound, "not_found", "Item not found")
return
}
// Parse multipart form (max 500MB)
if err := r.ParseMultipartForm(500 << 20); err != nil {
writeError(w, http.StatusBadRequest, "invalid_form", err.Error())
return
}
file, header, err := r.FormFile("file")
if err != nil {
writeError(w, http.StatusBadRequest, "missing_file", "File is required")
return
}
defer file.Close()
contentType := header.Header.Get("Content-Type")
if contentType == "" {
contentType = "application/octet-stream"
}
// Generate permanent key
fileID := uuid.New().String()
permanentKey := fmt.Sprintf("items/%s/files/%s/%s", item.ID, fileID, header.Filename)
// Write directly to storage
result, err := s.storage.Put(ctx, permanentKey, file, header.Size, contentType)
if err != nil {
s.logger.Error().Err(err).Msg("failed to upload file")
writeError(w, http.StatusInternalServerError, "upload_failed", "Failed to store file")
return
}
// Create DB record
itemFile := &db.ItemFile{
ItemID: item.ID,
Filename: header.Filename,
ContentType: contentType,
Size: result.Size,
ObjectKey: permanentKey,
StorageBackend: s.storageBackend(),
}
if err := s.itemFiles.Create(ctx, itemFile); err != nil {
s.logger.Error().Err(err).Msg("failed to create item file record")
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to save file record")
return
}
s.logger.Info().
Str("part_number", partNumber).
Str("file_id", itemFile.ID).
Str("filename", header.Filename).
Int64("size", result.Size).
Msg("file uploaded to item")
writeJSON(w, http.StatusCreated, itemFileToResponse(itemFile))
}
// HandleUploadItemThumbnail accepts a multipart file upload and sets it as the item thumbnail.
func (s *Server) HandleUploadItemThumbnail(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
partNumber := chi.URLParam(r, "partNumber")
if s.storage == nil {
writeError(w, http.StatusServiceUnavailable, "storage_unavailable", "File storage not configured")
return
}
item, err := s.items.GetByPartNumber(ctx, partNumber)
if err != nil {
s.logger.Error().Err(err).Msg("failed to get item")
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get item")
return
}
if item == nil {
writeError(w, http.StatusNotFound, "not_found", "Item not found")
return
}
// Parse multipart form (max 10MB for thumbnails)
if err := r.ParseMultipartForm(10 << 20); err != nil {
writeError(w, http.StatusBadRequest, "invalid_form", err.Error())
return
}
file, header, err := r.FormFile("file")
if err != nil {
writeError(w, http.StatusBadRequest, "missing_file", "File is required")
return
}
defer file.Close()
contentType := header.Header.Get("Content-Type")
if contentType == "" {
contentType = "image/png"
}
thumbnailKey := fmt.Sprintf("items/%s/thumbnail.png", item.ID)
if _, err := s.storage.Put(ctx, thumbnailKey, file, header.Size, contentType); err != nil {
s.logger.Error().Err(err).Msg("failed to upload thumbnail")
writeError(w, http.StatusInternalServerError, "upload_failed", "Failed to store thumbnail")
return
}
if err := s.items.SetThumbnailKey(ctx, item.ID, thumbnailKey); err != nil {
s.logger.Error().Err(err).Msg("failed to update thumbnail key")
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to save thumbnail")
return
}
w.WriteHeader(http.StatusNoContent)
}
// HandleDownloadItemFile streams an item file attachment to the client.
func (s *Server) HandleDownloadItemFile(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
partNumber := chi.URLParam(r, "partNumber")
fileID := chi.URLParam(r, "fileId")
if s.storage == nil {
writeError(w, http.StatusServiceUnavailable, "storage_unavailable", "File storage not configured")
return
}
item, err := s.items.GetByPartNumber(ctx, partNumber)
if err != nil || item == nil {
writeError(w, http.StatusNotFound, "not_found", "Item not found")
return
}
file, err := s.itemFiles.Get(ctx, fileID)
if err != nil {
writeError(w, http.StatusNotFound, "not_found", "File not found")
return
}
if file.ItemID != item.ID {
writeError(w, http.StatusNotFound, "not_found", "File not found")
return
}
reader, err := s.storage.Get(ctx, file.ObjectKey)
if err != nil {
s.logger.Error().Err(err).Str("key", file.ObjectKey).Msg("failed to get file")
writeError(w, http.StatusInternalServerError, "download_failed", "Failed to retrieve file")
return
}
defer reader.Close()
w.Header().Set("Content-Type", file.ContentType)
w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, file.Filename))
if file.Size > 0 {
w.Header().Set("Content-Length", strconv.FormatInt(file.Size, 10))
}
io.Copy(w, reader)
}