Remove the MinIO/S3 storage backend entirely. The filesystem backend is fully implemented, already used in production, and a migrate-storage tool exists for any remaining MinIO deployments to migrate beforehand. Changes: - Delete MinIO client implementation (internal/storage/storage.go) - Delete migrate-storage tool (cmd/migrate-storage, scripts/migrate-storage.sh) - Remove MinIO service, volumes, and env vars from all Docker Compose files - Simplify StorageConfig: remove Endpoint, AccessKey, SecretKey, Bucket, UseSSL, Region fields; add SILO_STORAGE_ROOT_DIR env override - Change all SQL COALESCE defaults from 'minio' to 'filesystem' - Add migration 020 to update column defaults to 'filesystem' - Remove minio-go/v7 dependency (go mod tidy) - Update all config examples, setup scripts, docs, and tests
501 lines
15 KiB
Go
501 lines
15 KiB
Go
package api
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/go-chi/chi/v5"
|
|
"github.com/google/uuid"
|
|
"github.com/kindredsystems/silo/internal/db"
|
|
)
|
|
|
|
// presignUploadRequest is the request body for generating a presigned upload URL.
|
|
type presignUploadRequest struct {
|
|
Filename string `json:"filename"`
|
|
ContentType string `json:"content_type"`
|
|
Size int64 `json:"size"`
|
|
}
|
|
|
|
// HandlePresignUpload generates a presigned PUT URL for direct browser upload.
|
|
func (s *Server) HandlePresignUpload(w http.ResponseWriter, r *http.Request) {
|
|
if s.storage == nil {
|
|
writeError(w, http.StatusServiceUnavailable, "storage_unavailable", "File storage not configured")
|
|
return
|
|
}
|
|
|
|
var req presignUploadRequest
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid_json", err.Error())
|
|
return
|
|
}
|
|
|
|
if req.Filename == "" {
|
|
writeError(w, http.StatusBadRequest, "invalid_request", "Filename is required")
|
|
return
|
|
}
|
|
if req.Size > 500*1024*1024 {
|
|
writeError(w, http.StatusBadRequest, "invalid_request", "File size exceeds 500MB limit")
|
|
return
|
|
}
|
|
if req.ContentType == "" {
|
|
req.ContentType = "application/octet-stream"
|
|
}
|
|
|
|
// Generate a unique temp key
|
|
id := uuid.New().String()
|
|
objectKey := fmt.Sprintf("uploads/tmp/%s/%s", id, req.Filename)
|
|
|
|
expiry := 15 * time.Minute
|
|
u, err := s.storage.PresignPut(r.Context(), objectKey, expiry)
|
|
if err != nil {
|
|
s.logger.Error().Err(err).Msg("failed to generate presigned URL")
|
|
writeError(w, http.StatusInternalServerError, "presign_failed", "Failed to generate upload URL")
|
|
return
|
|
}
|
|
|
|
writeJSON(w, http.StatusOK, map[string]any{
|
|
"object_key": objectKey,
|
|
"upload_url": u.String(),
|
|
"expires_at": time.Now().Add(expiry).Format(time.RFC3339),
|
|
})
|
|
}
|
|
|
|
// itemFileResponse is the JSON representation of an item file attachment.
|
|
type itemFileResponse struct {
|
|
ID string `json:"id"`
|
|
Filename string `json:"filename"`
|
|
ContentType string `json:"content_type"`
|
|
Size int64 `json:"size"`
|
|
ObjectKey string `json:"object_key"`
|
|
CreatedAt string `json:"created_at"`
|
|
}
|
|
|
|
func itemFileToResponse(f *db.ItemFile) itemFileResponse {
|
|
return itemFileResponse{
|
|
ID: f.ID,
|
|
Filename: f.Filename,
|
|
ContentType: f.ContentType,
|
|
Size: f.Size,
|
|
ObjectKey: f.ObjectKey,
|
|
CreatedAt: f.CreatedAt.Format(time.RFC3339),
|
|
}
|
|
}
|
|
|
|
// HandleListItemFiles lists file attachments for an item.
|
|
func (s *Server) HandleListItemFiles(w http.ResponseWriter, r *http.Request) {
|
|
ctx := r.Context()
|
|
partNumber := chi.URLParam(r, "partNumber")
|
|
|
|
item, err := s.items.GetByPartNumber(ctx, partNumber)
|
|
if err != nil {
|
|
s.logger.Error().Err(err).Msg("failed to get item")
|
|
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get item")
|
|
return
|
|
}
|
|
if item == nil {
|
|
writeError(w, http.StatusNotFound, "not_found", "Item not found")
|
|
return
|
|
}
|
|
|
|
files, err := s.itemFiles.ListByItem(ctx, item.ID)
|
|
if err != nil {
|
|
s.logger.Error().Err(err).Msg("failed to list item files")
|
|
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to list files")
|
|
return
|
|
}
|
|
|
|
result := make([]itemFileResponse, 0, len(files))
|
|
for _, f := range files {
|
|
result = append(result, itemFileToResponse(f))
|
|
}
|
|
|
|
writeJSON(w, http.StatusOK, result)
|
|
}
|
|
|
|
// associateFileRequest is the request body for associating an uploaded file with an item.
|
|
type associateFileRequest struct {
|
|
ObjectKey string `json:"object_key"`
|
|
Filename string `json:"filename"`
|
|
ContentType string `json:"content_type"`
|
|
Size int64 `json:"size"`
|
|
}
|
|
|
|
// HandleAssociateItemFile moves a temp upload to permanent storage and creates a DB record.
|
|
func (s *Server) HandleAssociateItemFile(w http.ResponseWriter, r *http.Request) {
|
|
ctx := r.Context()
|
|
partNumber := chi.URLParam(r, "partNumber")
|
|
|
|
if s.storage == nil {
|
|
writeError(w, http.StatusServiceUnavailable, "storage_unavailable", "File storage not configured")
|
|
return
|
|
}
|
|
|
|
item, err := s.items.GetByPartNumber(ctx, partNumber)
|
|
if err != nil {
|
|
s.logger.Error().Err(err).Msg("failed to get item")
|
|
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get item")
|
|
return
|
|
}
|
|
if item == nil {
|
|
writeError(w, http.StatusNotFound, "not_found", "Item not found")
|
|
return
|
|
}
|
|
|
|
var req associateFileRequest
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid_json", err.Error())
|
|
return
|
|
}
|
|
|
|
if req.ObjectKey == "" || req.Filename == "" {
|
|
writeError(w, http.StatusBadRequest, "invalid_request", "object_key and filename are required")
|
|
return
|
|
}
|
|
|
|
// Security: only allow associating files from the temp upload prefix
|
|
if !strings.HasPrefix(req.ObjectKey, "uploads/tmp/") {
|
|
writeError(w, http.StatusBadRequest, "invalid_request", "object_key must be a temp upload")
|
|
return
|
|
}
|
|
|
|
if req.ContentType == "" {
|
|
req.ContentType = "application/octet-stream"
|
|
}
|
|
|
|
// Create DB record first to get the file ID
|
|
itemFile := &db.ItemFile{
|
|
ItemID: item.ID,
|
|
Filename: req.Filename,
|
|
ContentType: req.ContentType,
|
|
Size: req.Size,
|
|
ObjectKey: "", // will be set after copy
|
|
}
|
|
|
|
// Generate permanent key
|
|
fileID := uuid.New().String()
|
|
permanentKey := fmt.Sprintf("items/%s/files/%s/%s", item.ID, fileID, req.Filename)
|
|
|
|
// Copy from temp to permanent location
|
|
if err := s.storage.Copy(ctx, req.ObjectKey, permanentKey); err != nil {
|
|
s.logger.Error().Err(err).Str("src", req.ObjectKey).Str("dst", permanentKey).Msg("failed to copy file")
|
|
writeError(w, http.StatusInternalServerError, "copy_failed", "Failed to move file to permanent storage")
|
|
return
|
|
}
|
|
|
|
// Delete the temp object (best-effort)
|
|
_ = s.storage.Delete(ctx, req.ObjectKey)
|
|
|
|
// Save DB record with permanent key
|
|
itemFile.ObjectKey = permanentKey
|
|
if err := s.itemFiles.Create(ctx, itemFile); err != nil {
|
|
s.logger.Error().Err(err).Msg("failed to create item file record")
|
|
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to save file record")
|
|
return
|
|
}
|
|
|
|
s.logger.Info().
|
|
Str("part_number", partNumber).
|
|
Str("file_id", itemFile.ID).
|
|
Str("filename", req.Filename).
|
|
Int64("size", req.Size).
|
|
Msg("file associated with item")
|
|
|
|
writeJSON(w, http.StatusCreated, itemFileToResponse(itemFile))
|
|
}
|
|
|
|
// HandleDeleteItemFile deletes a file attachment and its storage object.
|
|
func (s *Server) HandleDeleteItemFile(w http.ResponseWriter, r *http.Request) {
|
|
ctx := r.Context()
|
|
partNumber := chi.URLParam(r, "partNumber")
|
|
fileID := chi.URLParam(r, "fileId")
|
|
|
|
item, err := s.items.GetByPartNumber(ctx, partNumber)
|
|
if err != nil {
|
|
s.logger.Error().Err(err).Msg("failed to get item")
|
|
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get item")
|
|
return
|
|
}
|
|
if item == nil {
|
|
writeError(w, http.StatusNotFound, "not_found", "Item not found")
|
|
return
|
|
}
|
|
|
|
// Get the file record to find the storage key
|
|
file, err := s.itemFiles.Get(ctx, fileID)
|
|
if err != nil {
|
|
writeError(w, http.StatusNotFound, "not_found", "File not found")
|
|
return
|
|
}
|
|
|
|
// Verify the file belongs to this item
|
|
if file.ItemID != item.ID {
|
|
writeError(w, http.StatusNotFound, "not_found", "File not found")
|
|
return
|
|
}
|
|
|
|
// Delete from storage (best-effort)
|
|
if s.storage != nil {
|
|
_ = s.storage.Delete(ctx, file.ObjectKey)
|
|
}
|
|
|
|
// Delete DB record
|
|
if err := s.itemFiles.Delete(ctx, fileID); err != nil {
|
|
s.logger.Error().Err(err).Msg("failed to delete item file record")
|
|
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to delete file")
|
|
return
|
|
}
|
|
|
|
w.WriteHeader(http.StatusNoContent)
|
|
}
|
|
|
|
// setThumbnailRequest is the request body for setting an item thumbnail.
|
|
type setThumbnailRequest struct {
|
|
ObjectKey string `json:"object_key"`
|
|
}
|
|
|
|
// HandleSetItemThumbnail copies a temp upload to the item thumbnail location.
|
|
func (s *Server) HandleSetItemThumbnail(w http.ResponseWriter, r *http.Request) {
|
|
ctx := r.Context()
|
|
partNumber := chi.URLParam(r, "partNumber")
|
|
|
|
if s.storage == nil {
|
|
writeError(w, http.StatusServiceUnavailable, "storage_unavailable", "File storage not configured")
|
|
return
|
|
}
|
|
|
|
item, err := s.items.GetByPartNumber(ctx, partNumber)
|
|
if err != nil {
|
|
s.logger.Error().Err(err).Msg("failed to get item")
|
|
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get item")
|
|
return
|
|
}
|
|
if item == nil {
|
|
writeError(w, http.StatusNotFound, "not_found", "Item not found")
|
|
return
|
|
}
|
|
|
|
var req setThumbnailRequest
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid_json", err.Error())
|
|
return
|
|
}
|
|
|
|
if req.ObjectKey == "" {
|
|
writeError(w, http.StatusBadRequest, "invalid_request", "object_key is required")
|
|
return
|
|
}
|
|
|
|
// Security: only allow from temp upload prefix
|
|
if !strings.HasPrefix(req.ObjectKey, "uploads/tmp/") {
|
|
writeError(w, http.StatusBadRequest, "invalid_request", "object_key must be a temp upload")
|
|
return
|
|
}
|
|
|
|
// Copy to permanent thumbnail location
|
|
thumbnailKey := fmt.Sprintf("items/%s/thumbnail.png", item.ID)
|
|
if err := s.storage.Copy(ctx, req.ObjectKey, thumbnailKey); err != nil {
|
|
s.logger.Error().Err(err).Msg("failed to copy thumbnail")
|
|
writeError(w, http.StatusInternalServerError, "copy_failed", "Failed to set thumbnail")
|
|
return
|
|
}
|
|
|
|
// Delete temp object (best-effort)
|
|
_ = s.storage.Delete(ctx, req.ObjectKey)
|
|
|
|
// Update DB
|
|
if err := s.items.SetThumbnailKey(ctx, item.ID, thumbnailKey); err != nil {
|
|
s.logger.Error().Err(err).Msg("failed to update thumbnail key")
|
|
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to save thumbnail")
|
|
return
|
|
}
|
|
|
|
w.WriteHeader(http.StatusNoContent)
|
|
}
|
|
|
|
// storageBackend returns the configured storage backend name.
|
|
func (s *Server) storageBackend() string {
|
|
return "filesystem"
|
|
}
|
|
|
|
// HandleUploadItemFile accepts a multipart file upload and stores it as an item attachment.
|
|
func (s *Server) HandleUploadItemFile(w http.ResponseWriter, r *http.Request) {
|
|
ctx := r.Context()
|
|
partNumber := chi.URLParam(r, "partNumber")
|
|
|
|
if s.storage == nil {
|
|
writeError(w, http.StatusServiceUnavailable, "storage_unavailable", "File storage not configured")
|
|
return
|
|
}
|
|
|
|
item, err := s.items.GetByPartNumber(ctx, partNumber)
|
|
if err != nil {
|
|
s.logger.Error().Err(err).Msg("failed to get item")
|
|
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get item")
|
|
return
|
|
}
|
|
if item == nil {
|
|
writeError(w, http.StatusNotFound, "not_found", "Item not found")
|
|
return
|
|
}
|
|
|
|
// Parse multipart form (max 500MB)
|
|
if err := r.ParseMultipartForm(500 << 20); err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid_form", err.Error())
|
|
return
|
|
}
|
|
|
|
file, header, err := r.FormFile("file")
|
|
if err != nil {
|
|
writeError(w, http.StatusBadRequest, "missing_file", "File is required")
|
|
return
|
|
}
|
|
defer file.Close()
|
|
|
|
contentType := header.Header.Get("Content-Type")
|
|
if contentType == "" {
|
|
contentType = "application/octet-stream"
|
|
}
|
|
|
|
// Generate permanent key
|
|
fileID := uuid.New().String()
|
|
permanentKey := fmt.Sprintf("items/%s/files/%s/%s", item.ID, fileID, header.Filename)
|
|
|
|
// Write directly to storage
|
|
result, err := s.storage.Put(ctx, permanentKey, file, header.Size, contentType)
|
|
if err != nil {
|
|
s.logger.Error().Err(err).Msg("failed to upload file")
|
|
writeError(w, http.StatusInternalServerError, "upload_failed", "Failed to store file")
|
|
return
|
|
}
|
|
|
|
// Create DB record
|
|
itemFile := &db.ItemFile{
|
|
ItemID: item.ID,
|
|
Filename: header.Filename,
|
|
ContentType: contentType,
|
|
Size: result.Size,
|
|
ObjectKey: permanentKey,
|
|
StorageBackend: s.storageBackend(),
|
|
}
|
|
if err := s.itemFiles.Create(ctx, itemFile); err != nil {
|
|
s.logger.Error().Err(err).Msg("failed to create item file record")
|
|
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to save file record")
|
|
return
|
|
}
|
|
|
|
s.logger.Info().
|
|
Str("part_number", partNumber).
|
|
Str("file_id", itemFile.ID).
|
|
Str("filename", header.Filename).
|
|
Int64("size", result.Size).
|
|
Msg("file uploaded to item")
|
|
|
|
writeJSON(w, http.StatusCreated, itemFileToResponse(itemFile))
|
|
}
|
|
|
|
// HandleUploadItemThumbnail accepts a multipart file upload and sets it as the item thumbnail.
|
|
func (s *Server) HandleUploadItemThumbnail(w http.ResponseWriter, r *http.Request) {
|
|
ctx := r.Context()
|
|
partNumber := chi.URLParam(r, "partNumber")
|
|
|
|
if s.storage == nil {
|
|
writeError(w, http.StatusServiceUnavailable, "storage_unavailable", "File storage not configured")
|
|
return
|
|
}
|
|
|
|
item, err := s.items.GetByPartNumber(ctx, partNumber)
|
|
if err != nil {
|
|
s.logger.Error().Err(err).Msg("failed to get item")
|
|
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get item")
|
|
return
|
|
}
|
|
if item == nil {
|
|
writeError(w, http.StatusNotFound, "not_found", "Item not found")
|
|
return
|
|
}
|
|
|
|
// Parse multipart form (max 10MB for thumbnails)
|
|
if err := r.ParseMultipartForm(10 << 20); err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid_form", err.Error())
|
|
return
|
|
}
|
|
|
|
file, header, err := r.FormFile("file")
|
|
if err != nil {
|
|
writeError(w, http.StatusBadRequest, "missing_file", "File is required")
|
|
return
|
|
}
|
|
defer file.Close()
|
|
|
|
contentType := header.Header.Get("Content-Type")
|
|
if contentType == "" {
|
|
contentType = "image/png"
|
|
}
|
|
|
|
thumbnailKey := fmt.Sprintf("items/%s/thumbnail.png", item.ID)
|
|
|
|
if _, err := s.storage.Put(ctx, thumbnailKey, file, header.Size, contentType); err != nil {
|
|
s.logger.Error().Err(err).Msg("failed to upload thumbnail")
|
|
writeError(w, http.StatusInternalServerError, "upload_failed", "Failed to store thumbnail")
|
|
return
|
|
}
|
|
|
|
if err := s.items.SetThumbnailKey(ctx, item.ID, thumbnailKey); err != nil {
|
|
s.logger.Error().Err(err).Msg("failed to update thumbnail key")
|
|
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to save thumbnail")
|
|
return
|
|
}
|
|
|
|
w.WriteHeader(http.StatusNoContent)
|
|
}
|
|
|
|
// HandleDownloadItemFile streams an item file attachment to the client.
|
|
func (s *Server) HandleDownloadItemFile(w http.ResponseWriter, r *http.Request) {
|
|
ctx := r.Context()
|
|
partNumber := chi.URLParam(r, "partNumber")
|
|
fileID := chi.URLParam(r, "fileId")
|
|
|
|
if s.storage == nil {
|
|
writeError(w, http.StatusServiceUnavailable, "storage_unavailable", "File storage not configured")
|
|
return
|
|
}
|
|
|
|
item, err := s.items.GetByPartNumber(ctx, partNumber)
|
|
if err != nil || item == nil {
|
|
writeError(w, http.StatusNotFound, "not_found", "Item not found")
|
|
return
|
|
}
|
|
|
|
file, err := s.itemFiles.Get(ctx, fileID)
|
|
if err != nil {
|
|
writeError(w, http.StatusNotFound, "not_found", "File not found")
|
|
return
|
|
}
|
|
|
|
if file.ItemID != item.ID {
|
|
writeError(w, http.StatusNotFound, "not_found", "File not found")
|
|
return
|
|
}
|
|
|
|
reader, err := s.storage.Get(ctx, file.ObjectKey)
|
|
if err != nil {
|
|
s.logger.Error().Err(err).Str("key", file.ObjectKey).Msg("failed to get file")
|
|
writeError(w, http.StatusInternalServerError, "download_failed", "Failed to retrieve file")
|
|
return
|
|
}
|
|
defer reader.Close()
|
|
|
|
w.Header().Set("Content-Type", file.ContentType)
|
|
w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, file.Filename))
|
|
if file.Size > 0 {
|
|
w.Header().Set("Content-Length", strconv.FormatInt(file.Size, 10))
|
|
}
|
|
|
|
io.Copy(w, reader)
|
|
}
|