kindred/silo
1
1
Fork 0
Code Issues 30 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(kc): checkout packing + ETag caching (Phase 2)

Browse Source 
Implements issue #142 — .kc checkout pipeline that repacks silo/ entries
with current DB state before serving downloads.

When a client downloads a .kc file via GET /api/items/{pn}/file/{rev},
the server now:
1. Reads the file from storage into memory
2. Checks for silo/ directory (plain .fcstd files bypass packing)
3. Repacks silo/ entries with current item_metadata + revision history
4. Streams the repacked ZIP to the client

New files:
- internal/kc/pack.go: Pack() replaces silo/ entries in ZIP, preserving
  all non-silo entries (FreeCAD files, thumbnails) with original
  compression and timestamps. HasSiloDir() for lightweight detection.
- internal/api/pack_handlers.go: packKCFile server helper, computeETag,
  canSkipRepack lazy optimization.

ETag caching:
- ETag computed from revision_number + metadata.updated_at
- If-None-Match support returns 304 Not Modified before reading storage
- Cache-Control: private, must-revalidate

Lazy packing optimization:
- Skips repack if revision_hash matches and metadata unchanged since upload

Phase 2 packs: manifest.json, metadata.json, history.json,
dependencies.json (empty []). Approvals, macros, jobs deferred to
Phase 3-5.

Closes #142
This commit is contained in:
Forbes
2026-02-18 17:01:26 -06:00
parent dd010331c0
commit c216d64702
5 changed files with 517 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
internal/api/handlers.go
View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
@@ -19,6 +20,7 @@ import (
"github.com/kindredsystems/silo/internal/config"
"github.com/kindredsystems/silo/internal/db"
"github.com/kindredsystems/silo/internal/jobdef"
"github.com/kindredsystems/silo/internal/kc"
"github.com/kindredsystems/silo/internal/modules"
"github.com/kindredsystems/silo/internal/partnum"
"github.com/kindredsystems/silo/internal/schema"
@@ -1662,6 +1664,7 @@ func (s *Server) HandleUploadFile(w http.ResponseWriter, r *http.Request) {
}
// HandleDownloadFile downloads the file for a specific revision.
// For .kc files, silo/ entries are repacked with current DB state.
func (s *Server) HandleDownloadFile(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
partNumber := chi.URLParam(r, "partNumber")
@@ -1716,18 +1719,23 @@ func (s *Server) HandleDownloadFile(w http.ResponseWriter, r *http.Request) {
return
}
// Get file from storage
var reader interface {
Read(p []byte) (n int, err error)
Close() error
// ETag: computed from revision + metadata freshness.
meta, _ := s.metadata.Get(ctx, item.ID) // nil is ok (plain .fcstd)
etag := computeETag(revision, meta)
if match := r.Header.Get("If-None-Match"); match == etag {
w.Header().Set("ETag", etag)
w.WriteHeader(http.StatusNotModified)
return
}
// Get file from storage
var reader io.ReadCloser
if revision.FileVersion != nil && *revision.FileVersion != "" {
reader, err = s.storage.GetVersion(ctx, *revision.FileKey, *revision.FileVersion)
} else {
reader, err = s.storage.Get(ctx, *revision.FileKey)
}
if err != nil {
s.logger.Error().Err(err).Str("key", *revision.FileKey).Msg("failed to get file")
writeError(w, http.StatusInternalServerError, "download_failed", err.Error())
@@ -1735,28 +1743,37 @@ func (s *Server) HandleDownloadFile(w http.ResponseWriter, r *http.Request) {
}
defer reader.Close()
// Read entire file for potential .kc repacking.
data, err := io.ReadAll(reader)
if err != nil {
s.logger.Error().Err(err).Msg("failed to read file")
writeError(w, http.StatusInternalServerError, "download_failed", "Failed to read file")
return
}
// Repack silo/ entries for .kc files with indexed metadata.
output := data
if meta != nil {
if hasSilo, chkErr := kc.HasSiloDir(data); chkErr == nil && hasSilo {
if !canSkipRepack(revision, meta) {
if packed, packErr := s.packKCFile(ctx, data, item, revision, meta); packErr != nil {
s.logger.Warn().Err(packErr).Str("part_number", partNumber).Msg("kc: packing failed, serving original")
} else {
output = packed
}
}
}
}
// Set response headers
filename := partNumber + "_rev" + strconv.Itoa(revNum) + ".FCStd"
w.Header().Set("Content-Type", "application/octet-stream")
w.Header().Set("Content-Disposition", "attachment; filename=\""+filename+"\"")
if revision.FileSize != nil {
w.Header().Set("Content-Length", strconv.FormatInt(*revision.FileSize, 10))
}
w.Header().Set("Content-Length", strconv.Itoa(len(output)))
w.Header().Set("ETag", etag)
w.Header().Set("Cache-Control", "private, must-revalidate")
// Stream file to response
buf := make([]byte, 32*1024)
for {
n, readErr := reader.Read(buf)
if n > 0 {
if _, writeErr := w.Write(buf[:n]); writeErr != nil {
s.logger.Error().Err(writeErr).Msg("failed to write response")
return
}
}
if readErr != nil {
break
}
}
w.Write(output)
}
// HandleDownloadLatestFile downloads the file for the latest revision.