package api import ( "encoding/csv" "encoding/json" "fmt" "io" "net/http" "sort" "strconv" "strings" "time" "github.com/kindredsystems/silo/internal/db" "github.com/kindredsystems/silo/internal/partnum" ) // CSV Export/Import handlers for bulk data management // CSVExportOptions controls what fields to include in export. type CSVExportOptions struct { IncludeProperties bool IncludeRevisions bool } // CSVImportResult represents the result of an import operation. type CSVImportResult struct { TotalRows int `json:"total_rows"` SuccessCount int `json:"success_count"` ErrorCount int `json:"error_count"` Errors []CSVImportErr `json:"errors,omitempty"` CreatedItems []string `json:"created_items,omitempty"` } // CSVImportErr represents an error on a specific row. type CSVImportErr struct { Row int `json:"row"` Field string `json:"field,omitempty"` Message string `json:"message"` } // Standard CSV columns for export/import var csvColumns = []string{ "part_number", "item_type", "description", "current_revision", "created_at", "updated_at", "project", "category", } // HandleExportCSV exports items to CSV format. func (s *Server) HandleExportCSV(w http.ResponseWriter, r *http.Request) { ctx := r.Context() // Parse query options opts := db.ListOptions{ ItemType: r.URL.Query().Get("type"), Search: r.URL.Query().Get("search"), Project: r.URL.Query().Get("project"), Limit: 10000, // Max export limit } includeProps := r.URL.Query().Get("include_properties") == "true" // Fetch items items, err := s.items.List(ctx, opts) if err != nil { s.logger.Error().Err(err).Msg("failed to list items for export") writeError(w, http.StatusInternalServerError, "export_failed", err.Error()) return } // Collect all property keys if including properties propKeys := make(map[string]bool) itemProps := make(map[string]map[string]any) // part_number -> properties if includeProps { for _, item := range items { revisions, err := s.items.GetRevisions(ctx, item.ID) if err != nil { continue } for _, rev := range revisions { if rev.RevisionNumber == item.CurrentRevision && rev.Properties != nil { itemProps[item.PartNumber] = rev.Properties for k := range rev.Properties { propKeys[k] = true } break } } } } // Build header row headers := make([]string, len(csvColumns)) copy(headers, csvColumns) // Add property columns (sorted for consistency) sortedPropKeys := make([]string, 0, len(propKeys)) for k := range propKeys { // Skip internal/system properties if !strings.HasPrefix(k, "_") { sortedPropKeys = append(sortedPropKeys, k) } } sort.Strings(sortedPropKeys) headers = append(headers, sortedPropKeys...) // Set response headers for CSV download filename := fmt.Sprintf("silo-export-%s.csv", time.Now().Format("2006-01-02")) w.Header().Set("Content-Type", "text/csv") w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) // Write CSV writer := csv.NewWriter(w) defer writer.Flush() // Write header if err := writer.Write(headers); err != nil { s.logger.Error().Err(err).Msg("failed to write CSV header") return } // Write data rows for _, item := range items { row := make([]string, len(headers)) // Parse part number to extract project and category project, category := parsePartNumber(item.PartNumber) // Standard columns row[0] = item.PartNumber row[1] = item.ItemType row[2] = item.Description row[3] = strconv.Itoa(item.CurrentRevision) row[4] = item.CreatedAt.Format(time.RFC3339) row[5] = item.UpdatedAt.Format(time.RFC3339) row[6] = project row[7] = category // Property columns if includeProps { props := itemProps[item.PartNumber] for i, key := range sortedPropKeys { colIdx := len(csvColumns) + i if props != nil { if val, ok := props[key]; ok { row[colIdx] = formatPropertyValue(val) } } } } if err := writer.Write(row); err != nil { s.logger.Error().Err(err).Str("part_number", item.PartNumber).Msg("failed to write CSV row") continue } } s.logger.Info().Int("count", len(items)).Msg("exported items to CSV") } // HandleImportCSV imports items from a CSV file. func (s *Server) HandleImportCSV(w http.ResponseWriter, r *http.Request) { ctx := r.Context() // Parse multipart form (max 10MB) if err := r.ParseMultipartForm(10 << 20); err != nil { writeError(w, http.StatusBadRequest, "invalid_form", err.Error()) return } // Get file file, _, err := r.FormFile("file") if err != nil { writeError(w, http.StatusBadRequest, "missing_file", "CSV file is required") return } defer file.Close() // Get options dryRun := r.FormValue("dry_run") == "true" schemaName := r.FormValue("schema") if schemaName == "" { schemaName = "kindred-rd" } // Parse CSV reader := csv.NewReader(file) reader.TrimLeadingSpace = true // Read header headers, err := reader.Read() if err != nil { writeError(w, http.StatusBadRequest, "invalid_csv", "Failed to read CSV header") return } // Build column index map colIndex := make(map[string]int) for i, h := range headers { colIndex[strings.ToLower(strings.TrimSpace(h))] = i } // Validate required columns requiredCols := []string{"project", "category"} for _, col := range requiredCols { if _, ok := colIndex[col]; !ok { writeError(w, http.StatusBadRequest, "missing_column", fmt.Sprintf("Required column '%s' not found", col)) return } } result := CSVImportResult{ Errors: make([]CSVImportErr, 0), CreatedItems: make([]string, 0), } // Process rows rowNum := 1 // Start at 1 (header is row 0) for { record, err := reader.Read() if err == io.EOF { break } if err != nil { result.Errors = append(result.Errors, CSVImportErr{ Row: rowNum, Message: fmt.Sprintf("Failed to parse row: %v", err), }) result.ErrorCount++ rowNum++ continue } result.TotalRows++ rowNum++ // Extract values project := getCSVValue(record, colIndex, "project") category := getCSVValue(record, colIndex, "category") description := getCSVValue(record, colIndex, "description") partNumber := getCSVValue(record, colIndex, "part_number") // Validate project if project == "" { result.Errors = append(result.Errors, CSVImportErr{ Row: rowNum, Field: "project", Message: "Project code is required", }) result.ErrorCount++ continue } // Validate category if category == "" { result.Errors = append(result.Errors, CSVImportErr{ Row: rowNum, Field: "category", Message: "Category code is required", }) result.ErrorCount++ continue } // Build properties from extra columns properties := make(map[string]any) properties["project"] = strings.ToUpper(project) properties["category"] = strings.ToUpper(category) for col, idx := range colIndex { // Skip standard columns if isStandardColumn(col) { continue } if idx < len(record) && record[idx] != "" { properties[col] = parsePropertyValue(record[idx]) } } // If part_number is provided, check if it exists if partNumber != "" { existing, _ := s.items.GetByPartNumber(ctx, partNumber) if existing != nil { result.Errors = append(result.Errors, CSVImportErr{ Row: rowNum, Field: "part_number", Message: fmt.Sprintf("Part number '%s' already exists", partNumber), }) result.ErrorCount++ continue } } if dryRun { // In dry-run mode, just validate result.SuccessCount++ continue } // Generate part number if not provided if partNumber == "" { input := partnum.Input{ SchemaName: schemaName, Values: map[string]string{ "project": strings.ToUpper(project), "category": strings.ToUpper(category), }, } partNumber, err = s.partgen.Generate(ctx, input) if err != nil { result.Errors = append(result.Errors, CSVImportErr{ Row: rowNum, Message: fmt.Sprintf("Failed to generate part number: %v", err), }) result.ErrorCount++ continue } } // Determine item type from category itemType := "part" if len(category) > 0 { switch category[0] { case 'A': itemType = "assembly" case 'T': itemType = "tooling" } } // Create item item := &db.Item{ PartNumber: partNumber, ItemType: itemType, Description: description, } if err := s.items.Create(ctx, item, properties); err != nil { result.Errors = append(result.Errors, CSVImportErr{ Row: rowNum, Message: fmt.Sprintf("Failed to create item: %v", err), }) result.ErrorCount++ continue } result.SuccessCount++ result.CreatedItems = append(result.CreatedItems, partNumber) } s.logger.Info(). Int("total", result.TotalRows). Int("success", result.SuccessCount). Int("errors", result.ErrorCount). Bool("dry_run", dryRun). Msg("CSV import completed") writeJSON(w, http.StatusOK, result) } // HandleCSVTemplate returns an empty CSV template with headers. func (s *Server) HandleCSVTemplate(w http.ResponseWriter, r *http.Request) { schemaName := r.URL.Query().Get("schema") if schemaName == "" { schemaName = "kindred-rd" } sch, ok := s.schemas[schemaName] if !ok { writeError(w, http.StatusNotFound, "not_found", "Schema not found") return } // Build headers: standard columns + default property columns from schema headers := []string{ "project", "category", "description", } // Add default property columns from schema if sch.PropertySchemas != nil && sch.PropertySchemas.Defaults != nil { propNames := make([]string, 0, len(sch.PropertySchemas.Defaults)) for name := range sch.PropertySchemas.Defaults { propNames = append(propNames, name) } sort.Strings(propNames) headers = append(headers, propNames...) } // Set response headers filename := fmt.Sprintf("silo-import-template-%s.csv", schemaName) w.Header().Set("Content-Type", "text/csv") w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) writer := csv.NewWriter(w) defer writer.Flush() // Write header row if err := writer.Write(headers); err != nil { s.logger.Error().Err(err).Msg("failed to write CSV template header") return } // Write example row exampleRow := make([]string, len(headers)) exampleRow[0] = "PROJ1" // project exampleRow[1] = "F01" // category exampleRow[2] = "Example Item Description" // Leave property columns empty if err := writer.Write(exampleRow); err != nil { s.logger.Error().Err(err).Msg("failed to write CSV template example") return } } // Helper functions func parsePartNumber(pn string) (project, category string) { parts := strings.Split(pn, "-") if len(parts) >= 2 { project = parts[0] category = parts[1] } return } func formatPropertyValue(v any) string { switch val := v.(type) { case string: return val case float64: if val == float64(int64(val)) { return strconv.FormatInt(int64(val), 10) } return strconv.FormatFloat(val, 'f', -1, 64) case int: return strconv.Itoa(val) case int64: return strconv.FormatInt(val, 10) case bool: return strconv.FormatBool(val) case nil: return "" default: // For complex types, use JSON b, _ := json.Marshal(val) return string(b) } } func parsePropertyValue(s string) any { s = strings.TrimSpace(s) if s == "" { return nil } // Try boolean if s == "true" { return true } if s == "false" { return false } // Try integer if i, err := strconv.ParseInt(s, 10, 64); err == nil { return i } // Try float if f, err := strconv.ParseFloat(s, 64); err == nil { return f } // Try JSON (for arrays/objects) if (strings.HasPrefix(s, "[") && strings.HasSuffix(s, "]")) || (strings.HasPrefix(s, "{") && strings.HasSuffix(s, "}")) { var v any if err := json.Unmarshal([]byte(s), &v); err == nil { return v } } // Default to string return s } func getCSVValue(record []string, colIndex map[string]int, column string) string { if idx, ok := colIndex[column]; ok && idx < len(record) { return strings.TrimSpace(record[idx]) } return "" } func isStandardColumn(col string) bool { standardCols := map[string]bool{ "part_number": true, "item_type": true, "description": true, "current_revision": true, "created_at": true, "updated_at": true, "project": true, "category": true, } return standardCols[col] }