From f9324686c54d0bd1e92a4471eb6f560e0ff268be Mon Sep 17 00:00:00 2001 From: Forbes Date: Mon, 26 Jan 2026 06:06:21 -0600 Subject: [PATCH] Add roadmap and deployment examples --- ROADMAP.md | 492 ++++++++++++++++++++++++++ deployments/config.prod.yaml | 37 ++ deployments/docker-compose.prod.yaml | 50 +++ deployments/systemd/silod.env.example | 13 + deployments/systemd/silod.service | 43 +++ docs/DEPLOYMENT.md | 399 +++++++++++++++++++++ internal/db/items.go | 50 ++- 7 files changed, 1073 insertions(+), 11 deletions(-) create mode 100644 ROADMAP.md create mode 100644 deployments/config.prod.yaml create mode 100644 deployments/docker-compose.prod.yaml create mode 100644 deployments/systemd/silod.env.example create mode 100644 deployments/systemd/silod.service create mode 100644 docs/DEPLOYMENT.md diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..36ddba3 --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,492 @@ +# Silo Roadmap + +**Version:** 1.0 +**Date:** January 2026 +**Purpose:** Project inventory, SOLIDWORKS PDM gap analysis, and development roadmap + +--- + +## Table of Contents + +1. [Executive Summary](#executive-summary) +2. [Current Project Inventory](#current-project-inventory) +3. [SOLIDWORKS PDM Gap Analysis](#solidworks-pdm-gap-analysis) +4. [Feature Roadmap](#feature-roadmap) +5. [Implementation Phases](#implementation-phases) + +--- + +## Executive Summary + +Silo is an R&D-oriented item database and part management system designed for FreeCAD integration. It provides configurable part number generation, revision tracking, BOM management, and file versioning through MinIO storage. + +This document compares Silo's current capabilities against SOLIDWORKS PDM—the industry-leading product data management solution—to identify gaps and prioritize future development. + +### Key Differentiators + +| Aspect | Silo | SOLIDWORKS PDM | +|--------|------|----------------| +| **Target CAD** | FreeCAD / Kindred Create (open source) | SOLIDWORKS (proprietary) | +| **Part Numbering** | Schema-as-configuration (YAML) | Fixed format with some customization | +| **Licensing** | Open source / Kindred Proprietary | Commercial ($3,000-$10,000+ per seat) | +| **Storage** | PostgreSQL + MinIO (S3-compatible) | SQL Server + File Archive | +| **Philosophy** | R&D-oriented, lightweight | Enterprise-grade, comprehensive | + +--- + +## Current Project Inventory + +### Implemented Features (MVP Complete) + +#### Core Database System +- PostgreSQL schema with 7 migrations +- UUID-based identifiers throughout +- Soft delete support via `archived_at` timestamps +- Atomic sequence generation for part numbers + +#### Part Number Generation +- YAML schema parser with validation +- Segment types: `string`, `enum`, `serial`, `constant` +- Scope templates for serial counters (e.g., `{category}`, `{project}`) +- Format templates for custom output + +#### Item Management +- Full CRUD operations for items +- Item types: part, assembly, drawing, document, tooling, purchased, electrical, software +- Custom properties via JSONB storage +- Project tagging with many-to-many relationships + +#### Revision Control +- Append-only revision history +- Revision metadata: properties, file reference, checksum, comment +- Status tracking: draft, review, released, obsolete +- Labels/tags per revision +- Revision comparison (diff) +- Rollback functionality + +#### File Management +- MinIO integration with versioning +- File upload/download via REST API +- SHA256 checksums for integrity +- Storage path: `items/{partNumber}/rev{N}.FCStd` + +#### Bill of Materials (BOM) +- Relationship types: component, alternate, reference +- Reference designators for electronics +- Quantity tracking with units +- Revision-specific child linking + +#### Project Management +- Project CRUD operations +- Unique project codes (2-10 characters) +- Item-to-project tagging +- Project-filtered queries + +#### Data Import/Export +- CSV export with configurable properties +- CSV import with dry-run validation +- Template generation for import formatting + +#### API & Web Interface +- Comprehensive REST API (30+ endpoints) +- Middleware: logging, CORS, recovery, request ID +- Web UI foundation with htmx +- Health and readiness probes + +#### Configuration +- YAML configuration with environment variable overrides +- Multi-schema support +- Docker Compose deployment ready + +### Partially Implemented + +| Feature | Status | Notes | +|---------|--------|-------| +| FreeCAD Workbench | ~60% | Commands defined, upload/commit working, needs testing | +| Date segment type | Not started | Schema parser placeholder exists | +| Part number validation | Not started | API accepts but doesn't validate format | +| Location hierarchy CRUD | Schema only | Tables exist, no API endpoints | +| Inventory tracking | Schema only | Tables exist, no API endpoints | +| Unit tests | Not started | Critical for production use | + +### Infrastructure Status + +| Component | Status | +|-----------|--------| +| PostgreSQL | Running | +| MinIO | Configured (CPU compatibility issue being resolved) | +| Silo API Server | Builds successfully | +| Docker Compose | Complete with all services | + +--- + +## SOLIDWORKS PDM Gap Analysis + +This section compares Silo's capabilities against SOLIDWORKS PDM features. Gaps are categorized by priority and implementation complexity. + +### Legend +- **Silo Status:** Full / Partial / None +- **Priority:** Critical / High / Medium / Low +- **Complexity:** Simple / Moderate / Complex + +--- + +### 1. Version Control & Revision Management + +| Feature | SOLIDWORKS PDM | Silo Status | Priority | Complexity | +|---------|---------------|-------------|----------|------------| +| Check-in/check-out | Full pessimistic locking | None | High | Moderate | +| Version history | Complete with branching | Full (linear) | - | - | +| Revision labels | A, B, C or custom schemes | Full (custom labels) | - | - | +| Rollback/restore | Full | Full | - | - | +| Compare revisions | Visual + metadata diff | Metadata diff only | Medium | Complex | +| Get Latest Revision | One-click retrieval | Partial (API only) | Medium | Simple | + +**Gap Analysis:** +Silo lacks pessimistic locking (check-out), which is critical for multi-user CAD environments where file merging is impractical. Visual diff comparison would require FreeCAD integration for CAD file visualization. + +--- + +### 2. Workflow Management + +| Feature | SOLIDWORKS PDM | Silo Status | Priority | Complexity | +|---------|---------------|-------------|----------|------------| +| Custom workflows | Full visual designer | None | Critical | Complex | +| State transitions | Configurable with permissions | Basic (status field only) | Critical | Complex | +| Parallel approvals | Multiple approvers required | None | High | Complex | +| Automatic transitions | Timer/condition-based | None | Medium | Moderate | +| Email notifications | On state change | None | High | Moderate | +| ECO process | Built-in change management | None | High | Complex | +| Child state conditions | Block parent if children invalid | None | Medium | Moderate | + +**Gap Analysis:** +Workflow management is the largest functional gap. SOLIDWORKS PDM offers sophisticated state machines with parallel approvals, automatic transitions, and deep integration with engineering change processes. Silo currently has only a simple status field (draft/review/released/obsolete) with no transition rules or approval processes. + +--- + +### 3. User Management & Security + +| Feature | SOLIDWORKS PDM | Silo Status | Priority | Complexity | +|---------|---------------|-------------|----------|------------| +| User authentication | Windows AD, LDAP | None (single-user) | Critical | Moderate | +| Role-based permissions | Granular per folder/state | None | Critical | Complex | +| Group management | Full | None | Critical | Moderate | +| Folder permissions | Read/write/delete per folder | None | High | Moderate | +| State permissions | Actions allowed per state | None | High | Moderate | +| Audit trail | Complete action logging | None | High | Moderate | +| Private files | Pre-check-in visibility control | None | Low | Simple | + +**Gap Analysis:** +Silo currently operates as single-user with no authentication. Multi-user deployment requires authentication (LDAP planned), authorization (role-based access), and audit logging. This is prerequisite for any team environment. + +--- + +### 4. Search & Discovery + +| Feature | SOLIDWORKS PDM | Silo Status | Priority | Complexity | +|---------|---------------|-------------|----------|------------| +| Metadata search | Full with custom cards | Partial (API query params) | High | Moderate | +| Full-text content search | iFilters for Office, CAD | None | Medium | Complex | +| Quick search | Toolbar with history | None (API only) | Medium | Simple | +| Saved searches | User-defined favorites | None | Medium | Simple | +| Advanced operators | AND, OR, NOT, wildcards | None | Medium | Simple | +| Multi-variable search | Search across multiple fields | None | Medium | Simple | +| Where-used search | Find all assemblies using part | None | High | Moderate | + +**Gap Analysis:** +Silo has basic API-level filtering but lacks a rich search interface. Content search (searching within CAD files) would require FreeCAD file parsing. "Where-used" queries are particularly valuable for impact analysis. + +--- + +### 5. BOM Management + +| Feature | SOLIDWORKS PDM | Silo Status | Priority | Complexity | +|---------|---------------|-------------|----------|------------| +| Single-level BOM | Yes | Full | - | - | +| Multi-level BOM | Indented/exploded views | Schema only (no API) | High | Moderate | +| BOM comparison | Between revisions | None | Medium | Moderate | +| BOM export | Excel, XML, ERP formats | None | High | Simple | +| Calculated BOMs | Quantities rolled up | None | Medium | Moderate | +| Reference designators | Full support | Full | - | - | +| Alternate parts | Substitute tracking | Full | - | - | + +**Gap Analysis:** +BOM structure exists in the database but API endpoints for multi-level BOM retrieval and export are missing. BOM comparison between revisions would be valuable for change impact analysis. + +--- + +### 6. CAD Integration + +| Feature | SOLIDWORKS PDM | Silo Status | Priority | Complexity | +|---------|---------------|-------------|----------|------------| +| Native CAD add-in | Deep SOLIDWORKS integration | FreeCAD workbench (partial) | High | Complex | +| Property mapping | Bi-directional sync | Planned | High | Moderate | +| Task pane | Embedded in CAD UI | None | Medium | Complex | +| Lightweight components | Handle without full load | N/A for FreeCAD | - | - | +| Drawing/model linking | Automatic association | Manual | Medium | Moderate | +| Multi-CAD support | Third-party formats | FreeCAD only | Low | - | + +**Gap Analysis:** +FreeCAD workbench commands exist but need thorough testing. Property synchronization between Silo database and FreeCAD document properties is planned but not implemented. + +--- + +### 7. External Integrations + +| Feature | SOLIDWORKS PDM | Silo Status | Priority | Complexity | +|---------|---------------|-------------|----------|------------| +| ERP integration | SAP, Dynamics, etc. | None | Medium | Complex | +| API access | Full COM/REST API | Full REST API | - | - | +| Dispatch scripts | Automation without coding | None | Medium | Moderate | +| Task scheduler | Background processing | None | Medium | Moderate | +| Email system | SMTP integration | None | High | Simple | +| Web portal | Browser access | Partial (basic UI) | High | Moderate | + +**Gap Analysis:** +Silo has a solid REST API foundation. ERP integration can be built on this API but would require specific adapters per target system. Email notifications and scheduled tasks would enhance workflow automation. + +--- + +### 8. Reporting & Analytics + +| Feature | SOLIDWORKS PDM | Silo Status | Priority | Complexity | +|---------|---------------|-------------|----------|------------| +| Standard reports | Inventory, usage, activity | None | Medium | Moderate | +| Custom reports | User-defined queries | None | Medium | Moderate | +| Dashboard | Visual KPIs | None | Low | Moderate | +| Export formats | PDF, Excel, CSV | CSV only | Medium | Simple | + +**Gap Analysis:** +Reporting capabilities are absent. Basic reports (item counts, revision activity, where-used) would provide immediate value. + +--- + +### 9. File Handling + +| Feature | SOLIDWORKS PDM | Silo Status | Priority | Complexity | +|---------|---------------|-------------|----------|------------| +| File versioning | Automatic | Full (MinIO) | - | - | +| File preview | Thumbnails, 3D preview | None | Medium | Complex | +| File conversion | PDF, DXF generation | None | Medium | Complex | +| Replication | Multi-site sync | None | Low | Complex | +| File copy with refs | Copy tree with references | None | Medium | Moderate | + +**Gap Analysis:** +File storage works well. Thumbnail generation and file preview would significantly improve the web UI experience. Automatic conversion to PDF/DXF is valuable for sharing with non-CAD users. + +--- + +### Gap Summary by Priority + +#### Critical Gaps (Required for Team Use) +1. **User authentication** - LDAP/SSO integration +2. **Role-based permissions** - Folder and state-level access control +3. **Workflow engine** - State machines with transitions and approvals +4. **Check-out locking** - Pessimistic locking for CAD files + +#### High Priority Gaps (Significant Value) +1. **Email notifications** - Alert users on state changes +2. **Where-used search** - Impact analysis for changes +3. **Multi-level BOM API** - Retrieve full assembly structure +4. **BOM export** - Excel/CSV/XML for downstream systems +5. **Audit trail** - Log all user actions +6. **Web UI search** - User-friendly search interface + +#### Medium Priority Gaps (Nice to Have) +1. **Saved searches** - Frequently used queries +2. **File preview/thumbnails** - Visual browsing +3. **Reporting** - Activity and inventory reports +4. **Scheduled tasks** - Background automation +5. **BOM comparison** - Revision diff for assemblies + +--- + +## Feature Roadmap + +### Phase 1: Foundation (Current - Q2 2026) +*Complete MVP and stabilize core functionality* + +| Feature | Description | Status | +|---------|-------------|--------| +| MinIO integration | Resolve CPU compatibility, test upload/download | In Progress | +| FreeCAD workbench | Complete and test checkout/commit/status commands | In Progress | +| Unit tests | Core API and database operations | Not Started | +| Date segment type | Support date-based part number segments | Not Started | +| Part number validation | Validate format on creation | Not Started | +| Location CRUD API | Expose location hierarchy via REST | Not Started | +| Inventory API | Expose inventory operations via REST | Not Started | + +### Phase 2: Multi-User (Q2-Q3 2026) +*Enable team collaboration* + +| Feature | Description | Complexity | +|---------|-------------|------------| +| LDAP authentication | Integrate with FreeIPA/Active Directory | Moderate | +| User/group management | Create, assign, manage users and groups | Moderate | +| Folder permissions | Read/write/delete per folder hierarchy | Moderate | +| Check-out locking | Pessimistic locks with timeout | Moderate | +| Audit logging | Record all user actions with timestamps | Moderate | +| Session management | Token-based API authentication | Moderate | + +### Phase 3: Workflow Engine (Q3-Q4 2026) +*Implement engineering change processes* + +| Feature | Description | Complexity | +|---------|-------------|------------| +| Workflow designer | YAML-defined state machines | Complex | +| State transitions | Configurable transition rules | Complex | +| Transition permissions | Who can execute which transitions | Moderate | +| Single approvals | Basic approval workflow | Moderate | +| Parallel approvals | Multi-approver gates | Complex | +| Automatic transitions | Timer and condition-based | Complex | +| Email notifications | SMTP integration for alerts | Simple | +| Child state conditions | Block parent transitions | Moderate | + +### Phase 4: Search & Discovery (Q4 2026 - Q1 2027) +*Improve findability and navigation* + +| Feature | Description | Complexity | +|---------|-------------|------------| +| Advanced search UI | Web interface with filters | Moderate | +| Search operators | AND, OR, NOT, wildcards | Simple | +| Saved searches | User favorites | Simple | +| Where-used queries | Find parent assemblies | Moderate | +| Quick search | Toolbar search box | Simple | +| Content search | Search within file content | Complex | + +### Phase 5: BOM & Reporting (Q1-Q2 2027) +*Enhanced BOM management and analytics* + +| Feature | Description | Complexity | +|---------|-------------|------------| +| Multi-level BOM API | Recursive assembly retrieval | Moderate | +| BOM export | Excel, CSV, XML formats | Simple | +| BOM comparison | Diff between revisions | Moderate | +| Standard reports | Activity, inventory, usage | Moderate | +| Custom queries | User-defined report builder | Moderate | +| Dashboard | Visual KPIs and metrics | Moderate | + +### Phase 6: Advanced Features (Q2-Q4 2027) +*Enterprise capabilities* + +| Feature | Description | Complexity | +|---------|-------------|------------| +| File preview | Thumbnail generation | Complex | +| File conversion | Auto-generate PDF/DXF | Complex | +| ERP integration | Adapter framework | Complex | +| Task scheduler | Background job processing | Moderate | +| Webhooks | Event notifications to external systems | Moderate | +| API rate limiting | Protect against abuse | Simple | + +--- + +## Implementation Phases + +### Phase 1 Detailed Tasks + +#### 1.1 MinIO Integration Completion +- [ ] Verify MinIO container runs on target VM +- [ ] Test file upload via REST API +- [ ] Test file download via REST API +- [ ] Test FreeCAD workbench upload +- [ ] Verify version history in MinIO console + +#### 1.2 FreeCAD Workbench Completion +- [ ] Test `silo checkout` command +- [ ] Test `silo commit` with file upload +- [ ] Test `silo status` for modification detection +- [ ] Test `silo log` for revision history +- [ ] Test `silo register` for new part creation +- [ ] Document workbench installation + +#### 1.3 Unit Test Suite +- [ ] Database connection and transaction tests +- [ ] Item CRUD operation tests +- [ ] Revision creation and retrieval tests +- [ ] Part number generation tests +- [ ] File upload/download tests +- [ ] CSV import/export tests +- [ ] API endpoint tests + +#### 1.4 Missing Segment Types +- [ ] Implement date segment type +- [ ] Add strftime-style format support +- [ ] Update schema documentation + +#### 1.5 Location & Inventory APIs +- [ ] `GET /api/locations` - List locations +- [ ] `POST /api/locations` - Create location +- [ ] `GET /api/locations/{path}` - Get location +- [ ] `DELETE /api/locations/{path}` - Delete location +- [ ] `GET /api/inventory/{partNumber}` - Get inventory +- [ ] `POST /api/inventory/{partNumber}/adjust` - Adjust quantity +- [ ] `POST /api/inventory/{partNumber}/move` - Move between locations + +--- + +## Success Metrics + +### Phase 1 (Foundation) +- All existing tests pass +- File upload/download works end-to-end +- FreeCAD users can checkout, modify, commit parts + +### Phase 2 (Multi-User) +- 5+ concurrent users supported +- No data corruption under concurrent access +- Audit log captures all modifications + +### Phase 3 (Workflow) +- Engineering change process completable in Silo +- Email notifications delivered reliably +- Workflow state visible in web UI + +### Phase 4+ (Advanced) +- Search returns results in <2 seconds +- Where-used queries complete in <5 seconds +- BOM export matches assembly structure + +--- + +## References + +### SOLIDWORKS PDM Documentation +- [SOLIDWORKS PDM Product Page](https://www.solidworks.com/product/solidworks-pdm) +- [What's New in SOLIDWORKS PDM 2025](https://blogs.solidworks.com/solidworksblog/2024/10/whats-new-in-solidworks-pdm-2025.html) +- [Top 5 Enhancements in SOLIDWORKS PDM 2024](https://blogs.solidworks.com/solidworksblog/2023/10/top-5-enhancements-in-solidworks-pdm-2024.html) +- [SOLIDWORKS PDM Workflow Transitions](https://help.solidworks.com/2023/english/EnterprisePDM/Admin/c_workflow_transition.htm) +- [Ultimate Guide to SOLIDWORKS PDM Permissions](https://www.goengineer.com/blog/ultimate-guide-to-solidworks-pdm-permissions) +- [Searching in SOLIDWORKS PDM](https://help.solidworks.com/2021/english/EnterprisePDM/fileexplorer/c_searches.htm) +- [SOLIDWORKS PDM API Getting Started](https://3dswym.3dexperience.3ds.com/wiki/solidworks-news-info/getting-started-with-the-solidworks-pdm-api-solidpractices_gBCYaM75RgORBcpSO1m_Mw) + +### Silo Documentation +- [Silo Specification](docs/SPECIFICATION.md) +- [Development Status](docs/STATUS.md) + +--- + +## Appendix: Feature Comparison Matrix + +| Category | Feature | SW PDM Standard | SW PDM Pro | Silo Current | Silo Planned | +|----------|---------|-----------------|------------|--------------|--------------| +| **Version Control** | Check-in/out | Yes | Yes | No | Phase 2 | +| | Version history | Yes | Yes | Yes | - | +| | Rollback | Yes | Yes | Yes | - | +| **Workflow** | Custom workflows | Limited | Yes | No | Phase 3 | +| | Parallel approval | No | Yes | No | Phase 3 | +| | Notifications | No | Yes | No | Phase 3 | +| **Security** | User auth | Windows | Windows/LDAP | No | Phase 2 | +| | Permissions | Basic | Granular | No | Phase 2 | +| | Audit trail | Basic | Full | No | Phase 2 | +| **Search** | Metadata search | Yes | Yes | Partial | Phase 4 | +| | Content search | No | Yes | No | Phase 4 | +| | Where-used | Yes | Yes | No | Phase 4 | +| **BOM** | Single-level | Yes | Yes | Yes | - | +| | Multi-level | Yes | Yes | Schema only | Phase 5 | +| | BOM export | Yes | Yes | No | Phase 5 | +| **Integration** | API | Limited | Full | Full REST | - | +| | ERP connectors | No | Yes | No | Phase 6 | +| | Web access | No | Yes | Partial | Phase 4 | +| **Files** | Versioning | Yes | Yes | Yes | - | +| | Preview | Yes | Yes | No | Phase 6 | +| | Multi-site | No | Yes | No | Not Planned | diff --git a/deployments/config.prod.yaml b/deployments/config.prod.yaml new file mode 100644 index 0000000..0d34498 --- /dev/null +++ b/deployments/config.prod.yaml @@ -0,0 +1,37 @@ +# Silo Production Configuration +# For deployment on dedicated VM using external PostgreSQL and MinIO +# +# Credentials are provided via environment variables: +# SILO_DB_PASSWORD +# SILO_MINIO_ACCESS_KEY +# SILO_MINIO_SECRET_KEY + +server: + host: "0.0.0.0" + port: 8080 + base_url: "http://silo.kindred.internal:8080" + +database: + host: "psql.kindred.internal" + port: 5432 + name: "silo" + user: "silo" + password: "" # Set via SILO_DB_PASSWORD + sslmode: "require" + max_connections: 20 + +storage: + endpoint: "minio.kindred.internal:9000" + access_key: "" # Set via SILO_MINIO_ACCESS_KEY + secret_key: "" # Set via SILO_MINIO_SECRET_KEY + bucket: "silo-files" + use_ssl: true + region: "us-east-1" + +schemas: + directory: "/etc/silo/schemas" + default: "kindred-rd" + +freecad: + uri_scheme: "silo" + executable: "/usr/bin/freecad" diff --git a/deployments/docker-compose.prod.yaml b/deployments/docker-compose.prod.yaml new file mode 100644 index 0000000..104c54b --- /dev/null +++ b/deployments/docker-compose.prod.yaml @@ -0,0 +1,50 @@ +# Production Docker Compose for Silo +# Uses external PostgreSQL (psql.kindred.internal) and MinIO (minio.kindred.internal) +# +# Usage: +# export SILO_DB_PASSWORD= +# export SILO_MINIO_ACCESS_KEY= +# export SILO_MINIO_SECRET_KEY= +# docker compose -f docker-compose.prod.yaml up -d + +services: + silo: + build: + context: .. + dockerfile: build/package/Dockerfile + container_name: silod + restart: unless-stopped + environment: + # Database connection (psql.kindred.internal) + SILO_DB_HOST: psql.kindred.internal + SILO_DB_PORT: 5432 + SILO_DB_NAME: silo + SILO_DB_USER: silo + SILO_DB_PASSWORD: ${SILO_DB_PASSWORD:?Database password required} + SILO_DB_SSLMODE: require + + # MinIO storage (minio.kindred.internal) + SILO_MINIO_ENDPOINT: minio.kindred.internal:9000 + SILO_MINIO_ACCESS_KEY: ${SILO_MINIO_ACCESS_KEY:?MinIO access key required} + SILO_MINIO_SECRET_KEY: ${SILO_MINIO_SECRET_KEY:?MinIO secret key required} + SILO_MINIO_BUCKET: silo-files + SILO_MINIO_USE_SSL: "true" + + # Server settings + SILO_SERVER_BASE_URL: ${SILO_BASE_URL:-http://silo.kindred.internal:8080} + ports: + - "8080:8080" + volumes: + - ../schemas:/etc/silo/schemas:ro + - ./config.prod.yaml:/etc/silo/config.yaml:ro + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:8080/health"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" diff --git a/deployments/systemd/silod.env.example b/deployments/systemd/silod.env.example new file mode 100644 index 0000000..4db8e7b --- /dev/null +++ b/deployments/systemd/silod.env.example @@ -0,0 +1,13 @@ +# Silo daemon environment variables +# Copy to /etc/silo/silod.env and fill in values +# Permissions: chmod 600 /etc/silo/silod.env + +# Database credentials (psql.kindred.internal) +SILO_DB_PASSWORD= + +# MinIO credentials (minio.kindred.internal) +SILO_MINIO_ACCESS_KEY= +SILO_MINIO_SECRET_KEY= + +# Optional: Override server base URL +# SILO_SERVER_BASE_URL=http://silo.kindred.internal:8080 diff --git a/deployments/systemd/silod.service b/deployments/systemd/silod.service new file mode 100644 index 0000000..a96a627 --- /dev/null +++ b/deployments/systemd/silod.service @@ -0,0 +1,43 @@ +[Unit] +Description=Silo Item Database Server +Documentation=https://github.com/kindred-systems/silo +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=silo +Group=silo + +# Working directory +WorkingDirectory=/opt/silo + +# Environment file for secrets +EnvironmentFile=/etc/silo/silod.env + +# Main process +ExecStart=/opt/silo/bin/silod -config /etc/silo/config.yaml + +# Restart policy +Restart=on-failure +RestartSec=5s + +# Security hardening +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes +ReadOnlyPaths=/etc/silo +ReadWritePaths=/var/log/silo + +# Resource limits +LimitNOFILE=65535 +LimitNPROC=4096 + +# Logging +StandardOutput=journal +StandardError=journal +SyslogIdentifier=silod + +[Install] +WantedBy=multi-user.target diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md new file mode 100644 index 0000000..7691e34 --- /dev/null +++ b/docs/DEPLOYMENT.md @@ -0,0 +1,399 @@ +# Silo Production Deployment Guide + +This guide covers deploying Silo to a dedicated VM using external PostgreSQL and MinIO services. + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ silo.kindred.internal │ +│ ┌───────────────────────────────────────────────────────────┐ │ +│ │ silod │ │ +│ │ (Silo API Server) │ │ +│ │ :8080 │ │ +│ └───────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────┘ + │ │ + ▼ ▼ +┌─────────────────────────┐ ┌─────────────────────────────────┐ +│ psql.kindred.internal │ │ minio.kindred.internal │ +│ PostgreSQL 16 │ │ MinIO S3 │ +│ :5432 │ │ :9000 (API) │ +│ │ │ :9001 (Console) │ +└─────────────────────────┘ └─────────────────────────────────┘ +``` + +## Prerequisites + +### On psql.kindred.internal + +1. Create the Silo database and user: + +```sql +-- Connect as postgres superuser +CREATE USER silo WITH PASSWORD 'your-secure-password'; +CREATE DATABASE silo OWNER silo; + +-- Grant necessary permissions +GRANT ALL PRIVILEGES ON DATABASE silo TO silo; + +-- Connect to silo database +\c silo + +-- Enable UUID extension +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; +``` + +2. Run migrations: + +```bash +# From the silo project directory +psql -h psql.kindred.internal -U silo -d silo -f migrations/001_initial.sql +psql -h psql.kindred.internal -U silo -d silo -f migrations/002_sequence_by_name.sql +psql -h psql.kindred.internal -U silo -d silo -f migrations/003_remove_material.sql +psql -h psql.kindred.internal -U silo -d silo -f migrations/004_cad_sync_state.sql +psql -h psql.kindred.internal -U silo -d silo -f migrations/005_property_schema_version.sql +psql -h psql.kindred.internal -U silo -d silo -f migrations/006_project_tags.sql +psql -h psql.kindred.internal -U silo -d silo -f migrations/007_revision_status.sql +``` + +Or run all at once: + +```bash +for f in migrations/*.sql; do + echo "Running $f..." + psql -h psql.kindred.internal -U silo -d silo -f "$f" +done +``` + +3. Allow connections from Silo VM in `pg_hba.conf`: + +``` +# Allow silo.kindred.internal to connect +hostssl silo silo silo.kindred.internal/32 scram-sha-256 +``` + +### On minio.kindred.internal + +1. Create the Silo bucket and access credentials: + +```bash +# Using mc (MinIO Client) +mc alias set kindred https://minio.kindred.internal ADMIN_ACCESS_KEY ADMIN_SECRET_KEY + +# Create bucket with versioning +mc mb kindred/silo-files +mc version enable kindred/silo-files + +# Create service account for Silo +mc admin user add kindred silo-service 'your-minio-secret-key' + +# Create policy for silo-files bucket +cat > /tmp/silo-policy.json << 'EOF' +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:ListBucket", + "s3:GetBucketVersioning", + "s3:GetObjectVersion", + "s3:DeleteObjectVersion" + ], + "Resource": [ + "arn:aws:s3:::silo-files", + "arn:aws:s3:::silo-files/*" + ] + } + ] +} +EOF + +mc admin policy create kindred silo-policy /tmp/silo-policy.json +mc admin policy attach kindred silo-policy --user silo-service +``` + +2. Verify SSL certificate is valid (or configure Silo to use non-SSL if internal). + +--- + +## Deployment Options + +### Option A: Systemd Service (Recommended for Production) + +#### 1. Prepare the Silo VM + +```bash +# Create silo user +sudo useradd -r -m -d /opt/silo -s /sbin/nologin silo + +# Create directories +sudo mkdir -p /opt/silo/bin +sudo mkdir -p /etc/silo/schemas +sudo mkdir -p /var/log/silo + +# Set ownership +sudo chown -R silo:silo /opt/silo /var/log/silo +sudo chown root:silo /etc/silo +sudo chmod 750 /etc/silo +``` + +#### 2. Build and Install Binary + +```bash +# On build machine (requires Go 1.23+) +cd /path/to/silo +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-w -s" -o silod ./cmd/silod + +# Copy to Silo VM +scp silod silo.kindred.internal:/tmp/ +ssh silo.kindred.internal "sudo mv /tmp/silod /opt/silo/bin/ && sudo chmod 755 /opt/silo/bin/silod" +``` + +#### 3. Install Configuration + +```bash +# Copy config file +scp deployments/config.prod.yaml silo.kindred.internal:/tmp/config.yaml +ssh silo.kindred.internal "sudo mv /tmp/config.yaml /etc/silo/config.yaml" + +# Copy schemas +scp -r schemas/* silo.kindred.internal:/tmp/schemas/ +ssh silo.kindred.internal "sudo mv /tmp/schemas/* /etc/silo/schemas/" + +# Create environment file with secrets +ssh silo.kindred.internal +sudo cat > /etc/silo/silod.env << 'EOF' +SILO_DB_PASSWORD=your-database-password +SILO_MINIO_ACCESS_KEY=silo-service +SILO_MINIO_SECRET_KEY=your-minio-secret-key +EOF +sudo chmod 600 /etc/silo/silod.env +sudo chown root:silo /etc/silo/silod.env +``` + +#### 4. Install Systemd Service + +```bash +# Copy service file +scp deployments/systemd/silod.service silo.kindred.internal:/tmp/ +ssh silo.kindred.internal "sudo mv /tmp/silod.service /etc/systemd/system/" + +# Enable and start +ssh silo.kindred.internal << 'EOF' +sudo systemctl daemon-reload +sudo systemctl enable silod +sudo systemctl start silod +sudo systemctl status silod +EOF +``` + +#### 5. Verify Deployment + +```bash +# Check service status +sudo systemctl status silod + +# Check logs +sudo journalctl -u silod -f + +# Test health endpoint +curl http://localhost:8080/health + +# Test readiness (verifies DB and MinIO connectivity) +curl http://localhost:8080/ready +``` + +--- + +### Option B: Docker Compose + +#### 1. Install Docker on Silo VM + +```bash +# Debian/Ubuntu +sudo apt-get update +sudo apt-get install -y docker.io docker-compose-plugin +sudo usermod -aG docker $USER +``` + +#### 2. Clone Repository + +```bash +git clone https://github.com/kindred-systems/silo.git /opt/silo +cd /opt/silo +``` + +#### 3. Configure Environment + +```bash +# Create .env file +cat > /opt/silo/deployments/.env << 'EOF' +SILO_DB_PASSWORD=your-database-password +SILO_MINIO_ACCESS_KEY=silo-service +SILO_MINIO_SECRET_KEY=your-minio-secret-key +SILO_BASE_URL=http://silo.kindred.internal:8080 +EOF +chmod 600 /opt/silo/deployments/.env +``` + +#### 4. Start Service + +```bash +cd /opt/silo/deployments +docker compose -f docker-compose.prod.yaml up -d +``` + +#### 5. Verify + +```bash +docker compose -f docker-compose.prod.yaml ps +docker compose -f docker-compose.prod.yaml logs -f +curl http://localhost:8080/ready +``` + +--- + +## Post-Deployment Configuration + +### DNS Setup + +Add DNS records for `silo.kindred.internal` pointing to the Silo VM IP address. + +### Firewall Rules + +```bash +# Allow incoming connections on port 8080 +sudo ufw allow 8080/tcp + +# Or with iptables +sudo iptables -A INPUT -p tcp --dport 8080 -j ACCEPT +``` + +### Reverse Proxy (Optional) + +For TLS termination, configure nginx or caddy: + +```nginx +# /etc/nginx/sites-available/silo +server { + listen 443 ssl http2; + server_name silo.kindred.internal; + + ssl_certificate /etc/ssl/certs/silo.kindred.internal.crt; + ssl_certificate_key /etc/ssl/private/silo.kindred.internal.key; + + location / { + proxy_pass http://127.0.0.1:8080; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} +``` + +--- + +## Maintenance + +### View Logs + +```bash +# Systemd +sudo journalctl -u silod -f +sudo journalctl -u silod --since "1 hour ago" + +# Docker +docker compose -f docker-compose.prod.yaml logs -f silo +``` + +### Restart Service + +```bash +# Systemd +sudo systemctl restart silod + +# Docker +docker compose -f docker-compose.prod.yaml restart silo +``` + +### Update Deployment + +```bash +# Systemd - rebuild and replace binary +go build -ldflags="-w -s" -o silod ./cmd/silod +sudo systemctl stop silod +sudo cp silod /opt/silo/bin/silod +sudo systemctl start silod + +# Docker - rebuild and restart +docker compose -f docker-compose.prod.yaml build +docker compose -f docker-compose.prod.yaml up -d +``` + +### Database Migrations + +When updating Silo, check for new migrations: + +```bash +# List migration files +ls -la migrations/ + +# Run new migrations +psql -h psql.kindred.internal -U silo -d silo -f migrations/008_new_feature.sql +``` + +--- + +## Troubleshooting + +### Connection Refused to PostgreSQL + +1. Verify network connectivity: `nc -zv psql.kindred.internal 5432` +2. Check `pg_hba.conf` allows connections from Silo VM +3. Verify firewall rules on PostgreSQL server +4. Check credentials in `/etc/silo/silod.env` + +### Connection Refused to MinIO + +1. Verify network connectivity: `nc -zv minio.kindred.internal 9000` +2. Check SSL settings match (use_ssl: true/false) +3. Verify access key and secret key +4. Check bucket exists: `mc ls kindred/silo-files` + +### Service Won't Start + +1. Check logs: `sudo journalctl -u silod -n 50` +2. Verify config syntax: `/opt/silo/bin/silod -config /etc/silo/config.yaml -validate` +3. Check file permissions on config and env files +4. Verify schemas directory exists and contains YAML files + +### Health Check Fails + +```bash +# Test each component +curl http://localhost:8080/health # Basic health +curl http://localhost:8080/ready # Full readiness (DB + MinIO) + +# If ready fails, check individual services +psql -h psql.kindred.internal -U silo -d silo -c "SELECT 1" +mc ls kindred/silo-files +``` + +--- + +## Security Checklist + +- [ ] Database password is strong and unique +- [ ] MinIO credentials are service-account specific +- [ ] `/etc/silo/silod.env` has mode 600 +- [ ] SSL/TLS enabled for PostgreSQL (`sslmode: require`) +- [ ] SSL/TLS enabled for MinIO (`use_ssl: true`) +- [ ] Firewall restricts access to port 8080 +- [ ] Silo runs as non-root user +- [ ] Logs don't contain sensitive information diff --git a/internal/db/items.go b/internal/db/items.go index 60505b3..ef23788 100644 --- a/internal/db/items.go +++ b/internal/db/items.go @@ -368,19 +368,47 @@ func (r *ItemRepository) GetRevisions(ctx context.Context, itemID string) ([]*Re // GetRevision retrieves a specific revision by item ID and revision number. func (r *ItemRepository) GetRevision(ctx context.Context, itemID string, revisionNumber int) (*Revision, error) { + // Check if status column exists (migration 007 applied) + var hasStatusColumn bool + err := r.db.pool.QueryRow(ctx, ` + SELECT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'revisions' AND column_name = 'status' + ) + `).Scan(&hasStatusColumn) + if err != nil { + return nil, fmt.Errorf("checking schema: %w", err) + } + rev := &Revision{} var propsJSON []byte - err := r.db.pool.QueryRow(ctx, ` - SELECT id, item_id, revision_number, properties, file_key, file_version, - file_checksum, file_size, thumbnail_key, created_at, created_by, comment, - COALESCE(status, 'draft') as status, COALESCE(labels, '{}') as labels - FROM revisions - WHERE item_id = $1 AND revision_number = $2 - `, itemID, revisionNumber).Scan( - &rev.ID, &rev.ItemID, &rev.RevisionNumber, &propsJSON, &rev.FileKey, &rev.FileVersion, - &rev.FileChecksum, &rev.FileSize, &rev.ThumbnailKey, &rev.CreatedAt, &rev.CreatedBy, &rev.Comment, - &rev.Status, &rev.Labels, - ) + + if hasStatusColumn { + err = r.db.pool.QueryRow(ctx, ` + SELECT id, item_id, revision_number, properties, file_key, file_version, + file_checksum, file_size, thumbnail_key, created_at, created_by, comment, + COALESCE(status, 'draft') as status, COALESCE(labels, ARRAY[]::TEXT[]) as labels + FROM revisions + WHERE item_id = $1 AND revision_number = $2 + `, itemID, revisionNumber).Scan( + &rev.ID, &rev.ItemID, &rev.RevisionNumber, &propsJSON, &rev.FileKey, &rev.FileVersion, + &rev.FileChecksum, &rev.FileSize, &rev.ThumbnailKey, &rev.CreatedAt, &rev.CreatedBy, &rev.Comment, + &rev.Status, &rev.Labels, + ) + } else { + err = r.db.pool.QueryRow(ctx, ` + SELECT id, item_id, revision_number, properties, file_key, file_version, + file_checksum, file_size, thumbnail_key, created_at, created_by, comment + FROM revisions + WHERE item_id = $1 AND revision_number = $2 + `, itemID, revisionNumber).Scan( + &rev.ID, &rev.ItemID, &rev.RevisionNumber, &propsJSON, &rev.FileKey, &rev.FileVersion, + &rev.FileChecksum, &rev.FileSize, &rev.ThumbnailKey, &rev.CreatedAt, &rev.CreatedBy, &rev.Comment, + ) + rev.Status = "draft" + rev.Labels = []string{} + } + if err == pgx.ErrNoRows { return nil, nil }