From f60c25983b9d94bdeaf1b9ab20b1eab1e23b925e Mon Sep 17 00:00:00 2001 From: Forbes Date: Sat, 14 Feb 2026 13:06:24 -0600 Subject: [PATCH] feat: add YAML job definition parser and example definitions New package internal/jobdef mirrors the schema package pattern: - Load/LoadAll/Validate for YAML job definitions - Supports trigger types: revision_created, bom_changed, manual, schedule - Supports scope types: item, assembly, project - Supports compute types: validate, rebuild, diff, export, custom - Defaults: timeout=600s, max_retries=1, priority=100 Example definitions in jobdefs/: - assembly-validate.yaml: incremental validation on revision_created - part-export-step.yaml: STEP export on manual trigger 11 unit tests, all passing. --- internal/jobdef/jobdef.go | 166 +++++++++++++++++ internal/jobdef/jobdef_test.go | 328 +++++++++++++++++++++++++++++++++ jobdefs/assembly-validate.yaml | 26 +++ jobdefs/part-export-step.yaml | 24 +++ 4 files changed, 544 insertions(+) create mode 100644 internal/jobdef/jobdef.go create mode 100644 internal/jobdef/jobdef_test.go create mode 100644 jobdefs/assembly-validate.yaml create mode 100644 jobdefs/part-export-step.yaml diff --git a/internal/jobdef/jobdef.go b/internal/jobdef/jobdef.go new file mode 100644 index 0000000..4be8d00 --- /dev/null +++ b/internal/jobdef/jobdef.go @@ -0,0 +1,166 @@ +// Package jobdef handles YAML job definition parsing and validation. +package jobdef + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "gopkg.in/yaml.v3" +) + +// Definition represents a compute job definition loaded from YAML. +type Definition struct { + Name string `yaml:"name" json:"name"` + Version int `yaml:"version" json:"version"` + Description string `yaml:"description" json:"description"` + Trigger TriggerConfig `yaml:"trigger" json:"trigger"` + Scope ScopeConfig `yaml:"scope" json:"scope"` + Compute ComputeConfig `yaml:"compute" json:"compute"` + Runner RunnerConfig `yaml:"runner" json:"runner"` + Timeout int `yaml:"timeout" json:"timeout"` + MaxRetries int `yaml:"max_retries" json:"max_retries"` + Priority int `yaml:"priority" json:"priority"` +} + +// TriggerConfig describes when a job is created. +type TriggerConfig struct { + Type string `yaml:"type" json:"type"` + Filter map[string]string `yaml:"filter,omitempty" json:"filter,omitempty"` +} + +// ScopeConfig describes what a job operates on. +type ScopeConfig struct { + Type string `yaml:"type" json:"type"` +} + +// ComputeConfig describes the computation to perform. +type ComputeConfig struct { + Type string `yaml:"type" json:"type"` + Command string `yaml:"command" json:"command"` + Args map[string]any `yaml:"args,omitempty" json:"args,omitempty"` +} + +// RunnerConfig describes runner requirements. +type RunnerConfig struct { + Tags []string `yaml:"tags" json:"tags"` +} + +// DefinitionFile wraps a definition for YAML parsing. +type DefinitionFile struct { + Job Definition `yaml:"job"` +} + +var validTriggerTypes = map[string]bool{ + "revision_created": true, + "bom_changed": true, + "manual": true, + "schedule": true, +} + +var validScopeTypes = map[string]bool{ + "item": true, + "assembly": true, + "project": true, +} + +var validComputeTypes = map[string]bool{ + "validate": true, + "rebuild": true, + "diff": true, + "export": true, + "custom": true, +} + +// Load reads a job definition from a YAML file. +func Load(path string) (*Definition, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading job definition file: %w", err) + } + + var df DefinitionFile + if err := yaml.Unmarshal(data, &df); err != nil { + return nil, fmt.Errorf("parsing job definition YAML: %w", err) + } + + def := &df.Job + + // Apply defaults + if def.Timeout <= 0 { + def.Timeout = 600 + } + if def.MaxRetries <= 0 { + def.MaxRetries = 1 + } + if def.Priority <= 0 { + def.Priority = 100 + } + if def.Version <= 0 { + def.Version = 1 + } + + if err := def.Validate(); err != nil { + return nil, fmt.Errorf("validating %s: %w", path, err) + } + + return def, nil +} + +// LoadAll reads all job definitions from a directory. +func LoadAll(dir string) (map[string]*Definition, error) { + defs := make(map[string]*Definition) + + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("reading job definitions directory: %w", err) + } + + for _, entry := range entries { + if entry.IsDir() { + continue + } + if !strings.HasSuffix(entry.Name(), ".yaml") && !strings.HasSuffix(entry.Name(), ".yml") { + continue + } + + path := filepath.Join(dir, entry.Name()) + def, err := Load(path) + if err != nil { + return nil, fmt.Errorf("loading %s: %w", entry.Name(), err) + } + defs[def.Name] = def + } + + return defs, nil +} + +// Validate checks that the definition is well-formed. +func (d *Definition) Validate() error { + if d.Name == "" { + return fmt.Errorf("job definition name is required") + } + if d.Trigger.Type == "" { + return fmt.Errorf("trigger type is required") + } + if !validTriggerTypes[d.Trigger.Type] { + return fmt.Errorf("invalid trigger type %q", d.Trigger.Type) + } + if d.Scope.Type == "" { + return fmt.Errorf("scope type is required") + } + if !validScopeTypes[d.Scope.Type] { + return fmt.Errorf("invalid scope type %q", d.Scope.Type) + } + if d.Compute.Type == "" { + return fmt.Errorf("compute type is required") + } + if !validComputeTypes[d.Compute.Type] { + return fmt.Errorf("invalid compute type %q", d.Compute.Type) + } + if d.Compute.Command == "" { + return fmt.Errorf("compute command is required") + } + return nil +} diff --git a/internal/jobdef/jobdef_test.go b/internal/jobdef/jobdef_test.go new file mode 100644 index 0000000..ba3fe5c --- /dev/null +++ b/internal/jobdef/jobdef_test.go @@ -0,0 +1,328 @@ +package jobdef + +import ( + "os" + "path/filepath" + "testing" +) + +func TestLoadValid(t *testing.T) { + dir := t.TempDir() + content := ` +job: + name: test-job + version: 1 + description: "A test job" + trigger: + type: manual + scope: + type: item + compute: + type: validate + command: create-validate + runner: + tags: [create] + timeout: 300 + max_retries: 2 + priority: 50 +` + path := filepath.Join(dir, "test-job.yaml") + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatalf("writing test file: %v", err) + } + + def, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + + if def.Name != "test-job" { + t.Errorf("name = %q, want %q", def.Name, "test-job") + } + if def.Version != 1 { + t.Errorf("version = %d, want 1", def.Version) + } + if def.Trigger.Type != "manual" { + t.Errorf("trigger type = %q, want %q", def.Trigger.Type, "manual") + } + if def.Scope.Type != "item" { + t.Errorf("scope type = %q, want %q", def.Scope.Type, "item") + } + if def.Compute.Type != "validate" { + t.Errorf("compute type = %q, want %q", def.Compute.Type, "validate") + } + if def.Compute.Command != "create-validate" { + t.Errorf("compute command = %q, want %q", def.Compute.Command, "create-validate") + } + if len(def.Runner.Tags) != 1 || def.Runner.Tags[0] != "create" { + t.Errorf("runner tags = %v, want [create]", def.Runner.Tags) + } + if def.Timeout != 300 { + t.Errorf("timeout = %d, want 300", def.Timeout) + } + if def.MaxRetries != 2 { + t.Errorf("max_retries = %d, want 2", def.MaxRetries) + } + if def.Priority != 50 { + t.Errorf("priority = %d, want 50", def.Priority) + } +} + +func TestLoadDefaults(t *testing.T) { + dir := t.TempDir() + content := ` +job: + name: minimal + trigger: + type: manual + scope: + type: item + compute: + type: custom + command: do-something +` + path := filepath.Join(dir, "minimal.yaml") + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatalf("writing test file: %v", err) + } + + def, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + + if def.Timeout != 600 { + t.Errorf("default timeout = %d, want 600", def.Timeout) + } + if def.MaxRetries != 1 { + t.Errorf("default max_retries = %d, want 1", def.MaxRetries) + } + if def.Priority != 100 { + t.Errorf("default priority = %d, want 100", def.Priority) + } + if def.Version != 1 { + t.Errorf("default version = %d, want 1", def.Version) + } +} + +func TestLoadInvalidTriggerType(t *testing.T) { + dir := t.TempDir() + content := ` +job: + name: bad-trigger + trigger: + type: invalid_trigger + scope: + type: item + compute: + type: validate + command: create-validate +` + path := filepath.Join(dir, "bad.yaml") + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatalf("writing test file: %v", err) + } + + _, err := Load(path) + if err == nil { + t.Fatal("expected error for invalid trigger type") + } +} + +func TestLoadMissingName(t *testing.T) { + dir := t.TempDir() + content := ` +job: + trigger: + type: manual + scope: + type: item + compute: + type: validate + command: create-validate +` + path := filepath.Join(dir, "no-name.yaml") + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatalf("writing test file: %v", err) + } + + _, err := Load(path) + if err == nil { + t.Fatal("expected error for missing name") + } +} + +func TestLoadMissingCommand(t *testing.T) { + dir := t.TempDir() + content := ` +job: + name: no-command + trigger: + type: manual + scope: + type: item + compute: + type: validate +` + path := filepath.Join(dir, "no-cmd.yaml") + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatalf("writing test file: %v", err) + } + + _, err := Load(path) + if err == nil { + t.Fatal("expected error for missing command") + } +} + +func TestLoadAllDirectory(t *testing.T) { + dir := t.TempDir() + + job1 := ` +job: + name: job-one + trigger: + type: manual + scope: + type: item + compute: + type: validate + command: create-validate +` + job2 := ` +job: + name: job-two + trigger: + type: revision_created + scope: + type: assembly + compute: + type: export + command: create-export +` + if err := os.WriteFile(filepath.Join(dir, "one.yaml"), []byte(job1), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "two.yml"), []byte(job2), 0644); err != nil { + t.Fatal(err) + } + // Non-YAML file should be ignored + if err := os.WriteFile(filepath.Join(dir, "readme.txt"), []byte("ignore me"), 0644); err != nil { + t.Fatal(err) + } + + defs, err := LoadAll(dir) + if err != nil { + t.Fatalf("LoadAll: %v", err) + } + + if len(defs) != 2 { + t.Fatalf("loaded %d definitions, want 2", len(defs)) + } + if _, ok := defs["job-one"]; !ok { + t.Error("job-one not found") + } + if _, ok := defs["job-two"]; !ok { + t.Error("job-two not found") + } +} + +func TestLoadAllEmptyDirectory(t *testing.T) { + dir := t.TempDir() + + defs, err := LoadAll(dir) + if err != nil { + t.Fatalf("LoadAll: %v", err) + } + if len(defs) != 0 { + t.Errorf("loaded %d definitions from empty dir, want 0", len(defs)) + } +} + +func TestLoadWithFilter(t *testing.T) { + dir := t.TempDir() + content := ` +job: + name: filtered-job + trigger: + type: revision_created + filter: + item_type: assembly + scope: + type: assembly + compute: + type: validate + command: create-validate +` + path := filepath.Join(dir, "filtered.yaml") + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatalf("writing test file: %v", err) + } + + def, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + + if def.Trigger.Filter["item_type"] != "assembly" { + t.Errorf("filter item_type = %q, want %q", def.Trigger.Filter["item_type"], "assembly") + } +} + +func TestLoadWithArgs(t *testing.T) { + dir := t.TempDir() + content := ` +job: + name: args-job + trigger: + type: manual + scope: + type: item + compute: + type: export + command: create-export + args: + format: step + include_mesh: true +` + path := filepath.Join(dir, "args.yaml") + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatalf("writing test file: %v", err) + } + + def, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + + if def.Compute.Args["format"] != "step" { + t.Errorf("args format = %v, want %q", def.Compute.Args["format"], "step") + } + if def.Compute.Args["include_mesh"] != true { + t.Errorf("args include_mesh = %v, want true", def.Compute.Args["include_mesh"]) + } +} + +func TestValidateInvalidScopeType(t *testing.T) { + d := &Definition{ + Name: "test", + Trigger: TriggerConfig{Type: "manual"}, + Scope: ScopeConfig{Type: "galaxy"}, + Compute: ComputeConfig{Type: "validate", Command: "create-validate"}, + } + if err := d.Validate(); err == nil { + t.Fatal("expected error for invalid scope type") + } +} + +func TestValidateInvalidComputeType(t *testing.T) { + d := &Definition{ + Name: "test", + Trigger: TriggerConfig{Type: "manual"}, + Scope: ScopeConfig{Type: "item"}, + Compute: ComputeConfig{Type: "teleport", Command: "beam-up"}, + } + if err := d.Validate(); err == nil { + t.Fatal("expected error for invalid compute type") + } +} diff --git a/jobdefs/assembly-validate.yaml b/jobdefs/assembly-validate.yaml new file mode 100644 index 0000000..abbee0a --- /dev/null +++ b/jobdefs/assembly-validate.yaml @@ -0,0 +1,26 @@ +job: + name: assembly-validate + version: 1 + description: "Validate assembly by rebuilding its dependency subgraph" + + trigger: + type: revision_created + filter: + item_type: assembly + + scope: + type: assembly + + compute: + type: validate + command: create-validate + args: + rebuild_mode: incremental + check_interference: true + + runner: + tags: [create] + + timeout: 900 + max_retries: 2 + priority: 50 diff --git a/jobdefs/part-export-step.yaml b/jobdefs/part-export-step.yaml new file mode 100644 index 0000000..aacbff3 --- /dev/null +++ b/jobdefs/part-export-step.yaml @@ -0,0 +1,24 @@ +job: + name: part-export-step + version: 1 + description: "Export a part to STEP format" + + trigger: + type: manual + + scope: + type: item + + compute: + type: export + command: create-export + args: + format: step + output_key_template: "exports/{part_number}_rev{revision}.step" + + runner: + tags: [create] + + timeout: 300 + max_retries: 1 + priority: 100