diff --git a/AGENTS.md b/AGENTS.md index e751f90aa..f9f93840a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -107,6 +107,28 @@ make test-regression make test ``` +### Iterative E2E Development + +```bash +# Set up a persistent e2e cluster (does not tear down after tests) +make e2e-setup # Standard features +make experimental-e2e-setup # Experimental features + +# Run e2e scenarios against the running cluster +make e2e/install # All scenarios in install.feature +make e2e/install/Install # Scenarios starting with "Install" +make "e2e/install/Install latest" # Exact prefix with spaces +make e2e/install E2E_TIMEOUT=30m # Override timeout +make e2e/install KUBECONFIG=~/.kube/config # Override kubeconfig + +# Run against experimental cluster (override KUBECONFIG) +make e2e/install/Install KUBECONFIG=.kubeconfig/operator-controller-experimental-e2e.kubeconfig + +# Tear down the e2e cluster when done +make e2e-teardown # Standard cluster +make experimental-e2e-teardown # Experimental cluster +``` + ### Linting & Verification ```bash diff --git a/Makefile b/Makefile index b7ae06825..18d4ab65e 100644 --- a/Makefile +++ b/Makefile @@ -328,6 +328,15 @@ test-experimental-e2e: export DEFAULT_CATALOG := $(CATALOGS_MANIFEST) test-experimental-e2e: export INSTALL_DEFAULT_CATALOGS := false test-experimental-e2e: E2E_PROMETHEUS_VALUES := testdata/prometheus/values-experimental.yaml test-experimental-e2e: E2E_TIMEOUT ?= 25m +e2e-setup: SOURCE_MANIFEST := $(STANDARD_E2E_MANIFEST) +e2e-setup: export MANIFEST := $(STANDARD_RELEASE_MANIFEST) +e2e-setup: export DEFAULT_CATALOG := $(CATALOGS_MANIFEST) +e2e-setup: export INSTALL_DEFAULT_CATALOGS := false +experimental-e2e-setup: KIND_CONFIG := ./kind-config/kind-config-2node.yaml +experimental-e2e-setup: SOURCE_MANIFEST := $(EXPERIMENTAL_E2E_MANIFEST) +experimental-e2e-setup: export MANIFEST := $(EXPERIMENTAL_RELEASE_MANIFEST) +experimental-e2e-setup: export DEFAULT_CATALOG := $(CATALOGS_MANIFEST) +experimental-e2e-setup: export INSTALL_DEFAULT_CATALOGS := false E2E_KUBECONFIG = $(KUBECONFIG_DIR)/$*.kubeconfig @@ -436,6 +445,26 @@ test-e2e: e2e-coverage-operator-controller-e2e #HELP Run e2e test suite on local test-experimental-e2e: e2e-coverage-operator-controller-experimental-e2e #HELP Run experimental e2e test suite on local kind cluster -$(KIND) delete cluster --name operator-controller-experimental-e2e +.PHONY: e2e-setup +e2e-setup: wait-operator-controller-e2e #EXHELP Create a KIND cluster with standard OLM deployed for iterative e2e testing. + +.PHONY: experimental-e2e-setup +experimental-e2e-setup: wait-operator-controller-experimental-e2e #EXHELP Create a KIND cluster with experimental OLM deployed for iterative e2e testing. + +.PHONY: e2e-teardown experimental-e2e-teardown +e2e-teardown: kind-clean-operator-controller-e2e #EXHELP Delete the standard e2e KIND cluster. +experimental-e2e-teardown: kind-clean-operator-controller-experimental-e2e #EXHELP Delete the experimental e2e KIND cluster. + +.PHONY: e2e/% +e2e/%: E2E_TIMEOUT ?= 20m +e2e/%: KUBECONFIG ?= $(KUBECONFIG_DIR)/operator-controller-e2e.kubeconfig +e2e/%: #EXHELP Run e2e scenario against a cluster created by e2e-setup or experimental-e2e-setup (tear down with e2e-teardown). + @feature=$$(echo "$*" | cut -d/ -f1); \ + scenario=$$(echo "$*" | cut -d/ -f2-); \ + if [ "$$scenario" = "$$feature" ]; then scenario=""; fi; \ + KUBECONFIG=$(KUBECONFIG) go test -count=1 -v ./test/e2e/features_test.go \ + -timeout $(E2E_TIMEOUT) \ + -args --godog.concurrency=1 --e2e.scenario="$$scenario" "features/$$feature.feature" .PHONY: test-extension-developer-e2e test-extension-developer-e2e: SOURCE_MANIFEST := $(STANDARD_E2E_MANIFEST) diff --git a/go.mod b/go.mod index 37682f515..805775096 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,9 @@ require ( github.com/blang/semver/v4 v4.0.0 github.com/cert-manager/cert-manager v1.20.2 github.com/containerd/containerd v1.7.33 + github.com/cucumber/gherkin/go/v26 v26.2.0 github.com/cucumber/godog v0.15.1 + github.com/cucumber/messages/go/v21 v21.0.1 github.com/evanphx/json-patch v5.9.11+incompatible github.com/fsnotify/fsnotify v1.10.1 github.com/go-logr/logr v1.4.3 @@ -91,8 +93,6 @@ require ( github.com/containerd/typeurl/v2 v2.2.3 // indirect github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 // indirect github.com/containers/ocicrypt v1.3.0 // indirect - github.com/cucumber/gherkin/go/v26 v26.2.0 // indirect - github.com/cucumber/messages/go/v21 v21.0.1 // indirect github.com/cyberphone/json-canonicalization v0.0.0-20241213102144-19d51d7fe467 // indirect github.com/cyphar/filepath-securejoin v0.6.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect diff --git a/test/e2e/README.md b/test/e2e/README.md index 8997948db..60a7b79d7 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -270,6 +270,55 @@ Note that when this is done the `make` target will no longer automatically split GODOG_ARGS="--godog.tags=~@Serial --godog.concurrency=100" make test-experimental-e2e ``` +### Iterative Development + +For iterating on individual scenarios without full suite setup/teardown each time, +use the persistent cluster and single-scenario targets: + +**1. Set up a persistent cluster (once):** + +```bash +make e2e-setup # Standard features +make experimental-e2e-setup # Experimental features +``` + +This builds images, creates a KIND cluster, deploys OLM, and waits for readiness. +The cluster persists until explicitly torn down. + +**2. Run individual scenarios:** + +```bash +# Run all scenarios in a feature file +make e2e/install + +# Run scenarios matching a name prefix (case-insensitive) +make e2e/install/Install # all "Install ..." scenarios +make "e2e/install/Install latest" # prefix with spaces (use quotes) +make e2e/install/Boxcutter # single matching scenario + +# Override timeout or kubeconfig +make e2e/install/Install E2E_TIMEOUT=30m +make e2e/install KUBECONFIG=~/.kube/config +``` + +The prefix matches scenario names from the start. If multiple scenarios match, all +of them run. If no scenario matches, the command fails with a list of available +scenario names. + +When using `experimental-e2e-setup`, override `KUBECONFIG` to point at the +experimental cluster: + +```bash +make e2e/install/Install KUBECONFIG=.kubeconfig/operator-controller-experimental-e2e.kubeconfig +``` + +**3. Tear down when done:** + +```bash +make e2e-teardown # Standard cluster +make experimental-e2e-teardown # Experimental cluster +``` + ### Run Specific Feature ```bash @@ -302,6 +351,8 @@ Available formats: `pretty`, `cucumber`, `progress`, `junit` **Custom Flags:** +- `--e2e.scenario=`: Run scenarios whose name starts with the given prefix (case-insensitive). + Used by `make e2e//` internally. - `--log.debug`: Enable debug logging (development mode) - `--k8s.cli=`: Specify path to Kubernetes CLI (default: `kubectl`) - Useful for using `oc` or a specific kubectl binary diff --git a/test/e2e/features_test.go b/test/e2e/features_test.go index 81f1a0934..05e91b4cf 100644 --- a/test/e2e/features_test.go +++ b/test/e2e/features_test.go @@ -4,10 +4,13 @@ import ( "fmt" "log" "os" + "strings" "testing" + gherkin "github.com/cucumber/gherkin/go/v26" "github.com/cucumber/godog" "github.com/cucumber/godog/colors" + messages "github.com/cucumber/messages/go/v21" "github.com/spf13/pflag" "github.com/operator-framework/operator-controller/test/e2e/steps" @@ -23,8 +26,11 @@ var opts = godog.Options{ Strict: true, } +var scenarioFilter string + func init() { godog.BindCommandLineFlags("godog.", &opts) + pflag.StringVar(&scenarioFilter, "e2e.scenario", "", "scenario name prefix (case-insensitive)") } func TestMain(m *testing.M) { @@ -32,6 +38,21 @@ func TestMain(m *testing.M) { pflag.Parse() opts.Paths = pflag.Args() + if scenarioFilter != "" { + if len(opts.Paths) != 1 { + log.Fatalf("--e2e.scenario requires exactly one feature file path, got %d", len(opts.Paths)) + } + lines, err := findScenarioFirstLineNumberByPrefix(opts.Paths[0], scenarioFilter) + if err != nil { + log.Fatal(err) + } + basePath := opts.Paths[0] + opts.Paths = make([]string, len(lines)) + for i, line := range lines { + opts.Paths[i] = fmt.Sprintf("%s:%d", basePath, line) + } + } + // run tests sc := godog.TestSuite{ TestSuiteInitializer: InitializeSuite, @@ -62,6 +83,56 @@ func TestMain(m *testing.M) { } } +func findScenarioFirstLineNumberByPrefix(featurePath, prefix string) ([]int, error) { + f, err := os.Open(featurePath) + if err != nil { + return nil, fmt.Errorf("failed to open %s: %w", featurePath, err) + } + defer f.Close() + + doc, err := gherkin.ParseGherkinDocument(f, (&messages.Incrementing{}).NewId) + if err != nil { + return nil, fmt.Errorf("failed to parse %s: %w", featurePath, err) + } + + if doc.Feature == nil { + return nil, fmt.Errorf("no Feature found in %s", featurePath) + } + + prefix = strings.TrimSpace(prefix) + if prefix == "" { + return nil, fmt.Errorf("scenario prefix must not be empty") + } + prefix = strings.ToLower(prefix) + var matches []int + var allNames []string + + matchScenario := func(sc *messages.Scenario) { + allNames = append(allNames, sc.Name) + if strings.HasPrefix(strings.ToLower(sc.Name), prefix) { + matches = append(matches, int(sc.Location.Line)) + } + } + for _, child := range doc.Feature.Children { + if child.Scenario != nil { + matchScenario(child.Scenario) + } + if child.Rule != nil { + for _, rc := range child.Rule.Children { + if rc.Scenario != nil { + matchScenario(rc.Scenario) + } + } + } + } + + if len(matches) == 0 { + return nil, fmt.Errorf("no scenario matching prefix %q in %s\navailable scenarios:\n %s", + prefix, featurePath, strings.Join(allNames, "\n ")) + } + return matches, nil +} + func InitializeSuite(tc *godog.TestSuiteContext) { tc.BeforeSuite(steps.BeforeSuite) }