feat(test): add --output (text/json/yaml/github-actions) for machine-readable results

README.md

@@ -159,6 +159,51 @@ $ docker run -it quay.io/microcks/microcks-cli:latest microcks test 'Beer Catalo
 ```
 
 
+## Machine-readable test output
+
+`microcks test` accepts `--output` to control how the result is rendered:
+
+| Value | Output |
+| --- | --- |
+| `text` (default) | Human-readable summary |
+| `json` | The full `TestResult` as JSON |
+| `yaml` | The full `TestResult` as YAML |
+| `github-actions` | GitHub Actions workflow commands (annotations + log groups + step summary) |
+
+For machine formats (`json`/`yaml`/`github-actions`), progress goes to **stderr**
+and only the formatted result is written to **stdout**, so it can be piped or
+parsed cleanly:
+
+```bash
+microcks test "Pastry API:1.0.0" http://localhost:8080/api OPEN_API_SCHEMA \
+  --microcksURL=http://localhost:8585/api --output=json > result.json
+```
+
+### GitHub Actions
+
+With `--output=github-actions`, failures surface as `::error::` annotations,
+each operation is wrapped in a collapsible `::group::`, and a per-operation table
+is appended to the job summary (`$GITHUB_STEP_SUMMARY`). Set
+`MICROCKS_ACTIONS_VERBOSE=true` to also emit `::notice::` for passing operations.
+
+```yaml
+# .github/workflows/contract-test.yml
+name: contract-test
+on: [pull_request]
+jobs:
+  contract-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Microcks contract test
+        run: |
+          microcks test "Pastry API:1.0.0" "${{ env.API_URL }}" OPEN_API_SCHEMA \
+            --microcksURL=${{ secrets.MICROCKS_URL }} \
+            --keycloakClientId=${{ secrets.MICROCKS_CLIENT_ID }} \
+            --keycloakClientSecret=${{ secrets.MICROCKS_CLIENT_SECRET }} \
+            --output=github-actions
+```
+
 ## Tekton tasks
 
 This repository also contains different [Tekton](https://tekton.dev/) tasks definitions and sample pipelines. You'll find under the `/tekton` folder the resource for current `v1beta1` Tekton API version and the older `v1alpha1` under `tekton/v1alpha1`.

cmd/test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/microcks/microcks-cli/pkg/config"
 	"github.com/microcks/microcks-cli/pkg/connectors"
 	"github.com/microcks/microcks-cli/pkg/errors"
+	"github.com/microcks/microcks-cli/pkg/output"
 	"github.com/spf13/cobra"
 )
 
@@ -45,6 +46,7 @@ func NewTestCommand(globalClientOpts *connectors.ClientOptions) *cobra.Command {
 		readyTimeout       time.Duration
 		watch              bool
 		driver             string
+		outputFormat       string
 	)
 	var testCmd = &cobra.Command{
 
@@ -82,6 +84,11 @@ func NewTestCommand(globalClientOpts *connectors.ClientOptions) *cobra.Command {
 				os.Exit(1)
 			}
 
+			if !output.IsValid(outputFormat) {
+				fmt.Fprintln(os.Stderr, "--output must be one of: text, json, yaml, github-actions")
+				os.Exit(1)
+			}
+
 			// Collect optional HTTPS transport flags.
 			config.InsecureTLS = globalClientOpts.InsecureTLS
 			config.CaCertPaths = globalClientOpts.CaCertPaths
@@ -121,6 +128,8 @@ func NewTestCommand(globalClientOpts *connectors.ClientOptions) *cobra.Command {
 				filteredOperations: filteredOperations,
 				operationsHeaders:  operationsHeaders,
 				oAuth2Context:      oAuth2Context,
+				outputFormat:       outputFormat,
+				artifactPath:       artifact,
 			}
 
 			if !dryRun {
@@ -214,11 +223,11 @@ func NewTestCommand(globalClientOpts *connectors.ClientOptions) *cobra.Command {
 
 			success, testResultID, err := runTestAndWait(mc, params)
 			if err != nil {
-				fmt.Print(err)
+				fmt.Fprintln(os.Stderr, err)
 				os.Exit(1)
 			}
 
-			fmt.Printf("Full TestResult details are available here: %s/#/tests/%s \n", serverAddr, testResultID)
+			fmt.Fprintf(progressWriter(outputFormat), "Full TestResult details are available here: %s/#/tests/%s \n", serverAddr, testResultID)
 
 			if !success {
 				os.Exit(1)
@@ -237,6 +246,7 @@ func NewTestCommand(globalClientOpts *connectors.ClientOptions) *cobra.Command {
 	testCmd.Flags().DurationVar(&readyTimeout, "ready-timeout", 90*time.Second, "How long to wait for the ephemeral container to be ready (--dry-run only)")
 	testCmd.Flags().BoolVar(&watch, "watch", false, "Watch the artifact file and re-run the test on change (--dry-run only)")
 	testCmd.Flags().StringVar(&driver, "driver", "", "Container runtime for --dry-run: 'docker' or 'podman' (default: auto-detect)")
+	testCmd.Flags().StringVar(&outputFormat, "output", "text", "Output format: text, json, yaml, or github-actions")
 
 	return testCmd
 }

cmd/testDryRun.go

@@ -18,6 +18,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"io"
 	"net/url"
 	"os"
 	"os/exec"
@@ -68,8 +69,6 @@ func configureDriver(driver string) error {
 	}
 }
 
-// shouldUsePodman auto-detects podman only when it's clearly the intended
-// runtime: no explicit DOCKER_HOST, podman on PATH, and docker absent.
 func shouldUsePodman() bool {
 	if os.Getenv("DOCKER_HOST") != "" {
 		return false // respect an explicitly configured endpoint
@@ -142,14 +141,18 @@ func rewriteLocalEndpoint(testEndpoint string) (string, int, bool) {
 }
 
 func runDryRunTest(opts dryRunOptions) bool {
+	// Progress/diagnostics go to stderr for machine output formats so stdout
+	// carries only the formatted result.
+	progress := progressWriter(opts.params.outputFormat)
+
 	if err := validateDryRunOptions(opts); err != nil {
-		fmt.Println(err)
+		fmt.Fprintln(os.Stderr, err)
 		return false
 	}
 
 	// Select the container runtime (docker default, podman wired via DOCKER_HOST).
 	if err := configureDriver(opts.driver); err != nil {
-		fmt.Println(err)
+		fmt.Fprintln(os.Stderr, err)
 		return false
 	}
 
@@ -164,32 +167,32 @@ func runDryRunTest(opts dryRunOptions) bool {
 	// A localhost test endpoint refers to the user's machine, not the
 	// container: expose the port and point Microcks at the host gateway.
 	if rewritten, hostPort, ok := rewriteLocalEndpoint(opts.params.testEndpoint); ok {
-		fmt.Printf("Test endpoint %s is local: reaching it from the container as %s\n", opts.params.testEndpoint, rewritten)
+		fmt.Fprintf(progress, "Test endpoint %s is local: reaching it from the container as %s\n", opts.params.testEndpoint, rewritten)
 		opts.params.testEndpoint = rewritten
 		containerOpts = append(containerOpts, testcontainers.WithHostPortAccess(hostPort))
 	}
 
-	fmt.Printf("Starting ephemeral Microcks container (%s)...\n", opts.image)
+	fmt.Fprintf(progress, "Starting ephemeral Microcks container (%s)...\n", opts.image)
 	startCtx, startCancel := context.WithTimeout(ctx, opts.readyTimeout)
 	defer startCancel()
 
 	container, err := microcks.Run(startCtx, opts.image, containerOpts...)
 	if err != nil {
-		fmt.Printf("Failed to start ephemeral Microcks container: %s\n", err)
-		fmt.Println("Check that the container runtime is running, the port is free and the image is reachable (or raise --ready-timeout).")
+		fmt.Fprintf(os.Stderr, "Failed to start ephemeral Microcks container: %s\n", err)
+		fmt.Fprintln(os.Stderr, "Check that the container runtime is running, the port is free and the image is reachable (or raise --ready-timeout).")
 		if container != nil {
-			terminateContainer(container)
+			terminateContainer(container, progress)
 		}
 		return false
 	}
-	defer terminateContainer(container)
+	defer terminateContainer(container, progress)
 
 	endpoint, err := container.HttpEndpoint(ctx)
 	if err != nil {
-		fmt.Printf("Failed to resolve ephemeral Microcks endpoint: %s\n", err)
+		fmt.Fprintf(os.Stderr, "Failed to resolve ephemeral Microcks endpoint: %s\n", err)
 		return false
 	}
-	fmt.Printf("Ephemeral Microcks is ready at %s\n", endpoint)
+	fmt.Fprintf(progress, "Ephemeral Microcks is ready at %s\n", endpoint)
 
 	// The uber-native image runs without Keycloak: a headless client with
 	// the unauthenticated token is enough.
@@ -198,30 +201,32 @@ func runDryRunTest(opts dryRunOptions) bool {
 
 	success, testResultID, err := runTestAndWait(mc, opts.params)
 	if err != nil {
-		fmt.Println(err)
+		fmt.Fprintln(os.Stderr, err)
 		return false
 	}
 
 	if !opts.watch {
 		return success
 	}
-	printDetailsLink(endpoint, testResultID)
+	printDetailsLink(progress, endpoint, testResultID)
 	return watchAndRerun(ctx, mc, endpoint, opts)
 }
 
-func terminateContainer(container *microcks.MicrocksContainer) {
+func terminateContainer(container *microcks.MicrocksContainer, progress io.Writer) {
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
-	fmt.Println("Tearing down ephemeral Microcks container...")
+	fmt.Fprintln(progress, "Tearing down ephemeral Microcks container...")
 	if err := container.Terminate(ctx); err != nil {
-		fmt.Printf("Failed to terminate container %s: %s\n", container.GetContainerID(), err)
+		fmt.Fprintf(os.Stderr, "Failed to terminate container %s: %s\n", container.GetContainerID(), err)
 	}
 }
 
 func watchAndRerun(ctx context.Context, mc connectors.MicrocksClient, serverAddr string, opts dryRunOptions) bool {
+	progress := progressWriter(opts.params.outputFormat)
+
 	watcher, err := fsnotify.NewWatcher()
 	if err != nil {
-		fmt.Printf("Failed to create file watcher: %s\n", err)
+		fmt.Fprintf(os.Stderr, "Failed to create file watcher: %s\n", err)
 		return false
 	}
 	defer watcher.Close()
@@ -230,23 +235,23 @@ func watchAndRerun(ctx context.Context, mc connectors.MicrocksClient, serverAddr
 	// (rename + create), which silently drops a watch set on the file itself.
 	artifactPath, err := filepath.Abs(opts.artifact)
 	if err != nil {
-		fmt.Printf("Failed to resolve artifact path: %s\n", err)
+		fmt.Fprintf(os.Stderr, "Failed to resolve artifact path: %s\n", err)
 		return false
 	}
 	if err := watcher.Add(filepath.Dir(artifactPath)); err != nil {
-		fmt.Printf("Failed to watch %s: %s\n", filepath.Dir(artifactPath), err)
+		fmt.Fprintf(os.Stderr, "Failed to watch %s: %s\n", filepath.Dir(artifactPath), err)
 		return false
 	}
 
-	fmt.Printf("\nWatching %s for changes — press Ctrl+C to stop.\n", opts.artifact)
+	fmt.Fprintf(progress, "\nWatching %s for changes — press Ctrl+C to stop.\n", opts.artifact)
 
 	rerun := make(chan struct{}, 1)
 	var debounce *time.Timer
 
 	for {
 		select {
 		case <-ctx.Done():
-			fmt.Println("\nStopping watch mode.")
+			fmt.Fprintln(progress, "\nStopping watch mode.")
 			return true
 
 		case event, ok := <-watcher.Events:
@@ -275,32 +280,32 @@ func watchAndRerun(ctx context.Context, mc connectors.MicrocksClient, serverAddr
 			if !ok {
 				return true
 			}
-			fmt.Printf("Watch error: %s\n", err)
+			fmt.Fprintf(os.Stderr, "Watch error: %s\n", err)
 
 		case <-rerun:
-			fmt.Println(strings.Repeat("-", 60))
-			fmt.Printf("Artifact changed, re-importing %s ...\n", opts.artifact)
+			fmt.Fprintln(progress, strings.Repeat("-", 60))
+			fmt.Fprintf(progress, "Artifact changed, re-importing %s ...\n", opts.artifact)
 			if _, err := mc.UploadArtifact(opts.artifact, true); err != nil {
 				// Invalid spec mid-edit is normal in a TDD loop: report and
 				// keep watching, the next valid save recovers.
-				fmt.Printf("Re-import failed, waiting for next change: %s\n", err)
+				fmt.Fprintf(os.Stderr, "Re-import failed, waiting for next change: %s\n", err)
 				continue
 			}
 			success, testResultID, err := runTestAndWait(mc, opts.params)
 			if err != nil {
-				fmt.Printf("Test run failed, waiting for next change: %s\n", err)
+				fmt.Fprintf(os.Stderr, "Test run failed, waiting for next change: %s\n", err)
 				continue
 			}
-			printDetailsLink(serverAddr, testResultID)
+			printDetailsLink(progress, serverAddr, testResultID)
 			if success {
-				fmt.Println("Contract test PASSED — waiting for next change.")
+				fmt.Fprintln(progress, "Contract test PASSED — waiting for next change.")
 			} else {
-				fmt.Println("Contract test FAILED — waiting for next change.")
+				fmt.Fprintln(progress, "Contract test FAILED — waiting for next change.")
 			}
 		}
 	}
 }
 
-func printDetailsLink(serverAddr, testResultID string) {
-	fmt.Printf("Test details (live while watching): %s/#/tests/%s\n", serverAddr, testResultID)
+func printDetailsLink(progress io.Writer, serverAddr, testResultID string) {
+	fmt.Fprintf(progress, "Test details (live while watching): %s/#/tests/%s\n", serverAddr, testResultID)
 }

cmd/testExecutor.go

@@ -17,9 +17,12 @@ package cmd
 
 import (
 	"fmt"
+	"io"
+	"os"
 	"time"
 
 	"github.com/microcks/microcks-cli/pkg/connectors"
+	"github.com/microcks/microcks-cli/pkg/output"
 )
 
 // testParams bundles the inputs needed to launch and poll a Microcks test.
@@ -33,11 +36,27 @@ type testParams struct {
 	filteredOperations string
 	operationsHeaders  string
 	oAuth2Context      string
+	outputFormat       string
+	artifactPath       string
 }
 
-// runTestAndWait creates a test on the Microcks server and polls its result
-// until completion or timeout. Shared by the regular and --dry-run paths.
+// progressWriter returns where human progress/diagnostics should go. For
+// machine-readable output formats they go to stderr, leaving stdout for the
+// formatted result only.
+func progressWriter(format string) io.Writer {
+	if format != "" && output.OutputFormat(format) != output.FormatText {
+		return os.Stderr
+	}
+	return os.Stdout
+}
+
+// runTestAndWait creates a test on the Microcks server, polls until completion
+// or timeout, then renders the result in the requested output format (result to
+// stdout, progress to stderr for machine formats). Shared by the regular and
+// --dry-run paths.
 func runTestAndWait(mc connectors.MicrocksClient, params testParams) (bool, string, error) {
+	progress := progressWriter(params.outputFormat)
+
 	testResultID, err := mc.CreateTestResult(params.serviceRef, params.testEndpoint, params.runnerType, params.secretName,
 		params.waitForMillis, params.filteredOperations, params.operationsHeaders, params.oAuth2Context)
 	if err != nil {
@@ -59,19 +78,48 @@ func runTestAndWait(mc connectors.MicrocksClient, params testParams) (bool, stri
 		}
 		success = testResultSummary.Success
 		inProgress := testResultSummary.InProgress
-		fmt.Printf("MicrocksClient got status for test \"%s\" - success: %s, inProgress: %s \n", testResultID, fmt.Sprint(success), fmt.Sprint(inProgress))
+		fmt.Fprintf(progress, "MicrocksClient got status for test \"%s\" - success: %s, inProgress: %s \n", testResultID, fmt.Sprint(success), fmt.Sprint(inProgress))
 
 		if !inProgress {
 			break
 		}
 
-		fmt.Println("MicrocksTester waiting for 2 seconds before checking again or exiting.")
+		fmt.Fprintln(progress, "MicrocksTester waiting for 2 seconds before checking again or exiting.")
 		time.Sleep(2 * time.Second)
 	}
 
+	if err := renderTestResult(mc, testResultID, params.outputFormat, params.artifactPath); err != nil {
+		return false, testResultID, err
+	}
+
 	return success, testResultID, nil
 }
 
+// renderTestResult fetches the full result and writes it to stdout in the
+// requested format. artifactPath (when set) lets the github-actions formatter
+// map failures to file:line.
+func renderTestResult(mc connectors.MicrocksClient, testResultID, format, artifactPath string) error {
+	if format == "" {
+		format = string(output.FormatText)
+	}
+	full, err := mc.GetFullTestResult(testResultID)
+	if err != nil {
+		return fmt.Errorf("Got error when retrieving full test result: %s", err)
+	}
+	formatter, err := output.NewFormatter(output.OutputFormat(format), output.WithArtifactPath(artifactPath))
+	if err != nil {
+		return err
+	}
+	rendered, err := formatter.Format(full)
+	if err != nil {
+		return fmt.Errorf("Got error when formatting test result: %s", err)
+	}
+	if rendered != "" {
+		fmt.Println(rendered)
+	}
+	return nil
+}
+
 func nowInMilliseconds() int64 {
 	return time.Now().UnixNano() / int64(time.Millisecond)
 }