diff --git a/README.md b/README.md index cff57558..81464d2e 100644 --- a/README.md +++ b/README.md @@ -80,8 +80,15 @@ To run these integration tests locally: docker pull ghcr.io/coder/coder-preview:main-x.y.z-devel-abcd1234 ``` +1. On macOS, build the provider for Linux, since the binary runs inside the Coder container. For example `GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build .` on Apple silicon. Repeat this after editing provider sources, otherwise `make` rebuilds the binary for macOS. + 1. Run `CODER_IMAGE=ghcr.io/coder/coder-preview CODER_VERSION=main-x.y.z-devel-abcd1234 make test-integration`. +The four agent cases (`coder-script-order`, `-skip`, `-stop` and `-cancel`) start a workspace agent inside the Coder container and read its results through the API. +While a run is in progress the agent's log is at `/tmp/-agent.log` inside the container. +They run only when `CODER_VERSION` is `latest` or a release tag at or above `v2.39.0`, the first release that enforces script ordering. A preview tag like the one above does not compare as a release and skips them, so use `CODER_VERSION=latest` to run them against the newest preview image. +The `coder-script-order-compat` case runs on every image. + > [!Note] > You can specify `CODER_IMAGE` if the Coder image you wish to test is hosted somewhere other than `ghcr.io/coder/coder`. > For example, `CODER_IMAGE=example.com/repo/coder CODER_VERSION=foobar make test-integration`. diff --git a/go.mod b/go.mod index 3b175959..a3d84d3d 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,6 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/robfig/cron/v3 v3.0.1 github.com/stretchr/testify v1.11.1 - golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 golang.org/x/mod v0.38.0 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 ) diff --git a/go.sum b/go.sum index be5dbdd5..52a811ad 100644 --- a/go.sum +++ b/go.sum @@ -230,8 +230,6 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= -golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= -golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= diff --git a/integration/agent_test.go b/integration/agent_test.go new file mode 100644 index 00000000..74d324a1 --- /dev/null +++ b/integration/agent_test.go @@ -0,0 +1,241 @@ +package integration + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// These mirror the codersdk types of the same name, keeping only the fields +// the script ordering cases read. The package does not import codersdk, see +// the note on TestIntegration. +type workspaceAgentScriptDependency struct { + PrerequisiteResourceAddress string `json:"prerequisite_resource_address"` + Requirement string `json:"requirement"` +} + +type workspaceAgentScript struct { + DisplayName string `json:"display_name"` + LogSourceID string `json:"log_source_id"` + ResourceAddress string `json:"resource_address"` + Dependencies []workspaceAgentScriptDependency `json:"dependencies"` +} + +type workspaceAgent struct { + LifecycleState string `json:"lifecycle_state"` + Scripts []workspaceAgentScript `json:"scripts"` +} + +type agentScriptTiming struct { + StartedAt time.Time `json:"started_at"` + EndedAt time.Time `json:"ended_at"` + ExitCode int32 `json:"exit_code"` + Stage string `json:"stage"` + Status string `json:"status"` + DisplayName string `json:"display_name"` +} + +type workspaceBuildTimings struct { + AgentScriptTimings []agentScriptTiming `json:"agent_script_timings"` +} + +const ( + agentWaitTimeout = 2 * time.Minute + reportWaitTimeout = 30 * time.Second +) + +// startAgent runs coder agent in the background inside the Coder container +// for the workspace the named case created. +func startAgent(ctx context.Context, t *testing.T, ctrID, name string) { + t.Helper() + // The agent reads its token from the file the template wrote, so the + // token never appears in the command the harness logs. The subshell + // execs the agent so the PID file holds the agent's own PID. + cmd := fmt.Sprintf( + `mkdir -p /tmp/%[1]s-agent && `+ + `( exec env CODER_AGENT_URL=http://localhost:3000 `+ + `CODER_AGENT_TOKEN_FILE=/tmp/%[1]s.agent-token `+ + `CODER_AGENT_LOG_DIR=/tmp/%[1]s-agent `+ + `CODER_AGENT_SCRIPT_DATA_DIR=/tmp/%[1]s-agent `+ + `coder agent >/tmp/%[1]s-agent.log 2>&1 ) & `+ + `echo $! >/tmp/%[1]s-agent.pid`, + name) + _, rc := execContainer(ctx, t, ctrID, cmd) + require.Equal(t, 0, rc, "start agent for %s", name) +} + +// stopAgent sends SIGTERM to the case's agent. +func stopAgent(ctx context.Context, t *testing.T, ctrID, name string) { + t.Helper() + // Never signal the process by name: the server in the same container is + // the same binary. + _, rc := execContainer(ctx, t, ctrID, fmt.Sprintf("kill -TERM $(cat /tmp/%s-agent.pid)", name)) + require.Equal(t, 0, rc, "stop agent for %s", name) +} + +// agentLogTail returns the last lines of the agent's own log file. +func agentLogTail(ctx context.Context, t *testing.T, ctrID, name string) string { + t.Helper() + tail, _ := execContainer(ctx, t, ctrID, fmt.Sprintf("tail -n 50 /tmp/%s-agent.log", name)) + return tail +} + +// apiGet performs a GET against the Coder in the container as the workspace +// owner and decodes the JSON body into out. +func apiGet(ctx context.Context, t *testing.T, ctrID, name, path string, out any) error { + t.Helper() + // The session token is read by the container's shell at the moment of + // the call, so it never appears in the command the harness logs. + cmd := fmt.Sprintf( + `curl -s -w '\n%%{http_code}' -H "Coder-Session-Token: $(cat /tmp/%s.session-token)" http://localhost:3000%s`, + name, path) + // Callers poll from testify's goroutine, where a failed require would + // end the poll early instead of the test. + body, rc, err := tryExecContainer(ctx, t, ctrID, cmd) + if err != nil { + return fmt.Errorf("GET %s: %w", path, err) + } + if rc != 0 { + return fmt.Errorf("GET %s: exit %d: %s", path, rc, body) + } + i := strings.LastIndex(body, "\n") + if i < 0 { + return fmt.Errorf("GET %s: no status line in %q", path, body) + } + status, body := body[i+1:], body[:i] + if status != "200" { + return fmt.Errorf("GET %s: HTTP %s: %s", path, status, body) + } + if err := json.Unmarshal([]byte(body), out); err != nil { + return fmt.Errorf("decode GET %s: %w: %s", path, err, body) + } + return nil +} + +// agentIDFor returns the ID coderd assigned to the workspace's one agent. +func agentIDFor(ctx context.Context, t *testing.T, ctrID, name, workspaceID string) string { + t.Helper() + // coderd generates its own agent ID on insert, so the ID in the + // Terraform state is not the one the API knows. + var ws struct { + LatestBuild struct { + Resources []struct { + Agents []struct { + ID string `json:"id"` + } `json:"agents"` + } `json:"resources"` + } `json:"latest_build"` + } + require.NoError(t, apiGet(ctx, t, ctrID, name, "/api/v2/workspaces/"+workspaceID, &ws)) + var ids []string + for _, r := range ws.LatestBuild.Resources { + for _, a := range r.Agents { + ids = append(ids, a.ID) + } + } + require.Lenf(t, ids, 1, "expected one agent on workspace %s, got %v", workspaceID, ids) + return ids[0] +} + +// waitForAgent polls the agent until its lifecycle state is one of want and +// returns the last response. +func waitForAgent(ctx context.Context, t *testing.T, ctrID, name, agentID string, want ...string) workspaceAgent { + t.Helper() + deadline := time.Now().Add(agentWaitTimeout) + var agent workspaceAgent + var lastErr error + for time.Now().Before(deadline) { + lastErr = apiGet(ctx, t, ctrID, name, "/api/v2/workspaceagents/"+agentID, &agent) + if lastErr == nil && slices.Contains(want, agent.LifecycleState) { + return agent + } + time.Sleep(time.Second) + } + t.Fatalf("agent %s did not reach %v in time: last state %q, last error %v\nagent log tail:\n%s", + name, want, agent.LifecycleState, lastErr, agentLogTail(ctx, t, ctrID, name)) + return agent +} + +// requireLifecycle fails with the agent log tail when the agent ended in a +// different state, since the log names the script that caused it. +func requireLifecycle(ctx context.Context, t *testing.T, ctrID, name string, agent workspaceAgent, want string) { + t.Helper() + if agent.LifecycleState != want { + t.Fatalf("agent %s ended in %q, want %q\nagent log tail:\n%s", + name, agent.LifecycleState, want, agentLogTail(ctx, t, ctrID, name)) + } +} + +// waitForTimings polls the latest build's timings until every named script +// has one timing for the stage. +func waitForTimings(ctx context.Context, t *testing.T, ctrID, name, workspaceID, stage string, displayNames ...string) workspaceBuildTimings { + t.Helper() + var ws struct { + LatestBuild struct { + ID string `json:"id"` + } `json:"latest_build"` + } + require.NoError(t, apiGet(ctx, t, ctrID, name, "/api/v2/workspaces/"+workspaceID, &ws)) + // The agent reports its lifecycle state before the goroutines that send + // each timing have finished, so the first read after a state change can + // be empty. + var timings workspaceBuildTimings + require.EventuallyWithT(t, func(c *assert.CollectT) { + timings = workspaceBuildTimings{} + if !assert.NoError(c, apiGet(ctx, t, ctrID, name, "/api/v2/workspacebuilds/"+ws.LatestBuild.ID+"/timings", &timings)) { + return + } + for _, dn := range displayNames { + n := 0 + for _, tm := range timings.AgentScriptTimings { + if tm.Stage == stage && tm.DisplayName == dn { + n++ + } + } + assert.Equalf(c, 1, n, "expected one %s timing for %q, got %+v", stage, dn, timings.AgentScriptTimings) + } + }, reportWaitTimeout, time.Second) + return timings +} + +// timingFor returns the one timing with the given stage and display name. +func timingFor(t *testing.T, timings workspaceBuildTimings, stage, displayName string) agentScriptTiming { + t.Helper() + var found []agentScriptTiming + for _, tm := range timings.AgentScriptTimings { + if tm.Stage == stage && tm.DisplayName == displayName { + found = append(found, tm) + } + } + require.Lenf(t, found, 1, "expected one %s timing for %q, got %+v", stage, displayName, found) + return found[0] +} + +func requireStatus(t *testing.T, tm agentScriptTiming, want string) { + t.Helper() + require.Equalf(t, want, tm.Status, "%s: %+v", tm.DisplayName, tm) +} + +func requireExitCode(t *testing.T, tm agentScriptTiming, want int32) { + t.Helper() + require.Equalf(t, want, tm.ExitCode, "%s: %+v", tm.DisplayName, tm) +} + +// scriptFor fails the test if no script has displayName. +func scriptFor(t *testing.T, agent workspaceAgent, displayName string) workspaceAgentScript { + t.Helper() + for _, s := range agent.Scripts { + if s.DisplayName == displayName { + return s + } + } + t.Fatalf("no script named %q in %+v", displayName, agent.Scripts) + return workspaceAgentScript{} +} diff --git a/integration/coder-script-order-cancel/main.tf b/integration/coder-script-order-cancel/main.tf new file mode 100644 index 00000000..a1ec5cc7 --- /dev/null +++ b/integration/coder-script-order-cancel/main.tf @@ -0,0 +1,105 @@ +terraform { + required_providers { + coder = { + source = "coder/coder" + } + local = { + source = "hashicorp/local" + } + } +} + +data "coder_provisioner" "me" {} +data "coder_workspace" "me" {} +data "coder_workspace_owner" "me" {} + +resource "coder_agent" "main" { + os = "linux" + arch = data.coder_provisioner.me.arch +} + +variable "output_path" { + type = string +} + +# Every file the test reads is named after the output path, so a copied +# template cannot share a file with another case. +locals { + case = trimsuffix(var.output_path, ".json") + order_log = "${local.case}.log" +} + +# Runs long enough for the test to stop the agent while the dependents are +# still waiting. +resource "coder_script" "clone_repo" { + agent_id = coder_agent.main.id + display_name = "Clone repo" + run_on_start = true + timeout = 120 + script = <<-EOT + echo "clone_repo start" >> ${local.order_log} + sleep 60 + echo "clone_repo end" >> ${local.order_log} + EOT +} + +resource "coder_script" "install_tools" { + agent_id = coder_agent.main.id + display_name = "Install tools" + run_on_start = true + script = <<-EOT + echo "install_tools start" >> ${local.order_log} + echo "install_tools end" >> ${local.order_log} + EOT +} + +resource "coder_script" "post_setup" { + agent_id = coder_agent.main.id + display_name = "Post setup" + run_on_start = true + script = <<-EOT + echo "post_setup start" >> ${local.order_log} + echo "post_setup end" >> ${local.order_log} + EOT +} + +resource "coder_script" "start_code_server" { + agent_id = coder_agent.main.id + display_name = "code-server" + run_on_start = true + script = <<-EOT + echo "start_code_server start" >> ${local.order_log} + echo "start_code_server end" >> ${local.order_log} + EOT +} + +data "coder_script_order" "bootstrap" { + rule { + run = ["coder_script.install_tools"] + after = ["coder_script.clone_repo"] + } + rule { + run = ["coder_script.post_setup"] + after = ["coder_script.clone_repo"] + requires = "completion" + } +} + +resource "local_file" "output" { + filename = var.output_path + content = jsonencode({ + "workspace.id" = data.coder_workspace.me.id + "script_order.id" = data.coder_script_order.bootstrap.id + "script_order.rule.#" = tostring(length(data.coder_script_order.bootstrap.rule)) + }) +} + +resource "local_sensitive_file" "agent_token" { + filename = "${local.case}.agent-token" + content = coder_agent.main.token +} + +resource "local_sensitive_file" "session_token" { + filename = "${local.case}.session-token" + content = data.coder_workspace_owner.me.session_token +} diff --git a/integration/coder-script-order-compat/main.tf b/integration/coder-script-order-compat/main.tf new file mode 100644 index 00000000..6d5a6aef --- /dev/null +++ b/integration/coder-script-order-compat/main.tf @@ -0,0 +1,56 @@ +terraform { + required_providers { + coder = { + source = "coder/coder" + } + local = { + source = "hashicorp/local" + } + } +} + +data "coder_provisioner" "me" {} + +resource "coder_agent" "main" { + os = "linux" + arch = data.coder_provisioner.me.arch +} + +resource "coder_script" "clone_repo" { + agent_id = coder_agent.main.id + display_name = "Clone repo" + run_on_start = true + script = "true" +} + +resource "coder_script" "install_tools" { + agent_id = coder_agent.main.id + display_name = "Install tools" + run_on_start = true + script = "true" +} + +data "coder_script_order" "bootstrap" { + rule { + run = ["coder_script.install_tools"] + after = ["coder_script.clone_repo"] + } +} + +variable "output_path" { + type = string +} + +# Coder does not count coder_script as a resource that references the +# agent, so without the agent ID here the agent is dropped. +resource "local_file" "output" { + filename = var.output_path + content = jsonencode({ + "agent.id" = coder_agent.main.id + "script_order.id" = data.coder_script_order.bootstrap.id + "script_order.rule.0.run.0" = data.coder_script_order.bootstrap.rule[0].run[0] + "script_order.rule.0.after.0" = data.coder_script_order.bootstrap.rule[0].after[0] + "script_order.rule.0.requires" = data.coder_script_order.bootstrap.rule[0].requires + "script_order.rule.0.phase" = data.coder_script_order.bootstrap.rule[0].phase + }) +} diff --git a/integration/coder-script-order-skip/main.tf b/integration/coder-script-order-skip/main.tf new file mode 100644 index 00000000..d76d7ebc --- /dev/null +++ b/integration/coder-script-order-skip/main.tf @@ -0,0 +1,103 @@ +terraform { + required_providers { + coder = { + source = "coder/coder" + } + local = { + source = "hashicorp/local" + } + } +} + +data "coder_provisioner" "me" {} +data "coder_workspace" "me" {} +data "coder_workspace_owner" "me" {} + +resource "coder_agent" "main" { + os = "linux" + arch = data.coder_provisioner.me.arch +} + +variable "output_path" { + type = string +} + +# Every file the test reads is named after the output path, so a copied +# template cannot share a file with another case. +locals { + case = trimsuffix(var.output_path, ".json") + order_log = "${local.case}.log" +} + +# Fails on purpose so that install_tools is skipped and post_setup still runs. +resource "coder_script" "clone_repo" { + agent_id = coder_agent.main.id + display_name = "Clone repo" + run_on_start = true + script = <<-EOT + echo "clone_repo start" >> ${local.order_log} + echo "clone_repo end" >> ${local.order_log} + exit 1 + EOT +} + +resource "coder_script" "install_tools" { + agent_id = coder_agent.main.id + display_name = "Install tools" + run_on_start = true + script = <<-EOT + echo "install_tools start" >> ${local.order_log} + echo "install_tools end" >> ${local.order_log} + EOT +} + +resource "coder_script" "post_setup" { + agent_id = coder_agent.main.id + display_name = "Post setup" + run_on_start = true + script = <<-EOT + echo "post_setup start" >> ${local.order_log} + echo "post_setup end" >> ${local.order_log} + EOT +} + +resource "coder_script" "start_code_server" { + agent_id = coder_agent.main.id + display_name = "code-server" + run_on_start = true + script = <<-EOT + echo "start_code_server start" >> ${local.order_log} + echo "start_code_server end" >> ${local.order_log} + EOT +} + +data "coder_script_order" "bootstrap" { + rule { + run = ["coder_script.install_tools"] + after = ["coder_script.clone_repo"] + } + rule { + run = ["coder_script.post_setup"] + after = ["coder_script.clone_repo"] + requires = "completion" + } +} + +resource "local_file" "output" { + filename = var.output_path + content = jsonencode({ + "workspace.id" = data.coder_workspace.me.id + "script_order.id" = data.coder_script_order.bootstrap.id + "script_order.rule.#" = tostring(length(data.coder_script_order.bootstrap.rule)) + }) +} + +resource "local_sensitive_file" "agent_token" { + filename = "${local.case}.agent-token" + content = coder_agent.main.token +} + +resource "local_sensitive_file" "session_token" { + filename = "${local.case}.session-token" + content = data.coder_workspace_owner.me.session_token +} diff --git a/integration/coder-script-order-stop/main.tf b/integration/coder-script-order-stop/main.tf new file mode 100644 index 00000000..c45def2d --- /dev/null +++ b/integration/coder-script-order-stop/main.tf @@ -0,0 +1,78 @@ +terraform { + required_providers { + coder = { + source = "coder/coder" + } + local = { + source = "hashicorp/local" + } + } +} + +data "coder_provisioner" "me" {} +data "coder_workspace" "me" {} +data "coder_workspace_owner" "me" {} + +resource "coder_agent" "main" { + os = "linux" + arch = data.coder_provisioner.me.arch +} + +variable "output_path" { + type = string +} + +# Every file the test reads is named after the output path, so a copied +# template cannot share a file with another case. +locals { + case = trimsuffix(var.output_path, ".json") + order_log = "${local.case}.log" +} + +resource "coder_script" "save_state" { + agent_id = coder_agent.main.id + display_name = "Save state" + run_on_stop = true + script = <<-EOT + echo "save_state start" >> ${local.order_log} + sleep 2 + echo "save_state end" >> ${local.order_log} + EOT +} + +resource "coder_script" "stop_db" { + agent_id = coder_agent.main.id + display_name = "Stop DB" + run_on_stop = true + script = <<-EOT + echo "stop_db start" >> ${local.order_log} + echo "stop_db end" >> ${local.order_log} + EOT +} + +data "coder_script_order" "shutdown" { + rule { + run = ["coder_script.stop_db"] + after = ["coder_script.save_state"] + phase = "stop" + } +} + +resource "local_file" "output" { + filename = var.output_path + content = jsonencode({ + "workspace.id" = data.coder_workspace.me.id + "script_order.id" = data.coder_script_order.shutdown.id + "script_order.rule.#" = tostring(length(data.coder_script_order.shutdown.rule)) + }) +} + +resource "local_sensitive_file" "agent_token" { + filename = "${local.case}.agent-token" + content = coder_agent.main.token +} + +resource "local_sensitive_file" "session_token" { + filename = "${local.case}.session-token" + content = data.coder_workspace_owner.me.session_token +} diff --git a/integration/coder-script-order/main.tf b/integration/coder-script-order/main.tf new file mode 100644 index 00000000..f925db64 --- /dev/null +++ b/integration/coder-script-order/main.tf @@ -0,0 +1,108 @@ +terraform { + required_providers { + coder = { + source = "coder/coder" + } + local = { + source = "hashicorp/local" + } + } +} + +data "coder_provisioner" "me" {} +data "coder_workspace" "me" {} +data "coder_workspace_owner" "me" {} + +resource "coder_agent" "main" { + os = "linux" + arch = data.coder_provisioner.me.arch +} + +variable "output_path" { + type = string +} + +# Every file the test reads is named after the output path, so a copied +# template cannot share a file with another case. +locals { + case = trimsuffix(var.output_path, ".json") + order_log = "${local.case}.log" +} + +# Waits for the unordered script so the test can assert it started before +# this one ended. The sleep keeps the timing interval measurable. +resource "coder_script" "clone_repo" { + agent_id = coder_agent.main.id + display_name = "Clone repo" + run_on_start = true + timeout = 30 + script = <<-EOT + echo "clone_repo start" >> ${local.order_log} + until grep -q "start_code_server start" ${local.order_log}; do sleep 0.1; done + sleep 3 + echo "clone_repo end" >> ${local.order_log} + EOT +} + +resource "coder_script" "install_tools" { + agent_id = coder_agent.main.id + display_name = "Install tools" + run_on_start = true + script = <<-EOT + echo "install_tools start" >> ${local.order_log} + echo "install_tools end" >> ${local.order_log} + EOT +} + +resource "coder_script" "post_setup" { + agent_id = coder_agent.main.id + display_name = "Post setup" + run_on_start = true + script = <<-EOT + echo "post_setup start" >> ${local.order_log} + echo "post_setup end" >> ${local.order_log} + EOT +} + +resource "coder_script" "start_code_server" { + agent_id = coder_agent.main.id + display_name = "code-server" + run_on_start = true + script = <<-EOT + echo "start_code_server start" >> ${local.order_log} + echo "start_code_server end" >> ${local.order_log} + EOT +} + +data "coder_script_order" "bootstrap" { + rule { + run = ["coder_script.install_tools"] + after = ["coder_script.clone_repo"] + } + rule { + run = ["coder_script.post_setup"] + after = ["coder_script.clone_repo"] + requires = "completion" + } +} + +resource "local_file" "output" { + filename = var.output_path + content = jsonencode({ + "workspace.id" = data.coder_workspace.me.id + "script_order.id" = data.coder_script_order.bootstrap.id + "script_order.rule.#" = tostring(length(data.coder_script_order.bootstrap.rule)) + }) +} + +# The test reads these files inside the container. Keeping the tokens out of +# the output JSON keeps them out of the test log. +resource "local_sensitive_file" "agent_token" { + filename = "${local.case}.agent-token" + content = coder_agent.main.token +} + +resource "local_sensitive_file" "session_token" { + filename = "${local.case}.session-token" + content = data.coder_workspace_owner.me.session_token +} diff --git a/integration/integration_test.go b/integration/integration_test.go index afe38e1e..3dff8f70 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -10,6 +10,7 @@ import ( "os/signal" "path/filepath" "runtime" + "slices" "strconv" "strings" "sync" @@ -23,7 +24,6 @@ import ( "github.com/docker/docker/pkg/stdcopy" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "golang.org/x/exp/slices" "golang.org/x/mod/semver" ) @@ -61,6 +61,9 @@ func TestIntegration(t *testing.T) { t.Cleanup(cancel) ctrID := setup(ctx, t, t.Name(), coderImg, coderVersion) + // Anticipated version, update as required. + const scriptOrderMinVersion = "v2.39.0" + for _, tt := range []struct { // Name of the folder under `integration/` containing a test template name string @@ -68,6 +71,8 @@ func TestIntegration(t *testing.T) { minVersion string // map of string to regex to be passed to assertOutput() expectedOutput map[string]string + // Optional. Runs after assertOutput. + verify func(ctx context.Context, t *testing.T, ctrID, name string, output map[string]string) }{ { name: "test-data-source", @@ -225,6 +230,60 @@ func TestIntegration(t *testing.T) { "task.enabled": "false", }, }, + { + name: "coder-script-order", + minVersion: scriptOrderMinVersion, + expectedOutput: map[string]string{ + "workspace.id": `^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`, + "script_order.id": `.+`, + "script_order.rule.#": `^2$`, + }, + verify: verifyScriptOrder, + }, + { + name: "coder-script-order-skip", + minVersion: scriptOrderMinVersion, + expectedOutput: map[string]string{ + "workspace.id": `^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`, + "script_order.id": `.+`, + "script_order.rule.#": `^2$`, + }, + verify: verifyScriptOrderSkip, + }, + { + name: "coder-script-order-stop", + minVersion: scriptOrderMinVersion, + expectedOutput: map[string]string{ + "workspace.id": `^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`, + "script_order.id": `.+`, + "script_order.rule.#": `^1$`, + }, + verify: verifyScriptOrderStop, + }, + { + name: "coder-script-order-cancel", + minVersion: scriptOrderMinVersion, + expectedOutput: map[string]string{ + "workspace.id": `^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`, + "script_order.id": `.+`, + "script_order.rule.#": `^2$`, + }, + verify: verifyScriptOrderCancel, + }, + { + // A Coder older than the data source ignores it and must still + // import and build the template. + name: "coder-script-order-compat", + minVersion: "v0.0.0", + expectedOutput: map[string]string{ + "agent.id": `^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`, + "script_order.id": `.+`, + "script_order.rule.0.run.0": `^coder_script\.install_tools$`, + "script_order.rule.0.after.0": `^coder_script\.clone_repo$`, + "script_order.rule.0.requires": `^success$`, + "script_order.rule.0.phase": `^$`, + }, + }, } { t.Run(tt.name, func(t *testing.T) { t.Parallel() @@ -263,6 +322,9 @@ func TestIntegration(t *testing.T) { actual := make(map[string]string) require.NoError(t, json.NewDecoder(strings.NewReader(out)).Decode(&actual)) assertOutput(t, tt.expectedOutput, actual) + if tt.verify != nil { + tt.verify(ctx, t, ctrID, tt.name, actual) + } }) } } @@ -395,10 +457,21 @@ func ensureImage(ctx context.Context, t *testing.T, cli *client.Client, ref stri // execContainer executes the given command in the given container and returns // the output and the exit code of the command. func execContainer(ctx context.Context, t *testing.T, containerID, command string) (string, int) { + t.Helper() + out, rc, err := tryExecContainer(ctx, t, containerID, command) + require.NoError(t, err) + return out, rc +} + +// tryExecContainer is execContainer for callers that cannot fail the test, +// such as a poll condition running on another goroutine. +func tryExecContainer(ctx context.Context, t *testing.T, containerID, command string) (string, int, error) { t.Helper() t.Logf("exec container cmd: %q", command) cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) - require.NoError(t, err, "connect to docker") + if err != nil { + return "", 0, fmt.Errorf("connect to docker: %w", err) + } defer cli.Close() execConfig := types.ExecConfig{ AttachStdout: true, @@ -406,18 +479,26 @@ func execContainer(ctx context.Context, t *testing.T, containerID, command strin Cmd: []string{"/bin/sh", "-c", command}, } ex, err := cli.ContainerExecCreate(ctx, containerID, execConfig) - require.NoError(t, err, "create container exec") + if err != nil { + return "", 0, fmt.Errorf("create container exec: %w", err) + } resp, err := cli.ContainerExecAttach(ctx, ex.ID, types.ExecStartCheck{}) - require.NoError(t, err, "attach to container exec") + if err != nil { + return "", 0, fmt.Errorf("attach to container exec: %w", err) + } defer resp.Close() var buf bytes.Buffer _, err = stdcopy.StdCopy(&buf, &buf, resp.Reader) - require.NoError(t, err, "read stdout") + if err != nil { + return "", 0, fmt.Errorf("read stdout: %w", err) + } out := buf.String() t.Log("exec container output:\n" + out) execResp, err := cli.ContainerExecInspect(ctx, ex.ID) - require.NoError(t, err, "get exec exit code") - return out, execResp.ExitCode + if err != nil { + return "", 0, fmt.Errorf("get exec exit code: %w", err) + } + return out, execResp.ExitCode, nil } // assertOutput asserts that, for each key-value pair in expected: diff --git a/integration/script_order_test.go b/integration/script_order_test.go new file mode 100644 index 00000000..78018de8 --- /dev/null +++ b/integration/script_order_test.go @@ -0,0 +1,201 @@ +package integration + +import ( + "context" + "fmt" + "slices" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Matches the sleep in coder-script-order/main.tf. +const cloneRepoSleep = 3 * time.Second + +// waitForOrderLog polls the case's order log until it contains every line in +// want. +func waitForOrderLog(ctx context.Context, t *testing.T, ctrID, name string, want ...string) { + t.Helper() + ok := assert.Eventually(t, func() bool { + // The condition runs on testify's goroutine, where a failed require + // would stall the poll instead of ending the test. + o, _, err := tryExecContainer(ctx, t, ctrID, fmt.Sprintf("cat /tmp/%s.log 2>/dev/null", name)) + if err != nil { + return false + } + for _, w := range want { + if !strings.Contains(o, w) { + return false + } + } + return true + }, agentWaitTimeout, time.Second) + if !ok { + t.Fatalf("order log for %s never contained all of %q\nagent log tail:\n%s", + name, want, agentLogTail(ctx, t, ctrID, name)) + } +} + +// orderLogLines returns the lines the case's scripts appended to the order +// log, in the order they were written. +func orderLogLines(ctx context.Context, t *testing.T, ctrID, name string) []string { + t.Helper() + out, rc := execContainer(ctx, t, ctrID, fmt.Sprintf("cat /tmp/%s.log", name)) + require.Equal(t, 0, rc, "read order log for %s", name) + return strings.Split(strings.TrimSpace(out), "\n") +} + +// requireLineAfter asserts that later appears exactly once and after earlier. +func requireLineAfter(t *testing.T, lines []string, later, earlier string) { + t.Helper() + // Only relative order is checked. Scripts that start together may write + // in either order. + laterIdx := slices.Index(lines, later) + earlierIdx := slices.Index(lines, earlier) + require.NotEqualf(t, -1, laterIdx, "line %q missing from %q", later, lines) + require.NotEqualf(t, -1, earlierIdx, "line %q missing from %q", earlier, lines) + require.Equalf(t, 1, countOf(lines, later), "line %q appears more than once in %q", later, lines) + require.Greaterf(t, laterIdx, earlierIdx, "%q (line %d) should come after %q (line %d): %q", later, laterIdx, earlier, earlierIdx, lines) +} + +func countOf(lines []string, want string) int { + n := 0 + for _, l := range lines { + if l == want { + n++ + } + } + return n +} + +// requireDependencies asserts a script's resource address and its stored +// dependencies. A field renamed on either side of the provider and Coder +// contract shows up here as a script with no dependencies. +func requireDependencies(t *testing.T, agent workspaceAgent, displayName, address string, want []workspaceAgentScriptDependency) { + t.Helper() + s := scriptFor(t, agent, displayName) + require.Equalf(t, address, s.ResourceAddress, "%s resource_address", displayName) + require.Equalf(t, want, s.Dependencies, "%s dependencies", displayName) +} + +func verifyScriptOrder(ctx context.Context, t *testing.T, ctrID, name string, out map[string]string) { + agentID := agentIDFor(ctx, t, ctrID, name, out["workspace.id"]) + startAgent(ctx, t, ctrID, name) + agent := waitForAgent(ctx, t, ctrID, name, agentID, "ready", "start_error") + + // Checked before the order so a contract break is named as such rather + // than as a script running too early. + requireDependencies(t, agent, "Clone repo", "coder_script.clone_repo", nil) + requireDependencies(t, agent, "Install tools", "coder_script.install_tools", + []workspaceAgentScriptDependency{{PrerequisiteResourceAddress: "coder_script.clone_repo", Requirement: "success"}}) + requireDependencies(t, agent, "Post setup", "coder_script.post_setup", + []workspaceAgentScriptDependency{{PrerequisiteResourceAddress: "coder_script.clone_repo", Requirement: "completion"}}) + requireDependencies(t, agent, "code-server", "coder_script.start_code_server", nil) + + requireLifecycle(ctx, t, ctrID, name, agent, "ready") + + lines := orderLogLines(ctx, t, ctrID, name) + require.Len(t, lines, 8, "expected two lines per script: %q", lines) + requireLineAfter(t, lines, "install_tools start", "clone_repo end") + requireLineAfter(t, lines, "post_setup start", "clone_repo end") + requireLineAfter(t, lines, "clone_repo end", "start_code_server start") + + timings := waitForTimings(ctx, t, ctrID, name, out["workspace.id"], "start", "Clone repo", "Install tools", "Post setup", "code-server") + clone := timingFor(t, timings, "start", "Clone repo") + install := timingFor(t, timings, "start", "Install tools") + post := timingFor(t, timings, "start", "Post setup") + code := timingFor(t, timings, "start", "code-server") + for _, tm := range []agentScriptTiming{clone, install, post, code} { + requireStatus(t, tm, "ok") + requireExitCode(t, tm, 0) + } + require.Falsef(t, install.StartedAt.Before(clone.EndedAt), "Install tools started %s before Clone repo ended %s", install.StartedAt, clone.EndedAt) + require.Falsef(t, post.StartedAt.Before(clone.EndedAt), "Post setup started %s before Clone repo ended %s", post.StartedAt, clone.EndedAt) + require.Truef(t, code.StartedAt.Before(clone.EndedAt), "code-server started %s after Clone repo ended %s", code.StartedAt, clone.EndedAt) + require.GreaterOrEqualf(t, clone.EndedAt.Sub(clone.StartedAt), cloneRepoSleep, "Clone repo duration: %+v", clone) +} + +func verifyScriptOrderSkip(ctx context.Context, t *testing.T, ctrID, name string, out map[string]string) { + agentID := agentIDFor(ctx, t, ctrID, name, out["workspace.id"]) + startAgent(ctx, t, ctrID, name) + agent := waitForAgent(ctx, t, ctrID, name, agentID, "ready", "start_error") + + requireDependencies(t, agent, "Install tools", "coder_script.install_tools", + []workspaceAgentScriptDependency{{PrerequisiteResourceAddress: "coder_script.clone_repo", Requirement: "success"}}) + + requireLifecycle(ctx, t, ctrID, name, agent, "start_error") + + lines := orderLogLines(ctx, t, ctrID, name) + require.NotContains(t, lines, "install_tools start", "skipped script must not run: %q", lines) + requireLineAfter(t, lines, "post_setup start", "clone_repo end") + + timings := waitForTimings(ctx, t, ctrID, name, out["workspace.id"], "start", "Clone repo", "Install tools", "Post setup", "code-server") + clone := timingFor(t, timings, "start", "Clone repo") + requireStatus(t, clone, "exit_failure") + requireExitCode(t, clone, 1) + install := timingFor(t, timings, "start", "Install tools") + requireStatus(t, install, "skipped") + require.Truef(t, install.StartedAt.Equal(install.EndedAt), "skipped timing must be zero-length: %+v", install) + requireStatus(t, timingFor(t, timings, "start", "Post setup"), "ok") + requireStatus(t, timingFor(t, timings, "start", "code-server"), "ok") +} + +func verifyScriptOrderStop(ctx context.Context, t *testing.T, ctrID, name string, out map[string]string) { + agentID := agentIDFor(ctx, t, ctrID, name, out["workspace.id"]) + startAgent(ctx, t, ctrID, name) + agent := waitForAgent(ctx, t, ctrID, name, agentID, "ready") + + requireDependencies(t, agent, "Stop DB", "coder_script.stop_db", + []workspaceAgentScriptDependency{{PrerequisiteResourceAddress: "coder_script.save_state", Requirement: "success"}}) + + stopAgent(ctx, t, ctrID, name) + agent = waitForAgent(ctx, t, ctrID, name, agentID, "off", "shutdown_error", "shutdown_timeout") + requireLifecycle(ctx, t, ctrID, name, agent, "off") + + lines := orderLogLines(ctx, t, ctrID, name) + requireLineAfter(t, lines, "stop_db start", "save_state end") + + timings := waitForTimings(ctx, t, ctrID, name, out["workspace.id"], "stop", "Save state", "Stop DB") + save := timingFor(t, timings, "stop", "Save state") + stop := timingFor(t, timings, "stop", "Stop DB") + requireStatus(t, save, "ok") + requireStatus(t, stop, "ok") + require.Falsef(t, stop.StartedAt.Before(save.EndedAt), "Stop DB started %s before Save state ended %s", stop.StartedAt, save.EndedAt) +} + +func verifyScriptOrderCancel(ctx context.Context, t *testing.T, ctrID, name string, out map[string]string) { + agentID := agentIDFor(ctx, t, ctrID, name, out["workspace.id"]) + startAgent(ctx, t, ctrID, name) + // code-server starts in parallel with clone_repo. Its timing proves the + // agent saw it exit, so the signal cannot cancel it as well. + waitForOrderLog(ctx, t, ctrID, name, "clone_repo start") + waitForTimings(ctx, t, ctrID, name, out["workspace.id"], "start", "code-server") + stopAgent(ctx, t, ctrID, name) + agent := waitForAgent(ctx, t, ctrID, name, agentID, "off", "shutdown_error", "shutdown_timeout") + + requireDependencies(t, agent, "Install tools", "coder_script.install_tools", + []workspaceAgentScriptDependency{{PrerequisiteResourceAddress: "coder_script.clone_repo", Requirement: "success"}}) + requireDependencies(t, agent, "Post setup", "coder_script.post_setup", + []workspaceAgentScriptDependency{{PrerequisiteResourceAddress: "coder_script.clone_repo", Requirement: "completion"}}) + + requireLifecycle(ctx, t, ctrID, name, agent, "off") + + lines := orderLogLines(ctx, t, ctrID, name) + require.NotContains(t, lines, "clone_repo end") + require.NotContains(t, lines, "install_tools start") + require.NotContains(t, lines, "post_setup start") + + timings := waitForTimings(ctx, t, ctrID, name, out["workspace.id"], "start", "Clone repo", "Install tools", "Post setup", "code-server") + clone := timingFor(t, timings, "start", "Clone repo") + requireStatus(t, clone, "canceled") + require.Truef(t, clone.EndedAt.After(clone.StartedAt), "canceled running script must keep its real interval: %+v", clone) + for _, n := range []string{"Install tools", "Post setup"} { + tm := timingFor(t, timings, "start", n) + requireStatus(t, tm, "canceled") + require.Truef(t, tm.StartedAt.Equal(tm.EndedAt), "%s timing must be zero-length: %+v", n, tm) + } + requireStatus(t, timingFor(t, timings, "start", "code-server"), "ok") +}