Make an explicitly selected profile take precedence over auth environment variables

NEXT_CHANGELOG.md

@@ -7,6 +7,7 @@
 ### CLI
 
  * `databricks aitools install` is now plugin-first: it installs the Databricks plugin through each agent's own CLI (Claude Code, Codex, GitHub Copilot) instead of copying raw skill files. Agents without a plugin (OpenCode, Antigravity) still get skill files, and Cursor prints the `/add-plugin databricks` step. Use `--skills-only` to force raw skill files for every agent, or `--path <dir>` to write skills to a directory ([#5738](https://github.com/databricks/cli/pull/5738)).
+* An explicitly selected profile (`--profile` or a bundle's `workspace.profile`) now takes precedence over auth environment variables (`DATABRICKS_HOST`, `DATABRICKS_TOKEN`, etc.) instead of being silently shadowed by them; env vars still fill auth fields the profile leaves empty ([#5096](https://github.com/databricks/cli/issues/5096)).
 
 ### Bundles
 

acceptance/cmd/api/profile-overrides-env/output.txt

@@ -0,0 +1,44 @@
+
+=== api --profile overrides auth env vars (#5096)
+
+>>> [CLI] api get /api/2.0/clusters/list --profile my-workspace
+{}
+
+>>> print_requests.py --get //api/2.0/clusters/list
+{
+  "headers": {
+    "Authorization": [
+      "Bearer [DATABRICKS_TOKEN]"
+    ],
+    "User-Agent": [
+      "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/api_get cmd-exec-id/[UUID] interactive/none auth/pat"
+    ],
+    "X-Databricks-Workspace-Id": [
+      "[NUMID]"
+    ]
+  },
+  "method": "GET",
+  "path": "/api/2.0/clusters/list"
+}
+
+=== api host-only --profile fills the token from the environment (#5096)
+
+>>> [CLI] api get /api/2.0/clusters/list --profile host-only
+{}
+
+>>> print_requests.py --get //api/2.0/clusters/list
+{
+  "headers": {
+    "Authorization": [
+      "Bearer [DATABRICKS_TOKEN]"
+    ],
+    "User-Agent": [
+      "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/api_get cmd-exec-id/[UUID] interactive/none auth/pat"
+    ],
+    "X-Databricks-Workspace-Id": [
+      "[NUMID]"
+    ]
+  },
+  "method": "GET",
+  "path": "/api/2.0/clusters/list"
+}

acceptance/cmd/api/profile-overrides-env/script

@@ -0,0 +1,29 @@
+sethome "./home"
+
+# One profile with full credentials, one host-only; both point at the test
+# server while the auth env vars below point elsewhere.
+cat > "./home/.databrickscfg" <<EOF
+[my-workspace]
+host  = $DATABRICKS_HOST
+token = $DATABRICKS_TOKEN
+
+[host-only]
+host  = $DATABRICKS_HOST
+EOF
+
+# direnv-style auth env vars for a different workspace; before #5096 these
+# shadowed the profile selected with --profile.
+real_token=$DATABRICKS_TOKEN
+export DATABRICKS_HOST=https://dev.cloud.databricks.test
+export DATABRICKS_TOKEN=dev-token
+
+title "api --profile overrides auth env vars (#5096)\n"
+MSYS_NO_PATHCONV=1 trace $CLI api get /api/2.0/clusters/list --profile my-workspace
+trace print_requests.py --get //api/2.0/clusters/list
+
+# Host-only profile: the profile wins for the host, but env fills the token it
+# omits (#5096).
+export DATABRICKS_TOKEN=$real_token
+title "api host-only --profile fills the token from the environment (#5096)\n"
+MSYS_NO_PATHCONV=1 trace $CLI api get /api/2.0/clusters/list --profile host-only
+trace print_requests.py --get //api/2.0/clusters/list

acceptance/cmd/api/profile-overrides-env/test.toml

@@ -0,0 +1,3 @@
+Ignore = [
+    "home",
+]

acceptance/cmd/auth/describe/account-host-with-workspace-id/output.txt

@@ -7,10 +7,10 @@ Account ID: acct-123
 Authenticated with: pat
 -----
 Current configuration:
-  ✓ host: [DATABRICKS_URL] (from DATABRICKS_HOST environment variable)
+  ✓ host: [DATABRICKS_URL] (from [TEST_TMP_DIR]/home/.databrickscfg config file)
   ✓ account_id: acct-123 (from [TEST_TMP_DIR]/home/.databrickscfg config file)
   ✓ workspace_id: [NUMID] (from [TEST_TMP_DIR]/home/.databrickscfg config file)
-  ✓ token: ******** (from DATABRICKS_TOKEN environment variable)
+  ✓ token: ******** (from [TEST_TMP_DIR]/home/.databrickscfg config file)
   ✓ profile: acct-with-ws (from --profile flag)
   ✓ databricks_cli_path: [CLI]
   ✓ auth_type: pat

acceptance/cmd/auth/describe/profile-overrides-env/output.txt

@@ -0,0 +1,36 @@
+
+=== Describe with --profile overrides auth env vars (#5096)
+
+>>> [CLI] auth describe --profile my-workspace
+Host: [DATABRICKS_URL]
+User: [USERNAME]
+Authenticated with: pat
+-----
+Current configuration:
+  ✓ host: [DATABRICKS_URL] (from [TEST_TMP_DIR]/home/.databrickscfg config file)
+  ✓ workspace_id: [NUMID]
+  ✓ token: ******** (from [TEST_TMP_DIR]/home/.databrickscfg config file)
+  ✓ profile: my-workspace (from --profile flag)
+  ✓ databricks_cli_path: [CLI]
+  ✓ auth_type: pat
+  ✓ rate_limit: [NUMID] (from DATABRICKS_RATE_LIMIT environment variable)
+  ✓ cloud: AWS
+  ✓ discovery_url: [DATABRICKS_URL]/oidc/.well-known/oauth-authorization-server
+
+=== Describe with a host-only --profile fills the token from the environment (#5096)
+
+>>> [CLI] auth describe --profile host-only
+Host: [DATABRICKS_URL]
+User: [USERNAME]
+Authenticated with: pat
+-----
+Current configuration:
+  ✓ host: [DATABRICKS_URL] (from [TEST_TMP_DIR]/home/.databrickscfg config file)
+  ✓ workspace_id: [NUMID]
+  ✓ token: ******** (from DATABRICKS_TOKEN environment variable)
+  ✓ profile: host-only (from --profile flag)
+  ✓ databricks_cli_path: [CLI]
+  ✓ auth_type: pat
+  ✓ rate_limit: [NUMID] (from DATABRICKS_RATE_LIMIT environment variable)
+  ✓ cloud: AWS
+  ✓ discovery_url: [DATABRICKS_URL]/oidc/.well-known/oauth-authorization-server

acceptance/cmd/auth/describe/profile-overrides-env/script

@@ -0,0 +1,26 @@
+sethome "./home"
+
+# A profile carries full credentials; a second profile carries only a host.
+cat > "./home/.databrickscfg" <<EOF
+[my-workspace]
+host  = $DATABRICKS_HOST
+token = $DATABRICKS_TOKEN
+
+[host-only]
+host  = $DATABRICKS_HOST
+EOF
+
+# direnv-style auth env vars for a different workspace; before #5096 these
+# shadowed the profile selected with --profile.
+real_token=$DATABRICKS_TOKEN
+export DATABRICKS_HOST=https://dev.cloud.databricks.test
+export DATABRICKS_TOKEN=dev-token
+
+title "Describe with --profile overrides auth env vars (#5096)\n"
+trace $CLI auth describe --profile my-workspace
+
+# Host-only profile: the profile wins for the host, but env fills the token it
+# omits (#5096).
+export DATABRICKS_TOKEN=$real_token
+title "Describe with a host-only --profile fills the token from the environment (#5096)\n"
+trace $CLI auth describe --profile host-only

acceptance/cmd/auth/describe/profile-overrides-env/test.toml

@@ -0,0 +1,3 @@
+Ignore = [
+    "home"
+]

bundle/config/workspace.go

@@ -182,9 +182,14 @@ func (w *Workspace) Client(ctx context.Context) (*databricks.WorkspaceClient, er
 
 	cfg := w.Config(ctx)
 
-	// If only the host is configured, we try and unambiguously match it to
-	// a profile in the user's databrickscfg file. Override the default loaders.
-	if w.Host != "" && w.Profile == "" {
+	switch {
+	case w.Profile != "":
+		// An explicit profile wins over auth env vars (#5096).
+		// ValidateConfigAndProfileHost below still checks host agreement.
+		cfg.Loaders = databrickscfg.ProfileAuthLoaders
+	case w.Host != "":
+		// If only the host is configured, we try and unambiguously match it to
+		// a profile in the user's databrickscfg file. Override the default loaders.
 		cfg.Loaders = []config.Loader{
 			// Load auth creds from env vars
 			config.ConfigAttributes,

bundle/config/workspace_test.go

@@ -155,6 +155,78 @@ func TestWorkspaceClientNormalizesHostBeforeProfileResolution(t *testing.T) {
 	assert.Equal(t, "ws2", client.Config.Profile)
 }
 
+func TestWorkspaceClientProfileOverridesAuthEnv(t *testing.T) {
+	// An explicit profile must win over auth env vars (#5096).
+	setupWorkspaceTest(t)
+
+	err := databrickscfg.SaveToProfile(t.Context(), &config.Config{
+		Profile: "tst",
+		Host:    "https://tst.cloud.databricks.test",
+		Token:   "tst-token",
+	})
+	require.NoError(t, err)
+
+	// direnv-style auth env vars pointing at a different (dev) workspace.
+	t.Setenv("DATABRICKS_HOST", "https://dev.cloud.databricks.test")
+	t.Setenv("DATABRICKS_TOKEN", "dev-token")
+
+	w := Workspace{Profile: "tst"}
+	client, err := w.Client(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, "tst", client.Config.Profile)
+	assert.Equal(t, "https://tst.cloud.databricks.test", client.Config.Host)
+	assert.Equal(t, "tst-token", client.Config.Token)
+}
+
+func TestWorkspaceClientProfileFillsAuthFromEnv(t *testing.T) {
+	// Host-only profile: the profile wins for the host, but env fills the token
+	// it omits (#5096).
+	setupWorkspaceTest(t)
+
+	err := databrickscfg.SaveToProfile(t.Context(), &config.Config{
+		Profile: "host-only",
+		Host:    "https://tst.cloud.databricks.test",
+	})
+	require.NoError(t, err)
+
+	t.Setenv("DATABRICKS_TOKEN", "env-token")
+
+	w := Workspace{Profile: "host-only"}
+	client, err := w.Client(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, "host-only", client.Config.Profile)
+	assert.Equal(t, "https://tst.cloud.databricks.test", client.Config.Host)
+	// The token is not in the profile, so it is filled from the environment.
+	assert.Equal(t, "env-token", client.Config.Token)
+}
+
+func TestWorkspaceClientHostAndProfileOverridesAuthEnv(t *testing.T) {
+	// Bundle pins both workspace.host and workspace.profile: the profile wins
+	// for auth and the host check passes because they agree (#5096).
+	setupWorkspaceTest(t)
+
+	err := databrickscfg.SaveToProfile(t.Context(), &config.Config{
+		Profile: "tst",
+		Host:    "https://tst.cloud.databricks.test",
+		Token:   "tst-token",
+	})
+	require.NoError(t, err)
+
+	// direnv-style auth env vars pointing at a different (dev) workspace.
+	t.Setenv("DATABRICKS_HOST", "https://dev.cloud.databricks.test")
+	t.Setenv("DATABRICKS_TOKEN", "dev-token")
+
+	w := Workspace{
+		Host:    "https://tst.cloud.databricks.test",
+		Profile: "tst",
+	}
+	client, err := w.Client(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, "tst", client.Config.Profile)
+	assert.Equal(t, "https://tst.cloud.databricks.test", client.Config.Host)
+	assert.Equal(t, "tst-token", client.Config.Token)
+}
+
 func TestWorkspaceConfigHTTPTimeout(t *testing.T) {
 	for _, tc := range []struct {
 		envVal string

cmd/api/api.go

@@ -85,7 +85,9 @@ func makeCommand(method string) *cobra.Command {
 			// also reads it, but setting cfg.Profile here keeps any error
 			// messages we render referring to the same name), 3.
 			// [__settings__].default_profile in the config file.
-			if profileFlag := cmd.Flag("profile"); profileFlag != nil {
+			profileFlag := cmd.Flag("profile")
+			hasProfileFlag := profileFlag != nil && profileFlag.Value.String() != ""
+			if hasProfileFlag {
 				cfg.Profile = profileFlag.Value.String()
 			}
 			if cfg.Profile == "" {
@@ -95,7 +97,13 @@ func makeCommand(method string) *cobra.Command {
 				cfg.Profile = databrickscfg.ResolveDefaultProfile(cmd.Context())
 			}
 
-			auth.NormalizeDatabricksConfigFromEnv(cmd.Context(), cfg)
+			if hasProfileFlag {
+				// An explicit --profile wins over auth env vars (#5096); the host
+				// comes from the profile, so skip env host normalization.
+				cfg.Loaders = databrickscfg.ProfileAuthLoaders
+			} else {
+				auth.NormalizeDatabricksConfigFromEnv(cmd.Context(), cfg)
+			}
 
 			api, err := client.New(cfg)
 			if err != nil {

cmd/root/auth.go

@@ -110,6 +110,17 @@ func profileFlagValue(cmd *cobra.Command) (string, bool) {
 	return value, value != ""
 }
 
+// applyProfileAuthPrecedence makes an explicit --profile win over auth env vars
+// via ProfileAuthLoaders (#5096), skipping env host normalization since the host
+// comes from the profile. Without a profile flag, env-first behavior is kept.
+func applyProfileAuthPrecedence(ctx context.Context, cfg *config.Config, hasProfileFlag bool) {
+	if hasProfileFlag {
+		cfg.Loaders = databrickscfg.ProfileAuthLoaders
+		return
+	}
+	auth.NormalizeDatabricksConfigFromEnv(ctx, cfg)
+}
+
 // Helper function to create an account client or prompt once if the given configuration is not valid.
 func accountClientOrPrompt(ctx context.Context, cfg *config.Config, allowPrompt bool) (*databricks.AccountClient, error) {
 	a, err := databricks.NewAccountClient((*databricks.Config)(cfg))
@@ -195,15 +206,15 @@ func MustAnyClient(cmd *cobra.Command, args []string) (bool, error) {
 
 func MustAccountClient(cmd *cobra.Command, args []string) error {
 	cfg := &config.Config{}
+	ctx := cmd.Context()
 
 	// The command-line profile flag takes precedence over DATABRICKS_CONFIG_PROFILE.
 	pr, hasProfileFlag := profileFlagValue(cmd)
 	if hasProfileFlag {
 		cfg.Profile = pr
 	}
+	applyProfileAuthPrecedence(ctx, cfg, hasProfileFlag)
 
-	ctx := cmd.Context()
-	auth.NormalizeDatabricksConfigFromEnv(ctx, cfg)
 	ctx = cmdctx.SetConfigUsed(ctx, cfg)
 	cmd.SetContext(ctx)
 
@@ -325,8 +336,7 @@ func MustWorkspaceClient(cmd *cobra.Command, args []string) error {
 	if hasProfileFlag {
 		cfg.Profile = profile
 	}
-
-	auth.NormalizeDatabricksConfigFromEnv(ctx, cfg)
+	applyProfileAuthPrecedence(ctx, cfg, hasProfileFlag)
 	resolveDefaultProfile(ctx, cfg)
 
 	_, isTargetFlagSet := targetFlagValue(cmd)