auth: wire secure-storage cache into login/token/logout

NEXT_CHANGELOG.md

@@ -7,6 +7,7 @@
 ### CLI
 
 * Added `--limit` flag to all paginated list commands for client-side result capping ([#4984](https://github.com/databricks/cli/pull/4984)).
+* Added experimental OS-native secure token storage opt-in via `DATABRICKS_AUTH_STORAGE=secure` or `[__settings__].auth_storage = secure` in `.databrickscfg`. Legacy file-backed token storage remains the default.
 * Accept `yes` in addition to `y` for confirmation prompts, and show `[y/N]` to indicate that no is the default.
 
 ### Bundles

acceptance/cmd/auth/storage-modes/invalid-env/output.txt

@@ -0,0 +1,5 @@
+
+>>> [CLI] auth token --profile nonexistent
+Error: DATABRICKS_AUTH_STORAGE: unknown storage mode "bogus" (want legacy, secure, or plaintext)
+
+Exit code: 1

acceptance/cmd/auth/storage-modes/invalid-env/script

@@ -0,0 +1,6 @@
+export DATABRICKS_AUTH_STORAGE=bogus
+
+# Any auth command that resolves the storage mode must surface the error.
+# auth token is the smallest reproducer because it doesn't perform any
+# network I/O before resolving the mode.
+trace $CLI auth token --profile nonexistent

acceptance/cmd/auth/storage-modes/legacy-env-default/output.txt

@@ -0,0 +1,11 @@
+
+=== Token cache keys before logout
+[
+  "dev"
+]
+
+>>> [CLI] auth logout --profile dev --auto-approve
+Logged out of profile "dev". Use --delete to also remove it from the config file.
+
+=== Token cache keys after logout (should be empty)
+[]

acceptance/cmd/auth/storage-modes/legacy-env-default/script

@@ -0,0 +1,29 @@
+export DATABRICKS_AUTH_STORAGE=legacy
+
+cat > "./home/.databrickscfg" <<ENDCFG
+[dev]
+host = https://accounts.cloud.databricks.com
+account_id = account-id
+auth_type  = databricks-cli
+ENDCFG
+
+mkdir -p "./home/.databricks"
+cat > "./home/.databricks/token-cache.json" <<ENDCACHE
+{
+  "version": 1,
+  "tokens": {
+    "dev": {
+      "access_token": "dev-cached-token",
+      "token_type": "Bearer"
+    }
+  }
+}
+ENDCACHE
+
+title "Token cache keys before logout\n"
+jq -S '.tokens | keys' "./home/.databricks/token-cache.json"
+
+trace $CLI auth logout --profile dev --auto-approve
+
+title "Token cache keys after logout (should be empty)\n"
+jq -S '.tokens | keys' "./home/.databricks/token-cache.json"

acceptance/cmd/auth/storage-modes/script.prepare

@@ -0,0 +1,8 @@
+# Shared setup: isolate HOME and clear inherited auth env vars so each
+# storage-mode test is deterministic regardless of the runner.
+sethome "./home"
+
+unset DATABRICKS_HOST
+unset DATABRICKS_TOKEN
+unset DATABRICKS_CONFIG_PROFILE
+unset DATABRICKS_AUTH_STORAGE

acceptance/cmd/auth/storage-modes/test.toml

@@ -0,0 +1,3 @@
+Ignore = [
+    "home"
+]

cmd/auth/login.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/databricks/cli/libs/auth"
+	"github.com/databricks/cli/libs/auth/storage"
 	"github.com/databricks/cli/libs/browser"
 	"github.com/databricks/cli/libs/cmdio"
 	"github.com/databricks/cli/libs/databrickscfg"
@@ -21,6 +22,7 @@ import (
 	"github.com/databricks/databricks-sdk-go/config"
 	"github.com/databricks/databricks-sdk-go/config/experimental/auth/authconv"
 	"github.com/databricks/databricks-sdk-go/credentials/u2m"
+	"github.com/databricks/databricks-sdk-go/credentials/u2m/cache"
 	"github.com/spf13/cobra"
 	"golang.org/x/oauth2"
 )
@@ -145,6 +147,11 @@ a new profile is created.
 		ctx := cmd.Context()
 		profileName := cmd.Flag("profile").Value.String()
 
+		tokenCache, _, err := storage.ResolveCache(ctx, "")
+		if err != nil {
+			return err
+		}
+
 		// Cluster and Serverless are mutually exclusive.
 		if configureCluster && configureServerless {
 			return errors.New("please either configure serverless or cluster, not both")
@@ -195,7 +202,15 @@ a new profile is created.
 			if err := validateDiscoveryFlagCompatibility(cmd); err != nil {
 				return err
 			}
-			return discoveryLogin(ctx, &defaultDiscoveryClient{}, profileName, loginTimeout, scopes, existingProfile, getBrowserFunc(cmd))
+			return discoveryLogin(ctx, discoveryLoginInputs{
+				dc:              &defaultDiscoveryClient{},
+				profileName:     profileName,
+				timeout:         loginTimeout,
+				scopes:          scopes,
+				existingProfile: existingProfile,
+				browserFunc:     getBrowserFunc(cmd),
+				tokenCache:      tokenCache,
+			})
 		}
 
 		// Load unified host flag from the profile if not explicitly set via CLI flag.
@@ -230,6 +245,7 @@ a new profile is created.
 		persistentAuthOpts := []u2m.PersistentAuthOption{
 			u2m.WithOAuthArgument(oauthArgument),
 			u2m.WithBrowser(getBrowserFunc(cmd)),
+			u2m.WithTokenCache(tokenCache),
 		}
 		if len(scopesList) > 0 {
 			persistentAuthOpts = append(persistentAuthOpts, u2m.WithScopes(scopesList))
@@ -559,34 +575,47 @@ func validateDiscoveryFlagCompatibility(cmd *cobra.Command) error {
 	return nil
 }
 
+// discoveryLoginInputs groups the dependencies of discoveryLogin.
+// See https://google.github.io/styleguide/go/best-practices#option-structure.
+type discoveryLoginInputs struct {
+	dc              discoveryClient
+	profileName     string
+	timeout         time.Duration
+	scopes          string
+	existingProfile *profile.Profile
+	browserFunc     func(string) error
+	tokenCache      cache.TokenCache
+}
+
 // discoveryLogin runs the login.databricks.com discovery flow. The user
 // authenticates in the browser, selects a workspace, and the CLI receives
 // the workspace host from the OAuth callback's iss parameter.
-func discoveryLogin(ctx context.Context, dc discoveryClient, profileName string, timeout time.Duration, scopes string, existingProfile *profile.Profile, browserFunc func(string) error) error {
-	arg, err := dc.NewOAuthArgument(profileName)
+func discoveryLogin(ctx context.Context, in discoveryLoginInputs) error {
+	arg, err := in.dc.NewOAuthArgument(in.profileName)
 	if err != nil {
 		return discoveryErr("setting up login.databricks.com", err)
 	}
 
-	scopesList := splitScopes(scopes)
-	if len(scopesList) == 0 && existingProfile != nil && existingProfile.Scopes != "" {
-		scopesList = splitScopes(existingProfile.Scopes)
+	scopesList := splitScopes(in.scopes)
+	if len(scopesList) == 0 && in.existingProfile != nil && in.existingProfile.Scopes != "" {
+		scopesList = splitScopes(in.existingProfile.Scopes)
 	}
 
 	opts := []u2m.PersistentAuthOption{
 		u2m.WithOAuthArgument(arg),
-		u2m.WithBrowser(browserFunc),
+		u2m.WithBrowser(in.browserFunc),
 		u2m.WithDiscoveryLogin(),
+		u2m.WithTokenCache(in.tokenCache),
 	}
 	if len(scopesList) > 0 {
 		opts = append(opts, u2m.WithScopes(scopesList))
 	}
 
 	// Apply timeout before creating PersistentAuth so Challenge() respects it.
-	ctx, cancel := context.WithTimeout(ctx, timeout)
+	ctx, cancel := context.WithTimeout(ctx, in.timeout)
 	defer cancel()
 
-	persistentAuth, err := dc.NewPersistentAuth(ctx, opts...)
+	persistentAuth, err := in.dc.NewPersistentAuth(ctx, opts...)
 	if err != nil {
 		return discoveryErr("setting up login.databricks.com", err)
 	}
@@ -618,7 +647,7 @@ func discoveryLogin(ctx context.Context, dc discoveryClient, profileName string,
 		return fmt.Errorf("retrieving token after login: %w", err)
 	}
 
-	introspection, err := dc.IntrospectToken(ctx, discoveredHost, tok.AccessToken)
+	introspection, err := in.dc.IntrospectToken(ctx, discoveredHost, tok.AccessToken)
 	if err != nil {
 		log.Debugf(ctx, "token introspection failed (non-fatal): %v", err)
 	} else {
@@ -629,10 +658,10 @@ func discoveryLogin(ctx context.Context, dc discoveryClient, profileName string,
 			accountID = introspection.AccountID
 		}
 
-		if existingProfile != nil && existingProfile.AccountID != "" && introspection.AccountID != "" &&
-			existingProfile.AccountID != introspection.AccountID {
+		if in.existingProfile != nil && in.existingProfile.AccountID != "" && introspection.AccountID != "" &&
+			in.existingProfile.AccountID != introspection.AccountID {
 			log.Warnf(ctx, "detected account ID %q differs from existing profile account ID %q",
-				introspection.AccountID, existingProfile.AccountID)
+				introspection.AccountID, in.existingProfile.AccountID)
 		}
 	}
 
@@ -651,7 +680,7 @@ func discoveryLogin(ctx context.Context, dc discoveryClient, profileName string,
 		"serverless_compute_id",
 	)
 	err = databrickscfg.SaveToProfile(ctx, &config.Config{
-		Profile:     profileName,
+		Profile:     in.profileName,
 		Host:        discoveredHost,
 		AuthType:    authTypeDatabricksCLI,
 		AccountID:   accountID,
@@ -661,12 +690,12 @@ func discoveryLogin(ctx context.Context, dc discoveryClient, profileName string,
 	}, clearKeys...)
 	if err != nil {
 		if configFile != "" {
-			return fmt.Errorf("saving profile %q to %s: %w", profileName, configFile, err)
+			return fmt.Errorf("saving profile %q to %s: %w", in.profileName, configFile, err)
 		}
-		return fmt.Errorf("saving profile %q: %w", profileName, err)
+		return fmt.Errorf("saving profile %q: %w", in.profileName, err)
 	}
 
-	cmdio.LogString(ctx, fmt.Sprintf("Profile %s was successfully saved", profileName))
+	cmdio.LogString(ctx, fmt.Sprintf("Profile %s was successfully saved", in.profileName))
 	return nil
 }
 

cmd/auth/login_test.go

@@ -623,7 +623,13 @@ func TestDiscoveryLogin_IntrospectionFailureStillSavesProfile(t *testing.T) {
 	}
 
 	ctx, _ := cmdio.NewTestContextWithStdout(t.Context())
-	err = discoveryLogin(ctx, dc, "DISCOVERY", time.Second, "all-apis, ,sql,", nil, func(string) error { return nil })
+	err = discoveryLogin(ctx, discoveryLoginInputs{
+		dc:          dc,
+		profileName: "DISCOVERY",
+		timeout:     time.Second,
+		scopes:      "all-apis, ,sql,",
+		browserFunc: func(string) error { return nil },
+	})
 	require.NoError(t, err)
 
 	assert.Equal(t, "https://workspace.example.com", dc.introspectHost)
@@ -671,7 +677,13 @@ func TestDiscoveryLogin_AccountIDMismatchWarning(t *testing.T) {
 		AccountID: "old-account-id",
 	}
 
-	err = discoveryLogin(ctx, dc, "DISCOVERY", time.Second, "", existingProfile, func(string) error { return nil })
+	err = discoveryLogin(ctx, discoveryLoginInputs{
+		dc:              dc,
+		profileName:     "DISCOVERY",
+		timeout:         time.Second,
+		existingProfile: existingProfile,
+		browserFunc:     func(string) error { return nil },
+	})
 	require.NoError(t, err)
 
 	// Verify warning about mismatched account IDs was logged.
@@ -719,7 +731,13 @@ func TestDiscoveryLogin_NoWarningWhenAccountIDsMatch(t *testing.T) {
 		AccountID: "same-account-id",
 	}
 
-	err = discoveryLogin(ctx, dc, "DISCOVERY", time.Second, "", existingProfile, func(string) error { return nil })
+	err = discoveryLogin(ctx, discoveryLoginInputs{
+		dc:              dc,
+		profileName:     "DISCOVERY",
+		timeout:         time.Second,
+		existingProfile: existingProfile,
+		browserFunc:     func(string) error { return nil },
+	})
 	require.NoError(t, err)
 
 	// No warning should be logged when account IDs match.
@@ -739,7 +757,12 @@ func TestDiscoveryLogin_EmptyDiscoveredHostReturnsError(t *testing.T) {
 	}
 
 	ctx, _ := cmdio.NewTestContextWithStdout(t.Context())
-	err = discoveryLogin(ctx, dc, "DISCOVERY", time.Second, "", nil, func(string) error { return nil })
+	err = discoveryLogin(ctx, discoveryLoginInputs{
+		dc:          dc,
+		profileName: "DISCOVERY",
+		timeout:     time.Second,
+		browserFunc: func(string) error { return nil },
+	})
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "no workspace host was discovered")
 }
@@ -771,7 +794,13 @@ func TestDiscoveryLogin_ReloginPreservesExistingProfileScopes(t *testing.T) {
 
 	// No --scopes flag (empty string), should fall back to existing profile scopes.
 	ctx, _ := cmdio.NewTestContextWithStdout(t.Context())
-	err = discoveryLogin(ctx, dc, "DISCOVERY", time.Second, "", existingProfile, func(string) error { return nil })
+	err = discoveryLogin(ctx, discoveryLoginInputs{
+		dc:              dc,
+		profileName:     "DISCOVERY",
+		timeout:         time.Second,
+		existingProfile: existingProfile,
+		browserFunc:     func(string) error { return nil },
+	})
 	require.NoError(t, err)
 
 	savedProfile, err := loadProfileByName(ctx, "DISCOVERY", profile.DefaultProfiler)
@@ -808,7 +837,14 @@ func TestDiscoveryLogin_ExplicitScopesOverrideExistingProfile(t *testing.T) {
 
 	// Explicit --scopes flag should override existing profile scopes.
 	ctx, _ := cmdio.NewTestContextWithStdout(t.Context())
-	err = discoveryLogin(ctx, dc, "DISCOVERY", time.Second, "all-apis", existingProfile, func(string) error { return nil })
+	err = discoveryLogin(ctx, discoveryLoginInputs{
+		dc:              dc,
+		profileName:     "DISCOVERY",
+		timeout:         time.Second,
+		scopes:          "all-apis",
+		existingProfile: existingProfile,
+		browserFunc:     func(string) error { return nil },
+	})
 	require.NoError(t, err)
 
 	savedProfile, err := loadProfileByName(ctx, "DISCOVERY", profile.DefaultProfiler)
@@ -848,7 +884,12 @@ func TestDiscoveryLogin_SPOGHostPopulatesAccountIDFromDiscovery(t *testing.T) {
 	}
 
 	ctx, _ := cmdio.NewTestContextWithStdout(t.Context())
-	err = discoveryLogin(ctx, dc, "DISCOVERY", time.Second, "", nil, func(string) error { return nil })
+	err = discoveryLogin(ctx, discoveryLoginInputs{
+		dc:          dc,
+		profileName: "DISCOVERY",
+		timeout:     time.Second,
+		browserFunc: func(string) error { return nil },
+	})
 	require.NoError(t, err)
 
 	savedProfile, err := loadProfileByName(ctx, "DISCOVERY", profile.DefaultProfiler)
@@ -883,7 +924,12 @@ func TestDiscoveryLogin_IntrospectionFallsBackWhenDiscoveryFails(t *testing.T) {
 	}
 
 	ctx, _ := cmdio.NewTestContextWithStdout(t.Context())
-	err = discoveryLogin(ctx, dc, "DISCOVERY", time.Second, "", nil, func(string) error { return nil })
+	err = discoveryLogin(ctx, discoveryLoginInputs{
+		dc:          dc,
+		profileName: "DISCOVERY",
+		timeout:     time.Second,
+		browserFunc: func(string) error { return nil },
+	})
 	require.NoError(t, err)
 
 	savedProfile, err := loadProfileByName(ctx, "DISCOVERY", profile.DefaultProfiler)
@@ -932,7 +978,13 @@ auth_type = databricks-cli
 	}
 
 	ctx, _ := cmdio.NewTestContextWithStdout(t.Context())
-	err = discoveryLogin(ctx, dc, "DISCOVERY", time.Second, "", existingProfile, func(string) error { return nil })
+	err = discoveryLogin(ctx, discoveryLoginInputs{
+		dc:              dc,
+		profileName:     "DISCOVERY",
+		timeout:         time.Second,
+		existingProfile: existingProfile,
+		browserFunc:     func(string) error { return nil },
+	})
 	require.NoError(t, err)
 
 	savedProfile, err := loadProfileByName(ctx, "DISCOVERY", profile.DefaultProfiler)
@@ -982,7 +1034,13 @@ auth_type = databricks-cli
 	}
 
 	ctx, _ := cmdio.NewTestContextWithStdout(t.Context())
-	err = discoveryLogin(ctx, dc, "DISCOVERY", time.Second, "", existingProfile, func(string) error { return nil })
+	err = discoveryLogin(ctx, discoveryLoginInputs{
+		dc:              dc,
+		profileName:     "DISCOVERY",
+		timeout:         time.Second,
+		existingProfile: existingProfile,
+		browserFunc:     func(string) error { return nil },
+	})
 	require.NoError(t, err)
 
 	savedProfile, err := loadProfileByName(ctx, "DISCOVERY", profile.DefaultProfiler)