Make ciphertool-encrypted secrets work with ICP's config layout

distribution/scripts/ciphertool.bat

@@ -28,8 +28,10 @@ REM
 REM   JAVA_OPTS     (Optional) Java runtime options
 REM
 REM Usage:
-REM   ciphertool.bat -Dconfigure          Encrypt passwords interactively
-REM   ciphertool.bat -Dconfigure -Dvalue=<plaintext>   Encrypt a specific value
+REM   ciphertool.bat -Dconfigure   Encrypt the [plain-text] values in the top-level
+REM                                [secrets] table of conf\deployment.toml
+REM   ciphertool.bat               Prompt for a single value and print it encrypted
+REM   ciphertool.bat -help         Show all options
 REM ---------------------------------------------------------------------------
 
 REM Make sure prerequisite environment variables are set

distribution/scripts/ciphertool.sh

@@ -27,8 +27,10 @@
 #   JAVA_HOME   Must point at your Java Development Kit installation.
 #
 # Usage:
-#   ./ciphertool.sh -Dconfigure          Encrypt passwords interactively
-#   ./ciphertool.sh -Dconfigure -Dvalue=<plaintext>   Encrypt a specific value
+#   ./ciphertool.sh -Dconfigure   Encrypt the [plain-text] values in the top-level
+#                                 [secrets] table of conf/deployment.toml
+#   ./ciphertool.sh               Prompt for a single value and print it encrypted
+#   ./ciphertool.sh -help         Show all options
 
 # if JAVA_HOME is not set we're not happy
 if [ -z "$JAVA_HOME" ]; then
@@ -106,8 +108,8 @@ do
 done
 ICP_CLASSPATH=$ICP_CLASSPATH:$CLASSPATH
 
-# For Cygwin, switch paths to Windows format before running java
-if $cygwin; then
+# For Cygwin and Mingw (Git Bash), switch paths to Windows format before running java
+if $cygwin || $mingw; then
   JAVA_HOME=`cygpath --absolute --windows "$JAVA_HOME"`
   ICP_HOME=`cygpath --absolute --windows "$ICP_HOME"`
   CLASSPATH=`cygpath --path --windows "$CLASSPATH"`

distribution/src/main/resources/conf/security/cipher-text.properties

@@ -24,6 +24,6 @@
 #   Linux/Mac : ./bin/ciphertool.sh -Dconfigure
 #   Windows   : bin\ciphertool.bat -Dconfigure
 #
-# To encrypt a single value:
-#   Linux/Mac : ./bin/ciphertool.sh -Dconfigure -Dvalue=<plaintext>
-#   Windows   : bin\ciphertool.bat -Dconfigure -Dvalue=<plaintext>
+# To encrypt a single value (prompts for the value and prints the encrypted form):
+#   Linux/Mac : ./bin/ciphertool.sh
+#   Windows   : bin\ciphertool.bat

distribution/src/main/resources/conf/security/cipher-tool.properties

@@ -22,12 +22,18 @@
 #
 # 1. TOML mode (active for ICP): When conf/deployment.toml exists, the tool reads
 #    secrets directly from the [secrets] table in that file. This file is ignored.
-#    To encrypt secrets, add a [secrets] section to conf/deployment.toml:
+#    To encrypt secrets, reference each one by an alias and add the alias to a
+#    top-level [secrets] table in conf/deployment.toml (not [icp_server.secrets]
+#    or [icp_server.storage.secrets], which the tool does not process):
+#
+#      frontendJwtHMACSecret = "$secret{secretJWTsecret}"
+#
+#      [icp_server.storage]
+#      dbPassword = "$secret{secDbPassword}"
 #
 #      [secrets]
-#      frontendJwtHMACSecret = "[plain-text-value]"
-#      dbPassword = "[plain-text-value]"
-#      ssoClientSecret = "[plain-text-value]"
+#      secretJWTsecret = "[plain-text-value]"
+#      secDbPassword = "[plain-text-value]"
 #
 #    Then run: ./bin/ciphertool.sh -Dconfigure
 #    The tool encrypts each [plain-text-value] and rewrites it in deployment.toml.

docs/ldap-user-store.md

@@ -196,17 +196,19 @@ ldapTrustStorePassword = "truststore-password"
 
 ## Secrets
 
-Passwords can be encrypted using the WSO2 cipher tool and referenced via the secrets map:
+Passwords can be encrypted using the WSO2 cipher tool and referenced via the secrets map. Add the plain-text values, wrapped in `[]`, to a top-level `[secrets]` table in `conf/deployment.toml`:
 
 ```toml
 ldapConnectionPassword = "$secret{ldapConnectionPassword}"
 ldapTrustStorePassword = "$secret{ldapTrustStorePassword}"
 
 [secrets]
-ldapConnectionPassword = "<encrypted-value>"
-ldapTrustStorePassword = "<encrypted-value>"
+ldapConnectionPassword = "[connection-password]"
+ldapTrustStorePassword = "[truststore-password]"
 ```
 
+Then run `bin/ciphertool.sh -Dconfigure` (`bin\ciphertool.bat -Dconfigure` on Windows). The tool replaces each `[plain-text]` value with its encrypted form. Keep the table as a top-level `[secrets]`: the cipher tool does not process `[icp_server.secrets]`, and ICP fails to start if both tables are present.
+
 ---
 
 ## How it works

icp_server/Config.toml

@@ -135,7 +135,7 @@ frontendJwtHMACSecret = "default-secret-key-at-least-32-characters-long-for-hs25
 # ldapTrustStorePath = ""             # Path to a JKS truststore for verifying the LDAP server's CA cert
 # ldapTrustStorePassword = ""
 
-# Secrets alias form (use with [icp_server.secrets] block below):
+# Secrets alias form (use with the [secrets] table at the end of this file):
 # ldapConnectionPassword = "$secret{ldapConnectionPassword}"
 # ldapTrustStorePassword = "$secret{ldapTrustStorePassword}"
 
@@ -203,18 +203,11 @@ frontendJwtHMACSecret = "default-secret-key-at-least-32-characters-long-for-hs25
 # =============================================================================
 # SECRETS ALIAS CONFIGURATION
 # =============================================================================
+# Reference an encrypted value as "$secret{alias}". The alias is defined in the
+# [secrets] table at the end of this file.
 # frontendJwtHMACSecret = "$secret{secretJWTsecret}"
 # credentialsDbPassword = "$secret{secCredsDbPassword}"
 
-# =============================================================================
-# ENCRYPTED SECRETS CONFIGURATION
-# =============================================================================
-[icp_server.secrets]
-# secretJWTsecret = "encrypted-value-for-jwt-secret"
-# secCredsDbPassword = "encrypted-value-for-credentials-db-password"
-# ldapConnectionPassword = "encrypted-value-for-ldap-connection-password"
-# ldapTrustStorePassword = "encrypted-value-for-ldap-truststore-password"
-
 # =============================================================================
 # DATABASE CONFIGURATION
 # =============================================================================
@@ -278,14 +271,9 @@ frontendJwtHMACSecret = "default-secret-key-at-least-32-characters-long-for-hs25
 # =============================================================================
 # DATABASE SECRETS ALIAS CONFIGURATION
 # =============================================================================
+# The alias is defined in the [secrets] table at the end of this file.
 # dbPassword = "$secret{secDbPassword}"
 
-# =============================================================================
-# DATABASE ENCRYPTED SECRETS CONFIGURATION
-# =============================================================================
-[icp_server.storage.secrets]
-# secDbPassword = "encrypted-value-for-db-password"
-
 
 [[ballerina.log.modules]]
 name = "wso2/icp_server"
@@ -294,3 +282,20 @@ level = "INFO"
 [ballerina.http.traceLogAdvancedConfig]
 # Enable printing trace logs on the Console. The default value is `false`.
 console = false
+
+# =============================================================================
+# ENCRYPTED SECRETS CONFIGURATION
+# =============================================================================
+# Holds every value referenced as "$secret{alias}" above, for all sections.
+# 1. Uncomment [secrets] and add each value as plain text wrapped in [], e.g.
+#      secretJWTsecret = "[my-jwt-secret-at-least-32-characters-long]"
+# 2. Run bin/ciphertool.sh -Dconfigure (bin\ciphertool.bat on Windows).
+#    The tool replaces each [plain-text] value with its encrypted form.
+# Keep this as a top-level [secrets] table: the cipher tool only processes this
+# table, and ICP fails to start if [icp_server.secrets] is also present.
+# [secrets]
+# secretJWTsecret = "encrypted-value-for-jwt-secret"
+# secCredsDbPassword = "encrypted-value-for-credentials-db-password"
+# secDbPassword = "encrypted-value-for-db-password"
+# ldapConnectionPassword = "encrypted-value-for-ldap-connection-password"
+# ldapTrustStorePassword = "encrypted-value-for-ldap-truststore-password"

icp_server/Dependencies.toml

@@ -423,6 +423,22 @@ modules = [
 	{org = "ballerina", packageName = "time", moduleName = "time"}
 ]
 
+[[package]]
+org = "ballerina"
+name = "toml"
+version = "0.8.0"
+dependencies = [
+	{org = "ballerina", name = "file"},
+	{org = "ballerina", name = "io"},
+	{org = "ballerina", name = "time"}
+]
+modules = [
+	{org = "ballerina", packageName = "toml", moduleName = "toml"},
+	{org = "ballerina", packageName = "toml", moduleName = "toml.lexer"},
+	{org = "ballerina", packageName = "toml", moduleName = "toml.parser"},
+	{org = "ballerina", packageName = "toml", moduleName = "toml.writer"}
+]
+
 [[package]]
 org = "ballerina"
 name = "url"
@@ -656,6 +672,7 @@ dependencies = [
 	{org = "ballerina", name = "task"},
 	{org = "ballerina", name = "test"},
 	{org = "ballerina", name = "time"},
+	{org = "ballerina", name = "toml"},
 	{org = "ballerina", name = "url"},
 	{org = "ballerina", name = "uuid"},
 	{org = "ballerina", name = "websocket"},

icp_server/config.bal

@@ -175,6 +175,8 @@ configurable string opensearchPassword = "Ballerina@123";
 configurable boolean artifactsApiAllowInsecureTLS = true;
 
 // Secrets map containing values encrypted by the WSO2 cipher tool.
+// Populated from the top-level [secrets] table (the table the cipher tool encrypts) or, equivalently,
+// from [icp_server.secrets]. Use only one of the two; Ballerina rejects the top-level table if both exist.
 // All values present in this table are expected to be encrypted; plaintext values will cause an error.
 configurable map<string> secrets = {};
 

icp_server/init.bal

@@ -108,7 +108,7 @@ function init() returns error? {
 }
 
 // Resolves a configurable value for the default module.
-// If the value matches "$secret{alias}", looks up the alias in the [icp_server.secrets] map and decrypts it.
+// If the value matches "$secret{alias}", looks up the alias in the [secrets] map and decrypts it.
 // Otherwise returns the value unchanged (plain-text config).
 function resolveSecret(string configValue) returns string|error {
     return utils:resolveConfig(configValue, secrets);

icp_server/modules/storage/config.bal

@@ -46,6 +46,7 @@ configurable string artifactsApiTrustStorePath = "";
 configurable string artifactsApiTrustStorePassword = "";
 
 // Secrets map for the storage module — populated from [icp_server.storage.secrets] in Config.toml.
+// Aliases are also resolved from the top-level [secrets] table (see createDbClient).
 configurable map<string> secrets = {};
 
 // Runtime auth configuration (runtime and server communication)

icp_server/modules/storage/init.bal

@@ -31,20 +31,35 @@ final sql:Client dbClient = check createDbClient();
 final string resolvedArtifactsApiTrustStorePassword = check loadArtifactsApiTrustStore();
 
 function createDbClient() returns sql:Client|error {
-    string resolvedUser = check utils:resolveConfig(dbUser, secrets);
-    string resolvedPassword = check utils:resolveConfig(dbPassword, secrets);
+    map<string> dbSecrets = check resolveSecretsTable(dbUser, dbPassword);
+    string resolvedUser = check utils:resolveConfig(dbUser, dbSecrets);
+    string resolvedPassword = check utils:resolveConfig(dbPassword, dbSecrets);
     DatabaseConnectionManager dbManager = check new (dbType, dbHost, dbPort, dbName, resolvedUser, resolvedPassword, dbUseTLS);
     return dbManager.getClient();
 }
 
+// Aliases come from the top-level [secrets] table (the one the cipher tool encrypts), with
+// [icp_server.storage.secrets] taking precedence. The TOML config is read only when an alias is used.
+function resolveSecretsTable(string... configValues) returns map<string>|error {
+    if !configValues.some(v => utils:isSecretAlias(v)) {
+        return secrets;
+    }
+    map<string> merged = check utils:readTopLevelSecrets();
+    foreach [string, string] [alias, value] in secrets.entries() {
+        merged[alias] = value;
+    }
+    return merged;
+}
+
 function loadArtifactsApiTrustStore() returns string|error {
     if artifactsApiTrustStorePath.trim() == "" {
         return "";
     }
     if !check file:test(artifactsApiTrustStorePath, file:EXISTS) {
         return error(string `artifactsApiTrustStorePath does not exist: ${artifactsApiTrustStorePath}`);
     }
-    string password = check utils:resolveConfig(artifactsApiTrustStorePassword, secrets);
+    map<string> trustStoreSecrets = check resolveSecretsTable(artifactsApiTrustStorePassword);
+    string password = check utils:resolveConfig(artifactsApiTrustStorePassword, trustStoreSecrets);
     check validateTrustStore(artifactsApiTrustStorePath, password);
     if artifactsApiAllowInsecureTLS {
         log:printWarn("artifactsApiTrustStorePath is set, but artifactsApiAllowInsecureTLS is true in "

icp_server/modules/utils/cipher.bal

@@ -17,6 +17,7 @@
 import ballerina/crypto;
 import ballerina/file;
 import ballerina/os;
+import ballerina/toml;
 import ballerina/lang.array;
 import ballerina/log;
 
@@ -65,7 +66,7 @@ function resolvePassword(string envVar, string fallback, string errorMsg) return
 // Resolves a configurable value that may reference an encrypted secret.
 // If configValue matches "$secret{alias}", looks up alias in secrets, decrypts, and returns the plaintext.
 public isolated function resolveConfig(string configValue, map<string> secrets) returns string|error {
-    if configValue.startsWith(CIPHER_SECRET_PREFIX) && configValue.endsWith(CIPHER_SECRET_SUFFIX) {
+    if isSecretAlias(configValue) {
         string alias = configValue.substring(CIPHER_SECRET_PREFIX.length(), configValue.length() - CIPHER_SECRET_SUFFIX.length());
         string? encrypted = secrets[alias];
         if encrypted is () {
@@ -79,6 +80,60 @@ public isolated function resolveConfig(string configValue, map<string> secrets)
     return configValue;
 }
 
+// Returns true if configValue is a "$secret{alias}" reference.
+public isolated function isSecretAlias(string configValue) returns boolean {
+    return configValue.startsWith(CIPHER_SECRET_PREFIX) && configValue.endsWith(CIPHER_SECRET_SUFFIX);
+}
+
+// Reads the top-level [secrets] table from the TOML config ICP was started with. The sources match
+// the Ballerina runtime's: the files in BAL_CONFIG_FILES if set, otherwise the inline BAL_CONFIG_DATA,
+// otherwise ./Config.toml. With several files, the first file that defines an alias wins.
+// The WSO2 cipher tool only encrypts a top-level [secrets] table, and Ballerina binds that table only
+// to the root module's `secrets` configurable, so non-root modules use this to resolve the same aliases.
+public function readTopLevelSecrets() returns map<string>|error {
+    string configFiles = os:getEnv("BAL_CONFIG_FILES");
+    if configFiles != "" {
+        map<string> result = {};
+        foreach string path in re `${os:getEnv("OS") == "Windows_NT" ? ";" : ":"}`.split(configFiles) {
+            if path.trim() == "" || !check file:test(path, file:EXISTS) {
+                continue;
+            }
+            foreach [string, string] [alias, value] in secretsTable(check toml:readFile(path)).entries() {
+                if !result.hasKey(alias) {
+                    result[alias] = value;
+                }
+            }
+        }
+        return result;
+    }
+    string configData = os:getEnv("BAL_CONFIG_DATA");
+    if configData != "" {
+        return parseTopLevelSecrets(configData);
+    }
+    if check file:test("Config.toml", file:EXISTS) {
+        return secretsTable(check toml:readFile("Config.toml"));
+    }
+    return {};
+}
+
+// Returns the string entries of the top-level [secrets] table in TOML content.
+public isolated function parseTopLevelSecrets(string tomlContent) returns map<string>|error {
+    return secretsTable(check toml:readString(tomlContent));
+}
+
+isolated function secretsTable(map<json> config) returns map<string> {
+    json secrets = config["secrets"];
+    map<string> result = {};
+    if secrets is map<json> {
+        foreach [string, json] [alias, value] in secrets.entries() {
+            if value is string {
+                result[alias] = value;
+            }
+        }
+    }
+    return result;
+}
+
 // Decrypts a value encrypted by the WSO2 cipher tool using asymmetric RSA/ECB/OAEPwithSHA1andMGF1Padding.
 // Returns an error if decryption fails (including if the value is not a valid encrypted secret).
 public isolated function decrypt(string encryptedValue) returns string|error {