Support configuration overrides for troubleshooting

README.md

@@ -36,6 +36,7 @@ updates in a FIFO order.
 - [Site Level CRR](/docs/site-level-crr.md)
 - [Transient Source](/docs/transient-crr-source.md)
 - [Out of Band updates from RING](/docs/oob-s3-ring.md)
+- [Configuration overrides](/docs/config-overrides.md)
 
 ## QUICKSTART
 

docs/config-overrides.md

@@ -0,0 +1,60 @@
+# Configuration Overrides
+
+`BACKBEAT_CONFIG_OVERRIDES` holds a JSON document applied on top of the
+configuration file at startup, so that any field can be changed per process
+without a new image or release.
+
+This is a troubleshooting escape hatch, meant for support: the environment
+variables named after the configuration fields, described in
+[Configuration](/docs/configuration.md), remain the supported way to configure
+backbeat, and should be preferred whenever one exists for the field at hand.
+
+## Semantics
+
+The document is applied as a [JSON Merge Patch](https://www.rfc-editor.org/rfc/rfc7386),
+which is to say:
+
+- objects are merged recursively, so only the fields mentioned are changed;
+- arrays and scalars replace the value they override — an array is never merged
+  element-wise, so overriding a list with a shorter one drops the extra entries;
+- `null` deletes a field, restoring the default the schema defines for it.
+
+The overrides are applied last, over the configuration file and any other
+setting, so that nothing silently overrides them.
+
+The result is validated as a whole, exactly like the configuration file: an
+unknown field, a wrong type or a deleted mandatory field fails at startup,
+rather than leaving a setting silently ignored. Values are coerced by the
+schema, so `"250"` is accepted for a numeric field.
+
+## Examples
+
+Raise the log level of a single process:
+
+```sh
+BACKBEAT_CONFIG_OVERRIDES='{"log":{"logLevel":"debug"}}'
+```
+
+Set librdkafka producer parameters, whose dotted keys need no escaping, being
+plain JSON object keys:
+
+```sh
+BACKBEAT_CONFIG_OVERRIDES='{"kafka":{"producerParams":{"linger.ms":10}}}'
+```
+
+Change a field of an extension, and restore another to its default:
+
+```sh
+BACKBEAT_CONFIG_OVERRIDES='{"extensions":{"lifecycle":{"conductor":{"concurrency":20}}}}'
+BACKBEAT_CONFIG_OVERRIDES='{"queuePopulator":{"batchMaxRead":null}}'
+```
+
+Several changes are applied in a single document:
+
+```sh
+BACKBEAT_CONFIG_OVERRIDES='{
+    "log": { "logLevel": "debug" },
+    "queuePopulator": { "batchMaxRead": 250 },
+    "extensions": { "gc": { "consumer": { "concurrency": 5 } } }
+}'
+```

docs/configuration.md

@@ -88,6 +88,10 @@ setting a field of the configuration file. No schema declares them, so no name
 is derived for them, and their value is not validated.
 
 - `BACKBEAT_CONFIG_FILE`: path of the configuration file.
+- `BACKBEAT_CONFIG_OVERRIDES`: JSON merge patch applied over the whole
+  configuration, for the fields no name above reaches. The result is validated,
+  unlike the other variables of this section — see
+  [Configuration overrides](/docs/config-overrides.md).
 - `BACKBEAT_QUEUEPOPULATOR_EXTENSIONS`: extensions run by this queue populator,
   comma separated.
 - `BOOTSTRAP_SITE_NAME`: restricts the replication bootstrap list to one site.

lib/Config.js

@@ -66,9 +66,8 @@ class Config extends EventEmitter {
      * @returns {undefined}
      */
     _parseConfig(config) {
-        applyEnvOverrides(config, backbeatConfigJoi);
-
-        const parsedConfig = joi.attempt(config, backbeatConfigJoi);
+        const parsedConfig = joi.attempt(
+            applyEnvOverrides(config, backbeatConfigJoi), backbeatConfigJoi);
 
         // Strip config to ensure extensions cannot use another extension's config
         const globalConfig = { ...parsedConfig };
@@ -93,7 +92,7 @@ class Config extends EventEmitter {
         const lifecycleConfig = parsedConfig.extensions?.lifecycle;
         const backbeatSupportsTransition = lifecycleConfig?.supportedLifecycleRules?.includes('Transition');
         const replicationConfig = parsedConfig.extensions?.replication;
-        if (backbeatSupportsTransition && !replicationConfig.dataMoverTopic) {
+        if (backbeatSupportsTransition && !replicationConfig?.dataMoverTopic) {
             throw new Error('dataMoverTopic is required when lifecycle transitions is supported');
         }
 

lib/config/configOverrides.js

@@ -0,0 +1,131 @@
+'use strict';
+
+/**
+ * Generic configuration overrides, from the BACKBEAT_CONFIG_OVERRIDES
+ * environment variable.
+ *
+ * The variable holds a JSON document applied to the configuration as a JSON
+ * Merge Patch (RFC 7386): objects are merged recursively, arrays and scalars
+ * replace the value they override, and `null` deletes a field, restoring the
+ * default the schema defines for it.
+ *
+ * This is a troubleshooting escape hatch, for the fields no named setting
+ * reaches — e.g. the librdkafka parameters of `kafka.producerParams`, whose
+ * dotted keys need no escaping here, being plain JSON object keys:
+ *
+ *     BACKBEAT_CONFIG_OVERRIDES='{"kafka":{"producerParams":{"linger.ms":10}}}'
+ *
+ * It is applied last, over the configuration file and any named setting, so
+ * that nothing silently overrides the hatch someone reached for precisely
+ * because the usual path did not work. The result is still validated against
+ * the schema, so a typo or a wrong type fails at startup rather than leaving a
+ * setting silently ignored.
+ */
+
+const { getField } = require('./fields');
+
+const CONFIG_OVERRIDES = 'BACKBEAT_CONFIG_OVERRIDES';
+
+/**
+ * @param {*} value - value to test
+ * @returns {boolean} true for a JSON object, excluding arrays and null
+ */
+function isObject(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+/**
+ * Applies a JSON Merge Patch to a target value, transcribed from the reference
+ * pseudocode of RFC 7386, section 2.
+ *
+ * lodash's `merge()` is deliberately not used: it merges arrays element-wise
+ * instead of replacing them, which would leave stale entries behind when
+ * overriding a list with a shorter one, and it assigns `null` instead of
+ * deleting the field.
+ *
+ * The target is updated in place when both sides are objects, so that the
+ * caller keeps its reference; any other patch replaces it, and the new value is
+ * returned.
+ *
+ * A `__proto__` member is ignored, whatever the patch holds for it: no
+ * configuration field is named that, and JS objects cannot hold such a member
+ * anyway, only reach through it to Object.prototype.
+ *
+ * @param {*} target - value to patch
+ * @param {*} patch - merge patch to apply
+ * @returns {*} patched value
+ */
+function mergePatch(target, patch) {
+    if (!isObject(patch)) {
+        return patch;
+    }
+    if (!isObject(target)) {
+        // the patch describes an object: whatever the target held is replaced
+        target = {}; // eslint-disable-line no-param-reassign
+    }
+    Object.entries(patch).forEach(([key, value]) => {
+        if (key === '__proto__') {
+            // whatever the caller passes, this is not a member to write: it
+            // would update Object.prototype instead of the target
+            return;
+        }
+
+        if (value === null) {
+            delete target[key]; // eslint-disable-line no-param-reassign
+        } else {
+            target[key] = mergePatch(target[key], value); // eslint-disable-line no-param-reassign
+        }
+    });
+    return target;
+}
+
+/**
+ * Applies the fraction of the BACKBEAT_CONFIG_OVERRIDES merge patch covering
+ * the fields of one schema, in place. The patch mirrors the configuration, so
+ * that fraction sits at the config path of the schema root.
+ *
+ * A fraction that is not an object replaces the whole section rather than
+ * updating it, and cannot be applied in place: the caller has to use the value
+ * returned, and leave the schema to report it if the section is mandatory.
+ *
+ * @param {Object} config - configuration to update, matching the schema
+ * @param {string[]} [prefix] - config path of the schema root
+ * @param {Object} [env] - environment to read the overrides from
+ * @returns {*} updated configuration
+ */
+function applyConfigOverrides(config, prefix = [], env = process.env) {
+    if (!env[CONFIG_OVERRIDES]) {
+        return config;
+    }
+
+    let patch;
+    try {
+        // JSON.parse sets a `__proto__` key as a plain member: the reviver drops
+        // it at any depth, so that the document handed over holds only the
+        // configuration fields it is meant to describe
+        patch = JSON.parse(env[CONFIG_OVERRIDES],
+            (key, value) => (key === '__proto__' ? undefined : value));
+    } catch (err) {
+        throw new Error(`invalid JSON value for ${CONFIG_OVERRIDES}: ${err.message}`);
+    }
+
+    if (!isObject(patch)) {
+        // any other document would replace the whole configuration instead of
+        // updating it in place, which the caller would silently drop
+        throw new Error(`${CONFIG_OVERRIDES} must hold a JSON object`);
+    }
+
+    // the patch may cover none of the fields of the schema, and then leaves the
+    // configuration alone
+    const fraction = getField(patch, prefix);
+    if (fraction === undefined) {
+        return config;
+    }
+
+    return mergePatch(config, fraction);
+}
+
+module.exports = {
+    applyConfigOverrides,
+    mergePatch,
+};

lib/config/envOverrides.js

@@ -4,6 +4,7 @@
  * variables derived from the joi schemas.
  */
 
+const { applyConfigOverrides } = require('./configOverrides');
 const { getField, setField } = require('./fields');
 
 // A container exposes a single probe endpoint, shared by all the probe servers
@@ -221,13 +222,15 @@ function applyLivenessProbePort(config, mappings, port) {
 }
 
 /**
- * Applies the env vars mapped to the fields of a schema, in place.
+ * Applies the env vars mapped to the fields of a schema, in place. A generic override
+ * replacing the whole section cannot be applied in place, so the caller has to use the
+ * value returned rather than its own reference.
  *
  * @param {Object} config - configuration to update, matching the schema
  * @param {joi.Schema} schema - configuration schema
  * @param {string[]} [prefix] - config path of the schema root
  * @param {Object} [env] - environment to read the overrides from
- * @returns {Object} updated configuration. Invalid config returned untouched for joi to report
+ * @returns {*} updated configuration. Invalid config returned untouched for joi to report
  */
 function applyEnvOverrides(config, schema, prefix = [], env = process.env) {
     if (config === null || typeof config !== 'object') {
@@ -248,7 +251,8 @@ function applyEnvOverrides(config, schema, prefix = [], env = process.env) {
         }
     });
 
-    return config;
+    // the generic overrides come last, and win over every name derived above
+    return applyConfigOverrides(config, prefix, env);
 }
 
 module.exports = {