feat: Add the override store, overlay, and data system wiring

Adds the override layer described by the OVERRIDE specification: an override
store that holds marked flag and segment definitions, an overlay at the store
read boundary that returns the override entry for a key in preference to
LaunchDarkly data and enumerates the union of both, and a sink that applies
each snapshot as a single replacement and notifies flag change listeners of
every flag whose evaluation may have changed, including flags that depend on
a changed prerequisite or segment.

The override source is an option of the FDv2 data system configuration:
DataSystemConfig.override_source and ConfigBuilder.overrides(). The source
is built with the data system, so invalid configuration raises from the
client constructor. It starts before the data system's own threads, so its
initial load completes before the constructor returns, and it is closed with
the client. OverrideSource and OverrideSink are public protocols so a custom
source can be supplied.

The client consults the override store before its not-initialized
short-circuit. An overridden flag is served before the client has
LaunchDarkly data, and a flag absent from the override store still returns
the client-not-ready default. The all-flags state reads through the overlay
and, while the client has no LaunchDarkly data, contains only the overridden
flags. The async client and data system receive the same changes, with
override change notifications delivered on the event loop.

The OVERRIDE specification test vectors run as a unit test through the full
client stack.

Author: kinyoklion

ldclient/async_client.py

@@ -95,6 +95,7 @@ def __init__(self, config: AsyncConfig):
         # these flags, so plain booleans are safe.
         self._eval_cached_data_warned = False
         self._all_flags_cached_data_warned = False
+        self._all_flags_overrides_only_warned = False
 
         # Build the object graph here (loop-free). start() supplies the loop-bound
         # resources: the HTTP session (created lazily), the data source, the
@@ -498,17 +499,26 @@ async def _evaluate_internal(self, key: str, context: Context, default: Any, eve
         if self._config.offline:
             return EvaluationDetail(default, None, error_reason('CLIENT_NOT_READY')), None
 
+        def not_ready() -> Tuple[EvaluationDetail, Optional[FeatureFlag]]:
+            log.warning("Feature Flag evaluation attempted before client has initialized! Feature store unavailable - returning default: " + str(default) + " for feature key: " + key)
+            reason = error_reason('CLIENT_NOT_READY')
+            self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason))
+            return EvaluationDetail(default, None, reason), None
+
+        no_launchdarkly_data = False
         availability = await self._data_system.data_availability()
         if availability != DataAvailability.REFRESHED:
             if availability == DataAvailability.CACHED:
                 if not self._eval_cached_data_warned:
                     self._eval_cached_data_warned = True
                     log.warning("Feature Flag evaluation attempted before client has initialized - using last known values from feature store for feature key: " + key + ". This message is logged once.")
+            elif self._data_system.override_source_configured:
+                # No data from LaunchDarkly is available. The store read below still finds an
+                # entry that the override layer holds, and the SDK serves it. A miss returns the
+                # not-ready default.
+                no_launchdarkly_data = True
             else:
-                log.warning("Feature Flag evaluation attempted before client has initialized! Feature store unavailable - returning default: " + str(default) + " for feature key: " + key)
-                reason = error_reason('CLIENT_NOT_READY')
-                self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason))
-                return EvaluationDetail(default, None, reason), None
+                return not_ready()
 
         if not context.valid:
             log.warning("Context was invalid for flag evaluation (%s); returning default value" % context.error)
@@ -523,6 +533,8 @@ async def _evaluate_internal(self, key: str, context: Context, default: Any, eve
             self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason))
             return EvaluationDetail(default, None, reason), None
         if not flag:
+            if no_launchdarkly_data:
+                return not_ready()
             reason = error_reason('FLAG_NOT_FOUND')
             self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason))
             return EvaluationDetail(default, None, reason), None
@@ -572,12 +584,17 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState
             log.warning("all_flags_state() called, but client is in offline mode. Returning empty state")
             return FeatureFlagsState(False)
 
+        overrides_only = False
         availability = await self._data_system.data_availability()
         if availability != DataAvailability.REFRESHED:
             if availability == DataAvailability.CACHED:
                 if not self._all_flags_cached_data_warned:
                     self._all_flags_cached_data_warned = True
                     log.warning("all_flags_state() called before client has finished initializing! Using last known values from feature store. This message is logged once.")
+            elif self._data_system.override_source_configured:
+                # No data from LaunchDarkly is available. The store read below returns only the
+                # entries that the override layer holds. The result decides the state.
+                overrides_only = True
             else:
                 log.warning("all_flags_state() called before client has finished initializing! Feature store unavailable - returning empty state")
                 return FeatureFlagsState(False)
@@ -598,6 +615,14 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState
             log.error("Unable to read flags for all_flag_state: %s" % repr(e))
             return FeatureFlagsState(False)
 
+        if overrides_only:
+            if len(flags_map) == 0:
+                log.warning("all_flags_state() called before client has finished initializing! Feature store unavailable - returning empty state")
+                return FeatureFlagsState(False)
+            if not self._all_flags_overrides_only_warned:
+                self._all_flags_overrides_only_warned = True
+                log.warning("all_flags_state() called before client has finished initializing! Returning only flags from the override layer. This message is logged once.")
+
         for key, flag in flags_map.items():
             if client_only and not flag.get('clientSide', False):
                 continue

ldclient/async_config.py

@@ -21,6 +21,7 @@
     DataSourceBuilder,
     DataSourceBuilderConfig,
     HTTPConfig,
+    OverrideSourceBuilder,
     PrivateAttributesConfig
 )
 from ldclient.hook import AsyncHook
@@ -127,6 +128,13 @@ class AsyncDataSystemConfig:
     fdv1_fallback_synchronizer: Optional[DataSourceBuilder[AsyncSynchronizer]] = None
     """An optional fallback synchronizer that will read from FDv1"""
 
+    override_source: Optional[OverrideSourceBuilder] = None
+    """
+    An optional override source. Its flag and segment definitions take precedence over
+    LaunchDarkly data at evaluation time. Flag overrides are currently experimental and subject
+    to change.
+    """
+
 
 class AsyncConfig(DataSourceBuilderConfig, PrivateAttributesConfig):
     """Advanced configuration options for the async SDK client.

ldclient/client.py

@@ -89,6 +89,7 @@ def __init__(self, config: Config, start_wait: float = 5):
         self._cached_data_warning_lock = threading.Lock()
         self._eval_cached_data_warned = False
         self._all_flags_cached_data_warned = False
+        self._all_flags_overrides_only_warned = False
 
         self._owner_pid = os.getpid()
         self._fork_warned_pids: Dict[int, object] = {}
@@ -467,6 +468,13 @@ def _evaluate_internal(self, key: str, context: Context, default: Any, event_fac
         if self._config.offline:
             return EvaluationDetail(default, None, error_reason('CLIENT_NOT_READY')), None
 
+        def not_ready() -> Tuple[EvaluationDetail, Optional[FeatureFlag]]:
+            log.warning("Feature Flag evaluation attempted before client has initialized! Feature store unavailable - returning default: " + str(default) + " for feature key: " + key)
+            reason = error_reason('CLIENT_NOT_READY')
+            self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason))
+            return EvaluationDetail(default, None, reason), None
+
+        no_launchdarkly_data = False
         availability = self._data_system.data_availability
         if availability != DataAvailability.REFRESHED:
             if availability == DataAvailability.CACHED:
@@ -475,11 +483,13 @@ def _evaluate_internal(self, key: str, context: Context, default: Any, event_fac
                         if not self._eval_cached_data_warned:
                             self._eval_cached_data_warned = True
                             log.warning("Feature Flag evaluation attempted before client has initialized - using last known values from feature store for feature key: " + key + ". This message is logged once.")
+            elif self._data_system.override_source_configured:
+                # No data from LaunchDarkly is available. The store read below still finds an
+                # entry that the override layer holds, and the SDK serves it. A miss returns the
+                # not-ready default.
+                no_launchdarkly_data = True
             else:
-                log.warning("Feature Flag evaluation attempted before client has initialized! Feature store unavailable - returning default: " + str(default) + " for feature key: " + key)
-                reason = error_reason('CLIENT_NOT_READY')
-                self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason))
-                return EvaluationDetail(default, None, reason), None
+                return not_ready()
 
         if not context.valid:
             log.warning("Context was invalid for flag evaluation (%s); returning default value" % context.error)
@@ -494,6 +504,8 @@ def _evaluate_internal(self, key: str, context: Context, default: Any, event_fac
             self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason))
             return EvaluationDetail(default, None, reason), None
         if not flag:
+            if no_launchdarkly_data:
+                return not_ready()
             reason = error_reason('FLAG_NOT_FOUND')
             self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason))
             return EvaluationDetail(default, None, reason), None
@@ -545,6 +557,7 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState:
 
         self._check_forked()
 
+        overrides_only = False
         availability = self._data_system.data_availability
         if availability != DataAvailability.REFRESHED:
             if availability == DataAvailability.CACHED:
@@ -553,6 +566,10 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState:
                         if not self._all_flags_cached_data_warned:
                             self._all_flags_cached_data_warned = True
                             log.warning("all_flags_state() called before client has finished initializing! Using last known values from feature store. This message is logged once.")
+            elif self._data_system.override_source_configured:
+                # No data from LaunchDarkly is available. The store read below returns only the
+                # entries that the override layer holds. The result decides the state.
+                overrides_only = True
             else:
                 log.warning("all_flags_state() called before client has finished initializing! Feature store unavailable - returning empty state")
                 return FeatureFlagsState(False)
@@ -573,6 +590,16 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState:
             log.error("Unable to read flags for all_flag_state: %s" % repr(e))
             return FeatureFlagsState(False)
 
+        if overrides_only:
+            if len(flags_map) == 0:
+                log.warning("all_flags_state() called before client has finished initializing! Feature store unavailable - returning empty state")
+                return FeatureFlagsState(False)
+            if not self._all_flags_overrides_only_warned:
+                with self._cached_data_warning_lock:
+                    if not self._all_flags_overrides_only_warned:
+                        self._all_flags_overrides_only_warned = True
+                        log.warning("all_flags_state() called before client has finished initializing! Returning only flags from the override layer. This message is logged once.")
+
         for key, flag in flags_map.items():
             if client_only and not flag.get('clientSide', False):
                 continue

ldclient/config.py

@@ -24,6 +24,7 @@
     EventProcessor,
     FeatureStore,
     Initializer,
+    OverrideSource,
     Synchronizer,
     UpdateProcessor
 )
@@ -255,6 +256,23 @@ def build(self, config: DataSourceBuilderConfig) -> T_co:
         raise NotImplementedError
 
 
+class OverrideSourceBuilder(Protocol):  # pylint: disable=too-few-public-methods
+    """
+    Protocol for building an override source. Flag overrides are currently experimental and
+    subject to change.
+    """
+
+    def build(self, config: DataSourceBuilderConfig) -> OverrideSource:
+        """
+        Builds the override source. Invalid configuration raises an exception, which the client
+        constructor propagates.
+
+        :param config: the SDK configuration
+        :return: the built override source
+        """
+        raise NotImplementedError
+
+
 @dataclass(frozen=True)
 class DataSystemConfig:
     """Configuration for LaunchDarkly's data acquisition strategy."""
@@ -278,6 +296,13 @@ class DataSystemConfig:
     fdv1_fallback_synchronizer: Optional[DataSourceBuilder[Synchronizer]] = None
     """An optional fallback synchronizer that will read from FDv1"""
 
+    override_source: Optional[OverrideSourceBuilder] = None
+    """
+    An optional override source. Its flag and segment definitions take precedence over
+    LaunchDarkly data at evaluation time. Flag overrides are currently experimental and subject
+    to change.
+    """
+
 
 class Config(DataSourceBuilderConfig, PrivateAttributesConfig):
     """Advanced configuration options for the SDK client.
@@ -715,4 +740,4 @@ def _validate(self):
             log.warning("Missing or blank SDK key")
 
 
-__all__ = ['Config', 'BigSegmentsConfig', 'DataSourceBuilder', 'DataSystemConfig', 'HTTPConfig']
+__all__ = ['Config', 'BigSegmentsConfig', 'DataSourceBuilder', 'DataSystemConfig', 'HTTPConfig', 'OverrideSourceBuilder']

ldclient/datasystem.py

@@ -4,7 +4,11 @@
 
 from typing import List, Optional
 
-from ldclient.config import DataSourceBuilder, DataSystemConfig
+from ldclient.config import (
+    DataSourceBuilder,
+    DataSystemConfig,
+    OverrideSourceBuilder
+)
 from ldclient.impl.datasourcev2.polling import (
     FallbackToFDv1PollingDataSourceBuilder,
     PollingDataSourceBuilder
@@ -32,6 +36,7 @@ def __init__(self) -> None:
         self._fdv1_fallback_synchronizer: Optional[DataSourceBuilder[Synchronizer]] = None
         self._store_mode: DataStoreMode = DataStoreMode.READ_ONLY
         self._data_store: Optional[FeatureStore] = None
+        self._override_source: Optional[OverrideSourceBuilder] = None
 
     def initializers(self, initializers: Optional[List[DataSourceBuilder[Initializer]]]) -> "ConfigBuilder":
         """
@@ -80,6 +85,35 @@ def data_store(self, data_store: FeatureStore, store_mode: DataStoreMode) -> "Co
         self._store_mode = store_mode
         return self
 
+    def overrides(self, source: OverrideSourceBuilder) -> "ConfigBuilder":
+        """
+        Configures the SDK with an override source. Flag overrides are currently experimental
+        and subject to change.
+
+        The source supplies flag and segment definitions that take precedence over data
+        received from LaunchDarkly on a per-key basis. Overrides let an operator force one or
+        more flags to a known state on a running client, whether or not the client can reach
+        LaunchDarkly. Flags not present in the override data are unaffected.
+
+        The override source is not a data source. It has no effect on the client's
+        initialization status or data source status. Configuring it changes nothing until the
+        source actually supplies an override. At most one override source is accepted.
+
+        Example, with the file-based source and the SDK's default data sources:
+        ::
+
+            from ldclient import datasystem
+            from ldclient.integrations.overrides import FileOverrideSourceBuilder
+
+            source = FileOverrideSourceBuilder(['/etc/launchdarkly/overrides.json'])
+            config = Config(sdk_key, datasystem_config=datasystem.default().overrides(source).build())
+
+        :param source: a builder for the override source, such as
+          :class:`ldclient.integrations.overrides.FileOverrideSourceBuilder`
+        """
+        self._override_source = source
+        return self
+
     def build(self) -> DataSystemConfig:
         """
         Builds the data system configuration.
@@ -90,6 +124,7 @@ def build(self) -> DataSystemConfig:
             fdv1_fallback_synchronizer=self._fdv1_fallback_synchronizer,
             data_store_mode=self._store_mode,
             data_store=self._data_store,
+            override_source=self._override_source,
         )
 
 

ldclient/impl/datasystem/__init__.py

@@ -158,6 +158,17 @@ def environment_id(self) -> Optional[str]:
         """
         raise NotImplementedError
 
+    @property
+    @abstractmethod
+    def override_source_configured(self) -> bool:
+        """
+        Reports whether the data system was built with an override source. The value is
+        fixed at construction. When true, the store served by :attr:`store` overlays the
+        override layer, and the client serves an overridden flag before it has LaunchDarkly
+        data.
+        """
+        raise NotImplementedError
+
 
 class AsyncDataSystem(Protocol):
     """
@@ -232,6 +243,15 @@ def store(self) -> AsyncReadOnlyStore:
         """
         raise NotImplementedError
 
+    @property
+    @abstractmethod
+    def override_source_configured(self) -> bool:
+        """
+        Reports whether the data system was built with an override source. The value is
+        fixed at construction.
+        """
+        raise NotImplementedError
+
 
 class DiagnosticAccumulator(Protocol):
     def record_stream_init(self, timestamp, duration, failed):

ldclient/impl/datasystem/async_fdv1.py

@@ -124,6 +124,11 @@ async def stop(self):
     def store(self) -> AsyncReadOnlyStore:
         return self._store_view
 
+    @property
+    def override_source_configured(self) -> bool:
+        # Overrides are an option of the FDv2 data system only.
+        return False
+
     def set_diagnostic_accumulator(self, diagnostic_accumulator: DiagnosticAccumulator):
         """
         Sets the diagnostic accumulator for streaming initialization metrics.