feat: Add the override layer, the store overlay, and the override source configuration

The OVERRIDE specification defines an override layer: a runtime-mutable
collection of flag and segment definitions, supplied by an override
source as complete snapshots, that takes precedence over LaunchDarkly
data on a per-key basis at the store read boundary. Overrides are not a
data source. They have no effect on initialization status, data
availability, or data source status, and they are never persisted.

Public surface, all experimental and subject to change:
DataSystemBuilder.overrides(ComponentConfigurer<OverrideSource>),
DataSystemConfiguration.getOverrideSource(), and the
subsystems.OverrideSource and subsystems.OverrideSink interfaces. The
SDK builds the source like any other component, starts it before the
data source so its initial load completes during client construction,
and closes it with the client. An offline client starts no override
source. A source that cannot be built fails client construction.

OverrideLayer holds marked shallow copies in an immutable map swapped on
each update. OverrideOverlayStore implements the read boundary with
override precedence for per-key reads and enumeration, and serves the
layer alone when the base store fails. OverrideSinkImpl serializes
snapshot application and fires the normal flag change notifications for
every flag whose merged-view evaluation may have changed.

The not-initialized short-circuit consults the layer first, so a flag
that the layer holds is served before the client has LaunchDarkly data.
The all-flags state does the same and presents an override-affected
flag with event tracking off. The specification's test vectors run as a
unit test through the full client stack.

Author: kinyoklion

lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSystem.java

@@ -62,6 +62,15 @@ interface DataSystem {
    * @return the environment ID, or null
    */
   String getEnvironmentId();
+
+  /**
+   * Returns the override layer, or null if no override source is configured. The layer holds the
+   * entries that take precedence over LaunchDarkly data; {@link #getStore()} already applies them.
+   * The client consults the layer directly only for the not-initialized short-circuit.
+   *
+   * @return the override layer, or null
+   */
+  OverrideLayer getOverrideLayer();
 }
 
 /**

lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv1DataSystem.java

@@ -160,6 +160,11 @@ public String getEnvironmentId() {
     return dataStore.getEnvironmentId();
   }
 
+  @Override
+  public OverrideLayer getOverrideLayer() {
+    return null;
+  }
+
   @Override
   public void close() throws IOException {
     if (disposed) {

lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSystem.java

@@ -15,6 +15,7 @@
 import com.launchdarkly.sdk.server.subsystems.DataStore;
 import com.launchdarkly.sdk.server.subsystems.LoggingConfiguration;
 import com.launchdarkly.sdk.server.subsystems.DataSystemConfiguration;
+import com.launchdarkly.sdk.server.subsystems.OverrideSource;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -34,21 +35,44 @@ final class FDv2DataSystem implements DataSystem, Closeable {
   private final FlagChangeNotifier flagChanged;
   private final DataSourceStatusProvider dataSourceStatusProvider;
   private final DataStoreStatusProvider dataStoreStatusProvider;
+  private final LDLogger logger;
+  // The following are non-null only when an override source is configured. The layer holds the
+  // override entries. The overlay serves them in preference to the store's data at the store read
+  // boundary. The source populates the layer at runtime. None of them take part in
+  // initialization or data source status.
+  private final OverrideLayer overrideLayer;
+  private final OverrideSource overrideSource;
+  private final OverrideSinkImpl overrideSink;
   private boolean disposed = false;
 
   private FDv2DataSystem(
     DataStore store,
     DataSource dataSource,
     DataSourceStatusProvider dataSourceStatusProvider,
     DataStoreStatusProvider dataStoreStatusProvider,
-    FlagChangeNotifier flagChanged
+    FlagChangeNotifier flagChanged,
+    EventBroadcasterImpl<FlagChangeListener, FlagChangeEvent> flagChangeBroadcaster,
+    OverrideSource overrideSource,
+    LDLogger logger
   ) {
     this.store = store;
     this.dataSource = dataSource;
     this.dataStoreStatusProvider = dataStoreStatusProvider;
     this.dataSourceStatusProvider = dataSourceStatusProvider;
     this.flagChanged = flagChanged;
-    this.readOnlyStore = new ReadonlyStoreFacade(store);
+    this.logger = logger;
+    ReadOnlyStore baseStore = new ReadonlyStoreFacade(store);
+    this.overrideSource = overrideSource;
+    if (overrideSource == null) {
+      this.overrideLayer = null;
+      this.overrideSink = null;
+      this.readOnlyStore = baseStore;
+    } else {
+      this.overrideLayer = new OverrideLayer();
+      this.overrideSink = new OverrideSinkImpl(overrideLayer, baseStore, flagChangeBroadcaster,
+          logger.subLogger(Loggers.DATA_SOURCE_LOGGER_NAME));
+      this.readOnlyStore = new OverrideOverlayStore(baseStore, overrideLayer);
+    }
   }
 
   private static class FactoryWrapper<TDataSource> implements FDv2DataSource.DataSourceFactory<TDataSource> {
@@ -187,12 +211,23 @@ static FDv2DataSystem create(
 
     FlagChangeNotifier flagChanged = new FlagChangedFacade(dataSourceUpdates);
 
+    // The override source is built like any other component. Invalid configuration fails here, the
+    // same way an invalid data source or data store configuration fails. An offline client starts
+    // no data sources and no override source.
+    OverrideSource overrideSource = null;
+    if (dataSystemConfiguration.getOverrideSource() != null && !config.offline) {
+      overrideSource = dataSystemConfiguration.getOverrideSource().build(clientContext);
+    }
+
     return new FDv2DataSystem(
       store,
       dataSource,
       dataSourceStatusProvider,
       dataStoreStatusProvider,
-      flagChanged
+      flagChanged,
+      flagChangeBroadcaster,
+      overrideSource,
+      logger
     );
   }
 
@@ -203,6 +238,12 @@ public ReadOnlyStore getStore() {
 
   @Override
   public Future<Void> start() {
+    if (overrideSource != null) {
+      // The source starts before the data source, so a source that loads synchronously has its
+      // overrides in place before the client begins evaluating. Its initial load is part of
+      // starting the client.
+      overrideSource.start(overrideSink);
+    }
     return dataSource.start();
   }
 
@@ -231,12 +272,24 @@ public String getEnvironmentId() {
     return store.getEnvironmentId();
   }
 
+  @Override
+  public OverrideLayer getOverrideLayer() {
+    return overrideLayer;
+  }
+
   @Override
   public void close() throws IOException {
     if (disposed) {
       return;
     }
     try {
+      if (overrideSource != null) {
+        try {
+          overrideSource.close();
+        } catch (IOException | RuntimeException e) {
+          logger.warn("Error closing override source: {}", e.toString());
+        }
+      }
       dataSource.close();
       store.close();
     } finally {

lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FeatureFlagsState.java

@@ -276,15 +276,19 @@ public Builder add(
     }
 
     Builder addFlag(DataModel.FeatureFlag flag, EvalResult eval) {
+      // A consumer of this state sends individual events according to the tracking fields. An
+      // override-affected evaluation produces no individual events, so the state turns them off
+      // for this flag. The flag, its value, its version, and its reason stay.
+      boolean overrideAffected = eval.isOverrideAffected();
       return add(
           flag.getKey(),
           eval.getValue(),
           eval.isNoVariation() ? null : eval.getVariationIndex(),
           eval.getReason(),
           flag.getVersion(),
-          flag.isTrackEvents() || eval.isForceReasonTracking(),
-          eval.isForceReasonTracking(),
-          flag.getDebugEventsUntilDate(),
+          !overrideAffected && (flag.isTrackEvents() || eval.isForceReasonTracking()),
+          !overrideAffected && eval.isForceReasonTracking(),
+          overrideAffected ? null : flag.getDebugEventsUntilDate(),
           eval.getPrerequisiteEvalRecords().stream()
               .filter(record -> record.prereqOfFlag.getKey() == flag.getKey())  // only include top level prereqs
               .map(record -> record.flag.getKey()) // map from prereq record to prereq key

lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/InputValidatingEvaluator.java

@@ -14,6 +14,7 @@
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import static com.launchdarkly.sdk.EvaluationDetail.NO_VARIATION;
 import static com.launchdarkly.sdk.server.DataModel.FEATURES;
@@ -28,7 +29,11 @@ class InputValidatingEvaluator implements EvaluatorInterface {
 
   private final Evaluator evaluator;
   private final ReadOnlyStore store;
+  // Null when no override source is configured. The store already applies the layer. The layer is
+  // consulted directly only for the not-initialized short-circuit.
+  private final OverrideLayer overrideLayer;
   private final LDLogger logger;
+  private final AtomicBoolean allFlagsOverridesOnlyWarningLogged = new AtomicBoolean(false);
 
   // these are created at construction to avoid recreation during each evaluation
   private final EvaluationRecorder evaluationEventRecorderWithDetails;
@@ -40,11 +45,14 @@ class InputValidatingEvaluator implements EvaluatorInterface {
    * Creates an {@link InputValidatingEvaluator}
    *
    * @param store          will be used to get flag data
+   * @param overrideLayer  the override layer, or null if no override source is configured
    * @param segmentStore   will be used to get segment data
    * @param eventProcessor will be used to record events during evaluations as necessary
    * @param logger         for logging messages and errors during evaluations
    */
-  InputValidatingEvaluator(ReadOnlyStore store, BigSegmentStoreWrapper segmentStore, @Nonnull EventProcessor eventProcessor, LDLogger logger) {
+  InputValidatingEvaluator(ReadOnlyStore store, @Nullable OverrideLayer overrideLayer, BigSegmentStoreWrapper segmentStore,
+                           @Nonnull EventProcessor eventProcessor, LDLogger logger) {
+    this.overrideLayer = overrideLayer;
     this.evaluator = new Evaluator(new Evaluator.Getters() {
       public DataModel.FeatureFlag getFlag(String key) {
         return InputValidatingEvaluator.getFlag(store, key);
@@ -101,7 +109,9 @@ public EvalResultAndFlag evalAndFlag(String method, String flagKey, LDContext co
    */
   EvalResultAndFlag evaluate(String flagKey, LDContext context, LDValue defaultValue,
                              @Nullable LDValueType requireType, EvaluationRecorder recorder) {
-    if (!store.isInitialized()) {
+    // The override layer is consulted before the not-initialized short-circuit. A flag that the
+    // layer holds is served from it. Any other flag returns the not-ready default as before.
+    if (!store.isInitialized() && !hasOverrideFor(flagKey)) {
       logger.warn("Evaluation called before client initialized for feature flag \"{}\"; data store unavailable, returning default value", flagKey);
       recorder.recordEvaluationUnknownFlagError(flagKey, context, defaultValue, ErrorKind.CLIENT_NOT_READY);
       return new EvalResultAndFlag(EvalResult.error(ErrorKind.CLIENT_NOT_READY, defaultValue), null);
@@ -163,8 +173,16 @@ public FeatureFlagsState allFlagsState(LDContext context, FlagsStateOption... op
     FeatureFlagsState.Builder builder = FeatureFlagsState.builder(options);
 
     if (!store.isInitialized()) {
-      logger.warn("allFlagsState() was called before client initialized; data store unavailable, returning no data");
-      return builder.valid(false).build();
+      // With no LaunchDarkly data, the store read below returns only the entries that the override
+      // layer holds. The resulting state contains only those flags.
+      if (overrideLayer == null || overrideLayer.isEmpty()) {
+        logger.warn("allFlagsState() was called before client initialized; data store unavailable, returning no data");
+        return builder.valid(false).build();
+      }
+      if (allFlagsOverridesOnlyWarningLogged.compareAndSet(false, true)) {
+        logger.warn("allFlagsState() was called before client initialized; returning only flags from the override layer."
+            + " This message is logged once.");
+      }
     }
 
     if (context == null) {
@@ -214,6 +232,14 @@ private static DataModel.FeatureFlag getFlag(ReadOnlyStore store, String key) {
     return item == null ? null : (DataModel.FeatureFlag) item.getItem();
   }
 
+  private boolean hasOverrideFor(String flagKey) {
+    if (overrideLayer == null) {
+      return false;
+    }
+    DataStoreTypes.ItemDescriptor item = overrideLayer.get(FEATURES, flagKey);
+    return item != null && item.getItem() != null;
+  }
+
   private static DataModel.Segment getSegment(ReadOnlyStore store, String key) {
     DataStoreTypes.ItemDescriptor item = store.get(SEGMENTS, key);
     return item == null ? null : (DataModel.Segment) item.getItem();

lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/LDClient.java

@@ -209,7 +209,8 @@ public LDClient(String sdkKey, LDConfig config) {
       this.dataSystem = FDv1DataSystem.create(baseLogger, config, context, context.getLogging());
     }
 
-    EvaluatorInterface evaluator = new InputValidatingEvaluator(this.dataSystem.getStore(), bigSegmentStoreWrapper, eventProcessor, evaluationLogger);
+    EvaluatorInterface evaluator = new InputValidatingEvaluator(this.dataSystem.getStore(),
+        this.dataSystem.getOverrideLayer(), bigSegmentStoreWrapper, eventProcessor, evaluationLogger);
 
     // build environment metadata for plugins
     SdkMetadata sdkMetadata;

lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/OverrideLayer.java

@@ -0,0 +1,133 @@
+package com.launchdarkly.sdk.server;
+
+import com.google.common.collect.ImmutableMap;
+import com.launchdarkly.sdk.server.DataModel.FeatureFlag;
+import com.launchdarkly.sdk.server.DataModel.Segment;
+import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.DataKind;
+import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.ItemDescriptor;
+import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.KeyedItems;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * The override layer: a thread-safe store of override entries that an override source replaces
+ * wholesale on each update. Entries take precedence over LaunchDarkly data at the store read
+ * boundary; see {@link OverrideOverlayStore}.
+ * <p>
+ * Each entry is a marked shallow copy of the entity that the source supplied. The copy shares its
+ * nested collections with the source's entity, and the layer never writes to them. The source's
+ * entity is never marked, so a source may retain the entities it supplied and supply them again.
+ * <p>
+ * This class is package-private and should not be used by application code.
+ */
+final class OverrideLayer {
+  /**
+   * The previous and current contents of the layer after a replacement. The maps must not be
+   * modified.
+   */
+  static final class Replacement {
+    final ImmutableMap<DataKind, ImmutableMap<String, ItemDescriptor>> previous;
+    final ImmutableMap<DataKind, ImmutableMap<String, ItemDescriptor>> current;
+
+    Replacement(
+        ImmutableMap<DataKind, ImmutableMap<String, ItemDescriptor>> previous,
+        ImmutableMap<DataKind, ImmutableMap<String, ItemDescriptor>> current
+    ) {
+      this.previous = previous;
+      this.current = current;
+    }
+  }
+
+  // The contents are an immutable map that is swapped on each update, so the layer holds exactly
+  // one snapshot at any instant and readers never take a lock.
+  private volatile ImmutableMap<DataKind, ImmutableMap<String, ItemDescriptor>> contents = ImmutableMap.of();
+  // A single volatile read decides the common case of a configured but unpopulated layer.
+  private volatile boolean nonEmpty = false;
+  private final Object writeLock = new Object();
+
+  /**
+   * Atomically replaces the entire layer contents. A null or empty collection clears the layer.
+   *
+   * @param data the complete set of entries, grouped by data kind
+   * @return the previous and the new contents
+   */
+  Replacement setAll(Iterable<Map.Entry<DataKind, KeyedItems<ItemDescriptor>>> data) {
+    Map<DataKind, Map<String, ItemDescriptor>> collected = new HashMap<>();
+    int count = 0;
+    if (data != null) {
+      for (Map.Entry<DataKind, KeyedItems<ItemDescriptor>> kindEntry : data) {
+        Map<String, ItemDescriptor> items = collected.computeIfAbsent(kindEntry.getKey(), k -> new HashMap<>());
+        Iterable<Map.Entry<String, ItemDescriptor>> kindItems = kindEntry.getValue() == null
+            ? null : kindEntry.getValue().getItems();
+        if (kindItems == null) {
+          continue;
+        }
+        for (Map.Entry<String, ItemDescriptor> item : kindItems) {
+          items.put(item.getKey(), markedCopy(item.getValue()));
+          count++;
+        }
+      }
+    }
+    ImmutableMap.Builder<DataKind, ImmutableMap<String, ItemDescriptor>> builder = ImmutableMap.builder();
+    for (Map.Entry<DataKind, Map<String, ItemDescriptor>> e : collected.entrySet()) {
+      builder.put(e.getKey(), ImmutableMap.copyOf(e.getValue()));
+    }
+    ImmutableMap<DataKind, ImmutableMap<String, ItemDescriptor>> replacement = builder.build();
+
+    synchronized (writeLock) {
+      ImmutableMap<DataKind, ImmutableMap<String, ItemDescriptor>> previous = contents;
+      contents = replacement;
+      nonEmpty = count != 0;
+      return new Replacement(previous, replacement);
+    }
+  }
+
+  /**
+   * Returns the override entry for a key, or null if the layer has none.
+   *
+   * @param kind the data kind
+   * @param key the item key
+   * @return the marked entry or null
+   */
+  ItemDescriptor get(DataKind kind, String key) {
+    if (!nonEmpty) {
+      return null;
+    }
+    ImmutableMap<String, ItemDescriptor> items = contents.get(kind);
+    return items == null ? null : items.get(key);
+  }
+
+  /**
+   * Returns the entries of a kind. The map must not be modified.
+   *
+   * @param kind the data kind
+   * @return the entries, possibly empty
+   */
+  ImmutableMap<String, ItemDescriptor> all(DataKind kind) {
+    ImmutableMap<String, ItemDescriptor> items = contents.get(kind);
+    return items == null ? ImmutableMap.of() : items;
+  }
+
+  /**
+   * Reports whether the layer contains no entries.
+   *
+   * @return true if empty
+   */
+  boolean isEmpty() {
+    return !nonEmpty;
+  }
+
+  // Returns a copy of the descriptor whose item carries the override marker. An item of another
+  // type, or a deleted item placeholder, is returned as is.
+  static ItemDescriptor markedCopy(ItemDescriptor item) {
+    Object entity = item.getItem();
+    if (entity instanceof FeatureFlag) {
+      return new ItemDescriptor(item.getVersion(), ((FeatureFlag) entity).markedAsOverride());
+    }
+    if (entity instanceof Segment) {
+      return new ItemDescriptor(item.getVersion(), ((Segment) entity).markedAsOverride());
+    }
+    return item;
+  }
+}