feat: multi-address DNS resolution for contact points and connections (DRIVER-201)

core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java

@@ -27,6 +27,7 @@
 import com.datastax.oss.protocol.internal.util.Bytes;
 import edu.umd.cs.findbugs.annotations.NonNull;
 import edu.umd.cs.findbugs.annotations.Nullable;
+import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
 import java.security.PrivilegedActionException;
@@ -319,7 +320,7 @@ protected GssApiAuthenticator(
                 SUPPORTED_MECHANISMS,
                 options.getAuthorizationId(),
                 protocol,
-                ((InetSocketAddress) endPoint.resolve()).getAddress().getCanonicalHostName(),
+                serverName(endPoint),
                 options.getSaslProperties(),
                 null);
       } catch (LoginException | SaslException e) {
@@ -328,6 +329,23 @@ protected GssApiAuthenticator(
       this.endPoint = endPoint;
     }
 
+    /**
+     * The host name to build the Kerberos service principal from.
+     *
+     * <p>Prefers the canonical name of the resolved address, which is what Kerberos expects. The
+     * driver's own endpoints always hand this a resolved address — the channel carries an endpoint
+     * bound to the address it connected to (see {@code PinnableEndPoint}) — but a custom {@link
+     * EndPoint} implementation may still yield an unresolved one, in which case {@code
+     * getAddress()} is null. Fall back to the host string rather than throwing a {@link
+     * NullPointerException}: the hostname is usually the right service name anyway, and a failed
+     * reverse lookup should not take authentication down.
+     */
+    private static String serverName(EndPoint endPoint) {
+      InetSocketAddress address = (InetSocketAddress) endPoint.resolve();
+      InetAddress inetAddress = address.getAddress();
+      return inetAddress != null ? inetAddress.getCanonicalHostName() : address.getHostString();
+    }
+
     @NonNull
     @Override
     protected ByteBuffer getMechanism() {

core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java

@@ -135,6 +135,13 @@ public enum DefaultDriverOption implements DriverOption {
    * <p>Value-type: int
    */
   CONNECTION_MAX_ORPHAN_REQUESTS("advanced.connection.max-orphan-requests"),
+  /**
+   * The maximum number of addresses a single connection attempt will try, when the endpoint it
+   * connects to is a DNS name that resolves to several addresses.
+   *
+   * <p>Value-type: int
+   */
+  CONNECTION_MAX_CANDIDATE_ADDRESSES("advanced.connection.max-candidate-addresses"),
   /**
    * Whether to log non-fatal errors when the driver tries to open a new connection.
    *
@@ -701,8 +708,14 @@ public enum DefaultDriverOption implements DriverOption {
   CONTROL_CONNECTION_AGREEMENT_WARN("advanced.control-connection.schema-agreement.warn-on-failure"),
 
   /**
-   * Whether to forcibly add original contact points held by MetadataManager to the reconnection
-   * plan, in case there is no live nodes available according to LBP. Experimental.
+   * Whether to append the original contact points held by MetadataManager to the reconnection plan,
+   * after the live nodes reported by the load balancing policy. Defaults to {@code true}.
+   *
+   * <p>This is also the driver's DNS re-resolution path. Contact points are appended as-is, still
+   * unresolved hostnames, and each is expanded to its current DNS IPs at connection time through
+   * Netty's configured resolver. Metadata nodes, in contrast, hold an already-resolved endpoint
+   * that is never re-resolved. Keeping this enabled lets control-connection reconnects re-resolve
+   * the original hostnames and pick up new IPs once the live-node plan is exhausted.
    *
    * <p>Value-type: boolean
    */
@@ -837,7 +850,11 @@ public enum DefaultDriverOption implements DriverOption {
    * Whether to resolve the addresses passed to `basic.contact-points`.
    *
    * <p>Value-type: boolean
+   *
+   * @deprecated Contact points are now always kept as unresolved hostnames and expanded to all
+   *     their DNS-mapped IPs lazily at connection time. Setting this option has no effect.
    */
+  @Deprecated
   RESOLVE_CONTACT_POINTS("advanced.resolve-contact-points"),
 
   /**

core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java

@@ -245,6 +245,9 @@ private void readObject(ObjectInputStream stream) throws InvalidObjectException
     throw new InvalidObjectException("Proxy required");
   }
 
+  // RESOLVE_CONTACT_POINTS is deprecated and has no effect, but it is still a driver option, so the
+  // defaults map stays complete by carrying its reference.conf value.
+  @SuppressWarnings("deprecation")
   protected static void fillWithDriverDefaults(OptionsMap map) {
     Duration initQueryTimeout = Duration.ofSeconds(5);
     Duration requestTimeout = Duration.ofSeconds(2);
@@ -276,6 +279,7 @@ protected static void fillWithDriverDefaults(OptionsMap map) {
     map.put(TypedDriverOption.CONNECTION_POOL_INIT_BATCH_SIZE, 0);
     map.put(TypedDriverOption.CONNECTION_MAX_REQUESTS, 1024);
     map.put(TypedDriverOption.CONNECTION_MAX_ORPHAN_REQUESTS, 256);
+    map.put(TypedDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES, 5);
     map.put(TypedDriverOption.CONNECTION_WARN_INIT_ERROR, true);
     map.put(TypedDriverOption.CONNECTION_ADVANCED_SHARD_AWARENESS_ENABLED, true);
     map.put(TypedDriverOption.ADVANCED_SHARD_AWARENESS_PORT_LOW, 10000);
@@ -369,7 +373,7 @@ protected static void fillWithDriverDefaults(OptionsMap map) {
     map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_INTERVAL, Duration.ofMillis(200));
     map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_TIMEOUT, Duration.ofSeconds(10));
     map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_WARN, true);
-    map.put(TypedDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, false);
+    map.put(TypedDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, true);
     map.put(TypedDriverOption.PREPARE_ON_ALL_NODES, true);
     map.put(TypedDriverOption.REPREPARE_ENABLED, true);
     map.put(TypedDriverOption.REPREPARE_CHECK_SYSTEM_TABLE, false);

core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java

@@ -172,6 +172,13 @@ public String toString() {
   public static final TypedDriverOption<Integer> CONNECTION_MAX_ORPHAN_REQUESTS =
       new TypedDriverOption<>(
           DefaultDriverOption.CONNECTION_MAX_ORPHAN_REQUESTS, GenericType.INTEGER);
+  /**
+   * The maximum number of addresses a single connection attempt will try, when the endpoint it
+   * connects to is a DNS name that resolves to several addresses.
+   */
+  public static final TypedDriverOption<Integer> CONNECTION_MAX_CANDIDATE_ADDRESSES =
+      new TypedDriverOption<>(
+          DefaultDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES, GenericType.INTEGER);
   /** Whether to log non-fatal errors when the driver tries to open a new connection. */
   public static final TypedDriverOption<Boolean> CONNECTION_WARN_INIT_ERROR =
       new TypedDriverOption<>(DefaultDriverOption.CONNECTION_WARN_INIT_ERROR, GenericType.BOOLEAN);
@@ -600,7 +607,15 @@ public String toString() {
   public static final TypedDriverOption<Boolean> CONTROL_CONNECTION_AGREEMENT_WARN =
       new TypedDriverOption<>(
           DefaultDriverOption.CONTROL_CONNECTION_AGREEMENT_WARN, GenericType.BOOLEAN);
-  /** Whether to forcibly try original contacts if no live nodes are available */
+  /**
+   * Whether to append the original contact points to the control-connection reconnection plan,
+   * after the live nodes reported by the load balancing policy (defaults to {@code true}).
+   *
+   * <p>Contact points are appended as-is (unresolved hostnames); each is expanded to all of its
+   * current DNS IPs at connection time, which is also the driver's DNS re-resolution mechanism. The
+   * append is skipped for topology monitors that re-resolve node addresses themselves (such as the
+   * cloud/proxy monitors).
+   */
   public static final TypedDriverOption<Boolean> CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS =
       new TypedDriverOption<>(
           DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, GenericType.BOOLEAN);
@@ -664,7 +679,13 @@ public String toString() {
   /** The coalescer reschedule interval. */
   public static final TypedDriverOption<Duration> COALESCER_INTERVAL =
       new TypedDriverOption<>(DefaultDriverOption.COALESCER_INTERVAL, GenericType.DURATION);
-  /** Whether to resolve the addresses passed to `basic.contact-points`. */
+  /**
+   * Whether to resolve the addresses passed to `basic.contact-points`.
+   *
+   * @deprecated Contact points are now always kept as unresolved hostnames and expanded to all
+   *     their DNS-mapped IPs lazily at connection time. Setting this option has no effect.
+   */
+  @Deprecated
   public static final TypedDriverOption<Boolean> RESOLVE_CONTACT_POINTS =
       new TypedDriverOption<>(DefaultDriverOption.RESOLVE_CONTACT_POINTS, GenericType.BOOLEAN);
   /**

core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java

@@ -18,24 +18,59 @@
 package com.datastax.oss.driver.api.core.metadata;
 
 import edu.umd.cs.findbugs.annotations.NonNull;
-import java.net.InetSocketAddress;
 import java.net.SocketAddress;
 
 /**
  * Encapsulates the information needed to open connections to a node.
  *
  * <p>By default, the driver assumes plain TCP connections, and this is just a wrapper around an
- * {@link InetSocketAddress}. However, more complex deployment scenarios might use a custom
+ * {@link java.net.InetSocketAddress}. However, more complex deployment scenarios might use a custom
  * implementation that contains additional information; for example, if the nodes are accessed
  * through a proxy with SNI routing, an SNI server name is needed in addition to the proxy address.
  */
 public interface EndPoint {
 
   /**
-   * Resolves this instance to a socket address.
+   * Resolves this instance to the socket address connections should be opened to.
    *
    * <p>This will be called each time the driver opens a new connection to the node. The returned
    * address cannot be null.
+   *
+   * <p><b>Returning a hostname is fine, and is how multi-address support works.</b> The returned
+   * address need not be resolved: an {@linkplain java.net.InetSocketAddress#isUnresolved()
+   * unresolved} {@link java.net.InetSocketAddress} is expanded by the driver to <b>every</b>
+   * address the name maps to, and each one is tried in turn until a connection succeeds. That is
+   * what {@link com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint} does for contact
+   * points backed by a hostname, so a single unreachable IP behind a multi-record name no longer
+   * fails the connection.
+   *
+   * <p><b>Implementations must not resolve names themselves, and must not block.</b> The driver
+   * calls this from its admin event loop, and it performs the expansion through Netty's configured
+   * {@code AddressResolverGroup} — the same resolver an unresolved address reaches when it is
+   * handed to {@code Bootstrap.connect()}. Looking the name up here instead (for example with
+   * {@link java.net.InetAddress#getAllByName(String)}) would both block that loop and bypass a
+   * custom resolver installed via {@code NettyOptions#afterBootstrapInitialized(Bootstrap)}.
+   *
+   * <p><b>Callers must not assume the returned address is resolved.</b> It is for a node discovered
+   * from {@code system.peers} (built from that node's physical broadcast RPC address) and for the
+   * node the control connection is on (bound to the address that connection reached). It is
+   * <b>not</b> for a node reached through the Cloud SNI proxy, or through a cloud private-endpoint
+   * client route: there the address is the configured hostname, and {@link
+   * java.net.InetSocketAddress#getAddress()} returns {@code null}. Read the host with {@link
+   * java.net.InetSocketAddress#getHostString()}, which yields whichever of the two the address
+   * carries and never triggers a reverse lookup.
+   *
+   * @apiNote <b>Timeout note:</b> when a name expands to several addresses they are tried in
+   *     sequence, so the worst-case time before the node is declared unreachable is N times a full
+   *     attempt — and an attempt is more than a connect. Each address that accepts the TCP
+   *     connection then runs the init handshake, whose steps each arm their own {@code
+   *     advanced.connection.init-query-timeout}; those add up rather than sharing one deadline. An
+   *     address that stalls after accepting the connection can therefore burn {@code
+   *     advanced.connection.connect-timeout} plus several times {@code
+   *     advanced.connection.init-query-timeout} on its own. In practice DNS round-robin entries
+   *     have only a small number of records, so this is rarely a concern, but it is worth bearing
+   *     in mind when configuring timeouts — note also that session initialization has no overall
+   *     deadline of its own.
    */
   @NonNull
   SocketAddress resolve();

core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java

@@ -166,11 +166,15 @@ protected DriverConfigLoader defaultConfigLoader(@Nullable ClassLoader classLoad
    * <p>Contact points can also be provided statically in the configuration. If both are specified,
    * they will be merged. If both are absent, the driver will default to 127.0.0.1:9042.
    *
-   * <p>Contrary to the configuration, DNS names with multiple A-records will not be handled here.
-   * If you need that, extract them manually with {@link java.net.InetAddress#getAllByName(String)}
-   * before calling this method. Similarly, if you need connect addresses to stay unresolved, make
-   * sure you pass unresolved instances here (see {@code advanced.resolve-contact-points} in the
-   * configuration for more explanations).
+   * <p>The driver automatically expands any contact point backed by an unresolved hostname to all
+   * its DNS-mapped IPs at connection time (through Netty's configured resolver, so a custom {@code
+   * AddressResolverGroup} still applies), so passing a single hostname is sufficient to try all its
+   * IPs on initial connect. This applies equally to hostnames provided here programmatically (build
+   * an unresolved {@link InetSocketAddress} with {@link InetSocketAddress#createUnresolved(String,
+   * int)} to opt in) and to hostnames specified in the configuration. An already-resolved address
+   * passed here (the common case when constructing an {@code InetSocketAddress} directly from a
+   * hostname, which resolves eagerly) is used as provided, with no further expansion. The {@code
+   * advanced.resolve-contact-points} option is deprecated and has no effect.
    */
   @NonNull
   public SelfT addContactPoints(@NonNull Collection<InetSocketAddress> contactPoints) {
@@ -741,6 +745,23 @@ public SelfT withCloudSecureConnectBundle(@NonNull InputStream cloudConfigInputS
    *
    * <p>For more information, please refer to the DataStax Astra documentation.
    *
+   * <p>A proxy given as a hostname is resolved at connection time, to <b>all</b> of its addresses,
+   * and each is tried in turn. That holds however the {@link InetSocketAddress} was built: the
+   * driver keeps a proxy hostname unresolved internally, so passing one that the ordinary {@code
+   * InetSocketAddress(String, int)} constructor already resolved does not bind the session to that
+   * single address.
+   *
+   * <p>Prefer {@link InetSocketAddress#createUnresolved(String, int)} all the same, and especially
+   * for a proxy given as an <b>IP address</b>. Whether an address carries a name is read from
+   * {@code getHostString()}, which falls back to the underlying {@link java.net.InetAddress}'s
+   * cached host name -- and that field is filled in, on the very instance passed here, the first
+   * time anything calls {@code getHostName()} on it. The SNI SSL engine does exactly that while
+   * building an engine, unless reverse-lookup SANs are turned off. So an IP that has a {@code PTR}
+   * record can acquire a name mid-session, after which the driver treats that name as the proxy:
+   * the endpoints it builds from then on compare unequal to the earlier ones, report metrics under
+   * a different prefix, and connect to wherever that name resolves. An unresolved address is never
+   * subject to this, and is what the secure connect bundle produces.
+   *
    * @param cloudProxyAddress The address of the Cloud proxy to use.
    * @see <a href="https://en.wikipedia.org/wiki/Server_Name_Indication">Server Name Indication</a>
    */
@@ -957,11 +978,15 @@ protected final CompletionStage<CqlSession> buildDefaultSessionAsync() {
         programmaticArguments = programmaticArgumentsBuilder.build();
       }
 
-      boolean resolveAddresses =
-          defaultConfig.getBoolean(DefaultDriverOption.RESOLVE_CONTACT_POINTS, false);
-
+      // RESOLVE_CONTACT_POINTS is deprecated: contact points are always kept as unresolved
+      // hostnames, and expanded to all their DNS IPs at connection time by ChannelFactory.
+      // The value is still read, only to tell someone who set it that it no longer does anything.
+      // Note this tests the value rather than isDefined(): unlike the deprecated options warned
+      // about in DefaultDriverContext, this one ships uncommented in reference.conf, so it is
+      // always defined.
+      warnIfResolveContactPointsRequested(defaultConfig);
       Set<EndPoint> contactPoints =
-          ContactPoints.merge(programmaticContactPoints, configContactPoints, resolveAddresses);
+          ContactPoints.merge(programmaticContactPoints, configContactPoints, false);
 
       if (keyspace == null && defaultConfig.isDefined(DefaultDriverOption.SESSION_KEYSPACE)) {
         keyspace =
@@ -989,6 +1014,22 @@ private boolean anyProfileHasDatacenterDefined(DriverConfig driverConfig) {
     return false;
   }
 
+  /**
+   * Tells anyone who turned {@code advanced.resolve-contact-points} on that it no longer does
+   * anything, so the behaviour they configured does not disappear in silence.
+   */
+  @SuppressWarnings("deprecation")
+  private static void warnIfResolveContactPointsRequested(DriverExecutionProfile defaultConfig) {
+    if (defaultConfig.getBoolean(DefaultDriverOption.RESOLVE_CONTACT_POINTS, false)) {
+      LOG.warn(
+          "Option {} is deprecated and no longer has any effect. Contact points are now always kept"
+              + " as unresolved hostnames and expanded to all of their addresses at connection"
+              + " time, so a name that resolves to several nodes is one Node in the driver's"
+              + " metadata rather than one per address.",
+          DefaultDriverOption.RESOLVE_CONTACT_POINTS.getPath());
+    }
+  }
+
   /**
    * Returns URL based on the configUrl setting. If the configUrl has no protocol provided, the
    * method will fallback to file:// protocol and return URL that has file protocol specified.