[feature] Add a 'bom-action' attribute to the http request to allow control over how the BOM (if present) in the http response should be processed.

Closes #58

Author: adamretter

http-client-java/pom.xml

@@ -89,6 +89,11 @@
             <groupId>com.google.code.findbugs</groupId>
             <artifactId>jsr305</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.evolvedbinary.j8xu</groupId>
+            <artifactId>j8xu</artifactId>
+            <version>1.2.0</version>
+        </dependency>
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>

http-client-java/src/main/java/org/expath/httpclient/BomAction.java

@@ -0,0 +1,85 @@
+/****************************************************************************/
+/*  File:       org.expath.httpclient.BomAction.java                        */
+/*  Author:     Adam Retter - Evolved Binary Ltd                            */
+/*  Date:       2026-06-16                                                  */
+/*  Tags:                                                                   */
+/*      Copyright (c) 2026 Evolved Binary Ltd (see end of file.)            */
+/* ------------------------------------------------------------------------ */
+
+
+package org.expath.httpclient;
+
+/**
+ * Provides options for BOM (Byte Order Mark) handling.
+ */
+public enum BomAction {
+
+    /**
+     * Raise and error if a BOM is present.
+     */
+    ERROR,
+
+    /**
+     * Preserve the BOM in the output, and don't use it to inform the charset.
+     */
+    PRESERVE_IGNORE,
+
+    /**
+     * Preserve the BOM in the output, and use it to override the charset.
+     */
+    PRESERVE_OVERRIDE_CHARSET,
+
+    /**
+     * Drop the BOM in the output, and don't use it to inform the charset.
+     */
+    DROP_IGNORE,
+
+    /**
+     * Drop the BOM in the output, and use it to override the charset.
+     */
+    DROP_OVERRIDE_CHARSET;
+
+    /**
+     * Get the BOM Action from an Argument string.
+     *
+     * @param str the argument string.
+     *
+     * @return the BOM action.
+     *
+     * @throws IllegalArgumentException if there is no BOM action for the supplied argument string.
+     */
+    public static BomAction fromArgumentString(final String str) throws IllegalArgumentException {
+        return valueOf(str.toUpperCase().replace("-", "_"));
+    }
+
+    /**
+     * Get the argument version of the name.
+     *
+     * @return the argument name.
+     */
+    public String toArgumentString() {
+        return name().toLowerCase().replace('_', '-');
+    }
+
+}
+
+
+/* ------------------------------------------------------------------------ */
+/*  DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS COMMENT.               */
+/*                                                                          */
+/*  The contents of this file are subject to the Mozilla Public License     */
+/*  Version 1.0 (the "License"); you may not use this file except in        */
+/*  compliance with the License. You may obtain a copy of the License at    */
+/*  http://www.mozilla.org/MPL/.                                            */
+/*                                                                          */
+/*  Software distributed under the License is distributed on an "AS IS"     */
+/*  basis, WITHOUT WARRANTY OF ANY KIND, either express or implied.  See    */
+/*  the License for the specific language governing rights and limitations  */
+/*  under the License.                                                      */
+/*                                                                          */
+/*  The Original Code is: all this file.                                    */
+/*                                                                          */
+/*  The Initial Developer of the Original Code is Evolved Binary Ltd.       */
+/*                                                                          */
+/*  Contributor(s): none.                                                   */
+/* ------------------------------------------------------------------------ */

http-client-java/src/main/java/org/expath/httpclient/HttpRequest.java

@@ -28,6 +28,7 @@ public HttpResponse send(Result result, HttpConnection conn, HttpCredentials cre
     public void setHttpVersion(String ver)
             throws HttpClientException;
     public void setDefaultCharset(String charset);
+    public void setBomAction(BomAction bomAction);
     public void setOverrideType(String type);
     public void setHeaders(HeaderSet headers);
     public void setBody(HttpRequestBody body)
@@ -60,5 +61,5 @@ public void setBody(HttpRequestBody body)
 /*                                                                          */
 /*  The Initial Developer of the Original Code is Florent Georges.          */
 /*                                                                          */
-/*  Contributor(s): none.                                                   */
+/*  Contributor(s): Evolved Binary Ltd.                                     */
 /* ------------------------------------------------------------------------ */

http-client-java/src/main/java/org/expath/httpclient/impl/BodyFactory.java

@@ -17,6 +17,7 @@
 import org.apache.hc.core5.http.HeaderElement;
 import org.apache.hc.core5.http.message.BasicHeader;
 import org.apache.hc.core5.http.message.MessageSupport;
+import org.expath.httpclient.BomAction;
 import org.expath.httpclient.ContentType;
 import org.expath.httpclient.HeaderSet;
 import org.expath.httpclient.HttpClientError;
@@ -64,7 +65,7 @@ public static HttpRequestBody makeRequestBody(final Element elem, final Sequence
         }
     }
 
-    public static HttpResponseBody makeResponseBody(final Result result, final ContentType type, final HttpConnection conn)
+    public static HttpResponseBody makeResponseBody(final Result result, final ContentType type, final BomAction bomAction, final HttpConnection conn)
             throws HttpClientException {
         if (type == null) {
             // it is legitimate to not have a body in a response; for instance
@@ -80,26 +81,26 @@ public static HttpResponseBody makeResponseBody(final Result result, final Conte
             return null;
         }
         if (t.startsWith("multipart/")) {
-            return new MultipartResponseBody(result, in, type);
+            return new MultipartResponseBody(result, in, type, bomAction);
         } else {
-            return makeResponsePart(result, null, in, type);
+            return makeResponsePart(result, null, in, type, bomAction);
         }
     }
 
     // package-level to be used within MultipartResponseBody ctor
     // TODO: Take new methods into account (XHTML, BASE64 and HEX).
-    static HttpResponseBody makeResponsePart(final Result result, final HeaderSet headers, final InputStream in, final ContentType ctype)
+    static HttpResponseBody makeResponsePart(final Result result, final HeaderSet headers, final InputStream in, final ContentType ctype, final BomAction bomAction)
             throws HttpClientException {
         switch (parseType(ctype)) {
             case XML:
                 // TODO: 'content_type' is the header Content-Type without any param
                 // (i.e. "text/xml".)  Should we keep this, or put the whole header
                 // (i.e. "text/xml; charset=utf-8")? (and for other types as well...)
-                return new XmlResponseBody(result, in, ctype, headers, false);
+                return new XmlResponseBody(result, in, ctype, bomAction, headers, false);
             case HTML:
-                return new XmlResponseBody(result, in, ctype, headers, true);
+                return new XmlResponseBody(result, in, ctype, bomAction, headers, true);
             case TEXT:
-                return new TextResponseBody(result, in, ctype, headers);
+                return new TextResponseBody(result, in, ctype, bomAction, headers);
             case BINARY:
                 return new BinaryResponseBody(result, in, ctype, headers);
             default:
@@ -278,5 +279,5 @@ else if ("base64".equals(m)) {
 /*                                                                          */
 /*  The Initial Developer of the Original Code is Florent Georges.          */
 /*                                                                          */
-/*  Contributor(s): none.                                                   */
+/*  Contributor(s): Evolved Binary Ltd.                                                   */
 /* ------------------------------------------------------------------------ */

http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java

@@ -9,6 +9,7 @@
 
 package org.expath.httpclient.impl;
 
+import org.expath.httpclient.BomAction;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hc.core5.http.Header;
@@ -68,7 +69,7 @@ public HttpResponse send(Result result, HttpConnection conn, HttpCredentials cre
                 LOG.debug("There is no Content-Type, we assume there is no content");
             }
             else {
-                body = BodyFactory.makeResponseBody(result, type, conn);
+                body = BodyFactory.makeResponseBody(result, type, myBomAction, conn);
             }
         }
         long stop = System.currentTimeMillis();
@@ -135,6 +136,11 @@ public void setDefaultCharset(final String charset) {
         myDefaultCharset = charset;
     }
 
+    @Override
+    public void setBomAction(final BomAction bomAction) {
+        myBomAction = bomAction;
+    }
+
     @Override
     public void setOverrideType(String type)
     {
@@ -210,6 +216,7 @@ public void setPreemptiveAuthentication(final boolean preemptiveAuthentication)
     private String myHref;
     private String myHttpVer;
     private String myDefaultCharset;
+    private BomAction myBomAction = BomAction.PRESERVE_IGNORE;
     private String myOverrideType;
     private boolean myStatusOnly;
     private boolean myFollowRedirect = true;
@@ -240,5 +247,5 @@ public void setPreemptiveAuthentication(final boolean preemptiveAuthentication)
 /*                                                                          */
 /*  The Initial Developer of the Original Code is Florent Georges.          */
 /*                                                                          */
-/*  Contributor(s): none.                                                   */
+/*  Contributor(s): Evolved Binary Ltd.                                     */
 /* ------------------------------------------------------------------------ */

http-client-java/src/main/java/org/expath/httpclient/impl/MultipartResponseBody.java

@@ -35,13 +35,14 @@
  */
 public class MultipartResponseBody implements HttpResponseBody {
 
-    public MultipartResponseBody(final Result result, final InputStream in, final ContentType type)
+    public MultipartResponseBody(final Result result, final InputStream in, final ContentType type, final BomAction bomAction)
             throws HttpClientException {
         if (type == null || type.getType() == null) {
             throw new HttpClientException(HttpClientError.HC002, "No content type");
         }
 
         myContentType = type;
+        myBomAction = bomAction;
         myParts = new ArrayList<>();
 
         myBoundary = type.getBoundary();
@@ -167,34 +168,31 @@ private HttpResponseBody makeResponsePart(final Result result, final HeaderSet h
             throw new HttpClientException(HttpClientError.HC002, "impossible to find the content type");
         }
         final ContentType type = ContentType.parse(h, null, null);
-        try {
-            switch (BodyFactory.parseType(type)) {
-                case XML: {
-                    final Reader in = parser.getReader();
-                    return new XmlResponseBody(result, in, type, headers, false);
-                }
-                case HTML: {
-                    final Reader in = parser.getReader();
-                    return new XmlResponseBody(result, in, type, headers, true);
-                }
-                case TEXT: {
-                    final Reader in = parser.getReader();
-                    return new TextResponseBody(result, in, type, headers);
-                }
-                case BINARY: {
-                    final InputStream in = parser.getInputStream();
-                    return new BinaryResponseBody(result, in, type, headers);
-                }
-                default:
-                    throw new HttpClientException(HttpClientError.HC002, "INTERNAL ERROR: cannot happen");
+        switch (BodyFactory.parseType(type)) {
+            case XML: {
+                final InputStream in = parser.getInputStream();
+                return new XmlResponseBody(result, in, type, myBomAction, headers, false);
             }
-        } catch (final UnsupportedEncodingException ex) {
-            throw new HttpClientException(HttpClientError.HC002, "Unable to parse response part", ex);
+            case HTML: {
+                final InputStream in = parser.getInputStream();
+                return new XmlResponseBody(result, in, type, myBomAction, headers, true);
+            }
+            case TEXT: {
+                final InputStream in = parser.getInputStream();
+                return new TextResponseBody(result, in, type, myBomAction, headers);
+            }
+            case BINARY: {
+                final InputStream in = parser.getInputStream();
+                return new BinaryResponseBody(result, in, type, headers);
+            }
+            default:
+                throw new HttpClientException(HttpClientError.HC002, "INTERNAL ERROR: cannot happen");
         }
     }
 
     private List<HttpResponseBody> myParts;
     private ContentType myContentType;
+    private BomAction myBomAction;
     private String myBoundary;
     private static final Logger LOG = LoggerFactory.getLogger(MultipartResponseBody.class);
 }
@@ -217,5 +215,5 @@ private HttpResponseBody makeResponsePart(final Result result, final HeaderSet h
 /*                                                                          */
 /*  The Initial Developer of the Original Code is Florent Georges.          */
 /*                                                                          */
-/*  Contributor(s): none.                                                   */
+/*  Contributor(s): Evolved Binary Ltd.                                     */
 /* ------------------------------------------------------------------------ */

http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java

@@ -113,6 +113,13 @@ else if ( "send-authorization".equals(local) ) {
             else if ( "default-charset".equals(local) ) {
                 req.setDefaultCharset(a.getValue());
             }
+            else if ( "bom-action".equals(local)) {
+                try {
+                    req.setBomAction(BomAction.fromArgumentString(a.getValue()));
+                } catch (final IllegalArgumentException e) {
+                    throw new HttpClientException(HttpClientError.HC005, "Unknown http:request/@bom-action" + a.getValue());
+                }
+            }
             else if ( "override-media-type".equals(local) ) {
                 req.setOverrideType(a.getValue());
             }
@@ -298,5 +305,5 @@ private int toInteger(Attribute a)
 /*                                                                          */
 /*  The Initial Developer of the Original Code is Florent Georges.          */
 /*                                                                          */
-/*  Contributor(s): none.                                                   */
+/*  Contributor(s): Evolved Binary Ltd.                                     */
 /* ------------------------------------------------------------------------ */

http-client-java/src/main/java/org/expath/httpclient/impl/TextResponseBody.java

@@ -13,11 +13,15 @@
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 
+import com.evolvedbinary.j8xu.BOM;
+import com.evolvedbinary.j8xu.io.BomFilterInputStream;
 import org.expath.httpclient.*;
 import org.expath.httpclient.model.Result;
 import org.expath.httpclient.model.TreeBuilder;
 import org.expath.tools.ToolsException;
 
+import javax.annotation.Nullable;
+
 /**
  * A text body in the response.
  *
@@ -27,35 +31,46 @@ public class TextResponseBody implements HttpResponseBody {
 
     public static final Charset DEFAULT_HTTP_TEXT_CHARSET = StandardCharsets.ISO_8859_1;
 
-    public TextResponseBody(final Result result, final InputStream in, final ContentType type, final HeaderSet headers)
+    public TextResponseBody(final Result result, InputStream in, final ContentType type, final BomAction bomAction, final HeaderSet headers)
             throws HttpClientException {
         myContentType = type;
         myHeaders = headers;
 
-        final Charset contentCharset;
+        Charset contentCharset;
         if (type.getCharset() != null) {
             contentCharset = Charset.forName(type.getCharset());
         } else {
             contentCharset = DEFAULT_HTTP_TEXT_CHARSET;
         }
 
-        final Reader reader = new InputStreamReader(in, contentCharset);
-        result.add(reader, contentCharset);
-    }
-
-    public TextResponseBody(final Result result, final Reader in, final ContentType type, final HeaderSet headers)
-            throws HttpClientException {
-        myContentType = type;
-        myHeaders = headers;
-
-        final Charset contentCharset;
-        if (type.getCharset() != null) {
-            contentCharset = Charset.forName(type.getCharset());
-        } else {
-            contentCharset = DEFAULT_HTTP_TEXT_CHARSET;
+        if (bomAction != BomAction.PRESERVE_IGNORE) {
+            final BomFilterInputStream bomIn = new BomFilterInputStream(in);
+            try {
+                @Nullable final BOM bom = bomIn.parseBom();
+
+                if (bom != null) {
+                    if (bomAction == BomAction.ERROR) {
+                        throw new HttpClientException(HttpClientError.HC002, "bom-action='error' but found BOM: " + bom.name());
+                    }
+
+                    if (bomAction == BomAction.DROP_IGNORE) {
+                        bomIn.skip(bom.getBomBytes().length);
+                    } else if (bomAction == BomAction.DROP_OVERRIDE_CHARSET) {
+                        contentCharset = bom.getCharset();
+                        bomIn.skip(bom.getBomBytes().length);
+                    } else if (bomAction == BomAction.PRESERVE_OVERRIDE_CHARSET) {
+                        contentCharset = bom.getCharset();
+                    }
+                }
+            } catch (final IOException e) {
+                throw new HttpClientException(HttpClientError.HC002, "Unable to parser BOM: " + e.getMessage(), e);
+            }
+
+            in = bomIn;
         }
 
-        result.add(in, contentCharset);
+        final Reader reader = new InputStreamReader(in, contentCharset);
+        result.add(reader, contentCharset);
     }
 
     @Override
@@ -96,5 +111,5 @@ public void outputBody(final TreeBuilder b) throws HttpClientException {
 /*                                                                          */
 /*  The Initial Developer of the Original Code is Florent Georges.          */
 /*                                                                          */
-/*  Contributor(s): none.                                                   */
+/*  Contributor(s): Evolved Binary Ltd.                                     */
 /* ------------------------------------------------------------------------ */