4901 devcontainer does not build when behind an ssl proxy

.devcontainer/Dockerfile

@@ -19,6 +19,20 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 # Set env for tracking that we're running in a devcontainer
 ENV DEVCONTAINER=true
 
+# Add any trusted_ca files (may be needed for SSL proxying)
+COPY .devcontainer/trusted_ca/ /tmp/trusted_ca/
+RUN set -e; \
+    shopt -s nullglob; \
+    for cert in /tmp/trusted_ca/*.{crt,pem,cer}; do \
+        cert_name="$(basename "${cert%.*}")"; \
+        cp "$cert" "/usr/local/share/ca-certificates/${cert_name}.crt"; \
+    done; \
+    update-ca-certificates; \
+    rm -rf /tmp/trusted_ca
+ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
+ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
+ENV NODE_EXTRA_CA_CERTS=/etc/ssl/certs/ca-certificates.crt
+
 # Install Node.js for GH actions tests and UI
 ARG NODE_VERSION="lts/*"
 RUN su $USERNAME -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"

.devcontainer/devcontainer.json

@@ -9,6 +9,9 @@
   "build": {
     "context": "..",
     "dockerfile": "Dockerfile",
+    "options": [
+      "--platform=linux/amd64"
+    ],
     "args": {
       // To ensure that the group ID for the docker group in the container
       // matches the group ID on the host, add this to your .bash_profile on the host
@@ -18,6 +21,7 @@
     }
   },
   "runArgs": [
+    "--platform=linux/amd64",
     "--network",
     "host"
   ],
@@ -28,8 +32,10 @@
     "type=bind,source=${env:HOME}${env:USERPROFILE}/.azure,target=/home/vscode/.azure",
     // Mount docker socket for docker builds
     "type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock",
-    // Mounts the github cli login details from the host machine to the container (~/.config/gh/hosts.yml)
-    "type=bind,source=${env:HOME}${env:USERPROFILE}/.config,target=/home/vscode/.config"
+    // Mounts the vscode config folder from the host machine
+    "type=bind,source=${env:HOME}${env:USERPROFILE}/.config,target=/home/vscode/.config",
+    // Mount the codex folder from the host machine
+    "type=bind,source=${env:HOME}${env:USERPROFILE}/.codex,target=/home/vscode/.codex"
   ],
   "remoteUser": "vscode",
   "containerEnv": {

.devcontainer/scripts/initialize

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-mkdir -p "$HOME/.azure" "$HOME/.config" || true
+mkdir -p "$HOME/.azure" "$HOME/.config" "$HOME/.codex" || true

.devcontainer/scripts/initialize.cmd

@@ -1,2 +1,2 @@
 @echo off
-mkdir %USERPROFILE%\.azure %USERPROFILE%\.config || exit /b 0
+mkdir %USERPROFILE%\.azure %USERPROFILE%\.config %USERPROFILE%\.codex || exit /b 0

.devcontainer/trusted_ca/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

CHANGELOG.md

@@ -15,6 +15,7 @@ ENHANCEMENTS:
 * Strengthen TRE API authentication with a layered `auth/` package (typed exceptions, `PyJWKClient`-backed token validation, immutable `AuthenticatedUser` model, composable RBAC factories), remove the redundant `AccessService` abstraction, and add Event Grid publish resilience with distinct Graph/publish failure reporting. ([#4989](https://github.com/microsoft/AzureTRE/pull/4989))
 * Add support for formatting UI code via `pre-commit` and fix existing formatting issues. ([#4955](https://github.com/microsoft/AzureTRE/issues/4955))
 * Update the version of `super-linter` used in the `build_validation_develop` workflow to 8.7.0 ([#4957](https://github.com/microsoft/AzureTRE/issues/4957))
+* Add trusted CA support for devcontainer image builds and Porter bundle build and publish workflows.
 
 BUG FIXES:
 * Ignore changes to `ip_tags` on public IP resources to unblock deployments where these tags are set by Azure policy. (`core` 0.16.17, `tre-shared-service-certs` 0.7.11) ([#5019](https://github.com/microsoft/AzureTRE/issues/5019))

Makefile

@@ -78,6 +78,56 @@ mgmt-destroy:
 # The CI_CACHE_ACR_NAME is an optional container registry used for caching in addition to what's in ACR_NAME
 define build_image
 $(call target_title, "Building $(1) Image") \
+&& dockerfile="" dockerfile_backup="" dockerfile_backup_ready=false certificate_context_dir="" \
+&& certificate_build_context_args=() \
+&& cleanup_build_image() { \
+	if [ -n "$${dockerfile_backup}" ] && [ -f "$${dockerfile_backup}" ]; then \
+		if [ "$${dockerfile_backup_ready}" = "true" ]; then cp -p -- "$${dockerfile_backup}" "$${dockerfile}"; fi; \
+		rm -f -- "$${dockerfile_backup}"; \
+	fi; \
+	if [ -n "$${certificate_context_dir}" ] && [ -d "$${certificate_context_dir}" ]; then \
+		rm -rf -- "$${certificate_context_dir}"; \
+	fi; \
+} \
+&& if [ "$${DEVCONTAINER:-}" = "true" ]; then \
+	trusted_ca_source="${MAKEFILE_DIR}/.devcontainer/trusted_ca"; \
+	dockerfile=$(3); \
+	if [ ! -f "$${dockerfile}" ]; then echo "Unable to find Dockerfile $${dockerfile}" >&2; exit 1; fi; \
+	if [ ! -d "$${trusted_ca_source}" ]; then echo "Unable to find trusted CA directory $${trusted_ca_source}" >&2; exit 1; fi; \
+	if ! find "$${trusted_ca_source}" -maxdepth 1 -type f \( -name '*.crt' -o -name '*.pem' -o -name '*.cer' \) -print -quit | grep -q .; then echo "No .crt, .pem, or .cer files found in $${trusted_ca_source}" >&2; exit 1; fi; \
+	certificate_context_dir=$$(mktemp -d) \
+	&& dockerfile_backup=$$(mktemp) \
+	&& trap cleanup_build_image EXIT HUP INT TERM \
+	&& cp -p -- "$${dockerfile}" "$${dockerfile_backup}" \
+	&& dockerfile_backup_ready=true \
+	&& find "$${trusted_ca_source}" -maxdepth 1 -type f \( -name '*.crt' -o -name '*.pem' -o -name '*.cer' \) -exec cp -- {} "$${certificate_context_dir}" \; \
+	&& certificate_build_context_args=(--build-context "devcontainer-trusted-ca=$${certificate_context_dir}") \
+	&& awk ' \
+		!inserted && /^FROM([[:space:]]|$$)/ { \
+			print; \
+			print ""; \
+			print "COPY --from=devcontainer-trusted-ca . /tmp/devcontainer-trusted-ca/"; \
+			print "RUN { if [ -f /etc/ssl/certs/ca-certificates.crt ]; then cat /etc/ssl/certs/ca-certificates.crt; fi; for cert in /tmp/devcontainer-trusted-ca/*; do cat \"$$cert\"; echo; done; } > /tmp/devcontainer-ca-certificates.crt && mkdir -p /usr/local/share/ca-certificates && for cert in /tmp/devcontainer-trusted-ca/*; do cert_name=$$(basename \"$$cert\"); cp \"$$cert\" \"/usr/local/share/ca-certificates/azuretre-devcontainer-$${cert_name}.crt\"; done && if command -v update-ca-certificates >/dev/null 2>&1; then update-ca-certificates; fi"; \
+			print "RUN if command -v keytool >/dev/null 2>&1; then if [ -n \"$${JAVA_HOME:-}\" ] && [ -f \"$${JAVA_HOME}/lib/security/cacerts\" ]; then cp \"$${JAVA_HOME}/lib/security/cacerts\" /tmp/devcontainer-java-cacerts; fi; for cert in /tmp/devcontainer-trusted-ca/*; do cert_name=$$(basename \"$$cert\"); keytool -importcert -noprompt -alias \"azuretre-devcontainer-$${cert_name}\" -file \"$$cert\" -keystore /tmp/devcontainer-java-cacerts -storepass changeit; done; fi"; \
+			print "ARG SSL_CERT_FILE=/tmp/devcontainer-ca-certificates.crt"; \
+			print "ARG CURL_CA_BUNDLE=/tmp/devcontainer-ca-certificates.crt"; \
+			print "ARG MAVEN_OPTS=\"-Djavax.net.ssl.trustStore=/tmp/devcontainer-java-cacerts -Djavax.net.ssl.trustStorePassword=changeit\""; \
+			print "ARG NODE_EXTRA_CA_CERTS=/tmp/devcontainer-ca-certificates.crt"; \
+			inserted=1; \
+			next; \
+		} \
+		/^FROM([[:space:]]|$$)/ { final_user="" } \
+		/^USER([[:space:]]|$$)/ { final_user=$$0 } \
+		{ print } \
+		END { \
+			if (!inserted) exit 1; \
+			print ""; \
+			if (final_user != "") print "USER root"; \
+			print "RUN for cert in /tmp/devcontainer-trusted-ca/*; do cert_name=$$(basename \"$$cert\"); rm -f \"/usr/local/share/ca-certificates/azuretre-devcontainer-$${cert_name}.crt\"; done && if command -v update-ca-certificates >/dev/null 2>&1; then update-ca-certificates; fi && rm -rf /tmp/devcontainer-trusted-ca /tmp/devcontainer-ca-certificates.crt /tmp/devcontainer-java-cacerts"; \
+			if (final_user != "") print final_user; \
+		} \
+	' "$${dockerfile_backup}" > "$${dockerfile}"; \
+fi \
 && . ${MAKEFILE_DIR}/devops/scripts/bootstrap_azure_env.sh \
 && . ${MAKEFILE_DIR}/devops/scripts/set_docker_sock_permission.sh \
 && if [ "$${DISABLE_ACR_PUBLIC_ACCESS}" = "true" ]; then source ${MAKEFILE_DIR}/devops/scripts/mgmtacr_enable_public_access.sh; fi \
@@ -87,7 +137,7 @@ $(call target_title, "Building $(1) Image") \
 	az acr login -n $${CI_CACHE_ACR_NAME}; \
 	ci_cache="--cache-from $${CI_CACHE_ACR_NAME}${ACR_DOMAIN_SUFFIX}/${IMAGE_NAME_PREFIX}/$(1):$${__version__}"; fi \
 && docker build -t ${FULL_IMAGE_NAME_PREFIX}/$(1):$${__version__} --build-arg BUILDKIT_INLINE_CACHE=1 \
-	--cache-from ${FULL_IMAGE_NAME_PREFIX}/$(1):$${__version__} $${ci_cache:-} -f $(3) $(4)
+	--cache-from ${FULL_IMAGE_NAME_PREFIX}/$(1):$${__version__} $${ci_cache:-} "$${certificate_build_context_args[@]}" -f $(3) $(4)
 endef
 
 # Description: Build API image using the build_image method.
@@ -378,6 +428,8 @@ bundle-publish:
 	&& cd ${DIR} \
 	&& FULL_IMAGE_NAME_PREFIX=${FULL_IMAGE_NAME_PREFIX} \
 		${MAKEFILE_DIR}/devops/scripts/bundle_runtime_image_push.sh \
+	&& . ${MAKEFILE_DIR}/devops/scripts/porter_devcontainer_ca.sh \
+	&& porter_devcontainer_ca_patch \
 	&& porter publish --registry "${ACR_FQDN}" --force
 
 # Description: Register the bundle with the TRE API.

devops/scripts/bundle_runtime_image_build.sh

@@ -5,6 +5,10 @@ set -o nounset
 # Uncomment this line to see each command for debugging (careful: this will show secrets!)
 # set -o xtrace
 
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=docker_devcontainer_ca.sh
+source "${script_dir}/docker_devcontainer_ca.sh"
+
 # Check for import section (import image from external registry to ACR)
 if [ "$(yq eval ".custom.runtime_image.import" porter.yaml)" != "null" ]; then
   image_name=$(yq eval ".custom.runtime_image.name" porter.yaml)
@@ -47,6 +51,8 @@ if [ -n "${CI_CACHE_ACR_NAME:-}" ]; then
 	docker_cache+=("--cache-from" "${CI_CACHE_ACR_NAME}${acr_domain_suffix}/${IMAGE_NAME_PREFIX}/${image_name}:${version}")
 fi
 
+docker_devcontainer_ca_patch "${docker_file}"
+
 ARCHITECTURE=$(docker info --format "{{ .Architecture }}" )
 
 if [ "${ARCHITECTURE}" == "aarch64" ]; then
@@ -57,5 +63,5 @@ fi
 
 ${DOCKER_BUILD_COMMAND} --build-arg BUILDKIT_INLINE_CACHE=1 \
   -t "${FULL_IMAGE_NAME_PREFIX}/${image_name}:${version}" \
-  "${docker_cache[@]}" -f "${docker_file}" "${docker_context}"
+  "${docker_cache[@]}" "${DOCKER_DEVCONTAINER_BUILD_CONTEXT_ARGS[@]}" -f "${docker_file}" "${docker_context}"
 

devops/scripts/docker_devcontainer_ca.sh

@@ -0,0 +1,126 @@
+#!/bin/bash
+
+DOCKER_DEVCONTAINER_BUILD_CONTEXT_ARGS=()
+docker_devcontainer_ca_dockerfile=""
+docker_devcontainer_ca_dockerfile_backup=""
+docker_devcontainer_ca_dockerfile_backup_ready=false
+docker_devcontainer_ca_context_dir=""
+
+docker_devcontainer_ca_restore() {
+    if [ -n "${docker_devcontainer_ca_dockerfile_backup}" ] && [ -f "${docker_devcontainer_ca_dockerfile_backup}" ]; then
+        if [ "${docker_devcontainer_ca_dockerfile_backup_ready}" = "true" ]; then
+            cp -p -- "${docker_devcontainer_ca_dockerfile_backup}" "${docker_devcontainer_ca_dockerfile}"
+        fi
+        rm -f -- "${docker_devcontainer_ca_dockerfile_backup}"
+    fi
+
+    if [ -n "${docker_devcontainer_ca_context_dir}" ] && [ -d "${docker_devcontainer_ca_context_dir}" ]; then
+        rm -rf -- "${docker_devcontainer_ca_context_dir}"
+    fi
+
+    DOCKER_DEVCONTAINER_BUILD_CONTEXT_ARGS=()
+    docker_devcontainer_ca_dockerfile=""
+    docker_devcontainer_ca_dockerfile_backup=""
+    docker_devcontainer_ca_dockerfile_backup_ready=false
+    docker_devcontainer_ca_context_dir=""
+    trap - EXIT HUP INT TERM
+}
+
+docker_devcontainer_ca_patch() {
+    if [ "${DEVCONTAINER:-}" != "true" ]; then
+        return 0
+    fi
+
+    if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then
+        echo "Unable to find Dockerfile ${1:-}" >&2
+        return 1
+    fi
+
+    local helper_dir trusted_ca_source
+    helper_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+    trusted_ca_source="$(cd "${helper_dir}/../.." && pwd)/.devcontainer/trusted_ca"
+    docker_devcontainer_ca_dockerfile="$1"
+
+    if [ ! -d "${trusted_ca_source}" ]; then
+        echo "Unable to find trusted CA directory ${trusted_ca_source}" >&2
+        return 1
+    fi
+
+    if ! find "${trusted_ca_source}" -maxdepth 1 -type f \( -name '*.crt' -o -name '*.pem' -o -name '*.cer' \) -print -quit | grep -q .; then
+        echo "No .crt, .pem, or .cer files found in ${trusted_ca_source}" >&2
+        return 1
+    fi
+
+    docker_devcontainer_ca_context_dir=$(mktemp -d) || return 1
+    docker_devcontainer_ca_dockerfile_backup=$(mktemp) || {
+        docker_devcontainer_ca_restore
+        return 1
+    }
+    trap docker_devcontainer_ca_restore EXIT HUP INT TERM
+
+    if ! cp -p -- "${docker_devcontainer_ca_dockerfile}" "${docker_devcontainer_ca_dockerfile_backup}"; then
+        docker_devcontainer_ca_restore
+        return 1
+    fi
+    docker_devcontainer_ca_dockerfile_backup_ready=true
+
+    if ! find "${trusted_ca_source}" -maxdepth 1 -type f \( -name '*.crt' -o -name '*.pem' -o -name '*.cer' \) -exec cp -- {} "${docker_devcontainer_ca_context_dir}" \;; then
+        docker_devcontainer_ca_restore
+        return 1
+    fi
+    DOCKER_DEVCONTAINER_BUILD_CONTEXT_ARGS=(--build-context "devcontainer-trusted-ca=${docker_devcontainer_ca_context_dir}")
+
+    if ! awk '
+        function emit_setup() {
+            print ""
+            print "COPY --from=devcontainer-trusted-ca . /tmp/devcontainer-trusted-ca/"
+            print "RUN { if [ -f /etc/ssl/certs/ca-certificates.crt ]; then cat /etc/ssl/certs/ca-certificates.crt; fi; for cert in /tmp/devcontainer-trusted-ca/*; do cat \"$cert\"; echo; done; } > /tmp/devcontainer-ca-certificates.crt && mkdir -p /usr/local/share/ca-certificates && for cert in /tmp/devcontainer-trusted-ca/*; do cert_name=$(basename \"$cert\"); cp \"$cert\" \"/usr/local/share/ca-certificates/azuretre-devcontainer-${cert_name}.crt\"; done && if command -v update-ca-certificates >/dev/null 2>&1; then update-ca-certificates; fi"
+            print "RUN if command -v keytool >/dev/null 2>&1; then if [ -n \"${JAVA_HOME:-}\" ] && [ -f \"${JAVA_HOME}/lib/security/cacerts\" ]; then cp \"${JAVA_HOME}/lib/security/cacerts\" /tmp/devcontainer-java-cacerts; fi; for cert in /tmp/devcontainer-trusted-ca/*; do cert_name=$(basename \"$cert\"); keytool -importcert -noprompt -alias \"azuretre-devcontainer-${cert_name}\" -file \"$cert\" -keystore /tmp/devcontainer-java-cacerts -storepass changeit; done; fi"
+            print "ARG SSL_CERT_FILE=/tmp/devcontainer-ca-certificates.crt"
+            print "ARG CURL_CA_BUNDLE=/tmp/devcontainer-ca-certificates.crt"
+            print "ARG MAVEN_OPTS=\"-Djavax.net.ssl.trustStore=/tmp/devcontainer-java-cacerts -Djavax.net.ssl.trustStorePassword=changeit\""
+            print "ARG NODE_EXTRA_CA_CERTS=/tmp/devcontainer-ca-certificates.crt"
+        }
+        function emit_cleanup(final_user) {
+            print ""
+            if (final_user != "") print "USER root"
+            print "RUN for cert in /tmp/devcontainer-trusted-ca/*; do cert_name=$(basename \"$cert\"); rm -f \"/usr/local/share/ca-certificates/azuretre-devcontainer-${cert_name}.crt\"; done && if command -v update-ca-certificates >/dev/null 2>&1; then update-ca-certificates; fi && rm -rf /tmp/devcontainer-trusted-ca /tmp/devcontainer-ca-certificates.crt /tmp/devcontainer-java-cacerts"
+            if (final_user != "") print final_user
+        }
+        function flush_stage(    i, root_user_line, final_user, is_scratch) {
+            split(stage[1], from_fields, /[[:space:]]+/)
+            is_scratch = (tolower(from_fields[2]) == "scratch")
+            root_user_line = 0
+            final_user = ""
+
+            for (i = 1; i <= stage_line_count; i++) {
+                if (root_user_line == 0 && stage[i] ~ /^USER[[:space:]]+root([[:space:]:]|$)/) root_user_line = i
+                if (stage[i] ~ /^USER([[:space:]]|$)/) final_user = stage[i]
+            }
+
+            for (i = 1; i <= stage_line_count; i++) {
+                print stage[i]
+                if (!is_scratch && ((root_user_line == 0 && i == 1) || i == root_user_line)) emit_setup()
+                delete stage[i]
+            }
+
+            if (!is_scratch) emit_cleanup(final_user)
+            stage_line_count = 0
+        }
+        /^FROM([[:space:]]|$)/ {
+            if (seen_stage) flush_stage()
+            seen_stage = 1
+            stage[++stage_line_count] = $0
+            next
+        }
+        !seen_stage { print; next }
+        { stage[++stage_line_count] = $0 }
+        END {
+            if (!seen_stage) exit 1
+            flush_stage()
+        }
+    ' "${docker_devcontainer_ca_dockerfile_backup}" > "${docker_devcontainer_ca_dockerfile}"; then
+        docker_devcontainer_ca_restore
+        return 1
+    fi
+}

devops/scripts/porter_build_bundle.sh

@@ -1,5 +1,10 @@
 #!/bin/bash
 
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=porter_devcontainer_ca.sh
+source "${script_dir}/porter_devcontainer_ca.sh"
+porter_devcontainer_ca_patch
+
 if [ -f "porter-build-context.env" ]; then
 
     # shellcheck disable=SC1091
@@ -31,5 +36,5 @@ else
         cache=(--cache-to "type=registry,ref=${ref},mode=max" --cache-from "type=registry,ref=${ref}")
     fi
 
-    porter build "${cache[@]}"
+    porter build "${PORTER_DEVCONTAINER_BUILD_CONTEXT_ARGS[@]}" "${cache[@]}"
 fi