8367104: Check for RSASSA-PSS parameters when validating certificates against algorithm constraints

Reviewed-by: mullan

src/java.base/share/classes/sun/security/provider/certpath/AlgorithmChecker.java

@@ -221,7 +221,7 @@ public final class AlgorithmChecker extends PKIXCertPathChecker {
                     currSigAlg, prevPubKey, currSigAlgParams)) {
                     throw new CertPathValidatorException(
                         "Algorithm constraints check failed on " +
-                            currSigAlg + "signature and " +
+                            currSigAlg + " signature and " +
                             currPubKey.getAlgorithm() + " key with size of " +
                             sun.security.util.KeyUtil.getKeySize(currPubKey) +
                             "bits",

src/java.base/share/classes/sun/security/provider/certpath/PKIXCertPathValidator.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -212,6 +212,11 @@ public final class PKIXCertPathValidator extends CertPathValidatorSpi {
                     ((RevocationChecker)checker).init(anchor, params);
                 }
             }
+
+            // Set trust anchor for the user-specified AlgorithmChecker.
+            if (checker instanceof AlgorithmChecker algChecker) {
+                algChecker.trySetTrustAnchor(anchor);
+            }
         }
         // only add a RevocationChecker if revocation is enabled and
         // a PKIXRevocationChecker has not already been added

src/java.base/share/classes/sun/security/ssl/SSLAlgorithmConstraints.java

@@ -29,19 +29,30 @@ import java.security.AlgorithmConstraints;
 import java.security.AlgorithmParameters;
 import java.security.CryptoPrimitive;
 import java.security.Key;
+import java.security.spec.InvalidParameterSpecException;
+import java.security.spec.PSSParameterSpec;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 import java.util.Set;
+import java.util.TreeSet;
 import javax.net.ssl.*;
 import sun.security.util.DisabledAlgorithmConstraints;
 import static sun.security.util.DisabledAlgorithmConstraints.*;
 
 /**
  * Algorithm constraints for disabled algorithms property
- *
+ * <p>
  * See the "jdk.certpath.disabledAlgorithms" specification in java.security
  * for the syntax of the disabled algorithm string.
  */
 final class SSLAlgorithmConstraints implements AlgorithmConstraints {
 
+    public enum SIGNATURE_CONSTRAINTS_MODE {
+        PEER,  // Check against peer supported signatures
+        LOCAL  // Check against local supported signatures
+    }
+
     private static final DisabledAlgorithmConstraints tlsDisabledAlgConstraints =
             new DisabledAlgorithmConstraints(PROPERTY_TLS_DISABLED_ALGS,
                     new SSLAlgorithmDecomposer());
@@ -57,14 +68,15 @@ final class SSLAlgorithmConstraints implements AlgorithmConstraints {
 
     // the default algorithm constraints
     static final SSLAlgorithmConstraints DEFAULT =
-                        new SSLAlgorithmConstraints(null, true);
+            new SSLAlgorithmConstraints(null, true);
 
     // the default SSL only algorithm constraints
     static final SSLAlgorithmConstraints DEFAULT_SSL_ONLY =
-                        new SSLAlgorithmConstraints(null, false);
+            new SSLAlgorithmConstraints(null, false);
 
-    private SSLAlgorithmConstraints(AlgorithmConstraints userSpecifiedConstraints,
-                                    boolean enabledX509DisabledAlgConstraints) {
+    private SSLAlgorithmConstraints(
+            AlgorithmConstraints userSpecifiedConstraints,
+            boolean enabledX509DisabledAlgConstraints) {
         this(userSpecifiedConstraints, null, enabledX509DisabledAlgConstraints);
     }
 
@@ -81,10 +93,12 @@ final class SSLAlgorithmConstraints implements AlgorithmConstraints {
      * Returns a SSLAlgorithmConstraints instance that checks the provided
      * {@code userSpecifiedConstraints} in addition to standard checks.
      * Returns a singleton instance if parameter is null or DEFAULT.
+     *
      * @param userSpecifiedConstraints additional constraints to check
      * @return a SSLAlgorithmConstraints instance
      */
-    static SSLAlgorithmConstraints wrap(AlgorithmConstraints userSpecifiedConstraints) {
+    static SSLAlgorithmConstraints wrap(
+            AlgorithmConstraints userSpecifiedConstraints) {
         return wrap(userSpecifiedConstraints, true);
     }
 
@@ -102,23 +116,24 @@ final class SSLAlgorithmConstraints implements AlgorithmConstraints {
      * Returns a SSLAlgorithmConstraints instance that checks the constraints
      * configured for the given {@code socket} in addition to standard checks.
      * Returns a singleton instance if the constraints are null or DEFAULT.
+     *
      * @param socket socket with configured constraints
+     * @param mode SIGNATURE_CONSTRAINTS_MODE
      * @return a SSLAlgorithmConstraints instance
      */
-    static AlgorithmConstraints forSocket(SSLSocket socket,
-                                          boolean withDefaultCertPathConstraints) {
-        AlgorithmConstraints userSpecifiedConstraints =
-                getUserSpecifiedConstraints(socket);
-        return wrap(userSpecifiedConstraints, withDefaultCertPathConstraints);
-    }
-
     static SSLAlgorithmConstraints forSocket(
             SSLSocket socket,
-            String[] supportedAlgorithms,
+            SIGNATURE_CONSTRAINTS_MODE mode,
             boolean withDefaultCertPathConstraints) {
+
+        if (socket == null) {
+            return wrap(null, withDefaultCertPathConstraints);
+        }
+
         return new SSLAlgorithmConstraints(
                 nullIfDefault(getUserSpecifiedConstraints(socket)),
-                new SupportedSignatureAlgorithmConstraints(supportedAlgorithms),
+                new SupportedSignatureAlgorithmConstraints(
+                        socket.getHandshakeSession(), mode),
                 withDefaultCertPathConstraints);
     }
 
@@ -126,23 +141,24 @@ final class SSLAlgorithmConstraints implements AlgorithmConstraints {
      * Returns a SSLAlgorithmConstraints instance that checks the constraints
      * configured for the given {@code engine} in addition to standard checks.
      * Returns a singleton instance if the constraints are null or DEFAULT.
+     *
      * @param engine engine with configured constraints
+     * @param mode SIGNATURE_CONSTRAINTS_MODE
      * @return a SSLAlgorithmConstraints instance
      */
-    static AlgorithmConstraints forEngine(SSLEngine engine,
-                                          boolean withDefaultCertPathConstraints) {
-        AlgorithmConstraints userSpecifiedConstraints =
-                getUserSpecifiedConstraints(engine);
-        return wrap(userSpecifiedConstraints, withDefaultCertPathConstraints);
-    }
-
     static SSLAlgorithmConstraints forEngine(
             SSLEngine engine,
-            String[] supportedAlgorithms,
+            SIGNATURE_CONSTRAINTS_MODE mode,
             boolean withDefaultCertPathConstraints) {
+
+        if (engine == null) {
+            return wrap(null, withDefaultCertPathConstraints);
+        }
+
         return new SSLAlgorithmConstraints(
                 nullIfDefault(getUserSpecifiedConstraints(engine)),
-                new SupportedSignatureAlgorithmConstraints(supportedAlgorithms),
+                new SupportedSignatureAlgorithmConstraints(
+                        engine.getHandshakeSession(), mode),
                 withDefaultCertPathConstraints);
     }
 
@@ -159,7 +175,7 @@ final class SSLAlgorithmConstraints implements AlgorithmConstraints {
             // Please check the instance before casting to use SSLEngineImpl.
             if (engine instanceof SSLEngineImpl) {
                 HandshakeContext hc =
-                        ((SSLEngineImpl)engine).conContext.handshakeContext;
+                        ((SSLEngineImpl) engine).conContext.handshakeContext;
                 if (hc != null) {
                     return hc.sslConfig.userSpecifiedAlgorithmConstraints;
                 }
@@ -179,7 +195,7 @@ final class SSLAlgorithmConstraints implements AlgorithmConstraints {
             // Please check the instance before casting to use SSLSocketImpl.
             if (socket instanceof SSLSocketImpl) {
                 HandshakeContext hc =
-                        ((SSLSocketImpl)socket).conContext.handshakeContext;
+                        ((SSLSocketImpl) socket).conContext.handshakeContext;
                 if (hc != null) {
                     return hc.sslConfig.userSpecifiedAlgorithmConstraints;
                 }
@@ -279,15 +295,55 @@ final class SSLAlgorithmConstraints implements AlgorithmConstraints {
     }
 
     private static class SupportedSignatureAlgorithmConstraints
-                                    implements AlgorithmConstraints {
-        // supported signature algorithms
-        private final String[] supportedAlgorithms;
+            implements AlgorithmConstraints {
 
-        SupportedSignatureAlgorithmConstraints(String[] supportedAlgorithms) {
-            if (supportedAlgorithms != null) {
-                this.supportedAlgorithms = supportedAlgorithms.clone();
-            } else {
-                this.supportedAlgorithms = null;
+        // Supported signature algorithms
+        private Set<String> supportedAlgorithms;
+        // Supported signature schemes
+        private List<SignatureScheme> supportedSignatureSchemes;
+        private boolean checksDisabled;
+
+        SupportedSignatureAlgorithmConstraints(
+                SSLSession session, SIGNATURE_CONSTRAINTS_MODE mode) {
+
+            if (mode == null
+                    || !(session instanceof ExtendedSSLSession extSession
+                    // "signature_algorithms_cert" TLS extension is only
+                    // available starting with TLSv1.2.
+                    && ProtocolVersion.useTLS12PlusSpec(
+                    extSession.getProtocol()))) {
+
+                checksDisabled = true;
+                return;
+            }
+
+            supportedAlgorithms = new TreeSet<>(
+                    String.CASE_INSENSITIVE_ORDER);
+
+            switch (mode) {
+                case SIGNATURE_CONSTRAINTS_MODE.PEER:
+                    supportedAlgorithms.addAll(Arrays.asList(extSession
+                            .getPeerSupportedSignatureAlgorithms()));
+                    break;
+                case SIGNATURE_CONSTRAINTS_MODE.LOCAL:
+                    supportedAlgorithms.addAll(Arrays.asList(extSession
+                            .getLocalSupportedSignatureAlgorithms()));
+            }
+
+            // Do additional SignatureSchemes checks for in-house
+            // ExtendedSSLSession implementation.
+            if (extSession instanceof SSLSessionImpl sslSessionImpl) {
+                switch (mode) {
+                    case SIGNATURE_CONSTRAINTS_MODE.PEER:
+                        supportedSignatureSchemes = new ArrayList<>(
+                                sslSessionImpl
+                                        .getPeerSupportedSignatureSchemes());
+                        break;
+                    case SIGNATURE_CONSTRAINTS_MODE.LOCAL:
+                        supportedSignatureSchemes = new ArrayList<>(
+                                sslSessionImpl
+                                        .getLocalSupportedSignatureSchemes());
+                }
             }
         }
 
@@ -295,6 +351,10 @@ final class SSLAlgorithmConstraints implements AlgorithmConstraints {
         public boolean permits(Set<CryptoPrimitive> primitives,
                 String algorithm, AlgorithmParameters parameters) {
 
+            if (checksDisabled) {
+                return true;
+            }
+
             if (algorithm == null || algorithm.isEmpty()) {
                 throw new IllegalArgumentException(
                         "No algorithm name specified");
@@ -305,24 +365,11 @@ final class SSLAlgorithmConstraints implements AlgorithmConstraints {
                         "No cryptographic primitive specified");
             }
 
-            if (supportedAlgorithms == null ||
-                        supportedAlgorithms.length == 0) {
+            if (supportedAlgorithms == null || supportedAlgorithms.isEmpty()) {
                 return false;
             }
 
-            // trim the MGF part: <digest>with<encryption>and<mgf>
-            int position = algorithm.indexOf("and");
-            if (position > 0) {
-                algorithm = algorithm.substring(0, position);
-            }
-
-            for (String supportedAlgorithm : supportedAlgorithms) {
-                if (algorithm.equalsIgnoreCase(supportedAlgorithm)) {
-                    return true;
-                }
-            }
-
-            return false;
+            return supportedAlgorithms.contains(algorithm);
         }
 
         @Override
@@ -339,7 +386,41 @@ final class SSLAlgorithmConstraints implements AlgorithmConstraints {
                         "No algorithm name specified");
             }
 
-            return permits(primitives, algorithm, parameters);
+            return permits(primitives, algorithm, parameters)
+                    && checkRsaSsaPssParams(algorithm, key, parameters);
+        }
+
+        // Additional check for RSASSA-PSS signature algorithm parameters.
+        private boolean checkRsaSsaPssParams(
+                String algorithm, Key key, AlgorithmParameters parameters) {
+
+            if (supportedSignatureSchemes == null
+                    || key == null
+                    || parameters == null
+                    || !"RSASSA-PSS".equalsIgnoreCase(algorithm)) {
+                return true;
+            }
+
+            try {
+                String keyAlg = key.getAlgorithm();
+                String paramDigestAlg = parameters.getParameterSpec(
+                        PSSParameterSpec.class).getDigestAlgorithm();
+
+                return supportedSignatureSchemes.stream().anyMatch(ss ->
+                        ss.algorithm.equalsIgnoreCase(algorithm)
+                                && ss.keyAlgorithm.equalsIgnoreCase(keyAlg)
+                                && ((PSSParameterSpec) ss.signAlgParams.parameterSpec)
+                                .getDigestAlgorithm()
+                                .equalsIgnoreCase(paramDigestAlg));
+
+            } catch (InvalidParameterSpecException e) {
+                if (SSLLogger.isOn && SSLLogger.isOn("ssl")) {
+                    SSLLogger.warning("Invalid AlgorithmParameters: "
+                            + parameters + "; Error: " + e.getMessage());
+                }
+
+                return true;
+            }
         }
     }
 }

src/java.base/share/classes/sun/security/ssl/SSLContextImpl.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -33,6 +33,7 @@ import java.util.*;
 import java.util.concurrent.locks.ReentrantLock;
 import javax.net.ssl.*;
 import sun.security.provider.certpath.AlgorithmChecker;
+import sun.security.ssl.SSLAlgorithmConstraints.SIGNATURE_CONSTRAINTS_MODE;
 import sun.security.validator.Validator;
 
 /**
@@ -1449,22 +1450,8 @@ final class AbstractTrustManagerWrapper extends X509ExtendedTrustManager
                                     identityAlg, checkClientTrusted);
             }
 
-            // try the best to check the algorithm constraints
-            AlgorithmConstraints constraints;
-            if (ProtocolVersion.useTLS12PlusSpec(session.getProtocol())) {
-                if (session instanceof ExtendedSSLSession extSession) {
-                    String[] peerSupportedSignAlgs =
-                            extSession.getLocalSupportedSignatureAlgorithms();
-
-                    constraints = SSLAlgorithmConstraints.forSocket(
-                                    sslSocket, peerSupportedSignAlgs, true);
-                } else {
-                    constraints =
-                            SSLAlgorithmConstraints.forSocket(sslSocket, true);
-                }
-            } else {
-                constraints = SSLAlgorithmConstraints.forSocket(sslSocket, true);
-            }
+            AlgorithmConstraints constraints = SSLAlgorithmConstraints.forSocket(
+                    sslSocket, SIGNATURE_CONSTRAINTS_MODE.LOCAL, true);
 
             checkAlgorithmConstraints(chain, constraints, checkClientTrusted);
         }
@@ -1474,6 +1461,7 @@ final class AbstractTrustManagerWrapper extends X509ExtendedTrustManager
             String authType, SSLEngine engine,
             boolean checkClientTrusted) throws CertificateException {
         if (engine != null) {
+
             SSLSession session = engine.getHandshakeSession();
             if (session == null) {
                 throw new CertificateException("No handshake session");
@@ -1487,22 +1475,8 @@ final class AbstractTrustManagerWrapper extends X509ExtendedTrustManager
                                     identityAlg, checkClientTrusted);
             }
 
-            // try the best to check the algorithm constraints
-            AlgorithmConstraints constraints;
-            if (ProtocolVersion.useTLS12PlusSpec(session.getProtocol())) {
-                if (session instanceof ExtendedSSLSession extSession) {
-                    String[] peerSupportedSignAlgs =
-                            extSession.getLocalSupportedSignatureAlgorithms();
-
-                    constraints = SSLAlgorithmConstraints.forEngine(
-                                    engine, peerSupportedSignAlgs, true);
-                } else {
-                    constraints =
-                            SSLAlgorithmConstraints.forEngine(engine, true);
-                }
-            } else {
-                constraints = SSLAlgorithmConstraints.forEngine(engine, true);
-            }
+            AlgorithmConstraints constraints = SSLAlgorithmConstraints.forEngine(
+                    engine, SIGNATURE_CONSTRAINTS_MODE.LOCAL, true);
 
             checkAlgorithmConstraints(chain, constraints, checkClientTrusted);
         }

src/java.base/share/classes/sun/security/ssl/SSLSessionImpl.java

@@ -1388,10 +1388,10 @@ final class SSLSessionImpl extends ExtendedSSLSession {
     }
 
     /**
-     * Gets an array of supported signature schemes that the local side is
+     * Gets a collection of supported signature schemes that the local side is
      * willing to verify.
      */
-    public Collection<SignatureScheme> getLocalSupportedSignatureSchemes() {
+    Collection<SignatureScheme> getLocalSupportedSignatureSchemes() {
         return localSupportedSignAlgs;
     }
 
@@ -1404,6 +1404,15 @@ final class SSLSessionImpl extends ExtendedSSLSession {
         return SignatureScheme.getAlgorithmNames(peerSupportedSignAlgs);
     }
 
+    /**
+     * Gets a collection of supported signature schemes that the peer is
+     * willing to verify. Those are sent with the "signature_algorithms_cert"
+     * TLS extension.
+     */
+    Collection<SignatureScheme> getPeerSupportedSignatureSchemes() {
+        return peerSupportedSignAlgs;
+    }
+
     /**
      * Obtains a <code>List</code> containing all {@link SNIServerName}s
      * of the requested Server Name Indication (SNI) extension.

src/java.base/share/classes/sun/security/ssl/SignatureScheme.java

@@ -146,12 +146,12 @@ enum SignatureScheme {
                                     "RSA", 511,
                                     ProtocolVersion.PROTOCOLS_TO_12);
 
-    final int id;                       // hash + signature
-    final String name;                  // literal name
-    private final String algorithm;     // signature algorithm
-    final String keyAlgorithm;          // signature key algorithm
-    private final SigAlgParamSpec signAlgParams;    // signature parameters
-    private final NamedGroup namedGroup;    // associated named group
+    final int id;                         // hash + signature
+    final String name;                    // literal name
+    final String algorithm;               // signature algorithm
+    final String keyAlgorithm;            // signature key algorithm
+    final SigAlgParamSpec signAlgParams;  // signature parameters
+    private final NamedGroup namedGroup;  // associated named group
 
     // The minimal required key size in bits.
     //
@@ -185,7 +185,7 @@ enum SignatureScheme {
         RSA_PSS_SHA384 ("SHA-384", 48),
         RSA_PSS_SHA512 ("SHA-512", 64);
 
-        private final AlgorithmParameterSpec parameterSpec;
+        final AlgorithmParameterSpec parameterSpec;
         private final AlgorithmParameters parameters;
         private final boolean isAvailable;
 

src/java.base/share/classes/sun/security/ssl/X509KeyManagerCertChecking.java

@@ -39,16 +39,15 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Set;
-import javax.net.ssl.ExtendedSSLSession;
 import javax.net.ssl.SNIHostName;
 import javax.net.ssl.SNIServerName;
 import javax.net.ssl.SSLEngine;
-import javax.net.ssl.SSLSession;
 import javax.net.ssl.SSLSocket;
 import javax.net.ssl.StandardConstants;
 import javax.net.ssl.X509ExtendedKeyManager;
 import javax.security.auth.x500.X500Principal;
 import sun.security.provider.certpath.AlgorithmChecker;
+import sun.security.ssl.SSLAlgorithmConstraints.SIGNATURE_CONSTRAINTS_MODE;
 import sun.security.util.KnownOIDs;
 import sun.security.validator.Validator;
 
@@ -168,19 +167,8 @@ abstract class X509KeyManagerCertChecking extends X509ExtendedKeyManager {
         }
 
         if (socket instanceof SSLSocket sslSocket && sslSocket.isConnected()) {
-            SSLSession session = sslSocket.getHandshakeSession();
-
-            if (session instanceof ExtendedSSLSession extSession
-                    && ProtocolVersion.useTLS12PlusSpec(
-                    extSession.getProtocol())) {
-                // Use peer supported certificate signature algorithms
-                // sent with "signature_algorithms_cert" TLS extension.
-                return SSLAlgorithmConstraints.forSocket(sslSocket,
-                        extSession.getPeerSupportedSignatureAlgorithms(),
-                        true);
-            }
-
-            return SSLAlgorithmConstraints.forSocket(sslSocket, true);
+            return SSLAlgorithmConstraints.forSocket(
+                    sslSocket, SIGNATURE_CONSTRAINTS_MODE.PEER, true);
         }
 
         return SSLAlgorithmConstraints.DEFAULT;
@@ -193,21 +181,8 @@ abstract class X509KeyManagerCertChecking extends X509ExtendedKeyManager {
             return null;
         }
 
-        if (engine != null) {
-            SSLSession session = engine.getHandshakeSession();
-
-            if (session instanceof ExtendedSSLSession extSession
-                    && ProtocolVersion.useTLS12PlusSpec(
-                    extSession.getProtocol())) {
-                // Use peer supported certificate signature algorithms
-                // sent with "signature_algorithms_cert" TLS extension.
-                return SSLAlgorithmConstraints.forEngine(engine,
-                        extSession.getPeerSupportedSignatureAlgorithms(),
-                        true);
-            }
-        }
-
-        return SSLAlgorithmConstraints.forEngine(engine, true);
+        return SSLAlgorithmConstraints.forEngine(
+                engine, SIGNATURE_CONSTRAINTS_MODE.PEER, true);
     }
 
     // Algorithm constraints check.

src/java.base/share/classes/sun/security/ssl/X509TrustManagerImpl.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -31,6 +31,7 @@ import java.security.cert.*;
 import java.util.*;
 import java.util.concurrent.locks.ReentrantLock;
 import javax.net.ssl.*;
+import sun.security.ssl.SSLAlgorithmConstraints.SIGNATURE_CONSTRAINTS_MODE;
 import sun.security.util.AnchorCertificates;
 import sun.security.util.HostnameChecker;
 import sun.security.validator.*;
@@ -198,32 +199,19 @@ final class X509TrustManagerImpl extends X509ExtendedTrustManager
         Validator v = checkTrustedInit(chain, authType, checkClientTrusted);
 
         X509Certificate[] trustedChain;
-        if ((socket != null) && socket.isConnected() &&
-                (socket instanceof SSLSocket sslSocket)) {
+        if (socket instanceof SSLSocket sslSocket && sslSocket.isConnected()) {
 
             SSLSession session = sslSocket.getHandshakeSession();
             if (session == null) {
                 throw new CertificateException("No handshake session");
             }
 
-            // create the algorithm constraints
-            boolean isExtSession = (session instanceof ExtendedSSLSession);
-            AlgorithmConstraints constraints;
-            if (isExtSession &&
-                    ProtocolVersion.useTLS12PlusSpec(session.getProtocol())) {
-                ExtendedSSLSession extSession = (ExtendedSSLSession)session;
-                String[] localSupportedSignAlgs =
-                        extSession.getLocalSupportedSignatureAlgorithms();
-
-                constraints = SSLAlgorithmConstraints.forSocket(
-                                sslSocket, localSupportedSignAlgs, false);
-            } else {
-                constraints = SSLAlgorithmConstraints.forSocket(sslSocket, false);
-            }
+            AlgorithmConstraints constraints = SSLAlgorithmConstraints.forSocket(
+                    sslSocket, SIGNATURE_CONSTRAINTS_MODE.LOCAL, false);
 
             // Grab any stapled OCSP responses for use in validation
             List<byte[]> responseList = Collections.emptyList();
-            if (!checkClientTrusted && isExtSession) {
+            if (!checkClientTrusted && session instanceof ExtendedSSLSession) {
                 responseList =
                         ((ExtendedSSLSession)session).getStatusResponses();
             }
@@ -255,29 +243,18 @@ final class X509TrustManagerImpl extends X509ExtendedTrustManager
 
         X509Certificate[] trustedChain;
         if (engine != null) {
+
             SSLSession session = engine.getHandshakeSession();
             if (session == null) {
                 throw new CertificateException("No handshake session");
             }
 
-            // create the algorithm constraints
-            boolean isExtSession = (session instanceof ExtendedSSLSession);
-            AlgorithmConstraints constraints;
-            if (isExtSession &&
-                    ProtocolVersion.useTLS12PlusSpec(session.getProtocol())) {
-                ExtendedSSLSession extSession = (ExtendedSSLSession)session;
-                String[] localSupportedSignAlgs =
-                        extSession.getLocalSupportedSignatureAlgorithms();
-
-                constraints = SSLAlgorithmConstraints.forEngine(
-                                engine, localSupportedSignAlgs, false);
-            } else {
-                constraints = SSLAlgorithmConstraints.forEngine(engine, false);
-            }
+            AlgorithmConstraints constraints = SSLAlgorithmConstraints.forEngine(
+                    engine, SIGNATURE_CONSTRAINTS_MODE.LOCAL, false);
 
             // Grab any stapled OCSP responses for use in validation
             List<byte[]> responseList = Collections.emptyList();
-            if (!checkClientTrusted && isExtSession) {
+            if (!checkClientTrusted && session instanceof ExtendedSSLSession) {
                 responseList =
                         ((ExtendedSSLSession)session).getStatusResponses();
             }

src/java.base/share/classes/sun/security/validator/PKIXValidator.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -261,7 +261,6 @@ public final class PKIXValidator extends Validator {
         // apparently issued by trust anchor?
         X509Certificate last = chain[chain.length - 1];
         X500Principal issuer = last.getIssuerX500Principal();
-        X500Principal subject = last.getSubjectX500Principal();
         if (trustedSubjects.containsKey(issuer)) {
             return doValidate(chain, pkixParameters);
         }

test/jdk/sun/security/ssl/SignatureScheme/MD5NotAllowedInTLS13CertificateSignature.java

@@ -33,7 +33,6 @@
  * @run main/othervm MD5NotAllowedInTLS13CertificateSignature
  */
 
-import static jdk.test.lib.Asserts.assertEquals;
 import static jdk.test.lib.Asserts.assertTrue;
 import static jdk.test.lib.Utils.runAndCheckException;
 
@@ -99,11 +98,12 @@ public class MD5NotAllowedInTLS13CertificateSignature extends
                 serverEx -> {
                     Throwable clientEx = serverEx.getSuppressed()[0];
                     assertTrue(clientEx instanceof SSLHandshakeException);
-                    assertEquals(clientEx.getMessage(), "(bad_certificate) "
-                            + "PKIX path validation failed: "
-                            + "java.security.cert.CertPathValidatorException: "
-                            + "Algorithm constraints check failed on signature"
-                            + " algorithm: MD5withRSA");
+                    assertTrue(clientEx.getMessage().startsWith(
+                            "(bad_certificate)"
+                                    + " PKIX path validation failed: "
+                                    + "java.security.cert.CertPathValidatorException:"
+                                    + " Algorithm constraints check failed on "
+                                    + "MD5withRSA signature and RSA key"));
                 });
 
         // Should run fine on TLSv1.2.

test/jdk/sun/security/ssl/SignatureScheme/RsaSsaPssConstraints.java

@@ -0,0 +1,274 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import static jdk.test.lib.Asserts.assertEquals;
+import static jdk.test.lib.Asserts.assertTrue;
+import static jdk.test.lib.Utils.runAndCheckException;
+
+import java.io.IOException;
+import java.math.BigInteger;
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+import java.security.KeyStore;
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import java.security.SecureRandom;
+import java.security.Security;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.Date;
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLHandshakeException;
+import javax.net.ssl.TrustManagerFactory;
+import jdk.test.lib.security.CertificateBuilder;
+import sun.security.x509.AuthorityKeyIdentifierExtension;
+import sun.security.x509.GeneralName;
+import sun.security.x509.GeneralNames;
+import sun.security.x509.KeyIdentifier;
+import sun.security.x509.SerialNumber;
+import sun.security.x509.X500Name;
+
+
+/*
+ * @test
+ * @bug 8367104
+ * @summary Check for RSASSA-PSS parameters when validating certificates
+ *          against algorithm constraints.
+ * @modules java.base/sun.security.x509
+ *          java.base/sun.security.util
+ * @library /javax/net/ssl/templates
+ *          /test/lib
+ * @run main/othervm RsaSsaPssConstraints RSASSA-PSS RSASSA-PSS Rsa_pss_pss_Sha384 true
+ * @run main/othervm RsaSsaPssConstraints RSASSA-PSS RSASSA-PSS RsaSsa-Pss true
+ * @run main/othervm RsaSsaPssConstraints RSA RSASSA-PSS rsa_pss_Rsae_sha384 true
+ * @run main/othervm RsaSsaPssConstraints RSA RSASSA-PSS Rsa true
+ * @run main/othervm RsaSsaPssConstraints RSA RSASSA-PSS RSASSA-pSS true
+ * @run main/othervm RsaSsaPssConstraints RSA SHA384withRSA rsa_pkcs1_Sha384 true
+ * @run main/othervm RsaSsaPssConstraints EC SHA384withECDSA Ecdsa_Secp384r1_sha384 true
+ * @run main/othervm RsaSsaPssConstraints RSA SHA384withRSA SHA384withRsA true
+ * @run main/othervm RsaSsaPssConstraints RSASSA-PSS RSASSA-PSS rsa_pss_rsae_sha384 false
+ * @run main/othervm RsaSsaPssConstraints RSA RSASSA-PSS rsa_pss_pss_sha384 false
+ * @run main/othervm RsaSsaPssConstraints RSASSA-PSS RSASSA-PSS rsa_pss_pss_sha256 false
+ * @run main/othervm RsaSsaPssConstraints RSASSA-PSS RSASSA-PSS rsa_pss_pss_sha512 false
+ * @run main/othervm RsaSsaPssConstraints RSASSA-PSS RSASSA-PSS RSA false
+ * @run main/othervm RsaSsaPssConstraints RSA RSASSA-PSS rsa_pss_rsae_sha512 false
+ * @run main/othervm RsaSsaPssConstraints RSA SHA384withRSA rsa_pkcs1_sha256 false
+ * @run main/othervm RsaSsaPssConstraints EC SHA384withECDSA ecdsa_secp256r1_sha256 false
+ * @run main/othervm RsaSsaPssConstraints EC SHA384withECDSA SHA512withECDSA false
+ */
+
+public class RsaSsaPssConstraints extends SSLSocketTemplate {
+
+    private final String protocol;
+    private final String keyAlg;
+    private final String certSigAlg;
+    private X509Certificate trustedCert;
+    private X509Certificate serverCert;
+    private X509Certificate clientCert;
+    private KeyPair serverKeys;
+    private KeyPair clientKeys;
+
+    protected RsaSsaPssConstraints(
+            String protocol, String keyAlg,
+            String certSigAlg) throws Exception {
+        super();
+        this.protocol = protocol;
+        this.keyAlg = keyAlg;
+        this.certSigAlg = certSigAlg;
+        setupCertificates();
+    }
+
+    public static void main(String[] args) throws Exception {
+        if (args.length != 4) {
+            throw new RuntimeException("Wrong number of arguments");
+        }
+
+        String keyAlg = args[0];
+        String certSigAlg = args[1];
+        String constraintAlgo = args[2];
+        boolean fail = Boolean.parseBoolean(args[3]);
+
+        // Note: CertificateBuilder generates RSASSA-PSS certificate
+        // signature using SHA-384 digest algorithm by default.
+        Security.setProperty("jdk.tls.disabledAlgorithms",
+                constraintAlgo + " usage CertificateSignature");
+
+        for (String protocol : new String[]{"TLSv1.3", "TLSv1.2"}) {
+            var test = new RsaSsaPssConstraints(protocol, keyAlg, certSigAlg);
+
+            final String errorMsg = protocol.equals("TLSv1.2") ?
+                    "no cipher suites in common" :
+                    "No available authentication scheme";
+
+            if (fail) {
+                runAndCheckException(test::run,
+                        serverEx -> {
+                            assertTrue(
+                                    serverEx instanceof SSLHandshakeException);
+                            assertEquals(serverEx.getMessage(),
+                                    "(handshake_failure) " + errorMsg);
+                        });
+            } else {
+                test.run();
+            }
+        }
+
+        // Disable KeyManager's algorithm constraints checking and
+        // check against TrustManager's local supported signature
+        // algorithms on the client side.
+        System.setProperty(
+                "jdk.tls.SunX509KeyManager.certChecking", "false");
+
+        for (String protocol : new String[]{"TLSv1.3", "TLSv1.2"}) {
+            var test = new RsaSsaPssConstraints(protocol, keyAlg, certSigAlg);
+
+            if (fail) {
+                runAndCheckException(test::run,
+                        serverEx -> {
+                            Throwable clientEx = serverEx.getSuppressed()[0];
+                            assertTrue(clientEx instanceof SSLHandshakeException
+                                    || serverEx instanceof SSLHandshakeException);
+                        });
+            } else {
+                test.run();
+            }
+        }
+    }
+
+    @Override
+    public SSLContext createServerSSLContext() throws Exception {
+        return getSSLContext(
+                trustedCert, serverCert, serverKeys.getPrivate(), protocol);
+    }
+
+    @Override
+    public SSLContext createClientSSLContext() throws Exception {
+        return getSSLContext(
+                trustedCert, clientCert, clientKeys.getPrivate(), protocol);
+    }
+
+    private static SSLContext getSSLContext(
+            X509Certificate trustedCertificate, X509Certificate keyCertificate,
+            PrivateKey privateKey, String protocol)
+            throws Exception {
+
+        // create a key store
+        KeyStore ks = KeyStore.getInstance("PKCS12");
+        ks.load(null, null);
+
+        // import the trusted cert
+        ks.setCertificateEntry("TLS Signer", trustedCertificate);
+
+        // generate certificate chain
+        Certificate[] chain = new Certificate[2];
+        chain[0] = keyCertificate;
+        chain[1] = trustedCertificate;
+
+        // import the key entry.
+        final char[] passphrase = "passphrase".toCharArray();
+        ks.setKeyEntry("Whatever", privateKey, passphrase, chain);
+
+        TrustManagerFactory tmf = TrustManagerFactory.getInstance("PKIX");
+        tmf.init(ks);
+
+        // create SSL context
+        SSLContext ctx = SSLContext.getInstance(protocol);
+
+        KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509");
+        kmf.init(ks, passphrase);
+
+        ctx.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null);
+
+        return ctx;
+    }
+
+    // Certificate-building helper methods.
+
+    private void setupCertificates() throws Exception {
+        KeyPairGenerator kpg = KeyPairGenerator.getInstance(keyAlg);
+        KeyPair caKeys = kpg.generateKeyPair();
+        this.serverKeys = kpg.generateKeyPair();
+        this.clientKeys = kpg.generateKeyPair();
+
+        this.trustedCert = createTrustedCert(caKeys, certSigAlg);
+
+        this.serverCert = customCertificateBuilder(
+                "O=Some-Org, L=Some-City, ST=Some-State, C=US",
+                serverKeys.getPublic(), caKeys.getPublic())
+                .addBasicConstraintsExt(false, false, -1)
+                .build(trustedCert, caKeys.getPrivate(), certSigAlg);
+
+        this.clientCert = customCertificateBuilder(
+                "CN=localhost, OU=SSL-Client, O=Some-Org, L=Some-City, ST=Some-State, C=US",
+                clientKeys.getPublic(), caKeys.getPublic())
+                .addBasicConstraintsExt(false, false, -1)
+                .build(trustedCert, caKeys.getPrivate(), certSigAlg);
+    }
+
+    private static X509Certificate createTrustedCert(
+            KeyPair caKeys, String certSigAlg)
+            throws Exception {
+        SecureRandom random = new SecureRandom();
+
+        KeyIdentifier kid = new KeyIdentifier(caKeys.getPublic());
+        GeneralNames gns = new GeneralNames();
+        GeneralName name = new GeneralName(new X500Name(
+                "O=Trusted-Org, L=Some-City, ST=Some-State, C=US"));
+        gns.add(name);
+        BigInteger serialNumber = BigInteger.valueOf(
+                random.nextLong(1000000) + 1);
+        return customCertificateBuilder(
+                name.toString(),
+                caKeys.getPublic(), caKeys.getPublic())
+                .setSerialNumber(serialNumber)
+                .addExtension(new AuthorityKeyIdentifierExtension(kid, gns,
+                        new SerialNumber(serialNumber)))
+                .addBasicConstraintsExt(true, true, -1)
+                .build(null, caKeys.getPrivate(), certSigAlg);
+    }
+
+    private static CertificateBuilder customCertificateBuilder(
+            String subjectName, PublicKey publicKey, PublicKey caKey)
+            throws CertificateException, IOException {
+        SecureRandom random = new SecureRandom();
+
+        CertificateBuilder builder = new CertificateBuilder()
+                .setSubjectName(subjectName)
+                .setPublicKey(publicKey)
+                .setNotBefore(
+                        Date.from(Instant.now().minus(1, ChronoUnit.HOURS)))
+                .setNotAfter(Date.from(Instant.now().plus(1, ChronoUnit.HOURS)))
+                .setSerialNumber(
+                        BigInteger.valueOf(random.nextLong(1000000) + 1))
+                .addSubjectKeyIdExt(publicKey)
+                .addAuthorityKeyIdExt(caKey);
+        builder.addKeyUsageExt(
+                new boolean[]{true, true, true, true, true, true});
+
+        return builder;
+    }
+}

test/jdk/sun/security/ssl/X509TrustManagerImpl/CertChainAlgorithmConstraints.java

@@ -0,0 +1,284 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import static jdk.test.lib.Asserts.assertEquals;
+import static jdk.test.lib.Asserts.assertTrue;
+import static jdk.test.lib.Utils.runAndCheckException;
+
+import java.io.IOException;
+import java.math.BigInteger;
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+import java.security.KeyStore;
+import java.security.PublicKey;
+import java.security.SecureRandom;
+import java.security.Security;
+import java.security.cert.CertPathValidatorException;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.Date;
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.TrustManagerFactory;
+import javax.net.ssl.X509ExtendedTrustManager;
+import jdk.test.lib.security.CertificateBuilder;
+import sun.security.provider.certpath.SunCertPathBuilderException;
+import sun.security.validator.ValidatorException;
+import sun.security.x509.AuthorityKeyIdentifierExtension;
+import sun.security.x509.GeneralName;
+import sun.security.x509.GeneralNames;
+import sun.security.x509.KeyIdentifier;
+import sun.security.x509.SerialNumber;
+import sun.security.x509.X500Name;
+
+/*
+ * @test
+ * @bug 8367104
+ * @summary Check for RSASSA-PSS parameters when validating certificates
+ *          against algorithm constraints.
+ * @modules java.base/sun.security.x509
+ *          java.base/sun.security.util
+ *          java.base/sun.security.validator
+ *          java.base/sun.security.provider.certpath
+ * @library /javax/net/ssl/templates
+ *          /test/lib
+ * @run main/othervm CertChainAlgorithmConstraints RSASSA-PSS RSASSA-PSS Rsa_pss_pss_Sha384 true
+ * @run main/othervm CertChainAlgorithmConstraints RSASSA-PSS RSASSA-PSS RsaSsa-Pss true
+ * @run main/othervm CertChainAlgorithmConstraints RSA RSASSA-PSS rsa_pss_Rsae_sha384 true
+ * @run main/othervm CertChainAlgorithmConstraints RSA RSASSA-PSS Rsa true
+ * @run main/othervm CertChainAlgorithmConstraints RSA RSASSA-PSS RSASSA-pSS true
+ * @run main/othervm CertChainAlgorithmConstraints RSA SHA384withRSA rsa_pkcs1_Sha384 true
+ * @run main/othervm CertChainAlgorithmConstraints EC SHA384withECDSA Ecdsa_Secp384r1_sha384 true
+ * @run main/othervm CertChainAlgorithmConstraints RSA SHA384withRSA SHA384withRsA true
+ * @run main/othervm CertChainAlgorithmConstraints RSASSA-PSS RSASSA-PSS rsa_pss_rsae_sha384 false
+ * @run main/othervm CertChainAlgorithmConstraints RSA RSASSA-PSS rsa_pss_pss_sha384 false
+ * @run main/othervm CertChainAlgorithmConstraints RSASSA-PSS RSASSA-PSS rsa_pss_pss_sha256 false
+ * @run main/othervm CertChainAlgorithmConstraints RSASSA-PSS RSASSA-PSS rsa_pss_pss_sha512 false
+ * @run main/othervm CertChainAlgorithmConstraints RSASSA-PSS RSASSA-PSS RSA false
+ * @run main/othervm CertChainAlgorithmConstraints RSA RSASSA-PSS rsa_pss_rsae_sha512 false
+ * @run main/othervm CertChainAlgorithmConstraints RSA SHA384withRSA rsa_pkcs1_sha256 false
+ * @run main/othervm CertChainAlgorithmConstraints EC SHA384withECDSA ecdsa_secp256r1_sha256 false
+ * @run main/othervm CertChainAlgorithmConstraints EC SHA384withECDSA SHA512withECDSA false
+ */
+
+// Testing that an algorithm can be disabled with both a PKIXCertPathValidator
+// and a CertPathBuilder code paths. It is somewhat common for JSSE to fall
+// back to using a CertPathBuilder to find a valid chain.
+public class CertChainAlgorithmConstraints extends SSLEngineTemplate {
+
+    private final String keyAlg;
+    private final String certSigAlg;
+    private final boolean fail;
+
+    private X509Certificate trustedCert;
+    private X509Certificate serverCert;
+    private X509Certificate linkCert1;
+    private X509Certificate linkCert2;
+
+    protected CertChainAlgorithmConstraints(
+            String keyAlg, String certSigAlg, boolean fail)
+            throws Exception {
+        super();
+        this.keyAlg = keyAlg;
+        this.certSigAlg = certSigAlg;
+        this.fail = fail;
+        setupCertificates();
+    }
+
+    public static void main(String[] args) throws Exception {
+        if (args.length != 4) {
+            throw new RuntimeException("Wrong number of arguments");
+        }
+
+        String keyAlg = args[0];
+        String certSigAlg = args[1];
+        String constraintAlgo = args[2];
+        boolean fail = Boolean.parseBoolean(args[3]);
+
+        // Note: CertificateBuilder generates RSASSA-PSS certificate
+        // signature using SHA-384 digest algorithm by default.
+        Security.setProperty("jdk.tls.disabledAlgorithms",
+                constraintAlgo + " usage CertificateSignature");
+
+        new CertChainAlgorithmConstraints(keyAlg, certSigAlg, fail).run();
+    }
+
+    // Run things in TLS handshake order.
+    protected void run() throws Exception {
+
+        // Produce client_hello
+        clientEngine.wrap(clientOut, cTOs);
+        cTOs.flip();
+
+        // Consume client_hello.
+        serverEngine.unwrap(cTOs, serverIn);
+        runDelegatedTasks(serverEngine);
+
+        // Produce server_hello.
+        serverEngine.wrap(serverOut, sTOc);
+        sTOc.flip();
+
+        // Now that we have a Handshake session attached to the serverEngine,
+        // do the check.
+        checkChain(serverEngine);
+    }
+
+    protected void checkChain(SSLEngine engine) throws Exception {
+
+        // Create a key store.
+        KeyStore ks = KeyStore.getInstance("PKCS12");
+        ks.load(null, null);
+
+        // Import the trusted cert.
+        ks.setCertificateEntry("Trusted", trustedCert);
+
+        TrustManagerFactory tmf = TrustManagerFactory.getInstance("PKIX");
+        // Init TrustManager with a Key Store.
+        tmf.init(ks);
+
+        // Generate a mixed-up certificate chain.
+        X509Certificate[] mixedUpChain = new X509Certificate[3];
+        mixedUpChain[0] = linkCert2;
+        // Put EE cert between 2 link certs to mix up the chain.
+        mixedUpChain[1] = serverCert;
+        mixedUpChain[2] = linkCert1;
+
+        // Generate a valid certificate chain - we should get the same
+        // results with it but a different code path to be used.
+        X509Certificate[] validChain = new X509Certificate[3];
+        validChain[0] = serverCert;
+        validChain[1] = linkCert2;
+        validChain[2] = linkCert1;
+
+        var tm = (X509ExtendedTrustManager) tmf.getTrustManagers()[0];
+
+        if (fail) {
+            // Mixed-up chain: CertPathBuilder code path.
+            runAndCheckException(
+                    () -> tm.checkServerTrusted(mixedUpChain, "RSA", engine),
+                    ex -> {
+                        assertTrue(ex instanceof ValidatorException);
+                        assertTrue(
+                                ex.getCause() instanceof SunCertPathBuilderException);
+                        assertEquals(ex.getMessage(), "PKIX path "
+                                + "building failed: "
+                                + "sun.security.provider.certpath."
+                                + "SunCertPathBuilderException: unable to find "
+                                + "valid certification path to requested target");
+                    });
+
+            // Valid chain: PKIXCertPathValidator code path.
+            runAndCheckException(
+                    () -> tm.checkServerTrusted(validChain, "RSA", engine),
+                    ex -> {
+                        assertTrue(ex instanceof ValidatorException);
+                        assertTrue(
+                                ex.getCause() instanceof CertPathValidatorException);
+                        assertTrue(ex.getMessage().startsWith("PKIX path "
+                                + "validation failed: java.security.cert."
+                                + "CertPathValidatorException: Algorithm "
+                                + "constraints check failed on "
+                                + certSigAlg + " signature and "
+                                + keyAlg + " key"));
+                    });
+        } else {
+            tm.checkServerTrusted(mixedUpChain, "RSA", engine);
+            tm.checkServerTrusted(validChain, "RSA", engine);
+        }
+    }
+
+    // Certificate-building helper methods.
+
+    private void setupCertificates() throws Exception {
+        var kpg = KeyPairGenerator.getInstance(keyAlg);
+        var caKeys = kpg.generateKeyPair();
+        var serverKeys = kpg.generateKeyPair();
+        var linkKeys1 = kpg.generateKeyPair();
+        var linkKeys2 = kpg.generateKeyPair();
+
+        this.trustedCert = createTrustedCert(caKeys, certSigAlg);
+
+        this.linkCert1 = customCertificateBuilder(
+                "O=Link1, L=Some-City, ST=Some-State, C=US",
+                linkKeys1.getPublic(), caKeys.getPublic())
+                .addBasicConstraintsExt(true, true, -1)
+                .build(trustedCert, caKeys.getPrivate(), certSigAlg);
+
+        this.linkCert2 = customCertificateBuilder(
+                "O=Link2, L=Some-City, ST=Some-State, C=US",
+                linkKeys2.getPublic(), linkKeys1.getPublic())
+                .addBasicConstraintsExt(true, true, -1)
+                .build(linkCert1, linkKeys1.getPrivate(), certSigAlg);
+
+        this.serverCert = customCertificateBuilder(
+                "O=Some-Org, L=Some-City, ST=Some-State, C=US",
+                serverKeys.getPublic(), linkKeys2.getPublic())
+                .addBasicConstraintsExt(false, false, -1)
+                .build(linkCert2, linkKeys2.getPrivate(),
+                        certSigAlg);
+    }
+
+    private static X509Certificate createTrustedCert(
+            KeyPair caKeys, String certSigAlg)
+            throws Exception {
+        SecureRandom random = new SecureRandom();
+
+        KeyIdentifier kid = new KeyIdentifier(caKeys.getPublic());
+        GeneralNames gns = new GeneralNames();
+        GeneralName name = new GeneralName(new X500Name(
+                "O=Trusted-Org, L=Some-City, ST=Some-State, C=US"));
+        gns.add(name);
+        BigInteger serialNumber = BigInteger.valueOf(
+                random.nextLong(1000000) + 1);
+        return customCertificateBuilder(
+                name.toString(),
+                caKeys.getPublic(), caKeys.getPublic())
+                .setSerialNumber(serialNumber)
+                .addExtension(new AuthorityKeyIdentifierExtension(kid, gns,
+                        new SerialNumber(serialNumber)))
+                .addBasicConstraintsExt(true, true, -1)
+                .build(null, caKeys.getPrivate(), certSigAlg);
+    }
+
+    private static CertificateBuilder customCertificateBuilder(
+            String subjectName, PublicKey publicKey, PublicKey caKey)
+            throws CertificateException, IOException {
+        SecureRandom random = new SecureRandom();
+
+        CertificateBuilder builder = new CertificateBuilder()
+                .setSubjectName(subjectName)
+                .setPublicKey(publicKey)
+                .setNotBefore(
+                        Date.from(Instant.now().minus(1, ChronoUnit.HOURS)))
+                .setNotAfter(Date.from(Instant.now().plus(1, ChronoUnit.HOURS)))
+                .setSerialNumber(
+                        BigInteger.valueOf(random.nextLong(1000000) + 1))
+                .addSubjectKeyIdExt(publicKey)
+                .addAuthorityKeyIdExt(caKey);
+        builder.addKeyUsageExt(
+                new boolean[]{true, true, true, true, true, true});
+
+        return builder;
+    }
+}