wolfsshd: load PKCS#8 PEM host keys (e.g. ML-DSA)

wolfsshd could not load a host key stored as a PKCS#8 "-----BEGIN PRIVATE KEY-----"
PEM file (the form emitted for ML-DSA keys, and by `openssl genpkey`).

SetupCTX() pre-converted the file with wc_PemToDer(..., PRIVATEKEY_TYPE, ...),
which only recognizes the classic "RSA/EC PRIVATE KEY" headers. On a PKCS#8 body
that call returns success but produces a malformed DER (leading 0x04 rather than a
0x30 SEQUENCE); the bytes were then passed to wolfSSH_CTX_UsePrivateKey_buffer()
as WOLFSSH_FORMAT_ASN1, which rejected them with WS_BAD_FILETYPE_E.

Detect PEM vs DER by content and decode PEM with wc_KeyPemToDer(), which handles
PKCS#1, SEC1 and PKCS#8 private-key bodies. DER keys are unchanged.

Tested (master + wolfSSL master): an ML-DSA-65 PEM host key now loads
(wolfSSH_CTX_UsePrivateKey_buffer ret = 0, was WS_BAD_FILETYPE_E); ML-DSA-65 DER
and ECDSA SEC1 PEM host keys continue to load (no regression).

Also harden the host-key load path per review: reject an empty (0-byte)
key file explicitly instead of taking the PEM branch into WMALLOC(0), and
zeroize the raw file buffer (not just the decoded DER) before freeing so
private-key material does not linger in the heap.

Author: shaunchokshi

apps/wolfsshd/wolfsshd.c

@@ -349,9 +349,8 @@ static int SetupCTX(WOLFSSHD_CONFIG* conf, WOLFSSH_CTX** ctx,
         byte** banner)
 {
     int ret = WS_SUCCESS;
-    DerBuffer* der = NULL;
-    byte* privBuf;
-    word32 privBufSz;
+    byte* privBuf = NULL;
+    word32 privBufSz = 0;
     void* heap = NULL;
 
     if (ctx == NULL) {
@@ -412,28 +411,70 @@ static int SetupCTX(WOLFSSHD_CONFIG* conf, WOLFSSH_CTX** ctx,
             }
 
             if (ret == WS_SUCCESS) {
-                if (wc_PemToDer(data, dataSz, PRIVATEKEY_TYPE, &der, NULL,
-                                NULL, NULL) != 0) {
-                    wolfSSH_Log(WS_LOG_DEBUG, "[SSHD] Failed to convert host "
-                                "private key from PEM. Assuming key in DER "
-                                "format.");
+                /* Host keys may be PEM or DER. Detect by content: a DER key is
+                 * an ASN.1 SEQUENCE (leading 0x30); anything else is treated as
+                 * PEM text and decoded with wc_KeyPemToDer(), which handles
+                 * PKCS#1, SEC1 and PKCS#8 "PRIVATE KEY" bodies.
+                 *
+                 * The previous code used wc_PemToDer(..., PRIVATEKEY_TYPE, ...),
+                 * which only recognizes the classic "RSA/EC PRIVATE KEY" PEM
+                 * headers. On a PKCS#8 body (how ML-DSA host keys are emitted)
+                 * it returns *success* but yields a malformed body (leading
+                 * 0x04, not a 0x30 SEQUENCE), which
+                 * wolfSSH_CTX_UsePrivateKey_buffer() then rejects with
+                 * WS_BAD_FILETYPE_E, so ML-DSA PEM host keys could not load. */
+                byte* keyDer = NULL;
+
+                if (dataSz == 0) {
+                    /* An empty (0-byte) file passes the NULL check above but
+                     * carries no key material. Handle it explicitly as a file
+                     * error instead of falling into the PEM path, where
+                     * WMALLOC(0) is implementation-defined (may return NULL and
+                     * be misreported as WS_MEMORY_E). */
+                    wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Host key file is empty.");
+                    ret = WS_BAD_FILE_E;
+                }
+                else if (data[0] == 0x30) {
                     privBuf = data;
                     privBufSz = dataSz;
                 }
                 else {
-                    privBuf = der->buffer;
-                    privBufSz = der->length;
+                    keyDer = (byte*)WMALLOC(dataSz, heap, DYNTYPE_SSHD);
+                    if (keyDer == NULL) {
+                        ret = WS_MEMORY_E;
+                    }
+                    else {
+                        int keyDerSz = wc_KeyPemToDer(data, dataSz, keyDer,
+                                                      (int)dataSz, NULL);
+                        if (keyDerSz <= 0) {
+                            wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Failed to convert "
+                                        "host private key from PEM.");
+                            ret = WS_BAD_FILE_E;
+                        }
+                        else {
+                            privBuf = keyDer;
+                            privBufSz = (word32)keyDerSz;
+                        }
+                    }
                 }
 
-                if (wolfSSH_CTX_UsePrivateKey_buffer(*ctx, privBuf, privBufSz,
-                                                     WOLFSSH_FORMAT_ASN1) < 0) {
+                if (ret == WS_SUCCESS
+                        && wolfSSH_CTX_UsePrivateKey_buffer(*ctx, privBuf,
+                                privBufSz, WOLFSSH_FORMAT_ASN1) < 0) {
                     wolfSSH_Log(WS_LOG_ERROR,
                         "[SSHD] Failed to use host private key.");
                     ret = WS_BAD_ARGUMENT;
                 }
 
+                if (keyDer != NULL) {
+                    WS_FORCEZERO(keyDer, dataSz);
+                    WFREE(keyDer, heap, DYNTYPE_SSHD);
+                }
+                /* data held the raw private key — the DER bytes, or the PEM
+                 * text decoded into keyDer above. Zeroize before freeing so key
+                 * material does not linger in the heap after use. */
+                WS_FORCEZERO(data, dataSz);
                 freeBufferFromFile(data, heap);
-                wc_FreeDer(&der);
             }
         }
     }