Data Stuck in TLS Server Buffer

Hi all,

I’m encountering an issue while implementing a test for SMTP STARTTLS functionality. Specifically, when the TLS handshake fails, I want to test that a QUIT command is sent before gracefully closing the connection.

Issue:

  • The QUIT command appears to get stuck in the TLS connection handler’s buffer
  • This behavior is intermittent and seems to depend on how quickly the command is sent
  • Adding a small delay before sending the QUIT command resolves the issue

To isolate the problem, I’ve created a minimal reproduction case using simple plain text communication, without SMTP-specific code.

package main

import (
	"bufio"
	"bytes"
	"crypto/rand"
	"crypto/rsa"
	"crypto/tls"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"fmt"
	"math/big"
	"net"
	"strings"
	"sync"
	"testing"
	"time"
)

func TestTLSHandshake(t *testing.T) {
	ln := newLocalListener(t)
	defer ln.Close()

	wg := sync.WaitGroup{}
	wg.Add(1)

	go func() {
		// Client
		clientConn, _ := net.Dial("tcp", ln.Addr().String())
		defer clientConn.Close()

		_, _ = clientConn.Write([]byte("STARTTLS\r\n"))

		// Wait for readiness
		buf := make([]byte, 1024)
		_, _ = clientConn.Read(buf)
		fmt.Printf("[Client] Received reply: %s", string(buf))

		// Try TLS handshake
		tlsConn := tls.Client(clientConn, &tls.Config{
			ServerName: "localhost",
		})

		// Handshake should fail because client doesn't trust self-signed cert
		err := tlsConn.Handshake()
		if err != nil {
			fmt.Println("[Client] Handshake err: ", err)
		}

		// Delay to prevent QUIT from being consumed into TLS buffer
		time.Sleep(100 * time.Millisecond)

		_, _ = clientConn.Write([]byte("QUIT\r\n"))

		wg.Done()
	}()

	serverConn, _ := ln.Accept()
	defer serverConn.Close()

	quitReceived := false

	scanner := bufio.NewScanner(serverConn)
	for scanner.Scan() {
		input := scanner.Text()

		fmt.Printf("[Server] Received: '%s'\n", input)

		switch input {
		case "STARTTLS":
			// Generate test cert
			cert, privateKey, _ := generateTestCertificatePair()

			// Setup TLS
			keypair, _ := tls.X509KeyPair(cert, privateKey)
			config := &tls.Config{Certificates: []tls.Certificate{keypair}}

			// Reply readiness
			_, _ = serverConn.Write([]byte("Ready for handshake\r\n"))

			tlsConn := tls.Server(serverConn, config)

			handshakeErr := tlsConn.Handshake()
			if handshakeErr != nil {
				fmt.Println("[Server] Handshake error: ", handshakeErr)
				continue
			} else {
				fmt.Println("[Server] Handshake completed")
				t.Fatal("Handshake unexpectedly succeeded")
			}
		case "QUIT":
			quitReceived = true

			fmt.Println("[Server] Quit received")
		}
	}

	wg.Wait()

	if !quitReceived {
		t.Fatal("[Server] Quit not received")
	}
}

// Generate X509 encoded certificate for testing.
func generateTestCertificatePair() ([]byte, []byte, error) {
	organization := "example"
	host := "localhost"

	validFrom := time.Now()
	validTo := validFrom.Add(24 * time.Hour)

	// Generate private key
	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return nil, nil, err
	}

	// Create certificate serial number
	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)

	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
	if err != nil {
		return nil, nil, err
	}

	template := x509.Certificate{
		SerialNumber: serialNumber,
		Subject: pkix.Name{
			Organization: []string{organization},
			CommonName:   host,
		},
		NotBefore: validFrom,
		NotAfter:  validTo,

		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
		BasicConstraintsValid: true,
	}

	hosts := strings.Split(host, ",")
	template.DNSNames = append(template.DNSNames, hosts...)

	// Create certificate
	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey)
	if err != nil {
		return nil, nil, err
	}

	// Encode certificate to PEM
	certBuffer := &bytes.Buffer{}
	if err := pem.Encode(certBuffer, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
		return nil, nil, err
	}

	// Encode private key to PEM
	privBytes, err := x509.MarshalPKCS8PrivateKey(privateKey)
	if err != nil {
		return nil, nil, err
	}

	keyBuffer := &bytes.Buffer{}
	if err := pem.Encode(keyBuffer, &pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}); err != nil {
		return nil, nil, err
	}

	return certBuffer.Bytes(), keyBuffer.Bytes(), nil
}

func newLocalListener(t *testing.T) net.Listener {
	ln, _ := net.Listen("tcp", "localhost:0")
	return ln
}

The above code is also available here. Notice that if the delay is removed, the server will not receive the QUIT command.


I understand that TCP is a continuous stream and the application should have the message boundary by itself. However, I am using tls.Server() and do not have control over how it will handle the TCP connection.

Is there anything I can do to ensure that the server will be able to receive the QUIT command is received by the main handler instead of the TLS one?

Thank you for your help!

Are you wondering about this?

There is nothing to be confused about, because after the TLS handshake fails, the client needs to send a message to the server, and you send the message quickly to preempt the handshake completion process.

try it:

func TestTLSHandshake(t *testing.T) {
	ln := newLocalListener(t)
	defer ln.Close()

	wg := sync.WaitGroup{}
	wg.Add(1)

	go func() {
		// Client
		clientConn, _ := net.Dial("tcp", ln.Addr().String())
		defer clientConn.Close()

		_, _ = clientConn.Write([]byte("STARTTLS\r\n"))

		// Wait for readiness
		buf := make([]byte, 1024)
		_, _ = clientConn.Read(buf)
		fmt.Printf("[Client] Received reply: %s", string(buf))

		// Try TLS handshake
		tlsConn := tls.Client(clientConn, &tls.Config{
			ServerName: "localhost",
		})

		// Handshake should fail because client doesn't trust self-signed cert
		err := tlsConn.Handshake()
		if err != nil {
			fmt.Println("[Client] Handshake err: ", err)
		}

		// Delay to prevent QUIT from being consumed into TLS buffer
		//time.Sleep(100 * time.Millisecond)

		// wait server call handshake over
		_, _ = clientConn.Read(make([]byte, 4))

		_, _ = clientConn.Write([]byte("QUIT\r\n"))

		wg.Done()
	}()

	serverConn, _ := ln.Accept()
	defer serverConn.Close()

	quitReceived := false

	scanner := bufio.NewScanner(serverConn)
	for scanner.Scan() {
		input := scanner.Text()

		fmt.Printf("[Server] Received: '%s'\n", input)

		switch input {
		case "STARTTLS":
			// Generate test cert
			cert, privateKey, _ := generateTestCertificatePair()

			// Setup TLS
			keypair, _ := tls.X509KeyPair(cert, privateKey)
			config := &tls.Config{Certificates: []tls.Certificate{keypair}}

			// Reply readiness
			_, _ = serverConn.Write([]byte("Ready for handshake\r\n"))

			tlsConn := tls.Server(serverConn, config)

			handshakeErr := tlsConn.Handshake()
			if handshakeErr != nil {
				fmt.Println("[Server] Handshake error: ", handshakeErr)
			} else {
				fmt.Println("[Server] Handshake completed")
				t.Fatal("Handshake unexpectedly succeeded")
			}
			// tell client handshake over
			_, _ = serverConn.Write([]byte("QUIT"))
		case "QUIT":
			quitReceived = true

			fmt.Println("[Server] Quit received")
		}
	}

	wg.Wait()

	if !quitReceived {
		t.Fatal("[Server] Quit not received")
	}
}

Please refer to the TLS handshake process for more information so that you will not have these doubts.

1 Like

@peakedshout thank you for taking a look.

My original intention was to keep using the same TCP conn to send the mail in plaintext, especially when TLS handshake fail and it is not required by the user. As your code have shown, this requires a coordination between the server and client to avoid mixing application and TLS handshake data.

However, looking at the methods provided by Go TLS Conn, I believe we do not have the granular control for sending Closure Alerts. Looking at other SMTP clients in Go, I also found that the clients doesn’t try to send in plaintext when there is a TLS error. They instead just return error.

Therefore, the most practical solution seems to close the current connection, create a new TCP connection and then send in plaintext with the new connection.

Anyway, thanks again for checking in.

Haha.
In fact, the design of most communication protocols is simple and crude. By design, if you enter the parity-and-encrypt handshake, the usual approach is to drop the connection if encryption fails.
Let’s take a simple example: socks protocol. Initially, the protocol is also plaintext and chooses a parity-and-encryption method, but if the handshake fails, both the client and the server will actively close the connection and drop it.
The reason is simple: the logic to re-establish a connection is much simpler than the cryptographic fallback to re-align.