diff --git a/server.go b/server.go
index 54043bb..9914912 100644
--- a/server.go
+++ b/server.go
@@ -9,6 +9,7 @@ import (
 	"net"
 	"strings"
 	"sync/atomic"
+	"time"
 
 	"git.sr.ht/~emersion/go-scfg"
 	"github.com/caddyserver/certmagic"
@@ -16,6 +17,8 @@ import (
 	"github.com/pires/go-proxyproto/tlvparse"
 )
 
+const tlsHandshakeTimeout = 10 * time.Second
+
 type acmeCache struct {
 	config *certmagic.Config
 	cache  *certmagic.Cache
@@ -245,7 +248,6 @@ func (ln *Listener) handle(conn net.Conn) error {
 	defer conn.Close()
 	srv := ln.atomic.Load().(*listenerHandles).Server
 
-	// TODO: setup timeouts
 	tlsConfig := srv.ACMEConfig.TLSConfig()
 	getConfigForClient := tlsConfig.GetConfigForClient
 	tlsConfig.GetConfigForClient = func(hello *tls.ClientHelloInfo) (*tls.Config, error) {
@@ -270,9 +272,17 @@ func (ln *Listener) handle(conn net.Conn) error {
 		return tlsConfig, nil
 	}
 	tlsConn := tls.Server(conn, tlsConfig)
+
+	if err := tlsConn.SetDeadline(time.Now().Add(tlsHandshakeTimeout)); err != nil {
+		return fmt.Errorf("failed to set TLS handshake timeout: %v", err)
+	}
 	if err := tlsConn.Handshake(); err != nil {
 		return fmt.Errorf("TLS handshake failed: %v", err)
 	}
+	if err := tlsConn.SetDeadline(time.Time{}); err != nil {
+		return fmt.Errorf("failed to reset TLS handshake timeout: %v", err)
+	}
+	// TODO: allow setting custom downstream timeouts
 
 	tlsState := tlsConn.ConnectionState()
 	fe, err := ln.matchFrontend(tlsState.ServerName)
@@ -314,6 +324,8 @@ type Frontend struct {
 func (fe *Frontend) handle(downstream net.Conn, tlsState *tls.ConnectionState) error {
 	defer downstream.Close()
 
+	// TODO: setup upstream timeouts
+
 	be := &fe.Backend
 	upstream, err := net.Dial(be.Network, be.Address)
 	if err != nil {