Correct shutdown of net.Listener

[alexs (Alex)]

I’m working on a simple server that listens on a TCP socket, accepts connections, reads and process some data from the clients. There are nice examples of such servers in the official Go documentation as well as many posts here and there. But most of them does not give any information about how to close listener and cleanup existing connections.

As I understand, correct approach will be: on shutdown, stop accepting new connections; process existing connections and wait for them to close; finally, when all existing connections have closed, stop the server.

I’ve tried to implement this as follows

package main

import (
	"fmt"
	"net"
	"os"
	"sync"
	"time"
)

type Server struct {
	listener  net.Listener
	conns     []net.Conn
	mtx       sync.Mutex
	cancelled bool
}

func NewServer() *Server {
	addr, err := net.ResolveTCPAddr("tcp4", ":9999")
	if err != nil {
		fmt.Println("Failed to resolve address", err.Error())
		os.Exit(1)
	}

	listener, err := net.Listen("tcp", addr.String())
	if err != nil {
		fmt.Println("Failed to create listener", err.Error())
		os.Exit(1)
	}

	srv := &Server{
		listener,
		make([]net.Conn, 0, 50),
		sync.Mutex{},
		false,
	}
	go srv.Serve()
	return srv
}

func (srv *Server) Serve() {
	var stop bool
	for {
		srv.mtx.Lock()
		stop = srv.cancelled
		srv.mtx.Unlock()
		if stop {
			break
		}

		conn, err := srv.listener.Accept()
		if err != nil {
			fmt.Println("Failed to accept connection:", err.Error())
			continue
		}
		srv.mtx.Lock()
		srv.conns = append(srv.conns, conn)
		srv.mtx.Unlock()
		go srv.handleConnection(conn, len(srv.conns)-1)
	}
}

func (srv *Server) handleConnection(conn net.Conn, id int) error {
	fmt.Println("Accepted connection from", conn.RemoteAddr())

	defer func() {
		fmt.Println("Closing connection from", conn.RemoteAddr())
		conn.Close()
		srv.mtx.Lock()
		srv.conns[id] = nil
		srv.mtx.Unlock()
	}()

	buf := make([]byte, 1024)
	_, err := conn.Read(buf)
	if err != nil {
		fmt.Println("Read error", err.Error())
		return err
	}
	return nil
}

func (srv *Server) Stop() {
	fmt.Println("Stop requested")
	srv.mtx.Lock()
	srv.cancelled = true
	srv.mtx.Unlock()

	srv.listener.Close()
	ticker := time.NewTicker(500 * time.Millisecond)
	defer ticker.Stop()
	for {
		select {
		case <-ticker.C:
			fmt.Println("Waiting on connections", len(srv.conns))
		}
		if len(srv.conns) == 0 {
			fmt.Println("Stopped")
			return
		}
	}
}

func main() {
	s := NewServer()
	time.Sleep(2 * time.Second)
	s.Stop()
}

Is this correct, Go-ish approach or there is another, better solution?

And second question: when server stopped, there is an error “use of closed network connection”, e.g.

Stop requested
Failed to accept connection: accept tcp [::]:9999: use of closed network connection
Waiting on connections 0
Stopped

Is it possible to hide this error, but still catch all other errors?

[Giulio_Iotti]

Not very Go-ish, frankly. A more idiomatic solution is to have:

(1) one go-routine that Accept()s and gets a connection object. Just accept and pass them down to a channel.
(2) A number of workers that listen to the channel of connections. They get one connection from the channel, work on it, close it.

Shutdown means:
(1) You stop listening, by exiting from the loop where you have Accept().
(2) Close the connections channel. Now all workers at the next iteration of their loop over the connections channel will exit.
(3) Each worker exists from the loop over the connections channel and signals it is done (for example with sync.WaitGroup.Done())
(4) Wait for all workers to exit, for example with a sync.WaitGroup.Wait().

Let me know if you need more clarifications!

[alexs (Alex)]

Thanks for the help!

I tried to implement something like workflow you suggested, but stuck with putting/receiving connections to/from channel and handling them on shutdown. May I ask for some feedback/hints regarding following code?

package main

import (
	"fmt"
	"net"
	"os"
	"time"
)

type Server struct {
	listener net.Listener
	quit     chan bool
	conns    chan net.Conn
}

func NewServer() *Server {
	addr, err := net.ResolveTCPAddr("tcp4", ":9999")
	if err != nil {
		fmt.Println("Failed to resolve address", err.Error())
		os.Exit(1)
	}

	listener, err := net.Listen("tcp", addr.String())
	if err != nil {
		fmt.Println("Failed to create listener", err.Error())
		os.Exit(1)
	}

	srv := &Server{
		listener,
		make(chan bool),
		make(chan net.Conn, 10),
	}
	go srv.Serve()
	return srv
}

func (srv *Server) Serve() {
	for {
		select {
		case <-srv.quit:
			fmt.Println("Shutting down...")
			srv.listener.Close()
			// FIXME: check that all existing connections were processed
			// and closed correctly
			srv.quit <- true
			return
		default:
			//fmt.Println("Listening for clients")
			srv.listener.SetDeadline(time.Now().Add(1e9))
			conn, err := srv.listener.Accept()
			if err != nil {
				if opErr, ok := err.(*net.OpError); ok && opErr.Timeout() {
					continue
				}
				fmt.Println("Failed to accept connection:", err.Error())
			}
			srv.conns <- conn
			// FIXME: handle connections from the channel
		}
	}
}

func (srv *Server) handleConnection(conn net.Conn, id int) error {
	fmt.Println("Accepted connection from", conn.RemoteAddr())

	defer func() {
		fmt.Println("Closing connection from", conn.RemoteAddr())
		conn.Close()
	}()

	buf := make([]byte, 1024)
	_, err := conn.Read(buf)
	if err != nil {
		fmt.Println("Read error", err.Error())
		return err
	}
	return nil
}

func (srv *Server) Stop() {
	fmt.Println("Stop requested")
	srv.quit <- true
	<-srv.quit
	fmt.Println("Stopped successfully")
}

func main() {
	srv := NewServer()
	time.Sleep(2 * time.Second)
	srv.Stop()
}

[Giulio_Iotti]

Hi, have a look if the comments here help: https://play.golang.org/p/OqiSY2peFBN

[alexs (Alex)]

Thanks a lot for your help!