Memory Leak on FreeBSD Service

Getting Help Code Review
1

Using GoFiber to run an HTTP service. Running this code in the terminal on a Linux machine results in the GC working correctly and the memory being freed. Running it on FreeBSD as a startup service, the memory does not free, and leaks.

image
image556×838 62.4 KB


import (
	"bytes"
	"fmt"
	"runtime"

	"github.com/valyala/fastjson"
	"github.com/xuri/excelize/v2"
)

func convert_json_to_excel(request_data *fastjson.Value, buffer_pointer *bytes.Buffer) error {

	var err error
	// Remove redundant "data" key passed from body
	json_values := request_data.Get("data")
	if json_values == nil {
		err = fmt.Errorf("submitted json body must contain desired excel data in a key named \"data\": %d", err)
		fmt.Println(err)
		return err
	}

	json_values_array, _ := json_values.Array()

	// Initial no longer needed
	json_values.Del("data")

	// Create a working temporary Excel "file"
	excel_file := excelize.NewFile()
	defer excel_file.Close()

	sheet_index, _ := excel_file.NewSheet("Sheet1")

	excel_file.SetActiveSheet(sheet_index)

	//Embolden the first row
	header_style, _ := excel_file.NewStyle(&excelize.Style{
		Font: &excelize.Font{
			Bold: true,
		},
	})
	header_row_opts := excelize.RowOpts{StyleID: header_style}

	stream_writer, _ := excel_file.NewStreamWriter("Sheet1")

	// Widen all used columns based on headers
	header_row, _ := json_values_array[0].Array()
	stream_writer.SetColWidth(1, len(header_row), 30)

	// Iterate through data and write to file
	for row := 0; row < len(json_values_array); row++ {

		active_row, _ := json_values_array[row].Array()
		//fmt.Printf("%s\n", active_row)

		// Streamwriter is very performant, use of it requires type []interface{}
		interface_row := make([]interface{}, len(active_row))
		for cell := 0; cell < len(active_row); cell++ {
			// Free working memory
			json_values_array[row].Del(fmt.Sprintf("%d", cell))

			interface_row[cell] = active_row[cell].Get().MarshalTo(nil)
		}

		first_cell := fmt.Sprintf("A%d", (row + 1))

		if row == 0 {
			stream_writer.SetRow(first_cell, interface_row, header_row_opts)
		} else {
			stream_writer.SetRow(first_cell, interface_row)
		}
	}

	stream_writer.Flush()

	_, err = excel_file.WriteTo(buffer_pointer)
	if err != nil {
		err = fmt.Errorf("write fail: %d", err)
		fmt.Println(err)
		return err
	}

	// Force GC
	runtime.GC()

	return nil
}
2

Interesting. I see you are passing pointers; I wonder if your calling code isn’t releasing references to those pointers? Like maybe you’re blocking while waiting for input but the blocking code is holding a reference to buffer_pointer *bytes.Buffer or something, while in the web version that pointer is in a handlerFunc and thus is released when that func returns. What does your main() look like in the BSD service?

3

Thanks for replying,

Here is the handler function that calls convert_json_to_excel, main() in this service would just be initializing API routes and such:

func handle_excel_v1(fiber_context *fiber.Ctx) error {
	json_values_root, err := fastjson.ParseBytes(fiber_context.Body())
	if err != nil {
		err = fmt.Errorf("parse data error: %d", err)
		fmt.Println(err)
		return nil
	}

	// Buffer is set for convert_json_to_excel to write bytes to
	var excel_data bytes.Buffer

	err = convert_json_to_excel(json_values_root, &excel_data)
	if err != nil {
		err = fmt.Errorf("conversion error: %d", err)
		fmt.Println(err)
		return nil
	}

	if json_values_root.Exists("file_path") {
		file, _ := os.Create(fmt.Sprintf("%s.%s", string(json_values_root.GetStringBytes("file_path")), "xlsx"))
		defer file.Close()

		file.Write(excel_data.Bytes())
		fiber_context.JSON(fiber.Map{
			"success":   true,
			"file_path": string(json_values_root.GetStringBytes("file_path")),
		})
	} else {
		fiber_context.Response().Header.Set("Content-Disposition", "attachment; filename=response.xlsx")
		fiber_context.Write(excel_data.Bytes())
	}
	return nil
}
4

Oh interesting. I thought you maybe had some separate BSD-specific code to make it run as a service but you’re actually building the SAME code and it behaves differently on BSD? That might be a good candidate for opening an issue in the main repo.