[Potential Performance issue] Concurrent File upload time is linearly increasing; Need help

Getting Help
1 
package main

import (
	"bytes"
	"crypto/tls"
	"fmt"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"sync"
	"time"
)

func main() {
	test()

	time.Sleep(10 * time.Minute)
}

func test() {
	for i := 0; i < 50; i++ {
		go UploadFile("myFile", "C:\\XXXXX\\installation\\jmeter\\TinyFileUploadDocs\\file-example_PDF_500_kB.pdf", 0, 0)
	}
}
func UploadFile(key, filePath string, offset int64, limit int64) {

	url := "https://www.xyz.com/test"
	file, _ := os.Open(filePath)
	fi, _ := file.Stat()
	defer file.Close()

	//
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	part, err := writer.CreateFormFile(key, filepath.Base(filePath))
	if err != nil {
		fmt.Println(err)
	}
	_, err = io.Copy(part, file)

	err = writer.Close()
	if err != nil {
		fmt.Println(err)
	}
	if limit <= 0 {
		limit = fi.Size()
	}

	contentType := writer.FormDataContentType()
	req, err := http.NewRequest("POST", url, body)
	req.Header.Add("Content-Type", contentType)
	req.Header.Add("content-length", strconv.Itoa(int(limit)))
	transp, _ := createTransport()
	jobCreateTime := time.Now()
	resp, err := transp.RoundTrip(req)
	timeDifference := time.Now().Sub(jobCreateTime)
	fmt.Println("time difference is ", timeDifference)
	if resp != nil {
		defer resp.Body.Close()
		responseBody, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			fmt.Println("Response is %s %s", responseBody, err)
		}
	}

}

var m sync.Mutex
var defaultTransport *http.Transport

func createTransport() (*http.Transport, error) {
	m.Lock()
	defer m.Unlock()
	if defaultTransport != nil {
		fmt.Println("returning cached instance of httpclient", time.Now())
		return defaultTransport, nil
	}
	defaultRoundTripper := http.DefaultTransport
	defaultTransport = defaultRoundTripper.(*http.Transport).Clone()
	//if !ok {
	//	panic(fmt.Sprintf("defaultRoundTripper not an *http.Transport"))
	//}
	defaultTransport.TLSClientConfig = &tls.Config{
		InsecureSkipVerify: true,
	}
	defaultTransport.MaxIdleConns = 400
	defaultTransport.MaxIdleConnsPerHost = 400
	defaultTransport.IdleConnTimeout = 60 * time.Minute
	defaultTransport.ForceAttemptHTTP2 = false

	fmt.Println("returning newly created httpclient")
	return defaultTransport, nil
}

Response time is linearly increasingreturning newly created httpclient
returning cached instance of httpclient 2023-01-24 22:09:53.9637231 +0530 IST m=+0.014956001
returning cached instance of httpclient 2023-01-24 22:09:53.971713 +0530 IST m=+0.022946601
returning cached instance of httpclient 2023-01-24 22:09:53.971713 +0530 IST m=+0.022946601
returning cached instance of httpclient 2023-01-24 22:09:53.971713 +0530 IST m=+0.022946601
returning cached instance of httpclient 2023-01-24 22:09:53.9722399 +0530 IST m=+0.023473501
returning cached instance of httpclient 2023-01-24 22:09:53.9722399 +0530 IST m=+0.023473501
returning cached instance of httpclient 2023-01-24 22:09:53.9727521 +0530 IST m=+0.023985801
returning cached instance of httpclient 2023-01-24 22:09:53.9727617 +0530 IST m=+0.023995401
returning cached instance of httpclient 2023-01-24 22:09:53.9727617 +0530 IST m=+0.023995401
returning cached instance of httpclient 2023-01-24 22:09:53.9727617 +0530 IST m=+0.023995401
returning cached instance of httpclient 2023-01-24 22:09:53.9727617 +0530 IST m=+0.023995401
returning cached instance of httpclient 2023-01-24 22:09:53.9727617 +0530 IST m=+0.023995401
returning cached instance of httpclient 2023-01-24 22:09:53.9727617 +0530 IST m=+0.023995401
returning cached instance of httpclient 2023-01-24 22:09:53.9727617 +0530 IST m=+0.023995401
returning cached instance of httpclient 2023-01-24 22:09:53.9727617 +0530 IST m=+0.023995401
returning cached instance of httpclient 2023-01-24 22:09:53.9727617 +0530 IST m=+0.023995401
returning cached instance of httpclient 2023-01-24 22:09:53.9732854 +0530 IST m=+0.024519101
returning cached instance of httpclient 2023-01-24 22:09:53.9732854 +0530 IST m=+0.024519101
returning cached instance of httpclient 2023-01-24 22:09:53.9732854 +0530 IST m=+0.024519101
returning cached instance of httpclient 2023-01-24 22:09:53.9732854 +0530 IST m=+0.024519101
returning cached instance of httpclient 2023-01-24 22:09:53.9732854 +0530 IST m=+0.024519101
returning cached instance of httpclient 2023-01-24 22:09:53.9732854 +0530 IST m=+0.024519101
returning cached instance of httpclient 2023-01-24 22:09:53.9738049 +0530 IST m=+0.025038701
returning cached instance of httpclient 2023-01-24 22:09:53.9738049 +0530 IST m=+0.025038701
returning cached instance of httpclient 2023-01-24 22:09:53.9738049 +0530 IST m=+0.025038701
returning cached instance of httpclient 2023-01-24 22:09:53.9738049 +0530 IST m=+0.025038701
returning cached instance of httpclient 2023-01-24 22:09:53.9738049 +0530 IST m=+0.025038701
returning cached instance of httpclient 2023-01-24 22:09:53.9743254 +0530 IST m=+0.025559201
returning cached instance of httpclient 2023-01-24 22:09:53.9743254 +0530 IST m=+0.025559201
returning cached instance of httpclient 2023-01-24 22:09:53.9743254 +0530 IST m=+0.025559201
returning cached instance of httpclient 2023-01-24 22:09:53.9743254 +0530 IST m=+0.025559201
returning cached instance of httpclient 2023-01-24 22:09:53.9743254 +0530 IST m=+0.025559201
returning cached instance of httpclient 2023-01-24 22:09:53.9743254 +0530 IST m=+0.025559201
returning cached instance of httpclient 2023-01-24 22:09:53.9748602 +0530 IST m=+0.026094001
returning cached instance of httpclient 2023-01-24 22:09:53.9748602 +0530 IST m=+0.026094001
returning cached instance of httpclient 2023-01-24 22:09:53.9748602 +0530 IST m=+0.026094001
returning cached instance of httpclient 2023-01-24 22:09:53.9748602 +0530 IST m=+0.026094001
returning cached instance of httpclient 2023-01-24 22:09:53.9753908 +0530 IST m=+0.026624701
returning cached instance of httpclient 2023-01-24 22:09:53.9753908 +0530 IST m=+0.026624701
returning cached instance of httpclient 2023-01-24 22:09:53.9753908 +0530 IST m=+0.026624701
returning cached instance of httpclient 2023-01-24 22:09:53.9753908 +0530 IST m=+0.026624701
returning cached instance of httpclient 2023-01-24 22:09:53.9753908 +0530 IST m=+0.026624701
returning cached instance of httpclient 2023-01-24 22:09:53.9759277 +0530 IST m=+0.027161601
returning cached instance of httpclient 2023-01-24 22:09:53.9759277 +0530 IST m=+0.027161601
returning cached instance of httpclient 2023-01-24 22:09:53.9759277 +0530 IST m=+0.027161601
returning cached instance of httpclient 2023-01-24 22:09:53.9759277 +0530 IST m=+0.027161601
returning cached instance of httpclient 2023-01-24 22:09:53.9759277 +0530 IST m=+0.027161601
returning cached instance of httpclient 2023-01-24 22:09:53.9759277 +0530 IST m=+0.027161601
returning cached instance of httpclient 2023-01-24 22:09:53.9759277 +0530 IST m=+0.027161601
time difference is 2.2112483s
time difference is 2.2788669s
time difference is 2.682173s
time difference is 2.746857s
time difference is 2.848968s
time difference is 2.8817055s
time difference is 2.8923872s
time difference is 3.0118433s
time difference is 3.0381271s
time difference is 3.069961s
time difference is 3.2203927s
time difference is 3.3128912s
time difference is 3.3236621s
time difference is 3.3923878s
time difference is 3.4503079s
time difference is 3.4574394s
time difference is 3.4735691s
time difference is 3.4856638s
time difference is 3.4835582s
time difference is 3.4962755s
time difference is 3.5307242s
time difference is 3.5364922s
time difference is 3.5358658s
time difference is 3.5781682s
time difference is 3.5832667s
time difference is 3.5837974s
time difference is 3.6138955s
time difference is 3.6152253s
time difference is 3.6367311s
time difference is 3.6708537s
time difference is 3.6875839s
time difference is 3.6918802s
time difference is 3.7155376s
time difference is 3.7180674s
time difference is 3.7208604s
time difference is 3.7445374s
time difference is 3.7464671s
time difference is 3.7464801s
time difference is 3.7584079s
time difference is 3.8687172s
time difference is 3.8977421s
time difference is 3.9151934s
time difference is 3.9218858s
time difference is 3.9335794s
time difference is 3.9870066s
time difference is 4.0831174s
time difference is 4.2388037s
time difference is 4.2442227s
time difference is 5.8370773s
time difference is 6.4261305s

2

Have you tried any performance monitoring tools to see if you’re maxing out, your disk reads/network/CPU?

3

You are measuring a few different things there.
Some suggestions:

  • Use sync.Once to create the “cached httpclient” instead of taking a lock each time.
  • Use a single http.Client instead … no need to create a transport.
  • The limit logic has some problems in that the sizes will be wrong after passing through the CreateFormFile. You don’t have to set the content length with go as it will DTRT.
  • With an io.Pipe I don’t think you would need to read in the whole file up front.
  • time.Since > time.Now().Sub()

Hope some of that helps.

4

This topic was automatically closed 90 days after the last reply. New replies are no longer allowed.