Errors getting image and uploading it to s3

Getting Help
1

Hello There,

I’ve been attempting to speed up uploading images to a s3 bucket, without using goroutines everything works as expected but when I use the function with goroutines I get this error:

2017/10/23 12:40:20 There was an error decoding the image:  invalid JPEG format: missing 0xff00 sequence
2017/10/23 12:40:20 http: panic serving [::1]:45316: runtime error: invalid memory address or nil pointer dereference
goroutine 39 [running]:
net/http.(*conn).serve.func1(0xc420317c20)
        /usr/lib/go/src/net/http/server.go:1697 +0xd0
panic(0x9764a0, 0xd7b410)
        /usr/lib/go/src/runtime/panic.go:491 +0x283
image.image.Rectangle.Sub(...)
        /project/path/vendor/github.com/disintegration/imaging/helpers.go:171
/project/path/vendor/github.com/disintegration/imaging.Clone(0x0, 0x0, 0x80)
        /project/path/vendor/github.com/disintegration/imaging/helpers.go:171 +0x2c
/project/path/vendor/github.com/disintegration/imaging.toNRGBA(0x0, 0x0, 0xbe73af4927189325)
       /project/path/vendor/github.com/disintegration/imaging/helpers.go:432 +0x71
/project/path/vendor/github.com/disintegration/imaging.Resize(0x0, 0x0, 0x180, 0x0, 0x3ff0000000000000, 0xa352d8, 0xc4205700c0)
        /project/path/vendor/github.com/disintegration/imaging/resize.go:74 +0x82
/project/path/images.Thumbnail(0x0, 0x0, 0x2)
       /project/path/images/thumbnails.go:16 +0x6b
/project/path/storage.UploadImageAndThumbnail(0xc4200f0000, 0xa5, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /project/path/storage/upload.go:116 +0x5a6
/project/path/models.InsertImage(0xc4201990e0, 0xc42053eb70, 0xa, 0xc4205404d0)
        /project/path/models/image.go:95 +0x337
/project/path/handlers.GetImage.func1(0xd4eb80, 0xc42022e000, 0xc420186100, 0x0, 0x0, 0x0)
        /project/path/handlers/image.go:41 +0x49f
/project/path/vendor/github.com/julienschmidt/httprouter.(*Router).ServeHTTP(0xc420542a80, 0xd4eb80, 0xc42022e000, 0xc420186100)
        /project/path/vendor/github.com/julienschmidt/httprouter/router.go:299 +0x6f1
net/http.serverHandler.ServeHTTP(0xc42054aea0, 0xd4eb80, 0xc42022e000, 0xc420186100)
        /usr/lib/go/src/net/http/server.go:2619 +0xb4
net/http.(*conn).serve(0xc420317c20, 0xd4f300, 0xc42053c480)
        /usr/lib/go/src/net/http/server.go:1801 +0x71d
created by net/http.(*Server).Serve
        /usr/lib/go/src/net/http/server.go:2720 +0x288

For the most part I’m using the same exact code that I was before, just put it in goroutines. All the code does it gets a link to a url and gets the image, resizes the image and then sames it to a s3 bucket. If anyone could suggest something that will point me in the right direction to fixing it or doing it better that would be greatly appreciated.

UploadImageAndThumbnail()

func UploadImageAndThumbnail(url string) (string, string, string, string) {
	var (
		imageLocation     string
		thumbnailLocation string
		imageID           string
		thumbnailID       string
		wg                sync.WaitGroup
	)

	uuid, err := uuid.NewV4()
	if err != nil {
		log.Println("UUID failed to be initialized.", err)
	}

	client := &http.Client{Timeout: 5 * time.Second}
	resp, err := client.Get(url)
	if err != nil {
		log.Println("There was an error getting image: ", err)
	}
	defer resp.Body.Close()

	imageName := "public/images/" + uuid.String() + ".jpeg"
	thumbnailName := "public/thumbnails/" + uuid.String() + ".jpeg"

	wg.Add(2)
	go func(loc, id string) {
		defer wg.Done()
		loc, id = upload(bucket, imageName, "image/jpeg", resp.Body)
	}(imageLocation, imageID)

	buffer := streamToByte(resp.Body)
	r := bytes.NewReader(buffer)
	img, _, err := image.Decode(r)
	if err != nil {
		log.Println("There was an error decoding the image: ", err)
	}

	thumbnail := images.Thumbnail(img)

	go func(loc, id string) {
		defer wg.Done()
		loc, id = upload(bucket, thumbnailName, "image/jpeg", thumbnail)
	}(thumbnailLocation, thumbnailID)
	wg.Wait()

	fmt.Println(imageLocation, thumbnailLocation)
	return imageLocation, imageID, thumbnailLocation, thumbnailID
}

upload()

func upload(bucket, name, t string, file io.Reader) (string, string) {
	uploader, err := connectToSpaces()
	if err != nil {
		log.Println("The Uploader failed:", err)
	}

	result, err := uploader.Upload(&s3manager.UploadInput{
		Bucket:       aws.String(bucket),
		Key:          aws.String(name),
		Body:         file,
		CacheControl: aws.String("8600"),
		ContentType:  aws.String(t),
		ACL:          aws.String("public-read"),
	})
	if err != nil {
		log.Println("The upload function threw and error: ", err)
	}
	fmt.Printf("Successfully uploaded %q to %q\n", result.Location, bucket)

	return result.Location, result.UploadID
}
2

Sounds like a data race then.

In the code I notice two possible issues:

The two goroutines are closures accessing data from the outer context. Especially resp.Body is read twice, once inside and once outside the first goroutine.

And loc and id are passed by value but get new values assigned that are used outside the goroutine.

1 Like
3

Ah, I didn’t even think about then, let me rewrite it with I’m assuming channnels would help.

4

This topic was automatically closed 90 days after the last reply. New replies are no longer allowed.