Hello,
Can anyone help me to understand , how to tackle with golang performance .
Am inserting 5lakh documents into couchbase database,(tried both channels and waitsync group approach)
4kb document each (toatl 5lakh records) - 191 secs
1kb document each (toatl 5lakh records) - 53 secs
it’s huge difference in performance . whereas in C++ same is taking 7secs and 11secs respectively .
Below is my code :
package main
import (
"fmt"
"gopkg.in/couchbase/gocb.v1"
"strconv"
"time"
"sync"
"github.com/tkanos/gonfig"
"math/rand"
)
type Insert_doc struct {
Thread_id int
KTAB, SyncBuffer string
}
type Configuration struct {
NumofDoc int
ServerIp string
Username string
Password string
Randstr int
BucketName string
ThreadCount int
Port int
OP_TYPE int
}
func main() {
configuration := Configuration{}
_ = gonfig.GetConf("Couchbase_config.json", &configuration)
fmt.Println("Config File name Passed : Couchbase_config.json")
fmt.Println("ThreadCount : ",configuration.ThreadCount) //ThreadCount:2
fmt.Println("Server IP : ",configuration.ServerIp)
fmt.Println("Number of Requests per thread : ",configuration.NumofDoc) //NumofDoc: 250000
var wg sync.WaitGroup
for i := 0; i < configuration.ThreadCount; i++ { //ThreadCount : 2
wg.Add(1)
go worker(&wg,i,configuration.OP_TYPE)
}
wg.Wait()
}
func worker(wg *sync.WaitGroup,id int,s int) {
configuration := Configuration{}
_ = gonfig.GetConf("Couchbase_config.json", &configuration)
var insertCount int64 = 0
var readCount int64 = 0
var readproportion int
var updateproportion int
var opsSequence[100]int
operation_type := s
cluster, err := gocb.Connect(configuration.ServerIp) //Connects to the cluster
if err != nil {
fmt.Println(err.Error())
return
}
cluster.Authenticate(gocb.PasswordAuthenticator{
Username: configuration.Username,
Password: configuration.Password,
})
var bucket *gocb.Bucket
bucket, err = cluster.OpenBucket(configuration.BucketName, "") //Connects to the bucket
if err != nil {
fmt.Println(err.Error())
return
}
if operation_type == 1 {
updateproportion = 100
readproportion = 0
} else if operation_type == 2 {
updateproportion = 0
readproportion = 100
} else if operation_type == 3 {
updateproportion = 50
readproportion = 50
}
count:=0
for b := 0; b < updateproportion; b++ {
opsSequence[b] =1
count++
}
for b := 0; b < readproportion; b++ {
opsSequence[count+b]=2
}
Thread_Start := time.Now().Unix()
for j :=0; j < configuration.NumofDoc; j++ { //NumofDoc : 250000
k := j%100;
optype := opsSequence[k];
var x int = int(readCount % 5000);
switch(optype){
case 1:
document := Insert_doc{Thread_id: id, KTAB: "INSERT", SyncBuffer: RandomString(configuration.Randstr)} // Randstr - 4000
test := "Go_Demo_"+strconv.Itoa(id)+"_"+strconv.Itoa(int(insertCount))
createDocument(bucket,test, &document)
insertCount++;
break;
case 2:
test := "Go_Demo_"+strconv.Itoa(id)+"_"+strconv.Itoa(x)
getDocument(bucket,test)
readCount++;
break;
break;
default:
fmt.Println("Invalid Operation Type ",optype)
}
}
Thread_End := time.Now().Unix()
timediff := Thread_End - Thread_Start
var avgLatency float64 = float64(timediff)/float64(insertCount+readCount);
var opsPerSec float64 = 1/avgLatency;
fmt.Printf("THREAD_ID %d TOTAL WRITE : %d, TOTAL READ : %d, TOTAL OPERATION TIME : %d, AVG_LATENCY = %f S, OPS_PER_SECOND = %f \n",id, insertCount,readCount, timediff, avgLatency,opsPerSec);
wg.Done()
}
func createDocument(bucket *gocb.Bucket,documentId string, document *Insert_doc) {
_, error := bucket.Upsert(documentId, document, 0)
if error != nil {
fmt.Println(error.Error())
}
}
func RandomString(n int) string {
var letter = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
b := make([]rune, n)
for i := range b {
b[i] = letter[rand.Intn(len(letter))]
}
return string(b)
}
func getDocument(bucket *gocb.Bucket,documentId string) {
var get_data Insert_doc
_, error := bucket.Get(documentId, &get_data)
if error != nil {
fmt.Println(error.Error())
return
}
}