Database rows scan + unknown number of columns + JSON

Hey there, I have the following code for example;

err := rows.Scan(&v.IdVeiculoGrupo, &v.Identificacao)

Then after that I serialize the v struct like;

json.NewEncoder(w).Encode(&v)

I need something like when DBA select more columns, or remove some, there into the stored procedure inside database, serializing exactly the columns they have selected there, without having to create a fixed struct for that. Is there any simple way to do that?

Tnx.

Is there any simple way to do that?

Not sure it can be called simple, but I got something working.

main.go

package main

import (
	"database/sql"
	"encoding/json"
	"log"
	"os"
	"reflect"

	_ "github.com/mattn/go-sqlite3"
)

func main() {
	log.SetFlags(log.Lshortfile)

	db := initDb()

	b, err := queryToJson(db, "select * from test")
	if err != nil {
		log.Fatalln(err)
	}
	os.Stdout.Write(b)

}

func queryToJson(db *sql.DB, query string, args ...interface{}) ([]byte, error) {
	// an array of JSON objects
	// the map key is the field name
	var objects []map[string]interface{}

	rows, err := db.Query(query, args...)
	if err != nil {
		return nil, err
	}
	for rows.Next() {
		// figure out what columns were returned
		// the column names will be the JSON object field keys
		columns, err := rows.ColumnTypes()
		if err != nil {
			return nil, err
		}

		// Scan needs an array of pointers to the values it is setting
		// This creates the object and sets the values correctly
		values := make([]interface{}, len(columns))
		object := map[string]interface{}{}
		for i, column := range columns {
			object[column.Name()] = reflect.New(column.ScanType()).Interface()
			values[i] = object[column.Name()]
		}

		err = rows.Scan(values...)
		if err != nil {
			return nil, err
		}

		objects = append(objects, object)
	}

	// indent because I want to read the output
	return json.MarshalIndent(objects, "", "\t")
}

func initDb() *sql.DB {
	db, err := sql.Open("sqlite3", ":memory:")
	if err != nil {
		log.Fatalln(err)
	}

	_, err = db.Exec(`
create table test (
	id integer primary key,
	field1 integer,
	field2 text
);`)
	if err != nil {
		log.Fatalln(err)
	}

	_, err = db.Exec(`
insert into test (field1, field2) values
(1, 'first'),
(2, 'second')
`)
	if err != nil {
		log.Fatalln(err)
	}

	return db
}

output

$ go run main.go
[
	{
		"field1": 1,
		"field2": "first",
		"id": 1
	},
	{
		"field1": 2,
		"field2": "second",
		"id": 2
	}
]
1 Like

I used it, it writes the json with the fields but the string data, I mean the varchar fields, data comes encrypted;

[{"ATIVO":0,"ID":22,"IDCLIENTE":18,"SENHA":"MzM0NDU2Nw==","TIPOUSUARIO":1,"USUARIO":"MzM0NDU2Nw=="},{"ATIVO":1,"ID":23,"IDCLIENTE":19,"SENHA":"MTk3NQ==","TIPOUSUARIO":1,"USUARIO":"cG90ZW5jaWE="},{"ATIVO":1,"ID":24,"IDCLIENTE":20,"SENHA":"MzExNjM3Ng==","TIPOUSUARIO":1,"USUARIO":"Z2lsc29uMDE="}]

Why?

You need a pointer to RawMessage per this thread - https://groups.google.com/forum/#!topic/Golang-Nuts/38ShOlhxAYY

I’m here for more than 15 hours searching on whole Google and found nothing, below is my code, every data from varchar fields comes encrypted with base64, I really don’t understand why it happens and how to resolve that;

	rows, err := db.Query("EXEC PRCATUALIZACOES @IDVEICULO=" + urlqry.Get("idveiculo") + ", @TAMANHOPAGINA=1, @NUMEROPAGINA=1, @SATELITAL=" + Satelital)

	defer rows.Close()

	if err != nil {
		GeraLog(err.Error())
		return
	}

	if rows.Next() {
		columns, err := rows.ColumnTypes()
		if err != nil {
			GeraLog(err.Error())
			return
		}

		values := make([]interface{}, len(columns))
		object := map[string]interface{}{}

		for i, column := range columns {
			object[column.Name()] = reflect.New(column.ScanType()).Interface()
			values[i] = object[column.Name()]
		}

		err = rows.Scan(values...)
		if err != nil {
			GeraLog(err.Error())
			return
		}

		JsonUltimaPosicao.Status.Cod = "1"
		JsonUltimaPosicao.Status.Msg = "Sucesso."
		JsonUltimaPosicao.Info = object
	} else {
		JsonUltimaPosicao.Status.Cod = "-3"
		JsonUltimaPosicao.Status.Msg = "Nenhuma posicao ainda registrada para este alvo, ou o alvo informado e invalido."
	}

	json.NewEncoder(w).Encode(&JsonUltimaPosicao)

This is the output json;

{"STATUS":{"COD":"1","MSG":"Sucesso."},"INFO":{"ALTITUDE":"MDAwMA==","ANTIFURTO":"MA==","APELIDO":null,"BATERIACARREGANDO":"MA==","BLOQUEADO":"MA==","DATAHORAEXIB":"MzAvMTEvMjAxNyAwNDoyOTo1MQ==","DATAHORAGPS":"2017-11-30T04:30:00-02:00","DATAHORAORIGINAL":"2017-11-30T04:29:51-02:00","DATAHORASERVIDOR":"2017-11-30T04:29:46-02:00","DIRECAO":"Tg==","ENTRADA1LABEL":"RU5UUkFEQSAx","ENTRADA1SHOW":1,"ENTRADA1VALOFF":"T0ZG","ENTRADA1VALON":"T04=","ENTRADA2LABEL":"RU5UUkFEQSAy","ENTRADA2SHOW":1,"ENTRADA2VALOFF":"T0ZG","ENTRADA2VALON":"T04=","ENTRADA3LABEL":"RU5UUkFEQSAz","ENTRADA3SHOW":1,"ENTRADA3VALOFF":"T0ZG","ENTRADA3VALON":"T04=","ENTRADA4LABEL":"RU5UUkFEQSA0","ENTRADA4SHOW":1,"ENTRADA4VALOFF":"T0ZG","ENTRADA4VALON":"T04=","ENTRADAS1":"MA==","ENTRADAS2":"MA==","ENTRADAS3":"MA==","ENTRADAS4":"MA==","FALHAALIMENTACAOPRINCIPAL":"MA==","FALHABATERIAINTERNA":"MA==","HODOMETRO":7223162,"HORIMETRO":11633,"HORIMETROFORMATADO":"MTkzOjUzOjAw","ID":12910322,"IDCLIENTE":30,"IDENTIFICACAOVEICULO":"SExPNzkxOCAtIEZJQVQgRE9CTE8gQ0FSR08gMS40IC0gRkFOVEFTSUE=","IDMODULO":102,"IDMOTORISTA":0,"IDVEICULO":98,"IGNICAO":"MA==","INDICEPOSICAOMXT":"MA==","LATITUDE":"LTIzLjk2ODQ1MQ==","LONGITUDE":"LTQ2LjM1MTgyNQ==","MODELO":"U1RUM1hY","MOTORISTA":null,"MOTORLIGADO":"MA==","MOVENDO":"Mg==","NIVELSINALGSM":"LTE=","NUMSATELITES":"MDA=","PANICO":"MA==","PLACA":"SExPNzkxOA==","RPM":0,"SAIDA1HABILITAR":0,"SAIDA1LABEL":"U0FJREEgMQ==","SAIDA1SHOW":1,"SAIDA1VALOFF":"T0ZG","SAIDA1VALON":"T04=","SAIDA2HABILITAR":0,"SAIDA2LABEL":"U0FJREEgMg==","SAIDA2SHOW":1,"SAIDA2VALOFF":"T0ZG","SAIDA2VALON":"T04=","SAIDA3HABILITAR":1,"SAIDA3LABEL":"U0FJREEgMw==","SAIDA3SHOW":1,"SAIDA3VALOFF":"T0ZG","SAIDA3VALON":"T04=","SAIDA4HABILITAR":1,"SAIDA4LABEL":"U0FJREEgNA==","SAIDA4SHOW":1,"SAIDA4VALOFF":"T0ZG","SAIDA4VALON":"T04=","SAIDAS1":"MA==","SAIDAS2":"MA==","SAIDAS3":"MA==","SAIDAS4":"MA==","SATELITAL":0,"SERIEMODULO":"OTA3MDc2MTUx","SERIESATELITAL":null,"STATUSGPS":"MQ==","STATUSPOSICAO":"Mg==","TENSAO":"MTIuNzk=","TIPOMSG":"NA==","TIPORASTREADOR":"VkVJQ1VMQVI=","TIPOVEICULO":6,"TIPOVEICULODESCRICAO":"VkFOL0ZPUkdBTw==","VELOCIDADE":"MA==","VELOCIDADEMAXIMAEXCEDIDA":"MA=="}}

Could anyone help me resolving that?

Tnx.

What are the values of the strings if you run the stored procedure in the shell/management tools directly on the database?

1 Like

encoding/json encodes []byte (aka []unit8) as a base64-encoded string (source).

I was able to replicate this (and then fix it) in my example program by changing field2 in table test to a blob and then changing the type scanned into.

package main

import (
	"database/sql"
	"encoding/json"
	"log"
	"os"
	"reflect"

	_ "github.com/mattn/go-sqlite3"
)

func main() {
	log.SetFlags(log.Lshortfile)

	db := initDb()

	b, err := queryToJson(db, "select * from test")
	if err != nil {
		log.Fatalln(err)
	}
	os.Stdout.Write(b)

}

func queryToJson(db *sql.DB, query string, args ...interface{}) ([]byte, error) {
	// an array of JSON objects
	// the map key is the field name
	var objects []map[string]interface{}

	rows, err := db.Query(query, args...)
	if err != nil {
		return nil, err
	}
	for rows.Next() {
		// figure out what columns were returned
		// the column names will be the JSON object field keys
		columns, err := rows.ColumnTypes()
		if err != nil {
			return nil, err
		}

		// Scan needs an array of pointers to the values it is setting
		// This creates the object and sets the values correctly
		values := make([]interface{}, len(columns))
		object := map[string]interface{}{}
		for i, column := range columns {
			v := reflect.New(column.ScanType()).Interface()
			switch v.(type) {
			case *[]uint8:
				v = new(string)
			default:
				// use this to find the type for the field
				// you need to change
				// log.Printf("%v: %T", column.Name(), v)
			}

			object[column.Name()] = v
			values[i] = object[column.Name()]
		}

		err = rows.Scan(values...)
		if err != nil {
			return nil, err
		}

		objects = append(objects, object)
	}

	// indent because I want to read the output
	return json.MarshalIndent(objects, "", "\t")
}

func initDb() *sql.DB {
	db, err := sql.Open("sqlite3", ":memory:")
	if err != nil {
		log.Fatalln(err)
	}

	_, err = db.Exec(`
create table test (
	id integer primary key,
	field1 integer,
	field2 blob
);`)
	if err != nil {
		log.Fatalln(err)
	}

	_, err = db.Exec(`
insert into test (field1, field2) values
(1, ?),
(2, ?)
`, []byte("first"), []byte("second"))
	if err != nil {
		log.Fatalln(err)
	}

	return db
}

Output is the same as before.

After thinking a bit more, it is probably better to choose v based on column.DatabaseTypeName to differentiate between varchar and blobs (for example).

Updated:

  • removed direct dependency on reflect
  • switch on DatabaseTypeName to differentiate between text and blobs (base64 encoding blobs makes sense when encoding to JSON)
package main

import (
	"database/sql"
	"encoding/json"
	"log"
	"os"

	_ "github.com/mattn/go-sqlite3"
)

func main() {
	log.SetFlags(log.Lshortfile)

	db := initDb()

	b, err := queryToJson(db, "select * from test")
	if err != nil {
		log.Fatalln(err)
	}
	os.Stdout.Write(b)

}

func queryToJson(db *sql.DB, query string, args ...interface{}) ([]byte, error) {
	// an array of JSON objects
	// the map key is the field name
	var objects []map[string]interface{}

	rows, err := db.Query(query, args...)
	if err != nil {
		return nil, err
	}
	for rows.Next() {
		// figure out what columns were returned
		// the column names will be the JSON object field keys
		columns, err := rows.ColumnTypes()
		if err != nil {
			return nil, err
		}

		// Scan needs an array of pointers to the values it is setting
		// This creates the object and sets the values correctly
		values := make([]interface{}, len(columns))
		object := map[string]interface{}{}
		for i, column := range columns {
			var v interface{}

			switch column.DatabaseTypeName() {
			case "text":
				v = new(string)
			default:
				v = new(interface{}) // destination must be a pointer

				// use to figure out types for columns
				// log.Println(column.Name(), column.DatabaseTypeName())
			}

			object[column.Name()] = v
			values[i] = v
		}

		err = rows.Scan(values...)
		if err != nil {
			return nil, err
		}

		// use to see what is produced
		// log.Printf("%#v", object)

		objects = append(objects, object)
	}

	// indent because I want to read the output
	return json.MarshalIndent(objects, "", "\t")
}

func initDb() *sql.DB {
	db, err := sql.Open("sqlite3", ":memory:")
	if err != nil {
		log.Fatalln(err)
	}

	_, err = db.Exec(`
create table test (
	id integer primary key,
	field1 integer,
	field2 text,
	field3 blob
);`)
	if err != nil {
		log.Fatalln(err)
	}

	_, err = db.Exec(`
insert into test (field1, field2, field3) values
(1, ?, ?),
(2, ?, ?)
`, "first", "blob1", []byte("second"), "blob2")
	if err != nil {
		log.Fatalln(err)
	}

	return db
}

outputs:

$ go run main.go
[
	{
		"field1": 1,
		"field2": "first",
		"field3": "YmxvYjE=",
		"id": 1
	},
	{
		"field1": 2,
		"field2": "second",
		"field3": "YmxvYjI=",
		"id": 2
	}
]

Still not working, still showing the same encrypted data, now has completed the 3rd day I’m on here trying to resolve this problem.

Is your go 1.9?

1 Like

Yes 1.9

Hey there, myh code is that below now;

package main

import (
	//"database/sql"
	"encoding/json"
	"net/http"
	//	"reflect"
	"strings"
)

func ultimaposicao(w http.ResponseWriter, r *http.Request) {

	urlqry := r.URL.Query()

	JsonUltimaPosicao := ObjUltimaPosicao{}

	//faz a validação da chave do rest
	if !RestVerificaChave(urlqry.Get("chave")) {
		JsonUltimaPosicao.Status.Cod = "-2"
		JsonUltimaPosicao.Status.Msg = "Falha durante tentativa de autenticacao no REST! Credenciais invalidas."
		json.NewEncoder(w).Encode(&JsonUltimaPosicao)
		return
	}
	//faz a validação da chave do rest

	Satelital := "0" //PADRAO GPRS, ADAPTACAO PARA VERSOES ANTIGAS DO APLICATIVO
	if strings.TrimSpace(urlqry.Get("satelital")) != "" {
		Satelital = urlqry.Get("satelital")
	}

	rows, err := db.Query("EXEC PRCATUALIZACOES @IDVEICULO=" + urlqry.Get("idveiculo") + ", @TAMANHOPAGINA=1, @NUMEROPAGINA=1, @SATELITAL=" + Satelital)

	defer rows.Close()

	if err != nil {
		GeraLog(err.Error())
		return
	}

	if rows.Next() {
		columns, err := rows.ColumnTypes()
		if err != nil {
			GeraLog(err.Error())
			return
		}

		values := make([]interface{}, len(columns))
		object := map[string]interface{}{}

		for i, column := range columns {
			//object[column.Name()] = reflect.New(column.ScanType()).Interface()
			//values[i] = object[column.Name()]
			object[column.Name()] = new(GenericScanner)
			values[i] = object[column.Name()]
		}

		err = rows.Scan(values...)
		if err != nil {
			GeraLog(err.Error())
			return
		}

		JsonUltimaPosicao.Status.Cod = "1"
		JsonUltimaPosicao.Status.Msg = "Sucesso."
		JsonUltimaPosicao.Info = object
	} else {
		JsonUltimaPosicao.Status.Cod = "-3"
		JsonUltimaPosicao.Status.Msg = "Nenhuma posicao ainda registrada para este alvo, ou o alvo informado e invalido."
	}

	json.NewEncoder(w).Encode(&JsonUltimaPosicao)
}

The custom scanner code;

package main

import "time"

type GenericScanner struct {
	valid bool
	value interface{}
}

func (scanner *GenericScanner) getBytes(src interface{}) []byte {
	if a, ok := src.([]uint8); ok {
		return a
	}
	return nil
}

func (scanner *GenericScanner) Scan(src interface{}) error {
	switch src.(type) {
	case int64:
		if value, ok := src.(int64); ok {
			scanner.value = value
			scanner.valid = true
		}
	case float64:
		if value, ok := src.(float64); ok {
			scanner.value = value
			scanner.valid = true
		}
	case bool:
		if value, ok := src.(bool); ok {
			scanner.value = value
			scanner.valid = true
		}
	case string:
		value := scanner.getBytes(src)
		scanner.value = string(value)
		scanner.valid = true
	case []byte:
		value := scanner.getBytes(src)
		scanner.value = value
		scanner.valid = true
	case time.Time:
		if value, ok := src.(time.Time); ok {
			scanner.value = value
			scanner.valid = true
		}
	case nil:
		scanner.value = nil
		scanner.valid = true
	}
	return nil
}

I’ve found on my searchs I gotta implement a custom scanner, I did like above and my json now is worst than before, below you have it;

{"STATUS":{"COD":"1","MSG":"Sucesso."},"INFO":{"ALTITUDE":{},"ANTIFURTO":{},"APELIDO":{},"BATERIACARREGANDO":{},"BLOQUEADO":{},"DATAHORAEXIB":{},"DATAHORAGPS":{},"DATAHORAORIGINAL":{},"DATAHORASERVIDOR":{},"DIRECAO":{},"ENTRADA1LABEL":{},"ENTRADA1SHOW":{},"ENTRADA1VALOFF":{},"ENTRADA1VALON":{},"ENTRADA2LABEL":{},"ENTRADA2SHOW":{},"ENTRADA2VALOFF":{},"ENTRADA2VALON":{},"ENTRADA3LABEL":{},"ENTRADA3SHOW":{},"ENTRADA3VALOFF":{},"ENTRADA3VALON":{},"ENTRADA4LABEL":{},"ENTRADA4SHOW":{},"ENTRADA4VALOFF":{},"ENTRADA4VALON":{},"ENTRADAS1":{},"ENTRADAS2":{},"ENTRADAS3":{},"ENTRADAS4":{},"FALHAALIMENTACAOPRINCIPAL":{},"FALHABATERIAINTERNA":{},"HODOMETRO":{},"HORIMETRO":{},"HORIMETROFORMATADO":{},"ID":{},"IDCLIENTE":{},"IDENTIFICACAOVEICULO":{},"IDMODULO":{},"IDMOTORISTA":{},"IDVEICULO":{},"IGNICAO":{},"INDICEPOSICAOMXT":{},"LATITUDE":{},"LONGITUDE":{},"MODELO":{},"MOTORISTA":{},"MOTORLIGADO":{},"MOVENDO":{},"NIVELSINALGSM":{},"NUMSATELITES":{},"PANICO":{},"PLACA":{},"RPM":{},"SAIDA1HABILITAR":{},"SAIDA1LABEL":{},"SAIDA1SHOW":{},"SAIDA1VALOFF":{},"SAIDA1VALON":{},"SAIDA2HABILITAR":{},"SAIDA2LABEL":{},"SAIDA2SHOW":{},"SAIDA2VALOFF":{},"SAIDA2VALON":{},"SAIDA3HABILITAR":{},"SAIDA3LABEL":{},"SAIDA3SHOW":{},"SAIDA3VALOFF":{},"SAIDA3VALON":{},"SAIDA4HABILITAR":{},"SAIDA4LABEL":{},"SAIDA4SHOW":{},"SAIDA4VALOFF":{},"SAIDA4VALON":{},"SAIDAS1":{},"SAIDAS2":{},"SAIDAS3":{},"SAIDAS4":{},"SATELITAL":{},"SERIEMODULO":{},"SERIESATELITAL":{},"STATUSGPS":{},"STATUSPOSICAO":{},"TENSAO":{},"TIPOMSG":{},"TIPORASTREADOR":{},"TIPOVEICULO":{},"TIPOVEICULODESCRICAO":{},"VELOCIDADE":{},"VELOCIDADEMAXIMAEXCEDIDA":{}}}

I would really appreciate if anyone could help me with that, I just searched whole Google, but nothing.

Tnx.

Pieter, sorry my friend, I haven’t seen your reply, sorry… The values are normal, nothing encrypted, I use the same procedure in other modules of the project using other languages such as Java and PHP, and I get the values normally without encryptation…

Below goes one more try, at 5:12 in the morning of the 4th day;

package main

import (
	"database/sql"
	"encoding/json"
	"net/http"
	//	"reflect"
	"strings"
)

func ultimaposicao(w http.ResponseWriter, r *http.Request) {

	urlqry := r.URL.Query()

	JsonUltimaPosicao := ObjUltimaPosicao{}

	//faz a validação da chave do rest
	if !RestVerificaChave(urlqry.Get("chave")) {
		JsonUltimaPosicao.Status.Cod = "-2"
		JsonUltimaPosicao.Status.Msg = "Falha durante tentativa de autenticacao no REST! Credenciais invalidas."
		json.NewEncoder(w).Encode(&JsonUltimaPosicao)
		return
	}
	//faz a validação da chave do rest

	Satelital := "0" //PADRAO GPRS, ADAPTACAO PARA VERSOES ANTIGAS DO APLICATIVO
	if strings.TrimSpace(urlqry.Get("satelital")) != "" {
		Satelital = urlqry.Get("satelital")
	}

	rows, err := db.Query("EXEC PRCATUALIZACOES @IDVEICULO=" + urlqry.Get("idveiculo") + ", @TAMANHOPAGINA=1, @NUMEROPAGINA=1, @SATELITAL=" + Satelital)

	defer rows.Close()

	if err != nil {
		GeraLog(err.Error())
		return
	}

	if rows.Next() {
		columns, err := rows.ColumnTypes()
		if err != nil {
			GeraLog(err.Error())
			return
		}

		values := make([]interface{}, len(columns))
		object := map[string]interface{}{}

		for i, column := range columns {
			//object[column.Name()] = reflect.New(column.ScanType()).Interface()
			//values[i] = object[column.Name()]
			object[column.Name()] = new(sql.RawBytes)
			values[i] = object[column.Name()]
		}

		err = rows.Scan(values...)
		if err != nil {
			GeraLog(err.Error())
			return
		}

		JsonUltimaPosicao.Status.Cod = "1"
		JsonUltimaPosicao.Status.Msg = "Sucesso."
		JsonUltimaPosicao.Info = object
	} else {
		JsonUltimaPosicao.Status.Cod = "-3"
		JsonUltimaPosicao.Status.Msg = "Nenhuma posicao ainda registrada para este alvo, ou o alvo informado e invalido."
	}

	json.NewEncoder(w).Encode(&JsonUltimaPosicao)
}

According to the following tutorial, if I don’t know the columns types, I can use sql.RawBytes;
http://go-database-sql.org/varcols.html

I did it but I get the following exception;
Scan error on column index 5: unsupported Scan, storing driver.Value type time.Time into type *sql.RawBytes

Nobody has done this before??? Nobody to help…?

You’re scanning into sql.RawBytes. This is essentially a byte slice, and the JSON marshaller uses base64 for byte slices. Keep in mind that RawBytes will be things like eight separate bytes for a 64 bit integer, etc. There is no better way to represent that in JSON that base64.

If you want interpreted data you will need to use real types. Nathan has an example above using a switch on the column type to chose an appropriate type. You should try to read and understand that example, and apply it on your own code.

2 Likes

You mean that switch using column.DatabaseTypeName() ? That doesn’t work here for me, I even tried to print column.DatabaseTypeName() and it shows nothing, it shows up null for me, empty with no string…

You could investigate why that’s the case, check if your database and driver supports it (I’m not sure if you’ve mentioned what you use), or look for other methods to figure it out.

1 Like

Hey there, I did it! I did it using string pointer like below;

		for i, column := range columns {
			//object[column.Name()] = reflect.New(column.ScanType()).Interface()
			object[column.Name()] = new(*string)
			values[i] = object[column.Name()]
		}

But not any good, because I wanna respect datatypes such as int, and the way I did, every value(even int) is shown between quotes.

Isn’t really there a way to get field datatypes using GO + SQL Server 2008 + ODBC? Without the data types I can’t make a type assertion…