Hi folks, just wanted to understand if you guys build any datapipelines using golang ?
hi, here’s the example code for building a data pipeline using Golang
package main
import (
“fmt”
“log”
“time”
)
// Service for data collection
func collectData(input chan<- string) {
for i := 0; i < 10; i++ {
// Collect and send log data
data := fmt.Sprintf(“Log data %d”, i)
input ← data
time.Sleep(500 * time.Millisecond) // Arbitrary delay
}
close(input)
}
// Worker for data preprocessing
func preprocessData(input <-chan string, output chan<- string) {
for data := range input {
// Perform data preprocessing
preprocessedData := fmt.Sprintf(“Preprocessed: %s”, data)
// Send preprocessed data to the next stage
output <- preprocessedData
}
close(output)
}
// Service for data storage
func saveData(input <-chan string) {
for data := range input {
// Perform data storage operation
fmt.Printf(“Saving data: %s\n”, data)
}
// Data storage complete
}
// Service for displaying a dashboard
func displayDashboard(input <-chan string) {
for data := range input {
// Perform dashboard display operation
fmt.Printf(“Displaying data: %s\n”, data)
}
}
func main() {
// Create channels
dataChannel := make(chan string)
preprocessedDataChannel := make(chan string)
// Data collection
go collectData(dataChannel)
// Data preprocessing
go preprocessData(dataChannel, preprocessedDataChannel)
// Data storage
go saveData(preprocessedDataChannel)
// Display dashboard
displayDashboard(preprocessedDataChannel)
// Wait for a while to observe the results
time.Sleep(2 * time.Second)
}