I’m trying to make the web crawler exercise of Go tour. I know that there are these solutions https://gist.github.com/zyxar/2317744 but I’m trying to do it on my own way and I would like to know why these erros are happening
I tried to use WaitGroup and I continued getting errors, its commented how I used WaitGroup, also Mutex.
Why in the function Fetch I can’t send the value of i to the urls channel ?
Sorry in case of bad english.
[code]package main
import (
“fmt”
//“sync”
//“time”
)
type Fetcher interface {
// Fetch returns the body of URL and
// a slice of URLs found on that page.
Fetch(url string, body, urls chan<- string, err chan<- error, id int /,wg sync.WaitGroup/)
}
// Crawl uses fetcher to recursively crawl
// pages starting with url, to a maximum of depth.
func Crawl(url string, depth int, fetcher Fetcher, body, urls chan string, err chan error, id int/, wg sync.WaitGroup/) {
// TODO: Fetch URLs in parallel.
// TODO: Don’t fetch the same URL twice.
// This implementation doesn’t do either:
if depth <= 0 {
return
}
*id++
go fetcher.Fetch(url, body, urls, err, id/*, wg*/)
//time.Sleep(time.Millisecond)
//wg.Wait()
//I'm getting an error here because err never receive a value
if a := <-err; a != nil {
fmt.Println("test")
fmt.Println(err)
return
}
fmt.Printf("found: %s %q\n", url, body)
for u := range urls {
fmt.Println("Crawl")
Crawl(u, depth-1, fetcher, body, urls, err, id/*, wg*/)
}
return
}
func main() {
body := make(chan string)
urls := make(chan string)
err := make(chan error)
//var mutex sync.Mutex
//var wg sync.WaitGroup
id := 0
Crawl("http://golang.org/", 4, fetcher, body, urls, err, &id/*, &wg*/)
}
// fakeFetcher is Fetcher that returns canned results.
type fakeFetcher map[string]*fakeResult
type fakeResult struct {
body string
urls []string
}
func (f fakeFetcher) Fetch(url string, body, urls chan<- string, err chan<- error, id int/, wg sync.WaitGroup/){
//mutex.Lock()
//wg.Add(1)
if res, ok := f[url]; ok {
for _, i := range res.urls{
fmt.Printf("Check, id: %d \n", *id)
fmt.Printf("%T \n", i)
urls <- i //I can't send this value here for some reason
}
body <- res.body
err <- nil
} else{
body <- ""
urls <- ""
err <- fmt.Errorf("not found: %s", url)
}
//defer wg.Done()
//mutex.Unlock()
}
// fetcher is a populated fakeFetcher.
var fetcher = fakeFetcher{
“http://golang.org/”: &fakeResult{
“The Go Programming Language”,
[]string{
“http://golang.org/pkg/”,
“http://golang.org/cmd/”,
},
},
“http://golang.org/pkg/”: &fakeResult{
“Packages”,
[]string{
“http://golang.org/”,
“http://golang.org/cmd/”,
“http://golang.org/pkg/fmt/”,
“http://golang.org/pkg/os/”,
},
},
“http://golang.org/pkg/fmt/”: &fakeResult{
“Package fmt”,
[]string{
“http://golang.org/”,
“http://golang.org/pkg/”,
},
},
“http://golang.org/pkg/os/”: &fakeResult{
“Package os”,
[]string{
“http://golang.org/”,
“http://golang.org/pkg/”,
},
},
} [/code]
Output:
[code]Check, id: 1
string
fatal error: all goroutines are asleep - deadlock!
goroutine 1 [chan receive]:
main.Crawl(0x4b5615, 0x12, 0x4, 0x508340, 0xc04205c030, 0xc04200e300, 0xc04200e360, 0xc04200e3c0, 0xc04200a270)
C:/Users/kalem/Desktop/codigos/Go/webCrawlerTour.go:32 +0x117
main.main()
C:/Users/kalem/Desktop/codigos/Go/webCrawlerTour.go:56 +0x10a
goroutine 5 [chan send]:
main.fakeFetcher.Fetch(0xc04205c030, 0x4b5615, 0x12, 0xc04200e300, 0xc04200e360, 0xc04200e3c0, 0xc04200a270)
C:/Users/kalem/Desktop/codigos/Go/webCrawlerTour.go:77 +0x251
created by main.Crawl
C:/Users/kalem/Desktop/codigos/Go/webCrawlerTour.go:27 +0xd5[/code]