Commit 4b576c23 authored by Erick Hitter's avatar Erick Hitter
Browse files

Like event runners, leave event retreiver workers active, looping only to...

Like event runners, leave event retreiver workers active, looping only to retreive list of sites to check

Sites list is dynamic, need to retreive sites' events is not.
parent 3193fa7d
......@@ -63,43 +63,47 @@ func init() {
func main() {
logger.Println("Starting")
sites := make(chan Site)
events := make(chan Event)
go spawnEventRetrievers(events)
go spawnEventRetrievers(sites, events)
go spawnEventWorkers(events)
go retrieveSitesPeriodically(sites)
keepAlive()
logger.Println("Stopping")
}
func spawnEventRetrievers(queue chan Event) {
func spawnEventRetrievers(sites <-chan Site, queue chan<- Event) {
logger.Println("Spawning event-retrieval workers")
for w := 1; w <= workersGetEvents; w++ {
go queueSiteEvents(w, sites, queue)
}
logger.Println("Event-retrieval workers spawned")
}
func retrieveSitesPeriodically(sites chan<- Site) {
for true {
logger.Println("Spawning event-retrieval workers")
logger.Println("Retrieving sites")
sites, err := getSites()
siteList, err := getSites()
if err != nil {
time.Sleep(getEventsLoop)
continue
}
workerSites := make(chan string, len(sites))
for w := 1; w <= workersGetEvents; w++ {
go queueSiteEvents(w, workerSites, queue)
}
for _, site := range sites {
workerSites <- site.Url
for _, site := range siteList {
sites <- site
}
close(workerSites)
logger.Printf("Event-retrieval workers spawned, sleeping %d seconds", getEventsLoop / 1000 / 1000 / 1000)
logger.Printf("Sites sent to event retrievers, next site retreival in %d seconds", getEventsLoop / 1000 / 1000 / 1000)
time.Sleep(getEventsLoop)
}
}
func spawnEventWorkers(queue chan Event) {
func spawnEventWorkers(queue <-chan Event) {
logger.Println("Spawning event workers")
workerEvents := make(chan Event)
......@@ -187,18 +191,18 @@ func getMultisiteSites() ([]Site, error) {
return jsonRes, nil
}
func queueSiteEvents(workerId int, sites <-chan string, queue chan<- Event) {
func queueSiteEvents(workerId int, sites <-chan Site, queue chan<- Event) {
for site := range sites {
logger.Printf("getEvents-%d processing %s", workerId, site)
logger.Printf("getEvents-%d processing %s", workerId, site.Url)
siteEvents, err := getSiteEvents(site)
siteEvents, err := getSiteEvents(site.Url)
if err != nil {
time.Sleep(getEventsBreak)
continue
}
for _, event := range siteEvents {
event.Url = site
event.Url = site.Url
queue <- event
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment