diff --git a/goSpider.go b/goSpider.go index c5480f7..f4c71f8 100644 --- a/goSpider.go +++ b/goSpider.go @@ -578,11 +578,13 @@ func ParallelRequests(requests []Requests, numberOfWorkers int, duration time.Du var wg sync.WaitGroup resultCh := make(chan ResponseBody, len(requests)) // Buffered channel to hold all results + k := 0 for i := 0; i < numberOfWorkers; i++ { wg.Add(1) go func() { defer wg.Done() for input := range inputCh { + k++ time.Sleep(duration) cover, movements, people, err := crawlerFunc(input.ProcessNumber) resultCh <- ResponseBody{ @@ -595,6 +597,9 @@ func ParallelRequests(requests []Requests, numberOfWorkers int, duration time.Du log.Println(err) continue } + if k == len(requests)-1 { + break + } } }() } @@ -616,6 +621,10 @@ func ParallelRequests(requests []Requests, numberOfWorkers int, duration time.Du results = append(results, result) } + if k == len(requests)-1 { + return results, errorOnApiRequests + } + return results, errorOnApiRequests } diff --git a/goSpider_test.go b/goSpider_test.go index 9fee2c1..75abd72 100644 --- a/goSpider_test.go +++ b/goSpider_test.go @@ -1,6 +1,7 @@ package goSpider import ( + "log" "net/http" "os" "testing" @@ -153,3 +154,127 @@ func TestGetCurrentURL(t *testing.T) { t.Errorf("Expected URL: %s, but got: %s", expectedURL, currentURL) } } + +func TestParallelRequests(t *testing.T) { + users := []Requests{ + {ProcessNumber: "1017927-35.2023.8.26.0008"}, + {ProcessNumber: "0002396-75.2013.8.26.0201"}, + {ProcessNumber: "1551285-50.2021.8.26.0477"}, + {ProcessNumber: "0015386-82.2013.8.26.0562"}, + {ProcessNumber: "0007324-95.2015.8.26.0590"}, + {ProcessNumber: "1545639-85.2023.8.26.0090"}, + {ProcessNumber: "1557599-09.2021.8.26.0090"}, + {ProcessNumber: "1045142-72.2021.8.26.0002"}, + {ProcessNumber: "0208591-43.2009.8.26.0004"}, + {ProcessNumber: "1017927-35.2023.8.26.0008"}, + } + + numberOfWorkers := 3 + duration := 2 * time.Second + + results, err := ParallelRequests(users, numberOfWorkers, duration, Crawler) + if err != nil { + log.Printf("GetCurrentURL error: %v", err) + } + + log.Println("Finish Parallel Requests!") + + var found []string + for _, u := range users { + for _, result := range results { + for _, value := range result.Cover { + if value == u.ProcessNumber { + found = append(found, value) + } + } + } + } + + if len(found) != len(users) { + t.Errorf("Expected %d results, but got %d, List results: %v", len(users), len(found), found) + } + +} + +func Crawler(d string) (map[string]string, []map[int]map[string]interface{}, []map[int]map[string]interface{}, error) { + log.Printf("Crawling: %s", d) + + url := "https://esaj.tjsp.jus.br/sajcas/login" + nav := NewNavigator() + defer nav.Close() + + err := nav.OpenURL(url) + if err != nil { + log.Printf("OpenURL error: %v", err) + return nil, nil, nil, err + } + + usernameSelector := "#usernameForm" + passwordSelector := "#passwordForm" + username := "363.400.878-41" + password := "Remoto123*" + loginButtonSelector := "#pbEntrar" + messageFailedSuccess := "#mensagemRetorno > li" + + err = nav.Login(url, username, password, usernameSelector, passwordSelector, loginButtonSelector, messageFailedSuccess) + if err != nil { + log.Printf("Login error: %v", err) + return nil, nil, nil, err + } + + err = nav.ClickButton("#esajConteudoHome > table:nth-child(7) > tbody > tr > td.esajCelulaDescricaoServicos > a") + if err != nil { + log.Printf("ClickButton error: %v", err) + return nil, nil, nil, err + } + + err = nav.ClickButton("#esajConteudoHome > table:nth-child(3) > tbody > tr > td.esajCelulaDescricaoServicos > a") + if err != nil { + log.Printf("ClickButton error: %v", err) + return nil, nil, nil, err + } + + err = nav.CheckRadioButton("#interna_NUMPROC > div > fieldset > label:nth-child(5)") + if err != nil { + log.Printf("CheckRadioButton error: %v", err) + return nil, nil, nil, err + } + + err = nav.FillField("#nuProcessoAntigoFormatado", d) + if err != nil { + log.Printf("filling field error: %v", err) + return nil, nil, nil, err + } + + err = nav.ClickButton("#botaoConsultarProcessos") + if err != nil { + log.Printf("ClickButton error: %v", err) + return nil, nil, nil, err + } + + err = nav.ClickElement("#linkmovimentacoes") + if err != nil { + log.Printf("ClickElement error: %v", err) + return nil, nil, nil, err + } + + people, err := nav.ExtractTableData("#tablePartesPrincipais") + if err != nil { + log.Printf("ExtractTableData error: %v", err) + return nil, nil, nil, err + } + + movements, err := nav.ExtractTableData("#tabelaTodasMovimentacoes") + if err != nil { + log.Printf("ExtractTableData error: %v", err) + return nil, nil, nil, err + } + + cover, err := nav.ExtractDivText("#containerDadosPrincipaisProcesso", "#maisDetalhes") + if err != nil { + log.Printf("ExtractDivText error: %v", err) + return nil, nil, nil, err + } + + return cover, movements, people, nil +}