Merge pull request #11 from DanielFillol/daniel

ParallelRequests & TestParallelRequests

goSpider.go

@@ -578,11 +578,13 @@ func ParallelRequests(requests []Requests, numberOfWorkers int, duration time.Du
 	var wg sync.WaitGroup
 	resultCh := make(chan ResponseBody, len(requests)) // Buffered channel to hold all results
 
+	k := 0
 	for i := 0; i < numberOfWorkers; i++ {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
 			for input := range inputCh {
+				k++
 				time.Sleep(duration)
 				cover, movements, people, err := crawlerFunc(input.ProcessNumber)
 				resultCh <- ResponseBody{
@@ -595,6 +597,9 @@ func ParallelRequests(requests []Requests, numberOfWorkers int, duration time.Du
 					log.Println(err)
 					continue
 				}
+				if k == len(requests)-1 {
+					break
+				}
 			}
 		}()
 	}
@@ -616,6 +621,10 @@ func ParallelRequests(requests []Requests, numberOfWorkers int, duration time.Du
 		results = append(results, result)
 	}
 
+	if k == len(requests)-1 {
+		return results, errorOnApiRequests
+	}
+
 	return results, errorOnApiRequests
 }
 

goSpider_test.go

@@ -1,6 +1,7 @@
 package goSpider
 
 import (
+	"log"
 	"net/http"
 	"os"
 	"testing"
@@ -153,3 +154,127 @@ func TestGetCurrentURL(t *testing.T) {
 		t.Errorf("Expected URL: %s, but got: %s", expectedURL, currentURL)
 	}
 }
+
+func TestParallelRequests(t *testing.T) {
+	users := []Requests{
+		{ProcessNumber: "1017927-35.2023.8.26.0008"},
+		{ProcessNumber: "0002396-75.2013.8.26.0201"},
+		{ProcessNumber: "1551285-50.2021.8.26.0477"},
+		{ProcessNumber: "0015386-82.2013.8.26.0562"},
+		{ProcessNumber: "0007324-95.2015.8.26.0590"},
+		{ProcessNumber: "1545639-85.2023.8.26.0090"},
+		{ProcessNumber: "1557599-09.2021.8.26.0090"},
+		{ProcessNumber: "1045142-72.2021.8.26.0002"},
+		{ProcessNumber: "0208591-43.2009.8.26.0004"},
+		{ProcessNumber: "1017927-35.2023.8.26.0008"},
+	}
+
+	numberOfWorkers := 3
+	duration := 2 * time.Second
+
+	results, err := ParallelRequests(users, numberOfWorkers, duration, Crawler)
+	if err != nil {
+		log.Printf("GetCurrentURL error: %v", err)
+	}
+
+	log.Println("Finish Parallel Requests!")
+
+	var found []string
+	for _, u := range users {
+		for _, result := range results {
+			for _, value := range result.Cover {
+				if value == u.ProcessNumber {
+					found = append(found, value)
+				}
+			}
+		}
+	}
+
+	if len(found) != len(users) {
+		t.Errorf("Expected %d results, but got %d, List results: %v", len(users), len(found), found)
+	}
+
+}
+
+func Crawler(d string) (map[string]string, []map[int]map[string]interface{}, []map[int]map[string]interface{}, error) {
+	log.Printf("Crawling: %s", d)
+
+	url := "https://esaj.tjsp.jus.br/sajcas/login"
+	nav := NewNavigator()
+	defer nav.Close()
+
+	err := nav.OpenURL(url)
+	if err != nil {
+		log.Printf("OpenURL error: %v", err)
+		return nil, nil, nil, err
+	}
+
+	usernameSelector := "#usernameForm"
+	passwordSelector := "#passwordForm"
+	username := "363.400.878-41"
+	password := "Remoto123*"
+	loginButtonSelector := "#pbEntrar"
+	messageFailedSuccess := "#mensagemRetorno > li"
+
+	err = nav.Login(url, username, password, usernameSelector, passwordSelector, loginButtonSelector, messageFailedSuccess)
+	if err != nil {
+		log.Printf("Login error: %v", err)
+		return nil, nil, nil, err
+	}
+
+	err = nav.ClickButton("#esajConteudoHome > table:nth-child(7) > tbody > tr > td.esajCelulaDescricaoServicos > a")
+	if err != nil {
+		log.Printf("ClickButton error: %v", err)
+		return nil, nil, nil, err
+	}
+
+	err = nav.ClickButton("#esajConteudoHome > table:nth-child(3) > tbody > tr > td.esajCelulaDescricaoServicos > a")
+	if err != nil {
+		log.Printf("ClickButton error: %v", err)
+		return nil, nil, nil, err
+	}
+
+	err = nav.CheckRadioButton("#interna_NUMPROC > div > fieldset > label:nth-child(5)")
+	if err != nil {
+		log.Printf("CheckRadioButton error: %v", err)
+		return nil, nil, nil, err
+	}
+
+	err = nav.FillField("#nuProcessoAntigoFormatado", d)
+	if err != nil {
+		log.Printf("filling field error: %v", err)
+		return nil, nil, nil, err
+	}
+
+	err = nav.ClickButton("#botaoConsultarProcessos")
+	if err != nil {
+		log.Printf("ClickButton error: %v", err)
+		return nil, nil, nil, err
+	}
+
+	err = nav.ClickElement("#linkmovimentacoes")
+	if err != nil {
+		log.Printf("ClickElement error: %v", err)
+		return nil, nil, nil, err
+	}
+
+	people, err := nav.ExtractTableData("#tablePartesPrincipais")
+	if err != nil {
+		log.Printf("ExtractTableData error: %v", err)
+		return nil, nil, nil, err
+	}
+
+	movements, err := nav.ExtractTableData("#tabelaTodasMovimentacoes")
+	if err != nil {
+		log.Printf("ExtractTableData error: %v", err)
+		return nil, nil, nil, err
+	}
+
+	cover, err := nav.ExtractDivText("#containerDadosPrincipaisProcesso", "#maisDetalhes")
+	if err != nil {
+		log.Printf("ExtractDivText error: %v", err)
+		return nil, nil, nil, err
+	}
+
+	return cover, movements, people, nil
+}