Make collection retrievals parallel

This makes calls to get the objects from a collection link parallel. Some collections can contain a large number of items to retrieve. Doing this serially can have a noticable performance impact. This change allows three fetches in parallel so we don't overwhelm the Redfish service and cause a denial of service. With three concurrent operations we can be processing some responses while queuing up the more, making it a lot faster to process. Signed-off-by: Sean McGinnis <[email protected]> Co-Authored-by: Lorenzo Bolognesi <[email protected]>
stmcginnis · Nov 26, 2022 · 20391a5 · 20391a5
1 parent 3948add
commit 20391a5
Show file tree

Hide file tree

Showing 66 changed files with 1,511 additions and 517 deletions.
diff --git a/client.go b/client.go
@@ -417,7 +417,7 @@ func (c *APIClient) runRawRequestWithHeaders(method, url string, payloadBuffer i
 		// Set Content-Length custom headers on the request
 		// since its ignored when set using Header.Set()
 		if strings.EqualFold("Content-Length", k) {
-			req.ContentLength, err = strconv.ParseInt(v, 10, 64) //nolint:gomnd // base 10, 64 bit
+			req.ContentLength, err = strconv.ParseInt(v, 10, 64) // base 10, 64 bit
 			if err != nil {
 				return nil, common.ConstructError(0, []byte("error parsing custom Content-Length header"))
 			}

diff --git a/common/collection.go b/common/collection.go
@@ -7,6 +7,7 @@ package common
 import (
 	"encoding/json"
 	"fmt"
+	"sync"
 )
 
 // Collection represents a collection of entity references.
@@ -98,3 +99,35 @@ func (cr *CollectionError) Error() string {
 
 	return fmt.Sprintf("failed to retrieve some items: %s", errorsJSON)
 }
+
+// CollectList will retrieve a collection of entities from the Redfish service.
+func CollectList(get func(string), c Client, link string) error {
+	links, err := GetCollection(c, link)
+	if err != nil {
+		return err
+	}
+
+	CollectCollection(get, c, links.ItemLinks)
+	return nil
+}
+
+// CollectCollection will retrieve a collection of entitied from the Redfish service
+// when you already have the set of individual links in the collection.
+func CollectCollection(get func(string), c Client, links []string) {
+	// Only allow three concurrent requests to avoid overwhelming the service
+	limiter := make(chan struct{}, 3)
+	var wg sync.WaitGroup
+
+	for _, itemLink := range links {
+		wg.Add(1)
+		limiter <- struct{}{}
+
+		go func(itemLink string) {
+			defer wg.Done()
+			get(itemLink)
+			<-limiter
+		}(itemLink)
+	}
+
+	wg.Wait()
+}
diff --git a/common/message.go b/common/message.go
@@ -61,18 +61,32 @@ func ListReferencedMessages(c Client, link string) ([]*Message, error) {
 		return result, nil
 	}
 
-	links, err := GetCollection(c, link)
-	if err != nil {
-		return result, err
+	type GetResult struct {
+		Item  *Message
+		Link  string
+		Error error
 	}
 
+	ch := make(chan GetResult)
 	collectionError := NewCollectionError()
-	for _, messageLink := range links.ItemLinks {
-		message, err := GetMessage(c, messageLink)
+	get := func(link string) {
+		message, err := GetMessage(c, link)
+		ch <- GetResult{Item: message, Link: link, Error: err}
+	}
+
+	go func() {
+		err := CollectList(get, c, link)
 		if err != nil {
-			collectionError.Failures[messageLink] = err
+			collectionError.Failures[link] = err
+		}
+		close(ch)
+	}()
+
+	for r := range ch {
+		if r.Error != nil {
+			collectionError.Failures[r.Link] = r.Error
 		} else {
-			result = append(result, message)
+			result = append(result, r.Item)
 		}
 	}
 

diff --git a/redfish/assembly.go b/redfish/assembly.go
@@ -97,18 +97,32 @@ func ListReferencedAssemblys(c common.Client, link string) ([]*Assembly, error)
 		return result, nil
 	}
 
-	links, err := common.GetCollection(c, link)
-	if err != nil {
-		return result, err
+	type GetResult struct {
+		Item  *Assembly
+		Link  string
+		Error error
 	}
 
+	ch := make(chan GetResult)
 	collectionError := common.NewCollectionError()
-	for _, assemblyLink := range links.ItemLinks {
-		assembly, err := GetAssembly(c, assemblyLink)
+	get := func(link string) {
+		assembly, err := GetAssembly(c, link)
+		ch <- GetResult{Item: assembly, Link: link, Error: err}
+	}
+
+	go func() {
+		err := common.CollectList(get, c, link)
 		if err != nil {
-			collectionError.Failures[assemblyLink] = err
+			collectionError.Failures[link] = err
+		}
+		close(ch)
+	}()
+
+	for r := range ch {
+		if r.Error != nil {
+			collectionError.Failures[r.Link] = r.Error
 		} else {
-			result = append(result, assembly)
+			result = append(result, r.Item)
 		}
 	}
 

diff --git a/redfish/bios.go b/redfish/bios.go
@@ -165,18 +165,32 @@ func ListReferencedBioss(c common.Client, link string) ([]*Bios, error) { //noli
 		return result, nil
 	}
 
-	links, err := common.GetCollection(c, link)
-	if err != nil {
-		return result, err
+	type GetResult struct {
+		Item  *Bios
+		Link  string
+		Error error
 	}
 
+	ch := make(chan GetResult)
 	collectionError := common.NewCollectionError()
-	for _, biosLink := range links.ItemLinks {
-		bios, err := GetBios(c, biosLink)
+	get := func(link string) {
+		bios, err := GetBios(c, link)
+		ch <- GetResult{Item: bios, Link: link, Error: err}
+	}
+
+	go func() {
+		err := common.CollectList(get, c, link)
 		if err != nil {
-			collectionError.Failures[biosLink] = err
+			collectionError.Failures[link] = err
+		}
+		close(ch)
+	}()
+
+	for r := range ch {
+		if r.Error != nil {
+			collectionError.Failures[r.Link] = r.Error
 		} else {
-			result = append(result, bios)
+			result = append(result, r.Item)
 		}
 	}
 

diff --git a/redfish/chassis.go b/redfish/chassis.go
@@ -332,20 +332,38 @@ func GetChassis(c common.Client, uri string) (*Chassis, error) {
 }
 
 // ListReferencedChassis gets the collection of Chassis from a provided reference.
-func ListReferencedChassis(c common.Client, link string) ([]*Chassis, error) {
+func ListReferencedChassis(c common.Client, link string) ([]*Chassis, error) { //nolint:dupl
 	var result []*Chassis
-	links, err := common.GetCollection(c, link)
-	if err != nil {
-		return result, err
+	if link == "" {
+		return result, nil
+	}
+
+	type GetResult struct {
+		Item  *Chassis
+		Link  string
+		Error error
 	}
 
+	ch := make(chan GetResult)
 	collectionError := common.NewCollectionError()
-	for _, chassisLink := range links.ItemLinks {
-		chassis, err := GetChassis(c, chassisLink)
+	get := func(link string) {
+		chassis, err := GetChassis(c, link)
+		ch <- GetResult{Item: chassis, Link: link, Error: err}
+	}
+
+	go func() {
+		err := common.CollectList(get, c, link)
 		if err != nil {
-			collectionError.Failures[chassisLink] = err
+			collectionError.Failures[link] = err
+		}
+		close(ch)
+	}()
+
+	for r := range ch {
+		if r.Error != nil {
+			collectionError.Failures[r.Link] = r.Error
 		} else {
-			result = append(result, chassis)
+			result = append(result, r.Item)
 		}
 	}
 
@@ -367,22 +385,40 @@ func (chassis *Chassis) Drives() ([]*Drive, error) {
 	// In version v1.2.0 of the spec, Drives were added to the Chassis.Links
 	// property. But in v1.14.0 of the spec, Chassis.Drives was added as a
 	// direct property.
+	// TODO: Update this to use the concurrent collection method
+	collectionError := common.NewCollectionError()
 	driveLinks := chassis.linkedDrives
 	if chassis.drives != "" {
 		drives, err := common.GetCollection(chassis.Client, chassis.drives)
 		if err != nil {
-			return nil, err
+			collectionError.Failures[chassis.drives] = err
+			return nil, collectionError
 		}
 		driveLinks = drives.ItemLinks
 	}
 
-	collectionError := common.NewCollectionError()
-	for _, driveLink := range driveLinks {
-		drive, err := GetDrive(chassis.Client, driveLink)
-		if err != nil {
-			collectionError.Failures[driveLink] = err
+	type GetResult struct {
+		Item  *Drive
+		Link  string
+		Error error
+	}
+
+	ch := make(chan GetResult)
+	get := func(link string) {
+		drive, err := GetDrive(chassis.Client, link)
+		ch <- GetResult{Item: drive, Link: link, Error: err}
+	}
+
+	go func() {
+		common.CollectCollection(get, chassis.Client, driveLinks)
+		close(ch)
+	}()
+
+	for r := range ch {
+		if r.Error != nil {
+			collectionError.Failures[r.Link] = r.Error
 		} else {
-			result = append(result, drive)
+			result = append(result, r.Item)
 		}
 	}
 

diff --git a/redfish/compositionservice.go b/redfish/compositionservice.go
@@ -113,18 +113,32 @@ func ListReferencedCompositionServices(c common.Client, link string) ([]*Composi
 		return result, nil
 	}
 
-	links, err := common.GetCollection(c, link)
-	if err != nil {
-		return result, err
+	type GetResult struct {
+		Item  *CompositionService
+		Link  string
+		Error error
 	}
 
+	ch := make(chan GetResult)
 	collectionError := common.NewCollectionError()
-	for _, compositionserviceLink := range links.ItemLinks {
-		compositionservice, err := GetCompositionService(c, compositionserviceLink)
+	get := func(link string) {
+		compositionservice, err := GetCompositionService(c, link)
+		ch <- GetResult{Item: compositionservice, Link: link, Error: err}
+	}
+
+	go func() {
+		err := common.CollectList(get, c, link)
 		if err != nil {
-			collectionError.Failures[compositionserviceLink] = err
+			collectionError.Failures[link] = err
+		}
+		close(ch)
+	}()
+
+	for r := range ch {
+		if r.Error != nil {
+			collectionError.Failures[r.Link] = r.Error
 		} else {
-			result = append(result, compositionservice)
+			result = append(result, r.Item)
 		}
 	}
 

diff --git a/redfish/computersystem.go b/redfish/computersystem.go
@@ -675,20 +675,38 @@ func GetComputerSystem(c common.Client, uri string) (*ComputerSystem, error) {
 
 // ListReferencedComputerSystems gets the collection of ComputerSystem from
 // a provided reference.
-func ListReferencedComputerSystems(c common.Client, link string) ([]*ComputerSystem, error) {
+func ListReferencedComputerSystems(c common.Client, link string) ([]*ComputerSystem, error) { //nolint:dupl
 	var result []*ComputerSystem
-	links, err := common.GetCollection(c, link)
-	if err != nil {
-		return result, err
+	if link == "" {
+		return result, nil
+	}
+
+	type GetResult struct {
+		Item  *ComputerSystem
+		Link  string
+		Error error
 	}
 
+	ch := make(chan GetResult)
 	collectionError := common.NewCollectionError()
-	for _, computersystemLink := range links.ItemLinks {
-		computersystem, err := GetComputerSystem(c, computersystemLink)
+	get := func(link string) {
+		computersystem, err := GetComputerSystem(c, link)
+		ch <- GetResult{Item: computersystem, Link: link, Error: err}
+	}
+
+	go func() {
+		err := common.CollectList(get, c, link)
 		if err != nil {
-			collectionError.Failures[computersystemLink] = err
+			collectionError.Failures[link] = err
+		}
+		close(ch)
+	}()
+
+	for r := range ch {
+		if r.Error != nil {
+			collectionError.Failures[r.Link] = r.Error
 		} else {
-			result = append(result, computersystem)
+			result = append(result, r.Item)
 		}
 	}
 
@@ -711,18 +729,37 @@ func (computersystem *ComputerSystem) Bios() (*Bios, error) {
 // BootOptions gets all BootOption items for this system.
 func (computersystem *ComputerSystem) BootOptions() ([]*BootOption, error) {
 	var result []*BootOption
-	links, err := common.GetCollection(computersystem.Client, computersystem.Boot.bootOptions)
-	if err != nil {
-		return result, err
+	if computersystem.Boot.bootOptions == "" {
+		return result, nil
+	}
+	c := computersystem.Client
+
+	type GetResult struct {
+		Item  *BootOption
+		Link  string
+		Error error
 	}
 
+	ch := make(chan GetResult)
 	collectionError := common.NewCollectionError()
-	for _, bootOptionLink := range links.ItemLinks {
-		bootOption, err := GetBootOption(computersystem.Client, bootOptionLink)
+	get := func(link string) {
+		bootoption, err := GetBootOption(c, link)
+		ch <- GetResult{Item: bootoption, Link: link, Error: err}
+	}
+
+	go func() {
+		err := common.CollectList(get, c, computersystem.Boot.bootOptions)
 		if err != nil {
-			collectionError.Failures[bootOptionLink] = err
+			collectionError.Failures[computersystem.Boot.bootOptions] = err
+		}
+		close(ch)
+	}()
+
+	for r := range ch {
+		if r.Error != nil {
+			collectionError.Failures[r.Link] = r.Error
 		} else {
-			result = append(result, bootOption)
+			result = append(result, r.Item)
 		}
 	}