From d5e3b337b19b1fd99df3f2628d5efdf30a400a18 Mon Sep 17 00:00:00 2001 From: Lasse Hyldahl Jensen Date: Wed, 18 Sep 2024 14:26:38 +0200 Subject: [PATCH 1/7] Support binary image URL --- chat.go | 38 ++++++++++++++++++++++++++++++- chat_test.go | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 99 insertions(+), 2 deletions(-) diff --git a/chat.go b/chat.go index dc60f35b9..5f12e84e8 100644 --- a/chat.go +++ b/chat.go @@ -1,7 +1,9 @@ package openai import ( + "bytes" "context" + "encoding/base64" "encoding/json" "errors" "net/http" @@ -62,10 +64,44 @@ const ( ) type ChatMessageImageURL struct { - URL string `json:"url,omitempty"` + URL any `json:"url,omitempty"` Detail ImageURLDetail `json:"detail,omitempty"` } +type ImageURL string + +func (i ImageURL) String() string { + return string(i) +} + +type BinaryImageURL struct { + MimeType string + Data []byte +} + +func (b BinaryImageURL) MarshalJSON() ([]byte, error) { + encodedLength := base64.StdEncoding.EncodedLen(len(b.Data)) + buf := bytes.NewBuffer(make([]byte, 0, 15+len(b.MimeType)+encodedLength)) + + buf.WriteString(`"data:`) + buf.WriteString(b.MimeType) + buf.WriteString(`;base64,`) + + // base64 encode data and write it to the buffer + encoder := base64.NewEncoder(base64.StdEncoding, buf) + _, err := encoder.Write(b.Data) + if err != nil { + return nil, err + } + err = encoder.Close() + if err != nil { + return nil, err + } + + buf.WriteString(`"`) + return buf.Bytes(), nil +} + type ChatMessagePartType string const ( diff --git a/chat_test.go b/chat_test.go index 37dc09d4d..d2cb2aa60 100644 --- a/chat_test.go +++ b/chat_test.go @@ -2,6 +2,7 @@ package openai_test import ( "context" + "crypto/rand" "encoding/json" "errors" "fmt" @@ -342,7 +343,7 @@ func TestMultipartChatCompletions(t *testing.T) { { Type: openai.ChatMessagePartTypeImageURL, ImageURL: &openai.ChatMessageImageURL{ - URL: "URL", + URL: openai.ImageURL("URL"), Detail: openai.ImageURLDetailLow, }, }, @@ -553,3 +554,63 @@ func TestFinishReason(t *testing.T) { } } } + +func TestChatCompletionMessageMarshalJSONWithMultiContent(t *testing.T) { + // generate 20 mb of random data + imageData := generateEncodedData(t, 20*1024*1024) + //imageData = []byte("test") + //imageURL := openai.ImageURL(fmt.Sprintf("data:image/png;base64,%s", base64.StdEncoding.EncodeToString(imageData))) + imageURL := openai.BinaryImageURL{ + MimeType: "image/png", + Data: imageData, + } + + msg := openai.ChatCompletionMessage{ + Role: openai.ChatMessageRoleUser, + MultiContent: []openai.ChatMessagePart{ + { + Type: openai.ChatMessagePartTypeText, + Text: "Hello, world!", + }, + { + Type: openai.ChatMessagePartTypeImageURL, + ImageURL: &openai.ChatMessageImageURL{ + URL: imageURL, + Detail: openai.ImageURLDetailHigh, + }, + }, + }, + Name: "test-name", + FunctionCall: &openai.FunctionCall{ + Name: "test-function", + Arguments: `{"arg1":"value1"}`, + }, + ToolCalls: []openai.ToolCall{ + { + ID: "tool1", + Type: openai.ToolTypeFunction, + Function: openai.FunctionCall{ + Name: "tool-function", + Arguments: `{"arg2":"value2"}`, + }, + }, + }, + ToolCallID: "tool-call-id", + } + + data, err := json.Marshal(msg) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + t.Logf("marshaled message: %s", data) +} + +func generateEncodedData(t *testing.T, size int64) []byte { + data := make([]byte, size) + _, err := rand.Read(data) + if err != nil { + t.Fatalf("Failed to generate random data: %v", err) + } + return data +} From b2a7df602211775a4fc432bddbc87773267b2e78 Mon Sep 17 00:00:00 2001 From: Lasse Hyldahl Jensen Date: Wed, 18 Sep 2024 15:54:06 +0200 Subject: [PATCH 2/7] Use ChatCompletionRequest in test --- chat_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/chat_test.go b/chat_test.go index d2cb2aa60..53bce2a3f 100644 --- a/chat_test.go +++ b/chat_test.go @@ -555,7 +555,7 @@ func TestFinishReason(t *testing.T) { } } -func TestChatCompletionMessageMarshalJSONWithMultiContent(t *testing.T) { +func TestChatCompletionRequest_MarshalJSON_LargeImage(t *testing.T) { // generate 20 mb of random data imageData := generateEncodedData(t, 20*1024*1024) //imageData = []byte("test") @@ -597,8 +597,11 @@ func TestChatCompletionMessageMarshalJSONWithMultiContent(t *testing.T) { }, ToolCallID: "tool-call-id", } + req := openai.ChatCompletionRequest{ + Messages: []openai.ChatCompletionMessage{msg}, + } - data, err := json.Marshal(msg) + data, err := json.Marshal(req) if err != nil { t.Fatalf("Expected no error, got %v", err) } From 3950d62c29580454c06441c3764153fe63341bd7 Mon Sep 17 00:00:00 2001 From: Lasse Hyldahl Jensen Date: Wed, 18 Sep 2024 15:56:44 +0200 Subject: [PATCH 3/7] Cleanup --- chat.go | 7 +------ chat_test.go | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/chat.go b/chat.go index 5f12e84e8..61f9af335 100644 --- a/chat.go +++ b/chat.go @@ -64,16 +64,11 @@ const ( ) type ChatMessageImageURL struct { + // URL can be a string or a BinaryImageURL object. URL any `json:"url,omitempty"` Detail ImageURLDetail `json:"detail,omitempty"` } -type ImageURL string - -func (i ImageURL) String() string { - return string(i) -} - type BinaryImageURL struct { MimeType string Data []byte diff --git a/chat_test.go b/chat_test.go index 53bce2a3f..2caccae18 100644 --- a/chat_test.go +++ b/chat_test.go @@ -343,7 +343,7 @@ func TestMultipartChatCompletions(t *testing.T) { { Type: openai.ChatMessagePartTypeImageURL, ImageURL: &openai.ChatMessageImageURL{ - URL: openai.ImageURL("URL"), + URL: "URL", Detail: openai.ImageURLDetailLow, }, }, From 861974a45fffc3e12e20a775a9a0ccd2b22dab35 Mon Sep 17 00:00:00 2001 From: Lasse Hyldahl Jensen Date: Wed, 18 Sep 2024 16:12:52 +0200 Subject: [PATCH 4/7] Cleanup --- chat_test.go | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/chat_test.go b/chat_test.go index 2caccae18..15cac34f1 100644 --- a/chat_test.go +++ b/chat_test.go @@ -1,8 +1,10 @@ package openai_test import ( + "bytes" "context" "crypto/rand" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -555,11 +557,31 @@ func TestFinishReason(t *testing.T) { } } +func encodeImage(t *testing.T, mimeType string, data []byte) []byte { + encodedLength := base64.StdEncoding.EncodedLen(len(data)) + buf := bytes.NewBuffer(make([]byte, 0, 13+len(mimeType)+encodedLength)) + + buf.WriteString(`data:`) + buf.WriteString(mimeType) + buf.WriteString(`;base64,`) + + // base64 encode data and write it to the buffer + encoder := base64.NewEncoder(base64.StdEncoding, buf) + _, err := encoder.Write(data) + if err != nil { + t.Fatalf("Failed to encode image: %v", err) + } + err = encoder.Close() + if err != nil { + t.Fatalf("Failed to encode image: %v", err) + } + return buf.Bytes() +} + func TestChatCompletionRequest_MarshalJSON_LargeImage(t *testing.T) { // generate 20 mb of random data imageData := generateEncodedData(t, 20*1024*1024) - //imageData = []byte("test") - //imageURL := openai.ImageURL(fmt.Sprintf("data:image/png;base64,%s", base64.StdEncoding.EncodeToString(imageData))) + //imageURL := encodeImage(t, "image/png", imageData) imageURL := openai.BinaryImageURL{ MimeType: "image/png", Data: imageData, From 891008a82df3a967602b5dbcef43bc0d3c414c03 Mon Sep 17 00:00:00 2001 From: Lasse Hyldahl Jensen Date: Wed, 18 Sep 2024 16:22:24 +0200 Subject: [PATCH 5/7] Comment out encodeImage function --- chat_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/chat_test.go b/chat_test.go index 15cac34f1..4e746359d 100644 --- a/chat_test.go +++ b/chat_test.go @@ -1,10 +1,8 @@ package openai_test import ( - "bytes" "context" "crypto/rand" - "encoding/base64" "encoding/json" "errors" "fmt" @@ -557,7 +555,7 @@ func TestFinishReason(t *testing.T) { } } -func encodeImage(t *testing.T, mimeType string, data []byte) []byte { +/*func encodeImage(t *testing.T, mimeType string, data []byte) []byte { encodedLength := base64.StdEncoding.EncodedLen(len(data)) buf := bytes.NewBuffer(make([]byte, 0, 13+len(mimeType)+encodedLength)) @@ -576,7 +574,7 @@ func encodeImage(t *testing.T, mimeType string, data []byte) []byte { t.Fatalf("Failed to encode image: %v", err) } return buf.Bytes() -} +}*/ func TestChatCompletionRequest_MarshalJSON_LargeImage(t *testing.T) { // generate 20 mb of random data From b3320088664b8aecae04cf1632dd50f3f73f2a83 Mon Sep 17 00:00:00 2001 From: Lasse Hyldahl Jensen Date: Thu, 19 Sep 2024 08:41:11 +0200 Subject: [PATCH 6/7] Split test into 2 --- chat_test.go | 85 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 52 insertions(+), 33 deletions(-) diff --git a/chat_test.go b/chat_test.go index 4e746359d..08e334d24 100644 --- a/chat_test.go +++ b/chat_test.go @@ -1,8 +1,10 @@ package openai_test import ( + "bytes" "context" "crypto/rand" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -555,36 +557,47 @@ func TestFinishReason(t *testing.T) { } } -/*func encodeImage(t *testing.T, mimeType string, data []byte) []byte { - encodedLength := base64.StdEncoding.EncodedLen(len(data)) - buf := bytes.NewBuffer(make([]byte, 0, 13+len(mimeType)+encodedLength)) - - buf.WriteString(`data:`) - buf.WriteString(mimeType) - buf.WriteString(`;base64,`) - - // base64 encode data and write it to the buffer - encoder := base64.NewEncoder(base64.StdEncoding, buf) - _, err := encoder.Write(data) - if err != nil { - t.Fatalf("Failed to encode image: %v", err) +func TestChatCompletionRequest_MarshalJSON_LargeImage_Binary(t *testing.T) { + // generate 20 mb of random data + imageData := generateEncodedData(t, 20*1024*1024) + imageURL := openai.BinaryImageURL{ + MimeType: "image/png", + Data: imageData, } - err = encoder.Close() + req := generateRequestWithImage(imageURL) + + data, err := json.Marshal(req) if err != nil { - t.Fatalf("Failed to encode image: %v", err) + t.Fatalf("Expected no error, got %v", err) } - return buf.Bytes() -}*/ -func TestChatCompletionRequest_MarshalJSON_LargeImage(t *testing.T) { + t.Logf("marshaled message: %s", data) +} + +func TestChatCompletionRequest_MarshalJSON_LargeImage_Base64(t *testing.T) { // generate 20 mb of random data imageData := generateEncodedData(t, 20*1024*1024) - //imageURL := encodeImage(t, "image/png", imageData) - imageURL := openai.BinaryImageURL{ - MimeType: "image/png", - Data: imageData, + imageURL := encodeImage(t, "image/png", imageData) + req := generateRequestWithImage(imageURL) + + data, err := json.Marshal(req) + if err != nil { + t.Fatalf("Expected no error, got %v", err) } + t.Logf("marshaled message: %s", data) +} + +func generateEncodedData(t *testing.T, size int64) []byte { + data := make([]byte, size) + _, err := rand.Read(data) + if err != nil { + t.Fatalf("Failed to generate random data: %v", err) + } + return data +} + +func generateRequestWithImage(imageURL any) openai.ChatCompletionRequest { msg := openai.ChatCompletionMessage{ Role: openai.ChatMessageRoleUser, MultiContent: []openai.ChatMessagePart{ @@ -620,20 +633,26 @@ func TestChatCompletionRequest_MarshalJSON_LargeImage(t *testing.T) { req := openai.ChatCompletionRequest{ Messages: []openai.ChatCompletionMessage{msg}, } + return req +} - data, err := json.Marshal(req) - if err != nil { - t.Fatalf("Expected no error, got %v", err) - } +func encodeImage(t *testing.T, mimeType string, data []byte) []byte { + encodedLength := base64.StdEncoding.EncodedLen(len(data)) + buf := bytes.NewBuffer(make([]byte, 0, 13+len(mimeType)+encodedLength)) - t.Logf("marshaled message: %s", data) -} + buf.WriteString(`data:`) + buf.WriteString(mimeType) + buf.WriteString(`;base64,`) -func generateEncodedData(t *testing.T, size int64) []byte { - data := make([]byte, size) - _, err := rand.Read(data) + // base64 encode data and write it to the buffer + encoder := base64.NewEncoder(base64.StdEncoding, buf) + _, err := encoder.Write(data) if err != nil { - t.Fatalf("Failed to generate random data: %v", err) + t.Fatalf("Failed to encode image: %v", err) } - return data + err = encoder.Close() + if err != nil { + t.Fatalf("Failed to encode image: %v", err) + } + return buf.Bytes() } From aa19584146c90e130c68df88f87e95517e097426 Mon Sep 17 00:00:00 2001 From: Lasse Hyldahl Jensen Date: Thu, 19 Sep 2024 08:43:34 +0200 Subject: [PATCH 7/7] Don't log request --- chat_test.go | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/chat_test.go b/chat_test.go index 08e334d24..e24b24b0f 100644 --- a/chat_test.go +++ b/chat_test.go @@ -557,35 +557,31 @@ func TestFinishReason(t *testing.T) { } } -func TestChatCompletionRequest_MarshalJSON_LargeImage_Binary(t *testing.T) { +func TestChatCompletionRequest_MarshalJSON_LargeImage_Base64(t *testing.T) { // generate 20 mb of random data imageData := generateEncodedData(t, 20*1024*1024) - imageURL := openai.BinaryImageURL{ - MimeType: "image/png", - Data: imageData, - } + imageURL := encodeImage(t, "image/png", imageData) req := generateRequestWithImage(imageURL) - data, err := json.Marshal(req) + _, err := json.Marshal(req) if err != nil { t.Fatalf("Expected no error, got %v", err) } - - t.Logf("marshaled message: %s", data) } -func TestChatCompletionRequest_MarshalJSON_LargeImage_Base64(t *testing.T) { +func TestChatCompletionRequest_MarshalJSON_LargeImage_Binary(t *testing.T) { // generate 20 mb of random data imageData := generateEncodedData(t, 20*1024*1024) - imageURL := encodeImage(t, "image/png", imageData) + imageURL := openai.BinaryImageURL{ + MimeType: "image/png", + Data: imageData, + } req := generateRequestWithImage(imageURL) - data, err := json.Marshal(req) + _, err := json.Marshal(req) if err != nil { t.Fatalf("Expected no error, got %v", err) } - - t.Logf("marshaled message: %s", data) } func generateEncodedData(t *testing.T, size int64) []byte {