Skip to content

Commit

Permalink
feat multi files and retrieval
Browse files Browse the repository at this point in the history
  • Loading branch information
@xqdoo00o
xqdoo00o committed Jun 3, 2024
1 parent 02a7b24 commit 273c4b5
Showing 1 changed file with 84 additions and 24 deletions.
108 changes: 84 additions & 24 deletions typings/chatgpt/request.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,12 +85,13 @@ type UploadResp struct {
}

type FileResult struct {
Mime string
Filename string
Fileid string
Filesize int
Isimage bool
Bounds [2]int
Mime string
Filename string
Fileid string
Filesize int
Isimage bool
Bounds [2]int
TokenSize int
// Current file max-age 30 days
Upload int64
}
Expand All @@ -104,12 +105,18 @@ type ImgPart struct {
}

type ImgMeta struct {
Id string `json:"id"`
MimeType string `json:"mimeType"`
Name string `json:"name"`
Size int `json:"size"`
Width int `json:"width,omitempty"`
Height int `json:"height,omitempty"`
Id string `json:"id"`
MimeType string `json:"mimeType"`
Name string `json:"name"`
Size int `json:"size"`
Width int `json:"width,omitempty"`
Height int `json:"height,omitempty"`
TokenSize int `json:"file_token_size,omitempty"`
}

type RetrievalResult struct {
FileSizeTokens int `json:"file_size_tokens,omitempty"`
Status string `json:"retrieval_index_status"`
}

var (
Expand All @@ -118,8 +125,9 @@ var (
tls_client.WithTimeoutSeconds(600),
tls_client.WithClientProfile(profiles.Okhttp4Android13),
}...)
userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
fileHashPool = map[string]*FileResult{}
userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
fileHashPool = map[string]*FileResult{}
retrievalMime = map[string]bool{}
)

func init() {
Expand All @@ -128,6 +136,8 @@ func init() {
Name: "oai-dm-tgt-c-240329",
Value: "2024-04-02",
}})
// from https://chatgpt.com/backend-api/models?history_and_training_disabled=false models product_features
retrievalMime = map[string]bool{"text/rtf": true, "application/javascript": true, "text/x-tex": true, "text/css": true, "text/xml": true, "message/rfc822": true, "text/javascript": true, "application/rtf": true, "text/x-typescript": true, "application/x-powershell": true, "application/x-sql": true, "text/x-shellscript": true, "text/x-c++": true, "text/markdown": true, "text/x-php": true, "text/x-script.python": true, "text/vbscript": true, "text/x-asm": true, "application/vnd.oasis.opendocument.text": true, "text/x-lisp": true, "application/vnd.openxmlformats-officedocument.wordprocessingml.document": true, "application/x-rust": true, "text/x-diff": true, "text/x-python": true, "application/vnd.apple.keynote": true, "application/vnd.ms-powerpoint": true, "application/x-yaml": true, "application/msword": true, "application/x-scala": true, "text/plain": true, "text/html": true, "application/json": true, "text/calendar": true, "text/x-csharp": true, "text/x-rst": true, "text/x-java": true, "text/x-makefile": true, "application/pdf": true, "text/x-c": true, "text/x-vcard": true, "application/vnd.apple.pages": true, "application/vnd.openxmlformats-officedocument.presentationml.presentation": true, "text/x-ruby": true, "text/x-sh": true}
// from https://chromium.googlesource.com/chromium/src/+/HEAD/net/base/mime_util.cc
mimeMap := map[string]string{
"webm": "video/webm",
Expand Down Expand Up @@ -273,6 +283,10 @@ func processUrl(urlstr string, account string, secret *tokens.Secret, deviceId s
fileName := path.Base(u.Path)
extIndex := strings.Index(fileName, ".")
mimeType := mime.TypeByExtension(fileName[extIndex:])
mimeIdx := strings.Index(mimeType, ";")
if mimeIdx != -1 {
mimeType = mimeType[:mimeIdx]
}
request, err := http.NewRequest(http.MethodGet, urlstr, nil)
if err != nil {
return nil
Expand Down Expand Up @@ -305,7 +319,11 @@ func processUrl(urlstr string, account string, secret *tokens.Secret, deviceId s
if fileid == "" {
return nil
} else {
result := FileResult{Mime: mimeType, Filename: fileName, Filesize: len(binary), Fileid: fileid, Isimage: isImg, Bounds: bounds, Upload: time.Now().Unix()}
tokenSize := 0
if !isImg && retrievalMime[mimeType] {
tokenSize = getRetrievalToken(fileid, 10, secret, deviceId, proxy)
}
result := FileResult{Mime: mimeType, Filename: fileName, Filesize: len(binary), Fileid: fileid, Isimage: isImg, Bounds: bounds, TokenSize: tokenSize, Upload: time.Now().Unix()}
fileHashPool[hash] = &result
return &result
}
Expand All @@ -325,6 +343,10 @@ func processDataUrl(data string, account string, secret *tokens.Secret, deviceId
startIdx := strings.Index(data, ":")
endIdx := strings.Index(data, ";")
mimeType := data[startIdx+1 : endIdx]
mimeIdx := strings.Index(mimeType, ";")
if mimeIdx != -1 {
mimeType = mimeType[:mimeIdx]
}
var fileName string
extensions, _ := mime.ExtensionsByType(mimeType)
if len(extensions) > 0 {
Expand All @@ -346,7 +368,11 @@ func processDataUrl(data string, account string, secret *tokens.Secret, deviceId
if fileid == "" {
return nil
} else {
result := FileResult{Mime: mimeType, Filename: fileName, Filesize: len(binary), Fileid: fileid, Isimage: isImg, Bounds: bounds, Upload: time.Now().Unix()}
tokenSize := 0
if !isImg && retrievalMime[mimeType] {
tokenSize = getRetrievalToken(fileid, 10, secret, deviceId, proxy)
}
result := FileResult{Mime: mimeType, Filename: fileName, Filesize: len(binary), Fileid: fileid, Isimage: isImg, Bounds: bounds, TokenSize: tokenSize, Upload: time.Now().Unix()}
fileHashPool[hash] = &result
return &result
}
Expand All @@ -358,6 +384,8 @@ func uploadBinary(data []byte, mime string, name string, isImg bool, secret *tok
var fileCase string
if isImg {
fileCase = "multimodal"
} else if retrievalMime[mime] {
fileCase = "my_files"
} else {
fileCase = "ace_upload"
}
Expand Down Expand Up @@ -409,10 +437,40 @@ func uploadBinary(data []byte, mime string, name string, isImg bool, secret *tok
}
return fileResp.File_id
}

func getRetrievalToken(fileid string, retry int, secret *tokens.Secret, deviceId string, proxy string) int {
if proxy != "" {
client.SetProxy(proxy)
}
request, err := newRequest(http.MethodGet, "https://chatgpt.com/backend-api/files/"+fileid, nil, secret, deviceId)
if err != nil {
return 0
}
response, err := client.Do(request)
if err != nil {
return 0
}
defer response.Body.Close()
var evalResp RetrievalResult
err = json.NewDecoder(response.Body).Decode(&evalResp)
if err != nil {
return 0
}
if evalResp.Status == "success" {
return evalResp.FileSizeTokens
} else {
retry = retry - 1
if retry == 0 {
return 0
} else {
time.Sleep(time.Millisecond * 500)
return getRetrievalToken(fileid, retry, secret, deviceId, proxy)
}
}
}
func (c *ChatGPTRequest) AddMessage(role string, content interface{}, multimodal bool, account string, secret *tokens.Secret, deviceId string, proxy string) {
parts := []interface{}{}
var result *FileResult
var metadatas Chatgpt_metadata
msgType := "text"
switch v := content.(type) {
case string:
parts = append(parts, v)
Expand All @@ -438,6 +496,7 @@ func (c *ChatGPTRequest) AddMessage(role string, content interface{}, multimodal
continue
}
data := item.Image.Url
var result *FileResult
if strings.HasPrefix(data, "data:") {
result = processDataUrl(data, account, secret, deviceId, proxy)
if result == nil {
Expand All @@ -450,7 +509,11 @@ func (c *ChatGPTRequest) AddMessage(role string, content interface{}, multimodal
}
}
if result.Isimage {
msgType = "multimodal_text"
parts = append(parts, ImgPart{Asset_pointer: "file-service://" + result.Fileid, Content_type: "image_asset_pointer", Size_bytes: result.Filesize, Width: result.Bounds[0], Height: result.Bounds[1]})
metadatas.Attachments = append(metadatas.Attachments, ImgMeta{Id: result.Fileid, Name: result.Filename, Size: result.Filesize, MimeType: result.Mime, Width: result.Bounds[0], Height: result.Bounds[1]})
} else {
metadatas.Attachments = append(metadatas.Attachments, ImgMeta{Id: result.Fileid, Name: result.Filename, Size: result.Filesize, MimeType: result.Mime, TokenSize: result.TokenSize})
}
} else {
parts = append(parts, item.Text)
Expand All @@ -460,14 +523,11 @@ func (c *ChatGPTRequest) AddMessage(role string, content interface{}, multimodal
var msg = chatgpt_message{
ID: uuid.New(),
Author: chatgpt_author{Role: role},
Content: chatgpt_content{ContentType: "text", Parts: parts},
Content: chatgpt_content{ContentType: msgType, Parts: parts},
Metadata: nil,
}
if result != nil {
if result.Isimage {
msg.Content.ContentType = "multimodal_text"
}
msg.Metadata = &Chatgpt_metadata{Attachments: []ImgMeta{{Id: result.Fileid, Name: result.Filename, Size: result.Filesize, MimeType: result.Mime, Width: result.Bounds[0], Height: result.Bounds[1]}}}
if metadatas.Attachments != nil {
msg.Metadata = &metadatas
}
c.Messages = append(c.Messages, msg)
}
Expand Down

0 comments on commit 273c4b5

Please sign in to comment.