Skip to content

Commit

Permalink
feat: add image url base64 mode support
Browse files Browse the repository at this point in the history
  • Loading branch information
trevorwang committed Nov 26, 2023
1 parent b963a8f commit 3690c48
Show file tree
Hide file tree
Showing 5 changed files with 71 additions and 21 deletions.
32 changes: 31 additions & 1 deletion lib/src/chat/completion.dart
Original file line number Diff line number Diff line change
Expand Up @@ -364,15 +364,45 @@ class ImageContent with _$ImageContent {
_$ImageContentFromJson(json);
}

///By controlling the detail parameter, which has three options, low, high, or
///auto, you have control over how the model processes the image and generates
///its textual understanding. By default, the model will use the auto setting
/// which will look at the image input size and decide if it should use the
/// low or high setting.
@JsonEnum(fieldRename: FieldRename.snake)
enum ImageDetail {
auto,

/// low will disable the “high res” model. The model will receive a low-res
/// 512px x 512px version of the image, and represent the image with a budget
/// of 65 tokens. This allows the API to return faster responses and consume
/// fewer input tokens for use cases that do not require high detail.
low,

/// high will enable “high res” mode, which first allows the model to see the
/// low res image and then creates detailed crops of input images as 512px
/// squares based on the input image size. Each of the detailed crops uses
/// twice the token budget (65 tokens) for a total of 129 tokens.
high,
}

@freezed
class ImageUrl with _$ImageUrl {
const factory ImageUrl({
/// Either a URL of the image or the base64 encoded image data.
required String url,

/// Specifies the detail level of the image.
@Default("auto") String detail,
@Default(ImageDetail.auto) ImageDetail detail,
}) = _ImageUrl;
factory ImageUrl.base64({
required List<int> image,
String mimeType = "image/jpeg",
ImageDetail detail = ImageDetail.auto,
}) {
final url = "data:$mimeType;base64,${base64Encode(image)}";
return ImageUrl(url: url, detail: detail);
}

factory ImageUrl.fromJson(Map<String, dynamic> json) =>
_$ImageUrlFromJson(json);
Expand Down
20 changes: 10 additions & 10 deletions lib/src/chat/completion.freezed.dart
Original file line number Diff line number Diff line change
Expand Up @@ -3096,7 +3096,7 @@ mixin _$ImageUrl {
String get url => throw _privateConstructorUsedError;

/// Specifies the detail level of the image.
String get detail => throw _privateConstructorUsedError;
ImageDetail get detail => throw _privateConstructorUsedError;

Map<String, dynamic> toJson() => throw _privateConstructorUsedError;
@JsonKey(ignore: true)
Expand All @@ -3109,7 +3109,7 @@ abstract class $ImageUrlCopyWith<$Res> {
factory $ImageUrlCopyWith(ImageUrl value, $Res Function(ImageUrl) then) =
_$ImageUrlCopyWithImpl<$Res, ImageUrl>;
@useResult
$Res call({String url, String detail});
$Res call({String url, ImageDetail detail});
}

/// @nodoc
Expand All @@ -3136,7 +3136,7 @@ class _$ImageUrlCopyWithImpl<$Res, $Val extends ImageUrl>
detail: null == detail
? _value.detail
: detail // ignore: cast_nullable_to_non_nullable
as String,
as ImageDetail,
) as $Val);
}
}
Expand All @@ -3149,7 +3149,7 @@ abstract class _$$ImageUrlImplCopyWith<$Res>
__$$ImageUrlImplCopyWithImpl<$Res>;
@override
@useResult
$Res call({String url, String detail});
$Res call({String url, ImageDetail detail});
}

/// @nodoc
Expand All @@ -3174,15 +3174,15 @@ class __$$ImageUrlImplCopyWithImpl<$Res>
detail: null == detail
? _value.detail
: detail // ignore: cast_nullable_to_non_nullable
as String,
as ImageDetail,
));
}
}

/// @nodoc
@JsonSerializable()
class _$ImageUrlImpl implements _ImageUrl {
const _$ImageUrlImpl({required this.url, this.detail = "auto"});
const _$ImageUrlImpl({required this.url, this.detail = ImageDetail.auto});

factory _$ImageUrlImpl.fromJson(Map<String, dynamic> json) =>
_$$ImageUrlImplFromJson(json);
Expand All @@ -3194,7 +3194,7 @@ class _$ImageUrlImpl implements _ImageUrl {
/// Specifies the detail level of the image.
@override
@JsonKey()
final String detail;
final ImageDetail detail;

@override
String toString() {
Expand Down Expand Up @@ -3229,8 +3229,8 @@ class _$ImageUrlImpl implements _ImageUrl {
}

abstract class _ImageUrl implements ImageUrl {
const factory _ImageUrl({required final String url, final String detail}) =
_$ImageUrlImpl;
const factory _ImageUrl(
{required final String url, final ImageDetail detail}) = _$ImageUrlImpl;

factory _ImageUrl.fromJson(Map<String, dynamic> json) =
_$ImageUrlImpl.fromJson;
Expand All @@ -3242,7 +3242,7 @@ abstract class _ImageUrl implements ImageUrl {
@override

/// Specifies the detail level of the image.
String get detail;
ImageDetail get detail;
@override
@JsonKey(ignore: true)
_$$ImageUrlImplCopyWith<_$ImageUrlImpl> get copyWith =>
Expand Down
11 changes: 9 additions & 2 deletions lib/src/chat/completion.g.dart

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

29 changes: 21 additions & 8 deletions test/chat_completion_test.dart
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import 'dart:convert';
import 'dart:io';

import 'package:cancellation_token_http/http.dart';
import 'package:cancellation_token_http/testing.dart';
Expand Down Expand Up @@ -151,14 +152,26 @@ void main() {
"https://pics0.baidu.com/feed/314e251f95cad1c84595a5ad29667204c83d51f7.jpeg@f_auto?token=8ad7fa4a5a222a9039b02f92114bfc07"),
);

final request =
ChatCompletionRequest(model: Models.gpt4_1106VisonPreview, messages: [
ChatMessage.system(content: "Hello, how are you?"),
ChatMessage.user(content: [
TextContent(text: "show me what you see in the following image"),
ic,
]),
]);
final f = File("test/gpt4v-demo.webp");
final ic2 = ImageContent(
imageUrl: ImageUrl.base64(
image: f.readAsBytesSync(),
));

final request = ChatCompletionRequest(
model: Models.gpt4_1106VisonPreview,
maxTokens: 500,
messages: [
ChatMessage.system(content: "Hello, how are you?"),
ChatMessage.user(content: [
TextContent(
text:
"show me what you see in the following image? Are the two images the same?",
),
ic,
ic2,
]),
]);
final result = await proxyedClient.sendChatCompletion(request);
print(result);
});
Expand Down
Binary file added test/gpt4v-demo.webp
Binary file not shown.

0 comments on commit 3690c48

Please sign in to comment.