From fb4ac30042967c403344aeec5484faee1ef6e9a0 Mon Sep 17 00:00:00 2001
From: Jeremiah Lowin <153965+jlowin@users.noreply.github.com>
Date: Fri, 12 Jan 2024 09:59:35 -0500
Subject: [PATCH 1/8] Add vision
---
docs/ai/{ => audio}/speech.md | 0
docs/ai/{ => images}/painting.md | 0
docs/ai/{ => text}/casting.md | 0
docs/ai/{ => text}/classification.md | 0
docs/ai/{ => text}/extraction.md | 0
docs/ai/{ => text}/function.md | 0
docs/ai/{ => text}/generation.md | 0
docs/ai/vision/captioning.md | 38 +++++++
docs/api_reference/ai/beta/vision.md | 6 ++
docs/api_reference/requests.md | 1 -
docs/api_reference/types.md | 1 +
docs/assets/images/core/vision/marvin.webp | Bin 0 -> 53182 bytes
mkdocs.yml | 23 +++--
src/marvin/__init__.py | 3 +
src/marvin/ai/beta/__init__.py | 0
src/marvin/ai/beta/vision.py | 115 +++++++++++++++++++++
src/marvin/ai/prompts/vision_prompts.py | 15 +++
src/marvin/ai/text.py | 5 +-
src/marvin/client/openai.py | 42 +++++++-
src/marvin/settings.py | 16 +++
src/marvin/types.py | 52 +++++++++-
src/marvin/utilities/images.py | 15 +++
22 files changed, 312 insertions(+), 20 deletions(-)
rename docs/ai/{ => audio}/speech.md (100%)
rename docs/ai/{ => images}/painting.md (100%)
rename docs/ai/{ => text}/casting.md (100%)
rename docs/ai/{ => text}/classification.md (100%)
rename docs/ai/{ => text}/extraction.md (100%)
rename docs/ai/{ => text}/function.md (100%)
rename docs/ai/{ => text}/generation.md (100%)
create mode 100644 docs/ai/vision/captioning.md
create mode 100644 docs/api_reference/ai/beta/vision.md
delete mode 100644 docs/api_reference/requests.md
create mode 100644 docs/api_reference/types.md
create mode 100644 docs/assets/images/core/vision/marvin.webp
create mode 100644 src/marvin/ai/beta/__init__.py
create mode 100644 src/marvin/ai/beta/vision.py
create mode 100644 src/marvin/ai/prompts/vision_prompts.py
create mode 100644 src/marvin/utilities/images.py
diff --git a/docs/ai/speech.md b/docs/ai/audio/speech.md
similarity index 100%
rename from docs/ai/speech.md
rename to docs/ai/audio/speech.md
diff --git a/docs/ai/painting.md b/docs/ai/images/painting.md
similarity index 100%
rename from docs/ai/painting.md
rename to docs/ai/images/painting.md
diff --git a/docs/ai/casting.md b/docs/ai/text/casting.md
similarity index 100%
rename from docs/ai/casting.md
rename to docs/ai/text/casting.md
diff --git a/docs/ai/classification.md b/docs/ai/text/classification.md
similarity index 100%
rename from docs/ai/classification.md
rename to docs/ai/text/classification.md
diff --git a/docs/ai/extraction.md b/docs/ai/text/extraction.md
similarity index 100%
rename from docs/ai/extraction.md
rename to docs/ai/text/extraction.md
diff --git a/docs/ai/function.md b/docs/ai/text/function.md
similarity index 100%
rename from docs/ai/function.md
rename to docs/ai/text/function.md
diff --git a/docs/ai/generation.md b/docs/ai/text/generation.md
similarity index 100%
rename from docs/ai/generation.md
rename to docs/ai/text/generation.md
diff --git a/docs/ai/vision/captioning.md b/docs/ai/vision/captioning.md
new file mode 100644
index 000000000..604c1f336
--- /dev/null
+++ b/docs/ai/vision/captioning.md
@@ -0,0 +1,38 @@
+# Generating images
+
+Marvin can use OpenAI's vision API to process images as inputs.
+
+
+
What it does
+
+ The caption
function generates text from images.
+
+
+
+
+
+!!! example
+
+ Generate a description of the following image, hypothetically available at `/path/to/marvin.webp`:
+
+ ![](/assets/images/core/vision/marvin.webp)
+
+
+ ```python
+ import marvin
+ from pathlib import Path
+
+ marvin.caption(image=Path('/path/to/marvin.webp'))
+ ```
+
+ !!! success "Result"
+ "This is a digital illustration featuring a stylized, cute character resembling a Funko Pop vinyl figure with large, shiny eyes and a square-shaped head, sitting on abstract wavy shapes that simulate a landscape. The whimsical figure is set against a dark background with sparkling, colorful bokeh effects, giving it a magical, dreamy atmosphere."
+
+
+
+
How it works
+
+ Marvin passes your images to the OpenAI vision API as part of a larger prompt.
+
+
+
diff --git a/docs/api_reference/ai/beta/vision.md b/docs/api_reference/ai/beta/vision.md
new file mode 100644
index 000000000..196493a7c
--- /dev/null
+++ b/docs/api_reference/ai/beta/vision.md
@@ -0,0 +1,6 @@
+# Vision tools
+
+!!! tip "Beta"
+ Please note that vision support in Marvin is still in beta, as OpenAI has not finalized the vision API yet. While it works as expected, it is subject to change.
+
+::: marvin.ai.beta.vision
\ No newline at end of file
diff --git a/docs/api_reference/requests.md b/docs/api_reference/requests.md
deleted file mode 100644
index 962a508d5..000000000
--- a/docs/api_reference/requests.md
+++ /dev/null
@@ -1 +0,0 @@
-::: marvin.requests
\ No newline at end of file
diff --git a/docs/api_reference/types.md b/docs/api_reference/types.md
new file mode 100644
index 000000000..c9091398e
--- /dev/null
+++ b/docs/api_reference/types.md
@@ -0,0 +1 @@
+::: marvin.types
\ No newline at end of file
diff --git a/docs/assets/images/core/vision/marvin.webp b/docs/assets/images/core/vision/marvin.webp
new file mode 100644
index 0000000000000000000000000000000000000000..f40ea7d152536dde3642120915eb4f8b0ef96f98
GIT binary patch
literal 53182
zcmd42^K&Ln^zZq^oY=NG(ZsgxiJeSr+qP}nww+9D+s6b9Zm;eyjEm*jH8e4_$S-
zPoHX4_q&xO#l%`>0RVMTAq6#sUmCCg008=5s)YdXg8}}P5UtDr0Jk0B*&sA^5CUL)
zHteYqMHF-uq<7gImJt
ze=ayw`Zudu^*0p9BO?H}3PNTlp>C8GPkl%tIB2P;sHENcKkVWC=lMUD
z6D?XAS=iPwDFh56_2aS)vycH)Vwf!B4U=7|jQ+1-H)N8i?Ox?uTUQwWcLdBv99jN1
zH9BE2PnhwVIRkkmk6~~f(y9R3z*w=0tm+gMWQkR<`;K%xXQ(2N%pC%D
z{x8%2nTz3+`;6f)wx20PSuey<9nwZN6jRfV^DAnA^I21&e~@$pw}*W9zi-NHFEfsH
zG2SRJPE9G9sQw2f`ssQ_-vSAd!x3zv+0E8p!527eP9g#(D{mAIY$BF3v0wh8WM}f&
z+#c+5Q(JX(FIPdEb_4|K{d){~E;G-Pq#@l_H6zaaZ;|7*;nJg%pK4-|X88XCwe9A(
z2m}JpXGVxdN;_e=#=#2rnf(qxqjVGgVj)X`w6+7k?M#2qS#dwxai+ut=yoL4Q0(5w
zZOcIk7?4xyQvOAo>x8(%v
z^#4ecXC4oHT=X%u1uL#j;zVS!UMSM-$rZG*i&vJ6We|JGZ_bSLOR`FyN3=oDt&6H;
z@0Cw1SXTpkW=hNKg!DWp<40FG^?QBtEGQ`Rg=+9{+xmS`%rvlo>ot{3MDIasQ`N}H
zs5z2a^KYO&6kh=N+<~yoQUzam@4w7UduY;wlH)6j;q`WYgy(MAunm&O=zY+GIL2vz
z%~JlXn4cosUZ}!XyZG%c^Kxg!B=GYg$G6qBfb94-8
z-Sl*rZdM=oQaNo%2-HLg{MmYLYToK%Mw&W+>8ESIO9DnZ4O0_WLTjZ4cu}qxUWNEhx6e`5DWf3
zUZtZ|yAa(r=~fi-KkkeaT{D4+(4NF+wOf2H3+9f+a+9Ylh3P!KTi>nP;dF+uhHUHp
z#@3xXTXfgP3{M{0R?!{%VMNyRz0p0wuQFtw$*vr+fALn
zUNZntt&K3$j&@d6mFo0x>-&M~YSU+39(S10bR#608Pj>|NQZmXdu3zxW|d%iH%jef
zF?nsb@4o#he0S9qhWEAH5
z8$mvQLTfw`*vM#pH
zYMC};)xDZ~1;ZYrts2{d{g;*Jo4y)t4dyoK^F?rah9kvF4k6B=yE9Iy83_C6G-2KN
zg!eAD;J4pGd^!}N)1SDpg8F*CMKFneU3C)}gE^W5q@;7Zaz3O^t|XA%4GNx+u@(&
zZ@fg5jfucWY*`u$_A^t6Qxm4VN}DkxpZRgCu_#V2Wb`%QriBFPi0mA5sOaC({HRQkITcvIoO={~
zIC{&G%ktGmmnMrX6;RqN9O6%)KJ3&sGBSHB8v46+$j#l|gjWpR`q6sN?S7w)ag5g6
zrB0`m2KdURtzs>wE#q7+n|n41Nqz1RbF=rztIn9_Me-tnYzQuveg$#VNJ}hu-|nA-X&`ZFCx7IYgU%Ru
z;S_Cwl_a^pGWs)o?;^E_K0={Jsq5$~oK{)RWhgh~aFIesh2Cy6Gi~*CHx8a8Mbz1tQ8(mcJRefltU-*e-pwo2BHgrt>|S$%&M~vhd=t{K_C%%@KUg~h({b$C2xNkJ}#pTt30rwe=>KixV=jZgXU5nDhQvi4X#g{42nyCO-ckEcg
z$^XZabANZymt>i(!HdJZ95cQdFW^3DHvgsFVX`Us*-K?5%~U>d`p>H=hRcB>dLTRsMzKXhU@}L`MCMa>F_ECmcJI`$oxoh
z0Zx6%sm!#F>=G6c@^I#}c2e+d;d|8YhcvOZ(H)x&*=guPUzg&WtOJ
zQ;1=3t^q3x0Y@#Bc4mf^rbs#f=)OyR*3VG2el&tYAG|Y@cSlqz2I1(G?$
z@bH)K5Er_DkAgjOXVgEX8tLz|0C;z!NHPsTbV7z;(xzIe4Y5*nY+M10lBO%yV5m4r
zXS!_To_mCaNU!Mw;1v>C<9Wtnu?`~3epuidHY~Gsus32GuRg-uuc_IQf^{DCC^Wsa;hVtmyY~Ll@WK5y*zzW3XU(Q!Lzpx@0RBeTH5!jw
zy1O+on0#9^;P+Y&3g)I|rD!|G?{=+YB^C4f0}H=`4;Ea-CJvdOclj9z$|HTV8^B6(
zZ-5-pA840=%l2mf7ZUkL{-#FgmaM;L-{E&&c_>_N4&fr48L+X)+?X`z7E`L;P^Rb(
z818J>o+SU4Hu7{d%meRoJG5H$L>%EEiOP@|#;s3{NcI^=_CPfK^_{^&4ORdPLsZ1l
z?irbf811#)@~V`4FU8l*z)i|XNGNLi8k?d==j*Y;B=KsPSCxvwNx+X_M~rK`yV?7u
z7Iw9a)mxRFhh)$h12Z1zRU6xG9mR}?R$6B;ac~S}Hs@Dy6A&i}xY;4}fX`pIE6L<
zT^Q_IHqeZQReGyA{c2pHgQsrd4ZO=s=i))^L>J5%+535@kbyM336)S&cAlzp!~sSp
z10i!jCiyJZ`}t&OJ_LNZHa*X%rcqnDXWwOqcq9>&mgTrs>#_3dkWFZT22TM-!?3{+
z6F*mJ5j$Vf#;^-yJ6=2NO{qB<$HEge#$w4A6zCk)%!te|2&wh2Tp*YTR&3sYk!kh0
zi#0Z%2+d~sx@{U&>*;zGU8Uhgy7rG13~ug*!&~p1GB|TBO-Cph0uNshH}dS5>m*2&
z$7BTh%9zR!IDw}E;_E=i!-OGISGMfc&A@beh@H>Q!VFIPCpAL0(PT&nG-qn^mvg5xYt+rUD#FC(3EgH+@J+>dJ5+PGiNihr696uMp_olv>9LI4~eANI4g
zY@>JHZlRL7tg){O#y?P!!YoK~HGH4MWUsO>F$2N^P4u0}!dPStWQq}rKsfh?n2?(t
z2*T;UL!8Ge6kD&L*y=e&L7d)lher>IPoTHS#ZSS+p*$YRjPQq8>f!-!^AMY>l
z?KmFu9#xCG9C5%Slo6))<5`Mm@3C;oXI6Ac%tm}R#0xpTw+(pt>~HAz9%kU82@lxGuEg7$iyIfAOC4ekX(iRIm?MiSVxsQVl!;-eO5yR@by2_Y=%@8#mk7}1<~NBB~l_^8K@gk
z>1wmGg>lTq-MeN?Vi92WhD2-7&lxMkeAR#n2d!Iy>Lg7m{@TLlyh=YG;eYaQY6S+6{Me;0&$<)e*15w?=lUSrJ^R&rf>81(Z^
zS}pJQM+%E}5`Bf~o*g-fsv+!!4cvc^(9&=73|7dM
z6MIwBB^8yJY>cBB-$pZogLM)I*tu~<-
z5#nNM^G|AV)T%i7ccMyc$z>iWzr;xEllextfgEV%6jd%L&L~Zw?V0;6mSE-_gzUV-
zfKRMUWDg!oisfV!H%)dzWXoVJh7A84g{x`ml8u^rfCUI|{iceqrsaZ(os_5o-9f8zP?k7Q6$mrMNDTWyq
z&!To2Lj(hh@5~^0i>lH)y>w|1x$3Cf8CHz%M>kce{(3mm4CVAmxE!P9vEb$;L5~_g
zOiAXGp4SnzLwW_sg6_9U?5DQ2^nUNJb9WRCrfZ&ZV{~lBg*M1%Vs}q0N6craxs7-zIPTxe3;K;nrLM_wnZ$IkR9+{w
z19R<+f<%(-{XSd}JmyJvwwRhu0ux8~T=mIPM&1fbeW%Ck9ym6@qs
zrA~E!4fek25BJ`9VqrgNJa>|d-`{MvQ?BY*WIvrm$*@?d`X?4|TBu~IB39gP&)t}<
z|8amQaI6jC9P6D<)9@;-&iqZ==_lw-7{l(hC?&bcR2U!0DflgJmQtY61g_ut;3m!$
z715g`ZJ6MeWD`|yE{ndaac&+8EK9KlLc!cinlszff0Rc@3v6ardkR
zHuIHIFa;9(g?1afKFJ6UuzXDOwf)RcVVy~{vVls=S&7K)J;12C`hlj|iviDiN243l
zr}&(sd!AD1^@(%Pm6D!1+FpG?m);hZgvI-HwMVMO+_6;`|=C3LyAJDs&2b2gMMr
z+CNTj#g%i*zlQ4A%_ij26x((*gR;vo6}D{C+Zg>tO28E^x6O$E6=+|)q1__Q{=mt?
zeiESm1+CnZdW53AyF)0NYxvj|*>&VGE?
zz5>n?{E_mQ2g}>7CNQyk?=BZ?i*)e(P!#uOH&_Ru1)XHO{h6upOdv5QO!32YD%XwU
z&P9O!$9vO<^Gsd?2~wZ`$B3oH3T5TVH@-C7(ODh6QX6RKi$ymks%O1m-Z_f|2j8{l
zYzk-S6X$kAL{{VqqVSn=?xGXeZ>CDt_;?}VjktS`pK9I5$w1&LV(MoK&Z5qZQvxQ$
z@>$_W**V7yvza0+kra3pg4b+I=G!ha^)0IT${N{GI!-t8dc>qt;e3RC??UQ1`x*Y;
zm<8d9gla1P@DbCg55+QVI%of-jh~^yPm>gxNa8x88C2RlUEBA5nKN`?
zds5pJ7yifvf9#~Jx(~lflfiP%!6?P!L&FXzmDUkBC7!}d)aste&ar$5ZJ~Ev9!^J+
z?f-DEl<8yPfj9dY8rO{{zmFW+2*1K(<5bb$$_i~$Zf-Enl_(wRZ;M|k;_n3+`1lyA
z{k8YH<#cvI*$r$q?M1;>m64;l3UsMcmuJR-Fi4^UJ560{RZ_JYqB?;YVkw|eEV^#O
zbJbNf&eQKr5qe2}3Eo9^+zd-~(
zU#g)C$yE!9AW%j7Llo+5bSFaf*hmHib64c#GqE{vz;Gq#g;)=*$NanYX~v4AEi
zGk&>B8uABB^s{0??tQA>YndS>$VHN&POdhCr$^QU=|pvQX1aS^UU`%2?x>9b0&_ay
z@qLe{Dzv+(+4n_ZBQF25a3q+UarWNa*t7QqHvT65*ta@R12ss&D6lB%CKx@B;={`?
z5XY$`jL18p?^@cNzr@qbWNVl)>q^&%vE>$-Z@S4&?U7|LNyYm)p>fSb-w?P!J{gUO
z$w|wFxdm06;Zh|Mqs#x8P{SXFP+A13&O1GS5;S-CdKbQapi9X}Ww|hq8YoTRoI<;S
zzPVeT)N)K#jEhpVBBJqQ5Np@nNa?CCt+`MXWbVB-eebbWV0oRIbKCHvm`jYvYuOlI
zQVidPrAncCa+F6VXlRMoC8o0kkDd@Tvv?UaOD-D6o};_wm@Ciw?xaeaJ?;uky=}r7
z$tvt61MZXh3o6}DzX8D!Ivm#qR?5XhqFOoX?|sOM5Wzu9PEeXOUb!H)@
z_#M#3shm9KUAKQYV?&k|um-WJ3>z6HvKF!be69;1@*(n=-U7Mm{Whc~G*_b}2T^vw
z7SH&Vym>{-m6ARw0(sW`D&9ng+GBf;+0YmieV6Bu^QM=C!@j!r&XJa}Ck#k_K-p@v
zsqPxYkd-25sZD!wRBOh~
zYj}qt+p(bju7(e$Z6s?}dUQK|}S~c^b=Xk21gwO83DjzW;2+0lZdCjM!
z{r*tV?e&cM7E^sG>htZP6E0F*z@XY6L-w25?l@pvB{_tlpiLZ4hBa(8N24<*3%dBk
zQr&Ar<~|YrGE;DmAl(y|IQSk4tC&9^Mmo>59lxvwzVDMRgm>O<<9CQl6}#tkl{uq+
ztwe^7bAw%9KCIG%Fji*_(Jg5<5ool%Y2H0NLL2pv1IPS5v=eN97sTq?M*m*eiJ@nALq#6CsIn+>gHmYR;&Mi8%)s*zM|((j4k;S(A|ewt>dNNl-8o$-lLqmjzo4Ke@lNw<93#}?!ow>HQy^
zUYW83bH5J*bK^)nIKy8Sd6{3vUF2NmEs&{`kY2fb_>6kHTo=7OHbb?#2`LGi+@q_T
zqAE6G#suq}+Bv7AP7r*k%2fbhcWRVL9ycOeYDC+%K!loM80H7^+6WW=<23eP1dxA*
z*tJKhBcLM@ZM7(5dfN@vL#(u|Uf?WOUxjGhRuxn4h4N!87tfMs@BH-HCo_-#7G)A2
zW`mE5D*TlJ*Go+sLi8G59vZI(rcs~#0Xq~4t9Q>UmeTs@_FyVX{fg!U@+@pAMwU%b
zuYYgqrVe_2SQI`n;IpcDL(Zf{Hc2af@JL(Q>5%6J}V&Jh4Dt4&`
z{@jP}yxPpibEvv*#wz37!dQFxH*&YE2=UWnQKdZjBX$Jk)>d
z$ez>o9)mOV*if+7dZ@w-P+VWgPvsB)1d;m?OE~?kcD@*{hVE`+l?K)G?`(cPo7bED
z_9d|d0;eL=p5(>oevP!Vrf!?@>qlv)VJ-iGrxH7eA>*O4PVX?<-BO+PV-ibUOw-NF
zohewQJ;AmcWZ@r0h}hu>vLTGzA)NC6(x4?mj3o57PtmxW^|;t*8i|3akV9mZ<*zHW
zjT(+yuwWLS#MV0=xLUQNS*rT;1llIC_1h`GJVmts`{7oMAad5Q_0#8YJQnZD
zXUGH&Imja^=|YZz}XmInJ20psc(YckG8&vRjCEQE?+X&tS+02lLCqyvEmk
zpwU`<<<6K~&ZseUr`SEOFd6cL&jnn)NL+@T(071il|{TCik(GqEwJJv^>d>aYbmg)
zP-C+}4u)bch0%MrbpR|RU^SwLxcOz1>?G*
zY;Ej2iqB|d17E@TUs&}~Lu{1zgHAURuykGhgDx{x0`Vky#OXv#mjQI9LOXw3>Fz&4
zHk_SW5T+W^ca!HAz*l~dT~o4)K(B5Yyj5|P;vny7gkC(sGkOPt07m|9`#eUnFq