From 6ef7931f6278bb47af5b1bd2ea59d19452c9574c Mon Sep 17 00:00:00 2001 From: Daniel Kaplan Date: Wed, 12 Jul 2023 23:05:13 -0400 Subject: [PATCH 1/9] PEFT v1, #1 --- NeoX/__init__.py | 0 NeoX/convert.py | 27 +++++++++++++++++++++++++++ NeoX/load_model.py | 27 +++++++++++++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 NeoX/__init__.py create mode 100644 NeoX/convert.py create mode 100644 NeoX/load_model.py diff --git a/NeoX/__init__.py b/NeoX/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/NeoX/convert.py b/NeoX/convert.py new file mode 100644 index 0000000..6d38a3a --- /dev/null +++ b/NeoX/convert.py @@ -0,0 +1,27 @@ +from torch import nn +import loralib as lora + +def convert_model_lora(model): + for child_name, child in model.named_children(): + if isinstance(child, nn.Linear) and child_name == "query_key_value": + weight = child.weight + bias = child.bias + new = lora.MergedLinear(child.in_features, child.out_features, r = 4) + new.weight = weight + new.bias = bias + setattr(model, child_name, new) + # elif isinstance(child, nn.Conv2d): + # weight = child.weight + # bias = child.bias + # new = lora.Conv2d(child.in_channels, child.out_channels, child.kernel_size[0], r = 4)#kernel size would + # new.weight = weight + # new.bias = bias + # setattr(model, child_name, new) + # elif isinstance(child, nn.Embedding): + # weight = child.weight + # new = lora.Embedding(child.num_embeddings, child.embedding_dim, r = 4) + # new.weight = weight + # setattr(model, child_name, new) + else: + convert_model_lora(child) + return model \ No newline at end of file diff --git a/NeoX/load_model.py b/NeoX/load_model.py new file mode 100644 index 0000000..a4167b9 --- /dev/null +++ b/NeoX/load_model.py @@ -0,0 +1,27 @@ + +#dummy config +from transformers import GPTNeoXForCausalLM, AutoTokenizer +from convert import convert_model_lora +import torch +initial = True +from safetensors.torch import save_file, load_file + +if initial:#Load up a GPT Neo-x model specified by the config, convert to the lora model desired. + + model = GPTNeoXForCausalLM.from_pretrained("EleutherAI/pythia-70m-deduped") + tokenizer = AutoTokenizer.from_pretrained("EleutherAI/pythia-70m-deduped") + model = convert_model_lora(model) + + # torch.save(model.state_dict(), "./model.pt") + model.save_pretrained("./", safe_serialization = "True") + +else: + #We want to load a model + + model = GPTNeoXForCausalLM.from_pretrained("EleutherAI/pythia-70m-deduped")#Is it possible to just load from config without this issue... + model = convert_model_lora(model) + #We could skip the above step if we coded something that has the new architecture - this seems bad though because we'd need to do per adapter method + + loaded = load_file("./model.safetensors") + model.load_state_dict(loaded) + \ No newline at end of file From af3fe29d52ecb92d7d412f20b9e2eb87f644e9b4 Mon Sep 17 00:00:00 2001 From: Daniel Kaplan Date: Wed, 12 Jul 2023 23:09:13 -0400 Subject: [PATCH 2/9] Initial perceiver resampler code + test #3 --- PerceiverResampler/perceiver.py | 281 ++++++++++++++++++++++++++++++++ PerceiverResampler/test.py | 59 +++++++ 2 files changed, 340 insertions(+) create mode 100644 PerceiverResampler/perceiver.py create mode 100644 PerceiverResampler/test.py diff --git a/PerceiverResampler/perceiver.py b/PerceiverResampler/perceiver.py new file mode 100644 index 0000000..21b0a10 --- /dev/null +++ b/PerceiverResampler/perceiver.py @@ -0,0 +1,281 @@ +""" +Based on: https://github.com/lucidrains/flamingo-pytorch +""" + +import torch +from einops import rearrange, repeat +from einops_exts import rearrange_many +from torch import einsum, nn + + +def exists(val): + return val is not None + + +def FeedForward(dim, mult=4): + inner_dim = int(dim * mult) + return nn.Sequential( + nn.LayerNorm(dim), + nn.Linear(dim, inner_dim, bias=False), + nn.GELU(), + nn.Linear(inner_dim, dim, bias=False), + ) + + +class PerceiverAttention(nn.Module): + def __init__(self, *, dim, dim_head=64, heads=8): + super().__init__() + self.scale = dim_head**-0.5 + self.heads = heads + inner_dim = dim_head * heads + + self.norm_media = nn.LayerNorm(dim) + self.norm_latents = nn.LayerNorm(dim) + + self.to_q = nn.Linear(dim, inner_dim, bias=False) + self.to_kv = nn.Linear(dim, inner_dim * 2, bias=False) + self.to_out = nn.Linear(inner_dim, dim, bias=False) + + def forward(self, x, latents): + """ + Args: + x (torch.Tensor): image features + shape (b, T, n1, D) + latent (torch.Tensor): latent features + shape (b, T, n2, D) + """ + x = self.norm_media(x) + latents = self.norm_latents(latents) + + h = self.heads + + q = self.to_q(latents) + kv_input = torch.cat((x, latents), dim=-2) + k, v = self.to_kv(kv_input).chunk(2, dim=-1) + q, k, v = rearrange_many((q, k, v), "b t n (h d) -> b h t n d", h=h) + q = q * self.scale + + # attention + sim = einsum("... i d, ... j d -> ... i j", q, k) + sim = sim - sim.amax(dim=-1, keepdim=True).detach() + attn = sim.softmax(dim=-1) + + out = einsum("... i j, ... j d -> ... i d", attn, v) + out = rearrange(out, "b h t n d -> b t n (h d)", h=h) + return self.to_out(out) + + +class PerceiverResampler(nn.Module): + def __init__( + self, + *, + dim, + depth=6, + dim_head=64, + heads=8, + num_latents=64, + max_num_media=None, + max_num_frames=None, + ff_mult=4, + ): + super().__init__() + self.latents = nn.Parameter(torch.randn(num_latents, dim)) + self.frame_embs = ( + nn.Parameter(torch.randn(max_num_frames, dim)) + if exists(max_num_frames) + else None + ) + self.media_time_embs = ( + nn.Parameter(torch.randn(max_num_media, 1, dim)) + if exists(max_num_media) + else None + ) + + self.layers = nn.ModuleList([]) + for _ in range(depth): + self.layers.append( + nn.ModuleList( + [ + PerceiverAttention(dim=dim, dim_head=dim_head, heads=heads), + FeedForward(dim=dim, mult=ff_mult), + ] + ) + ) + + self.norm = nn.LayerNorm(dim) + + def forward(self, x): + """ + Args: + x (torch.Tensor): image features + shape (b, T, F, v, D) + Returns: + shape (b, T, n, D) where n is self.num_latents + """ + b, T, F, v = x.shape[:4] + + # frame and media time embeddings + if exists(self.frame_embs): + frame_embs = repeat(self.frame_embs[:F], "F d -> b T F v d", b=b, T=T, v=v) + x = x + frame_embs + x = rearrange( + x, "b T F v d -> b T (F v) d" + ) # flatten the frame and spatial dimensions + if exists(self.media_time_embs): + x = x + self.media_time_embs[:T] + + # blocks + latents = repeat(self.latents, "n d -> b T n d", b=b, T=T) + for attn, ff in self.layers: + latents = attn(x, latents) + latents + latents = ff(latents) + latents + return self.norm(latents) + + +# gated cross attention +class MaskedCrossAttention(nn.Module): + def __init__( + self, + *, + dim, + dim_visual, + dim_head=64, + heads=8, + only_attend_immediate_media=True, + ): + super().__init__() + self.scale = dim_head**-0.5 + self.heads = heads + inner_dim = dim_head * heads + + self.norm = nn.LayerNorm(dim) + + self.to_q = nn.Linear(dim, inner_dim, bias=False) + self.to_kv = nn.Linear(dim_visual, inner_dim * 2, bias=False) + self.to_out = nn.Linear(inner_dim, dim, bias=False) + + # whether for text to only attend to immediate preceding image, or all previous images + self.only_attend_immediate_media = only_attend_immediate_media + + def forward(self, x, media, media_locations=None, use_cached_media=False): + """ + Args: + x (torch.Tensor): text features + shape (B, T_txt, D_txt) + media (torch.Tensor): image features + shape (B, T_img, n, D_img) where n is the dim of the latents + media_locations: boolean mask identifying the media tokens in x + shape (B, T_txt) + use_cached_media: bool + If true, treat all of x as if they occur after the last media + registered in media_locations. T_txt does not need to exactly + equal media_locations.shape[1] in this case + """ + + if not use_cached_media: + assert ( + media_locations.shape[1] == x.shape[1] + ), f"media_location.shape is {media_locations.shape} but x.shape is {x.shape}" + + T_txt = x.shape[1] + _, T_img, n = media.shape[:3] + h = self.heads + + x = self.norm(x) + + q = self.to_q(x) + media = rearrange(media, "b t n d -> b (t n) d") + + k, v = self.to_kv(media).chunk(2, dim=-1) + q, k, v = rearrange_many((q, k, v), "b n (h d) -> b h n d", h=h) + + q = q * self.scale + + sim = einsum("... i d, ... j d -> ... i j", q, k) + + if exists(media_locations): + media_time = torch.arange(T_img, device=x.device) + 1 + + if use_cached_media: + # text time is set to the last cached media location + text_time = repeat( + torch.count_nonzero(media_locations, dim=1), + "b -> b i", + i=T_txt, + ) + else: + # at each boolean of True, increment the time counter (relative to media time) + text_time = media_locations.cumsum(dim=-1) + + # text time must equal media time if only attending to most immediate image + # otherwise, as long as text time is greater than media time (if attending to all previous images / media) + mask_op = torch.eq if self.only_attend_immediate_media else torch.ge + + text_to_media_mask = mask_op( + rearrange(text_time, "b i -> b 1 i 1"), + repeat(media_time, "j -> 1 1 1 (j n)", n=n), + ) + sim = sim.masked_fill(~text_to_media_mask, -torch.finfo(sim.dtype).max) + + sim = sim - sim.amax(dim=-1, keepdim=True).detach() + attn = sim.softmax(dim=-1) + + if exists(media_locations) and self.only_attend_immediate_media: + # any text without a preceding media needs to have attention zeroed out + text_without_media_mask = text_time == 0 + text_without_media_mask = rearrange( + text_without_media_mask, "b i -> b 1 i 1" + ) + attn = attn.masked_fill(text_without_media_mask, 0.0) + + out = einsum("... i j, ... j d -> ... i d", attn, v) + out = rearrange(out, "b h n d -> b n (h d)") + return self.to_out(out) + + +class GatedCrossAttentionBlock(nn.Module): + def __init__( + self, + *, + dim, + dim_visual, + dim_head=64, + heads=8, + ff_mult=4, + only_attend_immediate_media=True, + ): + super().__init__() + self.attn = MaskedCrossAttention( + dim=dim, + dim_visual=dim_visual, + dim_head=dim_head, + heads=heads, + only_attend_immediate_media=only_attend_immediate_media, + ) + self.attn_gate = nn.Parameter(torch.tensor([0.0])) + + self.ff = FeedForward(dim, mult=ff_mult) + self.ff_gate = nn.Parameter(torch.tensor([0.0])) + + def forward( + self, + x, + media, + media_locations=None, + use_cached_media=False, + ): + x = ( + self.attn( + x, + media, + media_locations=media_locations, + use_cached_media=use_cached_media, + ) + * self.attn_gate.tanh() + + x + ) + x = self.ff(x) * self.ff_gate.tanh() + x + + return x + + \ No newline at end of file diff --git a/PerceiverResampler/test.py b/PerceiverResampler/test.py new file mode 100644 index 0000000..25878e6 --- /dev/null +++ b/PerceiverResampler/test.py @@ -0,0 +1,59 @@ +import open_clip +import torch +from einops import rearrange, repeat +from einops_exts import rearrange_many +from torch import einsum, nn + + + +from perceiver import PerceiverResampler + + +vision_encoder, _, image_processor = open_clip.create_model_and_transforms( + "ViT-L-14", pretrained="openai") + +vision_encoder = vision_encoder.visual +vis_dim=open_clip.get_model_config("ViT-L-14")["vision_cfg"]["width"] +perceiver = PerceiverResampler(dim=vis_dim) + + + +batch = 5 +num_images = 27 +channels = 3 +height = 512 +width = 512 +input_data = torch.randn((batch, num_images, 1, channels, height, width)) +# vision_x (torch.Tensor): Vision input +# shape (B, T_img, F, C, H, W) with F=1 + + + +def encode_vision_x(vision_x: torch.Tensor): + """ + Compute media tokens from vision input by passing it through vision encoder and conditioning language model. + Args: + vision_x (torch.Tensor): Vision input + shape (B, T_img, F, C, H, W) + Images in the same chunk are collated along T_img, and frames are collated along F + Currently only F=1 is supported (single-frame videos) + + rearrange code based on https://github.com/dhansmair/flamingo-mini + """ + + assert vision_x.ndim == 6, "vision_x should be of shape (b, T_img, F, C, H, W)" + b, T, F = vision_x.shape[:3] + assert F == 1, "Only single frame supported" + + vision_x = rearrange(vision_x, "b T F c h w -> (b T F) c h w") + with torch.no_grad(): + vision_x = vision_encoder(vision_x)[1] + + + vision_x = rearrange(vision_x, "(b T F) v d -> b T F v d", b=b, T=T, F=F) + vision_x = perceiver(vision_x) + + # for layer in lang_encoder._get_decoder_layers(): + # layer.condition_vis_x(vision_x) + +encode_vision_x(input_data) \ No newline at end of file From cfbc1806aeb19215f5373d78ba83fcea92d1b79d Mon Sep 17 00:00:00 2001 From: "connor.brennan" Date: Tue, 18 Jul 2023 22:25:49 -0400 Subject: [PATCH 3/9] Added basic logic for Layer Extensions --- LayerExtender/__init__.py | 1 + .../BaseLayerExtender.cpython-39.pyc | Bin 0 -> 809 bytes .../__pycache__/LayerExtender.cpython-39.pyc | Bin 0 -> 805 bytes .../__pycache__/__init__.cpython-39.pyc | Bin 0 -> 183 bytes .../__pycache__/convert.cpython-39.pyc | Bin 0 -> 529 bytes .../__pycache__/layer_extender.cpython-39.pyc | Bin 0 -> 3249 bytes .../__pycache__/print_model.cpython-39.pyc | Bin 0 -> 444 bytes .../__pycache__/utils.cpython-39.pyc | Bin 0 -> 2182 bytes LayerExtender/config.json | 13 +++ LayerExtender/layer_extender.py | 78 ++++++++++++++++++ LayerExtender/load_model.py | 61 ++++++++++++++ LayerExtender/utils.py | 60 ++++++++++++++ 12 files changed, 213 insertions(+) create mode 100644 LayerExtender/__init__.py create mode 100644 LayerExtender/__pycache__/BaseLayerExtender.cpython-39.pyc create mode 100644 LayerExtender/__pycache__/LayerExtender.cpython-39.pyc create mode 100644 LayerExtender/__pycache__/__init__.cpython-39.pyc create mode 100644 LayerExtender/__pycache__/convert.cpython-39.pyc create mode 100644 LayerExtender/__pycache__/layer_extender.cpython-39.pyc create mode 100644 LayerExtender/__pycache__/print_model.cpython-39.pyc create mode 100644 LayerExtender/__pycache__/utils.cpython-39.pyc create mode 100644 LayerExtender/config.json create mode 100644 LayerExtender/layer_extender.py create mode 100644 LayerExtender/load_model.py create mode 100644 LayerExtender/utils.py diff --git a/LayerExtender/__init__.py b/LayerExtender/__init__.py new file mode 100644 index 0000000..16281fe --- /dev/null +++ b/LayerExtender/__init__.py @@ -0,0 +1 @@ +from .utils import * diff --git a/LayerExtender/__pycache__/BaseLayerExtender.cpython-39.pyc b/LayerExtender/__pycache__/BaseLayerExtender.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..93e81645f5e61bc598613bfbf18fb06169a1f420 GIT binary patch literal 809 zcmZWoy>8S%5T5bh{pbz?iV)%fKyG4*8X<&)C<384DmN?R-AjDfYqNWokm$NZO2-qV z04dGA01vY*6|X?W%$g{NGSYrte>0w$Z+2Zwr(+ha1P(`2OzA2HHbjM3^sU>hAy_NP8l~8qSDn;2$N7a@sU;a z+$CkJ=hdMnspLqAvM#I@Vgr9ZWJhhIvZktwtjs7-w`T8?QCb%|>rA^)<% zv;Vr!?&#U;PR)oa^Q^2N7>O zYy5^tCp!Z(AeV^=($Mg&FgpY;`Fv8CLu>9a$$Rhgx(#=39x^@sH{?>)HJ!(8GZPNsjtoomXUm%* m?#7-uNMjc-=>i`4{C!Fug78S%5T5bh{pbz?iV)%fKyJW>DmN?R-AjDB}YQ!WoC>JTlhOqj+#m(RZ(V1o={-j=-Fqj>N=~FRyQk^cP2S5q-u1T z-0A-AY1W?V5mn_m88<8(duPw?X`bEm7^4k~x?OiUi@Y%w%WuzVnI{ArSiuIDSdi#z z{EkSsx&S>OmktSHU-K8C_XuqCg`kf;iFbDWX97F^Gc(44TX@8G%BYjJFuI z{D35DX-Q^I@k)jw7N97Y_~oXbk)NBYpPQMJsGqE#oS&DMU!<2*l$w{9n5SP*l%JKF zTvDuGmYI^8UzDTolUSKrrXoSI8X7iZ%wt~jCYnWr zBN;!zESAYBjC!9m2_xpXQdGANMTZmdi2CHgppvLd&-oh(0iQ>d-7ysl&+r1raK3xS zmwXu+W+THF(TeLG6I<$5d>Na>g5@?|(e|6Z>DkD#75CTzX<*aQn#+`EP!{F9s-;Hd z231$7&Inba_7YrgQYjQ#lW%Zq9U5VbcEhq&cc@L#v=Zx4;*tPdkIpZvD>n#)g(r7O zz;0dPeNlUlub?wx`#Q43@&Et; literal 0 HcmV?d00001 diff --git a/LayerExtender/__pycache__/layer_extender.cpython-39.pyc b/LayerExtender/__pycache__/layer_extender.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6ed1ec9c49314661c55ee7a9cd542d3fe775ec95 GIT binary patch literal 3249 zcmcguOLH7G5SBDE`|>&o!B7d$;ZeX+#cpxo1QZY+1?4y?VqSBZl1JM0*t0W3Yu3iD z@hPD=!7s=mp}6=L@H@Eb%89?giEhoVy^C{!Dwr+xO08E*{q@(9>+3y+->0Acs%~NJ zcWNwuJ~rONVgQ9=if3%fBR&<8;M5kGH}xZb8brag6}6ns%i2*J`@F0Zb=`k2>V3kL zuY!9_1)A@z-SwhAYFes|nzrs^dmY;y)x~z#wJ#~}4jc6Tz^Ju5S9gD-MyZzVK(`3vR^vxybWBCc2@6puXZ?*&TS=RdG9SDy0k^`n+< ztKdF&R{2p!cU52abWe5f3)NF=_kz1TTKj?dEb60W>-Opg(&(Fonq~TR}mL6)J+LkT-+CE+N{^I2)PzPJXcbDV=xjWsWlOE%dB4n6n8+-n5woh8$||MiQFxHbu)+n$g@!{3DlE@8 z`lguSq@4Ch$4741fy`#wxIws-GjP6Bn8_mzXX{Cr;Hgx&kBd%lJjkhW)}L~FY1O#I zTk;BlN?=*TVm44z>@Iu2A2QBrQL|Vmc88tvEp&8XW~Bz(iQ_a+t2l-(4j=f|rwg#H zq{zqV#3^mdnPG76ZXCP2jN?=G=jLd)nCj6q&EzN<;e@<^;ST6Lm-(oKVoxWP86Bib z7m$rM>3|Dwj@)r$J$LS~Jc2%Me}&AZGcev0$4_5G!Ej+M+7hIBIja&n?&2d%()3$c z%rhuzw#RBd;kEdgf6h`?d%y6X_=#WpcNkE7LI=fZ0P9qL0)_SQ#tmG?YIJ#V)5Wa9 ztqfeHQBFKgL)+uRd7b9X&A!HupL&GZ^MRI~3fs-)RL5CrD%-7&%BH$<3_22`o!wZh zt2l;**!sBH`zcia-IPvhVuWR~d!6REJ9+d4F2)T6&`>8%*9ivBtl0MAIqKus_T%{5 znat)r?KoCNf=Hp|ZL0}0=Qkh>!ZlL&O-0c$)zxmHtY<2KXGy<4M^2)%43{{+N%A?+ zbwV2jxx1ebilojyU;%I8pm8uJ#4e?~u!Xu5c#9s3yhY6Y3czqvTOaYi!e zI>$0)noq*b&6}YczF?E4f%JoHL_9Fd6?!+XkXvRLR!9sGmT9S73?qSLXDo!A<3fqZ zM`@Eza-;-gEMx@@W0@dJ%^*x>2KlC82NNEq)vnuTcQ!!+o48@yAk!SVqgjnu<3p{> zJEhiY=*}|Wc71_l7k3vZgD%ri2Dzo7j2dI+q02<`NI}5R5a;iK-vrDhQoKfj;su;5 z&%(ovdhf>K>BIue7;}~%TA#?yL4k2G+r51Y(IaKWNlb{lN9@eN8D6Yf=bN-M1 ziKtN%vpPV@m0x=Y3_h${HNFR~v%8$Ls*Q=Mvy8Gqu=NKyXdB-o9HKRy%bg6_5lysy zrA;;H+SZ{?C%cvPcT#DrPkmAUriO`Dn-~NV>{^<~V=b#0Xk+_G+~=*J1Prbv6t|X8 zOxdqN@f%bdl6VB3-$cD3J>xC>T_Y(vyf2QQSib825!mmLMQ6a?Sg~S`>4xbkZLEYC z`IIK!pcOtxQFHX0cz|!why27tUy1zyUaW-@wfDWa&Q65#PJArrRpsvs_^jd&@L8dP zhvLL{b$sJtJ*fTSb{!zawzn)F0wuGLZU2+o+grN$8WV@kj<3A_?M%Z_yswX9_bxha zzux!|A1D4e%@yX?+~qX8Hm@Ye%7Yd>jmrV0ciY3qmCju1iq^&{m@2M{eVyB$%UR9r zp3AbXz7(mB87MWqkHx)}nDBXzlQVtp)Bi=cJ}P?xMfuXg=M+GkLugtgG;gEt5*q0T zmu*lLknA?p;-Sj6&`@M`Ag2J)0+5UFl>nFyc4V?|+l2Tm-GLGHQtE3+iBhLy7ofG5 hj5%Xt+j$%B3Ln((k~0i3_Z7Z@r^KIk^~o!j{{oEn`5gcN literal 0 HcmV?d00001 diff --git a/LayerExtender/__pycache__/print_model.cpython-39.pyc b/LayerExtender/__pycache__/print_model.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c85b9ac276e5cb4125b67d17410f920d1d2e3311 GIT binary patch literal 444 zcmYjNy-EW?5T4z=i${J$tSl^2dQG;mQW5c|5G0LVP{Mh;kdwQ;b8i(BE>(Ps5YW;W z@L?9Ltb7G4XIF#_%+I$o^L@i6yyi!4ug%F|M*QV;HsUL}fGy1bXE zsrgeTWT|t0l}x3+cs3GgHMUXzPjXOAbqo2J`+x`okS(%Hwuyfo#>|GM)agXKU-NHX Wel2vdVWT6w`!Sct1X1l~yYv^>!fLSq literal 0 HcmV?d00001 diff --git a/LayerExtender/__pycache__/utils.cpython-39.pyc b/LayerExtender/__pycache__/utils.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..72cd200d2450ae3fac9b6e4448cf96478f522c52 GIT binary patch literal 2182 zcmZuyOOG5i5VqaV>3QtzMhOo=9)Ln7LNgHu5NH(zvXC626;gtPT5MXqZO?9R`Y~&J zLpJI;&7P1zfol?JSK`7Q34RDCUpa9@;>L+8&(7unZn?V3E|<%$ugYn+8xd%yZ~n@E z2?_ZFh3A_K!ZqmXTOgcpT9A^)lp@axRywg$y0L3zr|?QY_AT!gK^%gPSG39~j>>l2 zJ|6GH9T<0uZrl~o4Z?jM91tGN*o?+K9`e=!iI;d^z{lH981L}zffq0HW!~dU2Q*&c z?rkzy`4jvh18P`R4VYQ}G*#kOx-aBs545Pbkfw91mgzzQ#r};i?yQUYt4}m2=)A7h zVO@)-I?akyDdUZEQSc1SKHnS=u0dC~fCv&(PGa^Y5sq* z@OpnD;5r#x$yhumcD67Gm9BxWcRcSSv$a+U)T8-}KT-34{y%4t1p3Vz3oa{?LBWXcb2+)Xy zv;*G?wc{Csu|-3OV!sYuy#=J9oW4e;j5F>WkcO?18J#+sG%(XRGY9V8T?h18q|C2c zWfT@J8d~?E--fPU2eJigH&&)plQGG`%1`ur`UBeqAld^D=3W7w*zk&JS;H9BXICO$ z1uG_6Epnx^x}01+4f5a6$-@h9Klg>y7!y$pSBpAJ3w0Hkg)Rj9%nM3MUTGn#v>3R? zv&)PR3gdh$?wxFo2v8u7lXO+<22Ncb`L%79MLA+@n>w>f`1^NG_AuB_HmDx{Htj} zxcxDMfQ36P4B0f;z02Kc1aic^drbYN+lL*{2l*~eAEmVnfRJZ{>I_KcsyYXn*&)VhM?rK7(O zV;@03PQE@pOC^pCm_>;tP+$--gHy(XQlD*0tWfeI5`3Q|mcMwYWm>6mElVNQ(3S-? z+?wd567_?*oL5`PVnkvJ$3X$FBtR&3kumL)_fRN_)xef~iBXa7A;GF_YlA$69Nxe6 m_PJBp^VxbAuFl_?t5{ss86X|nhj4~KLnt$h_QNn-IrlG`%N85} literal 0 HcmV?d00001 diff --git a/LayerExtender/config.json b/LayerExtender/config.json new file mode 100644 index 0000000..d420510 --- /dev/null +++ b/LayerExtender/config.json @@ -0,0 +1,13 @@ +{ + "extend_layers" : [ + { + "new_layer": "LoraExtender", + "params": {"test": 1}, + "use_default_match": true + }, + { + "match_type": "*GPTNeoXAttention", + "new_layer": "BaseLayerExtender" + } +] +} \ No newline at end of file diff --git a/LayerExtender/layer_extender.py b/LayerExtender/layer_extender.py new file mode 100644 index 0000000..32513f5 --- /dev/null +++ b/LayerExtender/layer_extender.py @@ -0,0 +1,78 @@ +from torch import nn, Tensor +from typing import Optional, Tuple, Union +import traceback +import loralib as lora +import fnmatch + + +class BaseModuleExtender(nn.Module): + """ + Base code to wrap an arbitary nn.Module into a new class. + + The original module is saved as self.wrapped_module and called automatically via the foerward pass. + + You can modify the values passed into wrapped layer via the arg and kwarg params + """ + def __init__(self, config, wrapped_module: nn.Module, **kwargs): + super().__init__() + + self.wrapped_module = wrapped_module + + def forward(self, input: Tensor, * args, **kwargs): + wrapped_outputs = self.wrapped_module(input, * args, **kwargs) + return wrapped_outputs + + @staticmethod + def is_match(name_list: str = "", type_list: str = ""): + return False + + +class BaseLayerExtender(BaseModuleExtender): + """ + Case code for wrapping LLM Layers. + + Layers are the fundamental unit for pipe parallel. + + This class is designed to interface our custom layers with the huggingface Trainer class and DeepSpeed. + """ + def __init__(self, config, wrapped_layer: nn.Module, **kwargs): + super().__init__(config, wrapped_layer) + + def forward(self, input: Tensor, * args, **kwargs): + wrapped_outputs = self.wrapped_module(input, * args, **kwargs) + return wrapped_outputs + +class LoraExtender(BaseModuleExtender): + """ + Wrapper for Lora adapaters + """ + def __init__(self, config, wrapped_module: nn.Module, **kwargs): + weight = wrapped_module.weight + bias = wrapped_module.bias + wrapped_module = lora.MergedLinear(wrapped_module.in_features, wrapped_module.out_features, r = 4, enable_lora=[True]) + wrapped_module.weight = weight + wrapped_module.bias = bias + + if "test" in kwargs: + print(kwargs["test"]) + + super().__init__(config, wrapped_module) + + def forward(self, input: Tensor, * args, **kwargs): + wrapped_outputs = self.wrapped_module(input, * args, **kwargs) + return wrapped_outputs + + @staticmethod + def is_match(name_list: str = "", type_list: str = ""): + model_name = type_list + first_token = model_name.find('.') + if first_token >= 0: + model_name = model_name[0:first_token] + + name_match = False + type_match = False + if model_name == "GPTNeoXModel": + name_match = fnmatch.fnmatchcase(name_list, "*query_key_value") + type_match = fnmatch.fnmatchcase(type_list, "*Linear") + + return name_match and type_match diff --git a/LayerExtender/load_model.py b/LayerExtender/load_model.py new file mode 100644 index 0000000..b91a76c --- /dev/null +++ b/LayerExtender/load_model.py @@ -0,0 +1,61 @@ + +#dummy config +from transformers import GPTNeoXForCausalLM, AutoTokenizer +from utils import print_model, convert_model +import torch +from safetensors.torch import save_file, load_file +import json + +with open('videorl/LayerExtender/config.json', 'r') as file: + config = json.load(file) + +initial = True + +if initial:#Load up a GPT Neo-x model specified by the config, convert to the lora model desired. + + model = GPTNeoXForCausalLM.from_pretrained("EleutherAI/pythia-70m-deduped") + tokenizer = AutoTokenizer.from_pretrained("EleutherAI/pythia-70m-deduped") + model = convert_model(model, config) + + print(config) + + print_model(model) + + model.save_pretrained("./", safe_serialization = "True") + + prompt = "Test" + input_ids = tokenizer(prompt, return_tensors="pt").input_ids + + gen_tokens = model.generate( + input_ids, + do_sample=True, + temperature=0.000001, + max_length=100, + ) + + print(tokenizer.batch_decode(gen_tokens)[0]) + + +else: + #We want to load a model + + model = GPTNeoXForCausalLM.from_pretrained("EleutherAI/pythia-70m-deduped")#Is it possible to just load from config without this issue... + tokenizer = AutoTokenizer.from_pretrained("EleutherAI/pythia-70m-deduped") + model = convert_model(model, {}) + #We could skip the above step if we coded something that has the new architecture - this seems bad though because we'd need to do per adapter method + + loaded = load_file("./model.safetensors") + model.load_state_dict(loaded) + + prompt = "Test" + input_ids = tokenizer(prompt, return_tensors="pt").input_ids + + gen_tokens = model.generate( + input_ids, + do_sample=True, + temperature=0.000001, + max_length=100, + ) + + print(tokenizer.batch_decode(gen_tokens)[0]) + \ No newline at end of file diff --git a/LayerExtender/utils.py b/LayerExtender/utils.py new file mode 100644 index 0000000..df0724e --- /dev/null +++ b/LayerExtender/utils.py @@ -0,0 +1,60 @@ +from torch import nn +from layer_extender import BaseLayerExtender, LoraExtender +from transformers.models.gpt_neox.modeling_gpt_neox import GPTNeoXAttention +import loralib as lora +from dataclasses import dataclass, field +import importlib +import fnmatch + +@dataclass +class ExtensionDefinition: + new_layer: str + match_name: str = "*" + match_type: str = "*" + params: dict = field(default_factory=dict) + use_default_match: bool = False + + +def convert_model(model, config): + extension_defs = config["extend_layers"] + + extensions = [ExtensionDefinition(**extension) for extension in extension_defs] + + + return convert_model_internal(model, config, extensions) + +def convert_model_internal(model, config, extensions, name_list: str = "", type_list: str = ""): + for child_name, child in model.named_children(): + new_layer = None + + name_list += f'{child_name}' if name_list == "" else f'.{child_name}' + type_list += f'{type(child).__name__}' if type_list == "" else f'.{type(child).__name__}' + + for extension in extensions: + class_type = getattr(importlib.import_module("layer_extender"), extension.new_layer) + if extension.use_default_match: + if class_type.is_match(name_list, type_list): + new_layer = class_type(config, child, **extension.params) + else: + name_match = True + if not extension.match_name == "*": + name_match = fnmatch.fnmatchcase(name_list, extension.match_name) + + type_match = True + if not extension.match_type == "*": + type_match = fnmatch.fnmatchcase(type_list, extension.match_type) + + if type_match and name_match: + new_layer = class_type(config, child, **extension.params) + + if not new_layer is None: + setattr(model, child_name, new_layer) + + convert_model_internal(child, config, extensions, name_list, type_list) + return model + + +def print_model(model, indent = ""): + for child_name, child in model.named_children(): + print(f'{indent}{child_name} ({type(child).__name__}):') + print_model(child, f'{indent} ') From cb62b49662fab64f80ad5bc13e3957b111f7fa91 Mon Sep 17 00:00:00 2001 From: "connor.brennan" Date: Tue, 18 Jul 2023 22:30:41 -0400 Subject: [PATCH 4/9] Removed code from other packages --- NeoX/convert.py | 27 --- NeoX/load_model.py | 27 --- PerceiverResampler/perceiver.py | 281 ---------------------------- PerceiverResampler/test.py | 59 ------ NeoX/__init__.py => __init__.py | 0 __pycache__/__init__.cpython-39.pyc | Bin 0 -> 147 bytes 6 files changed, 394 deletions(-) delete mode 100644 NeoX/convert.py delete mode 100644 NeoX/load_model.py delete mode 100644 PerceiverResampler/perceiver.py delete mode 100644 PerceiverResampler/test.py rename NeoX/__init__.py => __init__.py (100%) create mode 100644 __pycache__/__init__.cpython-39.pyc diff --git a/NeoX/convert.py b/NeoX/convert.py deleted file mode 100644 index 6d38a3a..0000000 --- a/NeoX/convert.py +++ /dev/null @@ -1,27 +0,0 @@ -from torch import nn -import loralib as lora - -def convert_model_lora(model): - for child_name, child in model.named_children(): - if isinstance(child, nn.Linear) and child_name == "query_key_value": - weight = child.weight - bias = child.bias - new = lora.MergedLinear(child.in_features, child.out_features, r = 4) - new.weight = weight - new.bias = bias - setattr(model, child_name, new) - # elif isinstance(child, nn.Conv2d): - # weight = child.weight - # bias = child.bias - # new = lora.Conv2d(child.in_channels, child.out_channels, child.kernel_size[0], r = 4)#kernel size would - # new.weight = weight - # new.bias = bias - # setattr(model, child_name, new) - # elif isinstance(child, nn.Embedding): - # weight = child.weight - # new = lora.Embedding(child.num_embeddings, child.embedding_dim, r = 4) - # new.weight = weight - # setattr(model, child_name, new) - else: - convert_model_lora(child) - return model \ No newline at end of file diff --git a/NeoX/load_model.py b/NeoX/load_model.py deleted file mode 100644 index a4167b9..0000000 --- a/NeoX/load_model.py +++ /dev/null @@ -1,27 +0,0 @@ - -#dummy config -from transformers import GPTNeoXForCausalLM, AutoTokenizer -from convert import convert_model_lora -import torch -initial = True -from safetensors.torch import save_file, load_file - -if initial:#Load up a GPT Neo-x model specified by the config, convert to the lora model desired. - - model = GPTNeoXForCausalLM.from_pretrained("EleutherAI/pythia-70m-deduped") - tokenizer = AutoTokenizer.from_pretrained("EleutherAI/pythia-70m-deduped") - model = convert_model_lora(model) - - # torch.save(model.state_dict(), "./model.pt") - model.save_pretrained("./", safe_serialization = "True") - -else: - #We want to load a model - - model = GPTNeoXForCausalLM.from_pretrained("EleutherAI/pythia-70m-deduped")#Is it possible to just load from config without this issue... - model = convert_model_lora(model) - #We could skip the above step if we coded something that has the new architecture - this seems bad though because we'd need to do per adapter method - - loaded = load_file("./model.safetensors") - model.load_state_dict(loaded) - \ No newline at end of file diff --git a/PerceiverResampler/perceiver.py b/PerceiverResampler/perceiver.py deleted file mode 100644 index 21b0a10..0000000 --- a/PerceiverResampler/perceiver.py +++ /dev/null @@ -1,281 +0,0 @@ -""" -Based on: https://github.com/lucidrains/flamingo-pytorch -""" - -import torch -from einops import rearrange, repeat -from einops_exts import rearrange_many -from torch import einsum, nn - - -def exists(val): - return val is not None - - -def FeedForward(dim, mult=4): - inner_dim = int(dim * mult) - return nn.Sequential( - nn.LayerNorm(dim), - nn.Linear(dim, inner_dim, bias=False), - nn.GELU(), - nn.Linear(inner_dim, dim, bias=False), - ) - - -class PerceiverAttention(nn.Module): - def __init__(self, *, dim, dim_head=64, heads=8): - super().__init__() - self.scale = dim_head**-0.5 - self.heads = heads - inner_dim = dim_head * heads - - self.norm_media = nn.LayerNorm(dim) - self.norm_latents = nn.LayerNorm(dim) - - self.to_q = nn.Linear(dim, inner_dim, bias=False) - self.to_kv = nn.Linear(dim, inner_dim * 2, bias=False) - self.to_out = nn.Linear(inner_dim, dim, bias=False) - - def forward(self, x, latents): - """ - Args: - x (torch.Tensor): image features - shape (b, T, n1, D) - latent (torch.Tensor): latent features - shape (b, T, n2, D) - """ - x = self.norm_media(x) - latents = self.norm_latents(latents) - - h = self.heads - - q = self.to_q(latents) - kv_input = torch.cat((x, latents), dim=-2) - k, v = self.to_kv(kv_input).chunk(2, dim=-1) - q, k, v = rearrange_many((q, k, v), "b t n (h d) -> b h t n d", h=h) - q = q * self.scale - - # attention - sim = einsum("... i d, ... j d -> ... i j", q, k) - sim = sim - sim.amax(dim=-1, keepdim=True).detach() - attn = sim.softmax(dim=-1) - - out = einsum("... i j, ... j d -> ... i d", attn, v) - out = rearrange(out, "b h t n d -> b t n (h d)", h=h) - return self.to_out(out) - - -class PerceiverResampler(nn.Module): - def __init__( - self, - *, - dim, - depth=6, - dim_head=64, - heads=8, - num_latents=64, - max_num_media=None, - max_num_frames=None, - ff_mult=4, - ): - super().__init__() - self.latents = nn.Parameter(torch.randn(num_latents, dim)) - self.frame_embs = ( - nn.Parameter(torch.randn(max_num_frames, dim)) - if exists(max_num_frames) - else None - ) - self.media_time_embs = ( - nn.Parameter(torch.randn(max_num_media, 1, dim)) - if exists(max_num_media) - else None - ) - - self.layers = nn.ModuleList([]) - for _ in range(depth): - self.layers.append( - nn.ModuleList( - [ - PerceiverAttention(dim=dim, dim_head=dim_head, heads=heads), - FeedForward(dim=dim, mult=ff_mult), - ] - ) - ) - - self.norm = nn.LayerNorm(dim) - - def forward(self, x): - """ - Args: - x (torch.Tensor): image features - shape (b, T, F, v, D) - Returns: - shape (b, T, n, D) where n is self.num_latents - """ - b, T, F, v = x.shape[:4] - - # frame and media time embeddings - if exists(self.frame_embs): - frame_embs = repeat(self.frame_embs[:F], "F d -> b T F v d", b=b, T=T, v=v) - x = x + frame_embs - x = rearrange( - x, "b T F v d -> b T (F v) d" - ) # flatten the frame and spatial dimensions - if exists(self.media_time_embs): - x = x + self.media_time_embs[:T] - - # blocks - latents = repeat(self.latents, "n d -> b T n d", b=b, T=T) - for attn, ff in self.layers: - latents = attn(x, latents) + latents - latents = ff(latents) + latents - return self.norm(latents) - - -# gated cross attention -class MaskedCrossAttention(nn.Module): - def __init__( - self, - *, - dim, - dim_visual, - dim_head=64, - heads=8, - only_attend_immediate_media=True, - ): - super().__init__() - self.scale = dim_head**-0.5 - self.heads = heads - inner_dim = dim_head * heads - - self.norm = nn.LayerNorm(dim) - - self.to_q = nn.Linear(dim, inner_dim, bias=False) - self.to_kv = nn.Linear(dim_visual, inner_dim * 2, bias=False) - self.to_out = nn.Linear(inner_dim, dim, bias=False) - - # whether for text to only attend to immediate preceding image, or all previous images - self.only_attend_immediate_media = only_attend_immediate_media - - def forward(self, x, media, media_locations=None, use_cached_media=False): - """ - Args: - x (torch.Tensor): text features - shape (B, T_txt, D_txt) - media (torch.Tensor): image features - shape (B, T_img, n, D_img) where n is the dim of the latents - media_locations: boolean mask identifying the media tokens in x - shape (B, T_txt) - use_cached_media: bool - If true, treat all of x as if they occur after the last media - registered in media_locations. T_txt does not need to exactly - equal media_locations.shape[1] in this case - """ - - if not use_cached_media: - assert ( - media_locations.shape[1] == x.shape[1] - ), f"media_location.shape is {media_locations.shape} but x.shape is {x.shape}" - - T_txt = x.shape[1] - _, T_img, n = media.shape[:3] - h = self.heads - - x = self.norm(x) - - q = self.to_q(x) - media = rearrange(media, "b t n d -> b (t n) d") - - k, v = self.to_kv(media).chunk(2, dim=-1) - q, k, v = rearrange_many((q, k, v), "b n (h d) -> b h n d", h=h) - - q = q * self.scale - - sim = einsum("... i d, ... j d -> ... i j", q, k) - - if exists(media_locations): - media_time = torch.arange(T_img, device=x.device) + 1 - - if use_cached_media: - # text time is set to the last cached media location - text_time = repeat( - torch.count_nonzero(media_locations, dim=1), - "b -> b i", - i=T_txt, - ) - else: - # at each boolean of True, increment the time counter (relative to media time) - text_time = media_locations.cumsum(dim=-1) - - # text time must equal media time if only attending to most immediate image - # otherwise, as long as text time is greater than media time (if attending to all previous images / media) - mask_op = torch.eq if self.only_attend_immediate_media else torch.ge - - text_to_media_mask = mask_op( - rearrange(text_time, "b i -> b 1 i 1"), - repeat(media_time, "j -> 1 1 1 (j n)", n=n), - ) - sim = sim.masked_fill(~text_to_media_mask, -torch.finfo(sim.dtype).max) - - sim = sim - sim.amax(dim=-1, keepdim=True).detach() - attn = sim.softmax(dim=-1) - - if exists(media_locations) and self.only_attend_immediate_media: - # any text without a preceding media needs to have attention zeroed out - text_without_media_mask = text_time == 0 - text_without_media_mask = rearrange( - text_without_media_mask, "b i -> b 1 i 1" - ) - attn = attn.masked_fill(text_without_media_mask, 0.0) - - out = einsum("... i j, ... j d -> ... i d", attn, v) - out = rearrange(out, "b h n d -> b n (h d)") - return self.to_out(out) - - -class GatedCrossAttentionBlock(nn.Module): - def __init__( - self, - *, - dim, - dim_visual, - dim_head=64, - heads=8, - ff_mult=4, - only_attend_immediate_media=True, - ): - super().__init__() - self.attn = MaskedCrossAttention( - dim=dim, - dim_visual=dim_visual, - dim_head=dim_head, - heads=heads, - only_attend_immediate_media=only_attend_immediate_media, - ) - self.attn_gate = nn.Parameter(torch.tensor([0.0])) - - self.ff = FeedForward(dim, mult=ff_mult) - self.ff_gate = nn.Parameter(torch.tensor([0.0])) - - def forward( - self, - x, - media, - media_locations=None, - use_cached_media=False, - ): - x = ( - self.attn( - x, - media, - media_locations=media_locations, - use_cached_media=use_cached_media, - ) - * self.attn_gate.tanh() - + x - ) - x = self.ff(x) * self.ff_gate.tanh() + x - - return x - - \ No newline at end of file diff --git a/PerceiverResampler/test.py b/PerceiverResampler/test.py deleted file mode 100644 index 25878e6..0000000 --- a/PerceiverResampler/test.py +++ /dev/null @@ -1,59 +0,0 @@ -import open_clip -import torch -from einops import rearrange, repeat -from einops_exts import rearrange_many -from torch import einsum, nn - - - -from perceiver import PerceiverResampler - - -vision_encoder, _, image_processor = open_clip.create_model_and_transforms( - "ViT-L-14", pretrained="openai") - -vision_encoder = vision_encoder.visual -vis_dim=open_clip.get_model_config("ViT-L-14")["vision_cfg"]["width"] -perceiver = PerceiverResampler(dim=vis_dim) - - - -batch = 5 -num_images = 27 -channels = 3 -height = 512 -width = 512 -input_data = torch.randn((batch, num_images, 1, channels, height, width)) -# vision_x (torch.Tensor): Vision input -# shape (B, T_img, F, C, H, W) with F=1 - - - -def encode_vision_x(vision_x: torch.Tensor): - """ - Compute media tokens from vision input by passing it through vision encoder and conditioning language model. - Args: - vision_x (torch.Tensor): Vision input - shape (B, T_img, F, C, H, W) - Images in the same chunk are collated along T_img, and frames are collated along F - Currently only F=1 is supported (single-frame videos) - - rearrange code based on https://github.com/dhansmair/flamingo-mini - """ - - assert vision_x.ndim == 6, "vision_x should be of shape (b, T_img, F, C, H, W)" - b, T, F = vision_x.shape[:3] - assert F == 1, "Only single frame supported" - - vision_x = rearrange(vision_x, "b T F c h w -> (b T F) c h w") - with torch.no_grad(): - vision_x = vision_encoder(vision_x)[1] - - - vision_x = rearrange(vision_x, "(b T F) v d -> b T F v d", b=b, T=T, F=F) - vision_x = perceiver(vision_x) - - # for layer in lang_encoder._get_decoder_layers(): - # layer.condition_vis_x(vision_x) - -encode_vision_x(input_data) \ No newline at end of file diff --git a/NeoX/__init__.py b/__init__.py similarity index 100% rename from NeoX/__init__.py rename to __init__.py diff --git a/__pycache__/__init__.cpython-39.pyc b/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..aeea9e8a0b6224bdd0fb6c64b1086d8762d8f0f6 GIT binary patch literal 147 zcmYe~<>g`kf;Gb1Q$X}%5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!H8enx(7s(x-} zPNIIYesX?ZUVf2YQc-GNUSghpK~a8IYH~@jepzNpYJO3UetdjpUS>&ryk0@&Ee@O9 Q{FKt1R6CG~pMjVG0JWnca{vGU literal 0 HcmV?d00001 From b2d5ffbd09337de52e79233521094b37189d3b01 Mon Sep 17 00:00:00 2001 From: Connor Date: Thu, 20 Jul 2023 03:48:43 -0400 Subject: [PATCH 5/9] Remvoed pycache --- __pycache__/__init__.cpython-39.pyc | Bin 147 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 __pycache__/__init__.cpython-39.pyc diff --git a/__pycache__/__init__.cpython-39.pyc b/__pycache__/__init__.cpython-39.pyc deleted file mode 100644 index aeea9e8a0b6224bdd0fb6c64b1086d8762d8f0f6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 147 zcmYe~<>g`kf;Gb1Q$X}%5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!H8enx(7s(x-} zPNIIYesX?ZUVf2YQc-GNUSghpK~a8IYH~@jepzNpYJO3UetdjpUS>&ryk0@&Ee@O9 Q{FKt1R6CG~pMjVG0JWnca{vGU From 62700f318b51dfc8f8307e947a964d8029ef955b Mon Sep 17 00:00:00 2001 From: Connor Date: Thu, 20 Jul 2023 03:51:57 -0400 Subject: [PATCH 6/9] Added pycache #2 --- .gitignore | 1 + .../__pycache__/BaseLayerExtender.cpython-39.pyc | Bin 809 -> 0 bytes .../__pycache__/LayerExtender.cpython-39.pyc | Bin 805 -> 0 bytes .../__pycache__/__init__.cpython-39.pyc | Bin 183 -> 0 bytes LayerExtender/__pycache__/convert.cpython-39.pyc | Bin 529 -> 0 bytes .../__pycache__/layer_extender.cpython-39.pyc | Bin 3249 -> 0 bytes .../__pycache__/print_model.cpython-39.pyc | Bin 444 -> 0 bytes LayerExtender/__pycache__/utils.cpython-39.pyc | Bin 2182 -> 0 bytes 8 files changed, 1 insertion(+) create mode 100644 .gitignore delete mode 100644 LayerExtender/__pycache__/BaseLayerExtender.cpython-39.pyc delete mode 100644 LayerExtender/__pycache__/LayerExtender.cpython-39.pyc delete mode 100644 LayerExtender/__pycache__/__init__.cpython-39.pyc delete mode 100644 LayerExtender/__pycache__/convert.cpython-39.pyc delete mode 100644 LayerExtender/__pycache__/layer_extender.cpython-39.pyc delete mode 100644 LayerExtender/__pycache__/print_model.cpython-39.pyc delete mode 100644 LayerExtender/__pycache__/utils.cpython-39.pyc diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ba0430d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__/ \ No newline at end of file diff --git a/LayerExtender/__pycache__/BaseLayerExtender.cpython-39.pyc b/LayerExtender/__pycache__/BaseLayerExtender.cpython-39.pyc deleted file mode 100644 index 93e81645f5e61bc598613bfbf18fb06169a1f420..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 809 zcmZWoy>8S%5T5bh{pbz?iV)%fKyG4*8X<&)C<384DmN?R-AjDfYqNWokm$NZO2-qV z04dGA01vY*6|X?W%$g{NGSYrte>0w$Z+2Zwr(+ha1P(`2OzA2HHbjM3^sU>hAy_NP8l~8qSDn;2$N7a@sU;a z+$CkJ=hdMnspLqAvM#I@Vgr9ZWJhhIvZktwtjs7-w`T8?QCb%|>rA^)<% zv;Vr!?&#U;PR)oa^Q^2N7>O zYy5^tCp!Z(AeV^=($Mg&FgpY;`Fv8CLu>9a$$Rhgx(#=39x^@sH{?>)HJ!(8GZPNsjtoomXUm%* m?#7-uNMjc-=>i`4{C!Fug78S%5T5bh{pbz?iV)%fKyJW>DmN?R-AjDB}YQ!WoC>JTlhOqj+#m(RZ(V1o={-j=-Fqj>N=~FRyQk^cP2S5q-u1T z-0A-AY1W?V5mn_m88<8(duPw?X`bEm7^4k~x?OiUi@Y%w%WuzVnI{ArSiuIDSdi#z z{EkSsx&S>OmktSHU-K8C_XuqCg`kf;iFbDWX97F^Gc(44TX@8G%BYjJFuI z{D35DX-Q^I@k)jw7N97Y_~oXbk)NBYpPQMJsGqE#oS&DMU!<2*l$w{9n5SP*l%JKF zTvDuGmYI^8UzDTolUSKrrXoSI8X7iZ%wt~jCYnWr zBN;!zESAYBjC!9m2_xpXQdGANMTZmdi2CHgppvLd&-oh(0iQ>d-7ysl&+r1raK3xS zmwXu+W+THF(TeLG6I<$5d>Na>g5@?|(e|6Z>DkD#75CTzX<*aQn#+`EP!{F9s-;Hd z231$7&Inba_7YrgQYjQ#lW%Zq9U5VbcEhq&cc@L#v=Zx4;*tPdkIpZvD>n#)g(r7O zz;0dPeNlUlub?wx`#Q43@&Et; diff --git a/LayerExtender/__pycache__/layer_extender.cpython-39.pyc b/LayerExtender/__pycache__/layer_extender.cpython-39.pyc deleted file mode 100644 index 6ed1ec9c49314661c55ee7a9cd542d3fe775ec95..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3249 zcmcguOLH7G5SBDE`|>&o!B7d$;ZeX+#cpxo1QZY+1?4y?VqSBZl1JM0*t0W3Yu3iD z@hPD=!7s=mp}6=L@H@Eb%89?giEhoVy^C{!Dwr+xO08E*{q@(9>+3y+->0Acs%~NJ zcWNwuJ~rONVgQ9=if3%fBR&<8;M5kGH}xZb8brag6}6ns%i2*J`@F0Zb=`k2>V3kL zuY!9_1)A@z-SwhAYFes|nzrs^dmY;y)x~z#wJ#~}4jc6Tz^Ju5S9gD-MyZzVK(`3vR^vxybWBCc2@6puXZ?*&TS=RdG9SDy0k^`n+< ztKdF&R{2p!cU52abWe5f3)NF=_kz1TTKj?dEb60W>-Opg(&(Fonq~TR}mL6)J+LkT-+CE+N{^I2)PzPJXcbDV=xjWsWlOE%dB4n6n8+-n5woh8$||MiQFxHbu)+n$g@!{3DlE@8 z`lguSq@4Ch$4741fy`#wxIws-GjP6Bn8_mzXX{Cr;Hgx&kBd%lJjkhW)}L~FY1O#I zTk;BlN?=*TVm44z>@Iu2A2QBrQL|Vmc88tvEp&8XW~Bz(iQ_a+t2l-(4j=f|rwg#H zq{zqV#3^mdnPG76ZXCP2jN?=G=jLd)nCj6q&EzN<;e@<^;ST6Lm-(oKVoxWP86Bib z7m$rM>3|Dwj@)r$J$LS~Jc2%Me}&AZGcev0$4_5G!Ej+M+7hIBIja&n?&2d%()3$c z%rhuzw#RBd;kEdgf6h`?d%y6X_=#WpcNkE7LI=fZ0P9qL0)_SQ#tmG?YIJ#V)5Wa9 ztqfeHQBFKgL)+uRd7b9X&A!HupL&GZ^MRI~3fs-)RL5CrD%-7&%BH$<3_22`o!wZh zt2l;**!sBH`zcia-IPvhVuWR~d!6REJ9+d4F2)T6&`>8%*9ivBtl0MAIqKus_T%{5 znat)r?KoCNf=Hp|ZL0}0=Qkh>!ZlL&O-0c$)zxmHtY<2KXGy<4M^2)%43{{+N%A?+ zbwV2jxx1ebilojyU;%I8pm8uJ#4e?~u!Xu5c#9s3yhY6Y3czqvTOaYi!e zI>$0)noq*b&6}YczF?E4f%JoHL_9Fd6?!+XkXvRLR!9sGmT9S73?qSLXDo!A<3fqZ zM`@Eza-;-gEMx@@W0@dJ%^*x>2KlC82NNEq)vnuTcQ!!+o48@yAk!SVqgjnu<3p{> zJEhiY=*}|Wc71_l7k3vZgD%ri2Dzo7j2dI+q02<`NI}5R5a;iK-vrDhQoKfj;su;5 z&%(ovdhf>K>BIue7;}~%TA#?yL4k2G+r51Y(IaKWNlb{lN9@eN8D6Yf=bN-M1 ziKtN%vpPV@m0x=Y3_h${HNFR~v%8$Ls*Q=Mvy8Gqu=NKyXdB-o9HKRy%bg6_5lysy zrA;;H+SZ{?C%cvPcT#DrPkmAUriO`Dn-~NV>{^<~V=b#0Xk+_G+~=*J1Prbv6t|X8 zOxdqN@f%bdl6VB3-$cD3J>xC>T_Y(vyf2QQSib825!mmLMQ6a?Sg~S`>4xbkZLEYC z`IIK!pcOtxQFHX0cz|!why27tUy1zyUaW-@wfDWa&Q65#PJArrRpsvs_^jd&@L8dP zhvLL{b$sJtJ*fTSb{!zawzn)F0wuGLZU2+o+grN$8WV@kj<3A_?M%Z_yswX9_bxha zzux!|A1D4e%@yX?+~qX8Hm@Ye%7Yd>jmrV0ciY3qmCju1iq^&{m@2M{eVyB$%UR9r zp3AbXz7(mB87MWqkHx)}nDBXzlQVtp)Bi=cJ}P?xMfuXg=M+GkLugtgG;gEt5*q0T zmu*lLknA?p;-Sj6&`@M`Ag2J)0+5UFl>nFyc4V?|+l2Tm-GLGHQtE3+iBhLy7ofG5 hj5%Xt+j$%B3Ln((k~0i3_Z7Z@r^KIk^~o!j{{oEn`5gcN diff --git a/LayerExtender/__pycache__/print_model.cpython-39.pyc b/LayerExtender/__pycache__/print_model.cpython-39.pyc deleted file mode 100644 index c85b9ac276e5cb4125b67d17410f920d1d2e3311..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 444 zcmYjNy-EW?5T4z=i${J$tSl^2dQG;mQW5c|5G0LVP{Mh;kdwQ;b8i(BE>(Ps5YW;W z@L?9Ltb7G4XIF#_%+I$o^L@i6yyi!4ug%F|M*QV;HsUL}fGy1bXE zsrgeTWT|t0l}x3+cs3GgHMUXzPjXOAbqo2J`+x`okS(%Hwuyfo#>|GM)agXKU-NHX Wel2vdVWT6w`!Sct1X1l~yYv^>!fLSq diff --git a/LayerExtender/__pycache__/utils.cpython-39.pyc b/LayerExtender/__pycache__/utils.cpython-39.pyc deleted file mode 100644 index 72cd200d2450ae3fac9b6e4448cf96478f522c52..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2182 zcmZuyOOG5i5VqaV>3QtzMhOo=9)Ln7LNgHu5NH(zvXC626;gtPT5MXqZO?9R`Y~&J zLpJI;&7P1zfol?JSK`7Q34RDCUpa9@;>L+8&(7unZn?V3E|<%$ugYn+8xd%yZ~n@E z2?_ZFh3A_K!ZqmXTOgcpT9A^)lp@axRywg$y0L3zr|?QY_AT!gK^%gPSG39~j>>l2 zJ|6GH9T<0uZrl~o4Z?jM91tGN*o?+K9`e=!iI;d^z{lH981L}zffq0HW!~dU2Q*&c z?rkzy`4jvh18P`R4VYQ}G*#kOx-aBs545Pbkfw91mgzzQ#r};i?yQUYt4}m2=)A7h zVO@)-I?akyDdUZEQSc1SKHnS=u0dC~fCv&(PGa^Y5sq* z@OpnD;5r#x$yhumcD67Gm9BxWcRcSSv$a+U)T8-}KT-34{y%4t1p3Vz3oa{?LBWXcb2+)Xy zv;*G?wc{Csu|-3OV!sYuy#=J9oW4e;j5F>WkcO?18J#+sG%(XRGY9V8T?h18q|C2c zWfT@J8d~?E--fPU2eJigH&&)plQGG`%1`ur`UBeqAld^D=3W7w*zk&JS;H9BXICO$ z1uG_6Epnx^x}01+4f5a6$-@h9Klg>y7!y$pSBpAJ3w0Hkg)Rj9%nM3MUTGn#v>3R? zv&)PR3gdh$?wxFo2v8u7lXO+<22Ncb`L%79MLA+@n>w>f`1^NG_AuB_HmDx{Htj} zxcxDMfQ36P4B0f;z02Kc1aic^drbYN+lL*{2l*~eAEmVnfRJZ{>I_KcsyYXn*&)VhM?rK7(O zV;@03PQE@pOC^pCm_>;tP+$--gHy(XQlD*0tWfeI5`3Q|mcMwYWm>6mElVNQ(3S-? z+?wd567_?*oL5`PVnkvJ$3X$FBtR&3kumL)_fRN_)xef~iBXa7A;GF_YlA$69Nxe6 m_PJBp^VxbAuFl_?t5{ss86X|nhj4~KLnt$h_QNn-IrlG`%N85} From 06e31e9a9853232f5d9987e2145c70465861a982 Mon Sep 17 00:00:00 2001 From: Connor Date: Thu, 20 Jul 2023 03:53:00 -0400 Subject: [PATCH 7/9] TTake 3 --- .gitignore | 2 +- LayerExtender/.giteignore | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 LayerExtender/.giteignore diff --git a/.gitignore b/.gitignore index ba0430d..ed8ebf5 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -__pycache__/ \ No newline at end of file +__pycache__ \ No newline at end of file diff --git a/LayerExtender/.giteignore b/LayerExtender/.giteignore new file mode 100644 index 0000000..ed8ebf5 --- /dev/null +++ b/LayerExtender/.giteignore @@ -0,0 +1 @@ +__pycache__ \ No newline at end of file From 4dbbe5f9134d17071443b9567f252efdb94ecb17 Mon Sep 17 00:00:00 2001 From: Connor Date: Thu, 20 Jul 2023 03:53:34 -0400 Subject: [PATCH 8/9] Name fix --- LayerExtender/{.giteignore => .gitingore} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename LayerExtender/{.giteignore => .gitingore} (100%) diff --git a/LayerExtender/.giteignore b/LayerExtender/.gitingore similarity index 100% rename from LayerExtender/.giteignore rename to LayerExtender/.gitingore From 606f0d924ff5429cc404e97a8083e9a4d55213f2 Mon Sep 17 00:00:00 2001 From: Connor Date: Thu, 20 Jul 2023 03:54:14 -0400 Subject: [PATCH 9/9] Name fix #2 --- LayerExtender/{.gitingore => .gitignore} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename LayerExtender/{.gitingore => .gitignore} (100%) diff --git a/LayerExtender/.gitingore b/LayerExtender/.gitignore similarity index 100% rename from LayerExtender/.gitingore rename to LayerExtender/.gitignore