Does offline mode support v2?

[Weiwenliang123]

def build_cotracker(checkpoint=None, offline=True, window_len=16, v2=False):
if offline:
cotracker = CoTrackerThreeOffline(
stride=4, corr_radius=3, window_len=window_len
)
else:
if v2:
cotracker = CoTracker2(stride=4, window_len=window_len)
else:
cotracker = CoTrackerThreeOnline(
stride=4, corr_radius=3, window_len=window_len
)

if checkpoint is not None:
    with open(checkpoint, "rb") as f:
        state_dict = torch.load(f, map_location="cpu")
        if "model" in state_dict:
            state_dict = state_dict["model"]
    cotracker.load_state_dict(state_dict)
return cotracker

The above is the code for build_cotracker.py from the website.
When I call the offline mode for v2, the following error occurs:
RuntimeError: Error(s) in loading state_dict for CoTrackerThreeOffline:
Missing key(s) in state_dict: "updateformer.vis_conf_head.weight", "updateformer.vis_conf_head.bias", "corr_mlp.fc1.weight", "corr_mlp.fc1.bias", "corr_mlp.fc2.weight", "corr_mlp.fc2.bias".
Unexpected key(s) in state_dict: "pos_emb", "norm.weight", "norm.bias", "track_feat_updater.0.weight", "track_feat_updater.0.bias", "vis_predictor.0.weight", "vis_predictor.0.bias", "updateformer.time_blocks.3.attn.to_q.weight", "updateformer.time_blocks.3.attn.to_q.bias", "updateformer.time_blocks.3.attn.to_kv.weight", "updateformer.time_blocks.3.attn.to_kv.bias", "updateformer.time_blocks.3.attn.to_out.weight", "updateformer.time_blocks.3.attn.to_out.bias", "updateformer.time_blocks.3.mlp.fc1.weight", "updateformer.time_blocks.3.mlp.fc1.bias", "updateformer.time_blocks.3.mlp.fc2.weight", "updateformer.time_blocks.3.mlp.fc2.bias", "updateformer.time_blocks.4.attn.to_q.weight", "updateformer.time_blocks.4.attn.to_q.bias", "updateformer.time_blocks.4.attn.to_kv.weight", "updateformer.time_blocks.4.attn.to_kv.bias", "updateformer.time_blocks.4.attn.to_out.weight", "updateformer.time_blocks.4.attn.to_out.bias", "updateformer.time_blocks.4.mlp.fc1.weight", "updateformer.time_blocks.4.mlp.fc1.bias", "updateformer.time_blocks.4.mlp.fc2.weight", "updateformer.time_blocks.4.mlp.fc2.bias", "updateformer.time_blocks.5.attn.to_q.weight", "updateformer.time_blocks.5.attn.to_q.bias", "updateformer.time_blocks.5.attn.to_kv.weight", "updateformer.time_blocks.5.attn.to_kv.bias", "updateformer.time_blocks.5.attn.to_out.weight", "updateformer.time_blocks.5.attn.to_out.bias", "updateformer.time_blocks.5.mlp.fc1.weight", "updateformer.time_blocks.5.mlp.fc1.bias", "updateformer.time_blocks.5.mlp.fc2.weight", "updateformer.time_blocks.5.mlp.fc2.bias", "updateformer.space_virtual_blocks.3.attn.to_q.weight", "updateformer.space_virtual_blocks.3.attn.to_q.bias", "updateformer.space_virtual_blocks.3.attn.to_kv.weight", "updateformer.space_virtual_blocks.3.attn.to_kv.bias", "updateformer.space_virtual_blocks.3.attn.to_out.weight", "updateformer.space_virtual_blocks.3.attn.to_out.bias", "updateformer.space_virtual_blocks.3.mlp.fc1.weight", "updateformer.space_virtual_blocks.3.mlp.fc1.bias", "updateformer.space_virtual_blocks.3.mlp.fc2.weight", "updateformer.space_virtual_blocks.3.mlp.fc2.bias", "updateformer.space_virtual_blocks.4.attn.to_q.weight", "updateformer.space_virtual_blocks.4.attn.to_q.bias", "updateformer.space_virtual_blocks.4.attn.to_kv.weight", "updateformer.space_virtual_blocks.4.attn.to_kv.bias", "updateformer.space_virtual_blocks.4.attn.to_out.weight", "updateformer.space_virtual_blocks.4.attn.to_out.bias", "updateformer.space_virtual_blocks.4.mlp.fc1.weight", "updateformer.space_virtual_blocks.4.mlp.fc1.bias", "updateformer.space_virtual_blocks.4.mlp.fc2.weight", "updateformer.space_virtual_blocks.4.mlp.fc2.bias", "updateformer.space_virtual_blocks.5.attn.to_q.weight", "updateformer.space_virtual_blocks.5.attn.to_q.bias", "updateformer.space_virtual_blocks.5.attn.to_kv.weight", "updateformer.space_virtual_blocks.5.attn.to_kv.bias", "updateformer.space_virtual_blocks.5.attn.to_out.weight", "updateformer.space_virtual_blocks.5.attn.to_out.bias", "updateformer.space_virtual_blocks.5.mlp.fc1.weight", "updateformer.space_virtual_blocks.5.mlp.fc1.bias", "updateformer.space_virtual_blocks.5.mlp.fc2.weight", "updateformer.space_virtual_blocks.5.mlp.fc2.bias", "updateformer.space_point2virtual_blocks.3.norm_context.weight", "updateformer.space_point2virtual_blocks.3.norm_context.bias", "updateformer.space_point2virtual_blocks.3.cross_attn.to_q.weight", "updateformer.space_point2virtual_blocks.3.cross_attn.to_q.bias", "updateformer.space_point2virtual_blocks.3.cross_attn.to_kv.weight", "updateformer.space_point2virtual_blocks.3.cross_attn.to_kv.bias", "updateformer.space_point2virtual_blocks.3.cross_attn.to_out.weight", "updateformer.space_point2virtual_blocks.3.cross_attn.to_out.bias", "updateformer.space_point2virtual_blocks.3.mlp.fc1.weight", "updateformer.space_point2virtual_blocks.3.mlp.fc1.bias", "updateformer.space_point2virtual_blocks.3.mlp.fc2.weight", "updateformer.space_point2virtual_blocks.3.mlp.fc2.bias", "updateformer.space_point2virtual_blocks.4.norm_context.weight", "updateformer.space_point2virtual_blocks.4.norm_context.bias", "updateformer.space_point2virtual_blocks.4.cross_attn.to_q.weight", "updateformer.space_point2virtual_blocks.4.cross_attn.to_q.bias", "updateformer.space_point2virtual_blocks.4.cross_attn.to_kv.weight", "updateformer.space_point2virtual_blocks.4.cross_attn.to_kv.bias", "updateformer.space_point2virtual_blocks.4.cross_attn.to_out.weight", "updateformer.space_point2virtual_blocks.4.cross_attn.to_out.bias", "updateformer.space_point2virtual_blocks.4.mlp.fc1.weight", "updateformer.space_point2virtual_blocks.4.mlp.fc1.bias", "updateformer.space_point2virtual_blocks.4.mlp.fc2.weight", "updateformer.space_point2virtual_blocks.4.mlp.fc2.bias", "updateformer.space_point2virtual_blocks.5.norm_context.weight", "updateformer.space_point2virtual_blocks.5.norm_context.bias", "updateformer.space_point2virtual_blocks.5.cross_attn.to_q.weight", "updateformer.space_point2virtual_blocks.5.cross_attn.to_q.bias", "updateformer.space_point2virtual_blocks.5.cross_attn.to_kv.weight", "updateformer.space_point2virtual_blocks.5.cross_attn.to_kv.bias", "updateformer.space_point2virtual_blocks.5.cross_attn.to_out.weight", "updateformer.space_point2virtual_blocks.5.cross_attn.to_out.bias", "updateformer.space_point2virtual_blocks.5.mlp.fc1.weight", "updateformer.space_point2virtual_blocks.5.mlp.fc1.bias", "updateformer.space_point2virtual_blocks.5.mlp.fc2.weight", "updateformer.space_point2virtual_blocks.5.mlp.fc2.bias", "updateformer.space_virtual2point_blocks.3.norm_context.weight", "updateformer.space_virtual2point_blocks.3.norm_context.bias", "updateformer.space_virtual2point_blocks.3.cross_attn.to_q.weight", "updateformer.space_virtual2point_blocks.3.cross_attn.to_q.bias", "updateformer.space_virtual2point_blocks.3.cross_attn.to_kv.weight", "updateformer.space_virtual2point_blocks.3.cross_attn.to_kv.bias", "updateformer.space_virtual2point_blocks.3.cross_attn.to_out.weight", "updateformer.space_virtual2point_blocks.3.cross_attn.to_out.bias", "updateformer.space_virtual2point_blocks.3.mlp.fc1.weight", "updateformer.space_virtual2point_blocks.3.mlp.fc1.bias", "updateformer.space_virtual2point_blocks.3.mlp.fc2.weight", "updateformer.space_virtual2point_blocks.3.mlp.fc2.bias", "updateformer.space_virtual2point_blocks.4.norm_context.weight", "updateformer.space_virtual2point_blocks.4.norm_context.bias", "updateformer.space_virtual2point_blocks.4.cross_attn.to_q.weight", "updateformer.space_virtual2point_blocks.4.cross_attn.to_q.bias", "updateformer.space_virtual2point_blocks.4.cross_attn.to_kv.weight", "updateformer.space_virtual2point_blocks.4.cross_attn.to_kv.bias", "updateformer.space_virtual2point_blocks.4.cross_attn.to_out.weight", "updateformer.space_virtual2point_blocks.4.cross_attn.to_out.bias", "updateformer.space_virtual2point_blocks.4.mlp.fc1.weight", "updateformer.space_virtual2point_blocks.4.mlp.fc1.bias", "updateformer.space_virtual2point_blocks.4.mlp.fc2.weight", "updateformer.space_virtual2point_blocks.4.mlp.fc2.bias", "updateformer.space_virtual2point_blocks.5.norm_context.weight", "updateformer.space_virtual2point_blocks.5.norm_context.bias", "updateformer.space_virtual2point_blocks.5.cross_attn.to_q.weight", "updateformer.space_virtual2point_blocks.5.cross_attn.to_q.bias", "updateformer.space_virtual2point_blocks.5.cross_attn.to_kv.weight", "updateformer.space_virtual2point_blocks.5.cross_attn.to_kv.bias", "updateformer.space_virtual2point_blocks.5.cross_attn.to_out.weight", "updateformer.space_virtual2point_blocks.5.cross_attn.to_out.bias", "updateformer.space_virtual2point_blocks.5.mlp.fc1.weight", "updateformer.space_virtual2point_blocks.5.mlp.fc1.bias", "updateformer.space_virtual2point_blocks.5.mlp.fc2.weight", "updateformer.space_virtual2point_blocks.5.mlp.fc2.bias".
size mismatch for time_emb: copying a param with shape torch.Size([1, 8, 456]) from checkpoint, the shape in current model is torch.Size([1, 8, 1110]).
size mismatch for updateformer.input_transform.weight: copying a param with shape torch.Size([384, 456]) from checkpoint, the shape in current model is torch.Size([384, 1110]).
size mismatch for updateformer.flow_head.weight: copying a param with shape torch.Size([130, 384]) from checkpoint, the shape in current model is torch.Size([2, 384]).
size mismatch for updateformer.flow_head.bias: copying a param with shape torch.Size([130]) from checkpoint, the shape in current model is torch.Size([2]).

[RitianYu]

I think you can just download the branch "cotracker2v1_release", which will allow you to load "cotracker2.pth" offline.

[Weiwenliang123]

In the official provided build_cotracker code, the function definition is as follows:
def build_cotracker(checkpoint=None, offline=True, window_len=16, v2=False):
if offline:
cotracker = CoTrackerThreeOffline(
stride=4, corr_radius=3, window_len=window_len
)
else:
if v2:
cotracker = CoTracker2(stride=4, window_len=window_len)
else:
cotracker = CoTrackerThreeOnline(
stride=4, corr_radius=3, window_len=window_len
)
def build_cotracker(checkpoint=None, offline=True, window_len=16, v2=False):
if offline:
cotracker = CoTrackerThreeOffline(
stride=4, corr_radius=3, window_len=window_len
)
else:
if v2:
cotracker = CoTracker2(stride=4, window_len=window_len)
else:
cotracker = CoTrackerThreeOnline(
stride=4, corr_radius=3, window_len=window_len
)
This leads to a jump to CoTrackerThreeOffline when I use the official CoTracker2 in offline mode

[LZY-the-boys]

I encounter the same problem. torch.hub.load("facebookresearch/co-tracker","cotracker2") not work anymore

[RitianYu]

I encounter the same problem. torch.hub.load("facebookresearch/co-tracker","cotracker2") not work anymore

This will download the latest version. Trying to download the branch "cotracker2v1_release" and load the model from local.

[Karlyeto]

• Same problem

[TongHengcheng]

I download branch "cotracker2v1_release" code and copy it to /home/xxx/.cache/torch/hub/facebookresearch_co-tracker_main can solve the problem

[LZY-the-boys]

I download branch "cotracker2v1_release" code and copy it to /home/xxx/.cache/torch/hub/facebookresearch_co-tracker_main can solve the problem

The problem is this cache will be automatically updated from remote code base, so directly rewrite the code may not work always

[Karlyeto]

Ok

…

On Wed, Nov 6, 2024, 8:19 AM Zhenyi Lu ***@***.***> wrote: I download branch "cotracker2v1_release" code and copy it to /home/xxx/.cache/torch/hub/facebookresearch_co-tracker_main can solve the problem The problem is this cache will be automatically updated from remote code base, so directly rewrite the code may not work always — Reply to this email directly, view it on GitHub <#123 (comment)>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/BMPBMYRVKD6WKVHVNOMFB6LZ7G7I5AVCNFSM6AAAAABQTOJU66VHI2DSMVQWIX3LMV43OSLTON2WKQ3PNVWWK3TUHMZDINJYHA3TQOJVGI> . You are receiving this because you commented.Message ID: ***@***.***>