Hugging Face's logo

David-huang
/
VLAB

TensorBoard
Model card Files Metrics Community
huangzhicheng.01 commited on
Commit
8e5a8b6
·
1 Parent(s): 1cdebcf

model weight

Browse files
Files changed (41) hide show
  1. downstreams/giant/.keep +0 -0
  2. downstreams/giant/didemo_retrieval_giant.pt +3 -0
  3. downstreams/giant/msrvtt_caption_giant.pt +3 -0
  4. downstreams/giant/msrvtt_retrieval_giant.pt +3 -0
  5. downstreams/giant/msrvtt_vqa_giant.pt +3 -0
  6. downstreams/giant/msvd_caption_giant.pt +3 -0
  7. downstreams/giant/msvd_retrieval_giant.pt +3 -0
  8. downstreams/giant/msvd_vqa_giant.pt +3 -0
  9. downstreams/large/.keep +0 -0
  10. downstreams/large/didemo_retrieval_large.pt +3 -0
  11. downstreams/large/msrvtt_caption_large.pt +3 -0
  12. downstreams/large/msrvtt_retrieval_large.pt +3 -0
  13. downstreams/large/msrvtt_vqa_large.pt +3 -0
  14. downstreams/large/msvd_caption_large.pt +3 -0
  15. downstreams/large/msvd_retrieval_large.pt +3 -0
  16. downstreams/large/msvd_vqa_large.pt +3 -0
  17. pretrain/.keep +0 -0
  18. pretrain/giant/ckpt/model_step_195680_last.pt +3 -0
  19. pretrain/giant/ckpt/optimizer_step_195680.pt +3 -0
  20. pretrain/giant/log/events.out.tfevents.1682691509.n136-129-032 +3 -0
  21. pretrain/giant/log/hps.json +148 -0
  22. pretrain/giant/log/log.txt +0 -0
  23. pretrain/large/ckpt/model_step_195680_last.pt +3 -0
  24. pretrain/large/ckpt/optimizer_step_195680.pt +3 -0
  25. pretrain/large/log/events.out.tfevents.1681230810.n136-024-031 +3 -0
  26. pretrain/large/log/hps.json +149 -0
  27. pretrain/large/log/log.txt +0 -0
  28. pretrained_weights/.keep +0 -0
  29. pretrained_weights/bert/bert-base-cased-vocab.txt +0 -0
  30. pretrained_weights/bert/bert-base-uncased-vocab.txt +0 -0
  31. pretrained_weights/bert/bert-base-uncased.bin +3 -0
  32. pretrained_weights/bert/bert-base-uncased.json +23 -0
  33. pretrained_weights/clip/ViT-L-14-336px.pt +3 -0
  34. pretrained_weights/eva/eva_clip_psz14.pt +3 -0
  35. pretrained_weights/mf_clip/pretrain-webvid10m-va-512-512-last4-parallel_large_stage2_5witer/ckpt/model_opt.pth +3 -0
  36. pretrained_weights/mf_clip/pretrain-webvid10m-va-512-512-last4-parallel_large_stage2_5witer/ckpt/model_step_50000.pt +3 -0
  37. pretrained_weights/mf_clip/pretrain-webvid10m-va-512-512-last4-parallel_large_stage2_5witer/ckpt/optimizer_step_50000.pt +3 -0
  38. pretrained_weights/mf_clip/pretrain-webvid10m-va-512-512-last4-parallel_large_stage2_5witer/log/events.out.tfevents.1677305848.n136-034-083 +3 -0
  39. pretrained_weights/mf_clip/pretrain-webvid10m-va-512-512-last4-parallel_large_stage2_5witer/log/events.out.tfevents.1677308392.n136-052-072 +3 -0
  40. pretrained_weights/mf_clip/pretrain-webvid10m-va-512-512-last4-parallel_large_stage2_5witer/log/hps.json +97 -0
  41. pretrained_weights/mf_clip/pretrain-webvid10m-va-512-512-last4-parallel_large_stage2_5witer/log/log.txt +0 -0
downstreams/giant/.keep ADDED
File without changes
downstreams/giant/didemo_retrieval_giant.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3203b8c57fbfe8380853050343a665bf368aeefb9447c35c37811c72838ffa23
3
+ size 6773950189
downstreams/giant/msrvtt_caption_giant.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:42dec87f285841e1110fb5683e29c233029a3ae65c25cb22014d7f7fe4fb3c0e
3
+ size 7269085844
downstreams/giant/msrvtt_retrieval_giant.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e544d338846d0eccf93247a173710f4ab079c9b4d08d88ebb6a93e30e63ad2c6
3
+ size 6773950189
downstreams/giant/msrvtt_vqa_giant.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aaa97ed2b64b2cd90541d3ac0c16f966cee4e0f7c9c40abc29474e1e52ccb7ae
3
+ size 7269085844
downstreams/giant/msvd_caption_giant.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:69810ef1c6fee3fe9337e4481c7665573cf55b5b2acf0ddd94aeff877a2c9627
3
+ size 7269085844
downstreams/giant/msvd_retrieval_giant.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:45d5bb5daebe1cd1e8a7dc78c9cf07611ae8b43bb4cb4eefbdb0e112542a4a8e
3
+ size 6773950189
downstreams/giant/msvd_vqa_giant.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9d019b64d07a024045c86ef736369d2a8c620ffe99a5f6b71a2577be3d26a8ba
3
+ size 7269085844
downstreams/large/.keep ADDED
File without changes
downstreams/large/didemo_retrieval_large.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b175bc6d0b4bbea68ac4f261c3c96c8f32348bc989dbdcc4d71fa6cc04227554
3
+ size 3938590605
downstreams/large/msrvtt_caption_large.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1350c35dcccf0a8a128a5357761f0f3b1149fa7c5e1100645ce2b1e41895f54b
3
+ size 4286299064
downstreams/large/msrvtt_retrieval_large.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c6638a8ab3f41b62ece9c4bbecebc0b8d44550b0622261f865810cfa2513ec92
3
+ size 3938590605
downstreams/large/msrvtt_vqa_large.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7bee9590179a44aea2657f85832908ef4566819d7af7efccb53e4c26a92de98f
3
+ size 4286299064
downstreams/large/msvd_caption_large.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8ccebf55200dc04c67937b6a04074e972143246dcd61a1b5d380a76093258f67
3
+ size 4286299064
downstreams/large/msvd_retrieval_large.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:01205a554ca414a86259b50233042e0f29bece0cdfdc5467d7b8eeb2dbba8523
3
+ size 3938590605
downstreams/large/msvd_vqa_large.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:92d96302ca81a5cea0961e0b587f5296299839942d0758a7039a8319a3b7ab6a
3
+ size 4286299064
pretrain/.keep ADDED
File without changes
pretrain/giant/ckpt/model_step_195680_last.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e2eb7f26bb8c300a3e4d659740ede793c4704bdc8bec7d1e563ae57e9b2912e4
3
+ size 7426864284
pretrain/giant/ckpt/optimizer_step_195680.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5d462ed02ec56700c7ebdfc968a91b807a006f8593bbc45d1e607906bf43181e
3
+ size 11199462765
pretrain/giant/log/events.out.tfevents.1682691509.n136-129-032 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:67c8e65dbae52abea226cf4b3909111193f42c1bdd687db42fe63a5d53c0d532
3
+ size 303603707
pretrain/giant/log/hps.json ADDED
@@ -0,0 +1,148 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "local_rank": 0,
3
+ "checkpoint": null,
4
+ "resume": false,
5
+ "output_dir": "./output/experiments/ensemble_base/pretrain-cc12m-cc4m-webvid10m-mfbasegaintopen+clipvaccwebfreeze-20-20-10-b32_64gpus",
6
+ "config": "./config/byte/pretrain/ensemble_base/pretrain-cc12m-cc4m-webvid10m-mfbasegaintfreeze+clipvaccwebfreeze-20-20-10-b32.json",
7
+ "wandb": false,
8
+ "find_unused_parameters": false,
9
+ "vision_encoder.frozen_vision": true,
10
+ "video_cfg": {
11
+ "sample_num": 4,
12
+ "resolution": 224,
13
+ "mean": [
14
+ 0.48145466,
15
+ 0.4578275,
16
+ 0.40821073
17
+ ],
18
+ "std": [
19
+ 0.26862954,
20
+ 0.26130258,
21
+ 0.27577711
22
+ ]
23
+ },
24
+ "schedule": {
25
+ "use_validate": false,
26
+ "valid_steps": 5000
27
+ },
28
+ "optimizer": {
29
+ "learning_rate": 0.0001,
30
+ "vision_lr": 5e-07,
31
+ "optim": "adamw",
32
+ "scheduler": "warmup_linear",
33
+ "betas": [
34
+ 0.9,
35
+ 0.98
36
+ ],
37
+ "weight_decay": 0.01,
38
+ "grad_norm": 5.0,
39
+ "warmup_ratio": 0.1,
40
+ "seed": 60,
41
+ "fp16": true,
42
+ "gradient_accumulation_steps": 1,
43
+ "num_train_steps": 195680
44
+ },
45
+ "vision_encoder": {
46
+ "vision_type": "cb_clip",
47
+ "pretrained": {
48
+ "eva_clip1": "./output/pretrained_weights/mf_clip/pretrain-cc16m-webvid10m-evagaint_baseline_15witer/ckpt/model_opt.pth",
49
+ "video_adapter_clip2": "./output/pretrained_weights/mf_clip/pretrain-webvid10m-va-512-512-last4-parallel_large_stage2_5witer/ckpt/model_opt.pth"
50
+ },
51
+ "video_dim": [
52
+ 1408,
53
+ 1024
54
+ ],
55
+ "proj_dim": 768,
56
+ "grad_checkpointing": true,
57
+ "start_adapt_layer": 7,
58
+ "adapt_cls_dim": 512,
59
+ "adapt_patch_dim": 512,
60
+ "adapt_order": "adapt_first",
61
+ "frozen_vision": true
62
+ },
63
+ "txt_encoder": {
64
+ "txt_dim": 1024,
65
+ "proj_dim": 768
66
+ },
67
+ "multi_encoder": {
68
+ "multi_type": "bert",
69
+ "bert_config": "./output/pretrained_weights/bert/bert-base-uncased.json",
70
+ "pretrained": "./output/pretrained_weights/bert/bert-base-uncased.bin",
71
+ "multimodal_dim": 768,
72
+ "vocab_size": 30522,
73
+ "match_neg_prob": 0.5,
74
+ "multi_cross": 2,
75
+ "ensemble_type": "scale_share_sum"
76
+ },
77
+ "losses": {
78
+ "mask_prob": 0.15,
79
+ "mask_token": 103
80
+ },
81
+ "data_cfg": {
82
+ "train": [
83
+ {
84
+ "datasets": {
85
+ "name": "distcc12m",
86
+ "data_path": [
87
+ "hdfs://haruna/home/byte_arnold_hl_vc/guolongteng.lt/data/public/vlm/cc12m_bs64"
88
+ ],
89
+ "datatype": "image",
90
+ "tokenizer": {
91
+ "txt_encoder": "clip_tokenizer",
92
+ "multi_encoder": "bert_tokenizer"
93
+ }
94
+ },
95
+ "task": "contraTwo_unimlmTwo_mlmTwo",
96
+ "data_len": 11105356,
97
+ "epoch": 20,
98
+ "mix_ratio": 1,
99
+ "batch_size": 32,
100
+ "n_workers": 2
101
+ },
102
+ {
103
+ "datasets": {
104
+ "name": "distcc4m",
105
+ "data_path": [
106
+ "hdfs://haruna/home/byte_arnold_hl_vc/guolongteng.lt/data/public/vlm/coco_testset_filtered",
107
+ "hdfs://haruna/home/byte_arnold_hl_vc/guolongteng.lt/data/public/vlm/vg_testset_filtered",
108
+ "hdfs://haruna/home/byte_arnold_hl_vc/guolongteng.lt/data/public/vlm/sbu_bs64",
109
+ "hdfs://haruna/home/byte_arnold_hl_vc/guolongteng.lt/data/public/vlm/cc3m_bs64"
110
+ ],
111
+ "datatype": "image",
112
+ "tokenizer": {
113
+ "txt_encoder": "clip_tokenizer",
114
+ "multi_encoder": "bert_tokenizer"
115
+ }
116
+ },
117
+ "task": "contraTwo_unimlmTwo_mlmTwo",
118
+ "data_len": 5114489,
119
+ "epoch": 20,
120
+ "mix_ratio": 1,
121
+ "batch_size": 32,
122
+ "n_workers": 2
123
+ },
124
+ {
125
+ "datasets": [
126
+ {
127
+ "name": "webvid10m",
128
+ "ids_path": "",
129
+ "video": "/mnt/bn/vlp-v6/webvid10m",
130
+ "txt": "/mnt/bn/vlp-v6/webvid2.5M_package/webvid_pretrain_anno_new.pkl",
131
+ "datatype": "video",
132
+ "tokenizer": {
133
+ "txt_encoder": "clip_tokenizer",
134
+ "multi_encoder": "bert_tokenizer"
135
+ }
136
+ }
137
+ ],
138
+ "task": "contraTwo_unimlmTwo_mlmTwo",
139
+ "data_len": 7639040,
140
+ "epoch": 10,
141
+ "mix_ratio": 1,
142
+ "batch_size": 32,
143
+ "n_workers": 2
144
+ }
145
+ ],
146
+ "val": []
147
+ }
148
+ }
pretrain/giant/log/log.txt ADDED
The diff for this file is too large to render. See raw diff
 
pretrain/large/ckpt/model_step_195680_last.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c0ba178cd6fecde28748fa074fb3309581efa01968c14a22e32147734f81078e
3
+ size 4590325792
pretrain/large/ckpt/optimizer_step_195680.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:421fffa2d7e53b2abe61db392b99764cd4207fcb9c28b491940f00236c702a07
3
+ size 5526390473
pretrain/large/log/events.out.tfevents.1681230810.n136-024-031 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9af9ae6523fb5b967a490d31b0a382b2a77839639b6e5377857d0d8d7359e12e
3
+ size 303603707
pretrain/large/log/hps.json ADDED
@@ -0,0 +1,149 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "local_rank": 0,
3
+ "checkpoint": null,
4
+ "resume": false,
5
+ "output_dir": "./output/experiments/ensemble_base/pretrain-cc12m-cc4m-webvid10m-baseclipopen+vidadapterfreeze-20-20-10-ensemble-scalesharesum_64gpus",
6
+ "config": "./config/byte/pretrain/ensemble_base/pretrain-cc12m-cc4m-webvid10m-baseclipopen+vidadapterfreeze-20-20-10-ensemble-scalesharesum.json",
7
+ "wandb": false,
8
+ "find_unused_parameters": false,
9
+ "vision_encoder.frozen_vision": true,
10
+ "video_cfg": {
11
+ "sample_num": 4,
12
+ "resolution": 224,
13
+ "mean": [
14
+ 0.48145466,
15
+ 0.4578275,
16
+ 0.40821073
17
+ ],
18
+ "std": [
19
+ 0.26862954,
20
+ 0.26130258,
21
+ 0.27577711
22
+ ]
23
+ },
24
+ "schedule": {
25
+ "use_validate": false,
26
+ "valid_steps": 5000
27
+ },
28
+ "optimizer": {
29
+ "learning_rate": 0.0001,
30
+ "vision_lr": 5e-07,
31
+ "optim": "adamw",
32
+ "scheduler": "warmup_linear",
33
+ "betas": [
34
+ 0.9,
35
+ 0.98
36
+ ],
37
+ "weight_decay": 0.01,
38
+ "grad_norm": 5.0,
39
+ "warmup_ratio": 0.1,
40
+ "seed": 60,
41
+ "fp16": true,
42
+ "gradient_accumulation_steps": 1,
43
+ "num_train_steps": 195680
44
+ },
45
+ "vision_encoder": {
46
+ "vision_type": "cb_clip",
47
+ "pretrained": {
48
+ "zc_clip1": "/mnt/bn/hexingjian/VLP/output/experiments/cbconfig/pretrain-cc4m-cc12m-clip_large-20-20_64gpus/ckpt/model_opt.pth",
49
+ "video_adapter_clip2": "./output/pretrained_weights/mf_clip/pretrain-webvid10m-va-512-512-last4-parallel_large_stage2_5witer/ckpt/model_opt.pth"
50
+ },
51
+ "video_dim": 1024,
52
+ "proj_dim": 768,
53
+ "adaptor": [
54
+ false,
55
+ false
56
+ ],
57
+ "grad_checkpointing": true,
58
+ "start_adapt_layer": 7,
59
+ "adapt_cls_dim": 512,
60
+ "adapt_patch_dim": 512,
61
+ "adapt_order": "adapt_first",
62
+ "frozen_vision": true
63
+ },
64
+ "txt_encoder": {
65
+ "txt_dim": 768,
66
+ "proj_dim": 768
67
+ },
68
+ "multi_encoder": {
69
+ "multi_type": "bert",
70
+ "bert_config": "./output/pretrained_weights/bert/bert-base-uncased.json",
71
+ "pretrained": "./output/pretrained_weights/bert/bert-base-uncased.bin",
72
+ "multimodal_dim": 768,
73
+ "vocab_size": 30522,
74
+ "match_neg_prob": 0.5,
75
+ "multi_cross": 2,
76
+ "ensemble_type": "scale_share_sum"
77
+ },
78
+ "losses": {
79
+ "mask_prob": 0.15,
80
+ "mask_token": 103
81
+ },
82
+ "data_cfg": {
83
+ "train": [
84
+ {
85
+ "datasets": {
86
+ "name": "distcc12m",
87
+ "data_path": [
88
+ "hdfs://haruna/home/byte_arnold_hl_vc/guolongteng.lt/data/public/vlm/cc12m_bs64"
89
+ ],
90
+ "datatype": "image",
91
+ "tokenizer": {
92
+ "txt_encoder": "clip_tokenizer",
93
+ "multi_encoder": "bert_tokenizer"
94
+ }
95
+ },
96
+ "task": "contraTwo_unimlmTwo_mlmTwo",
97
+ "data_len": 11105356,
98
+ "epoch": 20,
99
+ "mix_ratio": 1,
100
+ "batch_size": 32,
101
+ "n_workers": 2
102
+ },
103
+ {
104
+ "datasets": {
105
+ "name": "distcc4m",
106
+ "data_path": [
107
+ "hdfs://haruna/home/byte_arnold_hl_vc/guolongteng.lt/data/public/vlm/coco_testset_filtered",
108
+ "hdfs://haruna/home/byte_arnold_hl_vc/guolongteng.lt/data/public/vlm/vg_testset_filtered",
109
+ "hdfs://haruna/home/byte_arnold_hl_vc/guolongteng.lt/data/public/vlm/sbu_bs64",
110
+ "hdfs://haruna/home/byte_arnold_hl_vc/guolongteng.lt/data/public/vlm/cc3m_bs64"
111
+ ],
112
+ "datatype": "image",
113
+ "tokenizer": {
114
+ "txt_encoder": "clip_tokenizer",
115
+ "multi_encoder": "bert_tokenizer"
116
+ }
117
+ },
118
+ "task": "contraTwo_unimlmTwo_mlmTwo",
119
+ "data_len": 5114489,
120
+ "epoch": 20,
121
+ "mix_ratio": 1,
122
+ "batch_size": 32,
123
+ "n_workers": 4
124
+ },
125
+ {
126
+ "datasets": [
127
+ {
128
+ "name": "webvid10m",
129
+ "ids_path": "",
130
+ "video": "/mnt/bn/vlp-v6/webvid10m",
131
+ "txt": "/mnt/bn/vlp-v6/webvid2.5M_package/webvid_pretrain_anno_new.pkl",
132
+ "datatype": "video",
133
+ "tokenizer": {
134
+ "txt_encoder": "clip_tokenizer",
135
+ "multi_encoder": "bert_tokenizer"
136
+ }
137
+ }
138
+ ],
139
+ "task": "contraTwo_unimlmTwo_mlmTwo",
140
+ "data_len": 7639040,
141
+ "epoch": 10,
142
+ "mix_ratio": 1,
143
+ "batch_size": 32,
144
+ "n_workers": 2
145
+ }
146
+ ],
147
+ "val": []
148
+ }
149
+ }
pretrain/large/log/log.txt ADDED
The diff for this file is too large to render. See raw diff
 
pretrained_weights/.keep ADDED
File without changes
pretrained_weights/bert/bert-base-cased-vocab.txt ADDED
The diff for this file is too large to render. See raw diff
 
pretrained_weights/bert/bert-base-uncased-vocab.txt ADDED
The diff for this file is too large to render. See raw diff
 
pretrained_weights/bert/bert-base-uncased.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:097417381d6c7230bd9e3557456d726de6e83245ec8b24f529f60198a67b203a
3
+ size 440473133
pretrained_weights/bert/bert-base-uncased.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "BertForMaskedLM"
4
+ ],
5
+ "attention_probs_dropout_prob": 0.1,
6
+ "gradient_checkpointing": false,
7
+ "hidden_act": "gelu",
8
+ "hidden_dropout_prob": 0.1,
9
+ "hidden_size": 768,
10
+ "initializer_range": 0.02,
11
+ "intermediate_size": 3072,
12
+ "layer_norm_eps": 1e-12,
13
+ "max_position_embeddings": 512,
14
+ "model_type": "bert",
15
+ "num_attention_heads": 12,
16
+ "num_hidden_layers": 12,
17
+ "pad_token_id": 0,
18
+ "position_embedding_type": "absolute",
19
+ "transformers_version": "4.6.0.dev0",
20
+ "type_vocab_size": 2,
21
+ "use_cache": true,
22
+ "vocab_size": 30522
23
+ }
pretrained_weights/clip/ViT-L-14-336px.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3035c92b350959924f9f00213499208652fc7ea050643e8b385c2dac08641f02
3
+ size 934088680
pretrained_weights/eva/eva_clip_psz14.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fe1d3a90bb88c3816a8c708b8a16c11c93cc323ec9e7bc7a0e8b979449fa9785
3
+ size 2273012515
pretrained_weights/mf_clip/pretrain-webvid10m-va-512-512-last4-parallel_large_stage2_5witer/ckpt/model_opt.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:71947bb8ddd7330c2af01816129b84dc85d85387e789fcb6e9438aa2f9eafa8b
3
+ size 2327365453
pretrained_weights/mf_clip/pretrain-webvid10m-va-512-512-last4-parallel_large_stage2_5witer/ckpt/model_step_50000.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a8f16c9311fe947b0f9237766725bebcca11ed15ab3f8c732120791a414b95e0
3
+ size 2876529986
pretrained_weights/mf_clip/pretrain-webvid10m-va-512-512-last4-parallel_large_stage2_5witer/ckpt/optimizer_step_50000.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:475d773ba07608d0d65c50bfd7a9ae77240db21d3cb755a4eed1ec8a68f66f7e
3
+ size 5565357061
pretrained_weights/mf_clip/pretrain-webvid10m-va-512-512-last4-parallel_large_stage2_5witer/log/events.out.tfevents.1677305848.n136-034-083 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fa8faf04d73423365d7a8d22fc7e5bce0a13c82f347c9b3e8858ccd7dfe568a7
3
+ size 40
pretrained_weights/mf_clip/pretrain-webvid10m-va-512-512-last4-parallel_large_stage2_5witer/log/events.out.tfevents.1677308392.n136-052-072 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b357298d0f44e4cc40b1d7e7f2ae3a0b77d8b793040e9c148fbf7625cd2d7585
3
+ size 24200964
pretrained_weights/mf_clip/pretrain-webvid10m-va-512-512-last4-parallel_large_stage2_5witer/log/hps.json ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "local_rank": -1,
3
+ "checkpoint": "/mnt/bn/vlp-mf/checkpoints/pretrain-webvid10m-va-512-512-last4-parallel_large_stage1_5witer/ckpt/model_step_50000.pt",
4
+ "except_key": "multi",
5
+ "resume": false,
6
+ "output_dir": "/mnt/bn/vlp-mf/checkpoints/pretrain-webvid10m-va-512-512-last4-parallel_large_stage2_5witer",
7
+ "config": "./config/byte/pretrain/webvid/pretrain-webvid10m-videoclip_large_stage2.json",
8
+ "video_cfg": {
9
+ "sample_num": 4,
10
+ "resolution": 224,
11
+ "mean": [
12
+ 0.48145466,
13
+ 0.4578275,
14
+ 0.40821073
15
+ ],
16
+ "std": [
17
+ 0.26862954,
18
+ 0.26130258,
19
+ 0.27577711
20
+ ]
21
+ },
22
+ "schedule": {
23
+ "use_validate": false,
24
+ "valid_steps": 5000
25
+ },
26
+ "optimizer": {
27
+ "learning_rate": 0.0001,
28
+ "vision_lr": 5e-07,
29
+ "adapter_lr": 5e-07,
30
+ "num_train_steps": 50000,
31
+ "optim": "adamw",
32
+ "scheduler": "warmup_linear",
33
+ "betas": [
34
+ 0.9,
35
+ 0.98
36
+ ],
37
+ "weight_decay": 0.01,
38
+ "grad_norm": 5.0,
39
+ "warmup_ratio": 0.1,
40
+ "seed": 60,
41
+ "fp16": true,
42
+ "gradient_accumulation_steps": 1
43
+ },
44
+ "vision_encoder": {
45
+ "vision_type": "videoclip",
46
+ "pretrained": "./output/pretrained_weights/clip/ViT-L-14-336px.pt",
47
+ "sample_num": 4,
48
+ "video_dim": 1024,
49
+ "proj_dim": 768,
50
+ "start_adapt_layer": 7,
51
+ "adapt_cls_dim": 512,
52
+ "adapt_patch_dim": 512,
53
+ "adapt_order": "adapt_first",
54
+ "use_checkpoint": true
55
+ },
56
+ "txt_encoder": {
57
+ "txt_dim": 768,
58
+ "proj_dim": 768
59
+ },
60
+ "multi_encoder": {
61
+ "multi_type": "bert",
62
+ "bert_config": "./output/pretrained_weights/bert/bert-base-uncased.json",
63
+ "pretrained": "./output/pretrained_weights/bert/bert-base-uncased.bin",
64
+ "multimodal_dim": 768,
65
+ "vocab_size": 30522,
66
+ "match_neg_prob": 0.5
67
+ },
68
+ "losses": {
69
+ "mask_prob": 0.15,
70
+ "mask_token": 103
71
+ },
72
+ "data_cfg": {
73
+ "train": [
74
+ {
75
+ "datasets": [
76
+ {
77
+ "name": "webvid10m",
78
+ "video": "/mnt/bn/vlp-v6/webvid10m",
79
+ "bk": "hdfs://haruna/home/byte_labcv_gan/public/multimodal_all/video/webvid/train10MAll_new/",
80
+ "txt": "/mnt/bn/vlp-mf/webvid2.5M_package/webvid_pretrain_anno_new.pkl",
81
+ "ids_path": "",
82
+ "datatype": "video",
83
+ "tokenizer": {
84
+ "txt_encoder": "clip_tokenizer",
85
+ "multi_encoder": "bert_tokenizer"
86
+ }
87
+ }
88
+ ],
89
+ "task": "contraTwo_unimlmTwo_mlmTwo",
90
+ "mix_ratio": 1,
91
+ "batch_size": 32,
92
+ "n_workers": 2
93
+ }
94
+ ],
95
+ "val": []
96
+ }
97
+ }
pretrained_weights/mf_clip/pretrain-webvid10m-va-512-512-last4-parallel_large_stage2_5witer/log/log.txt ADDED
The diff for this file is too large to render. See raw diff