update weights from bf16 to fp32

config.json

@@ -117,7 +117,7 @@
     }
   },
   "tie_word_embeddings": false,
-  "torch_dtype": "bfloat16",
+  "torch_dtype": "float32",
   "transformers_version": "4.52.3",
   "use_cache": true,
   "vit_config": {

model-00001-of-00004.safetensors → model-00001-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fc6cee860646e6245edd505fef7896a8ebc5fd47d6e260852699ffcb351d4119
-size 4975847688
+oid sha256:22b149604bdd0e6d5a1eab6a7764e9bc001ba2eaefd1f6506761746069027893
+size 4892921912

model-00002-of-00004.safetensors → model-00002-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b3b707bb662766ec4202e82fa9245ffefc1f9ec40e4aebe011426ea2459e463b
-size 4890972104
+oid sha256:4308afe5a8c8c3e9e17cae20b9a4ef5ec81bf9f959511047855b2e9ae617784a
+size 4857403152

model-00003-of-00004.safetensors → model-00003-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a772516b69576965758f7683d47a10ef81916bcf7c52e10836329e14524a6ff3
-size 4620250280
+oid sha256:6564f9fdaaa424eaadfb33cf7e1a10ad9c6c7a0af1b90085fe8aae833ed52792
+size 4857403200

model-00004-of-00004.safetensors → model-00004-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:65446424dc3ae3bfe8675c254936da139406793f5329b1db40466a6f9ba02b17
-size 826278016
+oid sha256:734aa8101f1689d7e41f8cbfaff539248712aa879d21a03578d37839b6107c9d
+size 4857403200

model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b024919352d9a1adddfe53bf0accf25a86ceb4168fd9c95d5ff434df4c4bad5
+size 4857403200

model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90ae62928101ae357c6c7eb3c89bf9c9919dc9e5056a4f8b4be9aa16b343f051
+size 4651517480

model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1656295cc35ed895cb50a7557fc13c5619718f01c1df02f32e80a4625011bf9e
+size 1652555904

model.yaml

@@ -0,0 +1,195 @@
+model_name: molmo
+llm:
+  d_model: 4096
+  n_heads: 32
+  n_kv_heads: null
+  head_dim: null
+  qkv_bias: false
+  clip_qkv: null
+  n_layers: 32
+  mlp_ratio: 4
+  mlp_hidden_size: 22016
+  activation_type: swiglu
+  block_type: sequential
+  rope: true
+  rope_full_precision: true
+  rope_theta: 500000.0
+  rope_type: default
+  rope_factor: null
+  rope_high_freq_factor: null
+  rope_low_freq_factor: null
+  rope_original_max_position_embeddings: null
+  attention_type: sdpa
+  float32_attention: true
+  attention_dropout: 0.0
+  attention_layer_norm: true
+  attention_layer_norm_type: olmo
+  residual_dropout: 0.1
+  response_residual_dropout: 0.0
+  layer_norm_type: rms
+  layer_norm_with_affine: true
+  layer_norm_eps: 1.0e-06
+  attention_layer_norm_with_affine: true
+  max_sequence_length: 4096
+  max_position_embeddings: null
+  include_bias: false
+  bias_for_layer_norm: false
+  norm_after: true
+  moe_num_experts: 8
+  moe_top_k: 2
+  moe_mlp_impl: sparse
+  moe_log_expert_assignment: false
+  moe_shared_expert: false
+  moe_lbl_in_fp32: false
+  moe_interleave: false
+  moe_loss_weight: 0.1
+  moe_zloss_weight: null
+  moe_dropless: true
+  moe_capacity_factor: 1.25
+  embedding_dropout: 0.0
+  scale_logits: false
+  vocab_size: 100278
+  additional_vocab_size: 128
+  weight_tying: false
+  embedding_size: 100864
+  use_position_ids: true
+  tokenizer:
+    identifier: allenai/OLMo-2-1124-7B
+    tokenizer_dir: null
+    depth_tokens: true
+  init_path: /weka/oe-training-default/mm-olmo/pretrained_llms/olmo2-1124-7b.pt
+  init_incremental: null
+  new_embedding_init_range: 0.02
+  initializer_range: 0.02
+  normalize_input_embeds: false
+  activation_checkpoint: whole_layer
+  compile: blocks
+  fix_pad_tokenizer: true
+  resize_vocab: true
+  init_std: 0.02
+  init_fn: normal
+  init_cutoff_factor: null
+vision_backbone:
+  vit:
+    image_model_type: openai
+    image_default_input_size:
+    - 336
+    - 336
+    image_patch_size: 14
+    image_pos_patch_size: 14
+    image_emb_dim: 1024
+    image_num_heads: 16
+    image_num_key_value_heads: 16
+    image_num_layers: 23
+    image_head_dim: 64
+    image_mlp_dim: 4096
+    image_mlp_activations: quick_gelu
+    image_dropout_rate: 0.0
+    image_num_pos: 577
+    image_norm_eps: 1.0e-05
+    attention_dropout: 0.0
+    residual_dropout: 0.0
+    initializer_range: 0.02
+    float32_attention: true
+    attention_type: sdpa
+    activation_checkpointing: true
+    init_path: /weka/oe-training-default/mm-olmo/pretrained_image_encoders/vit-l-14-336.pt
+    resize_mode: default
+    pad_value: 0.0
+    normalize: openai
+  image_pooling_2d: attention_meanq
+  pooling_attention_mask: false
+  image_projector: mlp
+  image_padding_embed: pad_and_partial_pad
+  vit_layers:
+  - -2
+  - -9
+  skip_unused_layers: true
+  image_feature_dropout: 0.0
+  connector_activation_checkpointing: true
+  compile_vit: blocks
+data_formatter:
+  prompt_templates: uber_model
+  message_format: role
+  system_prompt: demo_or_style
+  always_start_with_space: false
+  default_inference_len: 65
+  select_answer: best
+  debug: false
+  image_last: false
+  format_message_list: null
+  p_one_message: 0.0
+mm_preprocessor:
+  crop_mode: overlap-and-resize-c2
+  max_crops: 8
+  max_images: 2
+  max_multi_image_crops: 8
+  pooling_w: 2
+  pooling_h: 2
+  overlap_margins:
+  - 4
+  - 4
+  use_col_tokens: true
+  loss_token_weighting: root_subsegments
+  legacy_image_mask: false
+  max_answer_len: null
+  img_aug: false
+bi_directional_attn: null
+lora_enable: false
+lora_rank: 64
+lora_alpha: 16
+lora_dropout: 0.05
+lora_bias: none
+n_action_bins: 256
+norm_stats:
+  molmoact:
+    action:
+      mean:
+      - 0.0005706787342205644
+      - 0.0002448957529850304
+      - -3.5987635783385485e-05
+      - 0.00021597897284664214
+      - -0.0004896928439848125
+      - -0.000241481073317118
+      - 0.5570635199546814
+      std:
+      - 0.005207270849496126
+      - 0.007506529800593853
+      - 0.006415561307221651
+      - 0.013248044066131115
+      - 0.010928540490567684
+      - 0.014873150736093521
+      - 0.49715080857276917
+      min:
+      - -0.07434078305959702
+      - -0.07339745759963989
+      - -0.06539416313171387
+      - -0.1688285619020462
+      - -0.10289879888296127
+      - -0.2667275667190552
+      - 0.0
+      max:
+      - 0.06042003631591797
+      - 0.09417290985584259
+      - 0.07019275426864624
+      - 0.2616892158985138
+      - 0.11751057207584381
+      - 0.16968433558940887
+      - 1.0
+      q01:
+      - -0.01538565568625927
+      - -0.021047022193670273
+      - -0.01688069850206375
+      - -0.044314172118902206
+      - -0.03890235349535942
+      - -0.04788423702120781
+      - 0.0
+      q99:
+      - 0.014661382883787155
+      - 0.026515591889619827
+      - 0.021398313343524933
+      - 0.04216696694493294
+      - 0.03401297703385353
+      - 0.04957397282123566
+      - 1.0
+    num_entries: 1560068