Adding modes, graphs and metadata.

README.md

@@ -42,7 +42,7 @@ Here is a detailed view on how the remaining heads are distributed in the networ
 
 ## Density plot
 
-<script src="/madlag/bert-base-uncased-squad1.1-block-sparse-0.13-v1/raw/main/model_card/density.js" id="59895319-1808-4770-b2c0-616dd35ac762"></script>
+<script src="/madlag/bert-base-uncased-squad1.1-block-sparse-0.13-v1/raw/main/model_card/density.js" id="2087227b-6b81-4065-969b-41ea1f61c72e"></script>
 
 ## Details
 

model_meta.json

@@ -0,0 +1,160 @@
+{
+    "args": {
+        "adam_epsilon": 1e-08, 
+        "alpha_ce": 0.1, 
+        "alpha_distil": 0.9, 
+        "ampere_learning_rate": 0.01, 
+        "ampere_mask_init": "constant", 
+        "ampere_mask_scale": 0.0, 
+        "ampere_pruning_method": "disabled", 
+        "cache_dir": "", 
+        "config_name": "", 
+        "data_dir": "squad_data", 
+        "do_eval": true, 
+        "do_lower_case": true, 
+        "do_train": true, 
+        "doc_stride": 128, 
+        "eval_all_checkpoints": true, 
+        "eval_batch_size": 16, 
+        "evaluate_during_training": false, 
+        "final_ampere_temperature": 20, 
+        "final_lambda": 100, 
+        "final_shuffling_temperature": 20, 
+        "final_threshold": 0.1, 
+        "final_warmup": 10, 
+        "fp16": false, 
+        "fp16_opt_level": "O1", 
+        "global_topk": false, 
+        "global_topk_frequency_compute": 25, 
+        "gradient_accumulation_steps": 1, 
+        "in_shuffling_group": 4, 
+        "initial_ampere_temperature": 0.0, 
+        "initial_shuffling_temperature": 0.1, 
+        "initial_threshold": 0.0, 
+        "initial_warmup": 1, 
+        "lang_id": 0, 
+        "learning_rate": 3e-05, 
+        "local_rank": -1, 
+        "logging_steps": 500, 
+        "mask_block_cols": 32, 
+        "mask_block_rows": 32, 
+        "mask_init": "constant", 
+        "mask_scale": 0.0, 
+        "mask_scores_learning_rate": 0.01, 
+        "max_answer_length": 30, 
+        "max_grad_norm": 1.0, 
+        "max_query_length": 64, 
+        "max_seq_length": 384, 
+        "max_steps": -1, 
+        "model_name_or_path": "bert-base-uncased", 
+        "model_type": "masked_bert", 
+        "n_best_size": 20, 
+        "n_gpu": 1, 
+        "no_cuda": false, 
+        "null_score_diff_threshold": 0.0, 
+        "num_train_epochs": 20.0, 
+        "out_shuffling_group": 4, 
+        "overwrite_cache": false, 
+        "overwrite_output_dir": true, 
+        "per_gpu_eval_batch_size": 16, 
+        "per_gpu_train_batch_size": 16, 
+        "predict_file": "dev-v1.1.json", 
+        "pruning_method": "sigmoied_threshold", 
+        "pruning_submethod": "default", 
+        "regularization": "l1", 
+        "save_steps": 5000, 
+        "seed": 42, 
+        "server_ip": "", 
+        "server_port": "", 
+        "shuffling_learning_rate": 0.001, 
+        "shuffling_method": "disabled", 
+        "teacher_name_or_path": "csarron/bert-base-uncased-squad-v1", 
+        "teacher_type": "bert", 
+        "temperature": 2.0, 
+        "threads": 8, 
+        "tokenizer_name": "", 
+        "train_batch_size": 16, 
+        "train_file": "train-v1.1.json", 
+        "truncate_train_examples": -1, 
+        "verbose_logging": false, 
+        "version_2_with_negative": false, 
+        "warmup_steps": 5400, 
+        "weight_decay": 0.0
+    }, 
+    "config": {
+        "_name_or_path": "bert-base-uncased", 
+        "ampere_mask_init": "constant", 
+        "ampere_mask_scale": 0.0, 
+        "ampere_pruning_method": "disabled", 
+        "architectures": ["MaskedBertForQuestionAnswering"], 
+        "attention_probs_dropout_prob": 0.1, 
+        "hidden_act": "gelu", 
+        "hidden_dropout_prob": 0.1, 
+        "hidden_size": 768, 
+        "in_shuffling_group": 4, 
+        "initializer_range": 0.02, 
+        "intermediate_size": 3072, 
+        "layer_norm_eps": 1e-12, 
+        "mask_block_cols": 32, 
+        "mask_block_rows": 32, 
+        "mask_init": "constant", 
+        "mask_scale": 0.0, 
+        "max_position_embeddings": 512, 
+        "model_type": "masked_bert", 
+        "num_attention_heads": 12, 
+        "num_hidden_layers": 12, 
+        "out_shuffling_group": 4, 
+        "pad_token_id": 0, 
+        "pruning_method": "sigmoied_threshold", 
+        "pruning_submethod": "default", 
+        "shuffling_method": "disabled", 
+        "type_vocab_size": 2, 
+        "vocab_size": 30522
+    }, 
+    "packaging": {
+        "model_name": "madlag/bert-base-uncased-squad1.1-block-sparse-0.13-v1", 
+        "model_owner": "madlag", 
+        "pytorch_final_file_size": 359300023
+    }, 
+    "performance": {
+        "dense": {
+            "eval_elapsed_time": 43.292644419998396
+        }, 
+        "pytorch_block_sparse": {
+            "eval_elapsed_time": 26.211968197021633
+        }, 
+        "speedup": 1.6516365384922744
+    }, 
+    "precision": {
+        "exact": 74.38978576660156, 
+        "f1": 83.25814819335938
+    }, 
+    "sparsity": {
+        "ampere": false, 
+        "block_size": [32, 32], 
+        "block_sparse": true, 
+        "block_sparse_density": 0.12510850694444445, 
+        "block_sparse_nnz": 10377, 
+        "block_sparse_total": 82944, 
+        "global_density": 0.32128202590889765, 
+        "is_block_sparse_valid": true, 
+        "nnz_parameters": 35175170, 
+        "parameters": 109483778, 
+        "pruned_heads": {
+            "0": [0, 1, 2, 4, 5, 6, 7, 9, 11], 
+            "1": [0, 1, 2, 3, 5, 6, 7, 8, 9], 
+            "2": [1, 2, 3, 4, 5, 7, 8, 10, 11], 
+            "3": [2, 3, 4, 6, 7, 10], 
+            "4": [0, 1, 2, 4, 6, 7, 8, 10, 11], 
+            "5": [0, 1, 2, 4, 5, 6, 7, 11], 
+            "6": [0, 2, 3, 4, 6, 7, 10], 
+            "7": [1, 2, 3, 5, 6, 7, 11], 
+            "8": [0, 1, 2, 3, 4, 5, 6, 7, 8], 
+            "9": [1, 3, 4, 5, 7, 9, 10], 
+            "10": [0, 1, 2, 4, 5, 6, 7, 8, 9], 
+            "11": [0, 2, 3, 5, 7, 8, 10, 11]
+        }, 
+        "total_attention_heads": 144, 
+        "total_pruned_attention_heads": 97
+    }
+}