[Preset] Add model preset for model delivery (#2553)

[Preset] Add model preset for wasm delivery

Author: CharlieFRuan

python/mlc_llm/model/model_preset.py

@@ -153,6 +153,30 @@
         "context_window_size": 2048,
         "prefill_chunk_size": 2048,
     },
+    "tinyllama_1b_chat_v0.4": {
+        "_name_or_path": "/data/tianduo/tinyllama-ft/checkpoint-3890",
+        "architectures": ["LlamaForCausalLM"],
+        "bos_token_id": 1,
+        "eos_token_id": 2,
+        "hidden_act": "silu",
+        "hidden_size": 2048,
+        "initializer_range": 0.02,
+        "intermediate_size": 5632,
+        "max_position_embeddings": 2048,
+        "model_type": "llama",
+        "num_attention_heads": 32,
+        "num_hidden_layers": 22,
+        "num_key_value_heads": 4,
+        "pretraining_tp": 1,
+        "rms_norm_eps": 1e-05,
+        "rope_scaling": None,
+        "rope_theta": 10000.0,
+        "tie_word_embeddings": False,
+        "torch_dtype": "float32",
+        "transformers_version": "4.33.1",
+        "use_cache": False,
+        "vocab_size": 32003,
+    },
     "tinyllama_1b_chat_v1.0": {
         "architectures": ["LlamaForCausalLM"],
         "attention_bias": False,
@@ -201,23 +225,78 @@
         "prefill_chunk_size": 128,
         "attention_sink_size": 4,
     },
+    "mistral_7b_v03": {
+        "architectures": ["MistralForCausalLM"],
+        "attention_dropout": 0.0,
+        "bos_token_id": 1,
+        "eos_token_id": 2,
+        "hidden_act": "silu",
+        "hidden_size": 4096,
+        "initializer_range": 0.02,
+        "intermediate_size": 14336,
+        "max_position_embeddings": 32768,
+        "model_type": "mistral",
+        "num_attention_heads": 32,
+        "num_hidden_layers": 32,
+        "num_key_value_heads": 8,
+        "rms_norm_eps": 1e-05,
+        "rope_theta": 1000000.0,
+        "sliding_window": None,
+        "tie_word_embeddings": False,
+        "torch_dtype": "bfloat16",
+        "transformers_version": "4.42.0.dev0",
+        "use_cache": True,
+        "vocab_size": 32768,
+    },
     "gpt2": {
+        "activation_function": "gelu_new",
         "architectures": ["GPT2LMHeadModel"],
+        "attn_pdrop": 0.1,
         "bos_token_id": 50256,
+        "embd_pdrop": 0.1,
         "eos_token_id": 50256,
-        "hidden_act": "gelu_new",
-        "n_embd": 768,
         "initializer_range": 0.02,
-        "n_positions": 1024,
+        "layer_norm_epsilon": 1e-05,
         "model_type": "gpt2",
+        "n_ctx": 1024,
+        "n_embd": 768,
         "n_head": 12,
         "n_layer": 12,
+        "n_positions": 1024,
+        "resid_pdrop": 0.1,
+        "summary_activation": None,
+        "summary_first_dropout": 0.1,
+        "summary_proj_to_labels": True,
+        "summary_type": "cls_index",
+        "summary_use_proj": True,
+        "task_specific_params": {"text-generation": {"do_sample": True, "max_length": 50}},
+        "vocab_size": 50257,
+    },
+    "gpt2_medium": {
+        "activation_function": "gelu_new",
+        "architectures": ["GPT2LMHeadModel"],
+        "attn_pdrop": 0.1,
+        "bos_token_id": 50256,
+        "embd_pdrop": 0.1,
+        "eos_token_id": 50256,
+        "initializer_range": 0.02,
         "layer_norm_epsilon": 1e-05,
-        "transformers_version": "4.26.0.dev0",
-        "use_cache": True,
+        "model_type": "gpt2",
+        "n_ctx": 1024,
+        "n_embd": 1024,
+        "n_head": 16,
+        "n_layer": 24,
+        "n_positions": 1024,
+        "n_special": 0,
+        "predict_special_tokens": True,
+        "resid_pdrop": 0.1,
+        "summary_activation": None,
+        "summary_first_dropout": 0.1,
+        "summary_proj_to_labels": True,
+        "summary_type": "cls_index",
+        "summary_use_proj": True,
+        "task_specific_params": {"text-generation": {"do_sample": True, "max_length": 50}},
         "vocab_size": 50257,
-        "context_window_size": 2048,
-        "prefill_chunk_size": 2048,
     },
     "gpt_bigcode": {
         "activation_function": "gelu_pytorch_tanh",
@@ -796,4 +875,102 @@
         "type_vocab_size": 2,
         "vocab_size": 30522,
     },
+    "stablelm-2-zephyr-1_6b": {
+        "architectures": ["StableLmForCausalLM"],
+        "bos_token_id": 100257,
+        "eos_token_id": 100257,
+        "hidden_act": "silu",
+        "hidden_size": 2048,
+        "initializer_range": 0.02,
+        "intermediate_size": 5632,
+        "max_position_embeddings": 4096,
+        "model_type": "stablelm",
+        "layer_norm_eps": 1e-05,
+        "num_attention_heads": 32,
+        "num_hidden_layers": 24,
+        "num_key_value_heads": 32,
+        "partial_rotary_factor": 0.25,
+        "rope_theta": 10000,
+        "tie_word_embeddings": False,
+        "torch_dtype": "float16",
+        "transformers_version": "4.38.0",
+        "use_cache": True,
+        "use_qkv_bias": True,
+        "vocab_size": 100352,
+    },
+    "qwen2_0_5b": {
+        "architectures": ["Qwen2ForCausalLM"],
+        "attention_dropout": 0.0,
+        "bos_token_id": 151643,
+        "eos_token_id": 151645,
+        "hidden_act": "silu",
+        "hidden_size": 896,
+        "initializer_range": 0.02,
+        "intermediate_size": 4864,
+        "max_position_embeddings": 32768,
+        "max_window_layers": 24,
+        "model_type": "qwen2",
+        "num_attention_heads": 14,
+        "num_hidden_layers": 24,
+        "num_key_value_heads": 2,
+        "rms_norm_eps": 1e-06,
+        "rope_theta": 1000000.0,
+        "sliding_window": 32768,
+        "tie_word_embeddings": True,
+        "torch_dtype": "bfloat16",
+        "transformers_version": "4.40.1",
+        "use_cache": True,
+        "use_sliding_window": False,
+        "vocab_size": 151936,
+    },
+    "qwen2_1_5b": {
+        "architectures": ["Qwen2ForCausalLM"],
+        "attention_dropout": 0.0,
+        "bos_token_id": 151643,
+        "eos_token_id": 151645,
+        "hidden_act": "silu",
+        "hidden_size": 1536,
+        "initializer_range": 0.02,
+        "intermediate_size": 8960,
+        "max_position_embeddings": 32768,
+        "max_window_layers": 28,
+        "model_type": "qwen2",
+        "num_attention_heads": 12,
+        "num_hidden_layers": 28,
+        "num_key_value_heads": 2,
+        "rms_norm_eps": 1e-06,
+        "rope_theta": 1000000.0,
+        "sliding_window": 32768,
+        "tie_word_embeddings": True,
+        "torch_dtype": "bfloat16",
+        "transformers_version": "4.40.1",
+        "use_cache": True,
+        "use_sliding_window": False,
+        "vocab_size": 151936,
+    },
+    "qwen2_7b": {
+        "architectures": ["Qwen2ForCausalLM"],
+        "attention_dropout": 0.0,
+        "bos_token_id": 151643,
+        "eos_token_id": 151645,
+        "hidden_act": "silu",
+        "hidden_size": 3584,
+        "initializer_range": 0.02,
+        "intermediate_size": 18944,
+        "max_position_embeddings": 32768,
+        "max_window_layers": 28,
+        "model_type": "qwen2",
+        "num_attention_heads": 28,
+        "num_hidden_layers": 28,
+        "num_key_value_heads": 4,
+        "rms_norm_eps": 1e-06,
+        "rope_theta": 1000000.0,
+        "sliding_window": 131072,
+        "tie_word_embeddings": False,
+        "torch_dtype": "bfloat16",
+        "transformers_version": "4.41.2",
+        "use_cache": True,
+        "use_sliding_window": False,
+        "vocab_size": 152064,
+    },
 }