TICO/tico/quantization/examples/configs/qwen3_vl_eval_suite_mx.yaml at ad0eb263726eade891281b8dce0ea8e144b2b90c · Samsung/TICO · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
model:
  family: qwen3_vl
  name_or_path: Qwen/Qwen3-VL-2B-Instruct
  trust_remote_code: true
  hf_token: null
  cache_dir: null

runtime:
  device: cuda
  dtype: float32
  seed: 42
  show_progress: true

calibration:
  datasets:
    - dataset: vqav2
      split: testdev
      n_samples: 128
    - dataset: wikitext2
      split: train
      n_samples: 128
  seq_len: 2048

model_args:
  vision:
    grid_thw: [8, 24, 24]
    visual_start_idx: 0
    spatial_merge_size: 2

pipeline:
  - name: spinquant
    enabled: true
    init_method: random
    r1_path: null
    r2_map_path: null
    enable_r1: true
    enable_r2: true
    fuse_deepstack_visual_outputs: true
    vision_init_method: random
    vision_r1_path: null
    vision_r2_map_path: null
    enable_vision_r1: true
    enable_vision_r2: true
    fuse_vision_layer_norms: true
    require_vision_r1_layernorm_compatible: true
    vision_rotation_tolerance: 1.0e-4

  - name: smoothquant
    enabled: false
    alpha: 0.5
    components: both
    custom_alpha_map: null

  - name: gptq
    enabled: true
    weight_bits: 4
    weight_bits_overrides: {}
    perchannel: true
    symmetric: false
    mse: null
    sensitivity:
      mode: compute
      path: null
    percdamp: 0.01
    groupsize: -1
    actorder: true
    static_groups: false
    verbose: false
    show_progress: true
    quantize_vision: true
    quantize_text: true
    quantize_lm_head: false
    quantize_vision_patch_embed: true
    quantize_vision_blocks: true
    quantize_vision_merger: true
    quantize_vision_deepstack_mergers: true
    quantize_text_layers: true
    move_cache_to_cpu: false

  - name: ptq
    enabled: true
    # Print the full model after PTQ.
    print_model: true
    # Global default for ALL activation observers (int16).
    # Observers not explicitly overridden below will use this.
    activation: int16
    # Override: all nn.Linear act_in/act_out observers.
    # Set to null to disable (fall back to global ``activation``).
    linear_activation:
      kind: mx
      elem_format: int8
      axis: -1
      shared_exp_method: max
      round: nearest
    # Override: all attention softmax observers.
    # Set to null to disable (fall back to global ``activation``).
    softmax_activation:
      kind: mx
      elem_format: int8
      axis: -1
      shared_exp_method: max
      round: nearest
    linear_weight: uint4
    vision_patch_embed_weight: uint8
    embedding_weight: uint8
    lm_head_weight: uint8
    spin_rotation_weight: int16
    norm: int16
    norm_weight: int16
    strict_wrap: true

evaluation:
  enabled: true
  vlm_tasks:
    - vqav2
    - textvqa
  coco: true
  # LLaVA-Bench is open-ended natural QA. The legacy CIDEr/BLEU path is not
  # enabled by default here. Use mode=judge for benchmark-style scoring.
  llava_bench:
    enabled: false
    mode: judge
    dataset: lmms-lab/llava-bench-in-the-wild
    split: train
    n_samples: 50
    start_index: 0
    max_seq_len: 2048
    max_new_tokens: 512
    temperature: 0.0
    candidate_label: Qwen/Qwen3-VL-2B-Instruct
    baseline_label: reference
    candidate_answers: null
    baseline_answers: null
    regenerate: false
    output:
      dir: ./out/llava_bench
      answers: ./out/llava_bench/qwen3_vl_2b.answers.jsonl
      reviews: ./out/llava_bench/qwen3_vl_2b.llama3_2_3b.reviews.jsonl
      summary: ./out/llava_bench/qwen3_vl_2b.llama3_2_3b.summary.json
    judge:
      enabled: true
      model_id: meta-llama/Llama-3.2-3B-Instruct
      device: cuda
      dtype: float16
      max_new_tokens: 256
      temperature: 0.0
      swap_order: true
  videomme:
    enabled: true
    batch_size: 1
    max_new_tokens: 30
    n_samples: -1
    max_num_frames: 32
    use_cache: null
    verbose: false
  mmlu:
    enabled: true
    subjects: null
    n_shots: 5
    n_samples: -1
    batch_size: 1
  hellaswag:
    enabled: true
    n_shots: 10
    n_samples: -1
    batch_size: 1
  mmmu:
    enabled: true
    dataset: MMMU/MMMU
    subjects: null
    n_shots: 5
    n_samples: -1
    max_new_tokens: 16
    temperature: 0.0
    verbose: false
  ppl:
    enabled: true
    dataset: wikitext2
    split: test
    stride: 512
  n_samples: -1
  max_seq_len: 2048

export:
  enabled: false
  output_dir: ./out/qwen3_vl_eval_suite
  artifacts:
    - ptq_checkpoint