quan.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"TRANSFORMERS_CACHE\"] = \"/scratch/megathon/cache/\"\n",
    "\n",
    "from med_llama import *\n",
    "\n",
    "kargs = {\n",
    "    \"quant_method\": \"q5_k_m\",\n",
    "    \"n\": 4096,\n",
    "    \"penalty\": 1.1,\n",
    "    \"ngl\": 35,\n",
    "    \"device\": \"cuda\",\n",
    "    \"con_docs\": \"webmd_context_docs.txt\",\n",
    "    \"chunk_size\": 200,\n",
    "    \"chunk_overlap\": 10,\n",
    "    \"embed_file\": \"sentence-transformers/all-mpnet-base-v2\"\n",
    "}\n",
    "\n",
    "prompt = \"What should I do if I want to stop dialysis?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# to convert to quantized model, GGML, Q4 and Q5\n",
    "root = \"/scratch/megathon/cache/models--meta-llama--Llama-2-7b-chat-hf/snapshots/94b07a6e30c3292b8265ed32ffdeccfdadf434a8\"\n",
    "quantize_model(root, \"/scratch/megathon/quant/metaquant2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running with quantized model /scratch/megathon/quant/metaquant.gguf.q5_k_m.bin\n",
      "Log start\n",
      "main: build = 1441 (ff3bad8)\n",
      "main: built with cc (Ubuntu 7.5.0-3ubuntu1~18.04) 7.5.0 for x86_64-linux-gnu\n",
      "main: seed  = 1698566820\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 1 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6\n",
      "llama_model_loader: loaded meta data with 19 key-value pairs and 291 tensors from /scratch/megathon/quant/metaquant.gguf.q5_k_m.bin (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight q5_K     [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:           blk.0.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:            blk.0.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:            blk.0.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:            blk.0.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:              blk.0.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:         blk.0.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:              blk.0.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.1.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.1.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:            blk.1.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:            blk.1.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:              blk.1.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:         blk.1.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:              blk.1.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:          blk.10.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:           blk.10.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:           blk.10.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:             blk.10.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:           blk.10.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:             blk.10.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:        blk.10.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:             blk.10.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:             blk.10.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:          blk.11.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:           blk.11.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:           blk.11.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:             blk.11.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:           blk.11.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:             blk.11.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:        blk.11.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:             blk.11.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:             blk.11.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:          blk.12.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:           blk.12.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:           blk.12.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:             blk.12.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:           blk.12.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:             blk.12.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:        blk.12.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:             blk.12.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:             blk.12.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:          blk.13.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:           blk.13.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:           blk.13.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:             blk.13.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:           blk.13.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:             blk.13.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:        blk.13.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:             blk.13.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:             blk.13.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:          blk.14.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:           blk.14.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:           blk.14.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:             blk.14.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:           blk.14.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:             blk.14.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:        blk.14.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:             blk.14.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:             blk.14.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:          blk.15.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:           blk.15.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:           blk.15.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:             blk.15.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:           blk.15.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:             blk.15.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:        blk.15.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:             blk.15.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:             blk.15.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:          blk.16.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:           blk.16.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:           blk.16.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:             blk.16.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:           blk.16.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:             blk.16.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:        blk.16.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:             blk.16.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:             blk.16.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:          blk.17.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:           blk.17.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:           blk.17.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:             blk.17.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:           blk.17.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:             blk.17.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:        blk.17.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:             blk.17.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:             blk.17.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:          blk.18.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:           blk.18.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:           blk.18.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.18.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:           blk.18.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:             blk.18.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:        blk.18.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:             blk.18.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.18.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.19.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.19.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:           blk.19.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.19.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:           blk.19.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:             blk.19.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:        blk.19.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:             blk.19.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.19.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:           blk.2.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:            blk.2.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:            blk.2.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:              blk.2.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:            blk.2.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:              blk.2.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:         blk.2.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:              blk.2.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:              blk.2.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.20.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.20.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:           blk.20.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.20.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:           blk.20.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:             blk.20.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:        blk.20.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:             blk.20.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.20.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.21.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.21.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:           blk.21.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.21.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:           blk.21.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:             blk.21.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:        blk.21.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:             blk.21.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.21.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.22.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.22.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:           blk.22.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.22.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:           blk.22.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:             blk.22.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:        blk.22.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:             blk.22.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.22.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.23.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.23.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:           blk.23.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.23.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:           blk.23.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:             blk.23.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:        blk.23.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:             blk.23.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.23.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:           blk.3.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:            blk.3.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:            blk.3.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:              blk.3.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:            blk.3.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:              blk.3.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:         blk.3.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:              blk.3.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:              blk.3.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:           blk.4.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:            blk.4.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:            blk.4.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:              blk.4.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:            blk.4.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:              blk.4.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:         blk.4.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:              blk.4.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:              blk.4.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:           blk.5.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:            blk.5.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:            blk.5.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:              blk.5.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:            blk.5.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:              blk.5.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:         blk.5.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:              blk.5.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:              blk.5.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:           blk.6.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:            blk.6.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:            blk.6.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:              blk.6.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:            blk.6.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:              blk.6.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:         blk.6.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:              blk.6.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:              blk.6.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:           blk.7.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:            blk.7.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:            blk.7.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:              blk.7.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:            blk.7.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:              blk.7.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:         blk.7.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:              blk.7.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:              blk.7.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:           blk.8.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:            blk.8.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:            blk.8.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:              blk.8.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:            blk.8.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:              blk.8.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:         blk.8.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:              blk.8.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:              blk.8.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:           blk.9.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:            blk.9.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:            blk.9.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:              blk.9.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:            blk.9.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:              blk.9.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:         blk.9.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:              blk.9.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:              blk.9.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:                    output.weight q6_K     [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:          blk.24.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:           blk.24.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:           blk.24.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:           blk.24.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:             blk.24.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:        blk.24.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:             blk.24.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:          blk.25.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:           blk.25.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:           blk.25.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:           blk.25.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:             blk.25.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:        blk.25.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:             blk.25.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:          blk.26.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:           blk.26.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:           blk.26.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:           blk.26.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:             blk.26.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:        blk.26.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:             blk.26.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:          blk.27.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:           blk.27.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:           blk.27.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:           blk.27.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:             blk.27.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:        blk.27.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:             blk.27.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:          blk.28.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:           blk.28.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:           blk.28.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:           blk.28.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:             blk.28.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:        blk.28.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:             blk.28.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:          blk.29.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:           blk.29.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:           blk.29.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:           blk.29.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:             blk.29.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:        blk.29.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:             blk.29.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:          blk.30.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:           blk.30.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:           blk.30.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:           blk.30.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:             blk.30.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:        blk.30.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:             blk.30.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:          blk.31.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:           blk.31.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:           blk.31.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:           blk.31.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:             blk.31.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:        blk.31.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:             blk.31.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:               output_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - kv   0:                       general.architecture str     \n",
      "llama_model_loader: - kv   1:                               general.name str     \n",
      "llama_model_loader: - kv   2:                       llama.context_length u32     \n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32     \n",
      "llama_model_loader: - kv   4:                          llama.block_count u32     \n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32     \n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32     \n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32     \n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32     \n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32     \n",
      "llama_model_loader: - kv  10:                          general.file_type u32     \n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str     \n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr     \n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr     \n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr     \n",
      "llama_model_loader: - kv  15:                tokenizer.ggml.bos_token_id u32     \n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.eos_token_id u32     \n",
      "llama_model_loader: - kv  17:            tokenizer.ggml.unknown_token_id u32     \n",
      "llama_model_loader: - kv  18:               general.quantization_version u32     \n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type q5_K:  193 tensors\n",
      "llama_model_loader: - type q6_K:   33 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = mostly Q5_K - Medium\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 4.45 GiB (5.68 BPW) \n",
      "llm_load_print_meta: general.name   = LLaMA v2\n",
      "llm_load_print_meta: BOS token = 1 '<s>'\n",
      "llm_load_print_meta: EOS token = 2 '</s>'\n",
      "llm_load_print_meta: UNK token = 0 '<unk>'\n",
      "llm_load_print_meta: LF token  = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size =    0.10 MB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: mem required  =   86.04 MB\n",
      "llm_load_tensors: offloading 32 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 35/35 layers to GPU\n",
      "llm_load_tensors: VRAM used: 4474.93 MB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: offloading v cache to GPU\n",
      "llama_kv_cache_init: offloading k cache to GPU\n",
      "llama_kv_cache_init: VRAM kv self = 256.00 MB\n",
      "llama_new_context_with_model: kv self size  =  256.00 MB\n",
      "llama_new_context_with_model: compute buffer total size = 76.63 MB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 70.50 MB\n",
      "llama_new_context_with_model: total VRAM used: 4801.43 MB (model: 4474.93 MB, context: 326.50 MB)\n",
      "\n",
      "system_info: n_threads = 24 / 48 | AVX = 1 | AVX2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, typical_p = 1.000, temp = 0.800\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 4096, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m\u001b[0m [end of text]\n",
      "\n",
      "llama_print_timings:        load time =     755.81 ms\n",
      "llama_print_timings:      sample time =     111.82 ms /   533 runs   (    0.21 ms per token,  4766.55 tokens per second)\n",
      "llama_print_timings: prompt eval time =      53.31 ms /    17 tokens (    3.14 ms per token,   318.90 tokens per second)\n",
      "llama_print_timings:        eval time =    5044.79 ms /   532 runs   (    9.48 ms per token,   105.46 tokens per second)\n",
      "llama_print_timings:       total time =    5334.62 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "# outputs from vanilla llama\n",
    "model_name = \"/scratch/megathon/quant/metaquant\"\n",
    "ggml_version = \"gguf\"\n",
    "qtype = f\"{model_name}.{ggml_version}.{kargs['quant_method']}.bin\"\n",
    "print(f\"Running with quantized model {qtype}\")\n",
    "\n",
    "!/scratch/megathon/quant/llama.cpp/main -m {qtype} -n {kargs['n']} --repeat_penalty {kargs['penalty']} --color -ngl {kargs['ngl']} -p f\"\\'{prompt}\\'\" > output.txt\n",
    "with open('output.txt') as f:\n",
    "    output = f.read()\n",
    "os.remove(\"output.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f\\'What should I do if I want to stop dialysis?\\'\n",
      "Here are some steps you can take if you want to stop dialysis:\n",
      "\n",
      "1. Discuss your decision with your nephrologist or kidney doctor: It is important to discuss your decision with your nephrologist or kidney doctor before stopping dialysis. They can help you understand the potential risks and benefits of stopping dialysis and provide guidance on how to do so safely.\n",
      "2. Review your medical history: Your nephrologist or kidney doctor will review your medical history to determine if stopping dialysis is safe for you. They may consider factors such as your age, overall health, and the cause of your kidney failure.\n",
      "3. Check your blood chemistry and other test results: Your nephrologist or kidney doctor may perform additional tests to check your blood chemistry and other vital signs before stopping dialysis. This can help ensure that you are in a stable condition and that stopping dialysis will not cause any immediate harm.\n",
      "4. Consider hospice care: If you have a terminal illness and are not likely to recover, your nephrologist or kidney doctor may recommend hospice care. Hospice care provides comfort measures and supportive care to help you manage your symptoms and improve your quality of life.\n",
      "5. Make arrangements for end-of-life care: If you decide to stop dialysis, it is important to make arrangements for end-of-life care. This can include advance directives, such as a living will or healthcare power of attorney, which can help ensure that your wishes are respected.\n",
      "6. Seek support from family and friends: Stopping dialysis can be emotionally challenging, so it is important to seek support from family and friends during this time. They can provide emotional support and help with practical tasks, such as managing your medications and caring for you at home.\n",
      "7. Consider alternative options: Depending on your specific situation, there may be alternative options available to help manage your kidney failure. These can include peritoneal dialysis, hemodialysis, or even a kidney transplant. Your nephrologist or kidney doctor can help you determine if these options are appropriate for you.\n",
      "\n",
      "It is important to note that stopping dialysis is a complex decision that should be made with the guidance of your healthcare provider and support from loved ones. It is also important to consider your personal preferences and values when making this decision.\n"
     ]
    }
   ],
   "source": [
    "print(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running with quantized model /scratch/megathon/quant/metaquant.gguf.q5_k_m.bin\n",
      "Generating vectorstore\n",
      "Getting context\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1441 (ff3bad8)\n",
      "main: built with cc (Ubuntu 7.5.0-3ubuntu1~18.04) 7.5.0 for x86_64-linux-gnu\n",
      "main: seed  = 1698567038\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 1 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6\n",
      "llama_model_loader: loaded meta data with 19 key-value pairs and 291 tensors from /scratch/megathon/quant/metaquant.gguf.q5_k_m.bin (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight q5_K     [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:           blk.0.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:            blk.0.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:            blk.0.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:            blk.0.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:              blk.0.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:         blk.0.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:              blk.0.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.1.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.1.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:            blk.1.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:            blk.1.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:              blk.1.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:         blk.1.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:              blk.1.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:          blk.10.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:           blk.10.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:           blk.10.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:             blk.10.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:           blk.10.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:             blk.10.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:        blk.10.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:             blk.10.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:             blk.10.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:          blk.11.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:           blk.11.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:           blk.11.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:             blk.11.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:           blk.11.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:             blk.11.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:        blk.11.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:             blk.11.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:             blk.11.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:          blk.12.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:           blk.12.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:           blk.12.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:             blk.12.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:           blk.12.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:             blk.12.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:        blk.12.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:             blk.12.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:             blk.12.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:          blk.13.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:           blk.13.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:           blk.13.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:             blk.13.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:           blk.13.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:             blk.13.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:        blk.13.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:             blk.13.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:             blk.13.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:          blk.14.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:           blk.14.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:           blk.14.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:             blk.14.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:           blk.14.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:             blk.14.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:        blk.14.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:             blk.14.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:             blk.14.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:          blk.15.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:           blk.15.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:           blk.15.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:             blk.15.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:           blk.15.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:             blk.15.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:        blk.15.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:             blk.15.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:             blk.15.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:          blk.16.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:           blk.16.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:           blk.16.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:             blk.16.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:           blk.16.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:             blk.16.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:        blk.16.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:             blk.16.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:             blk.16.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:          blk.17.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:           blk.17.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:           blk.17.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:             blk.17.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:           blk.17.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:             blk.17.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:        blk.17.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:             blk.17.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:             blk.17.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:          blk.18.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:           blk.18.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:           blk.18.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.18.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:           blk.18.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:             blk.18.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:        blk.18.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:             blk.18.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.18.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.19.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.19.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:           blk.19.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.19.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:           blk.19.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:             blk.19.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:        blk.19.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:             blk.19.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.19.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:           blk.2.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:            blk.2.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:            blk.2.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:              blk.2.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:            blk.2.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:              blk.2.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:         blk.2.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:              blk.2.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:              blk.2.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.20.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.20.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:           blk.20.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.20.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:           blk.20.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:             blk.20.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:        blk.20.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:             blk.20.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.20.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.21.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.21.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:           blk.21.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.21.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:           blk.21.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:             blk.21.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:        blk.21.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:             blk.21.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.21.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.22.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.22.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:           blk.22.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.22.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:           blk.22.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:             blk.22.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:        blk.22.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:             blk.22.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.22.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.23.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.23.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:           blk.23.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.23.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:           blk.23.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:             blk.23.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:        blk.23.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:             blk.23.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.23.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:           blk.3.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:            blk.3.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:            blk.3.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:              blk.3.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:            blk.3.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:              blk.3.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:         blk.3.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:              blk.3.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:              blk.3.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:           blk.4.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:            blk.4.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:            blk.4.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:              blk.4.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:            blk.4.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:              blk.4.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:         blk.4.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:              blk.4.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:              blk.4.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:           blk.5.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:            blk.5.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:            blk.5.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:              blk.5.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:            blk.5.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:              blk.5.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:         blk.5.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:              blk.5.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:              blk.5.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:           blk.6.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:            blk.6.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:            blk.6.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:              blk.6.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:            blk.6.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:              blk.6.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:         blk.6.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:              blk.6.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:              blk.6.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:           blk.7.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:            blk.7.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:            blk.7.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:              blk.7.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:            blk.7.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:              blk.7.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:         blk.7.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:              blk.7.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:              blk.7.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:           blk.8.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:            blk.8.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:            blk.8.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:              blk.8.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:            blk.8.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:              blk.8.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:         blk.8.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:              blk.8.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:              blk.8.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:           blk.9.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:            blk.9.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:            blk.9.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:              blk.9.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:            blk.9.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:              blk.9.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:         blk.9.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:              blk.9.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:              blk.9.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:                    output.weight q6_K     [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:          blk.24.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:           blk.24.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:           blk.24.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:           blk.24.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:             blk.24.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:        blk.24.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:             blk.24.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:          blk.25.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:           blk.25.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:           blk.25.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:           blk.25.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:             blk.25.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:        blk.25.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:             blk.25.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:          blk.26.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:           blk.26.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:           blk.26.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:           blk.26.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:             blk.26.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:        blk.26.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:             blk.26.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:          blk.27.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:           blk.27.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:           blk.27.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:           blk.27.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:             blk.27.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:        blk.27.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:             blk.27.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:          blk.28.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:           blk.28.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:           blk.28.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:           blk.28.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:             blk.28.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:        blk.28.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:             blk.28.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:          blk.29.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:           blk.29.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:           blk.29.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:           blk.29.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:             blk.29.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:        blk.29.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:             blk.29.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:          blk.30.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:           blk.30.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:           blk.30.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:           blk.30.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:             blk.30.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:        blk.30.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:             blk.30.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:          blk.31.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:           blk.31.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:           blk.31.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:           blk.31.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:             blk.31.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:        blk.31.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:             blk.31.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:               output_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - kv   0:                       general.architecture str     \n",
      "llama_model_loader: - kv   1:                               general.name str     \n",
      "llama_model_loader: - kv   2:                       llama.context_length u32     \n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32     \n",
      "llama_model_loader: - kv   4:                          llama.block_count u32     \n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32     \n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32     \n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32     \n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32     \n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32     \n",
      "llama_model_loader: - kv  10:                          general.file_type u32     \n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str     \n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr     \n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr     \n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr     \n",
      "llama_model_loader: - kv  15:                tokenizer.ggml.bos_token_id u32     \n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.eos_token_id u32     \n",
      "llama_model_loader: - kv  17:            tokenizer.ggml.unknown_token_id u32     \n",
      "llama_model_loader: - kv  18:               general.quantization_version u32     \n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type q5_K:  193 tensors\n",
      "llama_model_loader: - type q6_K:   33 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = mostly Q5_K - Medium\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 4.45 GiB (5.68 BPW) \n",
      "llm_load_print_meta: general.name   = LLaMA v2\n",
      "llm_load_print_meta: BOS token = 1 '<s>'\n",
      "llm_load_print_meta: EOS token = 2 '</s>'\n",
      "llm_load_print_meta: UNK token = 0 '<unk>'\n",
      "llm_load_print_meta: LF token  = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size =    0.10 MB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: mem required  =   86.04 MB\n",
      "llm_load_tensors: offloading 32 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 35/35 layers to GPU\n",
      "llm_load_tensors: VRAM used: 4474.93 MB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: offloading v cache to GPU\n",
      "llama_kv_cache_init: offloading k cache to GPU\n",
      "llama_kv_cache_init: VRAM kv self = 256.00 MB\n",
      "llama_new_context_with_model: kv self size  =  256.00 MB\n",
      "llama_new_context_with_model: compute buffer total size = 76.63 MB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 70.50 MB\n",
      "llama_new_context_with_model: total VRAM used: 4801.43 MB (model: 4474.93 MB, context: 326.50 MB)\n",
      "\n",
      "system_info: n_threads = 24 / 48 | AVX = 1 | AVX2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, typical_p = 1.000, temp = 0.800\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 4096, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m\u001b[0m [end of text]\n",
      "\n",
      "llama_print_timings:        load time =     929.50 ms\n",
      "llama_print_timings:      sample time =      43.74 ms /   173 runs   (    0.25 ms per token,  3955.10 tokens per second)\n",
      "llama_print_timings: prompt eval time =      77.01 ms /   231 tokens (    0.33 ms per token,  2999.73 tokens per second)\n",
      "llama_print_timings:        eval time =    1675.67 ms /   172 runs   (    9.74 ms per token,   102.65 tokens per second)\n",
      "llama_print_timings:       total time =    1847.89 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "# outputs from llama with vectorstore\n",
    "model_name = \"/scratch/megathon/quant/metaquant\"\n",
    "ggml_version = \"gguf\"\n",
    "qtype = f\"{model_name}.{ggml_version}.{kargs['quant_method']}.bin\"\n",
    "print(f\"Running with quantized model {qtype}\")\n",
    "\n",
    "print(\"Generating vectorstore\")\n",
    "vstore = get_vector_store(kargs, prompt)\n",
    "\n",
    "print(\"Getting context\")\n",
    "new_prompt = get_context(vstore, prompt)\n",
    "\n",
    "!/scratch/megathon/quant/llama.cpp/main -m {qtype} -n {kargs['n']} --repeat_penalty {kargs['penalty']} --color -ngl {kargs['ngl']} -p f\"\\'{new_prompt}\\'\" > output.txt\n",
    "with open('output.txt') as f:\n",
    "    output = f.read()\n",
    "os.remove(\"output.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ":\\'stop hospice care, and start back on treatments. You can also keep up these habits to stay well: Eat right. Stick to a healthy, well-balanced diet that's low in salt, fat, and sugar; drink plenty of water; and avoid foods high in phosphorus. Exercise regularly. Doctors often recommend exercise for people with kidney disease because it can help keep your body healthy. Don't smoke. Smoking raises blood pressure and can damage blood vessels, which can make kidney disease worse. Limit alcohol to 2 drinks a day if you are an adult age 18 or older; limit is lower for children and teens. Don't take too many nonsteroidal anti-inflammatory drugs (NSAIDs), such as ibuprofen or naproxen, unless your doctor says it's okay. These meds can harm the kidneys if you take too much of them. Get enough sleep. Most adults need 7 to 8 hours of sleep each night. Follow these steps and talk with your healthcare provider about any changes in your lifestyle or treatment plan. And remember, it's important to keep following your doctor's advice so that you can stay well and prevent complications from getting worse.\n"
     ]
    }
   ],
   "source": [
    "print(output.split(\"Answer\")[-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rough Work :)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading model file /scratch/megathon/cache/models--meta-llama--Llama-2-7b-chat-hf/snapshots/94b07a6e30c3292b8265ed32ffdeccfdadf434a8/model-00001-of-00002.safetensors\n",
      "Loading model file /scratch/megathon/cache/models--meta-llama--Llama-2-7b-chat-hf/snapshots/94b07a6e30c3292b8265ed32ffdeccfdadf434a8/model-00001-of-00002.safetensors\n",
      "Loading model file /scratch/megathon/cache/models--meta-llama--Llama-2-7b-chat-hf/snapshots/94b07a6e30c3292b8265ed32ffdeccfdadf434a8/model-00002-of-00002.safetensors\n",
      "params = Params(n_vocab=32000, n_embd=4096, n_layer=32, n_ctx=4096, n_ff=11008, n_head=32, n_head_kv=32, f_norm_eps=1e-05, f_rope_freq_base=None, f_rope_scale=None, ftype=<GGMLFileType.MostlyF16: 1>, path_model=PosixPath('/scratch/megathon/cache/models--meta-llama--Llama-2-7b-chat-hf/snapshots/94b07a6e30c3292b8265ed32ffdeccfdadf434a8'))\n",
      "Loading vocab file '/scratch/megathon/cache/models--meta-llama--Llama-2-7b-chat-hf/snapshots/94b07a6e30c3292b8265ed32ffdeccfdadf434a8/tokenizer.model', type 'spm'\n",
      "Permuting layer 0\n",
      "Permuting layer 1\n",
      "Permuting layer 2\n",
      "Permuting layer 3\n",
      "Permuting layer 4\n",
      "Permuting layer 5\n",
      "Permuting layer 6\n",
      "Permuting layer 7\n",
      "Permuting layer 8\n",
      "Permuting layer 9\n",
      "Permuting layer 10\n",
      "Permuting layer 11\n",
      "Permuting layer 12\n",
      "Permuting layer 13\n",
      "Permuting layer 14\n",
      "Permuting layer 15\n",
      "Permuting layer 16\n",
      "Permuting layer 17\n",
      "Permuting layer 18\n",
      "Permuting layer 19\n",
      "Permuting layer 20\n",
      "Permuting layer 21\n",
      "Permuting layer 22\n",
      "Permuting layer 23\n",
      "Permuting layer 24\n",
      "Permuting layer 25\n",
      "Permuting layer 26\n",
      "Permuting layer 27\n",
      "Permuting layer 28\n",
      "Permuting layer 29\n",
      "Permuting layer 30\n",
      "Permuting layer 31\n",
      "model.embed_tokens.weight                        -> token_embd.weight                        | F16    | [32000, 4096]\n",
      "model.layers.0.input_layernorm.weight            -> blk.0.attn_norm.weight                   | F16    | [4096]\n",
      "model.layers.0.mlp.down_proj.weight              -> blk.0.ffn_down.weight                    | F16    | [4096, 11008]\n",
      "model.layers.0.mlp.gate_proj.weight              -> blk.0.ffn_gate.weight                    | F16    | [11008, 4096]\n",
      "model.layers.0.mlp.up_proj.weight                -> blk.0.ffn_up.weight                      | F16    | [11008, 4096]\n",
      "model.layers.0.post_attention_layernorm.weight   -> blk.0.ffn_norm.weight                    | F16    | [4096]\n",
      "model.layers.0.self_attn.k_proj.weight           -> blk.0.attn_k.weight                      | F16    | [4096, 4096]\n",
      "model.layers.0.self_attn.o_proj.weight           -> blk.0.attn_output.weight                 | F16    | [4096, 4096]\n",
      "model.layers.0.self_attn.q_proj.weight           -> blk.0.attn_q.weight                      | F16    | [4096, 4096]\n",
      "skipping tensor blk.0.attn_rot_embd\n",
      "model.layers.0.self_attn.v_proj.weight           -> blk.0.attn_v.weight                      | F16    | [4096, 4096]\n",
      "model.layers.1.input_layernorm.weight            -> blk.1.attn_norm.weight                   | F16    | [4096]\n",
      "model.layers.1.mlp.down_proj.weight              -> blk.1.ffn_down.weight                    | F16    | [4096, 11008]\n",
      "model.layers.1.mlp.gate_proj.weight              -> blk.1.ffn_gate.weight                    | F16    | [11008, 4096]\n",
      "model.layers.1.mlp.up_proj.weight                -> blk.1.ffn_up.weight                      | F16    | [11008, 4096]\n",
      "model.layers.1.post_attention_layernorm.weight   -> blk.1.ffn_norm.weight                    | F16    | [4096]\n",
      "model.layers.1.self_attn.k_proj.weight           -> blk.1.attn_k.weight                      | F16    | [4096, 4096]\n",
      "model.layers.1.self_attn.o_proj.weight           -> blk.1.attn_output.weight                 | F16    | [4096, 4096]\n",
      "model.layers.1.self_attn.q_proj.weight           -> blk.1.attn_q.weight                      | F16    | [4096, 4096]\n",
      "skipping tensor blk.1.attn_rot_embd\n",
      "model.layers.1.self_attn.v_proj.weight           -> blk.1.attn_v.weight                      | F16    | [4096, 4096]\n",
      "model.layers.10.input_layernorm.weight           -> blk.10.attn_norm.weight                  | F16    | [4096]\n",
      "model.layers.10.mlp.down_proj.weight             -> blk.10.ffn_down.weight                   | F16    | [4096, 11008]\n",
      "model.layers.10.mlp.gate_proj.weight             -> blk.10.ffn_gate.weight                   | F16    | [11008, 4096]\n",
      "model.layers.10.mlp.up_proj.weight               -> blk.10.ffn_up.weight                     | F16    | [11008, 4096]\n",
      "model.layers.10.post_attention_layernorm.weight  -> blk.10.ffn_norm.weight                   | F16    | [4096]\n",
      "model.layers.10.self_attn.k_proj.weight          -> blk.10.attn_k.weight                     | F16    | [4096, 4096]\n",
      "model.layers.10.self_attn.o_proj.weight          -> blk.10.attn_output.weight                | F16    | [4096, 4096]\n",
      "model.layers.10.self_attn.q_proj.weight          -> blk.10.attn_q.weight                     | F16    | [4096, 4096]\n",
      "skipping tensor blk.10.attn_rot_embd\n",
      "model.layers.10.self_attn.v_proj.weight          -> blk.10.attn_v.weight                     | F16    | [4096, 4096]\n",
      "model.layers.11.input_layernorm.weight           -> blk.11.attn_norm.weight                  | F16    | [4096]\n",
      "model.layers.11.mlp.down_proj.weight             -> blk.11.ffn_down.weight                   | F16    | [4096, 11008]\n",
      "model.layers.11.mlp.gate_proj.weight             -> blk.11.ffn_gate.weight                   | F16    | [11008, 4096]\n",
      "model.layers.11.mlp.up_proj.weight               -> blk.11.ffn_up.weight                     | F16    | [11008, 4096]\n",
      "model.layers.11.post_attention_layernorm.weight  -> blk.11.ffn_norm.weight                   | F16    | [4096]\n",
      "model.layers.11.self_attn.k_proj.weight          -> blk.11.attn_k.weight                     | F16    | [4096, 4096]\n",
      "model.layers.11.self_attn.o_proj.weight          -> blk.11.attn_output.weight                | F16    | [4096, 4096]\n",
      "model.layers.11.self_attn.q_proj.weight          -> blk.11.attn_q.weight                     | F16    | [4096, 4096]\n",
      "skipping tensor blk.11.attn_rot_embd\n",
      "model.layers.11.self_attn.v_proj.weight          -> blk.11.attn_v.weight                     | F16    | [4096, 4096]\n",
      "model.layers.12.input_layernorm.weight           -> blk.12.attn_norm.weight                  | F16    | [4096]\n",
      "model.layers.12.mlp.down_proj.weight             -> blk.12.ffn_down.weight                   | F16    | [4096, 11008]\n",
      "model.layers.12.mlp.gate_proj.weight             -> blk.12.ffn_gate.weight                   | F16    | [11008, 4096]\n",
      "model.layers.12.mlp.up_proj.weight               -> blk.12.ffn_up.weight                     | F16    | [11008, 4096]\n",
      "model.layers.12.post_attention_layernorm.weight  -> blk.12.ffn_norm.weight                   | F16    | [4096]\n",
      "model.layers.12.self_attn.k_proj.weight          -> blk.12.attn_k.weight                     | F16    | [4096, 4096]\n",
      "model.layers.12.self_attn.o_proj.weight          -> blk.12.attn_output.weight                | F16    | [4096, 4096]\n",
      "model.layers.12.self_attn.q_proj.weight          -> blk.12.attn_q.weight                     | F16    | [4096, 4096]\n",
      "skipping tensor blk.12.attn_rot_embd\n",
      "model.layers.12.self_attn.v_proj.weight          -> blk.12.attn_v.weight                     | F16    | [4096, 4096]\n",
      "model.layers.13.input_layernorm.weight           -> blk.13.attn_norm.weight                  | F16    | [4096]\n",
      "model.layers.13.mlp.down_proj.weight             -> blk.13.ffn_down.weight                   | F16    | [4096, 11008]\n",
      "model.layers.13.mlp.gate_proj.weight             -> blk.13.ffn_gate.weight                   | F16    | [11008, 4096]\n",
      "model.layers.13.mlp.up_proj.weight               -> blk.13.ffn_up.weight                     | F16    | [11008, 4096]\n",
      "model.layers.13.post_attention_layernorm.weight  -> blk.13.ffn_norm.weight                   | F16    | [4096]\n",
      "model.layers.13.self_attn.k_proj.weight          -> blk.13.attn_k.weight                     | F16    | [4096, 4096]\n",
      "model.layers.13.self_attn.o_proj.weight          -> blk.13.attn_output.weight                | F16    | [4096, 4096]\n",
      "model.layers.13.self_attn.q_proj.weight          -> blk.13.attn_q.weight                     | F16    | [4096, 4096]\n",
      "skipping tensor blk.13.attn_rot_embd\n",
      "model.layers.13.self_attn.v_proj.weight          -> blk.13.attn_v.weight                     | F16    | [4096, 4096]\n",
      "model.layers.14.input_layernorm.weight           -> blk.14.attn_norm.weight                  | F16    | [4096]\n",
      "model.layers.14.mlp.down_proj.weight             -> blk.14.ffn_down.weight                   | F16    | [4096, 11008]\n",
      "model.layers.14.mlp.gate_proj.weight             -> blk.14.ffn_gate.weight                   | F16    | [11008, 4096]\n",
      "model.layers.14.mlp.up_proj.weight               -> blk.14.ffn_up.weight                     | F16    | [11008, 4096]\n",
      "model.layers.14.post_attention_layernorm.weight  -> blk.14.ffn_norm.weight                   | F16    | [4096]\n",
      "model.layers.14.self_attn.k_proj.weight          -> blk.14.attn_k.weight                     | F16    | [4096, 4096]\n",
      "model.layers.14.self_attn.o_proj.weight          -> blk.14.attn_output.weight                | F16    | [4096, 4096]\n",
      "model.layers.14.self_attn.q_proj.weight          -> blk.14.attn_q.weight                     | F16    | [4096, 4096]\n",
      "skipping tensor blk.14.attn_rot_embd\n",
      "model.layers.14.self_attn.v_proj.weight          -> blk.14.attn_v.weight                     | F16    | [4096, 4096]\n",
      "model.layers.15.input_layernorm.weight           -> blk.15.attn_norm.weight                  | F16    | [4096]\n",
      "model.layers.15.mlp.down_proj.weight             -> blk.15.ffn_down.weight                   | F16    | [4096, 11008]\n",
      "model.layers.15.mlp.gate_proj.weight             -> blk.15.ffn_gate.weight                   | F16    | [11008, 4096]\n",
      "model.layers.15.mlp.up_proj.weight               -> blk.15.ffn_up.weight                     | F16    | [11008, 4096]\n",
      "model.layers.15.post_attention_layernorm.weight  -> blk.15.ffn_norm.weight                   | F16    | [4096]\n",
      "model.layers.15.self_attn.k_proj.weight          -> blk.15.attn_k.weight                     | F16    | [4096, 4096]\n",
      "model.layers.15.self_attn.o_proj.weight          -> blk.15.attn_output.weight                | F16    | [4096, 4096]\n",
      "model.layers.15.self_attn.q_proj.weight          -> blk.15.attn_q.weight                     | F16    | [4096, 4096]\n",
      "skipping tensor blk.15.attn_rot_embd\n",
      "model.layers.15.self_attn.v_proj.weight          -> blk.15.attn_v.weight                     | F16    | [4096, 4096]\n",
      "model.layers.16.input_layernorm.weight           -> blk.16.attn_norm.weight                  | F16    | [4096]\n",
      "model.layers.16.mlp.down_proj.weight             -> blk.16.ffn_down.weight                   | F16    | [4096, 11008]\n",
      "model.layers.16.mlp.gate_proj.weight             -> blk.16.ffn_gate.weight                   | F16    | [11008, 4096]\n",
      "model.layers.16.mlp.up_proj.weight               -> blk.16.ffn_up.weight                     | F16    | [11008, 4096]\n",
      "model.layers.16.post_attention_layernorm.weight  -> blk.16.ffn_norm.weight                   | F16    | [4096]\n",
      "model.layers.16.self_attn.k_proj.weight          -> blk.16.attn_k.weight                     | F16    | [4096, 4096]\n",
      "model.layers.16.self_attn.o_proj.weight          -> blk.16.attn_output.weight                | F16    | [4096, 4096]\n",
      "model.layers.16.self_attn.q_proj.weight          -> blk.16.attn_q.weight                     | F16    | [4096, 4096]\n",
      "skipping tensor blk.16.attn_rot_embd\n",
      "model.layers.16.self_attn.v_proj.weight          -> blk.16.attn_v.weight                     | F16    | [4096, 4096]\n",
      "model.layers.17.input_layernorm.weight           -> blk.17.attn_norm.weight                  | F16    | [4096]\n",
      "model.layers.17.mlp.down_proj.weight             -> blk.17.ffn_down.weight                   | F16    | [4096, 11008]\n",
      "model.layers.17.mlp.gate_proj.weight             -> blk.17.ffn_gate.weight                   | F16    | [11008, 4096]\n",
      "model.layers.17.mlp.up_proj.weight               -> blk.17.ffn_up.weight                     | F16    | [11008, 4096]\n",
      "model.layers.17.post_attention_layernorm.weight  -> blk.17.ffn_norm.weight                   | F16    | [4096]\n",
      "model.layers.17.self_attn.k_proj.weight          -> blk.17.attn_k.weight                     | F16    | [4096, 4096]\n",
      "model.layers.17.self_attn.o_proj.weight          -> blk.17.attn_output.weight                | F16    | [4096, 4096]\n",
      "model.layers.17.self_attn.q_proj.weight          -> blk.17.attn_q.weight                     | F16    | [4096, 4096]\n",
      "skipping tensor blk.17.attn_rot_embd\n",
      "model.layers.17.self_attn.v_proj.weight          -> blk.17.attn_v.weight                     | F16    | [4096, 4096]\n",
      "model.layers.18.input_layernorm.weight           -> blk.18.attn_norm.weight                  | F16    | [4096]\n",
      "model.layers.18.mlp.down_proj.weight             -> blk.18.ffn_down.weight                   | F16    | [4096, 11008]\n",
      "model.layers.18.mlp.gate_proj.weight             -> blk.18.ffn_gate.weight                   | F16    | [11008, 4096]\n",
      "model.layers.18.mlp.up_proj.weight               -> blk.18.ffn_up.weight                     | F16    | [11008, 4096]\n",
      "model.layers.18.post_attention_layernorm.weight  -> blk.18.ffn_norm.weight                   | F16    | [4096]\n",
      "model.layers.18.self_attn.k_proj.weight          -> blk.18.attn_k.weight                     | F16    | [4096, 4096]\n",
      "model.layers.18.self_attn.o_proj.weight          -> blk.18.attn_output.weight                | F16    | [4096, 4096]\n",
      "model.layers.18.self_attn.q_proj.weight          -> blk.18.attn_q.weight                     | F16    | [4096, 4096]\n",
      "skipping tensor blk.18.attn_rot_embd\n",
      "model.layers.18.self_attn.v_proj.weight          -> blk.18.attn_v.weight                     | F16    | [4096, 4096]\n",
      "model.layers.19.input_layernorm.weight           -> blk.19.attn_norm.weight                  | F16    | [4096]\n",
      "model.layers.19.mlp.down_proj.weight             -> blk.19.ffn_down.weight                   | F16    | [4096, 11008]\n",
      "model.layers.19.mlp.gate_proj.weight             -> blk.19.ffn_gate.weight                   | F16    | [11008, 4096]\n",
      "model.layers.19.mlp.up_proj.weight               -> blk.19.ffn_up.weight                     | F16    | [11008, 4096]\n",
      "model.layers.19.post_attention_layernorm.weight  -> blk.19.ffn_norm.weight                   | F16    | [4096]\n",
      "model.layers.19.self_attn.k_proj.weight          -> blk.19.attn_k.weight                     | F16    | [4096, 4096]\n",
      "model.layers.19.self_attn.o_proj.weight          -> blk.19.attn_output.weight                | F16    | [4096, 4096]\n",
      "model.layers.19.self_attn.q_proj.weight          -> blk.19.attn_q.weight                     | F16    | [4096, 4096]\n",
      "skipping tensor blk.19.attn_rot_embd\n",
      "model.layers.19.self_attn.v_proj.weight          -> blk.19.attn_v.weight                     | F16    | [4096, 4096]\n",
      "model.layers.2.input_layernorm.weight            -> blk.2.attn_norm.weight                   | F16    | [4096]\n",
      "model.layers.2.mlp.down_proj.weight              -> blk.2.ffn_down.weight                    | F16    | [4096, 11008]\n",
      "model.layers.2.mlp.gate_proj.weight              -> blk.2.ffn_gate.weight                    | F16    | [11008, 4096]\n",
      "model.layers.2.mlp.up_proj.weight                -> blk.2.ffn_up.weight                      | F16    | [11008, 4096]\n",
      "model.layers.2.post_attention_layernorm.weight   -> blk.2.ffn_norm.weight                    | F16    | [4096]\n",
      "model.layers.2.self_attn.k_proj.weight           -> blk.2.attn_k.weight                      | F16    | [4096, 4096]\n",
      "model.layers.2.self_attn.o_proj.weight           -> blk.2.attn_output.weight                 | F16    | [4096, 4096]\n",
      "model.layers.2.self_attn.q_proj.weight           -> blk.2.attn_q.weight                      | F16    | [4096, 4096]\n",
      "skipping tensor blk.2.attn_rot_embd\n",
      "model.layers.2.self_attn.v_proj.weight           -> blk.2.attn_v.weight                      | F16    | [4096, 4096]\n",
      "model.layers.20.input_layernorm.weight           -> blk.20.attn_norm.weight                  | F16    | [4096]\n",
      "model.layers.20.mlp.down_proj.weight             -> blk.20.ffn_down.weight                   | F16    | [4096, 11008]\n",
      "model.layers.20.mlp.gate_proj.weight             -> blk.20.ffn_gate.weight                   | F16    | [11008, 4096]\n",
      "model.layers.20.mlp.up_proj.weight               -> blk.20.ffn_up.weight                     | F16    | [11008, 4096]\n",
      "model.layers.20.post_attention_layernorm.weight  -> blk.20.ffn_norm.weight                   | F16    | [4096]\n",
      "model.layers.20.self_attn.k_proj.weight          -> blk.20.attn_k.weight                     | F16    | [4096, 4096]\n",
      "model.layers.20.self_attn.o_proj.weight          -> blk.20.attn_output.weight                | F16    | [4096, 4096]\n",
      "model.layers.20.self_attn.q_proj.weight          -> blk.20.attn_q.weight                     | F16    | [4096, 4096]\n",
      "skipping tensor blk.20.attn_rot_embd\n",
      "model.layers.20.self_attn.v_proj.weight          -> blk.20.attn_v.weight                     | F16    | [4096, 4096]\n",
      "model.layers.21.input_layernorm.weight           -> blk.21.attn_norm.weight                  | F16    | [4096]\n",
      "model.layers.21.mlp.down_proj.weight             -> blk.21.ffn_down.weight                   | F16    | [4096, 11008]\n",
      "model.layers.21.mlp.gate_proj.weight             -> blk.21.ffn_gate.weight                   | F16    | [11008, 4096]\n",
      "model.layers.21.mlp.up_proj.weight               -> blk.21.ffn_up.weight                     | F16    | [11008, 4096]\n",
      "model.layers.21.post_attention_layernorm.weight  -> blk.21.ffn_norm.weight                   | F16    | [4096]\n",
      "model.layers.21.self_attn.k_proj.weight          -> blk.21.attn_k.weight                     | F16    | [4096, 4096]\n",
      "model.layers.21.self_attn.o_proj.weight          -> blk.21.attn_output.weight                | F16    | [4096, 4096]\n",
      "model.layers.21.self_attn.q_proj.weight          -> blk.21.attn_q.weight                     | F16    | [4096, 4096]\n",
      "skipping tensor blk.21.attn_rot_embd\n",
      "model.layers.21.self_attn.v_proj.weight          -> blk.21.attn_v.weight                     | F16    | [4096, 4096]\n",
      "model.layers.22.input_layernorm.weight           -> blk.22.attn_norm.weight                  | F16    | [4096]\n",
      "model.layers.22.mlp.down_proj.weight             -> blk.22.ffn_down.weight                   | F16    | [4096, 11008]\n",
      "model.layers.22.mlp.gate_proj.weight             -> blk.22.ffn_gate.weight                   | F16    | [11008, 4096]\n",
      "model.layers.22.mlp.up_proj.weight               -> blk.22.ffn_up.weight                     | F16    | [11008, 4096]\n",
      "model.layers.22.post_attention_layernorm.weight  -> blk.22.ffn_norm.weight                   | F16    | [4096]\n",
      "model.layers.22.self_attn.k_proj.weight          -> blk.22.attn_k.weight                     | F16    | [4096, 4096]\n",
      "model.layers.22.self_attn.o_proj.weight          -> blk.22.attn_output.weight                | F16    | [4096, 4096]\n",
      "model.layers.22.self_attn.q_proj.weight          -> blk.22.attn_q.weight                     | F16    | [4096, 4096]\n",
      "skipping tensor blk.22.attn_rot_embd\n",
      "model.layers.22.self_attn.v_proj.weight          -> blk.22.attn_v.weight                     | F16    | [4096, 4096]\n",
      "model.layers.23.input_layernorm.weight           -> blk.23.attn_norm.weight                  | F16    | [4096]\n",
      "model.layers.23.mlp.down_proj.weight             -> blk.23.ffn_down.weight                   | F16    | [4096, 11008]\n",
      "model.layers.23.mlp.gate_proj.weight             -> blk.23.ffn_gate.weight                   | F16    | [11008, 4096]\n",
      "model.layers.23.mlp.up_proj.weight               -> blk.23.ffn_up.weight                     | F16    | [11008, 4096]\n",
      "model.layers.23.post_attention_layernorm.weight  -> blk.23.ffn_norm.weight                   | F16    | [4096]\n",
      "model.layers.23.self_attn.k_proj.weight          -> blk.23.attn_k.weight                     | F16    | [4096, 4096]\n",
      "model.layers.23.self_attn.o_proj.weight          -> blk.23.attn_output.weight                | F16    | [4096, 4096]\n",
      "model.layers.23.self_attn.q_proj.weight          -> blk.23.attn_q.weight                     | F16    | [4096, 4096]\n",
      "skipping tensor blk.23.attn_rot_embd\n",
      "model.layers.23.self_attn.v_proj.weight          -> blk.23.attn_v.weight                     | F16    | [4096, 4096]\n",
      "model.layers.3.input_layernorm.weight            -> blk.3.attn_norm.weight                   | F16    | [4096]\n",
      "model.layers.3.mlp.down_proj.weight              -> blk.3.ffn_down.weight                    | F16    | [4096, 11008]\n",
      "model.layers.3.mlp.gate_proj.weight              -> blk.3.ffn_gate.weight                    | F16    | [11008, 4096]\n",
      "model.layers.3.mlp.up_proj.weight                -> blk.3.ffn_up.weight                      | F16    | [11008, 4096]\n",
      "model.layers.3.post_attention_layernorm.weight   -> blk.3.ffn_norm.weight                    | F16    | [4096]\n",
      "model.layers.3.self_attn.k_proj.weight           -> blk.3.attn_k.weight                      | F16    | [4096, 4096]\n",
      "model.layers.3.self_attn.o_proj.weight           -> blk.3.attn_output.weight                 | F16    | [4096, 4096]\n",
      "model.layers.3.self_attn.q_proj.weight           -> blk.3.attn_q.weight                      | F16    | [4096, 4096]\n",
      "skipping tensor blk.3.attn_rot_embd\n",
      "model.layers.3.self_attn.v_proj.weight           -> blk.3.attn_v.weight                      | F16    | [4096, 4096]\n",
      "model.layers.4.input_layernorm.weight            -> blk.4.attn_norm.weight                   | F16    | [4096]\n",
      "model.layers.4.mlp.down_proj.weight              -> blk.4.ffn_down.weight                    | F16    | [4096, 11008]\n",
      "model.layers.4.mlp.gate_proj.weight              -> blk.4.ffn_gate.weight                    | F16    | [11008, 4096]\n",
      "model.layers.4.mlp.up_proj.weight                -> blk.4.ffn_up.weight                      | F16    | [11008, 4096]\n",
      "model.layers.4.post_attention_layernorm.weight   -> blk.4.ffn_norm.weight                    | F16    | [4096]\n",
      "model.layers.4.self_attn.k_proj.weight           -> blk.4.attn_k.weight                      | F16    | [4096, 4096]\n",
      "model.layers.4.self_attn.o_proj.weight           -> blk.4.attn_output.weight                 | F16    | [4096, 4096]\n",
      "model.layers.4.self_attn.q_proj.weight           -> blk.4.attn_q.weight                      | F16    | [4096, 4096]\n",
      "skipping tensor blk.4.attn_rot_embd\n",
      "model.layers.4.self_attn.v_proj.weight           -> blk.4.attn_v.weight                      | F16    | [4096, 4096]\n",
      "model.layers.5.input_layernorm.weight            -> blk.5.attn_norm.weight                   | F16    | [4096]\n",
      "model.layers.5.mlp.down_proj.weight              -> blk.5.ffn_down.weight                    | F16    | [4096, 11008]\n",
      "model.layers.5.mlp.gate_proj.weight              -> blk.5.ffn_gate.weight                    | F16    | [11008, 4096]\n",
      "model.layers.5.mlp.up_proj.weight                -> blk.5.ffn_up.weight                      | F16    | [11008, 4096]\n",
      "model.layers.5.post_attention_layernorm.weight   -> blk.5.ffn_norm.weight                    | F16    | [4096]\n",
      "model.layers.5.self_attn.k_proj.weight           -> blk.5.attn_k.weight                      | F16    | [4096, 4096]\n",
      "model.layers.5.self_attn.o_proj.weight           -> blk.5.attn_output.weight                 | F16    | [4096, 4096]\n",
      "model.layers.5.self_attn.q_proj.weight           -> blk.5.attn_q.weight                      | F16    | [4096, 4096]\n",
      "skipping tensor blk.5.attn_rot_embd\n",
      "model.layers.5.self_attn.v_proj.weight           -> blk.5.attn_v.weight                      | F16    | [4096, 4096]\n",
      "model.layers.6.input_layernorm.weight            -> blk.6.attn_norm.weight                   | F16    | [4096]\n",
      "model.layers.6.mlp.down_proj.weight              -> blk.6.ffn_down.weight                    | F16    | [4096, 11008]\n",
      "model.layers.6.mlp.gate_proj.weight              -> blk.6.ffn_gate.weight                    | F16    | [11008, 4096]\n",
      "model.layers.6.mlp.up_proj.weight                -> blk.6.ffn_up.weight                      | F16    | [11008, 4096]\n",
      "model.layers.6.post_attention_layernorm.weight   -> blk.6.ffn_norm.weight                    | F16    | [4096]\n",
      "model.layers.6.self_attn.k_proj.weight           -> blk.6.attn_k.weight                      | F16    | [4096, 4096]\n",
      "model.layers.6.self_attn.o_proj.weight           -> blk.6.attn_output.weight                 | F16    | [4096, 4096]\n",
      "model.layers.6.self_attn.q_proj.weight           -> blk.6.attn_q.weight                      | F16    | [4096, 4096]\n",
      "skipping tensor blk.6.attn_rot_embd\n",
      "model.layers.6.self_attn.v_proj.weight           -> blk.6.attn_v.weight                      | F16    | [4096, 4096]\n",
      "model.layers.7.input_layernorm.weight            -> blk.7.attn_norm.weight                   | F16    | [4096]\n",
      "model.layers.7.mlp.down_proj.weight              -> blk.7.ffn_down.weight                    | F16    | [4096, 11008]\n",
      "model.layers.7.mlp.gate_proj.weight              -> blk.7.ffn_gate.weight                    | F16    | [11008, 4096]\n",
      "model.layers.7.mlp.up_proj.weight                -> blk.7.ffn_up.weight                      | F16    | [11008, 4096]\n",
      "model.layers.7.post_attention_layernorm.weight   -> blk.7.ffn_norm.weight                    | F16    | [4096]\n",
      "model.layers.7.self_attn.k_proj.weight           -> blk.7.attn_k.weight                      | F16    | [4096, 4096]\n",
      "model.layers.7.self_attn.o_proj.weight           -> blk.7.attn_output.weight                 | F16    | [4096, 4096]\n",
      "model.layers.7.self_attn.q_proj.weight           -> blk.7.attn_q.weight                      | F16    | [4096, 4096]\n",
      "skipping tensor blk.7.attn_rot_embd\n",
      "model.layers.7.self_attn.v_proj.weight           -> blk.7.attn_v.weight                      | F16    | [4096, 4096]\n",
      "model.layers.8.input_layernorm.weight            -> blk.8.attn_norm.weight                   | F16    | [4096]\n",
      "model.layers.8.mlp.down_proj.weight              -> blk.8.ffn_down.weight                    | F16    | [4096, 11008]\n",
      "model.layers.8.mlp.gate_proj.weight              -> blk.8.ffn_gate.weight                    | F16    | [11008, 4096]\n",
      "model.layers.8.mlp.up_proj.weight                -> blk.8.ffn_up.weight                      | F16    | [11008, 4096]\n",
      "model.layers.8.post_attention_layernorm.weight   -> blk.8.ffn_norm.weight                    | F16    | [4096]\n",
      "model.layers.8.self_attn.k_proj.weight           -> blk.8.attn_k.weight                      | F16    | [4096, 4096]\n",
      "model.layers.8.self_attn.o_proj.weight           -> blk.8.attn_output.weight                 | F16    | [4096, 4096]\n",
      "model.layers.8.self_attn.q_proj.weight           -> blk.8.attn_q.weight                      | F16    | [4096, 4096]\n",
      "skipping tensor blk.8.attn_rot_embd\n",
      "model.layers.8.self_attn.v_proj.weight           -> blk.8.attn_v.weight                      | F16    | [4096, 4096]\n",
      "model.layers.9.input_layernorm.weight            -> blk.9.attn_norm.weight                   | F16    | [4096]\n",
      "model.layers.9.mlp.down_proj.weight              -> blk.9.ffn_down.weight                    | F16    | [4096, 11008]\n",
      "model.layers.9.mlp.gate_proj.weight              -> blk.9.ffn_gate.weight                    | F16    | [11008, 4096]\n",
      "model.layers.9.mlp.up_proj.weight                -> blk.9.ffn_up.weight                      | F16    | [11008, 4096]\n",
      "model.layers.9.post_attention_layernorm.weight   -> blk.9.ffn_norm.weight                    | F16    | [4096]\n",
      "model.layers.9.self_attn.k_proj.weight           -> blk.9.attn_k.weight                      | F16    | [4096, 4096]\n",
      "model.layers.9.self_attn.o_proj.weight           -> blk.9.attn_output.weight                 | F16    | [4096, 4096]\n",
      "model.layers.9.self_attn.q_proj.weight           -> blk.9.attn_q.weight                      | F16    | [4096, 4096]\n",
      "skipping tensor blk.9.attn_rot_embd\n",
      "model.layers.9.self_attn.v_proj.weight           -> blk.9.attn_v.weight                      | F16    | [4096, 4096]\n",
      "lm_head.weight                                   -> output.weight                            | F16    | [32000, 4096]\n",
      "model.layers.24.input_layernorm.weight           -> blk.24.attn_norm.weight                  | F16    | [4096]\n",
      "model.layers.24.mlp.down_proj.weight             -> blk.24.ffn_down.weight                   | F16    | [4096, 11008]\n",
      "model.layers.24.mlp.gate_proj.weight             -> blk.24.ffn_gate.weight                   | F16    | [11008, 4096]\n",
      "model.layers.24.mlp.up_proj.weight               -> blk.24.ffn_up.weight                     | F16    | [11008, 4096]\n",
      "model.layers.24.post_attention_layernorm.weight  -> blk.24.ffn_norm.weight                   | F16    | [4096]\n",
      "model.layers.24.self_attn.k_proj.weight          -> blk.24.attn_k.weight                     | F16    | [4096, 4096]\n",
      "model.layers.24.self_attn.o_proj.weight          -> blk.24.attn_output.weight                | F16    | [4096, 4096]\n",
      "model.layers.24.self_attn.q_proj.weight          -> blk.24.attn_q.weight                     | F16    | [4096, 4096]\n",
      "skipping tensor blk.24.attn_rot_embd\n",
      "model.layers.24.self_attn.v_proj.weight          -> blk.24.attn_v.weight                     | F16    | [4096, 4096]\n",
      "model.layers.25.input_layernorm.weight           -> blk.25.attn_norm.weight                  | F16    | [4096]\n",
      "model.layers.25.mlp.down_proj.weight             -> blk.25.ffn_down.weight                   | F16    | [4096, 11008]\n",
      "model.layers.25.mlp.gate_proj.weight             -> blk.25.ffn_gate.weight                   | F16    | [11008, 4096]\n",
      "model.layers.25.mlp.up_proj.weight               -> blk.25.ffn_up.weight                     | F16    | [11008, 4096]\n",
      "model.layers.25.post_attention_layernorm.weight  -> blk.25.ffn_norm.weight                   | F16    | [4096]\n",
      "model.layers.25.self_attn.k_proj.weight          -> blk.25.attn_k.weight                     | F16    | [4096, 4096]\n",
      "model.layers.25.self_attn.o_proj.weight          -> blk.25.attn_output.weight                | F16    | [4096, 4096]\n",
      "model.layers.25.self_attn.q_proj.weight          -> blk.25.attn_q.weight                     | F16    | [4096, 4096]\n",
      "skipping tensor blk.25.attn_rot_embd\n",
      "model.layers.25.self_attn.v_proj.weight          -> blk.25.attn_v.weight                     | F16    | [4096, 4096]\n",
      "model.layers.26.input_layernorm.weight           -> blk.26.attn_norm.weight                  | F16    | [4096]\n",
      "model.layers.26.mlp.down_proj.weight             -> blk.26.ffn_down.weight                   | F16    | [4096, 11008]\n",
      "model.layers.26.mlp.gate_proj.weight             -> blk.26.ffn_gate.weight                   | F16    | [11008, 4096]\n",
      "model.layers.26.mlp.up_proj.weight               -> blk.26.ffn_up.weight                     | F16    | [11008, 4096]\n",
      "model.layers.26.post_attention_layernorm.weight  -> blk.26.ffn_norm.weight                   | F16    | [4096]\n",
      "model.layers.26.self_attn.k_proj.weight          -> blk.26.attn_k.weight                     | F16    | [4096, 4096]\n",
      "model.layers.26.self_attn.o_proj.weight          -> blk.26.attn_output.weight                | F16    | [4096, 4096]\n",
      "model.layers.26.self_attn.q_proj.weight          -> blk.26.attn_q.weight                     | F16    | [4096, 4096]\n",
      "skipping tensor blk.26.attn_rot_embd\n",
      "model.layers.26.self_attn.v_proj.weight          -> blk.26.attn_v.weight                     | F16    | [4096, 4096]\n",
      "model.layers.27.input_layernorm.weight           -> blk.27.attn_norm.weight                  | F16    | [4096]\n",
      "model.layers.27.mlp.down_proj.weight             -> blk.27.ffn_down.weight                   | F16    | [4096, 11008]\n",
      "model.layers.27.mlp.gate_proj.weight             -> blk.27.ffn_gate.weight                   | F16    | [11008, 4096]\n",
      "model.layers.27.mlp.up_proj.weight               -> blk.27.ffn_up.weight                     | F16    | [11008, 4096]\n",
      "model.layers.27.post_attention_layernorm.weight  -> blk.27.ffn_norm.weight                   | F16    | [4096]\n",
      "model.layers.27.self_attn.k_proj.weight          -> blk.27.attn_k.weight                     | F16    | [4096, 4096]\n",
      "model.layers.27.self_attn.o_proj.weight          -> blk.27.attn_output.weight                | F16    | [4096, 4096]\n",
      "model.layers.27.self_attn.q_proj.weight          -> blk.27.attn_q.weight                     | F16    | [4096, 4096]\n",
      "skipping tensor blk.27.attn_rot_embd\n",
      "model.layers.27.self_attn.v_proj.weight          -> blk.27.attn_v.weight                     | F16    | [4096, 4096]\n",
      "model.layers.28.input_layernorm.weight           -> blk.28.attn_norm.weight                  | F16    | [4096]\n",
      "model.layers.28.mlp.down_proj.weight             -> blk.28.ffn_down.weight                   | F16    | [4096, 11008]\n",
      "model.layers.28.mlp.gate_proj.weight             -> blk.28.ffn_gate.weight                   | F16    | [11008, 4096]\n",
      "model.layers.28.mlp.up_proj.weight               -> blk.28.ffn_up.weight                     | F16    | [11008, 4096]\n",
      "model.layers.28.post_attention_layernorm.weight  -> blk.28.ffn_norm.weight                   | F16    | [4096]\n",
      "model.layers.28.self_attn.k_proj.weight          -> blk.28.attn_k.weight                     | F16    | [4096, 4096]\n",
      "model.layers.28.self_attn.o_proj.weight          -> blk.28.attn_output.weight                | F16    | [4096, 4096]\n",
      "model.layers.28.self_attn.q_proj.weight          -> blk.28.attn_q.weight                     | F16    | [4096, 4096]\n",
      "skipping tensor blk.28.attn_rot_embd\n",
      "model.layers.28.self_attn.v_proj.weight          -> blk.28.attn_v.weight                     | F16    | [4096, 4096]\n",
      "model.layers.29.input_layernorm.weight           -> blk.29.attn_norm.weight                  | F16    | [4096]\n",
      "model.layers.29.mlp.down_proj.weight             -> blk.29.ffn_down.weight                   | F16    | [4096, 11008]\n",
      "model.layers.29.mlp.gate_proj.weight             -> blk.29.ffn_gate.weight                   | F16    | [11008, 4096]\n",
      "model.layers.29.mlp.up_proj.weight               -> blk.29.ffn_up.weight                     | F16    | [11008, 4096]\n",
      "model.layers.29.post_attention_layernorm.weight  -> blk.29.ffn_norm.weight                   | F16    | [4096]\n",
      "model.layers.29.self_attn.k_proj.weight          -> blk.29.attn_k.weight                     | F16    | [4096, 4096]\n",
      "model.layers.29.self_attn.o_proj.weight          -> blk.29.attn_output.weight                | F16    | [4096, 4096]\n",
      "model.layers.29.self_attn.q_proj.weight          -> blk.29.attn_q.weight                     | F16    | [4096, 4096]\n",
      "skipping tensor blk.29.attn_rot_embd\n",
      "model.layers.29.self_attn.v_proj.weight          -> blk.29.attn_v.weight                     | F16    | [4096, 4096]\n",
      "model.layers.30.input_layernorm.weight           -> blk.30.attn_norm.weight                  | F16    | [4096]\n",
      "model.layers.30.mlp.down_proj.weight             -> blk.30.ffn_down.weight                   | F16    | [4096, 11008]\n",
      "model.layers.30.mlp.gate_proj.weight             -> blk.30.ffn_gate.weight                   | F16    | [11008, 4096]\n",
      "model.layers.30.mlp.up_proj.weight               -> blk.30.ffn_up.weight                     | F16    | [11008, 4096]\n",
      "model.layers.30.post_attention_layernorm.weight  -> blk.30.ffn_norm.weight                   | F16    | [4096]\n",
      "model.layers.30.self_attn.k_proj.weight          -> blk.30.attn_k.weight                     | F16    | [4096, 4096]\n",
      "model.layers.30.self_attn.o_proj.weight          -> blk.30.attn_output.weight                | F16    | [4096, 4096]\n",
      "model.layers.30.self_attn.q_proj.weight          -> blk.30.attn_q.weight                     | F16    | [4096, 4096]\n",
      "skipping tensor blk.30.attn_rot_embd\n",
      "model.layers.30.self_attn.v_proj.weight          -> blk.30.attn_v.weight                     | F16    | [4096, 4096]\n",
      "model.layers.31.input_layernorm.weight           -> blk.31.attn_norm.weight                  | F16    | [4096]\n",
      "model.layers.31.mlp.down_proj.weight             -> blk.31.ffn_down.weight                   | F16    | [4096, 11008]\n",
      "model.layers.31.mlp.gate_proj.weight             -> blk.31.ffn_gate.weight                   | F16    | [11008, 4096]\n",
      "model.layers.31.mlp.up_proj.weight               -> blk.31.ffn_up.weight                     | F16    | [11008, 4096]\n",
      "model.layers.31.post_attention_layernorm.weight  -> blk.31.ffn_norm.weight                   | F16    | [4096]\n",
      "model.layers.31.self_attn.k_proj.weight          -> blk.31.attn_k.weight                     | F16    | [4096, 4096]\n",
      "model.layers.31.self_attn.o_proj.weight          -> blk.31.attn_output.weight                | F16    | [4096, 4096]\n",
      "model.layers.31.self_attn.q_proj.weight          -> blk.31.attn_q.weight                     | F16    | [4096, 4096]\n",
      "skipping tensor blk.31.attn_rot_embd\n",
      "model.layers.31.self_attn.v_proj.weight          -> blk.31.attn_v.weight                     | F16    | [4096, 4096]\n",
      "model.norm.weight                                -> output_norm.weight                       | F16    | [4096]\n",
      "Writing /scratch/megathon/quant/metaquant.gguf.fp16.bin, format 1\n",
      "This gguf file is for Little Endian only\n",
      "gguf: Setting special token type bos to 1\n",
      "gguf: Setting special token type eos to 2\n",
      "gguf: Setting special token type unk to 0\n",
      "[  1/291] Writing tensor token_embd.weight                      | size  32000 x   4096  | type F16  | T+   3\n",
      "[  2/291] Writing tensor blk.0.attn_norm.weight                 | size   4096           | type F32  | T+   3\n",
      "[  3/291] Writing tensor blk.0.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   3\n",
      "[  4/291] Writing tensor blk.0.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   3\n",
      "[  5/291] Writing tensor blk.0.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   3\n",
      "[  6/291] Writing tensor blk.0.ffn_norm.weight                  | size   4096           | type F32  | T+   3\n",
      "[  7/291] Writing tensor blk.0.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   3\n",
      "[  8/291] Writing tensor blk.0.attn_output.weight               | size   4096 x   4096  | type F16  | T+   3\n",
      "[  9/291] Writing tensor blk.0.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   3\n",
      "[ 10/291] Writing tensor blk.0.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   4\n",
      "[ 11/291] Writing tensor blk.1.attn_norm.weight                 | size   4096           | type F32  | T+   4\n",
      "[ 12/291] Writing tensor blk.1.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   8\n",
      "[ 13/291] Writing tensor blk.1.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   9\n",
      "[ 14/291] Writing tensor blk.1.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+  11\n",
      "[ 15/291] Writing tensor blk.1.ffn_norm.weight                  | size   4096           | type F32  | T+  11\n",
      "[ 16/291] Writing tensor blk.1.attn_k.weight                    | size   4096 x   4096  | type F16  | T+  11\n",
      "[ 17/291] Writing tensor blk.1.attn_output.weight               | size   4096 x   4096  | type F16  | T+  11\n",
      "[ 18/291] Writing tensor blk.1.attn_q.weight                    | size   4096 x   4096  | type F16  | T+  11\n",
      "[ 19/291] Writing tensor blk.1.attn_v.weight                    | size   4096 x   4096  | type F16  | T+  11\n",
      "[ 20/291] Writing tensor blk.10.attn_norm.weight                | size   4096           | type F32  | T+  11\n",
      "[ 21/291] Writing tensor blk.10.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  12\n",
      "[ 22/291] Writing tensor blk.10.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  14\n",
      "[ 23/291] Writing tensor blk.10.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  15\n",
      "[ 24/291] Writing tensor blk.10.ffn_norm.weight                 | size   4096           | type F32  | T+  15\n",
      "[ 25/291] Writing tensor blk.10.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  15\n",
      "[ 26/291] Writing tensor blk.10.attn_output.weight              | size   4096 x   4096  | type F16  | T+  15\n",
      "[ 27/291] Writing tensor blk.10.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  15\n",
      "[ 28/291] Writing tensor blk.10.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  15\n",
      "[ 29/291] Writing tensor blk.11.attn_norm.weight                | size   4096           | type F32  | T+  15\n",
      "[ 30/291] Writing tensor blk.11.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  17\n",
      "[ 31/291] Writing tensor blk.11.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  17\n",
      "[ 32/291] Writing tensor blk.11.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  17\n",
      "[ 33/291] Writing tensor blk.11.ffn_norm.weight                 | size   4096           | type F32  | T+  17\n",
      "[ 34/291] Writing tensor blk.11.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  17\n",
      "[ 35/291] Writing tensor blk.11.attn_output.weight              | size   4096 x   4096  | type F16  | T+  17\n",
      "[ 36/291] Writing tensor blk.11.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  17\n",
      "[ 37/291] Writing tensor blk.11.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  17\n",
      "[ 38/291] Writing tensor blk.12.attn_norm.weight                | size   4096           | type F32  | T+  17\n",
      "[ 39/291] Writing tensor blk.12.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  19\n",
      "[ 40/291] Writing tensor blk.12.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  19\n",
      "[ 41/291] Writing tensor blk.12.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  19\n",
      "[ 42/291] Writing tensor blk.12.ffn_norm.weight                 | size   4096           | type F32  | T+  19\n",
      "[ 43/291] Writing tensor blk.12.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  19\n",
      "[ 44/291] Writing tensor blk.12.attn_output.weight              | size   4096 x   4096  | type F16  | T+  19\n",
      "[ 45/291] Writing tensor blk.12.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  19\n",
      "[ 46/291] Writing tensor blk.12.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  19\n",
      "[ 47/291] Writing tensor blk.13.attn_norm.weight                | size   4096           | type F32  | T+  19\n",
      "[ 48/291] Writing tensor blk.13.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  21\n",
      "[ 49/291] Writing tensor blk.13.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  21\n",
      "[ 50/291] Writing tensor blk.13.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  21\n",
      "[ 51/291] Writing tensor blk.13.ffn_norm.weight                 | size   4096           | type F32  | T+  22\n",
      "[ 52/291] Writing tensor blk.13.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  22\n",
      "[ 53/291] Writing tensor blk.13.attn_output.weight              | size   4096 x   4096  | type F16  | T+  22\n",
      "[ 54/291] Writing tensor blk.13.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  23\n",
      "[ 55/291] Writing tensor blk.13.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  23\n",
      "[ 56/291] Writing tensor blk.14.attn_norm.weight                | size   4096           | type F32  | T+  23\n",
      "[ 57/291] Writing tensor blk.14.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  31\n",
      "[ 58/291] Writing tensor blk.14.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  31\n",
      "[ 59/291] Writing tensor blk.14.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  34\n",
      "[ 60/291] Writing tensor blk.14.ffn_norm.weight                 | size   4096           | type F32  | T+  34\n",
      "[ 61/291] Writing tensor blk.14.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  34\n",
      "[ 62/291] Writing tensor blk.14.attn_output.weight              | size   4096 x   4096  | type F16  | T+  34\n",
      "[ 63/291] Writing tensor blk.14.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  34\n",
      "[ 64/291] Writing tensor blk.14.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  34\n",
      "[ 65/291] Writing tensor blk.15.attn_norm.weight                | size   4096           | type F32  | T+  34\n",
      "[ 66/291] Writing tensor blk.15.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  34\n",
      "[ 67/291] Writing tensor blk.15.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  36\n",
      "[ 68/291] Writing tensor blk.15.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  36\n",
      "[ 69/291] Writing tensor blk.15.ffn_norm.weight                 | size   4096           | type F32  | T+  36\n",
      "[ 70/291] Writing tensor blk.15.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  36\n",
      "[ 71/291] Writing tensor blk.15.attn_output.weight              | size   4096 x   4096  | type F16  | T+  36\n",
      "[ 72/291] Writing tensor blk.15.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  36\n",
      "[ 73/291] Writing tensor blk.15.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  36\n",
      "[ 74/291] Writing tensor blk.16.attn_norm.weight                | size   4096           | type F32  | T+  36\n",
      "[ 75/291] Writing tensor blk.16.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  38\n",
      "[ 76/291] Writing tensor blk.16.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  38\n",
      "[ 77/291] Writing tensor blk.16.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  39\n",
      "[ 78/291] Writing tensor blk.16.ffn_norm.weight                 | size   4096           | type F32  | T+  39\n",
      "[ 79/291] Writing tensor blk.16.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  39\n",
      "[ 80/291] Writing tensor blk.16.attn_output.weight              | size   4096 x   4096  | type F16  | T+  39\n",
      "[ 81/291] Writing tensor blk.16.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  39\n",
      "[ 82/291] Writing tensor blk.16.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  39\n",
      "[ 83/291] Writing tensor blk.17.attn_norm.weight                | size   4096           | type F32  | T+  39\n",
      "[ 84/291] Writing tensor blk.17.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  40\n",
      "[ 85/291] Writing tensor blk.17.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  41\n",
      "[ 86/291] Writing tensor blk.17.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  41\n",
      "[ 87/291] Writing tensor blk.17.ffn_norm.weight                 | size   4096           | type F32  | T+  41\n",
      "[ 88/291] Writing tensor blk.17.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  41\n",
      "[ 89/291] Writing tensor blk.17.attn_output.weight              | size   4096 x   4096  | type F16  | T+  41\n",
      "[ 90/291] Writing tensor blk.17.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  41\n",
      "[ 91/291] Writing tensor blk.17.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  41\n",
      "[ 92/291] Writing tensor blk.18.attn_norm.weight                | size   4096           | type F32  | T+  41\n",
      "[ 93/291] Writing tensor blk.18.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  47\n",
      "[ 94/291] Writing tensor blk.18.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  48\n",
      "[ 95/291] Writing tensor blk.18.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  48\n",
      "[ 96/291] Writing tensor blk.18.ffn_norm.weight                 | size   4096           | type F32  | T+  50\n",
      "[ 97/291] Writing tensor blk.18.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  50\n",
      "[ 98/291] Writing tensor blk.18.attn_output.weight              | size   4096 x   4096  | type F16  | T+  50\n",
      "[ 99/291] Writing tensor blk.18.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  51\n",
      "[100/291] Writing tensor blk.18.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  51\n",
      "[101/291] Writing tensor blk.19.attn_norm.weight                | size   4096           | type F32  | T+  51\n",
      "[102/291] Writing tensor blk.19.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  56\n",
      "[103/291] Writing tensor blk.19.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  56\n",
      "[104/291] Writing tensor blk.19.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  57\n",
      "[105/291] Writing tensor blk.19.ffn_norm.weight                 | size   4096           | type F32  | T+  57\n",
      "[106/291] Writing tensor blk.19.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  57\n",
      "[107/291] Writing tensor blk.19.attn_output.weight              | size   4096 x   4096  | type F16  | T+  57\n",
      "[108/291] Writing tensor blk.19.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  57\n",
      "[109/291] Writing tensor blk.19.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  57\n",
      "[110/291] Writing tensor blk.2.attn_norm.weight                 | size   4096           | type F32  | T+  57\n",
      "[111/291] Writing tensor blk.2.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+  57\n",
      "[112/291] Writing tensor blk.2.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+  58\n",
      "[113/291] Writing tensor blk.2.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+  59\n",
      "[114/291] Writing tensor blk.2.ffn_norm.weight                  | size   4096           | type F32  | T+  59\n",
      "[115/291] Writing tensor blk.2.attn_k.weight                    | size   4096 x   4096  | type F16  | T+  59\n",
      "[116/291] Writing tensor blk.2.attn_output.weight               | size   4096 x   4096  | type F16  | T+  59\n",
      "[117/291] Writing tensor blk.2.attn_q.weight                    | size   4096 x   4096  | type F16  | T+  59\n",
      "[118/291] Writing tensor blk.2.attn_v.weight                    | size   4096 x   4096  | type F16  | T+  59\n",
      "[119/291] Writing tensor blk.20.attn_norm.weight                | size   4096           | type F32  | T+  59\n",
      "[120/291] Writing tensor blk.20.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  61\n",
      "[121/291] Writing tensor blk.20.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  61\n",
      "[122/291] Writing tensor blk.20.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  61\n",
      "[123/291] Writing tensor blk.20.ffn_norm.weight                 | size   4096           | type F32  | T+  61\n",
      "[124/291] Writing tensor blk.20.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  61\n",
      "[125/291] Writing tensor blk.20.attn_output.weight              | size   4096 x   4096  | type F16  | T+  61\n",
      "[126/291] Writing tensor blk.20.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  61\n",
      "[127/291] Writing tensor blk.20.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  61\n",
      "[128/291] Writing tensor blk.21.attn_norm.weight                | size   4096           | type F32  | T+  61\n",
      "[129/291] Writing tensor blk.21.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  66\n",
      "[130/291] Writing tensor blk.21.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  66\n",
      "[131/291] Writing tensor blk.21.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  67\n",
      "[132/291] Writing tensor blk.21.ffn_norm.weight                 | size   4096           | type F32  | T+  67\n",
      "[133/291] Writing tensor blk.21.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  67\n",
      "[134/291] Writing tensor blk.21.attn_output.weight              | size   4096 x   4096  | type F16  | T+  67\n",
      "[135/291] Writing tensor blk.21.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  68\n",
      "[136/291] Writing tensor blk.21.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  68\n",
      "[137/291] Writing tensor blk.22.attn_norm.weight                | size   4096           | type F32  | T+  68\n",
      "[138/291] Writing tensor blk.22.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  73\n",
      "[139/291] Writing tensor blk.22.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  75\n",
      "[140/291] Writing tensor blk.22.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  76\n",
      "[141/291] Writing tensor blk.22.ffn_norm.weight                 | size   4096           | type F32  | T+  76\n",
      "[142/291] Writing tensor blk.22.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  76\n",
      "[143/291] Writing tensor blk.22.attn_output.weight              | size   4096 x   4096  | type F16  | T+  76\n",
      "[144/291] Writing tensor blk.22.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  76\n",
      "[145/291] Writing tensor blk.22.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  76\n",
      "[146/291] Writing tensor blk.23.attn_norm.weight                | size   4096           | type F32  | T+  76\n",
      "[147/291] Writing tensor blk.23.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  77\n",
      "[148/291] Writing tensor blk.23.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  78\n",
      "[149/291] Writing tensor blk.23.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  78\n",
      "[150/291] Writing tensor blk.23.ffn_norm.weight                 | size   4096           | type F32  | T+  78\n",
      "[151/291] Writing tensor blk.23.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  78\n",
      "[152/291] Writing tensor blk.23.attn_output.weight              | size   4096 x   4096  | type F16  | T+  78\n",
      "[153/291] Writing tensor blk.23.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  78\n",
      "[154/291] Writing tensor blk.23.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  78\n",
      "[155/291] Writing tensor blk.3.attn_norm.weight                 | size   4096           | type F32  | T+  78\n",
      "[156/291] Writing tensor blk.3.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+  80\n",
      "[157/291] Writing tensor blk.3.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+  80\n",
      "[158/291] Writing tensor blk.3.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+  80\n",
      "[159/291] Writing tensor blk.3.ffn_norm.weight                  | size   4096           | type F32  | T+  80\n",
      "[160/291] Writing tensor blk.3.attn_k.weight                    | size   4096 x   4096  | type F16  | T+  80\n",
      "[161/291] Writing tensor blk.3.attn_output.weight               | size   4096 x   4096  | type F16  | T+  80\n",
      "[162/291] Writing tensor blk.3.attn_q.weight                    | size   4096 x   4096  | type F16  | T+  80\n",
      "[163/291] Writing tensor blk.3.attn_v.weight                    | size   4096 x   4096  | type F16  | T+  80\n",
      "[164/291] Writing tensor blk.4.attn_norm.weight                 | size   4096           | type F32  | T+  80\n",
      "[165/291] Writing tensor blk.4.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+  86\n",
      "[166/291] Writing tensor blk.4.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+  86\n",
      "[167/291] Writing tensor blk.4.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+  89\n",
      "[168/291] Writing tensor blk.4.ffn_norm.weight                  | size   4096           | type F32  | T+  89\n",
      "[169/291] Writing tensor blk.4.attn_k.weight                    | size   4096 x   4096  | type F16  | T+  89\n",
      "[170/291] Writing tensor blk.4.attn_output.weight               | size   4096 x   4096  | type F16  | T+  89\n",
      "[171/291] Writing tensor blk.4.attn_q.weight                    | size   4096 x   4096  | type F16  | T+  89\n",
      "[172/291] Writing tensor blk.4.attn_v.weight                    | size   4096 x   4096  | type F16  | T+  89\n",
      "[173/291] Writing tensor blk.5.attn_norm.weight                 | size   4096           | type F32  | T+  89\n",
      "[174/291] Writing tensor blk.5.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+  90\n",
      "[175/291] Writing tensor blk.5.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+  94\n",
      "[176/291] Writing tensor blk.5.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+  95\n",
      "[177/291] Writing tensor blk.5.ffn_norm.weight                  | size   4096           | type F32  | T+  95\n",
      "[178/291] Writing tensor blk.5.attn_k.weight                    | size   4096 x   4096  | type F16  | T+  95\n",
      "[179/291] Writing tensor blk.5.attn_output.weight               | size   4096 x   4096  | type F16  | T+  95\n",
      "[180/291] Writing tensor blk.5.attn_q.weight                    | size   4096 x   4096  | type F16  | T+  95\n",
      "[181/291] Writing tensor blk.5.attn_v.weight                    | size   4096 x   4096  | type F16  | T+  95\n",
      "[182/291] Writing tensor blk.6.attn_norm.weight                 | size   4096           | type F32  | T+  95\n",
      "[183/291] Writing tensor blk.6.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+  97\n",
      "[184/291] Writing tensor blk.6.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+  97\n",
      "[185/291] Writing tensor blk.6.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+  97\n",
      "[186/291] Writing tensor blk.6.ffn_norm.weight                  | size   4096           | type F32  | T+  97\n",
      "[187/291] Writing tensor blk.6.attn_k.weight                    | size   4096 x   4096  | type F16  | T+  97\n",
      "[188/291] Writing tensor blk.6.attn_output.weight               | size   4096 x   4096  | type F16  | T+  97\n",
      "[189/291] Writing tensor blk.6.attn_q.weight                    | size   4096 x   4096  | type F16  | T+  97\n",
      "[190/291] Writing tensor blk.6.attn_v.weight                    | size   4096 x   4096  | type F16  | T+  97\n",
      "[191/291] Writing tensor blk.7.attn_norm.weight                 | size   4096           | type F32  | T+  97\n",
      "[192/291] Writing tensor blk.7.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+  99\n",
      "[193/291] Writing tensor blk.7.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+  99\n",
      "[194/291] Writing tensor blk.7.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+  99\n",
      "[195/291] Writing tensor blk.7.ffn_norm.weight                  | size   4096           | type F32  | T+  99\n",
      "[196/291] Writing tensor blk.7.attn_k.weight                    | size   4096 x   4096  | type F16  | T+  99\n",
      "[197/291] Writing tensor blk.7.attn_output.weight               | size   4096 x   4096  | type F16  | T+  99\n",
      "[198/291] Writing tensor blk.7.attn_q.weight                    | size   4096 x   4096  | type F16  | T+  99\n",
      "[199/291] Writing tensor blk.7.attn_v.weight                    | size   4096 x   4096  | type F16  | T+  99\n",
      "[200/291] Writing tensor blk.8.attn_norm.weight                 | size   4096           | type F32  | T+  99\n",
      "[201/291] Writing tensor blk.8.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+ 101\n",
      "[202/291] Writing tensor blk.8.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+ 103\n",
      "[203/291] Writing tensor blk.8.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+ 103\n",
      "[204/291] Writing tensor blk.8.ffn_norm.weight                  | size   4096           | type F32  | T+ 103\n",
      "[205/291] Writing tensor blk.8.attn_k.weight                    | size   4096 x   4096  | type F16  | T+ 103\n",
      "[206/291] Writing tensor blk.8.attn_output.weight               | size   4096 x   4096  | type F16  | T+ 103\n",
      "[207/291] Writing tensor blk.8.attn_q.weight                    | size   4096 x   4096  | type F16  | T+ 104\n",
      "[208/291] Writing tensor blk.8.attn_v.weight                    | size   4096 x   4096  | type F16  | T+ 104\n",
      "[209/291] Writing tensor blk.9.attn_norm.weight                 | size   4096           | type F32  | T+ 104\n",
      "[210/291] Writing tensor blk.9.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+ 112\n",
      "[211/291] Writing tensor blk.9.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+ 113\n",
      "[212/291] Writing tensor blk.9.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+ 113\n",
      "[213/291] Writing tensor blk.9.ffn_norm.weight                  | size   4096           | type F32  | T+ 113\n",
      "[214/291] Writing tensor blk.9.attn_k.weight                    | size   4096 x   4096  | type F16  | T+ 113\n",
      "[215/291] Writing tensor blk.9.attn_output.weight               | size   4096 x   4096  | type F16  | T+ 113\n",
      "[216/291] Writing tensor blk.9.attn_q.weight                    | size   4096 x   4096  | type F16  | T+ 113\n",
      "[217/291] Writing tensor blk.9.attn_v.weight                    | size   4096 x   4096  | type F16  | T+ 113\n",
      "[218/291] Writing tensor output.weight                          | size  32000 x   4096  | type F16  | T+ 116\n",
      "[219/291] Writing tensor blk.24.attn_norm.weight                | size   4096           | type F32  | T+ 116\n",
      "[220/291] Writing tensor blk.24.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+ 116\n",
      "[221/291] Writing tensor blk.24.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+ 116\n",
      "[222/291] Writing tensor blk.24.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+ 116\n",
      "[223/291] Writing tensor blk.24.ffn_norm.weight                 | size   4096           | type F32  | T+ 116\n",
      "[224/291] Writing tensor blk.24.attn_k.weight                   | size   4096 x   4096  | type F16  | T+ 116\n",
      "[225/291] Writing tensor blk.24.attn_output.weight              | size   4096 x   4096  | type F16  | T+ 116\n",
      "[226/291] Writing tensor blk.24.attn_q.weight                   | size   4096 x   4096  | type F16  | T+ 116\n",
      "[227/291] Writing tensor blk.24.attn_v.weight                   | size   4096 x   4096  | type F16  | T+ 117\n",
      "[228/291] Writing tensor blk.25.attn_norm.weight                | size   4096           | type F32  | T+ 117\n",
      "[229/291] Writing tensor blk.25.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+ 118\n",
      "[230/291] Writing tensor blk.25.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+ 119\n",
      "[231/291] Writing tensor blk.25.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+ 120\n",
      "[232/291] Writing tensor blk.25.ffn_norm.weight                 | size   4096           | type F32  | T+ 121\n",
      "[233/291] Writing tensor blk.25.attn_k.weight                   | size   4096 x   4096  | type F16  | T+ 121\n",
      "[234/291] Writing tensor blk.25.attn_output.weight              | size   4096 x   4096  | type F16  | T+ 121\n",
      "[235/291] Writing tensor blk.25.attn_q.weight                   | size   4096 x   4096  | type F16  | T+ 121\n",
      "[236/291] Writing tensor blk.25.attn_v.weight                   | size   4096 x   4096  | type F16  | T+ 121\n",
      "[237/291] Writing tensor blk.26.attn_norm.weight                | size   4096           | type F32  | T+ 121\n",
      "[238/291] Writing tensor blk.26.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+ 129\n",
      "[239/291] Writing tensor blk.26.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+ 129\n",
      "[240/291] Writing tensor blk.26.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+ 130\n",
      "[241/291] Writing tensor blk.26.ffn_norm.weight                 | size   4096           | type F32  | T+ 130\n",
      "[242/291] Writing tensor blk.26.attn_k.weight                   | size   4096 x   4096  | type F16  | T+ 130\n",
      "[243/291] Writing tensor blk.26.attn_output.weight              | size   4096 x   4096  | type F16  | T+ 130\n",
      "[244/291] Writing tensor blk.26.attn_q.weight                   | size   4096 x   4096  | type F16  | T+ 130\n",
      "[245/291] Writing tensor blk.26.attn_v.weight                   | size   4096 x   4096  | type F16  | T+ 130\n",
      "[246/291] Writing tensor blk.27.attn_norm.weight                | size   4096           | type F32  | T+ 130\n",
      "[247/291] Writing tensor blk.27.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+ 131\n",
      "[248/291] Writing tensor blk.27.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+ 132\n",
      "[249/291] Writing tensor blk.27.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+ 133\n",
      "[250/291] Writing tensor blk.27.ffn_norm.weight                 | size   4096           | type F32  | T+ 133\n",
      "[251/291] Writing tensor blk.27.attn_k.weight                   | size   4096 x   4096  | type F16  | T+ 133\n",
      "[252/291] Writing tensor blk.27.attn_output.weight              | size   4096 x   4096  | type F16  | T+ 133\n",
      "[253/291] Writing tensor blk.27.attn_q.weight                   | size   4096 x   4096  | type F16  | T+ 133\n",
      "[254/291] Writing tensor blk.27.attn_v.weight                   | size   4096 x   4096  | type F16  | T+ 133\n",
      "[255/291] Writing tensor blk.28.attn_norm.weight                | size   4096           | type F32  | T+ 133\n",
      "[256/291] Writing tensor blk.28.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+ 135\n",
      "[257/291] Writing tensor blk.28.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+ 135\n",
      "[258/291] Writing tensor blk.28.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+ 135\n",
      "[259/291] Writing tensor blk.28.ffn_norm.weight                 | size   4096           | type F32  | T+ 135\n",
      "[260/291] Writing tensor blk.28.attn_k.weight                   | size   4096 x   4096  | type F16  | T+ 135\n",
      "[261/291] Writing tensor blk.28.attn_output.weight              | size   4096 x   4096  | type F16  | T+ 135\n",
      "[262/291] Writing tensor blk.28.attn_q.weight                   | size   4096 x   4096  | type F16  | T+ 135\n",
      "[263/291] Writing tensor blk.28.attn_v.weight                   | size   4096 x   4096  | type F16  | T+ 135\n",
      "[264/291] Writing tensor blk.29.attn_norm.weight                | size   4096           | type F32  | T+ 135\n",
      "[265/291] Writing tensor blk.29.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+ 141\n",
      "[266/291] Writing tensor blk.29.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+ 141\n",
      "[267/291] Writing tensor blk.29.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+ 144\n",
      "[268/291] Writing tensor blk.29.ffn_norm.weight                 | size   4096           | type F32  | T+ 144\n",
      "[269/291] Writing tensor blk.29.attn_k.weight                   | size   4096 x   4096  | type F16  | T+ 144\n",
      "[270/291] Writing tensor blk.29.attn_output.weight              | size   4096 x   4096  | type F16  | T+ 144\n",
      "[271/291] Writing tensor blk.29.attn_q.weight                   | size   4096 x   4096  | type F16  | T+ 144\n",
      "[272/291] Writing tensor blk.29.attn_v.weight                   | size   4096 x   4096  | type F16  | T+ 144\n",
      "[273/291] Writing tensor blk.30.attn_norm.weight                | size   4096           | type F32  | T+ 144\n",
      "[274/291] Writing tensor blk.30.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+ 144\n",
      "[275/291] Writing tensor blk.30.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+ 150\n",
      "[276/291] Writing tensor blk.30.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+ 150\n",
      "[277/291] Writing tensor blk.30.ffn_norm.weight                 | size   4096           | type F32  | T+ 150\n",
      "[278/291] Writing tensor blk.30.attn_k.weight                   | size   4096 x   4096  | type F16  | T+ 150\n",
      "[279/291] Writing tensor blk.30.attn_output.weight              | size   4096 x   4096  | type F16  | T+ 150\n",
      "[280/291] Writing tensor blk.30.attn_q.weight                   | size   4096 x   4096  | type F16  | T+ 150\n",
      "[281/291] Writing tensor blk.30.attn_v.weight                   | size   4096 x   4096  | type F16  | T+ 150\n",
      "[282/291] Writing tensor blk.31.attn_norm.weight                | size   4096           | type F32  | T+ 150\n",
      "[283/291] Writing tensor blk.31.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+ 152\n",
      "[284/291] Writing tensor blk.31.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+ 152\n",
      "[285/291] Writing tensor blk.31.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+ 152\n",
      "[286/291] Writing tensor blk.31.ffn_norm.weight                 | size   4096           | type F32  | T+ 152\n",
      "[287/291] Writing tensor blk.31.attn_k.weight                   | size   4096 x   4096  | type F16  | T+ 152\n",
      "[288/291] Writing tensor blk.31.attn_output.weight              | size   4096 x   4096  | type F16  | T+ 152\n",
      "[289/291] Writing tensor blk.31.attn_q.weight                   | size   4096 x   4096  | type F16  | T+ 152\n",
      "[290/291] Writing tensor blk.31.attn_v.weight                   | size   4096 x   4096  | type F16  | T+ 152\n",
      "[291/291] Writing tensor output_norm.weight                     | size   4096           | type F32  | T+ 152\n",
      "Wrote /scratch/megathon/quant/metaquant.gguf.fp16.bin\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ['TRANSFORMERS_CACHE'] = '/scratch/megathon/cache/'\n",
    "\n",
    "MODEL_ID = \"/scratch/megathon/cache/models--meta-llama--Llama-2-7b-chat-hf/snapshots/94b07a6e30c3292b8265ed32ffdeccfdadf434a8\"\n",
    "\n",
    "MODEL_NAME = MODEL_ID.split('/')[-1]\n",
    "GGML_VERSION = \"gguf\"\n",
    "\n",
    "# Convert to fp16\n",
    "fp16 = f\"/scratch/megathon/quant/metaquant.{GGML_VERSION}.fp16.bin\"\n",
    "!python /scratch/megathon/quant/llama.cpp/convert.py {MODEL_ID} --outtype f16 --outfile {fp16}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 1 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6\n",
      "main: build = 1441 (ff3bad8)\n",
      "main: built with cc (Ubuntu 7.5.0-3ubuntu1~18.04) 7.5.0 for x86_64-linux-gnu\n",
      "main: quantizing '/scratch/megathon/quant/metaquant.gguf.fp16.bin' to '/scratch/megathon/quant/metaquant.gguf.q4_k_m.bin' as Q4_K_M\n",
      "llama_model_loader: loaded meta data with 18 key-value pairs and 291 tensors from /scratch/megathon/quant/metaquant.gguf.fp16.bin (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight f16      [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:           blk.0.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:            blk.0.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:            blk.0.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:            blk.0.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:              blk.0.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:         blk.0.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:              blk.0.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.1.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.1.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:            blk.1.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:            blk.1.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:              blk.1.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:         blk.1.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:              blk.1.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:          blk.10.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:           blk.10.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:           blk.10.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:             blk.10.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:           blk.10.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:             blk.10.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:        blk.10.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:             blk.10.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:             blk.10.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:          blk.11.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:           blk.11.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:           blk.11.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:             blk.11.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:           blk.11.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:             blk.11.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:        blk.11.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:             blk.11.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:             blk.11.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:          blk.12.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:           blk.12.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:           blk.12.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:             blk.12.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:           blk.12.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:             blk.12.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:        blk.12.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:             blk.12.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:             blk.12.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:          blk.13.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:           blk.13.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:           blk.13.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:             blk.13.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:           blk.13.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:             blk.13.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:        blk.13.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:             blk.13.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:             blk.13.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:          blk.14.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:           blk.14.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:           blk.14.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:             blk.14.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:           blk.14.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:             blk.14.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:        blk.14.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:             blk.14.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:             blk.14.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:          blk.15.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:           blk.15.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:           blk.15.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:             blk.15.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:           blk.15.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:             blk.15.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:        blk.15.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:             blk.15.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:             blk.15.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:          blk.16.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:           blk.16.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:           blk.16.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:             blk.16.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:           blk.16.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:             blk.16.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:        blk.16.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:             blk.16.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:             blk.16.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:          blk.17.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:           blk.17.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:           blk.17.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:             blk.17.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:           blk.17.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:             blk.17.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:        blk.17.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:             blk.17.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:             blk.17.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:          blk.18.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:           blk.18.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:           blk.18.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.18.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:           blk.18.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:             blk.18.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:        blk.18.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:             blk.18.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.18.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.19.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.19.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:           blk.19.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.19.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:           blk.19.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:             blk.19.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:        blk.19.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:             blk.19.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.19.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:           blk.2.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:            blk.2.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:            blk.2.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:              blk.2.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:            blk.2.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:              blk.2.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:         blk.2.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:              blk.2.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:              blk.2.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.20.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.20.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:           blk.20.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.20.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:           blk.20.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:             blk.20.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:        blk.20.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:             blk.20.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.20.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.21.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.21.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:           blk.21.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.21.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:           blk.21.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:             blk.21.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:        blk.21.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:             blk.21.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.21.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.22.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.22.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:           blk.22.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.22.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:           blk.22.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:             blk.22.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:        blk.22.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:             blk.22.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.22.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.23.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.23.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:           blk.23.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.23.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:           blk.23.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:             blk.23.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:        blk.23.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:             blk.23.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.23.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:           blk.3.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:            blk.3.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:            blk.3.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:              blk.3.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:            blk.3.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:              blk.3.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:         blk.3.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:              blk.3.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:              blk.3.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:           blk.4.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:            blk.4.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:            blk.4.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:              blk.4.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:            blk.4.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:              blk.4.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:         blk.4.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:              blk.4.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:              blk.4.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:           blk.5.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:            blk.5.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:            blk.5.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:              blk.5.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:            blk.5.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:              blk.5.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:         blk.5.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:              blk.5.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:              blk.5.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:           blk.6.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:            blk.6.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:            blk.6.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:              blk.6.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:            blk.6.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:              blk.6.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:         blk.6.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:              blk.6.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:              blk.6.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:           blk.7.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:            blk.7.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:            blk.7.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:              blk.7.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:            blk.7.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:              blk.7.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:         blk.7.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:              blk.7.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:              blk.7.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:           blk.8.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:            blk.8.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:            blk.8.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:              blk.8.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:            blk.8.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:              blk.8.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:         blk.8.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:              blk.8.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:              blk.8.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:           blk.9.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:            blk.9.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:            blk.9.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:              blk.9.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:            blk.9.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:              blk.9.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:         blk.9.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:              blk.9.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:              blk.9.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:                    output.weight f16      [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:          blk.24.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:           blk.24.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:           blk.24.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:           blk.24.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:             blk.24.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:        blk.24.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:             blk.24.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:          blk.25.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:           blk.25.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:           blk.25.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:           blk.25.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:             blk.25.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:        blk.25.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:             blk.25.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:          blk.26.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:           blk.26.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:           blk.26.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:           blk.26.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:             blk.26.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:        blk.26.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:             blk.26.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:          blk.27.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:           blk.27.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:           blk.27.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:           blk.27.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:             blk.27.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:        blk.27.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:             blk.27.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:          blk.28.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:           blk.28.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:           blk.28.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:           blk.28.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:             blk.28.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:        blk.28.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:             blk.28.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:          blk.29.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:           blk.29.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:           blk.29.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:           blk.29.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:             blk.29.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:        blk.29.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:             blk.29.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:          blk.30.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:           blk.30.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:           blk.30.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:           blk.30.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:             blk.30.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:        blk.30.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:             blk.30.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:          blk.31.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:           blk.31.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:           blk.31.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:           blk.31.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:             blk.31.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:        blk.31.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:             blk.31.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:               output_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - kv   0:                       general.architecture str     \n",
      "llama_model_loader: - kv   1:                               general.name str     \n",
      "llama_model_loader: - kv   2:                       llama.context_length u32     \n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32     \n",
      "llama_model_loader: - kv   4:                          llama.block_count u32     \n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32     \n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32     \n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32     \n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32     \n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32     \n",
      "llama_model_loader: - kv  10:                          general.file_type u32     \n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str     \n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr     \n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr     \n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr     \n",
      "llama_model_loader: - kv  15:                tokenizer.ggml.bos_token_id u32     \n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.eos_token_id u32     \n",
      "llama_model_loader: - kv  17:            tokenizer.ggml.unknown_token_id u32     \n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type  f16:  226 tensors\n",
      "llama_model_quantize_internal: meta size = 741056 bytes\n",
      "[   1/ 291]                    token_embd.weight - [ 4096, 32000,     1,     1], type =    f16, quantizing to q4_K .. size =   250.00 MB ->    70.31 MB | hist: \n",
      "[   2/ 291]               blk.0.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[   3/ 291]                blk.0.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[   4/ 291]                blk.0.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[   5/ 291]                  blk.0.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[   6/ 291]                blk.0.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[   7/ 291]                  blk.0.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[   8/ 291]             blk.0.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[   9/ 291]                  blk.0.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  10/ 291]                  blk.0.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[  11/ 291]               blk.1.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  12/ 291]                blk.1.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[  13/ 291]                blk.1.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[  14/ 291]                  blk.1.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[  15/ 291]                blk.1.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  16/ 291]                  blk.1.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  17/ 291]             blk.1.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  18/ 291]                  blk.1.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  19/ 291]                  blk.1.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[  20/ 291]              blk.10.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  21/ 291]               blk.10.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[  22/ 291]               blk.10.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[  23/ 291]                 blk.10.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[  24/ 291]               blk.10.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  25/ 291]                 blk.10.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  26/ 291]            blk.10.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  27/ 291]                 blk.10.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  28/ 291]                 blk.10.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[  29/ 291]              blk.11.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  30/ 291]               blk.11.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[  31/ 291]               blk.11.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[  32/ 291]                 blk.11.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[  33/ 291]               blk.11.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  34/ 291]                 blk.11.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  35/ 291]            blk.11.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  36/ 291]                 blk.11.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  37/ 291]                 blk.11.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[  38/ 291]              blk.12.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  39/ 291]               blk.12.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[  40/ 291]               blk.12.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[  41/ 291]                 blk.12.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[  42/ 291]               blk.12.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  43/ 291]                 blk.12.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  44/ 291]            blk.12.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  45/ 291]                 blk.12.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  46/ 291]                 blk.12.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  47/ 291]              blk.13.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  48/ 291]               blk.13.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[  49/ 291]               blk.13.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[  50/ 291]                 blk.13.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[  51/ 291]               blk.13.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  52/ 291]                 blk.13.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  53/ 291]            blk.13.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  54/ 291]                 blk.13.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  55/ 291]                 blk.13.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  56/ 291]              blk.14.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  57/ 291]               blk.14.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[  58/ 291]               blk.14.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[  59/ 291]                 blk.14.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[  60/ 291]               blk.14.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  61/ 291]                 blk.14.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  62/ 291]            blk.14.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  63/ 291]                 blk.14.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  64/ 291]                 blk.14.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[  65/ 291]              blk.15.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  66/ 291]               blk.15.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[  67/ 291]               blk.15.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[  68/ 291]                 blk.15.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[  69/ 291]               blk.15.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  70/ 291]                 blk.15.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  71/ 291]            blk.15.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  72/ 291]                 blk.15.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  73/ 291]                 blk.15.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  74/ 291]              blk.16.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  75/ 291]               blk.16.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[  76/ 291]               blk.16.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[  77/ 291]                 blk.16.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[  78/ 291]               blk.16.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  79/ 291]                 blk.16.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  80/ 291]            blk.16.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  81/ 291]                 blk.16.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  82/ 291]                 blk.16.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  83/ 291]              blk.17.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  84/ 291]               blk.17.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[  85/ 291]               blk.17.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[  86/ 291]                 blk.17.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[  87/ 291]               blk.17.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  88/ 291]                 blk.17.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  89/ 291]            blk.17.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  90/ 291]                 blk.17.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  91/ 291]                 blk.17.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[  92/ 291]              blk.18.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  93/ 291]               blk.18.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[  94/ 291]               blk.18.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[  95/ 291]                 blk.18.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[  96/ 291]               blk.18.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  97/ 291]                 blk.18.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  98/ 291]            blk.18.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[  99/ 291]                 blk.18.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 100/ 291]                 blk.18.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 101/ 291]              blk.19.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 102/ 291]               blk.19.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 103/ 291]               blk.19.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 104/ 291]                 blk.19.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 105/ 291]               blk.19.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 106/ 291]                 blk.19.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 107/ 291]            blk.19.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 108/ 291]                 blk.19.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 109/ 291]                 blk.19.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 110/ 291]               blk.2.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 111/ 291]                blk.2.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[ 112/ 291]                blk.2.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 113/ 291]                  blk.2.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 114/ 291]                blk.2.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 115/ 291]                  blk.2.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 116/ 291]             blk.2.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 117/ 291]                  blk.2.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 118/ 291]                  blk.2.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[ 119/ 291]              blk.20.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 120/ 291]               blk.20.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 121/ 291]               blk.20.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 122/ 291]                 blk.20.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 123/ 291]               blk.20.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 124/ 291]                 blk.20.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 125/ 291]            blk.20.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 126/ 291]                 blk.20.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 127/ 291]                 blk.20.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 128/ 291]              blk.21.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 129/ 291]               blk.21.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 130/ 291]               blk.21.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 131/ 291]                 blk.21.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 132/ 291]               blk.21.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 133/ 291]                 blk.21.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 134/ 291]            blk.21.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 135/ 291]                 blk.21.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 136/ 291]                 blk.21.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 137/ 291]              blk.22.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 138/ 291]               blk.22.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[ 139/ 291]               blk.22.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 140/ 291]                 blk.22.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 141/ 291]               blk.22.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 142/ 291]                 blk.22.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 143/ 291]            blk.22.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 144/ 291]                 blk.22.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 145/ 291]                 blk.22.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[ 146/ 291]              blk.23.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 147/ 291]               blk.23.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 148/ 291]               blk.23.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 149/ 291]                 blk.23.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 150/ 291]               blk.23.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 151/ 291]                 blk.23.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 152/ 291]            blk.23.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 153/ 291]                 blk.23.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 154/ 291]                 blk.23.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 155/ 291]               blk.3.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 156/ 291]                blk.3.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 157/ 291]                blk.3.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 158/ 291]                  blk.3.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 159/ 291]                blk.3.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 160/ 291]                  blk.3.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 161/ 291]             blk.3.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 162/ 291]                  blk.3.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 163/ 291]                  blk.3.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 164/ 291]               blk.4.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 165/ 291]                blk.4.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[ 166/ 291]                blk.4.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 167/ 291]                  blk.4.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 168/ 291]                blk.4.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 169/ 291]                  blk.4.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 170/ 291]             blk.4.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 171/ 291]                  blk.4.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 172/ 291]                  blk.4.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[ 173/ 291]               blk.5.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 174/ 291]                blk.5.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 175/ 291]                blk.5.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 176/ 291]                  blk.5.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 177/ 291]                blk.5.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 178/ 291]                  blk.5.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 179/ 291]             blk.5.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 180/ 291]                  blk.5.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 181/ 291]                  blk.5.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 182/ 291]               blk.6.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 183/ 291]                blk.6.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 184/ 291]                blk.6.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 185/ 291]                  blk.6.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 186/ 291]                blk.6.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 187/ 291]                  blk.6.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 188/ 291]             blk.6.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 189/ 291]                  blk.6.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 190/ 291]                  blk.6.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 191/ 291]               blk.7.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 192/ 291]                blk.7.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[ 193/ 291]                blk.7.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 194/ 291]                  blk.7.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 195/ 291]                blk.7.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 196/ 291]                  blk.7.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 197/ 291]             blk.7.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 198/ 291]                  blk.7.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 199/ 291]                  blk.7.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[ 200/ 291]               blk.8.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 201/ 291]                blk.8.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 202/ 291]                blk.8.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 203/ 291]                  blk.8.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 204/ 291]                blk.8.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 205/ 291]                  blk.8.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 206/ 291]             blk.8.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 207/ 291]                  blk.8.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 208/ 291]                  blk.8.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 209/ 291]               blk.9.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 210/ 291]                blk.9.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 211/ 291]                blk.9.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 212/ 291]                  blk.9.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 213/ 291]                blk.9.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 214/ 291]                  blk.9.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 215/ 291]             blk.9.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 216/ 291]                  blk.9.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 217/ 291]                  blk.9.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 218/ 291]                        output.weight - [ 4096, 32000,     1,     1], type =    f16, quantizing to q6_K .. size =   250.00 MB ->   102.54 MB | hist: \n",
      "[ 219/ 291]              blk.24.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 220/ 291]               blk.24.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[ 221/ 291]               blk.24.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 222/ 291]                 blk.24.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 223/ 291]               blk.24.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 224/ 291]                 blk.24.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 225/ 291]            blk.24.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 226/ 291]                 blk.24.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 227/ 291]                 blk.24.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[ 228/ 291]              blk.25.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 229/ 291]               blk.25.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 230/ 291]               blk.25.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 231/ 291]                 blk.25.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 232/ 291]               blk.25.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 233/ 291]                 blk.25.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 234/ 291]            blk.25.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 235/ 291]                 blk.25.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 236/ 291]                 blk.25.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 237/ 291]              blk.26.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 238/ 291]               blk.26.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 239/ 291]               blk.26.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 240/ 291]                 blk.26.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 241/ 291]               blk.26.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 242/ 291]                 blk.26.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 243/ 291]            blk.26.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 244/ 291]                 blk.26.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 245/ 291]                 blk.26.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 246/ 291]              blk.27.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 247/ 291]               blk.27.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[ 248/ 291]               blk.27.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 249/ 291]                 blk.27.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 250/ 291]               blk.27.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 251/ 291]                 blk.27.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 252/ 291]            blk.27.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 253/ 291]                 blk.27.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 254/ 291]                 blk.27.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[ 255/ 291]              blk.28.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 256/ 291]               blk.28.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[ 257/ 291]               blk.28.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 258/ 291]                 blk.28.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 259/ 291]               blk.28.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 260/ 291]                 blk.28.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 261/ 291]            blk.28.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 262/ 291]                 blk.28.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 263/ 291]                 blk.28.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[ 264/ 291]              blk.29.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 265/ 291]               blk.29.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[ 266/ 291]               blk.29.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 267/ 291]                 blk.29.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 268/ 291]               blk.29.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 269/ 291]                 blk.29.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 270/ 291]            blk.29.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 271/ 291]                 blk.29.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 272/ 291]                 blk.29.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[ 273/ 291]              blk.30.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 274/ 291]               blk.30.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[ 275/ 291]               blk.30.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 276/ 291]                 blk.30.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 277/ 291]               blk.30.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 278/ 291]                 blk.30.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 279/ 291]            blk.30.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 280/ 291]                 blk.30.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 281/ 291]                 blk.30.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[ 282/ 291]              blk.31.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 283/ 291]               blk.31.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[ 284/ 291]               blk.31.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 285/ 291]                 blk.31.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_K .. size =    86.00 MB ->    24.19 MB | hist: \n",
      "[ 286/ 291]               blk.31.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 287/ 291]                 blk.31.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 288/ 291]            blk.31.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 289/ 291]                 blk.31.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_K .. size =    32.00 MB ->     9.00 MB | hist: \n",
      "[ 290/ 291]                 blk.31.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[ 291/ 291]                   output_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "llama_model_quantize_internal: model size  = 12853.02 MB\n",
      "llama_model_quantize_internal: quant size  =  3891.24 MB\n",
      "\n",
      "main: quantize time = 259001.39 ms\n",
      "main:    total time = 259001.39 ms\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 1 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6\n",
      "main: build = 1441 (ff3bad8)\n",
      "main: built with cc (Ubuntu 7.5.0-3ubuntu1~18.04) 7.5.0 for x86_64-linux-gnu\n",
      "main: quantizing '/scratch/megathon/quant/metaquant.gguf.fp16.bin' to '/scratch/megathon/quant/metaquant.gguf.q5_k_m.bin' as Q5_K_M\n",
      "llama_model_loader: loaded meta data with 18 key-value pairs and 291 tensors from /scratch/megathon/quant/metaquant.gguf.fp16.bin (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight f16      [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:           blk.0.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:            blk.0.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:            blk.0.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:            blk.0.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:              blk.0.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:         blk.0.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:              blk.0.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.1.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.1.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:            blk.1.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:            blk.1.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:              blk.1.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:         blk.1.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:              blk.1.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:          blk.10.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:           blk.10.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:           blk.10.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:             blk.10.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:           blk.10.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:             blk.10.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:        blk.10.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:             blk.10.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:             blk.10.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:          blk.11.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:           blk.11.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:           blk.11.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:             blk.11.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:           blk.11.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:             blk.11.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:        blk.11.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:             blk.11.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:             blk.11.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:          blk.12.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:           blk.12.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:           blk.12.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:             blk.12.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:           blk.12.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:             blk.12.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:        blk.12.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:             blk.12.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:             blk.12.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:          blk.13.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:           blk.13.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:           blk.13.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:             blk.13.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:           blk.13.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:             blk.13.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:        blk.13.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:             blk.13.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:             blk.13.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:          blk.14.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:           blk.14.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:           blk.14.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:             blk.14.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:           blk.14.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:             blk.14.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:        blk.14.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:             blk.14.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:             blk.14.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:          blk.15.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:           blk.15.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:           blk.15.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:             blk.15.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:           blk.15.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:             blk.15.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:        blk.15.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:             blk.15.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:             blk.15.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:          blk.16.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:           blk.16.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:           blk.16.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:             blk.16.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:           blk.16.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:             blk.16.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:        blk.16.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:             blk.16.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:             blk.16.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:          blk.17.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:           blk.17.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:           blk.17.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:             blk.17.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:           blk.17.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:             blk.17.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:        blk.17.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:             blk.17.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:             blk.17.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:          blk.18.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:           blk.18.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:           blk.18.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.18.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:           blk.18.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:             blk.18.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:        blk.18.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:             blk.18.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.18.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.19.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.19.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:           blk.19.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.19.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:           blk.19.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:             blk.19.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:        blk.19.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:             blk.19.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.19.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:           blk.2.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:            blk.2.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:            blk.2.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:              blk.2.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:            blk.2.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:              blk.2.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:         blk.2.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:              blk.2.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:              blk.2.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.20.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.20.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:           blk.20.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.20.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:           blk.20.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:             blk.20.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:        blk.20.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:             blk.20.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.20.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.21.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.21.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:           blk.21.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.21.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:           blk.21.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:             blk.21.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:        blk.21.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:             blk.21.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.21.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.22.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.22.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:           blk.22.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.22.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:           blk.22.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:             blk.22.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:        blk.22.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:             blk.22.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.22.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.23.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.23.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:           blk.23.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.23.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:           blk.23.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:             blk.23.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:        blk.23.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:             blk.23.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.23.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:           blk.3.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:            blk.3.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:            blk.3.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:              blk.3.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:            blk.3.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:              blk.3.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:         blk.3.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:              blk.3.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:              blk.3.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:           blk.4.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:            blk.4.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:            blk.4.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:              blk.4.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:            blk.4.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:              blk.4.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:         blk.4.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:              blk.4.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:              blk.4.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:           blk.5.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:            blk.5.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:            blk.5.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:              blk.5.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:            blk.5.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:              blk.5.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:         blk.5.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:              blk.5.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:              blk.5.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:           blk.6.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:            blk.6.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:            blk.6.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:              blk.6.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:            blk.6.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:              blk.6.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:         blk.6.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:              blk.6.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:              blk.6.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:           blk.7.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:            blk.7.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:            blk.7.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:              blk.7.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:            blk.7.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:              blk.7.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:         blk.7.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:              blk.7.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:              blk.7.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:           blk.8.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:            blk.8.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:            blk.8.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:              blk.8.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:            blk.8.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:              blk.8.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:         blk.8.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:              blk.8.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:              blk.8.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:           blk.9.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:            blk.9.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:            blk.9.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:              blk.9.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:            blk.9.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:              blk.9.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:         blk.9.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:              blk.9.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:              blk.9.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:                    output.weight f16      [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:          blk.24.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:           blk.24.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:           blk.24.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:           blk.24.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:             blk.24.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:        blk.24.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:             blk.24.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:          blk.25.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:           blk.25.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:           blk.25.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:           blk.25.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:             blk.25.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:        blk.25.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:             blk.25.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:          blk.26.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:           blk.26.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:           blk.26.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:           blk.26.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:             blk.26.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:        blk.26.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:             blk.26.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:          blk.27.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:           blk.27.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:           blk.27.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:           blk.27.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:             blk.27.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:        blk.27.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:             blk.27.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:          blk.28.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:           blk.28.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:           blk.28.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:           blk.28.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:             blk.28.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:        blk.28.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:             blk.28.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:          blk.29.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:           blk.29.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:           blk.29.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:           blk.29.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:             blk.29.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:        blk.29.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:             blk.29.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:          blk.30.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:           blk.30.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:           blk.30.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:           blk.30.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:             blk.30.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:        blk.30.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:             blk.30.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:          blk.31.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:           blk.31.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:           blk.31.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:           blk.31.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:             blk.31.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:        blk.31.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:             blk.31.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:               output_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - kv   0:                       general.architecture str     \n",
      "llama_model_loader: - kv   1:                               general.name str     \n",
      "llama_model_loader: - kv   2:                       llama.context_length u32     \n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32     \n",
      "llama_model_loader: - kv   4:                          llama.block_count u32     \n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32     \n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32     \n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32     \n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32     \n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32     \n",
      "llama_model_loader: - kv  10:                          general.file_type u32     \n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str     \n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr     \n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr     \n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr     \n",
      "llama_model_loader: - kv  15:                tokenizer.ggml.bos_token_id u32     \n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.eos_token_id u32     \n",
      "llama_model_loader: - kv  17:            tokenizer.ggml.unknown_token_id u32     \n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type  f16:  226 tensors\n",
      "llama_model_quantize_internal: meta size = 741056 bytes\n",
      "[   1/ 291]                    token_embd.weight - [ 4096, 32000,     1,     1], type =    f16, quantizing to q5_K .. size =   250.00 MB ->    85.94 MB | hist: \n",
      "[   2/ 291]               blk.0.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[   3/ 291]                blk.0.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[   4/ 291]                blk.0.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[   5/ 291]                  blk.0.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[   6/ 291]                blk.0.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[   7/ 291]                  blk.0.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[   8/ 291]             blk.0.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[   9/ 291]                  blk.0.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  10/ 291]                  blk.0.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[  11/ 291]               blk.1.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  12/ 291]                blk.1.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[  13/ 291]                blk.1.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[  14/ 291]                  blk.1.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[  15/ 291]                blk.1.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  16/ 291]                  blk.1.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  17/ 291]             blk.1.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  18/ 291]                  blk.1.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  19/ 291]                  blk.1.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[  20/ 291]              blk.10.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  21/ 291]               blk.10.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[  22/ 291]               blk.10.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[  23/ 291]                 blk.10.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[  24/ 291]               blk.10.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  25/ 291]                 blk.10.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  26/ 291]            blk.10.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  27/ 291]                 blk.10.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  28/ 291]                 blk.10.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[  29/ 291]              blk.11.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  30/ 291]               blk.11.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[  31/ 291]               blk.11.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[  32/ 291]                 blk.11.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[  33/ 291]               blk.11.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  34/ 291]                 blk.11.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  35/ 291]            blk.11.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  36/ 291]                 blk.11.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  37/ 291]                 blk.11.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[  38/ 291]              blk.12.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  39/ 291]               blk.12.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[  40/ 291]               blk.12.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[  41/ 291]                 blk.12.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[  42/ 291]               blk.12.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  43/ 291]                 blk.12.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  44/ 291]            blk.12.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  45/ 291]                 blk.12.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  46/ 291]                 blk.12.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  47/ 291]              blk.13.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  48/ 291]               blk.13.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[  49/ 291]               blk.13.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[  50/ 291]                 blk.13.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[  51/ 291]               blk.13.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  52/ 291]                 blk.13.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  53/ 291]            blk.13.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  54/ 291]                 blk.13.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  55/ 291]                 blk.13.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  56/ 291]              blk.14.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  57/ 291]               blk.14.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[  58/ 291]               blk.14.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[  59/ 291]                 blk.14.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[  60/ 291]               blk.14.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  61/ 291]                 blk.14.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  62/ 291]            blk.14.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  63/ 291]                 blk.14.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  64/ 291]                 blk.14.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[  65/ 291]              blk.15.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  66/ 291]               blk.15.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[  67/ 291]               blk.15.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[  68/ 291]                 blk.15.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[  69/ 291]               blk.15.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  70/ 291]                 blk.15.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  71/ 291]            blk.15.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  72/ 291]                 blk.15.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  73/ 291]                 blk.15.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  74/ 291]              blk.16.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  75/ 291]               blk.16.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[  76/ 291]               blk.16.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[  77/ 291]                 blk.16.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[  78/ 291]               blk.16.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  79/ 291]                 blk.16.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  80/ 291]            blk.16.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  81/ 291]                 blk.16.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  82/ 291]                 blk.16.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  83/ 291]              blk.17.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  84/ 291]               blk.17.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[  85/ 291]               blk.17.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[  86/ 291]                 blk.17.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[  87/ 291]               blk.17.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  88/ 291]                 blk.17.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  89/ 291]            blk.17.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  90/ 291]                 blk.17.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  91/ 291]                 blk.17.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[  92/ 291]              blk.18.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  93/ 291]               blk.18.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[  94/ 291]               blk.18.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[  95/ 291]                 blk.18.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[  96/ 291]               blk.18.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  97/ 291]                 blk.18.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  98/ 291]            blk.18.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[  99/ 291]                 blk.18.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 100/ 291]                 blk.18.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 101/ 291]              blk.19.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 102/ 291]               blk.19.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 103/ 291]               blk.19.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 104/ 291]                 blk.19.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 105/ 291]               blk.19.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 106/ 291]                 blk.19.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 107/ 291]            blk.19.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 108/ 291]                 blk.19.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 109/ 291]                 blk.19.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 110/ 291]               blk.2.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 111/ 291]                blk.2.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[ 112/ 291]                blk.2.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 113/ 291]                  blk.2.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 114/ 291]                blk.2.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 115/ 291]                  blk.2.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 116/ 291]             blk.2.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 117/ 291]                  blk.2.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 118/ 291]                  blk.2.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[ 119/ 291]              blk.20.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 120/ 291]               blk.20.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 121/ 291]               blk.20.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 122/ 291]                 blk.20.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 123/ 291]               blk.20.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 124/ 291]                 blk.20.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 125/ 291]            blk.20.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 126/ 291]                 blk.20.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 127/ 291]                 blk.20.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 128/ 291]              blk.21.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 129/ 291]               blk.21.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 130/ 291]               blk.21.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 131/ 291]                 blk.21.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 132/ 291]               blk.21.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 133/ 291]                 blk.21.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 134/ 291]            blk.21.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 135/ 291]                 blk.21.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 136/ 291]                 blk.21.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 137/ 291]              blk.22.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 138/ 291]               blk.22.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[ 139/ 291]               blk.22.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 140/ 291]                 blk.22.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 141/ 291]               blk.22.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 142/ 291]                 blk.22.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 143/ 291]            blk.22.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 144/ 291]                 blk.22.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 145/ 291]                 blk.22.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[ 146/ 291]              blk.23.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 147/ 291]               blk.23.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 148/ 291]               blk.23.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 149/ 291]                 blk.23.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 150/ 291]               blk.23.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 151/ 291]                 blk.23.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 152/ 291]            blk.23.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 153/ 291]                 blk.23.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 154/ 291]                 blk.23.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 155/ 291]               blk.3.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 156/ 291]                blk.3.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 157/ 291]                blk.3.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 158/ 291]                  blk.3.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 159/ 291]                blk.3.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 160/ 291]                  blk.3.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 161/ 291]             blk.3.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 162/ 291]                  blk.3.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 163/ 291]                  blk.3.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 164/ 291]               blk.4.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 165/ 291]                blk.4.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[ 166/ 291]                blk.4.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 167/ 291]                  blk.4.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 168/ 291]                blk.4.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 169/ 291]                  blk.4.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 170/ 291]             blk.4.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 171/ 291]                  blk.4.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 172/ 291]                  blk.4.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[ 173/ 291]               blk.5.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 174/ 291]                blk.5.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 175/ 291]                blk.5.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 176/ 291]                  blk.5.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 177/ 291]                blk.5.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 178/ 291]                  blk.5.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 179/ 291]             blk.5.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 180/ 291]                  blk.5.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 181/ 291]                  blk.5.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 182/ 291]               blk.6.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 183/ 291]                blk.6.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 184/ 291]                blk.6.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 185/ 291]                  blk.6.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 186/ 291]                blk.6.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 187/ 291]                  blk.6.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 188/ 291]             blk.6.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 189/ 291]                  blk.6.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 190/ 291]                  blk.6.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 191/ 291]               blk.7.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 192/ 291]                blk.7.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[ 193/ 291]                blk.7.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 194/ 291]                  blk.7.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 195/ 291]                blk.7.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 196/ 291]                  blk.7.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 197/ 291]             blk.7.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 198/ 291]                  blk.7.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 199/ 291]                  blk.7.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[ 200/ 291]               blk.8.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 201/ 291]                blk.8.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 202/ 291]                blk.8.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 203/ 291]                  blk.8.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 204/ 291]                blk.8.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 205/ 291]                  blk.8.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 206/ 291]             blk.8.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 207/ 291]                  blk.8.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 208/ 291]                  blk.8.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 209/ 291]               blk.9.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 210/ 291]                blk.9.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 211/ 291]                blk.9.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 212/ 291]                  blk.9.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 213/ 291]                blk.9.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 214/ 291]                  blk.9.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 215/ 291]             blk.9.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 216/ 291]                  blk.9.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 217/ 291]                  blk.9.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 218/ 291]                        output.weight - [ 4096, 32000,     1,     1], type =    f16, quantizing to q6_K .. size =   250.00 MB ->   102.54 MB | hist: \n",
      "[ 219/ 291]              blk.24.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 220/ 291]               blk.24.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[ 221/ 291]               blk.24.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 222/ 291]                 blk.24.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 223/ 291]               blk.24.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 224/ 291]                 blk.24.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 225/ 291]            blk.24.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 226/ 291]                 blk.24.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 227/ 291]                 blk.24.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[ 228/ 291]              blk.25.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 229/ 291]               blk.25.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 230/ 291]               blk.25.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 231/ 291]                 blk.25.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 232/ 291]               blk.25.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 233/ 291]                 blk.25.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 234/ 291]            blk.25.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 235/ 291]                 blk.25.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 236/ 291]                 blk.25.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 237/ 291]              blk.26.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 238/ 291]               blk.26.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 239/ 291]               blk.26.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 240/ 291]                 blk.26.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 241/ 291]               blk.26.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 242/ 291]                 blk.26.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 243/ 291]            blk.26.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 244/ 291]                 blk.26.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 245/ 291]                 blk.26.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 246/ 291]              blk.27.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 247/ 291]               blk.27.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[ 248/ 291]               blk.27.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 249/ 291]                 blk.27.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 250/ 291]               blk.27.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 251/ 291]                 blk.27.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 252/ 291]            blk.27.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 253/ 291]                 blk.27.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 254/ 291]                 blk.27.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[ 255/ 291]              blk.28.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 256/ 291]               blk.28.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[ 257/ 291]               blk.28.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 258/ 291]                 blk.28.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 259/ 291]               blk.28.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 260/ 291]                 blk.28.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 261/ 291]            blk.28.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 262/ 291]                 blk.28.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 263/ 291]                 blk.28.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[ 264/ 291]              blk.29.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 265/ 291]               blk.29.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[ 266/ 291]               blk.29.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 267/ 291]                 blk.29.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 268/ 291]               blk.29.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 269/ 291]                 blk.29.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 270/ 291]            blk.29.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 271/ 291]                 blk.29.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 272/ 291]                 blk.29.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[ 273/ 291]              blk.30.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 274/ 291]               blk.30.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[ 275/ 291]               blk.30.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 276/ 291]                 blk.30.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 277/ 291]               blk.30.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 278/ 291]                 blk.30.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 279/ 291]            blk.30.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 280/ 291]                 blk.30.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 281/ 291]                 blk.30.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[ 282/ 291]              blk.31.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 283/ 291]               blk.31.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    86.00 MB ->    35.27 MB | hist: \n",
      "[ 284/ 291]               blk.31.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 285/ 291]                 blk.31.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q5_K .. size =    86.00 MB ->    29.56 MB | hist: \n",
      "[ 286/ 291]               blk.31.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 287/ 291]                 blk.31.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 288/ 291]            blk.31.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 289/ 291]                 blk.31.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q5_K .. size =    32.00 MB ->    11.00 MB | hist: \n",
      "[ 290/ 291]                 blk.31.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q6_K .. size =    32.00 MB ->    13.12 MB | hist: \n",
      "[ 291/ 291]                   output_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "llama_model_quantize_internal: model size  = 12853.02 MB\n",
      "llama_model_quantize_internal: quant size  =  4560.87 MB\n",
      "\n",
      "main: quantize time = 242082.18 ms\n",
      "main:    total time = 242082.18 ms\n"
     ]
    }
   ],
   "source": [
    "QUANTIZATION_METHODS = [\"q4_k_m\", \"q5_k_m\"]\n",
    "\n",
    "for method in QUANTIZATION_METHODS:\n",
    "    qtype = f\"/scratch/megathon/quant/metaquant.{GGML_VERSION}.{method}.bin\"\n",
    "    !/scratch/megathon/quant/llama.cpp/quantize {fp16} {qtype} {method}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1441 (ff3bad8)\n",
      "main: built with cc (Ubuntu 7.5.0-3ubuntu1~18.04) 7.5.0 for x86_64-linux-gnu\n",
      "main: seed  = 1698532407\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 1 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6\n",
      "llama_model_loader: loaded meta data with 19 key-value pairs and 291 tensors from /scratch/megathon/quant/metaquant.gguf.q4_k_m.bin (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight q4_K     [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:           blk.0.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:            blk.0.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:            blk.0.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:            blk.0.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:              blk.0.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:         blk.0.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:              blk.0.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.1.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.1.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:            blk.1.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:            blk.1.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:              blk.1.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:         blk.1.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:              blk.1.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:          blk.10.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:           blk.10.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:           blk.10.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:             blk.10.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:           blk.10.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:             blk.10.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:        blk.10.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:             blk.10.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:             blk.10.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:          blk.11.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:           blk.11.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:           blk.11.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:             blk.11.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:           blk.11.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:             blk.11.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:        blk.11.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:             blk.11.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:             blk.11.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:          blk.12.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:           blk.12.ffn_down.weight q4_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:           blk.12.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:             blk.12.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:           blk.12.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:             blk.12.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:        blk.12.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:             blk.12.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:             blk.12.attn_v.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:          blk.13.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:           blk.13.ffn_down.weight q4_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:           blk.13.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:             blk.13.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:           blk.13.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:             blk.13.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:        blk.13.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:             blk.13.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:             blk.13.attn_v.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:          blk.14.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:           blk.14.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:           blk.14.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:             blk.14.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:           blk.14.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:             blk.14.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:        blk.14.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:             blk.14.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:             blk.14.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:          blk.15.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:           blk.15.ffn_down.weight q4_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:           blk.15.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:             blk.15.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:           blk.15.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:             blk.15.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:        blk.15.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:             blk.15.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:             blk.15.attn_v.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:          blk.16.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:           blk.16.ffn_down.weight q4_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:           blk.16.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:             blk.16.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:           blk.16.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:             blk.16.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:        blk.16.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:             blk.16.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:             blk.16.attn_v.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:          blk.17.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:           blk.17.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:           blk.17.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:             blk.17.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:           blk.17.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:             blk.17.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:        blk.17.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:             blk.17.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:             blk.17.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:          blk.18.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:           blk.18.ffn_down.weight q4_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:           blk.18.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.18.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:           blk.18.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:             blk.18.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:        blk.18.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:             blk.18.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.18.attn_v.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.19.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.19.ffn_down.weight q4_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:           blk.19.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.19.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:           blk.19.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:             blk.19.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:        blk.19.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:             blk.19.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.19.attn_v.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:           blk.2.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:            blk.2.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:            blk.2.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:              blk.2.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:            blk.2.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:              blk.2.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:         blk.2.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:              blk.2.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:              blk.2.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.20.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.20.ffn_down.weight q4_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:           blk.20.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.20.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:           blk.20.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:             blk.20.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:        blk.20.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:             blk.20.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.20.attn_v.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.21.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.21.ffn_down.weight q4_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:           blk.21.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.21.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:           blk.21.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:             blk.21.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:        blk.21.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:             blk.21.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.21.attn_v.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.22.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.22.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:           blk.22.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.22.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:           blk.22.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:             blk.22.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:        blk.22.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:             blk.22.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.22.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.23.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.23.ffn_down.weight q4_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:           blk.23.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.23.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:           blk.23.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:             blk.23.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:        blk.23.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:             blk.23.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.23.attn_v.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:           blk.3.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:            blk.3.ffn_down.weight q4_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:            blk.3.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:              blk.3.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:            blk.3.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:              blk.3.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:         blk.3.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:              blk.3.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:              blk.3.attn_v.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:           blk.4.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:            blk.4.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:            blk.4.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:              blk.4.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:            blk.4.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:              blk.4.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:         blk.4.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:              blk.4.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:              blk.4.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:           blk.5.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:            blk.5.ffn_down.weight q4_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:            blk.5.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:              blk.5.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:            blk.5.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:              blk.5.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:         blk.5.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:              blk.5.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:              blk.5.attn_v.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:           blk.6.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:            blk.6.ffn_down.weight q4_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:            blk.6.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:              blk.6.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:            blk.6.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:              blk.6.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:         blk.6.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:              blk.6.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:              blk.6.attn_v.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:           blk.7.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:            blk.7.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:            blk.7.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:              blk.7.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:            blk.7.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:              blk.7.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:         blk.7.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:              blk.7.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:              blk.7.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:           blk.8.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:            blk.8.ffn_down.weight q4_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:            blk.8.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:              blk.8.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:            blk.8.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:              blk.8.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:         blk.8.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:              blk.8.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:              blk.8.attn_v.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:           blk.9.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:            blk.9.ffn_down.weight q4_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:            blk.9.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:              blk.9.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:            blk.9.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:              blk.9.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:         blk.9.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:              blk.9.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:              blk.9.attn_v.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:                    output.weight q6_K     [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:          blk.24.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:           blk.24.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:           blk.24.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:           blk.24.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:             blk.24.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:        blk.24.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:             blk.24.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:          blk.25.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:           blk.25.ffn_down.weight q4_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:           blk.25.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:           blk.25.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:             blk.25.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:        blk.25.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:             blk.25.attn_v.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:          blk.26.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:           blk.26.ffn_down.weight q4_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:           blk.26.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:           blk.26.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:             blk.26.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:        blk.26.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:             blk.26.attn_v.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:          blk.27.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:           blk.27.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:           blk.27.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:           blk.27.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:             blk.27.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:        blk.27.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:             blk.27.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:          blk.28.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:           blk.28.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:           blk.28.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:           blk.28.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:             blk.28.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:        blk.28.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:             blk.28.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:          blk.29.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:           blk.29.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:           blk.29.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:           blk.29.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:             blk.29.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:        blk.29.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:             blk.29.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:          blk.30.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:           blk.30.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:           blk.30.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:           blk.30.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:             blk.30.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:        blk.30.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:             blk.30.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:          blk.31.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:           blk.31.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:           blk.31.ffn_gate.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.ffn_up.weight q4_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:           blk.31.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:             blk.31.attn_k.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:        blk.31.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:             blk.31.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:               output_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - kv   0:                       general.architecture str     \n",
      "llama_model_loader: - kv   1:                               general.name str     \n",
      "llama_model_loader: - kv   2:                       llama.context_length u32     \n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32     \n",
      "llama_model_loader: - kv   4:                          llama.block_count u32     \n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32     \n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32     \n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32     \n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32     \n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32     \n",
      "llama_model_loader: - kv  10:                          general.file_type u32     \n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str     \n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr     \n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr     \n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr     \n",
      "llama_model_loader: - kv  15:                tokenizer.ggml.bos_token_id u32     \n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.eos_token_id u32     \n",
      "llama_model_loader: - kv  17:            tokenizer.ggml.unknown_token_id u32     \n",
      "llama_model_loader: - kv  18:               general.quantization_version u32     \n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type q4_K:  193 tensors\n",
      "llama_model_loader: - type q6_K:   33 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = mostly Q4_K - Medium\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 3.80 GiB (4.84 BPW) \n",
      "llm_load_print_meta: general.name   = LLaMA v2\n",
      "llm_load_print_meta: BOS token = 1 '<s>'\n",
      "llm_load_print_meta: EOS token = 2 '</s>'\n",
      "llm_load_print_meta: UNK token = 0 '<unk>'\n",
      "llm_load_print_meta: LF token  = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size =    0.10 MB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: mem required  =   70.41 MB\n",
      "llm_load_tensors: offloading 32 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 35/35 layers to GPU\n",
      "llm_load_tensors: VRAM used: 3820.93 MB\n",
      "..................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: offloading v cache to GPU\n",
      "llama_kv_cache_init: offloading k cache to GPU\n",
      "llama_kv_cache_init: VRAM kv self = 256.00 MB\n",
      "llama_new_context_with_model: kv self size  =  256.00 MB\n",
      "llama_new_context_with_model: compute buffer total size = 76.63 MB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 70.50 MB\n",
      "llama_new_context_with_model: total VRAM used: 4147.43 MB (model: 3820.93 MB, context: 326.50 MB)\n",
      "\n",
      "system_info: n_threads = 24 / 48 | AVX = 1 | AVX2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, typical_p = 1.000, temp = 0.800\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 128, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33mWhat should I do if I want to stop dialysis?\u001b[0m\n",
      "Do I have the right to refuse treatment and stop dialysis?\n",
      "How can I stop dialysis safely?\n",
      "Who can help me make decisions about my treatment options?\n",
      "It's important to understand that stopping dialysis is a complex decision that requires careful consideration. Here are some steps you can take to explore your options:\n",
      "Talk to your healthcare team: Your nephrologist or other healthcare providers can provide information about the potential risks and benefits of stopping dialysis. They can also help you understand the impact on your overall health and well-being.\n",
      "Res\n",
      "llama_print_timings:        load time =   19382.56 ms\n",
      "llama_print_timings:      sample time =      29.42 ms /   128 runs   (    0.23 ms per token,  4350.19 tokens per second)\n",
      "llama_print_timings: prompt eval time =      39.33 ms /    14 tokens (    2.81 ms per token,   355.94 tokens per second)\n",
      "llama_print_timings:        eval time =    1118.39 ms /   127 runs   (    8.81 ms per token,   113.56 tokens per second)\n",
      "llama_print_timings:       total time =    1222.73 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "model_list = [file for file in os.listdir(\"/scratch/megathon/quant/\") if GGML_VERSION in file]\n",
    "prompt = \"What should I do if I want to stop dialysis?\"\n",
    "chosen_method = \"q4_k_m\" #input(\"Please specify the quantization method to run the model (options: \" + \", \".join(model_list) + \"): \")\n",
    "\n",
    "qtype = f\"/scratch/megathon/quant/metaquant.{GGML_VERSION}.{chosen_method}.bin\"\n",
    "!/scratch/megathon/quant/llama.cpp/main -m {qtype} -n 128 --repeat_penalty 1.1 --color -ngl 35 -p \"{prompt}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1441 (ff3bad8)\n",
      "main: built with cc (Ubuntu 7.5.0-3ubuntu1~18.04) 7.5.0 for x86_64-linux-gnu\n",
      "main: seed  = 1698532562\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 1 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6\n",
      "llama_model_loader: loaded meta data with 19 key-value pairs and 291 tensors from /scratch/megathon/quant/metaquant.gguf.q5_k_m.bin (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight q5_K     [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:           blk.0.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:            blk.0.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:            blk.0.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:            blk.0.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:              blk.0.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:         blk.0.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:              blk.0.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.1.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.1.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:            blk.1.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:            blk.1.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:              blk.1.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:         blk.1.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:              blk.1.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:          blk.10.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:           blk.10.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:           blk.10.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:             blk.10.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:           blk.10.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:             blk.10.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:        blk.10.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:             blk.10.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:             blk.10.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:          blk.11.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:           blk.11.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:           blk.11.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:             blk.11.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:           blk.11.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:             blk.11.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:        blk.11.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:             blk.11.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:             blk.11.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:          blk.12.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:           blk.12.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:           blk.12.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:             blk.12.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:           blk.12.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:             blk.12.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:        blk.12.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:             blk.12.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:             blk.12.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:          blk.13.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:           blk.13.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:           blk.13.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:             blk.13.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:           blk.13.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:             blk.13.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:        blk.13.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:             blk.13.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:             blk.13.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:          blk.14.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:           blk.14.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:           blk.14.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:             blk.14.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:           blk.14.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:             blk.14.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:        blk.14.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:             blk.14.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:             blk.14.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:          blk.15.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:           blk.15.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:           blk.15.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:             blk.15.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:           blk.15.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:             blk.15.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:        blk.15.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:             blk.15.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:             blk.15.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:          blk.16.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:           blk.16.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:           blk.16.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:             blk.16.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:           blk.16.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:             blk.16.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:        blk.16.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:             blk.16.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:             blk.16.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:          blk.17.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:           blk.17.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:           blk.17.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:             blk.17.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:           blk.17.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:             blk.17.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:        blk.17.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:             blk.17.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:             blk.17.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:          blk.18.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:           blk.18.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:           blk.18.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.18.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:           blk.18.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:             blk.18.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:        blk.18.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:             blk.18.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.18.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.19.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.19.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:           blk.19.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.19.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:           blk.19.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:             blk.19.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:        blk.19.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:             blk.19.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.19.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:           blk.2.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:            blk.2.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:            blk.2.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:              blk.2.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:            blk.2.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:              blk.2.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:         blk.2.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:              blk.2.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:              blk.2.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.20.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.20.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:           blk.20.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.20.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:           blk.20.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:             blk.20.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:        blk.20.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:             blk.20.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.20.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.21.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.21.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:           blk.21.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.21.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:           blk.21.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:             blk.21.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:        blk.21.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:             blk.21.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.21.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.22.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.22.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:           blk.22.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.22.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:           blk.22.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:             blk.22.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:        blk.22.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:             blk.22.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.22.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.23.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.23.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:           blk.23.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.23.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:           blk.23.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:             blk.23.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:        blk.23.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:             blk.23.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.23.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:           blk.3.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:            blk.3.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:            blk.3.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:              blk.3.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:            blk.3.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:              blk.3.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:         blk.3.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:              blk.3.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:              blk.3.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:           blk.4.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:            blk.4.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:            blk.4.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:              blk.4.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:            blk.4.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:              blk.4.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:         blk.4.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:              blk.4.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:              blk.4.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:           blk.5.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:            blk.5.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:            blk.5.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:              blk.5.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:            blk.5.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:              blk.5.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:         blk.5.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:              blk.5.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:              blk.5.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:           blk.6.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:            blk.6.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:            blk.6.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:              blk.6.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:            blk.6.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:              blk.6.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:         blk.6.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:              blk.6.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:              blk.6.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:           blk.7.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:            blk.7.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:            blk.7.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:              blk.7.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:            blk.7.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:              blk.7.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:         blk.7.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:              blk.7.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:              blk.7.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:           blk.8.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:            blk.8.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:            blk.8.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:              blk.8.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:            blk.8.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:              blk.8.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:         blk.8.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:              blk.8.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:              blk.8.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:           blk.9.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:            blk.9.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:            blk.9.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:              blk.9.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:            blk.9.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:              blk.9.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:         blk.9.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:              blk.9.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:              blk.9.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:                    output.weight q6_K     [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:          blk.24.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:           blk.24.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:           blk.24.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:           blk.24.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:             blk.24.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:        blk.24.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:             blk.24.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:          blk.25.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:           blk.25.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:           blk.25.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:           blk.25.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:             blk.25.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:        blk.25.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:             blk.25.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:          blk.26.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:           blk.26.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:           blk.26.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:           blk.26.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:             blk.26.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:        blk.26.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:             blk.26.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:          blk.27.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:           blk.27.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:           blk.27.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:           blk.27.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:             blk.27.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:        blk.27.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:             blk.27.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:          blk.28.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:           blk.28.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:           blk.28.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:           blk.28.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:             blk.28.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:        blk.28.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:             blk.28.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:          blk.29.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:           blk.29.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:           blk.29.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:           blk.29.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:             blk.29.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:        blk.29.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:             blk.29.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:          blk.30.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:           blk.30.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:           blk.30.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:           blk.30.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:             blk.30.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:        blk.30.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:             blk.30.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:          blk.31.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:           blk.31.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:           blk.31.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:           blk.31.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:             blk.31.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:        blk.31.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:             blk.31.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:               output_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - kv   0:                       general.architecture str     \n",
      "llama_model_loader: - kv   1:                               general.name str     \n",
      "llama_model_loader: - kv   2:                       llama.context_length u32     \n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32     \n",
      "llama_model_loader: - kv   4:                          llama.block_count u32     \n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32     \n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32     \n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32     \n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32     \n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32     \n",
      "llama_model_loader: - kv  10:                          general.file_type u32     \n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str     \n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr     \n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr     \n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr     \n",
      "llama_model_loader: - kv  15:                tokenizer.ggml.bos_token_id u32     \n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.eos_token_id u32     \n",
      "llama_model_loader: - kv  17:            tokenizer.ggml.unknown_token_id u32     \n",
      "llama_model_loader: - kv  18:               general.quantization_version u32     \n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type q5_K:  193 tensors\n",
      "llama_model_loader: - type q6_K:   33 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = mostly Q5_K - Medium\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 4.45 GiB (5.68 BPW) \n",
      "llm_load_print_meta: general.name   = LLaMA v2\n",
      "llm_load_print_meta: BOS token = 1 '<s>'\n",
      "llm_load_print_meta: EOS token = 2 '</s>'\n",
      "llm_load_print_meta: UNK token = 0 '<unk>'\n",
      "llm_load_print_meta: LF token  = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size =    0.10 MB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: mem required  = 4560.96 MB\n",
      "llm_load_tensors: offloading 0 repeating layers to GPU\n",
      "llm_load_tensors: offloaded 0/35 layers to GPU\n",
      "llm_load_tensors: VRAM used: 0.00 MB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_new_context_with_model: kv self size  =  256.00 MB\n",
      "llama_new_context_with_model: compute buffer total size = 76.63 MB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 70.50 MB\n",
      "llama_new_context_with_model: total VRAM used: 70.50 MB (model: 0.00 MB, context: 70.50 MB)\n",
      "\n",
      "system_info: n_threads = 24 / 48 | AVX = 1 | AVX2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, typical_p = 1.000, temp = 0.800\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 128, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33mWhat should I do if I want to stop dialysis?\u001b[0m\n",
      " hopefully, you will be able to make an informed decision about stopping dialysis with the help of your healthcare team. Here are some steps you can take:\n",
      "\n",
      "1. Discuss your options with your nephrologist or primary care physician: Your doctor can provide information on the potential risks and benefits of stopping dialysis, as well as any alternative treatments that may be available to you.\n",
      "2. Consider your quality of life: Stopping dialysis can have significant implications for your quality of life, including changes in your energy levels, ability to perform daily activities, and overall well\n",
      "llama_print_timings:        load time =   14420.20 ms\n",
      "llama_print_timings:      sample time =      28.35 ms /   128 runs   (    0.22 ms per token,  4514.67 tokens per second)\n",
      "llama_print_timings: prompt eval time =   17441.99 ms /    14 tokens ( 1245.86 ms per token,     0.80 tokens per second)\n",
      "llama_print_timings:        eval time = 2056328.81 ms /   127 runs   (16191.57 ms per token,     0.06 tokens per second)\n",
      "llama_print_timings:       total time = 2073847.38 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "model_list = [file for file in os.listdir(\"/scratch/megathon/quant/\") if GGML_VERSION in file]\n",
    "prompt = \"What should I do if I want to stop dialysis?\"\n",
    "chosen_method = \"q5_k_m\" #input(\"Please specify the quantization method to run the model (options: \" + \", \".join(model_list) + \"): \")\n",
    "\n",
    "qtype = f\"/scratch/megathon/quant/metaquant.{GGML_VERSION}.{chosen_method}.bin\"\n",
    "!/scratch/megathon/quant/llama.cpp/main -m {qtype} -n 128 --repeat_penalty 1.1 --color -p \"{prompt}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['TRANSFORMERS_CACHE'] = '/scratch/megathon/cache/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.document_loaders import TextLoader\n",
    "\n",
    "loader = TextLoader(\"train_webmd_squad_v2_consec.txt\")\n",
    "documents = loader.load()\n",
    "\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "\n",
    "text_splitter = RecursiveCharacterTextSplitter(chunk_size=200, chunk_overlap=10)\n",
    "all_splits = text_splitter.split_documents(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.embeddings import HuggingFaceEmbeddings\n",
    "from langchain.vectorstores import FAISS\n",
    "\n",
    "model_name = \"sentence-transformers/all-mpnet-base-v2\"\n",
    "model_kwargs = {\"device\": \"cuda\"}\n",
    "\n",
    "embeddings = HuggingFaceEmbeddings(model_name=model_name, model_kwargs=model_kwargs)\n",
    "\n",
    "# storing embeddings in the vector store\n",
    "vectorstore = FAISS.from_documents(all_splits, embeddings)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "query = \"What should I do if I want to stop dialysis?\"\n",
    "docs = vectorstore.similarity_search_with_score(query)\n",
    "\n",
    "context = \"\"\n",
    "for doc in docs:\n",
    "    context += doc[0].page_content + \" \""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "197 0.6995704\n",
      "197 0.7874575\n",
      "195 0.85200596\n",
      "195 0.87893635\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(docs)):\n",
    "    print(len(docs[i][0].page_content), docs[i][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"kidneys are failing, you might choose the hospice program rather than continuing with dialysis. But you can still change your mind, stop hospice care, and start back on treatments. Other people may kidneys and keep them working as long as you can. Follow your doctor's advice carefully. You can also keep up these habits to stay well: Eat right. Stick to a healthy, well-balanced diet that's low Pain medicines If your kidneys fail, you'll need dialysis, which uses a machine to filter your blood and remove waste, like salt, extra water, and certain chemicals. You can also get on a waiting may eventually stop working. That's serious, and it can be life-threatening. Healthy kidneys: Keep a balance of water and minerals (such as sodium, potassium, and phosphorus) in your blood Remove \""
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1441 (ff3bad8)\n",
      "main: built with cc (Ubuntu 7.5.0-3ubuntu1~18.04) 7.5.0 for x86_64-linux-gnu\n",
      "main: seed  = 1698540553\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 1 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6\n",
      "llama_model_loader: loaded meta data with 19 key-value pairs and 291 tensors from /scratch/megathon/quant/metaquant.gguf.q5_k_m.bin (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight q5_K     [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:           blk.0.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:            blk.0.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:            blk.0.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:            blk.0.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:              blk.0.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:         blk.0.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:              blk.0.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.1.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.1.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:            blk.1.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:            blk.1.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:              blk.1.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:         blk.1.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:              blk.1.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:          blk.10.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:           blk.10.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:           blk.10.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:             blk.10.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:           blk.10.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:             blk.10.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:        blk.10.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:             blk.10.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:             blk.10.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:          blk.11.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:           blk.11.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:           blk.11.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:             blk.11.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:           blk.11.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:             blk.11.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:        blk.11.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:             blk.11.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:             blk.11.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:          blk.12.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:           blk.12.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:           blk.12.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:             blk.12.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:           blk.12.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:             blk.12.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:        blk.12.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:             blk.12.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:             blk.12.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:          blk.13.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:           blk.13.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:           blk.13.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:             blk.13.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:           blk.13.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:             blk.13.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:        blk.13.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:             blk.13.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:             blk.13.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:          blk.14.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:           blk.14.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:           blk.14.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:             blk.14.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:           blk.14.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:             blk.14.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:        blk.14.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:             blk.14.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:             blk.14.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:          blk.15.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:           blk.15.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:           blk.15.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:             blk.15.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:           blk.15.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:             blk.15.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:        blk.15.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:             blk.15.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:             blk.15.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:          blk.16.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:           blk.16.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:           blk.16.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:             blk.16.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:           blk.16.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:             blk.16.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:        blk.16.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:             blk.16.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:             blk.16.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:          blk.17.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:           blk.17.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:           blk.17.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:             blk.17.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:           blk.17.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:             blk.17.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:        blk.17.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:             blk.17.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:             blk.17.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:          blk.18.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:           blk.18.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:           blk.18.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.18.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:           blk.18.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:             blk.18.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:        blk.18.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:             blk.18.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.18.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.19.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.19.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:           blk.19.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.19.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:           blk.19.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:             blk.19.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:        blk.19.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:             blk.19.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.19.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:           blk.2.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:            blk.2.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:            blk.2.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:              blk.2.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:            blk.2.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:              blk.2.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:         blk.2.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:              blk.2.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:              blk.2.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.20.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.20.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:           blk.20.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.20.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:           blk.20.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:             blk.20.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:        blk.20.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:             blk.20.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.20.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.21.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.21.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:           blk.21.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.21.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:           blk.21.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:             blk.21.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:        blk.21.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:             blk.21.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.21.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.22.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.22.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:           blk.22.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.22.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:           blk.22.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:             blk.22.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:        blk.22.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:             blk.22.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.22.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.23.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.23.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:           blk.23.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.23.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:           blk.23.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:             blk.23.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:        blk.23.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:             blk.23.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.23.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:           blk.3.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:            blk.3.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:            blk.3.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:              blk.3.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:            blk.3.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:              blk.3.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:         blk.3.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:              blk.3.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:              blk.3.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:           blk.4.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:            blk.4.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:            blk.4.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:              blk.4.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:            blk.4.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:              blk.4.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:         blk.4.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:              blk.4.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:              blk.4.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:           blk.5.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:            blk.5.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:            blk.5.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:              blk.5.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:            blk.5.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:              blk.5.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:         blk.5.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:              blk.5.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:              blk.5.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:           blk.6.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:            blk.6.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:            blk.6.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:              blk.6.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:            blk.6.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:              blk.6.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:         blk.6.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:              blk.6.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:              blk.6.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:           blk.7.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:            blk.7.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:            blk.7.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:              blk.7.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:            blk.7.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:              blk.7.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:         blk.7.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:              blk.7.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:              blk.7.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:           blk.8.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:            blk.8.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:            blk.8.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:              blk.8.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:            blk.8.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:              blk.8.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:         blk.8.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:              blk.8.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:              blk.8.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:           blk.9.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:            blk.9.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:            blk.9.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:              blk.9.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:            blk.9.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:              blk.9.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:         blk.9.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:              blk.9.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:              blk.9.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:                    output.weight q6_K     [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:          blk.24.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:           blk.24.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:           blk.24.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:           blk.24.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:             blk.24.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:        blk.24.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:             blk.24.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:          blk.25.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:           blk.25.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:           blk.25.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:           blk.25.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:             blk.25.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:        blk.25.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:             blk.25.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:          blk.26.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:           blk.26.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:           blk.26.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:           blk.26.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:             blk.26.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:        blk.26.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:             blk.26.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:          blk.27.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:           blk.27.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:           blk.27.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:           blk.27.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:             blk.27.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:        blk.27.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:             blk.27.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:          blk.28.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:           blk.28.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:           blk.28.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:           blk.28.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:             blk.28.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:        blk.28.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:             blk.28.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:          blk.29.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:           blk.29.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:           blk.29.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:           blk.29.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:             blk.29.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:        blk.29.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:             blk.29.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:          blk.30.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:           blk.30.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:           blk.30.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:           blk.30.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:             blk.30.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:        blk.30.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:             blk.30.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:          blk.31.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:           blk.31.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:           blk.31.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:           blk.31.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:             blk.31.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:        blk.31.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:             blk.31.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:               output_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - kv   0:                       general.architecture str     \n",
      "llama_model_loader: - kv   1:                               general.name str     \n",
      "llama_model_loader: - kv   2:                       llama.context_length u32     \n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32     \n",
      "llama_model_loader: - kv   4:                          llama.block_count u32     \n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32     \n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32     \n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32     \n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32     \n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32     \n",
      "llama_model_loader: - kv  10:                          general.file_type u32     \n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str     \n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr     \n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr     \n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr     \n",
      "llama_model_loader: - kv  15:                tokenizer.ggml.bos_token_id u32     \n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.eos_token_id u32     \n",
      "llama_model_loader: - kv  17:            tokenizer.ggml.unknown_token_id u32     \n",
      "llama_model_loader: - kv  18:               general.quantization_version u32     \n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type q5_K:  193 tensors\n",
      "llama_model_loader: - type q6_K:   33 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = mostly Q5_K - Medium\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 4.45 GiB (5.68 BPW) \n",
      "llm_load_print_meta: general.name   = LLaMA v2\n",
      "llm_load_print_meta: BOS token = 1 '<s>'\n",
      "llm_load_print_meta: EOS token = 2 '</s>'\n",
      "llm_load_print_meta: UNK token = 0 '<unk>'\n",
      "llm_load_print_meta: LF token  = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size =    0.10 MB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: mem required  =   86.04 MB\n",
      "llm_load_tensors: offloading 32 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 35/35 layers to GPU\n",
      "llm_load_tensors: VRAM used: 4474.93 MB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: offloading v cache to GPU\n",
      "llama_kv_cache_init: offloading k cache to GPU\n",
      "llama_kv_cache_init: VRAM kv self = 256.00 MB\n",
      "llama_new_context_with_model: kv self size  =  256.00 MB\n",
      "llama_new_context_with_model: compute buffer total size = 76.63 MB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 70.50 MB\n",
      "llama_new_context_with_model: total VRAM used: 4801.43 MB (model: 4474.93 MB, context: 326.50 MB)\n",
      "\n",
      "system_info: n_threads = 24 / 48 | AVX = 1 | AVX2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, typical_p = 1.000, temp = 0.800\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 4096, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33mWhat should I do if I want to stop dialysis?\u001b[0m\n",
      "There are some things you can consider when deciding whether or not to continue dialysis. Here are a few:\n",
      "1. Discuss your options with your healthcare provider: It's important to talk to your doctor or nephrologist about your feelings and concerns about continuing dialysis. They can help you understand the potential risks and benefits of stopping treatment and provide guidance on how to make an informed decision.\n",
      "2. Consider your overall health: If you have other serious health problems, such as heart disease, lung disease, or dementia, it may be more difficult for your body to handle the changes that come with stopping dialysis. In these cases, it's important to weigh the potential risks and benefits of continuing treatment against the potential impact of stopping it.\n",
      "3. Think about your quality of life: Dialysis can be a significant burden, both physically and emotionally. If you feel that the benefits of dialysis no longer outweigh the burdens, it may be worth considering stopping treatment. However, it's important to make sure you have a good quality of life before making this decision.\n",
      "4. Explore alternative treatments: There are several alternative treatments for kidney failure that may be an option if you decide to stop dialysis. These include transplantation and conservative management, which involves monitoring your condition closely without undergoing dialysis.\n",
      "5. Seek support from family and friends: It can be helpful to talk to loved ones about your decision and get their perspective on the situation. They may be able to provide emotional support and help you make an informed decision.\n",
      "6. Consider palliative care: Palliative care is a type of care that focuses on relieving symptoms and improving quality of life for people with serious illnesses, including kidney failure. If you decide to stop dialysis, your healthcare provider may recommend palliative care to help manage any symptoms or complications that arise.\n",
      "7. Consider hospice care: Hospice care is a type of care that focuses on providing comfort and support to people who are nearing the end of life. If you decide to stop dialysis, your healthcare provider may recommend hospice care to help manage any symptoms or complications that arise.\n",
      "8. Seek legal and financial advice: Stopping dialysis can have significant legal and financial implications, including changes to insurance coverage and potential costs associated with alternative treatments. It's important to seek advice from a lawyer or financial advisor before making a decision.\n",
      "9. Consider the emotional impact: Stopping dialysis can be a difficult decision, both emotionally and physically. It's important to consider the emotional impact on yourself and your loved ones before making a decision.\n",
      "10. Seek professional help: If you're struggling with the decision to stop dialysis, it may be helpful to speak with a mental health professional who can provide emotional support and guidance.\n",
      "\n",
      "It's important to note that stopping dialysis is a complex decision that should be made after careful consideration of all the available options. It's important to work closely with your healthcare provider and loved ones to make an informed decision that aligns with your values and goals. [end of text]\n",
      "\n",
      "llama_print_timings:        load time =    3506.72 ms\n",
      "llama_print_timings:      sample time =     150.01 ms /   701 runs   (    0.21 ms per token,  4672.99 tokens per second)\n",
      "llama_print_timings: prompt eval time =      45.58 ms /    14 tokens (    3.26 ms per token,   307.19 tokens per second)\n",
      "llama_print_timings:        eval time =    6674.42 ms /   700 runs   (    9.53 ms per token,   104.88 tokens per second)\n",
      "llama_print_timings:       total time =    7037.90 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "# model_list = [file for file in os.listdir(\"/scratch/megathon/quant/\") if GGML_VERSION in file]\n",
    "prompt = \"What should I do if I want to stop dialysis?\"\n",
    "chosen_method = \"q5_k_m\" #input(\"Please specify the quantization method to run the model (options: \" + \", \".join(model_list) + \"): \")\n",
    "\n",
    "qtype = f\"/scratch/megathon/quant/metaquant.gguf.{q5_k_m}.bin\"\n",
    "!/scratch/megathon/quant/llama.cpp/main -m {qtype} -n 4096 --repeat_penalty 1.1 --color -ngl 35 -p \"{prompt}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1441 (ff3bad8)\n",
      "main: built with cc (Ubuntu 7.5.0-3ubuntu1~18.04) 7.5.0 for x86_64-linux-gnu\n",
      "main: seed  = 1698540614\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 1 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "llama_model_loader: loaded meta data with 19 key-value pairs and 291 tensors from /scratch/megathon/quant/metaquant.gguf.q5_k_m.bin (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight q5_K     [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:           blk.0.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:            blk.0.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:            blk.0.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:            blk.0.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:              blk.0.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:         blk.0.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:              blk.0.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.1.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.1.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:            blk.1.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:            blk.1.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:              blk.1.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:         blk.1.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:              blk.1.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:          blk.10.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:           blk.10.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:           blk.10.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:             blk.10.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:           blk.10.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:             blk.10.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:        blk.10.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:             blk.10.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:             blk.10.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:          blk.11.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:           blk.11.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:           blk.11.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:             blk.11.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:           blk.11.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:             blk.11.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:        blk.11.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:             blk.11.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:             blk.11.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:          blk.12.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:           blk.12.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:           blk.12.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:             blk.12.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:           blk.12.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:             blk.12.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:        blk.12.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:             blk.12.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:             blk.12.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:          blk.13.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:           blk.13.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:           blk.13.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:             blk.13.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:           blk.13.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:             blk.13.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:        blk.13.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:             blk.13.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:             blk.13.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:          blk.14.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:           blk.14.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:           blk.14.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:             blk.14.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:           blk.14.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:             blk.14.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:        blk.14.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:             blk.14.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:             blk.14.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:          blk.15.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:           blk.15.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:           blk.15.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:             blk.15.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:           blk.15.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:             blk.15.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:        blk.15.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:             blk.15.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:             blk.15.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:          blk.16.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:           blk.16.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:           blk.16.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:             blk.16.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:           blk.16.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:             blk.16.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:        blk.16.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:             blk.16.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:             blk.16.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:          blk.17.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:           blk.17.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:           blk.17.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:             blk.17.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:           blk.17.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:             blk.17.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:        blk.17.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:             blk.17.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:             blk.17.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:          blk.18.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:           blk.18.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:           blk.18.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.18.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:           blk.18.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:             blk.18.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:        blk.18.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:             blk.18.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.18.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.19.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.19.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:           blk.19.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.19.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:           blk.19.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:             blk.19.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:        blk.19.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:             blk.19.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.19.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:           blk.2.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:            blk.2.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:            blk.2.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:              blk.2.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:            blk.2.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:              blk.2.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:         blk.2.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:              blk.2.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:              blk.2.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.20.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.20.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:           blk.20.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.20.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:           blk.20.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:             blk.20.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:        blk.20.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:             blk.20.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.20.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.21.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.21.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:           blk.21.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.21.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:           blk.21.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:             blk.21.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:        blk.21.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:             blk.21.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.21.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.22.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.22.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:           blk.22.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.22.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:           blk.22.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:             blk.22.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:        blk.22.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:             blk.22.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.22.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.23.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.23.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:           blk.23.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.23.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:           blk.23.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:             blk.23.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:        blk.23.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:             blk.23.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.23.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:           blk.3.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:            blk.3.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:            blk.3.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:              blk.3.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:            blk.3.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:              blk.3.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:         blk.3.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:              blk.3.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:              blk.3.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:           blk.4.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:            blk.4.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:            blk.4.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:              blk.4.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:            blk.4.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:              blk.4.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:         blk.4.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:              blk.4.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:              blk.4.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:           blk.5.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:            blk.5.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:            blk.5.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:              blk.5.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:            blk.5.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:              blk.5.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:         blk.5.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:              blk.5.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:              blk.5.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:           blk.6.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:            blk.6.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:            blk.6.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:              blk.6.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:            blk.6.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:              blk.6.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:         blk.6.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:              blk.6.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:              blk.6.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:           blk.7.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:            blk.7.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:            blk.7.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:              blk.7.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:            blk.7.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:              blk.7.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:         blk.7.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:              blk.7.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:              blk.7.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:           blk.8.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:            blk.8.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:            blk.8.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:              blk.8.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:            blk.8.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:              blk.8.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:         blk.8.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:              blk.8.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:              blk.8.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:           blk.9.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:            blk.9.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:            blk.9.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:              blk.9.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:            blk.9.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:              blk.9.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:         blk.9.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:              blk.9.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:              blk.9.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:                    output.weight q6_K     [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:          blk.24.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:           blk.24.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:           blk.24.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:           blk.24.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:             blk.24.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:        blk.24.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:             blk.24.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:          blk.25.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:           blk.25.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:           blk.25.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:           blk.25.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:             blk.25.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:        blk.25.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:             blk.25.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:          blk.26.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:           blk.26.ffn_down.weight q5_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:           blk.26.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:           blk.26.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:             blk.26.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:        blk.26.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:             blk.26.attn_v.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:          blk.27.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:           blk.27.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:           blk.27.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:           blk.27.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:             blk.27.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:        blk.27.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:             blk.27.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:          blk.28.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:           blk.28.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:           blk.28.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:           blk.28.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:             blk.28.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:        blk.28.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:             blk.28.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:          blk.29.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:           blk.29.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:           blk.29.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:           blk.29.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:             blk.29.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:        blk.29.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:             blk.29.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:          blk.30.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:           blk.30.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:           blk.30.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:           blk.30.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:             blk.30.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:        blk.30.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:             blk.30.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:          blk.31.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:           blk.31.ffn_down.weight q6_K     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:           blk.31.ffn_gate.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.ffn_up.weight q5_K     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:           blk.31.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:             blk.31.attn_k.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:        blk.31.attn_output.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.attn_q.weight q5_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:             blk.31.attn_v.weight q6_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:               output_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - kv   0:                       general.architecture str     \n",
      "llama_model_loader: - kv   1:                               general.name str     \n",
      "llama_model_loader: - kv   2:                       llama.context_length u32     \n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32     \n",
      "llama_model_loader: - kv   4:                          llama.block_count u32     \n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32     \n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32     \n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32     \n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32     \n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32     \n",
      "llama_model_loader: - kv  10:                          general.file_type u32     \n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str     \n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr     \n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr     \n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr     \n",
      "llama_model_loader: - kv  15:                tokenizer.ggml.bos_token_id u32     \n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.eos_token_id u32     \n",
      "llama_model_loader: - kv  17:            tokenizer.ggml.unknown_token_id u32     \n",
      "llama_model_loader: - kv  18:               general.quantization_version u32     \n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type q5_K:  193 tensors\n",
      "llama_model_loader: - type q6_K:   33 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = mostly Q5_K - Medium\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 4.45 GiB (5.68 BPW) \n",
      "llm_load_print_meta: general.name   = LLaMA v2\n",
      "llm_load_print_meta: BOS token = 1 '<s>'\n",
      "llm_load_print_meta: EOS token = 2 '</s>'\n",
      "llm_load_print_meta: UNK token = 0 '<unk>'\n",
      "llm_load_print_meta: LF token  = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size =    0.10 MB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: mem required  =   86.04 MB\n",
      "llm_load_tensors: offloading 32 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 35/35 layers to GPU\n",
      "llm_load_tensors: VRAM used: 4474.93 MB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: offloading v cache to GPU\n",
      "llama_kv_cache_init: offloading k cache to GPU\n",
      "llama_kv_cache_init: VRAM kv self = 256.00 MB\n",
      "llama_new_context_with_model: kv self size  =  256.00 MB\n",
      "llama_new_context_with_model: compute buffer total size = 76.63 MB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 70.50 MB\n",
      "llama_new_context_with_model: total VRAM used: 4801.43 MB (model: 4474.93 MB, context: 326.50 MB)\n",
      "\n",
      "system_info: n_threads = 24 / 48 | AVX = 1 | AVX2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, typical_p = 1.000, temp = 0.800\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 4096, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33mkidneys are failing, you might choose the hospice program rather than continuing with dialysis. But you can still change your mind, stop hospice care, and start back on treatments. Other people may kidneys and keep them working as long as you can. Follow your doctor's advice carefully. You can also keep up these habits to stay well: Eat right. Stick to a healthy, well-balanced diet that's low Pain medicines If your kidneys fail, you'll need dialysis, which uses a machine to filter your blood and remove waste, like salt, extra water, and certain chemicals. You can also get on a waiting may eventually stop working. That's serious, and it can be life-threatening. Healthy kidneys: Keep a balance of water and minerals (such as sodium, potassium, and phosphorus) in your blood Remove  \n",
      "\n",
      " Question: What should I do if I want to stop dialysis? \n",
      "\n",
      " Answer:\u001b[0m If you are on dialysis and you decide that you no longer want to continue treatment, it is important to speak with your doctor or healthcare provider as soon as possible. They can help you understand the potential risks and consequences of stopping dialysis, and they can also provide support and guidance throughout the process. It is important to note that stopping dialysis without proper medical supervision and care can be dangerous and may lead to serious complications, such as a buildup of waste products in the body or fluid overload. Therefore, it is crucial to work closely with your healthcare provider to ensure a safe and smooth transition off of dialysis.\n",
      "\n",
      "Here are some steps you can take if you want to stop dialysis:\n",
      "\n",
      "1. Discuss your decision with your doctor: It's important to have an open and honest conversation with your doctor about your decision to stop dialysis. They can help you understand the potential risks and consequences of stopping treatment and provide guidance on how to proceed.\n",
      "2. Get a thorough medical evaluation: Before stopping dialysis, it is important to get a thorough medical evaluation to ensure that you are in good overall health and that your body is functioning properly. This may involve blood tests, imaging studies, and other diagnostic tests.\n",
      "3. Consider hospice care: If you have advanced kidney disease or another serious illness, hospice care may be an option. Hospice care provides a range of services, including medical, emotional, and spiritual support, to help you manage your symptoms and improve your quality of life.\n",
      "4. Make necessary arrangements: If you decide to stop dialysis, you will need to make arrangements for ongoing medical care and support. This may involve working with a home healthcare agency or hospice provider, as well as making sure that you have access to necessary medications and supplies.\n",
      "5. Emotional support: Stopping dialysis can be an emotionally challenging experience, so it's important to have a strong support system in place. This may involve talking to family members, friends, or a mental health professional, as well as joining a support group for people with kidney disease.\n",
      "\n",
      "Overall, stopping dialysis without proper medical supervision and care can be dangerous and may lead to serious complications. Therefore, it is crucial to work closely with your healthcare provider to ensure a safe and smooth transition off of dialysis. [end of text]\n",
      "\n",
      "llama_print_timings:        load time =     762.00 ms\n",
      "llama_print_timings:      sample time =     109.65 ms /   519 runs   (    0.21 ms per token,  4733.24 tokens per second)\n",
      "llama_print_timings: prompt eval time =      79.78 ms /   228 tokens (    0.35 ms per token,  2857.79 tokens per second)\n",
      "llama_print_timings:        eval time =    4979.13 ms /   518 runs   (    9.61 ms per token,   104.03 tokens per second)\n",
      "llama_print_timings:       total time =    5298.97 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "# model_list = [file for file in os.listdir(\"/scratch/megathon/quant/\") if GGML_VERSION in file]\n",
    "prompt = f\"{context} \\n\\n Question: What should I do if I want to stop dialysis? \\n\\n Answer:\"\n",
    "chosen_method = \"q5_k_m\" #input(\"Please specify the quantization method to run the model (options: \" + \", \".join(model_list) + \"): \")\n",
    "\n",
    "qtype = f\"/scratch/megathon/quant/metaquant.gguf.{chosen_method}.bin\"\n",
    "!/scratch/megathon/quant/llama.cpp/main -m {qtype} -n 4096 --repeat_penalty 1.1 --color -ngl 35 -p \"{prompt}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "meg",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}