Vllm version update 29 04 25

refact-server/Dockerfile.base

@@ -28,13 +28,10 @@ RUN git clone https://github.com/smallcloudai/linguist.git /tmp/linguist \
     && rake build_gem
 ENV PATH="${PATH}:/tmp/linguist/bin"
 
-RUN pip install --no-cache-dir torch==2.5.0
-RUN pip install --no-cache-dir xformers==v0.0.28.post2
+RUN pip install --no-cache-dir torch==2.6.0
+RUN pip install --no-cache-dir xformers==0.0.29.post2
 RUN pip install ninja
 RUN pip install packaging==24.1 setuptools==70.0.0 setuptools-scm==8.1.0
+# new cuda caps require cuda version >=12.8: 10.0, 10.1, 12.0
 ENV CMAKE_ARGS="-DLLAMA_CUBLAS=on -DCMAKE_CUDA_ARCHITECTURES=60;61;70;75;80;86;89;90+PTX"
-RUN pip install -v --no-build-isolation git+https://github.com/smallcloudai/vllm@refact_v0.7.3
-
-# there is no prebuild auto-gptq with torch 2.5.0 support
-ENV TORCH_CUDA_ARCH_LIST="6.0;6.1;7.0;7.5;8.0;8.6;8.9;9.0+PTX"
-RUN BUILD_CUDA_EXT=1 pip install -v --no-build-isolation git+https://github.com/PanQiWei/AutoGPTQ.git@v0.7.1
+RUN pip install -v --no-build-isolation vllm==v0.8.5

refact-server/refact_known_models/__init__.py

@@ -1,5 +1,4 @@
-from refact_known_models.refact import refact_mini_db
 from refact_known_models.huggingface import huggingface_mini_db
 from refact_known_models.passthrough import passthrough_mini_db
 
-models_mini_db = {**refact_mini_db, **huggingface_mini_db}
+models_mini_db = huggingface_mini_db

refact-server/refact_known_models/huggingface.py

@@ -1,19 +1,3 @@
-starcoder_supports_scratchpads = {
-    "FIM-PSM": {
-        "context_format": "starcoder",
-        "rag_ratio": 0.5,
-    },
-}
-
-deepseek_coder_supports_scratchpads = {
-    "FIM-PSM": {
-        "fim_prefix": "<｜fim▁begin｜>",
-        "fim_suffix": "<｜fim▁hole｜>",
-        "fim_middle": "<｜fim▁end｜>",
-        "eot": "<|EOT|>",
-    },
-}
-
 llama_supports_scratchpads = {
     "REPLACE": {
         "token_bos": "<|begin_of_text|>",
@@ -81,66 +65,6 @@
 }
 
 huggingface_mini_db = {
-    # starcoder2
-    "starcoder2/3b/base": {
-        "backend": "transformers",
-        "model_path": "bigcode/starcoder2-3b",
-        "model_class_kwargs": {},
-        "required_memory_mb": 8000,
-        "T": 4096,
-        "supports_scratchpads": {
-            "completion": starcoder_supports_scratchpads,
-        },
-        "deprecated": True,
-        "filter_caps": ["completion", "finetune"],
-    },
-    "starcoder2/7b/base": {
-        "backend": "transformers",
-        "model_path": "bigcode/starcoder2-7b",
-        "model_class_kwargs": {},
-        "required_memory_mb": 16000,
-        "T": 2048,
-        "supports_scratchpads": {
-            "completion": starcoder_supports_scratchpads,
-        },
-        "deprecated": True,
-        "filter_caps": ["completion", "finetune"],
-    },
-    "starcoder2/15b/base": {
-        "backend": "transformers",
-        "model_path": "bigcode/starcoder2-15b",
-        "model_class_kwargs": {},
-        "required_memory_mb": 20000,
-        "T": 4096,
-        "supports_scratchpads": {
-            "completion": starcoder_supports_scratchpads,
-        },
-        "deprecated": True,
-        "filter_caps": ["completion", "finetune"],
-    },
-    # deepseek-coder
-    "deepseek-coder/1.3b/base": {
-        "backend": "transformers",
-        "model_path": "deepseek-ai/deepseek-coder-1.3b-base",
-        "model_class_kwargs": {},
-        "T": 4096,
-        "supports_scratchpads": {
-            "completion": deepseek_coder_supports_scratchpads,
-        },
-        "deprecated": True,
-        "filter_caps": ["completion", "finetune"],
-    },
-    "deepseek-coder/5.7b/mqa-base": {
-        "backend": "transformers",
-        "model_path": "deepseek-ai/deepseek-coder-5.7bmqa-base",
-        "model_class_kwargs": {},
-        "T": 4096,
-        "supports_scratchpads": {
-            "completion": deepseek_coder_supports_scratchpads,
-        },
-        "deprecated": True,
-        "filter_caps": ["completion", "finetune"],
-    },
     # llama
     "llama3.1/8b/instruct": {
         "backend": "transformers",

refact-server/setup.py

@@ -36,17 +36,17 @@ class PyPackage:
         data=["git_command.exp"],
     ),
     "refact_webgui": PyPackage(
-        requires=["cryptography", "fastapi>=0.111.0", "giturlparse",
+        requires=["cryptography", "fastapi>=0.115.2", "giturlparse",
                   "starlette==0.37.2", "uvicorn", "uvloop", "termcolor", "python-multipart", "more_itertools",
                   "scyllapy==1.3.0", "pandas>=2.0.3"],
         requires_packages=["refact_known_models", "refact_utils"],
         data=["webgui/static/*", "webgui/static/components/modals/*",
               "webgui/static/dashboards/*", "webgui/static/assets/*", "webgui/static/utils/*",
               "webgui/static/assets/fonts/*"]),
     "self_hosting_machinery": PyPackage(
-        requires=["python-multipart", "auto-gptq==0.7.1", "accelerate",
-                  "termcolor", "torch", "transformers==4.47.1",  # Qwen2 is completely changed in transformers>=4.48
-                  "bitsandbytes", "safetensors", "peft", "triton",
+        requires=["python-multipart", "accelerate",
+                  "termcolor", "torch", "transformers>=4.51.0",
+                  "bitsandbytes>=0.45.5", "safetensors", "peft", "triton",
                   "torchinfo", "mpi4py", "deepspeed>=0.15.3",
                   "sentence-transformers>=3.4.1", "huggingface-hub>=0.29.2",
                   "aiohttp", "setproctitle", "google-auth>=2.37.0"],
@@ -95,7 +95,7 @@ def get_install_requires(packages):
 
 setup(
     name="refact-self-hosting",
-    version="1.10.0",
+    version="1.11.0",
     py_modules=list(setup_packages.keys()),
     package_data={
         name: py_package.data