Skip to content

Commit 34ac71a

Browse files
sinkingsugarsinkingsugar
committed
Merge commit '7a84777f42a9b3ba47db5d20b7662f8ddf92f652' into shards-master
2 parents e814c86 + 7a84777 commit 34ac71a

File tree

187 files changed

+12162
-10490
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

187 files changed

+12162
-10490
lines changed

‎.github/workflows/build-linux-cross.yml‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
name: Build on Linux using cross-compiler
2+
on:
3+
workflow_dispatch:
4+
workflow_call:
5+
6+
jobs:
7+
ubuntu-latest-riscv64-cpu-cross:
8+
runs-on: ubuntu-latest
9+
10+
steps:
11+
- uses: actions/checkout@v4
12+
- name: Setup Riscv
13+
run: |
14+
sudo dpkg --add-architecture riscv64
15+
sudo sed -i 's|http://azure.archive.ubuntu.com/ubuntu|http://ports.ubuntu.com/ubuntu-ports|g' \
16+
/etc/apt/sources.list /etc/apt/apt-mirrors.txt
17+
sudo apt-get clean
18+
sudo apt-get update
19+
sudo apt-get install -y --no-install-recommends \
20+
build-essential \
21+
gcc-14-riscv64-linux-gnu \
22+
g++-14-riscv64-linux-gnu
23+
24+
- name: Build
25+
run: |
26+
cmake -B build -DCMAKE_BUILD_TYPE=Release \
27+
-DGGML_OPENMP=OFF \
28+
-DLLAMA_BUILD_EXAMPLES=ON \
29+
-DLLAMA_BUILD_TESTS=OFF \
30+
-DCMAKE_SYSTEM_NAME=Linux \
31+
-DCMAKE_SYSTEM_PROCESSOR=riscv64 \
32+
-DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
33+
-DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14 \
34+
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
35+
-DCMAKE_FIND_ROOT_PATH=/usr/lib/riscv64-linux-gnu \
36+
-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \
37+
-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \
38+
-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH
39+
40+
cmake --build build --config Release -j $(nproc)
41+
42+
ubuntu-latest-riscv64-vulkan-cross:
43+
runs-on: ubuntu-latest
44+
45+
steps:
46+
- uses: actions/checkout@v4
47+
with:
48+
fetch-depth: 0
49+
50+
- name: Setup Riscv
51+
run: |
52+
sudo dpkg --add-architecture riscv64
53+
sudo sed -i 's|http://azure.archive.ubuntu.com/ubuntu|http://ports.ubuntu.com/ubuntu-ports|g' \
54+
/etc/apt/sources.list /etc/apt/apt-mirrors.txt
55+
sudo apt-get clean
56+
sudo apt-get update
57+
sudo apt-get install -y --no-install-recommends \
58+
build-essential \
59+
glslc \
60+
gcc-14-riscv64-linux-gnu \
61+
g++-14-riscv64-linux-gnu \
62+
libvulkan-dev:riscv64
63+
64+
- name: Build
65+
run: |
66+
cmake -B build -DCMAKE_BUILD_TYPE=Release \
67+
-DGGML_VULKAN=ON \
68+
-DGGML_OPENMP=OFF \
69+
-DLLAMA_BUILD_EXAMPLES=ON \
70+
-DLLAMA_BUILD_TESTS=OFF \
71+
-DCMAKE_SYSTEM_NAME=Linux \
72+
-DCMAKE_SYSTEM_PROCESSOR=riscv64 \
73+
-DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
74+
-DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14 \
75+
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
76+
-DCMAKE_FIND_ROOT_PATH=/usr/lib/riscv64-linux-gnu \
77+
-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \
78+
-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \
79+
-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH
80+
81+
cmake --build build --config Release -j $(nproc)
82+
83+
ubuntu-latest-arm64-vulkan-cross:
84+
runs-on: ubuntu-latest
85+
86+
steps:
87+
- uses: actions/checkout@v4
88+
with:
89+
fetch-depth: 0
90+
91+
- name: Setup Arm64
92+
run: |
93+
sudo dpkg --add-architecture arm64
94+
sudo sed -i 's|http://azure.archive.ubuntu.com/ubuntu|http://ports.ubuntu.com/ubuntu-ports|g' \
95+
/etc/apt/sources.list /etc/apt/apt-mirrors.txt
96+
sudo apt-get clean
97+
sudo apt-get update
98+
sudo apt-get install -y --no-install-recommends \
99+
build-essential \
100+
glslc \
101+
crossbuild-essential-arm64 \
102+
libvulkan-dev:arm64
103+
104+
- name: Build
105+
run: |
106+
cmake -B build -DCMAKE_BUILD_TYPE=Release \
107+
-DGGML_VULKAN=ON \
108+
-DGGML_OPENMP=OFF \
109+
-DLLAMA_BUILD_EXAMPLES=ON \
110+
-DLLAMA_BUILD_TESTS=OFF \
111+
-DCMAKE_SYSTEM_NAME=Linux \
112+
-DCMAKE_SYSTEM_PROCESSOR=aarch64 \
113+
-DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc \
114+
-DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++ \
115+
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
116+
-DCMAKE_FIND_ROOT_PATH=/usr/lib/aarch64-linux-gnu \
117+
-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \
118+
-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \
119+
-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH
120+
121+
cmake --build build --config Release -j $(nproc)

‎.github/workflows/build.yml‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ on:
1010
push:
1111
branches:
1212
- master
13-
paths: ['.github/workflows/build.yml', '**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.cuh', '**/*.swift', '**/*.m', '**/*.metal', '**/*.comp']
13+
paths: ['.github/workflows/build.yml', '.github/workflows/build-linux-cross.yml', '**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.cuh', '**/*.swift', '**/*.m', '**/*.metal', '**/*.comp']
1414
pull_request:
1515
types: [opened, synchronize, reopened]
1616
paths: ['.github/workflows/build.yml', '**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.cuh', '**/*.swift', '**/*.m', '**/*.metal', '**/*.comp']
@@ -606,6 +606,9 @@ jobs:
606606
-DGGML_SYCL_F16=ON
607607
cmake --build build --config Release -j $(nproc)
608608
609+
build-linux-cross:
610+
uses: ./.github/workflows/build-linux-cross.yml
611+
609612
macOS-latest-cmake-ios:
610613
runs-on: macos-latest
611614

@@ -803,7 +806,7 @@ jobs:
803806
env:
804807
OPENBLAS_VERSION: 0.3.23
805808
SDE_VERSION: 9.33.0-2024-01-07
806-
VULKAN_VERSION: 1.4.304.1
809+
VULKAN_VERSION: 1.4.309.0
807810

808811
strategy:
809812
matrix:

‎README.md‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,8 @@ Instructions for adding support for new models: [HOWTO-add-model.md](docs/develo
112112
- [x] [RWKV-6](https://github.com/BlinkDL/RWKV-LM)
113113
- [x] [QRWKV-6](https://huggingface.co/recursal/QRWKV6-32B-Instruct-Preview-v0.1)
114114
- [x] [GigaChat-20B-A3B](https://huggingface.co/ai-sage/GigaChat-20B-A3B-instruct)
115+
- [X] [Trillion-7B-preview](https://huggingface.co/trillionlabs/Trillion-7B-preview)
116+
- [x] [Ling models](https://huggingface.co/collections/inclusionAI/ling-67c51c85b34a7ea0aba94c32)
115117

116118
#### Multimodal
117119

@@ -528,6 +530,35 @@ If your issue is with model generation quality, then please at least scan the fo
528530
- [Aligning language models to follow instructions](https://openai.com/research/instruction-following)
529531
- [Training language models to follow instructions with human feedback](https://arxiv.org/abs/2203.02155)
530532
533+
## XCFramework
534+
The XCFramework is a precompiled version of the library for iOS, visionOS, tvOS,
535+
and macOS. It can be used in Swift projects without the need to compile the
536+
library from source. For example:
537+
```swift
538+
// swift-tools-version: 5.10
539+
// The swift-tools-version declares the minimum version of Swift required to build this package.
540+
541+
import PackageDescription
542+
543+
let package = Package(
544+
name: "MyLlamaPackage",
545+
targets: [
546+
.executableTarget(
547+
name: "MyLlamaPackage",
548+
dependencies: [
549+
"LlamaFramework"
550+
]),
551+
.binaryTarget(
552+
name: "LlamaFramework",
553+
url: "https://github.com/ggml-org/llama.cpp/releases/download/b5046/llama-b5046-xcframework.zip",
554+
checksum: "c19be78b5f00d8d29a25da41042cb7afa094cbf6280a225abe614b03b20029ab"
555+
)
556+
]
557+
)
558+
```
559+
The above example is using an intermediate build `b5046` of the library. This can be modified
560+
to use a different version by changing the URL and checksum.
561+
531562
## Completions
532563
Command-line completion is available for some environments.
533564

‎ci/README.md‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,43 @@ GG_BUILD_CUDA=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
2626
# with SYCL support
2727
source /opt/intel/oneapi/setvars.sh
2828
GG_BUILD_SYCL=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
29+
30+
# with MUSA support
31+
GG_BUILD_MUSA=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
32+
```
33+
34+
## Running MUSA CI in a Docker Container
35+
36+
Assuming `$PWD` is the root of the `llama.cpp` repository, follow these steps to set up and run MUSA CI in a Docker container:
37+
38+
### 1. Create a local directory to store cached models, configuration files and venv:
39+
40+
```bash
41+
mkdir -p $HOME/llama.cpp/ci-cache
42+
```
43+
44+
### 2. Create a local directory to store CI run results:
45+
46+
```bash
47+
mkdir -p $HOME/llama.cpp/ci-results
48+
```
49+
50+
### 3. Start a Docker container and run the CI:
51+
52+
```bash
53+
docker run --privileged -it \
54+
-v $HOME/llama.cpp/ci-cache:/ci-cache \
55+
-v $HOME/llama.cpp/ci-results:/ci-results \
56+
-v $PWD:/ws -w /ws \
57+
mthreads/musa:rc3.1.1-devel-ubuntu22.04
2958
```
59+
60+
Inside the container, execute the following commands:
61+
62+
```bash
63+
apt update -y && apt install -y bc cmake ccache git python3.10-venv time unzip wget
64+
git config --global --add safe.directory /ws
65+
GG_BUILD_MUSA=1 bash ./ci/run.sh /ci-results /ci-cache
66+
```
67+
68+
This setup ensures that the CI runs within an isolated Docker environment while maintaining cached files and results across runs.

‎ci/run.sh‎

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@
1616
# # with VULKAN support
1717
# GG_BUILD_VULKAN=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
1818
#
19+
# # with MUSA support
20+
# GG_BUILD_MUSA=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
21+
#
1922

2023
if [ -z "$2" ]; then
2124
echo "usage: $0 <output-dir> <mnt-dir>"
@@ -52,13 +55,24 @@ if [ ! -z ${GG_BUILD_SYCL} ]; then
5255
echo "source /opt/intel/oneapi/setvars.sh"
5356
exit 1
5457
fi
55-
58+
# Use only main GPU
59+
export ONEAPI_DEVICE_SELECTOR="level_zero:0"
60+
# Enable sysman for correct memory reporting
61+
export ZES_ENABLE_SYSMAN=1
62+
# to circumvent precision issues on CPY operations
63+
export SYCL_PROGRAM_COMPILE_OPTIONS="-cl-fp32-correctly-rounded-divide-sqrt"
5664
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_SYCL=1 -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_F16=ON"
5765
fi
5866

5967
if [ ! -z ${GG_BUILD_VULKAN} ]; then
6068
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_VULKAN=1"
6169
fi
70+
71+
if [ ! -z ${GG_BUILD_MUSA} ]; then
72+
# Use qy1 by default (MTT S80)
73+
MUSA_ARCH=${MUSA_ARCH:-21}
74+
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_MUSA=ON -DMUSA_ARCHITECTURES=${MUSA_ARCH}"
75+
fi
6276
## helpers
6377

6478
# download a file if it does not exist or if it is outdated
@@ -808,7 +822,7 @@ export LLAMA_LOG_PREFIX=1
808822
export LLAMA_LOG_TIMESTAMPS=1
809823

810824
if [ -z ${GG_BUILD_LOW_PERF} ]; then
811-
# Create symlink: ./llama.cpp/models-mnt -> $MNT/models/models-mnt
825+
# Create symlink: ./llama.cpp/models-mnt -> $MNT/models
812826
rm -rf ${SRC}/models-mnt
813827
mnt_models=${MNT}/models
814828
mkdir -p ${mnt_models}
@@ -826,16 +840,20 @@ if [ -z ${GG_BUILD_LOW_PERF} ]; then
826840
fi
827841

828842
ret=0
829-
830-
test $ret -eq 0 && gg_run ctest_debug
843+
if [ -z ${GG_BUILD_SYCL} ]; then
844+
# SYCL build breaks with debug build flags
845+
test $ret -eq 0 && gg_run ctest_debug
846+
fi
831847
test $ret -eq 0 && gg_run ctest_release
832848

833849
if [ -z ${GG_BUILD_LOW_PERF} ]; then
834850
test $ret -eq 0 && gg_run embd_bge_small
835851
test $ret -eq 0 && gg_run rerank_tiny
836852

837853
if [ -z ${GG_BUILD_CLOUD} ] || [ ${GG_BUILD_EXTRA_TESTS_0} ]; then
838-
test $ret -eq 0 && gg_run test_scripts_debug
854+
if [ -z ${GG_BUILD_SYCL} ]; then
855+
test $ret -eq 0 && gg_run test_scripts_debug
856+
fi
839857
test $ret -eq 0 && gg_run test_scripts_release
840858
fi
841859

@@ -846,7 +864,9 @@ if [ -z ${GG_BUILD_LOW_PERF} ]; then
846864
test $ret -eq 0 && gg_run pythia_2_8b
847865
#test $ret -eq 0 && gg_run open_llama_7b_v2
848866
fi
849-
test $ret -eq 0 && gg_run ctest_with_model_debug
867+
if [ -z ${GG_BUILD_SYCL} ]; then
868+
test $ret -eq 0 && gg_run ctest_with_model_debug
869+
fi
850870
test $ret -eq 0 && gg_run ctest_with_model_release
851871
fi
852872
fi

‎common/CMakeLists.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,8 +114,8 @@ if (LLAMA_LLGUIDANCE)
114114

115115
ExternalProject_Add(llguidance_ext
116116
GIT_REPOSITORY https://github.com/guidance-ai/llguidance
117-
# v0.6.12:
118-
GIT_TAG ced1c9023d47ec194fa977932d35ce65c2ebfc09
117+
# v0.7.10:
118+
GIT_TAG 0309d2a6bf40abda35344a362edc71e06d5009f8
119119
PREFIX ${CMAKE_BINARY_DIR}/llguidance
120120
SOURCE_DIR ${LLGUIDANCE_SRC}
121121
BUILD_IN_SOURCE TRUE

0 commit comments

Comments
 (0)