Skip to content

Commit cc38757

Browse files
DevinTDHaDevinTDHa
authored
[SPARKNLP-1027] Fix issue with pretrained model (#14413)
* [SPARKNLP-1027] Change Default AutoGGUF pretrained model * [SPARKNLP-1027] Change default model in example * [SPARKNLP-1027] Fix issue with pretrained model
1 parent 1a112cd commit cc38757

File tree

3 files changed

+21
-21
lines changed

3 files changed

+21
-21
lines changed

‎examples/python/llama.cpp/llama.cpp_in_Spark_NLP_AutoGGUFModel.ipynb

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
"source": [
2323
"## Download a GGUF Model\n",
2424
"\n",
25-
"Lets download a GGUF model to test it out. For this, we will use [microsoft/Phi-3-mini-4k-instruct-gguf](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf). It is a 3.8B parameter model which also is available in 4-bit quantization. \n",
25+
"Lets download a GGUF model to test it out. For this, we will use [bartowski/Phi-3.5-mini-instruct-GGUF](https://huggingface.co/bartowski/Phi-3.5-mini-instruct-GGUF). It is a 3.8B parameter model which also is available in 4-bit quantization. \n",
2626
"\n",
2727
"We can download the model by selecting the q4 GGUF file from the \"Files and versions\" tab.\n",
2828
"\n",
@@ -38,28 +38,28 @@
3838
"name": "stdout",
3939
"output_type": "stream",
4040
"text": [
41-
"--2024-07-20 11:11:30-- https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf/resolve/main/Phi-3-mini-4k-instruct-q4.gguf?download=true\n",
41+
"--2024-07-20 11:11:30-- https://huggingface.co/bartowski/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q4_K_M.gguf?download=true\n",
4242
"Resolving huggingface.co (huggingface.co)... 2600:9000:275f:7600:17:b174:6d00:93a1, 2600:9000:275f:3800:17:b174:6d00:93a1, 2600:9000:275f:6e00:17:b174:6d00:93a1, ...\n",
4343
"Connecting to huggingface.co (huggingface.co)|2600:9000:275f:7600:17:b174:6d00:93a1|:443... connected.\n",
4444
"HTTP request sent, awaiting response... 302 Found\n",
45-
"Location: https://cdn-lfs-us-1.huggingface.co/repos/41/c8/41c860f65b01de5dc4c68b00d84cead799d3e7c48e38ee749f4c6057776e2e9e/8a83c7fb9049a9b2e92266fa7ad04933bb53aa1e85136b7b30f1b8000ff2edef?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27Phi-3-mini-4k-instruct-q4.gguf%3B+filename%3D%22Phi-3-mini-4k-instruct-q4.gguf%22%3B&Expires=1721725890&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcyMTcyNTg5MH19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zLzQxL2M4LzQxYzg2MGY2NWIwMWRlNWRjNGM2OGIwMGQ4NGNlYWQ3OTlkM2U3YzQ4ZTM4ZWU3NDlmNGM2MDU3Nzc2ZTJlOWUvOGE4M2M3ZmI5MDQ5YTliMmU5MjI2NmZhN2FkMDQ5MzNiYjUzYWExZTg1MTM2YjdiMzBmMWI4MDAwZmYyZWRlZj9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=joXQf4QRpEhtFeQ3r3gJ0zyJ3bXReb9OxM%7EZit3GJ3355ycKQzemJ%7E6eD-J7%7EkphnsPpRpUDhQkCr2-Oidqo7dgltmFsWAX4SmQLn65R1yjO%7EsMvi%7E4vOUpaRPYlSMCyWWJpiZZjQYVH4Uk0o-G62ALFXKGaDfr627kvahP-fJYwNNP1riTrH8hbbah28ZKRAQjUGI1aNqerG0jojudnGOagawISAnudkAOFZfxnN7Qw3CoMySZLj9Euu02RBv2A5Yy0uSjG7b8rilx-tU5HDR3ECohdQQ8yPXjYFU-LZi-zcG1wwBDF-S01qb%7EgPWsTorenxfRM2cG6J%7EvSziGCzA__&Key-Pair-Id=K24J24Z295AEI9 [following]\n",
46-
"--2024-07-20 11:11:30-- https://cdn-lfs-us-1.huggingface.co/repos/41/c8/41c860f65b01de5dc4c68b00d84cead799d3e7c48e38ee749f4c6057776e2e9e/8a83c7fb9049a9b2e92266fa7ad04933bb53aa1e85136b7b30f1b8000ff2edef?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27Phi-3-mini-4k-instruct-q4.gguf%3B+filename%3D%22Phi-3-mini-4k-instruct-q4.gguf%22%3B&Expires=1721725890&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcyMTcyNTg5MH19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zLzQxL2M4LzQxYzg2MGY2NWIwMWRlNWRjNGM2OGIwMGQ4NGNlYWQ3OTlkM2U3YzQ4ZTM4ZWU3NDlmNGM2MDU3Nzc2ZTJlOWUvOGE4M2M3ZmI5MDQ5YTliMmU5MjI2NmZhN2FkMDQ5MzNiYjUzYWExZTg1MTM2YjdiMzBmMWI4MDAwZmYyZWRlZj9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=joXQf4QRpEhtFeQ3r3gJ0zyJ3bXReb9OxM%7EZit3GJ3355ycKQzemJ%7E6eD-J7%7EkphnsPpRpUDhQkCr2-Oidqo7dgltmFsWAX4SmQLn65R1yjO%7EsMvi%7E4vOUpaRPYlSMCyWWJpiZZjQYVH4Uk0o-G62ALFXKGaDfr627kvahP-fJYwNNP1riTrH8hbbah28ZKRAQjUGI1aNqerG0jojudnGOagawISAnudkAOFZfxnN7Qw3CoMySZLj9Euu02RBv2A5Yy0uSjG7b8rilx-tU5HDR3ECohdQQ8yPXjYFU-LZi-zcG1wwBDF-S01qb%7EgPWsTorenxfRM2cG6J%7EvSziGCzA__&Key-Pair-Id=K24J24Z295AEI9\n",
45+
"Location: https://cdn-lfs-us-1.huggingface.co/repos/41/c8/41c860f65b01de5dc4c68b00d84cead799d3e7c48e38ee749f4c6057776e2e9e/8a83c7fb9049a9b2e92266fa7ad04933bb53aa1e85136b7b30f1b8000ff2edef?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27Phi-3.5-mini-instruct-Q4_K_M.gguf%3B+filename%3D%22P Phi-3.5-mini-instruct-Q4_K_M.gguf%22%3B&Expires=1721725890&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcyMTcyNTg5MH19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zLzQxL2M4LzQxYzg2MGY2NWIwMWRlNWRjNGM2OGIwMGQ4NGNlYWQ3OTlkM2U3YzQ4ZTM4ZWU3NDlmNGM2MDU3Nzc2ZTJlOWUvOGE4M2M3ZmI5MDQ5YTliMmU5MjI2NmZhN2FkMDQ5MzNiYjUzYWExZTg1MTM2YjdiMzBmMWI4MDAwZmYyZWRlZj9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=joXQf4QRpEhtFeQ3r3gJ0zyJ3bXReb9OxM%7EZit3GJ3355ycKQzemJ%7E6eD-J7%7EkphnsPpRpUDhQkCr2-Oidqo7dgltmFsWAX4SmQLn65R1yjO%7EsMvi%7E4vOUpaRPYlSMCyWWJpiZZjQYVH4Uk0o-G62ALFXKGaDfr627kvahP-fJYwNNP1riTrH8hbbah28ZKRAQjUGI1aNqerG0jojudnGOagawISAnudkAOFZfxnN7Qw3CoMySZLj9Euu02RBv2A5Yy0uSjG7b8rilx-tU5HDR3ECohdQQ8yPXjYFU-LZi-zcG1wwBDF-S01qb%7EgPWsTorenxfRM2cG6J%7EvSziGCzA__&Key-Pair-Id=K24J24Z295AEI9 [following]\n",
46+
"--2024-07-20 11:11:30-- https://cdn-lfs-us-1.huggingface.co/repos/41/c8/41c860f65b01de5dc4c68b00d84cead799d3e7c48e38ee749f4c6057776e2e9e/8a83c7fb9049a9b2e92266fa7ad04933bb53aa1e85136b7b30f1b8000ff2edef?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27Phi-3.5-mini-instruct-Q4_K_M.gguf%3B+filename%3D%22P Phi-3.5-mini-instruct-Q4_K_M.gguf%22%3B&Expires=1721725890&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcyMTcyNTg5MH19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zLzQxL2M4LzQxYzg2MGY2NWIwMWRlNWRjNGM2OGIwMGQ4NGNlYWQ3OTlkM2U3YzQ4ZTM4ZWU3NDlmNGM2MDU3Nzc2ZTJlOWUvOGE4M2M3ZmI5MDQ5YTliMmU5MjI2NmZhN2FkMDQ5MzNiYjUzYWExZTg1MTM2YjdiMzBmMWI4MDAwZmYyZWRlZj9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=joXQf4QRpEhtFeQ3r3gJ0zyJ3bXReb9OxM%7EZit3GJ3355ycKQzemJ%7E6eD-J7%7EkphnsPpRpUDhQkCr2-Oidqo7dgltmFsWAX4SmQLn65R1yjO%7EsMvi%7E4vOUpaRPYlSMCyWWJpiZZjQYVH4Uk0o-G62ALFXKGaDfr627kvahP-fJYwNNP1riTrH8hbbah28ZKRAQjUGI1aNqerG0jojudnGOagawISAnudkAOFZfxnN7Qw3CoMySZLj9Euu02RBv2A5Yy0uSjG7b8rilx-tU5HDR3ECohdQQ8yPXjYFU-LZi-zcG1wwBDF-S01qb%7EgPWsTorenxfRM2cG6J%7EvSziGCzA__&Key-Pair-Id=K24J24Z295AEI9\n",
4747
"Resolving cdn-lfs-us-1.huggingface.co (cdn-lfs-us-1.huggingface.co)... 2600:9000:275f:7a00:17:9a40:4dc0:93a1, 2600:9000:275f:fc00:17:9a40:4dc0:93a1, 2600:9000:275f:4800:17:9a40:4dc0:93a1, ...\n",
4848
"Connecting to cdn-lfs-us-1.huggingface.co (cdn-lfs-us-1.huggingface.co)|2600:9000:275f:7a00:17:9a40:4dc0:93a1|:443... connected.\n",
4949
"HTTP request sent, awaiting response... 200 OK\n",
5050
"Length: 2393231072 (2.2G) [binary/octet-stream]\n",
51-
"Saving to: ‘Phi-3-mini-4k-instruct-q4.gguf?download=true’\n",
51+
"Saving to: ‘Phi-3.5-mini-instruct-Q4_K_M.gguf?download=true’\n",
5252
"\n",
5353
"Phi-3-mini-4k-instr 100%[===================>] 2.23G 22.5MB/s in 96s \n",
5454
"\n",
55-
"2024-07-20 11:13:06 (23.7 MB/s) - ‘Phi-3-mini-4k-instruct-q4.gguf?download=true’ saved [2393231072/2393231072]\n",
55+
"2024-07-20 11:13:06 (23.7 MB/s) - ‘Phi-3.5-mini-instruct-Q4_K_M.gguf?download=true’ saved [2393231072/2393231072]\n",
5656
"\n"
5757
]
5858
}
5959
],
6060
"source": [
61-
"EXPORT_PATH = \"Phi-3-mini-4k-instruct-q4.gguf\"\n",
62-
"! wget \"https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf/resolve/main/Phi-3-mini-4k-instruct-q4.gguf?download=true\" -O {EXPORT_PATH}"
61+
"EXPORT_PATH = \"Phi-3.5-mini-instruct-Q4_K_M.gguf\"\n",
62+
"! wget \"https://huggingface.co/bartowski/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q4_K_M.gguf?download=true\" -O {EXPORT_PATH}"
6363
]
6464
},
6565
{
@@ -359,7 +359,7 @@
359359
"metadata": {},
360360
"outputs": [],
361361
"source": [
362-
"autoGGUFModel.write().overwrite().save(f\"Phi-3-mini-4k-instruct-q4_spark_nlp\")"
362+
"autoGGUFModel.write().overwrite().save(f\"Phi-3.5-mini-instruct-Q4_K_M_spark_nlp\")"
363363
]
364364
},
365365
{
@@ -398,12 +398,12 @@
398398
"text": [
399399
"total 2337168\n",
400400
"drwxr-xr-x 2 ducha ducha 4096 Jul 21 16:24 metadata\n",
401-
"-rwxrwxr-x 1 ducha ducha 2393231072 Jul 21 16:24 Phi-3-mini-4k-instruct-q4.gguf\n"
401+
"-rwxrwxr-x 1 ducha ducha 2393231072 Jul 21 16:24 Phi-3.5-mini-instruct-Q4_K_M.gguf\n"
402402
]
403403
}
404404
],
405405
"source": [
406-
"! ls -l Phi-3-mini-4k-instruct-q4_spark_nlp"
406+
"! ls -l Phi-3.5-mini-instruct-Q4_K_M_spark_nlp"
407407
]
408408
},
409409
{
@@ -437,7 +437,7 @@
437437
"name": "stderr",
438438
"output_type": "stream",
439439
"text": [
440-
"llama_model_loader: loaded meta data with 24 key-value pairs and 195 tensors from /tmp/spark-bbad4f64-91a7-4b6e-8242-7f91e6abca54/userFiles-f7d4e4e9-c02d-46e4-81b5-bf5a26d70930/Phi-3-mini-4k-instruct-q4.gguf (version GGUF V3 (latest))\n",
440+
"llama_model_loader: loaded meta data with 24 key-value pairs and 195 tensors from /tmp/spark-bbad4f64-91a7-4b6e-8242-7f91e6abca54/userFiles-f7d4e4e9-c02d-46e4-81b5-bf5a26d70930/Phi-3.5-mini-instruct-Q4_K_M.gguf (version GGUF V3 (latest))\n",
441441
"llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
442442
"llama_model_loader: - kv 0: general.architecture str = phi3\n",
443443
"llama_model_loader: - kv 1: general.name str = Phi3\n",
@@ -583,7 +583,7 @@
583583
"\n",
584584
"document_assembler = DocumentAssembler().setInputCol(\"text\").setOutputCol(\"document\")\n",
585585
"\n",
586-
"auto_gguf_model = AutoGGUFModel.load(\"Phi-3-mini-4k-instruct-q4_spark_nlp\")\n",
586+
"auto_gguf_model = AutoGGUFModel.load(\"Phi-3.5-mini-instruct-Q4_K_M_spark_nlp\")\n",
587587
"\n",
588588
"pipeline = Pipeline().setStages([document_assembler, auto_gguf_model])\n",
589589
"\n",

‎python/test/annotator/seq2seq/auto_gguf_model_test.py

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -43,14 +43,13 @@ def runTest(self):
4343
DocumentAssembler().setInputCol("text").setOutputCol("document")
4444
)
4545

46-
modelPath = "models/codellama-7b.Q2_K.gguf"
4746
model = (
48-
AutoGGUFModel.loadSavedModel(modelPath, self.spark)
47+
AutoGGUFModel.pretrained()
4948
.setInputCols("document")
5049
.setOutputCol("completions")
5150
.setBatchSize(4)
5251
.setNPredict(20)
53-
.setNGpuLayers(99)
52+
.setNGpuLayers(5)
5453
.setTemperature(0.4)
5554
.setTopK(40)
5655
.setTopP(0.9)
@@ -79,9 +78,8 @@ def runTest(self):
7978
DocumentAssembler().setInputCol("text").setOutputCol("document")
8079
)
8180

82-
modelPath = "models/codellama-7b.Q2_K.gguf"
8381
model = (
84-
AutoGGUFModel.loadSavedModel(modelPath, self.spark)
82+
AutoGGUFModel.pretrained()
8583
.setInputCols("document")
8684
.setOutputCol("completions")
8785
.setBatchSize(4)
@@ -182,9 +180,8 @@ def setUp(self):
182180
self.spark = SparkContextForTest.spark
183181

184182
def runTest(self):
185-
modelPath = "models/codellama-7b.Q2_K.gguf"
186183
model = (
187-
AutoGGUFModel.loadSavedModel(modelPath, self.spark)
184+
AutoGGUFModel.pretrained()
188185
.setInputCols("document")
189186
.setOutputCol("completions")
190187
)

‎src/main/scala/com/johnsnowlabs/nlp/annotators/seq2seq/AutoGGUFModel.scala

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,10 @@ trait ReadAutoGGUFModel {
233233

234234
def readModel(instance: AutoGGUFModel, path: String, spark: SparkSession): Unit = {
235235
def findGGUFModelInFolder(): String = {
236-
val folder = new java.io.File(path)
236+
val folder =
237+
new java.io.File(
238+
path.replace("file:", "")
239+
) // File should be local at this point. TODO: Except if its HDFS?
237240
if (folder.exists && folder.isDirectory) {
238241
folder.listFiles
239242
.filter(_.isFile)

0 commit comments

Comments
 (0)