Fix UT env and upgrade torch to 2.4.0 (#1978)

Signed-off-by: Sun, Xuehao <[email protected]>

Author: XuehaoSun

.azure-pipelines/scripts/fwk_version.sh

@@ -2,9 +2,9 @@
 
 echo "export FWs version..."
 export tensorflow_version='2.15.0-official'
-export pytorch_version='2.3.0+cpu'
-export torchvision_version='0.18.0+cpu'
-export ipex_version='2.3.0+cpu'
+export pytorch_version='2.4.0+cpu'
+export torchvision_version='0.19.0'
+export ipex_version='2.4.0+cpu'
 export onnx_version='1.16.0'
 export onnxruntime_version='1.18.0'
 export mxnet_version='1.9.1'

.azure-pipelines/scripts/install_nc.sh

@@ -5,14 +5,16 @@ cd /neural-compressor
 if [[ $1 = *"3x_pt"* ]]; then
     if [[ $1 != *"3x_pt_fp8"* ]]; then
         echo -e "\n Install torch CPU ... "
-        pip install torch==2.3.0 --index-url https://download.pytorch.org/whl/cpu
+        pip install torch==2.4.0 --index-url https://download.pytorch.org/whl/cpu
     fi
     python -m pip install --no-cache-dir -r requirements_pt.txt
-    python setup.py pt bdist_wheel
+    # python setup.py pt bdist_wheel
+    python setup.py bdist_wheel
     pip install --no-deps dist/neural_compressor*.whl --force-reinstall
 elif [[ $1 = *"3x_tf"* ]]; then
     python -m pip install --no-cache-dir -r requirements_tf.txt
-    python setup.py tf bdist_wheel
+    # python setup.py tf bdist_wheel
+    python setup.py bdist_wheel
     pip install dist/neural_compressor*.whl --force-reinstall
 else
     python -m pip install --no-cache-dir -r requirements.txt

.azure-pipelines/scripts/models/env_setup.sh

@@ -51,6 +51,10 @@ SCRIPTS_PATH="/neural-compressor/.azure-pipelines/scripts/models"
 log_dir="/neural-compressor/.azure-pipelines/scripts/models"
 if [[ "${inc_new_api}" == "3x"* ]]; then
     WORK_SOURCE_DIR="/neural-compressor/examples/3.x_api/${framework}"
+    git clone https://github.com/intel/intel-extension-for-transformers.git /itrex
+    cd /itrex
+    pip install -r requirements.txt
+    pip install -v .
 else
     WORK_SOURCE_DIR="/neural-compressor/examples/${framework}"
 fi
@@ -95,8 +99,8 @@ if [[ "${fwk_ver}" != "latest" ]]; then
             pip install intel-tensorflow==${fwk_ver}
         fi
     elif [[ "${framework}" == "pytorch" ]]; then
-        pip install torch==${fwk_ver} -f https://download.pytorch.org/whl/torch_stable.html
-        pip install torchvision==${torch_vision_ver} -f https://download.pytorch.org/whl/torch_stable.html
+        pip install torch==${fwk_ver} --index-url https://download.pytorch.org/whl/cpu
+        pip install torchvision==${torch_vision_ver} --index-url https://download.pytorch.org/whl/cpu
     elif [[ "${framework}" == "onnxrt" ]]; then
         pip install onnx==1.15.0
         pip install onnxruntime==${fwk_ver}

.azure-pipelines/scripts/ut/run_basic_pt_pruning.sh

@@ -4,9 +4,9 @@ test_case="run basic pt pruning"
 echo "${test_case}"
 
 echo "specify fwk version..."
-export pytorch_version='2.3.0+cpu'
+export pytorch_version='2.4.0+cpu'
 export torchvision_version='0.18.0+cpu'
-export ipex_version='2.3.0+cpu'
+export ipex_version='2.4.0+cpu'
 
 echo "set up UT env..."
 bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh "${test_case}"

.azure-pipelines/template/docker-template.yml

@@ -42,6 +42,7 @@ steps:
       - checkout: self
         clean: true
         displayName: "Checkout out Repo"
+        fetchDepth: 0
 
   - ${{ if eq(parameters.dockerConfigName, 'gitCloneDockerConfig') }}:
       - script: |

examples/3.x_api/pytorch/nlp/huggingface_models/language-modeling/quantization/weight_only/requirements.txt

@@ -9,7 +9,7 @@ wandb
 einops
 neural-compressor
 intel-extension-for-transformers
-lm_eval==0.4.2
+lm_eval==0.4.3
 peft
 auto_round
 intel_extension_for_pytorch