Merge branch 'main' into aqua/ADS_MS_changes

Author: kumar-shivam-ranjan

Diff for: .github/workflows/run-unittests-default_setup.yml

@@ -1,4 +1,4 @@
-name: "[Py3.9-3.11] - Default Tests"
+name: "[Py3.9-3.12] - Default Tests"
 
 on:
   workflow_dispatch:

Diff for: .github/workflows/run-unittests-py310-py311.yml

@@ -1,4 +1,4 @@
-name: "[Py3.10-3.11] - All Unit Tests"
+name: "[Py3.10-3.12] - All Unit Tests"
 
 on:
   workflow_dispatch:
@@ -23,6 +23,7 @@ permissions:
 # hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359
 env:
   SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
+  ENV TF_USE_LEGACY_KERAS:  1
 
 jobs:
   test:
@@ -33,7 +34,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
         name: ["unitary", "slow_tests"]
         include:
           - name: "unitary"

Diff for: pyproject.toml

@@ -48,6 +48,7 @@ classifiers = [
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
 ]
 
 # PEP 508 – Dependency specification for Python Software Packages - https://peps.python.org/pep-0508/
@@ -105,9 +106,11 @@ huggingface = [
 notebook = ["ipython>=7.23.1, <8.0", "ipywidgets~=7.6.3"]
 onnx = [
   "lightgbm",
-  "onnx>=1.12.0,<=1.15.0",  # v 1.15.0 set base on onnxrutime version and onnx opset support - https://onnxruntime.ai/docs/reference/compatibility.html#onnx-opset-support
+  "onnx>=1.12.0,<=1.15.0; python_version < '3.12'",  # v 1.15.0 set base on onnxrutime version and onnx opset support - https://onnxruntime.ai/docs/reference/compatibility.html#onnx-opset-support
+  "onnx>=1.12.0; python_version >= '3.12'",  # v 1.15.0 set base on onnxrutime version and onnx opset support - https://onnxruntime.ai/docs/reference/compatibility.html#onnx-opset-support
   "onnxmltools>=1.10.0",
-  "onnxruntime~=1.17.0,!=1.16.0",  # v1.17.0 used in Oracle Database 23ai; avoid v1.16 https://github.com/microsoft/onnxruntime/issues/17631, revealed by unit tests
+  "onnxruntime~=1.17.0,!=1.16.0; python_version < '3.12'",  # v1.17.0 used in Oracle Database 23ai; avoid v1.16 https://github.com/microsoft/onnxruntime/issues/17631, revealed by unit tests
+  "onnxruntime; python_version >= '3.12'",  # v1.17.0 used in Oracle Database 23ai; avoid v1.16 https://github.com/microsoft/onnxruntime/issues/17631, revealed by unit tests
   "oracle_ads[viz]",
   "protobuf",
   "skl2onnx>=1.10.4",
@@ -131,7 +134,8 @@ optuna = ["optuna==2.9.0", "oracle_ads[viz]"]
 spark = ["pyspark>=3.0.0"]
 tensorflow = [
   "oracle_ads[viz]",
-  "tensorflow<=2.15.1"  # v2.16.1 with consequence on tf2onnx v1.16.1 (latest) has an issue with Keras 3 installed in py3.11+ (https://github.com/onnx/tensorflow-onnx/issues/2319)
+  "tensorflow<=2.15.1; python_version < '3.12'",  # v2.16.1 with consequence on tf2onnx v1.16.1 (latest) has an issue with Keras 3 installed in py3.11+ (https://github.com/onnx/tensorflow-onnx/issues/2319)
+  "tensorflow; python_version >= '3.12'"  # v2.16.1 with consequence on tf2onnx v1.16.1 (latest) has an issue with Keras 3 installed in py3.11+ (https://github.com/onnx/tensorflow-onnx/issues/2319)
 ]
 text = [
   "spacy>=3.4.2,<3.8",   # the first version of spacy that supports python 3.11 is spacy v3.4.2; 3.8.2 has dependency conflict.

Diff for: test-requirements.txt

@@ -12,3 +12,4 @@ pytest-cov
 pytest-xdist
 pytest-asyncio
 ruff
+setuptools

Diff for: tests/unitary/default_setup/pipeline/test_pipeline_run.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-# Copyright (c) 2024 Oracle and/or its affiliates.
+# Copyright (c) 2024, 2025 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 from datetime import datetime
@@ -494,7 +494,6 @@ def test_logs_custom(self, mock_stop_condition):
         )
 
         with patch.object(OCIModelMixin, "deserialize") as mock_deserialize:
-
             mock_deserialize.return_value = oci.logging.models.Log(**OCI_LOG_DETAILS)
 
             consolidated_log = pipeline_run.logs(log_type="custom_log")
@@ -528,7 +527,6 @@ def test_logs_service(self, mock_get_service_logging):
             pipeline_run.logs(log_type="service_log")
 
         with patch.object(OCIModelMixin, "deserialize") as mock_deserialize:
-
             mock_deserialize.return_value = oci.logging.models.Log(**OCI_LOG_DETAILS)
             pipeline_run.log_details = oci.data_science.models.PipelineRunLogDetails(
                 **PIPELINE_RUN_LOG_DETAILS
@@ -558,7 +556,6 @@ def test_logs_service(self, mock_get_service_logging):
     @patch.object(OCIModelMixin, "sync")
     def test_logs_both(self, mock_sync, mock_stop_condition, mock_get_service_logging):
         with patch.object(OCIModelMixin, "deserialize") as mock_deserialize:
-
             mock_deserialize.return_value = oci.logging.models.Log(**OCI_LOG_DETAILS)
 
             pipeline_run = PipelineRun()
@@ -595,7 +592,6 @@ def test_logs_both(self, mock_sync, mock_stop_condition, mock_get_service_loggin
     @patch.object(PipelineRun, "_stop_condition", return_value=True)
     def test_custom_logging(self, mock_stop_condition):
         with patch.object(OCIModelMixin, "deserialize") as mock_deserialize:
-
             mock_deserialize.return_value = oci.logging.models.Log(**OCI_LOG_DETAILS)
             pipeline_run = PipelineRun()
             pipeline_run.log_details = oci.data_science.models.PipelineRunLogDetails(
@@ -617,7 +613,6 @@ def test_custom_logging(self, mock_stop_condition):
     @patch.object(PipelineRun, "_get_service_logging")
     def test_service_logging(self, mock_get_service_logging):
         with patch.object(OCIModelMixin, "deserialize") as mock_deserialize:
-
             mock_deserialize.return_value = oci.logging.models.Log(**OCI_LOG_DETAILS)
 
             pipeline_run = PipelineRun()
@@ -644,7 +639,6 @@ def test_service_logging(self, mock_get_service_logging):
     @patch.object(PipelineRun, "_search_service_logs")
     def test_get_service_logging(self, mock_search_service_logs):
         with patch.object(OCIModelMixin, "deserialize") as mock_deserialize:
-
             mock_deserialize.return_value = oci.logging.models.Log(**OCI_LOG_DETAILS)
             pipeline_run = PipelineRun()
             pipeline_run.id = PIPELINE_RUN_OCID
@@ -701,7 +695,7 @@ def test_watch_service_log(self, mock_logs, mock_stream_log):
         mock_logs.return_value = ConsolidatedLog(OCILog(log_type="SERVICE"))
 
         pipeline_run.watch(log_type="service_log")
-        mock_logs.called_with(log_type="service_log")
+        mock_logs.assert_called_with(log_type="service_log")
         mock_stream_log.assert_called_with(mock_logs.return_value, [], 3, "service_log")
 
     @patch.object(PipelineRun, "_PipelineRun__stream_log")
@@ -712,7 +706,7 @@ def test_watch_custom_log(self, mock_logs, mock_stream_log):
         mock_logs.return_value = ConsolidatedLog(OCILog(log_type="CUSTOM"))
 
         pipeline_run.watch(log_type="custom_log")
-        mock_logs.called_with(log_type="custom_log")
+        mock_logs.assert_called_with(log_type="custom_log")
         mock_stream_log.assert_called_with(mock_logs.return_value, [], 3, "custom_log")
 
     @patch.object(PipelineRun, "_PipelineRun__stream_log")
@@ -726,7 +720,7 @@ def test_watch_both_log(self, mock_logs, mock_stream_log):
         )
 
         pipeline_run.watch()
-        mock_logs.called_with(log_type=None)
+        mock_logs.assert_called_with(log_type=None)
         mock_stream_log.assert_called_with(mock_logs.return_value, [], 3, None)
 
     def test_build_filter_expression(self):

Diff for: tests/unitary/with_extras/model/test_model_framework_sklearn_model.py

@@ -1,11 +1,12 @@
 #!/usr/bin/env python
 
-# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
+# Copyright (c) 2021, 2025 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 """Unit tests for model frameworks. Includes tests for:
- - SklearnModel
+- SklearnModel
 """
+
 import base64
 import os
 import shutil
@@ -15,7 +16,7 @@
 import onnx
 import onnxruntime as rt
 import pandas as pd
-import pytest
+import pytest, sys
 from ads.model.framework.sklearn_model import SklearnModel
 from ads.model.serde.model_serializer import SklearnOnnxModelSerializer
 from joblib import load
@@ -325,6 +326,7 @@ def test_serialize_using_pipeline_joblib(self):
             loaded_model = load(file)
         assert len(loaded_model.predict(self.X_test)) != 0
 
+    @pytest.mark.skipif(sys.version_info >= (3, 12), reason="Skipped for Python 3.12+")
     def test_serialize_and_load_model_as_onnx_xgboost_pipeline(self):
         """
         Test serialize and load pipeline using Sklearn API with xgboost model.
@@ -365,6 +367,7 @@ def test_serialize_and_load_model_as_joblib_xgboost_pipeline(self):
             loaded_model = load(file)
         assert len(loaded_model.predict(self.X_iris)) != 0
 
+    @pytest.mark.skipif(sys.version_info >= (3, 12), reason="Skipped for Python 3.12+")
     def test_serialize_and_load_model_as_onnx_lgb_pipeline(self):
         """
         Test serialize and load pipeline using Sklearn API with lightgbm model.
@@ -405,6 +408,7 @@ def test_serialize_and_load_model_as_joblib_lgb_pipeline(self):
             loaded_model = load(file)
         assert len(loaded_model.predict(self.X_iris)) != 0
 
+    @pytest.mark.skipif(sys.version_info >= (3, 12), reason="Skipped for Python 3.12+")
     def test_serialize_and_load_model_as_onnx_lgb_reg_pipeline(self):
         """
         Test serialize and load pipeline using Sklearn API with lightgbm regressor model.

Diff for: tests/unitary/with_extras/model/test_model_framework_tensorflow_model.py

@@ -1,11 +1,12 @@
 #!/usr/bin/env python
 
-# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
+# Copyright (c) 2022, 2025 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 """Unit tests for model frameworks. Includes tests for:
- - TensorFlowModel
+- TensorFlowModel
 """
+
 import base64
 import os
 import shutil
@@ -17,7 +18,7 @@
 import numpy as np
 import onnxruntime as rt
 import pandas as pd
-import pytest
+import pytest, sys
 import tensorflow as tf
 import tempfile
 from ads.model.framework.tensorflow_model import TensorFlowModel
@@ -91,6 +92,7 @@ def setup_class(cls):
         cls.training_conda_env = CONDA_PACK_PATH
         cls.training_python_version = SUPPORTED_PYTHON_VERSION
 
+    @pytest.mark.skipif(sys.version_info >= (3, 12), reason="Skipped for Python 3.12+")
     def test_serialize_with_incorrect_model_file_name_onnx(self):
         """
         Test wrong model_file_name for onnx format.
@@ -139,6 +141,7 @@ def test_serialize_using_tf_with_modelname(self):
             os.path.join(tmp_model_dir, test_tf_model.model_file_name)
         )
 
+    @pytest.mark.skipif(sys.version_info >= (3, 12), reason="Skipped for Python 3.12+")
     def test_serialize_using_onnx_without_modelname(self):
         """
         Test serialize_model using onnx without model_file_name
@@ -156,6 +159,7 @@ def test_serialize_using_onnx_without_modelname(self):
         )
         assert os.path.exists(os.path.join(tmp_model_dir, "model.onnx"))
 
+    @pytest.mark.skipif(sys.version_info >= (3, 12), reason="Skipped for Python 3.12+")
     def test_serialize_using_onnx_with_modelname(self):
         """
         Test serialize_model using onnx with correct model_file_name
@@ -173,6 +177,7 @@ def test_serialize_using_onnx_with_modelname(self):
             os.path.join(tmp_model_dir, test_tf_model.model_file_name)
         )
 
+    @pytest.mark.skipif(sys.version_info >= (3, 12), reason="Skipped for Python 3.12+")
     def test_to_onnx(self):
         """
         Test if TensorFlowOnnxModelSerializer.serialize() generate onnx model result.
@@ -189,6 +194,7 @@ def test_to_onnx(self):
         )
         assert os.path.exists(os.path.join(tmp_model_dir, model_file_name))
 
+    @pytest.mark.skipif(sys.version_info >= (3, 12), reason="Skipped for Python 3.12+")
     def test_to_onnx_reload(self):
         """
         Test reloading the model generated by
@@ -209,6 +215,7 @@ def test_to_onnx_reload(self):
             is not None
         )
 
+    @pytest.mark.skipif(sys.version_info >= (3, 12), reason="Skipped for Python 3.12+")
     def test_to_onnx_without_dummy_input(self):
         """
         Test if TensorFlowOnnxModelSerializer.serialize() raise expected error.
@@ -224,6 +231,7 @@ def test_to_onnx_without_dummy_input(self):
         )
         assert os.path.isfile(os.path.join(tmp_model_dir, model_file_name))
 
+    @pytest.mark.skipif(sys.version_info >= (3, 12), reason="Skipped for Python 3.12+")
     def test_to_onnx_without_path(self):
         """
         Test if TensorFlowOnnxModelSerializer.serialize() raise expected error.
@@ -310,6 +318,7 @@ def test_prepare_default(self):
         )
         assert os.path.exists(os.path.join(tmp_model_dir, "model.h5"))
 
+    @pytest.mark.skipif(sys.version_info >= (3, 12), reason="Skipped for Python 3.12+")
     def test_prepare_onnx_with_input_signature(self):
         """
         Test if TensorFlowModel.prepare onnx serialization
@@ -326,6 +335,7 @@ def test_prepare_onnx_with_input_signature(self):
         )
         assert os.path.exists(os.path.join(tmp_model_dir, "model.onnx"))
 
+    @pytest.mark.skipif(sys.version_info >= (3, 12), reason="Skipped for Python 3.12+")
     def test_prepare_onnx_with_X_sample(self):
         """
         Test if TensorFlowModel.prepare raise expected error
@@ -342,6 +352,7 @@ def test_prepare_onnx_with_X_sample(self):
         )
         assert isinstance(test_tf_model.verify(self.x_test[:1].tolist()), dict)
 
+    @pytest.mark.skipif(sys.version_info >= (3, 12), reason="Skipped for Python 3.12+")
     def test_verify_onnx_without_input(self):
         """
         Test if TensorFlowModel.verify in onnx serialization

Diff for: tests/unitary/with_extras/model/test_model_framework_xgboost_model.py

@@ -1,12 +1,13 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
+# Copyright (c) 2021, 2025 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 """Unit tests for model frameworks. Includes tests for:
- - XGBoostModel
+- XGBoostModel
 """
+
 import base64
 import os
 import shutil
@@ -33,7 +34,6 @@ def setup_method(cls):
         os.makedirs(tmp_model_dir, exist_ok=True)
 
     def setup_class(cls):
-
         X, y = make_classification(n_samples=100, n_informative=5, n_classes=2)
         (
             X_train_classification,
@@ -167,6 +167,7 @@ def test_serialize_and_load_model_as_json_sklearn_api(self):
         pred_json = loaded_model.predict(self.X_test_classification)
         assert all(pred_sklearn == pred_json)
 
+    @pytest.mark.skipif(sys.version_info >= (3, 12), reason="Skipped for Python 3.12+")
     def test_serialize_and_load_model_as_onnx_sklearn_api(self):
         """
         Test serialize and load model using Sklearn API.
@@ -188,6 +189,7 @@ def test_serialize_and_load_model_as_onnx_sklearn_api(self):
         d = np.abs(pred_onx - pred_xgboost)
         assert d.max() <= 1
 
+    @pytest.mark.skipif(sys.version_info >= (3, 12), reason="Skipped for Python 3.12+")
     def test_serialize_with_model_file_name(self):
         """
         Test correct and wrong model_file_name format.