[core] Update sys.path to ensure that import lib works, thereby fixing test_job_isolation (#51765)

This PR fixes #51738.

I found that `sys.path` is different when we run `test_job.py` with
`pytest` and `bazel test //python/ray/tests:test_job` in CI runners (at
least the one for Python 3.12).

For the bazel command, `tmpdir/v1` is not in `sys.path`, so `import lib`
fails.

---------

Signed-off-by: kaihsun <[email protected]>

Author: kevin85421

Diff for: python/ray/tests/test_job.py

@@ -60,6 +60,7 @@ def test_job_isolation(ray_start_regular):
     # Make sure two jobs with same module name
     # don't interfere with each other
     # (https://github.com/ray-project/ray/issues/19358).
+
     gcs_address = ray_start_regular.address_info["gcs_address"]
 
     lib_template = """
@@ -74,27 +75,36 @@ def subtask():
 """
     driver_template = """
 import ray
+import os
+import sys
+
+# Add current directory to Python path so we can import lib.py
+current_dir = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, current_dir)
+
 import lib
 
 ray.init(address="{}")
 assert ray.get(lib.task.remote()) == {}
 """
+
+    def setup_driver_files(base_path: str, version: int) -> str:
+        version_path = os.path.join(base_path, f"v{version}")
+        os.makedirs(version_path)
+
+        lib_path = os.path.join(version_path, "lib.py")
+        driver_path = os.path.join(version_path, "driver.py")
+
+        with open(lib_path, "w") as f:
+            f.write(lib_template.format(version))
+        with open(driver_path, "w") as f:
+            f.write(driver_template.format(gcs_address, version))
+
+        return driver_path
+
     with tempfile.TemporaryDirectory() as tmpdir:
-        os.makedirs(os.path.join(tmpdir, "v1"))
-        v1_lib = os.path.join(tmpdir, "v1", "lib.py")
-        v1_driver = os.path.join(tmpdir, "v1", "driver.py")
-        with open(v1_lib, "w") as f:
-            f.write(lib_template.format(1))
-        with open(v1_driver, "w") as f:
-            f.write(driver_template.format(gcs_address, 1))
-
-        os.makedirs(os.path.join(tmpdir, "v2"))
-        v2_lib = os.path.join(tmpdir, "v2", "lib.py")
-        v2_driver = os.path.join(tmpdir, "v2", "driver.py")
-        with open(v2_lib, "w") as f:
-            f.write(lib_template.format(2))
-        with open(v2_driver, "w") as f:
-            f.write(driver_template.format(gcs_address, 2))
+        v1_driver = setup_driver_files(tmpdir, version=1)
+        v2_driver = setup_driver_files(tmpdir, version=2)
 
         subprocess.check_call([sys.executable, v1_driver])
         subprocess.check_call([sys.executable, v2_driver])