[ci] add flaky tags for flaky test groups (#51774)

this allows selecting all flaky test test groups with one tag

Signed-off-by: Lonnie Liu <[email protected]>

Author: aslonnie

.buildkite/core.rayci.yml

@@ -338,6 +338,7 @@ steps:
     key: core_flaky_tests
     tags:
       - python
+      - flaky
       - skip-on-premerge
     instance_type: large
     soft_fail: true

.buildkite/data.rayci.yml

@@ -221,6 +221,7 @@ steps:
     tags:
       - python
       - data
+      - flaky
       - skip-on-premerge
     instance_type: medium
     soft_fail: true

.buildkite/macos/macos.rayci.yml

@@ -111,6 +111,7 @@ steps:
       - python
       - macos_wheels
       - oss
+      - flaky
       - skip_on_premerge
     job_env: MACOS
     instance_type: macos

.buildkite/ml.rayci.yml

@@ -274,6 +274,7 @@ steps:
     key: ml_flaky_tests
     tags:
       - train
+      - flaky
       - skip-on-premerge
     instance_type: large
     commands:

.buildkite/rllib.rayci.yml

@@ -194,6 +194,7 @@ steps:
     tags:
       - rllib_gpu
       - gpu
+      - flaky
       - skip-on-premerge
     instance_type: gpu
     commands:
@@ -209,6 +210,7 @@ steps:
     key: rllib_flaky_tests_01
     tags:
       - rllib
+      - flaky
       - skip-on-premerge
     instance_type: large
     commands:
@@ -238,6 +240,7 @@ steps:
     key: rllib_flaky_tests_02
     tags:
       - rllib
+      - flaky
       - skip-on-premerge
     instance_type: large
     commands:

.buildkite/serve.rayci.yml

@@ -183,8 +183,9 @@ steps:
     key: serve_flaky_tests
     tags:
       - serve
-      - skip-on-premerge
       - python
+      - flaky
+      - skip-on-premerge
     instance_type: medium
     soft_fail: true
     commands:

.buildkite/windows.rayci.yml

@@ -128,7 +128,9 @@ steps:
 
   - label: "flaky :windows: core tests"
     key: windows_core_flaky_tests
-    tags: skip-on-premerge
+    tags:
+      - flaky
+      - skip-on-premerge
     job_env: WINDOWS
     instance_type: windows
     commands:
@@ -146,7 +148,9 @@ steps:
 
   - label: "flaky :windows: serverless tests"
     key: windows_serverless_flaky_tests
-    tags: skip-on-premerge
+    tags:
+      - flaky
+      - skip-on-premerge
     job_env: WINDOWS
     instance_type: windows
     commands:
@@ -164,7 +168,9 @@ steps:
 
   - label: "flaky :windows: serve tests"
     key: windows_serve_flaky_tests
-    tags: skip-on-premerge
+    tags:
+      - flaky
+      - skip-on-premerge
     job_env: WINDOWS
     instance_type: windows
     commands: