Internal change

PiperOrigin-RevId: 692963048

src/devtools/mobileharness/infra/controller/test/util/xtsdownloader/configs/mcts_list.txt

@@ -255,7 +255,6 @@ CtsTestAppWithQueryAllPackagesPermission
 CtsTetheringTest
 CtsTfliteNnapiDelegateTestCases
 CtsThreadNetworkTestCases
-CtsUsbTests
 CtsUwbMultiDeviceTestCase_FiraRangingTests
 CtsUwbMultiDeviceTestCase_UwbManagerTests
 CtsUwbTestCases

src/java/com/google/devtools/mobileharness/infra/ats/common/SessionRequestHandlerUtil.java

@@ -378,7 +378,12 @@ public ImmutableSet<String> getStaticMctsModules() throws MobileHarnessException
   /**
    * Gets a list of filtered tradefed modules.
    *
-   * <p>The list of modules is filtered by include/exclude filters and the given module names.
+   * <p>The list of modules is filtered by include/exclude filters and the given module names. This
+   * method only return the modules passing along with the run cts -m <MODULE> command; In CTS tool,
+   * the run cts -m <MODULE> command only support passing one module so currently this method will
+   * most likely return only one module. This method also made an assumption that the
+   * --include-filter <TESTS> and run cts -m <MODULE> can be used together which is not supported in
+   * CTS tool actually. If the given module names are empty, it will return all the modules.
    *
    * @return a list of filtered tradefed modules if the given modules are not empty. Or an empty
    *     list if the given modules are empty.

src/java/com/google/devtools/mobileharness/infra/ats/common/SessionRequestInfo.java

@@ -132,6 +132,8 @@ public abstract class SessionRequestInfo {
 
   public abstract Optional<Boolean> skipDeviceInfo();
 
+  public abstract Optional<Boolean> isEnableXtsDynamicDownload();
+
   public static Builder builder() {
     return new AutoValue_SessionRequestInfo.Builder()
         .setModuleNames(ImmutableList.of())
@@ -249,6 +251,8 @@ public abstract Builder setGivenMatchedNonTfModules(
 
     public abstract Builder setSkipDeviceInfo(boolean skipDeviceInfo);
 
+    public abstract Builder setIsEnableXtsDynamicDownload(boolean isEnableXtsDynamicDownload);
+
     protected abstract SessionRequestInfo autoBuild();
 
     public SessionRequestInfo build() {

src/java/com/google/devtools/mobileharness/infra/ats/common/jobcreator/BUILD

@@ -44,8 +44,10 @@ java_library(
         "//src/java/com/google/devtools/mobileharness/infra/ats/common:session_request_info",
         "//src/java/com/google/devtools/mobileharness/infra/ats/common:xts_property_name",
         "//src/java/com/google/devtools/mobileharness/infra/ats/common/plan:test_plan_parser",
+        "//src/java/com/google/devtools/mobileharness/platform/android/xts/common/util:xts_constants",
         "//src/java/com/google/devtools/mobileharness/platform/android/xts/common/util:xts_dir_util",
         "//src/java/com/google/devtools/mobileharness/platform/android/xts/suite:suite_common",
+        "//src/java/com/google/devtools/mobileharness/platform/android/xts/suite:suite_test_filter",
         "//src/java/com/google/devtools/mobileharness/platform/android/xts/suite/retry:retry_args",
         "//src/java/com/google/devtools/mobileharness/platform/android/xts/suite/retry:retry_generator",
         "//src/java/com/google/devtools/mobileharness/platform/android/xts/suite/subplan:sub_plan",

src/java/com/google/devtools/mobileharness/infra/ats/common/jobcreator/XtsJobCreator.java

@@ -36,9 +36,12 @@
 import com.google.devtools.mobileharness.infra.ats.common.XtsPropertyName;
 import com.google.devtools.mobileharness.infra.ats.common.XtsPropertyName.Job;
 import com.google.devtools.mobileharness.infra.ats.common.plan.TestPlanParser;
+import com.google.devtools.mobileharness.platform.android.xts.common.util.XtsConstants;
+import com.google.devtools.mobileharness.platform.android.xts.common.util.XtsConstants.XtsDynamicDownloadModuleMatchResult;
 import com.google.devtools.mobileharness.platform.android.xts.common.util.XtsDirUtil;
 import com.google.devtools.mobileharness.platform.android.xts.config.proto.ConfigurationProto.Configuration;
 import com.google.devtools.mobileharness.platform.android.xts.suite.SuiteCommon;
+import com.google.devtools.mobileharness.platform.android.xts.suite.SuiteTestFilter;
 import com.google.devtools.mobileharness.platform.android.xts.suite.retry.RetryArgs;
 import com.google.devtools.mobileharness.platform.android.xts.suite.retry.RetryGenerator;
 import com.google.devtools.mobileharness.platform.android.xts.suite.subplan.SubPlan;
@@ -101,17 +104,45 @@ public static boolean isSkippableException(MobileHarnessException e) {
    */
   public ImmutableList<JobInfo> createXtsTradefedTestJob(SessionRequestInfo sessionRequestInfo)
       throws MobileHarnessException, InterruptedException {
+    ImmutableSet<String> staticMctsModules = sessionRequestHandlerUtil.getStaticMctsModules();
     ImmutableList<String> tfModules =
         sessionRequestHandlerUtil.getFilteredTradefedModules(sessionRequestInfo);
+    ImmutableList<SuiteTestFilter> includeFilters =
+        sessionRequestInfo.includeFilters().stream()
+            .map(SuiteTestFilter::create)
+            .collect(toImmutableList());
 
     ImmutableList<TradefedJobInfo> tradefedJobInfoList =
         createXtsTradefedTestJobInfo(sessionRequestInfo, tfModules);
 
     ImmutableList.Builder<JobInfo> jobInfos = ImmutableList.builder();
+    XtsDynamicDownloadModuleMatchResult dynamicDownloadModuleMatchResult =
+        addEnableXtsDynamicDownloadToJob(
+            sessionRequestInfo, tfModules, staticMctsModules, includeFilters);
     for (TradefedJobInfo tradefedJobInfo : tradefedJobInfoList) {
-      JobInfo jobInfo =
-          sessionRequestHandlerUtil.createXtsTradefedTestJob(sessionRequestInfo, tradefedJobInfo);
-      jobInfos.add(jobInfo);
+      // Create two jobs if we enable dynamic download.
+      switch (dynamicDownloadModuleMatchResult) {
+        case SOME_MATCH:
+          // Create a job for non dynamic download test cases.
+          jobInfos.add(
+              createDynamicJobInfo(
+                  sessionRequestInfo, tradefedJobInfo, XtsConstants.STATIC_XTS_JOB));
+          // Create a job for dynamic download test cases.
+          jobInfos.add(
+              createDynamicJobInfo(
+                  sessionRequestInfo, tradefedJobInfo, XtsConstants.DYNAMIC_XTS_JOB));
+          break;
+        case ALL_MATCH:
+          // Create only one job for dynamic download test cases.
+          jobInfos.add(
+              createDynamicJobInfo(
+                  sessionRequestInfo, tradefedJobInfo, XtsConstants.DYNAMIC_XTS_JOB));
+          break;
+        default:
+          jobInfos.add(
+              sessionRequestHandlerUtil.createXtsTradefedTestJob(
+                  sessionRequestInfo, tradefedJobInfo));
+      }
     }
 
     return jobInfos.build();
@@ -412,6 +443,53 @@ private void injectBuildFingerprint(
         Job.PREV_SESSION_DEVICE_VENDOR_BUILD_FINGERPRINT, prevSessionDeviceVendorBuildFingerprint);
   }
 
+  private static XtsDynamicDownloadModuleMatchResult addEnableXtsDynamicDownloadToJob(
+      SessionRequestInfo sessionRequestInfo,
+      ImmutableList<String> tfModules,
+      ImmutableSet<String> staticMctsModules,
+      ImmutableList<SuiteTestFilter> includeFilters) {
+    if (sessionRequestInfo.isEnableXtsDynamicDownload().orElse(false)) {
+      // Consider independent two cases:
+      // 1. No -m and no --include-filter MCTS modules specified, dynamic download is disabled.
+      // 2. Some of the MCTS modules are specified, create a dynamic download job and a static job.
+      // 3. All of the MCTS modules are specified, create only one dynamic download job.
+      // 4. tfModules and includeFilters cannot be both non-empty.
+      if (!tfModules.isEmpty()) {
+        if (tfModules.stream().noneMatch(staticMctsModules::contains)) {
+          return XtsDynamicDownloadModuleMatchResult.NONE_MATCH;
+        } else if (tfModules.stream().allMatch(staticMctsModules::contains)) {
+          return XtsDynamicDownloadModuleMatchResult.ALL_MATCH;
+        } else {
+          return XtsDynamicDownloadModuleMatchResult.SOME_MATCH;
+        }
+      }
+      if (!includeFilters.isEmpty()) {
+        if (includeFilters.stream()
+            .noneMatch(filter -> staticMctsModules.stream().anyMatch(filter::matchModuleName))) {
+          return XtsDynamicDownloadModuleMatchResult.NONE_MATCH;
+        } else if (includeFilters.stream()
+            .allMatch(filter -> staticMctsModules.stream().anyMatch(filter::matchModuleName))) {
+          return XtsDynamicDownloadModuleMatchResult.ALL_MATCH;
+        } else {
+          return XtsDynamicDownloadModuleMatchResult.SOME_MATCH;
+        }
+      }
+      return XtsDynamicDownloadModuleMatchResult.SOME_MATCH;
+    } else {
+      return XtsDynamicDownloadModuleMatchResult.NONE_MATCH;
+    }
+  }
+
+  private JobInfo createDynamicJobInfo(
+      SessionRequestInfo sessionRequestInfo, TradefedJobInfo tradefedJobInfo, String jobType)
+      throws MobileHarnessException, InterruptedException {
+    JobInfo dynamicDownloadJobInfo =
+        sessionRequestHandlerUtil.createXtsTradefedTestJob(sessionRequestInfo, tradefedJobInfo);
+    dynamicDownloadJobInfo.properties().add(XtsConstants.IS_XTS_DYNAMIC_DOWNLOAD_ENABLED, "true");
+    dynamicDownloadJobInfo.properties().add(XtsConstants.XTS_DYNAMIC_DOWNLOAD_JOB_TYPE, jobType);
+    return dynamicDownloadJobInfo;
+  }
+
   protected static Properties loadTestReportProperties(Path testReportPropertiesFile)
       throws MobileHarnessException {
     Properties properties = new Properties();

src/java/com/google/devtools/mobileharness/infra/ats/console/controller/sessionplugin/BUILD

@@ -181,7 +181,6 @@ java_library(
         "//src/java/com/google/devtools/mobileharness/platform/android/xts/common/util:xts_constants",
         "//src/java/com/google/devtools/mobileharness/platform/android/xts/common/util:xts_dir_util",
         "//src/java/com/google/devtools/mobileharness/platform/android/xts/suite:suite_result_reporter",
-        "//src/java/com/google/devtools/mobileharness/platform/android/xts/suite:suite_test_filter",
         "//src/java/com/google/devtools/mobileharness/platform/android/xts/suite/retry:previous_result_loader",
         "//src/java/com/google/devtools/mobileharness/platform/android/xts/suite/retry:retry_type",
         "//src/java/com/google/devtools/mobileharness/shared/constant:log_record_importance",

src/java/com/google/devtools/mobileharness/infra/ats/console/controller/sessionplugin/RunCommandHandler.java

@@ -17,15 +17,13 @@
 package com.google.devtools.mobileharness.infra.ats.console.controller.sessionplugin;
 
 import static com.google.common.base.Ascii.toUpperCase;
-import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.devtools.mobileharness.shared.constant.LogRecordImportance.IMPORTANCE;
 import static com.google.devtools.mobileharness.shared.constant.LogRecordImportance.Importance.IMPORTANT;
 import static com.google.devtools.mobileharness.shared.util.base.ProtoTextFormat.shortDebugString;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
 import com.google.common.flogger.FluentLogger;
 import com.google.devtools.mobileharness.api.model.error.MobileHarnessException;
 import com.google.devtools.mobileharness.infra.ats.common.SessionRequestHandlerUtil;
@@ -45,7 +43,6 @@
 import com.google.devtools.mobileharness.platform.android.xts.common.util.XtsConstants;
 import com.google.devtools.mobileharness.platform.android.xts.common.util.XtsDirUtil;
 import com.google.devtools.mobileharness.platform.android.xts.suite.SuiteResultReporter;
-import com.google.devtools.mobileharness.platform.android.xts.suite.SuiteTestFilter;
 import com.google.devtools.mobileharness.platform.android.xts.suite.retry.PreviousResultLoader;
 import com.google.devtools.mobileharness.platform.android.xts.suite.retry.RetryType;
 import com.google.devtools.mobileharness.shared.util.file.local.LocalFileUtil;
@@ -135,14 +132,6 @@ ImmutableList<JobInfo> createTradefedJobs(RunCommand command)
       return ImmutableList.of();
     }
 
-    ImmutableSet<String> staticMctsModules = sessionRequestHandlerUtil.getStaticMctsModules();
-    ImmutableList<String> tfModules =
-        sessionRequestHandlerUtil.getFilteredTradefedModules(sessionRequestInfo);
-    ImmutableList<SuiteTestFilter> includeFilters =
-        sessionRequestInfo.includeFilters().stream()
-            .map(SuiteTestFilter::create)
-            .collect(toImmutableList());
-
     jobInfoList.forEach(
         jobInfo -> {
           jobInfo
@@ -155,8 +144,6 @@ ImmutableList<JobInfo> createTradefedJobs(RunCommand command)
                               .getSessionProperty(SESSION_PROPERTY_NAME_TIMESTAMP_DIR_NAME)
                               .orElseThrow())
                       .toString());
-          addEnableXtsDynamicDownloadToJob(
-              jobInfo, command, tfModules, staticMctsModules, includeFilters);
         });
     return jobInfoList;
   }
@@ -361,6 +348,9 @@ SessionRequestInfo generateSessionRequestInfo(RunCommand runCommand)
     if (runCommand.hasMaxSdkLevel()) {
       builder.setMaxSdkLevel(runCommand.getMaxSdkLevel());
     }
+    if (runCommand.getEnableXtsDynamicDownload()) {
+      builder.setIsEnableXtsDynamicDownload(runCommand.getEnableXtsDynamicDownload());
+    }
 
     sessionInfo
         .getSessionProperty(SessionProperties.PROPERTY_KEY_SESSION_CLIENT_ID)
@@ -384,29 +374,4 @@ private String createXtsTestResultSummary(
             : "")
         + "=================== End ====================\n";
   }
-
-  private static void addEnableXtsDynamicDownloadToJob(
-      JobInfo jobInfo,
-      RunCommand runCommand,
-      ImmutableList<String> tfModules,
-      ImmutableSet<String> staticMctsModules,
-      ImmutableList<SuiteTestFilter> includeFilters) {
-    if (runCommand.getEnableXtsDynamicDownload()) {
-      jobInfo.properties().add(XtsConstants.IS_XTS_DYNAMIC_DOWNLOAD_ENABLED, "true");
-    }
-    // Consider independent two cases:
-    // 1. No -m MCTS modules specified, dynamic download is disabled.
-    // 2. No include filtered MCTS modules, dynamic download is disabled.
-    if (!tfModules.isEmpty()) {
-      if (tfModules.stream().noneMatch(staticMctsModules::contains)) {
-        jobInfo.properties().add(XtsConstants.IS_XTS_DYNAMIC_DOWNLOAD_ENABLED, "false");
-      }
-    }
-    if (!includeFilters.isEmpty()) {
-      if (includeFilters.stream()
-          .noneMatch(filter -> staticMctsModules.stream().anyMatch(filter::matchModuleName))) {
-        jobInfo.properties().add(XtsConstants.IS_XTS_DYNAMIC_DOWNLOAD_ENABLED, "false");
-      }
-    }
-  }
 }

src/java/com/google/devtools/mobileharness/infra/controller/test/util/xtsdownloader/MctsDynamicDownloadPlugin.java

@@ -181,6 +181,11 @@ public XtsDynamicDownloadInfo parse(TestInfo test, String deviceId)
           String.format(
               "https://dl.google.com/dl/android/xts/mcts/tool/mcts_exclude/%s/%s/mcts-exclude.txt",
               aospVersion, preloadedMainlineVersion));
+      // Add the full MCTS list file link url to the second position of the list.
+      downloadLinkUrls.add(
+          String.format(
+              "https://dl.google.com/dl/android/xts/mcts/%s/%s/mcts_test_list.txt",
+              preloadedMainlineVersion, deviceAbi));
       for (String mctsNameAndVersioncode : mctsNamesOfPreloadedMainlineModules.get(PRELOADED_KEY)) {
         String moduleVersioncode =
             mctsNameAndVersioncode.substring(mctsNameAndVersioncode.indexOf(":") + 1);
@@ -214,8 +219,36 @@ public XtsDynamicDownloadInfo parse(TestInfo test, String deviceId)
   @Override
   public void downloadXtsFiles(XtsDynamicDownloadInfo xtsDynamicDownloadInfo, TestInfo testInfo)
       throws MobileHarnessException, InterruptedException {
-    Set<String> allTestModules = new HashSet<>();
     List<String> downloadUrlList = new ArrayList<>(xtsDynamicDownloadInfo.getDownloadUrlList());
+    // Download the MCTS full test list.
+    if (downloadUrlList.get(1).contains("mcts_test_list")) {
+      logger.atInfo().log("Start to download MCTS full test list.");
+      String mctsFullTestListUrl = downloadUrlList.get(1);
+      String mctsFullTestListFilePath =
+          downloadPublicUrlFiles(
+              mctsFullTestListUrl, mctsFullTestListUrl.replace("https://dl.google.com/dl", ""));
+      if (mctsFullTestListFilePath != null) {
+        testInfo
+            .properties()
+            .add(
+                XtsConstants.XTS_DYNAMIC_DOWNLOAD_PATH_TEST_LIST_PROPERTY_KEY,
+                mctsFullTestListFilePath);
+      }
+      downloadUrlList.remove(1);
+    }
+
+    // If the job type is static, skip downloading the MCTS files.
+    if (testInfo
+        .jobInfo()
+        .properties()
+        .get(XtsConstants.XTS_DYNAMIC_DOWNLOAD_JOB_TYPE)
+        .equals(XtsConstants.STATIC_XTS_JOB)) {
+      return;
+    }
+
+    // Download the MCTS files for dynamic download mcts job.
+    logger.atInfo().log("Start to download files for dynamic download MCTS job...");
+    Set<String> allTestModules = new HashSet<>();
     Set<String> excludeTestModules = new HashSet<>();
     // Get the exclude test modules.
     if (downloadUrlList.get(0).contains("mcts_exclude")) {

src/java/com/google/devtools/mobileharness/platform/android/xts/common/util/BUILD

@@ -53,6 +53,7 @@ java_library(
         "//src/java/com/google/devtools/mobileharness/infra/controller/test/util/xtsdownloader:__subpackages__",
         "//src/java/com/google/wireless/qa/mobileharness/shared/api/driver:__subpackages__",
         "//src/javatests/com/google/devtools/mobileharness/infra/ats:__subpackages__",
+        "//src/javatests/com/google/devtools/mobileharness/infra/controller/test/util/xtsdownloader:__subpackages__",
     ],
 )
 

src/java/com/google/devtools/mobileharness/platform/android/xts/common/util/XtsConstants.java

@@ -48,15 +48,35 @@ public class XtsConstants {
   /** A MH job property key to indicate whether xTS dynamic download is enabled. */
   public static final String IS_XTS_DYNAMIC_DOWNLOAD_ENABLED = "is_xts_dynamic_download_enabled";
 
+  /** The job type of dynamic download xTS jobs. */
+  public static final String XTS_DYNAMIC_DOWNLOAD_JOB_TYPE = "xts_dynamic_download_job_type";
+
+  public static final String STATIC_XTS_JOB = "static";
+  public static final String DYNAMIC_XTS_JOB = "dynamic";
+
   /** MH test property keys of path relative to the test temp dir. */
   public static final String XTS_DYNAMIC_DOWNLOAD_PATH_TEST_PROPERTY_KEY =
       "xts_dynamic_download_path";
 
+  /** MH test property keys of path relative to the test temp dir. */
+  public static final String XTS_DYNAMIC_DOWNLOAD_PATH_TEST_LIST_PROPERTY_KEY =
+      "xts_dynamic_download_test_list_path";
+
   public static final String XTS_DYNAMIC_DOWNLOAD_PATH_JDK_PROPERTY_KEY =
       "xts_dynamic_download_jdk_path";
 
   public static final Pattern RESULT_ZIP_FILENAME_PATTERN =
       Pattern.compile("^\\d{4}\\.\\d{2}\\.\\d{2}_\\d{2}\\.\\d{2}\\.\\d{2}\\.\\d{3}_\\d{4}\\.zip$");
 
+  /** The match result of filtered TF modules to define if dynamic download is enabled. */
+  public enum XtsDynamicDownloadModuleMatchResult {
+    /** All the filtered TF modules match the dynamic download modules. */
+    ALL_MATCH,
+    /** Some of the filtered TF modules match the dynamic download modules. */
+    SOME_MATCH,
+    /** None of the filtered TF modules match the dynamic download modules. */
+    NONE_MATCH,
+  }
+
   private XtsConstants() {}
 }