[DON'T MERGE] Stop listing all files POC

backend/core/src/main/java/org/sonarsource/sonarlint/core/ConnectionSuggestionProvider.java

@@ -20,6 +20,7 @@
 package org.sonarsource.sonarlint.core;
 
 import jakarta.inject.Inject;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -87,10 +88,7 @@ public void filesystemUpdated(FileSystemUpdatedEvent event) {
   @EventListener
   public void configurationScopesAdded(ConfigurationScopesAddedWithBindingEvent event) {
     var listConfigScopeIds = event.getConfigScopeIds().stream()
-      .map(clientFs::getFiles)
-      .flatMap(List::stream)
-      .filter(f -> ALL_BINDING_CLUE_FILENAMES.contains(f.getFileName()) || f.isSonarlintConfigurationFile())
-      .map(ClientFile::getConfigScopeId)
+      .filter(this::hasBindingClueFiles)
       .collect(Collectors.toSet());
 
     if (!listConfigScopeIds.isEmpty()) {
@@ -100,6 +98,22 @@ public void configurationScopesAdded(ConfigurationScopesAddedWithBindingEvent ev
     }
   }
 
+  /**
+   * Check if a configuration scope has binding clue files using lazy loading.
+   * Avoids loading all files by checking specific file paths.
+   */
+  private boolean hasBindingClueFiles(String configScopeId) {
+    // Check for sonar-project.properties and .sonarcloud.properties
+    for (String filename : ALL_BINDING_CLUE_FILENAMES) {
+      if (clientFs.getFileByIdePath(configScopeId, Path.of(filename)) != null) {
+        return true;
+      }
+    }
+    // Check for .sonarlint/*.json files
+    var sonarlintConfigFiles = clientFs.getFilesByPattern(configScopeId, ".sonarlint/*.json");
+    return !sonarlintConfigFiles.isEmpty();
+  }
+
   private void queueConnectionSuggestion(Set<String> listConfigScopeIds) {
     if (!listConfigScopeIds.isEmpty()) {
       var cancelMonitor = new SonarLintCancelMonitor();

backend/core/src/main/java/org/sonarsource/sonarlint/core/embedded/server/handler/ShowFixSuggestionRequestHandler.java

@@ -127,11 +127,11 @@ private boolean doesClientFileExists(String configScopeId, String filePath, Coll
     if (optTranslation.isPresent()) {
       var translation = optTranslation.get();
       var idePath = translation.serverToIdePath(Paths.get(filePath));
+      // Use lazy loading instead of iterating over all files
       for (var scope : boundScopes) {
-        for (var file : clientFs.getFiles(scope)) {
-          if (Path.of(file.getUri()).endsWith(idePath)) {
-            return true;
-          }
+        var file = clientFs.getFileByIdePath(scope, idePath);
+        if (file != null) {
+          return true;
         }
       }
     }

backend/core/src/main/java/org/sonarsource/sonarlint/core/file/PathTranslationService.java

@@ -32,7 +32,6 @@
 import org.sonarsource.sonarlint.core.commons.progress.SonarLintCancelMonitor;
 import org.sonarsource.sonarlint.core.event.BindingConfigChangedEvent;
 import org.sonarsource.sonarlint.core.event.ConfigurationScopeRemovedEvent;
-import org.sonarsource.sonarlint.core.fs.ClientFile;
 import org.sonarsource.sonarlint.core.fs.ClientFileSystemService;
 import org.sonarsource.sonarlint.core.repository.config.ConfigurationRepository;
 import org.sonarsource.sonarlint.core.serverconnection.prefix.FileTreeMatcher;
@@ -75,13 +74,20 @@ private FilePathTranslation computePaths(String configScopeId, SonarLintCancelMo
 
   private FilePathTranslation matchPaths(String configScopeId, FileTreeMatcher fileMatcher, List<Path> serverFilePaths) {
     LOG.debug("Starting matching paths for config scope '{}'...", configScopeId);
-    var localFilePaths = clientFs.getFiles(configScopeId);
-    if (localFilePaths.isEmpty()) {
-      LOG.debug("No client files for config scope '{}'. Skipping path matching.", configScopeId);
+    // Use directories instead of files for path matching optimization
+    var localDirectories = clientFs.getDirectories(configScopeId);
+    if (localDirectories.isEmpty()) {
+      LOG.debug("No client directories for config scope '{}'. Skipping path matching.", configScopeId);
       // Maybe a config scope without files, or the filesystem has not been initialized yet
       return new FilePathTranslation(Paths.get(""), Paths.get(""));
     }
-    var match = fileMatcher.match(serverFilePaths, localFilePaths.stream().map(ClientFile::getClientRelativePath).toList());
+    // Convert server file paths to directory paths
+    var serverDirectories = serverFilePaths.stream()
+      .map(Path::getParent)
+      .filter(java.util.Objects::nonNull)
+      .distinct()
+      .toList();
+    var match = fileMatcher.match(serverDirectories, localDirectories);
     LOG.debug("Matched paths for config scope '{}':\n  * idePrefix={}\n  * serverPrefix={}", configScopeId, match.idePrefix(), match.sqPrefix());
     return new FilePathTranslation(match.idePrefix(), match.sqPrefix());
   }

backend/core/src/main/java/org/sonarsource/sonarlint/core/fs/ClientFileSystemService.java

@@ -37,10 +37,16 @@
 import org.sonarsource.sonarlint.core.commons.log.SonarLintLogger;
 import org.sonarsource.sonarlint.core.commons.progress.SonarLintCancelMonitor;
 import org.sonarsource.sonarlint.core.event.ConfigurationScopeRemovedEvent;
+import org.sonarsource.sonarlint.core.event.ConfigurationScopesAddedWithBindingEvent;
 import org.sonarsource.sonarlint.core.rpc.protocol.SonarLintRpcClient;
 import org.sonarsource.sonarlint.core.rpc.protocol.backend.file.DidUpdateFileSystemParams;
 import org.sonarsource.sonarlint.core.rpc.protocol.client.fs.GetBaseDirParams;
+import org.sonarsource.sonarlint.core.rpc.protocol.client.fs.GetClientFileParams;
+import org.sonarsource.sonarlint.core.rpc.protocol.client.fs.GetFileByIdePathParams;
+import org.sonarsource.sonarlint.core.rpc.protocol.client.fs.GetFilesByPatternParams;
+import org.sonarsource.sonarlint.core.rpc.protocol.client.fs.ListDirectoriesParams;
 import org.sonarsource.sonarlint.core.rpc.protocol.client.fs.ListFilesParams;
+import org.sonarsource.sonarlint.core.rpc.protocol.client.fs.WarmUpFileSystemCacheParams;
 import org.sonarsource.sonarlint.core.rpc.protocol.common.ClientFileDto;
 import org.sonarsource.sonarlint.core.telemetry.TelemetryService;
 import org.springframework.context.ApplicationEventPublisher;
@@ -100,13 +106,39 @@ private static Charset charsetFromDto(@Nullable String dtoCharset) {
     }
   }
 
+  /**
+   * @deprecated Use getFileByIdePath for specific files instead. This method will try lazy loading first.
+   */
+  @Deprecated(since = "10.12", forRemoval = true)
   public List<ClientFile> findFilesByNamesInScope(String configScopeId, List<String> filenames) {
-    return getFiles(configScopeId).stream()
-      .filter(f -> filenames.contains(f.getClientRelativePath().getFileName().toString()))
-      .toList();
+    // Try lazy loading first
+    var result = new ArrayList<ClientFile>();
+    for (String filename : filenames) {
+      var file = getFileByIdePath(configScopeId, Path.of(filename));
+      if (file != null) {
+        result.add(file);
+      }
+    }
+    // Fall back to cached files if lazy loading didn't find anything
+    if (result.isEmpty()) {
+      return getFiles(configScopeId).stream()
+        .filter(f -> filenames.contains(f.getClientRelativePath().getFileName().toString()))
+        .toList();
+    }
+    return result;
   }
 
+  /**
+   * @deprecated Use getFilesByPattern for .sonarlint/**\/*.json instead. This method will try lazy loading first.
+   */
+  @Deprecated(since = "10.12", forRemoval = true)
   public List<ClientFile> findSonarlintConfigurationFilesByScope(String configScopeId) {
+    // Try lazy loading first with glob pattern for .sonarlint/**/*.json
+    var lazyFiles = getFilesByPattern(configScopeId, ".sonarlint/**/*.json");
+    if (!lazyFiles.isEmpty()) {
+      return lazyFiles;
+    }
+    // Fall back to cached files
     return getFiles(configScopeId).stream()
       .filter(ClientFile::isSonarlintConfigurationFile)
       .toList();
@@ -238,4 +270,176 @@ public Map<String, Set<URI>> groupFilesByConfigScope(Set<URI> fileUris) {
       ));
   }
 
+  /**
+   * Get files matching a glob pattern using lazy loading.
+   * Tries the new RPC method first, falls back to cache.
+   * @since 10.12
+   */
+  public List<ClientFile> getFilesByPattern(String configScopeId, String globPattern) {
+    try {
+      var response = rpcClient.getFilesByPattern(new GetFilesByPatternParams(configScopeId, globPattern)).join();
+      var files = response.getFiles();
+      if (!files.isEmpty()) {
+        // Cache the files we just loaded
+        addFilesToVirtualFileSystem(configScopeId, files);
+        return files.stream().map(ClientFileSystemService::fromDto).toList();
+      }
+    } catch (Exception e) {
+      logDebug("Error getting files by pattern, falling back to cache", e);
+    }
+    // Fall back to cache
+    return getFiles(configScopeId).stream()
+      .filter(f -> f.getClientRelativePath().toString().matches(globToRegex(globPattern)))
+      .toList();
+  }
+
+  private void addFilesToVirtualFileSystem(String configScopeId, List<ClientFileDto> files) {
+    files.forEach(dto -> {
+      var clientFile = fromDto(dto);
+      filesByUri.put(dto.getUri(), clientFile);
+      var byScope = filesByConfigScopeIdCache.get(configScopeId);
+      if (byScope != null) {
+        byScope.put(dto.getUri(), clientFile);
+      }
+    });
+  }
+
+  /**
+   * Get a file by its IDE path using lazy loading.
+   * Tries the new RPC method first, falls back to cache.
+   * @since 10.12
+   */
+  @CheckForNull
+  public ClientFile getFileByIdePath(String configScopeId, Path filePath) {
+    try {
+      var response = rpcClient.getFileByIdePath(new GetFileByIdePathParams(configScopeId, filePath)).join();
+      var fileDto = response.getFile();
+      if (fileDto != null) {
+        var clientFile = fromDto(fileDto);
+        // Cache the file we just loaded
+        filesByUri.put(fileDto.getUri(), clientFile);
+        var byScope = filesByConfigScopeIdCache.get(configScopeId);
+        if (byScope != null) {
+          byScope.put(fileDto.getUri(), clientFile);
+        }
+        return clientFile;
+      }
+    } catch (Exception e) {
+      logDebug("Error getting file by IDE path, falling back to cache", e);
+    }
+    // Fall back to cache
+    return getFiles(configScopeId).stream()
+      .filter(f -> f.getClientRelativePath().equals(filePath))
+      .findFirst()
+      .orElse(null);
+  }
+
+  /**
+   * Check if a file exists using lazy loading.
+   * Tries the new RPC method first, falls back to cache.
+   * @since 10.12
+   */
+  public boolean doesFileExist(URI fileUri) {
+    // First check cache
+    if (filesByUri.containsKey(fileUri)) {
+      return true;
+    }
+    // Try lazy loading
+    try {
+      var response = rpcClient.getClientFile(new GetClientFileParams(fileUri)).join();
+      var fileDto = response.getFile();
+      if (fileDto != null) {
+        var clientFile = fromDto(fileDto);
+        // Cache the file we just loaded
+        filesByUri.put(fileDto.getUri(), clientFile);
+        var byScope = filesByConfigScopeIdCache.get(clientFile.getConfigScopeId());
+        if (byScope != null) {
+          byScope.put(fileDto.getUri(), clientFile);
+        }
+        return true;
+      }
+    } catch (Exception e) {
+      logDebug("Error checking file existence", e);
+    }
+    return false;
+  }
+
+  /**
+   * Get directories (not files) for a configuration scope using lazy loading.
+   * Tries the new RPC method first, falls back to extracting directories from cached files.
+   * @since 10.12
+   */
+  public List<Path> getDirectories(String configScopeId) {
+    try {
+      var response = rpcClient.listDirectories(new ListDirectoriesParams(configScopeId)).join();
+      var directories = response.getDirectories();
+      if (!directories.isEmpty()) {
+        return directories.stream()
+          .map(dto -> fromDto(dto).getClientRelativePath())
+          .toList();
+      }
+    } catch (Exception e) {
+      logDebug("Error getting directories, falling back to cache", e);
+    }
+    // Fall back to extracting directories from cached files
+    return getFiles(configScopeId).stream()
+      .map(ClientFile::getClientRelativePath)
+      .map(Path::getParent)
+      .filter(Objects::nonNull)
+      .distinct()
+      .toList();
+  }
+
+  /**
+   * Listen for configuration scopes added and trigger cache warmup.
+   * @since 10.12
+   */
+  @EventListener
+  public void onConfigurationScopesAdded(ConfigurationScopesAddedWithBindingEvent event) {
+    var configScopeIds = event.getConfigScopeIds();
+    if (!configScopeIds.isEmpty()) {
+      logDebug("Triggering cache warmup for {} newly added configuration scopes", configScopeIds.size());
+      requestCacheWarmup(configScopeIds);
+    }
+  }
+
+  /**
+   * Request asynchronous cache warmup for the given configuration scopes.
+   * @since 10.12
+   */
+  public void requestCacheWarmup(Set<String> configScopeIds) {
+    try {
+      rpcClient.warmUpFileSystemCache(new WarmUpFileSystemCacheParams(configScopeIds));
+      logDebug("Requested cache warmup for {} configuration scopes", configScopeIds.size());
+    } catch (Exception e) {
+      logDebug("Error requesting cache warmup", e);
+    }
+  }
+
+  /**
+   * Submit a chunk of files to the cache during warmup.
+   * @since 10.12
+   */
+  public void submitFileCacheChunk(String configScopeId, List<ClientFileDto> files) {
+    logDebug("Received cache chunk with {} files for config scope '{}'", files.size(), configScopeId);
+    addFilesToVirtualFileSystem(configScopeId, files);
+  }
+
+  /**
+   * Simple glob to regex conversion for pattern matching.
+   */
+  private static String globToRegex(String glob) {
+    return glob.replace(".", "\\.")
+      .replace("*", ".*")
+      .replace("?", ".");
+  }
+
+  private static void logDebug(String message, Object... args) {
+    try {
+      LOG.debug(message, args);
+    } catch (Exception ignored) {
+      // Logging might not be configured yet during initialization
+    }
+  }
+
 }

backend/core/src/main/java/org/sonarsource/sonarlint/core/fs/FileExclusionService.java

@@ -98,11 +98,18 @@ public FileExclusionService(ConfigurationRepository configRepo, StorageService s
   }
 
   public boolean computeIfExcluded(URI fileUri, SonarLintCancelMonitor cancelMonitor) {
-    LOG.debug("Computing file exclusion for uri '{}'", fileUri);
     var clientFile = clientFileSystemService.getClientFile(fileUri);
     if (clientFile == null) {
-      LOG.debug("Unable to find client file for uri {}", fileUri);
-      return false;
+      // With lazy file system initialization, file might not be cached yet
+      // Ask the client if it exists before giving up
+      if (clientFileSystemService.doesFileExist(fileUri)) {
+        // Try to get it again now that we've loaded it
+        clientFile = clientFileSystemService.getClientFile(fileUri);
+      }
+      if (clientFile == null) {
+        LOG.debug("Unable to find client file for uri {}", fileUri);
+        return false;
+      }
     }
     var configScope = clientFile.getConfigScopeId();
     var effectiveBindingOpt = configRepo.getEffectiveBinding(configScope);
@@ -140,7 +147,6 @@ public boolean computeIfExcluded(URI fileUri, SonarLintCancelMonitor cancelMonit
     }
     var type = clientFile.isTest() ? InputFile.Type.TEST : InputFile.Type.MAIN;
     var result = !exclusionFilters.accept(serverPath.toString(), type);
-    LOG.debug("File exclusion for uri '{}' is {}", fileUri, result);
     return result;
   }
 

backend/core/src/main/java/org/sonarsource/sonarlint/core/remediation/aicodefix/AiCodeFixService.java

@@ -209,7 +209,13 @@ private AiSuggestionRequestBodyDto toDto(@Nullable String organizationKey, Strin
     // this is not perfect, the file content might have changed since the issue was detected
     var clientFile = clientFileSystemService.getClientFile(raisedIssue.fileUri());
     if (clientFile == null) {
-      throw new ResponseErrorException(new ResponseError(FILE_NOT_FOUND, "The provided issue ID corresponds to an unknown file", null));
+      // With lazy file system initialization, check if file exists before throwing error
+      if (clientFileSystemService.doesFileExist(raisedIssue.fileUri())) {
+        clientFile = clientFileSystemService.getClientFile(raisedIssue.fileUri());
+      }
+      if (clientFile == null) {
+        throw new ResponseErrorException(new ResponseError(FILE_NOT_FOUND, "The provided issue ID corresponds to an unknown file", null));
+      }
     }
     var issue = raisedIssue.issueDto();
     // the text range presence was checked earlier
@@ -225,6 +231,10 @@ private AiSuggestionRequestBodyDto toDto(@Nullable String organizationKey, Strin
     if (baseDir != null) {
       var fileUri = baseDir.resolve(taint.getIdeFilePath()).toUri();
       clientFile = clientFileSystemService.getClientFile(fileUri);
+      if (clientFile == null && clientFileSystemService.doesFileExist(fileUri)) {
+        // With lazy file system initialization, check if file exists before throwing error
+        clientFile = clientFileSystemService.getClientFile(fileUri);
+      }
     }
     if (clientFile == null) {
       throw new ResponseErrorException(new ResponseError(FILE_NOT_FOUND, "The provided taint ID corresponds to an unknown file", null));