Remove ASTUnit from indexer and clean up

Author: MaskRay

index_tests/_empty_test.cc

@@ -1,10 +1,4 @@
 /*
 OUTPUT:
-{
-  "includes": [],
-  "skipped_ranges": [],
-  "usr2func": [],
-  "usr2type": [],
-  "usr2var": []
-}
+{}
 */

src/clang_complete.cc

@@ -9,7 +9,6 @@
 #include "platform.h"
 
 #include <clang/Frontend/CompilerInstance.h>
-#include <clang/Frontend/FrontendDiagnostic.h>
 #include <clang/Lex/PreprocessorOptions.h>
 #include <clang/Sema/CodeCompleteConsumer.h>
 #include <llvm/ADT/Twine.h>
@@ -385,10 +384,6 @@ class CaptureCompletionResults : public CodeCompleteConsumer {
     }
   }
 
-  void ProcessOverloadCandidates(Sema &S, unsigned CurrentArg,
-                                 OverloadCandidate *Candidates,
-                                 unsigned NumCandidates) override {}
-
   CodeCompletionAllocator &getAllocator() override { return *Alloc; }
 
   CodeCompletionTUInfo &getCodeCompletionTUInfo() override { return CCTUInfo; }
@@ -472,30 +467,10 @@ class StoreDiags : public DiagnosticConsumer {
   }
 };
 
-std::unique_ptr<CompilerInvocation>
-buildCompilerInvocation(const std::vector<std::string> &args,
-                        IntrusiveRefCntPtr<vfs::FileSystem> VFS) {
-  std::vector<const char *> cargs;
-  for (auto &arg : args)
-    cargs.push_back(arg.c_str());
-  IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
-      CompilerInstance::createDiagnostics(new DiagnosticOptions));
-  std::unique_ptr<CompilerInvocation> CI =
-      createInvocationFromCommandLine(cargs, Diags, VFS);
-  if (CI) {
-    CI->getFrontendOpts().DisableFree = false;
-    CI->getLangOpts()->CommentOpts.ParseAllComments = true;
-    CI->getLangOpts()->SpellChecking = false;
-  }
-  return CI;
-}
-
-std::unique_ptr<CompilerInstance>
-BuildCompilerInstance(CompletionSession &session,
-                      std::unique_ptr<CompilerInvocation> CI,
-  DiagnosticConsumer &DC,
-                      const WorkingFiles::Snapshot &snapshot,
-                      std::vector<std::unique_ptr<llvm::MemoryBuffer>> &Bufs) {
+std::unique_ptr<CompilerInstance> BuildCompilerInstance(
+    CompletionSession &session, std::unique_ptr<CompilerInvocation> CI,
+    DiagnosticConsumer &DC, const WorkingFiles::Snapshot &snapshot,
+    std::vector<std::unique_ptr<llvm::MemoryBuffer>> &Bufs) {
   for (auto &file : snapshot.files) {
     Bufs.push_back(llvm::MemoryBuffer::getMemBuffer(file.content));
     if (file.filename == session.file.filename) {
@@ -558,7 +533,7 @@ void CompletionPreloadMain(ClangCompleteManager *completion_manager) {
 
     LOG_S(INFO) << "create completion session for " << session->file.filename;
     if (std::unique_ptr<CompilerInvocation> CI =
-            buildCompilerInvocation(args, session->FS))
+            BuildCompilerInvocation(args, session->FS))
       session->BuildPreamble(*CI);
   }
 }
@@ -583,10 +558,12 @@ void CompletionQueryMain(ClangCompleteManager *completion_manager) {
                                           true /*create_if_needed*/);
 
     std::unique_ptr<CompilerInvocation> CI =
-        buildCompilerInvocation(session->file.args, session->FS);
+        BuildCompilerInvocation(session->file.args, session->FS);
     if (!CI)
       continue;
+    CI->getDiagnosticOpts().IgnoreWarnings = true;
     clang::CodeCompleteOptions CCOpts;
+    CCOpts.IncludeBriefComments = true;
 #if LLVM_VERSION_MAJOR >= 7
     CCOpts.IncludeFixIts = true;
 #endif
@@ -596,6 +573,8 @@ void CompletionQueryMain(ClangCompleteManager *completion_manager) {
     FOpts.CodeCompletionAt.FileName = session->file.filename;
     FOpts.CodeCompletionAt.Line = request->position.line + 1;
     FOpts.CodeCompletionAt.Column = request->position.character + 1;
+    FOpts.SkipFunctionBodies = true;
+    CI->getLangOpts()->CommentOpts.ParseAllComments = true;
 
     StoreDiags DC;
     WorkingFiles::Snapshot snapshot =
@@ -629,7 +608,7 @@ void DiagnosticQueryMain(ClangCompleteManager *manager) {
         path, true /*mark_as_completion*/, true /*create_if_needed*/);
 
     std::unique_ptr<CompilerInvocation> CI =
-        buildCompilerInvocation(session->file.args, session->FS);
+        BuildCompilerInvocation(session->file.args, session->FS);
     if (!CI)
       continue;
     StoreDiags DC;
@@ -690,6 +669,7 @@ void CompletionSession::BuildPreamble(CompilerInvocation &CI) {
     return;
   CI.getFrontendOpts().SkipFunctionBodies = true;
   CI.getLangOpts()->RetainCommentsFromSystemHeaders = true;
+  CI.getLangOpts()->CommentOpts.ParseAllComments = true;
 #if LLVM_VERSION_MAJOR >= 7
   CI.getPreprocessorOpts().WriteCommentListToPCH = false;
 #endif

src/clang_tu.cc

@@ -51,3 +51,20 @@ Range FromTokenRange(const SourceManager &SM, const LangOptions &LangOpts,
   return FromCharSourceRange(SM, LangOpts, CharSourceRange::getTokenRange(R),
                              UniqueID);
 }
+
+std::unique_ptr<CompilerInvocation>
+BuildCompilerInvocation(const std::vector<std::string> &args,
+                        IntrusiveRefCntPtr<vfs::FileSystem> VFS) {
+  std::vector<const char *> cargs;
+  for (auto &arg : args)
+    cargs.push_back(arg.c_str());
+  IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
+      CompilerInstance::createDiagnostics(new DiagnosticOptions));
+  std::unique_ptr<CompilerInvocation> CI =
+      createInvocationFromCommandLine(cargs, Diags, VFS);
+  if (CI) {
+    CI->getFrontendOpts().DisableFree = false;
+    CI->getLangOpts()->SpellChecking = false;
+  }
+  return CI;
+}

src/clang_tu.h

@@ -6,6 +6,7 @@
 
 #include <clang/Basic/LangOptions.h>
 #include <clang/Basic/SourceManager.h>
+#include <clang/Frontend/CompilerInstance.h>
 
 #include <stdlib.h>
 
@@ -21,3 +22,7 @@ Range FromCharRange(const clang::SourceManager &SM,
 Range FromTokenRange(const clang::SourceManager &SM,
                      const clang::LangOptions &LangOpts, clang::SourceRange R,
                      llvm::sys::fs::UniqueID *UniqueID = nullptr);
+
+std::unique_ptr<clang::CompilerInvocation>
+BuildCompilerInvocation(const std::vector<std::string> &args,
+                        llvm::IntrusiveRefCntPtr<clang::vfs::FileSystem> VFS);

src/indexer.cc

@@ -10,8 +10,6 @@
 using ccls::Intern;
 
 #include <clang/AST/AST.h>
-#include <clang/Frontend/ASTUnit.h>
-#include <clang/Frontend/CompilerInstance.h>
 #include <clang/Frontend/FrontendAction.h>
 #include <clang/Index/IndexDataConsumer.h>
 #include <clang/Index/IndexingAction.h>
@@ -45,13 +43,10 @@ struct IndexParam {
   };
   std::unordered_map<const Decl *, DeclInfo> Decl2Info;
 
-  ASTUnit &Unit;
   ASTContext *Ctx;
-
   FileConsumer *file_consumer = nullptr;
 
-  IndexParam(ASTUnit &Unit, FileConsumer *file_consumer)
-      : Unit(Unit), file_consumer(file_consumer) {}
+  IndexParam(FileConsumer *file_consumer) : file_consumer(file_consumer) {}
 
   IndexFile *ConsumeFile(const FileEntry &File) {
     IndexFile *db = file_consumer->TryConsumeFile(File, &file_contents);
@@ -1158,14 +1153,9 @@ Index(VFS *vfs, const std::string &opt_wdir, const std::string &file,
   if (!g_config->index.enabled)
     return {};
 
-  std::vector<const char *> Args;
-  for (auto &arg : args)
-    Args.push_back(arg.c_str());
-  auto PCHCO = std::make_shared<PCHContainerOperations>();
-  IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
-      CompilerInstance::createDiagnostics(new DiagnosticOptions));
-  std::shared_ptr<CompilerInvocation> CI =
-      createInvocationFromCommandLine(Args, Diags);
+  auto PCH = std::make_shared<PCHContainerOperations>();
+  llvm::IntrusiveRefCntPtr<vfs::FileSystem> FS = vfs::getRealFileSystem();
+  std::shared_ptr<CompilerInvocation> CI = BuildCompilerInvocation(args, FS);
   if (!CI)
     return {};
   // -fparse-all-comments enables documentation in the indexer and in
@@ -1175,20 +1165,18 @@ Index(VFS *vfs, const std::string &opt_wdir, const std::string &file,
   CI->getLangOpts()->RetainCommentsFromSystemHeaders = true;
   CI->getLangOpts()->SpellChecking = false;
 
-  std::vector<std::unique_ptr<llvm::MemoryBuffer>> BufOwner;
-  for (auto &c : file_contents) {
-    std::unique_ptr<llvm::MemoryBuffer> MB =
-        llvm::MemoryBuffer::getMemBufferCopy(c.content, c.path);
-    CI->getPreprocessorOpts().addRemappedFile(c.path, MB.get());
-    BufOwner.push_back(std::move(MB));
-  }
-
-  auto Unit = ASTUnit::create(CI, Diags, true, true);
-  if (!Unit)
+  DiagnosticConsumer DC;
+  auto Clang = std::make_unique<CompilerInstance>(PCH);
+  Clang->setInvocation(std::move(CI));
+  Clang->setVirtualFileSystem(FS);
+  Clang->createDiagnostics(&DC, false);
+  Clang->setTarget(TargetInfo::CreateTargetInfo(
+      Clang->getDiagnostics(), Clang->getInvocation().TargetOpts));
+  if (!Clang->hasTarget())
     return {};
 
   FileConsumer file_consumer(vfs, file);
-  IndexParam param(*Unit, &file_consumer);
+  IndexParam param(&file_consumer);
   auto DataConsumer = std::make_shared<IndexDataConsumer>(param);
 
   index::IndexingOptions IndexOpts;
@@ -1199,36 +1187,30 @@ Index(VFS *vfs, const std::string &opt_wdir, const std::string &file,
   IndexOpts.IndexImplicitInstantiation = true;
 #endif
 
-  std::unique_ptr<FrontendAction> IndexAction = createIndexingAction(
+  std::unique_ptr<FrontendAction> Action = createIndexingAction(
       DataConsumer, IndexOpts, std::make_unique<IndexFrontendAction>(param));
 
-  DiagnosticErrorTrap DiagTrap(*Diags);
-  llvm::CrashRecoveryContext CRC;
-  auto compile = [&]() {
-    ASTUnit::LoadFromCompilerInvocationAction(
-        std::move(CI), PCHCO, Diags, IndexAction.get(), Unit.get(),
-        /*Persistent=*/true, /*ResourceDir=*/"",
-        /*OnlyLocalDecls=*/true,
-        /*CaptureDiagnostics=*/true, 0, false, false,
-        /*UserFilesAreVolatile=*/true);
-  };
-  if (!CRC.RunSafely(compile)) {
-    LOG_S(ERROR) << "clang crashed for " << file;
-    return {};
+  bool ok = false;
+  {
+    llvm::CrashRecoveryContext CRC;
+    auto parse = [&]() {
+                   if (!Action->BeginSourceFile(*Clang, Clang->getFrontendOpts().Inputs[0]))
+                     return;
+                   if (!Action->Execute())
+                     return;
+                   Action->EndSourceFile();
+                   ok = true;
+                 };
+    if (!CRC.RunSafely(parse)) {
+      LOG_S(ERROR) << "clang crashed for " << file;
+      return {};
+    }
   }
-  if (!Unit) {
+  if (!ok) {
     LOG_S(ERROR) << "failed to index " << file;
     return {};
   }
 
-  const SourceManager &SM = Unit->getSourceManager();
-  const FileEntry *FE = SM.getFileEntryForID(SM.getMainFileID());
-  IndexFile *main_file = param.ConsumeFile(*FE);
-  std::unordered_map<std::string, int> inc_to_line;
-  if (main_file)
-    for (auto &inc : main_file->includes)
-      inc_to_line[inc.resolved_path] = inc.line;
-
   auto result = param.file_consumer->TakeLocalState();
   for (std::unique_ptr<IndexFile> &entry : result) {
     entry->import_file = file;
@@ -1251,22 +1233,6 @@ Index(VFS *vfs, const std::string &opt_wdir, const std::string &file,
     for (auto &it : entry->usr2var)
       Uniquify(it.second.uses);
 
-    if (main_file) {
-      // If there are errors, show at least one at the include position.
-      auto it = inc_to_line.find(entry->path);
-      if (it != inc_to_line.end()) {
-        int line = it->second;
-        for (auto ls_diagnostic : entry->diagnostics_) {
-          if (ls_diagnostic.severity != lsDiagnosticSeverity::Error)
-            continue;
-          ls_diagnostic.range =
-              lsRange{lsPosition{line, 10}, lsPosition{line, 10}};
-          main_file->diagnostics_.push_back(ls_diagnostic);
-          break;
-        }
-      }
-    }
-
     // Update file contents and modification time.
     entry->last_write_time = param.file2write_time[entry->path];
 

src/indexer.h

@@ -251,8 +251,6 @@ struct IndexFile {
   std::unordered_map<Usr, IndexType> usr2type;
   std::unordered_map<Usr, IndexVar> usr2var;
 
-  // Diagnostics found when indexing this file. Not serialized.
-  std::vector<lsDiagnostic> diagnostics_;
   // File contents at the time of index. Not serialized.
   std::string file_contents;
 

src/pipeline.cc

@@ -243,12 +243,6 @@ bool Indexer_Parse(DiagnosticsPublisher *diag_pub, WorkingFiles *working_files,
   }
 
   for (std::unique_ptr<IndexFile> &curr : indexes) {
-    // Only emit diagnostics for non-interactive sessions, which makes it easier
-    // to identify indexing problems. For interactive sessions, diagnostics are
-    // handled by code completion.
-    if (!request.is_interactive)
-      diag_pub->Publish(working_files, curr->path, curr->diagnostics_);
-
     std::string path = curr->path;
     if (!(vfs->Stamp(path, curr->last_write_time) || path == path_to_index))
       continue;

src/project.cc

@@ -116,9 +116,6 @@ struct ProjectProcessor {
 
     args.push_back("-resource-dir=" + g_config->clang.resourceDir);
     args.push_back("-working-directory=" + entry.directory);
-    // There could be a clang version mismatch between what the project uses and
-    // what ccls uses. Make sure we do not emit warnings for mismatched options.
-    args.push_back("-Wno-unknown-warning-option");
 
     if (!command_set.insert(hash).second) {
       entry.args = std::move(args);

src/test.cc

@@ -295,42 +295,8 @@ bool RunIndexTests(const std::string &filter_path, bool enable_update) {
           const std::string &expected_path = entry.first;
           std::string expected_output = text_replacer.Apply(entry.second);
 
-          // FIXME: promote to utils, find and remove duplicates (ie,
-          // ccls_call_tree.cc, maybe something in project.cc).
-          auto basename = [](const std::string &path) -> std::string {
-            size_t last_index = path.find_last_of('/');
-            if (last_index == std::string::npos)
-              return path;
-            return path.substr(last_index + 1);
-          };
-
           // Get output from index operation.
           IndexFile *db = FindDbForPathEnding(expected_path, dbs);
-          if (db && !db->diagnostics_.empty()) {
-            printf("For %s\n", path.c_str());
-            for (const lsDiagnostic &diagnostic : db->diagnostics_) {
-              printf("  ");
-              if (diagnostic.severity)
-                switch (*diagnostic.severity) {
-                case lsDiagnosticSeverity::Error:
-                  printf("error ");
-                  break;
-                case lsDiagnosticSeverity::Warning:
-                  printf("warning ");
-                  break;
-                case lsDiagnosticSeverity::Information:
-                  printf("information ");
-                  break;
-                case lsDiagnosticSeverity::Hint:
-                  printf("hint ");
-                  break;
-                }
-              printf("%s:%s-%s:%s\n", basename(db->path).c_str(),
-                     diagnostic.range.start.ToString().c_str(),
-                     diagnostic.range.end.ToString().c_str(),
-                     diagnostic.message.c_str());
-            }
-          }
           std::string actual_output = "{}";
           if (db) {
             VerifySerializeToFrom(db);