[Parser] Eliminate 'CommentRetentionMode::None' in Lexer

Lexer should always set `Token.CommentLength` correctly because it
necessary for restoring to the token position with comments.
Otherwise, 'Token::isAtStartOfLine()' might not correctly set.

Author: rintaro

include/swift/Parse/Lexer.h

@@ -39,7 +39,6 @@ namespace swift {
   template<typename ...T> struct Diag;
 
 enum class CommentRetentionMode {
-  None,
   AttachToNextToken,
   ReturnAsTokens,
 };
@@ -185,7 +184,7 @@ class Lexer {
       const LangOptions &Options, const SourceManager &SourceMgr,
       unsigned BufferID, DiagnosticEngine *Diags, LexerMode LexMode,
       HashbangMode HashbangAllowed = HashbangMode::Disallowed,
-      CommentRetentionMode RetainComments = CommentRetentionMode::None);
+      CommentRetentionMode RetainComments = CommentRetentionMode::AttachToNextToken);
 
   /// Create a lexer that scans a subrange of the source buffer.
   Lexer(const LangOptions &Options, const SourceManager &SourceMgr,

include/swift/Parse/Parser.h

@@ -693,6 +693,10 @@ class Parser {
     return Context.LangOpts.EnableExperimentalConcurrency;
   }
 
+  bool shouldAttachCommentToDecl() {
+    return Context.LangOpts.AttachCommentsToDecls;
+  }
+
   /// Returns true if a Swift declaration starts after the current token,
   /// otherwise returns false.
   bool isNextStartOfSwiftDecl() {

lib/IDE/Formatting.cpp

@@ -49,7 +49,7 @@ static void widenOrSet(SourceRange &First, SourceRange Second) {
 static bool isFirstTokenOnLine(SourceManager &SM, SourceLoc Loc) {
   assert(Loc.isValid());
   SourceLoc LineStart = Lexer::getLocForStartOfLine(SM, Loc);
-  CommentRetentionMode SkipComments = CommentRetentionMode::None;
+  CommentRetentionMode SkipComments = CommentRetentionMode::AttachToNextToken;
   Token First = Lexer::getTokenAtLocation(SM, LineStart, SkipComments);
   return First.getLoc() == Loc;
 }
@@ -76,8 +76,8 @@ static std::optional<Token> getTokenAfter(SourceManager &SM, SourceLoc Loc,
                                           bool SkipComments = true) {
   assert(Loc.isValid());
   CommentRetentionMode Mode = SkipComments
-    ? CommentRetentionMode::None
-    : CommentRetentionMode::ReturnAsTokens;
+                                  ? CommentRetentionMode::AttachToNextToken
+                                  : CommentRetentionMode::ReturnAsTokens;
   assert(Lexer::getTokenAtLocation(SM, Loc, Mode).getLoc() == Loc);
   SourceLoc End = Lexer::getLocForEndOfToken(SM, Loc);
   Token Next = Lexer::getTokenAtLocation(SM, End, Mode);
@@ -814,8 +814,8 @@ class OutdentChecker: protected RangeWalker {
     } else if (R.isValid()) {
       // Check condition 2a.
       SourceLoc LineStart = Lexer::getLocForStartOfLine(SM, R);
-      Token First = Lexer::getTokenAtLocation(SM, LineStart,
-                                              CommentRetentionMode::None);
+      Token First = Lexer::getTokenAtLocation(
+          SM, LineStart, CommentRetentionMode::AttachToNextToken);
       IsOutdenting |= First.getLoc() == R;
     }
 

lib/Parse/Lexer.cpp

@@ -270,7 +270,7 @@ Token Lexer::getTokenAt(SourceLoc Loc) {
          "location from the wrong buffer");
 
   Lexer L(LangOpts, SourceMgr, BufferID, getUnderlyingDiags(), LexMode,
-          HashbangMode::Allowed, CommentRetentionMode::None);
+          HashbangMode::Allowed, CommentRetentionMode::AttachToNextToken);
   L.restoreState(State(Loc));
   return L.peekNextToken();
 }
@@ -3029,7 +3029,7 @@ static SourceLoc getLocForStartOfTokenInBuf(SourceManager &SM,
   LangOptions FakeLangOptions;
 
   Lexer L(FakeLangOptions, SM, BufferID, nullptr, LexerMode::Swift,
-          HashbangMode::Allowed, CommentRetentionMode::None,
+          HashbangMode::Allowed, CommentRetentionMode::AttachToNextToken,
           BufferStart, BufferEnd);
 
   // Lex tokens until we find the token that contains the source location.

lib/Parse/ParseDecl.cpp

@@ -6204,7 +6204,7 @@ ParserStatus Parser::parseDecl(bool IsAtStartOfLineOrPreviousHadSemi,
 
   // Parse attributes.
   DeclAttributes Attributes;
-  if (Tok.hasComment())
+  if (Tok.hasComment() && shouldAttachCommentToDecl())
     Attributes.add(new (Context) RawDocCommentAttr(Tok.getCommentRange()));
   ParserStatus AttrStatus = parseDeclAttributeList(
       Attributes, IfConfigsAreDeclAttrs);

lib/Parse/ParseGeneric.cpp

@@ -56,7 +56,7 @@ Parser::parseGenericParametersBeforeWhere(SourceLoc LAngleLoc,
 
     // Parse attributes.
     DeclAttributes attributes;
-    if (Tok.hasComment())
+    if (Tok.hasComment() && shouldAttachCommentToDecl())
       attributes.add(new (Context) RawDocCommentAttr(Tok.getCommentRange()));
     parseDeclAttributeList(attributes);
 

lib/Parse/Parser.cpp

@@ -341,21 +341,15 @@ Parser::Parser(unsigned BufferID, SourceFile &SF, SILParserStateBase *SIL,
                PersistentParserState *PersistentState)
     : Parser(BufferID, SF, &SF.getASTContext().Diags, SIL, PersistentState) {}
 
-Parser::Parser(unsigned BufferID, SourceFile &SF, DiagnosticEngine* LexerDiags,
-               SILParserStateBase *SIL,
-               PersistentParserState *PersistentState)
-    : Parser(
-          std::unique_ptr<Lexer>(new Lexer(
-              SF.getASTContext().LangOpts, SF.getASTContext().SourceMgr,
-              BufferID, LexerDiags,
-              sourceFileKindToLexerMode(SF.Kind),
-              SF.Kind == SourceFileKind::Main
-                  ? HashbangMode::Allowed
-                  : HashbangMode::Disallowed,
-              SF.getASTContext().LangOpts.AttachCommentsToDecls
-                  ? CommentRetentionMode::AttachToNextToken
-                  : CommentRetentionMode::None)),
-          SF, SIL, PersistentState) {}
+Parser::Parser(unsigned BufferID, SourceFile &SF, DiagnosticEngine *LexerDiags,
+               SILParserStateBase *SIL, PersistentParserState *PersistentState)
+    : Parser(std::unique_ptr<Lexer>(new Lexer(
+                 SF.getASTContext().LangOpts, SF.getASTContext().SourceMgr,
+                 BufferID, LexerDiags, sourceFileKindToLexerMode(SF.Kind),
+                 SF.Kind == SourceFileKind::Main ? HashbangMode::Allowed
+                                                 : HashbangMode::Disallowed,
+                 CommentRetentionMode::AttachToNextToken)),
+             SF, SIL, PersistentState) {}
 
 namespace {
 
@@ -1243,12 +1237,10 @@ ParserUnit::ParserUnit(SourceManager &SM, SourceFileKind SFKind,
     : Impl(*new Implementation(SM, SFKind, BufferID, LangOptions(), "input")) {
 
   std::unique_ptr<Lexer> Lex;
-  Lex.reset(new Lexer(Impl.LangOpts, SM,
-                      BufferID, &Impl.Diags,
-                      LexerMode::Swift,
-                      HashbangMode::Allowed,
-                      CommentRetentionMode::None,
-                      Offset, EndOffset));
+  Lex.reset(new Lexer(Impl.LangOpts, SM, BufferID, &Impl.Diags,
+                      LexerMode::Swift, HashbangMode::Allowed,
+                      CommentRetentionMode::AttachToNextToken, Offset,
+                      EndOffset));
   Impl.TheParser.reset(new Parser(std::move(Lex), *Impl.SF, /*SIL=*/nullptr,
                                   /*PersistentState=*/nullptr));
 }

lib/Refactoring/Async/AsyncConverter.cpp

@@ -651,8 +651,8 @@ void AsyncConverter::addFuncDecl(const FuncDecl *FD) {
     LeftEndLoc = Lexer::getLocForEndOfToken(
         SM, Params->get(TopHandler.Index - 1)->getEndLoc());
     // Skip to the end of any comments
-    Token Next =
-        Lexer::getTokenAtLocation(SM, LeftEndLoc, CommentRetentionMode::None);
+    Token Next = Lexer::getTokenAtLocation(
+        SM, LeftEndLoc, CommentRetentionMode::AttachToNextToken);
     if (Next.getKind() != tok::NUM_TOKENS)
       LeftEndLoc = Next.getLoc();
     break;

lib/Sema/MiscDiagnostics.cpp

@@ -2559,8 +2559,9 @@ class ImplicitSelfUsageChecker : public BaseDiagnosticWalker {
     // insert 'self,'. If it wasn't a valid entry, then we will at least not
     // be introducing any new errors/warnings...
     const auto locAfterBracket = brackets.Start.getAdvancedLoc(1);
-    const auto nextAfterBracket = Lexer::getTokenAtLocation(
-        Ctx.SourceMgr, locAfterBracket, CommentRetentionMode::None);
+    const auto nextAfterBracket =
+        Lexer::getTokenAtLocation(Ctx.SourceMgr, locAfterBracket,
+                                  CommentRetentionMode::AttachToNextToken);
     if (nextAfterBracket.getLoc() != brackets.End)
       diag.fixItInsertAfter(brackets.Start, "self, ");
     else
@@ -2581,8 +2582,8 @@ class ImplicitSelfUsageChecker : public BaseDiagnosticWalker {
     // opening brace of the closure, we may need to pad the fix-it
     // with a space.
     const auto nextLoc = closureExpr->getLoc().getAdvancedLoc(1);
-    const auto next = Lexer::getTokenAtLocation(Ctx.SourceMgr, nextLoc,
-                                                CommentRetentionMode::None);
+    const auto next = Lexer::getTokenAtLocation(
+        Ctx.SourceMgr, nextLoc, CommentRetentionMode::AttachToNextToken);
     std::string trailing = next.getLoc() == nextLoc ? " " : "";
 
     diag.fixItInsertAfter(closureExpr->getLoc(), " [self] in" + trailing);

test/Parse/backgtrack-startofline.swift

@@ -0,0 +1,7 @@
+// RUN: %target-typecheck-verify-swift -verify-ignore-unrelated
+
+func test() {
+  _ = "test"
+  /* comment */ #if true
+  #endif
+}