Re-apply "[Parse] Split incremental-extensions"

[bnbarham]

Re-applies #65683 with a fix to always run
Actions.ActOnEndOfTranslationUnit regardless of incremental processing.

[llvmbot]

@llvm/pr-subscribers-clang

Changes

Re-applies #65683 with a fix to always run `Actions.ActOnEndOfTranslationUnit` regardless of incremental processing. -- Full diff: https://github.com//pull/66446.diff

4 Files Affected:

• (modified) clang/include/clang/Lex/Preprocessor.h (+5-5)
• (modified) clang/lib/Lex/PPLexerChange.cpp (+1-1)
• (modified) clang/lib/Lex/Preprocessor.cpp (+4)
• (modified) clang/lib/Parse/Parser.cpp (+6-1)

diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index bc1d94a61508d8d..575d08b83fd3a02 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -277,6 +277,9 @@ class Preprocessor {
   /// Empty line handler.
   EmptylineHandler *Emptyline = nullptr;
 
+  /// True to avoid tearing down the lexer etc on EOF
+  bool IncrementalProcessing = false;
+
 public:
   /// The kind of translation unit we are processing.
   const TranslationUnitKind TUKind;
@@ -1910,14 +1913,11 @@ class Preprocessor {
   void recomputeCurLexerKind();
 
   /// Returns true if incremental processing is enabled
-  bool isIncrementalProcessingEnabled() const {
-    return getLangOpts().IncrementalExtensions;
-  }
+  bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
 
   /// Enables the incremental processing
   void enableIncrementalProcessing(bool value = true) {
-    // FIXME: Drop this interface.
-    const_cast<LangOptions &>(getLangOpts()).IncrementalExtensions = value;
+    IncrementalProcessing = value;
   }
 
   /// Specify the point at which code-completion will be performed.
diff --git a/clang/lib/Lex/PPLexerChange.cpp b/clang/lib/Lex/PPLexerChange.cpp
index ab005381adfaf2c..811a760420e0a2d 100644
--- a/clang/lib/Lex/PPLexerChange.cpp
+++ b/clang/lib/Lex/PPLexerChange.cpp
@@ -541,7 +541,7 @@ bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) {
   Result.startToken();
   CurLexer->BufferPtr = EndPos;
 
-  if (isIncrementalProcessingEnabled()) {
+  if (getLangOpts().IncrementalExtensions) {
     CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_repl_input_end);
     Result.setAnnotationEndLoc(Result.getLocation());
     Result.setAnnotationValue(nullptr);
diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index 8de78a13930ed62..f0381c18a8b6f77 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -146,6 +146,10 @@ Preprocessor::Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
     Ident_AbnormalTermination = nullptr;
   }
 
+  // Default incremental processing to -fincremental-extensions, clients can
+  // override with `enableIncrementalProcessing` if desired.
+  IncrementalProcessing = LangOpts.IncrementalExtensions;
+
   // If using a PCH where a #pragma hdrstop is expected, start skipping tokens.
   if (usingPCHWithPragmaHdrStop())
     SkippingUntilPragmaHdrStop = true;
diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp
index 09215b8303ecf9c..dda1dbaa0c21aa9 100644
--- a/clang/lib/Parse/Parser.cpp
+++ b/clang/lib/Parse/Parser.cpp
@@ -615,6 +615,11 @@ bool Parser::ParseTopLevelDecl(DeclGroupPtrTy &Result,
                                Sema::ModuleImportState &ImportState) {
   DestroyTemplateIdAnnotationsRAIIObj CleanupRAII(*this);
 
+  // Skip over the EOF token, flagging end of previous input for incremental
+  // processing
+  if (PP.isIncrementalProcessingEnabled() && Tok.is(tok::eof))
+    ConsumeToken();
+
   Result = nullptr;
   switch (Tok.getKind()) {
   case tok::annot_pragma_unused:
@@ -1038,7 +1043,7 @@ Parser::ParseExternalDeclaration(ParsedAttributes &Attrs,
       ConsumeToken();
       return nullptr;
     }
-    if (PP.isIncrementalProcessingEnabled() &&
+    if (getLangOpts().IncrementalExtensions &&
         !isDeclarationStatement(/*DisambiguatingWithExpression=*/true))
       return ParseTopLevelStmtDecl();
 

[bnbarham]

Any objections to this @vgvassilev?

[vgvassilev]

Any objections to this @vgvassilev?

I have no objections to this if that makes your workflow work. However I think this cures the symptom and not the real cause. It might be worth investigating how to adapt lldb to the incremental processing change more deeply.

[bnbarham]

I have no objections to this if that makes your workflow work. However I think this cures the symptom and not the real cause. It might be worth investigating how to adapt lldb to the incremental processing change more deeply.

To be clear, this is re-applying #65683 such that it is actually the NFC it was intended to be. I've closed the LLDB PR (#66271), so it's still using -fincremental-extensions.

The actual use case for splitting these is eg. Swift's clang importer, where the need is very similar to #62413 and IMO it makes sense to keep these two as separate, with -fincremental-extensions implying incremental processing.

[bnbarham]

Friendly ping @cor3ntin / @vgvassilev 🙏

[cor3ntin]

LGTM