fix(javac): add Java compiler flag to override -XDkeepComments flag

When the 'keepComments' flag is set, the compiler uses JavadocTokenizer for tokenizing the source code. While creating the LineMap instance, the JavadocTokenizer sets the 'expandTabs' flag to true which expands '\t' characters to spaces. This is needed while generating HTML pages for the Javadoc. However, if the tabs are expanded to spaces, then getPosition(...) and getColumnNumber(...) methods in the LineMap class return indices which cannot be directly used to insert text in the source code. This results in StringIndexOutOfBoundsException when trying to insert text in the editor (#1127, and probably #916 as well).

This commit adds the 'keepCommentsOverride' option to the compiler which is used by NBParserFactory to override the 'keepComments' flag. The CompileBatch class sets this option to 'ignore' which ignores the 'keepComments' flag so that JavadocTokenizer is not used for tokenization.

Author: itsaky

lsp/java/src/main/java/com/itsaky/androidide/lsp/java/compiler/CompileBatch.java

@@ -168,6 +168,7 @@ private List<String> options() {
         "-XDcompilePolicy=byfile",
         "-XD-Xprefer=source",
         "-XDide",
+        "-XDkeepCommentsOverride=ignore",
         "-XDsuppressAbortOnBadClassFile",
         "-XDshould-stop.at=GENERATE",
         "-XDdiags.formatterOptions=-source",

subprojects/javac-services/src/main/java/com/itsaky/androidide/javac/services/NBParserFactory.java

@@ -35,6 +35,7 @@
  */
 package com.itsaky.androidide.javac.services;
 
+import com.itsaky.androidide.utils.ILogger;
 import openjdk.tools.javac.parser.JavacParser;
 import openjdk.tools.javac.parser.Lexer;
 import openjdk.tools.javac.parser.ParserFactory;
@@ -52,17 +53,26 @@
 import openjdk.tools.javac.util.List;
 import openjdk.tools.javac.util.Name;
 import openjdk.tools.javac.util.Names;
+import openjdk.tools.javac.util.Options;
 import openjdk.tools.javac.util.Position;
 
 /**
  * @author lahvac
  */
 public class NBParserFactory extends ParserFactory {
 
+  public static final String KEEP_COMMENTS_OVERRIDE = "keepCommentsOverride";
+  public static final String KEEP_COMMENTS_OVERRIDE_KEEP = "keep";
+  public static final String KEEP_COMMENTS_OVERRIDE_IGNORE = "ignore";
+
+  private static final ILogger LOG = ILogger.newInstance("NBParserFactory");
+
   protected final ScannerFactory scannerFactory;
   protected final Names names;
   protected final CancelService cancelService;
 
+  protected final String keepCommentsOverride;
+
   public static void preRegister(Context context) {
     context.put(parserFactoryKey, (Context.Factory<ParserFactory>) NBParserFactory::new);
   }
@@ -72,6 +82,8 @@ protected NBParserFactory(Context context) {
     this.scannerFactory = ScannerFactory.instance(context);
     this.names = Names.instance(context);
     this.cancelService = CancelService.instance(context);
+    this.keepCommentsOverride = Options.instance(context).get(KEEP_COMMENTS_OVERRIDE);
+
   }
 
   @Override
@@ -81,9 +93,20 @@ public JavacParser newParser(
       boolean keepEndPos,
       boolean keepLineMap,
       boolean parseModuleInfo) {
-    Lexer lexer = scannerFactory.newScanner(input, keepDocComments);
+    var keepDocCommentsOverride = keepDocComments;
+    if (this.keepCommentsOverride != null) {
+      if (KEEP_COMMENTS_OVERRIDE_IGNORE.equals(this.keepCommentsOverride)) {
+        keepDocCommentsOverride = false;
+      } else if (KEEP_COMMENTS_OVERRIDE_KEEP.equals(this.keepCommentsOverride)) {
+        keepDocCommentsOverride = true;
+      }
+      LOG.debug("'keepComments' overridden to ", this.keepCommentsOverride);
+    }
+
+    Lexer lexer = scannerFactory.newScanner(input, keepDocCommentsOverride);
     return new NBJavacParser(
-        this, lexer, keepDocComments, keepLineMap, keepEndPos, parseModuleInfo, cancelService);
+        this, lexer, keepDocCommentsOverride, keepLineMap, keepEndPos, parseModuleInfo,
+        cancelService);
   }
 
   public static class NBJavacParser extends JavacParser {
@@ -201,7 +224,9 @@ public int getEndPos(JCTree jctree) {
 
       @Override
       public void storeEnd(JCTree tree, int endpos) {
-        if (endpos >= 0) delegate.storeEnd(tree, endpos);
+        if (endpos >= 0) {
+          delegate.storeEnd(tree, endpos);
+        }
       }
 
       @Override