Experimental incremental DOM parser

org.eclipse.lemminx/pom.xml

@@ -227,12 +227,14 @@
 			<groupId>xerces</groupId>
 			<artifactId>xercesImpl</artifactId>
 			<version>2.12.2</version>
+			<exclusions>
+			<exclusion>
+					<groupId>xml-apis</groupId>
+					<artifactId>xml-apis</artifactId>
+				</exclusion>
+			</exclusions>
 		</dependency>
-		<dependency>
-			<groupId>xml-apis</groupId>
-			<artifactId>xml-apis</artifactId>
-			<version>1.4.01</version>
-		</dependency>
+
 		<dependency>
 			<groupId>com.kotcrab.remark</groupId>
 			<artifactId>remark</artifactId>

org.eclipse.lemminx/src/main/java/org/eclipse/lemminx/XMLLanguageServer.java

@@ -50,6 +50,7 @@
 import org.eclipse.lemminx.settings.XMLFoldingSettings;
 import org.eclipse.lemminx.settings.XMLFormattingOptions;
 import org.eclipse.lemminx.settings.XMLGeneralClientSettings;
+import org.eclipse.lemminx.settings.XMLIncrementalParserSettings;
 import org.eclipse.lemminx.settings.XMLPreferences;
 import org.eclipse.lemminx.settings.XMLSymbolSettings;
 import org.eclipse.lemminx.settings.XMLTelemetrySettings;
@@ -225,6 +226,9 @@ private synchronized void updateSettings(Object initOptions, boolean initLogs) {
 				String workDir = serverSettings.getNormalizedWorkDir();
 				FilesUtils.setCachePathSetting(workDir);
 			}
+
+			XMLIncrementalParserSettings incrementalParserSettings = xmlClientSettings.getIncrementalParser();			
+			xmlTextDocumentService.setIncrementalParserSettings(incrementalParserSettings);
 		}
 		ContentModelSettings cmSettings = ContentModelSettings.getContentModelXMLSettings(initSettings);
 		if (cmSettings != null) {

org.eclipse.lemminx/src/main/java/org/eclipse/lemminx/XMLTextDocumentService.java

@@ -36,8 +36,11 @@
 import org.eclipse.lemminx.commons.ModelTextDocuments;
 import org.eclipse.lemminx.commons.ModelValidatorDelayer;
 import org.eclipse.lemminx.commons.TextDocument;
+import org.eclipse.lemminx.commons.TextDocumentChange;
 import org.eclipse.lemminx.dom.DOMDocument;
 import org.eclipse.lemminx.dom.DOMParser;
+import org.eclipse.lemminx.dom.IncrementalDOMParser;
+import org.eclipse.lemminx.dom.IncrementalDOMParserTestGenerator;
 import org.eclipse.lemminx.extensions.contentmodel.settings.XMLValidationRootSettings;
 import org.eclipse.lemminx.services.DocumentSymbolsResult;
 import org.eclipse.lemminx.services.SymbolInformationResult;
@@ -50,6 +53,7 @@
 import org.eclipse.lemminx.settings.XMLCompletionSettings;
 import org.eclipse.lemminx.settings.XMLFoldingSettings;
 import org.eclipse.lemminx.settings.XMLFormattingOptions;
+import org.eclipse.lemminx.settings.XMLIncrementalParserSettings;
 import org.eclipse.lemminx.settings.XMLPreferences;
 import org.eclipse.lemminx.settings.XMLSymbolSettings;
 import org.eclipse.lemminx.utils.XMLPositionUtility;
@@ -127,7 +131,7 @@ public class XMLTextDocumentService implements TextDocumentService {
 
 	private SharedSettings sharedSettings;
 	private LimitExceededWarner limitExceededWarner;
-
+	
 	/**
 	 * Enumeration for Validation triggered by.
 	 *
@@ -194,12 +198,30 @@ public void triggerValidationIfNeeded() {
 
 	private Boolean clientConfigurationSupport;
 
+	private XMLIncrementalParserSettings incrementalParser;
+
 	public XMLTextDocumentService(XMLLanguageServer xmlLanguageServer) {
 		this.xmlLanguageServer = xmlLanguageServer;
 		DOMParser parser = DOMParser.getInstance();
 		this.documents = new ModelTextDocuments<DOMDocument>((document, cancelChecker) -> {
 			return parser.parse(document, getXMLLanguageService().getResolverExtensionManager(), true, cancelChecker);
-		});
+		}, //
+				(document, changes, oldText) -> {
+					IncrementalDOMParser p = IncrementalDOMParser.getInstance();
+					p.parseIncremental(document, changes);
+
+					String generateTestWhen = incrementalParser != null ? incrementalParser.getGenerateTestWhen() : null;
+					if (generateTestWhen != null) {
+						if ("always".equals(generateTestWhen)) {
+							generateTest(document, changes, oldText);
+						} else if ("error".equals(generateTestWhen)) {
+							DOMDocument newDoc = parser.parse(document.getTextDocument(), getXMLLanguageService().getResolverExtensionManager(), true, () -> {});
+							if (!newDoc.toString().equals(document.toString())) {
+								generateTest(document, changes, oldText);
+							}
+						}
+					}					
+				});
 		this.sharedSettings = new SharedSettings();
 		this.limitExceededWarner = null;
 		this.xmlValidatorDelayer = new ModelValidatorDelayer<DOMDocument>((document) -> {
@@ -217,6 +239,20 @@ public XMLTextDocumentService(XMLLanguageServer xmlLanguageServer) {
 		});
 	}
 
+
+	private void generateTest(DOMDocument document, List<TextDocumentChange> changes, String oldText) {
+		IncrementalDOMParserTestGenerator.getInstance().generateTest(document, changes, oldText);
+	}
+
+	public void setIncrementalParsing(boolean incrementalParsing) {
+		this.documents.setIncrementalModel(incrementalParsing);
+	}
+
+	public void setIncrementalParserSettings(XMLIncrementalParserSettings incrementalParser) {
+		this.incrementalParser = incrementalParser;
+		this.documents.setIncrementalModel(incrementalParser != null ? incrementalParser.isEnabled() : false);
+	}
+
 	public void updateClientCapabilities(ClientCapabilities capabilities,
 			ExtendedClientCapabilities extendedClientCapabilities) {
 		if (capabilities != null) {
@@ -719,8 +755,7 @@ void validate(DOMDocument xmlDocument, Map<String, Object> validationArgs) throw
 		cancelChecker.checkCanceled();
 		getXMLLanguageService().publishDiagnostics(xmlDocument,
 				params -> xmlLanguageServer.getLanguageClient().publishDiagnostics(params),
-				(doc) -> triggerValidationFor(doc, TriggeredBy.Other),
-				sharedSettings.getValidationSettings(),
+				(doc) -> triggerValidationFor(doc, TriggeredBy.Other), sharedSettings.getValidationSettings(),
 				validationArgs, cancelChecker);
 	}
 
@@ -827,4 +862,5 @@ public LimitExceededWarner getLimitExceededWarner() {
 		}
 		return this.limitExceededWarner;
 	}
+
 }

org.eclipse.lemminx/src/main/java/org/eclipse/lemminx/commons/ModelTextDocument.java

@@ -11,10 +11,12 @@
 *******************************************************************************/
 package org.eclipse.lemminx.commons;
 
+import java.util.List;
 import java.util.concurrent.CancellationException;
 import java.util.function.BiFunction;
 import java.util.logging.Logger;
 
+import org.eclipse.lsp4j.TextDocumentContentChangeEvent;
 import org.eclipse.lsp4j.TextDocumentItem;
 import org.eclipse.lsp4j.jsonrpc.CancelChecker;
 
@@ -33,14 +35,21 @@ public class ModelTextDocument<T> extends TextDocument {
 
 	private T model;
 
-	public ModelTextDocument(TextDocumentItem document, BiFunction<TextDocument, CancelChecker, T> parse) {
+	private final ModelUpdater<T> modelUpdater;
+
+	private boolean incrementalModel;
+
+	public ModelTextDocument(TextDocumentItem document, BiFunction<TextDocument, CancelChecker, T> parse,
+			ModelUpdater<T> updateModel) {
 		super(document);
 		this.parse = parse;
+		this.modelUpdater = updateModel;
 	}
 
 	public ModelTextDocument(String text, String uri, BiFunction<TextDocument, CancelChecker, T> parse) {
 		super(text, uri);
 		this.parse = parse;
+		this.modelUpdater = null;
 	}
 
 	/**
@@ -83,7 +92,8 @@ private synchronized T getSynchronizedModel() {
 			LOGGER.fine("Start parsing of model with version '" + version);
 			// Stop of parse process can be done when completable future is canceled or when
 			// version of document changes
-			CancelChecker cancelChecker = new TextDocumentVersionChecker(this, version);
+			CancelChecker cancelChecker = isIncrementalModel() ? ModelTextDocuments.NO_CANCELLABLE
+					: new TextDocumentVersionChecker(this, version);
 			// parse the model
 			model = parse.apply(this, cancelChecker);
 		} catch (CancellationException e) {
@@ -101,14 +111,32 @@ private synchronized T getSynchronizedModel() {
 	public void setText(String text) {
 		super.setText(text);
 		// text changed, cancel the completable future which load the model
-		cancelModel();
+		if (!isIncrementalModel()) {
+			cancelModel();
+		}
 	}
 
 	@Override
 	public void setVersion(int version) {
 		super.setVersion(version);
 		// version changed, mark the model as dirty
-		cancelModel();
+		if (!isIncrementalModel()) {
+			cancelModel();
+		}
+	}
+
+	@Override
+	public List<TextDocumentChange> update(List<TextDocumentContentChangeEvent> changes) {
+		String oldText = super.getText();
+		List<TextDocumentChange> result = super.update(changes);
+		if (isIncrementalModel() && model != null && !result.isEmpty()) {
+			updateModel(model, oldText, result);
+		}
+		return result;
+	}
+
+	private void updateModel(T model, String oldText, List<TextDocumentChange> changes) {
+		modelUpdater.updateModel(model, changes, oldText);
 	}
 
 	/**
@@ -118,4 +146,12 @@ private void cancelModel() {
 		model = null;
 	}
 
+	public boolean isIncrementalModel() {
+		return incrementalModel;
+	}
+
+	public void setIncrementalModel(boolean incrementalModel) {
+		this.incrementalModel = incrementalModel;
+	}
+
 }

org.eclipse.lemminx/src/main/java/org/eclipse/lemminx/commons/ModelTextDocuments.java

@@ -30,16 +30,26 @@
  */
 public class ModelTextDocuments<T> extends TextDocuments<ModelTextDocument<T>> {
 
+	final static CancelChecker NO_CANCELLABLE = () -> {
+	};
+
 	private final BiFunction<TextDocument, CancelChecker, T> parse;
 
-	public ModelTextDocuments(BiFunction<TextDocument, CancelChecker, T> parse) {
+	private final ModelUpdater<T> modelUpdater;
+
+	private boolean incrementalModel;
+
+	public ModelTextDocuments(BiFunction<TextDocument, CancelChecker, T> parse,
+			ModelUpdater<T> updateModel) {
 		this.parse = parse;
+		this.modelUpdater = updateModel;
 	}
 
 	@Override
 	public ModelTextDocument<T> createDocument(TextDocumentItem document) {
-		ModelTextDocument<T> doc = new ModelTextDocument<T>(document, parse);
+		ModelTextDocument<T> doc = new ModelTextDocument<T>(document, parse, modelUpdater);
 		doc.setIncremental(isIncremental());
+		doc.setIncrementalModel(isIncrementalModel());
 		return doc;
 	}
 
@@ -115,7 +125,7 @@ public <R> CompletableFuture<R> computeModelAsync(TextDocumentIdentifier documen
 				return null;
 			}
 			// Apply the function code by using the parsed model.
-			return code.apply(model, cancelChecker);
+			return code.apply(model, isIncrementalModel() ? NO_CANCELLABLE : cancelChecker);
 		});
 	}
 
@@ -150,4 +160,12 @@ private static <R> CompletableFuture<R> computeAsyncCompose(Function<CancelCheck
 		start.complete(new FutureCancelChecker(result));
 		return result;
 	}
+
+	public boolean isIncrementalModel() {
+		return incrementalModel;
+	}
+
+	public void setIncrementalModel(boolean incrementalModel) {
+		this.incrementalModel = incrementalModel;
+	}
 }

org.eclipse.lemminx/src/main/java/org/eclipse/lemminx/commons/ModelUpdater.java

@@ -0,0 +1,9 @@
+package org.eclipse.lemminx.commons;
+
+import java.util.List;
+
+@FunctionalInterface
+public interface ModelUpdater<T> {
+
+	void updateModel(T model, List<TextDocumentChange> changes, String oldText);
+}

org.eclipse.lemminx/src/main/java/org/eclipse/lemminx/commons/TextDocument.java

@@ -12,6 +12,8 @@
  */
 package org.eclipse.lemminx.commons;
 
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.logging.Logger;
 import java.util.regex.Matcher;
@@ -156,17 +158,20 @@ private synchronized ILineTracker createLineTracker() {
 	/**
 	 * Update text of the document by using the changes and according the
 	 * incremental support.
-	 * 
+	 *
 	 * @param changes the text document changes.
+	 * @return list of changes with pre-calculated offsets (calculated before text update),
+	 *         or empty list if no changes or not incremental
 	 */
-	public void update(List<TextDocumentContentChangeEvent> changes) {
+	public List<TextDocumentChange> update(List<TextDocumentContentChangeEvent> changes) {
 		if (changes.size() < 1) {
 			// no changes, ignore it.
-			return;
+			return Collections.emptyList();
 		}
 		if (isIncremental()) {
 			try {
 				long start = System.currentTimeMillis();
+				List<TextDocumentChange> result = new ArrayList<>(changes.size());
 				synchronized (lock) {
 					// Initialize buffer and line tracker from the current text document
 					StringBuilder buffer = new StringBuilder(getText());
@@ -188,15 +193,23 @@ public void update(List<TextDocumentContentChangeEvent> changes) {
 						}
 						String text = changeEvent.getText();
 						int startOffset = offsetAt(range.getStart());
+						int newLength = text != null ? text.length() : 0;
+
+						// Store the change with pre-calculated offsets (before text update)
+						result.add(new TextDocumentChange(changeEvent, startOffset, length, newLength));
+
 						buffer.replace(startOffset, startOffset + length, text);
 						lineTracker.replace(startOffset, length, text);
 					}
 					// Update the new text content from the updated buffer
 					setText(buffer.toString());
 				}
 				LOGGER.fine("Text document content updated in " + (System.currentTimeMillis() - start) + "ms");
+				return result;
 			} catch (BadLocationException e) {
+				e.printStackTrace();
 				// Should never occur.
+				return Collections.emptyList();
 			}
 		} else {
 			// like vscode does, get the last changes
@@ -207,6 +220,7 @@ public void update(List<TextDocumentContentChangeEvent> changes) {
 				setText(last.getText());
 				lineTracker.set(last.getText());
 			}
+			return Collections.emptyList();
 		}
 	}
 }