dart/editor/tools/plugins/com.google.dart.engine/src/com/google/dart/engine/scanner/IncrementalScanner.java - external/dart - Git at Google

 /*
  * Copyright (c) 2012, the Dart project authors.
  *
  * Licensed under the Eclipse Public License v1.0 (the "License"); you may not use this file except
  * in compliance with the License. You may obtain a copy of the License at
  *
  * http://www.eclipse.org/legal/epl-v10.html
  *
  * Unless required by applicable law or agreed to in writing, software distributed under the License
  * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
  * or implied. See the License for the specific language governing permissions and limitations under
  * the License.
  */
 package com.google.dart.engine.scanner;

 import com.google.dart.engine.error.AnalysisErrorListener;
 import com.google.dart.engine.source.Source;
 import com.google.dart.engine.utilities.collection.TokenMap;

 /**
  * Instances of the class {@code IncrementalScanner} implement a scanner that scans a subset of a
  * string and inserts the resulting tokens into the middle of an existing token stream.
  *
  * @coverage dart.engine.parser
  */
 public class IncrementalScanner extends Scanner {
   /**
    * The reader used to access the characters in the source.
    */
   private CharacterReader reader;

   /**
    * A map from tokens that were copied to the copies of the tokens.
    */
   private TokenMap tokenMap = new TokenMap();

   /**
    * The token in the new token stream immediately to the left of the range of tokens that were
    * inserted, or the token immediately to the left of the modified region if there were no new
    * tokens.
    */
   private Token leftToken;

   /**
    * The token in the new token stream immediately to the right of the range of tokens that were
    * inserted, or the token immediately to the right of the modified region if there were no new
    * tokens.
    */
   private Token rightToken;

   /**
    * A flag indicating whether there were any tokens changed as a result of the modification.
    */
   private boolean hasNonWhitespaceChange = false;

   /**
    * Initialize a newly created scanner.
    *
    * @param source the source being scanned
    * @param reader the character reader used to read the characters in the source
    * @param errorListener the error listener that will be informed of any errors that are found
    */
   public IncrementalScanner(Source source, CharacterReader reader,
       AnalysisErrorListener errorListener) {
     super(source, reader, errorListener);
     this.reader = reader;
   }

   /**
    * Return the token in the new token stream immediately to the left of the range of tokens that
    * were inserted, or the token immediately to the left of the modified region if there were no new
    * tokens.
    *
    * @return the token to the left of the inserted tokens
    */
   public Token getLeftToken() {
     return leftToken;
   }

   /**
    * Return the token in the new token stream immediately to the right of the range of tokens that
    * were inserted, or the token immediately to the right of the modified region if there were no
    * new tokens.
    *
    * @return the token to the right of the inserted tokens
    */
   public Token getRightToken() {
     return rightToken;
   }

   /**
    * Return a map from tokens that were copied to the copies of the tokens.
    *
    * @return a map from tokens that were copied to the copies of the tokens
    */
   public TokenMap getTokenMap() {
     return tokenMap;
   }

   /**
    * Return {@code true} if there were any tokens either added or removed (or both) as a result of
    * the modification.
    *
    * @return {@code true} if there were any tokens changed as a result of the modification
    */
   public boolean hasNonWhitespaceChange() {
     return hasNonWhitespaceChange;
   }

   /**
    * Given the stream of tokens scanned from the original source, the modified source (the result of
    * replacing one contiguous range of characters with another string of characters), and a
    * specification of the modification that was made, return a stream of tokens scanned from the
    * modified source. The original stream of tokens will not be modified.
    *
    * @param originalStream the stream of tokens scanned from the original source
    * @param index the index of the first character in both the original and modified source that was
    *          affected by the modification
    * @param removedLength the number of characters removed from the original source
    * @param insertedLength the number of characters added to the modified source
    */
   public Token rescan(Token originalStream, int index, int removedLength, int insertedLength) {
     //
     // Copy all of the tokens in the originalStream whose end is less than the replacement start.
     // (If the replacement start is equal to the end of an existing token, then it means that the
     // existing token might have been modified, so we need to rescan it.)
     //
     while (originalStream.getType() != TokenType.EOF && originalStream.getEnd() < index) {
       originalStream = copyAndAdvance(originalStream, 0);
     }
     Token oldFirst = originalStream;
     Token oldLeftToken = originalStream.getPrevious();
     leftToken = getTail();
     //
     // Skip tokens in the original stream until we find a token whose offset is greater than the end
     // of the removed region. (If the end of the removed region is equal to the beginning of an
     // existing token, then it means that the existing token might have been modified, so we need to
     // rescan it.)
     //
     int removedEnd = index + (removedLength == 0 ? 0 : removedLength - 1);
     while (originalStream.getType() != TokenType.EOF && originalStream.getOffset() <= removedEnd) {
       originalStream = originalStream.getNext();
     }
     Token oldLast;
     Token oldRightToken;
     if (originalStream.getType() != TokenType.EOF && removedEnd + 1 == originalStream.getOffset()) {
       oldLast = originalStream;
       originalStream = originalStream.getNext();
       oldRightToken = originalStream;
     } else {
       oldLast = originalStream.getPrevious();
       oldRightToken = originalStream;
     }
     //
     // Compute the delta between the character index of characters after the modified region in the
     // original source and the index of the corresponding character in the modified source.
     //
     int delta = insertedLength - removedLength;
     //
     // Compute the range of characters that are known to need to be rescanned. If the index is
     // within an existing token, then we need to start at the beginning of the token.
     //
     int scanStart = Math.min(oldFirst.getOffset(), index);
     int oldEnd = oldLast.getEnd() + delta - 1;
     int newEnd = index + insertedLength - 1;
     int scanEnd = Math.max(newEnd, oldEnd);
     //
     // Starting at the start of the scan region, scan tokens from the modifiedSource until the end
     // of the just scanned token is greater than or equal to end of the scan region in the modified
     // source. Include trailing characters of any token that was split as a result of inserted text,
     // as in "ab" --> "a.b".
     //
     reader.setOffset(scanStart - 1);
     int next = reader.advance();
     while (next != -1 && reader.getOffset() <= scanEnd) {
       next = bigSwitch(next);
     }
     //
     // Copy the remaining tokens in the original stream, but apply the delta to the token's offset.
     //
     if (originalStream.getType() == TokenType.EOF) {
       copyAndAdvance(originalStream, delta);
       rightToken = getTail();
       rightToken.setNextWithoutSettingPrevious(rightToken);
     } else {
       originalStream = copyAndAdvance(originalStream, delta);
       rightToken = getTail();
       while (originalStream.getType() != TokenType.EOF) {
         originalStream = copyAndAdvance(originalStream, delta);
       }
       Token eof = copyAndAdvance(originalStream, delta);
       eof.setNextWithoutSettingPrevious(eof);
     }
     //
     // If the index is immediately after an existing token and the inserted characters did not
     // change that original token, then adjust the leftToken to be the next token. For example, in
     // "a; c;" --> "a;b c;", the leftToken was ";", but this code advances it to "b" since "b" is
     // the first new token.
     //
     Token newFirst = leftToken.getNext();
     while (newFirst != rightToken && oldFirst != oldRightToken
         && newFirst.getType() != TokenType.EOF && equalTokens(oldFirst, newFirst)) {
       tokenMap.put(oldFirst, newFirst);
       oldLeftToken = oldFirst;
       oldFirst = oldFirst.getNext();
       leftToken = newFirst;
       newFirst = newFirst.getNext();
     }
     Token newLast = rightToken.getPrevious();
     while (newLast != leftToken && oldLast != oldLeftToken && newLast.getType() != TokenType.EOF
         && equalTokens(oldLast, newLast)) {
       tokenMap.put(oldLast, newLast);
       oldRightToken = oldLast;
       oldLast = oldLast.getPrevious();
       rightToken = newLast;
       newLast = newLast.getPrevious();
     }
     hasNonWhitespaceChange = leftToken.getNext() != rightToken
         || oldLeftToken.getNext() != oldRightToken;
     //
     // TODO(brianwilkerson) Begin tokens are not getting associated with the corresponding end
     //     tokens (because the end tokens have not been copied when we're copying the begin tokens).
     //     This could have implications for parsing.
     // TODO(brianwilkerson) Update the lineInfo.
     //
     return getFirstToken();
   }

   private Token copyAndAdvance(Token originalToken, int delta) {
     Token copiedToken = originalToken.copy();
     tokenMap.put(originalToken, copiedToken);
     copiedToken.applyDelta(delta);
     appendToken(copiedToken);

     Token originalComment = originalToken.getPrecedingComments();
     Token copiedComment = originalToken.getPrecedingComments();
     while (originalComment != null) {
       tokenMap.put(originalComment, copiedComment);
       originalComment = originalComment.getNext();
       copiedComment = copiedComment.getNext();
     }
     return originalToken.getNext();
   }

   /**
    * Return {@code true} if the two tokens are equal to each other. For the purposes of the
    * incremental scanner, two tokens are equal if they have the same type and lexeme.
    *
    * @param oldToken the token from the old stream that is being compared
    * @param newToken the token from the new stream that is being compared
    * @return {@code true} if the two tokens are equal to each other
    */
   private boolean equalTokens(Token oldToken, Token newToken) {
     return oldToken.getType() == newToken.getType() && oldToken.getLength() == newToken.getLength()
         && oldToken.getLexeme().equals(newToken.getLexeme());
   }
 }
	/*
	* Copyright (c) 2012, the Dart project authors.
	*
	* Licensed under the Eclipse Public License v1.0 (the "License"); you may not use this file except
	* in compliance with the License. You may obtain a copy of the License at
	*
	* http://www.eclipse.org/legal/epl-v10.html
	*
	* Unless required by applicable law or agreed to in writing, software distributed under the License
	* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
	* or implied. See the License for the specific language governing permissions and limitations under
	* the License.
	*/
	package com.google.dart.engine.scanner;

	import com.google.dart.engine.error.AnalysisErrorListener;
	import com.google.dart.engine.source.Source;
	import com.google.dart.engine.utilities.collection.TokenMap;

	/**
	* Instances of the class {@code IncrementalScanner} implement a scanner that scans a subset of a
	* string and inserts the resulting tokens into the middle of an existing token stream.
	*
	* @coverage dart.engine.parser
	*/
	public class IncrementalScanner extends Scanner {
	/**
	* The reader used to access the characters in the source.
	*/
	private CharacterReader reader;

	/**
	* A map from tokens that were copied to the copies of the tokens.
	*/
	private TokenMap tokenMap = new TokenMap();

	/**
	* The token in the new token stream immediately to the left of the range of tokens that were
	* inserted, or the token immediately to the left of the modified region if there were no new
	* tokens.
	*/
	private Token leftToken;

	/**
	* The token in the new token stream immediately to the right of the range of tokens that were
	* inserted, or the token immediately to the right of the modified region if there were no new
	* tokens.
	*/
	private Token rightToken;

	/**
	* A flag indicating whether there were any tokens changed as a result of the modification.
	*/
	private boolean hasNonWhitespaceChange = false;

	/**
	* Initialize a newly created scanner.
	*
	* @param source the source being scanned
	* @param reader the character reader used to read the characters in the source
	* @param errorListener the error listener that will be informed of any errors that are found
	*/
	public IncrementalScanner(Source source, CharacterReader reader,
	AnalysisErrorListener errorListener) {
	super(source, reader, errorListener);
	this.reader = reader;
	}

	/**
	* Return the token in the new token stream immediately to the left of the range of tokens that
	* were inserted, or the token immediately to the left of the modified region if there were no new
	* tokens.
	*
	* @return the token to the left of the inserted tokens
	*/
	public Token getLeftToken() {
	return leftToken;
	}

	/**
	* Return the token in the new token stream immediately to the right of the range of tokens that
	* were inserted, or the token immediately to the right of the modified region if there were no
	* new tokens.
	*
	* @return the token to the right of the inserted tokens
	*/
	public Token getRightToken() {
	return rightToken;
	}

	/**
	* Return a map from tokens that were copied to the copies of the tokens.
	*
	* @return a map from tokens that were copied to the copies of the tokens
	*/
	public TokenMap getTokenMap() {
	return tokenMap;
	}

	/**
	* Return {@code true} if there were any tokens either added or removed (or both) as a result of
	* the modification.
	*
	* @return {@code true} if there were any tokens changed as a result of the modification
	*/
	public boolean hasNonWhitespaceChange() {
	return hasNonWhitespaceChange;
	}

	/**
	* Given the stream of tokens scanned from the original source, the modified source (the result of
	* replacing one contiguous range of characters with another string of characters), and a
	* specification of the modification that was made, return a stream of tokens scanned from the
	* modified source. The original stream of tokens will not be modified.
	*
	* @param originalStream the stream of tokens scanned from the original source
	* @param index the index of the first character in both the original and modified source that was
	* affected by the modification
	* @param removedLength the number of characters removed from the original source
	* @param insertedLength the number of characters added to the modified source
	*/
	public Token rescan(Token originalStream, int index, int removedLength, int insertedLength) {
	//
	// Copy all of the tokens in the originalStream whose end is less than the replacement start.
	// (If the replacement start is equal to the end of an existing token, then it means that the
	// existing token might have been modified, so we need to rescan it.)
	//
	while (originalStream.getType() != TokenType.EOF && originalStream.getEnd() < index) {
	originalStream = copyAndAdvance(originalStream, 0);
	}
	Token oldFirst = originalStream;
	Token oldLeftToken = originalStream.getPrevious();
	leftToken = getTail();
	//
	// Skip tokens in the original stream until we find a token whose offset is greater than the end
	// of the removed region. (If the end of the removed region is equal to the beginning of an
	// existing token, then it means that the existing token might have been modified, so we need to
	// rescan it.)
	//
	int removedEnd = index + (removedLength == 0 ? 0 : removedLength - 1);
	while (originalStream.getType() != TokenType.EOF && originalStream.getOffset() <= removedEnd) {
	originalStream = originalStream.getNext();
	}
	Token oldLast;
	Token oldRightToken;
	if (originalStream.getType() != TokenType.EOF && removedEnd + 1 == originalStream.getOffset()) {
	oldLast = originalStream;
	originalStream = originalStream.getNext();
	oldRightToken = originalStream;
	} else {
	oldLast = originalStream.getPrevious();
	oldRightToken = originalStream;
	}
	//
	// Compute the delta between the character index of characters after the modified region in the
	// original source and the index of the corresponding character in the modified source.
	//
	int delta = insertedLength - removedLength;
	//
	// Compute the range of characters that are known to need to be rescanned. If the index is
	// within an existing token, then we need to start at the beginning of the token.
	//
	int scanStart = Math.min(oldFirst.getOffset(), index);
	int oldEnd = oldLast.getEnd() + delta - 1;
	int newEnd = index + insertedLength - 1;
	int scanEnd = Math.max(newEnd, oldEnd);
	//
	// Starting at the start of the scan region, scan tokens from the modifiedSource until the end
	// of the just scanned token is greater than or equal to end of the scan region in the modified
	// source. Include trailing characters of any token that was split as a result of inserted text,
	// as in "ab" --> "a.b".
	//
	reader.setOffset(scanStart - 1);
	int next = reader.advance();
	while (next != -1 && reader.getOffset() <= scanEnd) {
	next = bigSwitch(next);
	}
	//
	// Copy the remaining tokens in the original stream, but apply the delta to the token's offset.
	//
	if (originalStream.getType() == TokenType.EOF) {
	copyAndAdvance(originalStream, delta);
	rightToken = getTail();
	rightToken.setNextWithoutSettingPrevious(rightToken);
	} else {
	originalStream = copyAndAdvance(originalStream, delta);
	rightToken = getTail();
	while (originalStream.getType() != TokenType.EOF) {
	originalStream = copyAndAdvance(originalStream, delta);
	}
	Token eof = copyAndAdvance(originalStream, delta);
	eof.setNextWithoutSettingPrevious(eof);
	}
	//
	// If the index is immediately after an existing token and the inserted characters did not
	// change that original token, then adjust the leftToken to be the next token. For example, in
	// "a; c;" --> "a;b c;", the leftToken was ";", but this code advances it to "b" since "b" is
	// the first new token.
	//
	Token newFirst = leftToken.getNext();
	while (newFirst != rightToken && oldFirst != oldRightToken
	&& newFirst.getType() != TokenType.EOF && equalTokens(oldFirst, newFirst)) {
	tokenMap.put(oldFirst, newFirst);
	oldLeftToken = oldFirst;
	oldFirst = oldFirst.getNext();
	leftToken = newFirst;
	newFirst = newFirst.getNext();
	}
	Token newLast = rightToken.getPrevious();
	while (newLast != leftToken && oldLast != oldLeftToken && newLast.getType() != TokenType.EOF
	&& equalTokens(oldLast, newLast)) {
	tokenMap.put(oldLast, newLast);
	oldRightToken = oldLast;
	oldLast = oldLast.getPrevious();
	rightToken = newLast;
	newLast = newLast.getPrevious();
	}
	hasNonWhitespaceChange = leftToken.getNext() != rightToken
	\|\| oldLeftToken.getNext() != oldRightToken;
	//
	// TODO(brianwilkerson) Begin tokens are not getting associated with the corresponding end
	// tokens (because the end tokens have not been copied when we're copying the begin tokens).
	// This could have implications for parsing.
	// TODO(brianwilkerson) Update the lineInfo.
	//
	return getFirstToken();
	}

	private Token copyAndAdvance(Token originalToken, int delta) {
	Token copiedToken = originalToken.copy();
	tokenMap.put(originalToken, copiedToken);
	copiedToken.applyDelta(delta);
	appendToken(copiedToken);

	Token originalComment = originalToken.getPrecedingComments();
	Token copiedComment = originalToken.getPrecedingComments();
	while (originalComment != null) {
	tokenMap.put(originalComment, copiedComment);
	originalComment = originalComment.getNext();
	copiedComment = copiedComment.getNext();
	}
	return originalToken.getNext();
	}

	/**
	* Return {@code true} if the two tokens are equal to each other. For the purposes of the
	* incremental scanner, two tokens are equal if they have the same type and lexeme.
	*
	* @param oldToken the token from the old stream that is being compared
	* @param newToken the token from the new stream that is being compared
	* @return {@code true} if the two tokens are equal to each other
	*/
	private boolean equalTokens(Token oldToken, Token newToken) {
	return oldToken.getType() == newToken.getType() && oldToken.getLength() == newToken.getLength()
	&& oldToken.getLexeme().equals(newToken.getLexeme());
	}
	}