private int getLastSimpleNameStart(String reference) {
	fScanner.setSource(reference.toCharArray());
	int lastIdentifierStart = -1;
	try {
		int tokenType = fScanner.getNextToken();
		while (tokenType != ITerminalSymbols.TokenNameEOF) {
			if (tokenType == ITerminalSymbols.TokenNameIdentifier) {
				lastIdentifierStart = fScanner.getCurrentTokenStartPosition();
			}
			tokenType = fScanner.getNextToken();
		}
	} catch (InvalidInputException e) {
		JavaLanguageServerPlugin.logException(e.getMessage(), e);
	}
	return lastIdentifierStart;
}
 

/**
 * @param source
 *            must be not null
 * @param range
 *            can be null
 * @return may return null, otherwise an initialized scanner which may
 *         answer which source offset index belongs to which source line
 * @throws JavaModelException
 */
private static IScanner initScanner(IType source, ISourceRange range) throws JavaModelException {
    if (range == null) {
        return null;
    }
    char[] charContent = getContent(source);
    if (charContent == null) {
        return null;
    }
    IScanner scanner = ToolFactory.createScanner(false, false, false, true);
    scanner.setSource(charContent);
    int offset = range.getOffset();
    try {
        while (scanner.getNextToken() != ITerminalSymbols.TokenNameEOF) {
            // do nothing, just wait for the end of stream
            if (offset <= scanner.getCurrentTokenEndPosition()) {
                break;
            }
        }
    } catch (InvalidInputException e) {
        FindbugsPlugin.getDefault().logException(e, "Could not init scanner for type: " + source);
    }
    return scanner;
}
 

@Override
public List<String> tokenListFromCode(final char[] code) {
	final List<String> tokens = Lists.newArrayList();
	tokens.add(SENTENCE_START);
	final PublicScanner scanner = prepareScanner(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			}
			tokens.addAll(getConvertedToken(scanner, token));
		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(SENTENCE_END);
	return tokens;
}
 

@Override
public List<String> tokenListFromCode(final char[] code) {
	final List<String> tokens = Lists.newArrayList();
	tokens.add(SENTENCE_START);
	final PublicScanner scanner = prepareScanner(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			}
			tokens.addAll(getConvertedToken(scanner, token));
		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(SENTENCE_END);
	return tokens;
}
 

/**
 * Sets the string value of this literal node. The value is the sequence
 * of characters that would appear in the source program, including
 * enclosing single quotes and embedded escapes. For example,
 * <ul>
 * <li><code>'a'</code> <code>setEscapedValue("\'a\'")</code></li>
 * <li><code>'\n'</code> <code>setEscapedValue("\'\\n\'")</code></li>
 * </ul>
 *
 * @param value the string value, including enclosing single quotes
 *    and embedded escapes
 * @exception IllegalArgumentException if the argument is incorrect
 */
public void setEscapedValue(String value) {
	// check setInternalEscapedValue(String) if this method is changed
	if (value == null) {
		throw new IllegalArgumentException();
	}
	Scanner scanner = this.ast.scanner;
	char[] source = value.toCharArray();
	scanner.setSource(source);
	scanner.resetTo(0, source.length);
	try {
		int tokenType = scanner.getNextToken();
		switch(tokenType) {
			case TerminalTokens.TokenNameCharacterLiteral:
				break;
			default:
				throw new IllegalArgumentException();
		}
	} catch(InvalidInputException e) {
		throw new IllegalArgumentException();
	}
	preValueChange(ESCAPED_VALUE_PROPERTY);
	this.escapedValue = value;
	postValueChange(ESCAPED_VALUE_PROPERTY);
}
 

@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final PublicScanner scanner = prepareScanner();
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				}
				final String nxtToken = transformToken(token,
						scanner.getCurrentTokenString());
				final int position = scanner.getCurrentTokenStartPosition();
				tokens.put(position, stripTokenIfNeeded(nxtToken));
			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}
 

@Override
public List<FullToken> getTokenListFromCode(final char[] code) {
	final List<FullToken> tokens = Lists.newArrayList();
	tokens.add(new FullToken(SENTENCE_START, SENTENCE_START));
	final PublicScanner scanner = prepareScanner(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			}
			for (final String cToken : getConvertedToken(scanner, token)) {
				tokens.add(new FullToken(cToken, ""));
			}
		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(new FullToken(SENTENCE_END, SENTENCE_END));
	return tokens;
}
 

@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final PublicScanner scanner = prepareScanner();
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				}
				final String nxtToken = transformToken(token,
						scanner.getCurrentTokenString());
				final int position = scanner.getCurrentTokenStartPosition();
				tokens.put(position, stripTokenIfNeeded(nxtToken));
			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}
 

/**
 * Removes comments and whitespace
 * @param reference the type reference
 * @return the reference only consisting of dots and java identifier characters
 */
public static String normalizeReference(String reference) {
	IScanner scanner= ToolFactory.createScanner(false, false, false, false);
	scanner.setSource(reference.toCharArray());
	StringBuffer sb= new StringBuffer();
	try {
		int tokenType= scanner.getNextToken();
		while (tokenType != ITerminalSymbols.TokenNameEOF) {
			sb.append(scanner.getRawTokenSource());
			tokenType= scanner.getNextToken();
		}
	} catch (InvalidInputException e) {
		Assert.isTrue(false, reference);
	}
	reference= sb.toString();
	return reference;
}
 

protected boolean parseThrows() {
	int start = this.scanner.currentPosition;
	try {
		Object typeRef = parseQualifiedName(true);
		if (this.abort) return false; // May be aborted by specialized parser
		if (typeRef == null) {
			if (this.reportProblems)
				this.sourceParser.problemReporter().javadocMissingThrowsClassName(this.tagSourceStart, this.tagSourceEnd, this.sourceParser.modifiers);
		} else {
			return pushThrowName(typeRef);
		}
	} catch (InvalidInputException ex) {
		if (this.reportProblems) this.sourceParser.problemReporter().javadocInvalidThrowsClass(start, getTokenEndPosition());
	}
	return false;
}
 

private final int consumeDigits0(int radix, int usingUnderscore, int invalidPosition, boolean expectingDigitFirst) throws InvalidInputException {
	int kind = 0;
	if (getNextChar('_')) {
		if (expectingDigitFirst) {
			return invalidPosition;
		}
		kind = usingUnderscore;
		while (getNextChar('_')) {/*empty */}
	}
	if (getNextCharAsDigit(radix)) {
		// continue to read digits or underscore
		while (getNextCharAsDigit(radix)) {/*empty */}
		int kind2 = consumeDigits0(radix, usingUnderscore, invalidPosition, false);
		if (kind2 == 0) {
			return kind;
		}
		return kind2;
	}
	if (kind == usingUnderscore) return invalidPosition;
	return kind;
}
 

private int getSurroundingComment(IScanner scanner) {
	try {
		int start= fLocation.getOffset();
		int end= start + fLocation.getLength();

		int token= scanner.getNextToken();
		while (token != ITerminalSymbols.TokenNameEOF) {
			if (TokenScanner.isComment(token)) {
				int currStart= scanner.getCurrentTokenStartPosition();
				int currEnd= scanner.getCurrentTokenEndPosition() + 1;
				if (currStart <= start && end <= currEnd) {
					return token;
				}
			}
			token= scanner.getNextToken();
		}

	} catch (InvalidInputException e) {
		// ignore
	}
	return ITerminalSymbols.TokenNameEOF;
}
 

private final int consumeDigits0(int radix, int usingUnderscore, int invalidPosition, boolean expectingDigitFirst) throws InvalidInputException {
	int kind = 0;
	if (getNextChar('_')) {
		if (expectingDigitFirst) {
			return invalidPosition;
		}
		kind = usingUnderscore;
		while (getNextChar('_')) {/*empty */}
	}
	if (getNextCharAsDigit(radix)) {
		// continue to read digits or underscore
		while (getNextCharAsDigit(radix)) {/*empty */}
		int kind2 = consumeDigits0(radix, usingUnderscore, invalidPosition, false);
		if (kind2 == 0) {
			return kind;
		}
		return kind2;
	}
	if (kind == usingUnderscore) return invalidPosition;
	return kind;
}
 

protected Object createArgumentReference(char[] name, int dim, boolean isVarargs, Object typeRef, long[] dimPositions, long argNamePos) throws InvalidInputException {
	// Create argument as we may need it after
	char[] argName = name==null ? CharOperation.NO_CHAR : name;
	Expression expression = (Expression) super.createArgumentReference(argName, dim, isVarargs, typeRef, dimPositions, argNamePos);
	// See if completion location is in argument
	int refStart = ((TypeReference)typeRef).sourceStart;
	int refEnd = ((TypeReference)typeRef).sourceEnd;
	boolean inCompletion = (refStart <= this.cursorLocation && this.cursorLocation <= refEnd) // completion cursor is between first and last stacked identifiers
		|| ((refStart == (refEnd+1) && refEnd == this.cursorLocation)); // or it's a completion on empty token
	if (this.completionNode == null && inCompletion) {
		JavadocArgumentExpression javadocArgument = (JavadocArgumentExpression) expression;
		TypeReference expressionType = javadocArgument.argument.type;
		if (expressionType instanceof JavadocSingleTypeReference) {
			this.completionNode = new CompletionOnJavadocSingleTypeReference((JavadocSingleTypeReference) expressionType);
		} else if (expressionType instanceof JavadocQualifiedTypeReference) {
			this.completionNode = new CompletionOnJavadocQualifiedTypeReference((JavadocQualifiedTypeReference) expressionType);
		}
		if (CompletionEngine.DEBUG) {
			System.out.println("	completion argument="+this.completionNode); //$NON-NLS-1$
		}
		return this.completionNode;
	}
	return expression;
}
 

protected Object createFieldReference(Object receiver) throws InvalidInputException {
	int refStart = (int) (this.identifierPositionStack[0] >>> 32);
	int refEnd = (int) this.identifierPositionStack[0];
	boolean inCompletion = (refStart <= (this.cursorLocation+1) && this.cursorLocation <= refEnd) // completion cursor is between first and last stacked identifiers
		|| ((refStart == (refEnd+1) && refEnd == this.cursorLocation)) // or it's a completion on empty token
		|| (this.memberStart == this.cursorLocation); // or it's a completion just after the member separator with an identifier after the cursor
	if (inCompletion) {
		JavadocFieldReference fieldRef = (JavadocFieldReference) super.createFieldReference(receiver);
		char[] name = this.sourceParser.compilationUnit.getMainTypeName();
		TypeDeclaration typeDecl = getParsedTypeDeclaration();
		if (typeDecl != null) {
			name = typeDecl.name;
		}
		this.completionNode = new CompletionOnJavadocFieldReference(fieldRef, this.memberStart, name);
		if (CompletionEngine.DEBUG) {
			System.out.println("	completion field="+this.completionNode); //$NON-NLS-1$
		}
		return this.completionNode;
	}
	return super.createFieldReference(receiver);
}
 

protected boolean parseTag(int previousPosition) throws InvalidInputException {
	int startPosition = this.inlineTagStarted ? this.inlineTagStart : previousPosition;
	boolean newLine = !this.lineStarted;
	boolean valid = super.parseTag(previousPosition);
	boolean inCompletion = (this.tagSourceStart <= (this.cursorLocation+1) && this.cursorLocation <= this.tagSourceEnd) // completion cursor is between first and last stacked identifiers
		|| ((this.tagSourceStart == (this.tagSourceEnd+1) && this.tagSourceEnd == this.cursorLocation)); // or it's a completion on empty token
	if (inCompletion) {
		int end = this.tagSourceEnd;
		if (this.inlineTagStarted && this.scanner.currentCharacter == '}') {
			end = this.scanner.currentPosition;
		}
		long position = (((long)startPosition)<<32) + end;
		int length = this.cursorLocation+1-this.tagSourceStart;
		char[] tag = new char[length];
		System.arraycopy(this.source, this.tagSourceStart, tag, 0, length);
		char[][][] tags = possibleTags(tag, newLine);
		if (tags != null) {
			this.completionNode = new CompletionOnJavadocTag(tag, position, startPosition, end, tags, this.allPossibleTags);
		}
	}
	return valid;
}
 

protected Object syntaxRecoverQualifiedName(int primitiveToken) throws InvalidInputException {
	if (this.cursorLocation == ((int)this.identifierPositionStack[this.identifierPtr])) {
		// special case of completion just before the dot.
		return createTypeReference(primitiveToken);
	}
	int idLength = this.identifierLengthStack[this.identifierLengthPtr];
	char[][] tokens = new char[idLength][];
	int startPtr = this.identifierPtr-idLength+1;
	System.arraycopy(this.identifierStack, startPtr, tokens, 0, idLength);
	long[] positions = new long[idLength+1];
	System.arraycopy(this.identifierPositionStack, startPtr, positions, 0, idLength);
	positions[idLength] = (((long)this.tokenPreviousPosition)<<32) + this.tokenPreviousPosition;
	this.completionNode = new CompletionOnJavadocQualifiedTypeReference(tokens, CharOperation.NO_CHAR, positions, this.tagSourceStart, this.tagSourceEnd);

	if (CompletionEngine.DEBUG) {
		System.out.println("	completion partial qualified type="+this.completionNode); //$NON-NLS-1$
	}
	return this.completionNode;
}
 

protected Object syntaxRecoverEmptyArgumentType(Object methodRef) throws InvalidInputException {
	if (methodRef instanceof JavadocMessageSend) {
		JavadocMessageSend msgSend = (JavadocMessageSend) methodRef;
		if (this.index > this.cursorLocation) {
			msgSend.sourceEnd = this.tokenPreviousPosition-1;
		}
		this.completionNode = new CompletionOnJavadocMessageSend(msgSend, this.memberStart);
	} else if (methodRef instanceof JavadocAllocationExpression) {
		JavadocAllocationExpression allocExp = (JavadocAllocationExpression) methodRef;
		if (this.index > this.cursorLocation) {
			allocExp.sourceEnd = this.tokenPreviousPosition-1;
		}
		this.completionNode = new CompletionOnJavadocAllocationExpression(allocExp, this.memberStart);
	}
	if (CompletionEngine.DEBUG) {
		System.out.println("	completion method="+this.completionNode); //$NON-NLS-1$
	}
	return this.completionNode;
}
 

@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final PublicScanner scanner = prepareScanner();
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				}
				final String nxtToken = transformToken(token,
						scanner.getCurrentTokenString());
				final int position = scanner.getCurrentTokenStartPosition();
				tokens.put(position, stripTokenIfNeeded(nxtToken));
			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}
 

public void printNextToken(int[] expectedTokenTypes, boolean considerSpaceIfAny){
	printComment(CodeFormatter.K_UNKNOWN, NO_TRAILING_COMMENT);
	try {
		this.currentToken = this.scanner.getNextToken();
		if (Arrays.binarySearch(expectedTokenTypes, this.currentToken) < 0) {
			StringBuffer expectations = new StringBuffer(5);
			for (int i = 0; i < expectedTokenTypes.length; i++){
				if (i > 0) {
					expectations.append(',');
				}
				expectations.append(expectedTokenTypes[i]);
			}
			throw new AbortFormatting("unexpected token type, expecting:["+expectations.toString()+"], actual:"+this.currentToken);//$NON-NLS-1$//$NON-NLS-2$
		}
		print(this.scanner.currentPosition - this.scanner.startPosition, considerSpaceIfAny);
	} catch (InvalidInputException e) {
		throw new AbortFormatting(e);
	}
}
 

protected Object createArgumentReference(char[] name, int dim, boolean isVarargs, Object typeRef, long[] dimPositions, long argNamePos) throws InvalidInputException {
	try {
		TypeReference argTypeRef = (TypeReference) typeRef;
		if (dim > 0) {
			long pos = (((long) argTypeRef.sourceStart) << 32) + argTypeRef.sourceEnd;
			if (typeRef instanceof JavadocSingleTypeReference) {
				JavadocSingleTypeReference singleRef = (JavadocSingleTypeReference) typeRef;
				argTypeRef = new JavadocArraySingleTypeReference(singleRef.token, dim, pos);
			} else {
				JavadocQualifiedTypeReference qualifRef = (JavadocQualifiedTypeReference) typeRef;
				argTypeRef = new JavadocArrayQualifiedTypeReference(qualifRef, dim);
			}
		}
		int argEnd = argTypeRef.sourceEnd;
		if (dim > 0) {
			argEnd = (int) dimPositions[dim-1];
			if (isVarargs) {
				argTypeRef.bits |= ASTNode.IsVarArgs; // set isVarArgs
			}
		}
		if (argNamePos >= 0) argEnd = (int) argNamePos;
		return new JavadocArgumentExpression(name, argTypeRef.sourceStart, argEnd, argTypeRef);
	}
	catch (ClassCastException ex) {
		throw new InvalidInputException();
	}
}
 

private boolean isIdentifier() throws InvalidInputException {
	switch(this.scanner.scanIdentifier()) {
		// assert and enum will not be recognized as java identifiers 
		// in 1.7 mode, which are in 1.3.
		case TerminalTokens.TokenNameIdentifier:
		case TerminalTokens.TokenNameassert:
		case TerminalTokens.TokenNameenum:
			return true;
		default:
			return false;
	}
}
 

public static NLSLine[] scan(ICompilationUnit cu) throws JavaModelException, BadLocationException, InvalidInputException {
	IJavaProject javaProject= cu.getJavaProject();
	IScanner scanner= null;
	if (javaProject != null) {
		String complianceLevel= javaProject.getOption(JavaCore.COMPILER_COMPLIANCE, true);
		String sourceLevel= javaProject.getOption(JavaCore.COMPILER_SOURCE, true);
		scanner= ToolFactory.createScanner(true, true, true, sourceLevel, complianceLevel);
	} else {
		scanner= ToolFactory.createScanner(true, true, false, true);
	}
	return scan(scanner, cu.getBuffer().getCharacters());
}
 

private int getNextToken(IScanner scanner) {
	int token = 0;
	while (token == 0) {
		try {
			token = scanner.getNextToken();
		} catch (InvalidInputException e) {
			// ignore
			// JavaLanguageServerPlugin.logException("Problem with folding range", e);
		}
	}
	return token;
}
 

/**
 * Creates a token comparator for the given string.
 *
 * @param text the text to be tokenized
 * @param textTokenComparatorFactory a factory to create text token comparators
 */
public SJavaTokenComparator(String text) {


  fText= text;

  int length= fText.length();
  fStarts= new int[length];
  fLengths= new int[length];
  fTokens= new String[length];
  fCount= 0;

  IScanner scanner= ToolFactory.createScanner(true, false, false, false); // returns comments & whitespace
  scanner.setSource(fText.toCharArray());
  int endPos= 0;
  try {
    int tokenType;
    while ((tokenType= scanner.getNextToken()) != ITerminalSymbols.TokenNameEOF) {
      int start= scanner.getCurrentTokenStartPosition();
      int end= scanner.getCurrentTokenEndPosition()+1;
      // Comments are treated as a single token (see bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=78063)
      if (!TokenScanner.isComment(tokenType)) {
        recordTokenRange(start, end - start);
      }
      endPos= end;
    }
  } catch (InvalidInputException ex) {
    // We couldn't parse part of the input. Fall through and make the rest a single token
  }
  // Workaround for https://bugs.eclipse.org/bugs/show_bug.cgi?id=13907
  if (endPos < length) {
    recordTokenRange(endPos, length - endPos);
  }
}
 

@Override
public List<FullToken> getTokenListFromCode(final char[] code) {
	final List<FullToken> tokens = Lists.newArrayList();
	tokens.add(new FullToken(SENTENCE_START, SENTENCE_START));
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			} else if (token == ITerminalSymbols.TokenNameIdentifier) {
				tokens.add(new FullToken(IDENTIFIER_TOKEN, ""));
			} else if (isLiteralToken(token)) {
				tokens.add(new FullToken(LITERAL_TOKEN, ""));
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
				tokens.add(new FullToken(COMMENT_BLOCK, ""));
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
				tokens.add(new FullToken(COMMENT_JAVADOC, ""));
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
				tokens.add(new FullToken(COMMENT_LINE, ""));
			} else {
				tokens.add(new FullToken(scanner.getCurrentTokenString(),
						""));
			}

		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(new FullToken(SENTENCE_END, SENTENCE_END));
	return tokens;
}
 

@Override
public List<String> tokenListFromCode(final char[] code) {
	final List<String> tokens = Lists.newArrayList();
	tokens.add(SENTENCE_START);
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			} else if (token == ITerminalSymbols.TokenNameIdentifier) {
				tokens.add(IDENTIFIER_TOKEN);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
				tokens.add(COMMENT_BLOCK);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
				tokens.add(COMMENT_LINE);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
				tokens.add(COMMENT_JAVADOC);
			} else if (isLiteralToken(token)) {
				tokens.add(LITERAL_TOKEN);
			} else {
				tokens.add(scanner.getCurrentTokenString());
			}

		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(SENTENCE_END);
	return tokens;
}
 

@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				final int position = scanner.getCurrentTokenStartPosition();

				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				} else if (token == ITerminalSymbols.TokenNameIdentifier) {
					tokens.put(position, IDENTIFIER_TOKEN);
				} else if (isLiteralToken(token)) {
					tokens.put(position, LITERAL_TOKEN);
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
					tokens.put(position, COMMENT_BLOCK);
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
					tokens.put(position, COMMENT_JAVADOC);
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
					tokens.put(position, COMMENT_LINE);
				} else {
					tokens.put(position, scanner.getCurrentTokenString());
				}

			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}
 

public List<WhitespaceAnnotatedToken> getTokensWithWidthData(
		final char[] code) {
	final List<WhitespaceAnnotatedToken> tokens = Lists.newArrayList();
	tokens.add(new WhitespaceAnnotatedToken(SENTENCE_START,
			SENTENCE_START, 0, 0));
	final PublicScanner scanner = prepareScanner(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			}
			for (final String cToken : getConvertedToken(scanner, token)) {
				final int currentPosition = scanner
						.getCurrentTokenStartPosition();
				final int currentLine = scanner
						.getLineNumber(currentPosition);
				final int lineStart = scanner.getLineStart(currentLine);
				tokens.add(new WhitespaceAnnotatedToken(cToken, "",
						currentPosition - lineStart, scanner
						.getCurrentTokenString().length()));
			}
		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(new WhitespaceAnnotatedToken(SENTENCE_END, SENTENCE_END,
			0, 0));
	return tokens;
}
 

@Override
public SortedMap<Integer, FullToken> fullTokenListWithPos(final char[] code) {
	// TODO Duplicate Code
	final PublicScanner scanner = prepareScanner();
	final SortedMap<Integer, FullToken> tokens = Maps.newTreeMap();
	tokens.put(-1, new FullToken(SENTENCE_START, SENTENCE_START));
	tokens.put(Integer.MAX_VALUE, new FullToken(SENTENCE_END, SENTENCE_END));
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				}
				final String nxtToken = transformToken(token,
						scanner.getCurrentTokenString());
				final int position = scanner.getCurrentTokenStartPosition();
				tokens.put(position,
						new FullToken(nxtToken, Integer.toString(token)));
			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}