源码类别
热门源码
 当前位置：源码 > Java源码 >

Java源码示例：codemining.languagetools.tokenizers.whitespace.WhitespaceToTokenConverter

示例1
@Override
public SortedMap<Integer, FullToken> fullTokenListWithPos(final char[] code) {
	final SortedMap<Integer, FullToken> tokens = Maps.newTreeMap();
	tokens.put(-1, new FullToken(SENTENCE_START, SENTENCE_START));
	tokens.put(Integer.MAX_VALUE, new FullToken(SENTENCE_END, SENTENCE_END));

	final Scanner scanner = new Scanner();
	scanner.setSource(code);
	final WhitespaceToTokenConverter wsConverter = new WhitespaceToTokenConverter();
	do {
		final int token = scanner.getNextToken();
		if (token == Token.tWHITESPACE) {
			final String wsToken = wsConverter
					.toWhiteSpaceSymbol(new String(scanner
							.getCurrentTokenSource()));
			tokens.put(scanner.getCurrentPosition(), new FullToken(wsToken,
					Integer.toString(token)));
		} else {
			final String nxtToken = new String(
					scanner.getCurrentTokenSource());
			tokens.put(scanner.getCurrentPosition(), new FullToken(
					getTokenType(token, nxtToken), Integer.toString(token)));
		}
	} while (!scanner.atEnd());
	return tokens;
}
示例2
@Override
public List<FullToken> getTokenListFromCode(final char[] code) {
	final List<FullToken> tokens = Lists.newArrayList();
	tokens.add(new FullToken(SENTENCE_START, SENTENCE_START));

	final Scanner scanner = new Scanner();
	scanner.setSource(code);
	final WhitespaceToTokenConverter wsConverter = new WhitespaceToTokenConverter();
	do {
		final int token = scanner.getNextToken();
		if (token == Token.tWHITESPACE) {
			final String wsToken = wsConverter
					.toWhiteSpaceSymbol(new String(scanner
							.getCurrentTokenSource()));
			tokens.add(new FullToken(wsToken, Integer.toString(token)));
		} else {
			final String nxtToken = new String(
					scanner.getCurrentTokenSource());
			tokens.add(new FullToken(getTokenType(token, nxtToken), Integer
					.toString(token)));
		}
	} while (!scanner.atEnd());

	tokens.add(new FullToken(SENTENCE_END, SENTENCE_END));
	return tokens;
}
示例3
@Override
public List<String> tokenListFromCode(final char[] code) {
	final List<String> tokens = Lists.newArrayList();
	tokens.add(SENTENCE_START);

	final Scanner scanner = new Scanner();
	scanner.setSource(code);
	final WhitespaceToTokenConverter wsConverter = new WhitespaceToTokenConverter();
	do {
		final int token = scanner.getNextToken();
		if (token == Token.tWHITESPACE) {
			final String wsToken = wsConverter
					.toWhiteSpaceSymbol(new String(scanner
							.getCurrentTokenSource()));
			tokens.add(wsToken);
		} else {
			final String nxtToken = new String(
					scanner.getCurrentTokenSource());
			tokens.add(getTokenType(token, nxtToken));
		}
	} while (!scanner.atEnd());

	tokens.add(SENTENCE_END);
	return tokens;
}
示例4
@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);

	final Scanner scanner = new Scanner();
	scanner.setSource(code);
	final WhitespaceToTokenConverter wsConverter = new WhitespaceToTokenConverter();
	do {
		final int token = scanner.getNextToken();
		if (token == Token.tWHITESPACE) {
			final String wsToken = wsConverter
					.toWhiteSpaceSymbol(new String(scanner
							.getCurrentTokenSource()));
			tokens.put(scanner.getCurrentPosition(), wsToken);
		} else {
			final String nxtToken = new String(
					scanner.getCurrentTokenSource());
			tokens.put(scanner.getCurrentPosition(),
					getTokenType(token, nxtToken));
		}
	} while (!scanner.atEnd());
	return tokens;
}
示例5
@Override
public SortedMap<Integer, FullToken> fullTokenListWithPos(final char[] code) {
	final SortedMap<Integer, FullToken> tokens = Maps.newTreeMap();
	tokens.put(-1, new FullToken(SENTENCE_START, SENTENCE_START));
	tokens.put(Integer.MAX_VALUE, new FullToken(SENTENCE_END, SENTENCE_END));

	final Scanner scanner = new Scanner();
	scanner.setSource(code);
	final WhitespaceToTokenConverter wsConverter = new WhitespaceToTokenConverter();
	do {
		final int token = scanner.getNextToken();
		if (token == Token.tWHITESPACE) {
			final String wsToken = wsConverter
					.toWhiteSpaceSymbol(new String(scanner
							.getCurrentTokenSource()));
			tokens.put(scanner.getCurrentPosition(), new FullToken(wsToken,
					Integer.toString(token)));
		} else {
			final String nxtToken = new String(
					scanner.getCurrentTokenSource());
			tokens.put(scanner.getCurrentPosition(), new FullToken(
					getTokenType(token, nxtToken), Integer.toString(token)));
		}
	} while (!scanner.atEnd());
	return tokens;
}
示例6
@Override
public List<FullToken> getTokenListFromCode(final char[] code) {
	final List<FullToken> tokens = Lists.newArrayList();
	tokens.add(new FullToken(SENTENCE_START, SENTENCE_START));

	final Scanner scanner = new Scanner();
	scanner.setSource(code);
	final WhitespaceToTokenConverter wsConverter = new WhitespaceToTokenConverter();
	do {
		final int token = scanner.getNextToken();
		if (token == Token.tWHITESPACE) {
			final String wsToken = wsConverter
					.toWhiteSpaceSymbol(new String(scanner
							.getCurrentTokenSource()));
			tokens.add(new FullToken(wsToken, Integer.toString(token)));
		} else {
			final String nxtToken = new String(
					scanner.getCurrentTokenSource());
			tokens.add(new FullToken(getTokenType(token, nxtToken), Integer
					.toString(token)));
		}
	} while (!scanner.atEnd());

	tokens.add(new FullToken(SENTENCE_END, SENTENCE_END));
	return tokens;
}
示例7
@Override
public List<String> tokenListFromCode(final char[] code) {
	final List<String> tokens = Lists.newArrayList();
	tokens.add(SENTENCE_START);

	final Scanner scanner = new Scanner();
	scanner.setSource(code);
	final WhitespaceToTokenConverter wsConverter = new WhitespaceToTokenConverter();
	do {
		final int token = scanner.getNextToken();
		if (token == Token.tWHITESPACE) {
			final String wsToken = wsConverter
					.toWhiteSpaceSymbol(new String(scanner
							.getCurrentTokenSource()));
			tokens.add(wsToken);
		} else {
			final String nxtToken = new String(
					scanner.getCurrentTokenSource());
			tokens.add(getTokenType(token, nxtToken));
		}
	} while (!scanner.atEnd());

	tokens.add(SENTENCE_END);
	return tokens;
}
示例8
@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);

	final Scanner scanner = new Scanner();
	scanner.setSource(code);
	final WhitespaceToTokenConverter wsConverter = new WhitespaceToTokenConverter();
	do {
		final int token = scanner.getNextToken();
		if (token == Token.tWHITESPACE) {
			final String wsToken = wsConverter
					.toWhiteSpaceSymbol(new String(scanner
							.getCurrentTokenSource()));
			tokens.put(scanner.getCurrentPosition(), wsToken);
		} else {
			final String nxtToken = new String(
					scanner.getCurrentTokenSource());
			tokens.put(scanner.getCurrentPosition(),
					getTokenType(token, nxtToken));
		}
	} while (!scanner.atEnd());
	return tokens;
}
示例9
@Override
public SortedMap<Integer, FullToken> fullTokenListWithPos(final char[] code) {
	final SortedMap<Integer, FullToken> tokens = Maps.newTreeMap();
	tokens.put(-1, new FullToken(SENTENCE_START, SENTENCE_START));
	tokens.put(Integer.MAX_VALUE, new FullToken(SENTENCE_END, SENTENCE_END));

	final Scanner scanner = new Scanner();
	scanner.setSource(code);
	final WhitespaceToTokenConverter wsConverter = new WhitespaceToTokenConverter();
	do {
		final int token = scanner.getNextToken();
		if (token == Token.tWHITESPACE) {
			final String wsToken = wsConverter
					.toWhiteSpaceSymbol(new String(scanner
							.getCurrentTokenSource()));
			tokens.put(scanner.getCurrentPosition(), new FullToken(wsToken,
					Integer.toString(token)));
		} else {
			final String nxtToken = new String(
					scanner.getCurrentTokenSource());
			tokens.put(scanner.getCurrentPosition(), new FullToken(
					getTokenType(token, nxtToken), Integer.toString(token)));
		}
	} while (!scanner.atEnd());
	return tokens;
}
示例10
@Override
public List<FullToken> getTokenListFromCode(final char[] code) {
	final List<FullToken> tokens = Lists.newArrayList();
	tokens.add(new FullToken(SENTENCE_START, SENTENCE_START));

	final Scanner scanner = new Scanner();
	scanner.setSource(code);
	final WhitespaceToTokenConverter wsConverter = new WhitespaceToTokenConverter();
	do {
		final int token = scanner.getNextToken();
		if (token == Token.tWHITESPACE) {
			final String wsToken = wsConverter
					.toWhiteSpaceSymbol(new String(scanner
							.getCurrentTokenSource()));
			tokens.add(new FullToken(wsToken, Integer.toString(token)));
		} else {
			final String nxtToken = new String(
					scanner.getCurrentTokenSource());
			tokens.add(new FullToken(getTokenType(token, nxtToken), Integer
					.toString(token)));
		}
	} while (!scanner.atEnd());

	tokens.add(new FullToken(SENTENCE_END, SENTENCE_END));
	return tokens;
}
示例11
@Override
public List<String> tokenListFromCode(final char[] code) {
	final List<String> tokens = Lists.newArrayList();
	tokens.add(SENTENCE_START);

	final Scanner scanner = new Scanner();
	scanner.setSource(code);
	final WhitespaceToTokenConverter wsConverter = new WhitespaceToTokenConverter();
	do {
		final int token = scanner.getNextToken();
		if (token == Token.tWHITESPACE) {
			final String wsToken = wsConverter
					.toWhiteSpaceSymbol(new String(scanner
							.getCurrentTokenSource()));
			tokens.add(wsToken);
		} else {
			final String nxtToken = new String(
					scanner.getCurrentTokenSource());
			tokens.add(getTokenType(token, nxtToken));
		}
	} while (!scanner.atEnd());

	tokens.add(SENTENCE_END);
	return tokens;
}
示例12
@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);

	final Scanner scanner = new Scanner();
	scanner.setSource(code);
	final WhitespaceToTokenConverter wsConverter = new WhitespaceToTokenConverter();
	do {
		final int token = scanner.getNextToken();
		if (token == Token.tWHITESPACE) {
			final String wsToken = wsConverter
					.toWhiteSpaceSymbol(new String(scanner
							.getCurrentTokenSource()));
			tokens.put(scanner.getCurrentPosition(), wsToken);
		} else {
			final String nxtToken = new String(
					scanner.getCurrentTokenSource());
			tokens.put(scanner.getCurrentPosition(),
					getTokenType(token, nxtToken));
		}
	} while (!scanner.atEnd());
	return tokens;
}
示例13
/**
 * @param code
 * @return
 */
private PublicScanner prepareScanner(final char[] code) {
	final PublicScanner scanner = new PublicScanner();
	scanner.setSource(code);
	scanner.tokenizeWhiteSpace = true;
	scanner.recordLineSeparator = true;
	scanner.tokenizeComments = true;
	whitespaceConverter = new WhitespaceToTokenConverter();
	return scanner;
}
示例14
/**
 * @param code
 * @return
 */
private PublicScanner prepareScanner(final char[] code) {
	final PublicScanner scanner = new PublicScanner();
	scanner.setSource(code);
	scanner.tokenizeWhiteSpace = true;
	scanner.recordLineSeparator = true;
	scanner.tokenizeComments = true;
	whitespaceConverter = new WhitespaceToTokenConverter();
	return scanner;
}
示例15
/**
 * @param code
 * @return
 */
private PublicScanner prepareScanner(final char[] code) {
	final PublicScanner scanner = new PublicScanner();
	scanner.setSource(code);
	scanner.tokenizeWhiteSpace = true;
	scanner.recordLineSeparator = true;
	scanner.tokenizeComments = true;
	whitespaceConverter = new WhitespaceToTokenConverter();
	return scanner;
}