|
1 |
| |
|
2 |
| |
|
3 |
| |
|
4 |
| package net.sourceforge.pmd.cpd; |
|
5 |
| |
|
6 |
| import java.io.BufferedReader; |
|
7 |
| import java.io.CharArrayReader; |
|
8 |
| import java.util.NoSuchElementException; |
|
9 |
| import java.util.StringTokenizer; |
|
10 |
| |
|
11 |
| |
|
12 |
| |
|
13 |
| |
|
14 |
| |
|
15 |
| |
|
16 |
| public class AnyTokenizer implements Tokenizer { |
|
17 |
| public static final String TOKENS = " \t!#$%^&*(){}-=+<>/\\`~;:"; |
|
18 |
| |
|
19 |
1
| public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
|
|
20 |
1
| StringBuffer sb = sourceCode.getCodeBuffer();
|
|
21 |
1
| BufferedReader reader = new BufferedReader(new CharArrayReader(sb.toString().toCharArray()));
|
|
22 |
1
| try {
|
|
23 |
1
| int lineNumber = 1;
|
|
24 |
1
| String line = reader.readLine();
|
|
25 |
1
| while (line != null) {
|
|
26 |
9
| StringTokenizer tokenizer = new StringTokenizer(line, TOKENS, true);
|
|
27 |
9
| try {
|
|
28 |
9
| String token = tokenizer.nextToken();
|
|
29 |
76
| while (token != null) {
|
|
30 |
76
| if (!token.equals(" ") && !token.equals("\t")) {
|
|
31 |
29
| tokenEntries.add(new TokenEntry(token, sourceCode.getFileName(), lineNumber));
|
|
32 |
| } |
|
33 |
76
| token = tokenizer.nextToken();
|
|
34 |
| } |
|
35 |
| } catch (NoSuchElementException ex) { |
|
36 |
| |
|
37 |
| } |
|
38 |
| |
|
39 |
9
| line = reader.readLine();
|
|
40 |
9
| lineNumber++;
|
|
41 |
| } |
|
42 |
| } catch (Exception ex) { |
|
43 |
0
| ex.printStackTrace();
|
|
44 |
| } finally { |
|
45 |
1
| try {
|
|
46 |
1
| reader.close();
|
|
47 |
| } catch (Exception ex) { |
|
48 |
| } |
|
49 |
1
| tokenEntries.add(TokenEntry.getEOF());
|
|
50 |
| } |
|
51 |
| } |
|
52 |
| } |