|
1 |
| |
|
2 |
| |
|
3 |
| |
|
4 |
| package net.sourceforge.pmd.cpd; |
|
5 |
| |
|
6 |
| import net.sourceforge.pmd.cpd.cppast.CPPParserTokenManager; |
|
7 |
| import net.sourceforge.pmd.cpd.cppast.SimpleCharStream; |
|
8 |
| import net.sourceforge.pmd.cpd.cppast.Token; |
|
9 |
| import net.sourceforge.pmd.cpd.cppast.TokenMgrError; |
|
10 |
| |
|
11 |
| import java.io.StringReader; |
|
12 |
| import java.util.ArrayList; |
|
13 |
| import java.util.List; |
|
14 |
| |
|
15 |
| public class CPPTokenizer implements Tokenizer { |
|
16 |
| protected String EOL = System.getProperty("line.separator", "\n"); |
|
17 |
| |
|
18 |
| private static SimpleCharStream charStream; |
|
19 |
| |
|
20 |
4
| public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
|
|
21 |
4
| StringBuffer sb = sourceCode.getCodeBuffer();
|
|
22 |
4
| try {
|
|
23 |
4
| if (charStream == null) {
|
|
24 |
1
| charStream = new SimpleCharStream(new StringReader(sb.toString()));
|
|
25 |
| } else { |
|
26 |
3
| charStream.ReInit(new StringReader(sb.toString()));
|
|
27 |
| } |
|
28 |
4
| CPPParserTokenManager.ReInit(charStream);
|
|
29 |
4
| CPPParserTokenManager.setFileName(sourceCode.getFileName());
|
|
30 |
4
| Token currToken = CPPParserTokenManager.getNextToken();
|
|
31 |
4
| while (currToken.image.length() > 0) {
|
|
32 |
39
| tokenEntries.add(new TokenEntry(currToken.image, sourceCode.getFileName(), currToken.beginLine));
|
|
33 |
39
| currToken = CPPParserTokenManager.getNextToken();
|
|
34 |
| } |
|
35 |
4
| tokenEntries.add(TokenEntry.getEOF());
|
|
36 |
4
| System.out.println("Added " + sourceCode.getFileName());
|
|
37 |
| } catch (TokenMgrError err) { |
|
38 |
0
| err.printStackTrace();
|
|
39 |
0
| System.out.println("Skipping " + sourceCode.getFileName() + " due to parse error");
|
|
40 |
0
| List emptyCode = new ArrayList();
|
|
41 |
0
| emptyCode.add("");
|
|
42 |
0
| tokenEntries.add(TokenEntry.getEOF());
|
|
43 |
| } |
|
44 |
| } |
|
45 |
| } |