001/*-------------------------------------------------------------------------+ 002| | 003| Copyright 2005-2011 the ConQAT Project | 004| | 005| Licensed under the Apache License, Version 2.0 (the "License"); | 006| you may not use this file except in compliance with the License. | 007| You may obtain a copy of the License at | 008| | 009| http://www.apache.org/licenses/LICENSE-2.0 | 010| | 011| Unless required by applicable law or agreed to in writing, software | 012| distributed under the License is distributed on an "AS IS" BASIS, | 013| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | 014| See the License for the specific language governing permissions and | 015| limitations under the License. | 016+-------------------------------------------------------------------------*/ 017package eu.cqse.check.framework.util; 018 019import static eu.cqse.check.framework.scanner.ETokenType.AND; 020import static eu.cqse.check.framework.scanner.ETokenType.COMMA; 021import static eu.cqse.check.framework.scanner.ETokenType.CONST; 022import static eu.cqse.check.framework.scanner.ETokenType.DOT; 023import static eu.cqse.check.framework.scanner.ETokenType.ELLIPSIS; 024import static eu.cqse.check.framework.scanner.ETokenType.EQ; 025import static eu.cqse.check.framework.scanner.ETokenType.GT; 026import static eu.cqse.check.framework.scanner.ETokenType.LBRACE; 027import static eu.cqse.check.framework.scanner.ETokenType.LBRACK; 028import static eu.cqse.check.framework.scanner.ETokenType.LPAREN; 029import static eu.cqse.check.framework.scanner.ETokenType.LT; 030import static eu.cqse.check.framework.scanner.ETokenType.MULT; 031import static eu.cqse.check.framework.scanner.ETokenType.RBRACE; 032import static eu.cqse.check.framework.scanner.ETokenType.RBRACK; 033import static eu.cqse.check.framework.scanner.ETokenType.RPAREN; 034import static eu.cqse.check.framework.scanner.ETokenType.SEMICOLON; 035import static eu.cqse.check.framework.shallowparser.TokenStreamUtils.NOT_FOUND; 036 037import java.util.ArrayList; 038import java.util.Arrays; 039import java.util.Collection; 040import java.util.EnumSet; 041import java.util.List; 042import java.util.Optional; 043 044import org.apache.logging.log4j.LogManager; 045import org.apache.logging.log4j.Logger; 046import org.conqat.lib.commons.assertion.CCSMAssert; 047import org.conqat.lib.commons.collections.CollectionUtils; 048import org.conqat.lib.commons.collections.Pair; 049import org.conqat.lib.commons.collections.UnmodifiableList; 050import org.conqat.lib.commons.string.StringUtils; 051 052import eu.cqse.check.framework.scanner.ELanguage; 053import eu.cqse.check.framework.scanner.ETokenType; 054import eu.cqse.check.framework.scanner.IToken; 055import eu.cqse.check.framework.shallowparser.SubTypeNames; 056import eu.cqse.check.framework.shallowparser.TokenStreamTextUtils; 057import eu.cqse.check.framework.shallowparser.TokenStreamUtils; 058import eu.cqse.check.framework.shallowparser.framework.EShallowEntityType; 059import eu.cqse.check.framework.shallowparser.framework.ShallowEntity; 060import eu.cqse.check.framework.util.variable.CLikeVariableUseExtractor; 061 062/** 063 * Base class for clike language feature parsers. 064 */ 065public abstract class CLikeLanguageFeatureParserBase implements ILanguageFeatureParser { 066 067 /** Conditional operator types. */ 068 public static final EnumSet<ETokenType> CONDITIONAL_OPERATOR_TYPES = EnumSet.of(ETokenType.EQEQ, ETokenType.LTEQ, 069 ETokenType.GTEQ, ETokenType.LT, ETokenType.GT, ETokenType.NOTEQ); 070 071 /** Token types that are used to declare a pointer or reference (C++). */ 072 public static final EnumSet<ETokenType> POINTER_TYPES = EnumSet.of(MULT, AND); 073 074 /** 075 * All strings that modify a type name. This includes pointer, reference and 076 * array declarations. 077 */ 078 protected static final String[] TYPE_MODIFICATIONS = { "*", "&", "[]" }; 079 080 private static final Logger LOGGER = LogManager.getLogger(); 081 082 /** The language this parser is used for. */ 083 private final ELanguage language; 084 085 /** All token types that can be used as valid identifiers. */ 086 protected final EnumSet<ETokenType> validIdentifiers; 087 088 /** 089 * All token types that can be used to declare a primitive type. Some primitive 090 * types can consist of multiple tokens. 091 */ 092 protected final EnumSet<ETokenType> primitiveTypeTokens; 093 094 /** All token types that can be used to declare a type. */ 095 protected final EnumSet<ETokenType> typeTokens; 096 097 /** 098 * The variable use extractor to find uses of names (e.g. variables or fields). 099 */ 100 protected final CLikeVariableUseExtractor useExtractor; 101 102 /** The token type to separate packages/namespaces/scopes. */ 103 protected final ETokenType packageSeparator; 104 105 /** The text of the package separator token. */ 106 protected final String packageSeparatorText; 107 108 /** The type of variable split of splitVariableTokens method */ 109 protected ETokenType variableSplitType = COMMA; 110 111 /** 112 * Constructor. 113 * 114 * @param language 115 * the concrete language of this parser 116 * @param validIdentifiers 117 * all token types that can be used as identifiers 118 * @param primitiveTypeTokens 119 * all token types that can be part of a primitive type 120 * @param additionalTypeTokens 121 * token types besides primitive ones that can be used to specify a 122 * type 123 * @param packageSeparator 124 * the token that is used to separate packages/scopes/namespaces 125 * @param packageSeparatorText 126 * the string representation of the package separator 127 */ 128 protected CLikeLanguageFeatureParserBase(ELanguage language, EnumSet<ETokenType> validIdentifiers, 129 EnumSet<ETokenType> primitiveTypeTokens, EnumSet<ETokenType> additionalTypeTokens, 130 ETokenType packageSeparator, String packageSeparatorText, CLikeVariableUseExtractor useExtractor) { 131 this.language = language; 132 this.validIdentifiers = validIdentifiers; 133 this.primitiveTypeTokens = primitiveTypeTokens; 134 this.typeTokens = EnumSet.copyOf(additionalTypeTokens); 135 this.typeTokens.addAll(primitiveTypeTokens); 136 this.packageSeparator = packageSeparator; 137 this.packageSeparatorText = packageSeparatorText; 138 this.useExtractor = useExtractor; // new CLikeVariableUseExtractor(DOT, noVariableSuccessorTypes); 139 140 } 141 142 /** Returns whether the given tokens are a variable declaration. */ 143 public boolean isVariableDeclaration(List<IToken> variableTokens) { 144 int equalIndex = TokenStreamUtils.firstTokenOfType(variableTokens, EQ); 145 // If there is an equals sign we assume the variable name to be in front 146 // it. Otherwise the last valid identifier will be used. 147 if (equalIndex == TokenStreamUtils.NOT_FOUND) { 148 return TokenStreamUtils.lastTokenOfType(variableTokens, 0, variableTokens.size(), validIdentifiers) > 0; 149 } 150 151 return (equalIndex - 1) > 0; 152 } 153 154 /** 155 * Returns whether the given type name matches the given full qualified type 156 * name considering the given imported namespaces. If the given type name is 157 * null, false is returned. 158 */ 159 public boolean matchesFullQualifiedTypeName(String typeName, Collection<String> importedNamespaces, 160 String fullQualifiedTypeName) { 161 if (typeName == null) { 162 return false; 163 } else if (typeName.equals(fullQualifiedTypeName)) { 164 return true; 165 } 166 167 for (String namespace : importedNamespaces) { 168 String potentialFullQualifiedTypeName = namespace + packageSeparatorText + typeName; 169 if (potentialFullQualifiedTypeName.equals(fullQualifiedTypeName)) { 170 return true; 171 } 172 } 173 174 return false; 175 } 176 177 /** Extracts all tokens that are part of a condition from a statement. */ 178 public List<IToken> extractConditionTokens(ShallowEntity statement) { 179 String subtype = statement.getSubtype(); 180 if (subtype.equals(SubTypeNames.FOR)) { 181 return TokenStreamUtils.tokensBetween(statement.ownStartTokens(), ETokenType.SEMICOLON, 182 ETokenType.SEMICOLON); 183 } else if (subtype.equals(SubTypeNames.DO)) { 184 List<IToken> tokens = statement.includedTokens(); 185 int whileOffset = TokenStreamUtils.lastTokenOfType(tokens, ETokenType.WHILE); 186 return TokenStreamUtils.tokensBetweenWithNesting(tokens, whileOffset, ETokenType.LPAREN, ETokenType.RPAREN); 187 } else { 188 return TokenStreamUtils.tokensBetweenWithNesting(statement.ownStartTokens(), ETokenType.LPAREN, 189 ETokenType.RPAREN); 190 } 191 } 192 193 /** Returns whether the given entity is static. */ 194 public boolean isStatic(ShallowEntity entity) { 195 return TokenStreamUtils.firstTokenOfType(entity.ownStartTokens(), 196 ETokenType.STATIC) != TokenStreamUtils.NOT_FOUND; 197 } 198 199 /** 200 * Returns whether the given entity only has static members. If there are no 201 * attribute and methods members, <code>false</code> is returned. 202 */ 203 public boolean hasOnlyStaticMembers(ShallowEntity clazz) { 204 CCSMAssert.isTrue(clazz.getType() == EShallowEntityType.TYPE, "clazz.getType() must be \"TYPE\""); 205 CCSMAssert.isTrue(clazz.getSubtype().equals("class"), "clazz.getSubType() must be \"class\""); 206 207 List<ShallowEntity> children = clazz.getChildren(); 208 int childCount = 0; 209 for (ShallowEntity child : children) { 210 if (child.getType() == EShallowEntityType.ATTRIBUTE || child.getType() == EShallowEntityType.METHOD 211 || child.getType() == EShallowEntityType.TYPE) { 212 childCount++; 213 if (!isStatic(child)) { 214 return false; 215 } 216 } 217 } 218 219 return childCount > 0; 220 } 221 222 /** Returns a list of all generic names for the given type. */ 223 public List<String> getGenericNamesForType(ShallowEntity type) { 224 CCSMAssert.isTrue(type.getType() == EShallowEntityType.TYPE, "Expected type entity"); 225 List<IToken> typeTokens = type.ownStartTokens(); 226 // find the type's name 227 int nameIndex = TokenStreamUtils.firstTokenOfType(typeTokens, ETokenType.IDENTIFIER); 228 if (nameIndex == TokenStreamUtils.NOT_FOUND || nameIndex >= typeTokens.size() - 1) { 229 return CollectionUtils.emptyList(); 230 } 231 232 // check if the name is followed by a '<' 233 if (typeTokens.get(nameIndex + 1).getType() != ETokenType.LT) { 234 return CollectionUtils.emptyList(); 235 } 236 237 return extractGenericTokens(typeTokens); 238 } 239 240 /** Returns a list of all generic names for the given method. */ 241 public List<String> getGenericNamesForMethod(ShallowEntity method) { 242 CCSMAssert.isTrue(method.getType() == EShallowEntityType.METHOD, "Expected method entity"); 243 List<IToken> methodTokens = method.ownStartTokens(); 244 245 int nameIndex = TokenStreamUtils.firstTokenOfType(methodTokens, LT); 246 if (nameIndex == TokenStreamUtils.NOT_FOUND || nameIndex >= methodTokens.size() - 1) { 247 return CollectionUtils.emptyList(); 248 } 249 250 return extractGenericTokens(methodTokens); 251 } 252 253 private List<String> extractGenericTokens(List<IToken> tokens) { 254 List<IToken> genericTokens = TokenStreamUtils.tokensBetweenWithNesting(tokens, ETokenType.LT, ETokenType.GT); 255 List<List<IToken>> splitTokens = TokenStreamUtils.split(genericTokens, ETokenType.COMMA); 256 List<List<IToken>> filteredTokens = CollectionUtils.map(splitTokens, this::filterGenericTokens); 257 return TokenStreamTextUtils.concatAllTokenTexts(filteredTokens); 258 } 259 260 /** 261 * Filters the generic tokens before converting them to their token texts. The 262 * default implementation doesn't do any filtering. 263 */ 264 protected List<IToken> filterGenericTokens(List<IToken> genericTokens) { 265 return genericTokens; 266 } 267 268 /** Returns whether the given method has a void return type. */ 269 public boolean hasVoidReturnType(ShallowEntity method) { 270 return (ETokenType.VOID.toString().toLowerCase()).equals(getReturnType(method)); 271 } 272 273 /** Returns the type name of the return type of the given method. */ 274 public String getReturnType(ShallowEntity method) { 275 CCSMAssert.isTrue(method.getType() == EShallowEntityType.METHOD, "method.getType() must be \"METHOD\""); 276 List<IToken> methodTokens = method.ownStartTokens(); 277 int parameterParenthesisIndex = getMethodOpeningParenthesisIndex(methodTokens); 278 if (parameterParenthesisIndex < 0) { 279 return null; 280 } 281 return getModifiersAndTypeFromTokens(methodTokens.subList(0, parameterParenthesisIndex)).getSecond(); 282 } 283 284 /** 285 * Returns the index of the opening parenthesis of a method from the given 286 * tokens. This method may be overwritten by subclasses to handle special 287 * nesting cases like in C++ templates. 288 */ 289 protected int getMethodOpeningParenthesisIndex(List<IToken> methodTokens) { 290 return TokenStreamUtils.findFirstTopLevel(methodTokens, LPAREN, Arrays.asList(LT), Arrays.asList(GT)); 291 292 } 293 294 /** 295 * Returns a list of parameter tokens for the given method. Given an anonymous 296 * method, i.e., lambda, we return an empty list of parameters. 297 */ 298 public List<IToken> getParameterTokens(ShallowEntity method) { 299 CCSMAssert.isTrue(method.getType() == EShallowEntityType.METHOD, "method.getType() must be \"METHOD\""); 300 UnmodifiableList<IToken> methodStartTokens = method.ownStartTokens(); 301 302 int indexOfMethodName = TokenStreamTextUtils.findFirst(methodStartTokens, method.getName()); 303 if (method.getSubtype().equals(SubTypeNames.OPERATOR)) { 304 indexOfMethodName = TokenStreamTextUtils.findFirst(methodStartTokens, 305 StringUtils.stripPrefix(method.getName(), SubTypeNames.OPERATOR)); 306 } 307 308 if (indexOfMethodName != NOT_FOUND) { 309 List<IToken> tokensAfterMethodName = methodStartTokens.subList(indexOfMethodName, methodStartTokens.size()); 310 return extractParameterTokens(tokensAfterMethodName); 311 } 312 return new ArrayList<>(); 313 } 314 315 /** 316 * Returns a list of parameter tokens for the given tokens after the method 317 * name. 318 */ 319 protected List<IToken> extractParameterTokens(List<IToken> tokensAfterMethodName) { 320 return TokenStreamUtils.tokensBetweenWithNesting(tokensAfterMethodName, ETokenType.LPAREN, ETokenType.RPAREN); 321 } 322 323 /** 324 * Returns value of {@link #getModifiersAndTypeFromTokens(List)} if no modifier 325 * and type information could be extracted from the given tokens. 326 */ 327 private static final Pair<List<IToken>, String> NO_MODIFIERS_AND_TYPE_RESULT = new Pair<>( 328 CollectionUtils.emptyList(), null); 329 330 /** 331 * Extracts a list of modifiers and a type name from the given token list. The 332 * given tokens must have the pattern of a variable declaration. If no type can 333 * be found, an empty list and null is returned. The generic part of types is 334 * omitted. 335 */ 336 public Pair<List<IToken>, String> getModifiersAndTypeFromTokens(List<IToken> tokens) { 337 int endIndex = tokens.size(); 338 if (TokenStreamUtils.endsWith(tokens, SEMICOLON)) { 339 endIndex--; 340 } 341 // skip and ignore LBRACK/RBRACK (initialization of array-type variables) 342 while (endIndex > 0 && TokenStreamUtils.hasTokenTypeSequence(tokens, endIndex - 1, RBRACK)) { 343 int lbrackIndex = TokenStreamUtils.findMatchingOpeningToken(tokens, endIndex - 2, LBRACK, RBRACK); 344 if (lbrackIndex == NOT_FOUND) { 345 LOGGER.error(() -> "Could not extract modifiers and type from tokens: " 346 + TokenStreamTextUtils.concatTokenTexts(tokens, " ")); 347 return NO_MODIFIERS_AND_TYPE_RESULT; 348 } 349 endIndex = lbrackIndex; 350 } 351 endIndex = reverseSkipToType(tokens.subList(0, endIndex)); 352 353 if (endIndex < 0) { 354 return NO_MODIFIERS_AND_TYPE_RESULT; 355 } 356 357 TypeNameTokenCollection typeNameTokenCollection = new TypeNameTokenCollection(); 358 List<IToken> modifierTokens = new ArrayList<>(); 359 360 endIndex = collectTokensFromTypeAndModifications(tokens, endIndex, modifierTokens, typeNameTokenCollection); 361 if (endIndex < 0) { 362 return NO_MODIFIERS_AND_TYPE_RESULT; 363 } 364 endIndex = collectTokensFromModifiers(tokens, endIndex, modifierTokens); 365 366 String typeName = typeNameTokenCollection.formatTypeName(); 367 return new Pair<>(modifierTokens, typeName); 368 } 369 370 /** 371 * Collects modifier and type tokens from type modifications like pointer or 372 * reference declarations and from the type type name tokens itself. 373 */ 374 protected int collectTokensFromTypeAndModifications(List<IToken> tokens, int endIndex, List<IToken> modifierTokens, 375 TypeNameTokenCollection typeName) { 376 endIndex = collectTokensFromPointers(tokens, endIndex, modifierTokens, typeName); 377 if (endIndex != -1) { 378 endIndex = collectTokensFromType(tokens, endIndex, typeName); 379 } 380 return endIndex; 381 } 382 383 /** 384 * Collects modifier tokens from pointer or reference declarations (mainly const 385 * in C++). 386 */ 387 protected int collectTokensFromPointers(List<IToken> tokens, int endIndex, List<IToken> modifierTokens, 388 TypeNameTokenCollection typeName) { 389 int offset; 390 391 for (offset = endIndex; offset > 0; offset--) { 392 IToken token = tokens.get(offset - 1); 393 if (token.getType() == CONST) { 394 modifierTokens.add(token); 395 } else if (token.getType() == RBRACK) { 396 if (offset > 1 && tokens.get(offset - 2).getType() == LBRACK) { 397 typeName.addToken(token); 398 typeName.addToken(tokens.get(offset - 2)); 399 offset--; 400 } 401 } else if (POINTER_TYPES.contains(token.getType())) { 402 typeName.addToken(token); 403 } else { 404 return offset; 405 } 406 } 407 return offset; 408 } 409 410 /** 411 * Collects type name tokens from a type name within a variable declaration. 412 * Generic/templated parts of the type name are filtered. 413 */ 414 protected int collectTokensFromType(List<IToken> tokens, int endIndex, TypeNameTokenCollection typeName) { 415 // if there is a primitive type 416 if (endIndex > 0 && isPrimitiveTypeToken(tokens.get(endIndex - 1))) { 417 return collectTokensFromPrimitiveType(tokens, endIndex, typeName); 418 } 419 420 endIndex = collectTokenFromTypePart(tokens, endIndex, typeName); 421 if (endIndex == -1) { 422 return -1; 423 } 424 425 while (endIndex > 1 && tokens.get(endIndex - 1).getType() == packageSeparator) { 426 typeName.addToken(tokens.get(endIndex - 1)); 427 endIndex = collectTokenFromTypePart(tokens, endIndex - 1, typeName); 428 if (endIndex == -1) { 429 return -1; 430 } 431 } 432 return endIndex; 433 } 434 435 /** 436 * Collects type name tokens from a primitive type within a variable 437 * declaration. The assumption is that a primitive type can consists of multiple 438 * {@link #primitiveTypeTokens} like "unsigned long long" in C++. 439 */ 440 protected int collectTokensFromPrimitiveType(List<IToken> tokens, int endIndex, TypeNameTokenCollection typeName) { 441 typeName.markPrimitive(); 442 while (endIndex > 0 && isPrimitiveTypeToken(tokens.get(endIndex - 1))) { 443 typeName.addToken(tokens.get(endIndex - 1)); 444 endIndex--; 445 } 446 return endIndex; 447 } 448 449 /** 450 * Collects a type name token from a part of a type name within a variable 451 * declaration. A part is the name of a namespace/package or the type's name 452 * itself. E.g. the complete typename std::vector<int> consists of the 453 * parts std and vector. Generic/templated parts are skipped. 454 */ 455 protected int collectTokenFromTypePart(List<IToken> tokens, int endIndex, TypeNameTokenCollection typeName) { 456 if (endIndex > 0 && tokens.get(endIndex - 1).getType() == GT) { 457 endIndex = skipGenericReverse(tokens, endIndex - 1); 458 } 459 if (endIndex <= 0 || !isTypeToken(tokens.get(endIndex - 1))) { 460 return -1; 461 } 462 typeName.addToken(tokens.get(endIndex - 1)); 463 return endIndex - 1; 464 } 465 466 /** 467 * Collects modifier tokens from modifier specifications in front of the type 468 * name within a variable declaration. 469 */ 470 protected int collectTokensFromModifiers(List<IToken> tokens, int endIndex, List<IToken> modifierTokens) { 471 modifierTokens.addAll(0, tokens.subList(0, endIndex)); 472 return 0; 473 } 474 475 /** Returns whether the given token is a type token. */ 476 public boolean isTypeToken(IToken token) { 477 return typeTokens.contains(token.getType()); 478 } 479 480 /** Returns whether the given token is a primitive type token. */ 481 public boolean isPrimitiveTypeToken(IToken token) { 482 return primitiveTypeTokens.contains(token.getType()); 483 } 484 485 /** 486 * Skips from the end of the given token list (that represents a variable 487 * definition) to the token after the last token that specifies the variables 488 * type. 489 */ 490 protected int reverseSkipToType(List<IToken> tokens) { 491 int endIndex = TokenStreamUtils.firstTokenOfType(tokens, EQ); 492 if (endIndex == TokenStreamUtils.NOT_FOUND) { 493 // if there is no assignment, we search for the variable name 494 endIndex = TokenStreamUtils.lastTokenOfType(tokens, validIdentifiers); 495 } else { 496 // otherwise we set the end index to the variable name which is in 497 // front of the equals sign 498 endIndex -= 1; 499 } 500 return endIndex; 501 } 502 503 /** 504 * Skips a generic or template declaration in the given token list that begins 505 * at the given start index. The token at the given index must be of type 506 * {@link ETokenType#LT}. The index of the matching closing token is returned. 507 */ 508 public int skipGeneric(List<IToken> tokens, int startIndex) { 509 return TokenStreamUtils.findMatchingClosingToken(tokens, startIndex + 1, LT, GT); 510 } 511 512 /** 513 * Skips a generic parameter (nested < and >) in reverse direction and 514 * returns the index of the first opening < token. Tje token at the given 515 * index must be of type {@link ETokenType#GT}. The index of the matching 516 * opening token is returned. 517 */ 518 public int skipGenericReverse(List<IToken> tokens, int startIndex) { 519 return TokenStreamUtils.findMatchingOpeningToken(tokens, startIndex - 1, LT, GT); 520 } 521 522 /** 523 * Returns a list of lists of tokens that represent the parameters of the given 524 * method. 525 */ 526 public List<List<IToken>> getSplitParameterTokens(ShallowEntity method) { 527 List<IToken> parameterTokens = getParameterTokens(method); 528 if (parameterTokens.isEmpty()) { 529 return CollectionUtils.emptyList(); 530 } 531 return splitVariableTokens(parameterTokens); 532 } 533 534 /** 535 * Returns all parameter type names of the given method. If a parameter has no 536 * name, the entry in the list is empty. 537 */ 538 public List<Optional<String>> getParameterTypeNames(ShallowEntity method) { 539 return CollectionUtils.map(getSplitParameterTokens(method), 540 tokens -> Optional.ofNullable(getModifiersAndTypeFromTokens(tokens).getSecond())); 541 } 542 543 /** 544 * Return the start index of the type name that ends at the given index within 545 * the given token list. This method is used to find type names that consist of 546 * multiple tokens, e. g. names that are prefixed with a namespace. 547 */ 548 protected int getTypeNameStartIndex(List<IToken> tokens, int lastTypeTokenIndex) { 549 int i = lastTypeTokenIndex; 550 while (i > 1) { 551 ETokenType first = tokens.get(i - 1).getType(); 552 ETokenType second = tokens.get(i - 2).getType(); 553 554 if (first != DOT || !validIdentifiers.contains(second)) { 555 break; 556 } 557 558 i -= 2; 559 } 560 return i; 561 } 562 563 /** Extracts type names that are used in generics from the given tokens. */ 564 public List<String> getGenericTypeNames(List<IToken> tokens) { 565 List<String> typeNames = new ArrayList<>(); 566 List<IToken> tokenDumpList = new ArrayList<>(); 567 int startIndex = 0; 568 while (true) { 569 int openingIndex = TokenStreamUtils.firstTokenOfType(tokens, startIndex, LT); 570 if (openingIndex < 0) { 571 break; 572 } 573 int endIndex = skipGeneric(tokens, openingIndex); 574 if (endIndex <= 0) { 575 break; 576 } 577 578 List<IToken> subList = tokens.subList(openingIndex + 1, endIndex); 579 TypeNameTokenCollection typeNameTokenCollection = new TypeNameTokenCollection(); 580 int offset = collectTokensFromTypeAndModifications(subList, subList.size(), tokenDumpList, 581 typeNameTokenCollection); 582 if (offset >= 0) { 583 typeNames.add(typeNameTokenCollection.formatTypeName()); 584 } 585 586 startIndex = endIndex + 1; 587 588 } 589 return typeNames; 590 591 } 592 593 /** 594 * Splits the given variable declaration tokens at commas regarding nesting 595 * between parenthesis, brackets, braces and angle brackets. 596 */ 597 public List<List<IToken>> splitVariableTokens(List<IToken> variableTokens) { 598 List<List<IToken>> splitTokens = TokenStreamUtils.splitWithNesting(variableTokens, variableSplitType, 599 Arrays.asList(LPAREN, LBRACK, LBRACE, LT), Arrays.asList(RPAREN, RBRACK, RBRACE, GT)); 600 return splitTokens; 601 } 602 603 /** Returns all variable name tokens from the given variable tokens. */ 604 public List<IToken> getVariableNamesFromTokens(List<IToken> variableTokens) { 605 List<List<IToken>> splitTokens = splitVariableTokens(variableTokens); 606 607 List<IToken> variableNames = new ArrayList<>(); 608 for (List<IToken> tokens : splitTokens) { 609 IToken name = getVariableNameFromTokens(tokens); 610 if (name != null) { 611 variableNames.add(name); 612 } 613 } 614 return variableNames; 615 } 616 617 /** 618 * Returns the variable name token from the given token list or 619 * <code>null</code> if none is found. The list of tokens must have the pattern 620 * of a variable declaration. 621 */ 622 public IToken getVariableNameFromTokens(List<IToken> tokens) { 623 int equalIndex = TokenStreamUtils.firstTokenOfType(tokens, EQ); 624 // If there is an equals sign we assume the variable name to be in front 625 // it. Otherwise the last valid identifier will be used. 626 int nameIndex = equalIndex - 1; 627 if (equalIndex == TokenStreamUtils.NOT_FOUND) { 628 nameIndex = TokenStreamUtils.lastTokenOfType(tokens, 0, tokens.size(), validIdentifiers); 629 } 630 631 if (nameIndex < 0) { 632 return null; 633 } 634 return tokens.get(nameIndex); 635 } 636 637 /** 638 * Returns the type name of a variable from the declaring tokens. If none is 639 * found, null is returned. 640 */ 641 public String getVariableTypeFromTokens(List<IToken> tokens) { 642 int reverseSkipToType = reverseSkipToType(tokens); 643 if (reverseSkipToType == NOT_FOUND) { 644 return null; 645 } 646 return TokenStreamTextUtils.concatTokenTexts(tokens.subList(0, reverseSkipToType), " "); 647 } 648 649 /** 650 * Returns the variable name of an exception within a catch statement or null if 651 * the exception is not named. 652 */ 653 public IToken getVariableNameFromCatchTokens(List<IToken> catchTokens) { 654 List<IToken> exceptionTokens = TokenStreamUtils.tokensBetween(catchTokens, ETokenType.LPAREN, 655 ETokenType.RPAREN); 656 657 int doubleIdentifierIndex = TokenStreamUtils.firstTokenOfTypeSequence(exceptionTokens, 0, ETokenType.IDENTIFIER, 658 ETokenType.IDENTIFIER); 659 if (doubleIdentifierIndex == TokenStreamUtils.NOT_FOUND) { 660 return null; 661 } 662 return exceptionTokens.get(doubleIdentifierIndex + 1); 663 } 664 665 /** 666 * Returns the type name of an exception within a catch statement or null if 667 * none is found. 668 */ 669 public String getTypeNameFromCatchTokens(List<IToken> catchTokens) { 670 List<IToken> exceptionTokens = TokenStreamUtils.tokensBetween(catchTokens, ETokenType.LPAREN, 671 ETokenType.RPAREN); 672 if (exceptionTokens.isEmpty()) { 673 return null; 674 } 675 676 if (exceptionTokens.size() == 1) { 677 return exceptionTokens.get(0).getText(); 678 } 679 680 int typeEndIndex = TokenStreamUtils.firstTokenOfTypeSequence(exceptionTokens, 0, ETokenType.IDENTIFIER, 681 ETokenType.IDENTIFIER); 682 if (typeEndIndex == TokenStreamUtils.NOT_FOUND) { 683 typeEndIndex = exceptionTokens.size() - 1; 684 } 685 686 int typeStartIndex = getTypeNameStartIndex(exceptionTokens, typeEndIndex); 687 return TokenStreamTextUtils.concatTokenTexts(exceptionTokens.subList(typeStartIndex, typeEndIndex + 1)); 688 } 689 690 /** 691 * Returns all variable declaration tokens within a for-like statement. For-like 692 * statements are for loops and similar constructs like using and catch 693 * statements in Cs. The given end token type specifies the token where the 694 * variable declaration tokens end. 695 */ 696 public List<IToken> getVariableTokensFromForLikeTokens(List<IToken> forLikeTokens, ETokenType endToken) { 697 int leftParenIndex = TokenStreamUtils.firstTokenOfType(forLikeTokens, LPAREN); 698 if (leftParenIndex == TokenStreamUtils.NOT_FOUND) { 699 return CollectionUtils.emptyList(); 700 } 701 702 int endIndex; 703 if (endToken == RPAREN) { 704 endIndex = TokenStreamUtils.findMatchingClosingToken(forLikeTokens, leftParenIndex + 1, LPAREN, RPAREN); 705 } else { 706 endIndex = TokenStreamUtils.firstTokenOfType(forLikeTokens, endToken); 707 } 708 709 if (endIndex == TokenStreamUtils.NOT_FOUND) { 710 return CollectionUtils.emptyList(); 711 } 712 713 return forLikeTokens.subList(leftParenIndex + 1, endIndex); 714 } 715 716 /** 717 * Returns all token indices from the given tokens that are uses of the variable 718 * with the given name. The heuristic is to look for all occurrences of the 719 * variable name in the given tokens. An occurrence is considered as variable 720 * usage, if the following token is not in {@link #validIdentifiers}. Those 721 * token types are set from the subclass for a concrete language. Internally a 722 * {@link CLikeVariableUseExtractor} is used. 723 * 724 * @param isField 725 * if the variable is a field 726 * @param isShadowed 727 * if the variable is a field and shadowed by another local variable 728 */ 729 public List<Integer> getVariableUsesFromTokens(List<IToken> tokens, String variableName, boolean isField, 730 boolean isShadowed) { 731 return useExtractor.extractVariableUses(tokens, variableName, isField, isShadowed); 732 } 733 734 /** 735 * Returns all token indices from the given tokens that read the value of the 736 * variable with the given name. 737 * 738 * @param isField 739 * if the variable is a field 740 * @param isShadowed 741 * if the variable is a field and shadowed by another local variable 742 */ 743 public List<Integer> getVariableReadsFromTokens(List<IToken> tokens, String variableName, boolean isField, 744 boolean isShadowed) { 745 return useExtractor.extractVariableReads(tokens, variableName, isField, isShadowed); 746 } 747 748 /** 749 * Returns all token indices from the given tokens that change the value of the 750 * variable with the given name. 751 * 752 * @param isField 753 * if the variable is a field 754 * @param isShadowed 755 * if the variable is a field and shadowed by another local variable 756 */ 757 public List<Integer> getVariableWritesFromTokens(List<IToken> tokens, String variableName, boolean isField, 758 boolean isShadowed) { 759 return useExtractor.extractVariableWrites(tokens, variableName, isField, isShadowed); 760 } 761 762 /** Returns whether the given entity is an import. */ 763 public abstract boolean isImport(ShallowEntity entity); 764 765 /** Returns the imported name of the given import shallow entity. */ 766 public abstract String getImportName(ShallowEntity entity); 767 768 /** Returns whether the given tokens represent a variable length argument. */ 769 public boolean hasVariableLengthArgumentList(List<IToken> tokens) { 770 return TokenStreamUtils.containsAll(tokens, ELLIPSIS); 771 } 772 773 /** 774 * Returns a list of names that are imported by the given "import" entities. If 775 * the import is an aliasing import (in the case of C#) it is ignored. 776 */ 777 public List<String> getImportedNames(List<ShallowEntity> importEntities) { 778 List<String> importedNames = new ArrayList<>(); 779 for (ShallowEntity usingEntity : importEntities) { 780 String importName = getImportName(usingEntity); 781 if (importName != null) { 782 importedNames.add(importName); 783 } 784 } 785 return importedNames; 786 } 787 788 /** 789 * Removes all pointer, array and reference declarations from the given 790 * typename. 791 */ 792 public String getPlainTypeName(String typeName) { 793 return StringUtils.removeAll(typeName, TYPE_MODIFICATIONS); 794 } 795 796 /** Returns the base name of the given type name. */ 797 public String getBaseTypeName(String typeName) { 798 int index = typeName.lastIndexOf(packageSeparatorText); 799 if (index == -1) { 800 return typeName; 801 } 802 return typeName.substring(index + packageSeparatorText.length()); 803 } 804 805 /** {@inheritDoc} */ 806 @Override 807 public ELanguage getLanguage() { 808 return language; 809 } 810 811}