001/*-------------------------------------------------------------------------+ 002| | 003| Copyright 2005-2011 the ConQAT Project | 004| | 005| Licensed under the Apache License, Version 2.0 (the "License"); | 006| you may not use this file except in compliance with the License. | 007| You may obtain a copy of the License at | 008| | 009| http://www.apache.org/licenses/LICENSE-2.0 | 010| | 011| Unless required by applicable law or agreed to in writing, software | 012| distributed under the License is distributed on an "AS IS" BASIS, | 013| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | 014| See the License for the specific language governing permissions and | 015| limitations under the License. | 016+-------------------------------------------------------------------------*/ 017package eu.cqse.check.framework.util.tokens; 018 019import static eu.cqse.check.framework.scanner.ETokenType.IDENTIFIER; 020 021import java.util.ArrayList; 022import java.util.Arrays; 023import java.util.EnumSet; 024import java.util.HashMap; 025import java.util.List; 026import java.util.Map; 027import java.util.regex.Matcher; 028import java.util.regex.Pattern; 029 030import org.conqat.lib.commons.assertion.CCSMAssert; 031 032import eu.cqse.check.framework.core.CheckException; 033import eu.cqse.check.framework.scanner.ELanguage; 034import eu.cqse.check.framework.scanner.ETokenType; 035import eu.cqse.check.framework.scanner.IToken; 036import eu.cqse.check.framework.scanner.ScannerUtils; 037import eu.cqse.check.framework.shallowparser.TokenStreamUtils; 038 039/** 040 * Transforms a token stream according to a search and a replace pattern. Both 041 * patterns can contain variables in the form of <code>$name</code>. 042 * 043 * <h1>Example</h1> 044 * <ul> 045 * <li>Language: Java 046 * <li>Search Pattern: <code>assertNotNull($a)</code> 047 * <li>Replacement Pattern: <code>$a != null</code> 048 * <li>Input: <code>assertNotNull(foo)</code> 049 * <li>Output: <code>foo != null</code> 050 * </ul> 051 * 052 * Additionally, variables can be postfixed with a number, e.g. <code>$a1</code> 053 * to signal that the code matched to <code>$a1</code> should only be exactly 1 054 * token in length. 055 */ 056public class TokenStreamTransformationPattern { 057 058 /** 059 * Prefix of pattern variables. 060 */ 061 private static final String VARIABLE_PREFIX = "$"; 062 063 /** The search pattern transformed into matchers. */ 064 private final List<IMatcher> matchers = new ArrayList<>(); 065 066 /** The replacement pattern as tokens of the input language. */ 067 private final List<IToken> replacementPatternTokens; 068 069 private final ELanguage language; 070 071 /** Returned if the matcher did not match. */ 072 public static final int NO_MATCH = -1; 073 074 /** 075 * Constructor. 076 * 077 * @throws CheckException 078 * if the search pattern has invalid syntax. 079 */ 080 public TokenStreamTransformationPattern(String searchPatternString, String replacementPatternString, 081 ELanguage language) throws CheckException { 082 List<IToken> searchPatternTokens = ScannerUtils.getTokens(searchPatternString, language); 083 createMatchers(searchPatternTokens); 084 this.replacementPatternTokens = ScannerUtils.getTokens(replacementPatternString, language); 085 this.language = language; 086 } 087 088 /** 089 * Creates matchers from the given search pattern tokens. 090 * 091 * @throws CheckException 092 * if the pattern has invalid syntax. 093 */ 094 private void createMatchers(List<IToken> searchPatternTokens) throws CheckException { 095 for (int i = 0; i < searchPatternTokens.size(); i++) { 096 IToken token = searchPatternTokens.get(i); 097 String text = token.getText(); 098 ETokenType type = token.getType(); 099 if (type == IDENTIFIER && text.startsWith(VARIABLE_PREFIX)) { 100 if (i + 1 < searchPatternTokens.size()) { 101 ETokenType endTokenType = searchPatternTokens.get(i + 1).getType(); 102 matchers.add(new VariableMatcher(text, endTokenType)); 103 } else { 104 throw new CheckException("The last token in the search pattern may not be a variable!"); 105 } 106 } else { 107 matchers.add(new TokenTypeMatcher(type, text)); 108 } 109 } 110 } 111 112 /** 113 * Checks if the given tokens after the given offset contain more variables that 114 * should be replaced. 115 */ 116 public int containsMoreVariables(int offset, List<IToken> tokens) { 117 for (int i = offset; i < tokens.size(); i++) { 118 IToken token = tokens.get(i); 119 if (token.getType() == ETokenType.IDENTIFIER && token.getText().startsWith(VARIABLE_PREFIX)) { 120 return i; 121 } 122 } 123 return NO_MATCH; 124 } 125 126 /** 127 * Applies the pattern to the given tokens and returns the transformed token 128 * list. In case the pattern does not match, <code>null</code> is returned. 129 */ 130 private Result apply(List<IToken> tokens, int position) { 131 Map<String, List<IToken>> variables = new HashMap<>(); 132 int matchedTokens = matchSearchPattern(tokens, position, variables); 133 if (matchedTokens == NO_MATCH) { 134 return null; 135 } 136 List<IToken> result = createResult(variables); 137 return new Result(result, matchedTokens); 138 } 139 140 /** 141 * Applies the given patterns on the token stream and replaces all matches with 142 * the transformed tokens. 143 */ 144 public static List<IToken> applyPatterns(List<IToken> tokens, List<TokenStreamTransformationPattern> patterns) { 145 List<IToken> transformedTokens = new ArrayList<>(); 146 int position = 0; 147 while (position < tokens.size()) { 148 Result result = applyPatterns(tokens, patterns, position); 149 if (result == null) { 150 transformedTokens.add(tokens.get(position)); 151 position += 1; 152 } else { 153 transformedTokens.addAll(result.getTransformedTokens()); 154 position += result.getMatchedTokens(); 155 } 156 } 157 return transformedTokens; 158 } 159 160 /** 161 * Applies the given patterns at the given position. The result of the first 162 * match is returned. If no pattern matches, <code>null</code> is returned. 163 */ 164 private static Result applyPatterns(List<IToken> tokens, List<TokenStreamTransformationPattern> patterns, 165 int position) { 166 for (TokenStreamTransformationPattern pattern : patterns) { 167 Result result = pattern.apply(tokens, position); 168 if (result != null) { 169 return result; 170 } 171 } 172 return null; 173 } 174 175 /** 176 * Matches the matchers against the given tokens and returns the variable map 177 * created by the matchers. Returns <code>{@link #NO_MATCH}</code> if the 178 * matchers do not match the token stream. 179 */ 180 private int matchSearchPattern(List<IToken> tokens, int startPosition, Map<String, List<IToken>> variables) { 181 int tokenPosition = startPosition; 182 for (IMatcher matcher : matchers) { 183 if (tokenPosition >= tokens.size()) { 184 return NO_MATCH; 185 } 186 int nextPosition = matcher.apply(tokens, tokenPosition, variables); 187 if (nextPosition == NO_MATCH) { 188 return NO_MATCH; 189 } 190 CCSMAssert.isTrue(nextPosition > tokenPosition, "Matcher did not advance token stream."); 191 tokenPosition = nextPosition; 192 } 193 return tokenPosition - startPosition; 194 } 195 196 /** 197 * Creates the result token list based on the given variable map. 198 */ 199 private List<IToken> createResult(Map<String, List<IToken>> variables) { 200 List<IToken> result = new ArrayList<>(); 201 for (int i = 0; i < replacementPatternTokens.size(); i++) { 202 IToken token = replacementPatternTokens.get(i); 203 String text = token.getText(); 204 if (token.getType() == IDENTIFIER && text.startsWith(VARIABLE_PREFIX)) { 205 List<IToken> variableMatch = variables.get(text); 206 CCSMAssert.isNotNull(variableMatch, "Variable " + text + " was not matched"); 207 result.addAll(variableMatch); 208 } else { 209 result.add(token); 210 } 211 } 212 return result; 213 } 214 215 public ELanguage getLanguage() { 216 return language; 217 } 218 219 /** Result of successfully matching one pattern. */ 220 private static class Result { 221 222 /** The transformed tokens. */ 223 private final List<IToken> transformedTokens; 224 225 /** The number of tokens that matched in the input token stream. */ 226 private final int matchedTokens; 227 228 /** Constructor. */ 229 public Result(List<IToken> transformedTokens, int matchedTokens) { 230 this.transformedTokens = transformedTokens; 231 this.matchedTokens = matchedTokens; 232 } 233 234 /** 235 * @see #matchedTokens 236 */ 237 public int getMatchedTokens() { 238 return matchedTokens; 239 } 240 241 /** 242 * @see #transformedTokens 243 */ 244 public List<IToken> getTransformedTokens() { 245 return transformedTokens; 246 } 247 248 } 249 250 /** Matches part of the search pattern against the token stream. */ 251 private static interface IMatcher { 252 253 /** 254 * Tries to match this matcher at the given position in the token stream. May 255 * modify the given variables map. Returns 256 * {@link TokenStreamTransformationPattern#NO_MATCH} if the matcher does not 257 * apply at this position. Otherwise returns the position where the next matcher 258 * should be applied. 259 */ 260 public int apply(List<IToken> tokens, int position, Map<String, List<IToken>> variables); 261 262 } 263 264 /** 265 * Matches if the token at the current position has a certain type and text. 266 */ 267 private static class TokenTypeMatcher implements IMatcher { 268 269 /** The token type to match. */ 270 private final ETokenType type; 271 272 /** The expected text. */ 273 private final String text; 274 275 /** Constructor. */ 276 public TokenTypeMatcher(ETokenType type, String text) { 277 this.type = type; 278 this.text = text; 279 } 280 281 /** {@inheritDoc} */ 282 @Override 283 public int apply(List<IToken> tokens, int position, Map<String, List<IToken>> variables) { 284 IToken token = tokens.get(position); 285 if (token.getType() == type && token.getText().equals(text)) { 286 return position + 1; 287 } 288 return NO_MATCH; 289 } 290 291 /** {@inheritDoc} */ 292 @Override 293 public String toString() { 294 return "TokenTypeMatcher[type=" + type + ",text=" + text + "]"; 295 } 296 297 } 298 299 /** 300 * Matches a variable from the current position in the token stream to the first 301 * occurrence of the end token. 302 */ 303 private static class VariableMatcher implements IMatcher { 304 305 /** The pattern to find out how many tokens long a match should be */ 306 private static final Pattern MATCH_LENGTH_PATTERN = Pattern.compile("\\$[a-zA-Z]+([0-9]+)"); 307 308 /** The variable to match. */ 309 private final String variableName; 310 311 /** 312 * The token type that signals the end of the variable match. 313 */ 314 private final ETokenType endTokenType; 315 316 /** 317 * The number of tokens that the variable should match. This may be null to 318 * express that the current variable does not contain a variable count. This 319 * means, it will greedily match as many tokens as possible. 320 */ 321 private Integer numberOfTokensToMatch = null; 322 323 /** Constructor. */ 324 public VariableMatcher(String variableName, ETokenType endTokenType) { 325 this.variableName = variableName; 326 this.endTokenType = endTokenType; 327 328 Matcher matcher = MATCH_LENGTH_PATTERN.matcher(variableName); 329 if (matcher.matches()) { 330 this.numberOfTokensToMatch = Integer.parseInt(matcher.group(1)); 331 } 332 } 333 334 /** {@inheritDoc} */ 335 @Override 336 public int apply(List<IToken> tokens, int position, Map<String, List<IToken>> variables) { 337 338 int endIndex = TokenStreamUtils.findFirstTopLevel(tokens, position, EnumSet.of(endTokenType), 339 Arrays.asList(ETokenType.LPAREN), Arrays.asList(ETokenType.RPAREN)); 340 if (endIndex == TokenStreamUtils.NOT_FOUND || position == endIndex) { 341 return NO_MATCH; 342 } 343 344 if (numberOfTokensToMatch != null && endIndex > position + numberOfTokensToMatch) { 345 endIndex = position + numberOfTokensToMatch; 346 } 347 348 variables.put(variableName, tokens.subList(position, endIndex)); 349 return endIndex; 350 } 351 352 /** {@inheritDoc} */ 353 @Override 354 public String toString() { 355 return "VariableMatcher[variableName=" + variableName + ",endTokenType=" + endTokenType + "]"; 356 } 357 358 } 359 360}