001/*-------------------------------------------------------------------------+
002|                                                                          |
003| Copyright 2005-2011 the ConQAT Project                                   |
004|                                                                          |
005| Licensed under the Apache License, Version 2.0 (the "License");          |
006| you may not use this file except in compliance with the License.         |
007| You may obtain a copy of the License at                                  |
008|                                                                          |
009|    http://www.apache.org/licenses/LICENSE-2.0                            |
010|                                                                          |
011| Unless required by applicable law or agreed to in writing, software      |
012| distributed under the License is distributed on an "AS IS" BASIS,        |
013| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
014| See the License for the specific language governing permissions and      |
015| limitations under the License.                                           |
016+-------------------------------------------------------------------------*/
017package eu.cqse.check.framework.util.tokens;
018
019import static eu.cqse.check.framework.scanner.ETokenType.IDENTIFIER;
020
021import java.util.ArrayList;
022import java.util.Arrays;
023import java.util.EnumSet;
024import java.util.HashMap;
025import java.util.List;
026import java.util.Map;
027import java.util.regex.Matcher;
028import java.util.regex.Pattern;
029
030import org.conqat.lib.commons.assertion.CCSMAssert;
031
032import eu.cqse.check.framework.core.CheckException;
033import eu.cqse.check.framework.scanner.ELanguage;
034import eu.cqse.check.framework.scanner.ETokenType;
035import eu.cqse.check.framework.scanner.IToken;
036import eu.cqse.check.framework.scanner.ScannerUtils;
037import eu.cqse.check.framework.shallowparser.TokenStreamUtils;
038
039/**
040 * Transforms a token stream according to a search and a replace pattern. Both
041 * patterns can contain variables in the form of <code>$name</code>.
042 *
043 * <h1>Example</h1>
044 * <ul>
045 * <li>Language: Java
046 * <li>Search Pattern: <code>assertNotNull($a)</code>
047 * <li>Replacement Pattern: <code>$a != null</code>
048 * <li>Input: <code>assertNotNull(foo)</code>
049 * <li>Output: <code>foo != null</code>
050 * </ul>
051 * 
052 * Additionally, variables can be postfixed with a number, e.g. <code>$a1</code>
053 * to signal that the code matched to <code>$a1</code> should only be exactly 1
054 * token in length.
055 */
056public class TokenStreamTransformationPattern {
057
058        /**
059         * Prefix of pattern variables.
060         */
061        private static final String VARIABLE_PREFIX = "$";
062
063        /** The search pattern transformed into matchers. */
064        private final List<IMatcher> matchers = new ArrayList<>();
065
066        /** The replacement pattern as tokens of the input language. */
067        private final List<IToken> replacementPatternTokens;
068
069        private final ELanguage language;
070
071        /** Returned if the matcher did not match. */
072        public static final int NO_MATCH = -1;
073
074        /**
075         * Constructor.
076         * 
077         * @throws CheckException
078         *             if the search pattern has invalid syntax.
079         */
080        public TokenStreamTransformationPattern(String searchPatternString, String replacementPatternString,
081                        ELanguage language) throws CheckException {
082                List<IToken> searchPatternTokens = ScannerUtils.getTokens(searchPatternString, language);
083                createMatchers(searchPatternTokens);
084                this.replacementPatternTokens = ScannerUtils.getTokens(replacementPatternString, language);
085                this.language = language;
086        }
087
088        /**
089         * Creates matchers from the given search pattern tokens.
090         * 
091         * @throws CheckException
092         *             if the pattern has invalid syntax.
093         */
094        private void createMatchers(List<IToken> searchPatternTokens) throws CheckException {
095                for (int i = 0; i < searchPatternTokens.size(); i++) {
096                        IToken token = searchPatternTokens.get(i);
097                        String text = token.getText();
098                        ETokenType type = token.getType();
099                        if (type == IDENTIFIER && text.startsWith(VARIABLE_PREFIX)) {
100                                if (i + 1 < searchPatternTokens.size()) {
101                                        ETokenType endTokenType = searchPatternTokens.get(i + 1).getType();
102                                        matchers.add(new VariableMatcher(text, endTokenType));
103                                } else {
104                                        throw new CheckException("The last token in the search pattern may not be a variable!");
105                                }
106                        } else {
107                                matchers.add(new TokenTypeMatcher(type, text));
108                        }
109                }
110        }
111
112        /**
113         * Checks if the given tokens after the given offset contain more variables that
114         * should be replaced.
115         */
116        public int containsMoreVariables(int offset, List<IToken> tokens) {
117                for (int i = offset; i < tokens.size(); i++) {
118                        IToken token = tokens.get(i);
119                        if (token.getType() == ETokenType.IDENTIFIER && token.getText().startsWith(VARIABLE_PREFIX)) {
120                                return i;
121                        }
122                }
123                return NO_MATCH;
124        }
125
126        /**
127         * Applies the pattern to the given tokens and returns the transformed token
128         * list. In case the pattern does not match, <code>null</code> is returned.
129         */
130        private Result apply(List<IToken> tokens, int position) {
131                Map<String, List<IToken>> variables = new HashMap<>();
132                int matchedTokens = matchSearchPattern(tokens, position, variables);
133                if (matchedTokens == NO_MATCH) {
134                        return null;
135                }
136                List<IToken> result = createResult(variables);
137                return new Result(result, matchedTokens);
138        }
139
140        /**
141         * Applies the given patterns on the token stream and replaces all matches with
142         * the transformed tokens.
143         */
144        public static List<IToken> applyPatterns(List<IToken> tokens, List<TokenStreamTransformationPattern> patterns) {
145                List<IToken> transformedTokens = new ArrayList<>();
146                int position = 0;
147                while (position < tokens.size()) {
148                        Result result = applyPatterns(tokens, patterns, position);
149                        if (result == null) {
150                                transformedTokens.add(tokens.get(position));
151                                position += 1;
152                        } else {
153                                transformedTokens.addAll(result.getTransformedTokens());
154                                position += result.getMatchedTokens();
155                        }
156                }
157                return transformedTokens;
158        }
159
160        /**
161         * Applies the given patterns at the given position. The result of the first
162         * match is returned. If no pattern matches, <code>null</code> is returned.
163         */
164        private static Result applyPatterns(List<IToken> tokens, List<TokenStreamTransformationPattern> patterns,
165                        int position) {
166                for (TokenStreamTransformationPattern pattern : patterns) {
167                        Result result = pattern.apply(tokens, position);
168                        if (result != null) {
169                                return result;
170                        }
171                }
172                return null;
173        }
174
175        /**
176         * Matches the matchers against the given tokens and returns the variable map
177         * created by the matchers. Returns <code>{@link #NO_MATCH}</code> if the
178         * matchers do not match the token stream.
179         */
180        private int matchSearchPattern(List<IToken> tokens, int startPosition, Map<String, List<IToken>> variables) {
181                int tokenPosition = startPosition;
182                for (IMatcher matcher : matchers) {
183                        if (tokenPosition >= tokens.size()) {
184                                return NO_MATCH;
185                        }
186                        int nextPosition = matcher.apply(tokens, tokenPosition, variables);
187                        if (nextPosition == NO_MATCH) {
188                                return NO_MATCH;
189                        }
190                        CCSMAssert.isTrue(nextPosition > tokenPosition, "Matcher did not advance token stream.");
191                        tokenPosition = nextPosition;
192                }
193                return tokenPosition - startPosition;
194        }
195
196        /**
197         * Creates the result token list based on the given variable map.
198         */
199        private List<IToken> createResult(Map<String, List<IToken>> variables) {
200                List<IToken> result = new ArrayList<>();
201                for (int i = 0; i < replacementPatternTokens.size(); i++) {
202                        IToken token = replacementPatternTokens.get(i);
203                        String text = token.getText();
204                        if (token.getType() == IDENTIFIER && text.startsWith(VARIABLE_PREFIX)) {
205                                List<IToken> variableMatch = variables.get(text);
206                                CCSMAssert.isNotNull(variableMatch, "Variable " + text + " was not matched");
207                                result.addAll(variableMatch);
208                        } else {
209                                result.add(token);
210                        }
211                }
212                return result;
213        }
214
215        public ELanguage getLanguage() {
216                return language;
217        }
218
219        /** Result of successfully matching one pattern. */
220        private static class Result {
221
222                /** The transformed tokens. */
223                private final List<IToken> transformedTokens;
224
225                /** The number of tokens that matched in the input token stream. */
226                private final int matchedTokens;
227
228                /** Constructor. */
229                public Result(List<IToken> transformedTokens, int matchedTokens) {
230                        this.transformedTokens = transformedTokens;
231                        this.matchedTokens = matchedTokens;
232                }
233
234                /**
235                 * @see #matchedTokens
236                 */
237                public int getMatchedTokens() {
238                        return matchedTokens;
239                }
240
241                /**
242                 * @see #transformedTokens
243                 */
244                public List<IToken> getTransformedTokens() {
245                        return transformedTokens;
246                }
247
248        }
249
250        /** Matches part of the search pattern against the token stream. */
251        private static interface IMatcher {
252
253                /**
254                 * Tries to match this matcher at the given position in the token stream. May
255                 * modify the given variables map. Returns
256                 * {@link TokenStreamTransformationPattern#NO_MATCH} if the matcher does not
257                 * apply at this position. Otherwise returns the position where the next matcher
258                 * should be applied.
259                 */
260                public int apply(List<IToken> tokens, int position, Map<String, List<IToken>> variables);
261
262        }
263
264        /**
265         * Matches if the token at the current position has a certain type and text.
266         */
267        private static class TokenTypeMatcher implements IMatcher {
268
269                /** The token type to match. */
270                private final ETokenType type;
271
272                /** The expected text. */
273                private final String text;
274
275                /** Constructor. */
276                public TokenTypeMatcher(ETokenType type, String text) {
277                        this.type = type;
278                        this.text = text;
279                }
280
281                /** {@inheritDoc} */
282                @Override
283                public int apply(List<IToken> tokens, int position, Map<String, List<IToken>> variables) {
284                        IToken token = tokens.get(position);
285                        if (token.getType() == type && token.getText().equals(text)) {
286                                return position + 1;
287                        }
288                        return NO_MATCH;
289                }
290
291                /** {@inheritDoc} */
292                @Override
293                public String toString() {
294                        return "TokenTypeMatcher[type=" + type + ",text=" + text + "]";
295                }
296
297        }
298
299        /**
300         * Matches a variable from the current position in the token stream to the first
301         * occurrence of the end token.
302         */
303        private static class VariableMatcher implements IMatcher {
304
305                /** The pattern to find out how many tokens long a match should be */
306                private static final Pattern MATCH_LENGTH_PATTERN = Pattern.compile("\\$[a-zA-Z]+([0-9]+)");
307
308                /** The variable to match. */
309                private final String variableName;
310
311                /**
312                 * The token type that signals the end of the variable match.
313                 */
314                private final ETokenType endTokenType;
315
316                /**
317                 * The number of tokens that the variable should match. This may be null to
318                 * express that the current variable does not contain a variable count. This
319                 * means, it will greedily match as many tokens as possible.
320                 */
321                private Integer numberOfTokensToMatch = null;
322
323                /** Constructor. */
324                public VariableMatcher(String variableName, ETokenType endTokenType) {
325                        this.variableName = variableName;
326                        this.endTokenType = endTokenType;
327
328                        Matcher matcher = MATCH_LENGTH_PATTERN.matcher(variableName);
329                        if (matcher.matches()) {
330                                this.numberOfTokensToMatch = Integer.parseInt(matcher.group(1));
331                        }
332                }
333
334                /** {@inheritDoc} */
335                @Override
336                public int apply(List<IToken> tokens, int position, Map<String, List<IToken>> variables) {
337
338                        int endIndex = TokenStreamUtils.findFirstTopLevel(tokens, position, EnumSet.of(endTokenType),
339                                        Arrays.asList(ETokenType.LPAREN), Arrays.asList(ETokenType.RPAREN));
340                        if (endIndex == TokenStreamUtils.NOT_FOUND || position == endIndex) {
341                                return NO_MATCH;
342                        }
343
344                        if (numberOfTokensToMatch != null && endIndex > position + numberOfTokensToMatch) {
345                                endIndex = position + numberOfTokensToMatch;
346                        }
347
348                        variables.put(variableName, tokens.subList(position, endIndex));
349                        return endIndex;
350                }
351
352                /** {@inheritDoc} */
353                @Override
354                public String toString() {
355                        return "VariableMatcher[variableName=" + variableName + ",endTokenType=" + endTokenType + "]";
356                }
357
358        }
359
360}