Source code

001/*-------------------------------------------------------------------------+
002|                                                                          |
003| Copyright (c) 2009-2018 CQSE GmbH                                        |
004|                                                                          |
005+-------------------------------------------------------------------------*/
006package eu.cqse.check.framework.checktest;
007
008import static java.util.Objects.requireNonNull;
009import static org.assertj.core.api.Assertions.assertThat;
010
011import java.util.ArrayList;
012import java.util.Collections;
013import java.util.EnumSet;
014import java.util.HashMap;
015import java.util.List;
016import java.util.Map;
017import java.util.regex.Matcher;
018import java.util.regex.Pattern;
019
020import org.conqat.engine.core.core.ConQATException;
021import org.conqat.lib.commons.assertion.CCSMAssert;
022import org.conqat.lib.commons.collections.CollectionUtils;
023import org.conqat.lib.commons.collections.Pair;
024import org.conqat.lib.commons.filesystem.FileSystemUtils;
025import org.conqat.lib.commons.resources.Resource;
026import org.conqat.lib.commons.string.StringUtils;
027
028import eu.cqse.check.framework.core.CheckException;
029import eu.cqse.check.framework.core.phase.ECodeViewOption;
030import eu.cqse.check.framework.core.phase.ECodeViewOption.ETextViewOption;
031import eu.cqse.check.framework.core.phase.ITokenElementContext;
032import eu.cqse.check.framework.core.xpath.DocumentRootShallowEntity;
033import eu.cqse.check.framework.preprocessor.c.CPreprocessor;
034import eu.cqse.check.framework.scanner.ELanguage;
035import eu.cqse.check.framework.scanner.IToken;
036import eu.cqse.check.framework.scanner.ScannerUtils;
037import eu.cqse.check.framework.shallowparser.ShallowParserException;
038import eu.cqse.check.framework.shallowparser.ShallowParserFactory;
039import eu.cqse.check.framework.shallowparser.framework.ShallowEntity;
040import eu.cqse.check.framework.typetracker.ITypeResolution;
041import eu.cqse.check.framework.typetracker.TypeTrackerFactory;
042import eu.cqse.check.util.clang.ClangTranslationUnitWrapper;
043
044/**
045 * The context implementation used during testing.
046 *
047 * This context can maintain filtered and unfiltered code separate from each
048 * other. However, if the filtered and unfiltered content is different, the line
049 * numbers of created findings might not be correct because we cannot convert
050 * between filtered and unfiltered offsets like normal teamscale.
051 *
052 * Therefore, line numbers for reported findings might be wrong in the test
053 * context depending on which code (filtered or unfiltered) the custom check
054 * computes its finding offsets.
055 */
056public class TestTokenElementContext implements ITokenElementContext {
057
058        /** The underlying code's path. */
059        private final String uniformPath;
060
061        /** The underlying, unfiltered code. */
062        private final String unfilteredCode;
063
064        /** The underlying, filtered code. */
065        private final String filteredCode;
066
067        private final ELanguage language;
068
069        /**
070         * Filtered code offsets. Each pairs of offsets denotes the start (inclusive)
071         * and end (exclusive) offset of a section of filtered code. Offsets refer to
072         * {@link #unfilteredCode}.
073         *
074         * Sorted by Pair::getFirst. Regions are non-overlapping.
075         */
076        private final List<Pair<Integer, Integer>> filteredRegions;
077
078        private final Map<ECodeViewOption, List<IToken>> tokensCache = new HashMap<>();
079        private final Map<ECodeViewOption, ShallowEntity> rootEntityCache = new HashMap<>();
080        private final Map<ECodeViewOption, List<ShallowEntity>> abstractSyntaxTreeCache = new HashMap<>();
081
082        private final Map<ECodeViewOption, ITypeResolution> typeResolutionCache = new HashMap<>();
083
084        public TestTokenElementContext(Resource codeFile, ELanguage language) {
085                this(codeFile.getPath(), language, codeFile.getContent(), Collections.emptyList());
086        }
087
088        public TestTokenElementContext(String uniformPath, ELanguage language, String unfilteredCode,
089                        List<Pattern> filterPatterns) {
090                this.uniformPath = uniformPath;
091                this.unfilteredCode = unfilteredCode;
092                this.language = requireNonNull(language);
093                filteredRegions = new ArrayList<>();
094                for (Pattern filterPattern : filterPatterns) {
095                        Matcher matcher = filterPattern.matcher(unfilteredCode);
096                        while (matcher.find()) {
097                                filteredRegions.add(new Pair<>(matcher.start(), matcher.end()));
098                        }
099                }
100                filteredRegions.sort(CollectionUtils.getPairComparator());
101                assertFilterRegionsValid(filteredRegions);
102                filteredCode = filterContent(unfilteredCode, filteredRegions);
103        }
104
105        /**
106         * filterRegions. Returns the filtered code. Filters character sequences from
107         * the given unfilteredCode based on the
108         *
109         * Assumes that the filter regions are sorted, non-empty and non-overlapping.
110         */
111        private static String filterContent(String unfilteredCode, List<Pair<Integer, Integer>> filterRegions) {
112                if (filterRegions.isEmpty()) {
113                        return unfilteredCode;
114                }
115                StringBuilder filteredCode = new StringBuilder();
116                if (filterRegions.get(0).getFirst() != 0) {
117                        filteredCode.append(unfilteredCode.subSequence(0, filterRegions.get(0).getFirst()));
118                }
119                for (int i = 0; i < filterRegions.size() - 1; i++) {
120                        filteredCode.append(
121                                        unfilteredCode.subSequence(filterRegions.get(i).getSecond(), filterRegions.get(i + 1).getFirst()));
122                }
123                if (CollectionUtils.getLast(filterRegions).getSecond() != unfilteredCode.length()) {
124
125                        filteredCode.append(unfilteredCode.subSequence(CollectionUtils.getLast(filterRegions).getSecond(),
126                                        unfilteredCode.length()));
127                }
128                return filteredCode.toString();
129        }
130
131        /**
132         * Asserts that the {@link #filteredRegions} are valid (each region is non-empty
133         * and regions do not overlap).
134         */
135        private static void assertFilterRegionsValid(List<Pair<Integer, Integer>> filteredRegions) {
136                filteredRegions.forEach(region -> assertThat(region.getFirst() < region.getSecond()).isTrue());
137                for (int i = 0; i < filteredRegions.size() - 1; i++) {
138                        assertThat(filteredRegions.get(i).getSecond() <= filteredRegions.get(i + 1).getFirst())
139                                        .as("Overlapping filter regions.").isTrue();
140                }
141        }
142
143        /** {@inheritDoc} */
144        @Override
145        public ELanguage getLanguage() {
146
147                return language;
148        }
149
150        /** {@inheritDoc} */
151        @Override
152        public String getUniformPath() {
153                return uniformPath;
154        }
155
156        /** Template method that is called before accessing the AST. */
157        protected void accessAst() {
158                // does nothing
159        }
160
161        /** Template method called before accessing type resolution. */
162        protected void accessTypeResolution() {
163                // does nothing
164        }
165
166        /**
167         * Returns the offset in filtered code that represents the same character as the
168         * given offset in the unfiltered Code.
169         */
170        protected int getFilteredOffset(int unfilteredCodeOffset) {
171                int filteredOffset = unfilteredCodeOffset;
172                for (Pair<Integer, Integer> filterRegion : filteredRegions) {
173                        if (filterRegion.getFirst() < unfilteredCodeOffset && filterRegion.getSecond() <= unfilteredCodeOffset) {
174                                // unfilteredCodeOffset is after this filtered region.
175                                // Subtract size of filtered region
176                                filteredOffset -= (filterRegion.getSecond() - filterRegion.getFirst());
177                        } else if (filterRegion.getFirst() < unfilteredCodeOffset) {
178                                // unfilteredCodeOffset is in this filtered region.
179                                // Return the first offset after this region and skip the
180                                // assertion.
181                                return getFilteredOffset(filterRegion.getSecond());
182                        } else {
183                                // unfilteredCodeOffset is before this region
184                                break;
185                        }
186                }
187                CCSMAssert.isTrue(filteredCode.codePointAt(filteredOffset) == unfilteredCode.codePointAt(unfilteredCodeOffset),
188                                "Computation of filtered offset in Custom check test context is wrong.");
189                return filteredOffset;
190        }
191
192        /**
193         * Returns whether the given offset in unfiltered code is in a filtered region.
194         */
195        protected boolean isInFilteredRegion(int unfilteredCodeOffset) {
196                for (Pair<Integer, Integer> filterRegion : filteredRegions) {
197                        if (filterRegion.getFirst() <= unfilteredCodeOffset && unfilteredCodeOffset < filterRegion.getSecond()) {
198                                return true;
199                        }
200                }
201                return false;
202        }
203
204        /** {@inheritDoc} */
205        @Override
206        public String getTextContent(ETextViewOption view) {
207                switch (view) {
208                case UNFILTERED_CONTENT:
209                        return unfilteredCode;
210                case FILTERED_CONTENT:
211                        return filteredCode;
212                default:
213                        throw new AssertionError("Missing case for " + view);
214                }
215        }
216
217        /** {@inheritDoc} */
218        @Override
219        public List<IToken> getTokens(ECodeViewOption view) {
220                if (tokensCache.containsKey(view)) {
221                        return tokensCache.get(view);
222                }
223                List<IToken> tokens = computeTokens(view);
224                tokensCache.put(view, tokens);
225                return tokens;
226        }
227
228        /**
229         * Computes a new {@link IToken} list for the given {@link ECodeViewOption}.
230         * Throws an {@link AssertionError} if the given {@link ECodeViewOption} is not
231         * supported.
232         */
233        private List<IToken> computeTokens(ECodeViewOption view) throws AssertionError {
234                switch (view) {
235                case UNFILTERED:
236                        return CollectionUtils.asUnmodifiable(ScannerUtils
237                                        .getTokens(getTextContent(ETextViewOption.UNFILTERED_CONTENT), getLanguage(), getUniformPath()));
238                case FILTERED:
239                        return CollectionUtils.asUnmodifiable(ScannerUtils
240                                        .getTokens(getTextContent(ETextViewOption.FILTERED_CONTENT), getLanguage(), getUniformPath()));
241                case UNFILTERED_PREPROCESSED:
242                        return calculateLocalPreprocessedTokens(ScannerUtils
243                                        .getTokens(getTextContent(ETextViewOption.UNFILTERED_CONTENT), getLanguage(), getUniformPath()));
244                case FILTERED_PREPROCESSED:
245                        return calculateLocalPreprocessedTokens(ScannerUtils
246                                        .getTokens(getTextContent(ETextViewOption.FILTERED_CONTENT), getLanguage(), getUniformPath()));
247                default:
248                        throw new AssertionError("Missing case for " + view);
249                }
250        }
251
252        /** {@inheritDoc} */
253        @Override
254        public ShallowEntity getRootEntity(ECodeViewOption view) throws CheckException {
255                if (rootEntityCache.containsKey(view)) {
256                        return rootEntityCache.get(view);
257                }
258                ShallowEntity rootEntity = computeRootEntity(view);
259                rootEntityCache.put(view, rootEntity);
260                return rootEntity;
261        }
262
263        /**
264         * Computes a new root entity for the given {@link ECodeViewOption}. Throws an
265         * {@link AssertionError} if the given {@link ECodeViewOption} is not supported.
266         */
267        private ShallowEntity computeRootEntity(ECodeViewOption view) throws CheckException, AssertionError {
268                /*
269                 * This might violate an assumption that each ShallowEntity is only part of one
270                 * AST (after this method it is part of the original AST and of the "rooted"
271                 * tree). However, this is necessary as some checks (e.g.,
272                 * AvoidUsingSystemThreadingThreadAbort) rely on the fact that the same (Object
273                 * equality) Entities are returned by this method and used to construct the
274                 * ITypeResolution.
275                 */
276                DocumentRootShallowEntity rootEntity = new DocumentRootShallowEntity(getElementName());
277                switch (view) {
278                case UNFILTERED_PREPROCESSED:
279                        rootEntity.setChildren(getAbstractSyntaxTree(ECodeViewOption.UNFILTERED_PREPROCESSED));
280                        break;
281                case FILTERED_PREPROCESSED:
282                        rootEntity.setChildren(getAbstractSyntaxTree(ECodeViewOption.FILTERED_PREPROCESSED));
283                        break;
284                case FILTERED:
285                        rootEntity.setChildren(getAbstractSyntaxTree(ECodeViewOption.FILTERED));
286                        break;
287                default:
288                        throw new AssertionError("The ECodeViewOption " + view + " is not supported.");
289                }
290                return rootEntity;
291        }
292
293        /** {@inheritDoc} */
294        @Override
295        public List<ShallowEntity> getAbstractSyntaxTree(ECodeViewOption view) throws CheckException {
296                if (abstractSyntaxTreeCache.containsKey(view)) {
297                        return abstractSyntaxTreeCache.get(view);
298                }
299                List<ShallowEntity> typeResolution = computeAbstractSyntaxTree(view);
300                abstractSyntaxTreeCache.put(view, typeResolution);
301                return typeResolution;
302        }
303
304        /**
305         * Computes a new Abstract syntax Tree for the given {@link ECodeViewOption}.
306         * Throws an {@link AssertionError} if the given {@link ECodeViewOption} is not
307         * supported.
308         */
309        private List<ShallowEntity> computeAbstractSyntaxTree(ECodeViewOption view) throws CheckException, AssertionError {
310                switch (view) {
311                case UNFILTERED_PREPROCESSED:
312                        try {
313                                return parseTokens(ECodeViewOption.UNFILTERED_PREPROCESSED, false);
314                        } catch (ShallowParserException e) {
315                                throw new CheckException(e);
316                        }
317                case FILTERED:
318                        try {
319                                return parseTokens(ECodeViewOption.FILTERED_PREPROCESSED, true);
320                        } catch (ShallowParserException e) {
321                                throw new CheckException(e);
322                        }
323                case FILTERED_PREPROCESSED:
324                        try {
325                                return parseTokens(ECodeViewOption.FILTERED_PREPROCESSED, false);
326                        } catch (ShallowParserException e) {
327                                throw new CheckException(e);
328                        }
329                default:
330                        throw new AssertionError("The ECodeViewOption " + view + " is not supported.");
331                }
332        }
333
334        /** {@inheritDoc} */
335        @Override
336        public ITypeResolution getTypeResolution(ECodeViewOption view) throws CheckException {
337                if (typeResolutionCache.containsKey(view)) {
338                        return typeResolutionCache.get(view);
339                }
340                ITypeResolution typeResolution = computeTypeResolution(view);
341                typeResolutionCache.put(view, typeResolution);
342                return typeResolution;
343        }
344
345        /**
346         * Computes a new {@link ITypeResolution} for the given {@link ECodeViewOption}.
347         * Throws an {@link AssertionError} if the given {@link ECodeViewOption} is not
348         * supported.
349         */
350        private ITypeResolution computeTypeResolution(ECodeViewOption view) throws CheckException {
351                switch (view) {
352                case UNFILTERED_PREPROCESSED:
353                        accessTypeResolution();
354                        return TypeTrackerFactory.createTypeTracker(getLanguage())
355                                        .createTypeResolution(getAbstractSyntaxTree(ECodeViewOption.UNFILTERED_PREPROCESSED));
356                case FILTERED:
357                        accessTypeResolution();
358                        return TypeTrackerFactory.createTypeTracker(getLanguage())
359                                        .createTypeResolution(getAbstractSyntaxTree(ECodeViewOption.FILTERED));
360                case FILTERED_PREPROCESSED:
361                        accessTypeResolution();
362                        return TypeTrackerFactory.createTypeTracker(getLanguage())
363                                        .createTypeResolution(getAbstractSyntaxTree(ECodeViewOption.FILTERED_PREPROCESSED));
364                default:
365                        throw new AssertionError("The ECodeViewOption " + view + " is not supported.");
366                }
367        }
368
369        /** Returns the file name of the current uniform path. */
370        private String getElementName() {
371                return StringUtils.getLastPart(getUniformPath(), FileSystemUtils.UNIX_SEPARATOR);
372        }
373
374        /**
375         * Parses the given Tokens and returns the shallow entities.
376         */
377        private List<ShallowEntity> parseTokens(ECodeViewOption codeViewOption, boolean removePreprocessorTokens)
378                        throws ShallowParserException {
379                CCSMAssert.isTrue(EnumSet.of(ECodeViewOption.FILTERED_PREPROCESSED, ECodeViewOption.UNFILTERED_PREPROCESSED)
380                                .contains(codeViewOption), "can parse preprocessed tokens only: " + codeViewOption);
381                List<ShallowEntity> entities = ShallowParserFactory.createParser(getLanguage())
382                                .parseTopLevel(getTokens(codeViewOption));
383                if (getLanguage() == ELanguage.CPP && removePreprocessorTokens) {
384                        CPreprocessor.removePreprocessorTokens(entities);
385                }
386                return entities;
387        }
388
389        @Override
390        public ClangTranslationUnitWrapper getClangTranslationUnitWrapper() {
391                try {
392                        return ClangTranslationUnitWrapper.createForContext(this);
393                } catch (ConQATException e) {
394                        CCSMAssert.fail("Exception while parsing with clang: " + e.getMessage(), e);
395                        return null;
396                }
397        }
398}