Source code

001/*-------------------------------------------------------------------------+
002|                                                                          |
003| Copyright 2005-2011 The ConQAT Project                                   |
004|                                                                          |
005| Licensed under the Apache License, Version 2.0 (the "License");          |
006| you may not use this file except in compliance with the License.         |
007| You may obtain a copy of the License at                                  |
008|                                                                          |
009|    http://www.apache.org/licenses/LICENSE-2.0                            |
010|                                                                          |
011| Unless required by applicable law or agreed to in writing, software      |
012| distributed under the License is distributed on an "AS IS" BASIS,        |
013| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
014| See the License for the specific language governing permissions and      |
015| limitations under the License.                                           |
016+-------------------------------------------------------------------------*/
017package eu.cqse.check.framework.checktest;
018
019import static org.assertj.core.api.Assertions.assertThat;
020import static org.assertj.core.api.Assertions.fail;
021
022import java.io.File;
023import java.io.IOException;
024import java.lang.reflect.InvocationTargetException;
025import java.util.HashMap;
026import java.util.HashSet;
027import java.util.List;
028import java.util.Map;
029import java.util.Set;
030import java.util.stream.Collectors;
031
032import org.assertj.core.api.Assertions;
033import org.conqat.lib.commons.collections.CollectionUtils;
034import org.conqat.lib.commons.filesystem.ClassPathUtils;
035import org.conqat.lib.commons.filesystem.FileSystemUtils;
036import org.conqat.lib.commons.resources.Resource;
037import org.conqat.lib.commons.string.StringUtils;
038import org.conqat.lib.commons.test.ExpectedDataContainer;
039import org.conqat.lib.commons.test.ExpectedResourceSource;
040import org.junit.jupiter.api.BeforeEach;
041import org.junit.jupiter.api.TestInstance;
042import org.junit.jupiter.params.ParameterizedTest;
043
044import eu.cqse.check.framework.core.CheckException;
045import eu.cqse.check.framework.core.CheckInfo;
046import eu.cqse.check.framework.core.CheckInstance;
047import eu.cqse.check.framework.core.option.CheckOptionWrapper;
048import eu.cqse.check.framework.core.phase.IExtractedValue;
049import eu.cqse.check.framework.core.phase.IGlobalExtractionPhase;
050import eu.cqse.check.framework.core.registry.CheckRegistry;
051import eu.cqse.check.framework.scanner.ELanguage;
052
053/**
054 * Parameterized test for check implementations. This base class is based on a
055 * mockup of the teamscale infrastructure used in production.
056 */
057@TestInstance(TestInstance.Lifecycle.PER_CLASS)
058public abstract class CheckTestBase {
059
060        /**
061         * We have to load Check classes into the static singleton
062         * {@link CheckRegistry#getInstance()}. This field is to remember which we
063         * already have loaded and to avoid "is already registered" warnings.
064         */
065        private static final HashSet<String> LOADED_DIRECTORIES = new HashSet<>();
066
067        /** Extension of the check parameter files. */
068        private static final String PARAMETERS_EXTENSION = "parameters";
069
070        /** Maps simple class names to the corresponding check info. */
071        protected Map<String, CheckInfo> checkInfoBySimpleClassName;
072
073        protected final Class<?> markerClass;
074
075        protected CheckTestBase(Class<?> markerClass) {
076                this.markerClass = markerClass;
077        }
078
079        @BeforeEach
080        protected void setUpCheckInfoMap() throws IOException {
081                if (checkInfoBySimpleClassName == null) {
082                        checkInfoBySimpleClassName = buildCheckInfoMap(markerClass);
083                }
084        }
085
086        /**
087         * Run test by parsing whole file and checking against reference.
088         *
089         * @param expectedDataContainer
090         *            Container holding an expected resource and their associated code
091         *            and parameter resources.
092         *            test-data/eu.cqse.check/./abap/AbapHardCodedUsernameCheck/hard_coded_username.abap.expected
093         */
094        @ParameterizedTest(name = "{0}")
095        @ExpectedResourceSource
096        void test(ExpectedDataContainer expectedDataContainer) throws CheckException {
097
098                // simple check name is the name of the parent directory
099                Resource expectedFile = expectedDataContainer.getExpectedResource();
100                String checkSimpleName = determineCheckNameFromPath(expectedFile);
101                Resource parametersResource = expectedDataContainer.getTestResources().stream()
102                                .filter(resource -> resource.hasExtension(PARAMETERS_EXTENSION)).findFirst().orElse(null);
103                Map<String, Object> optionValues = parseParameterFile(checkSimpleName, parametersResource,
104                                checkInfoBySimpleClassName);
105
106                runCheckAndAssertFindings(checkSimpleName, expectedDataContainer, optionValues, checkInfoBySimpleClassName);
107        }
108
109        /**
110         * Determines which check should be executed with the given expected file.
111         * Searches backwards from the end of the path. The first folder name that
112         * equals the simple name of a custom check class determines which check is
113         * executed.
114         */
115        private String determineCheckNameFromPath(Resource expectedFile) throws CheckException {
116                String[] segments = FileSystemUtils.getPathSegments(expectedFile.getPath());
117                for (int i = segments.length - 2; i >= 0; i--) {
118                        if (checkInfoBySimpleClassName.containsKey(segments[i])) {
119                                return segments[i];
120                        }
121                }
122                throw new CheckException("Could not determine check name from path " + expectedFile.getPath()
123                                + ". One folder name in the path must be equal to the check's simple class name.");
124        }
125
126        /**
127         * Determines the language to use for the test file based on the
128         * {@link ELanguage}s of the check and the ending of the uniform path.
129         */
130        public static ELanguage determineLanguage(CheckInfo checkInfo, String uniformPath) {
131                Set<ELanguage> matchingLanguages = CollectionUtils
132                                .intersectionSet(ELanguage.getAllLanguagesForPath(uniformPath), checkInfo.getSupportedLanguages());
133                if (matchingLanguages.size() != 1) {
134                        Assertions.assertThat(checkInfo.getSupportedLanguages().size()).isEqualTo(1).withFailMessage(
135                                        "Languages for check test filename extension couldn't be uniquely determined, but the check supports multiple languages.");
136                        return checkInfo.getSupportedLanguages().iterator().next();
137                }
138                return matchingLanguages.iterator().next();
139        }
140
141        /**
142         * Builds a map from simple class names to check infos. The simple class name
143         * must be unique. The rationale here is avoid classes with same name in
144         * different packages (confusing) but also make the test-data more readable. The
145         * clazz will be used to resolve the classpath to search for checks.
146         */
147        public static Map<String, CheckInfo> buildCheckInfoMap(Class<?> clazz) throws IOException {
148                // we expect the checks to reside in the same class folder as the tests
149                String classPath = ClassPathUtils.createClassPath(null, clazz);
150                if (!LOADED_DIRECTORIES.contains(classPath)) {
151                        if (StringUtils.endsWithOneOf(classPath, ".jar")) {
152                                CheckRegistry.getInstance().loadChecksFromJar(new File(classPath));
153                        }
154                        CheckRegistry.getInstance().registerChecksFromClasspathDirectory(new File(classPath));
155                        LOADED_DIRECTORIES.add(classPath);
156                }
157                Map<String, CheckInfo> checkInfoBySimpleClassName = new HashMap<>();
158                for (CheckInfo checkInfo : CheckRegistry.getInstance().getChecksInfos()) {
159                        if (checkInfoBySimpleClassName.put(checkInfo.getSimpleClassName(), checkInfo) != null) {
160                                fail("Duplicate simple class name: " + checkInfo.getSimpleClassName());
161                        }
162                }
163                return checkInfoBySimpleClassName;
164        }
165
166        /**
167         * Parses the check parameter files if it is available and return a map of
168         * parameter name to parameter values.
169         */
170        private static Map<String, Object> parseParameterFile(String checkSimpleName, Resource parameterFile,
171                        Map<String, CheckInfo> checkInfoBySimpleClassName) {
172                Map<String, Object> optionValues = new HashMap<>();
173
174                if (parameterFile != null) {
175                        CheckInfo check = checkInfoBySimpleClassName.get(checkSimpleName);
176                        Map<String, CheckOptionWrapper<?>> options = check.getOptions();
177                        List<String> lines = parameterFile.getLines();
178                        for (String line : lines) {
179                                String[] parts = line.split(":", 2);
180                                String parameterName = parts[0].trim();
181                                String parameterValue = parts[1].trim();
182                                if (!options.containsKey(parameterName)) {
183                                        Assertions.fail("Unknown parameter name " + parameterName + " in parameters file: "
184                                                        + parameterFile.getPath());
185                                }
186                                CheckOptionWrapper<?> checkOption = options.get(parameterName);
187                                optionValues.put(parameterName, parseCheckOptionValue(checkOption.getType(), parameterValue));
188
189                        }
190                }
191                return optionValues;
192        }
193
194        /** Parses a check option value of the given type from a String */
195        private static Object parseCheckOptionValue(Class<?> type, String parameterValue) {
196                if (type == String.class) {
197                        return parameterValue;
198                } else if (type == Integer.class) {
199                        return Integer.parseInt(parameterValue);
200                } else if (type == Boolean.class) {
201                        return Boolean.valueOf(parameterValue);
202                } else if (type == List.class) {
203                        return StringUtils.splitWithEscapeCharacter(parameterValue, ",");
204                } else if (type == Set.class) {
205                        List<String> parts = StringUtils.splitWithEscapeCharacter(parameterValue, ",");
206                        return new HashSet<>(parts);
207                } else {
208                        throw new IllegalStateException("Check option type not supported: " + type);
209                }
210        }
211
212        /**
213         * Asserts that the check executed on the given code file returns the findings
214         * as described by the findings string.
215         */
216        protected void runCheckAndAssertFindings(String checkSimpleName, ExpectedDataContainer expectedDataContainer,
217                        Map<String, Object> options, Map<String, CheckInfo> checkInfoBySimpleClassName) throws CheckException {
218                runCheckWithMockupContextAndAssertFindings(checkSimpleName, expectedDataContainer, options,
219                                checkInfoBySimpleClassName);
220        }
221
222        /**
223         * Asserts that the check executed on the given code returns the findings as
224         * described by the findings string.
225         */
226        private static void runCheckWithMockupContextAndAssertFindings(String checkSimpleName,
227                        ExpectedDataContainer expectedDataContainer, Map<String, Object> options,
228                        Map<String, CheckInfo> checkInfoBySimpleClassName) throws CheckException {
229                List<Resource> parsableFiles = CollectionUtils.filter(expectedDataContainer.getTestResources(),
230                                file -> ELanguage.fromResource(file) != null);
231
232                runCheckWithMockupContextAndAssertFindings(checkSimpleName, expectedDataContainer, checkInfoBySimpleClassName,
233                                options, parsableFiles);
234        }
235
236        /**
237         * Asserts that the check executed on the given code returns the findings as
238         * described by the findings string.
239         */
240        private static void runCheckWithMockupContextAndAssertFindings(String checkSimpleName,
241                        ExpectedDataContainer expectedDataContainer, Map<String, CheckInfo> checks, Map<String, Object> options,
242                        List<Resource> allFiles) throws CheckException {
243                CheckInfo checkInfo = checks.get(checkSimpleName);
244                TestCheckContext context = generateCheckContext(expectedDataContainer.getPrimaryTestResource(), checkInfo);
245                for (Class<? extends IGlobalExtractionPhase<?, ?>> phase : checkInfo.getRequiredPhases()) {
246                        preloadPhase(phase, allFiles, context);
247                }
248                runCheckWithMockupContextAndAssertFindings(checkSimpleName,
249                                expectedDataContainer.getExpectedResource().getContent(), options, context);
250        }
251
252        /**
253         * Creates a {@link TestCheckContext} that is provided to the tested
254         * custom-check instance.
255         */
256        private static TestCheckContext generateCheckContext(Resource code, CheckInfo checkInfo) {
257                return new TestCheckContext(code.getPath(), code.getContent(), checkInfo);
258        }
259
260        @SuppressWarnings("unchecked")
261        private static void preloadPhase(Class<? extends IGlobalExtractionPhase<?, ?>> phase, List<Resource> resources,
262                        TestCheckContext context) throws CheckException {
263                IGlobalExtractionPhase<?, ?> phaseInstance;
264                try {
265                        phaseInstance = phase.getConstructor().newInstance();
266                } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException
267                                | NoSuchMethodException | SecurityException e) {
268                        throw new CheckException("Could not initialize phase " + phase, e);
269                }
270
271                Map<String, List<IExtractedValue<?>>> result = new HashMap<>();
272                for (Resource resource : resources) {
273                        TestTokenElementContext phaseContext = new TestTokenElementContext(resource,
274                                        determineLanguage(context.getCheckInfo(), resource.getPath()));
275                        List<IExtractedValue<?>> values = (List<IExtractedValue<?>>) phaseInstance.extract(phaseContext);
276                        result.put(phaseContext.getUniformPath(), values);
277                }
278                context.addPhaseResult(phase, result);
279
280                if (phaseInstance.needsAccessUniformPathByValue()) {
281                        Map<String, List<IExtractedValue<?>>> byValue = result.values().stream().flatMap(List::stream)
282                                        .collect(Collectors.groupingBy(IExtractedValue::getValue));
283                        context.addInvertedPhaseResult(phase, byValue);
284                }
285        }
286
287        /**
288         * Asserts that the check executed on the given code returns the findings as
289         * described by the findings string.
290         */
291        private static void runCheckWithMockupContextAndAssertFindings(String checkSimpleName,
292                        String expectedFindingsString, Map<String, Object> options, TestCheckContext context)
293                        throws CheckException {
294                assertThat(context.getCheckInfo()).as("No check with simple class name " + checkSimpleName + " found!")
295                                .isNotNull();
296
297                CheckInstance instance = new CheckInstance(context.getCheckInfo());
298                for (String optionName : options.keySet()) {
299                        instance.setOption(optionName, options.get(optionName));
300                }
301                instance.initializeAndSetContext(context);
302                instance.execute();
303
304                String actualFindings = sortFindings(context.getFindingsString());
305                String expectedFindings = sortFindings(expectedFindingsString);
306
307                assertThat(actualFindings).as("Unexpected check result for " + context.getUniformPath())
308                                .isEqualTo(expectedFindings);
309        }
310
311        /**
312         * Sorts the lines within the findings to stay independent of finding creation
313         * order.
314         */
315        private static String sortFindings(String findingsText) {
316                return StringUtils.concat(
317                                CollectionUtils.sort(StringUtils
318                                                .splitLinesAsList(StringUtils.normalizeLineSeparatorsPlatformSpecific(findingsText))),
319                                StringUtils.LINE_SEPARATOR);
320        }
321}