diff --git a/.github/workflows/test-windows-webdriver.yml b/.github/workflows/test-windows-webdriver.yml index 4d8ec0c04..0c5a59850 100644 --- a/.github/workflows/test-windows-webdriver.yml +++ b/.github/workflows/test-windows-webdriver.yml @@ -49,15 +49,15 @@ jobs: name: Java${{ matrix.java }}-runTestWebdriverSuspiciousTagStateModel-artifact path: D:/a/TESTAR_dev/TESTAR_dev/testar/target/install/testar/bin/webdriver_and_suspicious - - name: Run webdriver to detect a browser console error in parabank - run: ./gradlew runTestWebdriverParabankConsoleError - - name: Save runTestWebdriverParabankConsoleError HTML report artifact + - name: Run webdriver to detect a browser console error and accessibility warning in parabank + run: ./gradlew runTestWebdriverParabankConsoleErrorAndAccessibilityWarning + - name: Save runTestWebdriverParabankConsoleErrorAndAccessibilityWarning HTML report artifact uses: actions/upload-artifact@v4 # Only upload GitHub Actions results if this task fails (Can be replaced with 'if: always()') if: failure() with: - name: Java${{ matrix.java }}-runTestWebdriverParabankConsoleError-artifact - path: D:/a/TESTAR_dev/TESTAR_dev/testar/target/install/testar/bin/webdriver_console_error + name: Java${{ matrix.java }}-runTestWebdriverParabankConsoleErrorAndAccessibilityWarning-artifact + path: D:/a/TESTAR_dev/TESTAR_dev/testar/target/install/testar/bin/webdriver_console_error_and_accessibility_warning - name: Run webdriver to login in parabank and detect a welcome suspicious tag run: ./gradlew runTestWebdriverParabankLogin diff --git a/README.md b/README.md index 75add0464..f831c95ae 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ Some of the most interesting parameters that can help to integrate TESTAR as an ShowVisualSettingsDialogOnStartup -> To run TESTAR without the GUI - Mode -> TESTAR execution Mode (Spy, Generate, Record, Replay, View) + Mode -> TESTAR execution Mode (Spy, Generate, Replay, View) SUTConnector & SUTConnectorValue -> The way to link with the desired application to be tested diff --git a/core/src/org/testar/ProtocolUtil.java b/core/src/org/testar/ProtocolUtil.java index 909da8326..e72aa47f7 100644 --- a/core/src/org/testar/ProtocolUtil.java +++ b/core/src/org/testar/ProtocolUtil.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * -* Copyright (c) 2016 - 2025 Universitat Politecnica de Valencia - www.upv.es -* Copyright (c) 2019 - 2025 Open Universiteit - www.ou.nl +* Copyright (c) 2016 - 2026 Universitat Politecnica de Valencia - www.upv.es +* Copyright (c) 2019 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -49,6 +49,7 @@ import org.testar.monkey.alayer.Widget; import java.awt.*; +import java.util.Collections; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; @@ -251,7 +252,8 @@ public static AWTCanvas getStateshotBinary(State state) { } //If the state Shape is not properly obtained, or the State has an error, use full monitor screen - if (viewPort == null || (state.get(Tags.OracleVerdict, Verdict.OK).severity() > Verdict.Severity.OK.getValue())) + List verdicts = state.get(Tags.OracleVerdicts, Collections.singletonList(Verdict.OK)); + if (viewPort == null || !Verdict.helperAreAllVerdictsOK(verdicts)) viewPort = state.get(Tags.Shape, null); // get the SUT process canvas (usually, full monitor screen) // Validate viewport dimensions before taking the screenshot diff --git a/core/src/org/testar/monkey/alayer/Tags.java b/core/src/org/testar/monkey/alayer/Tags.java index 2beb8ecc8..07b7837c5 100644 --- a/core/src/org/testar/monkey/alayer/Tags.java +++ b/core/src/org/testar/monkey/alayer/Tags.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * -* Copyright (c) 2013 - 2025 Universitat Politecnica de Valencia - www.upv.es -* Copyright (c) 2018 - 2025 Open Universiteit - www.ou.nl +* Copyright (c) 2013 - 2026 Universitat Politecnica de Valencia - www.upv.es +* Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -28,10 +28,6 @@ * POSSIBILITY OF SUCH DAMAGE. *******************************************************************************************************/ -/** - * @author Sebastian Bauersfeld - */ - package org.testar.monkey.alayer; import org.testar.CodingManager; @@ -161,9 +157,9 @@ private Tags() {} /** Usually attached to an object of {@link State}. The value is a screenshot of the state. */ public static final Tag ScreenshotPath = from("ScreenshotPath", String.class); - /** Usually attached to a {@link State} object. The value is an outcome of a test oracle for that state. It is - * used to mark states as 'suspicious' or 'erroneous' */ - public static final Tag OracleVerdict = from("OracleVerdict", Verdict.class); + /** Usually attached to a {@link State} object. The value is a list of outcomes of test oracles for that state. */ + @SuppressWarnings("unchecked") + public static final Tag> OracleVerdicts = from("OracleVerdicts", (Class>)(Class)List.class); /** The standard mouse object. Usually attached to systems */ public static final Tag StandardMouse = from("StandardMouse", Mouse.class); diff --git a/core/src/org/testar/monkey/alayer/Verdict.java b/core/src/org/testar/monkey/alayer/Verdict.java index edf29514b..f25d0528a 100644 --- a/core/src/org/testar/monkey/alayer/Verdict.java +++ b/core/src/org/testar/monkey/alayer/Verdict.java @@ -31,7 +31,9 @@ package org.testar.monkey.alayer; import java.io.Serializable; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import org.testar.monkey.Assert; import org.testar.monkey.Util; @@ -93,12 +95,16 @@ public enum Severity { // WARNING GROUP: ACCESSIBILITY WARNING_ACCESSIBILITY_FAULT(0.4, "WARNING_ACCESSIBILITY_FAULT"), - /** FAIL (0.5 - 1.0) **/ + /** FAIL (0.5 - 0.899) **/ UNREPLAYABLE(0.5, "UNREPLAYABLE"), // Sequence not replayable SUSPICIOUS_TAG(0.8, "SUSPICIOUS_TAG"), // Suspicious tag SUSPICIOUS_PROCESS(0.87, "SUSPICIOUS_PROCESS"), // Suspicious message in the process standard output/error SUSPICIOUS_LOG(0.89, "SUSPICIOUS_LOG"), // Suspicious message in log file or command output (LogOracle) + + /** CRITICAL (0.9 - 1.0) **/ + + CRITICAL(0.9, "CRITICAL"), NOT_RESPONDING(0.99999990, "NOT_RESPONDING"), // Unresponsive UNEXPECTEDCLOSE(0.99999999, "UNEXPECTEDCLOSE"), // Crash? Unexpected close? FAIL(1.0, "FAIL"); @@ -193,29 +199,18 @@ public Visualizer visualizer() { return visualizer; } - @Override - public String toString() { - return "severity: " + severity + " info: " + info; - } - /** - * Retrieves the verdict result of joining two verdicts. - * @param verdict A verdict to join with current verdict. - * - * @return A new verdict that is the result of joining the current verdict with the provided verdict. + * Indicates if this verdict is critical (SUT froze or crashed). + * + * @return true if verdict is critical */ - public Verdict join(Verdict verdict) { - Severity joinedSeverity = Arrays.stream(Severity.values()) - .filter(s -> s.getValue() == Math.max(this.severity, verdict.severity())) - .findFirst() - .orElse(Severity.FAIL); - - String joinedInfo = this.info.contains(verdict.info()) ? this.info - : (this.severity == Severity.OK.getValue() ? "" : this.info + "\n") + verdict.info(); - - Visualizer joinedVisualizer = Visualizer.join(this.visualizer(), verdict.visualizer()); + public boolean isCritical() { + return this.severity() >= Severity.CRITICAL.getValue(); + } - return new Verdict(joinedSeverity, joinedInfo, joinedVisualizer); + @Override + public String toString() { + return "severity: " + severity + " info: " + info; } /** @@ -234,4 +229,15 @@ public boolean equals(Object o) { && this.visualizer.equals(other.visualizer); } + public static boolean helperAreAllVerdictsOK(List verdicts) { + if(verdicts == null || verdicts.isEmpty()) return true; + + for (Verdict verdict : verdicts) { + if (verdict.severity() > Severity.OK.getValue()) { + return false; + } + } + return true; + } + } \ No newline at end of file diff --git a/core/test/org/testar/monkey/alayer/VerdictTest.java b/core/test/org/testar/monkey/alayer/VerdictTest.java index 9d9ac2bb2..9910d5e01 100644 --- a/core/test/org/testar/monkey/alayer/VerdictTest.java +++ b/core/test/org/testar/monkey/alayer/VerdictTest.java @@ -35,30 +35,16 @@ import java.util.Arrays; import org.junit.Test; -import org.testar.monkey.alayer.visualizers.RegionsVisualizer; import org.testar.monkey.alayer.visualizers.ShapeVisualizer; public class VerdictTest { - private final double DELTA = 0; - - private final Visualizer dummyVisualizer = new Visualizer(){ - private static final long serialVersionUID = -7830649624698071090L; - public void run(State s, Canvas c, Pen pen) {} - }; - private final Visualizer failVisualizer = new ShapeVisualizer( Pen.PEN_RED, Rect.from(0, 0, 10, 10), "Fail Visualizer", 0.5, 0.5); - private final Visualizer issueVisualizer = new RegionsVisualizer( - Pen.PEN_RED, - Arrays.asList(Rect.from(0, 0, 10, 10)), - "Issue Visualizer", - 0.5, 0.5); - @Test public void testToString() { Verdict v = new Verdict(Verdict.Severity.OK, "This is a test verdict"); @@ -67,54 +53,12 @@ public void testToString() { } @Test - public void testJoin() { - Verdict v1 = new Verdict(Verdict.Severity.OK, "Foo Bar"); - Verdict v2 = new Verdict(Verdict.Severity.FAIL, "Bar", failVisualizer); - Verdict v3 = new Verdict(Verdict.Severity.OK, "Baz", dummyVisualizer); - Verdict v4 = new Verdict(Verdict.Severity.FAIL, "Exception", issueVisualizer); - Verdict emptyVisualizerVerdict = new Verdict(Verdict.Severity.FAIL, "Empty"); - - assertTrue("Joining two Verdicts shall create a new Verdict", - v1 != v1.join(v2)); - - assertEquals("Joining two Verdicts shall set the severity to the maximum of both", - Verdict.Severity.FAIL.getValue(), v3.join(v2).severity(), DELTA); - - assertEquals("If a Verdict's info contains the info of the Verdict to be joined with, " + - "then only the containing info shall be used", - "Foo Bar", v1.join(v2).info()); - - assertEquals("If a Verdict is OK and its info does not contain the info of the Verdict to be joined with, " + - "then the containing info shall be discarded", - "Baz", v1.join(v3).info()); - - assertEquals("If a Verdict is not OK and its info does not contain the info of the Verdict to be joined with, " + - "then both infos shall be included separated by a line break", - "Bar\nBaz", v2.join(v3).info()); - - assertTrue("Joining an OK and Fail Verdicts shall use the Visualizer of the Verdict with high severity", - v2.join(v1).visualizer() == failVisualizer); - - assertTrue("Joining an OK and Fail Verdicts shall use the Visualizer of the Verdict with high severity", - v1.join(v2).visualizer() == failVisualizer); - - assertTrue("Joining Fail and Issue Verdicts must contain Fail Shapes", - v2.join(v4).visualizer().getShapes().containsAll(failVisualizer.getShapes())); - - assertTrue("Joining Issue and Fail Verdicts must contain Fail Shapes", - v4.join(v2).visualizer().getShapes().containsAll(failVisualizer.getShapes())); - - assertTrue("Joining Fail and Issue Verdicts must contain Issue Shapes", - v2.join(v4).visualizer().getShapes().containsAll(issueVisualizer.getShapes())); - - assertTrue("Joining Issue and Fail Verdicts must contain Issue Shapes", - v4.join(v2).visualizer().getShapes().containsAll(issueVisualizer.getShapes())); - - assertTrue("Joining Fail and emptyVisualizerVerdict Verdicts must equal Fail visualizer", - v2.join(emptyVisualizerVerdict).visualizer() == failVisualizer); + public void testVerdictHelperAreOK() { + Verdict ok = Verdict.OK; + Verdict warn = new Verdict(Verdict.Severity.SUSPICIOUS_TAG, "Issue", failVisualizer); - assertTrue("Joining emptyVisualizerVerdict and Fail Verdicts must equal Fail visualizer", - emptyVisualizerVerdict.join(v2).visualizer() == failVisualizer); + assertTrue(Verdict.helperAreAllVerdictsOK(Arrays.asList(ok))); + assertFalse(Verdict.helperAreAllVerdictsOK(Arrays.asList(warn))); } @Test diff --git a/statemodel/src/org/testar/statemodel/persistence/orientdb/hydrator/ConcreteStateHydrator.java b/statemodel/src/org/testar/statemodel/persistence/orientdb/hydrator/ConcreteStateHydrator.java index 78dbdd667..344079dd9 100644 --- a/statemodel/src/org/testar/statemodel/persistence/orientdb/hydrator/ConcreteStateHydrator.java +++ b/statemodel/src/org/testar/statemodel/persistence/orientdb/hydrator/ConcreteStateHydrator.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2018 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2018 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2018 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -39,6 +39,10 @@ import org.testar.statemodel.persistence.orientdb.entity.TypeConvertor; import org.testar.statemodel.persistence.orientdb.entity.VertexEntity; import org.testar.statemodel.persistence.orientdb.util.Validation; + +import java.util.Collections; +import java.util.List; + import org.testar.monkey.alayer.Tag; import org.testar.monkey.alayer.TaggableBase; import org.testar.monkey.alayer.Tags; @@ -89,18 +93,13 @@ public void hydrate(VertexEntity target, Object source) throws HydrationExceptio } // get the oracle verdict and transform it into a custom code - Verdict verdict = attributes.get(Tags.OracleVerdict, null); + List verdicts = attributes.get(Tags.OracleVerdicts, Collections.singletonList(Verdict.OK)); int oracleVerdictCode = 4; // default value - if (verdict != null) { - if (verdict.severity() == Verdict.Severity.OK.getValue()) { - oracleVerdictCode = 1; - } - else if (verdict.severity() == Verdict.Severity.FAIL.getValue()) { - oracleVerdictCode = 2; - } - else if (verdict.severity() > Verdict.Severity.OK.getValue() && verdict.severity() < Verdict.Severity.FAIL.getValue()) { - oracleVerdictCode = 3; - } + if (Verdict.helperAreAllVerdictsOK(verdicts)) { + oracleVerdictCode = 1; + } + else { + oracleVerdictCode = 2; } target.addPropertyValue("oracleVerdictCode", new PropertyValue(OType.INTEGER, oracleVerdictCode)); } diff --git a/testar/resources/settings/01_desktop_calculator/Protocol_01_desktop_calculator.java b/testar/resources/settings/01_desktop_calculator/Protocol_01_desktop_calculator.java index cf98ad152..79a83518d 100644 --- a/testar/resources/settings/01_desktop_calculator/Protocol_01_desktop_calculator.java +++ b/testar/resources/settings/01_desktop_calculator/Protocol_01_desktop_calculator.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2020 - 2023 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2020 - 2023 Open Universiteit - www.ou.nl + * Copyright (c) 2020 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2020 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -41,6 +41,7 @@ import org.testar.protocols.DesktopProtocol; import org.testar.settings.Settings; +import java.util.List; import java.util.Set; /** @@ -118,18 +119,18 @@ protected State getState(SUT system) throws StateBuildException{ * @return oracle verdict, which determines whether the state is erroneous and why. */ @Override - protected Verdict getVerdict(State state){ + protected List getVerdicts(State state){ // The super methods implements the implicit online state oracles for: // system crashes // non-responsiveness // suspicious tags - Verdict verdict = super.getVerdict(state); + List verdicts = super.getVerdicts(state); //-------------------------------------------------------- // MORE SOPHISTICATED STATE ORACLES CAN BE PROGRAMMED HERE //-------------------------------------------------------- - return verdict; + return verdicts; } /** diff --git a/testar/resources/settings/02_webdriver_parabank/Protocol_02_webdriver_parabank.java b/testar/resources/settings/02_webdriver_parabank/Protocol_02_webdriver_parabank.java index cbbdd388c..cf0cd23cc 100644 --- a/testar/resources/settings/02_webdriver_parabank/Protocol_02_webdriver_parabank.java +++ b/testar/resources/settings/02_webdriver_parabank/Protocol_02_webdriver_parabank.java @@ -41,10 +41,7 @@ import org.testar.monkey.alayer.webdriver.WdDriver; import org.testar.monkey.alayer.webdriver.enums.WdRoles; import org.testar.monkey.alayer.webdriver.enums.WdTags; -import org.testar.oracles.Oracle; -import org.testar.oracles.OracleSelection; import org.testar.plugin.NativeLinker; -import org.testar.monkey.ConfigTags; import org.testar.monkey.Pair; import org.testar.protocols.WebdriverProtocol; import org.testar.settings.Settings; @@ -59,11 +56,6 @@ public class Protocol_02_webdriver_parabank extends WebdriverProtocol { - // This list tracks the detected erroneous verdicts to avoid duplicates - private List listOfDetectedErroneousVerdicts = new ArrayList<>(); - - private List extendedOraclesList = new ArrayList<>(); - /** * Called once during the life time of TESTAR * This method can be used to perform initial setup work @@ -73,9 +65,6 @@ public class Protocol_02_webdriver_parabank extends WebdriverProtocol { @Override protected void initialize(Settings settings) { super.initialize(settings); - - // Reset the list when we start a new TESTAR run with multiple sequences - listOfDetectedErroneousVerdicts = new ArrayList<>(); } /** @@ -87,7 +76,6 @@ protected void initialize(Settings settings) { @Override protected void preSequencePreparations() { super.preSequencePreparations(); - extendedOraclesList = OracleSelection.loadExtendedOracles(settings.get(ConfigTags.ExtendedOracles)); } /** @@ -172,37 +160,10 @@ protected State getState(SUT system) throws StateBuildException { * @return oracle verdict, which determines whether the state is erroneous and why. */ @Override - protected Verdict getVerdict(State state) { - + protected List getVerdicts(State state) { // System crashes, non-responsiveness and suspicious tags automatically detected! // For web applications, web browser errors and warnings can also be enabled via settings - Verdict verdict = super.getVerdict(state); - - // If the Verdict is not OK but was already detected in a previous sequence - // Consider as OK to avoid duplicates and continue testing - if (verdict != Verdict.OK && containsVerdictInfo(listOfDetectedErroneousVerdicts, verdict.info())) { - // Consider as OK to continue testing - verdict = Verdict.OK; - webConsoleVerdict = Verdict.OK; - } - // If the Verdict is not OK and was not duplicated... - // We found an issue we need to report - else if (verdict.severity() != Verdict.OK.severity()) { - return verdict; - } - - // "ExtendedOracles" offered by TESTAR in the test.settings or Oracles GUI dialog - for (Oracle extendedOracle : extendedOraclesList) { - Verdict extendedVerdict = extendedOracle.getVerdict(state); - - // If the Custom Verdict is not OK and was not detected in a previous sequence - // return verdict with failure state - if (extendedVerdict != Verdict.OK - && !containsVerdictInfo(listOfDetectedErroneousVerdicts, extendedVerdict.info())) { - return extendedVerdict; - } - - } + List verdicts = super.getVerdicts(state); //----------------------------------------------------------------------------- // MORE SOPHISTICATED ORACLES CAN BE PROGRAMMED HERE (the sky is the limit ;-) @@ -210,24 +171,14 @@ else if (verdict.severity() != Verdict.OK.severity()) { // ... YOU MAY WANT TO CHECK YOUR CUSTOM ORACLES HERE ... - Verdict leafWidgetsOverlappingVerdict = leafWidgetsOverlapping(state); - - // If the Custom Verdict is not OK but was already detected in a previous sequence - // Consider as OK to avoid duplicates - if (leafWidgetsOverlappingVerdict != Verdict.OK - && !containsVerdictInfo(listOfDetectedErroneousVerdicts, leafWidgetsOverlappingVerdict.info())) { - return leafWidgetsOverlappingVerdict; - } + List overlapVerdicts = leafWidgetsOverlapping(state); + verdicts.addAll(overlapVerdicts); - return Verdict.OK; + return verdicts; } - private boolean containsVerdictInfo(List listOfDetectedErroneousVerdicts, String currentVerdictInfo) { - return listOfDetectedErroneousVerdicts.stream().anyMatch(verdictInfo -> verdictInfo.contains(currentVerdictInfo.replace("\n", " "))); - } - - public Verdict leafWidgetsOverlapping(State state) { - Verdict finalVerdict = Verdict.OK; + public List leafWidgetsOverlapping(State state) { + List verdicts = new ArrayList<>(); // Prepare a list that contains all the Rectangles from the leaf widgets List> leafWidgetsRects = new ArrayList<>(); @@ -265,12 +216,12 @@ public Verdict leafWidgetsOverlapping(State state) { Verdict.Severity.WARNING_UI_VISUAL_OR_RENDERING_FAULT, verdictMsg, visualizer); - finalVerdict = finalVerdict.join(clashVerdict); + verdicts.add(clashVerdict); } } } - return finalVerdict; + return verdicts; } private Pen getRedPen() { @@ -550,12 +501,6 @@ protected boolean moreActions(State state) { @Override protected void finishSequence() { super.finishSequence(); - // If the final Verdict is not OK and the verdict is not saved in the list - // This is a new run fail verdict - Verdict finalVerdict = getVerdict(latestState); - if(finalVerdict.severity() > Verdict.Severity.OK.getValue() && !listOfDetectedErroneousVerdicts.contains(finalVerdict.info().replace("\n", " "))) { - listOfDetectedErroneousVerdicts.add(finalVerdict.info().replace("\n", " ")); - } } /** diff --git a/testar/resources/settings/02_webdriver_parabank/test.settings b/testar/resources/settings/02_webdriver_parabank/test.settings index eef0f5f20..cd551e260 100644 --- a/testar/resources/settings/02_webdriver_parabank/test.settings +++ b/testar/resources/settings/02_webdriver_parabank/test.settings @@ -38,6 +38,12 @@ SUTConnectorValue = "https://para.testar.org/" Sequences = 5 SequenceLength = 20 +################################################################# +# Ignore reporting duplicated verdicts for this protocol +################################################################# + +IgnoreDuplicatedVerdicts = true + ################################################################# # Oracles based on suspicious tag values # @@ -191,9 +197,9 @@ SwitchNewTabs = true # WebConsoleWarningPattern: Regular expressions ORACLE to find suspicious messages in the browser warning console ################################################################# -WebConsoleErrorOracle = false +WebConsoleErrorOracle = true WebConsoleErrorPattern = .*.* -WebConsoleWarningOracle = false +WebConsoleWarningOracle = true WebConsoleWarningPattern = .*.* ################################################################# diff --git a/testar/resources/settings/03_webdriver_llm_parabank/Protocol_03_webdriver_llm_parabank.java b/testar/resources/settings/03_webdriver_llm_parabank/Protocol_03_webdriver_llm_parabank.java index 3b5871b8e..c34fe87c8 100644 --- a/testar/resources/settings/03_webdriver_llm_parabank/Protocol_03_webdriver_llm_parabank.java +++ b/testar/resources/settings/03_webdriver_llm_parabank/Protocol_03_webdriver_llm_parabank.java @@ -1,6 +1,6 @@ /** - * Copyright (c) 2018 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2019 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2019 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -161,31 +161,29 @@ protected State getState(SUT system) throws StateBuildException { * @return oracle verdict, which determines whether the state is erroneous and why. */ @Override - protected Verdict getVerdict(State state) { - // System crashes, non-responsiveness and suspicious tags automatically detected! - // For web applications, web browser errors and warnings can also be enabled via settings - Verdict verdict = super.getVerdict(state); - + protected List getVerdicts(State state) { // Use the LLM as an Oracle to determine if the test goal has been completed - Verdict llmVerdict = llmOracle.getVerdict(state); - - if(llmVerdict.severity() == Verdict.Severity.LLM_COMPLETE.getValue()) { - // Test goal was completed, retrieve next test goal from queue. - currentTestGoal = testGoalQueue.poll(); - - // Poll returns null if there are no more items remaining in the queue. - if(currentTestGoal == null) { - // No more test goals remaining, terminate sequence. - System.out.println("Test goal completed, but no more test goals."); - return llmVerdict; - } else { - System.out.println("Test goal completed, moving to next test goal."); - llmActionSelector.reset(currentTestGoal, true); - llmOracle.reset(currentTestGoal, true); + List llmVerdicts = llmOracle.getVerdicts(state); + + for(Verdict llmVerdict : llmVerdicts) { + if(llmVerdict.severity() == Verdict.Severity.LLM_COMPLETE.getValue()) { + // Test goal was completed, retrieve next test goal from queue. + currentTestGoal = testGoalQueue.poll(); + + // Poll returns null if there are no more items remaining in the queue. + if(currentTestGoal == null) { + // No more test goals remaining, terminate sequence. + System.out.println("Test goal completed, but no more test goals."); + return Collections.singletonList(llmVerdict); + } else { + System.out.println("Test goal completed, moving to next test goal."); + llmActionSelector.reset(currentTestGoal, true); + llmOracle.reset(currentTestGoal, true); + } } } - return verdict; + return super.getVerdicts(state); } /** diff --git a/testar/resources/settings/android_generic/Protocol_android_generic.java b/testar/resources/settings/android_generic/Protocol_android_generic.java index 2a442171b..9c5d2c6fb 100644 --- a/testar/resources/settings/android_generic/Protocol_android_generic.java +++ b/testar/resources/settings/android_generic/Protocol_android_generic.java @@ -48,6 +48,7 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.List; import java.util.Set; public class Protocol_android_generic extends AndroidProtocol { @@ -156,18 +157,18 @@ protected State getState(SUT system) throws StateBuildException { * @return oracle verdict, which determines whether the state is erroneous and why. */ @Override - protected Verdict getVerdict(State state){ + protected List getVerdicts(State state){ // The super methods implements the implicit online state oracles for: // system crashes // non-responsiveness // suspicious tags - Verdict verdict = super.getVerdict(state); + List verdicts = super.getVerdicts(state); //-------------------------------------------------------- // MORE SOPHISTICATED STATE ORACLES CAN BE PROGRAMMED HERE //-------------------------------------------------------- - return verdict; + return verdicts; } /** diff --git a/testar/resources/settings/desktop_generic/Protocol_desktop_generic.java b/testar/resources/settings/desktop_generic/Protocol_desktop_generic.java index 964e963cf..adc317c13 100644 --- a/testar/resources/settings/desktop_generic/Protocol_desktop_generic.java +++ b/testar/resources/settings/desktop_generic/Protocol_desktop_generic.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2013 - 2024 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2018 - 2024 Open Universiteit - www.ou.nl + * Copyright (c) 2013 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -29,6 +29,7 @@ *******************************************************************************************************/ +import java.util.List; import java.util.Set; import org.testar.DerivedActions; @@ -125,18 +126,18 @@ protected State getState(SUT system) throws StateBuildException { * @return oracle verdict, which determines whether the state is erroneous and why. */ @Override - protected Verdict getVerdict(State state){ + protected List getVerdicts(State state){ // The super methods implements the implicit online state oracles for: // system crashes // non-responsiveness // suspicious tags - Verdict verdict = super.getVerdict(state); + List verdicts = super.getVerdicts(state); //-------------------------------------------------------- // MORE SOPHISTICATED STATE ORACLES CAN BE PROGRAMMED HERE //-------------------------------------------------------- - return verdict; + return verdicts; } /** diff --git a/testar/resources/settings/desktop_java_coverage/Protocol_desktop_java_coverage.java b/testar/resources/settings/desktop_java_coverage/Protocol_desktop_java_coverage.java index 0310d1480..d79c8b7c3 100644 --- a/testar/resources/settings/desktop_java_coverage/Protocol_desktop_java_coverage.java +++ b/testar/resources/settings/desktop_java_coverage/Protocol_desktop_java_coverage.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2024 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2024 Open Universiteit - www.ou.nl + * Copyright (c) 2024 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2024 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -41,6 +41,7 @@ import org.testar.protocols.DesktopProtocol; import org.testar.settings.Settings; +import java.util.List; import java.util.Set; public class Protocol_desktop_java_coverage extends DesktopProtocol { @@ -109,18 +110,18 @@ protected State getState(SUT system) throws StateBuildException{ * @return oracle verdict, which determines whether the state is erroneous and why. */ @Override - protected Verdict getVerdict(State state){ + protected List getVerdicts(State state){ // The super methods implements the implicit online state oracles for: // system crashes // non-responsiveness // suspicious tags - Verdict verdict = super.getVerdict(state); + List verdicts = super.getVerdicts(state); //-------------------------------------------------------- // MORE SOPHISTICATED STATE ORACLES CAN BE PROGRAMMED HERE //-------------------------------------------------------- - return verdict; + return verdicts; } /** diff --git a/testar/resources/settings/webdriver_generic/Protocol_webdriver_generic.java b/testar/resources/settings/webdriver_generic/Protocol_webdriver_generic.java index 8da5b56c1..202a71846 100644 --- a/testar/resources/settings/webdriver_generic/Protocol_webdriver_generic.java +++ b/testar/resources/settings/webdriver_generic/Protocol_webdriver_generic.java @@ -1,6 +1,6 @@ /** - * Copyright (c) 2018 - 2023 Open Universiteit - www.ou.nl - * Copyright (c) 2019 - 2023 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2019 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -129,9 +129,9 @@ protected State getState(SUT system) throws StateBuildException { * @return oracle verdict, which determines whether the state is erroneous and why. */ @Override - protected Verdict getVerdict(State state) { + protected List getVerdicts(State state) { - Verdict verdict = super.getVerdict(state); + List verdicts = super.getVerdicts(state); // system crashes, non-responsiveness and suspicious tags automatically detected! //----------------------------------------------------------------------------- @@ -140,7 +140,7 @@ protected Verdict getVerdict(State state) { // ... YOU MAY WANT TO CHECK YOUR CUSTOM ORACLES HERE ... - return verdict; + return verdicts; } /** diff --git a/testar/resources/settings/webdriver_llm_state_model_evaluator/Protocol_webdriver_llm_state_model_evaluator.java b/testar/resources/settings/webdriver_llm_state_model_evaluator/Protocol_webdriver_llm_state_model_evaluator.java index 2956a69f3..5f6373116 100644 --- a/testar/resources/settings/webdriver_llm_state_model_evaluator/Protocol_webdriver_llm_state_model_evaluator.java +++ b/testar/resources/settings/webdriver_llm_state_model_evaluator/Protocol_webdriver_llm_state_model_evaluator.java @@ -1,6 +1,6 @@ /** - * Copyright (c) 2024 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2024 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2024 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2024 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -191,11 +191,7 @@ protected State getState(SUT system) throws StateBuildException { * @return oracle verdict, which determines whether the state is erroneous and why. */ @Override - protected Verdict getVerdict(State state) { - // System crashes, non-responsiveness and suspicious tags automatically detected! - // For web applications, web browser errors and warnings can also be enabled via settings - Verdict verdict = super.getVerdict(state); - + protected List getVerdicts(State state) { String modelIdentifier = stateModelManager.getModelIdentifier(); if(conditionEvaluator.evaluateConditions(modelIdentifier, stateModelManager)) { @@ -206,7 +202,7 @@ protected Verdict getVerdict(State state) { if(currentTestGoal == null) { // No more test goals remaining, terminate sequence. System.out.println("Test goal completed, but no more test goals."); - return new Verdict(Verdict.Severity.CONDITION_COMPLETE, "All test goal conditions completed."); + return Collections.singletonList(new Verdict(Verdict.Severity.CONDITION_COMPLETE, "All test goal conditions completed.")); } else { System.out.println("Test goal completed, moving to next test goal."); llmActionSelector.reset(currentTestGoal, true); @@ -215,7 +211,7 @@ protected Verdict getVerdict(State state) { } } - return verdict; + return super.getVerdicts(state); } /** diff --git a/testar/resources/settings/webdriver_llm_state_model_transition_evaluator/Protocol_webdriver_llm_state_model_transition_evaluator.java b/testar/resources/settings/webdriver_llm_state_model_transition_evaluator/Protocol_webdriver_llm_state_model_transition_evaluator.java index 9c8a7c664..ec24efdcb 100644 --- a/testar/resources/settings/webdriver_llm_state_model_transition_evaluator/Protocol_webdriver_llm_state_model_transition_evaluator.java +++ b/testar/resources/settings/webdriver_llm_state_model_transition_evaluator/Protocol_webdriver_llm_state_model_transition_evaluator.java @@ -1,6 +1,6 @@ /** - * Copyright (c) 2024 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2024 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2024 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2024 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -196,11 +196,7 @@ protected State getState(SUT system) throws StateBuildException { * @return oracle verdict, which determines whether the state is erroneous and why. */ @Override - protected Verdict getVerdict(State state) { - // System crashes, non-responsiveness and suspicious tags automatically detected! - // For web applications, web browser errors and warnings can also be enabled via settings - Verdict verdict = super.getVerdict(state); - + protected List getVerdicts(State state) { String modelIdentifier = stateModelManager.getModelIdentifier(); if(conditionEvaluator.evaluateConditions(modelIdentifier, stateModelManager)) { @@ -211,7 +207,7 @@ protected Verdict getVerdict(State state) { if(currentTestGoal == null) { // No more test goals remaining, terminate sequence. System.out.println("Test goal completed, but no more test goals."); - return new Verdict(Verdict.Severity.CONDITION_COMPLETE, "All test goal conditions completed."); + return Collections.singletonList(new Verdict(Verdict.Severity.CONDITION_COMPLETE, "All test goal conditions completed.")); } else { System.out.println("Test goal completed, moving to next test goal."); llmActionSelector.reset(currentTestGoal, true); @@ -220,7 +216,7 @@ protected Verdict getVerdict(State state) { } } - return verdict; + return super.getVerdicts(state); } /** diff --git a/testar/resources/settings/webdriver_llm_state_widgets_evaluator/Protocol_webdriver_llm_state_widgets_evaluator.java b/testar/resources/settings/webdriver_llm_state_widgets_evaluator/Protocol_webdriver_llm_state_widgets_evaluator.java index 20d6d8414..bdf51b222 100644 --- a/testar/resources/settings/webdriver_llm_state_widgets_evaluator/Protocol_webdriver_llm_state_widgets_evaluator.java +++ b/testar/resources/settings/webdriver_llm_state_widgets_evaluator/Protocol_webdriver_llm_state_widgets_evaluator.java @@ -1,6 +1,6 @@ /** - * Copyright (c) 2024 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2024 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2024 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2024 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -175,11 +175,7 @@ protected State getState(SUT system) throws StateBuildException { * @return oracle verdict, which determines whether the state is erroneous and why. */ @Override - protected Verdict getVerdict(State state) { - // System crashes, non-responsiveness and suspicious tags automatically detected! - // For web applications, web browser errors and warnings can also be enabled via settings - Verdict verdict = super.getVerdict(state); - + protected List getVerdicts(State state) { // Apply state conditions to check if the test goal has been completed if(conditionEvaluator.evaluateConditions(state)) { // Test goal was completed, retrieve next test goal from queue. @@ -188,7 +184,7 @@ protected Verdict getVerdict(State state) { // Poll returns null if there are no more items remaining in the queue. if(currentTestGoal == null) { // No more test goals remaining, terminate sequence. - return new Verdict(Verdict.Severity.CONDITION_COMPLETE, "All test goal conditions completed."); + return Collections.singletonList(new Verdict(Verdict.Severity.CONDITION_COMPLETE, "All test goal conditions completed.")); } else { llmActionSelector.reset(currentTestGoal, true); conditionEvaluator.clear(); @@ -197,7 +193,7 @@ protected Verdict getVerdict(State state) { } } - return verdict; + return super.getVerdicts(state); } /** diff --git a/testar/resources/settings/webdriver_remote_webcomponent/Protocol_webdriver_remote_webcomponent.java b/testar/resources/settings/webdriver_remote_webcomponent/Protocol_webdriver_remote_webcomponent.java index 4cf1a05c3..10c5d7ac0 100644 --- a/testar/resources/settings/webdriver_remote_webcomponent/Protocol_webdriver_remote_webcomponent.java +++ b/testar/resources/settings/webdriver_remote_webcomponent/Protocol_webdriver_remote_webcomponent.java @@ -1,6 +1,6 @@ /** - * Copyright (c) 2018 - 2023 Open Universiteit - www.ou.nl - * Copyright (c) 2019 - 2023 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2019 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -134,9 +134,9 @@ protected State getState(SUT system) throws StateBuildException { * @return oracle verdict, which determines whether the state is erroneous and why. */ @Override - protected Verdict getVerdict(State state) { + protected List getVerdicts(State state) { - Verdict verdict = super.getVerdict(state); + List verdicts = super.getVerdicts(state); // system crashes, non-responsiveness and suspicious titles automatically detected! //----------------------------------------------------------------------------- @@ -145,7 +145,7 @@ protected Verdict getVerdict(State state) { // ... YOU MAY WANT TO CHECK YOUR CUSTOM ORACLES HERE ... - return verdict; + return verdicts; } /** diff --git a/testar/resources/settings/webdriver_security_analysis/Protocol_webdriver_security_analysis.java b/testar/resources/settings/webdriver_security_analysis/Protocol_webdriver_security_analysis.java index db842800f..1ba87f6c8 100644 --- a/testar/resources/settings/webdriver_security_analysis/Protocol_webdriver_security_analysis.java +++ b/testar/resources/settings/webdriver_security_analysis/Protocol_webdriver_security_analysis.java @@ -1,6 +1,6 @@ /** - * Copyright (c) 2018 - 2024 Open Universiteit - www.ou.nl - * Copyright (c) 2018 - 2024 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2018 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -141,11 +141,13 @@ protected void beginSequence(SUT system, State state) { } @Override - protected Verdict getVerdict(State state) { + protected List getVerdicts(State state) { securityResultWriter.WriteVisit(WdDriver.getCurrentUrl()); - Verdict verdict = Verdict.OK; - oracleOrchestrator.getVerdict(verdict); - return verdict; + List verdicts = oracleOrchestrator.getVerdicts(); + if (verdicts == null || verdicts.isEmpty()) { + return Collections.singletonList(Verdict.OK); + } + return verdicts; } @Override diff --git a/testar/resources/workflow/settings/test_gradle_workflow_android_generic/Protocol_test_gradle_workflow_android_generic.java b/testar/resources/workflow/settings/test_gradle_workflow_android_generic/Protocol_test_gradle_workflow_android_generic.java index d468a12b2..458f1a1b1 100644 --- a/testar/resources/workflow/settings/test_gradle_workflow_android_generic/Protocol_test_gradle_workflow_android_generic.java +++ b/testar/resources/workflow/settings/test_gradle_workflow_android_generic/Protocol_test_gradle_workflow_android_generic.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2020 - 2025 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2020 - 2025 Open Universiteit - www.ou.nl + * Copyright (c) 2020 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2020 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -50,6 +50,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; +import java.util.List; import java.util.Scanner; import java.util.Set; @@ -139,19 +140,24 @@ protected void postSequenceProcessing() { // Verify html and txt report files were created Assert.isTrue(reportManager instanceof ReportManager); - File htmlReportFile = new File(((ReportManager)reportManager).getReportFileName().concat("_" + getFinalVerdict().verdictSeverityTitle() + ".html")); - File txtReportFile = new File(((ReportManager)reportManager).getReportFileName().concat("_" + getFinalVerdict().verdictSeverityTitle() + ".txt")); - System.out.println("htmlReportFile: " + htmlReportFile.getPath()); - System.out.println("txtReportFile: " + txtReportFile.getPath()); - Assert.isTrue(htmlReportFile.exists()); - Assert.isTrue(txtReportFile.exists()); - - // Verify report information - Assert.isTrue(fileContains("

TESTAR execution sequence report for sequence 1

", htmlReportFile)); - Assert.isTrue(fileContains("TESTAR execution sequence report for sequence 1", txtReportFile)); - - Assert.isTrue(fileContains("

Test verdict for this sequence:", htmlReportFile)); - Assert.isTrue(fileContains("Test verdict for this sequence:", txtReportFile)); + Assert.isTrue(getSequenceVerdicts().size() > 0); + int index = 1; + for (Verdict verdict : getSequenceVerdicts()) { + String suffixName = String.format("_V%03d_%s", index++, verdict.verdictSeverityTitle()); + File htmlReportFile = new File(((ReportManager)reportManager).getReportFileName().concat(suffixName + ".html")); + File txtReportFile = new File(((ReportManager)reportManager).getReportFileName().concat(suffixName + ".txt")); + System.out.println("htmlReportFile: " + htmlReportFile.getPath()); + System.out.println("txtReportFile: " + txtReportFile.getPath()); + Assert.isTrue(htmlReportFile.exists()); + Assert.isTrue(txtReportFile.exists()); + + // Verify report information + Assert.isTrue(fileContains("

TESTAR execution sequence report for sequence 1

", htmlReportFile)); + Assert.isTrue(fileContains("TESTAR execution sequence report for sequence 1", txtReportFile)); + + Assert.isTrue(fileContains("

Test verdict for this sequence:", htmlReportFile)); + Assert.isTrue(fileContains("Test verdict for this sequence:", txtReportFile)); + } } private boolean folderIsEmpty(Path path) { diff --git a/testar/resources/workflow/settings/test_gradle_workflow_desktop_generic/Protocol_test_gradle_workflow_desktop_generic.java b/testar/resources/workflow/settings/test_gradle_workflow_desktop_generic/Protocol_test_gradle_workflow_desktop_generic.java index 3a3e9c0ee..21313f4d7 100644 --- a/testar/resources/workflow/settings/test_gradle_workflow_desktop_generic/Protocol_test_gradle_workflow_desktop_generic.java +++ b/testar/resources/workflow/settings/test_gradle_workflow_desktop_generic/Protocol_test_gradle_workflow_desktop_generic.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2020 - 2023 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2020 - 2023 Open Universiteit - www.ou.nl + * Copyright (c) 2020 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2020 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -33,6 +33,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.Arrays; +import java.util.List; import java.util.Scanner; import java.util.Set; @@ -121,7 +122,7 @@ protected void postSequenceProcessing() { super.postSequenceProcessing(); // If OnlySaveFaultySequences is enabled and the sequence verdict is OK, sequence_ok must not exist - if(settings().get(ConfigTags.OnlySaveFaultySequences) && (getFinalVerdict()).severity() == Verdict.OK.severity()) { + if(settings().get(ConfigTags.OnlySaveFaultySequences) && Verdict.helperAreAllVerdictsOK(getSequenceVerdicts())) { String sequencesOkFolderName = OutputStructure.outerLoopOutputDir + File.separator + "sequences_ok"; File sequencesOkFolder = null; try { @@ -135,8 +136,13 @@ protected void postSequenceProcessing() { // Or if OnlySaveFaultySequences disabled, // sequence must have generated a .testar file else { - Assert.isTrue(getGeneratedSequenceName().endsWith(".testar")); - Assert.isTrue(new File(getGeneratedSequenceName()).exists()); + Assert.isTrue(getSequenceVerdicts().size() > 0); + for (Verdict verdict : getSequenceVerdicts()) { + String sequencesFolderName = OutputStructure.outerLoopOutputDir + File.separator + "sequences_" + verdict.verdictSeverityTitle().toLowerCase(); + File sequencesFolder = new File(sequencesFolderName); + File[] matchingFiles = sequencesFolder.listFiles((dir, name) -> name.contains("sequence_1") && name.endsWith(".testar")); + Assert.isTrue(matchingFiles != null && matchingFiles.length > 0); + } } // Verify the JsonUtils created a JSON State file @@ -159,19 +165,24 @@ protected void postSequenceProcessing() { // Verify html and txt report files were created Assert.isTrue(reportManager instanceof ReportManager); - File htmlReportFile = new File(((ReportManager)reportManager).getReportFileName().concat("_" + getFinalVerdict().verdictSeverityTitle() + ".html")); - File txtReportFile = new File(((ReportManager)reportManager).getReportFileName().concat("_" + getFinalVerdict().verdictSeverityTitle() + ".txt")); - System.out.println("htmlReportFile: " + htmlReportFile.getPath()); - System.out.println("txtReportFile: " + txtReportFile.getPath()); - Assert.isTrue(htmlReportFile.exists()); - Assert.isTrue(txtReportFile.exists()); - - // Verify report information - Assert.isTrue(fileContains("

TESTAR execution sequence report for sequence 1

", htmlReportFile)); - Assert.isTrue(fileContains("TESTAR execution sequence report for sequence 1", txtReportFile)); - - Assert.isTrue(fileContains("

Test verdict for this sequence:", htmlReportFile)); - Assert.isTrue(fileContains("Test verdict for this sequence:", txtReportFile)); + Assert.isTrue(getSequenceVerdicts().size() > 0); + int index = 1; + for (Verdict verdict : getSequenceVerdicts()) { + String suffixName = String.format("_V%03d_%s", index++, verdict.verdictSeverityTitle()); + File htmlReportFile = new File(((ReportManager)reportManager).getReportFileName().concat(suffixName + ".html")); + File txtReportFile = new File(((ReportManager)reportManager).getReportFileName().concat(suffixName + ".txt")); + System.out.println("htmlReportFile: " + htmlReportFile.getPath()); + System.out.println("txtReportFile: " + txtReportFile.getPath()); + Assert.isTrue(htmlReportFile.exists()); + Assert.isTrue(txtReportFile.exists()); + + // Verify report information + Assert.isTrue(fileContains("

TESTAR execution sequence report for sequence 1

", htmlReportFile)); + Assert.isTrue(fileContains("TESTAR execution sequence report for sequence 1", txtReportFile)); + + Assert.isTrue(fileContains("

Test verdict for this sequence:", htmlReportFile)); + Assert.isTrue(fileContains("Test verdict for this sequence:", txtReportFile)); + } } private boolean fileContains(String searchText, File file) { diff --git a/testar/resources/workflow/settings/test_gradle_workflow_webdriver_form_filling/Protocol_test_gradle_workflow_webdriver_form_filling.java b/testar/resources/workflow/settings/test_gradle_workflow_webdriver_form_filling/Protocol_test_gradle_workflow_webdriver_form_filling.java index 36af47369..2dac5d028 100644 --- a/testar/resources/workflow/settings/test_gradle_workflow_webdriver_form_filling/Protocol_test_gradle_workflow_webdriver_form_filling.java +++ b/testar/resources/workflow/settings/test_gradle_workflow_webdriver_form_filling/Protocol_test_gradle_workflow_webdriver_form_filling.java @@ -1,6 +1,6 @@ /** - * Copyright (c) 2021 - 2023 Open Universiteit - www.ou.nl - * Copyright (c) 2021 - 2023 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2021 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2021 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -49,7 +49,6 @@ import java.io.IOException; import java.io.ObjectInputStream; import java.util.*; -import java.util.zip.GZIPInputStream; /** * This protocol is used to test TESTAR by executing a gradle CI workflow. @@ -59,9 +58,9 @@ public class Protocol_test_gradle_workflow_webdriver_form_filling extends WebdriverProtocol { @Override - protected Verdict getVerdict(State state) { + protected List getVerdicts(State state) { // For custom CI testing purposes, force these generated sequences be OK - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } @Override @@ -126,7 +125,7 @@ protected void closeTestSession() { String sequencesOkFolderName = OutputStructure.outerLoopOutputDir + File.separator + "sequences_ok"; File sequencesOkFolder = new File(sequencesOkFolderName).getCanonicalFile(); System.out.println("sequencesFolder: " + sequencesOkFolder); - File[] matchingFiles = sequencesOkFolder.listFiles((dir, name) -> name.endsWith("sequence_1.testar")); + File[] matchingFiles = sequencesOkFolder.listFiles((dir, name) -> name.contains("sequence_1") && name.endsWith(".testar")); Assert.isTrue(matchingFiles.length == 1, "One replayable testar file was not created"); System.out.println("matchingFiles[0]: " + matchingFiles[0]); Assert.isTrue(isValidReplayFile(matchingFiles[0]), "Replayable testar file was not serialized correctly!"); @@ -147,8 +146,7 @@ private boolean isValidReplayFile(File replayFile){ try { FileInputStream fis = new FileInputStream(replayFile); BufferedInputStream bis = new BufferedInputStream(fis); - GZIPInputStream gis = new GZIPInputStream(bis); - ObjectInputStream ois = new ObjectInputStream(gis); + ObjectInputStream ois = new ObjectInputStream(bis); ois.readObject(); ois.close(); diff --git a/testar/resources/workflow/settings/test_gradle_workflow_webdriver_form_filling/test.settings b/testar/resources/workflow/settings/test_gradle_workflow_webdriver_form_filling/test.settings index 49daebddd..f2218004d 100644 --- a/testar/resources/workflow/settings/test_gradle_workflow_webdriver_form_filling/test.settings +++ b/testar/resources/workflow/settings/test_gradle_workflow_webdriver_form_filling/test.settings @@ -164,7 +164,7 @@ LogLevel = 1 FaultThreshold = 1.0E-9 MyClassPath = ./settings OnlySaveFaultySequences = false -ActionDuration = 0.1 +ActionDuration = 1 ShowVisualSettingsDialogOnStartup = true ReplayRetryTime = 30.0 Delete = diff --git a/testar/resources/workflow/settings/test_gradle_workflow_webdriver_generic/Protocol_test_gradle_workflow_webdriver_generic.java b/testar/resources/workflow/settings/test_gradle_workflow_webdriver_generic/Protocol_test_gradle_workflow_webdriver_generic.java index e55dc03e2..6c777ef2d 100644 --- a/testar/resources/workflow/settings/test_gradle_workflow_webdriver_generic/Protocol_test_gradle_workflow_webdriver_generic.java +++ b/testar/resources/workflow/settings/test_gradle_workflow_webdriver_generic/Protocol_test_gradle_workflow_webdriver_generic.java @@ -1,6 +1,6 @@ /** - * Copyright (c) 2021 - 2023 Open Universiteit - www.ou.nl - * Copyright (c) 2021 - 2023 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2021 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2021 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -170,19 +170,24 @@ protected void postSequenceProcessing() { // Verify html and txt report files were created Assert.isTrue(reportManager instanceof ReportManager); - File htmlReportFile = new File(((ReportManager)reportManager).getReportFileName().concat("_" + getFinalVerdict().verdictSeverityTitle() + ".html")); - File txtReportFile = new File(((ReportManager)reportManager).getReportFileName().concat("_" + getFinalVerdict().verdictSeverityTitle() + ".txt")); - System.out.println("htmlReportFile: " + htmlReportFile.getPath()); - System.out.println("txtReportFile: " + txtReportFile.getPath()); - Assert.isTrue(htmlReportFile.exists()); - Assert.isTrue(txtReportFile.exists()); - - // Verify report information - Assert.isTrue(fileContains("

TESTAR execution sequence report for sequence 1

", htmlReportFile)); - Assert.isTrue(fileContains("TESTAR execution sequence report for sequence 1", txtReportFile)); - - Assert.isTrue(fileContains("

Test verdict for this sequence:", htmlReportFile)); - Assert.isTrue(fileContains("Test verdict for this sequence:", txtReportFile)); + Assert.isTrue(getSequenceVerdicts().size() > 0); + int index = 1; + for (Verdict verdict : getSequenceVerdicts()) { + String suffixName = String.format("_V%03d_%s", index++, verdict.verdictSeverityTitle()); + File htmlReportFile = new File(((ReportManager)reportManager).getReportFileName().concat(suffixName + ".html")); + File txtReportFile = new File(((ReportManager)reportManager).getReportFileName().concat(suffixName + ".txt")); + System.out.println("htmlReportFile: " + htmlReportFile.getPath()); + System.out.println("txtReportFile: " + txtReportFile.getPath()); + Assert.isTrue(htmlReportFile.exists()); + Assert.isTrue(txtReportFile.exists()); + + // Verify report information + Assert.isTrue(fileContains("

TESTAR execution sequence report for sequence 1

", htmlReportFile)); + Assert.isTrue(fileContains("TESTAR execution sequence report for sequence 1", txtReportFile)); + + Assert.isTrue(fileContains("

Test verdict for this sequence:", htmlReportFile)); + Assert.isTrue(fileContains("Test verdict for this sequence:", txtReportFile)); + } } private boolean fileContains(String searchText, File file) { diff --git a/testar/resources/workflow/settings/test_gradle_workflow_webdriver_replay/Protocol_test_gradle_workflow_webdriver_replay.java b/testar/resources/workflow/settings/test_gradle_workflow_webdriver_replay/Protocol_test_gradle_workflow_webdriver_replay.java index 983536998..975a5111b 100644 --- a/testar/resources/workflow/settings/test_gradle_workflow_webdriver_replay/Protocol_test_gradle_workflow_webdriver_replay.java +++ b/testar/resources/workflow/settings/test_gradle_workflow_webdriver_replay/Protocol_test_gradle_workflow_webdriver_replay.java @@ -1,6 +1,6 @@ /** - * Copyright (c) 2021 - 2023 Open Universiteit - www.ou.nl - * Copyright (c) 2021 - 2023 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2021 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2021 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -131,19 +131,25 @@ protected void postSequenceProcessing() { // Verify html and txt report files were created Assert.isTrue(reportManager instanceof ReportManager); - File htmlReportFile = new File(((ReportManager)reportManager).getReportFileName().concat("_" + getReplayVerdict().verdictSeverityTitle() + ".html")); - File txtReportFile = new File(((ReportManager)reportManager).getReportFileName().concat("_" + getReplayVerdict().verdictSeverityTitle() + ".txt")); - System.out.println("htmlReportFile: " + htmlReportFile.getPath()); - System.out.println("txtReportFile: " + txtReportFile.getPath()); - Assert.isTrue(htmlReportFile.exists()); - Assert.isTrue(txtReportFile.exists()); - - // Verify report information - Assert.isTrue(fileContains("

TESTAR replay sequence report", htmlReportFile)); - Assert.isTrue(fileContains("TESTAR replay sequence report", txtReportFile)); - - Assert.isTrue(fileContains("

Test verdict for this sequence:", htmlReportFile)); - Assert.isTrue(fileContains("Test verdict for this sequence:", txtReportFile)); + List verdicts = getReplayVerdicts(); + Assert.isTrue(verdicts.size() > 0); + int index = 1; + for (Verdict verdict : verdicts) { + String suffixName = String.format("_V%03d_%s", index++, verdict.verdictSeverityTitle()); + File htmlReportFile = new File(((ReportManager)reportManager).getReportFileName().concat(suffixName + ".html")); + File txtReportFile = new File(((ReportManager)reportManager).getReportFileName().concat(suffixName + ".txt")); + System.out.println("htmlReportFile: " + htmlReportFile.getPath()); + System.out.println("txtReportFile: " + txtReportFile.getPath()); + Assert.isTrue(htmlReportFile.exists()); + Assert.isTrue(txtReportFile.exists()); + + // Verify report information + Assert.isTrue(fileContains("

TESTAR replay sequence report", htmlReportFile)); + Assert.isTrue(fileContains("TESTAR replay sequence report", txtReportFile)); + + Assert.isTrue(fileContains("

Test verdict for this sequence:", htmlReportFile)); + Assert.isTrue(fileContains("Test verdict for this sequence:", txtReportFile)); + } } private boolean fileContains(String searchText, File file) { diff --git a/testar/resources/workflow/settings/test_gradle_workflow_webdriver_replay/paris_parisone.testar b/testar/resources/workflow/settings/test_gradle_workflow_webdriver_replay/paris_parisone.testar index 7f0b0b0bf..e2d1e956f 100644 Binary files a/testar/resources/workflow/settings/test_gradle_workflow_webdriver_replay/paris_parisone.testar and b/testar/resources/workflow/settings/test_gradle_workflow_webdriver_replay/paris_parisone.testar differ diff --git a/testar/src/org/testar/FileHandling.java b/testar/src/org/testar/FileHandling.java index c7c1a3b0d..bf0c7c0fd 100644 --- a/testar/src/org/testar/FileHandling.java +++ b/testar/src/org/testar/FileHandling.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2013 - 2025 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2018 - 2025 Open Universiteit - www.ou.nl + * Copyright (c) 2013 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,15 +32,16 @@ package org.testar; import org.testar.serialisation.LogSerialiser; -import org.testar.monkey.Util; import org.testar.monkey.alayer.Verdict; import org.testar.monkey.alayer.exceptions.NoSuchTagException; import java.io.*; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; public class FileHandling { - public static String copyClassifiedSequence(String generatedSequence, File currentSeq, Verdict verdict) { + public static String copyClassifiedSequence(String generatedSequence, File currentSeq, Verdict verdict, String suffixName) { // Generate the target folder names based on the severity title String targetFolder = "sequences_" + verdict.verdictSeverityTitle().toLowerCase(); String targetSequence = targetFolder + File.separator + generatedSequence; @@ -74,9 +75,9 @@ public static String copyClassifiedSequence(String generatedSequence, File curre // If it exists, copy to the specific classification folder try { - copyToOutputDir(currentSeq, targetFolder); + copyToOutputDirWithSuffix(currentSeq, targetFolder, suffixName); } catch (NoSuchTagException | IOException e) { - LogSerialiser.log("Error copying classified test sequence: " + e.getMessage() + "\n", + LogSerialiser.log("Error copying classified test sequence: " + e.getMessage() + "\n", LogSerialiser.LogLevel.Critical ); } @@ -94,14 +95,24 @@ public static String copyClassifiedSequence(String generatedSequence, File curre * * @param file The sequence file to copy. * @param folderName The target folder name. + * @param suffixName The verdict suffix name (number + title). * @throws IOException If an I/O error occurs. * @throws NoSuchTagException If the specified tag does not exist. */ - private static void copyToOutputDir(File file, String folderName) throws IOException, NoSuchTagException { - Util.copyToDirectory( - file.getCanonicalPath(), - OutputStructure.outerLoopOutputDir + File.separator + folderName, - true - ); + private static void copyToOutputDirWithSuffix(File file, String folderName, String suffixName) throws IOException, NoSuchTagException { + String outputDir = OutputStructure.outerLoopOutputDir + File.separator + folderName; + File targetDir = new File(outputDir); + if (!targetDir.exists()) { + targetDir.mkdirs(); + } + + String fileName = file.getName(); + int dotIndex = fileName.lastIndexOf('.'); + String newName = (dotIndex == -1) + ? fileName + suffixName + : fileName.substring(0, dotIndex) + suffixName + fileName.substring(dotIndex); + + File destination = new File(outputDir + File.separator + newName); + Files.copy(file.toPath(), destination.toPath(), StandardCopyOption.REPLACE_EXISTING); } } diff --git a/testar/src/org/testar/monkey/AbstractProtocol.java b/testar/src/org/testar/monkey/AbstractProtocol.java index 104bc275b..b091174c2 100644 --- a/testar/src/org/testar/monkey/AbstractProtocol.java +++ b/testar/src/org/testar/monkey/AbstractProtocol.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2013 - 2023 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2018 - 2023 Open Universiteit - www.ou.nl + * Copyright (c) 2013 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -40,6 +40,7 @@ import org.testar.settings.Settings; +import java.util.List; import java.util.Set; /** @@ -53,7 +54,7 @@ * BeginSequence (starting "script" on the GUI of the SUT, for example login) * INNER LOOP * GetState - * GetVerdict + * GetVerdicts * StopCriteria (moreActions/moreSequences/time?) * DeriveActions * SelectAction @@ -127,12 +128,12 @@ public abstract class AbstractProtocol implements UnProc { protected abstract State getState(SUT system) throws StateBuildException; /** - * The getVerdict methods implements the online state oracles that - * examine the SUT's current state and returns an oracle verdict. + * The getVerdicts methods implements the online state oracles that + * examine the SUT's current state and returns oracle verdicts. * - * @return oracle verdict, which determines whether the state is erroneous and why. + * @return list of oracle verdicts, which determine whether the state is erroneous and why. */ - protected abstract Verdict getVerdict(State state); + protected abstract List getVerdicts(State state); /** * This method is used by TESTAR to determine the set of currently available actions. diff --git a/testar/src/org/testar/monkey/ConfigTags.java b/testar/src/org/testar/monkey/ConfigTags.java index 00674fdd5..3fff862fc 100644 --- a/testar/src/org/testar/monkey/ConfigTags.java +++ b/testar/src/org/testar/monkey/ConfigTags.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2013 - 2025 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2018 - 2025 Open Universiteit - www.ou.nl + * Copyright (c) 2013 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -56,6 +56,9 @@ private ConfigTags() {} public static final Tag SequenceLength = Tag.from("SequenceLength", Integer.class, "For each test sequence, the number of GUI actions to perform"); + public static final Tag IgnoreDuplicatedVerdicts = Tag.from("IgnoreDuplicatedVerdicts", Boolean.class, + "Sets whether to ignore reporting duplicate verdicts across sequences for the same protocol"); + public static final Tag SuspiciousTags = Tag.from("SuspiciousTags", String.class, "Regular expressions ORACLE to find suspicious messages in the GUI Tags"); @@ -346,8 +349,6 @@ private ConfigTags() {} * Other settings */ - public static final Tag FaultThreshold = Tag.from("FaultThreshold", Double.class); - @SuppressWarnings("unchecked") public static final Tag> Delete = Tag.from("Delete", (Class>) (Class) List.class); diff --git a/testar/src/org/testar/monkey/DefaultProtocol.java b/testar/src/org/testar/monkey/DefaultProtocol.java index 9c5359d1b..113dbb219 100644 --- a/testar/src/org/testar/monkey/DefaultProtocol.java +++ b/testar/src/org/testar/monkey/DefaultProtocol.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2013 - 2025 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2018 - 2025 Open Universiteit - www.ou.nl + * Copyright (c) 2013 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -34,9 +34,6 @@ import org.apache.logging.log4j.Logger; import org.testar.*; import org.testar.managers.NativeHookManager; -import org.testar.monkey.alayer.Action; -import org.testar.monkey.alayer.Canvas; -import org.testar.monkey.alayer.Color; import org.testar.monkey.alayer.*; import org.testar.monkey.alayer.actions.ActivateSystem; import org.testar.monkey.alayer.actions.AnnotatingActionCompiler; @@ -52,6 +49,7 @@ import org.testar.monkey.alayer.webdriver.WdProtocolUtil; import org.testar.monkey.alayer.windows.WinApiException; import org.testar.oracles.Oracle; +import org.testar.oracles.OracleSelection; import org.testar.oracles.log.LogOracle; import org.testar.oracles.log.ProcessListenerOracle; import org.testar.plugin.NativeLinker; @@ -66,14 +64,33 @@ import org.testar.statemodel.StateModelManager; import org.testar.statemodel.StateModelManagerFactory; -import javax.swing.*; -import java.awt.*; -import java.io.*; +import java.awt.Desktop; +import java.awt.GraphicsEnvironment; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; import java.util.List; -import java.util.*; +import java.util.Map; +import java.util.Set; +import java.util.WeakHashMap; +import java.util.StringJoiner; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.zip.GZIPInputStream; + +import javax.swing.JFrame; +import javax.swing.JOptionPane; +import javax.swing.JScrollPane; +import javax.swing.JTextArea; import static org.testar.monkey.alayer.Tags.*; @@ -84,6 +101,9 @@ public class DefaultProtocol extends RuntimeControlsProtocol { protected boolean processListenerOracleEnabled; protected Oracle processListenerOracle; + protected List extendedOraclesList = Collections.emptyList(); + + private VerdictProcessing verdictProcessing; private State stateForClickFilterLayerProtocol; @@ -97,27 +117,29 @@ public void setStateForClickFilterLayerProtocol(State stateForClickFilterLayerPr } String generatedSequence; - public String getGeneratedSequenceName() { - return generatedSequence; - } private File currentSeq; protected Mouse mouse; - private Verdict replayVerdict; + private List replayVerdicts = Collections.singletonList(Verdict.OK); + + public List getReplayVerdicts() { + return replayVerdicts; + } - public Verdict getReplayVerdict() { - return replayVerdict; + public void setReplayVerdicts(List replayVerdicts) { + this.replayVerdicts = replayVerdicts; } - public void setReplayVerdict(Verdict replayVerdict) { - this.replayVerdict = replayVerdict; + private List sequenceVerdicts = Collections.singletonList(Verdict.OK); + + public List getSequenceVerdicts() { + return sequenceVerdicts; } - Verdict finalVerdict = Verdict.OK; - public Verdict getFinalVerdict() { - return finalVerdict; + protected void updateSequenceVerdicts(List stateVerdicts) { + sequenceVerdicts = verdictProcessing.filterDuplicates(stateVerdicts); } protected String lastPrintParentsOf = "null-id"; @@ -239,9 +261,6 @@ public final void run(final Settings settings) { new ReplayMode().runReplayLoop(this); } else if (mode() == Modes.Spy) { new SpyMode().runSpyLoop(this); - } else if(mode() == Modes.Record) { - //new RecordMode().runRecordLoop(this); - System.out.println("Dear User, TESTAR Record mode is disabled temporarily."); } else if (mode() == Modes.Generate) { new GenerateMode().runGenerateOuterLoop(this); } @@ -279,6 +298,7 @@ protected void initialize(Settings settings) { startTime = Util.time(); this.settings = settings; mode = settings.get(ConfigTags.Mode); + verdictProcessing = new VerdictProcessing(settings); builder = NativeLinker.getNativeStateBuilder( settings.get(ConfigTags.TimeToFreeze), @@ -289,7 +309,7 @@ protected void initialize(Settings settings) { logOracleEnabled = settings.get(ConfigTags.LogOracleEnabled, false); processListenerOracleEnabled = settings.get(ConfigTags.ProcessListenerEnabled, false); - if ( mode() == Modes.Generate || /*mode() == Modes.Record ||*/ mode() == Modes.Replay ) { + if ( mode() == Modes.Generate || mode() == Modes.Replay ) { //Create the output folders OutputStructure.calculateOuterLoopDateString(); OutputStructure.sequenceInnerLoopCount = 0; @@ -323,8 +343,7 @@ private boolean isValidFile(){ FileInputStream fis = new FileInputStream(seqFile); BufferedInputStream bis = new BufferedInputStream(fis); - GZIPInputStream gis = new GZIPInputStream(bis); - ObjectInputStream ois = new ObjectInputStream(gis); + ObjectInputStream ois = new ObjectInputStream(bis); ois.readObject(); ois.close(); @@ -563,12 +582,30 @@ void emergencyTerminateTestSequence(SUT system, Exception e){ } } - void classifyAndCopySequenceIntoAppropriateDirectory(Verdict finalVerdict){ - // Check if user wants to save or not the sequences without faults - if (settings.get(ConfigTags.OnlySaveFaultySequences, false) && finalVerdict.severity() == Verdict.OK.severity()) { - LogSerialiser.log("Skipped generated sequence OK (\"" + this.generatedSequence + "\")\n", LogSerialiser.LogLevel.Info); - } else { - this.generatedSequence = FileHandling.copyClassifiedSequence(this.generatedSequence, currentSeq, finalVerdict); + void classifyAndCopySequenceIntoAppropriateDirectory(List verdicts){ + List verdictList = (verdicts == null || verdicts.isEmpty()) + ? Collections.singletonList(Verdict.OK) + : verdicts; + + int index = 1; + String firstTargetSequence = null; + for (Verdict verdict : verdictList) { + if (verdict == null) { + continue; + } + if (settings.get(ConfigTags.OnlySaveFaultySequences, false) + && verdict.severity() == Verdict.OK.severity()) { + LogSerialiser.log("Skipped generated sequence OK (\"" + this.generatedSequence + "\")\n", LogSerialiser.LogLevel.Info); + continue; + } + String suffixName = String.format("_V%03d_%s", index++, verdict.verdictSeverityTitle()); + String targetSequence = FileHandling.copyClassifiedSequence(this.generatedSequence, currentSeq, verdict, suffixName); + if (firstTargetSequence == null) { + firstTargetSequence = targetSequence; + } + } + if (firstTargetSequence != null) { + this.generatedSequence = firstTargetSequence; } } @@ -590,7 +627,7 @@ void saveActionIntoFragmentForReplayableSequence(Action action, State state, Set fragment.set(ActionDuration, settings().get(ConfigTags.ActionDuration)); fragment.set(ActionDelay, settings().get(ConfigTags.TimeToWaitAfterAction)); fragment.set(SystemState, state); - fragment.set(OracleVerdict, getFinalVerdict()); + fragment.set(OracleVerdicts, getSequenceVerdicts()); //Find the target widget of the current action, and save the title into the fragment if (state != null && action.get(Tags.OriginWidget, null) != null){ @@ -669,6 +706,10 @@ protected void preSequencePreparations() { logOracle = createLogOracle(settings); logOracle.initialize(); } + extendedOraclesList = OracleSelection.loadExtendedOracles(settings.get(ConfigTags.ExtendedOracles, "")); + for (Oracle oracle : extendedOraclesList) { + oracle.initialize(); + } } /** @@ -686,8 +727,8 @@ protected Canvas buildCanvas() { @Override protected void beginSequence(SUT system, State state){ - // Reset the final verdict for the new sequence - finalVerdict = Verdict.OK; + // Reset the sequence verdict for the new sequence + sequenceVerdicts = Collections.singletonList(Verdict.OK); } @Override @@ -738,7 +779,7 @@ protected SUT startSystem() throws SystemStartException { /** * This method gets the state of the SUT - * It also call getVerdict() and saves it into the state + * It also call getVerdicts() and saves it into the state * * @param system * @return @@ -757,20 +798,22 @@ protected State getState(SUT system) throws StateBuildException { if(settings.get(ConfigTags.Mode) == Modes.Spy) return state; + List verdicts = getVerdicts(state); + updateSequenceVerdicts(verdictProcessing.filterDuplicates(verdicts)); + state.set(Tags.OracleVerdicts, sequenceVerdicts); + + // State screenshot is taken after the Verdicts are computed + // This might be relevant to determine the viewPort of the screenshot depending on the Verdict (e.g., ProtocolUtil) setStateScreenshot(state); - Verdict verdict = getVerdict(state); - state.set(Tags.OracleVerdict, verdict); - - if(mode() != Modes.Spy && verdict.severity() >= settings().get(ConfigTags.FaultThreshold)) - { - LogSerialiser.log("Detected fault: " + verdict + "\n", LogSerialiser.LogLevel.Critical); - // this was added to kill the SUT if it is frozen: - if(verdict.severity() == Verdict.Severity.NOT_RESPONDING.getValue()) - { - //if the SUT is frozen, we should kill it! - LogSerialiser.log("SUT frozen, trying to kill it!\n", LogSerialiser.LogLevel.Critical); - SystemProcessHandling.killRunningProcesses(system, 100); + if (mode() != Modes.Spy) { + for (Verdict verdict : sequenceVerdicts) { + // this was added to kill the SUT if it is frozen: + if (verdict.severity() == Verdict.Severity.NOT_RESPONDING.getValue()) { + //if the SUT is frozen, we should kill it! + LogSerialiser.log("SUT frozen, trying to kill it!\n", LogSerialiser.LogLevel.Critical); + SystemProcessHandling.killRunningProcesses(system, 100); + } } } @@ -808,26 +851,33 @@ else if(NativeLinker.getPLATFORM_OS().contains(OperatingSystems.IOS)) { } @Override - protected Verdict getVerdict(State state){ + protected List getVerdicts(State state) { Assert.notNull(state); + List verdicts = new ArrayList<>(); + //------------------- // ORACLES FOR FREE //------------------- if ( processListenerOracleEnabled ) { - Verdict processListenerVerdict = processListenerOracle.getVerdict(state); - if ( processListenerVerdict.severity() == Verdict.Severity.SUSPICIOUS_PROCESS.getValue() ) { - return processListenerVerdict; + List processListenerVerdicts = processListenerOracle.getVerdicts(state); + for (Verdict processListenerVerdict : processListenerVerdicts) { + if ( processListenerVerdict.severity() == Verdict.Severity.SUSPICIOUS_PROCESS.getValue() ) { + verdicts.add(processListenerVerdict); + } } } // if the SUT is not running and closed unexpectedly, we assume it crashed - if(!state.get(IsRunning, false)) - return new Verdict(Verdict.Severity.UNEXPECTEDCLOSE, "System is offline! Closed Unexpectedly! I assume it crashed!"); + if(!state.get(IsRunning, false)) { + verdicts.add(new Verdict(Verdict.Severity.UNEXPECTEDCLOSE, "System is offline! Closed Unexpectedly! I assume it crashed!")); + return verdicts; + } // if the SUT does not respond within a given amount of time, we assume it crashed if(state.get(Tags.NotResponding, false)){ - return new Verdict(Verdict.Severity.NOT_RESPONDING, "System is unresponsive! I assume something is wrong!"); + verdicts.add(new Verdict(Verdict.Severity.NOT_RESPONDING, "System is unresponsive! I assume something is wrong!")); + return verdicts; } //------------------------ @@ -838,23 +888,37 @@ protected Verdict getVerdict(State state){ this.suspiciousTitlesPattern = Pattern.compile(settings().get(ConfigTags.SuspiciousTags), Pattern.UNICODE_CHARACTER_CLASS); // search all widgets for suspicious String Values - Verdict suspiciousValueVerdict = Verdict.OK; for(Widget w : state) { - suspiciousValueVerdict = suspiciousStringValueMatcher(w); + Verdict suspiciousValueVerdict = suspiciousStringValueMatcher(w); if(suspiciousValueVerdict.severity() == Verdict.Severity.SUSPICIOUS_TAG.getValue()) { - return suspiciousValueVerdict; + verdicts.add(suspiciousValueVerdict); } } if ( logOracleEnabled ) { - Verdict logVerdict = logOracle.getVerdict(state); - if ( logVerdict.severity() == Verdict.Severity.SUSPICIOUS_LOG.getValue() ) { - return logVerdict; + List logVerdicts = logOracle.getVerdicts(state); + for (Verdict logVerdict : logVerdicts) { + if ( logVerdict.severity() == Verdict.Severity.SUSPICIOUS_LOG.getValue() ) { + verdicts.add(logVerdict); + } } } - // if everything was OK ... - return Verdict.OK; + if (extendedOraclesList != null) { + for (Oracle extendedOracle : extendedOraclesList) { + List extendedVerdicts = extendedOracle.getVerdicts(state); + if (extendedVerdicts != null) { + verdicts.addAll(extendedVerdicts); + } + } + } + + // if empty at this point, everything was OK + if (verdicts.isEmpty()) { + verdicts.add(Verdict.OK); + } + + return verdicts; } private Verdict suspiciousStringValueMatcher(Widget w) { @@ -1105,8 +1169,8 @@ protected Action selectAction(State state, Set actions){ * @return */ protected boolean moreActions(State state) { - Verdict stateVerdict = state.get(Tags.OracleVerdict, Verdict.OK); - boolean faultySequence = stateVerdict.severity() != Verdict.OK.severity(); + List stateVerdicts = state.get(Tags.OracleVerdicts, Collections.singletonList(Verdict.OK)); + boolean faultySequence = !Verdict.helperAreAllVerdictsOK(stateVerdicts); return (!settings().get(ConfigTags.StopGenerationOnFault) || !faultySequence) && state.get(Tags.IsRunning, false) && !state.get(Tags.NotResponding, false) && //actionCount() < settings().get(ConfigTags.SequenceLength) && @@ -1141,20 +1205,14 @@ protected void stopSystem(SUT system) { */ @Override protected void postSequenceProcessing() { - - String status = ""; String statusInfo = ""; - if(mode() == Modes.Replay) { - reportManager.addTestVerdict(getReplayVerdict()); - status = (getReplayVerdict()).verdictSeverityTitle(); - statusInfo = (getReplayVerdict()).info(); - } - else { - reportManager.addTestVerdict(getFinalVerdict()); - status = (getFinalVerdict()).verdictSeverityTitle(); - statusInfo = (getFinalVerdict()).info(); - } + List verdicts = (mode() == Modes.Replay) + ? getReplayVerdicts() + : getSequenceVerdicts(); + + reportManager.addTestVerdicts(verdicts); + statusInfo = buildStatusInfo(verdicts); this.generatedSequence = OutputStructure.outerLoopOutputDir + File.separator + this.generatedSequence; try { @@ -1163,17 +1221,31 @@ protected void postSequenceProcessing() { LogSerialiser.log("Error generating sequence canonical path for the index logger\n", LogSerialiser.LogLevel.Critical); } - statusInfo = statusInfo.replace("\n"+Verdict.OK.info(), ""); - - //Timestamp(generated by logger) SUTname Mode SequenceFileObject Status "StatusInfo" + //Timestamp(generated by logger) SUTname Mode SequenceFileObject "StatusInfo" INDEXLOG.info(OutputStructure.executedSUTname + " " + settings.get(ConfigTags.Mode, mode()) + " " + this.generatedSequence - + " " + status + " \"" + statusInfo + "\"" ); + + " " + statusInfo); + + if(mode() == Modes.Generate) verdictProcessing.storeNewVerdicts(verdicts); reportManager.finishReport(); } + private String buildStatusInfo(List verdicts) { + List normalized = (verdicts == null || verdicts.isEmpty()) + ? Collections.singletonList(Verdict.OK) + : verdicts; + if (normalized.size() == 1) { + return normalized.get(0).info(); + } + StringJoiner joiner = new StringJoiner(" | "); + for (Verdict verdict : normalized) { + joiner.add("[" + verdict.verdictSeverityTitle() + "] " + verdict.info()); + } + return joiner.toString(); + } + /** * method for closing the internal TESTAR test session */ diff --git a/testar/src/org/testar/monkey/GenerateMode.java b/testar/src/org/testar/monkey/GenerateMode.java index 4ef6541df..35c09df20 100644 --- a/testar/src/org/testar/monkey/GenerateMode.java +++ b/testar/src/org/testar/monkey/GenerateMode.java @@ -38,10 +38,12 @@ import org.testar.monkey.alayer.State; import org.testar.monkey.alayer.Tags; import org.testar.monkey.alayer.Verdict; +import org.testar.monkey.alayer.actions.NOP; import org.testar.serialisation.LogSerialiser; import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Set; import java.util.StringJoiner; @@ -97,18 +99,20 @@ public void runGenerateOuterLoop(DefaultProtocol protocol) { // Initial getState() called before beginSequence: LogSerialiser.log("Obtaining system initial state before beginSequence...\n", LogSerialiser.LogLevel.Debug); State initialState = protocol.getState(system); - protocol.finalVerdict = initialState.get(Tags.OracleVerdict, Verdict.OK); + protocol.updateSequenceVerdicts(initialState.get(Tags.OracleVerdicts, Collections.singletonList(Verdict.OK))); // If the SUT does not contain initial failures, start the inner loop test sequence - if(protocol.finalVerdict.severity() == Verdict.OK.severity()) { + if(Verdict.helperAreAllVerdictsOK(protocol.getSequenceVerdicts())) { // beginSequence() - a script to interact with GUI, for example login screen LogSerialiser.log("Invoking begin sequence in the initial state...\n", LogSerialiser.LogLevel.Debug); protocol.beginSequence(system, initialState); // starting the INNER LOOP with the updated state after SUT modification - protocol.finalVerdict = runGenerateInnerLoop(protocol, system, protocol.getState(system)); + protocol.updateSequenceVerdicts(runGenerateInnerLoop(protocol, system, protocol.getState(system))); } else { // If failure exists in the initial state + // Saving the state with empty actions into replayable test sequence: + protocol.saveActionIntoFragmentForReplayableSequence(new NOP(), initialState, Collections.emptySet()); // Save initial state information in the state model before finishing protocol.stateModelManager.notifyNewStateReached(initialState, Collections.emptySet()); } @@ -151,7 +155,7 @@ public void runGenerateOuterLoop(DefaultProtocol protocol) { * * @param system */ - private Verdict runGenerateInnerLoop(DefaultProtocol protocol, SUT system, State state) { + private List runGenerateInnerLoop(DefaultProtocol protocol, SUT system, State state) { // Deriving actions from the state after begin sequence: Set actions = protocol.deriveActions(system, state); @@ -212,7 +216,7 @@ private Verdict runGenerateInnerLoop(DefaultProtocol protocol, SUT system, State protocol.stateModelManager.notifyNewStateReached(state, actions); } - return state.get(Tags.OracleVerdict, Verdict.OK); + return state.get(Tags.OracleVerdicts, Collections.singletonList(Verdict.OK)); } private void finishGeneratedSequence(DefaultProtocol protocol, SUT system) { @@ -224,11 +228,11 @@ private void finishGeneratedSequence(DefaultProtocol protocol, SUT system) { protocol.writeAndCloseFragmentForReplayableSequence(); - if (protocol.finalVerdict.severity() != Verdict.OK.severity()) + if (!Verdict.helperAreAllVerdictsOK(protocol.getSequenceVerdicts())) LogSerialiser.log("Sequence contained faults!\n", LogSerialiser.LogLevel.Critical); // Copy sequence file into proper directory: - protocol.classifyAndCopySequenceIntoAppropriateDirectory(protocol.getFinalVerdict()); + protocol.classifyAndCopySequenceIntoAppropriateDirectory(protocol.getSequenceVerdicts()); // Calling postSequenceProcessing() to allow resetting test environment after test sequence, etc protocol.postSequenceProcessing(); diff --git a/testar/src/org/testar/monkey/RecordMode.java b/testar/src/org/testar/monkey/RecordMode.java deleted file mode 100644 index d34bda23d..000000000 --- a/testar/src/org/testar/monkey/RecordMode.java +++ /dev/null @@ -1,248 +0,0 @@ -/*************************************************************************************************** - * - * Copyright (c) 2022 - 2023 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2022 - 2023 Open Universiteit - www.ou.nl - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - *******************************************************************************************************/ - -package org.testar.monkey; - -import org.testar.ActionStatus; -import org.testar.OutputStructure; -import org.testar.SutVisualization; -import org.testar.monkey.RuntimeControlsProtocol.Modes; -import org.testar.monkey.alayer.*; -import org.testar.monkey.alayer.actions.AnnotatingActionCompiler; -import org.testar.monkey.alayer.devices.KBKeys; -import org.testar.monkey.alayer.devices.MouseButtons; -import org.testar.monkey.alayer.exceptions.WidgetNotFoundException; -import org.testar.serialisation.LogSerialiser; - -import java.util.Collections; -import java.util.List; -import java.util.Set; - -public class RecordMode { - - private static final int MAX_ESC_ATTEMPTS = 99; - - /** - * Method to run TESTAR on Record User Actions Mode. - * @param system - */ - public void runRecordLoop(DefaultProtocol protocol) { - // Prepare the output folders structure - synchronized(this){ - OutputStructure.calculateInnerLoopDateString(); - OutputStructure.sequenceInnerLoopCount++; - } - - //empty method in defaultProtocol - allowing implementation in application specific protocols - //HTML report is created here in DefaultProtocol - protocol.preSequencePreparations(); - - //We need to invoke the SUT & the canvas representation - SUT system = protocol.startSUTandLogger(); - protocol.cv = protocol.buildCanvas(); - protocol.actionCount = 1; - - //Generating the sequence file that can be replayed: - protocol.getAndStoreGeneratedSequence(); - protocol.getAndStoreSequenceFile(); - - // notify the statemodelmanager - protocol.stateModelManager.notifyTestSequencedStarted(); - - /** - * Start Record User Action Loop - */ - while(protocol.mode() == Modes.Record && system.isRunning()) { - State state = protocol.getState(system); - protocol.cv.begin(); - Util.clear(protocol.cv); - - Set actions = protocol.deriveActions(system, state); - protocol.buildStateActionsIdentifiers(state, actions); - - //notify the state model manager of the new state - protocol.stateModelManager.notifyNewStateReached(state, actions); - - // If no actions are derived, create an ESC action - if(actions.isEmpty()){ - if (protocol.escAttempts >= MAX_ESC_ATTEMPTS){ - LogSerialiser.log("No available actions to execute! Tried ESC <" + MAX_ESC_ATTEMPTS + "> times. Stopping sequence generation!\n", LogSerialiser.LogLevel.Critical); - } - //---------------------------------- - // THERE MUST ALMOST BE ONE ACTION! - //---------------------------------- - // if we did not find any actions, then we just hit escape, maybe that works ;-) - Action escAction = new AnnotatingActionCompiler().hitKey(KBKeys.VK_ESCAPE); - protocol.buildEnvironmentActionIdentifiers(state, escAction); - actions.add(escAction); - protocol.escAttempts++; - } else - protocol.escAttempts = 0; - - ActionStatus actionStatus = new ActionStatus(); - - //Start Wait User Action Loop to obtain the Action did by the User - waitUserActionLoop(protocol, system, state, actionStatus); - - //Save the user action information into the logs - if (actionStatus.isUserEventAction()) { - - protocol.buildStateActionsIdentifiers(state, Collections.singleton(actionStatus.getAction())); - - //notify the state model manager of the executed action - protocol.stateModelManager.notifyActionExecution(actionStatus.getAction()); - - protocol.saveActionInfoInLogs(state, actionStatus.getAction(), "RecordedAction"); - DefaultProtocol.lastExecutedAction = actionStatus.getAction(); - protocol.actionCount++; - } - - /** - * When we close TESTAR with Shift+down arrow, last actions is detected like null - */ - if(actionStatus.getAction()!=null) { - protocol.saveActionIntoFragmentForReplayableSequence(actionStatus.getAction(), state, actions); - } else { - //If null just ignore - } - - Util.clear(protocol.cv); - protocol.cv.end(); - } - - //If user closes the SUT while in Record-mode, TESTAR will close (or go back to SettingsDialog): - if(!system.isRunning()){ - protocol.mode = Modes.Quit; - } - - if(protocol.mode() == Modes.Quit){ - // notify the statemodelmanager - protocol.stateModelManager.notifyTestSequenceStopped(); - - // notify the state model manager of the sequence end - protocol.stateModelManager.notifyTestingEnded(); - - //Closing fragment for recording replayable test sequence: - protocol.writeAndCloseFragmentForReplayableSequence(); - - //Copy sequence file into proper directory: - protocol.classifyAndCopySequenceIntoAppropriateDirectory(Verdict.OK); - - protocol.postSequenceProcessing(); - - //If we want to Quit the current execution we stop the system - protocol.stopSystem(system); - } - } - - /** - * Waits for an user UI action. - * Requirement: Mode must be Record. - */ - private void waitUserActionLoop(DefaultProtocol protocol, SUT system, State state, ActionStatus actionStatus){ - while (protocol.mode() == Modes.Record && !actionStatus.isUserEventAction()){ - if (protocol.userEvent != null){ - actionStatus.setAction(mapUserEvent(protocol, state)); - actionStatus.setUserEventAction((actionStatus.getAction() != null)); - protocol.userEvent = null; - } - synchronized(this){ - try { - this.wait(100); - } catch (InterruptedException e) {} - } - state = protocol.getState(system); - protocol.cv.begin(); - Util.clear(protocol.cv); - - //In Record-mode, we activate the visualization with Shift+ArrowUP: - if(protocol.visualizationOn) { - SutVisualization.visualizeState(false, - protocol.markParentWidget, - protocol.mouse, - protocol.lastPrintParentsOf, - protocol.cv, - state); - } - - Set actions = protocol.deriveActions(system, state); - protocol.buildStateActionsIdentifiers(state, actions); - - //In Record-mode, we activate the visualization with Shift+ArrowUP: - if(protocol.visualizationOn) { - protocol.visualizeActions(protocol.cv, state, actions); - } - - protocol.cv.end(); - } - } - - /** - * Records user action. - * - * @param state - * @return - */ - private Action mapUserEvent(DefaultProtocol protocol, State state){ - Assert.notNull(protocol.userEvent); - if (protocol.userEvent[0] instanceof MouseButtons){ // mouse events - double x = ((Double) protocol.userEvent[1]).doubleValue(); - double y = ((Double) protocol.userEvent[2]).doubleValue(); - Widget w = null; - try { - w = Util.widgetFromPoint(state, x, y); - x = 0.5; y = 0.5; - if (protocol.userEvent[0] == MouseButtons.BUTTON1) // left click - return (new AnnotatingActionCompiler()).leftClickAt(w,x,y); - else if (protocol.userEvent[0] == MouseButtons.BUTTON3) // right click - return (new AnnotatingActionCompiler()).rightClickAt(w,x,y); - } catch (WidgetNotFoundException we){ - System.out.println("Mapping user event ... widget not found @(" + x + "," + y + ")"); - return null; - } - } else if (protocol.userEvent[0] instanceof KBKeys) // key events - return (new AnnotatingActionCompiler()).hitKey((KBKeys) protocol.userEvent[0]); - else if (protocol.userEvent[0] instanceof String){ // type events - if (DefaultProtocol.lastExecutedAction == null) - return null; - List targets = DefaultProtocol.lastExecutedAction.get(Tags.Targets,null); - if (targets == null || targets.size() != 1) - return null; - try { - Widget w = targets.get(0).apply(state); - return (new AnnotatingActionCompiler()).clickTypeInto(w, (String) protocol.userEvent[0], true); - } catch (WidgetNotFoundException we){ - return null; - } - } - return null; - } - -} diff --git a/testar/src/org/testar/monkey/ReplayMode.java b/testar/src/org/testar/monkey/ReplayMode.java index ee140d975..bb82f41e7 100644 --- a/testar/src/org/testar/monkey/ReplayMode.java +++ b/testar/src/org/testar/monkey/ReplayMode.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2022 - 2023 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2022 - 2023 Open Universiteit - www.ou.nl + * Copyright (c) 2022 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2022 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -38,9 +38,9 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.ObjectInputStream; +import java.util.Collections; import java.util.List; import java.util.Set; -import java.util.zip.GZIPInputStream; import org.testar.OutputStructure; import org.testar.SutVisualization; @@ -65,10 +65,10 @@ public class ReplayMode { * Read the replayable file, repeat saved actions and generate new sequences, oracles and logs */ public void runReplayLoop(DefaultProtocol protocol) { + VerdictProcessing verdictProcessing = new VerdictProcessing(protocol.settings()); FileInputStream fis = null; BufferedInputStream bis = null; - GZIPInputStream gis = null; ObjectInputStream ois = null; protocol.actionCount = 1; @@ -92,8 +92,7 @@ public void runReplayLoop(DefaultProtocol protocol) { fis = new FileInputStream(seqFile); bis = new BufferedInputStream(fis); - gis = new GZIPInputStream(bis); - ois = new ObjectInputStream(gis); + ois = new ObjectInputStream(bis); /** * Initialize the fragment to create a new sequence and logs @@ -106,7 +105,7 @@ public void runReplayLoop(DefaultProtocol protocol) { protocol.cv = protocol.buildCanvas(); State state = protocol.getState(system); - protocol.setReplayVerdict(protocol.getVerdict(state)); + protocol.setReplayVerdicts(verdictProcessing.filterDuplicates(protocol.getVerdicts(state))); // notify the statemodelmanager protocol.stateModelManager.notifyTestSequencedStarted(); @@ -114,7 +113,7 @@ public void runReplayLoop(DefaultProtocol protocol) { double rrt = protocol.settings().get(ConfigTags.ReplayRetryTime); while(success - && protocol.getReplayVerdict().severity() == Verdict.OK.severity() + && Verdict.helperAreAllVerdictsOK(protocol.getReplayVerdicts()) && protocol.mode() == Modes.Replay) { //Initialize local fragment and read saved action of PathToReplaySequence File @@ -131,14 +130,14 @@ public void runReplayLoop(DefaultProtocol protocol) { } else { success = false; String msg = "Exception " + ioe.getMessage() + " reading TESTAR replayableFragment: " + seqFile; - protocol.setReplayVerdict(new Verdict(Verdict.Severity.UNREPLAYABLE, msg)); + protocol.setReplayVerdicts(Collections.singletonList(new Verdict(Verdict.Severity.UNREPLAYABLE, msg))); protocol.stateModelManager.notifyTestSequenceInterruptedBySystem(ioe.toString()); break; } } catch(NullPointerException npe) { success = false; String msg = "Null exception replaying TESTAR action"; - protocol.setReplayVerdict(new Verdict(Verdict.Severity.UNREPLAYABLE, msg)); + protocol.setReplayVerdicts(Collections.singletonList(new Verdict(Verdict.Severity.UNREPLAYABLE, msg))); protocol.stateModelManager.notifyTestSequenceInterruptedBySystem(npe.toString()); break; } @@ -200,7 +199,7 @@ public void runReplayLoop(DefaultProtocol protocol) { + " of the replayed sequence can not been replayed into " + " the State " + state.get(Tags.ConcreteID, state.toString()); - protocol.setReplayVerdict(new Verdict(Verdict.Severity.UNREPLAYABLE, msg)); + protocol.setReplayVerdicts(Collections.singletonList(new Verdict(Verdict.Severity.UNREPLAYABLE, msg))); break; } @@ -249,7 +248,7 @@ public void runReplayLoop(DefaultProtocol protocol) { state = protocol.getState(system); - protocol.setReplayVerdict(protocol.getVerdict(state)); + protocol.setReplayVerdicts(verdictProcessing.filterDuplicates(protocol.getVerdicts(state))); } } @@ -269,9 +268,6 @@ public void runReplayLoop(DefaultProtocol protocol) { if (ois != null){ try { ois.close(); } catch (IOException e) { e.printStackTrace(); } } - if (gis != null){ - try { gis.close(); } catch (IOException e) { e.printStackTrace(); } - } if (bis != null){ try { bis.close(); } catch (IOException e) { e.printStackTrace(); } } @@ -284,8 +280,10 @@ public void runReplayLoop(DefaultProtocol protocol) { system.stop(); } - if(protocol.getReplayVerdict().severity() != Verdict.OK.severity()) { - String msg = "Replayed Sequence contains Errors: "+ protocol.getReplayVerdict().info(); + List replayVerdicts = verdictProcessing.filterDuplicates(protocol.getReplayVerdicts()); + protocol.setReplayVerdicts(replayVerdicts); + if(!Verdict.helperAreAllVerdictsOK(replayVerdicts)) { + String msg = "Replayed Sequence contains Errors: " + buildVerdictsInfo(replayVerdicts); System.out.println(msg); LogSerialiser.log(msg, LogSerialiser.LogLevel.Info); @@ -294,9 +292,10 @@ public void runReplayLoop(DefaultProtocol protocol) { System.out.println(msg); LogSerialiser.log(msg, LogSerialiser.LogLevel.Info); - }else if(protocol.getReplayVerdict().severity() == Verdict.Severity.UNREPLAYABLE.getValue()){ - System.out.println(protocol.getReplayVerdict().info()); - LogSerialiser.log(protocol.getReplayVerdict().info(), LogSerialiser.LogLevel.Critical); + }else if(replayVerdicts.stream().anyMatch(v -> v.severity() == Verdict.Severity.UNREPLAYABLE.getValue())){ + String info = buildVerdictsInfo(replayVerdicts); + System.out.println(info); + LogSerialiser.log(info, LogSerialiser.LogLevel.Critical); }else{ String msg = "Fail replaying sequence.\n"; @@ -314,7 +313,7 @@ public void runReplayLoop(DefaultProtocol protocol) { protocol.writeAndCloseFragmentForReplayableSequence(); //Copy sequence file into proper directory: - protocol.classifyAndCopySequenceIntoAppropriateDirectory(protocol.getReplayVerdict()); + protocol.classifyAndCopySequenceIntoAppropriateDirectory(replayVerdicts); LogSerialiser.finish(); @@ -329,4 +328,19 @@ public void runReplayLoop(DefaultProtocol protocol) { // Going back to TESTAR settings dialog if it was used to start replay: protocol.mode = Modes.Quit; } + + private String buildVerdictsInfo(List verdicts) { + if (verdicts == null || verdicts.isEmpty()) return ""; + if (verdicts.size() == 1) { + return verdicts.get(0).info(); + } + StringBuilder builder = new StringBuilder(); + for (Verdict verdict : verdicts) { + if (builder.length() > 0) { + builder.append(" | "); + } + builder.append("[").append(verdict.verdictSeverityTitle()).append("] ").append(verdict.info()); + } + return builder.toString(); + } } diff --git a/testar/src/org/testar/monkey/RuntimeControlsProtocol.java b/testar/src/org/testar/monkey/RuntimeControlsProtocol.java index 9eb14501a..879e05951 100644 --- a/testar/src/org/testar/monkey/RuntimeControlsProtocol.java +++ b/testar/src/org/testar/monkey/RuntimeControlsProtocol.java @@ -48,7 +48,6 @@ public abstract class RuntimeControlsProtocol extends AbstractProtocol implement public enum Modes{ Spy, - Record, Generate, Quit, View, @@ -80,8 +79,7 @@ protected synchronized void setMode(Modes mode){ private final static double SLOW_MOTION = 2.0; //TODO: key commands come through java.awt.event but are the key codes same for all OS? if they are the same, then move to platform independent protocol? //TODO: Investigate better shortcut combinations to control TESTAR that does not interfere with SUT - // (e.g. SHIFT + 1 puts an ! in the notepad and hence interferes with SUT state, but the - // event is not recorded as a user event). + // (e.g. SHIFT + 1 puts an ! in the notepad and hence interferes with SUT state). /** * Override the default keylistener to implement the TESTAR shortcuts * SHIFT + SPACE @@ -129,15 +127,6 @@ else if (key == KBKeys.VK_0 && pressed.contains(KBKeys.VK_SHIFT)) { System.setProperty("DEBUG_WINDOWS_PROCESS_NAMES","true"); } - // In Record mode you can press any key except SHIFT to add a user keyboard - // This is because SHIFT is used for the TESTAR shortcuts - // This is not ideal, because now special characters and capital letters and other events that needs SHIFT - // cannot be recorded as an user event in Record.... - else if (!pressed.contains(KBKeys.VK_SHIFT) && mode() == Modes.Record && userEvent == null) { - //System.out.println("USER_EVENT key_down! " + key.toString()); - userEvent = new Object[]{key}; // would be ideal to set it up at keyUp - } - // SHIFT + ALT --> Toggle widget-tree hierarchy display if (pressed.contains(KBKeys.VK_ALT) && pressed.contains(KBKeys.VK_SHIFT)) { markParentWidget = !markParentWidget; @@ -162,23 +151,8 @@ public void keyUp(KBKeys key){ @Override public void mouseDown(MouseButtons btn, double x, double y){} - /** - * In Record mode the user can add user events by clicking and the event is added when releasing the mouse - * @param btn - * @param x - * @param y - */ @Override - public void mouseUp(MouseButtons btn, double x, double y){ - // In GenerateManual the user can add user events by clicking - if (mode() == Modes.Record && userEvent == null){ - userEvent = new Object[]{ - btn, - Double.valueOf(x), - Double.valueOf(y) - }; - } - } + public void mouseUp(MouseButtons btn, double x, double y){} @Override public void mouseMoved(double x, double y) {} diff --git a/testar/src/org/testar/monkey/VerdictProcessing.java b/testar/src/org/testar/monkey/VerdictProcessing.java new file mode 100644 index 000000000..afc2974b9 --- /dev/null +++ b/testar/src/org/testar/monkey/VerdictProcessing.java @@ -0,0 +1,205 @@ +/*************************************************************************************************** + * + * Copyright (c) 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2026 Open Universiteit - www.ou.nl + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + *******************************************************************************************************/ + +package org.testar.monkey; + +import org.testar.monkey.alayer.Verdict; +import org.testar.serialisation.LogSerialiser; +import org.testar.settings.Settings; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class VerdictProcessing { + private static final String LIST_VERDICTS_FAILURES_FILENAME = "list_of_verdicts_with_failures.txt"; + + private final boolean ignoreDuplicatedVerdicts; + private final List verdictIgnoreList = new ArrayList<>(); + private final File verdictIgnoreFile; + + public VerdictProcessing(Settings settings) { + ignoreDuplicatedVerdicts = settings.get(ConfigTags.IgnoreDuplicatedVerdicts, false); + verdictIgnoreFile = resolveVerdictIgnoreFile(); + + if (!ignoreDuplicatedVerdicts || verdictIgnoreFile == null || !verdictIgnoreFile.exists()) { + return; + } + + try { + Path path = verdictIgnoreFile.toPath(); + for (String line : Files.readAllLines(path, StandardCharsets.UTF_8)) { + String trimmed = line.trim(); + if (!trimmed.isEmpty() && !verdictIgnoreList.contains(trimmed)) { + verdictIgnoreList.add(trimmed); + } + } + } catch (IOException e) { + LogSerialiser.log("Unable to read verdict ignore file: " + verdictIgnoreFile.getAbsolutePath() + "\n", + LogSerialiser.LogLevel.Info); + } + } + + public List filterDuplicates(List verdicts) { + if (verdicts == null || verdicts.isEmpty()) { + return Collections.singletonList(Verdict.OK); + } + + List filtered = new ArrayList<>(); + + for (Verdict verdict : verdicts) { + if (verdict == null) { + continue; + } + + if (shouldIgnorePersistedDuplicate(verdict)) { + continue; + } + + filtered.add(verdict); + } + + if (filtered.isEmpty()) { + return Collections.singletonList(Verdict.OK); + } + + if (areAllVerdictsOk(filtered)) { + return Collections.singletonList(Verdict.OK); + } + + return clearOkIfFailurePresent(filtered); + } + + public void storeNewVerdicts(List verdicts) { + if (verdicts == null || verdicts.isEmpty()) { + return; + } + for (Verdict verdict : verdicts) { + storeVerdictFailInfoIfNew(verdict); + } + } + + public static File resolveVerdictIgnoreFile() { + if (Main.SSE_ACTIVATED != null && !Main.SSE_ACTIVATED.isEmpty()) { + return new File(Main.settingsDir + Main.SSE_ACTIVATED, LIST_VERDICTS_FAILURES_FILENAME); + } + String settingsPath = Settings.getSettingsPath(); + if (settingsPath != null && !settingsPath.isEmpty()) { + return new File(settingsPath, LIST_VERDICTS_FAILURES_FILENAME); + } + return null; + } + + private boolean isDuplicateVerdictInfo(String verdictInfo) { + String normalized = normalizeVerdictInfo(verdictInfo); + if (normalized.isEmpty()) { + return false; + } + return verdictIgnoreList.stream().anyMatch(info -> info.contains(normalized)); + } + + private void storeVerdictFailInfoIfNew(Verdict verdict) { + if (!ignoreDuplicatedVerdicts || verdict == null) { + return; + } + if (verdict.isCritical()) { + return; + } + if (verdict.severity() <= Verdict.Severity.OK.getValue()) { + return; + } + String normalized = normalizeVerdictInfo(verdict.info()); + if (normalized.isEmpty() || verdictIgnoreList.contains(normalized)) { + return; + } + verdictIgnoreList.add(normalized); + + if (verdictIgnoreFile == null) { + return; + } + try { + Files.write(verdictIgnoreFile.toPath(), + Collections.singletonList(normalized), + StandardCharsets.UTF_8, + StandardOpenOption.CREATE, + StandardOpenOption.APPEND); + } catch (IOException e) { + LogSerialiser.log("Unable to update verdict ignore file: " + verdictIgnoreFile.getAbsolutePath() + "\n", + LogSerialiser.LogLevel.Info); + } + } + + private String normalizeVerdictInfo(String verdictInfo) { + return verdictInfo == null ? "" : verdictInfo.replace("\n", " ").trim(); + } + + private boolean shouldIgnorePersistedDuplicate(Verdict verdict) { + return ignoreDuplicatedVerdicts + && !verdict.isCritical() + && verdict.severity() > Verdict.Severity.OK.getValue() + && isDuplicateVerdictInfo(verdict.info()); + } + + private boolean areAllVerdictsOk(List verdicts) { + for (Verdict verdict : verdicts) { + if (verdict.severity() > Verdict.Severity.OK.getValue()) { + return false; + } + } + return true; + } + + private List clearOkIfFailurePresent(List verdicts) { + boolean hasFailureVerdict = false; + for (Verdict verdict : verdicts) { + if (verdict != null && verdict.severity() > Verdict.Severity.OK.getValue()) { + hasFailureVerdict = true; + break; + } + } + if (!hasFailureVerdict) { + return verdicts; + } + List filtered = new ArrayList<>(); + for (Verdict verdict : verdicts) { + if (verdict != null && verdict.severity() > Verdict.Severity.OK.getValue()) { + filtered.add(verdict); + } + } + return filtered.isEmpty() ? verdicts : filtered; + } + +} diff --git a/testar/src/org/testar/oracles/Oracle.java b/testar/src/org/testar/oracles/Oracle.java index 60ea8a165..69952a105 100644 --- a/testar/src/org/testar/oracles/Oracle.java +++ b/testar/src/org/testar/oracles/Oracle.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2022 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2022 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2022 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2022 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -60,13 +60,13 @@ public interface Oracle { public abstract void initialize(); /** - * Request that the Oracle determine a verdict about the current state of the SUT. - * This method would usually be called by the getVerdict method in the protocol. + * Request that the Oracle determine verdicts about the current state of the SUT. + * This method would usually be called by the getVerdicts method in the protocol. * * @param state - * @return verdict + * @return list of verdicts */ - public abstract Verdict getVerdict(State state); + public abstract List getVerdicts(State state); /** * Provides a standard red pen for visual annotations. diff --git a/testar/src/org/testar/oracles/generic/visual/GenericVisualAlignmentMetricOracle.java b/testar/src/org/testar/oracles/generic/visual/GenericVisualAlignmentMetricOracle.java index 142a0cdb4..88f0132f3 100644 --- a/testar/src/org/testar/oracles/generic/visual/GenericVisualAlignmentMetricOracle.java +++ b/testar/src/org/testar/oracles/generic/visual/GenericVisualAlignmentMetricOracle.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2023 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2023 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2023 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2023 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,8 @@ package org.testar.oracles.generic.visual; import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import org.testar.monkey.alayer.Shape; import org.testar.monkey.alayer.State; @@ -65,7 +67,7 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { + public List getVerdicts(State state) { ArrayList regions = MetricsHelper.getRegions(state); double alignmentMetric = MetricsHelper.calculateAlignmentMetric(regions); @@ -73,10 +75,10 @@ public Verdict getVerdict(State state) { if (alignmentMetric < thresholdValue) { String verdictMsg = String.format("Alignment metric with value %f is below threshold value %f!", alignmentMetric, thresholdValue); Visualizer visualizer = new RegionsVisualizer(getRedPen(), regions, "Alignment Metric Warning", 0.5, 0.5); - return new Verdict(Verdict.Severity.WARNING_UI_VISUAL_OR_RENDERING_FAULT, verdictMsg, visualizer); + return Collections.singletonList(new Verdict(Verdict.Severity.WARNING_UI_VISUAL_OR_RENDERING_FAULT, verdictMsg, visualizer)); } - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } } diff --git a/testar/src/org/testar/oracles/generic/visual/GenericVisualBalanceMetricOracle.java b/testar/src/org/testar/oracles/generic/visual/GenericVisualBalanceMetricOracle.java index 174ba1f96..b0ea6419e 100644 --- a/testar/src/org/testar/oracles/generic/visual/GenericVisualBalanceMetricOracle.java +++ b/testar/src/org/testar/oracles/generic/visual/GenericVisualBalanceMetricOracle.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2023 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2023 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2023 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2023 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,8 @@ package org.testar.oracles.generic.visual; import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import org.testar.monkey.alayer.Rect; import org.testar.monkey.alayer.Shape; @@ -67,19 +69,19 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { + public List getVerdicts(State state) { if (state.childCount() == 0) { - return Verdict.OK; // State has no children, no need for balance metric evaluation + return Collections.singletonList(Verdict.OK); // State has no children, no need for balance metric evaluation } Shape sutShape = state.child(0).get(Tags.Shape, null); if (sutShape == null) { - return Verdict.OK; // SUT has no shape, no need for balance metric evaluation + return Collections.singletonList(Verdict.OK); // SUT has no shape, no need for balance metric evaluation } Rect sutRect = (Rect) sutShape; if (sutRect.width() <= 0 || sutRect.height() <= 0) { - return Verdict.OK; // Invalid shape dimensions, skip evaluation + return Collections.singletonList(Verdict.OK); // Invalid shape dimensions, skip evaluation } ArrayList regions = MetricsHelper.getRegions(state); @@ -89,10 +91,10 @@ public Verdict getVerdict(State state) { if (balanceMetric < thresholdValue) { String verdictMsg = String.format("Balance metric with value %f is below treshold value %f!", balanceMetric, thresholdValue); Visualizer visualizer = new RegionsVisualizer(getRedPen(), regions, "Balance Metric Warning", 0.5, 0.5); - return new Verdict(Verdict.Severity.WARNING_UI_VISUAL_OR_RENDERING_FAULT, verdictMsg, visualizer); + return Collections.singletonList(new Verdict(Verdict.Severity.WARNING_UI_VISUAL_OR_RENDERING_FAULT, verdictMsg, visualizer)); } - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } } diff --git a/testar/src/org/testar/oracles/generic/visual/GenericVisualCenterAlignmentMetricOracle.java b/testar/src/org/testar/oracles/generic/visual/GenericVisualCenterAlignmentMetricOracle.java index 4a40298be..be59c401a 100644 --- a/testar/src/org/testar/oracles/generic/visual/GenericVisualCenterAlignmentMetricOracle.java +++ b/testar/src/org/testar/oracles/generic/visual/GenericVisualCenterAlignmentMetricOracle.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2023 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2023 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2023 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2023 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,8 @@ package org.testar.oracles.generic.visual; import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import org.testar.monkey.alayer.Shape; import org.testar.monkey.alayer.State; @@ -65,7 +67,7 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { + public List getVerdicts(State state) { ArrayList regions = MetricsHelper.getRegions(state); double centerAlignmentMetric = MetricsHelper.calculateCenterAlignment(regions); @@ -73,10 +75,10 @@ public Verdict getVerdict(State state) { if (centerAlignmentMetric < thresholdValue) { String verdictMsg = String.format("Center alignment metric with value %f is below threshold value %f!", centerAlignmentMetric, thresholdValue); Visualizer visualizer = new RegionsVisualizer(getRedPen(), regions, "Center Alignment Metric Warning", 0.5, 0.5); - return new Verdict(Verdict.Severity.WARNING_UI_VISUAL_OR_RENDERING_FAULT, verdictMsg, visualizer); + return Collections.singletonList(new Verdict(Verdict.Severity.WARNING_UI_VISUAL_OR_RENDERING_FAULT, verdictMsg, visualizer)); } - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } } diff --git a/testar/src/org/testar/oracles/generic/visual/GenericVisualConcentricityMetricOracle.java b/testar/src/org/testar/oracles/generic/visual/GenericVisualConcentricityMetricOracle.java index 8e28a7c3d..6ea6f22d5 100644 --- a/testar/src/org/testar/oracles/generic/visual/GenericVisualConcentricityMetricOracle.java +++ b/testar/src/org/testar/oracles/generic/visual/GenericVisualConcentricityMetricOracle.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2023 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2023 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2023 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2023 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,8 @@ package org.testar.oracles.generic.visual; import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import org.testar.monkey.alayer.Rect; import org.testar.monkey.alayer.Shape; @@ -67,19 +69,19 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { + public List getVerdicts(State state) { if (state.childCount() == 0) { - return Verdict.OK; // State has no children, no need for balance metric evaluation + return Collections.singletonList(Verdict.OK); // State has no children, no need for balance metric evaluation } Shape sutShape = state.child(0).get(Tags.Shape, null); if (sutShape == null) { - return Verdict.OK; // SUT has no shape, no need for balance metric evaluation + return Collections.singletonList(Verdict.OK); // SUT has no shape, no need for balance metric evaluation } Rect sutRect = (Rect) sutShape; if (sutRect.width() <= 0 || sutRect.height() <= 0) { - return Verdict.OK; // Invalid shape dimensions, skip evaluation + return Collections.singletonList(Verdict.OK); // Invalid shape dimensions, skip evaluation } ArrayList regions = MetricsHelper.getRegions(state); @@ -89,10 +91,10 @@ public Verdict getVerdict(State state) { if (concentricityMetric < thresholdValue) { String verdictMsg = String.format("Concentricity metric with value %f is below threshold value %f!", concentricityMetric, thresholdValue); Visualizer visualizer = new RegionsVisualizer(getRedPen(), regions, "Concentricity Metric Warning", 0.5, 0.5); - return new Verdict(Verdict.Severity.WARNING_UI_VISUAL_OR_RENDERING_FAULT, verdictMsg, visualizer); + return Collections.singletonList(new Verdict(Verdict.Severity.WARNING_UI_VISUAL_OR_RENDERING_FAULT, verdictMsg, visualizer)); } - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } } diff --git a/testar/src/org/testar/oracles/generic/visual/GenericVisualDensityMetricOracle.java b/testar/src/org/testar/oracles/generic/visual/GenericVisualDensityMetricOracle.java index 0f35e9c41..ba57c3053 100644 --- a/testar/src/org/testar/oracles/generic/visual/GenericVisualDensityMetricOracle.java +++ b/testar/src/org/testar/oracles/generic/visual/GenericVisualDensityMetricOracle.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2023 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2023 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2023 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2023 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,8 @@ package org.testar.oracles.generic.visual; import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import org.testar.monkey.alayer.Rect; import org.testar.monkey.alayer.Shape; @@ -69,43 +71,46 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { + public List getVerdicts(State state) { if (state.childCount() == 0) { - return Verdict.OK; // State has no children, no need for balance metric evaluation + return Collections.singletonList(Verdict.OK); // State has no children, no need for balance metric evaluation } Shape sutShape = state.child(0).get(Tags.Shape, null); if (sutShape == null) { - return Verdict.OK; // SUT has no shape, no need for balance metric evaluation + return Collections.singletonList(Verdict.OK); // SUT has no shape, no need for balance metric evaluation } Rect sutRect = (Rect) sutShape; if (sutRect.width() <= 0 || sutRect.height() <= 0) { - return Verdict.OK; // Invalid shape dimensions, skip evaluation + return Collections.singletonList(Verdict.OK); // Invalid shape dimensions, skip evaluation } ArrayList regions = MetricsHelper.getRegions(state); double densityMetric = MetricsHelper.calculateDensity(regions, sutRect.width(), sutRect.height()); - Verdict widgetDensityVerdict = Verdict.OK; + List verdicts = new ArrayList<>(); if (densityMetric < thresholdMinValue) { String verdictMsg = String.format("Density metric with value %f is below threshold minimum value %f! Design too simple.", densityMetric, thresholdMinValue); Visualizer visualizer = new RegionsVisualizer(getRedPen(), regions, "Density Warning - Too Simple", 0.5, 0.5); Verdict verdict = new Verdict(Verdict.Severity.WARNING_UI_VISUAL_OR_RENDERING_FAULT, verdictMsg, visualizer); - widgetDensityVerdict = widgetDensityVerdict.join(verdict); + verdicts.add(verdict); } if (densityMetric > thresholdMaxValue) { String verdictMsg = String.format("Density metric with value %f is higher than threshold maximum value %f! Design too complex.", densityMetric, thresholdMaxValue); Visualizer visualizer = new RegionsVisualizer(getRedPen(), regions, "Density Warning - Too Complex", 0.5, 0.5); Verdict verdict = new Verdict(Verdict.Severity.WARNING_UI_VISUAL_OR_RENDERING_FAULT, verdictMsg, visualizer); - widgetDensityVerdict = widgetDensityVerdict.join(verdict); + verdicts.add(verdict); } - return widgetDensityVerdict; + if (verdicts.isEmpty()) { + return Collections.singletonList(Verdict.OK); + } + return verdicts; } } diff --git a/testar/src/org/testar/oracles/generic/visual/GenericVisualSimplicityMetricOracle.java b/testar/src/org/testar/oracles/generic/visual/GenericVisualSimplicityMetricOracle.java index 659b1f6ba..2a5ee1183 100644 --- a/testar/src/org/testar/oracles/generic/visual/GenericVisualSimplicityMetricOracle.java +++ b/testar/src/org/testar/oracles/generic/visual/GenericVisualSimplicityMetricOracle.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2023 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2023 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2023 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2023 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,8 @@ package org.testar.oracles.generic.visual; import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import org.testar.monkey.alayer.Rect; import org.testar.monkey.alayer.Shape; @@ -67,19 +69,19 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { + public List getVerdicts(State state) { if (state.childCount() == 0) { - return Verdict.OK; // State has no children, no need for balance metric evaluation + return Collections.singletonList(Verdict.OK); // State has no children, no need for balance metric evaluation } Shape sutShape = state.child(0).get(Tags.Shape, null); if (sutShape == null) { - return Verdict.OK; // SUT has no shape, no need for balance metric evaluation + return Collections.singletonList(Verdict.OK); // SUT has no shape, no need for balance metric evaluation } Rect sutRect = (Rect) sutShape; if (sutRect.width() <= 0 || sutRect.height() <= 0) { - return Verdict.OK; // Invalid shape dimensions, skip evaluation + return Collections.singletonList(Verdict.OK); // Invalid shape dimensions, skip evaluation } ArrayList regions = MetricsHelper.getRegions(state); @@ -89,10 +91,10 @@ public Verdict getVerdict(State state) { if (simplicityMetric < thresholdValue) { String verdictMsg = String.format("Simplicity metric with value %f is below threshold value %f!", simplicityMetric, thresholdValue); Visualizer visualizer = new RegionsVisualizer(getRedPen(), regions, "Simplicity Warning", 0.5, 0.5); - return new Verdict(Verdict.Severity.WARNING_UI_VISUAL_OR_RENDERING_FAULT, verdictMsg, visualizer); + return Collections.singletonList(new Verdict(Verdict.Severity.WARNING_UI_VISUAL_OR_RENDERING_FAULT, verdictMsg, visualizer)); } - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } } diff --git a/testar/src/org/testar/oracles/llm/LlmOracle.java b/testar/src/org/testar/oracles/llm/LlmOracle.java index bf3bc4817..d3a01b395 100644 --- a/testar/src/org/testar/oracles/llm/LlmOracle.java +++ b/testar/src/org/testar/oracles/llm/LlmOracle.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2024 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2024 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2024 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2024 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -40,6 +40,8 @@ import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.Base64; +import java.util.Collections; +import java.util.List; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; @@ -127,11 +129,11 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { + public List getVerdicts(State state) { // If the stateless option is enabled, initialize a new prompt to reduce tokens usage if(this.stateless) initialize(); - return getVerdictWithLlm(state); + return Collections.singletonList(getVerdictWithLlm(state)); } private Verdict getVerdictWithLlm(State state) { diff --git a/testar/src/org/testar/oracles/log/AndroidLogcatOracle.java b/testar/src/org/testar/oracles/log/AndroidLogcatOracle.java index c10127f1f..79ae9d22e 100644 --- a/testar/src/org/testar/oracles/log/AndroidLogcatOracle.java +++ b/testar/src/org/testar/oracles/log/AndroidLogcatOracle.java @@ -46,6 +46,7 @@ import java.nio.file.Paths; import java.nio.file.StandardOpenOption; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Set; import java.util.TreeSet; @@ -104,9 +105,9 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { + public List getVerdicts(State state) { if (settings.get(ConfigTags.Mode) != RuntimeControlsProtocol.Modes.Generate) { - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } String packageName = AndroidAppiumFramework.getAppPackageFromCapabilitiesOrCurrent(); @@ -114,7 +115,7 @@ public Verdict getVerdict(State state) { if (dump == null || dump.isBlank()) { AndroidAppiumFramework.clearLogcat(); - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } List relevantLines = dump == null ? List.of() : List.of(dump.split("\\r?\\n")); @@ -133,7 +134,7 @@ public Verdict getVerdict(State state) { List matches = detectRegexMatches(newLines, regex); if (matches.isEmpty()) { AndroidAppiumFramework.clearLogcat(); - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } Set uniqueSorted = new TreeSet<>(matches); @@ -142,7 +143,7 @@ public Verdict getVerdict(State state) { info.append(String.join(" | ", uniqueSorted)); AndroidAppiumFramework.clearLogcat(); - return new Verdict(Verdict.Severity.SUSPICIOUS_LOG, info.toString().trim()); + return Collections.singletonList(new Verdict(Verdict.Severity.SUSPICIOUS_LOG, info.toString().trim())); } private void appendToSequenceLog(List lines) { diff --git a/testar/src/org/testar/oracles/log/LogOracle.java b/testar/src/org/testar/oracles/log/LogOracle.java index a59a0ca68..088c71e0b 100644 --- a/testar/src/org/testar/oracles/log/LogOracle.java +++ b/testar/src/org/testar/oracles/log/LogOracle.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2022 - 2023 Open Universiteit - www.ou.nl - * Copyright (c) 2022 - 2023 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2022 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2022 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,7 @@ package org.testar.oracles.log; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.testar.monkey.ConfigTags; @@ -63,13 +64,13 @@ public void initialize() { checker.initialRead(); } - public Verdict getVerdict(State state) { + public List getVerdicts(State state) { errorsList.addAll(checker.readAndCheck()); if ( errorsList.size() == 0 ) { - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } else { - return new Verdict(Verdict.Severity.SUSPICIOUS_LOG, String.join(";", errorsList)); + return Collections.singletonList(new Verdict(Verdict.Severity.SUSPICIOUS_LOG, String.join(";", errorsList))); } } diff --git a/testar/src/org/testar/oracles/log/ProcessListenerOracle.java b/testar/src/org/testar/oracles/log/ProcessListenerOracle.java index 8588d87ec..5fd36c54c 100644 --- a/testar/src/org/testar/oracles/log/ProcessListenerOracle.java +++ b/testar/src/org/testar/oracles/log/ProcessListenerOracle.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2018 - 2025 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2018 - 2025 Open Universiteit - www.ou.nl + * Copyright (c) 2018 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -36,6 +36,8 @@ import java.io.IOException; import java.io.InputStreamReader; import java.io.PrintWriter; +import java.util.Collections; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -109,9 +111,9 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { + public List getVerdicts(State state) { if (!processListenerEnabled) { - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } try { @@ -120,17 +122,17 @@ public Verdict getVerdict(State state) { Verdict standardOutputVerdict = checkStream(standardOutputReader, "_StdOut.log", "StdOut"); if (standardErrorVerdict.severity() != Verdict.Severity.OK.getValue()) - return standardErrorVerdict; + return Collections.singletonList(standardErrorVerdict); if (standardOutputVerdict.severity() != Verdict.Severity.OK.getValue()) - return standardOutputVerdict; + return Collections.singletonList(standardOutputVerdict); } catch (IOException e) { System.err.println("Unable to read the SUT process buffer"); - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } private Verdict checkStream(BufferedReader reader, String logSuffix, String streamLabel) throws IOException { diff --git a/testar/src/org/testar/oracles/log/WebBrowserConsoleOracle.java b/testar/src/org/testar/oracles/log/WebBrowserConsoleOracle.java new file mode 100644 index 000000000..0fff5750d --- /dev/null +++ b/testar/src/org/testar/oracles/log/WebBrowserConsoleOracle.java @@ -0,0 +1,117 @@ +/*************************************************************************************************** + * + * Copyright (c) 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2026 Open Universiteit - www.ou.nl + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + *******************************************************************************************************/ + +package org.testar.oracles.log; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.logging.Level; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.openqa.selenium.logging.LogEntries; +import org.openqa.selenium.logging.LogEntry; +import org.testar.monkey.ConfigTags; +import org.testar.monkey.alayer.State; +import org.testar.monkey.alayer.Verdict; +import org.testar.monkey.alayer.webdriver.WdDriver; +import org.testar.oracles.Oracle; +import org.testar.settings.Settings; + +/** + * Web browser console oracle (WebDriver). + * Scans browser logs for error/warning lines that match regular expression patterns. + */ +public class WebBrowserConsoleOracle implements Oracle { + + private final boolean errorEnabled; + private final boolean warningEnabled; + private final String errorPatternText; + private final String warningPatternText; + + public WebBrowserConsoleOracle(Settings settings) { + this.errorEnabled = settings.get(ConfigTags.WebConsoleErrorOracle, false); + this.warningEnabled = settings.get(ConfigTags.WebConsoleWarningOracle, false); + this.errorPatternText = settings.get(ConfigTags.WebConsoleErrorPattern, ""); + this.warningPatternText = settings.get(ConfigTags.WebConsoleWarningPattern, ""); + } + + @Override + public void initialize() { + // Nothing extra to initialize + } + + @Override + public List getVerdicts(State state) { + List verdicts = new ArrayList<>(); + + LogEntries logEntries = WdDriver.getBrowserLogs(); + + // If Web Console Error Oracle is enabled and we have some pattern to match + if (errorEnabled && !errorPatternText.isEmpty()) { + // Load the web console error pattern + Pattern errorPattern = Pattern.compile(errorPatternText, Pattern.UNICODE_CHARACTER_CLASS); + // Check Severe messages in the WebDriver logs + for (LogEntry logEntry : logEntries) { + if (logEntry.getLevel().equals(Level.SEVERE)) { + // Check if the severe error message matches with the web console error pattern + String consoleErrorMsg = logEntry.getMessage(); + Matcher matcherError = errorPattern.matcher(consoleErrorMsg); + if (matcherError.matches()) { + verdicts.add(new Verdict(Verdict.Severity.SUSPICIOUS_LOG, "Web Browser Console Error: " + consoleErrorMsg)); + } + } + } + } + + // If Web Console Warning Oracle is enabled and we have some pattern to match + if (warningEnabled && !warningPatternText.isEmpty()) { + // Load the web console warning pattern + Pattern warningPattern = Pattern.compile(warningPatternText, Pattern.UNICODE_CHARACTER_CLASS); + // Check Warning messages in the WebDriver logs + for (LogEntry logEntry : logEntries) { + if (logEntry.getLevel().equals(Level.WARNING)) { + // Check if the warning message matches with the web console error pattern + String consoleWarningMsg = logEntry.getMessage(); + Matcher matcherWarning = warningPattern.matcher(consoleWarningMsg); + if (matcherWarning.matches()) { + verdicts.add(new Verdict(Verdict.Severity.SUSPICIOUS_LOG, "Web Browser Console Warning: " + consoleWarningMsg)); + } + } + } + } + + if (verdicts.isEmpty()) { + return Collections.singletonList(Verdict.OK); + } + return verdicts; + } +} diff --git a/testar/src/org/testar/oracles/web/accessibility/WebAccessibilityClickableSizeOracle.java b/testar/src/org/testar/oracles/web/accessibility/WebAccessibilityClickableSizeOracle.java index 8843739c8..cb4d5d397 100644 --- a/testar/src/org/testar/oracles/web/accessibility/WebAccessibilityClickableSizeOracle.java +++ b/testar/src/org/testar/oracles/web/accessibility/WebAccessibilityClickableSizeOracle.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2025 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2025 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,6 +32,8 @@ package org.testar.oracles.web.accessibility; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import org.testar.monkey.alayer.Rect; @@ -71,8 +73,8 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { - List incorrectWidgets = new ArrayList<>(); + public List getVerdicts(State state) { + List verdicts = new ArrayList<>(); // Iterate over all widgets in the state for (Widget widget : state) { @@ -87,30 +89,27 @@ public Verdict getVerdict(State state) { // Check if the widget is smaller than the recommended pixels if (width < minClickableThreshold || height < minClickableThreshold) { - // If so, save it as incorrect widget - incorrectWidgets.add(widget); + String verdictMsg = String.format( + "Clickable web widget %s is too small (%sx%s px). Minimum: %s px.", + getDescriptionOfWidgets(Collections.singletonList(widget), WdTags.WebOuterHTML), + width.intValue(), + height.intValue(), + minClickableThreshold + ); + Visualizer visualizer = new RegionsVisualizer( + getRedPen(), + getWidgetRegions(Arrays.asList(widget)), + "Accessibility Fault", + 0.5, 0.5); + verdicts.add(new Verdict(Verdict.Severity.WARNING_ACCESSIBILITY_FAULT, verdictMsg, visualizer)); } } } - // If exists one or more incorrect widgets - if(!incorrectWidgets.isEmpty()) { - // Create and return a WARNING_ACCESSIBILITY_FAULT verdict - String verdictMsg = String.format( - "Clickable web widgets %s are too small (Minimum: %s px).", - getDescriptionOfWidgets(incorrectWidgets, WdTags.WebOuterHTML), - minClickableThreshold - ); - Visualizer visualizer = new RegionsVisualizer( - getRedPen(), - getWidgetRegions(incorrectWidgets), - "Accessibility Fault", - 0.5, 0.5); - return new Verdict(Verdict.Severity.WARNING_ACCESSIBILITY_FAULT, verdictMsg, visualizer); - } else { - return Verdict.OK; + if(!verdicts.isEmpty()) { + return verdicts; } - + return Collections.singletonList(Verdict.OK); } } diff --git a/testar/src/org/testar/oracles/web/accessibility/WebAccessibilityFontSizeOracle.java b/testar/src/org/testar/oracles/web/accessibility/WebAccessibilityFontSizeOracle.java index 4fa20b07f..7093e4ab8 100644 --- a/testar/src/org/testar/oracles/web/accessibility/WebAccessibilityFontSizeOracle.java +++ b/testar/src/org/testar/oracles/web/accessibility/WebAccessibilityFontSizeOracle.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2025 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2025 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,7 @@ package org.testar.oracles.web.accessibility; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.testar.monkey.alayer.State; @@ -65,8 +66,8 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { - List incorrectWidgets = new ArrayList<>(); + public List getVerdicts(State state) { + List verdicts = new ArrayList<>(); // Iterate over all widgets in the state for (Widget widget : state) { @@ -84,30 +85,26 @@ public Verdict getVerdict(State state) { // Check if the font size is below the minimum threshold if (fontSize > 0 && fontSize < minFontSizeThreshold) { - // If so, save it as incorrect widget - incorrectWidgets.add(widget); + String verdictMsg = String.format( + "Widget text %s is too small (%d px). Minimum recommended is %d px.", + getDescriptionOfWidgets(Collections.singletonList(widget), WdTags.WebTextContent), + fontSize, + minFontSizeThreshold + ); + Visualizer visualizer = new RegionsVisualizer( + getRedPen(), + getWidgetRegions(Collections.singletonList(widget)), + "Accessibility Fault", + 0.5, 0.5); + verdicts.add(new Verdict(Verdict.Severity.WARNING_ACCESSIBILITY_FAULT, verdictMsg, visualizer)); } } } - // If exists one or more incorrect widgets - if(!incorrectWidgets.isEmpty()) { - // Create and return a WARNING_ACCESSIBILITY_FAULT verdict - String verdictMsg = String.format( - "These widgets Text %s are too small. Minimum recommended is %s px.", - getDescriptionOfWidgets(incorrectWidgets, WdTags.WebTextContent), - minFontSizeThreshold - ); - Visualizer visualizer = new RegionsVisualizer( - getRedPen(), - getWidgetRegions(incorrectWidgets), - "Accessibility Fault", - 0.5, 0.5); - return new Verdict(Verdict.Severity.WARNING_ACCESSIBILITY_FAULT, verdictMsg, visualizer); - } else { - return Verdict.OK; + if(!verdicts.isEmpty()) { + return verdicts; } - + return Collections.singletonList(Verdict.OK); } /** diff --git a/testar/src/org/testar/oracles/web/accessibility/WebAccessibilityImagesAltOracle.java b/testar/src/org/testar/oracles/web/accessibility/WebAccessibilityImagesAltOracle.java index 587bdb59c..0f42e3a4c 100644 --- a/testar/src/org/testar/oracles/web/accessibility/WebAccessibilityImagesAltOracle.java +++ b/testar/src/org/testar/oracles/web/accessibility/WebAccessibilityImagesAltOracle.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2025 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2025 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,7 @@ package org.testar.oracles.web.accessibility; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.testar.monkey.alayer.Roles; @@ -58,36 +59,31 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { - List incorrectWidgets = new ArrayList<>(); + public List getVerdicts(State state) { + List verdicts = new ArrayList<>(); // Check if some widget of the state for(Widget widget : state) { // Is a widget image () and if it lacks alternative text if(widget.get(Tags.Role, Roles.Widget).equals(WdRoles.WdIMG) && (widget.get(WdTags.WebAlt, null) == null || widget.get(WdTags.WebAlt, "").isBlank())) { - // If so, save it as incorrect widget - incorrectWidgets.add(widget); + String verdictMsg = String.format( + "Detected web image widget %s without alternative text!", + getDescriptionOfWidgets(Collections.singletonList(widget), WdTags.WebOuterHTML) + ); + Visualizer visualizer = new RegionsVisualizer( + getRedPen(), + getWidgetRegions(Collections.singletonList(widget)), + "Accessibility Fault", + 0.5, 0.5); + verdicts.add(new Verdict(Verdict.Severity.WARNING_ACCESSIBILITY_FAULT, verdictMsg, visualizer)); } } - // If exists one or more incorrect widgets - if(!incorrectWidgets.isEmpty()) { - // Create and return a WARNING_ACCESSIBILITY_FAULT verdict - String verdictMsg = String.format( - "Detected web image widgets '%s' without alternative text!", - getDescriptionOfWidgets(incorrectWidgets, WdTags.WebOuterHTML) - ); - Visualizer visualizer = new RegionsVisualizer( - getRedPen(), - getWidgetRegions(incorrectWidgets), - "Accessibility Fault", - 0.5, 0.5); - return new Verdict(Verdict.Severity.WARNING_ACCESSIBILITY_FAULT, verdictMsg, visualizer); - } else { - return Verdict.OK; + if(!verdicts.isEmpty()) { + return verdicts; } - + return Collections.singletonList(Verdict.OK); } } diff --git a/testar/src/org/testar/oracles/web/invariants/WebInvariantDuplicateMenuItems.java b/testar/src/org/testar/oracles/web/invariants/WebInvariantDuplicateMenuItems.java index 5b3a5ca73..d3b34e6a8 100644 --- a/testar/src/org/testar/oracles/web/invariants/WebInvariantDuplicateMenuItems.java +++ b/testar/src/org/testar/oracles/web/invariants/WebInvariantDuplicateMenuItems.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2025 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2025 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,7 @@ package org.testar.oracles.web.invariants; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -55,8 +56,8 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { - List menuWidgetsWithDuplicates = new ArrayList<>(); + public List getVerdicts(State state) { + List verdicts = new ArrayList<>(); for (Widget w : state) { // Check for UL elements with at least two children @@ -78,29 +79,27 @@ public Verdict getVerdict(State state) { // If duplicates are found, prepare the verdict message if (duplicatesTexts.size() > 0) { - menuWidgetsWithDuplicates.add(w); + String verdictMsg = String.format( + "Detected a Unnumbered List (UL) web menu %s with duplicate option elements: %s", + getDescriptionOfWidgets(Collections.singletonList(w), WdTags.WebId), + duplicatesTexts + ); + + Visualizer visualizer = new RegionsVisualizer( + getRedPen(), + getWidgetRegions(Collections.singletonList(w)), + "Invariant Fault", + 0.5, 0.5); + + verdicts.add(new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer)); } } } - // If exists one or more incorrect widgets - if (!menuWidgetsWithDuplicates.isEmpty()) { - - String verdictMsg = String.format( - "Detected a Unnumbered List (UL) web menu %s with duplicate option elements!", - getDescriptionOfWidgets(menuWidgetsWithDuplicates, WdTags.WebId) - ); - - Visualizer visualizer = new RegionsVisualizer( - getRedPen(), - getWidgetRegions(menuWidgetsWithDuplicates), - "Invariant Fault", - 0.5, 0.5); - - return new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer); + if (!verdicts.isEmpty()) { + return verdicts; } - - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } // Helper method to find duplicates in a list diff --git a/testar/src/org/testar/oracles/web/invariants/WebInvariantDuplicateSelectItems.java b/testar/src/org/testar/oracles/web/invariants/WebInvariantDuplicateSelectItems.java index ae59ec056..3836fa8a4 100644 --- a/testar/src/org/testar/oracles/web/invariants/WebInvariantDuplicateSelectItems.java +++ b/testar/src/org/testar/oracles/web/invariants/WebInvariantDuplicateSelectItems.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2025 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2025 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -67,8 +67,8 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { - List selectWidgetsWithDuplicates = new ArrayList<>(); + public List getVerdicts(State state) { + List verdicts = new ArrayList<>(); for (Widget w : state) { if (roles.contains(w.get(Tags.Role, Roles.Widget)) && !w.get(WdTags.WebId, "").isEmpty()) { @@ -89,7 +89,19 @@ public Verdict getVerdict(State state) { .collect(Collectors.toSet()); if (!duplicatesTexts.isEmpty()) { - selectWidgetsWithDuplicates.add(w); + String verdictMsg = String.format( + "Detected Select widget %s with duplicate values: %s", + getDescriptionOfWidgets(Collections.singletonList(w), WdTags.WebId), + duplicatesTexts + ); + + Visualizer visualizer = new RegionsVisualizer( + getRedPen(), + getWidgetRegions(Collections.singletonList(w)), + "Invariant Fault", + 0.5, 0.5); + + verdicts.add(new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer)); } } } catch (Exception e) { @@ -98,23 +110,9 @@ public Verdict getVerdict(State state) { } } - // If exists one or more incorrect widgets - if (!selectWidgetsWithDuplicates.isEmpty()) { - - String verdictMsg = String.format( - "Detected Select widgets %s with duplicate values!", - getDescriptionOfWidgets(selectWidgetsWithDuplicates, WdTags.WebId) - ); - - Visualizer visualizer = new RegionsVisualizer( - getRedPen(), - getWidgetRegions(selectWidgetsWithDuplicates), - "Invariant Fault", - 0.5, 0.5); - - return new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer); + if (!verdicts.isEmpty()) { + return verdicts; } - - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } } diff --git a/testar/src/org/testar/oracles/web/invariants/WebInvariantDuplicatedRowsInTable.java b/testar/src/org/testar/oracles/web/invariants/WebInvariantDuplicatedRowsInTable.java index ffe5c89c1..e4b2d58d0 100644 --- a/testar/src/org/testar/oracles/web/invariants/WebInvariantDuplicatedRowsInTable.java +++ b/testar/src/org/testar/oracles/web/invariants/WebInvariantDuplicatedRowsInTable.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2025 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2025 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,7 @@ package org.testar.oracles.web.invariants; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -59,50 +60,51 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { - List incorrectWidgets = new ArrayList<>(); - List incorrectWidgetDescriptions = new ArrayList<>(); + public List getVerdicts(State state) { + List verdicts = new ArrayList<>(); for (Widget w : state) { if(w.get(Tags.Role, Roles.Widget).equals(WdRoles.WdTABLE)) { List> rowElementsDescription = new ArrayList<>(); extractAllRowDescriptionsFromTable(w, rowElementsDescription); - List> duplicatedDescriptions = + List>> duplicatedDescriptions = rowElementsDescription.stream() .collect(Collectors.groupingBy(Pair::right)) .entrySet().stream() .filter(e -> e.getValue().size() > 1) - .flatMap(e -> e.getValue().stream()) + .map(e -> e.getValue()) .collect(Collectors.toList()); // If the list of duplicated descriptions contains a matching prepare the verdict if(!duplicatedDescriptions.isEmpty()) { - for (Pair duplicatedWidget : duplicatedDescriptions) { + for (List> duplicatedWidgets : duplicatedDescriptions) { + Pair duplicatedWidget = duplicatedWidgets.get(0); // Ignore empty rows if (!duplicatedWidget.right().replaceAll("_","").isEmpty()) { - incorrectWidgets.add(duplicatedWidget.left()); - incorrectWidgetDescriptions.add(duplicatedWidget.right()); + String verdictMsg = String.format( + "Detected duplicated row in a Table for the widget: %s ", + duplicatedWidget.right() + ); + List widgets = duplicatedWidgets.stream() + .map(Pair::left) + .collect(Collectors.toList()); + Visualizer visualizer = new RegionsVisualizer( + getRedPen(), + getWidgetRegions(widgets), + "Invariant Fault", + 0.5, 0.5); + verdicts.add(new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer)); } } - - String verdictMsg = String.format( - "Detected a duplicated rows in a Table for the widgets: %s ", - incorrectWidgetDescriptions - ); - - Visualizer visualizer = new RegionsVisualizer( - getRedPen(), - getWidgetRegions(incorrectWidgets), - "Invariant Fault", - 0.5, 0.5); - - return new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer); } } } - return Verdict.OK; + if (!verdicts.isEmpty()) { + return verdicts; + } + return Collections.singletonList(Verdict.OK); } private void extractAllRowDescriptionsFromTable(Widget w, List> rowElementsDescription) { diff --git a/testar/src/org/testar/oracles/web/invariants/WebInvariantEmptySelectItems.java b/testar/src/org/testar/oracles/web/invariants/WebInvariantEmptySelectItems.java index 219e6357a..7f8dd1c95 100644 --- a/testar/src/org/testar/oracles/web/invariants/WebInvariantEmptySelectItems.java +++ b/testar/src/org/testar/oracles/web/invariants/WebInvariantEmptySelectItems.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2025 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2025 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,7 @@ package org.testar.oracles.web.invariants; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.testar.monkey.alayer.Role; @@ -64,8 +65,8 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { - List emptySelectWidgets = new ArrayList<>(); + public List getVerdicts(State state) { + List verdicts = new ArrayList<>(); for (Widget w : state) { if (roles.contains(w.get(Tags.Role, Roles.Widget)) && !w.get(WdTags.WebId, "").isEmpty()) { @@ -76,7 +77,19 @@ public Verdict getVerdict(State state) { // If the select contains 0 or 1 item if (selectItemsLength != null && selectItemsLength.intValue() <= 1) { - emptySelectWidgets.add(w); + String verdictMsg = String.format( + "Detected Select widget %s with empty or only one item (count: %d)!", + getDescriptionOfWidgets(Collections.singletonList(w), WdTags.WebId), + selectItemsLength.intValue() + ); + + Visualizer visualizer = new RegionsVisualizer( + getRedPen(), + getWidgetRegions(Collections.singletonList(w)), + "Invariant Fault", + 0.5, 0.5); + + verdicts.add(new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer)); } } catch (Exception e) { // Ignore webdriver execute script errors @@ -84,23 +97,9 @@ public Verdict getVerdict(State state) { } } - // If exists one or more incorrect widgets - if (!emptySelectWidgets.isEmpty()) { - - String verdictMsg = String.format( - "Detected Select widgets %s with empty or only one item!", - getDescriptionOfWidgets(emptySelectWidgets, WdTags.WebId) - ); - - Visualizer visualizer = new RegionsVisualizer( - getRedPen(), - getWidgetRegions(emptySelectWidgets), - "Invariant Fault", - 0.5, 0.5); - - return new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer); + if (!verdicts.isEmpty()) { + return verdicts; } - - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } } diff --git a/testar/src/org/testar/oracles/web/invariants/WebInvariantManySelectItems.java b/testar/src/org/testar/oracles/web/invariants/WebInvariantManySelectItems.java index d0e938628..5dbe61bf3 100644 --- a/testar/src/org/testar/oracles/web/invariants/WebInvariantManySelectItems.java +++ b/testar/src/org/testar/oracles/web/invariants/WebInvariantManySelectItems.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2025 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2025 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,7 @@ package org.testar.oracles.web.invariants; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.testar.monkey.alayer.Roles; @@ -63,8 +64,8 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { - List selectWidgetsWithManyItems = new ArrayList<>(); + public List getVerdicts(State state) { + List verdicts = new ArrayList<>(); for (Widget w : state) { if (w.get(Tags.Role, Roles.Widget).equals(WdRoles.WdSELECT) && !w.get(WdTags.WebId, "").isEmpty()) { @@ -75,7 +76,20 @@ public Verdict getVerdict(State state) { // Check if the items of the select widget are more than the thresholdValue if (selectItemsLength != null && selectItemsLength.intValue() > thresholdValue) { - selectWidgetsWithManyItems.add(w); + String verdictMsg = String.format( + "Detected Select widget %s which has %d items (threshold: %d)", + getDescriptionOfWidgets(Collections.singletonList(w), WdTags.WebId), + selectItemsLength.intValue(), + thresholdValue + ); + + Visualizer visualizer = new RegionsVisualizer( + getRedPen(), + getWidgetRegions(Collections.singletonList(w)), + "Invariant Fault", + 0.5, 0.5); + + verdicts.add(new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer)); } } catch (Exception e) { @@ -84,23 +98,9 @@ public Verdict getVerdict(State state) { } } - if (!selectWidgetsWithManyItems.isEmpty()) { - - String verdictMsg = String.format( - "Detected Select widgets %s which have more items than the threshold value of %d", - getDescriptionOfWidgets(selectWidgetsWithManyItems, WdTags.WebId), - thresholdValue - ); - - Visualizer visualizer = new RegionsVisualizer( - getRedPen(), - getWidgetRegions(selectWidgetsWithManyItems), - "Invariant Fault", - 0.5, 0.5); - - return new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer); + if (!verdicts.isEmpty()) { + return verdicts; } - - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } } diff --git a/testar/src/org/testar/oracles/web/invariants/WebInvariantNumberWithLotOfDecimals.java b/testar/src/org/testar/oracles/web/invariants/WebInvariantNumberWithLotOfDecimals.java index 26a76cb4b..820aa38d5 100644 --- a/testar/src/org/testar/oracles/web/invariants/WebInvariantNumberWithLotOfDecimals.java +++ b/testar/src/org/testar/oracles/web/invariants/WebInvariantNumberWithLotOfDecimals.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2025 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2025 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,7 @@ package org.testar.oracles.web.invariants; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.testar.monkey.alayer.State; @@ -59,8 +60,8 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { - List incorrectDecimalWidgets = new ArrayList<>(); + public List getVerdicts(State state) { + List verdicts = new ArrayList<>(); for (Widget w : state) { // If the widget contains a web text that is a numeric value @@ -72,31 +73,29 @@ public Verdict getVerdict(State state) { int decimalPlaces = number.length() - number.indexOf('.') - 1; if (decimalPlaces > maxDecimals) { - incorrectDecimalWidgets.add(w); + String verdictMsg = String.format( + "Detected widget %s with %d decimals (max: %d)!", + getDescriptionOfWidgets(Collections.singletonList(w), WdTags.WebTextContent), + decimalPlaces, + maxDecimals + ); + + Visualizer visualizer = new RegionsVisualizer( + getRedPen(), + getWidgetRegions(Collections.singletonList(w)), + "Invariant Fault", + 0.5, 0.5); + + verdicts.add(new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer)); } } } } - // If exists one or more incorrect widgets - if (!incorrectDecimalWidgets.isEmpty()) { - - String verdictMsg = String.format( - "Detected widgets %s with more than %d decimals!", - getDescriptionOfWidgets(incorrectDecimalWidgets, WdTags.WebTextContent), - maxDecimals - ); - - Visualizer visualizer = new RegionsVisualizer( - getRedPen(), - getWidgetRegions(incorrectDecimalWidgets), - "Invariant Fault", - 0.5, 0.5); - - return new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer); + if (!verdicts.isEmpty()) { + return verdicts; } - - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } private boolean isNumeric(String strNum) { diff --git a/testar/src/org/testar/oracles/web/invariants/WebInvariantTextAreaWithoutLength.java b/testar/src/org/testar/oracles/web/invariants/WebInvariantTextAreaWithoutLength.java index 493736868..69de672bd 100644 --- a/testar/src/org/testar/oracles/web/invariants/WebInvariantTextAreaWithoutLength.java +++ b/testar/src/org/testar/oracles/web/invariants/WebInvariantTextAreaWithoutLength.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2025 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2025 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,7 @@ package org.testar.oracles.web.invariants; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.testar.monkey.alayer.Role; @@ -63,32 +64,28 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { - List textAreasWithoutLength = new ArrayList<>(); + public List getVerdicts(State state) { + List verdicts = new ArrayList<>(); for (Widget w : state) { if (roles.contains(w.get(Tags.Role, Roles.Widget)) && w.get(WdTags.WebMaxLength, -1) == 0) { - textAreasWithoutLength.add(w); + String verdictMsg = String.format( + "Detected TextArea widget %s with 0 max length!", + getDescriptionOfWidgets(Collections.singletonList(w), WdTags.WebOuterHTML) + ); + Visualizer visualizer = new RegionsVisualizer( + getRedPen(), + getWidgetRegions(Collections.singletonList(w)), + "Invariant Fault", + 0.5, 0.5); + + verdicts.add(new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer)); } } - // If exists one or more incorrect widgets - if (!textAreasWithoutLength.isEmpty()) { - - String verdictMsg = String.format( - "Detected TextArea widgets %s with 0 max length!", - getDescriptionOfWidgets(textAreasWithoutLength, WdTags.WebOuterHTML) - ); - - Visualizer visualizer = new RegionsVisualizer( - getRedPen(), - getWidgetRegions(textAreasWithoutLength), - "Invariant Fault", - 0.5, 0.5); - - return new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer); + if (!verdicts.isEmpty()) { + return verdicts; } - - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } } diff --git a/testar/src/org/testar/oracles/web/invariants/WebInvariantUnsortedSelectItems.java b/testar/src/org/testar/oracles/web/invariants/WebInvariantUnsortedSelectItems.java index 526ce8698..acb24e1ff 100644 --- a/testar/src/org/testar/oracles/web/invariants/WebInvariantUnsortedSelectItems.java +++ b/testar/src/org/testar/oracles/web/invariants/WebInvariantUnsortedSelectItems.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2025 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2025 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -71,8 +71,8 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { - List unsortedSelectWidgets = new ArrayList<>(); + public List getVerdicts(State state) { + List verdicts = new ArrayList<>(); for (Widget w : state) { if (roles.contains(w.get(Tags.Role, Roles.Widget)) && !w.get(WdTags.WebId, "").isEmpty()) { @@ -89,7 +89,17 @@ public Verdict getVerdict(State state) { // Check if the options are sorted if (selectOptionsList != null && !isSorted(selectOptionsList)) { - unsortedSelectWidgets.add(w); + String verdictMsg = String.format( + "Detected Select widget %s with unsorted elements!", + getDescriptionOfWidgets(Collections.singletonList(w), WdTags.WebId) + ); + Visualizer visualizer = new RegionsVisualizer( + getRedPen(), + getWidgetRegions(Collections.singletonList(w)), + "Invariant Fault", + 0.5, 0.5); + + verdicts.add(new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer)); } } catch (Exception e) { // Ignore webdriver execute script errors @@ -97,23 +107,10 @@ public Verdict getVerdict(State state) { } } - if (!unsortedSelectWidgets.isEmpty()) { - - String verdictMsg = String.format( - "Detected Select widgets %s with unsorted elements!", - getDescriptionOfWidgets(unsortedSelectWidgets, WdTags.WebId) - ); - - Visualizer visualizer = new RegionsVisualizer( - getRedPen(), - getWidgetRegions(unsortedSelectWidgets), - "Invariant Fault", - 0.5, 0.5); - - return new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer); + if (!verdicts.isEmpty()) { + return verdicts; } - - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } // Helper method to check if the list of strings is sorted diff --git a/testar/src/org/testar/protocols/WebdriverProtocol.java b/testar/src/org/testar/protocols/WebdriverProtocol.java index 49934f1d2..113f3f822 100644 --- a/testar/src/org/testar/protocols/WebdriverProtocol.java +++ b/testar/src/org/testar/protocols/WebdriverProtocol.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2019 - 2025 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2019 - 2025 Open Universiteit - www.ou.nl + * Copyright (c) 2019 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2019 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -35,8 +35,6 @@ import org.apache.commons.lang3.ArrayUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.openqa.selenium.logging.LogEntries; -import org.openqa.selenium.logging.LogEntry; import org.testar.environment.Environment; import org.testar.monkey.*; import org.testar.monkey.alayer.*; @@ -53,6 +51,7 @@ import org.testar.monkey.alayer.windows.WinApiException; import org.testar.monkey.alayer.windows.WinProcess; import org.testar.monkey.alayer.windows.Windows; +import org.testar.oracles.log.WebBrowserConsoleOracle; import org.testar.plugin.NativeLinker; import org.testar.serialisation.LogSerialiser; import org.testar.settings.Settings; @@ -61,8 +60,6 @@ import java.net.MalformedURLException; import java.net.URL; import java.util.*; -import java.util.logging.Level; -import java.util.regex.Matcher; import java.util.regex.Pattern; import static org.testar.monkey.alayer.Tags.Blocked; import static org.testar.monkey.alayer.Tags.Enabled; @@ -88,9 +85,8 @@ public class WebdriverProtocol extends GenericUtilsProtocol { // List of attributes to identify and close policy popups protected Multimap policyAttributes = ArrayListMultimap.create(); - - // Verdict obtained from messages coming from the web browser console - protected Verdict webConsoleVerdict = Verdict.OK; + // Oracle to obtain verdicts from messages coming from the web browser console + protected WebBrowserConsoleOracle webBrowserConsoleOracle; /** * Called once during the life time of TESTAR @@ -139,6 +135,8 @@ protected void initialize(Settings settings){ // List of web attributes that TESTAR should ignore when obtaining the web state Constants.setIgnoredAttributes(settings.get(ConfigTags.WebIgnoredAttributes)); + + webBrowserConsoleOracle = new WebBrowserConsoleOracle(settings); } /** @@ -147,8 +145,6 @@ protected void initialize(Settings settings){ @Override protected void preSequencePreparations() { super.preSequencePreparations(); - // reset web browser console verdict - webConsoleVerdict = Verdict.OK; } /** @@ -282,8 +278,8 @@ protected State getState(SUT system) throws StateBuildException { WinProcess.toForeground(system.get(Tags.PID), 0.3, 100); } catch (WinApiException wae) { logger.log(org.apache.logging.log4j.Level.WARN, wae); - Verdict verdict = new Verdict(Verdict.Severity.NOT_RESPONDING, "Unable to bring the browser to foreground!"); - state.set(Tags.OracleVerdict, verdict); + Verdict notRespondingVerdict = new Verdict(Verdict.Severity.NOT_RESPONDING, "Unable to bring the browser to foreground!"); + state.set(Tags.OracleVerdicts, Collections.singletonList(notRespondingVerdict)); } } @@ -291,56 +287,22 @@ protected State getState(SUT system) throws StateBuildException { } /** - * The getVerdict methods implements the online state oracles that - * examine the SUT's current state and returns an oracle verdict. + * The getVerdicts methods implements the online state oracles that + * examine the SUT's current state and returns a list of oracle verdicts. * - * @return oracle verdict, which determines whether the state is erroneous and why. + * @return list of oracle verdicts */ @Override - protected Verdict getVerdict(State state) { - Verdict stateVerdict = super.getVerdict(state); - - LogEntries logEntries = WdDriver.getBrowserLogs(); - - // If Web Console Error Oracle is enabled and we have some pattern to match - if(settings.get(ConfigTags.WebConsoleErrorOracle, false) && !settings.get(ConfigTags.WebConsoleErrorPattern, "").isEmpty()) { - // Load the web console error pattern - Pattern errorPattern = Pattern.compile(settings.get(ConfigTags.WebConsoleErrorPattern), Pattern.UNICODE_CHARACTER_CLASS); - // Check Severe messages in the WebDriver logs - for(LogEntry logEntry : logEntries) { - if(logEntry.getLevel().equals(Level.SEVERE)) { - // Check if the severe error message matches with the web console error pattern - String consoleErrorMsg = logEntry.getMessage(); - Matcher matcherError = errorPattern.matcher(consoleErrorMsg); - if(matcherError.matches()) { - webConsoleVerdict = new Verdict(Verdict.Severity.SUSPICIOUS_TAG, "Web Browser Console Error: " + consoleErrorMsg); - } - } - } - // Join GUI verdict with WebDriver console verdict - stateVerdict = stateVerdict.join(webConsoleVerdict); - } - - // If Web Console Warning Oracle is enabled and we have some pattern to match - if(settings.get(ConfigTags.WebConsoleWarningOracle, false) && !settings.get(ConfigTags.WebConsoleWarningPattern, "").isEmpty()) { - // Load the web console warning pattern - Pattern warningPattern = Pattern.compile(settings.get(ConfigTags.WebConsoleWarningPattern), Pattern.UNICODE_CHARACTER_CLASS); - // Check Warning messages in the WebDriver logs - for(LogEntry logEntry : logEntries) { - if(logEntry.getLevel().equals(Level.WARNING)) { - // Check if the warning message matches with the web console error pattern - String consoleWarningMsg = logEntry.getMessage(); - Matcher matcherWarning = warningPattern.matcher(consoleWarningMsg); - if(matcherWarning.matches()) { - webConsoleVerdict = new Verdict(Verdict.Severity.SUSPICIOUS_TAG, "Web Browser Console Warning: " + consoleWarningMsg); - } - } - } - // Join GUI verdict with WebDriver console verdict - stateVerdict = stateVerdict.join(webConsoleVerdict); - } + protected List getVerdicts(State state) { + List verdicts = super.getVerdicts(state); + List browserConsoleVerdicts = webBrowserConsoleOracle.getVerdicts(state); + for (Verdict browserVerdict : browserConsoleVerdicts) { + if (browserVerdict.severity() > Verdict.OK.severity()) { + verdicts.add(browserVerdict); + } + } - return stateVerdict; + return verdicts; } /** diff --git a/testar/src/org/testar/reporting/DummyReportManager.java b/testar/src/org/testar/reporting/DummyReportManager.java index c82150089..eba382ae8 100644 --- a/testar/src/org/testar/reporting/DummyReportManager.java +++ b/testar/src/org/testar/reporting/DummyReportManager.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2025 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2025 Open Universiteit - www.ou.nl + * Copyright (c) 2025 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2025 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -30,6 +30,7 @@ package org.testar.reporting; +import java.util.List; import java.util.Set; import org.testar.monkey.alayer.Action; @@ -59,7 +60,7 @@ public void addSelectedAction(State state, Action action) { } @Override - public void addTestVerdict(Verdict verdict) { + public void addTestVerdicts(List verdicts) { } diff --git a/testar/src/org/testar/reporting/HtmlReporter.java b/testar/src/org/testar/reporting/HtmlReporter.java index 5639797d2..54ff6c473 100644 --- a/testar/src/org/testar/reporting/HtmlReporter.java +++ b/testar/src/org/testar/reporting/HtmlReporter.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2018 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2018 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2018 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -39,9 +39,11 @@ import org.testar.monkey.alayer.Verdict; import org.testar.monkey.alayer.webdriver.enums.WdTags; +import java.io.File; import java.time.Instant; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.List; import java.util.Set; import java.util.StringJoiner; @@ -49,6 +51,8 @@ public class HtmlReporter implements Reporting { private HtmlFormatUtil htmlReportUtil; private int innerLoopCounter = 0; + private boolean deleteBaseReport = false; + private String baseReportPath; private final String openStateBlockContainer = "
"; private final String openDerivedBlockContainer = "
"; @@ -252,25 +256,53 @@ public void addSelectedAction(State state, Action action) } @Override - public void addTestVerdict(Verdict verdict) + public void addTestVerdicts(List verdicts) { + String baseFilePath = htmlReportUtil.getFile().getAbsolutePath(); + deleteBaseReport = true; + baseReportPath = baseFilePath; + int index = 1; + for (Verdict verdict : verdicts) { + String suffixName = String.format("_V%03d_%s", index++, verdict.verdictSeverityTitle()); + String verdictFilePath = appendSuffixToFile(baseFilePath, suffixName); + + htmlReportUtil.duplicateFile(verdictFilePath); + + HtmlFormatUtil verdictUtil = new HtmlFormatUtil(verdictFilePath); + addVerdictBlock(verdictUtil, verdict); + verdictUtil.addContent("
"); + verdictUtil.addFooter(); + verdictUtil.writeToFile(); + } + } + + private void addVerdictBlock(HtmlFormatUtil util, Verdict verdict) { String verdictInfo = StringEscapeUtils.escapeHtml(verdict.info()); if(verdict.severity() > Verdict.OK.severity()) verdictInfo = verdictInfo.replace(Verdict.OK.info(), "").replace("\n", ""); - htmlReportUtil.addContent(openVerdictBlockContainer); // Open verdict block container - htmlReportUtil.addHeading(2, "Test verdict for this sequence: " + verdictInfo); - htmlReportUtil.addHeading(4, "Severity: " + verdict.severity()); - htmlReportUtil.addContent(""); - htmlReportUtil.addContent(closeContainer); // Close verdict block container + util.addContent(openVerdictBlockContainer); // Open verdict block container + util.addHeading(2, "Test verdict for this sequence: " + verdictInfo); + util.addHeading(4, "Severity: " + verdict.severity()); + util.addContent(""); + util.addContent(closeContainer); // Close verdict block container + } - htmlReportUtil.appendToFileName("_" + verdict.verdictSeverityTitle()); - htmlReportUtil.writeToFile(); + private String appendSuffixToFile(String filePath, String suffixName) { + int dotIndex = filePath.lastIndexOf('.'); + if (dotIndex == -1) { + return filePath + suffixName; + } + return filePath.substring(0, dotIndex) + suffixName + filePath.substring(dotIndex); } @Override public void finishReport() { + if (deleteBaseReport && baseReportPath != null) { + new File(baseReportPath).delete(); + return; + } htmlReportUtil.addContent("
"); // Close the main div container htmlReportUtil.addFooter(); diff --git a/testar/src/org/testar/reporting/PlainTextReporter.java b/testar/src/org/testar/reporting/PlainTextReporter.java index 5aca0f4d8..e887ca4e2 100644 --- a/testar/src/org/testar/reporting/PlainTextReporter.java +++ b/testar/src/org/testar/reporting/PlainTextReporter.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2024 Open Universiteit - www.ou.nl - * Copyright (c) 2024 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2024 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2024 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -39,6 +39,7 @@ import org.testar.monkey.alayer.Verdict; import java.util.ArrayList; +import java.util.List; import java.util.Set; import java.util.StringJoiner; @@ -46,6 +47,8 @@ public class PlainTextReporter implements Reporting { private PlainTextFormatUtil plainTextReportUtil; private int innerLoopCounter = 0; + private boolean deleteBaseReport = false; + private String baseReportPath; public PlainTextReporter(String fileName, boolean replay) //replay or generate mode { plainTextReportUtil = new PlainTextFormatUtil(fileName); @@ -193,25 +196,51 @@ public void addSelectedAction(State state, Action action) plainTextReportUtil.writeToFile(); } - + @Override - public void addTestVerdict(Verdict verdict) + public void addTestVerdicts(List verdicts) { + String baseFilePath = plainTextReportUtil.getFile().getAbsolutePath(); + deleteBaseReport = true; + baseReportPath = baseFilePath; + int index = 1; + for (Verdict verdict : verdicts) { + String suffixName = String.format("_V%03d_%s", index++, verdict.verdictSeverityTitle()); + String verdictFilePath = appendSuffixToFile(baseFilePath, suffixName); + + plainTextReportUtil.duplicateFile(verdictFilePath); + + PlainTextFormatUtil verdictUtil = new PlainTextFormatUtil(verdictFilePath); + addVerdictBlock(verdictUtil, verdict); + verdictUtil.writeToFile(); + } + } + + private void addVerdictBlock(PlainTextFormatUtil util, Verdict verdict) { String verdictInfo = verdict.info(); if(verdict.severity() > Verdict.OK.severity()) verdictInfo = verdictInfo.replace(Verdict.OK.info(), ""); - - plainTextReportUtil.addHorizontalLine(); - plainTextReportUtil.addHeading(3, "Test verdict for this sequence: " + verdictInfo); - plainTextReportUtil.addHeading(5, "Severity: " + verdict.severity()); - - plainTextReportUtil.appendToFileName("_" + verdict.verdictSeverityTitle()); - plainTextReportUtil.writeToFile(); + + util.addHorizontalLine(); + util.addHeading(3, "Test verdict for this sequence: " + verdictInfo); + util.addHeading(5, "Severity: " + verdict.severity()); + } + + private String appendSuffixToFile(String filePath, String suffixName) { + int dotIndex = filePath.lastIndexOf('.'); + if (dotIndex == -1) { + return filePath + suffixName; + } + return filePath.substring(0, dotIndex) + suffixName + filePath.substring(dotIndex); } @Override public void finishReport() { + if (deleteBaseReport && baseReportPath != null) { + new java.io.File(baseReportPath).delete(); + return; + } plainTextReportUtil.writeToFile(); } } diff --git a/testar/src/org/testar/reporting/ReportManager.java b/testar/src/org/testar/reporting/ReportManager.java index 9ca25097c..73357381b 100644 --- a/testar/src/org/testar/reporting/ReportManager.java +++ b/testar/src/org/testar/reporting/ReportManager.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2023 Open Universiteit - www.ou.nl - * Copyright (c) 2023 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2023 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2023 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -41,6 +41,8 @@ import java.io.File; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.Set; public class ReportManager implements Reporting @@ -94,7 +96,8 @@ public void addState(State state) { if(firstStateAdded) { - if(firstActionsAdded || (state.get(Tags.OracleVerdict, Verdict.OK).severity() > Verdict.Severity.OK.getValue())) + List verdicts = state.get(Tags.OracleVerdicts, Collections.singletonList(Verdict.OK)); + if(firstActionsAdded || !Verdict.helperAreAllVerdictsOK(verdicts)) { //if the first state contains a failure, write the same state in case it was a login for(Reporting reporter : reporters) reporter.addState(state); @@ -137,11 +140,11 @@ public void addSelectedAction(State state, Action action) for(Reporting reporter : reporters) reporter.addSelectedAction(state, action); } - - public void addTestVerdict(Verdict verdict) + + public void addTestVerdicts(List verdicts) { if(reportingEnabled) for(Reporting reporter : reporters) - reporter.addTestVerdict(verdict); + reporter.addTestVerdicts(verdicts); } } diff --git a/testar/src/org/testar/reporting/Reporting.java b/testar/src/org/testar/reporting/Reporting.java index bccbbb85f..96a14439b 100644 --- a/testar/src/org/testar/reporting/Reporting.java +++ b/testar/src/org/testar/reporting/Reporting.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2021 - 2023 Open Universiteit - www.ou.nl - * Copyright (c) 2021 - 2023 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2021 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2021 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -34,6 +34,7 @@ import org.testar.monkey.alayer.State; import org.testar.monkey.alayer.Verdict; +import java.util.List; import java.util.Set; public interface Reporting @@ -42,6 +43,6 @@ public interface Reporting public void addActions(Set actions); public void addActionsAndUnvisitedActions(Set actions, Set concreteIdsOfUnvisitedActions); public void addSelectedAction(State state, Action action); - public void addTestVerdict(Verdict verdict); + public void addTestVerdicts(List verdicts); public void finishReport(); } diff --git a/testar/src/org/testar/securityanalysis/helpers/SecurityOracleOrchestrator.java b/testar/src/org/testar/securityanalysis/helpers/SecurityOracleOrchestrator.java index 187d0bf65..57128a7d8 100644 --- a/testar/src/org/testar/securityanalysis/helpers/SecurityOracleOrchestrator.java +++ b/testar/src/org/testar/securityanalysis/helpers/SecurityOracleOrchestrator.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2022 Open Universiteit - www.ou.nl - * Copyright (c) 2022 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2022 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2022 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -38,6 +38,7 @@ import org.testar.securityanalysis.SecurityResultWriter; import org.testar.securityanalysis.oracles.*; +import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -118,16 +119,21 @@ private void addListeners(DevTools devTools) securityOracle.addListener(devTools); } - public Verdict getVerdict(Verdict verdict) + public List getVerdicts() { + List verdicts = new ArrayList<>(); for (BaseSecurityOracle securityOracle : securityOracles) { Verdict newVerdict = securityOracle.getVerdict(); - if (newVerdict != null) - verdict.join(newVerdict); + if (newVerdict != null && newVerdict.severity() > Verdict.Severity.OK.getValue()) { + verdicts.add(newVerdict); + } } - if (activeSecurityOracle != null) - verdict.join(activeSecurityOracle.getVerdict()); - - return verdict; + if (activeSecurityOracle != null) { + Verdict activeVerdict = activeSecurityOracle.getVerdict(); + if (activeVerdict != null && activeVerdict.severity() > Verdict.Severity.OK.getValue()) { + verdicts.add(activeVerdict); + } + } + return verdicts; } } diff --git a/testar/src/org/testar/settings/SettingsDefaults.java b/testar/src/org/testar/settings/SettingsDefaults.java index 4f7ca817b..eac5ab1d4 100644 --- a/testar/src/org/testar/settings/SettingsDefaults.java +++ b/testar/src/org/testar/settings/SettingsDefaults.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2023 - 2025 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2023 - 2025 Open Universiteit - www.ou.nl + * Copyright (c) 2023 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2023 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -49,7 +49,6 @@ private SettingsDefaults() {} defaults.add(Pair.from(ProcessesToKillDuringTest, "(?!x)x")); defaults.add(Pair.from(ShowVisualSettingsDialogOnStartup, true)); - defaults.add(Pair.from(FaultThreshold, 0.1)); defaults.add(Pair.from(LogLevel, 1)); defaults.add(Pair.from(Mode, RuntimeControlsProtocol.Modes.Spy)); defaults.add(Pair.from(OutputDir, Main.outputDir)); @@ -69,6 +68,7 @@ private SettingsDefaults() {} defaults.add(Pair.from(SUTConnectorValue, "")); defaults.add(Pair.from(Delete, new ArrayList())); defaults.add(Pair.from(CopyFromTo, new ArrayList>())); + defaults.add(Pair.from(IgnoreDuplicatedVerdicts, false)); defaults.add(Pair.from(SuspiciousTags, "(?!x)x")); defaults.add(Pair.from(ClickFilter, "(?!x)x")); defaults.add(Pair.from(MyClassPath, Arrays.asList(Main.settingsDir))); diff --git a/testar/src/org/testar/settings/SettingsFileStructure.java b/testar/src/org/testar/settings/SettingsFileStructure.java index 8097a4e84..e0f2e12db 100644 --- a/testar/src/org/testar/settings/SettingsFileStructure.java +++ b/testar/src/org/testar/settings/SettingsFileStructure.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2023 - 2025 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2023 - 2025 Open Universiteit - www.ou.nl + * Copyright (c) 2023 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2023 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -132,6 +132,12 @@ public static String getTestSettingsStructure() { , ConfigTags.ProcessesToKillDuringTest.name() + " = " , "" , "#################################################################" + , "# Ignore reporting duplicated verdicts for this protocol" + , "#################################################################" + , "" + , ConfigTags.IgnoreDuplicatedVerdicts.name() + " = " + , "" + , "#################################################################" , "# Oracles based on suspicious tag values" , "#" , "# Regular expression and Tags to apply them" diff --git a/testar/src/org/testar/settings/dialog/GeneralPanel.java b/testar/src/org/testar/settings/dialog/GeneralPanel.java index df85183f3..d05d40214 100644 --- a/testar/src/org/testar/settings/dialog/GeneralPanel.java +++ b/testar/src/org/testar/settings/dialog/GeneralPanel.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * -* Copyright (c) 2013 - 2024 Universitat Politecnica de Valencia - www.upv.es -* Copyright (c) 2018 - 2024 Open Universiteit - www.ou.nl +* Copyright (c) 2013 - 2026 Universitat Politecnica de Valencia - www.upv.es +* Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -33,6 +33,7 @@ import org.testar.monkey.*; import org.testar.settings.Settings; +import org.testar.settings.dialog.components.IgnoredVerdictsDialog; import org.testar.settings.dialog.components.UndoTextArea; import javax.swing.*; @@ -56,7 +57,8 @@ public class GeneralPanel extends SettingsPanel implements Observer { private JSpinner spnSequenceLength; //private JCheckBox checkStopOnFault; private JComboBox comboBoxProtocol; - private JCheckBox compileCheckBox, checkActionVisualization; + private JCheckBox compileCheckBox, checkActionVisualization, checkIgnoreDuplicatedVerdict; + private JButton btnManageIgnoredVerdicts; private JLabel labelAppName = new JLabel("Application name"); private JLabel labelAppVersion = new JLabel("Application version"); @@ -134,7 +136,18 @@ private void addGeneralControlsGlobal(SettingsDialog settingsDialog) { checkActionVisualization.setBounds(10, 240, 192, 21); //checkActionVisualization.setToolTipText(checkStopOnFaultTTT); add(checkActionVisualization); - + + checkIgnoreDuplicatedVerdict = new JCheckBox("Ignore duplicated verdicts"); + checkIgnoreDuplicatedVerdict.setBounds(10, 262, 192, 21); + checkIgnoreDuplicatedVerdict.setToolTipText(ConfigTags.IgnoreDuplicatedVerdicts.getDescription()); + add(checkIgnoreDuplicatedVerdict); + + btnManageIgnoredVerdicts = new JButton("Manage ignored verdicts"); + btnManageIgnoredVerdicts.setBounds(10, 286, 192, 25); + btnManageIgnoredVerdicts.setToolTipText("Open a modal dialog to check or clear existing ignored verdicts"); + btnManageIgnoredVerdicts.addActionListener(this::btnManageIgnoredVerdictsActionPerformed); + add(btnManageIgnoredVerdicts); + labelAppName.setBounds(330, 242, 150, 27); labelAppName.setToolTipText(ToolTipTexts.applicationNameTTT); add(labelAppName); @@ -242,6 +255,12 @@ private void btnEditProtocolActionPerformed(ActionEvent evt) { dialog.setVisible(true); } + private void btnManageIgnoredVerdictsActionPerformed(ActionEvent evt) { + JDialog dialog = new IgnoredVerdictsDialog(); + dialog.setModalityType(JDialog.ModalityType.APPLICATION_MODAL); + dialog.setVisible(true); + } + /** * Populate JPanelGeneral from Settings structure. * @@ -254,6 +273,7 @@ public void populateFrom(final Settings settings) { cboxSUTconnector.setSelectedItem(settings.get(ConfigTags.SUTConnector)); //checkStopOnFault.setSelected(settings.get(ConfigTags.StopGenerationOnFault)); checkActionVisualization.setSelected(settings.get(ConfigTags.VisualizeActions)); + checkIgnoreDuplicatedVerdict.setSelected(settings.get(ConfigTags.IgnoreDuplicatedVerdicts)); txtSutPath.setInitialText(settings.get(ConfigTags.SUTConnectorValue)); comboBoxProtocol.setSelectedItem(settings.get(ConfigTags.ProtocolClass).split("/")[0]); spnNumSequences.setValue(settings.get(ConfigTags.Sequences)); @@ -275,6 +295,7 @@ public void extractInformation(final Settings settings) { settings.set(ConfigTags.SUTConnectorValue, txtSutPath.getText()); //settings.set(ConfigTags.StopGenerationOnFault, checkStopOnFault.isSelected()); settings.set(ConfigTags.VisualizeActions, checkActionVisualization.isSelected()); + settings.set(ConfigTags.IgnoreDuplicatedVerdicts, checkIgnoreDuplicatedVerdict.isSelected()); settings.set(ConfigTags.Sequences, (Integer) spnNumSequences.getValue()); settings.set(ConfigTags.SequenceLength, (Integer) spnSequenceLength.getValue()); settings.set(ConfigTags.AlwaysCompile, compileCheckBox.isSelected()); diff --git a/testar/src/org/testar/settings/dialog/ToolTipTexts.java b/testar/src/org/testar/settings/dialog/ToolTipTexts.java index 79c10b66f..8aca78066 100644 --- a/testar/src/org/testar/settings/dialog/ToolTipTexts.java +++ b/testar/src/org/testar/settings/dialog/ToolTipTexts.java @@ -52,8 +52,6 @@ public class ToolTipTexts { "generation. This is ideal if a sequence turns out not to be reproducible.\n"; public static String btnModelTTT = "\nStart in State Model Analysis Mode:
\n" + "This mode allows you to connect with OrientDB to inspect the inferred models.\n"; - public static String btnRecordTTT = "\nStart in RECORD Mode:
\n" + - "This modes enables the tester to manually record (part of) a sequence.\n"; // TTTs for the general tab public static String sutConnectorTTT = "How does TESTAR connect to the SUT"; diff --git a/testar/src/org/testar/settings/dialog/components/IgnoredVerdictsDialog.java b/testar/src/org/testar/settings/dialog/components/IgnoredVerdictsDialog.java new file mode 100644 index 000000000..2f3ad0e9b --- /dev/null +++ b/testar/src/org/testar/settings/dialog/components/IgnoredVerdictsDialog.java @@ -0,0 +1,157 @@ +/*************************************************************************************************** + * + * Copyright (c) 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2026 Open Universiteit - www.ou.nl + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + *******************************************************************************************************/ + +package org.testar.settings.dialog.components; + +import org.testar.monkey.VerdictProcessing; + +import javax.swing.JButton; +import javax.swing.DefaultListModel; +import javax.swing.JDialog; +import javax.swing.JList; +import javax.swing.JOptionPane; +import javax.swing.JPanel; +import javax.swing.JScrollPane; +import javax.swing.ListSelectionModel; +import java.awt.BorderLayout; +import java.awt.Dimension; +import java.awt.FlowLayout; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.List; + +public class IgnoredVerdictsDialog extends JDialog { + + private final DefaultListModel listModel = new DefaultListModel<>(); + private final JList verdictList = new JList<>(listModel); + private File ignoreFile; + + public IgnoredVerdictsDialog() { + setTitle("Ignored Verdicts"); + setSize(new Dimension(700, 420)); + setLocationRelativeTo(null); + setDefaultCloseOperation(DISPOSE_ON_CLOSE); + setLayout(new BorderLayout()); + + verdictList.setSelectionMode(ListSelectionModel.MULTIPLE_INTERVAL_SELECTION); + add(new JScrollPane(verdictList), BorderLayout.CENTER); + + JPanel buttonPanel = new JPanel(new FlowLayout(FlowLayout.RIGHT)); + JButton removeSelectedButton = new JButton("Remove Selected"); + removeSelectedButton.addActionListener(e -> removeSelectedIgnoredVerdicts()); + JButton clearAllButton = new JButton("Clear All"); + clearAllButton.addActionListener(e -> clearAllIgnoredVerdicts()); + JButton closeButton = new JButton("Close"); + closeButton.addActionListener(e -> dispose()); + buttonPanel.add(removeSelectedButton); + buttonPanel.add(clearAllButton); + buttonPanel.add(closeButton); + add(buttonPanel, BorderLayout.SOUTH); + + loadIgnoredVerdicts(); + } + + private void loadIgnoredVerdicts() { + listModel.clear(); + ignoreFile = VerdictProcessing.resolveVerdictIgnoreFile(); + if (ignoreFile == null) { + listModel.addElement("Ignored verdict file path is not available."); + return; + } + try { + File parent = ignoreFile.getParentFile(); + if (parent != null && !parent.exists()) { + parent.mkdirs(); + } + if (!ignoreFile.exists()) { + ignoreFile.createNewFile(); + } + List lines = Files.readAllLines(ignoreFile.toPath(), StandardCharsets.UTF_8); + boolean hasEntries = false; + for (String line : lines) { + String trimmed = line.trim(); + if (!trimmed.isEmpty()) { + listModel.addElement(trimmed); + hasEntries = true; + } + } + if (!hasEntries) { + listModel.addElement("(No ignored verdicts yet)"); + } + } catch (IOException ex) { + listModel.addElement("Unable to load ignored verdicts file: " + ex.getMessage()); + } + } + + private void clearAllIgnoredVerdicts() { + if (ignoreFile == null) { + JOptionPane.showMessageDialog(this, "Ignored verdict file path is not available.", "Ignored Verdicts", JOptionPane.WARNING_MESSAGE); + return; + } + try { + Files.writeString(ignoreFile.toPath(), "", StandardCharsets.UTF_8); + loadIgnoredVerdicts(); + } catch (IOException ex) { + JOptionPane.showMessageDialog(this, "Unable to clear ignored verdicts:\n" + ex.getMessage(), "Ignored Verdicts", JOptionPane.ERROR_MESSAGE); + } + } + + private void removeSelectedIgnoredVerdicts() { + if (ignoreFile == null) { + JOptionPane.showMessageDialog(this, "Ignored verdict file path is not available.", "Ignored Verdicts", JOptionPane.WARNING_MESSAGE); + return; + } + List selected = verdictList.getSelectedValuesList(); + if (selected == null || selected.isEmpty() || selected.contains("(No ignored verdicts yet)")) { + return; + } + try { + List current = Files.readAllLines(ignoreFile.toPath(), StandardCharsets.UTF_8); + List remaining = new ArrayList<>(); + for (String line : current) { + String trimmed = line.trim(); + if (trimmed.isEmpty()) { + continue; + } + if (!selected.contains(trimmed)) { + remaining.add(trimmed); + } + } + Files.write(ignoreFile.toPath(), remaining, StandardCharsets.UTF_8, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE); + loadIgnoredVerdicts(); + } catch (IOException ex) { + JOptionPane.showMessageDialog(this, "Unable to remove selected verdicts:\n" + ex.getMessage(), "Ignored Verdicts", JOptionPane.ERROR_MESSAGE); + } + } +} diff --git a/testar/test/org/testar/TestFileHandling.java b/testar/test/org/testar/TestFileHandling.java index 116bfffdc..d1e54166b 100644 --- a/testar/test/org/testar/TestFileHandling.java +++ b/testar/test/org/testar/TestFileHandling.java @@ -28,31 +28,41 @@ public void setUp() throws IOException { @Test public void testCopyClassifiedSequence_okSeverity() throws IOException { - FileHandling.copyClassifiedSequence("okSequence", testFile, Verdict.OK); + String suffixName = "_V001_OK"; + FileHandling.copyClassifiedSequence("okSequence", testFile, Verdict.OK, suffixName); File targetDir = new File(outputDir, "sequences_ok"); assertTrue("OK sequence folder should be created", targetDir.exists()); - assertTrue("Copied file should exist in sequences_ok", new File(targetDir, testFile.getName()).exists()); + assertTrue("Copied file should exist in sequences_ok", new File(targetDir, appendSuffix(testFile.getName(), suffixName)).exists()); } @Test public void testCopyClassifiedSequence_failSeverity() throws IOException { - FileHandling.copyClassifiedSequence("failSequence", testFile, Verdict.FAIL); + String suffixName = "_V001_FAIL"; + FileHandling.copyClassifiedSequence("failSequence", testFile, Verdict.FAIL, suffixName); File targetDir = new File(outputDir, "sequences_fail"); assertTrue("Fail sequence folder should be created", targetDir.exists()); - assertTrue("Copied file should exist in sequences_fail", new File(targetDir, testFile.getName()).exists()); + assertTrue("Copied file should exist in sequences_fail", new File(targetDir, appendSuffix(testFile.getName(), suffixName)).exists()); } @Test public void testCopyClassifiedSequence_suspiciousLogSeverity() throws IOException { Verdict verdict = new Verdict(Verdict.Severity.SUSPICIOUS_LOG, "Suspicious log entry found"); + String suffixName = "_V001_SUSPICIOUS_LOG"; - FileHandling.copyClassifiedSequence("logSequence", testFile, verdict); + FileHandling.copyClassifiedSequence("logSequence", testFile, verdict, suffixName); File targetDir = new File(outputDir, "sequences_suspicious_log"); assertTrue("Suspicious log folder should be created", targetDir.exists()); - assertTrue("Copied file should exist in sequences_suspicious_log", new File(targetDir, testFile.getName()).exists()); + assertTrue("Copied file should exist in sequences_suspicious_log", new File(targetDir, appendSuffix(testFile.getName(), suffixName)).exists()); + } + + private String appendSuffix(String fileName, String suffix) { + int dotIndex = fileName.lastIndexOf('.'); + return (dotIndex == -1) + ? fileName + suffix + : fileName.substring(0, dotIndex) + suffix + fileName.substring(dotIndex); } } diff --git a/testar/test/org/testar/monkey/TestMoreActionsLogic.java b/testar/test/org/testar/monkey/TestMoreActionsLogic.java index 7d0f412a0..6b5302247 100644 --- a/testar/test/org/testar/monkey/TestMoreActionsLogic.java +++ b/testar/test/org/testar/monkey/TestMoreActionsLogic.java @@ -1,6 +1,7 @@ package org.testar.monkey; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Properties; @@ -27,7 +28,7 @@ public void test_continue_more_actions() { protocol.startTime = Util.time(); StateStub state = new StateStub(); - state.set(Tags.OracleVerdict, new Verdict(Verdict.Severity.OK, "State is OK")); + state.set(Tags.OracleVerdicts, Collections.singletonList(new Verdict(Verdict.Severity.OK, "State is OK"))); state.set(Tags.IsRunning, true); state.set(Tags.NotResponding, false); @@ -50,7 +51,7 @@ public void test_stop_max_time_reached() { protocol.startTime = Util.time() - 70.0; StateStub state = new StateStub(); - state.set(Tags.OracleVerdict, new Verdict(Verdict.Severity.OK, "State is OK")); + state.set(Tags.OracleVerdicts, Collections.singletonList(new Verdict(Verdict.Severity.OK, "State is OK"))); state.set(Tags.IsRunning, true); state.set(Tags.NotResponding, false); @@ -73,7 +74,7 @@ public void test_stop_more_actions_due_verdict() { protocol.startTime = Util.time(); StateStub state = new StateStub(); - state.set(Tags.OracleVerdict, new Verdict(Verdict.Severity.SUSPICIOUS_TAG, "State contains suspicious message")); + state.set(Tags.OracleVerdicts, Collections.singletonList(new Verdict(Verdict.Severity.SUSPICIOUS_TAG, "State contains suspicious message"))); state.set(Tags.IsRunning, true); state.set(Tags.NotResponding, false); @@ -96,7 +97,7 @@ public void test_more_actions_do_not_stop_fault() { protocol.startTime = Util.time(); StateStub state = new StateStub(); - state.set(Tags.OracleVerdict, new Verdict(Verdict.Severity.SUSPICIOUS_TAG, "State contains suspicious message")); + state.set(Tags.OracleVerdicts, Collections.singletonList(new Verdict(Verdict.Severity.SUSPICIOUS_TAG, "State contains suspicious message"))); state.set(Tags.IsRunning, true); state.set(Tags.NotResponding, false); @@ -119,7 +120,7 @@ public void test_stop_not_running() { protocol.startTime = Util.time(); StateStub state = new StateStub(); - state.set(Tags.OracleVerdict, new Verdict(Verdict.Severity.OK, "State is OK")); + state.set(Tags.OracleVerdicts, Collections.singletonList(new Verdict(Verdict.Severity.OK, "State is OK"))); state.set(Tags.IsRunning, false); state.set(Tags.NotResponding, false); @@ -142,7 +143,7 @@ public void test_stop_not_responding() { protocol.startTime = Util.time(); StateStub state = new StateStub(); - state.set(Tags.OracleVerdict, new Verdict(Verdict.Severity.OK, "State is OK")); + state.set(Tags.OracleVerdicts, Collections.singletonList(new Verdict(Verdict.Severity.OK, "State is OK"))); state.set(Tags.IsRunning, true); state.set(Tags.NotResponding, true); diff --git a/testar/test/org/testar/monkey/VerdictProcessingTest.java b/testar/test/org/testar/monkey/VerdictProcessingTest.java new file mode 100644 index 000000000..3b10f0381 --- /dev/null +++ b/testar/test/org/testar/monkey/VerdictProcessingTest.java @@ -0,0 +1,121 @@ +package org.testar.monkey; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.junit.After; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.testar.monkey.alayer.Verdict; +import org.testar.settings.Settings; + +public class VerdictProcessingTest { + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + @After + public void tearDown() { + Settings.setSettingsPath(null); + Main.SSE_ACTIVATED = null; + } + + @Test + public void testNullListReturnsOk() { + Settings settings = new Settings(); + settings.set(ConfigTags.IgnoreDuplicatedVerdicts, false); + VerdictProcessing processing = new VerdictProcessing(settings); + + List filtered = processing.filterDuplicates(null); + assertEquals(1, filtered.size()); + assertEquals(Verdict.OK.severity(), filtered.get(0).severity(), 0.0); + } + + @Test + public void testMultipleOkReturnsSingleOk() { + Settings settings = new Settings(); + settings.set(ConfigTags.IgnoreDuplicatedVerdicts, false); + VerdictProcessing processing = new VerdictProcessing(settings); + + List filtered = processing.filterDuplicates(Arrays.asList(Verdict.OK, Verdict.OK)); + assertEquals(1, filtered.size()); + assertEquals(Verdict.OK.severity(), filtered.get(0).severity(), 0.0); + } + + @Test + public void testOkAndFailureRemovesOk() { + Settings settings = new Settings(); + settings.set(ConfigTags.IgnoreDuplicatedVerdicts, false); + VerdictProcessing processing = new VerdictProcessing(settings); + + Verdict failure = new Verdict(Verdict.Severity.SUSPICIOUS_TAG, "Issue"); + Verdict logVerdict = new Verdict(Verdict.Severity.SUSPICIOUS_LOG, "Log exception"); + List filtered = processing.filterDuplicates(Arrays.asList(Verdict.OK, failure, logVerdict)); + assertEquals(2, filtered.size()); + assertTrue(filtered.get(0).severity() > Verdict.OK.severity()); + assertTrue(filtered.get(1).severity() > Verdict.OK.severity()); + } + + @Test + public void testIgnoresKnownDuplicate() throws Exception { + File settingsDir = tempFolder.newFolder("settings"); + Settings.setSettingsPath(settingsDir.getAbsolutePath()); + + File ignoreFile = new File(settingsDir, "list_of_verdicts_with_failures.txt"); + Files.write(ignoreFile.toPath(), Collections.singletonList("duplicate message"), StandardCharsets.UTF_8); + + Settings settings = new Settings(); + settings.set(ConfigTags.IgnoreDuplicatedVerdicts, true); + VerdictProcessing processing = new VerdictProcessing(settings); + + Verdict duplicate = new Verdict(Verdict.Severity.SUSPICIOUS_TAG, "duplicate message"); + List filtered = processing.filterDuplicates(Collections.singletonList(duplicate)); + assertEquals(1, filtered.size()); + assertEquals(Verdict.OK.severity(), filtered.get(0).severity(), 0.0); + } + + @Test + public void testVerdictIgnoreFile_PrioritizesSSE() throws Exception { + File tempSettingsDir = tempFolder.newFolder("tempSettingsDir"); + String originalSettingsDir = Main.settingsDir; + try { + Main.settingsDir = tempSettingsDir.getAbsolutePath() + File.separator; + Main.SSE_ACTIVATED = "protocol_selected"; + Settings.setSettingsPath(tempFolder.newFolder("otherProtocol").getAbsolutePath()); + + File verdictIgnoreFile = VerdictProcessing.resolveVerdictIgnoreFile(); + assertEquals(new File(Main.settingsDir + "protocol_selected", "list_of_verdicts_with_failures.txt").getAbsolutePath(), + verdictIgnoreFile.getAbsolutePath()); + } finally { + Main.settingsDir = originalSettingsDir; // cleanup to restore static global dir + } + } + + @Test + public void testVerdictIgnoreFile_UsesSettingsPathWhenNoSSE() throws Exception { + File tempSettingsDir = tempFolder.newFolder("tempSettingsDir"); + Main.SSE_ACTIVATED = null; + Settings.setSettingsPath(tempSettingsDir.getAbsolutePath()); + + File verdictIgnoreFile = VerdictProcessing.resolveVerdictIgnoreFile(); + assertEquals(new File(tempSettingsDir, "list_of_verdicts_with_failures.txt").getAbsolutePath(), + verdictIgnoreFile.getAbsolutePath()); + } + + @Test + public void testVerdictIgnoreFile_IsNullWhenNoContext() { + Main.SSE_ACTIVATED = null; + Settings.setSettingsPath(null); + + File verdictIgnoreFile = VerdictProcessing.resolveVerdictIgnoreFile(); + assertEquals(null, verdictIgnoreFile); + } +} diff --git a/testar/test/org/testar/oracles/log/TestAndroidLogcatOracle.java b/testar/test/org/testar/oracles/log/TestAndroidLogcatOracle.java index 1f249d798..d4b971b78 100644 --- a/testar/test/org/testar/oracles/log/TestAndroidLogcatOracle.java +++ b/testar/test/org/testar/oracles/log/TestAndroidLogcatOracle.java @@ -79,7 +79,9 @@ public void generateModeVerdict_DetectsRegexAndReturnsSuspiciousLog() { ); androidLogcatOracle.initialize(); - Verdict verdict = androidLogcatOracle.getVerdict(state); + List verdicts = androidLogcatOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.assertEquals(Verdict.Severity.SUSPICIOUS_LOG.getValue(), verdict.severity(), 0.0); Assert.assertEquals(verdict.info(), "Suspicious Android logcat line(s) detected AndroidRuntime: FATAL EXCEPTION: main"); @@ -108,12 +110,16 @@ public void generateModeVerdict_ProcessesOnlyNewLinesAcrossCalls() { androidLogcatOracle.initialize(); - Verdict first = androidLogcatOracle.getVerdict(state); - Assert.assertEquals(Verdict.Severity.OK.getValue(), first.severity(), 0.0); + List firstInvokationVerdicts = androidLogcatOracle.getVerdicts(state); + Assert.assertEquals(1, firstInvokationVerdicts.size()); + Verdict firstVerdict = firstInvokationVerdicts.get(0); + Assert.assertEquals(Verdict.Severity.OK.getValue(), firstVerdict.severity(), 0.0); - Verdict second = androidLogcatOracle.getVerdict(state); - Assert.assertEquals(Verdict.Severity.SUSPICIOUS_LOG.getValue(), second.severity(), 0.0); - Assert.assertEquals(second.info(), "Suspicious Android logcat line(s) detected MyTag: Exception happened"); + List secondInvokationVerdicts = androidLogcatOracle.getVerdicts(state); + Assert.assertEquals(1, secondInvokationVerdicts.size()); + Verdict secondVerdict = secondInvokationVerdicts.get(0); + Assert.assertEquals(Verdict.Severity.SUSPICIOUS_LOG.getValue(), secondVerdict.severity(), 0.0); + Assert.assertEquals(secondVerdict.info(), "Suspicious Android logcat line(s) detected MyTag: Exception happened"); } } @@ -136,7 +142,9 @@ public void generateModeVerdict_DeduplicatesAndOrdersMatches() { .thenReturn(lineB + "\n" + lineA + "\n" + lineB); androidLogcatOracle.initialize(); - Verdict verdict = androidLogcatOracle.getVerdict(state); + List verdicts = androidLogcatOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.assertEquals(Verdict.Severity.SUSPICIOUS_LOG.getValue(), verdict.severity(), 0.0); String expected = "Suspicious Android logcat line(s) detected " @@ -165,7 +173,9 @@ public void generateModeVerdict_DeduplicatesNumbersInMatches() { .thenReturn(line1 + "\n" + line2); androidLogcatOracle.initialize(); - Verdict verdict = androidLogcatOracle.getVerdict(state); + List verdicts = androidLogcatOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.assertEquals(Verdict.Severity.SUSPICIOUS_LOG.getValue(), verdict.severity(), 0.0); String expected = "Suspicious Android logcat line(s) detected " @@ -193,7 +203,9 @@ public void generateModeVerdict_KeepsHttpStatusCodes() { .thenReturn(line1 + "\n" + line2); androidLogcatOracle.initialize(); - Verdict verdict = androidLogcatOracle.getVerdict(state); + List verdicts = androidLogcatOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.assertEquals(Verdict.Severity.SUSPICIOUS_LOG.getValue(), verdict.severity(), 0.0); String expected = "Suspicious Android logcat line(s) detected " diff --git a/testar/test/org/testar/protocols/TestBrowserConsoleVerdict.java b/testar/test/org/testar/oracles/log/TestBrowserConsoleVerdict.java similarity index 58% rename from testar/test/org/testar/protocols/TestBrowserConsoleVerdict.java rename to testar/test/org/testar/oracles/log/TestBrowserConsoleVerdict.java index 4dc1edeaf..4dae11ba9 100644 --- a/testar/test/org/testar/protocols/TestBrowserConsoleVerdict.java +++ b/testar/test/org/testar/oracles/log/TestBrowserConsoleVerdict.java @@ -1,9 +1,10 @@ -package org.testar.protocols; +package org.testar.oracles.log; import static org.mockito.Mockito.*; import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.logging.Level; import org.junit.Before; import org.junit.Test; @@ -12,15 +13,15 @@ import org.openqa.selenium.logging.LogEntry; import org.testar.monkey.Assert; import org.testar.monkey.ConfigTags; -import org.testar.monkey.alayer.Tags; import org.testar.monkey.alayer.Verdict; import org.testar.monkey.alayer.webdriver.WdDriver; import org.testar.settings.Settings; import org.testar.stub.StateStub; -public class TestBrowserConsoleVerdict extends WebdriverProtocol { +public class TestBrowserConsoleVerdict { private StateStub state; + private Settings settings; @Before public void settings_setup() { @@ -29,8 +30,6 @@ public void settings_setup() { settings.set(ConfigTags.TagsForSuspiciousOracle, Collections.singletonList("Title")); state = new StateStub(); - state.set(Tags.IsRunning, true); - state.set(Tags.NotResponding, false); } @Test @@ -40,10 +39,13 @@ public void test_console_ok() { settings.set(ConfigTags.WebConsoleWarningOracle, true); settings.set(ConfigTags.WebConsoleWarningPattern, ".*.*"); + WebBrowserConsoleOracle oracle = new WebBrowserConsoleOracle(settings); LogEntry infoEntry = new LogEntry(Level.INFO, System.currentTimeMillis(), "this is an info message"); LogEntries logEntries = new LogEntries(Collections.singletonList(infoEntry)); - Verdict verdict = runWithMockedLogs(logEntries); + List verdicts = runWithMockedLogs(oracle, logEntries); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isEquals(Verdict.OK.severity(), verdict.severity()); Assert.isEquals("No problem detected.", verdict.info()); @@ -56,12 +58,15 @@ public void test_console_error() { settings.set(ConfigTags.WebConsoleWarningOracle, true); settings.set(ConfigTags.WebConsoleWarningPattern, ".*.*"); + WebBrowserConsoleOracle oracle = new WebBrowserConsoleOracle(settings); LogEntry severeEntry = new LogEntry(Level.SEVERE, System.currentTimeMillis(), "some severe error occurred"); LogEntries logEntries = new LogEntries(Collections.singletonList(severeEntry)); - Verdict verdict = runWithMockedLogs(logEntries); + List verdicts = runWithMockedLogs(oracle, logEntries); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); - Assert.isEquals(Verdict.Severity.SUSPICIOUS_TAG.getValue(), verdict.severity()); + Assert.isEquals(Verdict.Severity.SUSPICIOUS_LOG.getValue(), verdict.severity()); Assert.isEquals("Web Browser Console Error: some severe error occurred", verdict.info()); } @@ -72,21 +77,41 @@ public void test_console_warning() { settings.set(ConfigTags.WebConsoleWarningOracle, true); settings.set(ConfigTags.WebConsoleWarningPattern, ".*.*"); + WebBrowserConsoleOracle oracle = new WebBrowserConsoleOracle(settings); LogEntry severeEntry = new LogEntry(Level.SEVERE, System.currentTimeMillis(), "some severe error occurred"); LogEntry warningEntry = new LogEntry(Level.WARNING, System.currentTimeMillis(), "this is a warning message"); LogEntries logEntries = new LogEntries(Arrays.asList(severeEntry, warningEntry)); - Verdict verdict = runWithMockedLogs(logEntries); + List verdicts = runWithMockedLogs(oracle, logEntries); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); - Assert.isEquals(Verdict.Severity.SUSPICIOUS_TAG.getValue(), verdict.severity()); + Assert.isEquals(Verdict.Severity.SUSPICIOUS_LOG.getValue(), verdict.severity()); Assert.isEquals("Web Browser Console Warning: this is a warning message", verdict.info()); } - private Verdict runWithMockedLogs(LogEntries mockedEntries) { + @Test + public void test_console_error_and_warning() { + settings.set(ConfigTags.WebConsoleErrorOracle, true); + settings.set(ConfigTags.WebConsoleErrorPattern, ".*.*"); + settings.set(ConfigTags.WebConsoleWarningOracle, true); + settings.set(ConfigTags.WebConsoleWarningPattern, ".*.*"); + + WebBrowserConsoleOracle oracle = new WebBrowserConsoleOracle(settings); + LogEntry severeEntry = new LogEntry(Level.SEVERE, System.currentTimeMillis(), "some severe error occurred"); + LogEntry warningEntry = new LogEntry(Level.WARNING, System.currentTimeMillis(), "this is a warning message"); + LogEntries logEntries = new LogEntries(Arrays.asList(severeEntry, warningEntry)); + + List verdicts = runWithMockedLogs(oracle, logEntries); + Assert.isEquals(2, verdicts.size()); + Assert.isEquals("Web Browser Console Error: some severe error occurred", verdicts.get(0).info()); + Assert.isEquals("Web Browser Console Warning: this is a warning message", verdicts.get(1).info()); + } + + private List runWithMockedLogs(WebBrowserConsoleOracle oracle, LogEntries mockedEntries) { try (MockedStatic mockedStatic = mockStatic(WdDriver.class)) { mockedStatic.when(WdDriver::getBrowserLogs).thenReturn(mockedEntries); - return getVerdict(state); + return oracle.getVerdicts(state); } } } - diff --git a/testar/test/org/testar/oracles/log/TestLogOracles.java b/testar/test/org/testar/oracles/log/TestLogOracles.java index e5759a084..28de482d0 100644 --- a/testar/test/org/testar/oracles/log/TestLogOracles.java +++ b/testar/test/org/testar/oracles/log/TestLogOracles.java @@ -69,7 +69,9 @@ public void wrongConfigurationLogOracleFile() { logOracle.initialize(); State state = Mockito.mock(State.class); - Verdict logVerdict = logOracle.getVerdict(state); + List verdicts = logOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict logVerdict = verdicts.get(0); // Verify that the logVerdict is OK Assert.assertTrue(logVerdict.severity() == Verdict.Severity.OK.getValue()); @@ -98,7 +100,9 @@ public void initialSuspiciousLogOracleErrorMessage() { logOracle.initialize(); State state = Mockito.mock(State.class); - Verdict logVerdict = logOracle.getVerdict(state); + List verdicts = logOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict logVerdict = verdicts.get(0); System.out.println("initialSuspiciousLogOracleErrorMessage logVerdict: " + logVerdict.info()); // Verify that the logVerdict is OK @@ -128,7 +132,9 @@ public void runtimeValidLogOracleFile() { Assert.assertTrue(fileContent().contains(validMessage)); State state = Mockito.mock(State.class); - Verdict logVerdict = logOracle.getVerdict(state); + List verdicts = logOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict logVerdict = verdicts.get(0); System.out.println("runtimeValidLogOracleFile logVerdict: " + logVerdict.info()); // Verify that the logVerdict is OK @@ -158,7 +164,9 @@ public void runtimeSuspiciousLogOracleErrorMessage() { Assert.assertTrue(fileContent().contains(suspiciousMessage)); State state = Mockito.mock(State.class); - Verdict logVerdict = logOracle.getVerdict(state); + List verdicts = logOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict logVerdict = verdicts.get(0); System.out.println("runtimeSuspiciousLogOracleErrorMessage logVerdict: " + logVerdict.info()); // Verify that the logVerdict detected the suspiciousMessage diff --git a/testar/test/org/testar/oracles/log/TestProcessListenerOracle.java b/testar/test/org/testar/oracles/log/TestProcessListenerOracle.java index aa6c32c64..d36da60f0 100644 --- a/testar/test/org/testar/oracles/log/TestProcessListenerOracle.java +++ b/testar/test/org/testar/oracles/log/TestProcessListenerOracle.java @@ -75,7 +75,9 @@ public void spy_mode_is_always_ok_because_is_disabled() { processOracle.initialize(); State state = Mockito.mock(State.class); - Verdict processVerdict = processOracle.getVerdict(state); + List verdicts = processOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict processVerdict = verdicts.get(0); // Verify that the processVerdict is OK because we are in spy mode Assert.assertTrue(processVerdict.severity() == Verdict.Severity.OK.getValue()); @@ -98,7 +100,9 @@ public void connect_windows_title_is_always_ok_because_is_disabled() { processOracle.initialize(); State state = Mockito.mock(State.class); - Verdict processVerdict = processOracle.getVerdict(state); + List verdicts = processOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict processVerdict = verdicts.get(0); // Verify that the processVerdict is OK because we connect with title Assert.assertTrue(processVerdict.severity() == Verdict.Severity.OK.getValue()); @@ -121,7 +125,9 @@ public void connect_process_name_is_always_ok_because_is_disabled() { processOracle.initialize(); State state = Mockito.mock(State.class); - Verdict processVerdict = processOracle.getVerdict(state); + List verdicts = processOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict processVerdict = verdicts.get(0); // Verify that the processVerdict is OK because we connect with process name Assert.assertTrue(processVerdict.severity() == Verdict.Severity.OK.getValue()); @@ -144,7 +150,9 @@ public void webdriver_is_always_ok_because_is_disabled() { processOracle.initialize(); State state = Mockito.mock(State.class); - Verdict processVerdict = processOracle.getVerdict(state); + List verdicts = processOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict processVerdict = verdicts.get(0); // Verify that the processVerdict is OK because we connect with web apps Assert.assertTrue(processVerdict.severity() == Verdict.Severity.OK.getValue()); @@ -167,7 +175,9 @@ public void process_detection_for_error_buffer() { processOracle.initialize(); State state = Mockito.mock(State.class); - Verdict processVerdict = processOracle.getVerdict(state); + List verdicts = processOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict processVerdict = verdicts.get(0); // Verify that the processVerdict detects an error in the error buffer Assert.assertTrue(processVerdict.severity() == Verdict.Severity.SUSPICIOUS_PROCESS.getValue()); @@ -191,7 +201,9 @@ public void process_detection_for_output_buffer() { processOracle.initialize(); State state = Mockito.mock(State.class); - Verdict processVerdict = processOracle.getVerdict(state); + List verdicts = processOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict processVerdict = verdicts.get(0); // Verify that the processVerdict detects an error in the output buffer Assert.assertTrue(processVerdict.severity() == Verdict.Severity.SUSPICIOUS_PROCESS.getValue()); @@ -215,7 +227,9 @@ public void process_detection_for_both_buffers() { processOracle.initialize(); State state = Mockito.mock(State.class); - Verdict processVerdict = processOracle.getVerdict(state); + List verdicts = processOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict processVerdict = verdicts.get(0); // Verify that the processVerdict detects an error Assert.assertTrue(processVerdict.severity() == Verdict.Severity.SUSPICIOUS_PROCESS.getValue()); diff --git a/testar/test/org/testar/oracles/web/accessibility/TestWebAccessibilityClickableSizeOracle.java b/testar/test/org/testar/oracles/web/accessibility/TestWebAccessibilityClickableSizeOracle.java index ab0281a71..82a4b7d22 100644 --- a/testar/test/org/testar/oracles/web/accessibility/TestWebAccessibilityClickableSizeOracle.java +++ b/testar/test/org/testar/oracles/web/accessibility/TestWebAccessibilityClickableSizeOracle.java @@ -51,9 +51,11 @@ public void test_detection_accessibility_clickable_size() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebAccessibilityClickableSizeOracle); // Assert the oracle verdict is WARNING_ACCESSIBILITY_FAULT - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_ACCESSIBILITY_FAULT.getTitle())); - Assert.isTrue(verdict.info().equals("Clickable web widgets '<a>link</a>' , are too small (Minimum: 24 px).")); + Assert.isTrue(verdict.info().equals("Clickable web widget '<a>link</a>' , is too small (0x0 px). Minimum: 24 px.")); } @Test @@ -73,7 +75,9 @@ public void test_undetection_accessibility_clickable_size() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebAccessibilityClickableSizeOracle); // Assert the oracle verdict is OK - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.OK.getTitle())); Assert.isTrue(verdict.info().equals("No problem detected.")); } diff --git a/testar/test/org/testar/oracles/web/accessibility/TestWebAccessibilityFontSizeOracle.java b/testar/test/org/testar/oracles/web/accessibility/TestWebAccessibilityFontSizeOracle.java index 06d6117cd..4a313b45a 100644 --- a/testar/test/org/testar/oracles/web/accessibility/TestWebAccessibilityFontSizeOracle.java +++ b/testar/test/org/testar/oracles/web/accessibility/TestWebAccessibilityFontSizeOracle.java @@ -50,9 +50,11 @@ public void test_detection_accessibility_font_size() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebAccessibilityFontSizeOracle); // Assert the oracle verdict is WARNING_ACCESSIBILITY_FAULT - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_ACCESSIBILITY_FAULT.getTitle())); - Assert.isTrue(verdict.info().equals("These widgets Text 'widgettext' , are too small. Minimum recommended is 12 px.")); + Assert.isTrue(verdict.info().equals("Widget text 'widgettext' , is too small (11 px). Minimum recommended is 12 px.")); } @Test @@ -72,7 +74,9 @@ public void test_undetection_accessibility_font_size() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebAccessibilityFontSizeOracle); // Assert the oracle verdict is OK - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.OK.getTitle())); Assert.isTrue(verdict.info().equals("No problem detected.")); } diff --git a/testar/test/org/testar/oracles/web/accessibility/TestWebAccessibilityImagesAltOracle.java b/testar/test/org/testar/oracles/web/accessibility/TestWebAccessibilityImagesAltOracle.java index f26222672..95e73aabc 100644 --- a/testar/test/org/testar/oracles/web/accessibility/TestWebAccessibilityImagesAltOracle.java +++ b/testar/test/org/testar/oracles/web/accessibility/TestWebAccessibilityImagesAltOracle.java @@ -49,9 +49,11 @@ public void test_detection_accessibility_images_alt() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebAccessibilityImagesAltOracle); // Assert the oracle verdict is WARNING_ACCESSIBILITY_FAULT - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_ACCESSIBILITY_FAULT.getTitle())); - Assert.isTrue(verdict.info().equals("Detected web image widgets ''<img src='url'>whatever</img>' , ' without alternative text!")); + Assert.isTrue(verdict.info().equals("Detected web image widget '<img src='url'>whatever</img>' , without alternative text!")); } @Test @@ -70,7 +72,9 @@ public void test_undetection_accessibility_images_alt() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebAccessibilityImagesAltOracle); // Assert the oracle verdict is OK - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.OK.getTitle())); Assert.isTrue(verdict.info().equals("No problem detected.")); } diff --git a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantDuplicateMenuItems.java b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantDuplicateMenuItems.java index a564b0251..48bf2c7bd 100644 --- a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantDuplicateMenuItems.java +++ b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantDuplicateMenuItems.java @@ -57,9 +57,11 @@ public void test_detection_web_invariant_duplicated_menu_items() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantDuplicateMenuItems); // Assert the oracle verdict is WARNING_WEB_INVARIANT_FAULT - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT.getTitle())); - Assert.isTrue(verdict.info().equals("Detected a Unnumbered List (UL) web menu 'menuid' , with duplicate option elements!")); + Assert.isTrue(verdict.info().equals("Detected a Unnumbered List (UL) web menu 'menuid' , with duplicate option elements: [menu_element]")); } @Test @@ -86,7 +88,9 @@ public void test_undetection_web_invariant_duplicated_menu_items() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantDuplicateMenuItems); // Assert the oracle verdict is OK - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.OK.getTitle())); Assert.isTrue(verdict.info().equals("No problem detected.")); } diff --git a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantDuplicateSelectItems.java b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantDuplicateSelectItems.java index 175528814..1168fa3aa 100644 --- a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantDuplicateSelectItems.java +++ b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantDuplicateSelectItems.java @@ -59,9 +59,11 @@ public void test_detection_web_invariant_duplicate_select_items() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantDuplicateSelectItems); // Assert the oracle verdict is WARNING_WEB_INVARIANT_FAULT - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT.getTitle())); - Assert.isTrue(verdict.info().equals("Detected Select widgets 'selectid' , with duplicate values!")); + Assert.isTrue(verdict.info().equals("Detected Select widget 'selectid' , with duplicate values: [Renault]")); } } @@ -87,7 +89,9 @@ public void test_undetection_web_invariant_duplicate_select_items() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantDuplicateSelectItems); // Assert the oracle verdict is OK - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.OK.getTitle())); Assert.isTrue(verdict.info().equals("No problem detected.")); } diff --git a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantDuplicatedRowsInTable.java b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantDuplicatedRowsInTable.java index 53d7b05e0..72db2a122 100644 --- a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantDuplicatedRowsInTable.java +++ b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantDuplicatedRowsInTable.java @@ -9,6 +9,7 @@ import org.testar.monkey.Assert; import org.testar.monkey.ConfigTags; import org.testar.monkey.Pair; +import org.testar.monkey.alayer.Rect; import org.testar.monkey.alayer.Tags; import org.testar.monkey.alayer.Verdict; import org.testar.monkey.alayer.webdriver.enums.WdRoles; @@ -44,6 +45,7 @@ public void test_detection_web_invariant_duplicated_rows_table() { WidgetStub firstRow = new WidgetStub(); firstRow.set(Tags.Role, WdRoles.WdTR); + firstRow.set(Tags.Shape, Rect.fromCoordinates(0, 0, 10, 10)); widgetTable.addChild(firstRow); WidgetStub firstHeader = new WidgetStub(); @@ -58,6 +60,7 @@ public void test_detection_web_invariant_duplicated_rows_table() { WidgetStub secondRow = new WidgetStub(); secondRow.set(Tags.Role, WdRoles.WdTR); + secondRow.set(Tags.Shape, Rect.fromCoordinates(0, 0, 10, 10)); widgetTable.addChild(secondRow); WidgetStub secondHeader = new WidgetStub(); @@ -75,9 +78,13 @@ public void test_detection_web_invariant_duplicated_rows_table() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantDuplicatedRowsInTable); // Assert the oracle verdict is WARNING_WEB_INVARIANT_FAULT - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT.getTitle())); - Assert.isTrue(verdict.info().contains("Detected a duplicated rows in a Table for the widgets: [_header_content_data_content, _header_content_data_content]")); + Assert.isTrue(verdict.info().contains("Detected duplicated row in a Table for the widget: _header_content_data_content")); + Assert.isEquals(2, verdict.visualizer().getShapes().size()); } @Test @@ -122,8 +129,64 @@ public void test_undetection_invariant_duplicated_rows_table() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantDuplicatedRowsInTable); // Assert the oracle verdict is OK - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.OK.getTitle())); Assert.isTrue(verdict.info().equals("No problem detected.")); } + + @Test + public void test_detection_web_invariant_duplicated_rows_multiple_tables() { + StateStub state = new StateStub(); + + WidgetStub tableOne = createTable(state, "table1"); + addDuplicatedRowPair(tableOne, "headerA", "dataA"); + addDuplicatedRowPair(tableOne, "headerB", "dataB"); + + WidgetStub tableTwo = createTable(state, "table2"); + addDuplicatedRowPair(tableTwo, "headerC", "dataC"); + addDuplicatedRowPair(tableTwo, "headerD", "dataD"); + + Assert.isTrue(extendedOraclesList.size() == 1); + Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantDuplicatedRowsInTable); + + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(4, verdicts.size()); + for (Verdict verdict : verdicts) { + Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT.getTitle())); + Assert.isEquals(2, verdict.visualizer().getShapes().size()); + } + } + + private WidgetStub createTable(StateStub state, String tableId) { + WidgetStub widgetTable = new WidgetStub(); + widgetTable.set(Tags.Role, WdRoles.WdTABLE); + widgetTable.set(WdTags.WebId, tableId); + state.addChild(widgetTable); + widgetTable.setParent(state); + return widgetTable; + } + + private void addDuplicatedRowPair(WidgetStub table, String headerText, String dataText) { + addRow(table, headerText, dataText); + addRow(table, headerText, dataText); + } + + private void addRow(WidgetStub table, String headerText, String dataText) { + WidgetStub row = new WidgetStub(); + row.set(Tags.Role, WdRoles.WdTR); + row.set(Tags.Shape, Rect.fromCoordinates(0, 0, 10, 10)); + table.addChild(row); + + WidgetStub header = new WidgetStub(); + header.set(Tags.Role, WdRoles.WdTH); + header.set(WdTags.WebTextContent, headerText); + row.addChild(header); + + WidgetStub data = new WidgetStub(); + data.set(Tags.Role, WdRoles.WdTD); + data.set(WdTags.WebTextContent, dataText); + row.addChild(data); + } } diff --git a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantEmptySelectItems.java b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantEmptySelectItems.java index c632bf498..f1e761ced 100644 --- a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantEmptySelectItems.java +++ b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantEmptySelectItems.java @@ -57,9 +57,11 @@ public void test_detection_web_invariant_empty_select_items() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantEmptySelectItems); // Assert the oracle verdict is WARNING_WEB_INVARIANT_FAULT - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT.getTitle())); - Assert.isTrue(verdict.info().equals("Detected Select widgets 'selectid' , with empty or only one item!")); + Assert.isTrue(verdict.info().equals("Detected Select widget 'selectid' , with empty or only one item (count: 1)!")); } } @@ -84,7 +86,9 @@ public void test_undetection_web_invariant_empty_select_items() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantEmptySelectItems); // Assert the oracle verdict is OK - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.OK.getTitle())); Assert.isTrue(verdict.info().equals("No problem detected.")); } diff --git a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantManySelectItems.java b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantManySelectItems.java index 532db63af..8755bf6e5 100644 --- a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantManySelectItems.java +++ b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantManySelectItems.java @@ -57,9 +57,11 @@ public void test_detection_web_invariant_many_select_items() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantManySelectItems); // Assert the oracle verdict is WARNING_WEB_INVARIANT_FAULT - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT.getTitle())); - Assert.isTrue(verdict.info().equals("Detected Select widgets 'selectid' , which have more items than the threshold value of 100")); + Assert.isTrue(verdict.info().equals("Detected Select widget 'selectid' , which has 101 items (threshold: 100)")); } } @@ -84,7 +86,9 @@ public void test_undetection_web_invariant_many_select_items() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantManySelectItems); // Assert the oracle verdict is OK - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.OK.getTitle())); Assert.isTrue(verdict.info().equals("No problem detected.")); } diff --git a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantNumberWithLotOfDecimals.java b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantNumberWithLotOfDecimals.java index ea7a20c71..b9dfb9d80 100644 --- a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantNumberWithLotOfDecimals.java +++ b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantNumberWithLotOfDecimals.java @@ -45,9 +45,11 @@ public void test_detection_web_invariant_number_lot_decimals() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantNumberWithLotOfDecimals); // Assert the oracle verdict is WARNING_WEB_INVARIANT_FAULT - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT.getTitle())); - Assert.isTrue(verdict.info().equals("Detected widgets '30.123€' , with more than 2 decimals!")); + Assert.isTrue(verdict.info().equals("Detected widget '30.123€' , with 3 decimals (max: 2)!")); } @Test @@ -64,7 +66,9 @@ public void test_undetection_web_invariant_number_lot_decimals() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantNumberWithLotOfDecimals); // Assert the oracle verdict is OK - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.OK.getTitle())); Assert.isTrue(verdict.info().equals("No problem detected.")); } diff --git a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantTextAreaWithoutLength.java b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantTextAreaWithoutLength.java index fb768b123..73417f0a1 100644 --- a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantTextAreaWithoutLength.java +++ b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantTextAreaWithoutLength.java @@ -49,9 +49,11 @@ public void test_detection_web_invariant_textarea_without_length() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantTextAreaWithoutLength); // Assert the oracle verdict is WARNING_WEB_INVARIANT_FAULT - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT.getTitle())); - Assert.isTrue(verdict.info().equals("Detected TextArea widgets '<textarea maxlength=0></textarea>' , with 0 max length!")); + Assert.isTrue(verdict.info().equals("Detected TextArea widget '<textarea maxlength=0></textarea>' , with 0 max length!")); } @Test @@ -70,7 +72,9 @@ public void test_undetection_web_invariant_textarea_without_length() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantTextAreaWithoutLength); // Assert the oracle verdict is OK - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.OK.getTitle())); Assert.isTrue(verdict.info().equals("No problem detected.")); } diff --git a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantUnsortedSelectItems.java b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantUnsortedSelectItems.java index 165073698..fc98082d3 100644 --- a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantUnsortedSelectItems.java +++ b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantUnsortedSelectItems.java @@ -59,9 +59,11 @@ public void test_detection_web_invariant_unsorted_select_items_month() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantUnsortedSelectItems); // Assert the oracle verdict is WARNING_WEB_INVARIANT_FAULT - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT.getTitle())); - Assert.isTrue(verdict.info().equals("Detected Select widgets 'selectid' , with unsorted elements!")); + Assert.isTrue(verdict.info().equals("Detected Select widget 'selectid' , with unsorted elements!")); } } @@ -87,9 +89,11 @@ public void test_detection_web_invariant_unsorted_select_items_natural() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantUnsortedSelectItems); // Assert the oracle verdict is WARNING_WEB_INVARIANT_FAULT - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT.getTitle())); - Assert.isTrue(verdict.info().equals("Detected Select widgets 'selectid' , with unsorted elements!")); + Assert.isTrue(verdict.info().equals("Detected Select widget 'selectid' , with unsorted elements!")); } } @@ -115,9 +119,11 @@ public void test_detection_web_invariant_unsorted_select_items_numeric() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantUnsortedSelectItems); // Assert the oracle verdict is WARNING_WEB_INVARIANT_FAULT - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT.getTitle())); - Assert.isTrue(verdict.info().equals("Detected Select widgets 'selectid' , with unsorted elements!")); + Assert.isTrue(verdict.info().equals("Detected Select widget 'selectid' , with unsorted elements!")); } } @@ -143,7 +149,9 @@ public void test_undetection_web_invariant_unsorted_select_items_month() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantUnsortedSelectItems); // Assert the oracle verdict is OK - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.OK.getTitle())); Assert.isTrue(verdict.info().equals("No problem detected.")); } @@ -171,7 +179,9 @@ public void test_undetection_web_invariant_unsorted_select_items_natural() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantUnsortedSelectItems); // Assert the oracle verdict is OK - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.OK.getTitle())); Assert.isTrue(verdict.info().equals("No problem detected.")); } @@ -199,7 +209,9 @@ public void test_undetection_web_invariant_unsorted_select_items_numeric() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantUnsortedSelectItems); // Assert the oracle verdict is OK - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.OK.getTitle())); Assert.isTrue(verdict.info().equals("No problem detected.")); } diff --git a/testar/test/org/testar/reporting/TestReportManager.java b/testar/test/org/testar/reporting/TestReportManager.java index 238d49f37..dc4725152 100644 --- a/testar/test/org/testar/reporting/TestReportManager.java +++ b/testar/test/org/testar/reporting/TestReportManager.java @@ -4,6 +4,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Properties; @@ -34,7 +35,7 @@ public class TestReportManager { private static StateStub state; private static Set derivedActions; private static Action selectedAction; - private static Verdict finalVerdict = Verdict.OK; + private static List finalVerdicts = Collections.singletonList(Verdict.OK); @Before public void setUp() throws IOException { @@ -99,7 +100,7 @@ public void testHtmlReport() { ReportManager reportManager = createReportManager(settings); // Verify the html report file was created with the state and actions information - File htmlReportFile = new File(reportManager.getReportFileName().concat("_OK.html")); + File htmlReportFile = new File(reportManager.getReportFileName().concat("_V001_OK.html")); System.out.println("testHtmlReport: " + htmlReportFile.getPath()); Assert.assertTrue(htmlReportFile.exists()); @@ -120,7 +121,7 @@ public void testHtmlReport() { Assert.assertTrue(fileContains("

Test verdict for this sequence: No problem detected.

", htmlReportFile)); // Verify the plain txt report was not created - File txtReportFile = new File(reportManager.getReportFileName().concat("_OK.txt")); + File txtReportFile = new File(reportManager.getReportFileName().concat("_V001_OK.txt")); Assert.assertTrue(!txtReportFile.exists()); } @@ -142,11 +143,11 @@ public void testPlainReport() { ReportManager reportManager = createReportManager(settings); // Verify the html report was not created - File htmlReportFile = new File(reportManager.getReportFileName().concat("_OK.html")); + File htmlReportFile = new File(reportManager.getReportFileName().concat("_V001_OK.html")); Assert.assertTrue(!htmlReportFile.exists()); // Verify the txt report file was created with the state and actions information - File txtReportFile = new File(reportManager.getReportFileName().concat("_OK.txt")); + File txtReportFile = new File(reportManager.getReportFileName().concat("_V001_OK.txt")); System.out.println("testPlainReport: " + txtReportFile.getPath()); Assert.assertTrue(txtReportFile.exists()); @@ -187,7 +188,7 @@ public void testHtmlReportWithoutStateScreenshot() { ReportManager reportManager = createReportManager(settings); // Verify the html report file was created with the state and actions information - File htmlReportFile = new File(reportManager.getReportFileName().concat("_OK.html")); + File htmlReportFile = new File(reportManager.getReportFileName().concat("_V001_OK.html")); System.out.println("testHtmlReportWithoutScreenshot: " + htmlReportFile.getPath()); Assert.assertTrue(htmlReportFile.exists()); @@ -216,11 +217,12 @@ public void testHtmlReportParsesSpecialCharacters() { // Prepare a report only with the final verdict ReportManager reportManager = new ReportManager(false, settings); - reportManager.addTestVerdict(new Verdict(Verdict.Severity.FAIL, "Failure is ")); + Verdict failVerdict = new Verdict(Verdict.Severity.FAIL, "Failure is "); + reportManager.addTestVerdicts(Collections.singletonList(failVerdict)); reportManager.finishReport(); // Verify the html report file was created - File htmlReportFile = new File(reportManager.getReportFileName().concat("_FAIL.html")); + File htmlReportFile = new File(reportManager.getReportFileName().concat("_V001_FAIL.html")); System.out.println("testSpecialCharacters: " + htmlReportFile.getPath()); Assert.assertTrue(htmlReportFile.exists()); @@ -233,7 +235,7 @@ private ReportManager createReportManager(Settings settings) { reportManager.addState(state); reportManager.addActions(derivedActions); reportManager.addSelectedAction(state, selectedAction); - reportManager.addTestVerdict(finalVerdict); + reportManager.addTestVerdicts(finalVerdicts); reportManager.finishReport(); return reportManager; } diff --git a/testar/workflow_windows_webdriver.gradle b/testar/workflow_windows_webdriver.gradle index 331630a3f..cdd0f41fb 100644 --- a/testar/workflow_windows_webdriver.gradle +++ b/testar/workflow_windows_webdriver.gradle @@ -31,19 +31,25 @@ task runTestWebdriverSuspiciousTagStateModel(type: Exec, dependsOn: createDataba } // Connect to para.testar.org to detect the browser console error message -task runTestWebdriverParabankConsoleError(type: Exec) { +// And also add the extended oracle to detect the accessibility warning of an image without alternative text +task runTestWebdriverParabankConsoleErrorAndAccessibilityWarning(type: Exec) { // Read command line output standardOutput = new ByteArrayOutputStream() group = 'test_testar_workflow' - description ='runTestWebdriverParabankConsoleError' + description ='runTestWebdriverParabankConsoleErrorAndAccessibilityWarning' workingDir 'target/install/testar/bin' - commandLine 'cmd', '/c', 'testar sse=test_gradle_workflow_webdriver_parabank AlwaysCompile=true ApplicationName="webdriver_console_error" ShowVisualSettingsDialogOnStartup=false Mode=Generate Sequences=1 SequenceLength=1 WebConsoleErrorOracle=true' + commandLine 'cmd', '/c', 'testar sse=test_gradle_workflow_webdriver_parabank AlwaysCompile=true ApplicationName="webdriver_console_error_and_accessibility_warning" ShowVisualSettingsDialogOnStartup=false Mode=Generate Sequences=1 SequenceLength=1 WebConsoleErrorOracle=true ExtendedOracles=WebAccessibilityImagesAltOracle' doLast { String output = standardOutput.toString() - // Check that output detects the browser console error message - if(output.readLines().any{line->line.contains("Failed to load resource: the server responded with a status of 404")}) { - println "\n${output} \nTESTAR login and browser console error detection has been executed sucessfully" + def required = [ + "[WARNING_ACCESSIBILITY_FAULT] Detected web image widget", + "[SUSPICIOUS_LOG] Web Browser Console Error" + ] + + // Check that output detects both the browser console error message and the accessibility warning + if (required.every { s -> output.contains(s) }) { + println "\n${output} \nTESTAR browser console error and accessibility warning detection has been executed sucessfully" } else { throw new GradleException("\n${output} \nERROR: Executing TESTAR") } @@ -142,7 +148,7 @@ task runTestWebdriverUnreplayable(type: Exec) { String output = standardOutput.toString() // Check that TESTAR detects not replayable button - if(output.readLines().any{line->line.contains("Left Click at 'ParisOne' of the replayed sequence can not been replayed")}) { + if(output.readLines().any{line->line.contains("unreplayable_sequence_1.testar The Action Left Click at 'ParisOne' of the replayed sequence can not been replayed")}) { println "\n${output} \nTESTAR has sucessfully detected a not replayable sequence" } else { throw new GradleException("\n${output} \nERROR: Detecting not replayable button with TESTAR Replay mode")