From 67414362cc00e0b9a94899fe05716e7a2484404e Mon Sep 17 00:00:00 2001 From: ferpasri Date: Thu, 19 Feb 2026 19:41:15 +0100 Subject: [PATCH 01/10] Delete old RecordMode --- testar/src/org/testar/monkey/RecordMode.java | 248 ------------------- 1 file changed, 248 deletions(-) delete mode 100644 testar/src/org/testar/monkey/RecordMode.java diff --git a/testar/src/org/testar/monkey/RecordMode.java b/testar/src/org/testar/monkey/RecordMode.java deleted file mode 100644 index d34bda23d..000000000 --- a/testar/src/org/testar/monkey/RecordMode.java +++ /dev/null @@ -1,248 +0,0 @@ -/*************************************************************************************************** - * - * Copyright (c) 2022 - 2023 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2022 - 2023 Open Universiteit - www.ou.nl - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - *******************************************************************************************************/ - -package org.testar.monkey; - -import org.testar.ActionStatus; -import org.testar.OutputStructure; -import org.testar.SutVisualization; -import org.testar.monkey.RuntimeControlsProtocol.Modes; -import org.testar.monkey.alayer.*; -import org.testar.monkey.alayer.actions.AnnotatingActionCompiler; -import org.testar.monkey.alayer.devices.KBKeys; -import org.testar.monkey.alayer.devices.MouseButtons; -import org.testar.monkey.alayer.exceptions.WidgetNotFoundException; -import org.testar.serialisation.LogSerialiser; - -import java.util.Collections; -import java.util.List; -import java.util.Set; - -public class RecordMode { - - private static final int MAX_ESC_ATTEMPTS = 99; - - /** - * Method to run TESTAR on Record User Actions Mode. - * @param system - */ - public void runRecordLoop(DefaultProtocol protocol) { - // Prepare the output folders structure - synchronized(this){ - OutputStructure.calculateInnerLoopDateString(); - OutputStructure.sequenceInnerLoopCount++; - } - - //empty method in defaultProtocol - allowing implementation in application specific protocols - //HTML report is created here in DefaultProtocol - protocol.preSequencePreparations(); - - //We need to invoke the SUT & the canvas representation - SUT system = protocol.startSUTandLogger(); - protocol.cv = protocol.buildCanvas(); - protocol.actionCount = 1; - - //Generating the sequence file that can be replayed: - protocol.getAndStoreGeneratedSequence(); - protocol.getAndStoreSequenceFile(); - - // notify the statemodelmanager - protocol.stateModelManager.notifyTestSequencedStarted(); - - /** - * Start Record User Action Loop - */ - while(protocol.mode() == Modes.Record && system.isRunning()) { - State state = protocol.getState(system); - protocol.cv.begin(); - Util.clear(protocol.cv); - - Set actions = protocol.deriveActions(system, state); - protocol.buildStateActionsIdentifiers(state, actions); - - //notify the state model manager of the new state - protocol.stateModelManager.notifyNewStateReached(state, actions); - - // If no actions are derived, create an ESC action - if(actions.isEmpty()){ - if (protocol.escAttempts >= MAX_ESC_ATTEMPTS){ - LogSerialiser.log("No available actions to execute! Tried ESC <" + MAX_ESC_ATTEMPTS + "> times. Stopping sequence generation!\n", LogSerialiser.LogLevel.Critical); - } - //---------------------------------- - // THERE MUST ALMOST BE ONE ACTION! - //---------------------------------- - // if we did not find any actions, then we just hit escape, maybe that works ;-) - Action escAction = new AnnotatingActionCompiler().hitKey(KBKeys.VK_ESCAPE); - protocol.buildEnvironmentActionIdentifiers(state, escAction); - actions.add(escAction); - protocol.escAttempts++; - } else - protocol.escAttempts = 0; - - ActionStatus actionStatus = new ActionStatus(); - - //Start Wait User Action Loop to obtain the Action did by the User - waitUserActionLoop(protocol, system, state, actionStatus); - - //Save the user action information into the logs - if (actionStatus.isUserEventAction()) { - - protocol.buildStateActionsIdentifiers(state, Collections.singleton(actionStatus.getAction())); - - //notify the state model manager of the executed action - protocol.stateModelManager.notifyActionExecution(actionStatus.getAction()); - - protocol.saveActionInfoInLogs(state, actionStatus.getAction(), "RecordedAction"); - DefaultProtocol.lastExecutedAction = actionStatus.getAction(); - protocol.actionCount++; - } - - /** - * When we close TESTAR with Shift+down arrow, last actions is detected like null - */ - if(actionStatus.getAction()!=null) { - protocol.saveActionIntoFragmentForReplayableSequence(actionStatus.getAction(), state, actions); - } else { - //If null just ignore - } - - Util.clear(protocol.cv); - protocol.cv.end(); - } - - //If user closes the SUT while in Record-mode, TESTAR will close (or go back to SettingsDialog): - if(!system.isRunning()){ - protocol.mode = Modes.Quit; - } - - if(protocol.mode() == Modes.Quit){ - // notify the statemodelmanager - protocol.stateModelManager.notifyTestSequenceStopped(); - - // notify the state model manager of the sequence end - protocol.stateModelManager.notifyTestingEnded(); - - //Closing fragment for recording replayable test sequence: - protocol.writeAndCloseFragmentForReplayableSequence(); - - //Copy sequence file into proper directory: - protocol.classifyAndCopySequenceIntoAppropriateDirectory(Verdict.OK); - - protocol.postSequenceProcessing(); - - //If we want to Quit the current execution we stop the system - protocol.stopSystem(system); - } - } - - /** - * Waits for an user UI action. - * Requirement: Mode must be Record. - */ - private void waitUserActionLoop(DefaultProtocol protocol, SUT system, State state, ActionStatus actionStatus){ - while (protocol.mode() == Modes.Record && !actionStatus.isUserEventAction()){ - if (protocol.userEvent != null){ - actionStatus.setAction(mapUserEvent(protocol, state)); - actionStatus.setUserEventAction((actionStatus.getAction() != null)); - protocol.userEvent = null; - } - synchronized(this){ - try { - this.wait(100); - } catch (InterruptedException e) {} - } - state = protocol.getState(system); - protocol.cv.begin(); - Util.clear(protocol.cv); - - //In Record-mode, we activate the visualization with Shift+ArrowUP: - if(protocol.visualizationOn) { - SutVisualization.visualizeState(false, - protocol.markParentWidget, - protocol.mouse, - protocol.lastPrintParentsOf, - protocol.cv, - state); - } - - Set actions = protocol.deriveActions(system, state); - protocol.buildStateActionsIdentifiers(state, actions); - - //In Record-mode, we activate the visualization with Shift+ArrowUP: - if(protocol.visualizationOn) { - protocol.visualizeActions(protocol.cv, state, actions); - } - - protocol.cv.end(); - } - } - - /** - * Records user action. - * - * @param state - * @return - */ - private Action mapUserEvent(DefaultProtocol protocol, State state){ - Assert.notNull(protocol.userEvent); - if (protocol.userEvent[0] instanceof MouseButtons){ // mouse events - double x = ((Double) protocol.userEvent[1]).doubleValue(); - double y = ((Double) protocol.userEvent[2]).doubleValue(); - Widget w = null; - try { - w = Util.widgetFromPoint(state, x, y); - x = 0.5; y = 0.5; - if (protocol.userEvent[0] == MouseButtons.BUTTON1) // left click - return (new AnnotatingActionCompiler()).leftClickAt(w,x,y); - else if (protocol.userEvent[0] == MouseButtons.BUTTON3) // right click - return (new AnnotatingActionCompiler()).rightClickAt(w,x,y); - } catch (WidgetNotFoundException we){ - System.out.println("Mapping user event ... widget not found @(" + x + "," + y + ")"); - return null; - } - } else if (protocol.userEvent[0] instanceof KBKeys) // key events - return (new AnnotatingActionCompiler()).hitKey((KBKeys) protocol.userEvent[0]); - else if (protocol.userEvent[0] instanceof String){ // type events - if (DefaultProtocol.lastExecutedAction == null) - return null; - List targets = DefaultProtocol.lastExecutedAction.get(Tags.Targets,null); - if (targets == null || targets.size() != 1) - return null; - try { - Widget w = targets.get(0).apply(state); - return (new AnnotatingActionCompiler()).clickTypeInto(w, (String) protocol.userEvent[0], true); - } catch (WidgetNotFoundException we){ - return null; - } - } - return null; - } - -} From 9f4a0425be903fa1f53291d2eacff26db796eea9 Mon Sep 17 00:00:00 2001 From: ferpasri Date: Fri, 20 Feb 2026 16:51:49 +0100 Subject: [PATCH 02/10] Refactor getVerdict to List getVerdicts --- core/src/org/testar/ProtocolUtil.java | 8 +- core/src/org/testar/monkey/alayer/Tags.java | 14 +- .../src/org/testar/monkey/alayer/Verdict.java | 48 ++-- .../org/testar/monkey/alayer/VerdictTest.java | 66 +----- .../hydrator/ConcreteStateHydrator.java | 25 +-- .../Protocol_01_desktop_calculator.java | 11 +- .../Protocol_02_webdriver_parabank.java | 65 +----- .../02_webdriver_parabank/test.settings | 10 +- .../Protocol_03_webdriver_llm_parabank.java | 44 ++-- .../Protocol_android_generic.java | 7 +- .../Protocol_desktop_generic.java | 11 +- .../Protocol_desktop_java_coverage.java | 11 +- .../Protocol_webdriver_generic.java | 10 +- ...l_webdriver_llm_state_model_evaluator.java | 14 +- ..._llm_state_model_transition_evaluator.java | 14 +- ...webdriver_llm_state_widgets_evaluator.java | 14 +- ...rotocol_webdriver_remote_webcomponent.java | 10 +- .../Protocol_webdriver_security_analysis.java | 14 +- ..._test_gradle_workflow_android_generic.java | 37 ++-- ..._test_gradle_workflow_desktop_generic.java | 39 ++-- ...radle_workflow_webdriver_form_filling.java | 8 +- ...est_gradle_workflow_webdriver_generic.java | 36 +-- ...test_gradle_workflow_webdriver_replay.java | 36 +-- testar/src/org/testar/FileHandling.java | 35 ++- .../org/testar/monkey/AbstractProtocol.java | 15 +- testar/src/org/testar/monkey/ConfigTags.java | 9 +- .../org/testar/monkey/DefaultProtocol.java | 209 ++++++++++++------ .../src/org/testar/monkey/GenerateMode.java | 15 +- testar/src/org/testar/monkey/ReplayMode.java | 46 ++-- .../org/testar/monkey/VerdictProcessing.java | 189 ++++++++++++++++ testar/src/org/testar/oracles/Oracle.java | 12 +- .../GenericVisualAlignmentMetricOracle.java | 12 +- .../GenericVisualBalanceMetricOracle.java | 18 +- ...ericVisualCenterAlignmentMetricOracle.java | 12 +- ...enericVisualConcentricityMetricOracle.java | 18 +- .../GenericVisualDensityMetricOracle.java | 25 ++- .../GenericVisualSimplicityMetricOracle.java | 18 +- .../src/org/testar/oracles/llm/LlmOracle.java | 10 +- .../oracles/log/AndroidLogcatOracle.java | 11 +- .../src/org/testar/oracles/log/LogOracle.java | 11 +- .../oracles/log/ProcessListenerOracle.java | 18 +- .../oracles/log/WebBrowserConsoleOracle.java | 117 ++++++++++ .../WebAccessibilityClickableSizeOracle.java | 45 ++-- .../WebAccessibilityFontSizeOracle.java | 43 ++-- .../WebAccessibilityImagesAltOracle.java | 40 ++-- .../WebInvariantDuplicateMenuItems.java | 43 ++-- .../WebInvariantDuplicateSelectItems.java | 42 ++-- .../WebInvariantDuplicatedRowsInTable.java | 50 +++-- .../WebInvariantEmptySelectItems.java | 43 ++-- .../WebInvariantManySelectItems.java | 44 ++-- .../WebInvariantNumberWithLotOfDecimals.java | 45 ++-- .../WebInvariantTextAreaWithoutLength.java | 41 ++-- .../WebInvariantUnsortedSelectItems.java | 39 ++-- .../testar/protocols/WebdriverProtocol.java | 80 ++----- .../testar/reporting/DummyReportManager.java | 7 +- .../org/testar/reporting/HtmlReporter.java | 52 ++++- .../testar/reporting/PlainTextReporter.java | 51 ++++- .../org/testar/reporting/ReportManager.java | 15 +- .../src/org/testar/reporting/Reporting.java | 7 +- .../helpers/SecurityOracleOrchestrator.java | 24 +- .../org/testar/settings/SettingsDefaults.java | 6 +- .../settings/SettingsFileStructure.java | 10 +- .../testar/settings/dialog/GeneralPanel.java | 15 +- testar/test/org/testar/TestFileHandling.java | 22 +- .../testar/monkey/TestMoreActionsLogic.java | 13 +- .../testar/monkey/VerdictProcessingTest.java | 83 +++++++ .../oracles/log/TestAndroidLogcatOracle.java | 22 +- .../log}/TestBrowserConsoleVerdict.java | 51 +++-- .../testar/oracles/log/TestLogOracles.java | 16 +- .../log/TestProcessListenerOracle.java | 28 ++- ...stWebAccessibilityClickableSizeOracle.java | 10 +- .../TestWebAccessibilityFontSizeOracle.java | 10 +- .../TestWebAccessibilityImagesAltOracle.java | 10 +- .../TestWebInvariantDuplicateMenuItems.java | 10 +- .../TestWebInvariantDuplicateSelectItems.java | 10 +- ...TestWebInvariantDuplicatedRowsInTable.java | 69 +++++- .../TestWebInvariantEmptySelectItems.java | 10 +- .../TestWebInvariantManySelectItems.java | 10 +- ...stWebInvariantNumberWithLotOfDecimals.java | 10 +- ...TestWebInvariantTextAreaWithoutLength.java | 10 +- .../TestWebInvariantUnsortedSelectItems.java | 30 ++- .../testar/reporting/TestReportManager.java | 20 +- 82 files changed, 1592 insertions(+), 946 deletions(-) create mode 100644 testar/src/org/testar/monkey/VerdictProcessing.java create mode 100644 testar/src/org/testar/oracles/log/WebBrowserConsoleOracle.java create mode 100644 testar/test/org/testar/monkey/VerdictProcessingTest.java rename testar/test/org/testar/{protocols => oracles/log}/TestBrowserConsoleVerdict.java (58%) diff --git a/core/src/org/testar/ProtocolUtil.java b/core/src/org/testar/ProtocolUtil.java index 909da8326..e72aa47f7 100644 --- a/core/src/org/testar/ProtocolUtil.java +++ b/core/src/org/testar/ProtocolUtil.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * -* Copyright (c) 2016 - 2025 Universitat Politecnica de Valencia - www.upv.es -* Copyright (c) 2019 - 2025 Open Universiteit - www.ou.nl +* Copyright (c) 2016 - 2026 Universitat Politecnica de Valencia - www.upv.es +* Copyright (c) 2019 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -49,6 +49,7 @@ import org.testar.monkey.alayer.Widget; import java.awt.*; +import java.util.Collections; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; @@ -251,7 +252,8 @@ public static AWTCanvas getStateshotBinary(State state) { } //If the state Shape is not properly obtained, or the State has an error, use full monitor screen - if (viewPort == null || (state.get(Tags.OracleVerdict, Verdict.OK).severity() > Verdict.Severity.OK.getValue())) + List verdicts = state.get(Tags.OracleVerdicts, Collections.singletonList(Verdict.OK)); + if (viewPort == null || !Verdict.helperAreAllVerdictsOK(verdicts)) viewPort = state.get(Tags.Shape, null); // get the SUT process canvas (usually, full monitor screen) // Validate viewport dimensions before taking the screenshot diff --git a/core/src/org/testar/monkey/alayer/Tags.java b/core/src/org/testar/monkey/alayer/Tags.java index 2beb8ecc8..07b7837c5 100644 --- a/core/src/org/testar/monkey/alayer/Tags.java +++ b/core/src/org/testar/monkey/alayer/Tags.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * -* Copyright (c) 2013 - 2025 Universitat Politecnica de Valencia - www.upv.es -* Copyright (c) 2018 - 2025 Open Universiteit - www.ou.nl +* Copyright (c) 2013 - 2026 Universitat Politecnica de Valencia - www.upv.es +* Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -28,10 +28,6 @@ * POSSIBILITY OF SUCH DAMAGE. *******************************************************************************************************/ -/** - * @author Sebastian Bauersfeld - */ - package org.testar.monkey.alayer; import org.testar.CodingManager; @@ -161,9 +157,9 @@ private Tags() {} /** Usually attached to an object of {@link State}. The value is a screenshot of the state. */ public static final Tag ScreenshotPath = from("ScreenshotPath", String.class); - /** Usually attached to a {@link State} object. The value is an outcome of a test oracle for that state. It is - * used to mark states as 'suspicious' or 'erroneous' */ - public static final Tag OracleVerdict = from("OracleVerdict", Verdict.class); + /** Usually attached to a {@link State} object. The value is a list of outcomes of test oracles for that state. */ + @SuppressWarnings("unchecked") + public static final Tag> OracleVerdicts = from("OracleVerdicts", (Class>)(Class)List.class); /** The standard mouse object. Usually attached to systems */ public static final Tag StandardMouse = from("StandardMouse", Mouse.class); diff --git a/core/src/org/testar/monkey/alayer/Verdict.java b/core/src/org/testar/monkey/alayer/Verdict.java index edf29514b..f25d0528a 100644 --- a/core/src/org/testar/monkey/alayer/Verdict.java +++ b/core/src/org/testar/monkey/alayer/Verdict.java @@ -31,7 +31,9 @@ package org.testar.monkey.alayer; import java.io.Serializable; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import org.testar.monkey.Assert; import org.testar.monkey.Util; @@ -93,12 +95,16 @@ public enum Severity { // WARNING GROUP: ACCESSIBILITY WARNING_ACCESSIBILITY_FAULT(0.4, "WARNING_ACCESSIBILITY_FAULT"), - /** FAIL (0.5 - 1.0) **/ + /** FAIL (0.5 - 0.899) **/ UNREPLAYABLE(0.5, "UNREPLAYABLE"), // Sequence not replayable SUSPICIOUS_TAG(0.8, "SUSPICIOUS_TAG"), // Suspicious tag SUSPICIOUS_PROCESS(0.87, "SUSPICIOUS_PROCESS"), // Suspicious message in the process standard output/error SUSPICIOUS_LOG(0.89, "SUSPICIOUS_LOG"), // Suspicious message in log file or command output (LogOracle) + + /** CRITICAL (0.9 - 1.0) **/ + + CRITICAL(0.9, "CRITICAL"), NOT_RESPONDING(0.99999990, "NOT_RESPONDING"), // Unresponsive UNEXPECTEDCLOSE(0.99999999, "UNEXPECTEDCLOSE"), // Crash? Unexpected close? FAIL(1.0, "FAIL"); @@ -193,29 +199,18 @@ public Visualizer visualizer() { return visualizer; } - @Override - public String toString() { - return "severity: " + severity + " info: " + info; - } - /** - * Retrieves the verdict result of joining two verdicts. - * @param verdict A verdict to join with current verdict. - * - * @return A new verdict that is the result of joining the current verdict with the provided verdict. + * Indicates if this verdict is critical (SUT froze or crashed). + * + * @return true if verdict is critical */ - public Verdict join(Verdict verdict) { - Severity joinedSeverity = Arrays.stream(Severity.values()) - .filter(s -> s.getValue() == Math.max(this.severity, verdict.severity())) - .findFirst() - .orElse(Severity.FAIL); - - String joinedInfo = this.info.contains(verdict.info()) ? this.info - : (this.severity == Severity.OK.getValue() ? "" : this.info + "\n") + verdict.info(); - - Visualizer joinedVisualizer = Visualizer.join(this.visualizer(), verdict.visualizer()); + public boolean isCritical() { + return this.severity() >= Severity.CRITICAL.getValue(); + } - return new Verdict(joinedSeverity, joinedInfo, joinedVisualizer); + @Override + public String toString() { + return "severity: " + severity + " info: " + info; } /** @@ -234,4 +229,15 @@ public boolean equals(Object o) { && this.visualizer.equals(other.visualizer); } + public static boolean helperAreAllVerdictsOK(List verdicts) { + if(verdicts == null || verdicts.isEmpty()) return true; + + for (Verdict verdict : verdicts) { + if (verdict.severity() > Severity.OK.getValue()) { + return false; + } + } + return true; + } + } \ No newline at end of file diff --git a/core/test/org/testar/monkey/alayer/VerdictTest.java b/core/test/org/testar/monkey/alayer/VerdictTest.java index 9d9ac2bb2..9910d5e01 100644 --- a/core/test/org/testar/monkey/alayer/VerdictTest.java +++ b/core/test/org/testar/monkey/alayer/VerdictTest.java @@ -35,30 +35,16 @@ import java.util.Arrays; import org.junit.Test; -import org.testar.monkey.alayer.visualizers.RegionsVisualizer; import org.testar.monkey.alayer.visualizers.ShapeVisualizer; public class VerdictTest { - private final double DELTA = 0; - - private final Visualizer dummyVisualizer = new Visualizer(){ - private static final long serialVersionUID = -7830649624698071090L; - public void run(State s, Canvas c, Pen pen) {} - }; - private final Visualizer failVisualizer = new ShapeVisualizer( Pen.PEN_RED, Rect.from(0, 0, 10, 10), "Fail Visualizer", 0.5, 0.5); - private final Visualizer issueVisualizer = new RegionsVisualizer( - Pen.PEN_RED, - Arrays.asList(Rect.from(0, 0, 10, 10)), - "Issue Visualizer", - 0.5, 0.5); - @Test public void testToString() { Verdict v = new Verdict(Verdict.Severity.OK, "This is a test verdict"); @@ -67,54 +53,12 @@ public void testToString() { } @Test - public void testJoin() { - Verdict v1 = new Verdict(Verdict.Severity.OK, "Foo Bar"); - Verdict v2 = new Verdict(Verdict.Severity.FAIL, "Bar", failVisualizer); - Verdict v3 = new Verdict(Verdict.Severity.OK, "Baz", dummyVisualizer); - Verdict v4 = new Verdict(Verdict.Severity.FAIL, "Exception", issueVisualizer); - Verdict emptyVisualizerVerdict = new Verdict(Verdict.Severity.FAIL, "Empty"); - - assertTrue("Joining two Verdicts shall create a new Verdict", - v1 != v1.join(v2)); - - assertEquals("Joining two Verdicts shall set the severity to the maximum of both", - Verdict.Severity.FAIL.getValue(), v3.join(v2).severity(), DELTA); - - assertEquals("If a Verdict's info contains the info of the Verdict to be joined with, " + - "then only the containing info shall be used", - "Foo Bar", v1.join(v2).info()); - - assertEquals("If a Verdict is OK and its info does not contain the info of the Verdict to be joined with, " + - "then the containing info shall be discarded", - "Baz", v1.join(v3).info()); - - assertEquals("If a Verdict is not OK and its info does not contain the info of the Verdict to be joined with, " + - "then both infos shall be included separated by a line break", - "Bar\nBaz", v2.join(v3).info()); - - assertTrue("Joining an OK and Fail Verdicts shall use the Visualizer of the Verdict with high severity", - v2.join(v1).visualizer() == failVisualizer); - - assertTrue("Joining an OK and Fail Verdicts shall use the Visualizer of the Verdict with high severity", - v1.join(v2).visualizer() == failVisualizer); - - assertTrue("Joining Fail and Issue Verdicts must contain Fail Shapes", - v2.join(v4).visualizer().getShapes().containsAll(failVisualizer.getShapes())); - - assertTrue("Joining Issue and Fail Verdicts must contain Fail Shapes", - v4.join(v2).visualizer().getShapes().containsAll(failVisualizer.getShapes())); - - assertTrue("Joining Fail and Issue Verdicts must contain Issue Shapes", - v2.join(v4).visualizer().getShapes().containsAll(issueVisualizer.getShapes())); - - assertTrue("Joining Issue and Fail Verdicts must contain Issue Shapes", - v4.join(v2).visualizer().getShapes().containsAll(issueVisualizer.getShapes())); - - assertTrue("Joining Fail and emptyVisualizerVerdict Verdicts must equal Fail visualizer", - v2.join(emptyVisualizerVerdict).visualizer() == failVisualizer); + public void testVerdictHelperAreOK() { + Verdict ok = Verdict.OK; + Verdict warn = new Verdict(Verdict.Severity.SUSPICIOUS_TAG, "Issue", failVisualizer); - assertTrue("Joining emptyVisualizerVerdict and Fail Verdicts must equal Fail visualizer", - emptyVisualizerVerdict.join(v2).visualizer() == failVisualizer); + assertTrue(Verdict.helperAreAllVerdictsOK(Arrays.asList(ok))); + assertFalse(Verdict.helperAreAllVerdictsOK(Arrays.asList(warn))); } @Test diff --git a/statemodel/src/org/testar/statemodel/persistence/orientdb/hydrator/ConcreteStateHydrator.java b/statemodel/src/org/testar/statemodel/persistence/orientdb/hydrator/ConcreteStateHydrator.java index 78dbdd667..344079dd9 100644 --- a/statemodel/src/org/testar/statemodel/persistence/orientdb/hydrator/ConcreteStateHydrator.java +++ b/statemodel/src/org/testar/statemodel/persistence/orientdb/hydrator/ConcreteStateHydrator.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2018 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2018 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2018 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -39,6 +39,10 @@ import org.testar.statemodel.persistence.orientdb.entity.TypeConvertor; import org.testar.statemodel.persistence.orientdb.entity.VertexEntity; import org.testar.statemodel.persistence.orientdb.util.Validation; + +import java.util.Collections; +import java.util.List; + import org.testar.monkey.alayer.Tag; import org.testar.monkey.alayer.TaggableBase; import org.testar.monkey.alayer.Tags; @@ -89,18 +93,13 @@ public void hydrate(VertexEntity target, Object source) throws HydrationExceptio } // get the oracle verdict and transform it into a custom code - Verdict verdict = attributes.get(Tags.OracleVerdict, null); + List verdicts = attributes.get(Tags.OracleVerdicts, Collections.singletonList(Verdict.OK)); int oracleVerdictCode = 4; // default value - if (verdict != null) { - if (verdict.severity() == Verdict.Severity.OK.getValue()) { - oracleVerdictCode = 1; - } - else if (verdict.severity() == Verdict.Severity.FAIL.getValue()) { - oracleVerdictCode = 2; - } - else if (verdict.severity() > Verdict.Severity.OK.getValue() && verdict.severity() < Verdict.Severity.FAIL.getValue()) { - oracleVerdictCode = 3; - } + if (Verdict.helperAreAllVerdictsOK(verdicts)) { + oracleVerdictCode = 1; + } + else { + oracleVerdictCode = 2; } target.addPropertyValue("oracleVerdictCode", new PropertyValue(OType.INTEGER, oracleVerdictCode)); } diff --git a/testar/resources/settings/01_desktop_calculator/Protocol_01_desktop_calculator.java b/testar/resources/settings/01_desktop_calculator/Protocol_01_desktop_calculator.java index cf98ad152..79a83518d 100644 --- a/testar/resources/settings/01_desktop_calculator/Protocol_01_desktop_calculator.java +++ b/testar/resources/settings/01_desktop_calculator/Protocol_01_desktop_calculator.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2020 - 2023 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2020 - 2023 Open Universiteit - www.ou.nl + * Copyright (c) 2020 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2020 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -41,6 +41,7 @@ import org.testar.protocols.DesktopProtocol; import org.testar.settings.Settings; +import java.util.List; import java.util.Set; /** @@ -118,18 +119,18 @@ protected State getState(SUT system) throws StateBuildException{ * @return oracle verdict, which determines whether the state is erroneous and why. */ @Override - protected Verdict getVerdict(State state){ + protected List getVerdicts(State state){ // The super methods implements the implicit online state oracles for: // system crashes // non-responsiveness // suspicious tags - Verdict verdict = super.getVerdict(state); + List verdicts = super.getVerdicts(state); //-------------------------------------------------------- // MORE SOPHISTICATED STATE ORACLES CAN BE PROGRAMMED HERE //-------------------------------------------------------- - return verdict; + return verdicts; } /** diff --git a/testar/resources/settings/02_webdriver_parabank/Protocol_02_webdriver_parabank.java b/testar/resources/settings/02_webdriver_parabank/Protocol_02_webdriver_parabank.java index cbbdd388c..f6245a171 100644 --- a/testar/resources/settings/02_webdriver_parabank/Protocol_02_webdriver_parabank.java +++ b/testar/resources/settings/02_webdriver_parabank/Protocol_02_webdriver_parabank.java @@ -59,9 +59,6 @@ public class Protocol_02_webdriver_parabank extends WebdriverProtocol { - // This list tracks the detected erroneous verdicts to avoid duplicates - private List listOfDetectedErroneousVerdicts = new ArrayList<>(); - private List extendedOraclesList = new ArrayList<>(); /** @@ -73,9 +70,6 @@ public class Protocol_02_webdriver_parabank extends WebdriverProtocol { @Override protected void initialize(Settings settings) { super.initialize(settings); - - // Reset the list when we start a new TESTAR run with multiple sequences - listOfDetectedErroneousVerdicts = new ArrayList<>(); } /** @@ -172,36 +166,15 @@ protected State getState(SUT system) throws StateBuildException { * @return oracle verdict, which determines whether the state is erroneous and why. */ @Override - protected Verdict getVerdict(State state) { - + protected List getVerdicts(State state) { // System crashes, non-responsiveness and suspicious tags automatically detected! // For web applications, web browser errors and warnings can also be enabled via settings - Verdict verdict = super.getVerdict(state); - - // If the Verdict is not OK but was already detected in a previous sequence - // Consider as OK to avoid duplicates and continue testing - if (verdict != Verdict.OK && containsVerdictInfo(listOfDetectedErroneousVerdicts, verdict.info())) { - // Consider as OK to continue testing - verdict = Verdict.OK; - webConsoleVerdict = Verdict.OK; - } - // If the Verdict is not OK and was not duplicated... - // We found an issue we need to report - else if (verdict.severity() != Verdict.OK.severity()) { - return verdict; - } + List verdicts = super.getVerdicts(state); // "ExtendedOracles" offered by TESTAR in the test.settings or Oracles GUI dialog for (Oracle extendedOracle : extendedOraclesList) { - Verdict extendedVerdict = extendedOracle.getVerdict(state); - - // If the Custom Verdict is not OK and was not detected in a previous sequence - // return verdict with failure state - if (extendedVerdict != Verdict.OK - && !containsVerdictInfo(listOfDetectedErroneousVerdicts, extendedVerdict.info())) { - return extendedVerdict; - } - + List extendedVerdicts = extendedOracle.getVerdicts(state); + verdicts.addAll(extendedVerdicts); } //----------------------------------------------------------------------------- @@ -210,24 +183,14 @@ else if (verdict.severity() != Verdict.OK.severity()) { // ... YOU MAY WANT TO CHECK YOUR CUSTOM ORACLES HERE ... - Verdict leafWidgetsOverlappingVerdict = leafWidgetsOverlapping(state); - - // If the Custom Verdict is not OK but was already detected in a previous sequence - // Consider as OK to avoid duplicates - if (leafWidgetsOverlappingVerdict != Verdict.OK - && !containsVerdictInfo(listOfDetectedErroneousVerdicts, leafWidgetsOverlappingVerdict.info())) { - return leafWidgetsOverlappingVerdict; - } - - return Verdict.OK; - } + List overlapVerdicts = leafWidgetsOverlapping(state); + verdicts.addAll(overlapVerdicts); - private boolean containsVerdictInfo(List listOfDetectedErroneousVerdicts, String currentVerdictInfo) { - return listOfDetectedErroneousVerdicts.stream().anyMatch(verdictInfo -> verdictInfo.contains(currentVerdictInfo.replace("\n", " "))); + return verdicts; } - public Verdict leafWidgetsOverlapping(State state) { - Verdict finalVerdict = Verdict.OK; + public List leafWidgetsOverlapping(State state) { + List verdicts = new ArrayList<>(); // Prepare a list that contains all the Rectangles from the leaf widgets List> leafWidgetsRects = new ArrayList<>(); @@ -265,12 +228,12 @@ public Verdict leafWidgetsOverlapping(State state) { Verdict.Severity.WARNING_UI_VISUAL_OR_RENDERING_FAULT, verdictMsg, visualizer); - finalVerdict = finalVerdict.join(clashVerdict); + verdicts.add(clashVerdict); } } } - return finalVerdict; + return verdicts; } private Pen getRedPen() { @@ -550,12 +513,6 @@ protected boolean moreActions(State state) { @Override protected void finishSequence() { super.finishSequence(); - // If the final Verdict is not OK and the verdict is not saved in the list - // This is a new run fail verdict - Verdict finalVerdict = getVerdict(latestState); - if(finalVerdict.severity() > Verdict.Severity.OK.getValue() && !listOfDetectedErroneousVerdicts.contains(finalVerdict.info().replace("\n", " "))) { - listOfDetectedErroneousVerdicts.add(finalVerdict.info().replace("\n", " ")); - } } /** diff --git a/testar/resources/settings/02_webdriver_parabank/test.settings b/testar/resources/settings/02_webdriver_parabank/test.settings index eef0f5f20..cd551e260 100644 --- a/testar/resources/settings/02_webdriver_parabank/test.settings +++ b/testar/resources/settings/02_webdriver_parabank/test.settings @@ -38,6 +38,12 @@ SUTConnectorValue = "https://para.testar.org/" Sequences = 5 SequenceLength = 20 +################################################################# +# Ignore reporting duplicated verdicts for this protocol +################################################################# + +IgnoreDuplicatedVerdicts = true + ################################################################# # Oracles based on suspicious tag values # @@ -191,9 +197,9 @@ SwitchNewTabs = true # WebConsoleWarningPattern: Regular expressions ORACLE to find suspicious messages in the browser warning console ################################################################# -WebConsoleErrorOracle = false +WebConsoleErrorOracle = true WebConsoleErrorPattern = .*.* -WebConsoleWarningOracle = false +WebConsoleWarningOracle = true WebConsoleWarningPattern = .*.* ################################################################# diff --git a/testar/resources/settings/03_webdriver_llm_parabank/Protocol_03_webdriver_llm_parabank.java b/testar/resources/settings/03_webdriver_llm_parabank/Protocol_03_webdriver_llm_parabank.java index 3b5871b8e..c34fe87c8 100644 --- a/testar/resources/settings/03_webdriver_llm_parabank/Protocol_03_webdriver_llm_parabank.java +++ b/testar/resources/settings/03_webdriver_llm_parabank/Protocol_03_webdriver_llm_parabank.java @@ -1,6 +1,6 @@ /** - * Copyright (c) 2018 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2019 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2019 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -161,31 +161,29 @@ protected State getState(SUT system) throws StateBuildException { * @return oracle verdict, which determines whether the state is erroneous and why. */ @Override - protected Verdict getVerdict(State state) { - // System crashes, non-responsiveness and suspicious tags automatically detected! - // For web applications, web browser errors and warnings can also be enabled via settings - Verdict verdict = super.getVerdict(state); - + protected List getVerdicts(State state) { // Use the LLM as an Oracle to determine if the test goal has been completed - Verdict llmVerdict = llmOracle.getVerdict(state); - - if(llmVerdict.severity() == Verdict.Severity.LLM_COMPLETE.getValue()) { - // Test goal was completed, retrieve next test goal from queue. - currentTestGoal = testGoalQueue.poll(); - - // Poll returns null if there are no more items remaining in the queue. - if(currentTestGoal == null) { - // No more test goals remaining, terminate sequence. - System.out.println("Test goal completed, but no more test goals."); - return llmVerdict; - } else { - System.out.println("Test goal completed, moving to next test goal."); - llmActionSelector.reset(currentTestGoal, true); - llmOracle.reset(currentTestGoal, true); + List llmVerdicts = llmOracle.getVerdicts(state); + + for(Verdict llmVerdict : llmVerdicts) { + if(llmVerdict.severity() == Verdict.Severity.LLM_COMPLETE.getValue()) { + // Test goal was completed, retrieve next test goal from queue. + currentTestGoal = testGoalQueue.poll(); + + // Poll returns null if there are no more items remaining in the queue. + if(currentTestGoal == null) { + // No more test goals remaining, terminate sequence. + System.out.println("Test goal completed, but no more test goals."); + return Collections.singletonList(llmVerdict); + } else { + System.out.println("Test goal completed, moving to next test goal."); + llmActionSelector.reset(currentTestGoal, true); + llmOracle.reset(currentTestGoal, true); + } } } - return verdict; + return super.getVerdicts(state); } /** diff --git a/testar/resources/settings/android_generic/Protocol_android_generic.java b/testar/resources/settings/android_generic/Protocol_android_generic.java index 2a442171b..9c5d2c6fb 100644 --- a/testar/resources/settings/android_generic/Protocol_android_generic.java +++ b/testar/resources/settings/android_generic/Protocol_android_generic.java @@ -48,6 +48,7 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.List; import java.util.Set; public class Protocol_android_generic extends AndroidProtocol { @@ -156,18 +157,18 @@ protected State getState(SUT system) throws StateBuildException { * @return oracle verdict, which determines whether the state is erroneous and why. */ @Override - protected Verdict getVerdict(State state){ + protected List getVerdicts(State state){ // The super methods implements the implicit online state oracles for: // system crashes // non-responsiveness // suspicious tags - Verdict verdict = super.getVerdict(state); + List verdicts = super.getVerdicts(state); //-------------------------------------------------------- // MORE SOPHISTICATED STATE ORACLES CAN BE PROGRAMMED HERE //-------------------------------------------------------- - return verdict; + return verdicts; } /** diff --git a/testar/resources/settings/desktop_generic/Protocol_desktop_generic.java b/testar/resources/settings/desktop_generic/Protocol_desktop_generic.java index 964e963cf..adc317c13 100644 --- a/testar/resources/settings/desktop_generic/Protocol_desktop_generic.java +++ b/testar/resources/settings/desktop_generic/Protocol_desktop_generic.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2013 - 2024 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2018 - 2024 Open Universiteit - www.ou.nl + * Copyright (c) 2013 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -29,6 +29,7 @@ *******************************************************************************************************/ +import java.util.List; import java.util.Set; import org.testar.DerivedActions; @@ -125,18 +126,18 @@ protected State getState(SUT system) throws StateBuildException { * @return oracle verdict, which determines whether the state is erroneous and why. */ @Override - protected Verdict getVerdict(State state){ + protected List getVerdicts(State state){ // The super methods implements the implicit online state oracles for: // system crashes // non-responsiveness // suspicious tags - Verdict verdict = super.getVerdict(state); + List verdicts = super.getVerdicts(state); //-------------------------------------------------------- // MORE SOPHISTICATED STATE ORACLES CAN BE PROGRAMMED HERE //-------------------------------------------------------- - return verdict; + return verdicts; } /** diff --git a/testar/resources/settings/desktop_java_coverage/Protocol_desktop_java_coverage.java b/testar/resources/settings/desktop_java_coverage/Protocol_desktop_java_coverage.java index 0310d1480..d79c8b7c3 100644 --- a/testar/resources/settings/desktop_java_coverage/Protocol_desktop_java_coverage.java +++ b/testar/resources/settings/desktop_java_coverage/Protocol_desktop_java_coverage.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2024 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2024 Open Universiteit - www.ou.nl + * Copyright (c) 2024 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2024 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -41,6 +41,7 @@ import org.testar.protocols.DesktopProtocol; import org.testar.settings.Settings; +import java.util.List; import java.util.Set; public class Protocol_desktop_java_coverage extends DesktopProtocol { @@ -109,18 +110,18 @@ protected State getState(SUT system) throws StateBuildException{ * @return oracle verdict, which determines whether the state is erroneous and why. */ @Override - protected Verdict getVerdict(State state){ + protected List getVerdicts(State state){ // The super methods implements the implicit online state oracles for: // system crashes // non-responsiveness // suspicious tags - Verdict verdict = super.getVerdict(state); + List verdicts = super.getVerdicts(state); //-------------------------------------------------------- // MORE SOPHISTICATED STATE ORACLES CAN BE PROGRAMMED HERE //-------------------------------------------------------- - return verdict; + return verdicts; } /** diff --git a/testar/resources/settings/webdriver_generic/Protocol_webdriver_generic.java b/testar/resources/settings/webdriver_generic/Protocol_webdriver_generic.java index 8da5b56c1..202a71846 100644 --- a/testar/resources/settings/webdriver_generic/Protocol_webdriver_generic.java +++ b/testar/resources/settings/webdriver_generic/Protocol_webdriver_generic.java @@ -1,6 +1,6 @@ /** - * Copyright (c) 2018 - 2023 Open Universiteit - www.ou.nl - * Copyright (c) 2019 - 2023 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2019 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -129,9 +129,9 @@ protected State getState(SUT system) throws StateBuildException { * @return oracle verdict, which determines whether the state is erroneous and why. */ @Override - protected Verdict getVerdict(State state) { + protected List getVerdicts(State state) { - Verdict verdict = super.getVerdict(state); + List verdicts = super.getVerdicts(state); // system crashes, non-responsiveness and suspicious tags automatically detected! //----------------------------------------------------------------------------- @@ -140,7 +140,7 @@ protected Verdict getVerdict(State state) { // ... YOU MAY WANT TO CHECK YOUR CUSTOM ORACLES HERE ... - return verdict; + return verdicts; } /** diff --git a/testar/resources/settings/webdriver_llm_state_model_evaluator/Protocol_webdriver_llm_state_model_evaluator.java b/testar/resources/settings/webdriver_llm_state_model_evaluator/Protocol_webdriver_llm_state_model_evaluator.java index 2956a69f3..5f6373116 100644 --- a/testar/resources/settings/webdriver_llm_state_model_evaluator/Protocol_webdriver_llm_state_model_evaluator.java +++ b/testar/resources/settings/webdriver_llm_state_model_evaluator/Protocol_webdriver_llm_state_model_evaluator.java @@ -1,6 +1,6 @@ /** - * Copyright (c) 2024 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2024 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2024 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2024 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -191,11 +191,7 @@ protected State getState(SUT system) throws StateBuildException { * @return oracle verdict, which determines whether the state is erroneous and why. */ @Override - protected Verdict getVerdict(State state) { - // System crashes, non-responsiveness and suspicious tags automatically detected! - // For web applications, web browser errors and warnings can also be enabled via settings - Verdict verdict = super.getVerdict(state); - + protected List getVerdicts(State state) { String modelIdentifier = stateModelManager.getModelIdentifier(); if(conditionEvaluator.evaluateConditions(modelIdentifier, stateModelManager)) { @@ -206,7 +202,7 @@ protected Verdict getVerdict(State state) { if(currentTestGoal == null) { // No more test goals remaining, terminate sequence. System.out.println("Test goal completed, but no more test goals."); - return new Verdict(Verdict.Severity.CONDITION_COMPLETE, "All test goal conditions completed."); + return Collections.singletonList(new Verdict(Verdict.Severity.CONDITION_COMPLETE, "All test goal conditions completed.")); } else { System.out.println("Test goal completed, moving to next test goal."); llmActionSelector.reset(currentTestGoal, true); @@ -215,7 +211,7 @@ protected Verdict getVerdict(State state) { } } - return verdict; + return super.getVerdicts(state); } /** diff --git a/testar/resources/settings/webdriver_llm_state_model_transition_evaluator/Protocol_webdriver_llm_state_model_transition_evaluator.java b/testar/resources/settings/webdriver_llm_state_model_transition_evaluator/Protocol_webdriver_llm_state_model_transition_evaluator.java index 9c8a7c664..ec24efdcb 100644 --- a/testar/resources/settings/webdriver_llm_state_model_transition_evaluator/Protocol_webdriver_llm_state_model_transition_evaluator.java +++ b/testar/resources/settings/webdriver_llm_state_model_transition_evaluator/Protocol_webdriver_llm_state_model_transition_evaluator.java @@ -1,6 +1,6 @@ /** - * Copyright (c) 2024 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2024 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2024 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2024 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -196,11 +196,7 @@ protected State getState(SUT system) throws StateBuildException { * @return oracle verdict, which determines whether the state is erroneous and why. */ @Override - protected Verdict getVerdict(State state) { - // System crashes, non-responsiveness and suspicious tags automatically detected! - // For web applications, web browser errors and warnings can also be enabled via settings - Verdict verdict = super.getVerdict(state); - + protected List getVerdicts(State state) { String modelIdentifier = stateModelManager.getModelIdentifier(); if(conditionEvaluator.evaluateConditions(modelIdentifier, stateModelManager)) { @@ -211,7 +207,7 @@ protected Verdict getVerdict(State state) { if(currentTestGoal == null) { // No more test goals remaining, terminate sequence. System.out.println("Test goal completed, but no more test goals."); - return new Verdict(Verdict.Severity.CONDITION_COMPLETE, "All test goal conditions completed."); + return Collections.singletonList(new Verdict(Verdict.Severity.CONDITION_COMPLETE, "All test goal conditions completed.")); } else { System.out.println("Test goal completed, moving to next test goal."); llmActionSelector.reset(currentTestGoal, true); @@ -220,7 +216,7 @@ protected Verdict getVerdict(State state) { } } - return verdict; + return super.getVerdicts(state); } /** diff --git a/testar/resources/settings/webdriver_llm_state_widgets_evaluator/Protocol_webdriver_llm_state_widgets_evaluator.java b/testar/resources/settings/webdriver_llm_state_widgets_evaluator/Protocol_webdriver_llm_state_widgets_evaluator.java index 20d6d8414..bdf51b222 100644 --- a/testar/resources/settings/webdriver_llm_state_widgets_evaluator/Protocol_webdriver_llm_state_widgets_evaluator.java +++ b/testar/resources/settings/webdriver_llm_state_widgets_evaluator/Protocol_webdriver_llm_state_widgets_evaluator.java @@ -1,6 +1,6 @@ /** - * Copyright (c) 2024 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2024 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2024 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2024 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -175,11 +175,7 @@ protected State getState(SUT system) throws StateBuildException { * @return oracle verdict, which determines whether the state is erroneous and why. */ @Override - protected Verdict getVerdict(State state) { - // System crashes, non-responsiveness and suspicious tags automatically detected! - // For web applications, web browser errors and warnings can also be enabled via settings - Verdict verdict = super.getVerdict(state); - + protected List getVerdicts(State state) { // Apply state conditions to check if the test goal has been completed if(conditionEvaluator.evaluateConditions(state)) { // Test goal was completed, retrieve next test goal from queue. @@ -188,7 +184,7 @@ protected Verdict getVerdict(State state) { // Poll returns null if there are no more items remaining in the queue. if(currentTestGoal == null) { // No more test goals remaining, terminate sequence. - return new Verdict(Verdict.Severity.CONDITION_COMPLETE, "All test goal conditions completed."); + return Collections.singletonList(new Verdict(Verdict.Severity.CONDITION_COMPLETE, "All test goal conditions completed.")); } else { llmActionSelector.reset(currentTestGoal, true); conditionEvaluator.clear(); @@ -197,7 +193,7 @@ protected Verdict getVerdict(State state) { } } - return verdict; + return super.getVerdicts(state); } /** diff --git a/testar/resources/settings/webdriver_remote_webcomponent/Protocol_webdriver_remote_webcomponent.java b/testar/resources/settings/webdriver_remote_webcomponent/Protocol_webdriver_remote_webcomponent.java index 4cf1a05c3..10c5d7ac0 100644 --- a/testar/resources/settings/webdriver_remote_webcomponent/Protocol_webdriver_remote_webcomponent.java +++ b/testar/resources/settings/webdriver_remote_webcomponent/Protocol_webdriver_remote_webcomponent.java @@ -1,6 +1,6 @@ /** - * Copyright (c) 2018 - 2023 Open Universiteit - www.ou.nl - * Copyright (c) 2019 - 2023 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2019 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -134,9 +134,9 @@ protected State getState(SUT system) throws StateBuildException { * @return oracle verdict, which determines whether the state is erroneous and why. */ @Override - protected Verdict getVerdict(State state) { + protected List getVerdicts(State state) { - Verdict verdict = super.getVerdict(state); + List verdicts = super.getVerdicts(state); // system crashes, non-responsiveness and suspicious titles automatically detected! //----------------------------------------------------------------------------- @@ -145,7 +145,7 @@ protected Verdict getVerdict(State state) { // ... YOU MAY WANT TO CHECK YOUR CUSTOM ORACLES HERE ... - return verdict; + return verdicts; } /** diff --git a/testar/resources/settings/webdriver_security_analysis/Protocol_webdriver_security_analysis.java b/testar/resources/settings/webdriver_security_analysis/Protocol_webdriver_security_analysis.java index 142ae431c..c16e065af 100644 --- a/testar/resources/settings/webdriver_security_analysis/Protocol_webdriver_security_analysis.java +++ b/testar/resources/settings/webdriver_security_analysis/Protocol_webdriver_security_analysis.java @@ -1,6 +1,6 @@ /** - * Copyright (c) 2018 - 2024 Open Universiteit - www.ou.nl - * Copyright (c) 2018 - 2024 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2018 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -141,11 +141,13 @@ protected void beginSequence(SUT system, State state) { } @Override - protected Verdict getVerdict(State state) { + protected List getVerdicts(State state) { securityResultWriter.WriteVisit(WdDriver.getCurrentUrl()); - Verdict verdict = Verdict.OK; - oracleOrchestrator.getVerdict(verdict); - return verdict; + List verdicts = oracleOrchestrator.getVerdicts(); + if (verdicts == null || verdicts.isEmpty()) { + return Collections.singletonList(Verdict.OK); + } + return verdicts; } @Override diff --git a/testar/resources/workflow/settings/test_gradle_workflow_android_generic/Protocol_test_gradle_workflow_android_generic.java b/testar/resources/workflow/settings/test_gradle_workflow_android_generic/Protocol_test_gradle_workflow_android_generic.java index d468a12b2..1b33d827a 100644 --- a/testar/resources/workflow/settings/test_gradle_workflow_android_generic/Protocol_test_gradle_workflow_android_generic.java +++ b/testar/resources/workflow/settings/test_gradle_workflow_android_generic/Protocol_test_gradle_workflow_android_generic.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2020 - 2025 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2020 - 2025 Open Universiteit - www.ou.nl + * Copyright (c) 2020 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2020 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -50,6 +50,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; +import java.util.List; import java.util.Scanner; import java.util.Set; @@ -139,19 +140,25 @@ protected void postSequenceProcessing() { // Verify html and txt report files were created Assert.isTrue(reportManager instanceof ReportManager); - File htmlReportFile = new File(((ReportManager)reportManager).getReportFileName().concat("_" + getFinalVerdict().verdictSeverityTitle() + ".html")); - File txtReportFile = new File(((ReportManager)reportManager).getReportFileName().concat("_" + getFinalVerdict().verdictSeverityTitle() + ".txt")); - System.out.println("htmlReportFile: " + htmlReportFile.getPath()); - System.out.println("txtReportFile: " + txtReportFile.getPath()); - Assert.isTrue(htmlReportFile.exists()); - Assert.isTrue(txtReportFile.exists()); - - // Verify report information - Assert.isTrue(fileContains("

TESTAR execution sequence report for sequence 1

", htmlReportFile)); - Assert.isTrue(fileContains("TESTAR execution sequence report for sequence 1", txtReportFile)); - - Assert.isTrue(fileContains("

Test verdict for this sequence:", htmlReportFile)); - Assert.isTrue(fileContains("Test verdict for this sequence:", txtReportFile)); + List verdicts = getFinalVerdicts(); + Assert.isTrue(verdicts.size() > 0); + int index = 1; + for (Verdict verdict : verdicts) { + String suffixName = String.format("_V%03d_%s", index++, verdict.verdictSeverityTitle()); + File htmlReportFile = new File(((ReportManager)reportManager).getReportFileName().concat(suffixName + ".html")); + File txtReportFile = new File(((ReportManager)reportManager).getReportFileName().concat(suffixName + ".txt")); + System.out.println("htmlReportFile: " + htmlReportFile.getPath()); + System.out.println("txtReportFile: " + txtReportFile.getPath()); + Assert.isTrue(htmlReportFile.exists()); + Assert.isTrue(txtReportFile.exists()); + + // Verify report information + Assert.isTrue(fileContains("

TESTAR execution sequence report for sequence 1

", htmlReportFile)); + Assert.isTrue(fileContains("TESTAR execution sequence report for sequence 1", txtReportFile)); + + Assert.isTrue(fileContains("

Test verdict for this sequence:", htmlReportFile)); + Assert.isTrue(fileContains("Test verdict for this sequence:", txtReportFile)); + } } private boolean folderIsEmpty(Path path) { diff --git a/testar/resources/workflow/settings/test_gradle_workflow_desktop_generic/Protocol_test_gradle_workflow_desktop_generic.java b/testar/resources/workflow/settings/test_gradle_workflow_desktop_generic/Protocol_test_gradle_workflow_desktop_generic.java index 3a3e9c0ee..08733e20d 100644 --- a/testar/resources/workflow/settings/test_gradle_workflow_desktop_generic/Protocol_test_gradle_workflow_desktop_generic.java +++ b/testar/resources/workflow/settings/test_gradle_workflow_desktop_generic/Protocol_test_gradle_workflow_desktop_generic.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2020 - 2023 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2020 - 2023 Open Universiteit - www.ou.nl + * Copyright (c) 2020 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2020 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -33,6 +33,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.Arrays; +import java.util.List; import java.util.Scanner; import java.util.Set; @@ -121,7 +122,7 @@ protected void postSequenceProcessing() { super.postSequenceProcessing(); // If OnlySaveFaultySequences is enabled and the sequence verdict is OK, sequence_ok must not exist - if(settings().get(ConfigTags.OnlySaveFaultySequences) && (getFinalVerdict()).severity() == Verdict.OK.severity()) { + if(settings().get(ConfigTags.OnlySaveFaultySequences) && Verdict.helperAreAllVerdictsOK(getFinalVerdicts())) { String sequencesOkFolderName = OutputStructure.outerLoopOutputDir + File.separator + "sequences_ok"; File sequencesOkFolder = null; try { @@ -159,19 +160,25 @@ protected void postSequenceProcessing() { // Verify html and txt report files were created Assert.isTrue(reportManager instanceof ReportManager); - File htmlReportFile = new File(((ReportManager)reportManager).getReportFileName().concat("_" + getFinalVerdict().verdictSeverityTitle() + ".html")); - File txtReportFile = new File(((ReportManager)reportManager).getReportFileName().concat("_" + getFinalVerdict().verdictSeverityTitle() + ".txt")); - System.out.println("htmlReportFile: " + htmlReportFile.getPath()); - System.out.println("txtReportFile: " + txtReportFile.getPath()); - Assert.isTrue(htmlReportFile.exists()); - Assert.isTrue(txtReportFile.exists()); - - // Verify report information - Assert.isTrue(fileContains("

TESTAR execution sequence report for sequence 1

", htmlReportFile)); - Assert.isTrue(fileContains("TESTAR execution sequence report for sequence 1", txtReportFile)); - - Assert.isTrue(fileContains("

Test verdict for this sequence:", htmlReportFile)); - Assert.isTrue(fileContains("Test verdict for this sequence:", txtReportFile)); + List verdicts = getFinalVerdicts(); + Assert.isTrue(verdicts.size() > 0); + int index = 1; + for (Verdict verdict : verdicts) { + String suffixName = String.format("_V%03d_%s", index++, verdict.verdictSeverityTitle()); + File htmlReportFile = new File(((ReportManager)reportManager).getReportFileName().concat(suffixName + ".html")); + File txtReportFile = new File(((ReportManager)reportManager).getReportFileName().concat(suffixName + ".txt")); + System.out.println("htmlReportFile: " + htmlReportFile.getPath()); + System.out.println("txtReportFile: " + txtReportFile.getPath()); + Assert.isTrue(htmlReportFile.exists()); + Assert.isTrue(txtReportFile.exists()); + + // Verify report information + Assert.isTrue(fileContains("

TESTAR execution sequence report for sequence 1

", htmlReportFile)); + Assert.isTrue(fileContains("TESTAR execution sequence report for sequence 1", txtReportFile)); + + Assert.isTrue(fileContains("

Test verdict for this sequence:", htmlReportFile)); + Assert.isTrue(fileContains("Test verdict for this sequence:", txtReportFile)); + } } private boolean fileContains(String searchText, File file) { diff --git a/testar/resources/workflow/settings/test_gradle_workflow_webdriver_form_filling/Protocol_test_gradle_workflow_webdriver_form_filling.java b/testar/resources/workflow/settings/test_gradle_workflow_webdriver_form_filling/Protocol_test_gradle_workflow_webdriver_form_filling.java index 36af47369..68ea82e0c 100644 --- a/testar/resources/workflow/settings/test_gradle_workflow_webdriver_form_filling/Protocol_test_gradle_workflow_webdriver_form_filling.java +++ b/testar/resources/workflow/settings/test_gradle_workflow_webdriver_form_filling/Protocol_test_gradle_workflow_webdriver_form_filling.java @@ -1,6 +1,6 @@ /** - * Copyright (c) 2021 - 2023 Open Universiteit - www.ou.nl - * Copyright (c) 2021 - 2023 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2021 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2021 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -59,9 +59,9 @@ public class Protocol_test_gradle_workflow_webdriver_form_filling extends WebdriverProtocol { @Override - protected Verdict getVerdict(State state) { + protected List getVerdicts(State state) { // For custom CI testing purposes, force these generated sequences be OK - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } @Override diff --git a/testar/resources/workflow/settings/test_gradle_workflow_webdriver_generic/Protocol_test_gradle_workflow_webdriver_generic.java b/testar/resources/workflow/settings/test_gradle_workflow_webdriver_generic/Protocol_test_gradle_workflow_webdriver_generic.java index e55dc03e2..153982014 100644 --- a/testar/resources/workflow/settings/test_gradle_workflow_webdriver_generic/Protocol_test_gradle_workflow_webdriver_generic.java +++ b/testar/resources/workflow/settings/test_gradle_workflow_webdriver_generic/Protocol_test_gradle_workflow_webdriver_generic.java @@ -1,6 +1,6 @@ /** - * Copyright (c) 2021 - 2023 Open Universiteit - www.ou.nl - * Copyright (c) 2021 - 2023 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2021 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2021 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -170,19 +170,25 @@ protected void postSequenceProcessing() { // Verify html and txt report files were created Assert.isTrue(reportManager instanceof ReportManager); - File htmlReportFile = new File(((ReportManager)reportManager).getReportFileName().concat("_" + getFinalVerdict().verdictSeverityTitle() + ".html")); - File txtReportFile = new File(((ReportManager)reportManager).getReportFileName().concat("_" + getFinalVerdict().verdictSeverityTitle() + ".txt")); - System.out.println("htmlReportFile: " + htmlReportFile.getPath()); - System.out.println("txtReportFile: " + txtReportFile.getPath()); - Assert.isTrue(htmlReportFile.exists()); - Assert.isTrue(txtReportFile.exists()); - - // Verify report information - Assert.isTrue(fileContains("

TESTAR execution sequence report for sequence 1

", htmlReportFile)); - Assert.isTrue(fileContains("TESTAR execution sequence report for sequence 1", txtReportFile)); - - Assert.isTrue(fileContains("

Test verdict for this sequence:", htmlReportFile)); - Assert.isTrue(fileContains("Test verdict for this sequence:", txtReportFile)); + List verdicts = getFinalVerdicts(); + Assert.isTrue(verdicts.size() > 0); + int index = 1; + for (Verdict verdict : verdicts) { + String suffixName = String.format("_V%03d_%s", index++, verdict.verdictSeverityTitle()); + File htmlReportFile = new File(((ReportManager)reportManager).getReportFileName().concat(suffixName + ".html")); + File txtReportFile = new File(((ReportManager)reportManager).getReportFileName().concat(suffixName + ".txt")); + System.out.println("htmlReportFile: " + htmlReportFile.getPath()); + System.out.println("txtReportFile: " + txtReportFile.getPath()); + Assert.isTrue(htmlReportFile.exists()); + Assert.isTrue(txtReportFile.exists()); + + // Verify report information + Assert.isTrue(fileContains("

TESTAR execution sequence report for sequence 1

", htmlReportFile)); + Assert.isTrue(fileContains("TESTAR execution sequence report for sequence 1", txtReportFile)); + + Assert.isTrue(fileContains("

Test verdict for this sequence:", htmlReportFile)); + Assert.isTrue(fileContains("Test verdict for this sequence:", txtReportFile)); + } } private boolean fileContains(String searchText, File file) { diff --git a/testar/resources/workflow/settings/test_gradle_workflow_webdriver_replay/Protocol_test_gradle_workflow_webdriver_replay.java b/testar/resources/workflow/settings/test_gradle_workflow_webdriver_replay/Protocol_test_gradle_workflow_webdriver_replay.java index 983536998..975a5111b 100644 --- a/testar/resources/workflow/settings/test_gradle_workflow_webdriver_replay/Protocol_test_gradle_workflow_webdriver_replay.java +++ b/testar/resources/workflow/settings/test_gradle_workflow_webdriver_replay/Protocol_test_gradle_workflow_webdriver_replay.java @@ -1,6 +1,6 @@ /** - * Copyright (c) 2021 - 2023 Open Universiteit - www.ou.nl - * Copyright (c) 2021 - 2023 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2021 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2021 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -131,19 +131,25 @@ protected void postSequenceProcessing() { // Verify html and txt report files were created Assert.isTrue(reportManager instanceof ReportManager); - File htmlReportFile = new File(((ReportManager)reportManager).getReportFileName().concat("_" + getReplayVerdict().verdictSeverityTitle() + ".html")); - File txtReportFile = new File(((ReportManager)reportManager).getReportFileName().concat("_" + getReplayVerdict().verdictSeverityTitle() + ".txt")); - System.out.println("htmlReportFile: " + htmlReportFile.getPath()); - System.out.println("txtReportFile: " + txtReportFile.getPath()); - Assert.isTrue(htmlReportFile.exists()); - Assert.isTrue(txtReportFile.exists()); - - // Verify report information - Assert.isTrue(fileContains("

TESTAR replay sequence report", htmlReportFile)); - Assert.isTrue(fileContains("TESTAR replay sequence report", txtReportFile)); - - Assert.isTrue(fileContains("

Test verdict for this sequence:", htmlReportFile)); - Assert.isTrue(fileContains("Test verdict for this sequence:", txtReportFile)); + List verdicts = getReplayVerdicts(); + Assert.isTrue(verdicts.size() > 0); + int index = 1; + for (Verdict verdict : verdicts) { + String suffixName = String.format("_V%03d_%s", index++, verdict.verdictSeverityTitle()); + File htmlReportFile = new File(((ReportManager)reportManager).getReportFileName().concat(suffixName + ".html")); + File txtReportFile = new File(((ReportManager)reportManager).getReportFileName().concat(suffixName + ".txt")); + System.out.println("htmlReportFile: " + htmlReportFile.getPath()); + System.out.println("txtReportFile: " + txtReportFile.getPath()); + Assert.isTrue(htmlReportFile.exists()); + Assert.isTrue(txtReportFile.exists()); + + // Verify report information + Assert.isTrue(fileContains("

TESTAR replay sequence report", htmlReportFile)); + Assert.isTrue(fileContains("TESTAR replay sequence report", txtReportFile)); + + Assert.isTrue(fileContains("

Test verdict for this sequence:", htmlReportFile)); + Assert.isTrue(fileContains("Test verdict for this sequence:", txtReportFile)); + } } private boolean fileContains(String searchText, File file) { diff --git a/testar/src/org/testar/FileHandling.java b/testar/src/org/testar/FileHandling.java index c7c1a3b0d..bf0c7c0fd 100644 --- a/testar/src/org/testar/FileHandling.java +++ b/testar/src/org/testar/FileHandling.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2013 - 2025 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2018 - 2025 Open Universiteit - www.ou.nl + * Copyright (c) 2013 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,15 +32,16 @@ package org.testar; import org.testar.serialisation.LogSerialiser; -import org.testar.monkey.Util; import org.testar.monkey.alayer.Verdict; import org.testar.monkey.alayer.exceptions.NoSuchTagException; import java.io.*; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; public class FileHandling { - public static String copyClassifiedSequence(String generatedSequence, File currentSeq, Verdict verdict) { + public static String copyClassifiedSequence(String generatedSequence, File currentSeq, Verdict verdict, String suffixName) { // Generate the target folder names based on the severity title String targetFolder = "sequences_" + verdict.verdictSeverityTitle().toLowerCase(); String targetSequence = targetFolder + File.separator + generatedSequence; @@ -74,9 +75,9 @@ public static String copyClassifiedSequence(String generatedSequence, File curre // If it exists, copy to the specific classification folder try { - copyToOutputDir(currentSeq, targetFolder); + copyToOutputDirWithSuffix(currentSeq, targetFolder, suffixName); } catch (NoSuchTagException | IOException e) { - LogSerialiser.log("Error copying classified test sequence: " + e.getMessage() + "\n", + LogSerialiser.log("Error copying classified test sequence: " + e.getMessage() + "\n", LogSerialiser.LogLevel.Critical ); } @@ -94,14 +95,24 @@ public static String copyClassifiedSequence(String generatedSequence, File curre * * @param file The sequence file to copy. * @param folderName The target folder name. + * @param suffixName The verdict suffix name (number + title). * @throws IOException If an I/O error occurs. * @throws NoSuchTagException If the specified tag does not exist. */ - private static void copyToOutputDir(File file, String folderName) throws IOException, NoSuchTagException { - Util.copyToDirectory( - file.getCanonicalPath(), - OutputStructure.outerLoopOutputDir + File.separator + folderName, - true - ); + private static void copyToOutputDirWithSuffix(File file, String folderName, String suffixName) throws IOException, NoSuchTagException { + String outputDir = OutputStructure.outerLoopOutputDir + File.separator + folderName; + File targetDir = new File(outputDir); + if (!targetDir.exists()) { + targetDir.mkdirs(); + } + + String fileName = file.getName(); + int dotIndex = fileName.lastIndexOf('.'); + String newName = (dotIndex == -1) + ? fileName + suffixName + : fileName.substring(0, dotIndex) + suffixName + fileName.substring(dotIndex); + + File destination = new File(outputDir + File.separator + newName); + Files.copy(file.toPath(), destination.toPath(), StandardCopyOption.REPLACE_EXISTING); } } diff --git a/testar/src/org/testar/monkey/AbstractProtocol.java b/testar/src/org/testar/monkey/AbstractProtocol.java index 104bc275b..b091174c2 100644 --- a/testar/src/org/testar/monkey/AbstractProtocol.java +++ b/testar/src/org/testar/monkey/AbstractProtocol.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2013 - 2023 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2018 - 2023 Open Universiteit - www.ou.nl + * Copyright (c) 2013 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -40,6 +40,7 @@ import org.testar.settings.Settings; +import java.util.List; import java.util.Set; /** @@ -53,7 +54,7 @@ * BeginSequence (starting "script" on the GUI of the SUT, for example login) * INNER LOOP * GetState - * GetVerdict + * GetVerdicts * StopCriteria (moreActions/moreSequences/time?) * DeriveActions * SelectAction @@ -127,12 +128,12 @@ public abstract class AbstractProtocol implements UnProc { protected abstract State getState(SUT system) throws StateBuildException; /** - * The getVerdict methods implements the online state oracles that - * examine the SUT's current state and returns an oracle verdict. + * The getVerdicts methods implements the online state oracles that + * examine the SUT's current state and returns oracle verdicts. * - * @return oracle verdict, which determines whether the state is erroneous and why. + * @return list of oracle verdicts, which determine whether the state is erroneous and why. */ - protected abstract Verdict getVerdict(State state); + protected abstract List getVerdicts(State state); /** * This method is used by TESTAR to determine the set of currently available actions. diff --git a/testar/src/org/testar/monkey/ConfigTags.java b/testar/src/org/testar/monkey/ConfigTags.java index 00674fdd5..3fff862fc 100644 --- a/testar/src/org/testar/monkey/ConfigTags.java +++ b/testar/src/org/testar/monkey/ConfigTags.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2013 - 2025 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2018 - 2025 Open Universiteit - www.ou.nl + * Copyright (c) 2013 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -56,6 +56,9 @@ private ConfigTags() {} public static final Tag SequenceLength = Tag.from("SequenceLength", Integer.class, "For each test sequence, the number of GUI actions to perform"); + public static final Tag IgnoreDuplicatedVerdicts = Tag.from("IgnoreDuplicatedVerdicts", Boolean.class, + "Sets whether to ignore reporting duplicate verdicts across sequences for the same protocol"); + public static final Tag SuspiciousTags = Tag.from("SuspiciousTags", String.class, "Regular expressions ORACLE to find suspicious messages in the GUI Tags"); @@ -346,8 +349,6 @@ private ConfigTags() {} * Other settings */ - public static final Tag FaultThreshold = Tag.from("FaultThreshold", Double.class); - @SuppressWarnings("unchecked") public static final Tag> Delete = Tag.from("Delete", (Class>) (Class) List.class); diff --git a/testar/src/org/testar/monkey/DefaultProtocol.java b/testar/src/org/testar/monkey/DefaultProtocol.java index 9c5359d1b..37cfb4401 100644 --- a/testar/src/org/testar/monkey/DefaultProtocol.java +++ b/testar/src/org/testar/monkey/DefaultProtocol.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2013 - 2025 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2018 - 2025 Open Universiteit - www.ou.nl + * Copyright (c) 2013 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -34,9 +34,6 @@ import org.apache.logging.log4j.Logger; import org.testar.*; import org.testar.managers.NativeHookManager; -import org.testar.monkey.alayer.Action; -import org.testar.monkey.alayer.Canvas; -import org.testar.monkey.alayer.Color; import org.testar.monkey.alayer.*; import org.testar.monkey.alayer.actions.ActivateSystem; import org.testar.monkey.alayer.actions.AnnotatingActionCompiler; @@ -66,15 +63,35 @@ import org.testar.statemodel.StateModelManager; import org.testar.statemodel.StateModelManagerFactory; -import javax.swing.*; -import java.awt.*; -import java.io.*; +import java.awt.Desktop; +import java.awt.GraphicsEnvironment; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; import java.util.List; -import java.util.*; +import java.util.Map; +import java.util.Set; +import java.util.WeakHashMap; +import java.util.StringJoiner; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; +import javax.swing.JFrame; +import javax.swing.JOptionPane; +import javax.swing.JScrollPane; +import javax.swing.JTextArea; + import static org.testar.monkey.alayer.Tags.*; public class DefaultProtocol extends RuntimeControlsProtocol { @@ -85,6 +102,9 @@ public class DefaultProtocol extends RuntimeControlsProtocol { protected boolean processListenerOracleEnabled; protected Oracle processListenerOracle; + private List lastStateVerdicts = Collections.emptyList(); + private VerdictProcessing verdictProcessing; + private State stateForClickFilterLayerProtocol; protected Reporting reportManager = new DummyReportManager(); @@ -105,19 +125,25 @@ public String getGeneratedSequenceName() { protected Mouse mouse; - private Verdict replayVerdict; + private List replayVerdicts = Collections.singletonList(Verdict.OK); - public Verdict getReplayVerdict() { - return replayVerdict; + public List getReplayVerdicts() { + return replayVerdicts; } - public void setReplayVerdict(Verdict replayVerdict) { - this.replayVerdict = replayVerdict; + public void setReplayVerdicts(List replayVerdicts) { + this.replayVerdicts = replayVerdicts; } - Verdict finalVerdict = Verdict.OK; - public Verdict getFinalVerdict() { - return finalVerdict; + List finalVerdicts = Collections.singletonList(Verdict.OK); + + public List getFinalVerdicts() { + if (lastStateVerdicts == null || lastStateVerdicts.isEmpty()) { + return finalVerdicts == null || finalVerdicts.isEmpty() + ? Collections.singletonList(Verdict.OK) + : finalVerdicts; + } + return lastStateVerdicts; } protected String lastPrintParentsOf = "null-id"; @@ -279,6 +305,7 @@ protected void initialize(Settings settings) { startTime = Util.time(); this.settings = settings; mode = settings.get(ConfigTags.Mode); + verdictProcessing = new VerdictProcessing(settings); builder = NativeLinker.getNativeStateBuilder( settings.get(ConfigTags.TimeToFreeze), @@ -563,12 +590,30 @@ void emergencyTerminateTestSequence(SUT system, Exception e){ } } - void classifyAndCopySequenceIntoAppropriateDirectory(Verdict finalVerdict){ - // Check if user wants to save or not the sequences without faults - if (settings.get(ConfigTags.OnlySaveFaultySequences, false) && finalVerdict.severity() == Verdict.OK.severity()) { - LogSerialiser.log("Skipped generated sequence OK (\"" + this.generatedSequence + "\")\n", LogSerialiser.LogLevel.Info); - } else { - this.generatedSequence = FileHandling.copyClassifiedSequence(this.generatedSequence, currentSeq, finalVerdict); + void classifyAndCopySequenceIntoAppropriateDirectory(List verdicts){ + List verdictList = (verdicts == null || verdicts.isEmpty()) + ? Collections.singletonList(Verdict.OK) + : verdicts; + + int index = 1; + String firstTargetSequence = null; + for (Verdict verdict : verdictList) { + if (verdict == null) { + continue; + } + if (settings.get(ConfigTags.OnlySaveFaultySequences, false) + && verdict.severity() == Verdict.OK.severity()) { + LogSerialiser.log("Skipped generated sequence OK (\"" + this.generatedSequence + "\")\n", LogSerialiser.LogLevel.Info); + continue; + } + String suffixName = String.format("_V%03d_%s", index++, verdict.verdictSeverityTitle()); + String targetSequence = FileHandling.copyClassifiedSequence(this.generatedSequence, currentSeq, verdict, suffixName); + if (firstTargetSequence == null) { + firstTargetSequence = targetSequence; + } + } + if (firstTargetSequence != null) { + this.generatedSequence = firstTargetSequence; } } @@ -590,7 +635,7 @@ void saveActionIntoFragmentForReplayableSequence(Action action, State state, Set fragment.set(ActionDuration, settings().get(ConfigTags.ActionDuration)); fragment.set(ActionDelay, settings().get(ConfigTags.TimeToWaitAfterAction)); fragment.set(SystemState, state); - fragment.set(OracleVerdict, getFinalVerdict()); + fragment.set(OracleVerdicts, getFinalVerdicts()); //Find the target widget of the current action, and save the title into the fragment if (state != null && action.get(Tags.OriginWidget, null) != null){ @@ -687,7 +732,7 @@ protected Canvas buildCanvas() { @Override protected void beginSequence(SUT system, State state){ // Reset the final verdict for the new sequence - finalVerdict = Verdict.OK; + finalVerdicts = Collections.singletonList(Verdict.OK); } @Override @@ -738,7 +783,7 @@ protected SUT startSystem() throws SystemStartException { /** * This method gets the state of the SUT - * It also call getVerdict() and saves it into the state + * It also call getVerdicts() and saves it into the state * * @param system * @return @@ -757,20 +802,22 @@ protected State getState(SUT system) throws StateBuildException { if(settings.get(ConfigTags.Mode) == Modes.Spy) return state; + List verdicts = getVerdicts(state); + lastStateVerdicts = verdictProcessing.filterDuplicates(verdicts); + state.set(Tags.OracleVerdicts, lastStateVerdicts); + + // State screenshot is taken after the Verdicts are computed + // This might be relevant to determine the viewPort of the screenshot depending on the Verdict (e.g., ProtocolUtil) setStateScreenshot(state); - Verdict verdict = getVerdict(state); - state.set(Tags.OracleVerdict, verdict); - - if(mode() != Modes.Spy && verdict.severity() >= settings().get(ConfigTags.FaultThreshold)) - { - LogSerialiser.log("Detected fault: " + verdict + "\n", LogSerialiser.LogLevel.Critical); - // this was added to kill the SUT if it is frozen: - if(verdict.severity() == Verdict.Severity.NOT_RESPONDING.getValue()) - { - //if the SUT is frozen, we should kill it! - LogSerialiser.log("SUT frozen, trying to kill it!\n", LogSerialiser.LogLevel.Critical); - SystemProcessHandling.killRunningProcesses(system, 100); + if (mode() != Modes.Spy) { + for (Verdict verdict : lastStateVerdicts) { + // this was added to kill the SUT if it is frozen: + if (verdict.severity() == Verdict.Severity.NOT_RESPONDING.getValue()) { + //if the SUT is frozen, we should kill it! + LogSerialiser.log("SUT frozen, trying to kill it!\n", LogSerialiser.LogLevel.Critical); + SystemProcessHandling.killRunningProcesses(system, 100); + } } } @@ -808,26 +855,33 @@ else if(NativeLinker.getPLATFORM_OS().contains(OperatingSystems.IOS)) { } @Override - protected Verdict getVerdict(State state){ + protected List getVerdicts(State state) { Assert.notNull(state); + List verdicts = new ArrayList<>(); + //------------------- // ORACLES FOR FREE //------------------- if ( processListenerOracleEnabled ) { - Verdict processListenerVerdict = processListenerOracle.getVerdict(state); - if ( processListenerVerdict.severity() == Verdict.Severity.SUSPICIOUS_PROCESS.getValue() ) { - return processListenerVerdict; + List processListenerVerdicts = processListenerOracle.getVerdicts(state); + for (Verdict processListenerVerdict : processListenerVerdicts) { + if ( processListenerVerdict.severity() == Verdict.Severity.SUSPICIOUS_PROCESS.getValue() ) { + verdicts.add(processListenerVerdict); + } } } // if the SUT is not running and closed unexpectedly, we assume it crashed - if(!state.get(IsRunning, false)) - return new Verdict(Verdict.Severity.UNEXPECTEDCLOSE, "System is offline! Closed Unexpectedly! I assume it crashed!"); + if(!state.get(IsRunning, false)) { + verdicts.add(new Verdict(Verdict.Severity.UNEXPECTEDCLOSE, "System is offline! Closed Unexpectedly! I assume it crashed!")); + return verdicts; + } // if the SUT does not respond within a given amount of time, we assume it crashed if(state.get(Tags.NotResponding, false)){ - return new Verdict(Verdict.Severity.NOT_RESPONDING, "System is unresponsive! I assume something is wrong!"); + verdicts.add(new Verdict(Verdict.Severity.NOT_RESPONDING, "System is unresponsive! I assume something is wrong!")); + return verdicts; } //------------------------ @@ -838,23 +892,28 @@ protected Verdict getVerdict(State state){ this.suspiciousTitlesPattern = Pattern.compile(settings().get(ConfigTags.SuspiciousTags), Pattern.UNICODE_CHARACTER_CLASS); // search all widgets for suspicious String Values - Verdict suspiciousValueVerdict = Verdict.OK; for(Widget w : state) { - suspiciousValueVerdict = suspiciousStringValueMatcher(w); + Verdict suspiciousValueVerdict = suspiciousStringValueMatcher(w); if(suspiciousValueVerdict.severity() == Verdict.Severity.SUSPICIOUS_TAG.getValue()) { - return suspiciousValueVerdict; + verdicts.add(suspiciousValueVerdict); } } if ( logOracleEnabled ) { - Verdict logVerdict = logOracle.getVerdict(state); - if ( logVerdict.severity() == Verdict.Severity.SUSPICIOUS_LOG.getValue() ) { - return logVerdict; + List logVerdicts = logOracle.getVerdicts(state); + for (Verdict logVerdict : logVerdicts) { + if ( logVerdict.severity() == Verdict.Severity.SUSPICIOUS_LOG.getValue() ) { + verdicts.add(logVerdict); + } } } - // if everything was OK ... - return Verdict.OK; + // if empty at this point, everything was OK + if (verdicts.isEmpty()) { + verdicts.add(Verdict.OK); + } + + return verdicts; } private Verdict suspiciousStringValueMatcher(Widget w) { @@ -1105,8 +1164,8 @@ protected Action selectAction(State state, Set actions){ * @return */ protected boolean moreActions(State state) { - Verdict stateVerdict = state.get(Tags.OracleVerdict, Verdict.OK); - boolean faultySequence = stateVerdict.severity() != Verdict.OK.severity(); + List stateVerdicts = state.get(Tags.OracleVerdicts, Collections.singletonList(Verdict.OK)); + boolean faultySequence = !Verdict.helperAreAllVerdictsOK(stateVerdicts); return (!settings().get(ConfigTags.StopGenerationOnFault) || !faultySequence) && state.get(Tags.IsRunning, false) && !state.get(Tags.NotResponding, false) && //actionCount() < settings().get(ConfigTags.SequenceLength) && @@ -1141,20 +1200,14 @@ protected void stopSystem(SUT system) { */ @Override protected void postSequenceProcessing() { - - String status = ""; String statusInfo = ""; - if(mode() == Modes.Replay) { - reportManager.addTestVerdict(getReplayVerdict()); - status = (getReplayVerdict()).verdictSeverityTitle(); - statusInfo = (getReplayVerdict()).info(); - } - else { - reportManager.addTestVerdict(getFinalVerdict()); - status = (getFinalVerdict()).verdictSeverityTitle(); - statusInfo = (getFinalVerdict()).info(); - } + List verdicts = (lastStateVerdicts == null || lastStateVerdicts.isEmpty()) + ? (mode() == Modes.Replay ? getReplayVerdicts() : getFinalVerdicts()) + : lastStateVerdicts; + + reportManager.addTestVerdicts(verdicts); + statusInfo = buildStatusInfo(verdicts); this.generatedSequence = OutputStructure.outerLoopOutputDir + File.separator + this.generatedSequence; try { @@ -1163,17 +1216,31 @@ protected void postSequenceProcessing() { LogSerialiser.log("Error generating sequence canonical path for the index logger\n", LogSerialiser.LogLevel.Critical); } - statusInfo = statusInfo.replace("\n"+Verdict.OK.info(), ""); - - //Timestamp(generated by logger) SUTname Mode SequenceFileObject Status "StatusInfo" + //Timestamp(generated by logger) SUTname Mode SequenceFileObject "StatusInfo" INDEXLOG.info(OutputStructure.executedSUTname + " " + settings.get(ConfigTags.Mode, mode()) + " " + this.generatedSequence - + " " + status + " \"" + statusInfo + "\"" ); + + " " + statusInfo); + + if(mode() == Modes.Generate) verdictProcessing.storeNewVerdicts(verdicts); reportManager.finishReport(); } + private String buildStatusInfo(List verdicts) { + List normalized = (verdicts == null || verdicts.isEmpty()) + ? Collections.singletonList(Verdict.OK) + : verdicts; + if (normalized.size() == 1) { + return normalized.get(0).info(); + } + StringJoiner joiner = new StringJoiner(" | "); + for (Verdict verdict : normalized) { + joiner.add("[" + verdict.verdictSeverityTitle() + "] " + verdict.info()); + } + return joiner.toString(); + } + /** * method for closing the internal TESTAR test session */ diff --git a/testar/src/org/testar/monkey/GenerateMode.java b/testar/src/org/testar/monkey/GenerateMode.java index 4ef6541df..734795e33 100644 --- a/testar/src/org/testar/monkey/GenerateMode.java +++ b/testar/src/org/testar/monkey/GenerateMode.java @@ -42,6 +42,7 @@ import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Set; import java.util.StringJoiner; @@ -97,16 +98,16 @@ public void runGenerateOuterLoop(DefaultProtocol protocol) { // Initial getState() called before beginSequence: LogSerialiser.log("Obtaining system initial state before beginSequence...\n", LogSerialiser.LogLevel.Debug); State initialState = protocol.getState(system); - protocol.finalVerdict = initialState.get(Tags.OracleVerdict, Verdict.OK); + protocol.finalVerdicts = initialState.get(Tags.OracleVerdicts, Collections.singletonList(Verdict.OK)); // If the SUT does not contain initial failures, start the inner loop test sequence - if(protocol.finalVerdict.severity() == Verdict.OK.severity()) { + if(Verdict.helperAreAllVerdictsOK(protocol.finalVerdicts)) { // beginSequence() - a script to interact with GUI, for example login screen LogSerialiser.log("Invoking begin sequence in the initial state...\n", LogSerialiser.LogLevel.Debug); protocol.beginSequence(system, initialState); // starting the INNER LOOP with the updated state after SUT modification - protocol.finalVerdict = runGenerateInnerLoop(protocol, system, protocol.getState(system)); + protocol.finalVerdicts = runGenerateInnerLoop(protocol, system, protocol.getState(system)); } else { // If failure exists in the initial state // Save initial state information in the state model before finishing @@ -151,7 +152,7 @@ public void runGenerateOuterLoop(DefaultProtocol protocol) { * * @param system */ - private Verdict runGenerateInnerLoop(DefaultProtocol protocol, SUT system, State state) { + private List runGenerateInnerLoop(DefaultProtocol protocol, SUT system, State state) { // Deriving actions from the state after begin sequence: Set actions = protocol.deriveActions(system, state); @@ -212,7 +213,7 @@ private Verdict runGenerateInnerLoop(DefaultProtocol protocol, SUT system, State protocol.stateModelManager.notifyNewStateReached(state, actions); } - return state.get(Tags.OracleVerdict, Verdict.OK); + return state.get(Tags.OracleVerdicts, Collections.singletonList(Verdict.OK)); } private void finishGeneratedSequence(DefaultProtocol protocol, SUT system) { @@ -224,11 +225,11 @@ private void finishGeneratedSequence(DefaultProtocol protocol, SUT system) { protocol.writeAndCloseFragmentForReplayableSequence(); - if (protocol.finalVerdict.severity() != Verdict.OK.severity()) + if (!Verdict.helperAreAllVerdictsOK(protocol.finalVerdicts)) LogSerialiser.log("Sequence contained faults!\n", LogSerialiser.LogLevel.Critical); // Copy sequence file into proper directory: - protocol.classifyAndCopySequenceIntoAppropriateDirectory(protocol.getFinalVerdict()); + protocol.classifyAndCopySequenceIntoAppropriateDirectory(protocol.getFinalVerdicts()); // Calling postSequenceProcessing() to allow resetting test environment after test sequence, etc protocol.postSequenceProcessing(); diff --git a/testar/src/org/testar/monkey/ReplayMode.java b/testar/src/org/testar/monkey/ReplayMode.java index ee140d975..c83c07582 100644 --- a/testar/src/org/testar/monkey/ReplayMode.java +++ b/testar/src/org/testar/monkey/ReplayMode.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2022 - 2023 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2022 - 2023 Open Universiteit - www.ou.nl + * Copyright (c) 2022 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2022 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -38,6 +38,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.ObjectInputStream; +import java.util.Collections; import java.util.List; import java.util.Set; import java.util.zip.GZIPInputStream; @@ -106,7 +107,7 @@ public void runReplayLoop(DefaultProtocol protocol) { protocol.cv = protocol.buildCanvas(); State state = protocol.getState(system); - protocol.setReplayVerdict(protocol.getVerdict(state)); + protocol.setReplayVerdicts(protocol.getVerdicts(state)); // notify the statemodelmanager protocol.stateModelManager.notifyTestSequencedStarted(); @@ -114,7 +115,7 @@ public void runReplayLoop(DefaultProtocol protocol) { double rrt = protocol.settings().get(ConfigTags.ReplayRetryTime); while(success - && protocol.getReplayVerdict().severity() == Verdict.OK.severity() + && Verdict.helperAreAllVerdictsOK(protocol.getReplayVerdicts()) && protocol.mode() == Modes.Replay) { //Initialize local fragment and read saved action of PathToReplaySequence File @@ -131,14 +132,14 @@ public void runReplayLoop(DefaultProtocol protocol) { } else { success = false; String msg = "Exception " + ioe.getMessage() + " reading TESTAR replayableFragment: " + seqFile; - protocol.setReplayVerdict(new Verdict(Verdict.Severity.UNREPLAYABLE, msg)); + protocol.setReplayVerdicts(Collections.singletonList(new Verdict(Verdict.Severity.UNREPLAYABLE, msg))); protocol.stateModelManager.notifyTestSequenceInterruptedBySystem(ioe.toString()); break; } } catch(NullPointerException npe) { success = false; String msg = "Null exception replaying TESTAR action"; - protocol.setReplayVerdict(new Verdict(Verdict.Severity.UNREPLAYABLE, msg)); + protocol.setReplayVerdicts(Collections.singletonList(new Verdict(Verdict.Severity.UNREPLAYABLE, msg))); protocol.stateModelManager.notifyTestSequenceInterruptedBySystem(npe.toString()); break; } @@ -200,7 +201,7 @@ public void runReplayLoop(DefaultProtocol protocol) { + " of the replayed sequence can not been replayed into " + " the State " + state.get(Tags.ConcreteID, state.toString()); - protocol.setReplayVerdict(new Verdict(Verdict.Severity.UNREPLAYABLE, msg)); + protocol.setReplayVerdicts(Collections.singletonList(new Verdict(Verdict.Severity.UNREPLAYABLE, msg))); break; } @@ -249,7 +250,7 @@ public void runReplayLoop(DefaultProtocol protocol) { state = protocol.getState(system); - protocol.setReplayVerdict(protocol.getVerdict(state)); + protocol.setReplayVerdicts(protocol.getVerdicts(state)); } } @@ -284,8 +285,9 @@ public void runReplayLoop(DefaultProtocol protocol) { system.stop(); } - if(protocol.getReplayVerdict().severity() != Verdict.OK.severity()) { - String msg = "Replayed Sequence contains Errors: "+ protocol.getReplayVerdict().info(); + List replayVerdicts = protocol.getReplayVerdicts(); + if(!Verdict.helperAreAllVerdictsOK(replayVerdicts)) { + String msg = "Replayed Sequence contains Errors: " + buildVerdictsInfo(replayVerdicts); System.out.println(msg); LogSerialiser.log(msg, LogSerialiser.LogLevel.Info); @@ -294,9 +296,10 @@ public void runReplayLoop(DefaultProtocol protocol) { System.out.println(msg); LogSerialiser.log(msg, LogSerialiser.LogLevel.Info); - }else if(protocol.getReplayVerdict().severity() == Verdict.Severity.UNREPLAYABLE.getValue()){ - System.out.println(protocol.getReplayVerdict().info()); - LogSerialiser.log(protocol.getReplayVerdict().info(), LogSerialiser.LogLevel.Critical); + }else if(replayVerdicts.stream().anyMatch(v -> v.severity() == Verdict.Severity.UNREPLAYABLE.getValue())){ + String info = buildVerdictsInfo(replayVerdicts); + System.out.println(info); + LogSerialiser.log(info, LogSerialiser.LogLevel.Critical); }else{ String msg = "Fail replaying sequence.\n"; @@ -314,7 +317,7 @@ public void runReplayLoop(DefaultProtocol protocol) { protocol.writeAndCloseFragmentForReplayableSequence(); //Copy sequence file into proper directory: - protocol.classifyAndCopySequenceIntoAppropriateDirectory(protocol.getReplayVerdict()); + protocol.classifyAndCopySequenceIntoAppropriateDirectory(protocol.getReplayVerdicts()); LogSerialiser.finish(); @@ -329,4 +332,19 @@ public void runReplayLoop(DefaultProtocol protocol) { // Going back to TESTAR settings dialog if it was used to start replay: protocol.mode = Modes.Quit; } + + private String buildVerdictsInfo(List verdicts) { + if (verdicts == null || verdicts.isEmpty()) return ""; + if (verdicts.size() == 1) { + return verdicts.get(0).info(); + } + StringBuilder builder = new StringBuilder(); + for (Verdict verdict : verdicts) { + if (builder.length() > 0) { + builder.append(" | "); + } + builder.append("[").append(verdict.verdictSeverityTitle()).append("] ").append(verdict.info()); + } + return builder.toString(); + } } diff --git a/testar/src/org/testar/monkey/VerdictProcessing.java b/testar/src/org/testar/monkey/VerdictProcessing.java new file mode 100644 index 000000000..20d43899d --- /dev/null +++ b/testar/src/org/testar/monkey/VerdictProcessing.java @@ -0,0 +1,189 @@ +/*************************************************************************************************** + * + * Copyright (c) 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2026 Open Universiteit - www.ou.nl + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + *******************************************************************************************************/ + +package org.testar.monkey; + +import org.testar.monkey.alayer.Verdict; +import org.testar.serialisation.LogSerialiser; +import org.testar.settings.Settings; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class VerdictProcessing { + private static final String LIST_VERDICTS_FAILURES_FILENAME = "list_of_verdicts_with_failures.txt"; + + private final boolean ignoreDuplicatedVerdicts; + private final List verdictIgnoreList = new ArrayList<>(); + private final File verdictIgnoreFile; + + public VerdictProcessing(Settings settings) { + ignoreDuplicatedVerdicts = settings.get(ConfigTags.IgnoreDuplicatedVerdicts, false); + verdictIgnoreFile = resolveVerdictIgnoreFile(); + + if (!ignoreDuplicatedVerdicts || verdictIgnoreFile == null || !verdictIgnoreFile.exists()) { + return; + } + + try { + Path path = verdictIgnoreFile.toPath(); + for (String line : Files.readAllLines(path, StandardCharsets.UTF_8)) { + String trimmed = line.trim(); + if (!trimmed.isEmpty() && !verdictIgnoreList.contains(trimmed)) { + verdictIgnoreList.add(trimmed); + } + } + } catch (IOException e) { + LogSerialiser.log("Unable to read verdict ignore file: " + verdictIgnoreFile.getAbsolutePath() + "\n", + LogSerialiser.LogLevel.Info); + } + } + + public List filterDuplicates(List verdicts) { + if (verdicts == null || verdicts.isEmpty()) { + return Collections.singletonList(Verdict.OK); + } + List filtered = new ArrayList<>(); + for (Verdict verdict : verdicts) { + boolean shouldIgnore = ignoreDuplicatedVerdicts + && !verdict.isCritical() + && verdict.severity() > Verdict.Severity.OK.getValue() + && isDuplicateVerdictInfo(verdict.info()); + if (!shouldIgnore) { + filtered.add(verdict); + } + } + if (filtered.isEmpty()) { + return Collections.singletonList(Verdict.OK); + } + boolean allVerdictsOk = true; + for (Verdict verdict : filtered) { + if (verdict.severity() > Verdict.Severity.OK.getValue()) { + allVerdictsOk = false; + break; + } + } + if (allVerdictsOk) { + return Collections.singletonList(Verdict.OK); + } + return clearOkIfFailurePresent(filtered); + } + + public void storeNewVerdicts(List verdicts) { + if (verdicts == null || verdicts.isEmpty()) { + return; + } + for (Verdict verdict : verdicts) { + storeVerdictFailInfoIfNew(verdict); + } + } + + private File resolveVerdictIgnoreFile() { + String settingsPath = Settings.getSettingsPath(); + if (settingsPath != null && !settingsPath.isEmpty()) { + return new File(settingsPath, LIST_VERDICTS_FAILURES_FILENAME); + } + if (Main.SSE_ACTIVATED != null && !Main.SSE_ACTIVATED.isEmpty()) { + return new File(Main.settingsDir + Main.SSE_ACTIVATED, LIST_VERDICTS_FAILURES_FILENAME); + } + return null; + } + + private boolean isDuplicateVerdictInfo(String verdictInfo) { + String normalized = normalizeVerdictInfo(verdictInfo); + if (normalized.isEmpty()) { + return false; + } + return verdictIgnoreList.stream().anyMatch(info -> info.contains(normalized)); + } + + private void storeVerdictFailInfoIfNew(Verdict verdict) { + if (!ignoreDuplicatedVerdicts || verdict == null) { + return; + } + if (verdict.isCritical()) { + return; + } + if (verdict.severity() <= Verdict.Severity.OK.getValue()) { + return; + } + String normalized = normalizeVerdictInfo(verdict.info()); + if (normalized.isEmpty() || verdictIgnoreList.contains(normalized)) { + return; + } + verdictIgnoreList.add(normalized); + + if (verdictIgnoreFile == null) { + return; + } + try { + Files.write(verdictIgnoreFile.toPath(), + Collections.singletonList(normalized), + StandardCharsets.UTF_8, + StandardOpenOption.CREATE, + StandardOpenOption.APPEND); + } catch (IOException e) { + LogSerialiser.log("Unable to update verdict ignore file: " + verdictIgnoreFile.getAbsolutePath() + "\n", + LogSerialiser.LogLevel.Info); + } + } + + private String normalizeVerdictInfo(String verdictInfo) { + return verdictInfo == null ? "" : verdictInfo.replace("\n", " ").trim(); + } + + private List clearOkIfFailurePresent(List verdicts) { + boolean hasFailureVerdict = false; + for (Verdict verdict : verdicts) { + if (verdict != null && verdict.severity() > Verdict.Severity.OK.getValue()) { + hasFailureVerdict = true; + break; + } + } + if (!hasFailureVerdict) { + return verdicts; + } + List filtered = new ArrayList<>(); + for (Verdict verdict : verdicts) { + if (verdict != null && verdict.severity() > Verdict.Severity.OK.getValue()) { + filtered.add(verdict); + } + } + return filtered.isEmpty() ? verdicts : filtered; + } + +} diff --git a/testar/src/org/testar/oracles/Oracle.java b/testar/src/org/testar/oracles/Oracle.java index 60ea8a165..69952a105 100644 --- a/testar/src/org/testar/oracles/Oracle.java +++ b/testar/src/org/testar/oracles/Oracle.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2022 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2022 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2022 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2022 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -60,13 +60,13 @@ public interface Oracle { public abstract void initialize(); /** - * Request that the Oracle determine a verdict about the current state of the SUT. - * This method would usually be called by the getVerdict method in the protocol. + * Request that the Oracle determine verdicts about the current state of the SUT. + * This method would usually be called by the getVerdicts method in the protocol. * * @param state - * @return verdict + * @return list of verdicts */ - public abstract Verdict getVerdict(State state); + public abstract List getVerdicts(State state); /** * Provides a standard red pen for visual annotations. diff --git a/testar/src/org/testar/oracles/generic/visual/GenericVisualAlignmentMetricOracle.java b/testar/src/org/testar/oracles/generic/visual/GenericVisualAlignmentMetricOracle.java index 142a0cdb4..88f0132f3 100644 --- a/testar/src/org/testar/oracles/generic/visual/GenericVisualAlignmentMetricOracle.java +++ b/testar/src/org/testar/oracles/generic/visual/GenericVisualAlignmentMetricOracle.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2023 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2023 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2023 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2023 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,8 @@ package org.testar.oracles.generic.visual; import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import org.testar.monkey.alayer.Shape; import org.testar.monkey.alayer.State; @@ -65,7 +67,7 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { + public List getVerdicts(State state) { ArrayList regions = MetricsHelper.getRegions(state); double alignmentMetric = MetricsHelper.calculateAlignmentMetric(regions); @@ -73,10 +75,10 @@ public Verdict getVerdict(State state) { if (alignmentMetric < thresholdValue) { String verdictMsg = String.format("Alignment metric with value %f is below threshold value %f!", alignmentMetric, thresholdValue); Visualizer visualizer = new RegionsVisualizer(getRedPen(), regions, "Alignment Metric Warning", 0.5, 0.5); - return new Verdict(Verdict.Severity.WARNING_UI_VISUAL_OR_RENDERING_FAULT, verdictMsg, visualizer); + return Collections.singletonList(new Verdict(Verdict.Severity.WARNING_UI_VISUAL_OR_RENDERING_FAULT, verdictMsg, visualizer)); } - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } } diff --git a/testar/src/org/testar/oracles/generic/visual/GenericVisualBalanceMetricOracle.java b/testar/src/org/testar/oracles/generic/visual/GenericVisualBalanceMetricOracle.java index 174ba1f96..b0ea6419e 100644 --- a/testar/src/org/testar/oracles/generic/visual/GenericVisualBalanceMetricOracle.java +++ b/testar/src/org/testar/oracles/generic/visual/GenericVisualBalanceMetricOracle.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2023 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2023 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2023 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2023 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,8 @@ package org.testar.oracles.generic.visual; import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import org.testar.monkey.alayer.Rect; import org.testar.monkey.alayer.Shape; @@ -67,19 +69,19 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { + public List getVerdicts(State state) { if (state.childCount() == 0) { - return Verdict.OK; // State has no children, no need for balance metric evaluation + return Collections.singletonList(Verdict.OK); // State has no children, no need for balance metric evaluation } Shape sutShape = state.child(0).get(Tags.Shape, null); if (sutShape == null) { - return Verdict.OK; // SUT has no shape, no need for balance metric evaluation + return Collections.singletonList(Verdict.OK); // SUT has no shape, no need for balance metric evaluation } Rect sutRect = (Rect) sutShape; if (sutRect.width() <= 0 || sutRect.height() <= 0) { - return Verdict.OK; // Invalid shape dimensions, skip evaluation + return Collections.singletonList(Verdict.OK); // Invalid shape dimensions, skip evaluation } ArrayList regions = MetricsHelper.getRegions(state); @@ -89,10 +91,10 @@ public Verdict getVerdict(State state) { if (balanceMetric < thresholdValue) { String verdictMsg = String.format("Balance metric with value %f is below treshold value %f!", balanceMetric, thresholdValue); Visualizer visualizer = new RegionsVisualizer(getRedPen(), regions, "Balance Metric Warning", 0.5, 0.5); - return new Verdict(Verdict.Severity.WARNING_UI_VISUAL_OR_RENDERING_FAULT, verdictMsg, visualizer); + return Collections.singletonList(new Verdict(Verdict.Severity.WARNING_UI_VISUAL_OR_RENDERING_FAULT, verdictMsg, visualizer)); } - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } } diff --git a/testar/src/org/testar/oracles/generic/visual/GenericVisualCenterAlignmentMetricOracle.java b/testar/src/org/testar/oracles/generic/visual/GenericVisualCenterAlignmentMetricOracle.java index 4a40298be..be59c401a 100644 --- a/testar/src/org/testar/oracles/generic/visual/GenericVisualCenterAlignmentMetricOracle.java +++ b/testar/src/org/testar/oracles/generic/visual/GenericVisualCenterAlignmentMetricOracle.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2023 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2023 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2023 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2023 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,8 @@ package org.testar.oracles.generic.visual; import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import org.testar.monkey.alayer.Shape; import org.testar.monkey.alayer.State; @@ -65,7 +67,7 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { + public List getVerdicts(State state) { ArrayList regions = MetricsHelper.getRegions(state); double centerAlignmentMetric = MetricsHelper.calculateCenterAlignment(regions); @@ -73,10 +75,10 @@ public Verdict getVerdict(State state) { if (centerAlignmentMetric < thresholdValue) { String verdictMsg = String.format("Center alignment metric with value %f is below threshold value %f!", centerAlignmentMetric, thresholdValue); Visualizer visualizer = new RegionsVisualizer(getRedPen(), regions, "Center Alignment Metric Warning", 0.5, 0.5); - return new Verdict(Verdict.Severity.WARNING_UI_VISUAL_OR_RENDERING_FAULT, verdictMsg, visualizer); + return Collections.singletonList(new Verdict(Verdict.Severity.WARNING_UI_VISUAL_OR_RENDERING_FAULT, verdictMsg, visualizer)); } - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } } diff --git a/testar/src/org/testar/oracles/generic/visual/GenericVisualConcentricityMetricOracle.java b/testar/src/org/testar/oracles/generic/visual/GenericVisualConcentricityMetricOracle.java index 8e28a7c3d..6ea6f22d5 100644 --- a/testar/src/org/testar/oracles/generic/visual/GenericVisualConcentricityMetricOracle.java +++ b/testar/src/org/testar/oracles/generic/visual/GenericVisualConcentricityMetricOracle.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2023 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2023 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2023 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2023 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,8 @@ package org.testar.oracles.generic.visual; import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import org.testar.monkey.alayer.Rect; import org.testar.monkey.alayer.Shape; @@ -67,19 +69,19 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { + public List getVerdicts(State state) { if (state.childCount() == 0) { - return Verdict.OK; // State has no children, no need for balance metric evaluation + return Collections.singletonList(Verdict.OK); // State has no children, no need for balance metric evaluation } Shape sutShape = state.child(0).get(Tags.Shape, null); if (sutShape == null) { - return Verdict.OK; // SUT has no shape, no need for balance metric evaluation + return Collections.singletonList(Verdict.OK); // SUT has no shape, no need for balance metric evaluation } Rect sutRect = (Rect) sutShape; if (sutRect.width() <= 0 || sutRect.height() <= 0) { - return Verdict.OK; // Invalid shape dimensions, skip evaluation + return Collections.singletonList(Verdict.OK); // Invalid shape dimensions, skip evaluation } ArrayList regions = MetricsHelper.getRegions(state); @@ -89,10 +91,10 @@ public Verdict getVerdict(State state) { if (concentricityMetric < thresholdValue) { String verdictMsg = String.format("Concentricity metric with value %f is below threshold value %f!", concentricityMetric, thresholdValue); Visualizer visualizer = new RegionsVisualizer(getRedPen(), regions, "Concentricity Metric Warning", 0.5, 0.5); - return new Verdict(Verdict.Severity.WARNING_UI_VISUAL_OR_RENDERING_FAULT, verdictMsg, visualizer); + return Collections.singletonList(new Verdict(Verdict.Severity.WARNING_UI_VISUAL_OR_RENDERING_FAULT, verdictMsg, visualizer)); } - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } } diff --git a/testar/src/org/testar/oracles/generic/visual/GenericVisualDensityMetricOracle.java b/testar/src/org/testar/oracles/generic/visual/GenericVisualDensityMetricOracle.java index 0f35e9c41..ba57c3053 100644 --- a/testar/src/org/testar/oracles/generic/visual/GenericVisualDensityMetricOracle.java +++ b/testar/src/org/testar/oracles/generic/visual/GenericVisualDensityMetricOracle.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2023 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2023 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2023 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2023 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,8 @@ package org.testar.oracles.generic.visual; import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import org.testar.monkey.alayer.Rect; import org.testar.monkey.alayer.Shape; @@ -69,43 +71,46 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { + public List getVerdicts(State state) { if (state.childCount() == 0) { - return Verdict.OK; // State has no children, no need for balance metric evaluation + return Collections.singletonList(Verdict.OK); // State has no children, no need for balance metric evaluation } Shape sutShape = state.child(0).get(Tags.Shape, null); if (sutShape == null) { - return Verdict.OK; // SUT has no shape, no need for balance metric evaluation + return Collections.singletonList(Verdict.OK); // SUT has no shape, no need for balance metric evaluation } Rect sutRect = (Rect) sutShape; if (sutRect.width() <= 0 || sutRect.height() <= 0) { - return Verdict.OK; // Invalid shape dimensions, skip evaluation + return Collections.singletonList(Verdict.OK); // Invalid shape dimensions, skip evaluation } ArrayList regions = MetricsHelper.getRegions(state); double densityMetric = MetricsHelper.calculateDensity(regions, sutRect.width(), sutRect.height()); - Verdict widgetDensityVerdict = Verdict.OK; + List verdicts = new ArrayList<>(); if (densityMetric < thresholdMinValue) { String verdictMsg = String.format("Density metric with value %f is below threshold minimum value %f! Design too simple.", densityMetric, thresholdMinValue); Visualizer visualizer = new RegionsVisualizer(getRedPen(), regions, "Density Warning - Too Simple", 0.5, 0.5); Verdict verdict = new Verdict(Verdict.Severity.WARNING_UI_VISUAL_OR_RENDERING_FAULT, verdictMsg, visualizer); - widgetDensityVerdict = widgetDensityVerdict.join(verdict); + verdicts.add(verdict); } if (densityMetric > thresholdMaxValue) { String verdictMsg = String.format("Density metric with value %f is higher than threshold maximum value %f! Design too complex.", densityMetric, thresholdMaxValue); Visualizer visualizer = new RegionsVisualizer(getRedPen(), regions, "Density Warning - Too Complex", 0.5, 0.5); Verdict verdict = new Verdict(Verdict.Severity.WARNING_UI_VISUAL_OR_RENDERING_FAULT, verdictMsg, visualizer); - widgetDensityVerdict = widgetDensityVerdict.join(verdict); + verdicts.add(verdict); } - return widgetDensityVerdict; + if (verdicts.isEmpty()) { + return Collections.singletonList(Verdict.OK); + } + return verdicts; } } diff --git a/testar/src/org/testar/oracles/generic/visual/GenericVisualSimplicityMetricOracle.java b/testar/src/org/testar/oracles/generic/visual/GenericVisualSimplicityMetricOracle.java index 659b1f6ba..2a5ee1183 100644 --- a/testar/src/org/testar/oracles/generic/visual/GenericVisualSimplicityMetricOracle.java +++ b/testar/src/org/testar/oracles/generic/visual/GenericVisualSimplicityMetricOracle.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2023 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2023 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2023 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2023 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,8 @@ package org.testar.oracles.generic.visual; import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import org.testar.monkey.alayer.Rect; import org.testar.monkey.alayer.Shape; @@ -67,19 +69,19 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { + public List getVerdicts(State state) { if (state.childCount() == 0) { - return Verdict.OK; // State has no children, no need for balance metric evaluation + return Collections.singletonList(Verdict.OK); // State has no children, no need for balance metric evaluation } Shape sutShape = state.child(0).get(Tags.Shape, null); if (sutShape == null) { - return Verdict.OK; // SUT has no shape, no need for balance metric evaluation + return Collections.singletonList(Verdict.OK); // SUT has no shape, no need for balance metric evaluation } Rect sutRect = (Rect) sutShape; if (sutRect.width() <= 0 || sutRect.height() <= 0) { - return Verdict.OK; // Invalid shape dimensions, skip evaluation + return Collections.singletonList(Verdict.OK); // Invalid shape dimensions, skip evaluation } ArrayList regions = MetricsHelper.getRegions(state); @@ -89,10 +91,10 @@ public Verdict getVerdict(State state) { if (simplicityMetric < thresholdValue) { String verdictMsg = String.format("Simplicity metric with value %f is below threshold value %f!", simplicityMetric, thresholdValue); Visualizer visualizer = new RegionsVisualizer(getRedPen(), regions, "Simplicity Warning", 0.5, 0.5); - return new Verdict(Verdict.Severity.WARNING_UI_VISUAL_OR_RENDERING_FAULT, verdictMsg, visualizer); + return Collections.singletonList(new Verdict(Verdict.Severity.WARNING_UI_VISUAL_OR_RENDERING_FAULT, verdictMsg, visualizer)); } - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } } diff --git a/testar/src/org/testar/oracles/llm/LlmOracle.java b/testar/src/org/testar/oracles/llm/LlmOracle.java index bf3bc4817..d3a01b395 100644 --- a/testar/src/org/testar/oracles/llm/LlmOracle.java +++ b/testar/src/org/testar/oracles/llm/LlmOracle.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2024 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2024 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2024 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2024 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -40,6 +40,8 @@ import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.Base64; +import java.util.Collections; +import java.util.List; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; @@ -127,11 +129,11 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { + public List getVerdicts(State state) { // If the stateless option is enabled, initialize a new prompt to reduce tokens usage if(this.stateless) initialize(); - return getVerdictWithLlm(state); + return Collections.singletonList(getVerdictWithLlm(state)); } private Verdict getVerdictWithLlm(State state) { diff --git a/testar/src/org/testar/oracles/log/AndroidLogcatOracle.java b/testar/src/org/testar/oracles/log/AndroidLogcatOracle.java index 42650b4da..2f0586b43 100644 --- a/testar/src/org/testar/oracles/log/AndroidLogcatOracle.java +++ b/testar/src/org/testar/oracles/log/AndroidLogcatOracle.java @@ -46,6 +46,7 @@ import java.nio.file.Paths; import java.nio.file.StandardOpenOption; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Set; import java.util.TreeSet; @@ -104,9 +105,9 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { + public List getVerdicts(State state) { if (settings.get(ConfigTags.Mode) != RuntimeControlsProtocol.Modes.Generate) { - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } String packageName = AndroidAppiumFramework.getAppPackageFromCapabilitiesOrCurrent(); @@ -114,7 +115,7 @@ public Verdict getVerdict(State state) { if (dump == null || dump.isBlank()) { AndroidAppiumFramework.clearLogcat(); - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } List relevantLines = dump == null ? List.of() : List.of(dump.split("\\r?\\n")); @@ -133,7 +134,7 @@ public Verdict getVerdict(State state) { List matches = detectRegexMatches(newLines, regex); if (matches.isEmpty()) { AndroidAppiumFramework.clearLogcat(); - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } Set uniqueSorted = new TreeSet<>(matches); @@ -142,7 +143,7 @@ public Verdict getVerdict(State state) { info.append(String.join(" | ", uniqueSorted)); AndroidAppiumFramework.clearLogcat(); - return new Verdict(Verdict.Severity.SUSPICIOUS_LOG, info.toString().trim()); + return Collections.singletonList(new Verdict(Verdict.Severity.SUSPICIOUS_LOG, info.toString().trim())); } private void appendToSequenceLog(List lines) { diff --git a/testar/src/org/testar/oracles/log/LogOracle.java b/testar/src/org/testar/oracles/log/LogOracle.java index a59a0ca68..088c71e0b 100644 --- a/testar/src/org/testar/oracles/log/LogOracle.java +++ b/testar/src/org/testar/oracles/log/LogOracle.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2022 - 2023 Open Universiteit - www.ou.nl - * Copyright (c) 2022 - 2023 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2022 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2022 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,7 @@ package org.testar.oracles.log; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.testar.monkey.ConfigTags; @@ -63,13 +64,13 @@ public void initialize() { checker.initialRead(); } - public Verdict getVerdict(State state) { + public List getVerdicts(State state) { errorsList.addAll(checker.readAndCheck()); if ( errorsList.size() == 0 ) { - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } else { - return new Verdict(Verdict.Severity.SUSPICIOUS_LOG, String.join(";", errorsList)); + return Collections.singletonList(new Verdict(Verdict.Severity.SUSPICIOUS_LOG, String.join(";", errorsList))); } } diff --git a/testar/src/org/testar/oracles/log/ProcessListenerOracle.java b/testar/src/org/testar/oracles/log/ProcessListenerOracle.java index 8588d87ec..5fd36c54c 100644 --- a/testar/src/org/testar/oracles/log/ProcessListenerOracle.java +++ b/testar/src/org/testar/oracles/log/ProcessListenerOracle.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2018 - 2025 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2018 - 2025 Open Universiteit - www.ou.nl + * Copyright (c) 2018 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -36,6 +36,8 @@ import java.io.IOException; import java.io.InputStreamReader; import java.io.PrintWriter; +import java.util.Collections; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -109,9 +111,9 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { + public List getVerdicts(State state) { if (!processListenerEnabled) { - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } try { @@ -120,17 +122,17 @@ public Verdict getVerdict(State state) { Verdict standardOutputVerdict = checkStream(standardOutputReader, "_StdOut.log", "StdOut"); if (standardErrorVerdict.severity() != Verdict.Severity.OK.getValue()) - return standardErrorVerdict; + return Collections.singletonList(standardErrorVerdict); if (standardOutputVerdict.severity() != Verdict.Severity.OK.getValue()) - return standardOutputVerdict; + return Collections.singletonList(standardOutputVerdict); } catch (IOException e) { System.err.println("Unable to read the SUT process buffer"); - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } private Verdict checkStream(BufferedReader reader, String logSuffix, String streamLabel) throws IOException { diff --git a/testar/src/org/testar/oracles/log/WebBrowserConsoleOracle.java b/testar/src/org/testar/oracles/log/WebBrowserConsoleOracle.java new file mode 100644 index 000000000..0fff5750d --- /dev/null +++ b/testar/src/org/testar/oracles/log/WebBrowserConsoleOracle.java @@ -0,0 +1,117 @@ +/*************************************************************************************************** + * + * Copyright (c) 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2026 Open Universiteit - www.ou.nl + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + *******************************************************************************************************/ + +package org.testar.oracles.log; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.logging.Level; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.openqa.selenium.logging.LogEntries; +import org.openqa.selenium.logging.LogEntry; +import org.testar.monkey.ConfigTags; +import org.testar.monkey.alayer.State; +import org.testar.monkey.alayer.Verdict; +import org.testar.monkey.alayer.webdriver.WdDriver; +import org.testar.oracles.Oracle; +import org.testar.settings.Settings; + +/** + * Web browser console oracle (WebDriver). + * Scans browser logs for error/warning lines that match regular expression patterns. + */ +public class WebBrowserConsoleOracle implements Oracle { + + private final boolean errorEnabled; + private final boolean warningEnabled; + private final String errorPatternText; + private final String warningPatternText; + + public WebBrowserConsoleOracle(Settings settings) { + this.errorEnabled = settings.get(ConfigTags.WebConsoleErrorOracle, false); + this.warningEnabled = settings.get(ConfigTags.WebConsoleWarningOracle, false); + this.errorPatternText = settings.get(ConfigTags.WebConsoleErrorPattern, ""); + this.warningPatternText = settings.get(ConfigTags.WebConsoleWarningPattern, ""); + } + + @Override + public void initialize() { + // Nothing extra to initialize + } + + @Override + public List getVerdicts(State state) { + List verdicts = new ArrayList<>(); + + LogEntries logEntries = WdDriver.getBrowserLogs(); + + // If Web Console Error Oracle is enabled and we have some pattern to match + if (errorEnabled && !errorPatternText.isEmpty()) { + // Load the web console error pattern + Pattern errorPattern = Pattern.compile(errorPatternText, Pattern.UNICODE_CHARACTER_CLASS); + // Check Severe messages in the WebDriver logs + for (LogEntry logEntry : logEntries) { + if (logEntry.getLevel().equals(Level.SEVERE)) { + // Check if the severe error message matches with the web console error pattern + String consoleErrorMsg = logEntry.getMessage(); + Matcher matcherError = errorPattern.matcher(consoleErrorMsg); + if (matcherError.matches()) { + verdicts.add(new Verdict(Verdict.Severity.SUSPICIOUS_LOG, "Web Browser Console Error: " + consoleErrorMsg)); + } + } + } + } + + // If Web Console Warning Oracle is enabled and we have some pattern to match + if (warningEnabled && !warningPatternText.isEmpty()) { + // Load the web console warning pattern + Pattern warningPattern = Pattern.compile(warningPatternText, Pattern.UNICODE_CHARACTER_CLASS); + // Check Warning messages in the WebDriver logs + for (LogEntry logEntry : logEntries) { + if (logEntry.getLevel().equals(Level.WARNING)) { + // Check if the warning message matches with the web console error pattern + String consoleWarningMsg = logEntry.getMessage(); + Matcher matcherWarning = warningPattern.matcher(consoleWarningMsg); + if (matcherWarning.matches()) { + verdicts.add(new Verdict(Verdict.Severity.SUSPICIOUS_LOG, "Web Browser Console Warning: " + consoleWarningMsg)); + } + } + } + } + + if (verdicts.isEmpty()) { + return Collections.singletonList(Verdict.OK); + } + return verdicts; + } +} diff --git a/testar/src/org/testar/oracles/web/accessibility/WebAccessibilityClickableSizeOracle.java b/testar/src/org/testar/oracles/web/accessibility/WebAccessibilityClickableSizeOracle.java index 8843739c8..cb4d5d397 100644 --- a/testar/src/org/testar/oracles/web/accessibility/WebAccessibilityClickableSizeOracle.java +++ b/testar/src/org/testar/oracles/web/accessibility/WebAccessibilityClickableSizeOracle.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2025 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2025 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,6 +32,8 @@ package org.testar.oracles.web.accessibility; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import org.testar.monkey.alayer.Rect; @@ -71,8 +73,8 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { - List incorrectWidgets = new ArrayList<>(); + public List getVerdicts(State state) { + List verdicts = new ArrayList<>(); // Iterate over all widgets in the state for (Widget widget : state) { @@ -87,30 +89,27 @@ public Verdict getVerdict(State state) { // Check if the widget is smaller than the recommended pixels if (width < minClickableThreshold || height < minClickableThreshold) { - // If so, save it as incorrect widget - incorrectWidgets.add(widget); + String verdictMsg = String.format( + "Clickable web widget %s is too small (%sx%s px). Minimum: %s px.", + getDescriptionOfWidgets(Collections.singletonList(widget), WdTags.WebOuterHTML), + width.intValue(), + height.intValue(), + minClickableThreshold + ); + Visualizer visualizer = new RegionsVisualizer( + getRedPen(), + getWidgetRegions(Arrays.asList(widget)), + "Accessibility Fault", + 0.5, 0.5); + verdicts.add(new Verdict(Verdict.Severity.WARNING_ACCESSIBILITY_FAULT, verdictMsg, visualizer)); } } } - // If exists one or more incorrect widgets - if(!incorrectWidgets.isEmpty()) { - // Create and return a WARNING_ACCESSIBILITY_FAULT verdict - String verdictMsg = String.format( - "Clickable web widgets %s are too small (Minimum: %s px).", - getDescriptionOfWidgets(incorrectWidgets, WdTags.WebOuterHTML), - minClickableThreshold - ); - Visualizer visualizer = new RegionsVisualizer( - getRedPen(), - getWidgetRegions(incorrectWidgets), - "Accessibility Fault", - 0.5, 0.5); - return new Verdict(Verdict.Severity.WARNING_ACCESSIBILITY_FAULT, verdictMsg, visualizer); - } else { - return Verdict.OK; + if(!verdicts.isEmpty()) { + return verdicts; } - + return Collections.singletonList(Verdict.OK); } } diff --git a/testar/src/org/testar/oracles/web/accessibility/WebAccessibilityFontSizeOracle.java b/testar/src/org/testar/oracles/web/accessibility/WebAccessibilityFontSizeOracle.java index 4fa20b07f..7093e4ab8 100644 --- a/testar/src/org/testar/oracles/web/accessibility/WebAccessibilityFontSizeOracle.java +++ b/testar/src/org/testar/oracles/web/accessibility/WebAccessibilityFontSizeOracle.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2025 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2025 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,7 @@ package org.testar.oracles.web.accessibility; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.testar.monkey.alayer.State; @@ -65,8 +66,8 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { - List incorrectWidgets = new ArrayList<>(); + public List getVerdicts(State state) { + List verdicts = new ArrayList<>(); // Iterate over all widgets in the state for (Widget widget : state) { @@ -84,30 +85,26 @@ public Verdict getVerdict(State state) { // Check if the font size is below the minimum threshold if (fontSize > 0 && fontSize < minFontSizeThreshold) { - // If so, save it as incorrect widget - incorrectWidgets.add(widget); + String verdictMsg = String.format( + "Widget text %s is too small (%d px). Minimum recommended is %d px.", + getDescriptionOfWidgets(Collections.singletonList(widget), WdTags.WebTextContent), + fontSize, + minFontSizeThreshold + ); + Visualizer visualizer = new RegionsVisualizer( + getRedPen(), + getWidgetRegions(Collections.singletonList(widget)), + "Accessibility Fault", + 0.5, 0.5); + verdicts.add(new Verdict(Verdict.Severity.WARNING_ACCESSIBILITY_FAULT, verdictMsg, visualizer)); } } } - // If exists one or more incorrect widgets - if(!incorrectWidgets.isEmpty()) { - // Create and return a WARNING_ACCESSIBILITY_FAULT verdict - String verdictMsg = String.format( - "These widgets Text %s are too small. Minimum recommended is %s px.", - getDescriptionOfWidgets(incorrectWidgets, WdTags.WebTextContent), - minFontSizeThreshold - ); - Visualizer visualizer = new RegionsVisualizer( - getRedPen(), - getWidgetRegions(incorrectWidgets), - "Accessibility Fault", - 0.5, 0.5); - return new Verdict(Verdict.Severity.WARNING_ACCESSIBILITY_FAULT, verdictMsg, visualizer); - } else { - return Verdict.OK; + if(!verdicts.isEmpty()) { + return verdicts; } - + return Collections.singletonList(Verdict.OK); } /** diff --git a/testar/src/org/testar/oracles/web/accessibility/WebAccessibilityImagesAltOracle.java b/testar/src/org/testar/oracles/web/accessibility/WebAccessibilityImagesAltOracle.java index 587bdb59c..0f42e3a4c 100644 --- a/testar/src/org/testar/oracles/web/accessibility/WebAccessibilityImagesAltOracle.java +++ b/testar/src/org/testar/oracles/web/accessibility/WebAccessibilityImagesAltOracle.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2025 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2025 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,7 @@ package org.testar.oracles.web.accessibility; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.testar.monkey.alayer.Roles; @@ -58,36 +59,31 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { - List incorrectWidgets = new ArrayList<>(); + public List getVerdicts(State state) { + List verdicts = new ArrayList<>(); // Check if some widget of the state for(Widget widget : state) { // Is a widget image () and if it lacks alternative text if(widget.get(Tags.Role, Roles.Widget).equals(WdRoles.WdIMG) && (widget.get(WdTags.WebAlt, null) == null || widget.get(WdTags.WebAlt, "").isBlank())) { - // If so, save it as incorrect widget - incorrectWidgets.add(widget); + String verdictMsg = String.format( + "Detected web image widget %s without alternative text!", + getDescriptionOfWidgets(Collections.singletonList(widget), WdTags.WebOuterHTML) + ); + Visualizer visualizer = new RegionsVisualizer( + getRedPen(), + getWidgetRegions(Collections.singletonList(widget)), + "Accessibility Fault", + 0.5, 0.5); + verdicts.add(new Verdict(Verdict.Severity.WARNING_ACCESSIBILITY_FAULT, verdictMsg, visualizer)); } } - // If exists one or more incorrect widgets - if(!incorrectWidgets.isEmpty()) { - // Create and return a WARNING_ACCESSIBILITY_FAULT verdict - String verdictMsg = String.format( - "Detected web image widgets '%s' without alternative text!", - getDescriptionOfWidgets(incorrectWidgets, WdTags.WebOuterHTML) - ); - Visualizer visualizer = new RegionsVisualizer( - getRedPen(), - getWidgetRegions(incorrectWidgets), - "Accessibility Fault", - 0.5, 0.5); - return new Verdict(Verdict.Severity.WARNING_ACCESSIBILITY_FAULT, verdictMsg, visualizer); - } else { - return Verdict.OK; + if(!verdicts.isEmpty()) { + return verdicts; } - + return Collections.singletonList(Verdict.OK); } } diff --git a/testar/src/org/testar/oracles/web/invariants/WebInvariantDuplicateMenuItems.java b/testar/src/org/testar/oracles/web/invariants/WebInvariantDuplicateMenuItems.java index 5b3a5ca73..d3b34e6a8 100644 --- a/testar/src/org/testar/oracles/web/invariants/WebInvariantDuplicateMenuItems.java +++ b/testar/src/org/testar/oracles/web/invariants/WebInvariantDuplicateMenuItems.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2025 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2025 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,7 @@ package org.testar.oracles.web.invariants; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -55,8 +56,8 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { - List menuWidgetsWithDuplicates = new ArrayList<>(); + public List getVerdicts(State state) { + List verdicts = new ArrayList<>(); for (Widget w : state) { // Check for UL elements with at least two children @@ -78,29 +79,27 @@ public Verdict getVerdict(State state) { // If duplicates are found, prepare the verdict message if (duplicatesTexts.size() > 0) { - menuWidgetsWithDuplicates.add(w); + String verdictMsg = String.format( + "Detected a Unnumbered List (UL) web menu %s with duplicate option elements: %s", + getDescriptionOfWidgets(Collections.singletonList(w), WdTags.WebId), + duplicatesTexts + ); + + Visualizer visualizer = new RegionsVisualizer( + getRedPen(), + getWidgetRegions(Collections.singletonList(w)), + "Invariant Fault", + 0.5, 0.5); + + verdicts.add(new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer)); } } } - // If exists one or more incorrect widgets - if (!menuWidgetsWithDuplicates.isEmpty()) { - - String verdictMsg = String.format( - "Detected a Unnumbered List (UL) web menu %s with duplicate option elements!", - getDescriptionOfWidgets(menuWidgetsWithDuplicates, WdTags.WebId) - ); - - Visualizer visualizer = new RegionsVisualizer( - getRedPen(), - getWidgetRegions(menuWidgetsWithDuplicates), - "Invariant Fault", - 0.5, 0.5); - - return new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer); + if (!verdicts.isEmpty()) { + return verdicts; } - - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } // Helper method to find duplicates in a list diff --git a/testar/src/org/testar/oracles/web/invariants/WebInvariantDuplicateSelectItems.java b/testar/src/org/testar/oracles/web/invariants/WebInvariantDuplicateSelectItems.java index ae59ec056..3836fa8a4 100644 --- a/testar/src/org/testar/oracles/web/invariants/WebInvariantDuplicateSelectItems.java +++ b/testar/src/org/testar/oracles/web/invariants/WebInvariantDuplicateSelectItems.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2025 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2025 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -67,8 +67,8 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { - List selectWidgetsWithDuplicates = new ArrayList<>(); + public List getVerdicts(State state) { + List verdicts = new ArrayList<>(); for (Widget w : state) { if (roles.contains(w.get(Tags.Role, Roles.Widget)) && !w.get(WdTags.WebId, "").isEmpty()) { @@ -89,7 +89,19 @@ public Verdict getVerdict(State state) { .collect(Collectors.toSet()); if (!duplicatesTexts.isEmpty()) { - selectWidgetsWithDuplicates.add(w); + String verdictMsg = String.format( + "Detected Select widget %s with duplicate values: %s", + getDescriptionOfWidgets(Collections.singletonList(w), WdTags.WebId), + duplicatesTexts + ); + + Visualizer visualizer = new RegionsVisualizer( + getRedPen(), + getWidgetRegions(Collections.singletonList(w)), + "Invariant Fault", + 0.5, 0.5); + + verdicts.add(new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer)); } } } catch (Exception e) { @@ -98,23 +110,9 @@ public Verdict getVerdict(State state) { } } - // If exists one or more incorrect widgets - if (!selectWidgetsWithDuplicates.isEmpty()) { - - String verdictMsg = String.format( - "Detected Select widgets %s with duplicate values!", - getDescriptionOfWidgets(selectWidgetsWithDuplicates, WdTags.WebId) - ); - - Visualizer visualizer = new RegionsVisualizer( - getRedPen(), - getWidgetRegions(selectWidgetsWithDuplicates), - "Invariant Fault", - 0.5, 0.5); - - return new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer); + if (!verdicts.isEmpty()) { + return verdicts; } - - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } } diff --git a/testar/src/org/testar/oracles/web/invariants/WebInvariantDuplicatedRowsInTable.java b/testar/src/org/testar/oracles/web/invariants/WebInvariantDuplicatedRowsInTable.java index ffe5c89c1..e4b2d58d0 100644 --- a/testar/src/org/testar/oracles/web/invariants/WebInvariantDuplicatedRowsInTable.java +++ b/testar/src/org/testar/oracles/web/invariants/WebInvariantDuplicatedRowsInTable.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2025 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2025 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,7 @@ package org.testar.oracles.web.invariants; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -59,50 +60,51 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { - List incorrectWidgets = new ArrayList<>(); - List incorrectWidgetDescriptions = new ArrayList<>(); + public List getVerdicts(State state) { + List verdicts = new ArrayList<>(); for (Widget w : state) { if(w.get(Tags.Role, Roles.Widget).equals(WdRoles.WdTABLE)) { List> rowElementsDescription = new ArrayList<>(); extractAllRowDescriptionsFromTable(w, rowElementsDescription); - List> duplicatedDescriptions = + List>> duplicatedDescriptions = rowElementsDescription.stream() .collect(Collectors.groupingBy(Pair::right)) .entrySet().stream() .filter(e -> e.getValue().size() > 1) - .flatMap(e -> e.getValue().stream()) + .map(e -> e.getValue()) .collect(Collectors.toList()); // If the list of duplicated descriptions contains a matching prepare the verdict if(!duplicatedDescriptions.isEmpty()) { - for (Pair duplicatedWidget : duplicatedDescriptions) { + for (List> duplicatedWidgets : duplicatedDescriptions) { + Pair duplicatedWidget = duplicatedWidgets.get(0); // Ignore empty rows if (!duplicatedWidget.right().replaceAll("_","").isEmpty()) { - incorrectWidgets.add(duplicatedWidget.left()); - incorrectWidgetDescriptions.add(duplicatedWidget.right()); + String verdictMsg = String.format( + "Detected duplicated row in a Table for the widget: %s ", + duplicatedWidget.right() + ); + List widgets = duplicatedWidgets.stream() + .map(Pair::left) + .collect(Collectors.toList()); + Visualizer visualizer = new RegionsVisualizer( + getRedPen(), + getWidgetRegions(widgets), + "Invariant Fault", + 0.5, 0.5); + verdicts.add(new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer)); } } - - String verdictMsg = String.format( - "Detected a duplicated rows in a Table for the widgets: %s ", - incorrectWidgetDescriptions - ); - - Visualizer visualizer = new RegionsVisualizer( - getRedPen(), - getWidgetRegions(incorrectWidgets), - "Invariant Fault", - 0.5, 0.5); - - return new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer); } } } - return Verdict.OK; + if (!verdicts.isEmpty()) { + return verdicts; + } + return Collections.singletonList(Verdict.OK); } private void extractAllRowDescriptionsFromTable(Widget w, List> rowElementsDescription) { diff --git a/testar/src/org/testar/oracles/web/invariants/WebInvariantEmptySelectItems.java b/testar/src/org/testar/oracles/web/invariants/WebInvariantEmptySelectItems.java index 219e6357a..7f8dd1c95 100644 --- a/testar/src/org/testar/oracles/web/invariants/WebInvariantEmptySelectItems.java +++ b/testar/src/org/testar/oracles/web/invariants/WebInvariantEmptySelectItems.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2025 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2025 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,7 @@ package org.testar.oracles.web.invariants; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.testar.monkey.alayer.Role; @@ -64,8 +65,8 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { - List emptySelectWidgets = new ArrayList<>(); + public List getVerdicts(State state) { + List verdicts = new ArrayList<>(); for (Widget w : state) { if (roles.contains(w.get(Tags.Role, Roles.Widget)) && !w.get(WdTags.WebId, "").isEmpty()) { @@ -76,7 +77,19 @@ public Verdict getVerdict(State state) { // If the select contains 0 or 1 item if (selectItemsLength != null && selectItemsLength.intValue() <= 1) { - emptySelectWidgets.add(w); + String verdictMsg = String.format( + "Detected Select widget %s with empty or only one item (count: %d)!", + getDescriptionOfWidgets(Collections.singletonList(w), WdTags.WebId), + selectItemsLength.intValue() + ); + + Visualizer visualizer = new RegionsVisualizer( + getRedPen(), + getWidgetRegions(Collections.singletonList(w)), + "Invariant Fault", + 0.5, 0.5); + + verdicts.add(new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer)); } } catch (Exception e) { // Ignore webdriver execute script errors @@ -84,23 +97,9 @@ public Verdict getVerdict(State state) { } } - // If exists one or more incorrect widgets - if (!emptySelectWidgets.isEmpty()) { - - String verdictMsg = String.format( - "Detected Select widgets %s with empty or only one item!", - getDescriptionOfWidgets(emptySelectWidgets, WdTags.WebId) - ); - - Visualizer visualizer = new RegionsVisualizer( - getRedPen(), - getWidgetRegions(emptySelectWidgets), - "Invariant Fault", - 0.5, 0.5); - - return new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer); + if (!verdicts.isEmpty()) { + return verdicts; } - - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } } diff --git a/testar/src/org/testar/oracles/web/invariants/WebInvariantManySelectItems.java b/testar/src/org/testar/oracles/web/invariants/WebInvariantManySelectItems.java index d0e938628..5dbe61bf3 100644 --- a/testar/src/org/testar/oracles/web/invariants/WebInvariantManySelectItems.java +++ b/testar/src/org/testar/oracles/web/invariants/WebInvariantManySelectItems.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2025 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2025 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,7 @@ package org.testar.oracles.web.invariants; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.testar.monkey.alayer.Roles; @@ -63,8 +64,8 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { - List selectWidgetsWithManyItems = new ArrayList<>(); + public List getVerdicts(State state) { + List verdicts = new ArrayList<>(); for (Widget w : state) { if (w.get(Tags.Role, Roles.Widget).equals(WdRoles.WdSELECT) && !w.get(WdTags.WebId, "").isEmpty()) { @@ -75,7 +76,20 @@ public Verdict getVerdict(State state) { // Check if the items of the select widget are more than the thresholdValue if (selectItemsLength != null && selectItemsLength.intValue() > thresholdValue) { - selectWidgetsWithManyItems.add(w); + String verdictMsg = String.format( + "Detected Select widget %s which has %d items (threshold: %d)", + getDescriptionOfWidgets(Collections.singletonList(w), WdTags.WebId), + selectItemsLength.intValue(), + thresholdValue + ); + + Visualizer visualizer = new RegionsVisualizer( + getRedPen(), + getWidgetRegions(Collections.singletonList(w)), + "Invariant Fault", + 0.5, 0.5); + + verdicts.add(new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer)); } } catch (Exception e) { @@ -84,23 +98,9 @@ public Verdict getVerdict(State state) { } } - if (!selectWidgetsWithManyItems.isEmpty()) { - - String verdictMsg = String.format( - "Detected Select widgets %s which have more items than the threshold value of %d", - getDescriptionOfWidgets(selectWidgetsWithManyItems, WdTags.WebId), - thresholdValue - ); - - Visualizer visualizer = new RegionsVisualizer( - getRedPen(), - getWidgetRegions(selectWidgetsWithManyItems), - "Invariant Fault", - 0.5, 0.5); - - return new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer); + if (!verdicts.isEmpty()) { + return verdicts; } - - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } } diff --git a/testar/src/org/testar/oracles/web/invariants/WebInvariantNumberWithLotOfDecimals.java b/testar/src/org/testar/oracles/web/invariants/WebInvariantNumberWithLotOfDecimals.java index 26a76cb4b..820aa38d5 100644 --- a/testar/src/org/testar/oracles/web/invariants/WebInvariantNumberWithLotOfDecimals.java +++ b/testar/src/org/testar/oracles/web/invariants/WebInvariantNumberWithLotOfDecimals.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2025 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2025 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,7 @@ package org.testar.oracles.web.invariants; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.testar.monkey.alayer.State; @@ -59,8 +60,8 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { - List incorrectDecimalWidgets = new ArrayList<>(); + public List getVerdicts(State state) { + List verdicts = new ArrayList<>(); for (Widget w : state) { // If the widget contains a web text that is a numeric value @@ -72,31 +73,29 @@ public Verdict getVerdict(State state) { int decimalPlaces = number.length() - number.indexOf('.') - 1; if (decimalPlaces > maxDecimals) { - incorrectDecimalWidgets.add(w); + String verdictMsg = String.format( + "Detected widget %s with %d decimals (max: %d)!", + getDescriptionOfWidgets(Collections.singletonList(w), WdTags.WebTextContent), + decimalPlaces, + maxDecimals + ); + + Visualizer visualizer = new RegionsVisualizer( + getRedPen(), + getWidgetRegions(Collections.singletonList(w)), + "Invariant Fault", + 0.5, 0.5); + + verdicts.add(new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer)); } } } } - // If exists one or more incorrect widgets - if (!incorrectDecimalWidgets.isEmpty()) { - - String verdictMsg = String.format( - "Detected widgets %s with more than %d decimals!", - getDescriptionOfWidgets(incorrectDecimalWidgets, WdTags.WebTextContent), - maxDecimals - ); - - Visualizer visualizer = new RegionsVisualizer( - getRedPen(), - getWidgetRegions(incorrectDecimalWidgets), - "Invariant Fault", - 0.5, 0.5); - - return new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer); + if (!verdicts.isEmpty()) { + return verdicts; } - - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } private boolean isNumeric(String strNum) { diff --git a/testar/src/org/testar/oracles/web/invariants/WebInvariantTextAreaWithoutLength.java b/testar/src/org/testar/oracles/web/invariants/WebInvariantTextAreaWithoutLength.java index 493736868..69de672bd 100644 --- a/testar/src/org/testar/oracles/web/invariants/WebInvariantTextAreaWithoutLength.java +++ b/testar/src/org/testar/oracles/web/invariants/WebInvariantTextAreaWithoutLength.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2025 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2025 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,6 +31,7 @@ package org.testar.oracles.web.invariants; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.testar.monkey.alayer.Role; @@ -63,32 +64,28 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { - List textAreasWithoutLength = new ArrayList<>(); + public List getVerdicts(State state) { + List verdicts = new ArrayList<>(); for (Widget w : state) { if (roles.contains(w.get(Tags.Role, Roles.Widget)) && w.get(WdTags.WebMaxLength, -1) == 0) { - textAreasWithoutLength.add(w); + String verdictMsg = String.format( + "Detected TextArea widget %s with 0 max length!", + getDescriptionOfWidgets(Collections.singletonList(w), WdTags.WebOuterHTML) + ); + Visualizer visualizer = new RegionsVisualizer( + getRedPen(), + getWidgetRegions(Collections.singletonList(w)), + "Invariant Fault", + 0.5, 0.5); + + verdicts.add(new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer)); } } - // If exists one or more incorrect widgets - if (!textAreasWithoutLength.isEmpty()) { - - String verdictMsg = String.format( - "Detected TextArea widgets %s with 0 max length!", - getDescriptionOfWidgets(textAreasWithoutLength, WdTags.WebOuterHTML) - ); - - Visualizer visualizer = new RegionsVisualizer( - getRedPen(), - getWidgetRegions(textAreasWithoutLength), - "Invariant Fault", - 0.5, 0.5); - - return new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer); + if (!verdicts.isEmpty()) { + return verdicts; } - - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } } diff --git a/testar/src/org/testar/oracles/web/invariants/WebInvariantUnsortedSelectItems.java b/testar/src/org/testar/oracles/web/invariants/WebInvariantUnsortedSelectItems.java index 526ce8698..acb24e1ff 100644 --- a/testar/src/org/testar/oracles/web/invariants/WebInvariantUnsortedSelectItems.java +++ b/testar/src/org/testar/oracles/web/invariants/WebInvariantUnsortedSelectItems.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2025 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2025 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -71,8 +71,8 @@ public void initialize() { } @Override - public Verdict getVerdict(State state) { - List unsortedSelectWidgets = new ArrayList<>(); + public List getVerdicts(State state) { + List verdicts = new ArrayList<>(); for (Widget w : state) { if (roles.contains(w.get(Tags.Role, Roles.Widget)) && !w.get(WdTags.WebId, "").isEmpty()) { @@ -89,7 +89,17 @@ public Verdict getVerdict(State state) { // Check if the options are sorted if (selectOptionsList != null && !isSorted(selectOptionsList)) { - unsortedSelectWidgets.add(w); + String verdictMsg = String.format( + "Detected Select widget %s with unsorted elements!", + getDescriptionOfWidgets(Collections.singletonList(w), WdTags.WebId) + ); + Visualizer visualizer = new RegionsVisualizer( + getRedPen(), + getWidgetRegions(Collections.singletonList(w)), + "Invariant Fault", + 0.5, 0.5); + + verdicts.add(new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer)); } } catch (Exception e) { // Ignore webdriver execute script errors @@ -97,23 +107,10 @@ public Verdict getVerdict(State state) { } } - if (!unsortedSelectWidgets.isEmpty()) { - - String verdictMsg = String.format( - "Detected Select widgets %s with unsorted elements!", - getDescriptionOfWidgets(unsortedSelectWidgets, WdTags.WebId) - ); - - Visualizer visualizer = new RegionsVisualizer( - getRedPen(), - getWidgetRegions(unsortedSelectWidgets), - "Invariant Fault", - 0.5, 0.5); - - return new Verdict(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT, verdictMsg, visualizer); + if (!verdicts.isEmpty()) { + return verdicts; } - - return Verdict.OK; + return Collections.singletonList(Verdict.OK); } // Helper method to check if the list of strings is sorted diff --git a/testar/src/org/testar/protocols/WebdriverProtocol.java b/testar/src/org/testar/protocols/WebdriverProtocol.java index 49934f1d2..113f3f822 100644 --- a/testar/src/org/testar/protocols/WebdriverProtocol.java +++ b/testar/src/org/testar/protocols/WebdriverProtocol.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2019 - 2025 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2019 - 2025 Open Universiteit - www.ou.nl + * Copyright (c) 2019 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2019 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -35,8 +35,6 @@ import org.apache.commons.lang3.ArrayUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.openqa.selenium.logging.LogEntries; -import org.openqa.selenium.logging.LogEntry; import org.testar.environment.Environment; import org.testar.monkey.*; import org.testar.monkey.alayer.*; @@ -53,6 +51,7 @@ import org.testar.monkey.alayer.windows.WinApiException; import org.testar.monkey.alayer.windows.WinProcess; import org.testar.monkey.alayer.windows.Windows; +import org.testar.oracles.log.WebBrowserConsoleOracle; import org.testar.plugin.NativeLinker; import org.testar.serialisation.LogSerialiser; import org.testar.settings.Settings; @@ -61,8 +60,6 @@ import java.net.MalformedURLException; import java.net.URL; import java.util.*; -import java.util.logging.Level; -import java.util.regex.Matcher; import java.util.regex.Pattern; import static org.testar.monkey.alayer.Tags.Blocked; import static org.testar.monkey.alayer.Tags.Enabled; @@ -88,9 +85,8 @@ public class WebdriverProtocol extends GenericUtilsProtocol { // List of attributes to identify and close policy popups protected Multimap policyAttributes = ArrayListMultimap.create(); - - // Verdict obtained from messages coming from the web browser console - protected Verdict webConsoleVerdict = Verdict.OK; + // Oracle to obtain verdicts from messages coming from the web browser console + protected WebBrowserConsoleOracle webBrowserConsoleOracle; /** * Called once during the life time of TESTAR @@ -139,6 +135,8 @@ protected void initialize(Settings settings){ // List of web attributes that TESTAR should ignore when obtaining the web state Constants.setIgnoredAttributes(settings.get(ConfigTags.WebIgnoredAttributes)); + + webBrowserConsoleOracle = new WebBrowserConsoleOracle(settings); } /** @@ -147,8 +145,6 @@ protected void initialize(Settings settings){ @Override protected void preSequencePreparations() { super.preSequencePreparations(); - // reset web browser console verdict - webConsoleVerdict = Verdict.OK; } /** @@ -282,8 +278,8 @@ protected State getState(SUT system) throws StateBuildException { WinProcess.toForeground(system.get(Tags.PID), 0.3, 100); } catch (WinApiException wae) { logger.log(org.apache.logging.log4j.Level.WARN, wae); - Verdict verdict = new Verdict(Verdict.Severity.NOT_RESPONDING, "Unable to bring the browser to foreground!"); - state.set(Tags.OracleVerdict, verdict); + Verdict notRespondingVerdict = new Verdict(Verdict.Severity.NOT_RESPONDING, "Unable to bring the browser to foreground!"); + state.set(Tags.OracleVerdicts, Collections.singletonList(notRespondingVerdict)); } } @@ -291,56 +287,22 @@ protected State getState(SUT system) throws StateBuildException { } /** - * The getVerdict methods implements the online state oracles that - * examine the SUT's current state and returns an oracle verdict. + * The getVerdicts methods implements the online state oracles that + * examine the SUT's current state and returns a list of oracle verdicts. * - * @return oracle verdict, which determines whether the state is erroneous and why. + * @return list of oracle verdicts */ @Override - protected Verdict getVerdict(State state) { - Verdict stateVerdict = super.getVerdict(state); - - LogEntries logEntries = WdDriver.getBrowserLogs(); - - // If Web Console Error Oracle is enabled and we have some pattern to match - if(settings.get(ConfigTags.WebConsoleErrorOracle, false) && !settings.get(ConfigTags.WebConsoleErrorPattern, "").isEmpty()) { - // Load the web console error pattern - Pattern errorPattern = Pattern.compile(settings.get(ConfigTags.WebConsoleErrorPattern), Pattern.UNICODE_CHARACTER_CLASS); - // Check Severe messages in the WebDriver logs - for(LogEntry logEntry : logEntries) { - if(logEntry.getLevel().equals(Level.SEVERE)) { - // Check if the severe error message matches with the web console error pattern - String consoleErrorMsg = logEntry.getMessage(); - Matcher matcherError = errorPattern.matcher(consoleErrorMsg); - if(matcherError.matches()) { - webConsoleVerdict = new Verdict(Verdict.Severity.SUSPICIOUS_TAG, "Web Browser Console Error: " + consoleErrorMsg); - } - } - } - // Join GUI verdict with WebDriver console verdict - stateVerdict = stateVerdict.join(webConsoleVerdict); - } - - // If Web Console Warning Oracle is enabled and we have some pattern to match - if(settings.get(ConfigTags.WebConsoleWarningOracle, false) && !settings.get(ConfigTags.WebConsoleWarningPattern, "").isEmpty()) { - // Load the web console warning pattern - Pattern warningPattern = Pattern.compile(settings.get(ConfigTags.WebConsoleWarningPattern), Pattern.UNICODE_CHARACTER_CLASS); - // Check Warning messages in the WebDriver logs - for(LogEntry logEntry : logEntries) { - if(logEntry.getLevel().equals(Level.WARNING)) { - // Check if the warning message matches with the web console error pattern - String consoleWarningMsg = logEntry.getMessage(); - Matcher matcherWarning = warningPattern.matcher(consoleWarningMsg); - if(matcherWarning.matches()) { - webConsoleVerdict = new Verdict(Verdict.Severity.SUSPICIOUS_TAG, "Web Browser Console Warning: " + consoleWarningMsg); - } - } - } - // Join GUI verdict with WebDriver console verdict - stateVerdict = stateVerdict.join(webConsoleVerdict); - } + protected List getVerdicts(State state) { + List verdicts = super.getVerdicts(state); + List browserConsoleVerdicts = webBrowserConsoleOracle.getVerdicts(state); + for (Verdict browserVerdict : browserConsoleVerdicts) { + if (browserVerdict.severity() > Verdict.OK.severity()) { + verdicts.add(browserVerdict); + } + } - return stateVerdict; + return verdicts; } /** diff --git a/testar/src/org/testar/reporting/DummyReportManager.java b/testar/src/org/testar/reporting/DummyReportManager.java index c82150089..eba382ae8 100644 --- a/testar/src/org/testar/reporting/DummyReportManager.java +++ b/testar/src/org/testar/reporting/DummyReportManager.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2025 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2025 Open Universiteit - www.ou.nl + * Copyright (c) 2025 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2025 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -30,6 +30,7 @@ package org.testar.reporting; +import java.util.List; import java.util.Set; import org.testar.monkey.alayer.Action; @@ -59,7 +60,7 @@ public void addSelectedAction(State state, Action action) { } @Override - public void addTestVerdict(Verdict verdict) { + public void addTestVerdicts(List verdicts) { } diff --git a/testar/src/org/testar/reporting/HtmlReporter.java b/testar/src/org/testar/reporting/HtmlReporter.java index 5639797d2..54ff6c473 100644 --- a/testar/src/org/testar/reporting/HtmlReporter.java +++ b/testar/src/org/testar/reporting/HtmlReporter.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2018 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2018 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2018 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -39,9 +39,11 @@ import org.testar.monkey.alayer.Verdict; import org.testar.monkey.alayer.webdriver.enums.WdTags; +import java.io.File; import java.time.Instant; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.List; import java.util.Set; import java.util.StringJoiner; @@ -49,6 +51,8 @@ public class HtmlReporter implements Reporting { private HtmlFormatUtil htmlReportUtil; private int innerLoopCounter = 0; + private boolean deleteBaseReport = false; + private String baseReportPath; private final String openStateBlockContainer = "
"; private final String openDerivedBlockContainer = "
"; @@ -252,25 +256,53 @@ public void addSelectedAction(State state, Action action) } @Override - public void addTestVerdict(Verdict verdict) + public void addTestVerdicts(List verdicts) { + String baseFilePath = htmlReportUtil.getFile().getAbsolutePath(); + deleteBaseReport = true; + baseReportPath = baseFilePath; + int index = 1; + for (Verdict verdict : verdicts) { + String suffixName = String.format("_V%03d_%s", index++, verdict.verdictSeverityTitle()); + String verdictFilePath = appendSuffixToFile(baseFilePath, suffixName); + + htmlReportUtil.duplicateFile(verdictFilePath); + + HtmlFormatUtil verdictUtil = new HtmlFormatUtil(verdictFilePath); + addVerdictBlock(verdictUtil, verdict); + verdictUtil.addContent("
"); + verdictUtil.addFooter(); + verdictUtil.writeToFile(); + } + } + + private void addVerdictBlock(HtmlFormatUtil util, Verdict verdict) { String verdictInfo = StringEscapeUtils.escapeHtml(verdict.info()); if(verdict.severity() > Verdict.OK.severity()) verdictInfo = verdictInfo.replace(Verdict.OK.info(), "").replace("\n", ""); - htmlReportUtil.addContent(openVerdictBlockContainer); // Open verdict block container - htmlReportUtil.addHeading(2, "Test verdict for this sequence: " + verdictInfo); - htmlReportUtil.addHeading(4, "Severity: " + verdict.severity()); - htmlReportUtil.addContent(""); - htmlReportUtil.addContent(closeContainer); // Close verdict block container + util.addContent(openVerdictBlockContainer); // Open verdict block container + util.addHeading(2, "Test verdict for this sequence: " + verdictInfo); + util.addHeading(4, "Severity: " + verdict.severity()); + util.addContent(""); + util.addContent(closeContainer); // Close verdict block container + } - htmlReportUtil.appendToFileName("_" + verdict.verdictSeverityTitle()); - htmlReportUtil.writeToFile(); + private String appendSuffixToFile(String filePath, String suffixName) { + int dotIndex = filePath.lastIndexOf('.'); + if (dotIndex == -1) { + return filePath + suffixName; + } + return filePath.substring(0, dotIndex) + suffixName + filePath.substring(dotIndex); } @Override public void finishReport() { + if (deleteBaseReport && baseReportPath != null) { + new File(baseReportPath).delete(); + return; + } htmlReportUtil.addContent("
"); // Close the main div container htmlReportUtil.addFooter(); diff --git a/testar/src/org/testar/reporting/PlainTextReporter.java b/testar/src/org/testar/reporting/PlainTextReporter.java index 5aca0f4d8..e887ca4e2 100644 --- a/testar/src/org/testar/reporting/PlainTextReporter.java +++ b/testar/src/org/testar/reporting/PlainTextReporter.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2024 Open Universiteit - www.ou.nl - * Copyright (c) 2024 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2024 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2024 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -39,6 +39,7 @@ import org.testar.monkey.alayer.Verdict; import java.util.ArrayList; +import java.util.List; import java.util.Set; import java.util.StringJoiner; @@ -46,6 +47,8 @@ public class PlainTextReporter implements Reporting { private PlainTextFormatUtil plainTextReportUtil; private int innerLoopCounter = 0; + private boolean deleteBaseReport = false; + private String baseReportPath; public PlainTextReporter(String fileName, boolean replay) //replay or generate mode { plainTextReportUtil = new PlainTextFormatUtil(fileName); @@ -193,25 +196,51 @@ public void addSelectedAction(State state, Action action) plainTextReportUtil.writeToFile(); } - + @Override - public void addTestVerdict(Verdict verdict) + public void addTestVerdicts(List verdicts) { + String baseFilePath = plainTextReportUtil.getFile().getAbsolutePath(); + deleteBaseReport = true; + baseReportPath = baseFilePath; + int index = 1; + for (Verdict verdict : verdicts) { + String suffixName = String.format("_V%03d_%s", index++, verdict.verdictSeverityTitle()); + String verdictFilePath = appendSuffixToFile(baseFilePath, suffixName); + + plainTextReportUtil.duplicateFile(verdictFilePath); + + PlainTextFormatUtil verdictUtil = new PlainTextFormatUtil(verdictFilePath); + addVerdictBlock(verdictUtil, verdict); + verdictUtil.writeToFile(); + } + } + + private void addVerdictBlock(PlainTextFormatUtil util, Verdict verdict) { String verdictInfo = verdict.info(); if(verdict.severity() > Verdict.OK.severity()) verdictInfo = verdictInfo.replace(Verdict.OK.info(), ""); - - plainTextReportUtil.addHorizontalLine(); - plainTextReportUtil.addHeading(3, "Test verdict for this sequence: " + verdictInfo); - plainTextReportUtil.addHeading(5, "Severity: " + verdict.severity()); - - plainTextReportUtil.appendToFileName("_" + verdict.verdictSeverityTitle()); - plainTextReportUtil.writeToFile(); + + util.addHorizontalLine(); + util.addHeading(3, "Test verdict for this sequence: " + verdictInfo); + util.addHeading(5, "Severity: " + verdict.severity()); + } + + private String appendSuffixToFile(String filePath, String suffixName) { + int dotIndex = filePath.lastIndexOf('.'); + if (dotIndex == -1) { + return filePath + suffixName; + } + return filePath.substring(0, dotIndex) + suffixName + filePath.substring(dotIndex); } @Override public void finishReport() { + if (deleteBaseReport && baseReportPath != null) { + new java.io.File(baseReportPath).delete(); + return; + } plainTextReportUtil.writeToFile(); } } diff --git a/testar/src/org/testar/reporting/ReportManager.java b/testar/src/org/testar/reporting/ReportManager.java index 9ca25097c..73357381b 100644 --- a/testar/src/org/testar/reporting/ReportManager.java +++ b/testar/src/org/testar/reporting/ReportManager.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2023 Open Universiteit - www.ou.nl - * Copyright (c) 2023 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2023 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2023 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -41,6 +41,8 @@ import java.io.File; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.Set; public class ReportManager implements Reporting @@ -94,7 +96,8 @@ public void addState(State state) { if(firstStateAdded) { - if(firstActionsAdded || (state.get(Tags.OracleVerdict, Verdict.OK).severity() > Verdict.Severity.OK.getValue())) + List verdicts = state.get(Tags.OracleVerdicts, Collections.singletonList(Verdict.OK)); + if(firstActionsAdded || !Verdict.helperAreAllVerdictsOK(verdicts)) { //if the first state contains a failure, write the same state in case it was a login for(Reporting reporter : reporters) reporter.addState(state); @@ -137,11 +140,11 @@ public void addSelectedAction(State state, Action action) for(Reporting reporter : reporters) reporter.addSelectedAction(state, action); } - - public void addTestVerdict(Verdict verdict) + + public void addTestVerdicts(List verdicts) { if(reportingEnabled) for(Reporting reporter : reporters) - reporter.addTestVerdict(verdict); + reporter.addTestVerdicts(verdicts); } } diff --git a/testar/src/org/testar/reporting/Reporting.java b/testar/src/org/testar/reporting/Reporting.java index bccbbb85f..96a14439b 100644 --- a/testar/src/org/testar/reporting/Reporting.java +++ b/testar/src/org/testar/reporting/Reporting.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2021 - 2023 Open Universiteit - www.ou.nl - * Copyright (c) 2021 - 2023 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2021 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2021 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -34,6 +34,7 @@ import org.testar.monkey.alayer.State; import org.testar.monkey.alayer.Verdict; +import java.util.List; import java.util.Set; public interface Reporting @@ -42,6 +43,6 @@ public interface Reporting public void addActions(Set actions); public void addActionsAndUnvisitedActions(Set actions, Set concreteIdsOfUnvisitedActions); public void addSelectedAction(State state, Action action); - public void addTestVerdict(Verdict verdict); + public void addTestVerdicts(List verdicts); public void finishReport(); } diff --git a/testar/src/org/testar/securityanalysis/helpers/SecurityOracleOrchestrator.java b/testar/src/org/testar/securityanalysis/helpers/SecurityOracleOrchestrator.java index 187d0bf65..57128a7d8 100644 --- a/testar/src/org/testar/securityanalysis/helpers/SecurityOracleOrchestrator.java +++ b/testar/src/org/testar/securityanalysis/helpers/SecurityOracleOrchestrator.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2022 Open Universiteit - www.ou.nl - * Copyright (c) 2022 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2022 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2022 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -38,6 +38,7 @@ import org.testar.securityanalysis.SecurityResultWriter; import org.testar.securityanalysis.oracles.*; +import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -118,16 +119,21 @@ private void addListeners(DevTools devTools) securityOracle.addListener(devTools); } - public Verdict getVerdict(Verdict verdict) + public List getVerdicts() { + List verdicts = new ArrayList<>(); for (BaseSecurityOracle securityOracle : securityOracles) { Verdict newVerdict = securityOracle.getVerdict(); - if (newVerdict != null) - verdict.join(newVerdict); + if (newVerdict != null && newVerdict.severity() > Verdict.Severity.OK.getValue()) { + verdicts.add(newVerdict); + } } - if (activeSecurityOracle != null) - verdict.join(activeSecurityOracle.getVerdict()); - - return verdict; + if (activeSecurityOracle != null) { + Verdict activeVerdict = activeSecurityOracle.getVerdict(); + if (activeVerdict != null && activeVerdict.severity() > Verdict.Severity.OK.getValue()) { + verdicts.add(activeVerdict); + } + } + return verdicts; } } diff --git a/testar/src/org/testar/settings/SettingsDefaults.java b/testar/src/org/testar/settings/SettingsDefaults.java index 4f7ca817b..eac5ab1d4 100644 --- a/testar/src/org/testar/settings/SettingsDefaults.java +++ b/testar/src/org/testar/settings/SettingsDefaults.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2023 - 2025 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2023 - 2025 Open Universiteit - www.ou.nl + * Copyright (c) 2023 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2023 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -49,7 +49,6 @@ private SettingsDefaults() {} defaults.add(Pair.from(ProcessesToKillDuringTest, "(?!x)x")); defaults.add(Pair.from(ShowVisualSettingsDialogOnStartup, true)); - defaults.add(Pair.from(FaultThreshold, 0.1)); defaults.add(Pair.from(LogLevel, 1)); defaults.add(Pair.from(Mode, RuntimeControlsProtocol.Modes.Spy)); defaults.add(Pair.from(OutputDir, Main.outputDir)); @@ -69,6 +68,7 @@ private SettingsDefaults() {} defaults.add(Pair.from(SUTConnectorValue, "")); defaults.add(Pair.from(Delete, new ArrayList())); defaults.add(Pair.from(CopyFromTo, new ArrayList>())); + defaults.add(Pair.from(IgnoreDuplicatedVerdicts, false)); defaults.add(Pair.from(SuspiciousTags, "(?!x)x")); defaults.add(Pair.from(ClickFilter, "(?!x)x")); defaults.add(Pair.from(MyClassPath, Arrays.asList(Main.settingsDir))); diff --git a/testar/src/org/testar/settings/SettingsFileStructure.java b/testar/src/org/testar/settings/SettingsFileStructure.java index 8097a4e84..e0f2e12db 100644 --- a/testar/src/org/testar/settings/SettingsFileStructure.java +++ b/testar/src/org/testar/settings/SettingsFileStructure.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2023 - 2025 Universitat Politecnica de Valencia - www.upv.es - * Copyright (c) 2023 - 2025 Open Universiteit - www.ou.nl + * Copyright (c) 2023 - 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2023 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -132,6 +132,12 @@ public static String getTestSettingsStructure() { , ConfigTags.ProcessesToKillDuringTest.name() + " = " , "" , "#################################################################" + , "# Ignore reporting duplicated verdicts for this protocol" + , "#################################################################" + , "" + , ConfigTags.IgnoreDuplicatedVerdicts.name() + " = " + , "" + , "#################################################################" , "# Oracles based on suspicious tag values" , "#" , "# Regular expression and Tags to apply them" diff --git a/testar/src/org/testar/settings/dialog/GeneralPanel.java b/testar/src/org/testar/settings/dialog/GeneralPanel.java index df85183f3..188ecec13 100644 --- a/testar/src/org/testar/settings/dialog/GeneralPanel.java +++ b/testar/src/org/testar/settings/dialog/GeneralPanel.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * -* Copyright (c) 2013 - 2024 Universitat Politecnica de Valencia - www.upv.es -* Copyright (c) 2018 - 2024 Open Universiteit - www.ou.nl +* Copyright (c) 2013 - 2026 Universitat Politecnica de Valencia - www.upv.es +* Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -56,7 +56,7 @@ public class GeneralPanel extends SettingsPanel implements Observer { private JSpinner spnSequenceLength; //private JCheckBox checkStopOnFault; private JComboBox comboBoxProtocol; - private JCheckBox compileCheckBox, checkActionVisualization; + private JCheckBox compileCheckBox, checkActionVisualization, checkIgnoreDuplicatedVerdict; private JLabel labelAppName = new JLabel("Application name"); private JLabel labelAppVersion = new JLabel("Application version"); @@ -134,7 +134,12 @@ private void addGeneralControlsGlobal(SettingsDialog settingsDialog) { checkActionVisualization.setBounds(10, 240, 192, 21); //checkActionVisualization.setToolTipText(checkStopOnFaultTTT); add(checkActionVisualization); - + + checkIgnoreDuplicatedVerdict = new JCheckBox("Ignore duplicated verdicts"); + checkIgnoreDuplicatedVerdict.setBounds(10, 262, 192, 21); + checkIgnoreDuplicatedVerdict.setToolTipText(ConfigTags.IgnoreDuplicatedVerdicts.getDescription()); + add(checkIgnoreDuplicatedVerdict); + labelAppName.setBounds(330, 242, 150, 27); labelAppName.setToolTipText(ToolTipTexts.applicationNameTTT); add(labelAppName); @@ -254,6 +259,7 @@ public void populateFrom(final Settings settings) { cboxSUTconnector.setSelectedItem(settings.get(ConfigTags.SUTConnector)); //checkStopOnFault.setSelected(settings.get(ConfigTags.StopGenerationOnFault)); checkActionVisualization.setSelected(settings.get(ConfigTags.VisualizeActions)); + checkIgnoreDuplicatedVerdict.setSelected(settings.get(ConfigTags.IgnoreDuplicatedVerdicts)); txtSutPath.setInitialText(settings.get(ConfigTags.SUTConnectorValue)); comboBoxProtocol.setSelectedItem(settings.get(ConfigTags.ProtocolClass).split("/")[0]); spnNumSequences.setValue(settings.get(ConfigTags.Sequences)); @@ -275,6 +281,7 @@ public void extractInformation(final Settings settings) { settings.set(ConfigTags.SUTConnectorValue, txtSutPath.getText()); //settings.set(ConfigTags.StopGenerationOnFault, checkStopOnFault.isSelected()); settings.set(ConfigTags.VisualizeActions, checkActionVisualization.isSelected()); + settings.set(ConfigTags.IgnoreDuplicatedVerdicts, checkIgnoreDuplicatedVerdict.isSelected()); settings.set(ConfigTags.Sequences, (Integer) spnNumSequences.getValue()); settings.set(ConfigTags.SequenceLength, (Integer) spnSequenceLength.getValue()); settings.set(ConfigTags.AlwaysCompile, compileCheckBox.isSelected()); diff --git a/testar/test/org/testar/TestFileHandling.java b/testar/test/org/testar/TestFileHandling.java index 116bfffdc..d1e54166b 100644 --- a/testar/test/org/testar/TestFileHandling.java +++ b/testar/test/org/testar/TestFileHandling.java @@ -28,31 +28,41 @@ public void setUp() throws IOException { @Test public void testCopyClassifiedSequence_okSeverity() throws IOException { - FileHandling.copyClassifiedSequence("okSequence", testFile, Verdict.OK); + String suffixName = "_V001_OK"; + FileHandling.copyClassifiedSequence("okSequence", testFile, Verdict.OK, suffixName); File targetDir = new File(outputDir, "sequences_ok"); assertTrue("OK sequence folder should be created", targetDir.exists()); - assertTrue("Copied file should exist in sequences_ok", new File(targetDir, testFile.getName()).exists()); + assertTrue("Copied file should exist in sequences_ok", new File(targetDir, appendSuffix(testFile.getName(), suffixName)).exists()); } @Test public void testCopyClassifiedSequence_failSeverity() throws IOException { - FileHandling.copyClassifiedSequence("failSequence", testFile, Verdict.FAIL); + String suffixName = "_V001_FAIL"; + FileHandling.copyClassifiedSequence("failSequence", testFile, Verdict.FAIL, suffixName); File targetDir = new File(outputDir, "sequences_fail"); assertTrue("Fail sequence folder should be created", targetDir.exists()); - assertTrue("Copied file should exist in sequences_fail", new File(targetDir, testFile.getName()).exists()); + assertTrue("Copied file should exist in sequences_fail", new File(targetDir, appendSuffix(testFile.getName(), suffixName)).exists()); } @Test public void testCopyClassifiedSequence_suspiciousLogSeverity() throws IOException { Verdict verdict = new Verdict(Verdict.Severity.SUSPICIOUS_LOG, "Suspicious log entry found"); + String suffixName = "_V001_SUSPICIOUS_LOG"; - FileHandling.copyClassifiedSequence("logSequence", testFile, verdict); + FileHandling.copyClassifiedSequence("logSequence", testFile, verdict, suffixName); File targetDir = new File(outputDir, "sequences_suspicious_log"); assertTrue("Suspicious log folder should be created", targetDir.exists()); - assertTrue("Copied file should exist in sequences_suspicious_log", new File(targetDir, testFile.getName()).exists()); + assertTrue("Copied file should exist in sequences_suspicious_log", new File(targetDir, appendSuffix(testFile.getName(), suffixName)).exists()); + } + + private String appendSuffix(String fileName, String suffix) { + int dotIndex = fileName.lastIndexOf('.'); + return (dotIndex == -1) + ? fileName + suffix + : fileName.substring(0, dotIndex) + suffix + fileName.substring(dotIndex); } } diff --git a/testar/test/org/testar/monkey/TestMoreActionsLogic.java b/testar/test/org/testar/monkey/TestMoreActionsLogic.java index 7d0f412a0..6b5302247 100644 --- a/testar/test/org/testar/monkey/TestMoreActionsLogic.java +++ b/testar/test/org/testar/monkey/TestMoreActionsLogic.java @@ -1,6 +1,7 @@ package org.testar.monkey; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Properties; @@ -27,7 +28,7 @@ public void test_continue_more_actions() { protocol.startTime = Util.time(); StateStub state = new StateStub(); - state.set(Tags.OracleVerdict, new Verdict(Verdict.Severity.OK, "State is OK")); + state.set(Tags.OracleVerdicts, Collections.singletonList(new Verdict(Verdict.Severity.OK, "State is OK"))); state.set(Tags.IsRunning, true); state.set(Tags.NotResponding, false); @@ -50,7 +51,7 @@ public void test_stop_max_time_reached() { protocol.startTime = Util.time() - 70.0; StateStub state = new StateStub(); - state.set(Tags.OracleVerdict, new Verdict(Verdict.Severity.OK, "State is OK")); + state.set(Tags.OracleVerdicts, Collections.singletonList(new Verdict(Verdict.Severity.OK, "State is OK"))); state.set(Tags.IsRunning, true); state.set(Tags.NotResponding, false); @@ -73,7 +74,7 @@ public void test_stop_more_actions_due_verdict() { protocol.startTime = Util.time(); StateStub state = new StateStub(); - state.set(Tags.OracleVerdict, new Verdict(Verdict.Severity.SUSPICIOUS_TAG, "State contains suspicious message")); + state.set(Tags.OracleVerdicts, Collections.singletonList(new Verdict(Verdict.Severity.SUSPICIOUS_TAG, "State contains suspicious message"))); state.set(Tags.IsRunning, true); state.set(Tags.NotResponding, false); @@ -96,7 +97,7 @@ public void test_more_actions_do_not_stop_fault() { protocol.startTime = Util.time(); StateStub state = new StateStub(); - state.set(Tags.OracleVerdict, new Verdict(Verdict.Severity.SUSPICIOUS_TAG, "State contains suspicious message")); + state.set(Tags.OracleVerdicts, Collections.singletonList(new Verdict(Verdict.Severity.SUSPICIOUS_TAG, "State contains suspicious message"))); state.set(Tags.IsRunning, true); state.set(Tags.NotResponding, false); @@ -119,7 +120,7 @@ public void test_stop_not_running() { protocol.startTime = Util.time(); StateStub state = new StateStub(); - state.set(Tags.OracleVerdict, new Verdict(Verdict.Severity.OK, "State is OK")); + state.set(Tags.OracleVerdicts, Collections.singletonList(new Verdict(Verdict.Severity.OK, "State is OK"))); state.set(Tags.IsRunning, false); state.set(Tags.NotResponding, false); @@ -142,7 +143,7 @@ public void test_stop_not_responding() { protocol.startTime = Util.time(); StateStub state = new StateStub(); - state.set(Tags.OracleVerdict, new Verdict(Verdict.Severity.OK, "State is OK")); + state.set(Tags.OracleVerdicts, Collections.singletonList(new Verdict(Verdict.Severity.OK, "State is OK"))); state.set(Tags.IsRunning, true); state.set(Tags.NotResponding, true); diff --git a/testar/test/org/testar/monkey/VerdictProcessingTest.java b/testar/test/org/testar/monkey/VerdictProcessingTest.java new file mode 100644 index 000000000..cafb24211 --- /dev/null +++ b/testar/test/org/testar/monkey/VerdictProcessingTest.java @@ -0,0 +1,83 @@ +package org.testar.monkey; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.junit.After; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.testar.monkey.alayer.Verdict; +import org.testar.settings.Settings; + +public class VerdictProcessingTest { + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + @After + public void tearDown() { + Settings.setSettingsPath(null); + } + + @Test + public void testNullListReturnsOk() { + Settings settings = new Settings(); + settings.set(ConfigTags.IgnoreDuplicatedVerdicts, false); + VerdictProcessing processing = new VerdictProcessing(settings); + + List filtered = processing.filterDuplicates(null); + assertEquals(1, filtered.size()); + assertEquals(Verdict.OK.severity(), filtered.get(0).severity(), 0.0); + } + + @Test + public void testMultipleOkReturnsSingleOk() { + Settings settings = new Settings(); + settings.set(ConfigTags.IgnoreDuplicatedVerdicts, false); + VerdictProcessing processing = new VerdictProcessing(settings); + + List filtered = processing.filterDuplicates(Arrays.asList(Verdict.OK, Verdict.OK)); + assertEquals(1, filtered.size()); + assertEquals(Verdict.OK.severity(), filtered.get(0).severity(), 0.0); + } + + @Test + public void testOkAndFailureRemovesOk() { + Settings settings = new Settings(); + settings.set(ConfigTags.IgnoreDuplicatedVerdicts, false); + VerdictProcessing processing = new VerdictProcessing(settings); + + Verdict failure = new Verdict(Verdict.Severity.SUSPICIOUS_TAG, "Issue"); + Verdict logVerdict = new Verdict(Verdict.Severity.SUSPICIOUS_LOG, "Log exception"); + List filtered = processing.filterDuplicates(Arrays.asList(Verdict.OK, failure, logVerdict)); + assertEquals(2, filtered.size()); + assertTrue(filtered.get(0).severity() > Verdict.OK.severity()); + assertTrue(filtered.get(1).severity() > Verdict.OK.severity()); + } + + @Test + public void testIgnoresKnownDuplicate() throws Exception { + File settingsDir = tempFolder.newFolder("settings"); + Settings.setSettingsPath(settingsDir.getAbsolutePath()); + + File ignoreFile = new File(settingsDir, "list_of_verdicts_with_failures.txt"); + Files.write(ignoreFile.toPath(), Collections.singletonList("duplicate message"), StandardCharsets.UTF_8); + + Settings settings = new Settings(); + settings.set(ConfigTags.IgnoreDuplicatedVerdicts, true); + VerdictProcessing processing = new VerdictProcessing(settings); + + Verdict duplicate = new Verdict(Verdict.Severity.SUSPICIOUS_TAG, "duplicate message"); + List filtered = processing.filterDuplicates(Collections.singletonList(duplicate)); + assertEquals(1, filtered.size()); + assertEquals(Verdict.OK.severity(), filtered.get(0).severity(), 0.0); + } +} diff --git a/testar/test/org/testar/oracles/log/TestAndroidLogcatOracle.java b/testar/test/org/testar/oracles/log/TestAndroidLogcatOracle.java index f66fe80ed..bd592aad4 100644 --- a/testar/test/org/testar/oracles/log/TestAndroidLogcatOracle.java +++ b/testar/test/org/testar/oracles/log/TestAndroidLogcatOracle.java @@ -79,7 +79,9 @@ public void generateModeVerdict_DetectsRegexAndReturnsSuspiciousLog() { ); androidLogcatOracle.initialize(); - Verdict verdict = androidLogcatOracle.getVerdict(state); + List verdicts = androidLogcatOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.assertEquals(Verdict.Severity.SUSPICIOUS_LOG.getValue(), verdict.severity(), 0.0); Assert.assertEquals(verdict.info(), "Suspicious Android logcat line(s) detected AndroidRuntime: FATAL EXCEPTION: main"); @@ -108,12 +110,16 @@ public void generateModeVerdict_ProcessesOnlyNewLinesAcrossCalls() { androidLogcatOracle.initialize(); - Verdict first = androidLogcatOracle.getVerdict(state); - Assert.assertEquals(Verdict.Severity.OK.getValue(), first.severity(), 0.0); + List firstInvokationVerdicts = androidLogcatOracle.getVerdicts(state); + Assert.assertEquals(1, firstInvokationVerdicts.size()); + Verdict firstVerdict = firstInvokationVerdicts.get(0); + Assert.assertEquals(Verdict.Severity.OK.getValue(), firstVerdict.severity(), 0.0); - Verdict second = androidLogcatOracle.getVerdict(state); - Assert.assertEquals(Verdict.Severity.SUSPICIOUS_LOG.getValue(), second.severity(), 0.0); - Assert.assertEquals(second.info(), "Suspicious Android logcat line(s) detected MyTag: Exception happened"); + List secondInvokationVerdicts = androidLogcatOracle.getVerdicts(state); + Assert.assertEquals(1, secondInvokationVerdicts.size()); + Verdict secondVerdict = secondInvokationVerdicts.get(0); + Assert.assertEquals(Verdict.Severity.SUSPICIOUS_LOG.getValue(), secondVerdict.severity(), 0.0); + Assert.assertEquals(secondVerdict.info(), "Suspicious Android logcat line(s) detected MyTag: Exception happened"); } } @@ -136,7 +142,9 @@ public void generateModeVerdict_DeduplicatesAndOrdersMatches() { .thenReturn(lineB + "\n" + lineA + "\n" + lineB); androidLogcatOracle.initialize(); - Verdict verdict = androidLogcatOracle.getVerdict(state); + List verdicts = androidLogcatOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.assertEquals(Verdict.Severity.SUSPICIOUS_LOG.getValue(), verdict.severity(), 0.0); String expected = "Suspicious Android logcat line(s) detected " diff --git a/testar/test/org/testar/protocols/TestBrowserConsoleVerdict.java b/testar/test/org/testar/oracles/log/TestBrowserConsoleVerdict.java similarity index 58% rename from testar/test/org/testar/protocols/TestBrowserConsoleVerdict.java rename to testar/test/org/testar/oracles/log/TestBrowserConsoleVerdict.java index 4dc1edeaf..4dae11ba9 100644 --- a/testar/test/org/testar/protocols/TestBrowserConsoleVerdict.java +++ b/testar/test/org/testar/oracles/log/TestBrowserConsoleVerdict.java @@ -1,9 +1,10 @@ -package org.testar.protocols; +package org.testar.oracles.log; import static org.mockito.Mockito.*; import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.logging.Level; import org.junit.Before; import org.junit.Test; @@ -12,15 +13,15 @@ import org.openqa.selenium.logging.LogEntry; import org.testar.monkey.Assert; import org.testar.monkey.ConfigTags; -import org.testar.monkey.alayer.Tags; import org.testar.monkey.alayer.Verdict; import org.testar.monkey.alayer.webdriver.WdDriver; import org.testar.settings.Settings; import org.testar.stub.StateStub; -public class TestBrowserConsoleVerdict extends WebdriverProtocol { +public class TestBrowserConsoleVerdict { private StateStub state; + private Settings settings; @Before public void settings_setup() { @@ -29,8 +30,6 @@ public void settings_setup() { settings.set(ConfigTags.TagsForSuspiciousOracle, Collections.singletonList("Title")); state = new StateStub(); - state.set(Tags.IsRunning, true); - state.set(Tags.NotResponding, false); } @Test @@ -40,10 +39,13 @@ public void test_console_ok() { settings.set(ConfigTags.WebConsoleWarningOracle, true); settings.set(ConfigTags.WebConsoleWarningPattern, ".*.*"); + WebBrowserConsoleOracle oracle = new WebBrowserConsoleOracle(settings); LogEntry infoEntry = new LogEntry(Level.INFO, System.currentTimeMillis(), "this is an info message"); LogEntries logEntries = new LogEntries(Collections.singletonList(infoEntry)); - Verdict verdict = runWithMockedLogs(logEntries); + List verdicts = runWithMockedLogs(oracle, logEntries); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isEquals(Verdict.OK.severity(), verdict.severity()); Assert.isEquals("No problem detected.", verdict.info()); @@ -56,12 +58,15 @@ public void test_console_error() { settings.set(ConfigTags.WebConsoleWarningOracle, true); settings.set(ConfigTags.WebConsoleWarningPattern, ".*.*"); + WebBrowserConsoleOracle oracle = new WebBrowserConsoleOracle(settings); LogEntry severeEntry = new LogEntry(Level.SEVERE, System.currentTimeMillis(), "some severe error occurred"); LogEntries logEntries = new LogEntries(Collections.singletonList(severeEntry)); - Verdict verdict = runWithMockedLogs(logEntries); + List verdicts = runWithMockedLogs(oracle, logEntries); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); - Assert.isEquals(Verdict.Severity.SUSPICIOUS_TAG.getValue(), verdict.severity()); + Assert.isEquals(Verdict.Severity.SUSPICIOUS_LOG.getValue(), verdict.severity()); Assert.isEquals("Web Browser Console Error: some severe error occurred", verdict.info()); } @@ -72,21 +77,41 @@ public void test_console_warning() { settings.set(ConfigTags.WebConsoleWarningOracle, true); settings.set(ConfigTags.WebConsoleWarningPattern, ".*.*"); + WebBrowserConsoleOracle oracle = new WebBrowserConsoleOracle(settings); LogEntry severeEntry = new LogEntry(Level.SEVERE, System.currentTimeMillis(), "some severe error occurred"); LogEntry warningEntry = new LogEntry(Level.WARNING, System.currentTimeMillis(), "this is a warning message"); LogEntries logEntries = new LogEntries(Arrays.asList(severeEntry, warningEntry)); - Verdict verdict = runWithMockedLogs(logEntries); + List verdicts = runWithMockedLogs(oracle, logEntries); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); - Assert.isEquals(Verdict.Severity.SUSPICIOUS_TAG.getValue(), verdict.severity()); + Assert.isEquals(Verdict.Severity.SUSPICIOUS_LOG.getValue(), verdict.severity()); Assert.isEquals("Web Browser Console Warning: this is a warning message", verdict.info()); } - private Verdict runWithMockedLogs(LogEntries mockedEntries) { + @Test + public void test_console_error_and_warning() { + settings.set(ConfigTags.WebConsoleErrorOracle, true); + settings.set(ConfigTags.WebConsoleErrorPattern, ".*.*"); + settings.set(ConfigTags.WebConsoleWarningOracle, true); + settings.set(ConfigTags.WebConsoleWarningPattern, ".*.*"); + + WebBrowserConsoleOracle oracle = new WebBrowserConsoleOracle(settings); + LogEntry severeEntry = new LogEntry(Level.SEVERE, System.currentTimeMillis(), "some severe error occurred"); + LogEntry warningEntry = new LogEntry(Level.WARNING, System.currentTimeMillis(), "this is a warning message"); + LogEntries logEntries = new LogEntries(Arrays.asList(severeEntry, warningEntry)); + + List verdicts = runWithMockedLogs(oracle, logEntries); + Assert.isEquals(2, verdicts.size()); + Assert.isEquals("Web Browser Console Error: some severe error occurred", verdicts.get(0).info()); + Assert.isEquals("Web Browser Console Warning: this is a warning message", verdicts.get(1).info()); + } + + private List runWithMockedLogs(WebBrowserConsoleOracle oracle, LogEntries mockedEntries) { try (MockedStatic mockedStatic = mockStatic(WdDriver.class)) { mockedStatic.when(WdDriver::getBrowserLogs).thenReturn(mockedEntries); - return getVerdict(state); + return oracle.getVerdicts(state); } } } - diff --git a/testar/test/org/testar/oracles/log/TestLogOracles.java b/testar/test/org/testar/oracles/log/TestLogOracles.java index e5759a084..28de482d0 100644 --- a/testar/test/org/testar/oracles/log/TestLogOracles.java +++ b/testar/test/org/testar/oracles/log/TestLogOracles.java @@ -69,7 +69,9 @@ public void wrongConfigurationLogOracleFile() { logOracle.initialize(); State state = Mockito.mock(State.class); - Verdict logVerdict = logOracle.getVerdict(state); + List verdicts = logOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict logVerdict = verdicts.get(0); // Verify that the logVerdict is OK Assert.assertTrue(logVerdict.severity() == Verdict.Severity.OK.getValue()); @@ -98,7 +100,9 @@ public void initialSuspiciousLogOracleErrorMessage() { logOracle.initialize(); State state = Mockito.mock(State.class); - Verdict logVerdict = logOracle.getVerdict(state); + List verdicts = logOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict logVerdict = verdicts.get(0); System.out.println("initialSuspiciousLogOracleErrorMessage logVerdict: " + logVerdict.info()); // Verify that the logVerdict is OK @@ -128,7 +132,9 @@ public void runtimeValidLogOracleFile() { Assert.assertTrue(fileContent().contains(validMessage)); State state = Mockito.mock(State.class); - Verdict logVerdict = logOracle.getVerdict(state); + List verdicts = logOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict logVerdict = verdicts.get(0); System.out.println("runtimeValidLogOracleFile logVerdict: " + logVerdict.info()); // Verify that the logVerdict is OK @@ -158,7 +164,9 @@ public void runtimeSuspiciousLogOracleErrorMessage() { Assert.assertTrue(fileContent().contains(suspiciousMessage)); State state = Mockito.mock(State.class); - Verdict logVerdict = logOracle.getVerdict(state); + List verdicts = logOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict logVerdict = verdicts.get(0); System.out.println("runtimeSuspiciousLogOracleErrorMessage logVerdict: " + logVerdict.info()); // Verify that the logVerdict detected the suspiciousMessage diff --git a/testar/test/org/testar/oracles/log/TestProcessListenerOracle.java b/testar/test/org/testar/oracles/log/TestProcessListenerOracle.java index aa6c32c64..d36da60f0 100644 --- a/testar/test/org/testar/oracles/log/TestProcessListenerOracle.java +++ b/testar/test/org/testar/oracles/log/TestProcessListenerOracle.java @@ -75,7 +75,9 @@ public void spy_mode_is_always_ok_because_is_disabled() { processOracle.initialize(); State state = Mockito.mock(State.class); - Verdict processVerdict = processOracle.getVerdict(state); + List verdicts = processOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict processVerdict = verdicts.get(0); // Verify that the processVerdict is OK because we are in spy mode Assert.assertTrue(processVerdict.severity() == Verdict.Severity.OK.getValue()); @@ -98,7 +100,9 @@ public void connect_windows_title_is_always_ok_because_is_disabled() { processOracle.initialize(); State state = Mockito.mock(State.class); - Verdict processVerdict = processOracle.getVerdict(state); + List verdicts = processOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict processVerdict = verdicts.get(0); // Verify that the processVerdict is OK because we connect with title Assert.assertTrue(processVerdict.severity() == Verdict.Severity.OK.getValue()); @@ -121,7 +125,9 @@ public void connect_process_name_is_always_ok_because_is_disabled() { processOracle.initialize(); State state = Mockito.mock(State.class); - Verdict processVerdict = processOracle.getVerdict(state); + List verdicts = processOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict processVerdict = verdicts.get(0); // Verify that the processVerdict is OK because we connect with process name Assert.assertTrue(processVerdict.severity() == Verdict.Severity.OK.getValue()); @@ -144,7 +150,9 @@ public void webdriver_is_always_ok_because_is_disabled() { processOracle.initialize(); State state = Mockito.mock(State.class); - Verdict processVerdict = processOracle.getVerdict(state); + List verdicts = processOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict processVerdict = verdicts.get(0); // Verify that the processVerdict is OK because we connect with web apps Assert.assertTrue(processVerdict.severity() == Verdict.Severity.OK.getValue()); @@ -167,7 +175,9 @@ public void process_detection_for_error_buffer() { processOracle.initialize(); State state = Mockito.mock(State.class); - Verdict processVerdict = processOracle.getVerdict(state); + List verdicts = processOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict processVerdict = verdicts.get(0); // Verify that the processVerdict detects an error in the error buffer Assert.assertTrue(processVerdict.severity() == Verdict.Severity.SUSPICIOUS_PROCESS.getValue()); @@ -191,7 +201,9 @@ public void process_detection_for_output_buffer() { processOracle.initialize(); State state = Mockito.mock(State.class); - Verdict processVerdict = processOracle.getVerdict(state); + List verdicts = processOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict processVerdict = verdicts.get(0); // Verify that the processVerdict detects an error in the output buffer Assert.assertTrue(processVerdict.severity() == Verdict.Severity.SUSPICIOUS_PROCESS.getValue()); @@ -215,7 +227,9 @@ public void process_detection_for_both_buffers() { processOracle.initialize(); State state = Mockito.mock(State.class); - Verdict processVerdict = processOracle.getVerdict(state); + List verdicts = processOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict processVerdict = verdicts.get(0); // Verify that the processVerdict detects an error Assert.assertTrue(processVerdict.severity() == Verdict.Severity.SUSPICIOUS_PROCESS.getValue()); diff --git a/testar/test/org/testar/oracles/web/accessibility/TestWebAccessibilityClickableSizeOracle.java b/testar/test/org/testar/oracles/web/accessibility/TestWebAccessibilityClickableSizeOracle.java index ab0281a71..82a4b7d22 100644 --- a/testar/test/org/testar/oracles/web/accessibility/TestWebAccessibilityClickableSizeOracle.java +++ b/testar/test/org/testar/oracles/web/accessibility/TestWebAccessibilityClickableSizeOracle.java @@ -51,9 +51,11 @@ public void test_detection_accessibility_clickable_size() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebAccessibilityClickableSizeOracle); // Assert the oracle verdict is WARNING_ACCESSIBILITY_FAULT - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_ACCESSIBILITY_FAULT.getTitle())); - Assert.isTrue(verdict.info().equals("Clickable web widgets '<a>link</a>' , are too small (Minimum: 24 px).")); + Assert.isTrue(verdict.info().equals("Clickable web widget '<a>link</a>' , is too small (0x0 px). Minimum: 24 px.")); } @Test @@ -73,7 +75,9 @@ public void test_undetection_accessibility_clickable_size() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebAccessibilityClickableSizeOracle); // Assert the oracle verdict is OK - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.OK.getTitle())); Assert.isTrue(verdict.info().equals("No problem detected.")); } diff --git a/testar/test/org/testar/oracles/web/accessibility/TestWebAccessibilityFontSizeOracle.java b/testar/test/org/testar/oracles/web/accessibility/TestWebAccessibilityFontSizeOracle.java index 06d6117cd..4a313b45a 100644 --- a/testar/test/org/testar/oracles/web/accessibility/TestWebAccessibilityFontSizeOracle.java +++ b/testar/test/org/testar/oracles/web/accessibility/TestWebAccessibilityFontSizeOracle.java @@ -50,9 +50,11 @@ public void test_detection_accessibility_font_size() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebAccessibilityFontSizeOracle); // Assert the oracle verdict is WARNING_ACCESSIBILITY_FAULT - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_ACCESSIBILITY_FAULT.getTitle())); - Assert.isTrue(verdict.info().equals("These widgets Text 'widgettext' , are too small. Minimum recommended is 12 px.")); + Assert.isTrue(verdict.info().equals("Widget text 'widgettext' , is too small (11 px). Minimum recommended is 12 px.")); } @Test @@ -72,7 +74,9 @@ public void test_undetection_accessibility_font_size() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebAccessibilityFontSizeOracle); // Assert the oracle verdict is OK - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.OK.getTitle())); Assert.isTrue(verdict.info().equals("No problem detected.")); } diff --git a/testar/test/org/testar/oracles/web/accessibility/TestWebAccessibilityImagesAltOracle.java b/testar/test/org/testar/oracles/web/accessibility/TestWebAccessibilityImagesAltOracle.java index f26222672..95e73aabc 100644 --- a/testar/test/org/testar/oracles/web/accessibility/TestWebAccessibilityImagesAltOracle.java +++ b/testar/test/org/testar/oracles/web/accessibility/TestWebAccessibilityImagesAltOracle.java @@ -49,9 +49,11 @@ public void test_detection_accessibility_images_alt() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebAccessibilityImagesAltOracle); // Assert the oracle verdict is WARNING_ACCESSIBILITY_FAULT - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_ACCESSIBILITY_FAULT.getTitle())); - Assert.isTrue(verdict.info().equals("Detected web image widgets ''<img src='url'>whatever</img>' , ' without alternative text!")); + Assert.isTrue(verdict.info().equals("Detected web image widget '<img src='url'>whatever</img>' , without alternative text!")); } @Test @@ -70,7 +72,9 @@ public void test_undetection_accessibility_images_alt() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebAccessibilityImagesAltOracle); // Assert the oracle verdict is OK - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.OK.getTitle())); Assert.isTrue(verdict.info().equals("No problem detected.")); } diff --git a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantDuplicateMenuItems.java b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantDuplicateMenuItems.java index a564b0251..48bf2c7bd 100644 --- a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantDuplicateMenuItems.java +++ b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantDuplicateMenuItems.java @@ -57,9 +57,11 @@ public void test_detection_web_invariant_duplicated_menu_items() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantDuplicateMenuItems); // Assert the oracle verdict is WARNING_WEB_INVARIANT_FAULT - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT.getTitle())); - Assert.isTrue(verdict.info().equals("Detected a Unnumbered List (UL) web menu 'menuid' , with duplicate option elements!")); + Assert.isTrue(verdict.info().equals("Detected a Unnumbered List (UL) web menu 'menuid' , with duplicate option elements: [menu_element]")); } @Test @@ -86,7 +88,9 @@ public void test_undetection_web_invariant_duplicated_menu_items() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantDuplicateMenuItems); // Assert the oracle verdict is OK - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.OK.getTitle())); Assert.isTrue(verdict.info().equals("No problem detected.")); } diff --git a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantDuplicateSelectItems.java b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantDuplicateSelectItems.java index 175528814..1168fa3aa 100644 --- a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantDuplicateSelectItems.java +++ b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantDuplicateSelectItems.java @@ -59,9 +59,11 @@ public void test_detection_web_invariant_duplicate_select_items() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantDuplicateSelectItems); // Assert the oracle verdict is WARNING_WEB_INVARIANT_FAULT - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT.getTitle())); - Assert.isTrue(verdict.info().equals("Detected Select widgets 'selectid' , with duplicate values!")); + Assert.isTrue(verdict.info().equals("Detected Select widget 'selectid' , with duplicate values: [Renault]")); } } @@ -87,7 +89,9 @@ public void test_undetection_web_invariant_duplicate_select_items() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantDuplicateSelectItems); // Assert the oracle verdict is OK - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.OK.getTitle())); Assert.isTrue(verdict.info().equals("No problem detected.")); } diff --git a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantDuplicatedRowsInTable.java b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantDuplicatedRowsInTable.java index 53d7b05e0..72db2a122 100644 --- a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantDuplicatedRowsInTable.java +++ b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantDuplicatedRowsInTable.java @@ -9,6 +9,7 @@ import org.testar.monkey.Assert; import org.testar.monkey.ConfigTags; import org.testar.monkey.Pair; +import org.testar.monkey.alayer.Rect; import org.testar.monkey.alayer.Tags; import org.testar.monkey.alayer.Verdict; import org.testar.monkey.alayer.webdriver.enums.WdRoles; @@ -44,6 +45,7 @@ public void test_detection_web_invariant_duplicated_rows_table() { WidgetStub firstRow = new WidgetStub(); firstRow.set(Tags.Role, WdRoles.WdTR); + firstRow.set(Tags.Shape, Rect.fromCoordinates(0, 0, 10, 10)); widgetTable.addChild(firstRow); WidgetStub firstHeader = new WidgetStub(); @@ -58,6 +60,7 @@ public void test_detection_web_invariant_duplicated_rows_table() { WidgetStub secondRow = new WidgetStub(); secondRow.set(Tags.Role, WdRoles.WdTR); + secondRow.set(Tags.Shape, Rect.fromCoordinates(0, 0, 10, 10)); widgetTable.addChild(secondRow); WidgetStub secondHeader = new WidgetStub(); @@ -75,9 +78,13 @@ public void test_detection_web_invariant_duplicated_rows_table() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantDuplicatedRowsInTable); // Assert the oracle verdict is WARNING_WEB_INVARIANT_FAULT - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT.getTitle())); - Assert.isTrue(verdict.info().contains("Detected a duplicated rows in a Table for the widgets: [_header_content_data_content, _header_content_data_content]")); + Assert.isTrue(verdict.info().contains("Detected duplicated row in a Table for the widget: _header_content_data_content")); + Assert.isEquals(2, verdict.visualizer().getShapes().size()); } @Test @@ -122,8 +129,64 @@ public void test_undetection_invariant_duplicated_rows_table() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantDuplicatedRowsInTable); // Assert the oracle verdict is OK - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.OK.getTitle())); Assert.isTrue(verdict.info().equals("No problem detected.")); } + + @Test + public void test_detection_web_invariant_duplicated_rows_multiple_tables() { + StateStub state = new StateStub(); + + WidgetStub tableOne = createTable(state, "table1"); + addDuplicatedRowPair(tableOne, "headerA", "dataA"); + addDuplicatedRowPair(tableOne, "headerB", "dataB"); + + WidgetStub tableTwo = createTable(state, "table2"); + addDuplicatedRowPair(tableTwo, "headerC", "dataC"); + addDuplicatedRowPair(tableTwo, "headerD", "dataD"); + + Assert.isTrue(extendedOraclesList.size() == 1); + Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantDuplicatedRowsInTable); + + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(4, verdicts.size()); + for (Verdict verdict : verdicts) { + Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT.getTitle())); + Assert.isEquals(2, verdict.visualizer().getShapes().size()); + } + } + + private WidgetStub createTable(StateStub state, String tableId) { + WidgetStub widgetTable = new WidgetStub(); + widgetTable.set(Tags.Role, WdRoles.WdTABLE); + widgetTable.set(WdTags.WebId, tableId); + state.addChild(widgetTable); + widgetTable.setParent(state); + return widgetTable; + } + + private void addDuplicatedRowPair(WidgetStub table, String headerText, String dataText) { + addRow(table, headerText, dataText); + addRow(table, headerText, dataText); + } + + private void addRow(WidgetStub table, String headerText, String dataText) { + WidgetStub row = new WidgetStub(); + row.set(Tags.Role, WdRoles.WdTR); + row.set(Tags.Shape, Rect.fromCoordinates(0, 0, 10, 10)); + table.addChild(row); + + WidgetStub header = new WidgetStub(); + header.set(Tags.Role, WdRoles.WdTH); + header.set(WdTags.WebTextContent, headerText); + row.addChild(header); + + WidgetStub data = new WidgetStub(); + data.set(Tags.Role, WdRoles.WdTD); + data.set(WdTags.WebTextContent, dataText); + row.addChild(data); + } } diff --git a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantEmptySelectItems.java b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantEmptySelectItems.java index c632bf498..f1e761ced 100644 --- a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantEmptySelectItems.java +++ b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantEmptySelectItems.java @@ -57,9 +57,11 @@ public void test_detection_web_invariant_empty_select_items() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantEmptySelectItems); // Assert the oracle verdict is WARNING_WEB_INVARIANT_FAULT - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT.getTitle())); - Assert.isTrue(verdict.info().equals("Detected Select widgets 'selectid' , with empty or only one item!")); + Assert.isTrue(verdict.info().equals("Detected Select widget 'selectid' , with empty or only one item (count: 1)!")); } } @@ -84,7 +86,9 @@ public void test_undetection_web_invariant_empty_select_items() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantEmptySelectItems); // Assert the oracle verdict is OK - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.OK.getTitle())); Assert.isTrue(verdict.info().equals("No problem detected.")); } diff --git a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantManySelectItems.java b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantManySelectItems.java index 532db63af..8755bf6e5 100644 --- a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantManySelectItems.java +++ b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantManySelectItems.java @@ -57,9 +57,11 @@ public void test_detection_web_invariant_many_select_items() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantManySelectItems); // Assert the oracle verdict is WARNING_WEB_INVARIANT_FAULT - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT.getTitle())); - Assert.isTrue(verdict.info().equals("Detected Select widgets 'selectid' , which have more items than the threshold value of 100")); + Assert.isTrue(verdict.info().equals("Detected Select widget 'selectid' , which has 101 items (threshold: 100)")); } } @@ -84,7 +86,9 @@ public void test_undetection_web_invariant_many_select_items() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantManySelectItems); // Assert the oracle verdict is OK - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.OK.getTitle())); Assert.isTrue(verdict.info().equals("No problem detected.")); } diff --git a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantNumberWithLotOfDecimals.java b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantNumberWithLotOfDecimals.java index ea7a20c71..b9dfb9d80 100644 --- a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantNumberWithLotOfDecimals.java +++ b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantNumberWithLotOfDecimals.java @@ -45,9 +45,11 @@ public void test_detection_web_invariant_number_lot_decimals() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantNumberWithLotOfDecimals); // Assert the oracle verdict is WARNING_WEB_INVARIANT_FAULT - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT.getTitle())); - Assert.isTrue(verdict.info().equals("Detected widgets '30.123€' , with more than 2 decimals!")); + Assert.isTrue(verdict.info().equals("Detected widget '30.123€' , with 3 decimals (max: 2)!")); } @Test @@ -64,7 +66,9 @@ public void test_undetection_web_invariant_number_lot_decimals() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantNumberWithLotOfDecimals); // Assert the oracle verdict is OK - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.OK.getTitle())); Assert.isTrue(verdict.info().equals("No problem detected.")); } diff --git a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantTextAreaWithoutLength.java b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantTextAreaWithoutLength.java index fb768b123..73417f0a1 100644 --- a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantTextAreaWithoutLength.java +++ b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantTextAreaWithoutLength.java @@ -49,9 +49,11 @@ public void test_detection_web_invariant_textarea_without_length() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantTextAreaWithoutLength); // Assert the oracle verdict is WARNING_WEB_INVARIANT_FAULT - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT.getTitle())); - Assert.isTrue(verdict.info().equals("Detected TextArea widgets '<textarea maxlength=0></textarea>' , with 0 max length!")); + Assert.isTrue(verdict.info().equals("Detected TextArea widget '<textarea maxlength=0></textarea>' , with 0 max length!")); } @Test @@ -70,7 +72,9 @@ public void test_undetection_web_invariant_textarea_without_length() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantTextAreaWithoutLength); // Assert the oracle verdict is OK - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.OK.getTitle())); Assert.isTrue(verdict.info().equals("No problem detected.")); } diff --git a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantUnsortedSelectItems.java b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantUnsortedSelectItems.java index 165073698..fc98082d3 100644 --- a/testar/test/org/testar/oracles/web/invariants/TestWebInvariantUnsortedSelectItems.java +++ b/testar/test/org/testar/oracles/web/invariants/TestWebInvariantUnsortedSelectItems.java @@ -59,9 +59,11 @@ public void test_detection_web_invariant_unsorted_select_items_month() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantUnsortedSelectItems); // Assert the oracle verdict is WARNING_WEB_INVARIANT_FAULT - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT.getTitle())); - Assert.isTrue(verdict.info().equals("Detected Select widgets 'selectid' , with unsorted elements!")); + Assert.isTrue(verdict.info().equals("Detected Select widget 'selectid' , with unsorted elements!")); } } @@ -87,9 +89,11 @@ public void test_detection_web_invariant_unsorted_select_items_natural() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantUnsortedSelectItems); // Assert the oracle verdict is WARNING_WEB_INVARIANT_FAULT - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT.getTitle())); - Assert.isTrue(verdict.info().equals("Detected Select widgets 'selectid' , with unsorted elements!")); + Assert.isTrue(verdict.info().equals("Detected Select widget 'selectid' , with unsorted elements!")); } } @@ -115,9 +119,11 @@ public void test_detection_web_invariant_unsorted_select_items_numeric() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantUnsortedSelectItems); // Assert the oracle verdict is WARNING_WEB_INVARIANT_FAULT - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.WARNING_WEB_INVARIANT_FAULT.getTitle())); - Assert.isTrue(verdict.info().equals("Detected Select widgets 'selectid' , with unsorted elements!")); + Assert.isTrue(verdict.info().equals("Detected Select widget 'selectid' , with unsorted elements!")); } } @@ -143,7 +149,9 @@ public void test_undetection_web_invariant_unsorted_select_items_month() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantUnsortedSelectItems); // Assert the oracle verdict is OK - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.OK.getTitle())); Assert.isTrue(verdict.info().equals("No problem detected.")); } @@ -171,7 +179,9 @@ public void test_undetection_web_invariant_unsorted_select_items_natural() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantUnsortedSelectItems); // Assert the oracle verdict is OK - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.OK.getTitle())); Assert.isTrue(verdict.info().equals("No problem detected.")); } @@ -199,7 +209,9 @@ public void test_undetection_web_invariant_unsorted_select_items_numeric() { Assert.isTrue(extendedOraclesList.get(0) instanceof WebInvariantUnsortedSelectItems); // Assert the oracle verdict is OK - Verdict verdict = extendedOraclesList.get(0).getVerdict(state); + List verdicts = extendedOraclesList.get(0).getVerdicts(state); + Assert.isEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.isTrue(verdict.verdictSeverityTitle().equals(Verdict.Severity.OK.getTitle())); Assert.isTrue(verdict.info().equals("No problem detected.")); } diff --git a/testar/test/org/testar/reporting/TestReportManager.java b/testar/test/org/testar/reporting/TestReportManager.java index 238d49f37..dc4725152 100644 --- a/testar/test/org/testar/reporting/TestReportManager.java +++ b/testar/test/org/testar/reporting/TestReportManager.java @@ -4,6 +4,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Properties; @@ -34,7 +35,7 @@ public class TestReportManager { private static StateStub state; private static Set derivedActions; private static Action selectedAction; - private static Verdict finalVerdict = Verdict.OK; + private static List finalVerdicts = Collections.singletonList(Verdict.OK); @Before public void setUp() throws IOException { @@ -99,7 +100,7 @@ public void testHtmlReport() { ReportManager reportManager = createReportManager(settings); // Verify the html report file was created with the state and actions information - File htmlReportFile = new File(reportManager.getReportFileName().concat("_OK.html")); + File htmlReportFile = new File(reportManager.getReportFileName().concat("_V001_OK.html")); System.out.println("testHtmlReport: " + htmlReportFile.getPath()); Assert.assertTrue(htmlReportFile.exists()); @@ -120,7 +121,7 @@ public void testHtmlReport() { Assert.assertTrue(fileContains("

Test verdict for this sequence: No problem detected.

", htmlReportFile)); // Verify the plain txt report was not created - File txtReportFile = new File(reportManager.getReportFileName().concat("_OK.txt")); + File txtReportFile = new File(reportManager.getReportFileName().concat("_V001_OK.txt")); Assert.assertTrue(!txtReportFile.exists()); } @@ -142,11 +143,11 @@ public void testPlainReport() { ReportManager reportManager = createReportManager(settings); // Verify the html report was not created - File htmlReportFile = new File(reportManager.getReportFileName().concat("_OK.html")); + File htmlReportFile = new File(reportManager.getReportFileName().concat("_V001_OK.html")); Assert.assertTrue(!htmlReportFile.exists()); // Verify the txt report file was created with the state and actions information - File txtReportFile = new File(reportManager.getReportFileName().concat("_OK.txt")); + File txtReportFile = new File(reportManager.getReportFileName().concat("_V001_OK.txt")); System.out.println("testPlainReport: " + txtReportFile.getPath()); Assert.assertTrue(txtReportFile.exists()); @@ -187,7 +188,7 @@ public void testHtmlReportWithoutStateScreenshot() { ReportManager reportManager = createReportManager(settings); // Verify the html report file was created with the state and actions information - File htmlReportFile = new File(reportManager.getReportFileName().concat("_OK.html")); + File htmlReportFile = new File(reportManager.getReportFileName().concat("_V001_OK.html")); System.out.println("testHtmlReportWithoutScreenshot: " + htmlReportFile.getPath()); Assert.assertTrue(htmlReportFile.exists()); @@ -216,11 +217,12 @@ public void testHtmlReportParsesSpecialCharacters() { // Prepare a report only with the final verdict ReportManager reportManager = new ReportManager(false, settings); - reportManager.addTestVerdict(new Verdict(Verdict.Severity.FAIL, "Failure is ")); + Verdict failVerdict = new Verdict(Verdict.Severity.FAIL, "Failure is "); + reportManager.addTestVerdicts(Collections.singletonList(failVerdict)); reportManager.finishReport(); // Verify the html report file was created - File htmlReportFile = new File(reportManager.getReportFileName().concat("_FAIL.html")); + File htmlReportFile = new File(reportManager.getReportFileName().concat("_V001_FAIL.html")); System.out.println("testSpecialCharacters: " + htmlReportFile.getPath()); Assert.assertTrue(htmlReportFile.exists()); @@ -233,7 +235,7 @@ private ReportManager createReportManager(Settings settings) { reportManager.addState(state); reportManager.addActions(derivedActions); reportManager.addSelectedAction(state, selectedAction); - reportManager.addTestVerdict(finalVerdict); + reportManager.addTestVerdicts(finalVerdicts); reportManager.finishReport(); return reportManager; } From 1ab0abf8c0e2d964aa1662d9f62ac10b0965f3d8 Mon Sep 17 00:00:00 2001 From: ferpasri Date: Fri, 20 Feb 2026 16:59:55 +0100 Subject: [PATCH 03/10] Remove disabled Record mode --- README.md | 2 +- .../org/testar/monkey/DefaultProtocol.java | 5 +--- .../monkey/RuntimeControlsProtocol.java | 30 ++----------------- .../testar/settings/dialog/ToolTipTexts.java | 2 -- 4 files changed, 4 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 75add0464..f831c95ae 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ Some of the most interesting parameters that can help to integrate TESTAR as an ShowVisualSettingsDialogOnStartup -> To run TESTAR without the GUI - Mode -> TESTAR execution Mode (Spy, Generate, Record, Replay, View) + Mode -> TESTAR execution Mode (Spy, Generate, Replay, View) SUTConnector & SUTConnectorValue -> The way to link with the desired application to be tested diff --git a/testar/src/org/testar/monkey/DefaultProtocol.java b/testar/src/org/testar/monkey/DefaultProtocol.java index 37cfb4401..e3bdc6097 100644 --- a/testar/src/org/testar/monkey/DefaultProtocol.java +++ b/testar/src/org/testar/monkey/DefaultProtocol.java @@ -265,9 +265,6 @@ public final void run(final Settings settings) { new ReplayMode().runReplayLoop(this); } else if (mode() == Modes.Spy) { new SpyMode().runSpyLoop(this); - } else if(mode() == Modes.Record) { - //new RecordMode().runRecordLoop(this); - System.out.println("Dear User, TESTAR Record mode is disabled temporarily."); } else if (mode() == Modes.Generate) { new GenerateMode().runGenerateOuterLoop(this); } @@ -316,7 +313,7 @@ protected void initialize(Settings settings) { logOracleEnabled = settings.get(ConfigTags.LogOracleEnabled, false); processListenerOracleEnabled = settings.get(ConfigTags.ProcessListenerEnabled, false); - if ( mode() == Modes.Generate || /*mode() == Modes.Record ||*/ mode() == Modes.Replay ) { + if ( mode() == Modes.Generate || mode() == Modes.Replay ) { //Create the output folders OutputStructure.calculateOuterLoopDateString(); OutputStructure.sequenceInnerLoopCount = 0; diff --git a/testar/src/org/testar/monkey/RuntimeControlsProtocol.java b/testar/src/org/testar/monkey/RuntimeControlsProtocol.java index 9eb14501a..879e05951 100644 --- a/testar/src/org/testar/monkey/RuntimeControlsProtocol.java +++ b/testar/src/org/testar/monkey/RuntimeControlsProtocol.java @@ -48,7 +48,6 @@ public abstract class RuntimeControlsProtocol extends AbstractProtocol implement public enum Modes{ Spy, - Record, Generate, Quit, View, @@ -80,8 +79,7 @@ protected synchronized void setMode(Modes mode){ private final static double SLOW_MOTION = 2.0; //TODO: key commands come through java.awt.event but are the key codes same for all OS? if they are the same, then move to platform independent protocol? //TODO: Investigate better shortcut combinations to control TESTAR that does not interfere with SUT - // (e.g. SHIFT + 1 puts an ! in the notepad and hence interferes with SUT state, but the - // event is not recorded as a user event). + // (e.g. SHIFT + 1 puts an ! in the notepad and hence interferes with SUT state). /** * Override the default keylistener to implement the TESTAR shortcuts * SHIFT + SPACE @@ -129,15 +127,6 @@ else if (key == KBKeys.VK_0 && pressed.contains(KBKeys.VK_SHIFT)) { System.setProperty("DEBUG_WINDOWS_PROCESS_NAMES","true"); } - // In Record mode you can press any key except SHIFT to add a user keyboard - // This is because SHIFT is used for the TESTAR shortcuts - // This is not ideal, because now special characters and capital letters and other events that needs SHIFT - // cannot be recorded as an user event in Record.... - else if (!pressed.contains(KBKeys.VK_SHIFT) && mode() == Modes.Record && userEvent == null) { - //System.out.println("USER_EVENT key_down! " + key.toString()); - userEvent = new Object[]{key}; // would be ideal to set it up at keyUp - } - // SHIFT + ALT --> Toggle widget-tree hierarchy display if (pressed.contains(KBKeys.VK_ALT) && pressed.contains(KBKeys.VK_SHIFT)) { markParentWidget = !markParentWidget; @@ -162,23 +151,8 @@ public void keyUp(KBKeys key){ @Override public void mouseDown(MouseButtons btn, double x, double y){} - /** - * In Record mode the user can add user events by clicking and the event is added when releasing the mouse - * @param btn - * @param x - * @param y - */ @Override - public void mouseUp(MouseButtons btn, double x, double y){ - // In GenerateManual the user can add user events by clicking - if (mode() == Modes.Record && userEvent == null){ - userEvent = new Object[]{ - btn, - Double.valueOf(x), - Double.valueOf(y) - }; - } - } + public void mouseUp(MouseButtons btn, double x, double y){} @Override public void mouseMoved(double x, double y) {} diff --git a/testar/src/org/testar/settings/dialog/ToolTipTexts.java b/testar/src/org/testar/settings/dialog/ToolTipTexts.java index 79c10b66f..8aca78066 100644 --- a/testar/src/org/testar/settings/dialog/ToolTipTexts.java +++ b/testar/src/org/testar/settings/dialog/ToolTipTexts.java @@ -52,8 +52,6 @@ public class ToolTipTexts { "generation. This is ideal if a sequence turns out not to be reproducible.\n"; public static String btnModelTTT = "\nStart in State Model Analysis Mode:
\n" + "This mode allows you to connect with OrientDB to inspect the inferred models.\n"; - public static String btnRecordTTT = "\nStart in RECORD Mode:
\n" + - "This modes enables the tester to manually record (part of) a sequence.\n"; // TTTs for the general tab public static String sutConnectorTTT = "How does TESTAR connect to the SUT"; From db23f33cd5b6f378ea5a97dbf7cfce364ee8ece6 Mon Sep 17 00:00:00 2001 From: ferpasri Date: Fri, 20 Feb 2026 20:26:38 +0100 Subject: [PATCH 04/10] Update to VerdictProcessing filterDuplicates --- .../org/testar/monkey/VerdictProcessing.java | 44 +++++++++++++------ 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/testar/src/org/testar/monkey/VerdictProcessing.java b/testar/src/org/testar/monkey/VerdictProcessing.java index 20d43899d..868bffe2e 100644 --- a/testar/src/org/testar/monkey/VerdictProcessing.java +++ b/testar/src/org/testar/monkey/VerdictProcessing.java @@ -77,29 +77,29 @@ public List filterDuplicates(List verdicts) { if (verdicts == null || verdicts.isEmpty()) { return Collections.singletonList(Verdict.OK); } + List filtered = new ArrayList<>(); + for (Verdict verdict : verdicts) { - boolean shouldIgnore = ignoreDuplicatedVerdicts - && !verdict.isCritical() - && verdict.severity() > Verdict.Severity.OK.getValue() - && isDuplicateVerdictInfo(verdict.info()); - if (!shouldIgnore) { - filtered.add(verdict); + if (verdict == null) { + continue; } + + if (shouldIgnorePersistedDuplicate(verdict)) { + continue; + } + + filtered.add(verdict); } + if (filtered.isEmpty()) { return Collections.singletonList(Verdict.OK); } - boolean allVerdictsOk = true; - for (Verdict verdict : filtered) { - if (verdict.severity() > Verdict.Severity.OK.getValue()) { - allVerdictsOk = false; - break; - } - } - if (allVerdictsOk) { + + if (areAllVerdictsOk(filtered)) { return Collections.singletonList(Verdict.OK); } + return clearOkIfFailurePresent(filtered); } @@ -166,6 +166,22 @@ private String normalizeVerdictInfo(String verdictInfo) { return verdictInfo == null ? "" : verdictInfo.replace("\n", " ").trim(); } + private boolean shouldIgnorePersistedDuplicate(Verdict verdict) { + return ignoreDuplicatedVerdicts + && !verdict.isCritical() + && verdict.severity() > Verdict.Severity.OK.getValue() + && isDuplicateVerdictInfo(verdict.info()); + } + + private boolean areAllVerdictsOk(List verdicts) { + for (Verdict verdict : verdicts) { + if (verdict.severity() > Verdict.Severity.OK.getValue()) { + return false; + } + } + return true; + } + private List clearOkIfFailurePresent(List verdicts) { boolean hasFailureVerdict = false; for (Verdict verdict : verdicts) { From 0db51b5aa662139ea4be98be33b02f45a6574f17 Mon Sep 17 00:00:00 2001 From: ferpasri Date: Fri, 20 Feb 2026 20:27:40 +0100 Subject: [PATCH 05/10] Update sequence verdicts and replay mode --- ..._test_gradle_workflow_android_generic.java | 5 +-- ..._test_gradle_workflow_desktop_generic.java | 16 ++++--- ...radle_workflow_webdriver_form_filling.java | 6 +-- .../test.settings | 2 +- ...est_gradle_workflow_webdriver_generic.java | 5 +-- .../paris_parisone.testar | Bin 17074 -> 91355 bytes .../org/testar/monkey/DefaultProtocol.java | 41 ++++++++---------- .../src/org/testar/monkey/GenerateMode.java | 13 +++--- testar/src/org/testar/monkey/ReplayMode.java | 18 +++----- testar/workflow_windows_webdriver.gradle | 2 +- 10 files changed, 50 insertions(+), 58 deletions(-) diff --git a/testar/resources/workflow/settings/test_gradle_workflow_android_generic/Protocol_test_gradle_workflow_android_generic.java b/testar/resources/workflow/settings/test_gradle_workflow_android_generic/Protocol_test_gradle_workflow_android_generic.java index 1b33d827a..458f1a1b1 100644 --- a/testar/resources/workflow/settings/test_gradle_workflow_android_generic/Protocol_test_gradle_workflow_android_generic.java +++ b/testar/resources/workflow/settings/test_gradle_workflow_android_generic/Protocol_test_gradle_workflow_android_generic.java @@ -140,10 +140,9 @@ protected void postSequenceProcessing() { // Verify html and txt report files were created Assert.isTrue(reportManager instanceof ReportManager); - List verdicts = getFinalVerdicts(); - Assert.isTrue(verdicts.size() > 0); + Assert.isTrue(getSequenceVerdicts().size() > 0); int index = 1; - for (Verdict verdict : verdicts) { + for (Verdict verdict : getSequenceVerdicts()) { String suffixName = String.format("_V%03d_%s", index++, verdict.verdictSeverityTitle()); File htmlReportFile = new File(((ReportManager)reportManager).getReportFileName().concat(suffixName + ".html")); File txtReportFile = new File(((ReportManager)reportManager).getReportFileName().concat(suffixName + ".txt")); diff --git a/testar/resources/workflow/settings/test_gradle_workflow_desktop_generic/Protocol_test_gradle_workflow_desktop_generic.java b/testar/resources/workflow/settings/test_gradle_workflow_desktop_generic/Protocol_test_gradle_workflow_desktop_generic.java index 08733e20d..21313f4d7 100644 --- a/testar/resources/workflow/settings/test_gradle_workflow_desktop_generic/Protocol_test_gradle_workflow_desktop_generic.java +++ b/testar/resources/workflow/settings/test_gradle_workflow_desktop_generic/Protocol_test_gradle_workflow_desktop_generic.java @@ -122,7 +122,7 @@ protected void postSequenceProcessing() { super.postSequenceProcessing(); // If OnlySaveFaultySequences is enabled and the sequence verdict is OK, sequence_ok must not exist - if(settings().get(ConfigTags.OnlySaveFaultySequences) && Verdict.helperAreAllVerdictsOK(getFinalVerdicts())) { + if(settings().get(ConfigTags.OnlySaveFaultySequences) && Verdict.helperAreAllVerdictsOK(getSequenceVerdicts())) { String sequencesOkFolderName = OutputStructure.outerLoopOutputDir + File.separator + "sequences_ok"; File sequencesOkFolder = null; try { @@ -136,8 +136,13 @@ protected void postSequenceProcessing() { // Or if OnlySaveFaultySequences disabled, // sequence must have generated a .testar file else { - Assert.isTrue(getGeneratedSequenceName().endsWith(".testar")); - Assert.isTrue(new File(getGeneratedSequenceName()).exists()); + Assert.isTrue(getSequenceVerdicts().size() > 0); + for (Verdict verdict : getSequenceVerdicts()) { + String sequencesFolderName = OutputStructure.outerLoopOutputDir + File.separator + "sequences_" + verdict.verdictSeverityTitle().toLowerCase(); + File sequencesFolder = new File(sequencesFolderName); + File[] matchingFiles = sequencesFolder.listFiles((dir, name) -> name.contains("sequence_1") && name.endsWith(".testar")); + Assert.isTrue(matchingFiles != null && matchingFiles.length > 0); + } } // Verify the JsonUtils created a JSON State file @@ -160,10 +165,9 @@ protected void postSequenceProcessing() { // Verify html and txt report files were created Assert.isTrue(reportManager instanceof ReportManager); - List verdicts = getFinalVerdicts(); - Assert.isTrue(verdicts.size() > 0); + Assert.isTrue(getSequenceVerdicts().size() > 0); int index = 1; - for (Verdict verdict : verdicts) { + for (Verdict verdict : getSequenceVerdicts()) { String suffixName = String.format("_V%03d_%s", index++, verdict.verdictSeverityTitle()); File htmlReportFile = new File(((ReportManager)reportManager).getReportFileName().concat(suffixName + ".html")); File txtReportFile = new File(((ReportManager)reportManager).getReportFileName().concat(suffixName + ".txt")); diff --git a/testar/resources/workflow/settings/test_gradle_workflow_webdriver_form_filling/Protocol_test_gradle_workflow_webdriver_form_filling.java b/testar/resources/workflow/settings/test_gradle_workflow_webdriver_form_filling/Protocol_test_gradle_workflow_webdriver_form_filling.java index 68ea82e0c..2dac5d028 100644 --- a/testar/resources/workflow/settings/test_gradle_workflow_webdriver_form_filling/Protocol_test_gradle_workflow_webdriver_form_filling.java +++ b/testar/resources/workflow/settings/test_gradle_workflow_webdriver_form_filling/Protocol_test_gradle_workflow_webdriver_form_filling.java @@ -49,7 +49,6 @@ import java.io.IOException; import java.io.ObjectInputStream; import java.util.*; -import java.util.zip.GZIPInputStream; /** * This protocol is used to test TESTAR by executing a gradle CI workflow. @@ -126,7 +125,7 @@ protected void closeTestSession() { String sequencesOkFolderName = OutputStructure.outerLoopOutputDir + File.separator + "sequences_ok"; File sequencesOkFolder = new File(sequencesOkFolderName).getCanonicalFile(); System.out.println("sequencesFolder: " + sequencesOkFolder); - File[] matchingFiles = sequencesOkFolder.listFiles((dir, name) -> name.endsWith("sequence_1.testar")); + File[] matchingFiles = sequencesOkFolder.listFiles((dir, name) -> name.contains("sequence_1") && name.endsWith(".testar")); Assert.isTrue(matchingFiles.length == 1, "One replayable testar file was not created"); System.out.println("matchingFiles[0]: " + matchingFiles[0]); Assert.isTrue(isValidReplayFile(matchingFiles[0]), "Replayable testar file was not serialized correctly!"); @@ -147,8 +146,7 @@ private boolean isValidReplayFile(File replayFile){ try { FileInputStream fis = new FileInputStream(replayFile); BufferedInputStream bis = new BufferedInputStream(fis); - GZIPInputStream gis = new GZIPInputStream(bis); - ObjectInputStream ois = new ObjectInputStream(gis); + ObjectInputStream ois = new ObjectInputStream(bis); ois.readObject(); ois.close(); diff --git a/testar/resources/workflow/settings/test_gradle_workflow_webdriver_form_filling/test.settings b/testar/resources/workflow/settings/test_gradle_workflow_webdriver_form_filling/test.settings index 49daebddd..f2218004d 100644 --- a/testar/resources/workflow/settings/test_gradle_workflow_webdriver_form_filling/test.settings +++ b/testar/resources/workflow/settings/test_gradle_workflow_webdriver_form_filling/test.settings @@ -164,7 +164,7 @@ LogLevel = 1 FaultThreshold = 1.0E-9 MyClassPath = ./settings OnlySaveFaultySequences = false -ActionDuration = 0.1 +ActionDuration = 1 ShowVisualSettingsDialogOnStartup = true ReplayRetryTime = 30.0 Delete = diff --git a/testar/resources/workflow/settings/test_gradle_workflow_webdriver_generic/Protocol_test_gradle_workflow_webdriver_generic.java b/testar/resources/workflow/settings/test_gradle_workflow_webdriver_generic/Protocol_test_gradle_workflow_webdriver_generic.java index 153982014..6c777ef2d 100644 --- a/testar/resources/workflow/settings/test_gradle_workflow_webdriver_generic/Protocol_test_gradle_workflow_webdriver_generic.java +++ b/testar/resources/workflow/settings/test_gradle_workflow_webdriver_generic/Protocol_test_gradle_workflow_webdriver_generic.java @@ -170,10 +170,9 @@ protected void postSequenceProcessing() { // Verify html and txt report files were created Assert.isTrue(reportManager instanceof ReportManager); - List verdicts = getFinalVerdicts(); - Assert.isTrue(verdicts.size() > 0); + Assert.isTrue(getSequenceVerdicts().size() > 0); int index = 1; - for (Verdict verdict : verdicts) { + for (Verdict verdict : getSequenceVerdicts()) { String suffixName = String.format("_V%03d_%s", index++, verdict.verdictSeverityTitle()); File htmlReportFile = new File(((ReportManager)reportManager).getReportFileName().concat(suffixName + ".html")); File txtReportFile = new File(((ReportManager)reportManager).getReportFileName().concat(suffixName + ".txt")); diff --git a/testar/resources/workflow/settings/test_gradle_workflow_webdriver_replay/paris_parisone.testar b/testar/resources/workflow/settings/test_gradle_workflow_webdriver_replay/paris_parisone.testar index 7f0b0b0bfda09ea1b9dfdcdfffb0e805aa67d017..e2d1e956f14176ab524129cc0ef13434517cc5cc 100644 GIT binary patch literal 91355 zcmeHw3!GdL_zr6FYw{|EU*D2lEV(~If&y8#OUCIhQJ9bUBSf>({<9AJDr?Y)^ zy-@7iovmz~tteE(2?*>(VDI*9ZJZ+f!S8+QGpD@hYn@8RWlDFkoXuaB&FSUp4k~TD zs@2BJ#r$mLl70n$t7byvuAzTHD(s9_&3)wTM{fM|Ef4*=L+RR~gvP0|Tsg1pQvA7M z_V8g{IYG3qm@SR<4HUDr+QwbV%Dh&~RSOlpP%i1piMAAjdbLm*qaCRT|8JVE0L|Q5`>`4rx-pJSXe&qNk ze{f2N(z#6u)wF4?TF~dHM}^X8nR~QL>7Fjs>e*u9uvXQT=kKB-`^0eQ6T_iT42QmJ z%w%}4#-Fa7bY(eQsg`M+Ooj8BPHofjz0|hTn%j0Y4S{o`zM)qgnf;$*zwA&H>I53$ z|JvUDFDIg|VG7~x^lmHHsl+4qe)@_FCjZbyb!GFQrq{yyxE8KeG%deATquQWTCQBm*Vc!#dU$5MkQ)zY!}pR;m~dMYL?#-+IZ?EAH4&AZxSv3i`a$#n_7IF^-if$Nfi2>tsv7% zU0ET@LX(an7UZb{Eltg&9}y|h>?RBD=)^S$qS{HL8a z_B>ATu}uk#YlX3KoyOfvA+L|`QaUR%7nL);C&*r{M3fCx%C&+3lXF_6H4@QO>OWVE z8~ID^_of2Xaqgg=r=P!mBKMlRKU5#?P=Wxsa~|4P+~Xj4cp_hK-SoG94!}nJqit zyJaGukrG`JS%>kS(8Lshg!ei@w5#pC>Wi_&hN}bwRh2V6#|u+!?2dC+zx=C}f9_EH zI~0GmSQ*dmPy*TN*hnn8L+KhJSfcQ;s-~58D4kV})-HObPd=$UDc&sk;dx0_IoGoy z8?|Mo6A2UKO2eRdX{kPCkEE5xKUgjn@)i2yX>@8o)=K)LLA1`ZE+W*JrtdX|;M=bM zu&RVNf1i4Hoe~@`=tH!WZ&SJmA@+-(Yj-Ju(LyPY1>$V)*K!#rq4yE0jb|$wu@5b$ z3&n4w{$1eNObH#bq4(c)-q^Ig9S4*y;(=(WpLM{SIOy#GCBSPKC0%>K#%LHz`T-?4!z?+DNa4=OCS43+~C0iwQq=wsBKw94j-;9-@3uTFF-+k>D*mHpqyT62B8Mkt%TrLh3Dnr_=E^;3k{qk;VwBRavUTmc#TDJ7%aP`Qy-u!!FENFZf z+=8qLZbn}t;KR~Qp*!1jTh6)1sgBswy*4e+nU-1MJM$JuS*tB z*Okz4e(TjkLwl~IVfK8B9k8=_SC^*B`NC)cBa~=zDX9H14VS}uAQj^>qSXQ?52c4RM2teR-@^`hGA6(024Hvj!! zopdyJ9d(%aBO37jesb{F&#v`*$)h4b{&LO!=s*7aTpBCh_&x}v%A8UB-SF+-`UY2G z3<*ESh*EA^cf~dYcN=lRA;NxQ=;)2Fwcg&AYBf6t_G(Gq9*!K#%@6@|J&~h{TH6)VDH*SdlBN3c4?z}c%WFwO@d!t$K+Wj+VX?PbDyRj zo#i=Y#f#eUiYrciSK*QO+q<-CPqi>sC>d|Es$A$<#*8+SuM%@u?H$gG94}0e06IDp{et<{2J2=@dmYj=9_u0y<}|1Ky9Y=$%8|BzCMyUtmbmj zNHQ7C#8av&>e+SY-+j!UF-Y}nuM#XN8tosn4Sjum0~`9Tu4&a;-(|Fu$d>Zup-Tsc zw(RTMR?gKSwy5=8UeLGKNBTtg!Mrvt1q*pXNgm;}_a3}@sMc2#+|@zKSMMFyr^LeH ze^URzLo{oIu!UFTU1gHU-1^ho{?BX9`BxhESIBi9Z9NSR4GawDZM2?1iUs)6$GJ}* zlB5?s{K1YJhmk=Jrxm-iv-aW8ZzLC~-~Z|l@Y|8EsN1T8nD7y>4MfLcX9= zDIp1C^g@n=W^-b^no|Pxb0dst^zNH}`ws|{wfHIXjy!h@&H*^awA^riXpKE-fRb)0 z@(#@??D7ZxvWOJ2CC6P^*7s?(3duo8!BS1*+edKjeL^$%x0 z$KF!oz<0XL)npA zZ6CgJ8%K$8zr}CB==DE5V-jg33Gmi-)P`wGQ=Zvf&S!bd806}_%N$zf9LuozJHI_aN4onut@-wAqB%;RS(Rex$PsT)R9=ZLY z_c}5XQE118w@#!SDd*=zx^2&W-L>{spTfl3sm+a)v(@}%gnTu}vLY~9K%;>SD zDiXI;>j~2s04sfQPr0I%251L=KFPsL`ugxXk!9TlJ)4DZQ=cf(rF5Qf3o2Z2?Nl~< zS`^K*ctJ&T4m6uRR?(aKh)j}NAWJh%NDK~EK@e~}W_FysAZf)2fJ7W{l*?8M#8-uL zM7D5wG<+#3D`>6n<+tK|1%h;<#yurX(v-Yb1HE$$&6H&h^0t+OG(~1DLwK-qH4&oe z^)=GLcu6K1__>h^M1aex*-}p9{)ZdJpdbKe-^j+kbr|V6_ ztFi0GATqno&n{iRB3sFpyx%-WbH}Bxbw@ZAcU=0~NiRAsa(HyS#_GJy-J|uA(C*mg zJqw!O6ZHx{27i64PVDDZqB zlO*m1$W02R@c9MG8QB;vgg4nr>n&=fF@2nZFT7AR#9r5d!V7zASUr1*QwASPMf8-) zG^tSPjiO!L#x`3ekkG459P7~{jzfDGe9=v+lKR;k-dokEsW~D8Yt^Q;p7r6L9XQQ?91eGPhbe&9 zA+b_k`<+9)iA|%~A{W$vUF7Bz(2bt` z#%(10>+zJ)Zx^a+t@_EgZR%S4 zdQH1dSyiZ6GLh>@=`2N6+musC6Fo+xH&}|%IvI2nO8OOKqf*FUp{&p)PeW!ElJ7D% z??EUX+aie3{O3;`-}A?JzH(isvWpA^m?_h!mQ^EVC>yr3%K8drrFdj{DubLH%YjR}fMuH`hunEH>qsZ=@2}1~hD$*D;MuJsi4QdovFO6$OHYYIB zbkR1FE2g-P0!ts!+5*J1V8o4NA^?oM%vYk#kc=!#v6mTO$UXNr2C)2)Dljykh`yCf zn0V_1yH~AhJH0qtu#3jv0u#Kmcx^wG9t@GwC>zxaEJ2keQn5Iq?=e%~3*UWk?w6`bJbVHU7a}x{czgg3SMh1nCzOnMWL}PQ zd>+;AbF21Cy3%!h@7yh!g|}{+`*A zf3b!B8$!7)KZTE9$ImRC62cxflRQd1={U2jzaV3p;Q@W7kS<4}$-J73#?le`XJ{}V ze*B8dkV`5I?5W9*?pZp;Iv|&Je-PeT;L! zdBXJl!SwwRK0~iE7y9gjc$9J>=w4XC$Z6m_sVm1hv?>(n^K)xVX%vEBg2L_fbz8*r z9mhO7qF9BD0pL3snSH0j>49@%Q(?+M3YS`{8!o#4vv`zZv=ga`L^q*x7wib4?-K+% zXZ6`xJjxE4+CjMzkRG>2COUyJ1ek)Xoq}atGNhbuXwAeQy5==zu-`>5B2c>L1 z1bTZR>9m$>bxSV%LR6kpx z@q53Jn`W&N90>CzUknb$y6<9ZqFFvfoJ9`fhwlQiMec%xT84<=xRu`WAt%W1OUT4x zO7W`|4wWX=#hiv|w5YX;jHvOqY-?3xdB(G-#pb#$&QFiEDp}1>jHHvXNHmj)#N+7< zNZewrii&PEtyOy;;0WZtOTJ0vD0~dmv{s!!YgK)&8jD9`(L_3)R(V-G)=Sk1v{W_M z$}5%XSgCez{x~gP!}+~D&)Zz6*7JUmM9kqf%r!s^$AY%t+zVa!d_0wD-CH|6FrF%< zD$$VB2Ue4 z!y>j&0?xPCji6l5#SW1J#Ry}u6kI+}bIj8AfL4KO~3bex5$XS2a^z^e_$=@_INjMFw$!m;H; z;WxrTa|GP;5Mq)DXVEZD>lFXVUnL@*gC!%v`oZ2p{7F8WpXS+e7?@@*3r6{}DD#&p zl+1t`ChL(AOWtx=CtQ8R*s!9NaEV07vkFN2MmRhZJvJa~hxgVGkChH()A2|$mWn6R zY7BW=VEPWxnaH*D2J!(hwUvC>d z6YODr+m356_La799SM2NI?(z=Va;NOb(0}@_Au9{V|6h=B^613Obf8q`e3{lj|E8p zUbpT$nQR-LwM3JuP-#C0%F#B{e}GU7v(tU2c`IU;eer!AN?p+6+sXT}PKFCpXykhp zKQ{}Fm@~SaBc;&DLhw!t!Ev&Q4H9U(O;+fnmdTC?K%qs{FiO;w({0vCq`_n(@_-oK2q!Ej!P4Yp zt*B?CnM5?5QE8VGiGuKrP|h!kS?vFXtjh)E%7J4DJLrZ7^7+sUu;LaaG1a><)p@wP z&v(q1MeX1vYi4Yo#Ykn>jrF52AN^|~ZEh@2KHCZw-fO*CzkVaJC0_R(Wk9$I5r%XJ zJbTKOXZO7y#P}$j`zqv$#ROBq_Zc`Jhr?w4l*MmbKEwh#MYa#S^X#9n(%XH8oDD^M z(kz0X_cta4tNh@tkDTr&G>bKx7l8lb`*}p?vsgwuaytg6E!swO+{0Q4DcH%+5_*O zASr|Q58zuecpL0H%97yiXEGwhf!$W{(x4k`_{?NHk||;FN~DsBR3w&;QGMRn{d2Tt z*|2+m19o$Nm*Wfo`rrlJ=mLP-B;!+p+q=Z@w7E;eZCRctZtq$UZhvPXxcz-l@ZaJ5 z#KLXN0^cKW9)iPECjLOO5ta|Jw~`}p``_Gg`#-JpcApKmA2y5NF}YGck`aNGg990w z7lPY=q-;*yK52N1I#EAduH@4MPlbKqURkb$R&1*;8)LEMpG>?FilQzDHrEk>a21_hegd?KXzw-IhQ z*@EDPbHH$#J*G{zHQ+2C-kTWaJhkrZm%ELtj__;f-`_8OjRmWD9$00$`;5szEU3ki zV;Xd@8g{-3@oTzu!b_U!`)joR4WREzbB2rg`cA~Wg=64*2h}bFKBfo3mP+h(&HX|G z+OfIigX+@9q8?lGDU9oOnc4Hr4ms52uPI4o($C|(^l@7Pl+c2mwxyiUD+Q)zI$4j5 zSePWWsSw81AE4u z`XVCOXIOT4NmgSoM9O{S`0jx3K{z}>9emktLr!Y^%yV|oIKxaw#S9-E0+T)*=|UY1 z{2jNomA!{e#!CD0)W}qEWIUQ8X#wpt)MPRPX+a}z$A#WZ5rR++ydB3rhPJTp3Dd_m z8@#8-YALY6uKMx?S;YG6@k^qP7b_~LMy%)$AU&n?6Jq4pj#9KYl%D3tP(c9_jjV!b z#6TcSC|6-<$GM0xm^BHA6eKp;7;pAXHerV_m_hhDctLmXKJsU|zN$;MD zCDJJ*+DT4FAAAcEjrv)bA$Anz=DOoNUfG*PqZ`nY+c{E-Mi*{RN9wHNY-KX9XGypi ziKH`8hdJG1;TBC}A{~W~C-vQ?IbDq|ql~1Ipc&(*2hv*H*YcRtHCD^=%$O<9WI_&- zMcqE)SjeJI+*Y7*u=BF64)n=sQD<9R9AIq=Unt6UjK@$9ZEgr;>5?A(a4MP{rG0^# zO3<|+pqw|zvM{FsWEn;uGv7!1AgdhTfEuXUW8d3>nwtfdbZiCTYfwAJg*1T!bzAK_ zBuC}3w7UgcS>)j5R8IkC)o(P%5>$2!@bI|_5vHIrsHS^UQCC?d)28DJrcy=HN(Anw z#||(DOyQ1e+?aL50@*~4?XEm=$DmtIJH&MinMLq0;(08b1?_noVHB-{?Y|R3g>4~% zcOn7cRf49ZCxXnDv@+)CI3RYnPZ4&WIJ`HVh|Gi%rolGW)Nhzg?a`Yi+A)G4G zGqsPxC!zCcX5AGyK)`7jcu_Nbe2Ez;g3fD?l|auNIJtm)3&l~>NDG2UeQu(h)}q|AB(mi^>0icGplS%uxvPb(Sol0K*PLH7VQGcwofk<2DI9$qN^!EhYE-IiECV^471DTys3k6@B%uq10>0BW(TZv|2 z#6UvsL?)G_DjNY9lT2X%ek**-2EcuGO;E#&ls6*LC^IFlXqp!Uqh)|#2Z5FYWpQ&Y z{^YNCa&^6UT?#+OQpA3vJvLtVuNgF*D7!+9^~UJ01-JXRM8}RhOm4RWgS_M2a3G=W z0GT`9=?0jHu%Pyh?oj)7x10{BeFv2zpq9rP&n)3A8EP@joltw?@Rq!uN+;u^s+x$# z=@QU1((FWE-U#2KarZsP4tGgp*e34M={$YBP#>!h?8YNxZyAfRG;paraRcm9gNRnH zusc~FA=50CP6d*ZIl=t+B4PJasDfel>5!^gI(6Z$JAu4 zMz~AMVKhp{Q*rRT-njdD^!eE1F7;31E_vI+U9`-GyMjW`WZeDe;&AsfM8}R#S-6W) z-tlEP{{#nQ?jR9aGoy+K3wIxI$K5Zu<#gaK>qP*xT$SKxmI-&iPo+9>mjtzhySdoZ zsG5$^?aOL3nLr{2UN)~;H1K|PS%LQiJ6FPYCbbW}@k5o#=p23SC#KSE-q9$tR!fK8 z<|Yy4n!z`cnTyS;YKAOgGo%KJrD1ZmSe_Tg9Rhq|0J3!OjYIs%x-}DgR|=k&X^cn* z{5D1|9e@jkLOpson$1SZCNDt%oMe+cps-EyU@^YGM8vWIaLkMG*#=7$*#$O!~62K9!r9B#9|~av+jSCsQhtF%Yvg*P?;=akg;k zI9K2?_92Eojmq!9Ub*84ocF-_DxAk#*rs&604Xpu>Y zozl_g{&JEYIXf;v4U7!7*C8u>);{GWO0zOF1M)bPp_VkMrG1Jp<7#u2C3WecY$}=7 z^Q0}!5F@Qd_&ph?{}wM+60JZdj>3^*e=<%X!<2M=lmsRj5?RwBK(6l|b2MO&X^N5@ z#wnd_9tGy6^SMaWu!G?$+BQzH7SNW)DO=fM%{ar?V47lG=q%!4lfub~g*QCQV3Bgt z@W8~(*et!86dleXv*S#L?X4TPI^2DTm;`YeR$7jI3{+#^0(7NMm-y>_x2Lh3-Nu z6B3Ak#v`-ksW{!t6iq~;v<;3#!Nhpu;wCh!0T*WwwkTX&4I2_`+M{L<$$BR$23VLK zUyf~stNh$O&`YXLsh@JGdS}#STQQ#nNhfnd4eHCz0U~SX#c)92&Mh9?C!-Wkqt28o z-`cs^EvJ1`(Fq1t6v2at=cI6!ys3a8h;vinv>@pOep9 z3hunDL7H>SaWI{jJTz0w&}>YTMM6As>~QdDXF@_K=wd>0);w$wUY-LV+sUv{b4{&DD#AO4{^T=pq9+H4Fq&htT6%7KQwn3?9GSm*cKy6Di zk@R(apoMj6mSmSXM zm8NT#I*BIAeBbF(vQE=u;>kr>%)EC>GTN<)J{ zMh8zyS30Qm*=lVrt|m!a86o?W*b?OBoi9TTyw!({t8uH(T$^L7-|~(J!etj5k&vV= zDsTX>BW9sU>$DI7pBkOcWhSPg8BK-VPAo|dRoTd=a-Vqr{t7f`+2Hn~W=Xm1xN#n@ zO#_imvrF!mDH^ z#fQ}BJ4vpxCW-@AyCO4UXk(v_X|D|sHoq#J$ z0-S>2_eSndvDtsb*p%y(Q4)sy*ZYO;&ls)XvWLqOpv#P$MiIXI+d!1Y8(>aKTw4 zW@$J}Qql@tqn{(LR!fj5KuyxoAL@}e%65g9w*YoRQLcjrgod=L((-1%WQtVD!`vPa!f_FlJ~4wU^6l_OA=2ikI?EE^0t zI9Y+R)w!xhM~vxGMY@s%Y@a|`@M()j+549jW!KpF+JzW{(8-j{NF_Nnr)ud;IuT2y zz(Oq@Wt-ZAQXPyfa5gNxXah*hZCS&gZSsdnmn+VzYO$&1|)H{C!2g`mZ=-l~3H{?Wwg}L8w$K3C` z<#b@~zfd^>b9uNeC+4z!kOOn+ZW~>T%v5IS9xSpxCB`qN@*LQS-h6|c#e?oo*z%)u zt-xNVpYvJ4v=O-;JKRQoGinHOIWxX3{&^uuh(rMb#qN`&~CJKWCsS-BB>_GS`%klQ$1B(COaf9s-_rc+ae zN}i6c(2)i;qGs?CmZ-BNHJ5q++EJ7@3+Ha!Eij}_RnzHcHbzI~>1#KUSPW73z~O;$ zFMRjGp;JT-(~quhgkt^d3c%N}esUF#^|RHD&k56ZvZd+A7Vbt^xXnfk{1xfmF#o{v zLw<+a@GOJn2iDm`iSbH~E+e6J7Vn3odJ*b><2uXz$8K6sr(vDt*vCLN_B~Mz)kESc;8f`x~f$a*8)@1X%dP9=4XS4K~OL)Uo$%2!VR5#MwH(4#x z{2kF_i;bo=2n`xqeF(4TT)dDruJ9)gvX)(OP=&2w=` zmtajzrj;i}7g)-!^WlKNUF$6t((+M6CqPQ%=y7FQyDqTO+kMvENL}EMMG-uRcuoqZ ziE*Wr5)u2E%!-1{2jO6}`CzSLd zhU_s5h1q6{kE}j5Jw8FI$XJ|IvvdV3f99%j*Uo+7UAvARFB{gnSbWItxooScYEKRp z?P6V|%#UZeC28p)e+`Do?Z=CWrSPYoi5;uqMLC(+`7J)W4pFYIvI((W7~owu zz^TIlmAmHb@F>BSB5V*_a);Pyx14slcoz>e%t)?kSt0fnRH~DW#hvff$>~(QpwVYL z36aPW*gVbARYLkjbG0uyW{69IqXi%?N#ioPQ9@iYawfT{IFPlxh$|HTPQ)c6J3`$2 z;VPMYkTj6agVD!>h@f5~)FzXUParYl_b~ao{sxZ>xvzz9$&l-2{K3d!bjAg8jhLlD zF5M?OTS|;(i5^;%E<(t}$nZQ(UGhfUH>1PH9C4|4Qq0(%^+y-lWg)Jh&NCTtU$Z#G zy@SZu^*0vcVtjYK6V6-UfXZF(aKlZCup#an+!2?qb(aWj_t}`*w^2Eavs~4(A}-qp zIha}laYv#NjlSWQN#Xolf*Qc)A6;*NZ_$YR_sfd7hPj2n1PI>S#9T7}n61ag^_oh? zU9_Cgk^%xRnZ<2t4@x}>$HwH=s)hJ0-M>UvJjdyxl~e@ri)3;?iM))sP}?pYaox;7 zEb5{P2j+G=O(heXN+^3DOAi9{0)4}elVZ=Sv)Qq>bh8gSpAc@ z>rn4d^`5*g+6Lrx(&(9tz4t8+dp|{l?E0jIy%_0TUxM>FIG}Ua7wxzyv6Lcg*!u~0 z?4@r`$nosCcV$}i<;Re;huMkV; zn@$f^dH z(%VO-)s?z_J#KlrC=$IW8ab$@$7AWKSh{xb`dY4v#5MQmgEj3?T`T3ZgX;Bz1CxcZ z{7i{H%%H|0gHFVgvEE8)3~e%a6iF!rUBykZ#(#kCoJVVY=fcUuc@;Sw=aJLN1DO{d z-&sr;uaEFn@mwq2Z-ad==K&NQ5(aCn6!#V|}%Yg5^JEkWP|dZ zpCJ}6)&3=2gz+T#g|3j*e<`I-kI|PHqjWPTNg>s^N(nsoeWLyaaUe`(SNj;WWDRQb zIQP$$`Cz%!aU7D7;B3C{+#)T2+H+OnVeOpn9j75ft%cRqE+88fL*;j)*c*=jjga^O8(0WzW-b*d3U*9)2^;~ruK<;ytaDq<cliFNwwwRsMs=Zl3qF(L| zniY%wVeH+l-`_SU@0OHK=~7yp4y4%Bixk&L?A3`V;^1sl;?uIn%Sv%X*BXgV=npV9r$~b zNWW5lsH+p_x7SuI!BwY*4zMm%2-$8&%(5{9^DECR}j za87}91{`4) zMvfZr`7Zg9h?Kt^zFj`mk@iYT0sCrbX*v*a6$fYO(T_(u&fjn>oZlHQ-8>@>BNf;e z<)vIaz{1BY9AnS*oakmm!-NYK}AwbjdH>u1;TPqZIe(iZ)s( zY8(3c`UW=iU0u_vwZ6-=YAIXFmxnGL9NMz4Z(BK6pCa4vTHoabeS3YRPlO*NppnRy zFVysEfedL0YX`3$s`b?hQ-xx-dXVzfd&l*uA}YHV4zrFQVTzvGZ*&6Tq}Xluf~f;Y z&x6BaEkBDe?l0}@PZ`{&rpK~sqNt7#-$MVz;`CuKB>AG5?o04}8O}a9UxD*&I9&3B z@G&KAJf^Pv#ZY#nW(=@$DXfocVPQU83l~bYLSEw_`p({nmJg%z8;nm%8W;Zw%M~7$ zMo6A$p6e1I4=t_qJZ15VtVccM4PBrjg zX{w=B(VPRVXOGo*VT?|6C!2JpPBUqw8Gg8?&lRFimEITI^>S+DOq>c=+OQ-dJaQY4dw&QR5|d>)Ze!4h5bM zNRu?b0J%rY)$sWR${E=hE`&GPO6x6Zr7?Y+f-k&KG{j!lfx-)Wd6xGYlbnj^!At8w zsW*ytaU0ugm4H>RHgT*+i#QJLVI)U4sY>c+b9ir6BRt9x86?E65BKzNooKNO@wU$G z$e(Y%j^-Zo%;hd?(|T{NUais^AY>1==3Gd$ku`4YzLB6p4uEht+}$1S-jvIhrn9vE zW6|Lq>p>CH6P_vL_3?{)=(JU&Cp@kd#>VxFdoqbwj|ABwv8x-xk;?4GuwJeRU(e=E zeWC#J@Onw>t*?VQrVj~u?|;4>K_0>)=2}Sq6ujWsb+%B$VrXEpnCF{mu-F$B8f;C- zCdP9MHi2_IyUs#p!Q%=hXR(@+PNm5h**KUgMJ-6imw)?C$7<|*u2Zn)xa?czY54VZihy9HA%Yfm zGepXRH~_)k_Q{Rq$!D`H>7fdxlvM^P{nJlw%?x$^w+yHVNhx*BpJR3dVm3ol${jvq z;T=}a5T(!4Vc7)wZQbFf@{ICr1(+2!clcTnCnJmwb6;P{-M0GPEEROo^*UZT+;)S;npjV)f8^|GI-?o`51hh?g`WP2h(SYxt!M?9On>pQED%y-H&=< zG$woPibn?cP*VSmNbhxU9?i5L6Sdp$JN3KHV@&>+A){eFAtwoft|CY5D+bwT>2Yid zUH^^E!PKE>-1`%!{#&>>6L+`{KNK^5o{yl4-|GrX|6$IIBrhYwAM=_Io^){hu2)zh z;&pKTiX2fPzjQO(B0eJ|CZdZ=3OD>M8~QbtXFQ8qY_2QZKRwVdSGGy_wN}8~5{X!()=%xcB z7o2;cSZW^pT{}E5o+_m((UB1{w<18L)r*?r4W#cwW0ws`V~fxcpm`oL5g=IC_0+x?bg^jnj`~Q&w{qKhZ>iJ!7u0(_d%XhiM@;zjI zFkPq>Mv9u{L(Y%F`2aZrmU##*2Q16Gz2vVF5ttGziC>!w?o<3pPK=-CVM+~*GGX2* zUlwKlQiXC^HN#{*GGfV0sm-Q^csdrW65~aiz(>n#x!zo=B@P zypt9j>JHJHCXBwepCgdl1EayA=1bZ{IYAX==tfLF;QEwss4mqqhiaYoZF#HLi9(ZP zSL~vn9tay4FXZ!D2`L_i!^9?LwUab*cqVQYw{%S(o=8v5MUx{jvT3ESgva9viuFd> z?;!7Tp{y_VtWwoQC)`@ti`V0^$_S3RTlbyOEOQovXF$@$Hw^-k@Wp?C5DI%k z-x`c*z`hi=kk=0j(dA!Qzlu^9wBNDYtyv<2FY@-aS!BT5N2mFah0dmVIDH5~is;x# zoUAA5S-fsqAWd8G(`}kYr@6vB$j%#BA$%AgNfr_XHAUz95((N8M2*Y#oL9=%2T6RkDeSEKiniD_D51^=9k(jl?$L z>Q5LgZbAgxFe-dXDdozK`~QQ8;r|t!M=Ip<*ypL>hxEyhA;M(#KVk9GmJhLjPLb`G z?)*5JYs-Ob_gSZu{EwMM@VMmpCmbqK=H)b)01~HV*Ad}$#wf4?860O0=nnc)qB2%h z>C->-*|%6E#fNnHAe{e~NNymut^776RK^Lo`GI=_+S~?kA1{I1FcC2D8+gu&KcbEa z-={UfHzzqZM}2sR-PteE`>G=}*vNKtf4}&3qPN*x^!g|q3Em4l5#%jH4m=md0>=X< z!?$D{w`3_Mj+o!&|AZ$g6>VfCMd9DfYJVyesS`VBJ2g4}fbv3#L z@PX&J0(_vy0%^O?0{Fl=Q~|FMV#F>dz^ABaC&1&H3C}}if#)FoVi7*@0wL_0AK@3v zn&bWPj<7Yf8=J;98FO5|Hj=1U<4HQknTchRbPob6s@mbq2gS7L*Noq0Z0E+{4L4(Z z`A|Jkqa)|3Xo{@Ai8)8Iwm9>^4kTqnAAoPkh;GSUEJQb3?qJN#dvLv^Eq0dVVKmO2 zF9)K}OvWP_I`&B#(iDB|CK5}>D1$emUy0(D4bk^EAUgNA3DFl}@{K-tK{>j>P%f$a zlrZJH?C8z@W(a5Pf1404%kzZrT?>Nni`#^795@PWfwL3N4hyp}3j$Ze*#ieu4h-7y z);20_n7!Q{v-i5?v@3E0rc#!x5}RTH+)HN4bJTVh%&s3USMq6sr$joMPO#kqbL9c> z?~BLm{d_%z|DP;9g&}33QvH7d8D9Wre~Zf;{5Z(t|2jEMYK!k9=35*C-*<_ILKbU! z5PaB7@Kx9M;s3Uo-1HzH4&nH}BQx7y0*5udiTOAoIVa?=sj_IAJuU%H_3hk?Q(3gT z@^;foYuG*-+bTnhKujo`uhXePr!C|>t1T}Ah7Hbas#YqDYxHf|R6I_1;6?Dd3=M}Q zBZ3Sqvtel@x{cjJKqSwu&2zZNNJfV<3?CiNkv5B!hiVm^AwXd^2$V z2nQJIFyrx^`RutV<2)Q2GB+NI0lhqU4M9xWHvRxA5IO@S2A0iC*%)c_46&#A`7uy{ zw!^Ei9X24#eIrZ>1H1%y*$7}5wr9FUVlq0zJEmtNN%}%J?#fT6keS(%Bk;|u*a+Ax zwu~F0m9K_`6iltpo{%Pj&Y|mvQ_awi@>`nFz_}wpm5-w zZaWP`aD|vjfjeAz;=tS8a?%KGkz84uB ziUX%HYv2wfShOJNJq?1SV+)0JFKIdvnHw3+&?WDrAk8=~@sgv0I+XFw50i_W#oD7aR%s_11A@dZ=pD9T0=r`q|Z&1(^@o}SJiYXLTai+YnJpg zXz8-y=tX1@*&czwI#Jc0D2sLhW!tA4Se<7o3V+lEgiN~Dd`F5;u@NQW07fPa{L0o;QqLYfaKZ_!N4X)Dd$BWmc@TZ>F{hJi^kpG9y{D6$zYqfd(!Y;eY{W~s}StQBV-Cq z*RsZ`TT7J=G{CO-ForAaPS!`rPEw@~ zVRXk0c8#c|!7gh|V`{QiBiyCsFdD@t>r>RLMgamZCO4zcYtiRpkGs@AiM!-AWt3=| zhk$`wX2V@Up=U7e1_cN@WeoGV9L8NS)|L}@&!?iDxJz5 z60y7yvWQ9Zf$n>djNk3O@ZASz{?7phx#bMLT*{Kc*X2toD+SNXG)Ai&@Y@)j1(Hbb2Pka4++s3x;9v zf5D^$!x$C8t#CHM0ilEab_lf@q}X6M;SR$WyXCaAzd?wZ1Pt?tTTU3>OGP_j_+(Ol zmG!CI%p^%n=~mqc-CUYdk&J<*32smuH{K^Bv2$Gj19DIPO({uoL5vmXMxT+Hv^iU1@PmjF+K` zwaT_MRN2aw>`T5%xnWJ_y-SoGGpydh%hENDIE zydK6WCz!@5O~(N|Z;t!(zOwQquwsm_c{)7NrR#E8=8B6;NV3Ot2kj1^ewxe;WVt@;M(F*wn*#xnC` znxF*5)lkRK1Z6IhJDkkMaVdW?O}qrPsuk2G8Bvw~x9k^?n=qb7#V>N-1&4w2D11D7 zpPnWtaaR)*Yn^cHWUSzY;-zJP0s&GYqRq`5Cc{e-AQ9(=@4XY58uu?uHO9+ZLB!?4 zOJy}IP^@W>n!V&hs2E^jc6=$D5-~hn+*3qY&^TdjOCgyoI=Cl~pduE*EoAbI9qOqU4nFM+PzVKG3{cLRhZjH( z9r)NzhJ_-h!{lUqQZL6#_)2by6wxI5;g7a^11B@^%Lbh1k;%zCB$>z7B!~t9P}`u? zF@aj+y1ZqB+Q#yXX9GN~p9Mng-@8Dqmvlj(+uaz94R(G(NJxVpfCIV*|IrOC5dm&c z6m!%aXFuqc(}A=1nnm!KTTYz)92MOXXOAMmqD4I)1Gf7GlQ}=rnL$6j?gfyRCiZVx zkxr@2T|X&24~^)DW-@9{&CvXY+#T|)R=lX>JsXermtSOIu=CqmLdWIY|3l+bOSJ-3_=c>O+=BWmV>ww&<# z3o6Eb?24B=9X9b~N z59eceL+vR2eFG)23u{ zwTBsFpZE^3tWu3){GTU%{OB$@3t|zVq2{dx500V>AYrg%&1$H^g3yl#xm%cJ!5Mq2 zZ2h;i;K8ZJ?gb~1Q}@UjgpNE|C&%bQgO5tH)niEHT1^N!y5+B_1?P7$k0TEZa59w8 zf(@RhD0_61ELYYeBMezC6WZMA8ee*8RYE8cNftsX`@B_Wg9? zUyZ2TrX^p}VBH?%;LV@~pQ@(QwCPJm$drVveq$IX_n3p_Uij{VLuO}=I~qdfT}Yt; z#5Ztcvj}q>tZ1eBPP`Dw_mR_5^}U7392Rb~!jivX*&F5`0vF+-ZKL5?rh6IG%3SI2 zSTr4{Z>uEe&|*Yf3X?()xYxfieK(U&P^W=)<8;i(vhN8q2eY{|c=S+xuQpZROa=SB zNuH|jKjV*+5~aLUS&Vz&OTS|1y_sM(y?5(jsvS`Qzg5U6|7olM^J~iqlc!cqFnM=Dlbdv&jxveHz}o zkZ@Oe^AtcQ9GhKhwP@JqGFY`T5^^fNN63&%EzyUlqOk;hKZPy;a@*u`p^TJZmln3k zZGn_R>21wVRn2en^(yo*%_62{8kppz;K4AfKloN8 zP7|X`DJ3Gx-Xk;L*4&7Q6cM@s831x(rmv%6GQ1_iw`U0e%p$>}_p0IwY{leGk2wy~ z@wtgYf({y}xIl+2QP8y|a+oG{960#2Q+g8$x+uNRY|RLuZw`2DBmF{g)1mS{r01t* za?uRICd4A~bTokxmO?`qEE^ciTM^N~U@|a1X69(0!Bn14D(}_=nMdOtLVJLw^V8{> z46|;wMJ*NHAveRK5Dbe#Ff3xShihTzLYpK|43g={l zP^JTiX`4THxJ+N-P?Kaq!^VePvBx&QQ3}{_qeAF?sNex|LjM3?!^WSH$FcEmsq`jD zcDYd@Wbcaud>t_hg;|@J?-%0ZQ(1j#dVC_9(PDA30-#$e`CD9#Sj&B47WH2Ac-gSl zMMf$+?y|e2sy#VavosD(1MzSltLm#FJM9n~(QphXjDLL{<-1T!eOT9{WY9&tb3p!Hrp#3lL30uVPzW-Pf;LR>QOBynm+ zT=u?1Zr6#p)F8Ye2jbEPpa^mEhpS{=L_*vY-Qg_tNEw^0K_?$_a4GUU4188Px0 zopFI&BW7ukOPcT5Qereq^w6Soh&&S`v)eRAS_=KLxLXTxzmJGxj=0o27x^3-=3yuw zLc1))71VhqBkosfcu`KVK^utsAms{u+d^E7?+^xA=wIM~%Ap_4kGS7*M_g9}B?sdE zkSY*}%L8pW5%*Cl+7)p}q7jWgMn~q%BuM4G-A;7#TS&iX9{1mu6>$x_9)SsvKevgw zWM?#6kB#d!RU<>6DBWg&6=mrxZc}?ussnO~$*olj@madNCK^e_lXNgKf<%i%+&>{N zBku3vTQcIh*&A8ZMHLRr?RLg39d+|lqwxgH=;{1XCYmu$t$CsDicYj>*-+QT-pJ~o z#9fD$gR1vJUTzzZ*GZ#iGWPy%QP{geAu5OdWMMBx`igEiU2rf4R)prq-aoox?+RB- zB|G-62v7x#y<)g6BlfO1jf!r<-h81rKB1Al5wW{;ix>eEQZe>I+q!7%U2z=S8-+GD zsko@r5a=6vJ)Doj!98V^!P{yiv<1nyRo0hQSc@DQLd=!qh!ty}$GfqtMnX+rS_xf> zGWK}n4DCnkUReWIV|SW%AuamB`TWoUG%5!C(-i1ut{Lx~zHDLIqF6q2So$H`_aC;I zAD9EhZ)=a0(>})hRj%L7vE}9HSs(rNi<4c({`SmObn@`cFFyM}-!2R25~h8p%5(^4 z%>h_fyDpU;S73N4q%SySaEm&Y7@v)e&^b^wP9H;!rPL^}*BYW-kLgObx365+D|P*P zTyS$yBzjRaa!^f=$I?@=bnW2vwOq9}Ue;^w(FbeVp}JPeX$RHo2M5%AJ~BH(=NjU4 zOC(v7B$K_B5_B7&On4lI_MbP5SpeEB!A(sV@Am@K;&KI`1vIr#Uom*+|3{9 z#SxI&TwG1km3I-kfAhS>FpeHadR_fdrLa3r}GZNetUXAV`$WcCwH;PdlII+#o; zyfe@2pRB39*4Lnu0!}%n#p5s+b2%R9DJbB9&H8MAmOxNfZHaekz4;3ZTh+NO*Mf%Sb z<;!10ZRDr{pYM_%iAed&;oIdCfJ?cOQowEz|6o$BT*bjzdi3KlM~?AHN}ftQhz}#? z1~@O}BC*BwF+ie-75h2S&D8o`+Nd5LC>C;);jA8B$5!+h=*lhR1TY48AOtX~F%V9s zfiR;^);HHciUp|E0@N?WlL&hDkWJ|4uFGJa4o9o?WBs^K{;®PjAHR4$n7s79Sw?5&S-)rx$NX5?sNn$6`;<|nX5G( z&FWmtl;fnt|9f7R;lU~@lS5{=?ke!AQB0WXVi&Q?r1^2xogG^?tJV~h zXicJ<=+K=yI-t&7pH+o)qcx4AIa|LUnPv2FzI<^AvRH969iG4wc>OLv;o5W+h?$G9 z{8j9JyhXri|I|Mdx-ViKC_yJUz|PxeYfE!{^bXj5{z$F#Tq33)pgbR}rqo!e_mrHr zWYwUvouEptB*Q3AE@-CBTxggeEmjymuK%#z$`#G>&mgKWU(3+Wty;!7Dv+yO3)+!m zIp=&GGTC(h`p|>(t6`M-G(67e5hm^K-D69l-1}s;LbPs@gT=8BjMq#<>H6#)gz@`9 z_`dhC1}UJa0pDikWDM<(4Pon4w6$^WY*(*fqY@=fZ9!4n{91GAc~x6Gc>W!`<8gVmz08#M5=-D0jA}Xci{kItK+&%L1@87Lo+NE=3&bL^OTW zKy?LW^>7iypIK^@C`+#o%9p^vjK56KS)$8PYE}_j#4;zRAX~;dyXm>2ry%OMs3>zK4|kTFD^<*bGKBm+$arw_N)n%;LM4jHnS(aO zfC$>|uR6YD%pxM@it*?JQNFX3Kh&5vKy?bS^V#hZU6J#H#)6$iNLP8>B=l>}^Jd&# zk`lwN%>U7!f&ifN6E67c0%I-^b_&#mj5^m6uRTQ(K->by`XAHwGMWwD>%3J&8!L5stK9;^?WBRral>!5?I>qVUf^(sW_8T$9C>p7olRKutZ-aP{1r~D zAIOfjg|Czxb^FuR(i;1B*T+WrnPn4C)nds}$!|}RnMvRC-2~Q@ecw6^uz06rd6NsX zZuc8MKZ1uaN?P@{QrT~YgxB}>iCq88gXNX3BTEB8IwGkII3p5kw#ggs_tme9#)=8u zb3CrTRQmadNg0Qms)dCxTkyMOtQ{o#BoiZ`?)~L)!=|!HDg_XcMCR1Zhc>nD=gBIE z&7PK~*n`dO4S5R|DBq|e#&9j+Kit{Aa!&nDa)j?N4_Hbucl?Qe#2^$f*BBeCWY z*pl4Ur;?rR$L0PAX`A`XHKgyI3xbz#Y&hAT7j88g(Y+5;!enQMH*)3|?>mAm5K&}5 z#y@RV7XcBZFcA*Y0jOvsSH-1sW&WeV?goV@p7LEeS^*SO9ejRc6+dildV%5j4#L93 z6xGVs9-~BKzClrA{Ju`d9oM!DECFQj5`h^>W>1&y8k98tBpqldwiQT6R`RHPfl?kC zIIBB;F9D@pf^U5rTgf^;W!00EXbsU9gRJ=$G!t)zcoESQtciZhU)${87ry64j)4+H zhx9<`#1y^rB>iAmvM`7Yr-SAkEbA6AQ>b>(SAUZwe;mK!$MMvr)Zz?)o~*Q|SW~wS7;o zAxNj}-|vWLat?5ENDs6?*qy9%-Udg~{4V4z*wMpkcx!B3SGP+HCaWwiX&Q0TAd+aJ z@dxUW#+1w99{2q{GJR8~%*V`%_+^cdpmycU$qSJFjz@)&ov?xBl(={&e0BI=c7uAX zUtR=GBLsY9%5s-s@mr8KtYADeR`G=-LR))pm+Lp*-`u|B=GP1XY0zI5K%wz-;m$cig zyQZj<8LI;JEkZ9{po`}*Y0m(|wF%yT{7wp`@qs<+HPbil>g~R!`8)6LYYKJkdlNti zU&%0>1p(=E#!Cp6ULu!|3+k6do_C#`gWXU7(NB`Lt_5YMW5B zp0{4Az?5(PqRFOWY()MRV-#De^4b)|->$*4P^Kc|wNHc9UQYAbr%mM>*de!NGcimY zubJtz?ts{;y`L8aV%|(GZi#a57hF(Ph=r(ABPT0y@|ePXu0X3?7a9nNIUJ-pXO>bN zu8AW|P6cS3l5RAVQKKp<&=4AzR=`lJ5SQ8w(w*!cZ=wIS%V_iHU;2f7<|5X+@jWvH z^ZmS0GnMyO-sMfdDUG^go&N5eAhxN$YimL~%3}|Vn-{cCJHICBNZ4E?n^r9q2`d#< zc1-Ibz~c_0#+jVg(Wr@TW=ZhmNa~sQ0>6z56C7!vsekuF&;2+k=Q~oUz(Fn*4ii^T z;W%;vErWzIrC1~0$YVJCHOLF!C$p!p@C~hzi^!x~Q=740mghlbWb3F6G4+J=UaC8z z0>e%uyv(EhO~>A2b1^s?FVk&AC**+00TpRmC2XYv#_tT?6^XE(_qNGD0?FcM`)+SJ zl#7M?0`|dl)UE923)oF!GCqS--Z8|A&-WsX=N1U{4(?Awi;hb`ZU@3|hZ5g~$398k zr^sx9RvltuZs|w5-N$UBQ&VFz8#Q*^9;ZrhNz~+GYv9JK6&pdlmP_n0Q#WkJ1}UT- zX3t@CwKArJf<{`3T#!SztrSby>Zro>Zw(bZC?3exOgkeTP3-1GxE+^Rdu>$m>ZK=e--$Iz6G>oo8bF`g*)9D4A;As^{ zmrw^?*FeVkw6dqR71D)?|1E!8B3sCYtB}4FM4QbcZ3oV;V;H-%n+)?dVhocItu`EX ze+MGcPMwqntx2$9!TLWb0we7XdshF6bNFfq2pK?f_^RqS9p6=GeO}ff--~YNenC>P zGobfdk|nFJh{^p`?n|gPq0+B~?>CIra$P_jD7+V`<(3`HPIbKrp}bi;c!iyAqYO(O zSMig#{i_nT*VpCQgCe(Bo43Mxg3Gw?r4`~@AUFGE?V?^(X&+BisEYH|+)J$%4iUL? zTz0eK_p^0VPM4G*;~1U>X~#nHy17R6{s7Ly21iHLw8-N$Ot_nG0Hr4k{ucgdL;jfOgu}@@EUi&n_m=r@y~#spUh~OF#AGMH){?qH?w1s zS$F&re!0P2e?2VpyJqu&-Oy{|VqA}3-idJ#J?IxZ!W%=fj&6{_i@V-h4Mntdb}e-+ z(rnVam|4WwQhK|+jolYH2Cb`0T8QgZq&z+sm3FWXR6p_bs$HKwL>aUcb#f*m4>uV4 zqbZAsk&;izDqWOjlZ0@R6zS_Pv6!?EY^dI1oFTan0`R&V^TLiw#8c^``A&YYYWe(p zqTk9OB$G!x>&&O(5U8A1>w|PaQH>xVRK{Ss;5o{64Z>3HU&jgJgI_c zBac{tI^r21M8${J@infg1g1T5QRY3-crZQKn38$jU*u?u0u{Wgc zgj_fbydNkfXSe8NQn6k!Wu|TMsGgm@WE|!@T==BO3`=}Gf^>is6+s>j!_uOP?htav zqf?cy9OYGk#?lt`^+Bc(NOy*g<~vsMkdQkqt;;2k_;+E|&oDv#7h8F~_RQH5R{^AH zyQJZ2tu`k;a$Eh8N7D2OG>rJ6hP$^#MVo_8e!IF=9>}&i7e)G<7&IGf`tJp@nKWS4 znYT&m-6Q3MiPfuKZ+xMky-E)PwMbP9G-T;}x-62!eb~C=fJ=6Prp(x#Bcniaz^)9LN9!{!jZ(Kw| zzRq-Tk?CstP5G7~8v0xRgbr>0^{QCH-=Oh@@bgwd1lt`T(Dwq>{vItkva*Kz>Ef!h zIA%!J`lDJ_AP|8(c85!Pe|zOS!DB-Li!kxJgyVL@BJH;$JZ!h@ZJL#bu-ewOHhopa zRcBzDL8W%?l48J-ghkFk5ca)aXK}6)zQ-ud@TwR07s$Ja|c?ys+&fV_`SIO>u&yR4O z6ub4Kj}#-Nz!C8)!X0rY3n)Nk?AhRPx`U4fQU%Er24#+^2=W)RO?oV^!>(9sU377Z z4-5dy)sw&RrZ)#=03T6($Y@>4_H7Vnxjm85nySPsLuZp<dnHm#< z5ml=Vy*RD;K+zi~RfRY^=#i9rlj1FvNiutiLpYX}NxHs~RVAts&RuG0AK0dV;e{qL zv0QukXQvhaE5oL`Wh0#JMaS~fE6 zF&Z}TGS$?$3qgH7@U$s1-CQ~FrWTG2Q&$oDyiN!a?I1yn{8@ z8M$`d1sLRb!fKNaOACpX&AstRH<&*b+6#|xa5vhi4%wp1tv;dyTwF>bdLxVSdg@H!aU@^Ei0fku&}HCFXZjH^ z;JHNsUJkT2Ak&u|n3QCM+CwoqhUIlT9NQl}YKu-q z(2OyIOhlmKYZ5c~^k|^*mk{jOKt!kcp;F&_CktaqlFTq+D&|DKi3>DiE?Kab|BMV? z7S9p@P(ET^V=i2u@f){i&uvOWjU3C1PywOSFFwGy>s$4 z6nE8ALNbC6?tRXI95|b=I(b{2ylS!e2(gN$PV1&l&M*Y{moPmN$rXwAf^N3X*D$?r z6g%*|QQVVc-H~{n>k&ay3lMqs2;GN?^58v9@YpyRG=9MW$Ru6%3lwzCgT6B&z9ow2 zk32$@C;~`1V-769QHY_x?U?qy%icPM0)Lp{KRaMHnV_vR(&?1ZGfUAT3rIurv2;;` zV}^kRepbC`A7i_^h#z-2oQi(4oM5DgL>fz~8dJ{w;V+h(->5GEho&oK$SHi08{t=76%}Q9OID4f(x{n3co>sLmdr{H9&GBjfPnP7qkjX zo}LqvSHT8-c+D`oOS5oq-;+u_3f~ykNzBdU>ZYBBfdohTc1)Rxcj0?EYwUu81U~#U=y}fz{_c=s14X- z)8#6DVRku~%7;`;Em1-(d&a|3#^vR+B7X3e2d!2=a%a);Mv1oW@J*a%*A0uayYur} z3tS?0u+QL@+-nz3-=oe*L=8T`k_PTqq1@&$`I!LNXI2FX?32}=j9yGh`m9u%`Ycc- zVH$fS{dX)!cE8}3KyGTvgMjSX(K-`d$9Pw>t;qx3Ah!rZ>FH~Wlmq$zDDHBESV+e* zd#hhIE^(_qm)4THmZ0{+Xh5wtOnjuRyleNx{F^zCye*%yt|1$Gm*6cwSad&;+yRLo zkAbT%yZ3f%$pluh8?m4V&u;7c&xNk5c3uVWKrI1UU9DXfrUu$$2rP3Pf0FV|{-{5yd$K2@fs54Sw-i92zMPMX$MVR3t zPe>6tgjFiR^S$fwzOkt`_Dtaw>q=(miO!V0q`a@xGT_U98`kihHF}pZv2=?@D}?Ur z<53FFiDVWma`KAbvq&?83`F$KsOtjsDqJhDWewLz&O#1f(aaXa4Gt#v92=Sy7M)N< z6N@P3@ZH*V_m-z=MziINurg5H^RTc28iQO*AIt!ygHcs8EMKUP`vHOG$^<!;{eT0V470Lo?FI0(Zr@ zS3!w#SKXe;Z5>PwN^yPK3UGM74rWE6rvqb*yY?Z)4M`Z(`1f&dm83K6_yQ5&LlwGt z0>YRZ@X*2)S5m9LNE86+#IsP_6DH^x2unXA?auvjY@ohfe>6Mw!`Pa}uw6?AQ2M5j zT@&ug>j*9&$YEa~MZ~D&C0^m%Y#)Kz5q21AgynZrOgtjyG*9#tRr6v2G>gc1(1>Cl zTg|Qtq)*a6UlMTW1Rlj~EtN_V`bH9xc^2>|c!A3m$x<4BM^{4qU2&;@)yC|-EXoOR z&w~zvX84ElF5M9}yePHoZWs<>z(B85&T-lsB4R}(f>!RrH-HY^s6asV8tKUCE+lbO##0`b(9HNL0vJhJ*4z0V~{1mp-wDsY5{JC1GRt_ z&PyEEbcI#c3KY(~xIl&B5@J3xn@RhZ8ziVILHfEVI3 zQZu*%%+w(T87rrhGGhdVAGUL~tHf;g>ro)U%?Hrl;dE-Uv<}iGH1!8|$bm){CTR8$ z98D4qMDLHLK?W-6oWvn1We(C^!7hFuMiw{hEf_@HcVDUv*p5m!w8!qsPn{o-Dr&J{ zZiy3TMzI7uFhi`)y%6wg$2lQz+F4&gftWu-61>2wc&QUsXwNndEPv$MmL&KqFZ>n$ zPmbv6E5gRWZA*~B>xYfOD}?Lnjc36#!T{9e!Ct`L6th=ZdAGW{ZXSiyAx8p+2XKwg zus~E-nH$i7?uk*xh&BhqheIMg0!Izs+o@$6yX4HsN4)vUT5 zC>yo+p&B?_T((LL14;L+2Uc;B0q6Cm>5o#-*HTPdt?W$crL10nXMBMq6x2LwM10$g z;4($XIe|L+w9EvIM;&8kAyx!3m>f}|s;_&N5^@@_0ivvSQ8RqO7H4T~HfQN2tLjbD zS_+fZ6=!-ATjC^_mquqfuv+Y8X;O@)d73>0gi>2tHkf` zi>$3lCAd@!YmB6s?{XC#Qmz*+$x5{dL62;Sr=w3~@j4J>Qc;qNiet=-R&`SFJ(q-4 zcQ)G6U6*(?QR&zX#q*2 zrY`ZtZ$wk^EE(X5&P7+ub-X|rUbz94cdj3hwY^ve#VD#nGX@(bfv{<+jSqy9yUq9M z^eFzc5qRhUC@WzSxtPUKnA&-p8LMlyf8txG;XAhey<&;bmwKoM-7@+m*DlEMY36)G z92AZ)=k;Xy+Z2%?V+m#m5PoOc5~~6j2@~t1Ams!E+!3>>3lepVS~@A_7AYm_P<0uz z;2d?K7W%z!m(zp9tRM>c0zhSW3g_CP>sr6^*yd$P95S)}8+NfWTDJ>IC}mi zoT8skkO|mFk+kz#LD-}g0qJnYrR19Z}{Z^wZ_7%Tfz1Sl3-~f7i{8Xyc?4YKfd1; zG|e>*Zgso}nKXpcX52Z*nX33K?dhJa$c`hhy1TQ8mH$m(++niP;Q z${5IiN#B`rUna;wO~;0jMg@jr?3g8SPcz*~o%PjexksaJ`7yEv^b?C@N6RB|@v+<^ zLv8U?A~DaIQ|u9GIQNP|yWlH{N3#A$XM=SSaxkPGc;7R^na=2X=P9&JuQ%(O8GHJ$tR;&2ST&Qx-5;0 z*{MpG4AW&!5~VAHvZt$;O2=Z&d{7}gxGP~Cu@wcCE$EjtwBX{5{yCm8DI)2O_1FAHDcHfQ0}QjZO%2_-wvad6 zf%?F~1`-cB8@7J14j<>XAJbG@0*~N8Ww1>?%ox!G44f|hh1q1_xYdtEqv)0yaYC@^ zR+EuQr$uzm$4P%mQSO%5-@+_U8N^`%MYx{LqPzKeFetn~BQf!CsG4ET&6x_YIwoOV z^C(R}P(z5%P>~Gc@Fq)qrij8uzD-kW!-DZqeJweVE*9=tc-V%$XS`&pwG94@aD7*ACQAbN(yuXq=TfUF~z)k z9IN^u-2K~XFj9+qyWAb*zsfUgAZT&>_sfe4jALKIEdL=mr(Y{I3Q*fGhwoJ$=Iu)R z22K%#^b!v_Lr`TdR%i!@#2lAFxaFC@NZ~L?j)*T=FW;D+9(Lg$VQF3eRvbNH`Uv0o zE1Jsq?LEBv-Bg$kH#UIQXWdWmLO5iHIzI)({bP$irjl~V`d?XHSfjHQi@u02aKA)1 zU3_(%Pp%23z?J4%rH(~dvKS5t7h!;p6u;sC!P-^TwX!kD7Ju|TGLYX_gDfC(GeVFs zCY!&AZR2plTiA5j-yqvrIU-Mwn5Fr5cT^H(*7#V#|U=wt(p4*i_+JK|J!U zS78Q)3!SjMb~bRHeIc3K6JhP&XUX^O6#>)2ejE5+KUA1r1h#-Kf`$R#!GBZODu6fS zb5%=uyx1dMsCFG?m5ne3Y`0T zmu~Xl$-@!%WZK1fo`^fPBSwP<*(;=>->x&Q9h=EJqAG3wF#xRGP*rB<(Ue}lb{$sF zLPV1i1@l|DHs$z}g^40ty7}-(u_sXKckkg8OzE?NllI@SL&}&U^$0`EfC8|vu;?s1 z^UW~04Ydr;nsh3{%L?=2a$k)Y;rchs;%X?>9Nj+(%Qi;DJ4S>eH>~>k| z#ox8)pKnQI6>x2T;^Y-uWTWXRoQ95|Wee{8;WLapOdC}la^2+{A5p@7l+u3g9GqW! zVbtl+z0#q*k_6>}#pFIgUe2X&SuZh1Bn*c5hLPr{^vA%i#U4isai)l?x%gexNG|{| z*h9=DlzI6|kC#o?oNY*sBijzEgn^~ERpLx^$Jp3m!JrvL9>}r*xAipwlwb)sq1?Hm zpyW|46a#pF`PYJ7H$-ICM4F~o9mhy+;dkDQk+Qa0cBxs)wO|9>{K-pJ{6zCoj;R_& zn4?4-afkM|Qia@p54^dN2JChSK!MKx$|rPIVjQIb&yl`Up_-fXh(JX5Z_jjk*~j4G z)ochu&aEdiZnG@nx5VUl`5;L)dTWQ*n`X z9h59XHmElYyI-p(0+cRmK*WWU$S2QzIi~rb@S@5bqC(m+Z0_1~8p9XFD^8OZ3GeDm zm2wLrZzV#nV);bO1uHu7X>Q2UL{JxND%pwJ(TBxs(Q}awUHHfPdh5q?;$mOKJ5ar; zOqmS(2{$;MgsG{#q?gv1AyU5RcjupGE7J%s*D1akB>WL>dZ<~}JMl{l`0mpY%dBu& z2i0gE2kT;C#@xZT)B0$VDuy|YG%+by4OlCoSV*r=9e+! zM3NVJg5#9)!Qyq z+OCdtSMEr6|HLz62TJl98*5FYzzvwxEhaxRKD$Y0O99nOAXb0GaU|0luw#MMWySkb zf5kZJ|`Yz%IZ^0qt{YgX3olz zVKWoxYBa*>U|0s=%jZ^Uj)N#%iP2laqtPLO(&?QGHS!frVPAcmg?1?}x%tn!YhhDe zo`e=50v_z9X~)&fa>MR5#+Mq#n=6HBy<FPksXiRCFA1Q=hc zFX?qn*u|})9cG49#NgO9%@;2y67qxW(N3^Y=f@pJM)ATCXut(nF|o#CE@=*_q&cU6 zY-kX(nYOLR=%_`zoVU=W3dU&H5-KN?A1|6EaE;v3nUHl`%*HgG#8qpSqdI6iTw}1H z@^E-`3+9+jqqc6C9=lZ#O*g9OrSyD!vv~QZa|q04VjE6I)*cQn-yGNsvs8pPS(cly zQMYn%Fhaa+K`U%$PqP_(7n+vkpR-ECCY+n}|avQmC zq7Scp0T$e9$#K?D`-XPFTPAMMj+-xB{VGGb+e5$iK&Y*cQJSlS?p;f-%`Eb9tTW>N zza-f1AQRtmd_<0d^q3K}o-W4Ile4QqQm1{RY&m&CVJ} zwef5aF}#enobAH~Vx4x!P;Y(F)+PsQB^9ksYTi43-$IOJu7~iUx^UAoUc7B7*F%k* z?iO+vNr-zIBWt8mQ@kx&@O6RC+fqZ|rsXb_KJSIjOsf4w>C$?tbD}nrXX_nXCERpi z%RZqeL$b2muTC`ia6%|6FyT3Vc5JtbCXpTr0JYsY)^TM|e=l-jpp=dX+aJn~ z67}iVm7mMs1^0W;H)1Zf2IJ&`XeO?aB%X%4y(J1th<}}?i{J~M-x%U+r*4|vfk{99 zrrFl?u$+LnBW8;>WgW(!eL5;NI8veT!6U^KDeq4}hY^>+yQ)i^!1!ZchXU~~x)g-{ z#y@rHW5UhmaaSMC3FRHQ{|czZJ;F=>WM&0QF%TX<+@Lup-I>Er*+GS!KnNWQ-eOI3 z{b5lzN36yDA5uz4M&G@JOOcJLH%Nzt3S;8=sJ+x+HMn|87Kqw>(}j&*a6)5+*id2m zkCF<-j2?X(sC=XNx;Nmlf9Hc+H#{jzlo@n*&1GL*H{~zYR)i8;|EEPvuCL+iRaR$@ zovpP(4<4bWq5(!4Y>u%ZBy2he&?ZKc9wv4x6~zyIR~$peHAIBov&GR#~+|e6+i$b zN4OctiTQE|WQ{wZUC?=GhA~wj8?Ec_9+<*D!|kNwCEi2z$`31v?94H8Se)^ zd5_g^ulo(W-(K9Mj`c2mTH};$5PLc|9m$zm=d$`A%O;)fEYLOjmh~gd1snw>n_|UQVHwmmR&Wn_`4YD;K3l#E?hHb=#I{b z1MXW9DPqYDpoH%~TFE&m1=J+?fKz{JN3%X9)0Dqz@viFhJwEw2s#K-EN8vw&UQ7nO$cgw1-r_cDIhk^X#E`Lv&JrN0e5BJZ=F&?fkTj(x?z` zooxI&mFJB~!b1Vs2_nBwNlb8tmE4C8di6gSHP9ax?I}iBEjR<6gQ5>NCbegf3vR65 z<|9w*=n#E!Z`zLR^w`65G`D^ijPMrdjuT>$+`?{(W`X9%7U@eGp8o;7{-w16=6h-s z@R!$FhTm1v;JzgN*Z%-=@3gLb7&4(meDM-{U0jw!>Qld=CW0aI_+2z zB=qd{wc4@;-TYPXzetr1MShzlz(hpR(T{|8qsYS`!tdH#vSbOGB#frnjK+5%^kzJf z`W<{P`JW#dt08uG!Q}IR9m;M6|F2RlBDKnBD$C*7#;)~!=$61Y4_#(n)y~|B-~pNb zas=0RJ#cPPMMdIqAfk0#wd?D>>~8Lp*@|rWE5Wx}U=q+W>+nY_T7Sa^PR08<-oYLFi0R9Y_|)qG!i+R*fVU(PXP5 zf4H0tV%N|cW2l~%Z z4--)NdeCDV+~%#4c5jxdRV_9@b_Wnl;_IY5psMMX=qWLb4C_O{z)=- zK-|b%PyebX@a+f1_nq%zEuc-~8p*W=XUG4;PdocblW;&0fHHf?jVz^y15`YRk^@OW zy8JFc2*PbYU9}wKF_JI(1Lh;g+ zUD!Dd|Dh?r_}wxu-ACD}IJ||rD}R%iz|NK{MX|}?q<%P>{8=fYh)NHQ%nuo`IDL`2 zMvHu)?{9Y4z?}%I%rEDPR8w=H@R&II=E$3la4DmDuL@>p$TP``2#;>eEx+ z5Ma2hU*iV-(IG(GY{-MHu{U1w% zn%R^Fc|bei{Uwd40}FK>2wyG80o`NSw!;hmH2$@#uhe)+`oRCiII(BA442^935NaG zLd5b=G(&xy3lbb^UZGno*4vQ=?9uEG7dWoJVkEJi|LrJpg>Y1WZcbxDt+9Y~o3oc# zuy^KN<__7j^~`=x+O-WAlZPA8(PqN9&UpwcE<)kmvwa7dJp4EEBa2y=k`*muIXhcG z$636#9pm+1G(BHbLLTN(a#lfJe z6YcuNbbhLe70e?vt286(NF56;?6G(GGSOebU1=$5nZyDT?1)a1A!b7YGv(sC&Zs^N zIi6Dj?_Y8<8b=jz5WZgo9}xPYir9}9T&aZ`k%#&-pQFM`^Q(GJ`aqM9`h<$a0+`aY z>FED8+`MJ&7$yx||GAm3{J?ax#Tb$Ed;2qk48*|gb2CHd06ewlzvmsg7o|oIYnQVo ztnxI}e-9$6NHh}PISS)$2LU_SD*gyN=v5~!RgUg#FLfSm5A2tS%CS?LYajxt_N%xz zE}QFDDtERd_%u*6p~P3p*Y%K1>SD?DExR1FYoJ6I;Xlct4_SoqnL?UlxN1+?Vw}|5 z;6IW(Fw>W|^c#?w(fs3OCed2zE@@NbCpG{0*0QOz{O(GBu}N2c463^bK$Vq}7%Y*9 zT0`p2E5&D`y4Lg0^c#eK_uoL-zR4z{KO7pijxi;XLgk3Kv<^tID;m1Fl>nswtr2at zHjAP1f@nx~Y_Tn(yGi<&x0n}diWn5A6Ho#fdN41*X2IV`0Se;Mi5^h|xUS^x$kagi z{jcs^^@EW{_dNDwpld4>y*v6w6siM56%bwEE@tMaaXSb~&NThIw{L)bX?~WUH5D0jK5g(_XD76SErW9kD z_K6*wNQFhFk6_)lB>+a}I7nfY?~;=WukC?}&V8KwS2at@eLvFuTRDL^?-Ay9Gwc7E zpZ+1Sk;u_KFndq7Z`&ZUe?R(NPc24CJZuLhFR8okDs|7qeJRbO`0)}iKLFLGI6T8L z`qU*acx~~2%!iSL_CNEvvY*{fod^qp(cnJ)L5DzPn*~0qXRG%=9~qG&dh7(o05Y=> ztf@!ipjFc+2{<&T)138Iz$3B$$J2Kb(Ll-+h(MMQ|4M%aE1O()in6CtjCMRdH-(RP zpt5LZCdnRAxZbsw-lG}X_R^gte6t@KOtYPBbh90UFrV3f{X7mptV&G3#%g7yTb2)j zu9tJUaBZ}9hau(miaY7&bknt^u39Q-xqxJuU(j~f7P112)7QjDMazaDfjMRD`G}E1 z8+*pnQL$)bDM@I7WK2?&QQ>8QSjFbu<4;35)aaR?0yprXgeW(T`O#$S#crSOOe%2Y zgP!&u&g^MMnz38+#=PRuE6w>`C{~W5uI7rf0&L*cRblD_(iy_ICeeoO?N27PW=qer#%feW+1y(ej#y(LqZ zpgC};nxAU$$@7Y0F`r^O=EmNJ@E^RH1QwdQOFZkI?Y{~eNYm91R$^Rk%_;Y+z78P? zq8#rEh?_B_iX7ZFs3tAm!>>>K*3uq}ghgl=nJ_ZRfm3LlawHz;X4`1<{nPy5^WPG9 zp!48qEaII_xB68#T034wV=!>tT;eCtu=0Ok#Tj{$=y0w3Ff8r$rCqA5l!$>aJ}gW= zS-2|VylTNvK7uy2yw6%b@h+hq32xvVe{lY%3+rhLdF^X3$BANx(8doJx=FqvXd7BP z@V$xBx`Q5bF#%=8+|iP^$>LWZAxG$p+og6`%b6O3Hb zy@(&Zb1c<&Vn;*`r#kyQ2TUuU-6PW{yhiQ7PErl2#Q;R7wu>;M;;Z0Cc(~@t8eX!> znqcydw)3XXEIhrf^Vhe$-SkGQp7mZMFkNlgXW%q&AR# zSrke3-8})L_W}clmTRr>x6w^V64W{j)V~!*^#yKw zaWk}?P1l_F2a`fm^7t92rZHChAxRF&kc3i z2`7u)Fxs_q9TQ#0>q5mVr`f6I8RfIy@unQ;W3QE{EMqQl^@#pSbBUXfMJ!R*Y+o;jyscg_)**cw$Ws-U?u{)rOC_@*_WUam77|xtWDNH$muZ8?Q;jGH4UKu z_oo<`+J!_KPMfT$_?lF~$dqS|5WA5s1Or||h{X%O zu}TH|Kju>=svaM+Y4Urat40CUT6ZnpxJWGT?~%}Aj@${;B={S;D0U~aV2mi1JCG zaR$|FBT297IZaB+d(PGebkVy5rtE{7oSIXk$Q_6U2Z5OE!$u|D#4hvr?I6K{uU`o! zoxONSR0o}IVatc~&&9ao`~Dw`oD}2GWX#9{2{K448*UkJZXZ}!t6$DV|LZ)eOmg_eSJ+c#}?qj2}kj%s8;ju7>!{m1EdrXiVAzdc7;rks$8^XC65h64bO%Si0% z{H0=qbV(C}n@+w2A(J~Vp{;s@m;Y${GrKRP$*;LA>rK;Z@XMQFADoNA1M#K8b(Q`< z+;-?G4JEYAAu~xiBmq`MSi_+-?mhT##K!_iZrOSmTWz_D4Sh+@(FvSLaOAL@kWWGa z9?JZOM8DN4sI3zVMKVy;b4r5wo@>VRKYPOgbH#T5Jd6E)dHBL^F?XyS%>@8xpn}x6{cE^*XZUJ z@|k@+T<}{A>-J~Ve?js1m;GDGzo5xlBJlA{uOv0rNfDa0=(N6YhR?m^v4U1jyv&3o zb$XY5I;_=_)}G_+3O3T!{ImL#2XPKLp7on_%GHRbf(5QxiH3G@FNzfU#+`cox`&S8 zsohId$Ar$Cj&ibBOJ~WuBlY?$f-HXe9ixH&!p3bA*@X^AQ&`4Hs7v zt-HGO=?GUm4&FcN*A(u7V{2_oDXJk&S1M92SQFKpmD?&oJJawxOp+b`|#Uq+n8C+zod-yh|X)--@Bm$Gly_wm+35aF(;nV&Gr6W)?a3e$GNehr{ zJ`g3DSVbuZxOn<1{z@2kWz47kyD#O0rEE?k%fvDhT)h^Qw10;d3-d?J~7RPB7BNaza+^TTWmDlwqpnj^>gN;N2FrRW88U z!Re`)t5vT^0Piq6nn&}NmepF7&M`CxemFcWM7kdv1?Wl~2zNh*^x&Ak(3urv7R$ih zv()Hu9l~6sMhp!>uBh8o|IlD~|5qtAClwi%u?UG6@%u8Ise@{|BM>HISZU0RVn4LU{lH diff --git a/testar/src/org/testar/monkey/DefaultProtocol.java b/testar/src/org/testar/monkey/DefaultProtocol.java index e3bdc6097..50e793fb6 100644 --- a/testar/src/org/testar/monkey/DefaultProtocol.java +++ b/testar/src/org/testar/monkey/DefaultProtocol.java @@ -85,7 +85,6 @@ import java.util.StringJoiner; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.zip.GZIPInputStream; import javax.swing.JFrame; import javax.swing.JOptionPane; @@ -102,7 +101,6 @@ public class DefaultProtocol extends RuntimeControlsProtocol { protected boolean processListenerOracleEnabled; protected Oracle processListenerOracle; - private List lastStateVerdicts = Collections.emptyList(); private VerdictProcessing verdictProcessing; private State stateForClickFilterLayerProtocol; @@ -117,9 +115,6 @@ public void setStateForClickFilterLayerProtocol(State stateForClickFilterLayerPr } String generatedSequence; - public String getGeneratedSequenceName() { - return generatedSequence; - } private File currentSeq; @@ -135,15 +130,14 @@ public void setReplayVerdicts(List replayVerdicts) { this.replayVerdicts = replayVerdicts; } - List finalVerdicts = Collections.singletonList(Verdict.OK); + private List sequenceVerdicts = Collections.singletonList(Verdict.OK); - public List getFinalVerdicts() { - if (lastStateVerdicts == null || lastStateVerdicts.isEmpty()) { - return finalVerdicts == null || finalVerdicts.isEmpty() - ? Collections.singletonList(Verdict.OK) - : finalVerdicts; - } - return lastStateVerdicts; + public List getSequenceVerdicts() { + return sequenceVerdicts; + } + + protected void updateSequenceVerdicts(List stateVerdicts) { + sequenceVerdicts = verdictProcessing.filterDuplicates(stateVerdicts); } protected String lastPrintParentsOf = "null-id"; @@ -347,8 +341,7 @@ private boolean isValidFile(){ FileInputStream fis = new FileInputStream(seqFile); BufferedInputStream bis = new BufferedInputStream(fis); - GZIPInputStream gis = new GZIPInputStream(bis); - ObjectInputStream ois = new ObjectInputStream(gis); + ObjectInputStream ois = new ObjectInputStream(bis); ois.readObject(); ois.close(); @@ -632,7 +625,7 @@ void saveActionIntoFragmentForReplayableSequence(Action action, State state, Set fragment.set(ActionDuration, settings().get(ConfigTags.ActionDuration)); fragment.set(ActionDelay, settings().get(ConfigTags.TimeToWaitAfterAction)); fragment.set(SystemState, state); - fragment.set(OracleVerdicts, getFinalVerdicts()); + fragment.set(OracleVerdicts, getSequenceVerdicts()); //Find the target widget of the current action, and save the title into the fragment if (state != null && action.get(Tags.OriginWidget, null) != null){ @@ -728,8 +721,8 @@ protected Canvas buildCanvas() { @Override protected void beginSequence(SUT system, State state){ - // Reset the final verdict for the new sequence - finalVerdicts = Collections.singletonList(Verdict.OK); + // Reset the sequence verdict for the new sequence + sequenceVerdicts = Collections.singletonList(Verdict.OK); } @Override @@ -800,15 +793,15 @@ protected State getState(SUT system) throws StateBuildException { return state; List verdicts = getVerdicts(state); - lastStateVerdicts = verdictProcessing.filterDuplicates(verdicts); - state.set(Tags.OracleVerdicts, lastStateVerdicts); + updateSequenceVerdicts(verdictProcessing.filterDuplicates(verdicts)); + state.set(Tags.OracleVerdicts, sequenceVerdicts); // State screenshot is taken after the Verdicts are computed // This might be relevant to determine the viewPort of the screenshot depending on the Verdict (e.g., ProtocolUtil) setStateScreenshot(state); if (mode() != Modes.Spy) { - for (Verdict verdict : lastStateVerdicts) { + for (Verdict verdict : sequenceVerdicts) { // this was added to kill the SUT if it is frozen: if (verdict.severity() == Verdict.Severity.NOT_RESPONDING.getValue()) { //if the SUT is frozen, we should kill it! @@ -1199,9 +1192,9 @@ protected void stopSystem(SUT system) { protected void postSequenceProcessing() { String statusInfo = ""; - List verdicts = (lastStateVerdicts == null || lastStateVerdicts.isEmpty()) - ? (mode() == Modes.Replay ? getReplayVerdicts() : getFinalVerdicts()) - : lastStateVerdicts; + List verdicts = (mode() == Modes.Replay) + ? getReplayVerdicts() + : getSequenceVerdicts(); reportManager.addTestVerdicts(verdicts); statusInfo = buildStatusInfo(verdicts); diff --git a/testar/src/org/testar/monkey/GenerateMode.java b/testar/src/org/testar/monkey/GenerateMode.java index 734795e33..35c09df20 100644 --- a/testar/src/org/testar/monkey/GenerateMode.java +++ b/testar/src/org/testar/monkey/GenerateMode.java @@ -38,6 +38,7 @@ import org.testar.monkey.alayer.State; import org.testar.monkey.alayer.Tags; import org.testar.monkey.alayer.Verdict; +import org.testar.monkey.alayer.actions.NOP; import org.testar.serialisation.LogSerialiser; import java.util.Arrays; @@ -98,18 +99,20 @@ public void runGenerateOuterLoop(DefaultProtocol protocol) { // Initial getState() called before beginSequence: LogSerialiser.log("Obtaining system initial state before beginSequence...\n", LogSerialiser.LogLevel.Debug); State initialState = protocol.getState(system); - protocol.finalVerdicts = initialState.get(Tags.OracleVerdicts, Collections.singletonList(Verdict.OK)); + protocol.updateSequenceVerdicts(initialState.get(Tags.OracleVerdicts, Collections.singletonList(Verdict.OK))); // If the SUT does not contain initial failures, start the inner loop test sequence - if(Verdict.helperAreAllVerdictsOK(protocol.finalVerdicts)) { + if(Verdict.helperAreAllVerdictsOK(protocol.getSequenceVerdicts())) { // beginSequence() - a script to interact with GUI, for example login screen LogSerialiser.log("Invoking begin sequence in the initial state...\n", LogSerialiser.LogLevel.Debug); protocol.beginSequence(system, initialState); // starting the INNER LOOP with the updated state after SUT modification - protocol.finalVerdicts = runGenerateInnerLoop(protocol, system, protocol.getState(system)); + protocol.updateSequenceVerdicts(runGenerateInnerLoop(protocol, system, protocol.getState(system))); } else { // If failure exists in the initial state + // Saving the state with empty actions into replayable test sequence: + protocol.saveActionIntoFragmentForReplayableSequence(new NOP(), initialState, Collections.emptySet()); // Save initial state information in the state model before finishing protocol.stateModelManager.notifyNewStateReached(initialState, Collections.emptySet()); } @@ -225,11 +228,11 @@ private void finishGeneratedSequence(DefaultProtocol protocol, SUT system) { protocol.writeAndCloseFragmentForReplayableSequence(); - if (!Verdict.helperAreAllVerdictsOK(protocol.finalVerdicts)) + if (!Verdict.helperAreAllVerdictsOK(protocol.getSequenceVerdicts())) LogSerialiser.log("Sequence contained faults!\n", LogSerialiser.LogLevel.Critical); // Copy sequence file into proper directory: - protocol.classifyAndCopySequenceIntoAppropriateDirectory(protocol.getFinalVerdicts()); + protocol.classifyAndCopySequenceIntoAppropriateDirectory(protocol.getSequenceVerdicts()); // Calling postSequenceProcessing() to allow resetting test environment after test sequence, etc protocol.postSequenceProcessing(); diff --git a/testar/src/org/testar/monkey/ReplayMode.java b/testar/src/org/testar/monkey/ReplayMode.java index c83c07582..bb82f41e7 100644 --- a/testar/src/org/testar/monkey/ReplayMode.java +++ b/testar/src/org/testar/monkey/ReplayMode.java @@ -41,7 +41,6 @@ import java.util.Collections; import java.util.List; import java.util.Set; -import java.util.zip.GZIPInputStream; import org.testar.OutputStructure; import org.testar.SutVisualization; @@ -66,10 +65,10 @@ public class ReplayMode { * Read the replayable file, repeat saved actions and generate new sequences, oracles and logs */ public void runReplayLoop(DefaultProtocol protocol) { + VerdictProcessing verdictProcessing = new VerdictProcessing(protocol.settings()); FileInputStream fis = null; BufferedInputStream bis = null; - GZIPInputStream gis = null; ObjectInputStream ois = null; protocol.actionCount = 1; @@ -93,8 +92,7 @@ public void runReplayLoop(DefaultProtocol protocol) { fis = new FileInputStream(seqFile); bis = new BufferedInputStream(fis); - gis = new GZIPInputStream(bis); - ois = new ObjectInputStream(gis); + ois = new ObjectInputStream(bis); /** * Initialize the fragment to create a new sequence and logs @@ -107,7 +105,7 @@ public void runReplayLoop(DefaultProtocol protocol) { protocol.cv = protocol.buildCanvas(); State state = protocol.getState(system); - protocol.setReplayVerdicts(protocol.getVerdicts(state)); + protocol.setReplayVerdicts(verdictProcessing.filterDuplicates(protocol.getVerdicts(state))); // notify the statemodelmanager protocol.stateModelManager.notifyTestSequencedStarted(); @@ -250,7 +248,7 @@ public void runReplayLoop(DefaultProtocol protocol) { state = protocol.getState(system); - protocol.setReplayVerdicts(protocol.getVerdicts(state)); + protocol.setReplayVerdicts(verdictProcessing.filterDuplicates(protocol.getVerdicts(state))); } } @@ -270,9 +268,6 @@ public void runReplayLoop(DefaultProtocol protocol) { if (ois != null){ try { ois.close(); } catch (IOException e) { e.printStackTrace(); } } - if (gis != null){ - try { gis.close(); } catch (IOException e) { e.printStackTrace(); } - } if (bis != null){ try { bis.close(); } catch (IOException e) { e.printStackTrace(); } } @@ -285,7 +280,8 @@ public void runReplayLoop(DefaultProtocol protocol) { system.stop(); } - List replayVerdicts = protocol.getReplayVerdicts(); + List replayVerdicts = verdictProcessing.filterDuplicates(protocol.getReplayVerdicts()); + protocol.setReplayVerdicts(replayVerdicts); if(!Verdict.helperAreAllVerdictsOK(replayVerdicts)) { String msg = "Replayed Sequence contains Errors: " + buildVerdictsInfo(replayVerdicts); System.out.println(msg); @@ -317,7 +313,7 @@ public void runReplayLoop(DefaultProtocol protocol) { protocol.writeAndCloseFragmentForReplayableSequence(); //Copy sequence file into proper directory: - protocol.classifyAndCopySequenceIntoAppropriateDirectory(protocol.getReplayVerdicts()); + protocol.classifyAndCopySequenceIntoAppropriateDirectory(replayVerdicts); LogSerialiser.finish(); diff --git a/testar/workflow_windows_webdriver.gradle b/testar/workflow_windows_webdriver.gradle index 331630a3f..12c4a6c67 100644 --- a/testar/workflow_windows_webdriver.gradle +++ b/testar/workflow_windows_webdriver.gradle @@ -142,7 +142,7 @@ task runTestWebdriverUnreplayable(type: Exec) { String output = standardOutput.toString() // Check that TESTAR detects not replayable button - if(output.readLines().any{line->line.contains("Left Click at 'ParisOne' of the replayed sequence can not been replayed")}) { + if(output.readLines().any{line->line.contains("unreplayable_sequence_1.testar The Action Left Click at 'ParisOne' of the replayed sequence can not been replayed")}) { println "\n${output} \nTESTAR has sucessfully detected a not replayable sequence" } else { throw new GradleException("\n${output} \nERROR: Detecting not replayable button with TESTAR Replay mode") From 15f9b4c344ed5ace1d7ec191b4a3d9d9ab99cf81 Mon Sep 17 00:00:00 2001 From: ferpasri Date: Fri, 27 Feb 2026 14:16:04 +0100 Subject: [PATCH 06/10] Update resolveVerdictIgnoreFile to prioritize SSE --- .../org/testar/monkey/VerdictProcessing.java | 8 ++-- .../testar/monkey/VerdictProcessingTest.java | 38 +++++++++++++++++++ 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/testar/src/org/testar/monkey/VerdictProcessing.java b/testar/src/org/testar/monkey/VerdictProcessing.java index 868bffe2e..afc2974b9 100644 --- a/testar/src/org/testar/monkey/VerdictProcessing.java +++ b/testar/src/org/testar/monkey/VerdictProcessing.java @@ -112,14 +112,14 @@ public void storeNewVerdicts(List verdicts) { } } - private File resolveVerdictIgnoreFile() { + public static File resolveVerdictIgnoreFile() { + if (Main.SSE_ACTIVATED != null && !Main.SSE_ACTIVATED.isEmpty()) { + return new File(Main.settingsDir + Main.SSE_ACTIVATED, LIST_VERDICTS_FAILURES_FILENAME); + } String settingsPath = Settings.getSettingsPath(); if (settingsPath != null && !settingsPath.isEmpty()) { return new File(settingsPath, LIST_VERDICTS_FAILURES_FILENAME); } - if (Main.SSE_ACTIVATED != null && !Main.SSE_ACTIVATED.isEmpty()) { - return new File(Main.settingsDir + Main.SSE_ACTIVATED, LIST_VERDICTS_FAILURES_FILENAME); - } return null; } diff --git a/testar/test/org/testar/monkey/VerdictProcessingTest.java b/testar/test/org/testar/monkey/VerdictProcessingTest.java index cafb24211..3b10f0381 100644 --- a/testar/test/org/testar/monkey/VerdictProcessingTest.java +++ b/testar/test/org/testar/monkey/VerdictProcessingTest.java @@ -25,6 +25,7 @@ public class VerdictProcessingTest { @After public void tearDown() { Settings.setSettingsPath(null); + Main.SSE_ACTIVATED = null; } @Test @@ -80,4 +81,41 @@ public void testIgnoresKnownDuplicate() throws Exception { assertEquals(1, filtered.size()); assertEquals(Verdict.OK.severity(), filtered.get(0).severity(), 0.0); } + + @Test + public void testVerdictIgnoreFile_PrioritizesSSE() throws Exception { + File tempSettingsDir = tempFolder.newFolder("tempSettingsDir"); + String originalSettingsDir = Main.settingsDir; + try { + Main.settingsDir = tempSettingsDir.getAbsolutePath() + File.separator; + Main.SSE_ACTIVATED = "protocol_selected"; + Settings.setSettingsPath(tempFolder.newFolder("otherProtocol").getAbsolutePath()); + + File verdictIgnoreFile = VerdictProcessing.resolveVerdictIgnoreFile(); + assertEquals(new File(Main.settingsDir + "protocol_selected", "list_of_verdicts_with_failures.txt").getAbsolutePath(), + verdictIgnoreFile.getAbsolutePath()); + } finally { + Main.settingsDir = originalSettingsDir; // cleanup to restore static global dir + } + } + + @Test + public void testVerdictIgnoreFile_UsesSettingsPathWhenNoSSE() throws Exception { + File tempSettingsDir = tempFolder.newFolder("tempSettingsDir"); + Main.SSE_ACTIVATED = null; + Settings.setSettingsPath(tempSettingsDir.getAbsolutePath()); + + File verdictIgnoreFile = VerdictProcessing.resolveVerdictIgnoreFile(); + assertEquals(new File(tempSettingsDir, "list_of_verdicts_with_failures.txt").getAbsolutePath(), + verdictIgnoreFile.getAbsolutePath()); + } + + @Test + public void testVerdictIgnoreFile_IsNullWhenNoContext() { + Main.SSE_ACTIVATED = null; + Settings.setSettingsPath(null); + + File verdictIgnoreFile = VerdictProcessing.resolveVerdictIgnoreFile(); + assertEquals(null, verdictIgnoreFile); + } } From 806c5ba53a5e9f5521250e3768818aecdcfcd3b0 Mon Sep 17 00:00:00 2001 From: ferpasri Date: Fri, 27 Feb 2026 14:24:24 +0100 Subject: [PATCH 07/10] Add button to manage ignored verdicts --- .../testar/settings/dialog/GeneralPanel.java | 14 ++ .../components/IgnoredVerdictsDialog.java | 157 ++++++++++++++++++ 2 files changed, 171 insertions(+) create mode 100644 testar/src/org/testar/settings/dialog/components/IgnoredVerdictsDialog.java diff --git a/testar/src/org/testar/settings/dialog/GeneralPanel.java b/testar/src/org/testar/settings/dialog/GeneralPanel.java index 188ecec13..d05d40214 100644 --- a/testar/src/org/testar/settings/dialog/GeneralPanel.java +++ b/testar/src/org/testar/settings/dialog/GeneralPanel.java @@ -33,6 +33,7 @@ import org.testar.monkey.*; import org.testar.settings.Settings; +import org.testar.settings.dialog.components.IgnoredVerdictsDialog; import org.testar.settings.dialog.components.UndoTextArea; import javax.swing.*; @@ -57,6 +58,7 @@ public class GeneralPanel extends SettingsPanel implements Observer { //private JCheckBox checkStopOnFault; private JComboBox comboBoxProtocol; private JCheckBox compileCheckBox, checkActionVisualization, checkIgnoreDuplicatedVerdict; + private JButton btnManageIgnoredVerdicts; private JLabel labelAppName = new JLabel("Application name"); private JLabel labelAppVersion = new JLabel("Application version"); @@ -140,6 +142,12 @@ private void addGeneralControlsGlobal(SettingsDialog settingsDialog) { checkIgnoreDuplicatedVerdict.setToolTipText(ConfigTags.IgnoreDuplicatedVerdicts.getDescription()); add(checkIgnoreDuplicatedVerdict); + btnManageIgnoredVerdicts = new JButton("Manage ignored verdicts"); + btnManageIgnoredVerdicts.setBounds(10, 286, 192, 25); + btnManageIgnoredVerdicts.setToolTipText("Open a modal dialog to check or clear existing ignored verdicts"); + btnManageIgnoredVerdicts.addActionListener(this::btnManageIgnoredVerdictsActionPerformed); + add(btnManageIgnoredVerdicts); + labelAppName.setBounds(330, 242, 150, 27); labelAppName.setToolTipText(ToolTipTexts.applicationNameTTT); add(labelAppName); @@ -247,6 +255,12 @@ private void btnEditProtocolActionPerformed(ActionEvent evt) { dialog.setVisible(true); } + private void btnManageIgnoredVerdictsActionPerformed(ActionEvent evt) { + JDialog dialog = new IgnoredVerdictsDialog(); + dialog.setModalityType(JDialog.ModalityType.APPLICATION_MODAL); + dialog.setVisible(true); + } + /** * Populate JPanelGeneral from Settings structure. * diff --git a/testar/src/org/testar/settings/dialog/components/IgnoredVerdictsDialog.java b/testar/src/org/testar/settings/dialog/components/IgnoredVerdictsDialog.java new file mode 100644 index 000000000..2f3ad0e9b --- /dev/null +++ b/testar/src/org/testar/settings/dialog/components/IgnoredVerdictsDialog.java @@ -0,0 +1,157 @@ +/*************************************************************************************************** + * + * Copyright (c) 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2026 Open Universiteit - www.ou.nl + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + *******************************************************************************************************/ + +package org.testar.settings.dialog.components; + +import org.testar.monkey.VerdictProcessing; + +import javax.swing.JButton; +import javax.swing.DefaultListModel; +import javax.swing.JDialog; +import javax.swing.JList; +import javax.swing.JOptionPane; +import javax.swing.JPanel; +import javax.swing.JScrollPane; +import javax.swing.ListSelectionModel; +import java.awt.BorderLayout; +import java.awt.Dimension; +import java.awt.FlowLayout; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.List; + +public class IgnoredVerdictsDialog extends JDialog { + + private final DefaultListModel listModel = new DefaultListModel<>(); + private final JList verdictList = new JList<>(listModel); + private File ignoreFile; + + public IgnoredVerdictsDialog() { + setTitle("Ignored Verdicts"); + setSize(new Dimension(700, 420)); + setLocationRelativeTo(null); + setDefaultCloseOperation(DISPOSE_ON_CLOSE); + setLayout(new BorderLayout()); + + verdictList.setSelectionMode(ListSelectionModel.MULTIPLE_INTERVAL_SELECTION); + add(new JScrollPane(verdictList), BorderLayout.CENTER); + + JPanel buttonPanel = new JPanel(new FlowLayout(FlowLayout.RIGHT)); + JButton removeSelectedButton = new JButton("Remove Selected"); + removeSelectedButton.addActionListener(e -> removeSelectedIgnoredVerdicts()); + JButton clearAllButton = new JButton("Clear All"); + clearAllButton.addActionListener(e -> clearAllIgnoredVerdicts()); + JButton closeButton = new JButton("Close"); + closeButton.addActionListener(e -> dispose()); + buttonPanel.add(removeSelectedButton); + buttonPanel.add(clearAllButton); + buttonPanel.add(closeButton); + add(buttonPanel, BorderLayout.SOUTH); + + loadIgnoredVerdicts(); + } + + private void loadIgnoredVerdicts() { + listModel.clear(); + ignoreFile = VerdictProcessing.resolveVerdictIgnoreFile(); + if (ignoreFile == null) { + listModel.addElement("Ignored verdict file path is not available."); + return; + } + try { + File parent = ignoreFile.getParentFile(); + if (parent != null && !parent.exists()) { + parent.mkdirs(); + } + if (!ignoreFile.exists()) { + ignoreFile.createNewFile(); + } + List lines = Files.readAllLines(ignoreFile.toPath(), StandardCharsets.UTF_8); + boolean hasEntries = false; + for (String line : lines) { + String trimmed = line.trim(); + if (!trimmed.isEmpty()) { + listModel.addElement(trimmed); + hasEntries = true; + } + } + if (!hasEntries) { + listModel.addElement("(No ignored verdicts yet)"); + } + } catch (IOException ex) { + listModel.addElement("Unable to load ignored verdicts file: " + ex.getMessage()); + } + } + + private void clearAllIgnoredVerdicts() { + if (ignoreFile == null) { + JOptionPane.showMessageDialog(this, "Ignored verdict file path is not available.", "Ignored Verdicts", JOptionPane.WARNING_MESSAGE); + return; + } + try { + Files.writeString(ignoreFile.toPath(), "", StandardCharsets.UTF_8); + loadIgnoredVerdicts(); + } catch (IOException ex) { + JOptionPane.showMessageDialog(this, "Unable to clear ignored verdicts:\n" + ex.getMessage(), "Ignored Verdicts", JOptionPane.ERROR_MESSAGE); + } + } + + private void removeSelectedIgnoredVerdicts() { + if (ignoreFile == null) { + JOptionPane.showMessageDialog(this, "Ignored verdict file path is not available.", "Ignored Verdicts", JOptionPane.WARNING_MESSAGE); + return; + } + List selected = verdictList.getSelectedValuesList(); + if (selected == null || selected.isEmpty() || selected.contains("(No ignored verdicts yet)")) { + return; + } + try { + List current = Files.readAllLines(ignoreFile.toPath(), StandardCharsets.UTF_8); + List remaining = new ArrayList<>(); + for (String line : current) { + String trimmed = line.trim(); + if (trimmed.isEmpty()) { + continue; + } + if (!selected.contains(trimmed)) { + remaining.add(trimmed); + } + } + Files.write(ignoreFile.toPath(), remaining, StandardCharsets.UTF_8, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE); + loadIgnoredVerdicts(); + } catch (IOException ex) { + JOptionPane.showMessageDialog(this, "Unable to remove selected verdicts:\n" + ex.getMessage(), "Ignored Verdicts", JOptionPane.ERROR_MESSAGE); + } + } +} From 1143f62157b921be1f21fff0b4cf1da01b82496b Mon Sep 17 00:00:00 2001 From: ferpasri Date: Fri, 27 Feb 2026 14:28:54 +0100 Subject: [PATCH 08/10] Update TestAndroidLogcatOracle for list verdicts --- .../org/testar/oracles/log/TestAndroidLogcatOracle.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/testar/test/org/testar/oracles/log/TestAndroidLogcatOracle.java b/testar/test/org/testar/oracles/log/TestAndroidLogcatOracle.java index e9c1172b7..d4b971b78 100644 --- a/testar/test/org/testar/oracles/log/TestAndroidLogcatOracle.java +++ b/testar/test/org/testar/oracles/log/TestAndroidLogcatOracle.java @@ -173,7 +173,9 @@ public void generateModeVerdict_DeduplicatesNumbersInMatches() { .thenReturn(line1 + "\n" + line2); androidLogcatOracle.initialize(); - Verdict verdict = androidLogcatOracle.getVerdict(state); + List verdicts = androidLogcatOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.assertEquals(Verdict.Severity.SUSPICIOUS_LOG.getValue(), verdict.severity(), 0.0); String expected = "Suspicious Android logcat line(s) detected " @@ -201,7 +203,9 @@ public void generateModeVerdict_KeepsHttpStatusCodes() { .thenReturn(line1 + "\n" + line2); androidLogcatOracle.initialize(); - Verdict verdict = androidLogcatOracle.getVerdict(state); + List verdicts = androidLogcatOracle.getVerdicts(state); + Assert.assertEquals(1, verdicts.size()); + Verdict verdict = verdicts.get(0); Assert.assertEquals(Verdict.Severity.SUSPICIOUS_LOG.getValue(), verdict.severity(), 0.0); String expected = "Suspicious Android logcat line(s) detected " From fdaf053fcc6abee84e030d281ac6468dd939b503 Mon Sep 17 00:00:00 2001 From: ferpasri Date: Fri, 27 Feb 2026 16:19:06 +0100 Subject: [PATCH 09/10] Move ExtendedOracles logic to TESTAR internals --- .../Protocol_02_webdriver_parabank.java | 12 ------------ testar/src/org/testar/monkey/DefaultProtocol.java | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/testar/resources/settings/02_webdriver_parabank/Protocol_02_webdriver_parabank.java b/testar/resources/settings/02_webdriver_parabank/Protocol_02_webdriver_parabank.java index f6245a171..cf0cd23cc 100644 --- a/testar/resources/settings/02_webdriver_parabank/Protocol_02_webdriver_parabank.java +++ b/testar/resources/settings/02_webdriver_parabank/Protocol_02_webdriver_parabank.java @@ -41,10 +41,7 @@ import org.testar.monkey.alayer.webdriver.WdDriver; import org.testar.monkey.alayer.webdriver.enums.WdRoles; import org.testar.monkey.alayer.webdriver.enums.WdTags; -import org.testar.oracles.Oracle; -import org.testar.oracles.OracleSelection; import org.testar.plugin.NativeLinker; -import org.testar.monkey.ConfigTags; import org.testar.monkey.Pair; import org.testar.protocols.WebdriverProtocol; import org.testar.settings.Settings; @@ -59,8 +56,6 @@ public class Protocol_02_webdriver_parabank extends WebdriverProtocol { - private List extendedOraclesList = new ArrayList<>(); - /** * Called once during the life time of TESTAR * This method can be used to perform initial setup work @@ -81,7 +76,6 @@ protected void initialize(Settings settings) { @Override protected void preSequencePreparations() { super.preSequencePreparations(); - extendedOraclesList = OracleSelection.loadExtendedOracles(settings.get(ConfigTags.ExtendedOracles)); } /** @@ -171,12 +165,6 @@ protected List getVerdicts(State state) { // For web applications, web browser errors and warnings can also be enabled via settings List verdicts = super.getVerdicts(state); - // "ExtendedOracles" offered by TESTAR in the test.settings or Oracles GUI dialog - for (Oracle extendedOracle : extendedOraclesList) { - List extendedVerdicts = extendedOracle.getVerdicts(state); - verdicts.addAll(extendedVerdicts); - } - //----------------------------------------------------------------------------- // MORE SOPHISTICATED ORACLES CAN BE PROGRAMMED HERE (the sky is the limit ;-) //----------------------------------------------------------------------------- diff --git a/testar/src/org/testar/monkey/DefaultProtocol.java b/testar/src/org/testar/monkey/DefaultProtocol.java index 50e793fb6..113dbb219 100644 --- a/testar/src/org/testar/monkey/DefaultProtocol.java +++ b/testar/src/org/testar/monkey/DefaultProtocol.java @@ -49,6 +49,7 @@ import org.testar.monkey.alayer.webdriver.WdProtocolUtil; import org.testar.monkey.alayer.windows.WinApiException; import org.testar.oracles.Oracle; +import org.testar.oracles.OracleSelection; import org.testar.oracles.log.LogOracle; import org.testar.oracles.log.ProcessListenerOracle; import org.testar.plugin.NativeLinker; @@ -100,6 +101,7 @@ public class DefaultProtocol extends RuntimeControlsProtocol { protected boolean processListenerOracleEnabled; protected Oracle processListenerOracle; + protected List extendedOraclesList = Collections.emptyList(); private VerdictProcessing verdictProcessing; @@ -704,6 +706,10 @@ protected void preSequencePreparations() { logOracle = createLogOracle(settings); logOracle.initialize(); } + extendedOraclesList = OracleSelection.loadExtendedOracles(settings.get(ConfigTags.ExtendedOracles, "")); + for (Oracle oracle : extendedOraclesList) { + oracle.initialize(); + } } /** @@ -898,6 +904,15 @@ protected List getVerdicts(State state) { } } + if (extendedOraclesList != null) { + for (Oracle extendedOracle : extendedOraclesList) { + List extendedVerdicts = extendedOracle.getVerdicts(state); + if (extendedVerdicts != null) { + verdicts.addAll(extendedVerdicts); + } + } + } + // if empty at this point, everything was OK if (verdicts.isEmpty()) { verdicts.add(Verdict.OK); From b614fe94a2df2198316d43851f3bb658f84f65e7 Mon Sep 17 00:00:00 2001 From: ferpasri Date: Fri, 27 Feb 2026 18:14:20 +0100 Subject: [PATCH 10/10] Update CI task to check multiple verdicts --- .github/workflows/test-windows-webdriver.yml | 10 +++++----- testar/workflow_windows_webdriver.gradle | 18 ++++++++++++------ 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test-windows-webdriver.yml b/.github/workflows/test-windows-webdriver.yml index 4d8ec0c04..0c5a59850 100644 --- a/.github/workflows/test-windows-webdriver.yml +++ b/.github/workflows/test-windows-webdriver.yml @@ -49,15 +49,15 @@ jobs: name: Java${{ matrix.java }}-runTestWebdriverSuspiciousTagStateModel-artifact path: D:/a/TESTAR_dev/TESTAR_dev/testar/target/install/testar/bin/webdriver_and_suspicious - - name: Run webdriver to detect a browser console error in parabank - run: ./gradlew runTestWebdriverParabankConsoleError - - name: Save runTestWebdriverParabankConsoleError HTML report artifact + - name: Run webdriver to detect a browser console error and accessibility warning in parabank + run: ./gradlew runTestWebdriverParabankConsoleErrorAndAccessibilityWarning + - name: Save runTestWebdriverParabankConsoleErrorAndAccessibilityWarning HTML report artifact uses: actions/upload-artifact@v4 # Only upload GitHub Actions results if this task fails (Can be replaced with 'if: always()') if: failure() with: - name: Java${{ matrix.java }}-runTestWebdriverParabankConsoleError-artifact - path: D:/a/TESTAR_dev/TESTAR_dev/testar/target/install/testar/bin/webdriver_console_error + name: Java${{ matrix.java }}-runTestWebdriverParabankConsoleErrorAndAccessibilityWarning-artifact + path: D:/a/TESTAR_dev/TESTAR_dev/testar/target/install/testar/bin/webdriver_console_error_and_accessibility_warning - name: Run webdriver to login in parabank and detect a welcome suspicious tag run: ./gradlew runTestWebdriverParabankLogin diff --git a/testar/workflow_windows_webdriver.gradle b/testar/workflow_windows_webdriver.gradle index 12c4a6c67..cdd0f41fb 100644 --- a/testar/workflow_windows_webdriver.gradle +++ b/testar/workflow_windows_webdriver.gradle @@ -31,19 +31,25 @@ task runTestWebdriverSuspiciousTagStateModel(type: Exec, dependsOn: createDataba } // Connect to para.testar.org to detect the browser console error message -task runTestWebdriverParabankConsoleError(type: Exec) { +// And also add the extended oracle to detect the accessibility warning of an image without alternative text +task runTestWebdriverParabankConsoleErrorAndAccessibilityWarning(type: Exec) { // Read command line output standardOutput = new ByteArrayOutputStream() group = 'test_testar_workflow' - description ='runTestWebdriverParabankConsoleError' + description ='runTestWebdriverParabankConsoleErrorAndAccessibilityWarning' workingDir 'target/install/testar/bin' - commandLine 'cmd', '/c', 'testar sse=test_gradle_workflow_webdriver_parabank AlwaysCompile=true ApplicationName="webdriver_console_error" ShowVisualSettingsDialogOnStartup=false Mode=Generate Sequences=1 SequenceLength=1 WebConsoleErrorOracle=true' + commandLine 'cmd', '/c', 'testar sse=test_gradle_workflow_webdriver_parabank AlwaysCompile=true ApplicationName="webdriver_console_error_and_accessibility_warning" ShowVisualSettingsDialogOnStartup=false Mode=Generate Sequences=1 SequenceLength=1 WebConsoleErrorOracle=true ExtendedOracles=WebAccessibilityImagesAltOracle' doLast { String output = standardOutput.toString() - // Check that output detects the browser console error message - if(output.readLines().any{line->line.contains("Failed to load resource: the server responded with a status of 404")}) { - println "\n${output} \nTESTAR login and browser console error detection has been executed sucessfully" + def required = [ + "[WARNING_ACCESSIBILITY_FAULT] Detected web image widget", + "[SUSPICIOUS_LOG] Web Browser Console Error" + ] + + // Check that output detects both the browser console error message and the accessibility warning + if (required.every { s -> output.contains(s) }) { + println "\n${output} \nTESTAR browser console error and accessibility warning detection has been executed sucessfully" } else { throw new GradleException("\n${output} \nERROR: Executing TESTAR") }