diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index 744b12366d3..dde190e97a0 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -388,7 +388,7 @@ jobs: env: SHARDS: ${{ github.event.inputs.shards || '0' }} TARGET_PER_SHARD: 35 - MAX_SHARDS: 10 + MAX_SHARDS: 8 run: | mapfile -t its < <(find integration-tests/src/test/java -name '*IT.java' -printf '%P\n' | sed -e 's|/|.|g' -e 's|\.java$||' | sort) count=${#its[@]} @@ -489,7 +489,6 @@ jobs: -Dvaadin.productionMode \ -DskipFrontend \ -Dfailsafe.forkCount=$FORK_COUNT \ - -Dcom.vaadin.testbench.Parameters.testsInParallel=2 \ -Dfailsafe.rerunFailingTestsCount=2 \ -Dmaven.test.redirectTestOutputToFile=true \ -Dtest.reuseDriver=true \ diff --git a/vaadin-flow-components-shared-parent/vaadin-flow-components-test-util/src/main/java/com/vaadin/tests/AbstractComponentIT.java b/vaadin-flow-components-shared-parent/vaadin-flow-components-test-util/src/main/java/com/vaadin/tests/AbstractComponentIT.java index a99500735f1..d18cacb7fd5 100644 --- a/vaadin-flow-components-shared-parent/vaadin-flow-components-test-util/src/main/java/com/vaadin/tests/AbstractComponentIT.java +++ b/vaadin-flow-components-shared-parent/vaadin-flow-components-test-util/src/main/java/com/vaadin/tests/AbstractComponentIT.java @@ -25,20 +25,25 @@ import java.util.Arrays; import java.util.List; import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; import java.util.logging.Level; import java.util.stream.Collectors; import org.junit.After; import org.junit.AfterClass; +import org.junit.AssumptionViolatedException; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Rule; +import org.junit.rules.TestRule; import org.openqa.selenium.By; import org.openqa.selenium.Dimension; import org.openqa.selenium.TimeoutException; import org.openqa.selenium.WebDriver; import org.openqa.selenium.WebElement; +import org.openqa.selenium.WrapsDriver; import org.openqa.selenium.chrome.ChromeDriver; import org.openqa.selenium.chrome.ChromeDriverService; import org.openqa.selenium.chrome.ChromeOptions; @@ -60,52 +65,164 @@ /** * Base class for Flow component integration tests. *

- * Extends {@link TestBenchTestCase} directly and manages a headless Chrome - * driver that is reused across all test methods in the same test class. By - * default the driver is started locally; setting {@code -Dtest.use.hub=true} - * routes driver creation to a Selenium Grid endpoint configured via - * {@code -Dtest.hub.url} (default {@code http://localhost:4444/wd/hub}). - *

- * This test setup does not support running tests in parallel. The only way to - * parallelize tests is forking separate JVMs that run one suite at a time, for - * example using {@code failsafe.forkCount}. - *

- * Test classes must be annotated with {@link TestPath} to specify the URL path - * of the test view. + * Four independent opt-in features, each enabled by a system property: + *

+ * Subclasses can opt out of driver reuse on a per-class basis by overriding + * {@link #isReuseDriver()} to return {@code false} regardless of system + * properties. */ public abstract class AbstractComponentIT extends TestBenchTestCase { private static final Logger logger = LoggerFactory .getLogger(AbstractComponentIT.class); + // ----- Feature flags ----- + + /** One driver per class (reuse between methods). */ private static final boolean REUSE_DRIVER = Boolean .getBoolean("test.reuseDriver"); + /** + * One driver for the whole JVM fork (reuse between classes). Implies + * REUSE_DRIVER. + */ + private static final boolean SHARE_DRIVER = Boolean + .getBoolean("test.shareDriver"); + + /** Use Vaadin SPA navigation instead of full page loads. */ + private static final boolean USE_SPA = Boolean.getBoolean("test.useSpa"); + + /** Run Chrome in headed (visible) mode. Default is headless. */ + private static final boolean HEADED = Boolean.getBoolean("test.headed"); + private static final boolean USE_HUB = Boolean.getBoolean("test.use.hub"); private static final String HUB_URL = System.getProperty("test.hub.url", "http://" + Parameters.getHubHostname() + ":4444/wd/hub"); + // ----- Driver state ----- + + /** + * Per-thread driver. Persists across classes when SHARE_DRIVER is true, + * across methods when REUSE_DRIVER is true, created fresh per method + * otherwise. + */ private static final ThreadLocal sharedDriver = new ThreadLocal<>(); + /** + * Tracks all drivers created per thread so the JVM shutdown hook can quit + * them cleanly. Only used when SHARE_DRIVER is true. + */ + private static final ConcurrentHashMap allDrivers = new ConcurrentHashMap<>(); + + private static volatile boolean shutdownHookRegistered = false; + + // ----- Consecutive-failure abort (only meaningful with driver sharing) + // ----- + + private static final AtomicInteger consecutiveFailures = new AtomicInteger( + 0); + private static final int MAX_CONSECUTIVE_FAILURES = 5; + + /** + * JUnit rule that aborts the fork after too many consecutive failures when + * {@code test.shareDriver} is active, to avoid burning CI time on a dead + * driver. + */ + @Rule + public TestRule consecutiveFailureAbort = (base, + description) -> base == null ? null + : consecutiveFailureAbortStatement(base); + + private static org.junit.runners.model.Statement consecutiveFailureAbortStatement( + org.junit.runners.model.Statement base) { + return new org.junit.runners.model.Statement() { + @Override + public void evaluate() throws Throwable { + if (SHARE_DRIVER && consecutiveFailures + .get() >= MAX_CONSECUTIVE_FAILURES) { + throw new AssumptionViolatedException("Aborting: " + + consecutiveFailures.get() + + " consecutive failures, driver may be unstable"); + } + try { + base.evaluate(); + consecutiveFailures.set(0); + } catch (AssumptionViolatedException e) { + throw e; + } catch (Throwable t) { + consecutiveFailures.incrementAndGet(); + throw t; + } + } + }; + } + @Rule public ScreenshotOnFailureRule screenshotOnFailure = new ScreenshotOnFailureRule( this, false); + // ----- JUnit lifecycle ----- + @BeforeClass public static void createDriver() { - if (!REUSE_DRIVER) { + if (!REUSE_DRIVER && !SHARE_DRIVER) { + return; + } + if (SHARE_DRIVER && tryReuseExistingDriver()) { return; } WebDriver driver = createWebDriver(); driver.manage().window().setSize(new Dimension(1024, 800)); driver.manage().timeouts().pageLoadTimeout(Duration.ofSeconds(10)); sharedDriver.set(driver); + if (SHARE_DRIVER) { + allDrivers.put(Thread.currentThread().threadId(), driver); + } + } + + private static boolean tryReuseExistingDriver() { + ensureShutdownHook(); + WebDriver existing = sharedDriver.get(); + if (existing != null && isDriverAlive(existing)) { + return true; + } + if (existing != null) { + tryQuitDriver(existing); + sharedDriver.remove(); + } + return false; + } + + /** + * Returns whether this test class participates in driver reuse. Override + * and return {@code false} in subclasses that require a fresh browser per + * method regardless of {@code test.reuseDriver} / {@code test.shareDriver}. + */ + protected boolean isReuseDriver() { + return REUSE_DRIVER || SHARE_DRIVER; } @Before public void resetDriver() throws Exception { - if (!REUSE_DRIVER) { + if (!isReuseDriver()) { WebDriver driver = createWebDriver(); driver.manage().window().setSize(new Dimension(1024, 800)); driver.manage().timeouts().pageLoadTimeout(Duration.ofSeconds(10)); @@ -118,7 +235,7 @@ public void resetDriver() throws Exception { @After public void quitDriverPerMethod() { - if (!REUSE_DRIVER) { + if (!isReuseDriver()) { tryQuitDriver(sharedDriver.get()); sharedDriver.remove(); } @@ -126,6 +243,11 @@ public void quitDriverPerMethod() { @AfterClass public static void quitDriver() { + if (SHARE_DRIVER) { + // Keep the driver alive for the next test class. + // The shutdown hook will quit it when the JVM exits. + return; + } if (!REUSE_DRIVER) { return; } @@ -149,10 +271,7 @@ protected String getTestPath() { } protected String getRootURL() { - String host = "localhost"; - if (USE_HUB) { - host = findHostAddress(); - } + String host = USE_HUB ? findHostAddress() : "localhost"; return "http://" + host + ":8080"; } @@ -170,12 +289,10 @@ private String findHostAddress() { }).flatMap(NetworkInterface::inetAddresses) .filter(InetAddress::isSiteLocalAddress) .map(InetAddress::getHostAddress).findFirst() - .orElseThrow(() -> { - return new RuntimeException( - "No compatible (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) ip address found."); - }); + .orElseThrow(() -> new IllegalStateException( + "No compatible (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) ip address found.")); } catch (SocketException e) { - throw new RuntimeException("Could not find the host name", e); + throw new IllegalStateException("Could not find the host name", e); } } @@ -209,6 +326,15 @@ protected void open() { protected void open(String... parameters) { String url = getTestURL(parameters); + + // Pass the relative URL (path + query string) so SPA navigation + // preserves query parameters used to configure the test view's + // initial state. + String relativeUrl = url.substring(getRootURL().length()); + if (isReuseDriver() && USE_SPA && trySpaNavigation(relativeUrl)) { + return; + } + TimeoutException lastTimeout = null; for (int attempt = 1; attempt <= 3; attempt++) { try { @@ -225,40 +351,100 @@ protected void open(String... parameters) { throw lastTimeout; } + /** + * Attempts SPA navigation via the Vaadin client router. + * + * @return {@code true} if navigation succeeded, {@code false} if a full + * page load is required + */ + private boolean trySpaNavigation(String relativeUrl) { + try { + Boolean hasVaadin = (Boolean) executeScript( + "return !!(window.Vaadin && window.Vaadin.Flow)"); + if (!Boolean.TRUE.equals(hasVaadin)) { + return false; + } + // Compare full path+query so tests with different query parameters + // get a fresh server-side view even when the path is the same. + String currentFullPath = (String) executeScript( + "return window.location.pathname + window.location.search"); + String normalizedUrl = relativeUrl.startsWith("/") ? relativeUrl + : "/" + relativeUrl; + if (normalizedUrl.equals(currentFullPath)) { + // Same URL: clear cookies so the server creates a fresh + // session, then fall through to a full page load. + getDriver().manage().deleteAllCookies(); + return false; + } + executeScript( + "window.dispatchEvent(new CustomEvent('vaadin-navigate'," + + "{detail:{url:arguments[0],state:null,replace:false,callback:true}}))", + normalizedUrl); + getCommandExecutor().waitForVaadin(); + return true; + } catch (Exception e) { + logger.debug("SPA navigation failed, falling back to full load", e); + return false; + } + } + // ----- Driver management ----- + private static boolean isDriverAlive(WebDriver driver) { + try { + driver.getTitle(); + return true; + } catch (Exception e) { + logger.debug("Driver health check failed", e); + return false; + } + } + private static void tryQuitDriver(WebDriver driver) { try { driver.quit(); } catch (Exception e) { - // Ignore - driver may already be dead + logger.debug("Failed to quit driver", e); } } - private static boolean isJavaInDebugMode() { + private static boolean isDebugMode() { return ManagementFactory.getRuntimeMXBean().getInputArguments() .toString().contains("jdwp"); } + private static void ensureShutdownHook() { + if (shutdownHookRegistered) { + return; + } + synchronized (AbstractComponentIT.class) { + if (shutdownHookRegistered) { + return; + } + shutdownHookRegistered = true; + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + allDrivers.values().forEach(AbstractComponentIT::tryQuitDriver); + allDrivers.clear(); + }, "vaadin-test-driver-shutdown")); + } + } + private static WebDriver createWebDriver() { for (int i = 0; i < 3; i++) { try { - if (USE_HUB) { - return tryCreateRemoteDriver(); - } - - return tryCreateChromeDriver(); + return USE_HUB ? tryCreateRemoteDriver() + : tryCreateChromeDriver(); } catch (Exception e) { - logger.warn("Unable to create driver on attempt " + i, e); + logger.warn("Unable to create driver on attempt {}", i, e); } } - throw new RuntimeException( + throw new IllegalStateException( "Gave up trying to create a driver instance"); } private static ChromeOptions buildChromeOptions() { ChromeOptions options = new ChromeOptions(); - if (!isJavaInDebugMode()) { + if (!HEADED && !isDebugMode()) { options.addArguments("--headless=new", "--disable-gpu", "--disable-backgrounding-occluded-windows"); } @@ -292,28 +478,25 @@ private static WebDriver tryCreateRemoteDriver() return TestBench.createDriver(remoteDriver); } + private static WebDriver unwrap(WebDriver driver) { + WebDriver actual = driver; + while (actual instanceof WrapsDriver wrapsDriver) { + actual = wrapsDriver.getWrappedDriver(); + } + return actual; + } + // ----- Test helper methods ----- - /** - * Waits up to 10s for the given condition to become false. - */ protected void waitUntilNot(ExpectedCondition condition) { waitUntilNot(condition, 10); } - /** - * Waits the given number of seconds for the given condition to become - * false. - */ protected void waitUntilNot(ExpectedCondition condition, long timeoutInSeconds) { waitUntil(ExpectedConditions.not(condition), timeoutInSeconds); } - /** - * Returns true if an element can be found from the driver with given - * selector. - */ public boolean isElementPresent(By by) { try { WebElement element = getDriver().findElement(by); @@ -323,19 +506,11 @@ public boolean isElementPresent(By by) { } } - /** - * Clicks on the element, using JS. This method is more convenient than - * Selenium {@code findElement(By.id(urlId)).click()}, because Selenium - * method changes scroll position, which is not always needed. - */ protected void clickElementWithJs(String elementId) { executeScript(String.format("document.getElementById('%s').click();", elementId)); } - /** - * Clicks on the element, using JS. - */ protected void clickElementWithJs(WebElement element) { executeScript("arguments[0].click();", element); } @@ -352,9 +527,6 @@ protected void waitForElementVisible(final By by) { waitUntil(ExpectedConditions.visibilityOfElementLocated(by)); } - /** - * Scrolls the page to the element given using javascript. - */ protected void scrollToElement(WebElement element) { Objects.requireNonNull(element, "The element to scroll to should not be null"); @@ -362,18 +534,11 @@ protected void scrollToElement(WebElement element) { element); } - /** - * Scrolls the page to the element specified and clicks it. - */ protected void scrollIntoViewAndClick(WebElement element) { scrollToElement(element); element.click(); } - /** - * Gets the log entries from the browser that have the given logging level - * or higher. - */ protected List getLogEntries(Level level) { getCommandExecutor().waitForVaadin(); @@ -388,10 +553,6 @@ protected List getLogEntries(Level level) { private static final String WEB_SOCKET_CONNECTION_ERROR_PREFIX = "WebSocket connection to "; - /** - * Checks browser's log entries, throws an error for any client-side error - * and logs any client-side warnings. - */ protected void checkLogsForErrors( Predicate acceptableMessagePredicate) { getLogEntries(Level.WARNING).forEach(logEntry -> { @@ -415,18 +576,10 @@ protected void checkLogsForErrors( }); } - /** - * Checks browser's log entries, throws an error for any client-side error - * and logs any client-side warnings. - */ protected void checkLogsForErrors() { checkLogsForErrors(msg -> false); } - /** - * If dev server start in progress wait until it's started. Otherwise return - * immediately. - */ protected void waitForDevServer() { Object result; do { @@ -436,18 +589,11 @@ protected void waitForDevServer() { } while (Boolean.TRUE.equals(result)); } - /** - * Calls the {@code blur()} function on the current active element of the - * page, if any. - */ public void blur() { executeScript( "!!document.activeElement ? document.activeElement.blur() : 0"); } - /** - * Gets a property value from a web element using JavaScript. - */ public String getProperty(WebElement element, String propertyName) { Object result = executeScript( "return arguments[0]." + propertyName + ";", element);