Fix flaky SSH security integration tests on Windows CI

itests/test/src/test/java/org/apache/karaf/itests/ssh/SshCommandTestBase.java

@@ -22,6 +22,7 @@
 import java.util.EnumSet;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.karaf.itests.BaseTest;
 import org.apache.sshd.client.SshClient;
@@ -33,6 +34,7 @@
 import org.apache.sshd.client.session.ClientSession.ClientSessionEvent;
 import org.apache.sshd.common.channel.PtyMode;
 import org.awaitility.Awaitility;
+import org.awaitility.core.ConditionTimeoutException;
 import org.junit.Assert;
 import org.junit.runner.RunWith;
 import org.ops4j.pax.exam.junit.PaxExam;
@@ -52,41 +54,67 @@ enum Result { OK, NOT_FOUND, NO_CREDENTIALS }
     void addUsers(String manageruser, String vieweruser) throws Exception {
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         OutputStream pipe = openSshChannel("karaf", "karaf", out);
-        pipe.write(("jaas:realm-manage --realm=karaf"
+        writeCommandAndWait(pipe, out, "jaas:realm-manage --realm=karaf"
                 + ";jaas:user-add " + manageruser + " " + manageruser
                 + ";jaas:role-add " + manageruser + " manager"
                 + ";jaas:role-add " + manageruser + " viewer"
                 + ";jaas:role-add " + manageruser + " ssh"
                 + ";jaas:user-add " + vieweruser + " " + vieweruser
                 + ";jaas:role-add " + vieweruser + " viewer"
                 + ";jaas:role-add " + vieweruser + " ssh"
-                + ";jaas:update;jaas:realm-manage --realm=karaf;jaas:user-list\n").getBytes());
-        pipe.flush();
+                + ";jaas:update;jaas:realm-manage --realm=karaf;jaas:user-list");
         closeSshChannel(pipe);
         System.out.println(new String(out.toByteArray()));
     }
 
     void addViewer(String vieweruser) throws Exception {
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         OutputStream pipe = openSshChannel("karaf", "karaf", out);
-        pipe.write(("jaas:realm-manage --realm=karaf"
+        writeCommandAndWait(pipe, out, "jaas:realm-manage --realm=karaf"
                 + ";jaas:user-add " + vieweruser + " " + vieweruser
                 + ";jaas:role-add " + vieweruser + " viewer"
                 + ";jaas:role-add " + vieweruser + " ssh"
-                + ";jaas:update;jaas:realm-manage --realm=karaf;jaas:user-list\n").getBytes());
-        pipe.flush();
+                + ";jaas:update;jaas:realm-manage --realm=karaf;jaas:user-list");
         closeSshChannel(pipe);
         System.out.println(new String(out.toByteArray()));
     }
 
-    String assertCommand(String user, String command, Result result) throws Exception {
+    /**
+     * Writes the given command(s) to the SSH channel and blocks until they have been fully
+     * processed by the remote shell.
+     *
+     * <p>A sentinel {@code echo <marker>} command is appended after the supplied command and
+     * this method waits until the unique marker shows up in the captured output. Because the
+     * remote shell reads and executes its input line by line, the marker cannot appear before
+     * the supplied command has been fully executed and its output flushed back to the client.
+     * Without this synchronization the SSH session could be torn down (see
+     * {@link #closeSshChannel(OutputStream)}) while the command output is still in flight,
+     * producing truncated output and spurious assertion failures - regularly observed on slower
+     * Windows CI runners. {@code echo} is a gogo built-in that is not restricted by any command
+     * ACL, so it is safe to use for every test user.</p>
+     */
+    private void writeCommandAndWait(OutputStream pipe, ByteArrayOutputStream out, String command) throws IOException {
         if (!command.endsWith("\n"))
             command += "\n";
+        pipe.write(command.getBytes());
+        String marker = "KARAF_ITEST_MARKER_" + System.nanoTime();
+        pipe.write(("echo " + marker + "\n").getBytes());
+        pipe.flush();
 
+        try {
+            Awaitility.await().atMost(60, TimeUnit.SECONDS)
+                    .pollInterval(200, TimeUnit.MILLISECONDS)
+                    .until(() -> out.toString().contains(marker));
+        } catch (ConditionTimeoutException e) {
+            // Fall through: proceed with whatever output was captured so far. For assertions
+            // this yields a more useful failure message than the timeout itself.
+        }
+    }
+
+    String assertCommand(String user, String command, Result result) throws Exception {
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         OutputStream pipe = openSshChannel(user, user, out, out);
-        pipe.write(command.getBytes());
-        pipe.flush();
+        writeCommandAndWait(pipe, out, command);
 
         closeSshChannel(pipe);
         String output = new String(out.toByteArray());